Remove duplicate entries from array but keep the most complete entries

Question

I have a multidimensional array with values like this:

let unfilteredArray = [
  ["1234", "no email entered", "other value", null, 7, "another value"],
  ["3333", "b@example.com", "another value", 2, 10, "something else"],
  ["1234", "a@example.com", "random value", 2, null, "something else"],
  ["4444", "c@example.com", "another value", 29, 3, "xxx"],
  ["5555", "abcd", "another value", 3, 41, "yyy"],
  ["1234", "another random text", "another value", 4, 8, "zzz"],
  ["5555", "efgh", "another value", null, 0, null]
];

I would like to extract unique users ID but keep the entries with email in the column index 1. If given user ID has no email than use the last row with given user ID. All the other columns for selected rows should kept in the output.

The result for the values above should look like this:

let unfilteredArray = [
  ["1234", "a@example.com", "random value", 2, null, "something else"],
  ["3333", "b@example.com", "another value", 2, 10, "something else"],
  ["4444", "c@example.com", "another value", 29, 3, "xxx"],
  ["5555", "efgh", "another value", null, 0, null] // no email for User ID 5555 => last 5555 row used
];

So far I have a script that removes duplicities:

// index = index of the column with User ID
function removeDuplicates(array, index) {
  // Gets only the first entry - the rest are removed
  let filteredArray = [];
  for (let i = 1; i < array.length; i++) {
    let isDuplicate = false;
    for (let j = i-1; j >= 0; j--) {
      if (array[i][index] == array[j][index]) {
        isDuplicate = true;
        break;
      }
    }
    if (!isDuplicate) {
      filteredArray.push(array[i]);
    }
  }
  return filteredArray;
}

How can I ensure the rows with email are returned?

Can you group the array by User Id, then `.reduce` each group to one record? I'd give it a go in a fiddle if you include one with example data. — joshvito, Jan 29 '21 at 16:22
Some of the columns can contain null values. Maybe because of this the proposed solutions do not work? I updated the examples to be as close to the real case as possible. Thank you! — Nelie, Jan 31 '21 at 19:38

Ori Drori · Answer 1 · 2021-01-31T20:04:40.803

Reduce the array to a Map, and include in the Map only rows with an id (key) that doesn't exist on the Map, or ones that their value is an email. Convert the Map's .values() back to an array using array spread, or Array.from().

Note: the validateEmail(email) function taken from this answer.

function validateEmail(email) {
  const re = /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
  return re.test(String(email).toLowerCase());
}

function removeDuplicates(array) {
  return Array.from(
    array.reduce((acc, row) => {
      const [key, value] = row;
      
      if(!acc.has(key) || validateEmail(value)) {
        acc.set(key, row);
      }
     
      return acc;
    }, new Map()).values()
  )
}

const unfilteredArray = [["1234","no email entered","other value",null,7,"another value"],["3333","b@example.com","another value",2,10,"something else"],["1234","a@example.com","random value",2,null,"something else"],["4444","c@example.com","another value",29,3,"xxx"],["5555","abcd","another value",3,41,"yyy"],["1234","another random text","another value",4,8,"zzz"],["5555","efgh","another value",null,0,null]];

const result = removeDuplicates(unfilteredArray);

console.log(result);

Thank you so much for the answer! I tried your solution but it doesn't work with the real case. I have more columns in the multidimensional array which should be kept. Could you please update the solution so it keeps those? I updated the question so it is accurate with the real case. — Nelie, Jan 31 '21 at 19:29
This solution works no matter what the length of the array, as long as the `id` is the 1st item in a row, and the email is the 2nd. I've updated the example with your new data, but haven't changed the solution itself. — Ori Drori, Jan 31 '21 at 20:05

Ali Mustafa Fidancı · Answer 2 · 2021-01-29T17:03:11.027

I combined a few answers on Stackoverflow and got the piece of code. I changed your array to object. If it is not important, here my result code

var unfilteredArray = [
  {id:1234, text: "no email entered"},
  {id:3333, text: "b@example.com"},
  {id:1234, text: "a@example.com"},
  {id:4444, text: "c@example.com"},
  {id:5555, text: "abcd"},
  {id:1234, text: "another random text"},
  {id:5555,text: "efgh"}
];

function isValidEmail(email) {
    const re = /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
  return re.test(String(email).toLowerCase());
}

var arr2 = unfilteredArray.reduce( (a,b) => {
    var i = a.findIndex( x => x.id === b.id);
    return i === -1 ? a.push({ id : b.id, text : b.text }) : a[i].text = 
    this.isValidEmail(a[i].text) ? a[i].text: b.text, a;
}, []);

console.log(arr2)

*validation email function link => here

Thank you very much for the answer! I have this as an array so I updated your your solution to array I tried in the real case. Unfortunately it doesn't work. I have more columns in the multidimensional array which should be kept. Could you please update the solution so it keeps those? Maybe the issue is with some values to be null? I updated the question so it is accurate with the real case. — Nelie, Jan 31 '21 at 19:32

Remove duplicate entries from array but keep the most complete entries

2 Answers2