Removing duplicate objects with Underscore for Javascript

Question

I have this kind of array:

var foo = [ { "a" : "1" }, { "b" : "2" }, { "a" : "1" } ];

I'd like to filter it to have:

var bar = [ { "a" : "1" }, { "b" : "2" }];

I tried using _.uniq, but I guess because { "a" : "1" } is not equal to itself, it doesn't work. Is there any way to provide underscore uniq with an overriden equals function?

Answer 1

.uniq/.unique accepts a callback

var list = [{a:1,b:5},{a:1,c:5},{a:2},{a:3},{a:4},{a:3},{a:2}];

var uniqueList = _.uniq(list, function(item, key, a) { 
    return item.a;
});

// uniqueList = [Object {a=1, b=5}, Object {a=2}, Object {a=3}, Object {a=4}]

Notes:

1. Callback return value used for comparison
2. First comparison object with unique return value used as unique
3. underscorejs.org demonstrates no callback usage
4. lodash.com shows usage

Another example : using the callback to extract car makes, colors from a list

Answer 2

If you're looking to remove duplicates based on an id you could do something like this:

var res = [
  {id: 1, content: 'heeey'},
  {id: 2, content: 'woah'}, 
  {id: 1, content:'foo'},
  {id: 1, content: 'heeey'},
];
var uniques = _.map(_.groupBy(res,function(doc){
  return doc.id;
}),function(grouped){
  return grouped[0];
});

//uniques
//[{id: 1, content: 'heeey'},{id: 2, content: 'woah'}]

Answer 3

Implementation of Shiplu's answer.

var foo = [ { "a" : "1" }, { "b" : "2" }, { "a" : "1" } ];

var x = _.uniq( _.collect( foo, function( x ){
    return JSON.stringify( x );
}));

console.log( x ); // returns [ { "a" : "1" }, { "b" : "2" } ]

Answer 4

When I have an attribute id, this is my preffered way in underscore:

var x = [{i:2}, {i:2, x:42}, {i:4}, {i:3}];
_.chain(x).indexBy("i").values().value();
// > [{i:2, x:42}, {i:4}, {i:3}]

Answer 5

Using underscore unique lib following is working for me, I m making list unique on the based of _id then returning String value of _id:

var uniqueEntities = _.uniq(entities, function (item, key, a) {
                                    return item._id.toString();
                                });

Answer 6

Here is a simple solution, which uses a deep object comparison to check for duplicates (without resorting to converting to JSON, which is inefficient and hacky)

var newArr = _.filter(oldArr, function (element, index) {
    // tests if the element has a duplicate in the rest of the array
    for(index += 1; index < oldArr.length; index += 1) {
        if (_.isEqual(element, oldArr[index])) {
            return false;
        }
    }
    return true;
});

It filters out all elements if they have a duplicate later in the array - such that the last duplicate element is kept.

The testing for a duplicate uses _.isEqual which performs an optimised deep comparison between the two objects see the underscore isEqual documentation for more info.

edit: updated to use _.filter which is a cleaner approach

Answer 7

The lodash 4.6.1 docs have this as an example for object key equality:

_.uniqWith(objects, _.isEqual);

https://lodash.com/docs#uniqWith

Answer 8

Try iterator function

For example you can return first element

x = [['a',1],['b',2],['a',1]]

_.uniq(x,false,function(i){  

   return i[0]   //'a','b'

})

=> [['a',1],['b',2]]

Answer 9

here's my solution (coffeescript) :

_.mixin
  deepUniq: (coll) ->
    result = []
    remove_first_el_duplicates = (coll2) ->

      rest = _.rest(coll2)
      first = _.first(coll2)
      result.push first
      equalsFirst = (el) -> _.isEqual(el,first)

      newColl = _.reject rest, equalsFirst

      unless _.isEmpty newColl
        remove_first_el_duplicates newColl

    remove_first_el_duplicates(coll)
    result

example:

_.deepUniq([ {a:1,b:12}, [ 2, 1, 2, 1 ], [ 1, 2, 1, 2 ],[ 2, 1, 2, 1 ], {a:1,b:12} ]) 
//=> [ { a: 1, b: 12 }, [ 2, 1, 2, 1 ], [ 1, 2, 1, 2 ] ]

Answer 10

with underscore i had to use String() in the iteratee function

function isUniq(item) {
    return String(item.user);
}
var myUniqArray = _.uniq(myArray, isUniq);

Answer 11

I wanted to solve this simple solution in a straightforward way of writing, with a little bit of a pain of computational expenses... but isn't it a trivial solution with a minimum variable definition, is it?

function uniq(ArrayObjects){
  var out = []
  ArrayObjects.map(obj => {
    if(_.every(out, outobj => !_.isEqual(obj, outobj))) out.push(obj)
  })
  return out
}

Answer 12

var foo = [ { "a" : "1" }, { "b" : "2" }, { "a" : "1" } ];
var bar = _.map(_.groupBy(foo, function (f) { 
        return JSON.stringify(f); 
    }), function (gr) { 
        return gr[0]; 
    }
);

Lets break this down. First lets group the array items by their stringified value

var grouped = _.groupBy(foo, function (f) { 
    return JSON.stringify(f); 
});

grouped looks like:

{
    '{ "a" : "1" }' = [ { "a" : "1" } { "a" : "1" } ],
    '{ "b" : "2" }' = [ { "b" : "2" } ]
}

Then lets grab the first element from each group

var bar = _.map(grouped, function(gr)
    return gr[0]; 
});

bar looks like: [ { "a" : "1" }, { "b" : "2" } ]

Put it all together:

var foo = [ { "a" : "1" }, { "b" : "2" }, { "a" : "1" } ];
var bar = _.map(_.groupBy(foo, function (f) { 
        return JSON.stringify(f); 
    }), function (gr) { 
        return gr[0]; 
    }
);

Answer 13

You can do it in a shorthand as:

_.uniq(foo, 'a')