51

I'm trying to create a series of lists in HTML of the form:

<div>
    <h3>Math K</h3>
    <li>Counting & Cardinality</li>
    <li>Geometry</li>
</div>
<div>
    <h3>Math 1</h3>
    <li>Counting & Cardinality</li>
    <li>Orders of Operation</li>
</div>
<div>
    <h3>Math 2</h3>
    <li>Geometry</li>
</div>

My original though was to create an array and push that into the <div> element on the page with $("#divid").append(array). I created an array that looks like this:

var standardsList = [
    {"Grade": "Math K", "Domain": "Counting & Cardinality"},
    {"Grade": "Math K", "Domain": "Counting & Cardinality"},
    {"Grade": "Math K", "Domain": "Counting & Cardinality"},
    {"Grade": "Math K", "Domain": "Counting & Cardinality"},
    {"Grade": "Math K", "Domain": "Geometry"},
    {"Grade": "Math 1", "Domain": "Counting & Cardinality"},
    {"Grade": "Math 1", "Domain": "Counting & Cardinality"},
    {"Grade": "Math 1", "Domain": "Orders of Operation"},
    {"Grade": "Math 2", "Domain": "Geometry"},
    {"Grade": "Math 2", "Domain": "Geometry"}
];

And I need to remove the duplicates so that something like this remains:

var standardsList = [
    {"Grade": "Math K", "Domain": "Counting & Cardinality"},
    {"Grade": "Math K", "Domain": "Geometry"},
    {"Grade": "Math 1", "Domain": "Counting & Cardinality"},
    {"Grade": "Math 1", "Domain": "Orders of Operation"},
    {"Grade": "Math 2", "Domain": "Geometry"}
];

I've tried installing underscore.js and using ._uniq but that only seems to work when a single key:value pair appears in the object. I can't seem to get it to work across multiple keys.

When I try something like:

var uniqueStandards = _.uniq(standardsList, function(item, key, Domain){
    return item.Domain;
});

I only get the first three unique values (one per grade). But I need all the unique values across both grade and domain. Is there a simple way to feed both keys to the _.uniq function?

Ultimately, I need a list with the each unique grade as the header and the unique domains as the list items to pass into an HTML page.

Braiam
  • 1
  • 11
  • 47
  • 78
dchess
  • 587
  • 1
  • 7
  • 13
  • 2
    `return item.Grade + item.Domain`? – cookie monster May 07 '14 at 02:58
  • 1
    possible duplicate of [Underscore: Remove object having some duplicate properties](http://stackoverflow.com/questions/21426994/underscore-remove-object-having-some-duplicate-properties) – cookie monster May 07 '14 at 02:59
  • For those looking for a json deduplicator, as I was, this provides such a great resource using a hashTable lookup and array.filter() -> https://www.jstips.co/en/javascript/deduplicate-an-array/ – Kzqai Jan 10 '19 at 07:48

19 Answers19

42

I know there are many answers already but the best one that worked for me for a complex json structure is:

var arr = [{ "State": "RWCW", "desc": "WEST", "code": "RWCW", "level": 0, "save": "RWCW : WEST", "attribute1": "", "attribute2": "" }, { "State": "RSCW", "desc": "SOUTHEAST", "code": "RSCW", "level": 0, "save": "RSCW : SOUTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RCSW", "desc": "SOUTHWEST", "code": "RCSW", "level": 0, "save": "RCSW : SOUTHWEST", "attribute1": "", "attribute2": "" }, { "State": "RECW", "desc": "NORTHEAST", "code": "RECW", "level": 0, "save": "RECW : NORTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RWCW", "desc": "WEST", "code": "RWCW", "level": 0, "save": "RWCW : WEST", "attribute1": "", "attribute2": "" }, { "State": "RWCW", "desc": "WEST", "code": "RWCW", "level": 0, "save": "RWCW : WEST", "attribute1": "", "attribute2": "" }, { "State": "RSCW", "desc": "SOUTHEAST", "code": "RSCW", "level": 0, "save": "RSCW : SOUTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RCSW", "desc": "SOUTHWEST", "code": "RCSW", "level": 0, "save": "RCSW : SOUTHWEST", "attribute1": "", "attribute2": "" }, { "State": "RECW", "desc": "NORTHEAST", "code": "RECW", "level": 0, "save": "RECW : NORTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RWCW", "desc": "WEST", "code": "RWCW", "level": 0, "save": "RWCW : WEST", "attribute1": "", "attribute2": "" }, { "State": "RSCW", "desc": "SOUTHEAST", "code": "RSCW", "level": 0, "save": "RSCW : SOUTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RECW", "desc": "NORTHEAST", "code": "RECW", "level": 0, "save": "RECW : NORTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RWCW", "desc": "WEST", "code": "RWCW", "level": 0, "save": "RWCW : WEST", "attribute1": "", "attribute2": "" }, { "State": "RCSW", "desc": "SOUTHWEST", "code": "RCSW", "level": 0, "save": "RCSW : SOUTHWEST", "attribute1": "", "attribute2": "" }, { "State": "RWCW", "desc": "WEST", "code": "RWCW", "level": 0, "save": "RWCW : WEST", "attribute1": "", "attribute2": "" }, { "State": "RCNW", "desc": "MIDWEST", "code": "RCNW", "level": 0, "save": "RCNW : MIDWEST", "attribute1": "", "attribute2": "" }, { "State": "RCSW", "desc": "SOUTHWEST", "code": "RCSW", "level": 0, "save": "RCSW : SOUTHWEST", "attribute1": "", "attribute2": "" }, { "State": "RECW", "desc": "NORTHEAST", "code": "RECW", "level": 0, "save": "RECW : NORTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RCNW", "desc": "MIDWEST", "code": "RCNW", "level": 0, "save": "RCNW : MIDWEST", "attribute1": "", "attribute2": "" }, { "State": "RSCW", "desc": "SOUTHEAST", "code": "RSCW", "level": 0, "save": "RSCW : SOUTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RECW", "desc": "NORTHEAST", "code": "RECW", "level": 0, "save": "RECW : NORTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RCNW", "desc": "MIDWEST", "code": "RCNW", "level": 0, "save": "RCNW : MIDWEST", "attribute1": "", "attribute2": "" }, { "State": "RSCW", "desc": "SOUTHEAST", "code": "RSCW", "level": 0, "save": "RSCW : SOUTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RSCW", "desc": "SOUTHEAST", "code": "RSCW", "level": 0, "save": "RSCW : SOUTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RSCW", "desc": "SOUTHEAST", "code": "RSCW", "level": 0, "save": "RSCW : SOUTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RCNW", "desc": "MIDWEST", "code": "RCNW", "level": 0, "save": "RCNW : MIDWEST", "attribute1": "", "attribute2": "" }, { "State": "RSCW", "desc": "SOUTHEAST", "code": "RSCW", "level": 0, "save": "RSCW : SOUTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RECW", "desc": "NORTHEAST", "code": "RECW", "level": 0, "save": "RECW : NORTHEAST", "attribute1": "", "attribute2": "" }];

var clean = arr.filter((arr, index, self) =>
    index === self.findIndex((t) => (t.save === arr.save && t.State === arr.State)))

console.log(clean);

You can try this directly to chrome browser console and edit according to your need.

I hope this helps someone.

Tushar Walzade
  • 3,737
  • 4
  • 33
  • 56
Aarchie
  • 511
  • 1
  • 5
  • 10
29
function arrUnique(arr) {
    var cleaned = [];
    arr.forEach(function(itm) {
        var unique = true;
        cleaned.forEach(function(itm2) {
            if (_.isEqual(itm, itm2)) unique = false;
        });
        if (unique)  cleaned.push(itm);
    });
    return cleaned;
}

var standardsList = arrUnique(standardsList);

FIDDLE

This will return

var standardsList = [
    {"Grade": "Math K", "Domain": "Counting & Cardinality"},
    {"Grade": "Math K", "Domain": "Geometry"},
    {"Grade": "Math 1", "Domain": "Counting & Cardinality"},
    {"Grade": "Math 1", "Domain": "Orders of Operation"},
    {"Grade": "Math 2", "Domain": "Geometry"}
];

Which is exactly what you asked for ?

Shenal Silva
  • 1,955
  • 5
  • 26
  • 37
adeneo
  • 312,895
  • 29
  • 395
  • 388
  • 2
    what do you say for performance part, if I have long list of json (at least 500 json, with 10 property each). – maddy Aug 07 '15 at 18:16
20

Ultimately, I need a list with the each unique grade as the header and the unique domains as the list items to pass into an HTML page. I may be going about this wrong, so if there is an easier way to accomplish that end goal, I am open to ideas.

So you don't actually need that output array in the format you asked about.

That being the case, I would cut directly to the chase with a very simple and efficient solution:

var grades = {};
standardsList.forEach( function( item ) {
    var grade = grades[item.Grade] = grades[item.Grade] || {};
    grade[item.Domain] = true;
});

console.log( JSON.stringify( grades, null, 4 ) );

The resulting grades object is:

{
    "Math K": {
        "Counting & Cardinality": true,
        "Geometry": true
    },
    "Math 1": {
        "Counting & Cardinality": true,
        "Orders of Operation": true
    },
    "Math 2": {
        "Geometry": true
    }
}

One interesting thing about this approach is that it is very fast. Note that it makes only a single pass through the input array, unlike other solutions that require multiple passes (whether you write them yourself or whether _.uniq() does it for you). For a small number of items this won't matter, but it's good to keep in mind for larger lists.

And with this object you now have everything you need to run any code or generate any other format you want. For example, if you do need the exact array output format you mentioned, you can use:

var outputList = [];
for( var grade in grades ) {
    for( var domain in grades[grade] ) {
        outputList.push({ Grade: grade, Domain: domain });
    }
}

JSON.stringify( outputList, null, 4 );

This will log:

[
    {
        "Grade": "Math K",
        "Domain": "Counting & Cardinality"
    },
    {
        "Grade": "Math K",
        "Domain": "Geometry"
    },
    {
        "Grade": "Math 1",
        "Domain": "Counting & Cardinality"
    },
    {
        "Grade": "Math 1",
        "Domain": "Orders of Operation"
    },
    {
        "Grade": "Math 2",
        "Domain": "Geometry"
    }
]

Rai asks in a comment how this line of code works:

var grade = grades[item.Grade] = grades[item.Grade] || {};

This is a common idiom for fetching an object property or providing a default value if the property is missing. Note that the = assignments are done in right-to-left order. So we could translate it literally to use an if statement and a temp variable:

// Fetch grades[item.Grade] and save it in temp
var temp = grades[item.Grade];
if( ! temp ) {
    // It was missing, so use an empty object as the default value
    temp = {};
}
// Now save the result in grades[item.Grade] (in case it was missing)
// and in grade
grades[item.Grade] = temp;
var grade = temp;

You may notice that in the case where grades[item.Grade] already exists, we take the value we just fetched and store it back into the same property. This is unnecessary, of course, and you probably wouldn't do it if you were writing the code out like this. Instead, you would simplify it:

var grade = grades[item.Grade];
if( ! grade ) {
    grade = grades[item.Grade] = {};
}

That would be a perfectly reasonable way to write the same code, and it's more efficient too. It also gives you a way to do a more specific test than the "truthiness" that the || idiom relies on. For example instead of if( ! grade ) you might want to use if( grade === undefined ).

Michael Geary
  • 28,450
  • 9
  • 65
  • 75
  • This is very close to what I need. How would you modify this to make the result a simple array in the form of [grade,domain,domain,grade,domain,domain,grade,domain]? – dchess May 07 '14 at 03:47
  • The array format you're asking about here doesn't sound practical at all. How do you distinguish between grades and domains when they are intermixed like that? It would be very hard to work with. It's easy to turn this `grades` object into any format you want, or to work with directly and call any kind of functions as needed with the data. But don't get hung up on a data format like that array which just won't work. The real question to ask is what you want to do with the data from there. – Michael Geary May 07 '14 at 04:07
  • But if you want the output format you asked about in the original question, that's easy. I updated the answer with a few lines of code to generate that format. – Michael Geary May 07 '14 at 04:14
  • I'm hoping to generate some html with the Grade as the header and the Domains unique to that grade as on unordered list below it. I was thinking the array would let me inject some html tags in between and create an html string. Like this ["
    Grade1
  • Domain1
  • Domain2
Grade2
  • Domain1
  • etc..."]
    – dchess May 07 '14 at 04:15
  • @MichaelGeary Could you please explain the logic of `var grade = grades[item.Grade] = grades[item.Grade] || {};`? – SoundStage Aug 10 '17 at 16:38
  • 1
    @Rai Sure thing, I added a note at the end of the answer explaining it. – Michael Geary Aug 10 '17 at 21:24
  • @MichaelGeary Awesome! Thank you! – SoundStage Aug 11 '17 at 04:56