Parsing JSON to exclude duplicates Node.JS

Question

First off, I'm new to NodeJS and JavaScript, so any help would be greatly appreciated.

My goal is to be able to parse a csv file so that I can see how many users log in each month. In order for me to do that, I've used the npm "CSVtoJSON" to convert the csv file to JSON format. This what I've get from the JSON file but with over 8000+ entries. Below is a snippet.

[
 { date: '2015-04-09T19:30:22.213795+00:00',
   'email address': '',
   'full name': 'Canton Bashkin',
   action: 'LoggedInActivity',
   application: '',
   project: '',
   task: '',
   'other email address': '',
   'other full name': '',
   description: 'Canton Bashkin logged in' },
 { date: '2015-04-08T00:34:42.261728+00:00',
   'email address': '',
   'full name': 'Sha Phad',
   action: 'LoggedInActivity',
   application: '',
   project: '',
   task: '',
   'other email address': '',
   'other full name': '',
   description: 'Sha Phad logged in' },
 { date: '2015-04-07T23:31:32.559654+00:00',
   'email address': '',
   'full name': 'Canton Bashkin',
   action: 'LoggedInActivity',
   application: '',
   project: '',
   task: '',
   'other email address': '',
   'other full name': '',
   description: 'Canton Bashkin logged in' },
 { date: '2015-04-07T23:31:02.408628+00:00',
   'email address': '',
   'full name': 'Sha Phad',
   action: 'LoggedInActivity',
   application: '',
   project: '',
   task: '',
   'other email address': '',
   'other full name': '',
   description: 'Sneha Phadke logged in'}
]

My code:

//Converter Class 
var Converter = require("csvtojson").Converter;
var converter = new Converter({});
var allUsers = [];

//end_parsed will be emitted once parsing finished
function parseUsers() {

    converter.on("end_parsed", function (jsonArray) {
        for (var i = 0; i < jsonArray.length; i++) {
            var users = jsonArray[i]['full name'];
            if (jsonArray[i]['action'] === 'LoggedInActivity') {
                if (users != 'SD Elements Support' && users != 'Karan Sharala' && users != 'Rick Bogans'
                && users != 'Kanal Bhatya' && users != 'Sanka Saja' && users != 'Kiiko Plash') {
                        allUsers.push(jsonArray[i]);    
                } 
            }
        }

        console.log(allUsers);

    });
}

parseUsers();


//read from file 
require("fs").createReadStream("log.csv").pipe(converter);

I am stuck trying to figure out the best way to parse JSON so that I don't add duplicates of the same name into the array. If I'm doing this completely wrong, please point me in the right direction. Thank you guys for the help.

After I get the names, I need to separate them into individual months, and I'm not too sure how to do that. Any help would be greatly appreciated!

If _full name_ is a _String_ you can use the names as _Object keys_ and test the object for the existence of the key (much faster than iterating through an Array testing each item) — Paul S., Oct 27 '15 at 23:29
You should look at [this answer on how to unique an array](http://stackoverflow.com/a/14438954/691711) — zero298, Oct 27 '15 at 23:30
Be warned that the example @zero298 links to is for primitives only, for objects you'd need a bit more effort than a simple `indexOf` — Paul S., Oct 27 '15 at 23:31
Thanks for the comments. I will take a look at object keys then — Kelvin, Oct 27 '15 at 23:36

Paul S. · Accepted Answer · 2015-10-27T23:41:31.310

Using an Object called unique to keep track of already added Strings

var unique = Object.create(null); // empty object, no inheritance
for (var i = 0; i < jsonArray.length; i++) {
    var users = jsonArray[i]['full name'];
    if (jsonArray[i]['action'] === 'LoggedInActivity') {
        if (!(users in unique)) { // not seen this `full name` before
            unique[users] = allUsers.push(jsonArray[i]);
        } 
    }
}

You can combine this with filter if you find it easier to understand, in this particular example it is not much cleaner. A very large Array may also make the cost of function overhead with filter too expensive.

var unique = Object.create(null);
allUsers = jsonArray.filter(function (e) {
    if (e['action'] === 'LoggedInActivity' && !(e['full name'] in unique)) {
        return unique[e['full name']] = true;
    }
    return false;
});

Parsing JSON to exclude duplicates Node.JS

1 Answers1

Linked