Filter out duplicates on add to Mongoose MongoDB

Question

I have a .json file on my hard drive with about 70 entries.

This is the model i'm using:

var tweetModel = mongoose.model('tweet', {
    "correct":Boolean,
    "text": String,
    "id":{type:String, unique:true},
    "user":{
        "atNamn":String,
        "namn":String,
        "bild":String,
        "id":String
}
});

Some of the tweets are duplicates and therefore share the same "id" attribute. I want to filter these out when adding. At the moment i just go through the JSON like this.

var JSONData = require("../public/data/jsondata")
for(key in JSONData){
    new tweetModel(JSONData[key]).save(function(err,doc){
        if(err){console.log(err)}
        else{
            console.log(doc);
        }
    })
}

If i run this one time, they ALL get added. If i do it one more time a duplicate error is thrown. I want it to check for duplicates BEFORE adding!

Refer: http://stackoverflow.com/questions/7267102/how-do-i-update-upsert-a-document-in-mongoose — BatScream, Sep 07 '14 at 11:55

score 1 · Accepted Answer · answered Sep 07 '14 at 14:16

1

Use a simple JS object to keep track of which id values you've already seen:

var seenIds = {};
for(key in JSONData){
    var json = JSONData[key];
    if (!seenIds[json.id]) {
        seenIds[json.id] = true;
        new tweetModel(json).save(function(err,doc){
            if(err){console.log(err)}
            else{
                console.log(doc);
            }
        });
    }
}

answered Sep 07 '14 at 14:16

JohnnyHK

305,182
66
621
471

Thank you! Ofcourse it's not a mongoose feature, the filtering can be done beforehand. – Max Krog Sep 08 '14 at 13:32

Filter out duplicates on add to Mongoose MongoDB

1 Answers1