Update Entire Collection In MongoDB Node Js

Question

I have a program that receive a New file from FTP server after some specific time , I insert the new updated file's data into my database MongoDB the fields remain same , only data changes in new file.. Now the problem is that EVERY TIME I have to insert whole new collection into database, and the collection increases accordingly.Forexample- first time data is of 20 records , second time 40 and then 60 and so on .The thing I want to do is I want to check which field's data is updated in New file before inserting new FILE's data , and should only update these field's data in database instead of inserting whole new document.Does MONGOOSE or MONGODB provide solution for this , means IF I PASS A DATA AS PARAMETER , IT SOULD COMPARE MY EXISTING COLLECTION WITH MY NEW DATA and then Update Only UPDATED FIELDS ..Please help me i,m stuck , thanks :) . I,m using NODE JS ...

var c = new Client();
            var connectionProperties = {
                host: 'ABC',
                user: 'ABC',
                port: 'ABC',
                password: 'ABC',
            };
            c.connect(connectionProperties);
          c.on('ready', function () {
                c.get('path-to-excel-file', function (err, stream) {
                    if (err) throw err;
                    stream.once('close', function () {
                        const workBook = XLSX.readFile('Convertedfile.xlsx');
                        XLSX.writeFile(workBook, 'Convertedfile', { bookType: "csv" });
                        csv()
                            .fromFile("Convertedfile")
                            .then((jsonObj) => {
                                Model.collection.insert(jsonObj, function (err, docs) {
                                    if (err) {
                                        return console.error(err);
                                    } else {
                                        console.log("All Documents insterted");
                                    }
                                });
                            })
                        c.end()
                    });
                    stream.pipe(fs.createWriteStream('ConvertedFile.xlsx'))
                })
          })

Answer 1

It looks like you need upsert means update if document/record exists or insert/create it.

So this can be done either 1 document at a moment or in bulk, but it would need a query to find the matching document(s) first.

Since you've not provided any sample data so I can't write a sample code snippet for you but here is the link to get you started, for bulk: Bulk.find.upsert and for single document this thread is good : how-do-i-update-upsert-a-document-in-mongoose

Update: here is the mongodb bulk upsert in action:

const mongo = require('mongodb');
const MongoClient = mongo.MongoClient;

const client = new MongoClient('mongodb://127.0.0.1:27017', { useUnifiedTopology: true });
client.connect(async err => {
    if (err) {
        console.log('DB Connection Error ', err);
    } else {
        const db = client.db('node-cheat-db');

        // lets assume you've read all file contents and data is now ready for db operation
        let records = [
            {first: 'john', last: 'doe', email: 'johen@doe.com'},
            {first: 'jack', last: 'doe', email: 'jack@doe.com'},
            {first: 'jill', last: 'doe_updated', email: 'jill@doe.com'}
        ];

        // prepare bulk upsert so that new records are created and existing are updated
        let bulk = db.collection('users').initializeOrderedBulkOp();
        for (var i = 0; i < records.length; i++) {
            bulk.find({
        "email": records[i].email // at least 1 key should be treated as PK; in this example email is PK
        }).upsert(records[i]).replaceOne(records[i]);
        }
        bulk.execute(function (err,updateResult) {
                if (updateResult.result.ok != 1) {
                    console.log('Bulk Upsert Error');
                } else {
                    console.log(`Inserted: ${updateResult.result.nUpserted} and Updated: ${updateResult.result.nModified}`);
                }
        });
    }
});

sample output looks like:

Inserted: 0 and Updated: 3

Further Details:

Clone node-cheat bulk-update, run node bulk-update.js followed by npm install mongodb.