Making mongoose.js queries run synchronously

Question

I have two mongoose collections. The first stores a list of places, the second is visits to the places. My node code goes through and attempts to get the list of visits to each place and build a string that I output as JSON. The first query completes before the second ever starts - is there a way to make them run synchronously?

Answer 1

If you are using node.js then u should use https://github.com/caolan/async

when you have to fetch data from multiple collections you have to chain your queries multiple times.

It will make your code complex and difficult to read and no modularity. Use async to create modularity using mongodb and node.js

Example Code from my project :

var async = require('async');

var createGlobalGroup = function(socket, data) {
    async.waterfall(
    [
    /**
     * this function is required to pass data recieved from client
     * @param  {Function} callback To pass data recieved from client
     */

    function(callback) {
        callback(null, socket, data);
    },
    /**
     * Step 1: Verify User
     */
    verifyUser,
    /**
     * Step 2: Check User Access Rights And Roles
     */
    checkUserAccessRightsAndRoles,
    /**
     * Step 3: Create Project
     */
    createNewGlobalGroup], function(err, result) {
        /**
         * function to be called when all functions in async array has been called
         */
        console.log('project created ....')
    });
}
verifyUser = function(socket, data, callback) {
//do your query
    /**
     * call next function in series
     * provide sufficient input to next function
     */
    callback(null, socket, data, {
        "isValidUser": true,
    });
}

checkUserAccessRightsAndRoles = function(socket, data, asyncObj, callback) {
    //do your query
    if(condition) {
        callback(null, socket, data, {
            roles: result,
            "isValidUser": asyncObj.isValidUser,
            "userId": asyncObj.userId,
        });
    } else {
    //no call back
    }
}

var createNewGlobalGroup = function(socket, data, asyncObj, callback) {
//wanna stop then no callback
}

Answer 2

There is no native synchronous api for mongodb/mongoose queries (and your wouldn't want one in practicality). As WiredPrarie mentions, you should chain the queries, with the second one starting after the first completes and running a callback. Here is an example:

function findVisits(placesQuery,callback){
    Places.find(placesQuery).exec(function(err,places){
        if (err || !places.length){
            console.log('there was a problem');
            callback(err, null);
        }else{
            var visitQuery = ... //however you want to filter places
            Visits.find(visitQuery).exec(function(err2,visits){
                if (err2 || !visits.length){
                    console.log('there was a problem');
                    callback(err2,null);
                }else{
                    callback(null, visits)
                }
            });
        }
    });
}

Answer 3

If you're using Node 8.x you can utilize async/await: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await

await operator pauses execution of async function until the Promise is resolved and returns the value. This way your code will look more synchronous:

const query1 = MyModel.find({ name: /john/i }, null, { skip: 10 });
const result1 = await query1.exec();

const query2 = MyModel.find({ name: /john/i }, null, { skip: 100 });
const result2 = await query2.exec();

Queries will be executed in succession.

Answer 4

Old solution: Promises

Mongoose supports promises these days, so you can .then() your queries. For example:

app.get('/notifications', function (req, res, next) {
  Users.findOne({
    username: req.body.username,
    password: req.body.password,
  }).then(user => {
    if (!user) {
      res.json({success: false, message: "Username or password incorrect."});
      return;
    }

    return Notifications.find({
      user: user._id
    }).then(notifications => {
      res.json({success: true, notifications});
    });
  ).catch(error => {
    // Standard way to handle errors in express
    next(error);
    // Or custom handling
    //console.error(error);
    //res.json({success: false, error: error.message});
  });
});

New solution: async-await

Now that Javascript has async-await, you can use that, which will save a few lines, and flatten the code a bit:

app.get('/notifications', async (req, res, next) => {
  try {
    const user = await Users.findOne({
      username: req.body.username,
      password: req.body.password,
    });
    if (!user) {
      res.json({success: false, message: "Username or password incorrect."});
      return;
    }

    const notifications = await Notifications.find({
      user: user._id
    });
    res.json({success: true, notifications});
  } catch (error) {
    next(error);
  }
});

My preferred solution: Clean async-await

Personally, I dislike adding the async keyword to the express callback function, because that is not really supposed to be an async function: I am not intending to return a promise from it.

I prefer an explicit transition between synchronous and asynchronous code, using an IIAFE:

app.get('/notifications', (req, res, next) => {
  (async () => {
    const user = await Users.findOne({
      username: req.body.username,
      password: req.body.password,
    });
    if (!user) {
      res.json({success: false, message: "Username or password incorrect."});
      return;
    }

    const notifications = await Notifications.find({
      user: user._id
    });
    res.json({success: true, notifications});
  })().catch(error => {
    next(error);
  });
  // Remember to use () to call the async function!
  // You can also shrink the above to simply .catch(next);
});

Do not forget to catch the errors!

Whichever approach you use, if the async function returns an error (in the form of a rejected promise), and express does not handle that error, this is called an unhandled rejection, and Node may decide to crash your process!

Answer 5

To synchronize I used es6-promise.

var Promise = require('es6-promise').Promise
  , mongoose = require('mongoose')
  , Schema = mongoose.Schema;

// define schemas and models.
var placeSchema = new Schema({
        name: { type: String },
        memo: { type: String }
    })
  , Places = mongoose.model('place', placeSchema)
  , visitSchema = new Schema({
        placeName: { type: String }, // foreign key for place.
        visitor: { type: String },
        comment: { type: String }
    })
  , Visits = mongoose.model('visit', visitSchema);

// query for visits by visitor and place.
function findVisitsWithPlace(visitor, place) {
    return new Promise(function (resolve, reject) {
        Visits.find({
            visitor: visitor,
            placeName: place.name
        }, function (error, visits) {
            if (error) {
                reject(error);
                return;
            }

            // build a result object you want.
            // ()
            resolve({
                place: place,
                visits: visits
            });
        });
    });
}

// functions for node route.
module.exports = {
    // - access to "GET /placevisits/?visitor=Visitor-1".
    get: function (request, response) {
        var visitor = request.query.visitor;

        // - to get the places...
        Places.find({}, function (error, places) {
            Promise.all(places.map(function (place) {
                // - run the child queries with parent object...
                return findVisitsWithPlace(visitor, place);
            })).then(function (placeAndVisits) {
                // - and get result.
                // placeAndVisits have still contain visits empty.
                // exclude them.
                var result = [];
                placeAndVisits.forEach(function (placeandvisit) {
                    if (placeandvisit.visits.length != 0) {
                        result.push(placeandvisit);
                    }
                });
                response.json(result);
            });
        });
    }
};

and I got JSON like following.

[
    {
        "place": {
            "_id": "564e58a1dbed862155771d46",
            "name": "Place-A",
            "memo": "A memo for Place A."
        },
        "visits": [
            {
                "_id": "564e58cedbed862155771d49",
                "placeName": "Place-A",
                "visitor": "Visitor-1",
                "comment": "A comment for Place A by Visitor-1"
            },
            {
                "_id": "564e58dcdbed862155771d4a",
                "placeName": "Place-A",
                "visitor": "Visitor-1",
                "comment": "2nd visit. Again comment for Place A by Visitor-1"
            }
        ]
    },
    {
        "place": {
            "_id": "564e58afdbed862155771d47",
            "name": "Place-B",
            "memo": "A memo for Place B."
        },
        "visits": [
            {
                "_id": "564e58ebdbed862155771d4c",
                "placeName": "Place-B",
                "visitor": "Visitor-1",
                "comment": "A comment for Place B by Visitor-1"
            }
        ]
    }
]

Answer 6

Here is an alternate method for making pseudo synchronous requests using MongooseJS. The idea here is to create a queue of queries that need to be executed. Then create a function that is called recursively until the queue is exhausted. Once the queue is exhausted, the recursion stops and a response sent back for the original request. I am using Express Routes, so all of this code is encapsulated in my route handler. In this case an HTTP POST.

var express = require('express');
var router = express.Router();

//POST /auth/create
router.post('/create', function(req, res) {
    var queue = [
        {"schema": require('..\\models\\people.js'), "query": {username: req.body.username}},
        {"schema": require('..\\models\\members.js'), "query": {username: req.body.username}}
    ],
    retData = []; 

    var curTask = 0.


    function recurse()
    {   
        if(curTask < queue.length){
                var task = queue[curTask];
                task.schema.findOne(task.query, function(err, data){
                retData.push(err || data);
                curTask++;
                recurse();
            })
        }else{
            res.json(retData);
        }

    }

    recurse();

});



module.exports = router;

Answer 7

Here's what I ended up doing tonight.

mongoose.createConnection("mongodb://localhost:27017/chemresearch")
.then(async db => {
    const collection = db.collection("chemical");
    const all = collection.find({});

    while(all.hasNext()) {
        let chemical = await all.next();
        await work(chemical);
    }
});

The work method is just returning a promise.

const work = (chemical) => new Promise((resolve, reject) => {
    const casNumber = chemical.casRegistryNumber;
    analysis(casNumber, (error) => {
        error ? reject(error) : resolve(casNumber);
    })
});