Handling asynchronous calls on queries in node.js

Question

I am little bit struggling in handling the asynchronus calls on queries, how can I get the response in proper order.

I have a user array, it has a payload of json objects, what I want to do is inserting the user and its details in different tables such as emails, pictures etc. Below is my pseudo code snippet

var jsonObj = {};
var jsonArray = [];
for (var i = 0; i < userArray.length; i++) {
    var userJSON = userArray[i];
    if (typeof userJSONJSON.list_of_emails != 'undefined') {
        conn.query('insert into user_tble set ?', userJSON, function (err, ins) {
            conn.query('insert into user_emails (user_id,email_address,) values ?', [globalEmailAddressArray], function (err, ins2) {
                conn.query('insert into user_phones (user_id,phone_number) values ?', [globalPhoneNumberArray], function (err, ins3) {
                    conn.query('insert into user_pictures (user_id,pic_url) values ?', ['http://localhost:300/somepicture'], function (err, ins4) {
                        jsonObj["result"] = "executed1";
                        jsonArray.push(jsonObj);
                        res.status(200).json(arr: jsonArray)
                    })
                })
            })
        });
    } else if (typeof userJSONJSON.list_of_phones != 'undefined') {
        conn.query('insert into user_phones (user_id,phone_number) values ?', [globalPhoneNumberArray], function (err, ins3) {
                    conn.query('insert into user_pictures (user_id,pic_url) values ?', ['http://localhost:300/somepicture'], function (err, ins4) {
                        jsonObj["result"] = "executed2";
                        jsonArray.push(jsonObj);
                        res.status(200).json(arr: jsonArray)
                    })
                })
    }   
}

if i give a payload something like

{
   "pay_load":[ 
       { 
           "list_of_emails": [ 
               {"email":"user1@user1.com"} 
              ],
            "last_name": "",
            "first_name": "User1"

       },
  { 
      "list_of_email_addresses": [ 
          {"email":"user2@user2.com"} 
          ],
      "last_name": "",
        "first_name": "User2"

  },
  { 
      "list_of_email_addresses": [],
      "last_name": "",
        "first_name": "User3"

  }
    ]
}

If I execute the json code it returns e.g it returns me the output

{
 arr:[
      {
        result : "executed2"
      }
     ]
}

I want something like due to to asynchronous nature i think it skips out the remaining two.

{
 arr:[
      {
        result : "executed1"
      },
      {
        result : "executed1"
      },
      {
        result : "executed2"
      }
     ]
}

In short how can I handle the asynchronous calls to achieve the above output.

Answer 1

I'd personally recommend you look into using Promises. I've had great luck with bluebird and it simplifies the callback hell that node can turn into. However, if you'd rather stick with callbacks, check out the async library.

The problem you're seeing is that the for loop will continue to the next item without waiting for the callback. It will also exit without waiting for any callback. You want each iteration to run the query and then continue only once the query finishes.

Edit Additionally, you're seeing it return early because you send the response on the first call. You want to wait until the whole list has been populated and then send the response (see the Promises example below)

Here's an example of using async:

var async = require("async")

var userArray = ["mickey mouse", "donald duck", "goofy"];
async.eachSeries(userArray, function (user, cb) {
    doFakeQuery('insert into user_tble set ?', function (err, ins) {
            doFakeQuery('insert into user_emails (user_id,email_address,) values ?', function (err, ins2) {
                doFakeQuery('insert into user_phones (user_id,phone_number) values ?', function (err, ins3) {
                    doFakeQuery('insert into user_pictures (user_id,pic_url) values ?', function (err, ins4) {
                        console.log("Finished queries for " + user)
                        cb();
                    })
                })
            })
        });
}, function (err) {
    console.log("Finished async.map")
})

function doFakeQuery(query, callback) {
    console.log("Running query '" + query + "'")
    //Mock an asynchronous callback
    setTimeout(function () {
        callback();
    }, 300)
}

For comparison, here is your code rewritten with Promises.

var Promise = require("bluebird")
//Convert conn.query into a function that returns a promise, instead of a callback
var pQuery = Promise.promisify(conn.query)

var jsonArray = []

Promise.mapSeries(userArray, function (user) {
    if (user.list_of_emails != undefined) {
        return addUser(user)
            .then(addEmails)
            .then(addPhones)
            .then(addPictures)
            .then(function () {
                jsonArray.push({result: "executed1"})
            })
    }
    else if (user.list_of_phones != undefined) {
        return addPhones()
            .then(addPictures)
            .then(function () {
                jsonArray.push({result: "executed2"})
            })
    }
})
    .then(function () {
        console.log("Done!")
        res.status(200).json(jsonArray)
    })

function addUser(userJSON) {
    return pQuery('insert into user_tble set ?', userJSON)
}

function addEmails() {
    return pQuery('insert into user_emails (user_id,email_address,) values ?', [globalEmailAddressArray])
}

function addPhones() {
    return pQuery('insert into user_phones (user_id,phone_number) values ?', [globalPhoneNumberArray])
}

function addPictures() {
    return pQuery('insert into user_pictures (user_id,pic_url) values ?', ['http://localhost:300/somepicture'])
}