Construct array in nodejs through for loop

Question

I am using sails (0.11.0) running on nodejs (6.9.1). I am trying to construct an array by filling it through for loop. I would send this completed array in response to the client. I have tried various methods as suggested by people here on Stack Overflow, for example

the discussion here suggested

for (var i = yearStart; i < yearEnd+1; i++) {
    arr.push(i);
}

On this discussion, it is suggested to use:

var array = calendars.map(function(item) {
    return item.id;
});

console.log(array);

Similarly I tried many methods but I am coming across the same issue that during the loop, the array gets filled but as soon as the loop is completed, the array gets empty because of asynchronous process and therefore I can not send the response. To tackle with this I tried checking the index inside the loop body and send response from inside the loop body itself through

var userArray = [];
_.each(users, function(user, index){
  MySQLConnector.query('CALL user_image (?)', [user.id], function(err, userImage){
    if(err){
      return res.json({"status":"some_error"});
    }else{
      userID = user.id
      userImageID = userImage[0][0].id;
      var userInfo = {
        userID: userID,
        userImageID: userImageID
      }
      userArray.push(userInfo)
      if(index == users.length - 1){
          res.json({selectedUsers: userArray});
      }
    }
  });
});

I am initiating an empty userArray and then iterate through users object where each element of the object is characterized by name user and an index. Through a MySQL query I am fetching the userImage object and in each iteration, I am creating an object called userInfo that consists of userID and userImageID. I am pushing this object into userArray. And after each iteratio of the for loop (_.each), I check if last index is reached. Once last index is reached, the final array is sent as response before loop body is complete.

Here too I have an issue that the array body is not always completely filled. The reason is due to asynchronous process, the index does not always follow the order 0,1,2,3,4,.... and it can start with any number and can jump to any index in the next iteration, for example the first index to start would be 4, the second would be 0, third would be 2 and so on. This sequence would be different for every time we run this for loop. For a user, it will appear to be a total random process. Therefore if users.length is 8, and current index is randomly 7 at third iteration, the condition index == users.length - 1 will be met and response will be sent just with an array consisting of 3 elements rather than 8.

Can someone suggest me a better and robust way to fill an array through the for loop in nodejs and send that array in response, so that all items are included in the array in their original order?

Answer 1

As you are using node js , it is better to use any promises library like bluebird or async to handle Async requests. The reason your loop is not working as expected is because as you've pointed out, due to async requests taking time to resolve for which _.each loop is not waiting.

Using bluebird, it can be done with Promise.map method which works as explained below from the documentaion :

Given an Iterable(arrays are Iterable), or a promise of an Iterable, which produces promises (or a mix of promises and values), iterate over all the values in the Iterable into an array and map the array to another using the given mapper function.

Promises returned by the mapper function are awaited for and the returned promise doesn't fulfill until all mapped promises have fulfilled as well. If any promise in the array is rejected, or any promise returned by the mapper function is rejected, the returned promise is rejected as well.

Hence, Using Promise.map your code can be updated like below :

var Promise = require("bluebird");     

return Promise.map(users, function(user, index){
  return MySQLConnector.query('CALL user_image (?)', [user.id], function(err, userImage){
    if(err){
      return Promise.reject({"status":"some_error"});
    }else{
      userID = user.id
      userImageID = userImage[0][0].id;
      var userInfo = {
        userID: userID,
        userImageID: userImageID
      }
      return userInfo;
    }
  });
})
.then(function (usersArray){
  res.json({selectedUsers: usersArray});
})
.catch(function (err){
  res.json(err);
});

Answer 2

You can execute loops with functions with callbacks synchronously using SynJS:

var SynJS = require('synjs');
var mysql      = require('mysql');
var connection = mysql.createConnection({
  host     : 'localhost',
  user     : 'tracker',
  password : 'tracker123',
  database : 'tracker'
});


function myFunction1(modules,connection,users) {
    var ret=[];
    for(var i=0; i<users.length; i++) {
        connection.query("SELECT CONCAT('some image of user #',?) AS userImage", [users[i]], function(err, rows, fields) {
              if (err) throw err;

              ret.push({
                  id: users[i],
                  image:    rows[0].userImage
              });
              modules.SynJS.resume(_synjsContext); // <-- indicate that callback is finished
        });
        SynJS.wait(); // <-- wait for callback to finish
    }
    return ret;
};

var modules = {
        SynJS:  SynJS,
        mysql:  mysql,
};

var users = [1,5,7,9,20,21];

SynJS.run(myFunction1,null,modules,connection,users,function (ret) {
    console.log('done. result is:');
    console.log(ret);
});

Result would be following:

done. result is:
[ { id: 1, image: 'some image of user #1' },
  { id: 5, image: 'some image of user #5' },
  { id: 7, image: 'some image of user #7' },
  { id: 9, image: 'some image of user #9' },
  { id: 20, image: 'some image of user #20' },
  { id: 21, image: 'some image of user #21' } ]