How to make a for statement wait multiple mongoose queries?

Question

What I am currently trying to do is to display all the playlists with the songs in it. To do that I first find every playlists, then I do a for to loop through them all (in the same time I initialize globalArr and put the values then it will be sended as json because it's an API) and the problem is when I do another find in the loop(PlaylistSong.find or Song.find) well since it's asynchronous the find will be made when the for will be over, and I will have 0 results because they will take the value of increment when he will be at his maximum. I heard of async, I even googled but I really don't understand how to put through this code because it's a combination of for loops and async queries...

Thanks for your help.

router.get('/', function(req, res, next) {
    Playlist.find(function (err, playlists) {
        if (err) return next(err);
        /* Loop through every playlists */
        var globalArr = [];
        for (var increment = 0; increment < playlists.length; ++increment)
        {
            globalArr[increment] = [];
            globalArr[increment]["name"] = playlists[increment].name;
            /* Loop through every links between Songs and Playlist */
            PlaylistSong.find({idPlaylist: playlists[increment]._id}, function (err, songs) {
                if (err) return next(err);
                for (var songIncrement = 0; songIncrement < songs.length; ++songIncrement) {
                {
                    console.log("increment"+increment);
                    globalArr[increment][songIncrement] = [];
                    /* Getting the actual song by his ID */
                    Song.find({_id: song.idSong}, function (err, song) {
                        if (err) return next(err);
                        globalArr[increment][songIncrement]["name"] = songs[songIncrement].name;
                        globalArr[increment][songIncrement]["artist"] = songs[songIncrement].artist;
                        globalArr[increment][songIncrement]["picture"] = songs[songIncrement].picture;
                        globalArr[increment][songIncrement]["price"] = songs[songIncrement].price;
                        globalArr[increment][songIncrement]["file"] = songs[songIncrement].file;
                        globalArr[increment][songIncrement]["difficulty"] = songs[songIncrement].difficulty;
                        globalArr[increment][songIncrement]["downloaded"] = songs[songIncrement].downloaded;
                    });
                }

            }});
        }
        res.contentType('application/json');
        res.send(JSON.stringify(globalArr));
    });
});

score 1 · Accepted Answer · edited May 23 '17 at 12:09

See this question and the accepted answer: Simplest way to wait some asynchronous tasks complete, in Javascript?

It basically says to use the Async module, push all of your async function calls onto it and then use async.parallel() which gives you a callback when all of the async functions have completed.

I haven't tested it, but something like this seems like it might work:

var async = require('async');

var calls = [];

router.get('/', function(req, res, next) {
    Playlist.find(function (err, playlists) {
        if (err) return next(err);
        /* Loop through every playlists */
        var globalArr = [];
        for (var increment = 0; increment < playlists.length; ++increment)
        {
            (function() {
                var i = increment;
                calls.push(function(callback) {
                    globalArr[i] = [];
                    globalArr[i]["name"] = playlists[i].name;
                    /* Loop through every links between Songs and Playlist */
                    PlaylistSong.find({idPlaylist: playlists[increment]._id}, function (err, songs) {
                        if (err) return next(err);
                        for (var songIncrement = 0; songIncrement < songs.length; ++songIncrement) {
                        {
                            console.log("increment"+i);
                            globalArr[i][songIncrement] = [];
                            /* Getting the actual song by his ID */
                            Song.find({_id: song.idSong}, function (err, song) {
                                if (err) return next(err);
                                globalArr[i][songIncrement]["name"] = songs[songIncrement].name;
                                globalArr[i][songIncrement]["artist"] = songs[songIncrement].artist;
                                globalArr[i][songIncrement]["picture"] = songs[songIncrement].picture;
                                globalArr[i][songIncrement]["price"] = songs[songIncrement].price;
                                globalArr[i][songIncrement]["file"] = songs[songIncrement].file;
                                globalArr[i][songIncrement]["difficulty"] = songs[songIncrement].difficulty;
                                globalArr[i][songIncrement]["downloaded"] = songs[songIncrement].downloaded;
                            });
                        }
                        callback();
                    }});
                });
            })();
        }
        async.parallel(calls, function(err, result) {
            if (err) {
                // TODO: Handle error here
            }
            res.contentType('application/json');
            res.send(JSON.stringify(globalArr));
        });
    });
});

or if you don't want then to execute in parallel, you can use async.series() instead.

See this jsFiddle for a simplified example of your situation... https://jsfiddle.net/bpursley/fj22hf6g/

Still the same problem, when the function is called increment = 2 whereas it has to be 0 then 1 (I have 2 playlists) — Kangoo13, Apr 22 '16 at 20:35
I think it is because the loop is finishing before the functions are executed (defered execution) and increment is already complete. Try moving increment into a local variable and using that variable instead. I will edit my answer with what I'm talking about. — Brian Pursley, Apr 22 '16 at 22:26
OK, I updated the code example and also included a jsFiddle showing what I'm taking about. The trick is to get a new scope by using an IIFI and putting iteration into a local variable i so that it cannot be changed by the next iteration. I also forgot to put the callback on my previous example (I said I didn't run it). But check my jsfiddle and I think you will see a simple example of what I'm taking about. I don't think this is the ONLY way to solve the problem, but it is A way to solve it. — Brian Pursley, Apr 22 '16 at 22:49
Since you have multiple nested loops, you might have to do this in both the inner and outer loops. I would actually look at breaking this code up into functions to simplify it. My code example above is incomplete and probably won't work for you without making some modifications. To be honest I'm not really sure why you have to have two loops, one for increment and another for songIncrement, but you know your scenario better than I do. Just take a look at my jsFiddle though and you will see what I'm suggesting... https://jsfiddle.net/bpursley/fj22hf6g/ — Brian Pursley, Apr 22 '16 at 23:03

score 0 · Answer 2 · answered Apr 22 '16 at 19:27

Yeah, you should use async. I'll explain this in more detail later (son must go to bed...)

PlaylistSong.statics.findSongsByPlaylistId = function(id, done) {
  PlaylistSong.find({idPlaylist: id}, function(err, songs) {
    if (err) {
      done(err)
      return
    }
    var getSongsFns = songs.map(function(song) {
      return function(callback) {
        Song.find({_id: song.idSong}, callback)
      }
    })
    async.parallel(getSongsFns, done)
  })
}

router.get('/', function(req, res, next) {
    Playlist.find(function (err, playlists) {
        if (err) return next(err);
        var getSongsFns = playlists.map(function(playlist) {
          return function(callback) {
            PlaylistSong.findSongsByPlaylistId(playlist._id, callback)
          }
        })
        async.parallel(getSongsFns, function(err, songs) {
          if (err) {
            res.status(500).send()
            return
          }
          res.contentType('application/json');
          res.send(JSON.stringify(songs));
        }) 
    });
});

Well that's almost it, but there is the "name" field missing for the playlist (each playlist has a name), and there isn't the field name for the songs like I would like to have the songs : [here the result] you see ? `[ [ [ { "_id": "56d006faa96d4211007e7e50", "name": "Stressed out", "__v": 0, "updated_at": "2016-02-26T08:04:10.649Z", "created_at": "UDT" } ], [ { "_id": "ID", "name": "Test", "__v": 0, "updated_at": "CRT", "created_at": "UDT" } ] ], [] ]` — Kangoo13, Apr 22 '16 at 21:04

How to make a for statement wait multiple mongoose queries?

2 Answers2