Why is my express app returning an empty array?

Question

I am creating a CRUD api with express and mongodb. I have a specific route which queries one collection in my mongo db and retrieves whatever documents match the query criteria. My program then loops through these documents and trys to find the latest cross entry in another collection in my db

exports.findLatestCommitforAllRepos = function(req,res,next){


var githubCommitDataCollection = index.db.collection(githubCommitDataCollectionName);
var enabledRepoCollection = index.db.collection(enabledRepoCollectionName);
var latestCommits = []; 

enabledRepoCollection.find({enabled:true}).toArray(function(err,repos) {
    if (err) { next(err); }
    if (repos.length === 0 || repos === 'undefined') {
            res.status(404);
            res.send("There are no repos being tracked")
    }
    else {
        repos.forEach(function(enabledRepo) {
            var repo = enabledRepo.repo;
            var userOrOrg = enabledRepo.userOrOrg;

            githubCommitDataCollection.find({repo: repo, userOrOrg:userOrOrg}).sort({commitDate: -1}).limit(1).toArray(function(err,commit) {
                if (commit.length === 0 || repos === 'undefined') {
                    res.send("No commit found for repo " + repo);
                }
                // console.log(commit[0]);
                latestCommits.push(commit[0]);
                console.log(latestCommits);

            });
        });
        res.setHeader('Content-Type', 'application/json');    
        res.status(200);
        res.json(latestCommits);
        res.end();  
    }

 }); 
}

This results in an empty array being returned.

This looks like its a classic case of issuing an async request in a loop and expecting all the async operations to be done when the loop is done, but none of them have actually finished yet, thus your data structure is still empty. You will need to use something that tracks your async operations so you know when they are all done. This can be done with promises, something like the async library, or your own manual counters. — jfriend00, Mar 07 '16 at 16:30
Here's are examples of using a manual counter: http://stackoverflow.com/questions/35760831/how-to-do-first-n-times-async-loops-then-one-time-function-in-callback-way/35760975#35760975 and http://stackoverflow.com/questions/27119280/how-to-collect-the-value-to-an-array-in-nodejs-loop/27119344#27119344 and http://stackoverflow.com/questions/29186700/node-js-callback-function-at-the-after-loop-has-ended/29186825#29186825 and http://stackoverflow.com/questions/28122225/wait-for-several-db-connections-before-starting-express-server/28122510#28122510 — jfriend00, Mar 07 '16 at 16:32
Here's are examples of both a manual counter solution and a promise solution: http://stackoverflow.com/questions/34808482/calling-a-function-having-callback-in-for-loop/34814720#34814720 and http://stackoverflow.com/questions/33815807/node-js-request-for-loop-runs-twice/33816588#33816588 — jfriend00, Mar 07 '16 at 16:40

score 0 · Answer 1 · answered Mar 07 '16 at 16:34

One minor suggestion in code, use .findOne instead of .find

Means instead of

githubCommitDataCollection.find({repo: repo, userOrOrg:userOrOrg}).sort({commitDate: -1}).limit(1).toArray(function(err,commit) {

use

githubCommitDataCollection.findOne({repo: repo, userOrOrg:userOrOrg}).sort({commitDate: -1}).exec(function(err,commit) {

It will return only one commit and check console.log(commit) value to check what your are getting as result.

Or Please check share existing documents of githubCommitDataCollection

score 0 · Accepted Answer · answered Mar 07 '16 at 19:49

You can use the async libary especially the async.waterfall() method when you need to run a tasks array of functions in series, each passing their results to the next in the array.

Consider the following example:

// Include the async package
// Make sure you add "async" to your package.json
async = require("async");

exports.findLatestCommitforAllRepos = function(req,res,next){
    var latestCommits = [];
    async.waterfall([

        // Load all documents 
        function(callback) {
            index.db.collection(enabledRepoCollectionName).find({"enabled": true}).toArray(function(err,repos){
                if (err) return callback(err);                  
                callback(null, repos);
            });
        },

        // Get count of documents where price is empty 
        function(reposData, callback) {
            async.each(reposData, function(enabledRepo, callback) {
                index.db.collection(githubCommitDataCollectionName)
                        .findOne({repo: enabledRepo.repo, userOrOrg: enabledRepo.userOrOrg})
                        .sort({commitDate: -1}).limit(1)
                        .exec(function(err, commit) {
                            latestCommits.push(commit);
                            callback();
                        });
            }, callback);               
        }
    ], function(err, result) { //This function gets called after the three tasks have called their "task callbacks"
        if (err) return next(err);
        res.setHeader('Content-Type', 'application/json');    
        res.status(200);
        res.json(latestCommits);
        res.end();
    });
});

Why is my express app returning an empty array?

2 Answers2