Can't set headers after they are sent error with async

Question

I'm trying a pretty complex computation in my code below. I'm trying to get the list of bugs from github in the given project using the api https://api.github.com/repos/marklogic/java-client-api/issues?page=1&per_page=10. From the list of bugs I'm trying to get each issues' corresponding events and comments from their corresponding endpoints ex: https://api.github.com/repos/marklogic/java-client-api/issues/291/events and https://api.github.com/repos/marklogic/java-client-api/issues/291/comments. I'm using async library. I'm using waterfall function and parallel function to return a consolidated JSON for each bug such that each issue will have comment, & events in the same response for each issue. The problem is its throwing Can't set headers after they are sent error & its pointing to line 2 lines, I understand what the error is saying but I can't figure out how to fix it, because commenting out either of the offending lines results in request hang because the server is not sending the response. Please help! Thanks in advance

exports.listGitHubBugs = function(req, res) {
    var _page = req.query.page || 1;
    var _per_page = req.query.per_page || 25;
    var finalResult = []
        //console.log('url:', 'https://api.github.com/repos/marklogic/' + req.query.project + '/issues?page=' + _page + '&per_page=' + _per_page);
    var options = {
        url: 'https://api.github.com/repos/marklogic/' + req.query.project + '/issues?page=' + _page + '&per_page=' + _per_page,
        headers: {
            'User-Agent': req.query.project
        },
        auth: githubAuth
    };
    request(options, function(error, response, body) {
        if (error) {
            console.log(error);
            res.send(error);
        }

        if (!error && response.statusCode === 200) {
            var issues = JSON.parse(response.body)

            async.waterfall([
                 // get comments & events for all bugs and then send the response
                function(callback) {
                    issues.forEach(function(issue) {
                          // for each bug, get comments and events
                        async.parallel([

                            function(parallelCallback) {
                                var options = {
                                    url: issue.events_url,
                                    headers: {
                                        'User-Agent': getProjectNameFromURL(issue.events_url)
                                    },
                                    auth: githubAuth
                                };
                                request(options, function(error, response, body) {
                                    if (error) {
                                        console.log('ERROR', error);
                                        parallelCallback(error)
                                    }
                                    if (!error && response.statusCode === 200) {
                                        // console.log('events:', body);
                                        parallelCallback(null, body)

                                    }


                                })
                            },
                            function(parallelCallback) {
                                var options = {
                                    url: issue.comments_url,
                                    headers: {
                                        'User-Agent': getProjectNameFromURL(issue.comments_url)
                                    },
                                    auth: githubAuth
                                };
                                request(options, function(error, response, body) {
                                    if (error) {
                                        console.log('ERROR', error);
                                        parallelCallback(error)
                                    }
                                    if (!error && response.statusCode === 200) {
                                        //  console.log('comments:', body);
                                        parallelCallback(null, body)
                                    }


                                })
                            }
                        ], function(err, result) {
                            if (err) {
                                console.log('ERROR:', err);
                                callback(err);
                            }
                            console.log('parallel process done');
                            issue.events = JSON.parse(result[0]);
                            issue.comments = JSON.parse(result[1]);
                            finalResult.push(issue)
                            callback(null, finalResult) // offending line#1
                        })
                    }) // forEach end
                }
            ], function(err, result) {
                if (err) {
                    res.send(err);
                }
                console.log('waterfall done');
                console.log(result);
                res.send(result); // offending line#2
            })

        } // if end
    }) // reqest end

}

Error

UncaughtException: Can't set headers after they are sent.
ERROR Error: Can't set headers after they are sent.
    at ServerResponse.OutgoingMessage.setHeader (http.js:691:11)
    at ServerResponse.res.set.res.header (/Users/sreddy/space/angularjs/BugTrack/node_modules/express/lib/response.js:524:10)
    at ServerResponse.res.send (/Users/sreddy/space/angularjs/BugTrack/node_modules/express/lib/response.js:125:10)
    at ServerResponse.res.json (/Users/sreddy/space/angularjs/BugTrack/node_modules/express/lib/response.js:191:15)
    at /Users/sreddy/space/angularjs/BugTrack/server/api/common/common.controller.js:163:33
    at /Users/sreddy/space/angularjs/BugTrack/server/api/common/common.controller.js:153:29
    at /Users/sreddy/space/angularjs/BugTrack/node_modules/async/lib/async.js:254:17
    at done (/Users/sreddy/space/angularjs/BugTrack/node_modules/async/lib/async.js:135:19)
    at /Users/sreddy/space/angularjs/BugTrack/node_modules/async/lib/async.js:32:16
    at /Users/sreddy/space/angularjs/BugTrack/node_modules/async/lib/async.js:251:21

Final working code

exports.listGitHubBugs = function(req, res) {
var _page = req.query.page || 1;
var _per_page = req.query.per_page || 25;
var finalResult = []
    //console.log('url:', 'https://api.github.com/repos/marklogic/' + req.query.project + '/issues?page=' + _page + '&per_page=' + _per_page);
var options = {
    url: 'https://api.github.com/repos/marklogic/' + req.query.project + '/issues?page=' + _page + '&per_page=' + _per_page,
    headers: {
        'User-Agent': req.query.project
    },
    auth: githubAuth
};
request(options, function(error, response, body) {
    if (error) {
        console.log(error);
        return res.send(error);
    }

    if (!error && response.statusCode === 200) {
        var issues = JSON.parse(response.body)

        async.waterfall([
            // get events and comments for all bugs and return the final processes list of bugs
            function getEventsAndCommentsForAllBugs(callback) {
                issues.forEach(function getEventsAndComments(issue, index) {
                    // for each bug, get comments and events
                    async.parallel([

                        function getEvents(parallelCallback) {
                            var options = {
                                url: issue.events_url,
                                headers: {
                                    'User-Agent': getProjectNameFromURL(issue.events_url)
                                },
                                auth: githubAuth
                            };
                            request(options, function(error, response, body) {
                                if (error) {
                                    console.log('ERROR', error);
                                    parallelCallback(error)
                                }
                                if (response.statusCode === 200) {
                                    // console.log('events:', body);
                                    parallelCallback(null, body)

                                }


                            })
                        },
                        function getComments(parallelCallback) {
                            var options = {
                                url: issue.comments_url,
                                headers: {
                                    'User-Agent': getProjectNameFromURL(issue.comments_url)
                                },
                                auth: githubAuth
                            };
                            request(options, function(error, response, body) {
                                if (error) {
                                    console.log('ERROR', error);
                                    parallelCallback(error)
                                }
                                if (response.statusCode === 200) {
                                    //  console.log('comments:', body);
                                    parallelCallback(null, body)
                                }


                            })
                        }
                    ], function attachEventsAndComments(err, result) {
                        if (err) {
                            console.log('ERROR:', err);
                            callback(err);
                        }
                        console.log('parallel process done');
                        issue.eventList = JSON.parse(result[0]);
                        issue.commentList= JSON.parse(result[1]);
                        finalResult.push(issue)
                        if (index === (issues.length - 1)) {
                            callback(null, finalResult)
                        }
                        //
                    })
                }) // forEach end

            }
        ], function processedBugs(err, result) {
            if (err) {
                res.send(err);
            }
            console.log('waterfall done');
            console.log(result);
            res.send(result);
        })

    } // if end
}) // reqest end
}

Answer 1

could you provide a complete working example of the code, something we can try.

this said, there are several errors in this source code.

Onthe first request, if an error occurs, you write it in app.response,, but you don t stop execution. Thus, if an error occurs, you ll write twice the response object.

You should do if (error) { console.log(error); return res.send(error); }

instead of if (error) { console.log(error); res.send(error); }

Then, this can be changed if (!error && response.statusCode === 200) { to if (response.statusCode === 200) {

Same mistake occurs while fetching issues events and comments, please consider to fix it.

And also in the final callback of async.//

And in the final callback of async.waterfall.

finally, i suggest you to make use of named functions. That would help you to debug by providing more meaningfull error stack trace.

For example instead of doing, async.prallel([function(){/* code here*/}]);

You would write async.parallel([function nameOfTheTask(){/* code here*/}]);

consider also to use a linter such as eslint, several missing ; could break your code, see http://eslint.org/

Answer 2

I figured it out. I was calling the parellelCallback() for every iteration instead of calling it at the end of the loop. Simple if condition was all that was required.

if (index === (issues.length-1)) {
    callback(null, finalResult)
  }