I have yet another "Can't set headers after they are sent." problem. I've created a post router which receives a large object of urls, for which i need processing.
The processing consists of crawling all of the urls the object contains. I've set a bottleneck, with a maximum of 10 concurrent crawls and a wait time of 800 milliseconds.
Either way, it is possible for the data to arrive, and log it, but when I'm calling my function, which is suppose to process the data, my application throws an exception.
Post Router
router.post('/crawledUrls', function (req, res, err) {
crawler = new crawlerClass();
var urlArray = JSON.parse(req.body);
crawler.crawlArrayUrls(urlArray);
res.send({message: "Array sent successfully!"});
});
crawler.crawlArrayUrls
crawlerClass.prototype.crawlArrayUrls = function (arrayObject) {
var itemsProcessed = 0;
var emptyUrlArray = [];
var newArray = [];
for(keys in arrayObject){
newArray.push(arrayObject[keys]);
}
var limiter = new Bottleneck(10, 800);
newArray.forEach(function (listItem, indexArray, err) {
limiter.submit(err, function () {
if(err) throw err;
request({
encoding: null,
method: "GET",
headers: {
'User-Agent': 'request'
},
uri: listItem
}, function (error, response, body, err) {
if(err) throw err;
$ = cheerio.load(body);
if($('div.alert.alert-danger').html() == undefined) {
console.log(listItem);
emptyUrlArray.push(listItem);
} else {
console.log("Else");
//Not done.
}
});
});
});
};
App.js
var express = require('express');
var session = require('express-session');
var path = require('path');
var favicon = require('serve-favicon');
var logger = require('morgan');
//var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');
var scheduleObject = require('./UtilityLogic/scheduleObject');
var scheduleBidding = require("./GoogleAPILogic/scheduleBidding");
var dbModules = require("./DBLogic/dbModules");
var routes = require('./routes/index');
var app = express();
// view engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'jade');
app.enable('trust proxy');
app.use(logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(session({
secret: 'secret',
resave: false,
saveUninitialized: true,
cookie: {
httpOnly: true,
secure: true
}
}));
app.use(express.static(path.join(__dirname, 'public')));
app.use('/', routes);
// catch 404 and forward to error handler
app.use(function(req, res, next) {
var err = new Error('Not Found');
err.status = 404;
next(err);
});
// error handlers
// development error handler
// will print stacktrace
if (app.get('env') === 'development') {
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: err
});
throw err;
});
}
// production error handler
// no stacktraces leaked to user
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: {}
});
});
var scheduler = new scheduleObject();
var dbObject = new dbModules("./somePath");
var scheduleBidObject = new scheduleBidding("someKey");
//Test connection to DB
dbObject.testConnectionToDB();
scheduler.scheduleFunction('59 59 23 * * *',function (){
scheduleBidObject.getBiddingHeaders('/SomePath/');
});
scheduler.scheduleFunction('01 e 10 00 * * *',function (){
scheduleBidObject.getBiddingHeaders('/SomePath/');
});
module.exports = app;
Exception thrown
Error: Can't set headers after they are sent.
www-2 at ServerResponse.OutgoingMessage.setHeader (http.js:691:11)
www-2 at ServerResponse.header (/somePath/node_modules/express/lib/response.js:719:10)
www-2 at ServerResponse.send (/somePath/node_modules/express/lib/response.js:164:12)
www-2 at done (/somePath/node_modules/express/lib/response.js:956:10)
www-2 at Object.exports.renderFile (/somePath/node_modules/jade/lib/index.js:374:12)
www-2 at View.exports.__express [as engine] (/somePath/node_modules/jade/lib/index.js:417:11)
www-2 at View.render (/somePath/node_modules/express/lib/view.js:126:8)
www-2 at tryRender (/somePath/node_modules/express/lib/application.js:639:10)
www-2 at EventEmitter.render (/somePath/node_modules/express/lib/application.js:591:3)
www-2 at ServerResponse.render (/somePath/node_modules/express/lib/response.js:960:7)
