Nodejs send partial response

Question

There are 20,000 records in mongodb collection. I am exporting all these records in a csv. I am send partial response using this :

res.writeHead(200, {
                     "Content-Type": "application/csv",
                     "Content-disposition": "attachment; filename='import.csv'"
 });

res.write(data + '0', "binary");

Above code is executing in batch of 500. I am ending using this code when all records are processed.

if (++responseCount == loopCount) {
  res.end();
}

But I got this error :

Can't set headers after they are sent.

But I get the file downloaded with 500 records.

Here is my full code.

var exportData = function (req, res, next) {

var limit = 500;
var responseCount = 0;
var loopCount = 1;
var size = 30000;

//Get 500 records at one time
var getData = function (req, start, cb) {
    req.db.collection('items').find().skip(start).limit(limit).toArray(function (err, records) {
        if (err) throw err;
        cb(null, records);
    });
};

if (size > limit) {
    loopCount = parseInt(req.size / limit);

    if ((req.size % limit) != 0) {
        loopCount += 1;
    }
}

for (var j = 0; j < loopCount; j++) {

    getData(req, limit * j, function (err, records) {

        if (err) throw err;

        records.forEach(function (record) {
            //Process record one by one
        });

        res.write(records);

        if (++responseCount == loopCount) {
            res.setHeader('Content-type', 'application/csv');
            res.setHeader("Content-disposition", 'attachment; filename="import.csv"');
            res.end();

        }

    });
}
};

Can you show your full code? The problem is not in what you've put here. Ideally enough to run and see the issue (mocking in the data might be a good idea). — Aaron Dufour, Jul 13 '15 at 17:09
You are sending back the response before the batch operation is finished. — bluesman, Jul 13 '15 at 18:42
Problem is resolved now. http://stackoverflow.com/questions/31339652/nodejs-send-partial-response/31398864#31398864 — Rohit, Jul 14 '15 at 06:06
Obviously, you sent data before writing header. Anyway, why don't you mark your answer corrected, please. — Han, Jul 19 '15 at 16:55

score 2 · Answer 1 · answered Jul 10 '15 at 13:16

Why not just stream the data? You could use mongoose query.stream feature. An example from the docs:

// follows the nodejs 0.8 stream api
Thing.find({ name: /^hello/ }).stream().pipe(res)

The stream will take care of the data flow for you. As a node.js stream, you can also listen for events:

// manual streaming
var stream = Thing.find({ name: /^hello/ }).stream();

stream.on('data', function (doc) {
  // do something with the mongoose document
}).on('error', function (err) {
  // handle the error
}).on('close', function () {
  // the stream is closed
});

score 1 · Answer 2 · edited May 23 '17 at 10:34

1

This statement

res.writeHead(200, {
                     "Content-Type": "application/csv",
                     "Content-disposition": "attachment; filename='import.csv'"
 });

belongs to header section when you response is send.So,it send header.
res.end() also send header.So, in this way you sending header again.

Please refer this one stackoverlflow question

EDIT CODE :

data.pipe(resp);
resp.end();

Please refer this one for more pipe

edited May 23 '17 at 10:34

Community

1
1

answered Jul 10 '15 at 12:33

RIYAJ KHAN

15,032
5
31
53

I got error even when this condition is not executed. – Rohit Jul 10 '15 at 12:52

score 0 · Answer 3 · answered Jul 14 '15 at 06:08

var exportData = function (req, res, next) {

    res.writeHead(200, {
        'Content-Type': 'application/csv',
        'Content-Disposition': 'attachment; filename="import.csv"'
    });

    var limit = 500;
    var responseCount = 0;
    var loopCount = 1;
    var size = 30000;

    //Get 500 records at one time
    var getData = function (req, start, cb) {
        req.db.collection('items').find().skip(start).limit(limit).toArray(function (err, records) {
            if (err) throw err;
            cb(null, records);
        });
    };

    if (size > limit) {
        loopCount = parseInt(req.size / limit);

        if ((req.size % limit) != 0) {
            loopCount += 1;
        }
    }

    for (var j = 0; j < loopCount; j++) {

        getData(req, limit * j, function (err, records) {

            if (err) throw err;

            records.forEach(function (record) {
                //Process record one by one
            });

            res.write(records);

            if (++responseCount == loopCount) {
                res.end();
            }

        });
    }
};

When submiting answers, please add some explantion to your code. — bren, Jul 19 '15 at 22:51
Will write send the data instantly, as a partial response, or write the data to a buffer, waiting to be sent? — , Jun 05 '17 at 13:44

Nodejs send partial response

3 Answers3

Linked