Nodejs send partial response not working

Question

I need to send partial response from nodejs but this code is not working. At once it will fetch 500 records from the database and then process each record one by one. I want to send partial response from node.js. If i am storing data in an array then Buffer overflow error is occurred.

var exportData = function (req, res, next) {

    var limit = 500;
    var responseCount = 0;
    var loopCount = 1;
    var size = 30000;

    //Get 500 records at one time
    var getData = function (req, start, cb) {
        req.db.collection('items').find().skip(start).limit(limit).toArray(function (err, records) {
            if (err) throw err;
            cb(null, records);
        });
    };

    if (size > limit) {
        loopCount = parseInt(req.size / limit);

        if ((req.size % limit) != 0) {
            loopCount += 1;
        }
    }

    for (var j = 0; j < loopCount; j++) {

        getData(req, limit * j, function (err, records) {

            if (err) throw err;

            records.forEach(function (record) {
                //Process record one by one
            });

            res.write(records);

            if (++responseCount == loopCount) {
                res.setHeader('Content-type', 'application/csv');
                res.setHeader("Content-disposition", 'attachment; filename="import.csv"');
                res.end();

            }

        });
    }
};

you should try streaming the response instead. Unless you need to batch the requests together in groups of X, you can easily pipe each item from mongoose through your processing logic using `through`/`through2` and pipe that to the response. — Dan, Jul 13 '15 at 08:44
I'll give it a shot but it will still need modification on your end, likely, as I'm not testing this locally — Dan, Jul 13 '15 at 08:47
possible duplicate of [Nodejs send partial response](http://stackoverflow.com/questions/31339652/nodejs-send-partial-response) — Dan, Jul 14 '15 at 06:49

Dan · Answer 1 · 2015-07-13T12:18:19.053

0

From comments: You should try streaming. Your issue is likely because you are running out of memory loading so many records at once - you stated you get a overflow exception so this might be the case. Here's an example of streaming with express/http.Server. I can't guarantee it works because I've never used streaming with express/http.Server, but it should give you a starting point.

Note that anything you pipe to result must be a string.

var through = require('through');

function exportData(request, result) {
  var limit = 500;

  res.setHeader('Content-type', 'application/csv');
  res.setHeader('Content-disposition', 'attachment: filename="import.csv"');

  req.db.collection('items')
    .find()
    .limit(limit)
    .stream()
    .pipe(through(function(record) {
      var processedRecord = processRecord(record);
      this.write(processedRecord);
    }))
    .pipe(result);
}


function processRecord(record) {
  // process record one by one
  return record;
}

edited Jul 13 '15 at 12:18

answered Jul 13 '15 at 08:50

Dan

10,282
2
37
64

I got this error -> RangeError: Maximum call stack size exceeded – Rohit Jul 13 '15 at 10:53
Thats... impressive. What hardware are you testing this on? – Dan Jul 13 '15 at 11:23
Ubuntu with 4GB memory – Rohit Jul 13 '15 at 12:05
Please use the amended code - although I'm not entirely sure that will fix it. Call stack size exceeded would indicate that you are calling the same function over and over again. – Dan Jul 13 '15 at 12:23
Let us [continue this discussion in chat](http://chat.stackoverflow.com/rooms/83164/discussion-between-rohit-and-dan-pantry). – Rohit Jul 14 '15 at 05:44
Problem is resolved now http://stackoverflow.com/questions/31339652/nodejs-send-partial-response/31398864#31398864 – Rohit Jul 14 '15 at 06:11
1

Why did you ask this question twice? :S – Dan Jul 14 '15 at 06:49
Earlier both were posted differently, i changed it over the time. Should i remove the one? – Rohit Jul 14 '15 at 08:06

Nodejs send partial response not working

1 Answers1