Node.js: Memory usage keeps going up

Question

We are writing a script that reads a large set of JPG files on our server (infinite, since we have another process that keeps writing JPG files to the same directory) and send them to users' browsers as an MJPEG stream at fixed time interval (variable "frameDelay" in code below). This is similar to what an IP camera would do.

We found out the memory usage of this script keeps going up and always ends up being killed by the system (Ubuntu);

We have inspected this seemingly simple script many many times. Therefore I'm posting the code below. Any comments / suggestions are greatly appreciated!

app.get('/stream', function (req, res) {
    res.writeHead(200, {
        'Content-Type':'multipart/x-mixed-replace;boundary="' + boundary + '"',
        'Transfer-Encoding':'none',
        'Connection':'keep-alive',
        'Expires':'Fri, 01 Jan 1990 00:00:00 GMT',
        'Cache-Control':'no-cache, no-store, max-age=0, must-revalidate',
        'Pragma':'no-cache'
    });
res.write(CRLF + "--" + boundary + CRLF);

setInterval(function () {
    if(fileList.length<=1){
        fileList = fs.readdirSync(location).sort();
    }else{
        var fname = fileList.shift();
        if(fs.existsSync(location+fname)){
           var data = fs.readFileSync(location+fname);
            res.write('Content-Type:image/jpeg' + CRLF + 'Content-Length: ' + data.length + CRLF + CRLF);
            res.write(data);
            res.write(CRLF + '--' + boundary + CRLF);                   
            fs.unlinkSync(location+fname);
        }else{
            console.log("File doesn't find")
        }
    }
        console.log("new response:" + fname);
    }, frameDelay);
});

app.listen(port);
console.log("Server running at port " + port);

To facilitate the troubleshooting process, below is a stand-alone (no 3rd-party lib) test case.

It has exactly the same memory issue (memory usage keeps going up and finally got killed by the OS).

We believe the problem is in the setInterval () loop - maybe those images didn't get deleted from memory after being sent or something (maybe still stored in variable "res"?).

Any feedback / suggestions are greatly appreciated!

var http                = require('http');
var fs                  = require('fs');

var framedelay  = 40;
var port                = 3200;
var boundary    = 'myboundary';
var CR                  = '\r';
var LF                  = '\n';
var CRLF                = CR + LF;

function writeHttpHeader(res)
{
        res.writeHead(200,
        {
                'Content-Type': 'multipart/x-mixed-replace;boundary="' + boundary + '"',
                'Transfer-Encoding': 'none',
                'Connection': 'keep-alive',
                'Expires': 'Fri, 01 Jan 1990 00:00:00 GMT',
               'Cache-Control': 'no-cache, no-store, max-age=0, must-revalidate',
                'Pragma': 'no-cache',
        });

        res.write(CRLF + '--' + boundary + CRLF);
}

function writeJpegFrame(res, filename)
{
        fs.readFile('./videos-8081/frames/' + filename, function(err, data)
        {
                if (err)
                {
                        console.log(err);
                }
                else
                {
                        res.write('Content-Type:image/jpeg' + CRLF);
                        res.write('Content-Length:' + data.length + CRLF + CRLF);
                            res.write(data);
                        res.write(CRLF + '--' + boundary + CRLF);
                        console.log('Sent ' + filename);
                }
        });
}
http.createServer(function(req, res)
{
     writeHttpHeader(res)    
     fs.readdir('./videos-8081/frames', function(err, files)
     { 
         var i = -1;
         var sorted_files = files.sort();    
         setInterval(function()
         {
              if (++i >= sorted_files.length)
              {
                i = 0;
              }    
              writeJpegFrame(res, sorted_files[i]);
          }, framedelay);
        });
}).listen(port);
console.log('Server running at port ' + port);

Are you getting output at client? and what module are you using? — xiaoyi, Nov 12 '12 at 17:41
Do you mean to have `fileList` as a global variable (shared by all `/stream` requests)? Also, you should be using the async calls or this won't scale well at all. — JohnnyHK, Nov 12 '12 at 17:48
Is what you see a constant, steady 70 fps or something variable? — guillaume, Nov 12 '12 at 21:03
Thank you everybody for your quick feedback - really appreciate that. — user1673454, Nov 12 '12 at 21:26
The main issue is the memory usage. I edited my original post to include a simpler sample that doesn't use any 3rd party libs. — user1673454, Nov 12 '12 at 21:32
have you tried [finding memory leaks](http://stackoverflow.com/questions/4918557/debugging-memory-leaks-with-node-js-server)? — guillaume, Nov 12 '12 at 21:40

freakish · Answer 1 · 2012-11-12T22:53:17.660

Of course it leaks memory. You do

 setInterval(...)

on each request, but you never clean these intervals. Meaning that after ( for example ) 20 requests you have 20 intervals running in the background, which will run forever, even if clients/connections are long dead. One of the solutions is the following:

var my_interval = setInterval(function() {
    try {
        // all your code goes here
    } catch(e) {
        // res.write should throw an exception once the connection is dead
        // do the cleaning now
        clearInterval( my_interval );
    }
}, frameDelay);

req.on( "close", function() {
    // just in case
    clearInterval( my_interval );
});

which ensures that my_interval ( and all corresponding data ) will be cleaned once the connection is closed.

P.S. I advice using setTimeout instead of setInterval, because loading file may take more time then frameDelay which will cause problems.

P.S.2. Use asynchronous versions of fs functions. Entire power of Node.JS is in nonblocking operations and you are loosing the main advantage ( performance ) here.

also fs.unlinkSync should really be a cron job. – wayne Nov 13 '12 at 00:54 — wayne, Nov 13 '12 at 00:54

xiaoyi · Accepted Answer · 2012-11-13T03:57:53.930

There are a few things caused this

MJPEG is not designed to send high resolution motion pictures in a high frequency (25fps, in your case, may be ok for 320x240 frames, but not for 720p.) Just consider the output throughput of the payload 25fps*70KB = 1750KBps = 14Mbps which is higher than full HD video.
Node.js will cache the output in a buffer when the client is incapable to receive. Because you are sending to much data to client, so node saved them for you. That's why your memory usage never go down, and it's NOT memory leak. To detect if the output buffer is bulked up, check the return value of res.write().
setInterval() is OK to use, and will not cause any problem as long as the client keep the connection. But when the client is disconnected, you need to stop it. To do so, you need to monitor 'close' event.
You cannot maintain a stable fps by using MJPEG as it's not designed for this purpose, so no matter how hard you try, you cannot control the fps at client. But with carefully designed code, you can make the average fps stable by using setTimeout().

Here is the fixed code.

var http = require('http');
var fs = require('fs');
var framedelay = 40;
var port = 3200;
var boundary = 'myboundary';
var CR = '\r';
var LF = '\n';
var CRLF = CR + LF;

http.createServer(function(req, res) {

  var files = fs.readdirSync('./imgs');
  var i = -1;
  var timer;
  var sorted_files = files.sort();

  res.writeHead(200, {
    'Content-Type': 'multipart/x-mixed-replace;boundary="' + boundary + '"',
    'Transfer-Encoding': 'none',
    'Connection': 'keep-alive',
    'Expires': 'Fri, 01 Jan 1990 00:00:00 GMT',
    'Cache-Control': 'no-cache, no-store, max-age=0, must-revalidate',
    'Pragma': 'no-cache',
  });

  res.write(CRLF + '--' + boundary + CRLF);

  var writePic = function() {

    if (++i >= sorted_files.length)
      i = 0;

    var data = fs.readFileSync('./imgs/' + sorted_files[i]);
    res.write('Content-Type:image/jpeg' + CRLF);
    res.write('Content-Length:' + data.length + CRLF + CRLF);
    res.write(data);
    var ok = res.write(CRLF + '--' + boundary + CRLF);
    console.log('Sent ' + sorted_files[i], ok);

    if (ok)
      timer = setTimeout(writePic, framedelay);
  };

  res.on('close', function() {
    console.log('client closed');
    clearTimeout(timer);
  });

  res.on('drain', function() {
    console.log('drain');
    timer = setTimeout(writePic, framedelay);
  });

}).listen(port);

console.log('Server running at port ' + port);

Node.js: Memory usage keeps going up

2 Answers2