Express.js File Upload : 60s timeouts and HTTP connection timeout errors

Question

I'll explain very quickly what my applications does :

1. Client uploads a document
2. It gets processed
3. The server create's a snapshot image of the pdf
4. The server send's back the path to the snapshot image

Now the issue is that this whole process from step (1) through (4) takes more than 60s.

At first I had a code which used the old express.bodyParser(), but after 60s I would get the following error :

Error: Request aborted
    at IncomingMessage.onReqAborted (/DIR_TO_NODE/node/file-upload-error/node_modules/express/node_modules/connect/node_modules/multiparty/index.js:131:17)
    at IncomingMessage.EventEmitter.emit (events.js:92:17)
    at abortIncoming (http.js:1911:11)
    at Socket.serverSocketCloseListener (http.js:1923:5)
    at Socket.EventEmitter.emit (events.js:117:20)
    at TCP.close (net.js:466:12)

and the file in the OS's tmp directory was deleted.

So I decided to use the mulster middleware module that replaces express.multipart().

This change solved 1/2 of the problem because it removed the Express.js multipart error I would get earlier.

This means that my program keeps running now after 60s, which is good, but I still get an HTTP timeout from the client after those same 60s.

Now I know that browsers like Chrome have this fixes 60s timeout, but my question was the following :

• Is there anyway to prevent this HTTP timeout from happening after 60s ?

• If there isn't a way, how do other applications solve this problem ? I'm especially interested in cases where multiple concurrent users will have to fight for filesystem access.

Here a few test cases where I simulate a long processing by never answering the request with a res.send() and polling every 10s for the filenames in the tmp directory just to check when my file would disappear (sometimes 60s timeout, sometimes 120s timeout).

TEST CASE n°1 - with the legacy express.bodyParser() middleware :

var express = require('express');
var path = require('path');
var app = express();
var fs = require('fs');
var os = require('os');

var pid = process.pid.toString();
var interval;

app.use(express.bodyParser());

app.get('/', function (req, res) {
    res.send([
        '<form method="post" enctype="multipart/form-data">' +
            '<p>Image: <input type="file" name="pdf" /></p>' +
            '<p><input type="submit" value="Upload" /></p>' +
        '</form>'
    ].join(''));
});

app.post('/', function (req, res, next) {

    var file_path;
    if(req.files.pdf) {
        file_path = req.files.pdf.path;
    }
    else {
        file_path = os.tmpDir() + 'test';
    }

    var filename = path.basename(file_path);
    var dir = path.dirname(file_path);

    console.log('\n> PID: ' + pid);
    console.log('> Filepath: ' + dir);
    console.log('> Filename: ' + filename);

    var count = 0;
    clearInterval(interval);
    interval = setInterval(function () {
        count++;
        console.log('\n> Interval: ' + (count * 10) + ' seconds');

        var files = fs.readdirSync(dir);
        for(var i = 0; i < files.length; i++) {
            if(files[i].substr(0, pid.length) === pid) {
                console.log('\t-> File present: ' + files[i]);
            }
        }

    }, 10000);

});

app.listen(3000);
console.log('Express started on port 3000');

TEST CASE n°2 - with the mulster middleware :

var express = require('express');
var multer = require('multer');
var path = require('path');
var app = express();
var fs = require('fs');
var os = require('os');

var pid = process.pid.toString();
var interval;

app.use(express.json());
app.use(express.urlencoded());
app.use(multer({
    dest: os.tmpDir()
}));

app.get('/', function (req, res) {
    res.send([
        '<form method="post" enctype="multipart/form-data">' +
            '<p>Image: <input type="file" name="pdf" /></p>' +
            '<p><input type="submit" value="Upload" /></p>' +
        '</form>'
    ].join(''));
});

app.post('/', function (req, res, next) {

    var file_path;
    if(req.files.pdf) {
        file_path = req.files.pdf.path;
    }
    else {
        file_path = os.tmpDir() + 'test';
    }

    //var file_path = req.files.pdf.path;
    var filename = path.basename(file_path);
    var dir = path.dirname(file_path);

    console.log('\n> PID: ' + pid);
    console.log('> Filepath: ' + dir);
    console.log('> Filename: ' + filename);

    var count = 0;
    clearInterval(interval);
    interval = setInterval(function () {
        count++;
        console.log('\n> Interval: ' + (count * 10) + ' seconds');

        var files = fs.readdirSync(dir);
        for(var i = 0; i < files.length; i++) {
            if(files[i].split('.')[0].length === 32 || files[i].substr(0, pid.length) === pid) {
                console.log('\t-> File present: ' + files[i]);
            }
        }

    }, 10000);

});

app.listen(3000);
console.log('Express started on port 3000');

Answer 1

The problem looks very similar to this question. The 60 second timeout most likely comes from the underlying socket and can be fixed with:

app.post('/', function (req, res, next) { {
    // 10 minute timeout just for POST to /
    req.socket.setTimeout(10 * 60 * 1000);
    
    // ...

});

Answer 2

As far as I can see (in both cases) you have a route handler for the POST request, but in that handler you are not sending response. You should use res.send there too. Otherwise the user will not get response at all.