On many requests nodeJS hang up and promises won't resolve - especially on low bandwidth

Question

The specific code - multiple request in a loop hang out node especially on low bandwidth connection.

const baseRequest = request.defaults();
const specialRequest = baseRequest.defaults( {
    agent : false //// MUCH better but still errors sometimes happened (on high bandwidth connection - 1mB/s)
    //agent : false, pool : {maxSockets: 100} //// WORKS! - no errors on high bandwidth 1mB/s //// ERRORS - on low 50kB/s(down) 10kB/s(up) bandwidth connection - but moust of the requests are corectly served
    //agent : false, pool : {maxSockets: 500} //// ERRORS on high bandwidth 1mB/s //// ERRORS - on low 50kB/s(down) 10kB/s(up) bandwidth connection
    //agent : false, pool : {maxSockets: 500}, timeout : 5000 //// ERRORS on high bandwidth 1mB/s //// ERRORS - on low 50kB/s(down) 10kB/s(up) bandwidth connection
});

process.env.UV_THREADPOOL_SIZE = 128;

// promises array to store mutations (data)
var urlDataPromiseArray = [];
var sizeDataPromiseArray = [];

// here is the function (request) to grab the data:
function getData(url, i) {
    return new Promise(function (resolve) {
        var serverWork = 'serverWork/dataSave_' + i + '_.txt';
        var fs_cws_serverWork = fs.createWriteStream(serverWork, 'utf8');

        var requestData = specialRequest({url : url}, function(err, resp, data) {
            if (err || resp.statusCode >= 400) {
                if (err) {
                    console.log('something goes bad! - ' + err + ' --- ' + i);
                    sizeDataPromiseArray.push(getSize(i, 0));
                    return resolve(i + ' bad');
                }
                else {
                    console.log('something goes bad! - ' + i);
                    sizeDataPromiseArray.push(getSize(i, 0));
                    return resolve(i + ' bad');
                }
            }
            else if (!err && resp.statusCode === 200) {
                var dataLength = data.length;
                console.log('ok! - ' + i + ' - ' + dataLength);
                sizeDataPromiseArray.push(getSize(i, dataLength));
                return resolve(i + ' good - ' + dataLength);
            }
            else {
                console.log('need to check - ' + i);
                sizeDataPromiseArray.push(getSize(i, 0));
                return resolve(i + ' shit happens');
            }
        }).pipe(fs_cws_serverWork);
    })
}

// another function to calculate size of downloaded data
function getSize(i, size) {
    return new Promise(function(resolve) {
        resolve({
            [i]: size
        });
    });
}

// here we prepare the loop:
var loop = 1000;
for (var i = 0; i < loop; i++) {
    urlDataPromiseArray.push(getData('https://www.google.com', i));
    sizeDataPromiseArray.push(getSize(i, 0));
}

Promise.all(urlDataPromiseArray).then(function(url) {
    console.log();
    console.log('===============');
    console.log('===============');
    console.log('===============');
    console.log();
}).then(function() {
    return Promise.all(sizeDataPromiseArray).then(function(size) {
        size.forEach(function(item) {
            for (var key in item) {
                var value = item[key];
                console.log('--------');
                console.log(value + ' - size of request number: ' + key);
                console.log('--------');
            };
        });
    });
}).catch(function(err) {
    console.log('ERROR: ' + err);
});

console.log();
console.log('===============');
console.log('loop is finished - now we wait for async work to be done');
console.log('===============');
console.log();

ERRORS I get:

socket hang up

read ECONNRESET

connect ETIMEDOUT IP:PORT

ETIMEDOUT

ESOCKETTIMEDOUT

When I set agent : false this is MUCH better but still errors sometimes happen on high bandwidth connection - 1mB/s and there are lot of errors on LOW bandwidth [50kB/s(down) 10kB/s(up)] connection like: - socket hang up and read ECONNRESET and at the and all are connect ETIMEDOUT IP:PORT AND ->promises don't work<-

When I set agent : false, pool : {maxSockets: 100} it WORKS - no errors on HIGH bandwidth 1mB/s BUT there are ERRORS - on LOW 50kB/s(down) 10kB/s(up) bandwidth connection (but it is working better then with 'agent : false' only - moust of the requests are corectly served) like: - socket hang up and read ECONNRESET BUT promises resolve!

When I set agent : false, pool : {maxSockets: 500} it WORKS - no errors on HIGH bandwidth 1mB/s BUT there are ERRORS - on LOW 50kB/s(down) 10kB/s(up) bandwidth connection (actually this is much worse then 'pool : {maxSockets: 100}' option) like: - mainly socket hang up but also read ECONNRESET and at the and 'connect ETIMEDOUT IP:PORT' AND ->promises won't resolve<-

When I set agent : false, pool : {maxSockets: 500}, timeout : 5000 - there are errors on HIGH bandwidth (1mB/s) - on LOW 50kB/s(down) 10kB/s(up) bandwidth connection errors look like: - many ESOCKETTIMEDOUT and many ETIMEDOUT BUT promises resolve correctly

So the thing I want is promises to ALWAYS correctly resolve. All request must be FAST served (cause multiple clients connect to the server) but they CAN pass errors - I just need them to be fast - on low bandwidth, the can pass error only, but client must know that this is cause of low bandwidth of the server and must get this response ASAP.

agent : false, pool : {maxSockets: 100} - this is working very well on high bandwidth (all good, promises pass)

agent : false, pool : {maxSockets: 500}, timeout : 5000 - this is working best on low bandwidth (fast errors, promises are resolved)

How to merge them so it will works just fine on high bandwidth and on low - the last part is accepted like described above (errors) - but maybe there is nothing I can do better - and why promises do not work?

On many requests nodeJS hang up and promises won't resolve - especially on low bandwidth

0 Answers0

Linked