Who nodejs http request fails when there are too much tasks?

Question

I should login and call http some api points by nodejs,

var mins = 150;
var url = require('url');
const domainim= 'example.com';
const request = require('requestretry').defaults({
        jar : true,
        timeout : 120000
    });
const req = require('request');
var fs = require('fs');
var linkler = [];
var timeoutsure = 120 * 1000;
var separateReqPool = {
    maxSockets : 20
};
let success = 0, fail = 0, toplam = 0, plusplus = 0;
let adet = 0;
let jardata = {};
let baslama = Math.round(new Date().getTime() / 1000);

function texts(l) {
    var t = "";
    var p = "abcdefghijklmnopqrstuvwxyz0123456789";
    for (var i = 0; i < l; i++) {
        t += p.charAt(Math.floor(Math.random() * p.length));
    }
    return t;
}

fs.readFile("data.txt", function (err, data) {
    if (err)
        throw err;
    data = data.toString().replace("\r", "");
    data = data.split(/\r\n|\r|\n/g);
    console.log("toplam data : " + data.length);
    toplam = data.length;
    data.forEach(
        function (line) {
        isle(line);
    });
});

function onExit() {
    console.log('\n%d sec,\n%d total,\n%d suc,\n%d fai,\n%d plusplus,\n---------------------\n',
        (Math.round(new Date().getTime() / 1000)) - baslama, toplam, success, fail, plusplus);
}
process.on('exit', onExit);

function dataok(id, pass, bakiye) {
    console.log(id + " " + pass + " " + bakiye);
    var fd = fs.openSync("hardblock.txt", 'a+');
    fs.writeSync(fd, id + " " + pass + " " + bakiye + "\r\n");
    fs.closeSync(fd);

}

function extradata(id, pass, bakiye) {
    console.log(id + " " + pass + " " + bakiye);
    var fd = fs.openSync("harddata.txt", 'a+');
    fs.writeSync(fd, id + " " + pass + " " + bakiye + "\r\n");
    fs.closeSync(fd);

}

function isle(line) {
    jardata[line.split(" ")[0]] = req.jar();
    request({
        url : 'http://'+domainim+'/',
        jar : jardata[line.split(" ")[0]],
        headers : {
            'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
        },
        maxAttempts : 15,
        retryDelay : 2000,
        followRedirect : true,
        //  proxy :"http://127.0.0.1:38936",
        timeout : timeoutsure,
        pool : separateReqPool
    }, function optionalCallback(err, httpResponse, body) {
        if (httpResponse && httpResponse.attempts) {
            adet += httpResponse.attempts;
        }
        if (err) {
            return console.error('failed: 49', err);
        } else {

            data = {
                username : line.split(" ")[0],
                password : line.split(" ")[1],
                submit : "Giriş"
            };
            request.post({
                url : 'http://'+domainim+'/dologin/',
                jar : jardata[line.split(" ")[0]],
                header : httpResponse.headers,
                headers : {
                    'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
                },
                maxAttempts : 15,
                retryDelay : 2000,
                followRedirect : false,
                followAllRedirects : false,
                //  proxy :"http://127.0.0.1:38936",
                timeout : timeoutsure,
                pool : separateReqPool,
                form : data
            }, function optionalCallback(err, httpResponse, body) {
                if (httpResponse && httpResponse.attempts) {
                    adet += httpResponse.attempts;
                }
                if (err) {
                    return console.error('failed: 49', err);
                } else {

                    request({
                        url : 'http://'+domainim+'/',
                        jar : jardata[line.split(" ")[0]],
                        headers : {
                            'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
                        },
                        maxAttempts : 15,
                        retryDelay : 2000,
                        followRedirect : true,
                        //  proxy :"http://127.0.0.1:38936",
                        timeout : timeoutsure,
                        pool : separateReqPool
                    }, function optionalCallback(err, httpResponse, body) {
                        if (httpResponse && httpResponse.attempts) {
                            adet += httpResponse.attempts;
                        }
                        if (err) {}
                        else {
                            if (body.indexOf('class="success"') > -1) {
                                dataok(line.split(" ")[0], line.split(" ")[1], body.split('class="succes" title="')[1].split('"')[0]);
                                success++;
                                if (parseFloat(body.split('class="succes" title="')[1].split('"')[0]) >= mins) {
                                    plusplus++;
                                    extradata(line.split(" ")[0], line.split(" ")[1], body.split('class="succes" title="')[1].split('"')[0]);
                                } else {

                                    request({
                                        url : 'http://'+domainim+'/refresh/',
                                        jar : jardata[line.split(" ")[0]],
                                        headers : {
                                            'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
                                        },
                                        maxAttempts : 15,
                                        retryDelay : 2000,
                                        followRedirect : true,
                                        //  proxy :"http://127.0.0.1:38936",
                                        timeout : timeoutsure,
                                        pool : separateReqPool
                                    }, function optionalCallback(err, httpResponse, body) {
                                        if (httpResponse && httpResponse.attempts) {
                                            adet += httpResponse.attempts;
                                        }
                                        if (err) {}
                                        else {
                                        if (parseFloat(body)>= mins){
                                        plusplus++;
                                        extradata(line.split(" ")[0], line.split(" ")[1], parseFloat(body));
                                        }
                                        }
                                    });
                                }
                            } else {
                                fail++;
                            }
                        }
                    });

                }
            });

        }
    });

}

when my data in data.txt is like 5,10,1000 code works without a problem.

but when data gets bigger like 15,000 or 50,000 or bigger

it only can http calls with %10 success rate. and others fails and give me errors like this :

failed: 49  Error: connect EADDRINUSE 103.253.186.202:80
    at Object._errnoException (util.js:1031:13)
    at _exceptionWithHostPort (util.js:1052:20)
    at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1195:14)
  errno: 'EADDRINUSE',
  code: 'EADDRINUSE',
  syscall: 'connect',
  address: '1.2.3.4',
  port: 80 

what may be i doing wrong? is not nodejs capable of to do this?

and jar doesn't work as expected,it sometimes uses other's sessionas mistake

Answer 1

You should not be trying to make 15,000 http calls at once. That's just too much for your server to handle in terms of simultaneous resources used and if the requests are all going to the same host, that's probably more than that single host can handle at once anyway. I would guess that EADDRINUSE is just what you're getting because some resource has been exhausted when you attempt too many simultaneous requests.

Instead, you should run some much smaller number of requests at once (you can experiment with best performance of somewhere between 10 and 50). There are a number of other answers that show how to do that. Some use external libraries like Bluebird or Async and others code it manually. Here are several examples:

Run 1000 requests so that only 10 runs at a time

How to make millions of parallel http requests from nodejs app?

Node js socket explanation

Error: connect ETIMEDOUT when scraping

Make several requests to an API that can only handle 20 request a minute

Promise.all consumes all my RAM