Node js socket explanation

Question

I am building an application that will be making about a million calls to a remote api server. Will I be able to limit amount of connections to for example 10? Do I set max sockets to 10 will do it?

I am trying to understand what do these parameters do:

keepAlive: false,
maxSockets: 999,
maxFreeSockets: 1

In node http get function, in the following code:

var inputData = [];

for(i=1; i<=5000;i++){
    inputData.push('number' + i);
}

var options = {
    host: "localhost",
    port: 80,
    path: "/text.txt",
    keepAlive: false,
    maxSockets: 999,
    maxFreeSockets: 1
}


var limit = inputData.length;
var counter = 0;

function fetchData(number){

    return new Promise(function(resolve, reject){
        var http = require('http');

        fetch  = function(resp){
            var body = '';
            resp.on('data',function(chunk){
                body += chunk;
            })
            resp.on('end',function(){
                console.log(resp)
                resolve()
            })
            resp.on('error',function(err){
                console.log('error');
            })
        }
        var req = http.request(options, fetch);

        req.end();

    })
}



Promise.all(inputData.map(number => fetchData(number))).then(function(results) {
    console.log('finished');
    connection.end();

})
.catch(function(error) {
    console.log('there wa an error');
    console.log(error);
});

Have you tried reading the [documentation](https://nodejs.org/api/http.html)? — jcaron, Jan 15 '16 at 00:53
OFC. I'd like to find out what this means in real life. Explanation from docs doesn't tell me much.. — Adam, Jan 15 '16 at 00:57
The docs are actually quite explicit. "agent.maxSockets: By default set to Infinity. Determines how many concurrent sockets the agent can have open per origin. Origin is either a 'host:port' or 'host:port:localAddress' combination.". What is unclear in that? Do you have a specific scenario where things do not work as documented? — jcaron, Jan 15 '16 at 01:01
I need to make approx 1 million API calls and I need to limit amount of connection to 100 concurrent connections, not to DDOS the api server. Will setting this option do it? When they say Agent do they mean application? How is maxsockets different from maxFreeSockets, and how to put them together to achieve target as above? Also should I change keep alive flat? Will an implementation as above do the job? — Adam, Jan 15 '16 at 01:04
Again, this is all explicitly stated in the documentation. Note that according to that documentation, they're options of `Agent`, not `http.clientRequest`. If you don't trust the documentation (or your implementation), start with a limited number of requests and very small values for the limits, and monitor what happens using `netstat`, and slowly grow values. — jcaron, Jan 15 '16 at 01:33

jfriend00 · Accepted Answer · 2016-01-15T04:31:03.133

You really don't want to fire off 1,000,000 requests and somehow hope that maxSockets manages it to 100 at a time. There are a whole bunch of reasons why that is not a great way to do things. Instead, you should use your own code that manages the number of live connections to 100 at a time.

There are a number of ways to do that:

Write your own code that fires up 100 and then each time one finishes, it fires up the next one.
Use Bluebird's Promise.map() which has a built-in concurrency feature that will manage how many are inflight at the same time.
Use Async's async.mapLimit() which has a built-in concurrency feature that will manage how many are inflight at the same time.

As for writing code yourself to do this, you could do something like this;

function fetchAll() {
    var start = 1;
    var end = 1000000;
    var concurrentMax = 100;
    var concurrentCnt = 0;
    var cntr = start;
    return new Promise(function(resolve, reject) {

        // start up requests until the max concurrent requests are going
        function run() {
            while (cntr < end && concurrentCnt < concurrentMax) {
                ++concurrentCnt;
                fetchData(cntr++).then(function() {
                    --concurrentCnt;
                    run();
                }, function(err) {
                    --concurrentCnt;
                    // decide what to do with error here
                    // to continue processing more requests, call run() here
                    // to stop processing more requests, call reject(err) here
                });
            }
            if (cntr >= end && concurrentCnt === 0) {
                // all requests are done here
                resolve();
            }        
        }

        run();
    });

}

score 1 · Answer 2 · answered Jan 16 '16 at 00:26

I decided to use the async library.

Here is my complete solution to this:

var async = require('async')

var http = require('http');

var inputData = [];

for(i=1; i<=2000;i++){
    inputData.push('number' + i);
}

var options = {
    host: "o2.pl",
    path: "/static/desktop.css?v=0.0.417",
    port: 80
}

function fetchData(number, callback){

    return new Promise(function(resolve, reject){

        fetch  = function(resp){
            var body = '';
            resp.on('data',function(chunk){
                body += chunk;
            })
            process.stdout.write('.')

            callback()

            resp.on('error',function(err){
                console.log('error');
                console.log(err);

            })
        }
        var req = http.request(options, fetch);

        req.end();

    })
}

function foo(item, callback){

    return callback(false, 'foo');
}

async.mapLimit(inputData,100,fetchData,function(err, result){
    console.log('finished');
})

Thank you for your help.

Node js socket explanation

2 Answers2

Linked

Related