0

I getting a bit confuse about how to make post request one after one and not use up the server resource. I am trying to re-structure a huge amount of data and post it one by one to a new Mongoose database, I am using a mac pro to serve the gateway. Following are some of my code:

http.get(option, (res) => {
        
        res.on('data', function (chunk) {
            str += chunk;
        });

        
        res.on('end', async function() {
             var data = [];
             
            data = await JSON.parse(str);
            *// code is sucessfully till here, all data is received and parse properly*    
            data.forEach(item => {
                    var newData = item;
                    // console.log(newData);
                    postData(dataCat, newData);
    
                // console.log(item);
            })
            // console.log(str);
            });


            
        str = "";
    });
function postData(dataCat, newData) {
        
    // option = 'http://localhost:3008/pbapi/' + dataCat;
    // console.log(newData);

    var post_options = {
        host: '192.168.1.155',
        port: '3008',
        path: '/pbapi/' + dataCat,
        method: 'POST',
        headers: {
            'Content-Type': 'application/JSON'
        }
    };

    var post_req = http.request(post_options, function(res) {
        res.setEncoding('utf8');
        res.on('end', function (chunk) {
            console.log('Response: ' + chunk);
        });
    });

    post_req.write(newData);
    post_req.end();

}

** postData() is tested for single JSON object The forEach loop only work when the length of the data is below 2000(I only tested success till 2000 and failed at 2500).

Error code: 'ENFILE' If i put 2500 and above for the forEach loop

Jack Li
  • 1
  • 1
  • how i tested the allowrance: for (i=3000; i < 5000; i++ ) { postData(dataCat, JSON.stringify(data[i])); } – Jack Li Jul 04 '20 at 04:08
  • are you on a mac? – Karl L Jul 04 '20 at 04:18
  • Use ```Promise.all``` instead of sequential loop. Also in situations like this you need to get request and send a proper message to client: Hey i received your data and i'm going to process that. Then after process completion or failure send proper message to client through WebSocket or message brokers like google FCM or etc. – Vahid Alimohamadi Jul 04 '20 at 04:27
  • Just saying, ``JSON.parse()`` is not async. – Take-Some-Bytes Jul 04 '20 at 04:31

2 Answers2

0

If you want to make request one by one, you should wrap your post in Promise, and wait, when it done, after that do next Promise.

It calls promisification. You can read about that here

Alexander Aryutin
  • 66
  • 6
  • can you provide a sample based on my case? – Jack Li Jul 04 '20 at 10:17
  • still does not work when array size is above 2500 events.js:200 throw er; // Unhandled 'error' event ^ Error: connect ENOBUFS 192.168.1.155:3008 - Local (undefined:undefined) at internalConnect (net.js:913:16) at defaultTriggerAsyncIdScope (internal/async_hooks.js:305:12) at net.js:1004:9 – Jack Li Jul 11 '20 at 02:42
  • Emitted 'error' event on ClientRequest instance at: at Socket.socketErrorListener (_http_client.js:406:9) at Socket.emit (events.js:223:5) at emitErrorNT (internal/streams/destroy.js:92:8) at emitErrorAndCloseNT (internal/streams/destroy.js:60:3) at processTicksAndRejections (internal/process/task_queues.js:81:21) { errno: 'ENOBUFS', code: 'ENOBUFS', syscall: 'connect', address: '192.168.1.155', port: 3008 } – Jack Li Jul 11 '20 at 02:43
0
for (let i = 0, p = Promise.resolve(); i < data.length; i++) {
            p = p.then(_ => new Promise(resolve =>
                setTimeout(function () {
                    postData(dataCat, JSON.stringify(data[i]));
                    console.log(i);
                    resolve();
                }, Math.random() * 100)
            ));
        }

enter link description here

Jack Li
  • 1
  • 1