nodejs http.request how to collect data

Question

I am trying to invoke a rest API URL in one method and then use that response to form the next query to another URL, let's say git to query some information. I have seen numerous examples before posting this, in everyone people are simply printing the response to console or setting it to the server response.

I have tried using async/await but it just makes the whole thing really slow. So, I am not so keen on using that approach.

I am doing the callbacks but I am not able to collect data to actually loop over it. Here is my code snippet:-

function initialize(path) {
    // Setting URL and headers for request
    var options = {
        url: '',
        headers: {
            'Accept': '*/*',
            'User-Agent' : 'something something',
            'Authorization': 'Basic ' + new Buffer(process.env.username + ':' + process.env.password).toString('base64')
         },
         method: 'GET',
         port:443
    };
    options.url =  path;
    callback = function(err,response){
        var str= "";
        response.on('data',function(chunk){
            str+=chunk;
        });
        response.on('end',function(){
            return JSON.parse(str);
        });
    }
    http.request(options, callback).end();
}

And I call this from this method.

function collectData1(userid){
    var url = "example.com/users/user1";
    var results = initialize(url);
    console.log(results);
    return results;
}

But results always stays undefined.

So, how do I do this? Any suggestions?

Started getting this now. `events.js:136 throw er; // Unhandled 'error' event ^ Error: connect ECONNREFUSED 127.0.0.1:80 at Object._errnoException (util.js:1031:13) at _exceptionWithHostPort (util.js:1052:20) at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1195:14)` — MithunS, Jul 01 '18 at 02:02
Got through the port issue. I guess what I am looking for is a good example of either http.request or request.get. And I am not finding a decent example where the results are not straightaway returned back to the html or html-templating, without doing anything on the data itself. — MithunS, Jul 01 '18 at 03:13

score 3 · Accepted Answer · edited Aug 01 '20 at 11:07

function initialize(path, callback) {
// Setting URL and headers for request
var options = {
    host: "example.com"
    path: "/users/user1",
    headers: {
        'Accept': '*/*',
        'User-Agent': 'something something',
        'Authorization': 'Basic ' + new Buffer(process.env.username + ':' + process.env.password).toString('base64')
    },
    method: 'GET',
    port: 80
};

http.request(options, function (response) {
    response.setEncoding('utf8');
    var str = "";
    response.on('data', function (chunk) {
        str += chunk;
    });
    response.on('end', function () {
        callback(JSON.parse(str));
    });
}).end();
}

function collectData1(userid) {
    var url = "example.com/users/user1";
    initialize(url, function (result) {
        console.log(results);
        return results;
    });
}

Modified the code
Add response.setEncoding('utf8') inside request
http request is in port 80 and https request in 443
Refer http request docs

With your suggested code, I am getting this error. `TypeError [ERR_INVALID_ARG_TYPE]: The "chunk" argument must be one of type string or Buffer. Received type object` — MithunS, Jul 01 '18 at 05:41

Aypar Yılmazkaya · Answer 2 · 2018-07-01T04:54:06.907

There are three ways i can suggest;

Promises
Async / Await (which you have already tried)
Callback Functions (it's oldest way. Actually i don't suggest this :) )

Otherwise you can't return values directly from async functions.

Edit:

var http = require('http');

function initialize(path) {
    return new Promise(function (resolve, reject) {

        // Setting URL and headers for request
        var options = {
            hostname: "jsonplaceholder.typicode.com",
            path: path,
            // port:443,
            headers: {
                'Accept': '*/*',
                'User-Agent': 'something something',
                'Authorization': 'Basic ' + new Buffer(process.env.username + ':' + process.env.password).toString('base64')
            },
            method: 'GET',

        };

        var callback = function (response) {


            var str = "";
            response.on('data', function (chunk) {
                str += chunk;
            });

            response.on('end', function () {
                resolve(str);
            });

            request.on("error", function (error) {
                reject(error);
            })
        };

        var request = http.request(options, callback)
        request.on("error", function (error) {
            reject(error);
        });

        request.end();
    })
}

function collectData(userid) {
    var url = "/posts/" + userid;
    return initialize(url);
}


http.createServer(function (req, res) {


    collectData(1).then(function (data) {
        res.write(data);
        res.end();

    }).catch(function (error) {

        res.write(error.message);
        res.end();
    })


}).listen(8080);

How to use promises without async/await. Can you please post some sample code here? The callback `.then()` does not return values in time for me to process. And I keep getting undefined, thus I had to put await everywhere. — MithunS, Jul 01 '18 at 03:33
So, this is what I dont want to do, whatever I get back from promise, I dont just want to send that out to my display, I need to process it , (filter some values out, or use this response to make my next request). How would you do that ? — MithunS, Jul 01 '18 at 06:36

nodejs http.request how to collect data

2 Answers2