I'm reading the docs at http://nodejs.org/docs/v0.4.0/api/http.html#http.request, but for some reason, I can't seem to to actually find the body/data attribute on the returned, finished response object.
> var res = http.get({host:'www.somesite.com', path:'/'})
> res.finished
true
> res._hasBody
true
It's finished (http.get does that for you), so it should have some kind of content. But there's no body, no data, and I can't read from it. Where is the body hiding?
http.request docs contains example how to receive body of the response through handling data event:
var options = {
host: 'www.google.com',
port: 80,
path: '/upload',
method: 'POST'
};
var req = http.request(options, function(res) {
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers));
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log('BODY: ' + chunk);
});
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
// write data to request body
req.write('data\n');
req.write('data\n');
req.end();
http.get does the same thing as http.request except it calls req.end() automatically.
var options = {
host: 'www.google.com',
port: 80,
path: '/index.html'
};
http.get(options, function(res) {
console.log("Got response: " + res.statusCode);
res.on("data", function(chunk) {
console.log("BODY: " + chunk);
});
}).on('error', function(e) {
console.log("Got error: " + e.message);
});
I also want to add that the http.ClientResponse returned by http.get() has an end event, so here is another way that I receive the body response:
var options = {
host: 'www.google.com',
port: 80,
path: '/index.html'
};
http.get(options, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
console.log(body);
});
}).on('error', function(e) {
console.log("Got error: " + e.message);
});
The await keyword is the best way to get a response from an HTTP request, avoiding callbacks and .then()
You'll also need to use an HTTP client that returns Promises. http.get() still returns a Request object, so that won't work.
fetch is a low level client, that is both available from npm and nodeJS 17 onwards.superagent is a mature HTTP clients that features more reasonable defaults including simpler query string encoding, properly using mime types, JSON by default, and other common HTTP client features.axios is also quite popular and has similar advantages to superagentawait will wait until the Promise has a value - in this case, an HTTP response!
const superagent = require('superagent');
(async function(){
const response = await superagent.get('https://www.google.com')
console.log(response.text)
})();
Using await, control simply passes onto the next line once the promise returned by superagent.get() has a value.
The data event is fired multiple times with 'chunks' of the body as they are downloaded and an end event when all chunks have been downloaded.
With Node supporting Promises now, I created a simple wrapper to return the concatenated chunks through a Promise:
const httpGet = url => {
return new Promise((resolve, reject) => {
http.get(url, res => {
res.setEncoding('utf8');
let body = '';
res.on('data', chunk => body += chunk);
res.on('end', () => resolve(body));
}).on('error', reject);
});
};
You can call it from an async function with:
const body = await httpGet('http://www.somesite.com');
The body is not fully stored as part of the response, the reason for this is because the body can contain a very large amount of data, if it was to be stored on the response object, the program's memory would be consumed quite fast.
Instead, Node.js uses a mechanism called Stream. This mechanism allows holding only a small part of the data and allows the programmer to decide if to fully consume it in memory or use each part of the data as its flowing.
There are multiple ways how to fully consume the data into memory, since HTTP Response is a readable stream, all readable methods are available on the res object
listening to the "data" event and saving the chunks passed to the callback
const chunks = []
res.on("data", (chunk) => {
chunks.push(chunk)
});
res.on("end", () => {
const body = Buffer.concat(chunks);
});
When using this approach you do not interfere with the behavior of the stream and you are only gathering the data as it is available to the application.
using the "readble" event and calling res.read()
const chunks = [];
res.on("readable", () => {
let chunk;
while(null !== (chunk = res.read())){
chunks.push(chunk)
}
});
res.on("end", () => {
const body = Buffer.concat(chunks);
});
When going with this approach you are fully in charge of the stream flow and until res.read is called no more data will be passed into the stream.
using an async iterator
const chunks = [];
for await (const chunk of readable) {
chunks.push(chunk);
}
const body = Buffer.concat(chunks);
This approach is similar to the "data" event approach. It will just simplify the scoping and allow the entire process to happen in the same scope.
While as described, it is possible to fully consume data from the response it is always important to keep in mind if it is actually necessary to do so. In many cases, it is possible to simply direct the data to its destination without fully saving it into memory.
Node.js read streams, including HTTP response, have a built-in method for doing this, this method is called pipe. The usage is quite simple, readStream.pipe(writeStream);.
for example:
If the final destination of your data is the file system, you can simply open a write stream to the file system and then pipe the data to ts destination.
const { createWriteStream } = require("fs");
const writeStream = createWriteStream("someFile");
res.pipe(writeStream);
If you want to use .get you can do it like this
http.get(url, function(res){
res.setEncoding('utf8');
res.on('data', function(chunk){
console.log(chunk);
});
});
You need to add a listener to the request because node.js works asynchronous like that:
request.on('response', function (response) {
response.on('data', function (chunk) {
console.log('BODY: ' + chunk);
});
});
Just an improved version to nkron responce.
const httpGet = url => {
return new Promise((resolve, reject) => {
http.get(url, res => {
res.setEncoding('utf8');
const body = [];
res.on('data', chunk => body.push(chunk));
res.on('end', () => resolve(body.join('')));
}).on('error', reject);
});
};
Appending chunks in an string[] is better for memory usage, the join(''), will allocate new memory only once.
Needle module is also good, here is an example which uses needle module
var needle = require('needle');
needle.get('http://www.google.com', function(error, response) {
if (!error && response.statusCode == 200)
console.log(response.body);
});
A portion of Coffee here:
# My little helper
read_buffer = (buffer, callback) ->
data = ''
buffer.on 'readable', -> data += buffer.read().toString()
buffer.on 'end', -> callback data
# So request looks like
http.get 'http://i.want.some/stuff', (res) ->
read_buffer res, (response) ->
# Do some things with your response
# but don't do that exactly :D
eval(CoffeeScript.compile response, bare: true)
And compiled
var read_buffer;
read_buffer = function(buffer, callback) {
var data;
data = '';
buffer.on('readable', function() {
return data += buffer.read().toString();
});
return buffer.on('end', function() {
return callback(data);
});
};
http.get('http://i.want.some/stuff', function(res) {
return read_buffer(res, function(response) {
return eval(CoffeeScript.compile(response, {
bare: true
}));
});
});
You can't get the body of the response from the return value of http.get().
http.get() doesn't return a response object. It returns the request object (http.clientRequest). So, there isn't any way to get the body of the response from the return value of http.get().
I know it's an old question, but reading the documentation you linked to shows that this was the case even when you posted it.
