Read raw http message in Nodejs

Question

I'm sending an http request using the http.request function, and I would like to read the whole http response like text; that is, the raw http protocol text. Is it possible? I've written the below code but it's not working.

// Set up the request
console.log('Sending request');
var post_req = http.request(post_options, function(res) {
    res.setEncoding('utf8');
    console.log('Response statusCode: ' + res.statusCode);
//    res.on('data', function (chunk) {
//        console.log('Response: ' + chunk);
//    });
//    res.on('end', function() {});
});

post_req.on('socket', function (socket) {
    var response = "";
    socket.on('data', function(chunk){
    console.log(chunk);
    });
});

// post the data
post_req.write(post_data);
post_req.end();

Answer 1

If you want access to the raw http message, I'd suggest using the net module instead, and writing the request yourself. Something like this for a simple GET request:

var net = require('net');

var host = 'stackoverflow.com',
    port = 80,
    socket = net.connect(port, host, function() {

    var request = "GET / HTTP/1.1\r\nHost: " + host + "\r\n\r\n",
        rawResponse = "";

    // send http request:
    socket.end(request);

    // assume utf-8 encoding:
    socket.setEncoding('utf-8');

    // collect raw http message:
    socket.on('data', function(chunk) {
        rawResponse += chunk;
    });
    socket.on('end', function(){
        console.log(rawResponse);
    });


});

For a POST request sending application/x-www-form-urlencoded data, you could write the request using something like:

function writePOSTRequest (data, host, path) {
    return "POST " + path + " HTTP/1.1\r\n" +
            "Host: " + host + "\r\n" +
            "Content-Type: application/x-www-form-urlencoded\r\n" +
            "Content-Length: " + Buffer.byteLength(data) + "\r\n\r\n" +
            data + "\r\n\r\n";

}

var data = "name1=value1&name2=value2",
    request = writePOSTRequest(data, host, "/path/to/resource");

where I'm using Buffer.byteLength because Content-Length requires the length in bytes, not in characters. Also, remember that data must be URL encoded.

If you don't know much about the format of HTTP messages, then this is a decent place to start:

http://jmarshall.com/easy/http/

Also, if you don't know what the encoding of the response will be then you'll have to parse the headers first to find out, but UTF-8 is by far the most common so it's a pretty safe bet.

Answer 2

Streams2 and Streams1 not always able to inter-operate well, see "problem: streams1 and streams2 duality" in this video.

I tried to listen data at a bit lower level than streams and this code prints raw http response with headers for me:

var http = require('http');
var raw = '';

console.log('Sending request');
var req = http.request({host: 'stackoverflow.com'}, function(res) {
  watch(res, 'res');
  res.on('end', function() {
    console.log(raw);
  });
  res.on('data', function(data) {
    // if we don't attach 'data' handler here 'end' is not called
  });
});


req.on('socket', function (socket) {
  socket.resume();
  var oldOndata = socket.ondata;
  socket.ondata = function(buf, start, end) {
    raw += buf.slice(start, end).toString();
    oldOndata.call(socket, buf, start, end);
  };
});

req.end();

Answer 3

Assuming these kind of tools are allowed in your environment, you could run up an HTTP debug proxy such as Fiddler http://www.fiddler2.com/, which enables you to inspect the HTTP calls and responses.