How to use request or http module to read gzip page into a string

Question

I found the request module in js cannot handle gzip or inflate format http response correctly.

for example:

request({url:'some url'}, function (error, response, body) {
   //if the content-encoding is gzip, the body param here contains binaries other than readable string. And even worse after you convert the body to buffer, u even can not gunzip it.
}

so I want to use the sample code in official docs.

var request = http.get({ host: 'izs.me',
                         path: '/',
                         port: 80,
                         headers: { 'accept-encoding': 'gzip,deflate' } });
request.on('response', function(response) {
  var output = fs.createWriteStream('izs.me_index.html');

  switch (response.headers['content-encoding']) {
    // or, just use zlib.createUnzip() to handle both cases
    case 'gzip':
      response.pipe(zlib.createGunzip()).pipe(output);
      break;
    case 'deflate':
      response.pipe(zlib.createInflate()).pipe(output);
      break;
    default:
      response.pipe(output);
      break;
  }
});

The problem is that the code is writing the webpage into a file, I hope it can write the page into a string, so that i can process the page. I could not find any class like 'StringStream'.

If anyone has any idea on this,it will be great.

Answer 1

Pipe the response to the gzip stream and use it as you would use the original response object.

var req = http.request(options, function(res) {
    var body = "";

    res.on('error', function(err) {
       next(err);
    });

    var output;
    if( res.headers['content-encoding'] == 'gzip' ) {
      var gzip = zlib.createGunzip();
      res.pipe(gzip);
      output = gzip;
    } else {
      output = res;
    }

    output.on('data', function (data) {
       data = data.toString('utf-8');
       body += data;
    });

    output.on('end', function() {
        return next(false, body);
    });
 });

req.on('error', function(err) {
   next(err);
})

Answer 2

simplified example:

var https = require('https');
var gunzip = require('zlib').createGunzip();

var options = {
    host: 'api.stackexchange.com',
    path: '/2.1/info?site=stackoverflow'
};

https.get(options, function(res) {
  var body = '';

  res.pipe(gunzip);

  gunzip.on('data', function (data) {
      body += data;
  });

  gunzip.on('end', function() {
      console.log(JSON.parse(body));
  });
});

Answer 3

I ran into a similar issue and wanted to continue using the request library instead of the built-in http module. I've discussed two working approaches here: http://nickfishman.com/post/49533681471/nodejs-http-requests-with-gzip-deflate-compression. One of them is similar to @Teemu's answer, while the other uses streams.

Answer 4

request module handles the gzip responses. All we have to do is to set 'gzip' attribute in the opts. For detailed explaination please visit the below linke. There I have clearly explained with example.

https://stackoverflow.com/a/38582506/5878471

Answer 5

The answers of @Dawid and @Teemu sometimes brake the chars in the answer in case of utf-8 encoding. This code works much better:

function getGzipped(url, cb) {
    // downloads gzipped file
    http.get(url, function(res) {

        let chunks = [];

        res.on('data', function(chunk) {
            chunks.push(chunk);
        });
        res.on('end', function() {
            let buffer = Buffer.concat(chunks);
            zlib.gunzip(buffer, function(err, decoded) {
                if (err) throw err;
                cb(decoded && decoded.toString());
            });
        });
    });
}

Answer 6

Gathered it all together:

1. gzip, deflate support
2. JSON body decoding
3. Correct utf8 finish decoding
4. Promise-like function (async/await support)

Code:

import * as https from "https";
import * as zlib from "zlib";

const send = (url, config) => {
    return new Promise(async (resolve, reject) => {
        let req;
        try {
            req = https.request(url, config, (res) => {
                let body: any = [];
                let output;
                res.on("error", (e) => reject(e));

                if (res.headers["content-encoding"] == "gzip") {
                    const gzip = zlib.createGunzip();
                    res.pipe(gzip);
                    output = gzip;
                } else if (res.headers["content-encoding"] == "deflate") {
                    output = zlib.createInflate();
                    res.pipe(output);
                } else {
                    output = res;
                }

                output.on("data", (data) => body.push(data));

                output.on("end", () => {
                    try {
                        body = Buffer.concat(body).toString();
                        if (res.headers["content-type"].indexOf("application/json") > -1) body = JSON.parse(body);
                    } catch (e) {
                        reject(e);
                        return;
                    }
                    resolve({
                        data: body,
                        status: res.statusCode,
                        statusText: res.statusMessage,
                        headers: res.headers,
                        request: {
                            url,
                            config
                        }
                    });
                });
            });
            req.on("error", (e) => reject(e));
            req.end();
        } catch (e) {
            reject(e);
        }
    });
};

console.log(await send("https://www.google.com/", { headers: { "User-Agent": "Google" } }));