Save JSON data from Node.js Get request to global variable or file

Question

I need to save data from GET requests to a variable and then save it in a file. However, in some cases GET request does not save data to global variables.

var fs = require("fs");
var http = require("http");
var request = require('request');

var tmp_json = {};
var g_last = 0;
var data = {};

 //request 1
http.get('server:api', (resp) => {
  let data = '';

  resp.on('data', (chunk) => {
    data += chunk;
  });

  resp.on('end', () => {
    tmp_json.server1 = {};
    tmp_json.server1 = JSON.parse(data);
    g_last = tmp_json.height; // 100500
    console.log(g_last); // 100500
  });

}).on("error", (err) => {
  console.log("Error: " + err.message);
});

 //request 2
http.get('server2:api', (resp) => {
  let data = '';

  resp.on('data', (chunk) => {
    data += chunk;
  });

  resp.on('end', () => {
    tmp_json.server2 = {};    
    tmp_json.server2 = JSON.parse(data);
    g_last = tmp_json.height; // 256
    console.log(g_last); // 256
  });

}).on("error", (err) => {
  console.log("Error: " + err.message);
});

console.log(g_last); // 0

data = JSON.stringify(tmp_json);
fs.writeFile('data.json', data, 'utf8'); // empty file

Also I was trying to do it with fs.createWriteStream, but again I can save one request to file, but if there more then one I catch only buffer data.

Possible duplicate of [Why is my variable unaltered after I modify it inside of a function? - Asynchronous code reference](https://stackoverflow.com/questions/23667086/why-is-my-variable-unaltered-after-i-modify-it-inside-of-a-function-asynchron) — Heretic Monkey, May 09 '18 at 18:45
I don't think what you are attempting is possible due to those requests being asynchronous in nature. Remember, asynchronous code is run out-of-step, i.e. just because i call function1 before function2 doesn't necessarily mean that function1 will resolve before function2. — Jhecht, May 09 '18 at 19:26

Jtaks · Accepted Answer · 2018-05-09T20:31:09.383

Your problem is that request1 and request2 are happening while you are writing the file. This is because of the async nature of node. The execution order looks something like this:

Declare empty variables
Request1 goes out
Request2 goes out
Write empty variables to file
Request1 comes back and writes to variables
Request2 comes back and writes to variables

One way to fix this would be with Promises. The following allows for the function in then to be executed after the promises in Promise.all([ ... ]) have resolved:

var fs = require("fs");
var http = require("http");

var tmp_json = {};
var g_last = 0;
var data = {};

 //request 1
var req1 = new Promise((resolve, reject) => {
  http.get('server:api', (resp) => {
    let data = '';

    resp.on('data', (chunk) => {
      data += chunk;
    });

    resp.on('end', () => {
      tmp_json.server1 = {};
      tmp_json.server1 = JSON.parse(data);
      g_last = tmp_json.height; // 100500
      console.log(g_last); // 100500
      resolve()
    });

  }).on("error", (err) => {
    console.log("Error: " + err.message);
    reject(error)
  });
});

 //request 2
var req2 = new Promise((resolve, reject) => {
  http.get('server2:api', (resp) => {
    let data = '';

    resp.on('data', (chunk) => {
      data += chunk;
    });

    resp.on('end', () => {
      tmp_json.server2 = {};
      tmp_json.server2 = JSON.parse(data);
      g_last = tmp_json.height; // 256
      console.log(g_last); // 256
      resolve()
    });

  }).on("error", (err) => {
    console.log("Error: " + err.message);
    reject(error)
  });
});

Promise.all([ req1, req2 ]).then(() => {
  console.log(g_last);
  data = JSON.stringify(tmp_json);
  fs.writeFile('data.json', data, 'utf8');
})

Edit:

function handleGet (url) {
  return new Promise((resolve, reject) => {
    http.get(url, (resp) => {
      let data = '';

      resp.on('data', (chunk) => {
        data += chunk;
      });

      resp.on('end', () => {
        tmp_json.server1 = {};
        tmp_json.server1 = JSON.parse(data);
        g_last = tmp_json.height; // 100500
        console.log(g_last); // 100500
        resolve()
      });

    }).on("error", (err) => {
      console.log("Error: " + err.message);
      reject(error)
    });
  })
}

// Then use it
Promise.all([
  handleGet('http://google.ca'),
  handleGet('http://somesite.com')
])

Thanks it's work awesome! But can I call it like a function with parameters. I.e. if I need to set url adreess of server as a parameter? — MAx Shvedov, May 09 '18 at 20:22
@MAxShvedov You can write a function that returns a promise, I'll add an example to my answer. — Jtaks, May 09 '18 at 20:24
Node says me: `UnhandledPromiseRejectionWarning: Unhandled promise rejection (rejection id: 1): TypeError: http.get is not a function` — MAx Shvedov, May 10 '18 at 16:25

Save JSON data from Node.js Get request to global variable or file

1 Answers1