How to get JSON from URL and store it in a variable (Node.js)

Question

I want to get the JSON from a URL and store it in a variable that can be used in other functions as well but I can't figure out how to do this

I've tried using jquery and ajax but I don't know how to add jquery to a node app


var express = require("express");
var app = express();
var http = require("http");
var getJSON = require('get-json');

app.use(express.static(__dirname + '/public'));

app.get("/", function (req, res) {
    res.render("home.ejs");
});


app.get("/website", function (req, res) {
    res.render("website_pick.ejs");
});


app.get("/item", function (req, res) {
    var web1 = req.query.web1;
    var web2 = req.query.web2;
    var web3 = req.query.web3;

    if (web2 == undefined) {
        web2 = '';
    }

    if (web3 == undefined) {
        web3 = '';
    }

    app.set('web1', web1);
    app.set('web2', web2);
    app.set('web3', web3);

    res.render("item_pick.ejs");
});


app.get("/display", function (req, res) {
    var item = req.query.item;

    app.set('item', item);

    var web1 = app.get('web1');
    var web2 = app.get('web2');
    var web3 = app.get('web3');

    var item_split = item.split(" ");
    var item_join = item_split.join('+');

    var results_1 = [];

    for (var i = 0; i < 9; ++i) {
        var result_page = 1 + (10 * i);
        var result_url = 'https://www.googleapis.com/customsearch/v1?key=&cx=&num=10&siteSearch=' + web1 + '&siteSearchFilter=i&q=' + item_join + '&start=' + result_page;

        getJSON(result_url, function (error, response) {

            for (var i = 0; i < 9; ++i) {
                var result_item = response['items'][i];
                results_1.push([result_item['title'], result_item['link'], result_item['pagemap']['cse_image'][0]['src']]);
            }
        });
    }
    res.render("display.ejs", {result_list: results_1,});
});

app.listen(5000);

Within the getJSON block in the display route results_1 is storing the proper data, however in display.ejs result_list is empty

The line `results_1.push(//...` is executed only after the request is completed (obviously). If anything tries to grab the content of `results_1` sooner - which I'm guessing is exactly what's happening "outside that block" - it'll come up empty. — mbojko, Jul 31 '19 at 07:34

Tien Duong · Accepted Answer · 2019-07-31T08:39:19.747

0

results_1 is empty because you use it before getJSON completed and your callback which put data into result_1 executed

Updated

You need to use Promise.all to wait for all getJSON request complete then render the result_1.

app.get("/display", function (req, res) {
    var item = req.query.item;

    app.set('item', item);

    var web1 = app.get('web1');
    var web2 = app.get('web2');
    var web3 = app.get('web3');

    var item_split = item.split(" ");
    var item_join = item_split.join('+');

    var results_1 = [];
    var promises = [];

    for (var i = 0; i < 9; ++i) {
        var result_page = 1 + (10 * i);
        var result_url = 'https://www.googleapis.com/customsearch/v1?key=&cx=&num=10&siteSearch=' + web1 + '&siteSearchFilter=i&q=' + item_join + '&start=' + result_page;

        promises.push(new Promise(function(resolve, reject){
          getJSON(result_url, fuction (error, response) {
            if (error) {
              reject(error);
            }

            for (var i = 0; i < 9; ++i) {
                var result_item = response['items'][i];
                results_1.push([result_item['title'], result_item['link'], result_item['pagemap']['cse_image'][0]['src']]);
            }

            resolve('done');
          });
        }));
    }

    Promise.all(promises)
      .then(function() {
        res.render("display.ejs", {result_list: results_1});
      })
      .catch(function(error) {
        console.log(error);
      }) 
});

edited Jul 31 '19 at 08:39

answered Jul 31 '19 at 07:34

Tien Duong

2,517
1
10
27

So how can I fix this? – STRAWD Jul 31 '19 at 07:43
You need to share more code to describe what you want to do with ```results_1``` – Tien Duong Jul 31 '19 at 07:44
I added more code, hope it helps – STRAWD Jul 31 '19 at 07:53
The thing is I need my outer for loop to complete before I can send `results_1` to `display.ejs` – STRAWD Jul 31 '19 at 07:57
I see. Wait a minute – Tien Duong Jul 31 '19 at 07:57
I updated my answer. Please check it @STRAWD – Tien Duong Jul 31 '19 at 08:05
I tried this but the page never loads. Any suggestions? – STRAWD Jul 31 '19 at 08:31
Did you see any error in console? – Tien Duong Jul 31 '19 at 08:34
No it just keeps trying to load `display.ejs` – STRAWD Jul 31 '19 at 08:37
I updated my answer. I put callback of ```Promise.all``` in wrong position. I'm so sorry for my mistake. I also add an error handler to log error to console – Tien Duong Jul 31 '19 at 08:40
Thank you so much I'm so grateful! It works now! :) – STRAWD Jul 31 '19 at 08:52

score 0 · Answer 2 · answered Jul 31 '19 at 08:00

I'd probably refactor this into a function that can take a callback (if you're not using promises... which you probably should be :))

function getData(callback) {
  var results = [];
  var errors = [];
  var request_count = 9;

  for (var i = 1; i <= request_count; ++i) {
    var result_page = 10 * i;
    var result_url = 'https://www.googleapis.com/customsearch/v1?key=&cx=&num=10&siteSearch=' + web1 + '&siteSearchFilter=i&q=' + item_join + '&start=' + result_page;

    getJSON(result_url, function (error, response) {
      request_count++;

      if (error) {
        results.push(error)
      } else {
        for (var i = 0; i < 9; ++i) {
          var result_item = response['items'][i];
          results.push([result_item['title'], result_item['link'], result_item['pagemap']['cse_image'][0]['src']]);
        }
      }

      if (i == request_count) {
        callback(errors, results);
      }
    });
  }
}

And a promise-based version:

async function getData() {
  const request_count = 10;
  const requests = [];

  for (let i = 0; i < request_count - 1) {
    const result_url = 'https://www.googleapis.com/customsearch/v1?key=&cx=&num=10&siteSearch=' + web1 + '&siteSearchFilter=i&q=' + item_join + '&start=' + result_page;

    requests.push(new Promise((resolve, reject) => {
      getJSON(result_url, (error, response) => {
        if (error) return reject(error);

        for (let i2 = 0; i2 < 9; ++i) {
          var result_item = response['items'][i];
          resolve([result_item['title'], result_item['link'], result_item['pagemap']['cse_image'][0]['src']]);
        }
      })
    }))
  }

  const results = await Promise.all(requests)
  return results
}

This code is untested, but should get you going in the right direction.

How to get JSON from URL and store it in a variable (Node.js)

2 Answers2

Updated