How to use json response to create another URL. Nodejs

Question

I have the question, below code:

The problem is:

1. How can I send each line from response promiseGetCitiesData to promiseGetInformationDataPerCity.
2. Can I do it in one async.each functions?

Now, I created multiple Promise functions. One general function, which one start the program - getDataAndCloseDb().

Also I used async.each to call promise function with array parameter - locationArray.

Now, I would like to send each line from json response to next promise function (create get url), and collect the general response.

const MongoClient = require("mongodb").MongoClient;
    const request = require("request");
    const async = require("async");

    var locationsArray = [
      'location1',
      'location2',
      'location3'
    ];

    function promiseConnectToDatabase(urldb) {
      return new Promise(function(resolve, reject) {
        MongoClient.connect(urldb, (err, db) => {
          if (err) {
            console.log("MongoDb connection error.");
            reject(err);
          }
          console.log("Connected to MongoDb.");
          resolve(db);
        });
      });
    }

    function promiseGetCitiesData(location) {
      return new Promise(function(resolve, reject) {
        request({
          url: `https://example.com/${location}`,
          json: true
        }, (error, response, body) => {
          if (error) {
            console.log("Error connection to url.");
            reject();
          }
          console.log("location: " + location);
          console.log({location: location, cities: body.result.cities});
          resolve({location: location, cities: body.result.cities});
        });
      });
    }
    /*
    Example response from promiseGetCitiesData:

    Location: location1
    { location: 'location1',
      cities: 
       [ 'information1',
         'information2',
         'information3',
         'information4'' ] }

    */

    function promiseGetInformationDataPerCity(location, cities) {
      return new Promise(function(resolve, reject) {
        request({
          url: `https://example.com/${location}/${cities}`,
          //f.e https://example.com/location1/information1 etc.
          json: true
        }, (error, response, information) => {
          if (error) {
            console.log("Error connection to url.");
            reject();
          }
          console.log(information);
          resolve(information);
        });
      });
    }

    function promiseSaveDataToDatabase(db, body) {
      return new Promise(function(resolve, reject) {
        db.collection("testlocation").insert(body, function(dbError) {
          if (dbError) {
            reject(dbError);
          }
          resolve()
        });
      });
    }

    function promiseDisconnectDatabase(db) {
      return new Promise(function(resolve, reject) {
        db.close((err) => {
          if (err) {
            console.log("MongoDb disconnect error.");
            reject(err);
          }
          console.log("MongoDb disconnected.");
          resolve();
        });
      });
    }

    function promiseProvideDataFromEach(locationsArray, db) {
      return new Promise(function(resolve, reject) {
        async.each(locationsArray, function(loc, locProcessedCb) {
          promiseGetcitiesData(loc).then(function(resultscities) {
            promiseGetInformationDataPerCity(loc, resultscities).then(function(resultDetails) {
                promiseSaveDataToDatabase(db, resultDetails).then(function() {});
            locProcessedCb();
            });
          });
        }, function(err) {
          if (err) {
            locProcessedCb(err);
            reject(err);
          }
          console.log("All locations have been processed.");
          resolve();
        });
      });
    }

    function getDataAndCloseDb() {
      return new Promise(function(resolve, reject) {
        promiseConnectToDatabase("mongodb://127.0.0.1:27017/testApp").then(function(db) {
          promiseProvideDataFromEach(locationsArray, db).then(function() {
            promiseDisconnectDatabase(db).then(function() {});
          });
        });
      });
    }

    getDataAndCloseDb();

Answer 1

I think this is a lot simpler than the code in the question makes it appear. In particular, new Promise(...) can be completely avoided by :

• using require('async-request') instead of require('request').
• allowing MongoDb methods to return Promise, as many of them will do if no callback is passed.

Also

• by using the Promise.all(array.map(...)) pattern the need for require('async') disappears.
• https://stackoverflow.com/a/28915678/3478010 - provides a great little reusable disposer utility, which is useful here.
• Remember to return a promise/value from every .then() callback that is itself asynchronous and/or should deliver data.

With some guesswork, I think you want something like this :

const MongoClient = require('mongodb').MongoClient;
const request = require('async-request'); // just like `request()` but returns a promise

var locationsArray = [
    'location1',
    'location2',
    'location3'
];

function promiseGetCitiesData(loc) {
    return request({
        url: `https://example.com/${loc}`,
        json: true
    }).then(body => body.result.cities);
}

function promiseGetInformationDataPerCity(loc, cities) {
    return Promise.all(cities.map(city => {
        return request({
            'url': `https://example.com/${loc}/${city}`,
            'json': true
        }).then(cityInfo => ({ 'name':city, 'info':cityInfo }));
    }));
}

function promiseProvideDataFromEach(locationsArray, db) {
    return Promise.all(locationsArray.map(loc => {
        return promiseGetCitiesData(loc)
        .then(cities => promiseGetInformationDataPerCity(loc, cities)
        .then(citiesWithCityInfo => ({ 'location':loc, 'cities':citiesWithCityInfo }));
    }))
    .then(resultDetails => db.collection('testlocation').insertMany(resultDetails));
}

// disposer utility - credit: https://stackoverflow.com/a/28915678/3478010
function withDb(work) {
    var _db;
    return MongoClient.connect("mongodb://127.0.0.1:27017/testApp")
    .then((db) => {
        _db = db; // keep reference 
        return work(db); // perform work on db
    }).finally(() => {
        if (_db)
            _db.close();
    });
}

withDb(db => promiseProvideDataFromEach(locationsArray, db))
.then(() => {
    // connection released here
});

The guesswork centres mainly around what is to be inserted at db.collection('testlocation').insertMany(resultDetails). The code in the question gives no more than a clue. My attempt seems reasonable but may not be exactly what you want. Be prepared to make some changes in promiseProvideDataFromEach() and promiseGetInformationDataPerCity().

Answer 2

you can do something like this. Its a simpler code but I think you can map it to your current code.

const Promise = require('bluebird')

const cities = ['citya', 'cityb', 'cityc']

function resolveCities() {
  return new Promise(function(resolve, reject) {
    resolve(cities)
  })
}

function logCity(city) {
  console.log('city ', city)
}

return resolveCities() 
.then(function(cities) {
  return Promise.mapSeries(cities, function(city) {
    logCity(city);
  });
})