For of loop to make HTTPS geocoding api call and on result input the coordinates into SQL database, how to get Node's HTTPS module to work?

Question

I have 13.000 documents from MongoDB where I have address line + Postcode, im trying to make a request for each of them to Google's geocoding API and get LAT + LONG for them so I can have them appear dynamically on a map search.

I have designed the following for of loop and I'm testing with 10 items at a time but due to the async nature of both the writing to DB call and calling the API, the LAT/LONG coordinates from the HTTPS requests ends up being undefined/unavailable to knex's INSERT and the loop seems to keep going on and on...

Is it possible to write this in a blocking way? So the for loop doesn't go to the next item unless both promises have been resolved?

The code:

let results = [];
  await forLoop();

  async function forLoop() {
    for (job of allJobs) {
      const geoData = await getGeoData(
        job.site.addressLine1,
        job.site.postcode
      );
      const dbResult = await addToDb(geoData);
      results.push(dbResult);

      async function getGeoData(addressLine1, postcode) {
        const friendlyAddress = encodeURIComponent(addressLine1 + ' ' + postcode);
        https
          .get(
            'https://maps.googleapis.com/maps/api/geocode/json?key=<API_KEY_IGNORE_THIS_ITS_HARDCODED_IN_MY_REAL_CODE>&address=' +
              friendlyAddress,
            resp => {
              let data = '';
              resp.on('data', chunk => {
                data += chunk;
              });
              // The whole response has been received. Print out the result.
              resp.on('end', () => {
                console.log(JSON.parse(data).explanation);
                let result = JSON.parse(data);
                return result;
              });
            }
          )
          .on('error', err => {
            console.log('Error: ' + err.message);
          });
      }

      async function addToDb(geoData) {
        try {
          await knex('LOCATIONS')
            .returning('*')
            .insert({
              UPRN: job.site.UPRN,
              lat: geoData.results[0].geometry.location.lat,
              lng: geoData.results[0].geometry.location.lng
            });
        } catch (err) {
          err.name = 'database';
          next(err);
        }
      }
    }
  }
  res.send(results);

I've made sure the codebase has no nulls and have tested both the api call and the database call to make sure they work in isolation.

Answer 1

Just use promises (or callbacks)

I know that everyone hates JavaScript and so these anti-idiomatic transpilers and new language "features" exist to make JavaScript look like C# and whatnot but, honestly, it's just easier to use the language the way that it was originally designed (otherwise use Go or some other language that actually behaves the way that you want - and is more performant anyway). If you must expose async/await in your application, put it at the interface rather than littering it throughout.

My 2¢.

I'm just going to write some psuedo code to show you how easy this can be:

function doItAll(jobs) {
  var results = [];

  function next() {
    var job = jobs.shift();
    if (!job) {
      return Promise.resolve(results);
    }
    return makeRequest(job.url).then(function (stuff) {
      return updateDb().then(function (dbStuff) {
        results.push(dbStuff);
      }).then(next);
    });
  }

  return next();
}

function makeRequest() {
  return new Promise(function (resolve, reject) {
    var resp = http.get('...', {...});
    resp.on('end', function () {
      // ... do whatever
      resolve();
    });
  });
}

Simple. Easy to read. 1:1 correspondence between what the code looks like and what's actually happening. No trying to "force" JavaScript to behave counter to the way it was designed.

The longer you fight learning to understand async code, the longer it will take to to understand it.

Just dive in and learn to write JavaScript "the JavaScript way"! :D

Answer 2

Here's my updated function that works properly and synchronously, getting the data one by one and adding it to the database before moving to the next one.

I have made it by customizing @coolAJ86 answer and I've marked that as the correct one but thought it would be helpful for people stumbling across this thread to see my final, working & tested version.

var geoApiUrl =
    'https://maps.googleapis.com/maps/api/geocode/json?key=<<MY API KEY>>&address=';

  doItAll(allJobs)

  function doItAll(jobs) {
    var results = [];
    var errors = [];

    function nextJob() {
      var job = jobs.shift();
      if (!job) {
        return Promise.resolve(results);
      }
      var friendlyAddress =
        geoApiUrl +
        encodeURIComponent(job.addressLine1 + ' ' + job.postcode);

      return makeRequest(friendlyAddress).then(function(result) {
        if((result.results[0] === undefined) || (result.results[0].geometry === undefined)){
          nextJob();
        } else { 
        return knex('LOCATIONS')
          .returning('*')
          .insert({
            UPRN: job.UPRN,
            lat: result.results[0].geometry.location.lat,
            lng: result.results[0].geometry.location.lng,
            title: job.title,
            postcode: job.postcode,
            addressLine1: job.addressLine1,
            theo_id: job.clientId
          })
          .then(function(data) {
            // console.log('KNEX CALLBACK COMING')
            // console.log(data[0])
            console.log(data[0]);
            results.push(data[0]);
            nextJob();
          })
          .catch(function(err) {
            console.log(err);
            errors.push(job);
          });
        }
      });
    }
    return nextJob();
  }

  function makeRequest(url) {
    return new Promise(function(resolve, reject) {
      https
        .get(url, resp => {
          let data = '';
          resp.on('data', chunk => {
            data += chunk;
          });
          // The whole response has been received. Print out the result.
          resp.on('end', () => {
            let result = JSON.parse(data);
            resolve(result);
          });
        })
        .on('error', err => {
          console.log('Error: ' + err.message);
          reject(err);
        });
    });
  }