How to run multiple asynchronous functions then execute callbacks?

Question

In my Node.js code I need to make 2 or 3 API calls, and each will return some data. After all API calls are complete, I want to collect all the data into a single JSON object to send to the frontend.

I know how to do this using the API callbacks (the next call will happen in the previous call's callback) but this would be slow:

//1st request
request('http://www.example.com', function (err1, res1, body) {
  
  //2nd request
  request('http://www.example2.com', function (err2, res2, body2) {
  
    //combine data and do something with it

  });

});

I know you could also do something similar and neater with promises, but I think the same concept applies where the next call won't execute until the current one has finished.

Is there a way to call all functions at the same time, but for my final block of code to wait for all API calls to complete and supply data before executing?

Answer 1

Promises give you Promise.all() (this is true for native promises as well as library ones like bluebird's).

Update: Since Node 8, you can use util.promisify() like you would with Bluebird's .promisify()

var requestAsync = util.promisify(request); // const util = require('util')
var urls = ['url1', 'url2'];
Promise.all(urls.map(requestAsync)).then(allData => {
    // All data available here in the order of the elements in the array
});

So what you can do (native):

function requestAsync(url) {
    return new Promise(function(resolve, reject) {
        request(url, function(err, res, body) {
            if (err) { return reject(err); }
            return resolve([res, body]);
        });
    });
}
Promise.all([requestAsync('url1'), requestAsync('url2')])
    .then(function(allData) {
        // All data available here in the order it was called.
    });

If you have bluebird, this is even simpler:

var requestAsync = Promise.promisify(request);
var urls = ['url1', 'url2'];
Promise.all(urls.map(requestAsync)).then(allData => {
    // All data available here in the order of the elements in the array
});

Answer 2

Sounds like async.parallel() would also do the job if you'd like to use async:

var async = require('async');

async.parallel({
    one: function(parallelCb) {
        request('http://www.example1.com', function (err, res, body) {
            parallelCb(null, {err: err, res: res, body: body});
        });
    },
    two: function(parallelCb) {
        request('http://www.example2.com', function (err, res, body) {
            parallelCb(null, {err: err, res: res, body: body});
        });
    },
    three: function(parallelCb) {
        request('http://www.example3.com', function (err, res, body) {
            parallelCb(null, {err: err, res: res, body: body});
        });
    }
}, function(err, results) {
    // results will have the results of all 3
    console.log(results.one);
    console.log(results.two);
    console.log(results.three);
});

Answer 3

https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all

Promise.all is now included with ES6 so you don't need any 3rd party libraries at all.

"Promise.all waits for all fulfillments (or the first rejection)"

I've setup a gist to demonstrate Promise.all() with refactoring itterations at: https://gist.github.com/rainabba/21bf3b741c6f9857d741b69ba8ad78b1

I'm using an IIFE (Immediately Invoked Function Expression). If you're not familiar, you'll want to be for the example below though the gist shows how with using an IIFE. https://en.wikipedia.org/wiki/Immediately-invoked_function_expression

TL;DR

( function( promises ){
    return new Promise( ( resolve, reject ) => {
        Promise.all( promises )
            .then( values => {
                console.log("resolved all promises")
                console.dir( values );
                resolve( values.reduce( (sum,value) => { return sum+value }) ); //Use Array.prototype.reduce() to sum the values in the array
            })
            .catch( err => {
                console.dir( err );
                throw err;
            });

    });
})([ 
    new Promise( ( resolve, reject ) => {
        console.log("resolving 1");
        resolve( 1 );
    }),
    new Promise( ( resolve, reject ) => {
        console.log("resolving 2");
        resolve( 2 );
    })
 ]).then( sum => { console.dir( { sum: sum } ) } )

Answer 4

I had a similar use case where I had to do 10 concurrent calls. I did it with the combination of async/await and Promise.all.

async function getData() {
  try {
    let result = null
    const ids = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]

    let promises = ids.map(async (id) => {
        return fetch(
          `https://jsonplaceholder.typicode.com/todos/${id}`
        ).then((data) => data.json());
      });
    
    result = await Promise.all(promises)
    return result
  } catch(err) {
    console.log("error: ", err)
  }
}

getData().then(data => console.log(data))