promise.all inside a forEach loop — everything firing at once

Question

In a Node app, I need to iterate through some items in a synchronous fashion, but some of the operations inside the loop are asynchronous. My code right now looks like so:

someAPIpromise().then((items) => {
   items.forEach((item) => {
      Promise.all[myPromiseA(item), myPromiseB(item)]).then(() => {
         doSomethingSynchronouslyThatTakesAWhile();
      });
    }
}

This works wonders when the items is an array of 1. But, once there's more than one item, promise.all() will just fire off instantly for every item in the array, without waiting for the operation in the loop to end.

All that to say... how can I ensure that the entire operation for each item in the array is run synchronously (even if some operations are async and return a promise)?

Thanks so much!

N

Answer 1

You're constructing several promises, but they are all asynchronous. You construct Promise1, Promise2, Promise3, ... but once they're in the wild they are all firing simultaneously. If you want synchronous behavior you've got to chain them together so Promise1's .then() executes Promise2 and so on. In the past I've used Array.reduce for this.

someAPIpromise().then((items) => {
    items.reduce((accumulator, current) =>
        accumulator.then(() =>
             Promise.all[myPromiseA(item), myPromiseB(item)]).then(() => 
                 doSomethingSynchronouslyThatTakesAWhile();
             )
        )
    , Promise.resolve());

You can write this as a helper function if you like, which may make things clearer.

function execSequentially (arr, func) {
    return arr.reduce(
        (accumulator, current) => accumulator.then(() => func(current)), 
        Promise.resolve());
}

That function is executed as

execSequentially(items, item => console.log(item));

of course replacing console.log with what you want to do.

The helper function approach is also less invasive of a change. The helper applied to your original code:

someAPIpromise().then((items) => {
   execSequentially(items, (item) =>
      Promise.all[myPromiseA(item), myPromiseB(item)]).then(() => {
         doSomethingSynchronouslyThatTakesAWhile();
      });
   );
});

Answer 2

You should be able to remove .forEach(); use Array.prototype.reduce() to return an array of Promise values to Promise.all(). If element with items is a function, call function, else wrap within Promise.resolve(), which should return results in same order as in items array

See Promise.all()

Promise.all passes an array of values from all the promises in the iterable object that it was passed. The array of values maintains the order of the original iterable object, not the order that the promises were resolved in. If something passed in the iterable array is not a promise, it's converted to one by Promise.resolve.

var arr = [1, // not asynchronous
  function j() {
    return new Promise(function(resolve) {
      setTimeout(function() {
        resolve(2)
      }, Math.floor(Math.random() * 10000))
    })
  }, // asynchronous
  3, // not asynchronous
  function j() {
    return new Promise(function(resolve) {
      setTimeout(function() {
        resolve(4)
      }, Math.floor(Math.random() * 3500))
    })
  }, // asynchronous
  5, // not asynchronous
  Promise.resolve(6), // asynchronous
  7
];

Promise.all(arr.reduce(function(p, next) {
    var curr = Promise.resolve(typeof next === "function" ? next() : next);
    return p.concat.apply(p, [curr.then(function(data) {
      console.log(data);
      return data
    })]);
  }, []))
  .then(function(data) {
    console.log("complete", data)
  })

An alternative approach would be to use Array.prototype.shift() , Promise.resolve(), .then(), recursion

function re(items, res) {
  if (items.length) {
    var curr = items.shift();
    return Promise.resolve(
      typeof curr === "function" 
      ? curr() 
      : curr
    ).then(function(data) {
      // values from `arr` elements should be logged in sequential order
      console.log(data);
      res.push(data)
    }).then(re.bind(null, items, res))
  } else {
    return ["complete", res]
  }
}

var _items = arr.slice(0);

re(_items, [])
.then(function(complete) {
  console.log(complete)
})

var arr = [1, // not asynchronous
  function j() {
    return new Promise(function(resolve) {
      setTimeout(function() {
        resolve(2)
      }, Math.floor(Math.random() * 10000))
    })
  }, // asynchronous
  3, // not asynchronous
  function j() {
    return new Promise(function(resolve) {
      setTimeout(function() {
        resolve(4)
      }, Math.floor(Math.random() * 3500))
    })
  }, // asynchronous
  5, // not asynchronous
  Promise.resolve(6), // asynchronous
  7
];

function re(items, res) {
  if (items.length) {
    var curr = items.shift();
    return Promise.resolve(
      typeof curr === "function" 
      ? curr() 
      : curr
    ).then(function(data) {
      // values from `arr` elements should be logged in sequential order
      console.log(data);
      res.push(data)
    }).then(re.bind(null, items, res))
  } else {
    return ["complete", res]
  }
}
var _items = arr.slice(0);
re(_items, [])
  .then(function(complete) {
    console.log(complete)
  })

Answer 3

All righty... the way we were able to get it to work: array.reduce() with the help of Promises. The end result:

myAsyncAPIcall.then(items => {
    items.reduce((current, nextItem) => {
        return current.then(() => {
          return new Promise(res => {
             Promise.all([myPromiseA(nextItem), myPromiseB(nextItem]).then(() => {
               someSynchronousCallThatTakesAWhile(nextItem);
               res();
             }).catch(err => {
                   console.log(err);
             });
          });
        });
    }, Promise.resolve())
})

The way it works is, by wrapping each item of the array in its own Promise(resolve, reject), we can ensure that each iteration is run synchronously, as the completion of one iteration will trigger the need to resolve the next Promise, and so on and so forth. Within each promise resolving, calls can get kicked off asynchronously as much as you want, with the knowledge that they will only be scoped to the parent promise until it finishes.

I hope this helps folks!

Answer 4

How about keeping the forEach...

var stopAllProcessingOnServerLowValue= false;

function someAPIpromise(){
    var arr = [
        {id:123, urlVal:null},
        {id:456, urlVal:null},
        {id:789, urlVal:null},
        {id:101112, urlVal:null}
    ];

    return new Promise(function(resolve){
        setTimeout(function(){
            resolve(arr)
        }, 3000);
    })
}

function extractSomeValueRemotely(url){
    return new Promise(function(resolve, reject){
        console.log("simulate an async connection @ %s to request a value", url);
        setTimeout(function(){
            var someRandom = Math.round(Math.random()*7) + 1;
            console.log("%s responded with %s", url, someRandom);
            if(someRandom > 4){
                resolve(someRandom);
            }
            else{
                var issue = "Urls result is too low ("+someRandom+" <= 4).";
                console.warn(issue+".It will be set to -1");
                if(stopAllProcessingOnServerLowValue){
                    reject(issue+".Operation rejected because one or mole server results are too low ["+someRandom+"].");
                }
                else{
                    resolve(-1);
                }
            }
        }, 1500*Math.round(Math.random()*7) + 1);
    });
}

function addAnotherExtraParamToItem(_item){
    return new Promise(function(resolve, reject){
        setTimeout(function(){
            console.log("setting extra2 on %s", _item.id);
            _item['extra'] = "additional_processing_"+_item.id;
            resolve(_item);
        }, 1500*Math.round(Math.random()*5) + 1);
    });
}

function addOrderIndexToItem(_item, _order){
    return new Promise(function(resolve, reject){
        setTimeout(function(){
            console.log(">> setting order %s on %s",_order,  _item.id);
            _item['order'] = _order;
            resolve(_item);
        }, 1500*Math.round(Math.random()*3) + 1);
    });
}

someAPIpromise().then(function(items){

    var perItemPromises = [];
    items.forEach(function(item, idx){

        perItemPromises.push(

            new Promise(function(pulseItemResolve, pulseItemReject){
                var itemStepsPromises =  [];
                itemStepsPromises.push(addAnotherExtraParamToItem(item));

                itemStepsPromises.push(extractSomeValueRemotely("http://someservice:777/serve-me")
                    .catch(
                        function(reason){
                            //the entire item will be rejected id
                            pulseItemReject(reason);
                        })
                );

                itemStepsPromises.push(addOrderIndexToItem(item, idx));

                //promise that ensure order of execution on all previous async methods
                Promise.all(itemStepsPromises).then(function(values){
                    //0 - first is result from addAnotherExtraParamToItem
                    var theItem = values[0]; //it returns the item itself
                    //urlVal has not been set yet

                    // 1 - second promise return the url result
                    var serverResult = values[1];

                    //2 - third promise add the order index but we do not care to inspect it because theItem reference in value[0] has been already updated.
                    // console.info(values[2]);

                    //sets the url result in the item
                    theItem.urlVal = serverResult;
                    console.log("urlVal set to:", theItem.urlVal);

                    //resolve the prepared item
                    pulseItemResolve(theItem);

                });
            })
                .catch(function(reason){
                    //escalate error
                    throw new Error(reason);
                })
        )

    });

    Promise.all(perItemPromises).then(function(resultsInAllItems){
        console.info("Final results:");
        console.info(resultsInAllItems);
    }).catch(function(finalReject){
        console.error("Critical error:",finalReject);
    })


});

Answer 5

After much research, the definitive answer for me was here...

I've read bunches of solutions for having a useful pure JavaScript (no addons) -Promise Iterator- that could be easily used (one line) all over my projects, and finally I've found this solution by Salketer:

function one_by_one(objects_array, iterator, callback) {
    var start_promise = objects_array.reduce(function (prom, object) {
        return prom.then(function () {
            return iterator(object);
        });
    }, Promise.resolve()); // initial
    if(callback){
        start_promise.then(callback);
    }else{
        return start_promise;
    }
}

For details and example of usage visit the link.

It also allows to handle a callback directly.

It was simply the most logical and reusable method I've found after many days of struggling with Promise iterations and testing MULTIPLE solutions from many questions, blogs and official sites.

If you're also struggling for a definitive answer, give it a try.