Loops and Callback hell

Question

Suppose you have an Array/Object that contains a list of values. Lets say those a mysql commands or urls or filespaths. Now you want to iterate over all of them and execute some code over every entry.

for(let i = 0; i < urls.length; i++){
   doSthWith(urls[i]);
}

No Problem so far. But now lets say each function has a callback and needs the result of the last execution. e.g. you request something from one website and you want to use the results of this request for one of your following requests.

for(let i = 0; i < urls.length; i++){
    if(resultOfLastIteration.successful){  //or some other result besides the last one
        doSthWith(urls[i]);
    }
}

Now lets say the length of urls (or sth similar) is over 100. Thats why you normaly use a loop so you dont need to write the same function a 100 times. That also means that Promises wont do the trick either (except Im unaware trick a trick), because you have the same problem:

doSthWith(urls[0]).then(...
    doSthWith(urls[1]).then(... //either put them inside each other
).then(...
    doSthWith(urls[i])          //or in sequence 
    ...
).catch(err){...}

Either way I dont see a way to use a loop.

A way that I found but isnt really "good" is to use the package "wait.for"(https://www.npmjs.com/package/wait.for). But what makes this package tricky is to launch a fiber each time you want to use wait.for:

 //somewhere you use the function in a fiber Context
 wait.for(loopedExecutionOfUrls, urls); 
 //function declaration
 function loopedExecutionOfUrls(urls, cb){
     //variables: 
      for(let i = 0; i < urls.length; i++){
          if(someTempResultVar[i-1] === true){ 
            someTempResultVar = wait.for(doSthWith,urls[i]);
          } else if(...){...}
      }
 }

But Im not sure if this approach is really good, besides you always have to check if you have wrapped the whole thing in a Fiber so for each function that has loops with functions that have callbacks. Thus you have 3 levels: the lauchFiber level, wait.for(loopedFunction) level and the wait.for the callback function level. (Hope I that was formulated understandable)

So my questions is: Do you guys have a good approach where you can loop throw callback functions and can use results of those whenever you like?

good = easy to use, read, performant, not recursive,...

(Im sorry if this question is stupid, but I really have problems getting along with this asynchronous programming)

Answer 1

If you want to wait for doSthWith to finish before doing the same but with the nex url, you have to chain your promises and you can use array.prototype.reduce to do that:

urls = ["aaa", "bbb", "ccc", "ddd"];

urls.reduce((lastPromise, url) => lastPromise.then((resultOfPreviousPromise) => { 
    console.log("Result of previous request: ", resultOfPreviousPromise); // <-- Result of the previous request that you can use for the next request
    return doSthWith(url);
}), Promise.resolve());

function doSthWith(arg) {   // Simulate the doSthWith promise
   console.log("do something with: ", arg);
   return new Promise(resolve => {
      setTimeout(() => resolve("result of " + arg), 2000);
   });       
}

Answer 2

Use async, specifically async.each:

const async = require('async');

function doSthWith(url, cb) {
    console.log('doing something with ' + url);
    setTimeout(() => cb(), 2000);
}

const urls = ['https://stackoverflow.com/', 'https://phihag.de/'];
async.each(urls, doSthWith, (err) => {
    if (err) {
        // In practice, likely a callback or throw here
        console.error(err);
    } else {
        console.log('done!');
    }
});

Use async.map if you are interested in the result.

Answer 3

When I need to loop over promises I use my handy dandy ploop function. Here is an example:

// Function that returns a promise
var searchForNumber = function(number) {
    return new Promise(function(resolve, reject) {
      setTimeout(function() {
        var min = 1;
        var max = 10;
        var val = Math.floor(Math.random()*(max-min+1)+min);

        console.log('Value is: ' + val.toString());        

        return resolve(val);        
      }, 1000);
    });
};

// fn     : function that should return a promise.
// args   : the arguments that should be passed to fn.
// donefn : function that should check the result of the promise
//    and return true to indicate whether ploop should stop or not.
var ploop = function(fn, args, donefn) {
    return Promise.resolve(true)
      .then(function() {
          return(fn.apply(null, args));
      })
      .then(function(result) {
        var finished = donefn(result);
        if(finished === true){
           return result;
        } else {
          return ploop(fn, args, donefn);
        }
    });
};

var searchFor = 4;

var donefn = function(result) {
  return result === searchFor;
};

console.log('Searching for: ' + searchFor);
ploop(searchForNumber, [searchFor], donefn)
  .then(function(val) {
    console.log('Finally found! ' + val.toString());  
    process.exit(0);
  })
  .catch(function(err) {
    process.exit(1);
  });