Exiting Node Once All Promises Are Finished

Question

NOTE: This is a cronjob so I need to exit with a process.exit command when the processing is completed.

In this one particular case, I'm going to use some dummy code to represent the problem b/c I believe pasting the code from this script would generate more questions and cause confusion.

I'm curious about the framework choices I've made. I know there is a better way to organize the promises to create maximum parallel efficiency and, at the same time, know when to exit.

The code below may not be syntactically perfect since I'm just writing it by hand here for the purposes of showing the corner I painted myself into.

let promises = [];

promises.push(db.get_customers());

let cus_recs = await Promise.all(promises); promises = [];

cus_recs.map((cus_rec) => {

  promises.push(db.get_criteria(cus_rec));

});

let criteria_recs = await Promise.all(promises); promises = [];

cus_recs.map((cus_rec, a) => {

  let cus_criteria_recs = criteria_recs[a];

  cus_criteria_recs.map((cus_criteria_rec) => {

    ci.build_lis_match_data(cus_criteria_rec).then((lis_match_data) => {

      lis.get_matching_listings(cus_criteria_rec, lis_match_data).then((lis_recs) => {

        lis_recs.map((lis_rec) => {

          ci.build_lis_match_table(cus_criteria_rec, lis_rec).then((lis_match_table) => {

            promises.push(lis.do_the_thing(cus_rec, lis_rec));

          });

        });

      });

    });

  });

});

if (promises.length > 0) {

  await Promise.all(promises); promises = [];

}

process.exit(0);

The issue I am faced with is where to place process.exit(0); If placed as shown, node exists before all the promises are finished.

I am aware I can turn this into for loops and use async/await to just wait on each task.

I was hoping that this method would create extreme parallelism of these intensive tasks. Many of these tasks are > 0.1 secs and this will run for 600,000 (or so) records per day.

Answer 1

The problem is that you're doing promises.push() inside a .then() callback (actually, multiple nested ones even), which runs asynchronously. The promises.length is still 0 when your if statement runs.

Avoid using .push() with .map(), it already creates an array from the return values. So always return a promise from the callback.

let promises = [db.get_customers()];
let cus_recs = await Promise.all(promises);

promises = cus_recs.map((cus_rec) => {
  return db.get_criteria(cus_rec);
//^^^^^^
});
let criteria_recs = await Promise.all(promises);

??? = cus_recs.map((cus_rec, a) => { /*
^^^ what does this return? What *can* we return? */

  let cus_criteria_recs = criteria_recs[a];
  promises = cus_criteria_recs.map((cus_criteria_rec) => {
//^^^^^^^^ this creates an array of promises
    return ci.build_lis_match_data(cus_criteria_rec).then((lis_match_data) => {
//  ^^^^^^ (if they are returned here)

      return lis.get_matching_listings(cus_criteria_rec, lis_match_data).then((lis_recs) => {
//    ^^^^^^ need to use promise chaining
        promises = lis_recs.map((lis_rec) => {
//      ^^^^^^^^ same thing again
          return ci.build_lis_match_table(cus_criteria_rec, lis_rec).then((lis_match_table) => {^
//        ^^^^^^
            return lis.do_the_thing(cus_rec, lis_rec));
//          ^^^^^^
          });
        });
        …
      });
    });
  });
  … // now we have the `promises` array around - so we
  return Promise.all(promises)
});
… // and here again we need to deal with the ??? value

process.exit(0);

In fact I'd recommend not to even assign to a promises variable, just pass the result right into Promise.all - as that's the only reasonable thing you can do with an array of promises. So

const cus_recs = await Promise.all([
  db.get_customers()
]);

const criteria_recs = await Promise.all(cus_recs.map(cus_rec =>
  db.get_criteria(cus_rec)
));

await Promise.all(cus_recs.map((cus_rec, a) => { /*
^^^^^^^^^^^^^^^^^ this comes naturally now */

  let cus_criteria_recs = criteria_recs[a];
  return Promise.all(cus_criteria_recs.map(cus_criteria_rec =>
//^^^^^^^^^^^^^^^^^^
    ci.build_lis_match_data(cus_criteria_rec).then(lis_match_data =>  
      lis.get_matching_listings(cus_criteria_rec, lis_match_data).then(lis_recs =>
        Promise.all(lis_recs.map(lis_rec =>
//      ^^^^^^^^^^^
          ci.build_lis_match_table(cus_criteria_rec, lis_rec).then(lis_match_table =>
            lis.do_the_thing(cus_rec, lis_rec)
          )
        ))
      )
    )
  ));
}));

process.exit(0);

I would also recommend to use async/await everywhere, if you can easily replace the nested .then() calls by it:

await Promise.all(cus_recs.map((cus_rec, a) => {
  let cus_criteria_recs = criteria_recs[a];
  return Promise.all(cus_criteria_recs.map(async cus_criteria_rec => {
    const lis_match_data = await ci.build_lis_match_data(cus_criteria_rec);
    const lis_recs = await lis.get_matching_listings(cus_criteria_rec, lis_match_data);
    return Promise.all(lis_recs.map(async lis_rec => {
      const lis_match_table = await ci.build_lis_match_table(cus_criteria_rec, lis_rec);
      return lis.do_the_thing(cus_rec, lis_rec);
    }));
  }));
}));