NodeJS make promise wait for completion of foreach loop

Question

I have a NodeJS script that calls the API for users, gets multiple data for each user and writes it all to local file. I am trying to upload that file to server once all of the data is written into the file. The problem is that the code that should upload the file gets executed before the file is entirely populated. The code is written below. I can't figure out how to make promise wait for first function to complete.

var fs = require('fs');
var server = require('some-server');
var service = require('./some-service.js');
var moment = require('moment-timezone');
var csvWriter = require('csv-write-stream');
var writer = csvWriter({
sendHeaders: false
});

var users = require('./some-users')
writer.pipe(fs.createWriteStream('myFile' + '.txt'))

service.login().then(function (response) {

users.forEach(function (user) {

    service.getSpecificUser(user).then(function (response) {

        var myUser = JSON.parse(response)
        service.getDataForUser(user.Info).then(function (response) {
            var userData = JSON.parse(response);
            if (userData.IsValid) {
                userData.AdditionalInfo.forEach(function (additionalInfo) {
                    service.getAdditionalInfo(myUser.Info, userData.data).then(function (response) {

                        //Collect additional info and combine final results to write into file
                        // write to output csv file
                        writer.write({
                            //write information that is of interest
                        })

                    }, function (error) {
                        console.log('error getting additional data', error);
                    })
                }
                )
            }
        }, function (error) {
            console.log('error getting user data', error)
        })
    }, function (error) {
        console.log('error', myUser, error)
    })
});
}, function (error) {
 console.log('not logged', response);
}).then(function () {
//perform uploading to server
var fpath = 'path of file that contains downloaded data'
console.log("Trying to upload to file: " +fpath)
service.UploadFile(fpath, function (error, result, response) {
    if (!error) {
        console.log("Uploaded " + name);
    }
    else {
        console.log(error);
    }

})
})

Any help would be appreciated.

Which portion of `javascript` should be called last? Does `writer.write()` have a callback, or return a `Promise`? — guest271314, Dec 05 '16 at 21:04
@guest271314 Last function needs to upload the file to the server and needs to be called last — alminh, Dec 05 '16 at 21:06
[You cannot use `.forEach` with promises](http://stackoverflow.com/q/37576685/1048572). — Bergi, Dec 05 '16 at 21:11
@alminh Does `writer.write()` call have a completion event, callback, or return a `Promise`? — guest271314, Dec 05 '16 at 21:23
@guest271314 It uses pipe to continue writing to the same file with `fs.CreateWriteStream`. There are no callbacks or promises. — alminh, Dec 05 '16 at 21:27
@guest271314 Answer to that question would help me in this situation. Currently I am successfully getting all the data and writing it to the file. But I cannot determine when the write is finished, so that I can upload the file.. — alminh, Dec 05 '16 at 21:40
none of your `.then` callbacks return anything, therefore they will resolve (with undefined) as soon as they are called. As Bergi pointed out, .forEach with Promises makes little sense - not sure about the statement that *you cannot use .forEach with Promises*, because you can, you just have to write more complex code than if you used `.map` with `Promise.all` for instance - or use a library like bluebird that has `Promise.map` which is sugar for Array#map + Promise.all — Jaromanda X, Dec 05 '16 at 22:25
If you are going to catch/log intermediate errors, don't forget to re-throw them otherwise the promise chain will (try to) progress down its success path. — Roamer-1888, Dec 05 '16 at 23:37

guest271314 · Accepted Answer · 2016-12-05T23:17:17.553

1

You can substitute Promise.all(), Array.prototytpe.map() for .forEach(). The documentation for csv-write-steam appears to use .end() to complete call .write() at last .then().

  service.login().then(function(response) {
    return Promise.all(users.map(function(user) {
      return service.getSpecificUser(user).then(function(response) {
        var myUser = JSON.parse(response)
        return service.getDataForUser(user.Info).then(function(response) {
          var userData = JSON.parse(response);
          if (userData.IsValid) {
            return Promise.all(userData.AdditionalInfo.map(function(additionalInfo) {
              return service.getAdditionalInfo(myUser.Info, userData.data).then(function(response) {
                //Collect additional info and combine final results to write into file
                // write to output csv file
                writer.write({
                  //write information that is of interest
                });
              })
            }))
          }
        })
      })
    }));
  })
  .then(function() {
   writer.end();
    //perform uploading to server
    var fpath = 'path of file that contains downloaded data'
    console.log("Trying to upload to file: " + fpath)
    service.UploadFile(fpath, function(error, result, response) {
      if (!error) {
        console.log("Uploaded " + name);
      } else {
        console.log(error);
      }

    })
  })
  .catch(function(e) {
    console.log(e)
  })

edited Dec 05 '16 at 23:17

answered Dec 05 '16 at 22:17

guest271314

1
15
104
177

You must not call `writer.end()` there. The writing process is synchronous, you don't need to wait for anything. `writer.end()` does not return a promise. And even if it did, you wouldn't need the `Promise.resolve`. – Bergi Dec 05 '16 at 22:34
Why do you still upload the file when an error happened? You should move the `.catch` to the very end. – Bergi Dec 05 '16 at 22:35
@Bergi Have not tried `csv-write-stream`. Was about to read the script itself. `.end()` does not need to be called? Moved `.catch()` to chain to `.then()` – guest271314 Dec 05 '16 at 22:49
Maybe it does need to be called in the end, and maybe even some kind of stream finish event needs to be awaited, but surely it should not be called after every write. – Bergi Dec 05 '16 at 22:52
@Bergi The documentation does not describe what is returned. Have to do stuff now, but will read actual script later today. For now moved `.end()` call to last `.then()` – guest271314 Dec 05 '16 at 23:14
@Bergi I still need to upload the file with valid data, catch is being used to log invalid ones – alminh Dec 06 '16 at 09:32
@guest271314 Thanks for your effort! This code works, but as I stated, I need to log the errors for functions that collect user data, and upload the file with successfully retrieved data. I believe that this code resolves when at least one error occurrs during API calls, which is not what I need. Do you know what I need to change in it to write successful data and log errors? – alminh Dec 06 '16 at 09:59
@alminh You can adjust `javascript` to have same error handling pattern as at OP. Note the considerations mentioned by @Roamer-1888. – guest271314 Dec 06 '16 at 15:24

NodeJS make promise wait for completion of foreach loop

1 Answers1