0

I'm trying to execute a node script that downloads some urls. It looks like this:

const https = require('https');
const fs = require('fs');
const async = require('async');

function download_cb(err) {
    if (err) {
        console.log("Download error:"+err);
    }
}

function download_file(task) {
    const url = task.url;
    const dest = task.dest;
    var file = fs.createWriteStream(dest);
    var request = https.get(url, function(response) {
      response.pipe(file);
      file.on('finish', function() {
        file.close(download_cb);  // close() is async, call cb after close completes.
      });
    }).on('error', function(err) { // Handle errors
      fs.unlink(dest); // Delete the file async. (But we don't check the result)
      download_cb(err.message);
    });
  };

function get_urls() { 
  var queue = async.queue(download_file, 10);
  const urls = []; // some array of urls
  for (var i=0; i<urls.length; i++) {
    queue.push({url: urls[i], dest: dest/*whatever*/});
  }
  return queue.drain();
}

(async () {
    await get_urls().then(()=>{
        console.log("All done");
    })
  })();

This ends up downloading only the first 10 urls and then exits, while the "All done" message is never shown. It somehow seems that the promise returned by the function (queue.drain()) is never resolved even though it's being awaited for. What am I missing?

I also tried:

        queue.drain = function() {
            console.log("All files are downloaded");
        };

inside the get_urls function but it doesn't change anything and the code in it isn't get executed either.

shaimo
  • 376
  • 3
  • 17
  • Don't use `async.js` if you want to work with promises – Bergi Apr 20 '20 at 17:58
  • I tried without it earlier but ended up with an issue where too many files were trying to download at once. Isn't async.js a good way to solve this issue? Why wouldn't it work with promises? I actually don't care about promises here and I wouldn't care at all if it were synchronous to begin with - I just want it to download the entire list of files and only then exit. If you have any other suggestions on how to achieve this that would be great too :-) – shaimo Apr 20 '20 at 18:07
  • 1
    `async.js` is based on callback style, that's why it doesn't fit together with promises. If you [promisify](https://stackoverflow.com/q/22519784/1048572) your `download_file` function, you can just [use `await` in a loop](https://stackoverflow.com/a/37576787/1048572) – Bergi Apr 20 '20 at 18:15

0 Answers0