ForEach async js still not solved

Question

I still cannot grasp how to solve this. I tried solutions that were given in the How do I add a delay in a JavaScript loop? that supposedly has answers for my question but they have not worked for me.

Either I get the same thing that I tried (one wait function that runs once and that does not solve my problem). Or upon running the script, the script finishes with no output neither in the cli, nor in the Result.txt file that I want my results to be written into.

var path = require('path');
var fs = require('fs');
var VirusTotalApi = require("virustotal-api");
var virusTotal = new VirusTotalApi('<YOUR API KEY>');

fs.readdir('/home/username/Desktop/TEST/', function (err,files) {
  if (err) {
    return console.log('Unable to scan directory: ' +err);
  }

  files.forEach(function (file) {
    var directoryPath = path.join('/home/username/Desktop/TEST/', file);

    fs.readFile(directoryPath, (err, data) => {
      if (err) {
        console.log(`Cannot read file. ${err}`);
      } else {
        console.log(file);
        virusTotal
        .fileScan(data, `${file}`)
        .then(response => {
          let resource = response.resource;
      virusTotal.fileReport(resource).then(result => {
            fs.writeFile('Result.txt', `${file}: ` + JSON.stringify(result, ["verbose_msg","total","positives"]) + '\n', function (err) {
              if (err) throw err;
              if(!err) {
                console.log(`${file}: Saved!`);
              }
            }); 
          });
        })
        .catch(err => console.log(`Scan failed. ${err}`));
        if(!err) {
          console.log('Scan succesful');
        }
      }
    });
  });
});

Once more, I need a ForEach loop that would wait for 20 seconds, and then run the desired code. It would also need to wait before going on to the next for condition.

Answer 1

You don't solve asynchronous issues by inserting pauses and you can't pause a .forEach() loop anyway. Instead, you need to write the type of asynchronous code that gives you control over the sequencing and completion of your asynchronous operations.

Using promises for all asynchronous operations and avoiding the non-asynchronous-aware loop constructs such as .forEach() will make this a lot easier. Here's one way to approach it:

const path = require('path');
const fsp = require('fs').promises;
const VirusTotalApi = require("virustotal-api");
const virusTotal = new VirusTotalApi('<YOUR API KEY>');

const basePath = '/home/username/Desktop/TEST/';

async function scan() {
    const files = await fsp.readdir(basePath);
    let errors = [];
    for (let file of files) {
        const fullPath = path.join(basePath, file);
        try {
            const data = await fsp.readFile(fullPath);
            const response = await virusTotal.fileScan(data, file);
            const resource = response.resource;
            const result = await virusTotal.fileReport(resource);
            const resultLine = `${file}: ${JSON.stringify(result)}\n`;
            await fsp.appendFile('Result.txt', resultLine);
        } catch (e) {
            // collect the error, log the error and continue the loop
            e.fullPath = fullPath;
            errors.push(e);
            console.log(`Error processing ${fullPath}`, e);
            continue;
        }
    }
    // if there was an error, then reject with all the errors we got
    if (errors.length) {
        let e = new Error("Problems scanning files");
        e.allErrors = errors;
        throw e;
    }
}

scan().then(() => {
    console.log("all done scanning - no errors");
}).catch(err => {
    console.log(err);
});

Also, where you were using fs.writeFile('Result.txt', ...), I changed it to use .appendFile() because it appears you want to accumulate a log of results, not overwrite all previous results each time.

Answer 2

Based on the https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/forEach

As I see it is impossible to wait for 20 seconds between each callback call by using forEach function. The forEach does not work with Promises and thus it is impossible to wait inside of it.

There is exists a solution, it uses for loops and async container.

var path = require('path');
var fs = require('fs');
var VirusTotalApi = require("virustotal-api");
var virusTotal = new VirusTotalApi('<YOUR API KEY>');

const wait = (time) => new Promise((resolve) => setTimeout(resolve, time))

fs.readdir('/home/username/Desktop/TEST/', async function (err,files) {
  if (err) {
    return console.log('Unable to scan directory: ' +err);
  }

  for (let file of files) {
    var directoryPath = path.join('/home/username/Desktop/TEST/', file);

    fs.readFile(directoryPath, (err, data) => {
      if (err) {
        console.log(`Cannot read file. ${err}`);
      } else {
        console.log(file);
        virusTotal
        .fileScan(data, `${file}`)
        .then(response => {
          let resource = response.resource;
      virusTotal.fileReport(resource).then(result => {
            fs.writeFile('Result.txt', `${file}: ` + JSON.stringify(result, ["verbose_msg","total","positives"]) + '\n', function (err) {
              if (err) throw err;
              if(!err) {
                console.log(`${file}: Saved!`);
              }
            }); 
          });
        })
        .catch(err => console.log(`Scan failed. ${err}`));
        if(!err) {
          console.log('Scan succesful');
        }
      }
    });

    // Wait for 20 seconds
    await wait(20 * 1000)
  }
});