Await an iterative function without delimiter in JS

Question

I've got a directory with an unknown amount of subfolders. Each subfolder might have or not further subfolders. I am itterating through them using a recursive function. Due to the unknown amounts of subfolders I am missing a way to make sure that all folders have been checked before I continue. My knowledge on async and await is quiet limited. Is there any Way to handle this problem?

function searchForPackage(directory){
    fs.readdir(directory, function(err, files){
        if(err){
            return;
        }else{
            files.forEach(file => {
                var currentLocation = directory + "/" + file;
                if(fs.statSync(currentLocation).isDirectory() && file != 'bin' && file != '.bin'){
                    searchForPackage(currentLocation);
                    return;
                }else if(file == "package.json"){
                    var content = fs.readFileSync(currentLocation);
                    var jsonContent = JSON.parse(content);
                    var obj = {
                        name: jsonContent.name,
                        license: jsonContent.license,
                        version: jsonContent.version
                    }
                    jsonTable.push(obj);
                    jsonTable.push({name: jsonContent.name, license: jsonContent.license, version: jsonContent.version});
                    return;
                }
            })
        }
    })
  }

There is no async and await yet nor is there a promise...Thats why I'm asking...i have no clue how to apply asynchronity here — Jannik Mottulla, Mar 18 '19 at 11:52
@JannikMottulla [Find a way to get a promise](https://stackoverflow.com/q/22519784/1048572) for `fs.readdir(…)`, then [use `await` in the loop](https://stackoverflow.com/a/37576787/1048572). Please try something. — Bergi, Mar 18 '19 at 12:07
@Bergi The filestream is sync. My problem is that i dunno how to figure out when my function ran through all subfolders. — Jannik Mottulla, Mar 18 '19 at 12:19
`readFileSync` is sync, `readdir` is not. Make a promise and await it. — Bergi, Mar 18 '19 at 12:23

score 0 · Accepted Answer · answered Mar 18 '19 at 13:24

You have a few options:

1) Since everything else is done using fs's synchronous methods, you could change fs.readdir to fs.readdirSync:

function searchForPackage(directory) {
  fs.readdirSync(directory).forEach(file => {
    var currentLocation = directory + "/" + file;
    if (fs.statSync(currentLocation).isDirectory() && file != 'bin' && file != '.bin') {
      searchForPackage(currentLocation);
      return;
    } else if (file == "package.json") {
      var content = fs.readFileSync(currentLocation);
      var jsonContent = JSON.parse(content);
      var obj = {
        name: jsonContent.name,
        license: jsonContent.license,
        version: jsonContent.version
      }
      jsonTable.push(obj);
      jsonTable.push({name: jsonContent.name, license: jsonContent.license, version: jsonContent.version});
      return;
    }
  })
}

2) Convert fs.readdirSync to a Promise and then use async/await:

async function searchForPackage(directory) {
  const files = await new Promise((resolve, reject) => {
    fs.readdir(directory, (err, files) => {
      if (err) reject(err);
      else resolve(files);
    });
  });
  await Promise.all(files.map(async file => {
    var currentLocation = directory + "/" + file;
    if (fs.statSync(currentLocation).isDirectory() && file != 'bin' && file != '.bin') {
      await searchForPackage(currentLocation);
      return;
    } else if (file == "package.json") {
      var content = fs.readFileSync(currentLocation);
      var jsonContent = JSON.parse(content);
      var obj = {
        name: jsonContent.name,
        license: jsonContent.license,
        version: jsonContent.version
      }
      jsonTable.push(obj);
      jsonTable.push({name: jsonContent.name, license: jsonContent.license, version: jsonContent.version});
      return;
    }
  }))
}

3) Use a couple third-party modules to clean things up a bit (fs-extra takes care of promisifying asynchronous methods like fs.readdir for you. async-af provides chainable asynchronous JavaScript methods such as a parallel forEach.):

const fs = require('fs-extra');
const AsyncAF = require('async-af');

async function searchForPackage(directory) {
  await AsyncAF(fs.readdir(directory)).forEach(async file => {
    var currentLocation = directory + "/" + file;
    if (fs.statSync(currentLocation).isDirectory() && file != 'bin' && file != '.bin') {
      await searchForPackage(currentLocation);
    } else if (file == "package.json") {
      var content = fs.readFileSync(currentLocation);
      var jsonContent = JSON.parse(content);
      var obj = {
        name: jsonContent.name,
        license: jsonContent.license,
        version: jsonContent.version
      }
      jsonTable.push(obj);
      jsonTable.push({name: jsonContent.name, license: jsonContent.license, version: jsonContent.version});
    }
  });
}

Thanks for your help! But I guess i failed at explaining my Problem...My code is running fine so far. I am writing the extracted obj's into an array. If all folders have been checked I'd like to parse the array of obj's in .json. I need a way to wait for all folders to be checked (not loaded), otherwise i would get an empty array. — Jannik Mottulla, Mar 18 '19 at 13:47
I tested on one of my projects with multiple `package.json`s and it worked; the trick is (if you're using `fs.readdir` and not `fs.readdirSync`) you have to `await` running the function so the array has time to populate. For example: `(async () => { await searchForPackage('./'); console.log(jsonTable); })()` — Scott Rudiger, Mar 18 '19 at 13:54

Mulan · Answer 2 · 2019-05-25T17:46:11.003

I would suggest building smaller functions with isolated concerns. Start with a files function that simply returns all files and the files of all sub-directories -

const { readdir, stat } =
  require ("fs") .promises

const { join } =
  require ("path")

const files = async (path = ".") =>
  (await stat (path)) .isDirectory ()
    ? Promise
        .all
          ( (await readdir (path))
              .map (f => files (join (path, f)))
          )
        .then
          ( results =>
             [] .concat (...results)
          )
    : [ path ]

files () .then (console.log, console.error)

// [ './.main.js'
// , './node_modules/anotherpackage/README.md'
// , './node_modules/anotherpackage/package.json'
// , './node_modules/anotherpackage/index.js'
// , './node_modules/somepackage/.npmignore'
// , './node_modules/somepackage/LICENSE'
// , './node_modules/somepackage/README.md'
// , './node_modules/somepackage/package.json'
// , './node_modules/somepackage/index.js'
// , './node_modules/somepackage/test/test.js'
// , './package.json'
// ]

Then make a search function which depends on files and adds the capability to filter results -

const { basename } =
  require ("path")

const search = async (query, path = ".") =>
  (await files (path))
    .filter (x => basename (x) === query)

search ("package.json", ".")
  .then (console.log, console.error)

// [ './node_modules/anotherpackage/package.json'
// , './node_modules/somepackage/package.json'
// , './package.json'
// ]

Then make your readPackages function which depends on search and adds the capability to read/parse the packages -

const { readFile } =
  require ("fs") .promises

const readPackages = async (path = ".") =>
  Promise
    .all
      ( (await search ("package.json", path))
          .map (package => readFile (package))
      )
    .then
      ( buffers =>
          buffers .map (b => JSON .parse (String (b)))
      )

readPackages ('.')
  .then (console.log, console.error)

// [ <contents of anotherpackage/package.json>
// , <contents of somepackage/package.json>
// , <contents of package.json>
// ]

Finally, notice how jsonTable is no longer a global. Instead all data is nicely contained and flowing through our sequence of promises.

If you'd like the transform the packages as you're reading them, you can make transform a parameter of the readPackages function. This keeps it generic and allows you to read package contents in a user-specified way -

const readPackages = async (transform, path = ".") =>
  Promise
    .all
      ( (await search ("package.json", path))
          .map (package => readFile (package))
      )
    .then
      ( buffers =>
          buffers .map (b => transform (JSON .parse (String (b))))
      )

readPackages
  ( ({ name }) => ({ name }) 
  , '.'
  )
  .then (console.log, console.error)

// [ { name: 'anotherpackage' }
// , { name: 'somepackage' }
// , { name: 'mypackage' }
// ]

Or get name, version, and license -

readPackages
  ( ({ name, version, license = "None" }) =>
      ({ name, version, license }) 
  , '.'
  )
  .then (console.log, console.error)

// [ { name: 'anotherpackage', version: '1.0.0', license: 'None' }
// , { name: 'somepackage', version: '3.2.1', license: 'MIT' }
// , { name: 'mypackage', version: '1.2.3', license: 'BSD-3-Clause' }
// ]

Now in these simplified programs, we start to see some patterns emerging. To make our intentions more clear and avoid repeating ourselves, we design a reusable module -

const Parallel = p =>
  ( { map: async f =>
        Promise .all ((await p) .map (x => f (x)))
    , filter: async f =>
        (await p) .filter (x => f (x))
    , flatMap: async f =>
        Promise .all ((await p) .map (x => f (x))) .then (ys => [] .concat (...ys))
    , // ...
    }
  )

Now our files function is a lot nicer -

const files = async (path = ".") =>
  (await stat (path)) .isDirectory ()
    ? Parallel (readdir (path))
        .flatMap (f => files (join (path, f)))
    : [ path ]

Our search function is cleaned up a bit too -

const search = async (query, path = ".") =>
  Parallel (files (path))
    .filter (x => basename (x) === query)

Finally, readPackages -

const readPackages = async (f, path = ".") =>
  Parallel (search ("package.json", path))
    .map (readFile)
    .then
      ( buffers =>
          buffers .map (b => f (JSON .parse (String (b))))
      )

Behavior of each function is identical to the original implementations. Only now we have even more generic functions which can be reused in other areas of our program.

In this related Q&A, we use the Parallel module to implement a dirs function which recursively lists all directories at a given path.

Await an iterative function without delimiter in JS

2 Answers2

Linked