Asynchronously Perform Recursive Data Tree Construction?

Question

I am working on a web application that makes use of a file tree. The frontend JavaScript performs an ajax request to my Node.js server which calls my browse2 exported function. This function is then responsible for supplying the correct path to my function, getFolderContents(), that recursively builds the file system hierarchy object structure.

My issue is that I am currently doing things synchronously. Having done research into the inner workings of Node.js, it seems as though I should avoid synchronous operations at all costs. As such, I wanted to convert my code to work asynchronously. However, I couldn't get it working and all of my solutions were convoluted.

I have tried managing the flow using the "async" package. I had no luck with figuring that out. I tried implementing my own system of counters/loops/callbacks to determine when processes had finished executing. Ultimately, I suppose I can't wrap my mind around asynchronous execution flow.

I would like to ask two questions: 1. In this case, would it be detrimental to perform this request synchronously instead of asynchronously? 2. If yes to the first question, how should I go about converting this code to be asynchronous?

Note: When I tried to do things asynchronously, I used each synchronous function's asynchronous counterpart.

Below is my synchronous (working) code:

var path = require('path');
var fs = require('fs');

exports.browse2 = function(request, response) {
    var tree = getFolderContents('C:\\Users\\AccountName\\folder1\\folder2\\folder3\\test\\');

    response.send(tree);
};


function getFolderContents(route) {
    var branch = {};
    branch.title = path.basename(route);
    branch.folder = true;
    branch.children = [];

    var files = fs.readdirSync(route);
    var size = files.length;

    for (var i = 0; i < size; i++) {
        var file = files[i];
        var concatPath = path.join(route, file);

        if (fs.lstatSync(concatPath).isDirectory())
            branch.children.push(getFolderContents(concatPath));
        else
            branch.children.push({
                "title" : path.basename(file),
                "path" : file
            });
    }

    return branch;
}

I appreciate all input!

Edit:

Added asynchronous code attempt. Not fully working. Only a part of the tree is received.

    exports.browse2 = function(request, response) {
        getFolderContents(
                'C:\\Users\\AccountName\\folder1\\folder2\\folder3\\test\\',
                function(tree) {
                    response.send(tree);
                });
    };

function getFolderContents(route, callback) {
    var branch = {};
    branch.title = path.basename(route);
    branch.folder = true;
    branch.children = [];

    fs.readdir(route, function(err, files) {
        files.forEach(function(file) {
            var concatPath = path.join(route, file);

            fs.lstat(concatPath, function(err, stats) {
                if (stats.isDirectory())
                    branch.children.push(getFolderContents(concatPath, callback));
                else
                    branch.children.push({
                        "title" : path.basename(file),
                        "path" : file
                    });

                callback(branch);
            });         
        });
    });
}

Answer 1

The basic problem you're having is that when you use asynchronous calls, you can't just assign things to the return of the function. The entire point of async is that the function won't wait. So for example:

function get_data(a) {
    var data = some_async_call(a);

    //at this point, data is undefined because execution won't wait on the calls to finish

    data.do_something();  // this breaks because of the above
}

So instead what you do is pass an anonymous function to the asynchronous function called a callback, and the asynchronous function calls that function once the operations actually complete. The above example would become this:

function get_data(a) {
    some_async_call(a, function(data) {
        data.do_something();
    });
}

function some_async_call(variable, callback) {
    call_async({
        data: variable,
        success: callback
    });
}

And in your case that would look like this:

exports.browse2 = function(request, response) {
    getFolderContents('C:\\Users\\AccountName\\folder1\\folder2\\folder3\\test\\', function(tree) {
        response.send(tree);
    });
};

function getFolderContents(route, callback) {
    var branch = {};
    branch.title = path.basename(route);

    ...

    callback(branch);
}

If you're familiar with setTimetout, this is how that works - the design pattern is to pass an anonymous function that does the work, and that function then executes once the data/information is actually available.

Answer 2

I managed to get it working. Here are my answers to my own questions:

1. It is better to perform the tasks asynchronously because to do it otherwise would mean that the application would block other users from receiving their responses until subsequent requests have been responded to.

2. The way to convert the synchronous code to asynchronous code is to use a parallel loop. The code for my particular case is this:

   var path = require('path');
   var fs = require('fs');
   
   exports.browse2 = function(request, response) {
   getFolderContents(
           'C:\\Users\\AccountName\\folder1\\folder2\\folder3\\test\\',
           function(err, tree) {
               if (err)
                   throw err;
               response.send(tree);
           });
   };
   
   function getFolderContents(route, callback) {
   var branch = {};
   branch.title = path.basename(route);
   branch.folder = true;
   branch.children = [];
   
   fs.readdir(route, function(err, files) {
       if (err)
           return callback(err);
       var pending = files.length;
       if (!pending)
           return callback(null, branch);
       files.forEach(function(file) {
           var concatPath = path.join(route, file);
           fs.lstat(concatPath, function(err, stats) {
               if (stats && stats.isDirectory()) {
                   getFolderContents(concatPath, function(err, res) {
                       branch.children.push(res);
                       if (!--pending)
                           callback(null, branch);
                   });
               } else {
                   branch.children.push({
                       "title" : path.basename(file),
                       "path" : file
                   });
                   if (!--pending)
                       callback(null, branch);
               }
           });
       });
   });
   }
   

Thanks to user "chjj" with his response to a similar question on this thread: node.js fs.readdir recursive directory search

And thanks to user "Dan Smolinske" for directing me to the thread.