NodeJS Hash files using klaw recursive function in a directory

Question

I'm trying to hash every single file in a directory and then print the output into a .txt file on nodeJS. But the problem I'm facing is that the hashes are print at the wrong places in the .txt file.

To access each and every single files in the directory, I used Klaw API https://dustinpfister.github.io/2018/07/19/nodejs-klaw/?. I referred to NodeJS hash files recursively in a directory for hashing each file recursively. I suspected that the problem was because the console.log is not standardized especially when calling in a function console.log() async or sync??. I found a few solutions on the web which e.g setTimeout (but I do not know how to apply it in a case where the function has a throw and catch), I will appreciate if there is any help given!

This is my code on NodeJS

let klaw = require('klaw'),
path = require('path'),
// the dir to walk
dir_walk = process.argv[2] || process.cwd();

var crypto = require('crypto');
var fs = require('fs');

// walking dir_walk with the following options
klaw(dir_walk, {

    // default to full recursion, if now depth is given
    depthLimit: process.argv[3] || -1

}).on('data', function (item) {

    if (!item.stats.isDirectory()) {
        //hash function
        generateHash(item.path, function (e, hash) {
       if (e) done(e);
        console.log('Hash : ' + hash);
      });
        console.log('\nType : File');
        console.log('Name : ' + path.basename(item.path));
        console.log('Path: ' + item.path); // the full absolute path of of the item
        console.log('Mtime: ' + item.stats.mtime); 
        console.log('Ctime: ' + item.stats.ctime);// the stats of the item 
    }
    else{
        console.log('\nType : Folder');
        console.log('Name : ' + path.basename(item.path));
        console.log('Path: ' + item.path);
        console.log('Mtime: ' + item.stats.mtime); 
        console.log('Ctime: ' + item.stats.ctime);
    }

})
//Function for generating hashes

 function generateHash (filename, callback) {
    var algorithm = 'sha256';
    var hashTable = new Array();

    var hash = crypto.createHash(algorithm);
    var fileStream = fs.ReadStream(filename);

    fileStream.on('data', function(data) {
        hash.update(data);      
    });
    fileStream.on('end', function() {
        var digest = hash.digest('hex');
        callback(null, digest);
    });
}

The output for my directory is

Type : Folder
Name : TESTING
Path: /Users/*/Documents/TESTING
Mtime: Tue Jun 11 2019 16:09:29 GMT+0800 (Singapore Standard Time)
Ctime: Tue Jun 11 2019 16:09:29 GMT+0800 (Singapore Standard Time)


Type : File
Name : .DS_Store
Path: /Users/*/Documents/TESTING/.DS_Store
Mtime: Tue Jun 11 2019 15:44:03 GMT+0800 (Singapore Standard Time)
Ctime: Tue Jun 11 2019 15:44:03 GMT+0800 (Singapore Standard Time)


Type : File
Name : basic.js
Path: /Users/*/Documents/TESTING/basic.js
Mtime: Tue Jun 11 2019 16:09:08 GMT+0800 (Singapore Standard Time)
Ctime: Tue Jun 11 2019 16:09:08 GMT+0800 (Singapore Standard Time)


Type : File
Name : basicv1.js
Path: /Users/*/Documents/TESTING/basicv1.js
Mtime: Tue Jun 11 2019 17:37:02 GMT+0800 (Singapore Standard Time)
Ctime: Tue Jun 11 2019 17:37:02 GMT+0800 (Singapore Standard Time)
Hash : 1ebe514c6032f1bcb6c50a0e07fde487d8a38ca2a2a67948198bf48bc8877951
Hash : 56d7fadb2f8d37eda3986c73726fecb4469e84ac1fbe0b1108f2d141bb8a509f

Type : Folder
Name : folder
Path: /Users/*/Documents/TESTING/folder
Mtime: Sat Jun 08 2019 15:36:35 GMT+0800 (Singapore Standard Time)
Ctime: Tue Jun 11 2019 15:43:20 GMT+0800 (Singapore Standard Time)
Hash : 038789863d605a4f6219bbd0c087a32876e633ff4e678d1744d423e9abca1260

But the actual output I should be expecting is

Type : Folder
Name : TESTING
Path: /Users/*/Documents/TESTING
Mtime: Tue Jun 11 2019 16:09:29 GMT+0800 (Singapore Standard Time)
Ctime: Tue Jun 11 2019 16:09:29 GMT+0800 (Singapore Standard Time)


Type : File
Name : .DS_Store
Path: /Users/*/Documents/TESTING/.DS_Store
Mtime: Tue Jun 11 2019 15:44:03 GMT+0800 (Singapore Standard Time)
Ctime: Tue Jun 11 2019 15:44:03 GMT+0800 (Singapore Standard Time)
Hash : 1ebe514c6032f1bcb6c50a0e07fde487d8a38ca2a2a67948198bf48bc8877951


Type : File
Name : basic.js
Path: /Users/*/Documents/TESTING/basic.js
Mtime: Tue Jun 11 2019 16:09:08 GMT+0800 (Singapore Standard Time)
Ctime: Tue Jun 11 2019 16:09:08 GMT+0800 (Singapore Standard Time)
Hash : 56d7fadb2f8d37eda3986c73726fecb4469e84ac1fbe0b1108f2d141bb8a509f

Type : File
Name : basicv1.js
Path: /Users/*/Documents/TESTING/basicv1.js
Mtime: Tue Jun 11 2019 17:37:02 GMT+0800 (Singapore Standard Time)
Ctime: Tue Jun 11 2019 17:37:02 GMT+0800 (Singapore Standard Time)
Hash : 038789863d605a4f6219bbd0c087a32876e633ff4e678d1744d423e9abca1260

Type : Folder
Name : folder
Path: /Users/*/Documents/TESTING/folder
Mtime: Sat Jun 08 2019 15:36:35 GMT+0800 (Singapore Standard Time)
Ctime: Tue Jun 11 2019 15:43:20 GMT+0800 (Singapore Standard Time)

score 1 · Answer 1 · answered Jun 11 '19 at 10:27

1

generateHash is async. You must wait for result and after print like this

// walking dir_walk with the following options
klaw(dir_walk, {

    // default to full recursion, if now depth is given
    depthLimit: process.argv[3] || -1

}).on('data', function (item) {

    if (!item.stats.isDirectory()) {
        //hash function
        generateHash(item.path, function (e, hash) {
            if (e) done(e);
            console.log('\nType : File');
            console.log('Name : ' + path.basename(item.path));
            console.log('Path: ' + item.path); // the full absolute path of of the item
            console.log('Mtime: ' + item.stats.mtime);
            console.log('Ctime: ' + item.stats.ctime); // the stats of the item 
            console.log('Hash : ' + hash);
        });
    } else {
        console.log('\nType : Folder');
        console.log('Name : ' + path.basename(item.path));
        console.log('Path: ' + item.path);
        console.log('Mtime: ' + item.stats.mtime);
        console.log('Ctime: ' + item.stats.ctime);
    }

})

answered Jun 11 '19 at 10:27

Yaroslav Gaponov

1,997
13
12

Really thank you for your help! This works! I'm sorry if I'm slow in understanding async – Leow Zheng Xu Jun 11 '19 at 10:33
In general I propose switch it on promise or/and await/async for more clear code. – Yaroslav Gaponov Jun 11 '19 at 10:34
But in this case may be more suitable is using stream, and you code will be like klaw(dir_walk, {}).pipe(new YourLogStream()); – Yaroslav Gaponov Jun 11 '19 at 10:37
I will try to look up on that! Thanks again for the advice for code optimization – Leow Zheng Xu Jun 11 '19 at 10:39
klaw(dir_walk, {}).pipe(new CalcHashStream).pipe(new YourLogStream) – Yaroslav Gaponov Jun 11 '19 at 10:45

NodeJS Hash files using klaw recursive function in a directory

1 Answers1