Backup script writen in node.js for AWS S3 make synchronous to backup millions of files

Question

Many search on google, stackoverflow and bing I have made but I have no answer find for my problem.

How can I make my backup script to run synchronous. I have the problem with too many files in the folder and subfolders stop the script as result of too many open files (fs.createReadStream).

I hope somebody can help me. Thank you. Greetings Sven

var AWS = require('aws-sdk')
        , s3 = new AWS.S3()
        , fs = require('fs')
        , wrench = require('wrench')
        , util = require('util')

        var smadir = "/Users/myuser/myfolder"
          , smafiles = wrench.readdirSyncRecursive(smadir)

          smafiles.forEach (function (file) {
             var params = {Bucket: 'mybucked', Key: file, Body: fs.createReadStream(smadir + '/' + file)};
             var options = {partSize: 10 * 1024 * 1024, queueSize: 1};
             s3.upload(params, options, function(err, data) {
               console.log(err, data);
             })
          })

[async.eachSeries](https://github.com/caolan/async) – wayne Sep 04 '15 at 03:48 — wayne, Sep 04 '15 at 03:48

score 1 · Answer 1 · answered Sep 04 '15 at 05:11

You can manually write code that controls how many are uploading at once like this:

var AWS = require('aws-sdk')
    , s3 = new AWS.S3()
    , fs = require('fs')
    , wrench = require('wrench')
    , util = require('util')

var smadir = "/Users/myuser/myfolder"
    , smafiles = wrench.readdirSyncRecursive(smadir);

var index = 0, maxAtOnce = 10;

function uploadNext() {
    if (index >= smafiles.length) return;
    var file = smafiles[index++];
    var params = {Bucket: 'mybucked', Key: file, Body: fs.createReadStream(smadir + '/' + file)};
    var options = {partSize: 10 * 1024 * 1024, queueSize: 1};
    s3.upload(params, options, function(err, data) {
        console.log(err, data);
        // need to decide what to do here if an error occurred

        // since one just finished, launch another one
        uploadNext();
    });
}

// now launch the initial set
for (var i = 0; i < maxAtOnce; i++) {
    uploadNext();
}

Or, you can use a library like Async or Bluebird that has functions to manage the max number of parallel operations in flight at the same time.

For the Async library, you could use the .eachLimit() method which iterates an array, but limits the max number of operations in flight at the same time.

For the Bluebird promise library, you could use Promise.map() and specify the concurrency option to control the max number of operations in flight at the same time.

Thank you for your example. When I the Script run in a directory with very many files I get following error: myfile.js events.js:85 throw er; // Unhandled 'error' event ^ Error: EISDIR, read at Error (native) How I can fix this? — sm_a, Sep 04 '15 at 12:07
@sm_a - see http://stackoverflow.com/questions/20417118/using-node-js-i-get-error-eisdir-read. It sounds like you're trying to open a directory for reading - probably in the wrench code. — jfriend00, Sep 04 '15 at 15:20
@sm_a - In looking at the wrench code, it looks to me like `readdirSyncRecursive()` returns sub-directory names too,so you are likely calling `fs.createReadStream(smadir + '/' + file)` on a sub-directory which will return EISDIR. You will have to check to see if it's a directory before attempting to read it. This seems like a goof in the wrench library since the usual point of a recursive listing is to get all the files and to not have to check each one to see if it's a directory. — jfriend00, Sep 04 '15 at 15:44

score 0 · Answer 2 · answered Sep 04 '15 at 22:53

I have complete the solution. Here my code. Greetings Sven

var AWS = require('aws-sdk')
    , s3 = new AWS.S3()
    , fs = require('fs')
    , wrench = require('wrench')
    , util = require('util')

var smadir = "/home/user/folder"
  , smas3rootdir = "destination_folder"
  , smafiles = wrench.readdirSyncRecursive(smadir)

var index = 0, maxAtOnce = 1;

function uploadNext() {
    if (index >= smafiles.length) return;
    var file = smafiles[index++];
    var smafoldertocheck = smadir + '/' + file
    var smaprepare = fs.statSync(smafoldertocheck).isDirectory()

    if (!smaprepare) {
        var params = {Bucket: 'mybucked', Key: smas3rootdir + '/' + file, Body: fs.createReadStream(smadir + '/' + file)};
        var options = {partSize: 10 * 1024 * 1024, queueSize: 1};
        s3.upload(params, options, function(err, data) {
            console.log(err, data);
            // need to decide what to do here if an error occurred

            // since one just finished, launch another one
            uploadNext();
        })
    } else {
        uploadNext();
    }
}

// now launch the initial set
for (var i = 0; i < maxAtOnce; i++) {
    uploadNext();
}

Backup script writen in node.js for AWS S3 make synchronous to backup millions of files

2 Answers2