How to avoid blockin while uploading file using Meteor method

Question

I've created a Meteor method to upload a file, it's working well but until the file is fully uploaded, I cannot move around, all subscriptions seem to wait that the upload finishes... is there a way to avoid that ?

Here is the code on the server :

Meteor.publish('product-photo', function (productId) {
    return Meteor.photos.find({productId: productId}, {limit: 1});
});

Meteor.methods({
    /**
     * Creates an photo
     * @param obj
     * @return {*}
     */
    createPhoto: function (obj) {
        check(obj, Object);

        // Filter attributes
        obj = filter(obj, [
            'name',
            'productId',
            'size',
            'type',
            'url'
        ]);

        // Check user
        if (!this.userId) {
            throw new Meteor.Error('not-connected');
        }

        // Check file name
        if (typeof obj.name !== 'string' || obj.name.length > 255) {
            throw new Meteor.Error('invalid-file-name');
        }

        // Check file type
        if (typeof obj.type !== 'string' || [
                'image/gif',
                'image/jpg',
                'image/jpeg',
                'image/png'
            ].indexOf(obj.type) === -1) {
            throw new Meteor.Error('invalid-file-type');
        }

        // Check file url
        if (typeof obj.url !== 'string' || obj.url.length < 1) {
            throw new Meteor.Error('invalid-file-url');
        }

        // Check file size
        if (typeof obj.size !== 'number' || obj.size <= 0) {
            throw new Meteor.Error('invalid-file-size');
        }

        // Check file max size
        if (obj.size > 1024 * 1024) {
            throw new Meteor.Error('file-too-large');
        }

        // Check if product exists
        if (!obj.productId || Meteor.products.find({_id: obj.productId}).count() !== 1) {
            throw new Meteor.Error('product-not-found');
        }

        // Limit the number of photos per user
        if (Meteor.photos.find({productId: obj.productId}).count() >= 3) {
            throw new Meteor.Error('max-photos-reached');
        }

        // Resize the photo if the data is in base64
        if (typeof obj.url === 'string' && obj.url.indexOf('data:') === 0) {
            obj.url = resizeImage(obj.url, 400, 400);
            obj.size = obj.url.length;
            obj.type = 'image/png';
        }

        // Add info
        obj.createdAt = new Date();
        obj.userId = this.userId;

        return Meteor.photos.insert(obj);
    }
});

And the code on the client :

Template.product.events({
 'change [name=photo]': function (ev) {
        var self = this;
        readFilesAsDataURL(ev, function (event, file) {
            var photo = {
                name: file.name,
                productId: self._id,
                size: file.size,
                type: file.type,
                url: event.target.result
            };

            Session.set('uploadingPhoto', true);

            // Save the file
            Meteor.call('createPhoto', photo, function (err, photoId) {
                Session.set('uploadingPhoto', false);

                if (err) {
                    displayError(err);
                } else {
                    notify(i18n("Transfert terminé pour {{name}}", photo));
                }
            });
        });
    }
 });

Answer 1

I finally found the solution myself.

Explication : the code I used was blocking the subscriptions because it was using only one method call to transfer all the file from the first byte to the last one, that leads to block the thread (I think, the one reserved to each users on the server) until the transfer is complete.

Solution : I splitted the file into chunks of about 8KB, and send chunk by chunk, this way the thread or whatever was blocking the subscriptions is free after each chunk transfer.

The final working solution is on that post : How to write a file from an ArrayBuffer in JS

Client Code

// data comes from file.readAsArrayBuffer();
var total = data.byteLength;
var offset = 0;

var upload = function() {
  var length = 4096; // chunk size

  // adjust the last chunk size
  if (offset + length > total) {
     length = total - offset;
  }

  // I am using Uint8Array to create the chunk
  // because it can be passed to the Meteor.method natively
  var chunk = new Uint8Array(data, offset, length);

  if (offset < total) {
     // Send the chunk to the server and tell it what file to append to
     Meteor.call('uploadFileData', fileId, chunk, function (err, length) {
        if (!err) {
          offset += length;
          upload();
        }
     }
  }
};
upload();

Server code

var fs = Npm.require('fs');
var Future = Npm.require('fibers/future');

Meteor.methods({
  uploadFileData: function(fileId, chunk) {
    var fut = new Future();
    var path = '/uploads/' + fileId;

    // I tried that with no success
    chunk = String.fromCharCode.apply(null, chunk);

    // how to write the chunk that is an Uint8Array to the disk ?
    fs.appendFile(path, new Buffer(chunk), function (err) {
        if (err) {
          fut.throw(err);
        } else {
          fut.return(chunk.length);
        }
    });
    return fut.wait();
  }
});

Answer 2

Improving @Karl's code:

Client

This function breaks the file into chunks and sends them to the server one by one.

  function uploadFile(file) {
    const reader = new FileReader();
    let _offset = 0;
    let _total = file.size;

    return new Promise((resolve, reject) => {

      function readChunk() {
        var length = 10 * 1024; // chunk size

        // adjust the last chunk size
        if (_offset + length > _total) {
          length = _total - _offset;
        }

        if (_offset < _total) {
          const slice = file.slice(_offset, _offset + length);
          reader.readAsArrayBuffer(slice);
        } else {
          // EOF
          setProgress(100);
          resolve(true);
        }
      }

      reader.onload = function readerOnload() {
        let buffer = new Uint8Array(reader.result) // convert to binary
        Meteor.call('fileUpload', file.name, buffer, _offset,
          (error, length) => {
            if (error) {
              console.log('Oops, unable to import!');
              return false;
            } else {
              _offset += length;
              readChunk();
            }
          }
        );
      };

      reader.onloadend = function readerOnloadend() {
        setProgress(100 * _offset / _total);
      };

      readChunk();
    });
  }

Server

The server then writes to a file when offset is zero, or appends to its end otherwise, returning a promise, as I used an asynchronous function to write/append in order to avoid blocking the client.

if (Meteor.isServer) {
  var fs = require('fs');
  var Future = require('fibers/future');
}

Meteor.methods({
  // Upload file from client to server
  fileUpload(
    fileName: string,
    fileData: Uint8Array,
    offset: number) {
    check(fileName, String);
    check(fileData, Uint8Array);
    check(offset, Number);

    console.log(`[x] Received file ${fileName} data length: ${fileData.length}`);

    if (Meteor.isServer) {
      const fut = new Future();
      const filePath = '/tmp/' + fileName;
      const buffer = new Buffer(fileData);
      const jot = offset === 0 ? fs.writeFile : fs.appendFile;
      jot(filePath, buffer, 'binary', (err) => {
        if (err) {
          fut.throw(err);
        } else {
          fut.return(buffer.length);
        }
      });
      return fut.wait();
    }
  }
)};

Usage

uploadFile(file)
      .then(() => {
        /* do your stuff */
      });