Strongloop loopback-storage-service: how to use StorageService.downloadStream() to download a ZIP of all photos in container?

Question

I'm using loopback-component-storage with the filesystem provider to upload photos. It works great. But now I have a request to 'export/download' photos to a zip archive.

I've put together some code to add another method, downloadContainer() to Container from the file-storage example. It uses the Archiver module, everything seems to work fine, but the browser crashes after I call zip.finalize() I'm expecting to get a save file dialog box instead...

Here is my code so far:

Container.downloadContainer = function(container, files, res, cb) {
  var DELIM, _appendFile, _finalize, _oneComplete, _remaining, filenames, storageService, zip;
  zip = Archiver('zip');
  zip.pipe(res);
  storageService = this;
  _remaining = {};
  _appendFile = function(zip, container, filename) {
    var reader;
    console.log('appendFile=' + filename);
    reader = storageService.downloadStream(container, filename, function(stream) {
      return console.log('storageService.downloadStream() resp=', _.keys(stream));
    });
    zip.append(reader, {
      name: filename
    });
  };
  zip.on('error', function(err) {
    console.log('zip entry error', err);
    res.status(500).send({
      error: err.message
    });
  });
  zip.on('entry', function(o) {
    return _oneComplete(o.name);
  });
  _oneComplete = function(filename) {
    delete _remaining[filename];
    console.log('_oneComplete(): ', {
      remaining: _.keys(_remaining),
      size: zip.pointer()
    });
    if (_.isEmpty(_remaining)) {
      _finalize();
    }
  };
  _finalize = function() {
    console.log('calling zip.finalize() ...');
    res.on('close', function() {
      console.log('response closed');
      res.attachment(container + '.zip');
      return res.status(200).send('OK').end();
    });
    zip.finalize();
  };
  if (files === 'all' || _.isEmpty(files)) {
    console.log('files=', files);
    storageService.getFiles(container, function(err, ssFiles) {
      _remaining = _.object(_.pluck(ssFiles, 'name'));
      console.log('filenames=', _.keys(_remaining));
      return ssFiles.forEach(function(file) {
        _appendFile(zip, container, file.name);
      });
    });
  }
}

and here is what I see in the console

    // log
    files= all
    filenames= [ 'IMG_0799.PNG', 'IMG_0800.PNG', 'IMG_0801.PNG', 'IMG_0804.PNG' ]
    appendFile=IMG_0799.PNG
    appendFile=IMG_0800.PNG
    appendFile=IMG_0801.PNG
    appendFile=IMG_0804.PNG
    _oneComplete():  { remaining: [ 'IMG_0800.PNG', 'IMG_0801.PNG', 'IMG_0804.PNG' ],
      size: 336110 }
    _oneComplete():  { remaining: [ 'IMG_0801.PNG', 'IMG_0804.PNG' ], size: 460875 }
    _oneComplete():  { remaining: [ 'IMG_0804.PNG' ], size: 1506464 }
    _oneComplete():  { remaining: [], size: 1577608 }
    calling zip.finalize() ...
    // then browser crash

Answer 1

This code expands on the suggestion from Bryan above and works with provider=filesystem

Archiver = require('archiver')
module.exports = function(Container) {
  Container.downloadContainer = function(container, files, res, cb) {
    var DELIM, _appendFile, _finalize, _oneComplete, _remaining, filenames, storageService, zip, zipFilename;
    zip = Archiver('zip');
    zipFilename = container + '.zip';
    storageService = this;
    _remaining = {};
    _appendFile = function(zip, container, filename) {
      var reader;
      reader = storageService.downloadStream(container, filename);
      zip.append(reader, {
        name: filename
      });
      console.log("appending", {
        name: filename
      });
    };
    res.on('close', function() {
      console.log('Archive wrote %d bytes', zip.pointer());
      return res.status(200).send('OK').end();
    });
    res.attachment(zipFilename);
    zip.pipe(res);
    zip.on('error', function(err) {
      console.log('zip entry error', err);
      res.status(500).send({
        error: err.message
      });
    });
    zip.on('entry', function(o) {
      return _oneComplete(o.name);
    });
    _oneComplete = function(filename) {
      delete _remaining[filename];
      console.log('_oneComplete(): ', {
        remaining: _.keys(_remaining),
        size: zip.pointer()
      });
      if (_.isEmpty(_remaining)) {
        _finalize();
      }
    };
    _finalize = function() {
      console.log('calling zip.finalize() ...');
      zip.finalize();
    };
    if (files === 'all' || _.isEmpty(files)) {
      console.log('downloadContainer, files=', files);
      storageService.getFiles(container, function(err, ssFiles) {
        _remaining = _.object(_.pluck(ssFiles, 'name'));
        return ssFiles.forEach(function(file) {
          _appendFile(zip, container, file.name);
        });
      });
    } else {
      DELIM = ',';
      filenames = files.split(DELIM);
      _remaining = _.object(filenames);
      console.log('filenames=', _.keys(_remaining));
      _.each(filenames, function(filename) {
        _appendFile(zip, container, filename);
      });
    }
  };
  Container.remoteMethod('downloadContainer', {
    shared: true,
    accepts: [
      {
        arg: 'container',
        type: 'string',
        'http': {
          source: 'path'
        }
      }, {
        arg: 'files',
        type: 'string',
        required: false,
        'http': {
          source: 'path'
        }
      }, {
        arg: 'res',
        type: 'object',
        'http': {
          source: 'res'
        }
      }
    ],
    returns: [],
    http: {
      verb: 'get',
      path: '/:container/downloadContainer/:files'
    }
  });

Answer 2

I haven't done this before but I think you're on the right path from your comment.

Something like this should work:

var AWS = app.dataSources.amazon;
var container = 'c1';

app.get('/export/download', function (req, res) {
  var zip = Archiver('zip');
  // create the Archiver and pipe it to our client response.
  zip.pipe(res);
  // ask AWS for all the files in the container
  AWS.getFiles(container, function (err, files) {
    files.forEach(function (file) {
      // append each file stream to the zip archive
      zip.append(AWS.download({
        container: container,
        remote: file
      }), { name : file });
    });
    // I think finalize should end the stream and notify the client
    zip.finalize();
  });
});

Let me know how it goes!