Zipping multiple files in Nodejs having size ~ 300kb each and streaming to client

Question

My code is working fine when I zip 3 files around 300kb each and send it to client. Used following links for help:

But as soon as I try to zip 4th file I get "download - Failed Network error" in chrome.

Following is my code:

var express = require('express');
var app = express();
var fileSystem = require('fs');
var Archiver = require('archiver');
var util = require('util');
var AdmZip = require('adm-zip');
var config = require('./config');
var log_file = fileSystem.createWriteStream(__dirname + '/debug.log', {flags : 'a'});

logError = function(d) { //
  log_file.write('[' + new Date().toUTCString() + '] ' + util.format(d) + '\n');
};

app.get('/zip', function(req, res, next) {

 try {
  res = setHeaderOfRes(res);
  sendZip(req, res);
  }catch (err) {
  logError(err.message);
  next(err);        // This will call the error middleware for 500 error
  }

});



var setHeaderOfRes = function (res){
  res.setHeader("Access-Control-Allow-Origin", "*"); //Remove this when this is on production
  res.setHeader("Content-Type", "application/zip");
  res.setHeader("Content-disposition", "attachment;");  
  return res;
};

var sendZip = function (req, res) {

var filesNotFound = [];
  zip.pipe(res);
  if (req.query.leapIds) {
    var leapIdsArray = req.query.leapIds.split(',');
    var i, lengthi;
    for (i = 0, lengthi = leapIdsArray.length; i < lengthi; i++) {
      try {
          var t = config.web.sharedFilePath + leapIdsArray[i] + '.vsdx';
        if (fileSystem.statSync(t).isFile()) {        

            zip.append(new fileSystem.createReadStream(t), {
            name: leapIdsArray[i] + '.vsdx'
          });
        };
      } catch (err) {
      filesNotFound.push(leapIdsArray[i] + '.vsdx');
      }
    }

    var k, lengthk;
    var str = '';
      for (k = 0, lengthk = filesNotFound.length; k < lengthk; k++) {
      str += filesNotFound[k] +',';
      }
      if(filesNotFound.length > 0){
    zip.append('These file/files does not exist on server - ' + str , { name: 'logFile.log' });
    }
       zip.finalize();

  }
};

I tried zip.file instead of zip.append that didn't work. I want to zip minimum 10 files of 300kb each and send it to the client. Can anyone please let me know the approach. Thanks

/********************* Update ****************************************

I was only looking at server.js created in node. Actually the data is sent correctly to client. Angularjs client code seems to be not working for large files.

$http.get(env.nodeJsServerUrl + "zip?leapIds=" + nodeDetails, { responseType: "arraybuffer" }

                    ).then(function (response) {
                        nodesDetails = response.data;

                      var   base64String = _arrayBufferToBase64(nodesDetails);

                        function _arrayBufferToBase64(buffer) {
                            var binary = '';
                            var bytes = new Uint8Array(buffer);
                            var len = bytes.byteLength;
                            for (var i = 0; i < len; i++) {
                                binary += String.fromCharCode(bytes[i]);
                            }
                            return window.btoa(binary);
                        }
                        var anchor = angular.element('<a/>');
                        anchor.attr({
                            href: 'data:application/zip;base64,' + base64String,
                            target: '_blank',
                            download: $scope.main.routeParams.sectorId + "-ProcessFiles.zip"
                        })[0].click();
                    });

This part href: 'data:application/zip;base64,' + base64String, seems to be failing for large data received from server. For small files it is working. Large files it is failing.

Doubt this is your problem, but get rid of the `new` in `new fileSystem.createReadStream`. You should be adding error listeners to your read streams. Finally, you're creating a potentially large number of streams feeding into your zip archiver, which can be problematic (large number of file descriptors, stream backpressure, etc.). Usually it's wise to hook into some event from the zip archiver that is triggered when one file is done, and then add the next file. — ZachB, Aug 23 '16 at 04:28
This answer might help someone: https://stackoverflow.com/a/62639710/8612027 — J0nh1dd3n, Jun 29 '20 at 14:11

score 0 · Answer 1 · edited May 23 '17 at 10:28

Found out.

The problem was not in nodejs zipping logic. That worked perfect. Issue was in the way I was handling the received response data.

If the data that is received is too large then following code fails

 anchor.attr({
        href: 'data:application/zip;base64,' + base64String,
        target: '_blank',
        download: $scope.main.routeParams.sectorId + "-ProcessFiles.zip"
    })[0].click();

so the work around is to use blob:

function b64toBlob(b64Data, contentType, sliceSize) {
        contentType = contentType || '';
        sliceSize = sliceSize || 512;

        var byteCharacters = atob(b64Data);
        var byteArrays = [];

        for (var offset = 0; offset < byteCharacters.length; offset += sliceSize) {
            var slice = byteCharacters.slice(offset, offset + sliceSize);

            var byteNumbers = new Array(slice.length);
            for (var i = 0; i < slice.length; i++) {
                byteNumbers[i] = slice.charCodeAt(i);
            }
            var byteArray = new Uint8Array(byteNumbers);
            byteArrays.push(byteArray);
        }

        var blob = new Blob(byteArrays, { type: contentType });
        return blob;
    }
    var contentType = 'application/zip'
    var blob = b64toBlob(base64String, contentType);                       

    saveAs(blob, "hello world.zip");

This link helped me out: How to save binary data of zip file in Javascript?

score 0 · Answer 2 · answered Jun 29 '20 at 14:13

already answered here: https://stackoverflow.com/a/62639710/8612027

Sending a zip file as binary data with expressjs and node-zip:

app.get("/multipleinzip", (req, res) => {
    var zip = new require('node-zip')();
    var csv1 = "a,b,c,d,e,f,g,h\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8";
    zip.file('test1.file', csv1);
    var csv2 = "z,w,x,d,e,f,g,h\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8";
    zip.file('test2.file', csv2);
    var csv3 = "q,w,e,d,e,f,g,h\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8";
    zip.file('test3.file', csv3);
    var csv4 = "t,y,u,d,e,f,g,h\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8\n1,2,3,4,5,6,7,8";
    zip.file('test4.file', csv4);
    var data = zip.generate({base64:false,compression:'DEFLATE'});
    console.log(data); // ugly data
    res.type("zip")
    res.send(new Buffer(data, 'binary'));
})

Creating a download link for the zip file. Fetch data and convert the response to an arraybuffer with ->

    //get the response from fetch as arrayBuffer...
    var data = response.arrayBuffer();

    const blob = new Blob([data]);
    const fileName = `${filename}.${extension}`;
    
    if (navigator.msSaveBlob) {
      // IE 10+
      navigator.msSaveBlob(blob, fileName);
    } else {
      const link = document.createElement('a');
      // Browsers that support HTML5 download attribute
      if (link.download !== undefined) {
        const url = URL.createObjectURL(blob);
        link.setAttribute('href', url);
        link.setAttribute('download', fileName);
        link.style.visibility = 'hidden';
        document.body.appendChild(link);
        link.click();
        document.body.removeChild(link);
      }
    }

Zipping multiple files in Nodejs having size ~ 300kb each and streaming to client

2 Answers2