$http.post: Large files do not work

Question

I am trying to upload files through my web app using the following code.

View:

  <form name="uploadForm" class="form-horizontal col-sm-12">
    <div class="col-sm-4">
      <input type="file" ng-model="rsdCtrl.viewData.file" name="file"/>
    </div>
    <div class="col-sm-4">
      <button class="btn btn-success" type="submit" ng-click="uploadFile()">Upload</button>
    </div>
  </form>

Controller:

function uploadFile(){
  if (uploadForm.file.$valid && file) {
    return uploadService.upload(vd.file, "Convictions Calculator", "PCCS").then(function(response){
      /* Some stuff */
    }).catch(handleServiceError);
  }
}

uploadService:

(function (){
'use strict';
angular.module('cica.common').service('uploadService', ['$http', '$routeParams', uploadService]);

function uploadService($http, $routeParams) {

    this.upload = function (file, name, type) {
        const fd = new FormData();
        fd.append('document', file);
        fd.append('jobId', $routeParams.jobId);
        fd.append('documentRename', name);
        fd.append('documentType', type);

        return $http.post('/document/upload', fd, {
            transformRequest: angular.identity,
            headers: {'Content-Type': undefined}
        }).catch(function(err){
            handleHttpError('Unable to upload document.', err);
        });
    };
  }
})();

routes.js:

    'POST /document/upload': {controller: 'DocumentController', action: 'uploadDocument'},

DocumentController:

"use strict";
const fs = require('fs');

module.exports = {
  uploadDocument: function (req, res) {
    console.log(req.allParams());   //Inserted as part of debugging
    const params = req.allParams();
    req.file('document').upload({
        // don't allow the total upload size to exceed ~100MB
        maxBytes: 100000000
    }, function whenDone(err, uploadedFiles) {
        if (err) {
            return res.serverError(err);
        }
        // If no files were uploaded, respond with an error.
        else if (uploadedFiles.length === 0) {
            return res.serverError('No file was uploaded');
        } else {
            const filePath = uploadedFiles[0].fd;
            const filename = uploadedFiles[0].filename;
            return fs.readFile(filePath, function (err, data) {
                if (err) {
                    return res.serverError(err);
                } else {
                    const jobId = params.jobId;
                    const jobVars =
                        {
                            filePath: results.filePath,
                            fileName: params.documentRename,
                            fileType: params.documentType
                        };
                    return DocumentService.uploadConvictions(req.session.sessionId, jobId, jobVars).then(function (response) {
                        return res.send("Document uploaded.");
                    }).catch(function (err) {
                        return res.serverError(err);
                    });
                }
            });
        }
    });
},

If I upload a .jpeg (around 11kB) the upload works exactly as expected, however, if I try to upload a larger .jpeg (around 170kB) it falls over. There is no immediate error thrown/caught though, what happens is the formData object created in the upload service seems to lose its data. If I breakpoint on its value, it returns empty for the larger file, which eventually causes an error when the function tries to use these variables further on. Is there some kind of limit set to the size of a file you can upload via this method, or have I configured this incorrectly?

Answer 1

I take the chance and assume you are using bodyParser as middleware. bodyParser has a default limit of 100kb. Look at node_modules/body-parser/lib/types/urlencoded.js :

var limit = typeof options.limit !== 'number'
    ? bytes(options.limit || '100kb')
    : options.limit

You can change the limit in your app.js by

var bodyParser = require('body-parser');
...
app.use(bodyParser.urlencoded( { limit: 1048576 } )); //1mb

Answer 2

I use this workaround...

HTML:

<input type="file" style="display:none" value="" id="uploadNewAttachment"/>

JavaScript:

In JavaScript you can upload files using the 3 method:

var binBlob = []; // If you use AngularJS, better leave it out of the DOM
var fi = document.getElementById('uploadNewAttachment');
fi.onchange = function(e) {
    r = new FileReader();
    r.onloadend = function(ev) {
        binBlob[binBlob.length] = ev.target.result;
    };
    //r.readAsDataURL(e.target.files[0]); // Very slow due to Base64 encoding
    //r.readAsBinaryString(e.target.files[0]); // Slow and may result in incompatible chars with AJAX and PHP side
    r.readAsArrayBuffer(e.target.files[0]); // Fast and Furious!
};
$(fi).trigger('click');

What we have, javascript side is an Uint8Array of byte with values from 0 to 255 (or a Int8Array -128 to 127).

When this Array is sent via AJAX, it is "maximized" using signs and commas. This increases the number of total bytes sent.

EX:

[123, 38, 98, 240, 136, ...] or worse: [-123, 38, -81, 127, -127, ...]

As you can see, the number of characters transmitted is oversized.

We can instead proceed as follows:

Before send data over AJAX, do this:

var hexBlob = [];
for(var idx=0; idx<binBlob.length; idx++) {
    var ex = Array.from(new Uint8Array(binBlob[idx]));;
    for(var i=0;i<ex.length; i++) {
        ex[i] = ex[i].toString(16).padStart(2,'0');
    };
    hexBlob[idx] = ex.join('');
}

What you have now, is a string of hex bytes in chars!

Ex:

3a05f4c9...

that use less chars of a signed or unsigned javascript array.

PHP: On the PHP side, you can decode this array, directly to binary data, simply using:

for($idx=0; $idx<=count($hexBlob); $idx++) {
    // ...
    $binData = pack('H*',$hexBlob[$idx]);
    $bytesWritten = file_put_contents($path.'/'.$fileName[$idx], $binData);
    //...
}

This solution worked very well for me.

Answer 3

Avoid using the FormData API when Uploading Large Files¹

The FormData API encodes data in base64 which add 33% extra overhead.

Instead of sending FormData, send the file directly:

app.service('fileUpload', function ($http) {
    this.uploadFileToUrl = function (url, file) {
        ̶v̶a̶r̶ ̶f̶d̶ ̶=̶ ̶n̶e̶w̶ ̶F̶o̶r̶m̶D̶a̶t̶a̶(̶)̶;̶
        ̶f̶d̶.̶a̶p̶p̶e̶n̶d̶(̶'̶f̶i̶l̶e̶'̶,̶ ̶f̶i̶l̶e̶)̶;̶
        ̶r̶e̶t̶u̶r̶n̶ ̶$̶h̶t̶t̶p̶.̶p̶o̶s̶t̶(̶u̶r̶l̶,̶ ̶f̶d̶,̶ ̶{̶
        return $http.post(url, file, {
            transformRequest: angular.identity,
            headers: { 'Content-Type': undefined }
        });
    };
});

When the browser sends FormData, it uses 'Content-Type': multipart/formdata and encodes each part using base64.

When the browser sends a file (or blob), it sets the content type to the MIME-type of the file (or blob). It puts the binary data in the body of the request.

---

How to enable <input type="file"> to work with ng-model²

Out of the box, the ng-model directive does not work with input type="file". It needs a directive:

app.directive("selectNgFile", function() {
  return {
    require: "ngModel",
    link: function postLink(scope,elem,attrs,ngModel) {
      elem.on("change", function(e) {
        var files = elem[0].files[0];
        ngModel.$setViewValue(files);
      })
    }
  }
});

Usage:

<input type="file" select-ng-file ng-model="rsdCtrl.viewData.file" name="file"/>