Upload Progress — Request

Question

I'm uploading a file using Request.

req = request.post url: "http://foo.com", body: fileAsBuffer, (err, res, body) ->
    console.log "Uploaded!"

How do I know how much data has actually been uploaded? Is there some event that I can subscribe to or is there a property of the request that I can poll?

If none, what would be the best approach to upload data and know how much has been uploaded?

Answer 1

I spent a couple of hours to find anything valid in request and node sources, and finally found a different approach, which feels more correct to me.

We can rely on drain event and bytesWritten property:

request.put({
  url: 'https://example.org/api/upload',
  body: fs.createReadStream(path)
}).on('drain', () => {
  console.log(req.req.connection.bytesWritten);
});

Alternatively if you need to handle progress of file bytes, it's easier to use stream data event:

let size = fs.lstatSync(path).size;
let bytes = 0;

request.put({
  url: 'https://example.org/api/upload',
  body: fs.createReadStream(path).on('data', (chunk) => {
    console.log(bytes += chunk.length, size);
  })
});

Stream buffer size is 65536 bytes and read/drain procedure runs iteratively.

This seems to be working pretty well for me with node v4.5.0 and request v2.74.0.

Answer 2

I needed a handle on the upload progress for yet another project of mine.

What I found out is that you can poll the request's connection._bytesDispatched property.

For example:

r = request.post url: "http://foo.com", body: fileAsBuffer
setInterval (-> console.log "Uploaded: #{r.req.connection._bytesDispatched}"), 250

Note: If you were piping to r, poll r.req.connection.socket._bytesDispatched instead.

Answer 3

var request = require('request');
    var fs = require('fs');  
    let path ="C:/path/to/file";
    var formData = {
        vyapardb: fs.createReadStream(path)
    };

    let size = fs.lstatSync(path).size;

    var headers = {
        'Accept' : 'application/json',
        'Authorization' : 'Bearer '+token,
    };

    var r = request.post({url:'http://35.12.13/file/upload', formData: formData,  headers: headers}, function optionalCallback(err, httpResponse, body) {
        clearInterval(q);

    });
    var q = setInterval(function () {
        var dispatched = r.req.connection._bytesDispatched;
        let percent = dispatched*100/size;
         console.dir("Uploaded: " + percent + "%");

    }, 250);
}

Answer 4

I know this is old, but I just found a library 'progress-stream' that has not been mentioned and does this very nicely. https://www.npmjs.com/package/progress-stream

const streamProg = require('progress-stream');
const fs = require('fs');
const request = require('request');

const myFile = 'path/to/file.txt';
const fileSize = fs.statSync('path/to/file.txt').size;
const readStream = fs.createReadStream(myFile);
const progress = streamProg({time: 1000, length: fileSize})
    .on('progress', state => console.log(state));

readStream.pipe(progress).pipe(request.put({url: 'SOMEURL/file.txt'}));

Similarly, it can be used between a pipe on download as well.

Answer 5

Someone has created a nice module to accomplish this that has been running in the production stack for transloadit (so it is dependable and well maintained). You can find it here:

https://github.com/felixge/node-formidable

Code should look like:

var formidable = require('formidable'),
    http = require('http'),

    util = require('util');

http.createServer(function(req, res) {
  if (req.url == '/upload' && req.method.toLowerCase() == 'post') {
    // parse a file upload
    var form = new formidable.IncomingForm();
    form.parse(req, function(err, fields, files) {
      res.writeHead(200, {'content-type': 'text/plain'});
      res.write('received upload:\n\n');
      res.end(util.inspect({fields: fields, files: files}));
    });
    return;
  }

  // show a file upload form
  res.writeHead(200, {'content-type': 'text/html'});
  res.end(
    '<form action="/upload" enctype="multipart/form-data" method="post">'+
    '<input type="text" name="title"><br>'+
    '<input type="file" name="upload" multiple="multiple"><br>'+
    '<input type="submit" value="Upload">'+
    '</form>'
  );
}).listen(80);

You could then push status to a client using Socket.io

Interesting Note: This was one of the problems that led to Node's creation. In this video Ryan talks about how node was started trying to find the best way of notifying a user, real time, about the status of file upload over the web...anyway I digress, but the video is worth a watch if you are interested in Node's history