How to upload a file from the browser to Amazon S3 with node.js, Express, and knox?

Question

I'm trying to find some example code that utilizes node.js, Express, and knox.

The docs for Knox only give clear examples of how to upload a file already stored in the file system. https://github.com/learnboost/knox#readme

Additionally, there a number of simple tutorials (even in Express itself) on how to upload files directly to express and save to the file system.

What I'm having trouble finding is an example that lets you upload a client upload to a node server and have the data streamed directly to S3 rather than storing in the local file system first.

Can someone point me to a gist or other example that contains this kind of information?

Answer 1

All of the previous answers involve having the upload pass through your node.js server which is inefficient and unnecessary. Your node server does not have to handle the bandwidth or processing of uploaded files whatsoever because Amazon S3 allows uploads direct from the browser.

Have a look at this blog post: http://blog.tcs.de/post-file-to-s3-using-node/

I have not tried the code listed there, but having looked over it, it appears solid and I will be attempting an implementation of it shortly ad will update this answer with my findings.

Answer 2

Here is an example of streaming directly to s3 without ever touching your hard drive, using multiparty and knox:

var http = require('http')
  , util = require('util')
  , multiparty = require('multiparty')
  , knox = require('knox')
  , Batch = require('batch')
  , PORT = process.env.PORT || 27372

var s3Client = knox.createClient({
  secure: false,
  key: process.env.S3_KEY,
  secret: process.env.S3_SECRET,
  bucket: process.env.S3_BUCKET,
});

var Writable = require('readable-stream').Writable;
util.inherits(ByteCounter, Writable);
function ByteCounter(options) {
  Writable.call(this, options);
  this.bytes = 0;
}

ByteCounter.prototype._write = function(chunk, encoding, cb) {
  this.bytes += chunk.length;
  cb();
};

var server = http.createServer(function(req, res) {
  if (req.url === '/') {
    res.writeHead(200, {'content-type': 'text/html'});
    res.end(
      '<form action="/upload" enctype="multipart/form-data" method="post">'+
      '<input type="text" name="path"><br>'+
      '<input type="file" name="upload"><br>'+
      '<input type="submit" value="Upload">'+
      '</form>'
    );
  } else if (req.url === '/upload') {
    var headers = {
      'x-amz-acl': 'public-read',
    };
    var form = new multiparty.Form();
    var batch = new Batch();
    batch.push(function(cb) {
      form.on('field', function(name, value) {
        if (name === 'path') {
          var destPath = value;
          if (destPath[0] !== '/') destPath = '/' + destPath;
          cb(null, destPath);
        }
      });
    });
    batch.push(function(cb) {
      form.on('part', function(part) {
        if (! part.filename) return;
        cb(null, part);
      });
    });
    batch.end(function(err, results) {
      if (err) throw err;
      form.removeListener('close', onEnd);
      var destPath = results[0]
        , part = results[1];

      var counter = new ByteCounter();
      part.pipe(counter); // need this until knox upgrades to streams2
      headers['Content-Length'] = part.byteCount;
      s3Client.putStream(part, destPath, headers, function(err, s3Response) {
        if (err) throw err;
        res.statusCode = s3Response.statusCode;
        s3Response.pipe(res);
        console.log("https://s3.amazonaws.com/" + process.env.S3_BUCKET + destPath);
      });
      part.on('end', function() {
        console.log("part end");
        console.log("size", counter.bytes);
      });
    });
    form.on('close', onEnd);
    form.parse(req);

  } else {
    res.writeHead(404, {'content-type': 'text/plain'});
    res.end('404');
  }

  function onEnd() {
    throw new Error("no uploaded file");
  }
});
server.listen(PORT, function() {
  console.info('listening on http://0.0.0.0:'+PORT+'/');
});

example taken from https://github.com/superjoe30/node-multiparty/blob/master/examples/s3.js

Answer 3

the node/express code doesn't work with nodejs v0.4.7

here is the updated code for nodejs v0.4.7

app.post('/upload', function (req, res) {
  // connect-form additions
  req.form.complete(function (err, fields, files) {
    // here lies your uploaded file:
    var path = files['upload-file']['path'];
    // do knox stuff here
  });
});

Answer 4

* update *

as of mid 2009 amazon supports CORS and the upload via your node.js server isn't needed anymore. you can directly upload the file to S3.

---

with the help of the "connect-form" module you could just upload the file to your server (through normal multipart FORM) and then handle the S3 stuff afterwards ...

<form action="/upload" method="POST" id="addContentForm" enctype="multipart/form-data">
  <p><label for="media">File<br/><input type="file" name="media" /></label></p>
  <p><button type="submit">upload</button></p>
</form>

node/express code:

app.post('/upload', function (req, res) {
  // connect-form additions
  req.form.complete(function (err, fields, files) {
    // here lies your uploaded file:
    var path = files['media']['path'];
    // do knox stuff here
  });
});

you have to add the following line to the app configuration:

app.configure(function(){
  // rest of the config stuff ...
  app.use(form({ keepExtensions: true }));
  // ...
});

Answer 5

The connect-stream-s3 library can upload all of your forms files to S3 as part of middleware so you don't have to do any logic yourself. It needs express.bodyParser() for it to work at the moment, but I'm working on a version that will stream files direct to Amazon S3 prior to being written to disk:

• https://github.com/appsattic/connect-stream-s3

Please let me know how you get on. Hopefully it's a lot less hassle than doing it yourself once you're in your page handler. :)

Answer 6

I made this to upload directly from the Jquery File Upload plugin to S3 with file being public - it should point you in the right direction.

https://gist.github.com/3995819