21

I want to upload a file in my app.js server , which should pipe that file to a crossdomain server like my upload.js server.

The full code can be found under the following link

The upload.js server is working. My problem is the app.js server. Request seems to be capable of streaming files (https://github.com/request/request#streaming). But I dont get it working. I am always getting : [Error: Invalid protocol] in my app.js.

It musst be this line:

fs.createReadStream(file.path).pipe(request.post('localhost:4000/upload'))

I changed post to put and in my upload.js the post method also to put, but it results in the same error.

My Goal is to upload a file from the html page to localhost:3000/upload which pipes the file to localhost:4000/upload (ideally with same filename) . But I dont get it working (this post doesn't helped me).

app.js:

var express = require('express')
    , multiparty = require('multiparty')
    , request = require('request')
    , fs = require('fs')
    , util = require('util')
    , http = require('http');

var app = express();
app.use('/static', express.static('static'));

process.on('uncaughtException', function (err) {
    console.log(err);
});

app.get('/', function (req, res) {
    res.redirect('static/index.html');
});

app.post('/upload', function(req, res, next){

    //https://github.com/request/request#streaming

    var form = new multiparty.Form();

    form.parse(req, function(err, fields, files) {
        res.writeHead(200, {'content-type': 'text/plain'});
        res.write('received upload:\n\n');
        res.end(util.inspect({fields: fields, files: files}));
    });
    form.on('file', function(name,file) {

        //stream it to localhost:4000 with same name
        fs.createReadStream(file.path).pipe(request.post('localhost:4000/upload'))

        console.log(file.path);
    });

});

var server = app.listen(3000, '0.0.0.0' ,function () {
    var host = server.address().address;
    var port = server.address().port;

    console.log('Example app listening at http://%s:%s', host, port);
});

index.html:

<!DOCTYPE html>
<html>
<head lang="en">
    <meta charset="UTF-8">
    <title>Upload</title>
</head>
<body>
  <h2>Upload</h2>
  <form method="post" action="/upload" enctype="multipart/form-data">
    <input type="file" id="file" name="file" />
    <button>upload</button>
  </form>
</body>
</html>

upload.js:

var express = require('express')
    , multiparty = require('multiparty')
    , cors = require('cors')
    , util = require('util')
    , app = express();

app.use(cors());
process.on('uncaughtException', function (err) {
    console.log(err);
});

app.get('/', cors(), function(req, res, next){
    res.json({msg: 'This is CORS-enabled for all origins!'});
});

app.post('/upload', cors(), function(req, res, next){
    var form = new multiparty.Form();

    form.parse(req, function(err, fields, files) {
        res.writeHead(200, {'content-type': 'text/plain'});
        res.write('received upload:\n\n');
        res.end(util.inspect({fields: fields, files: files}));
    });
    form.on('file', function(name,file) {
        console.log(file);
        console.log(name);
    });

});

app.listen(4000, function(){
    console.log('CORS-enabled web server listening on port 4000');
});
svenhornberg
  • 14,376
  • 9
  • 40
  • 56

3 Answers3

10

Updated

I believe you're missing the protocol from the url. It should work, if you add the http protocol to the url:

fs.createReadStream(file.path).pipe(request.post('http://localhost:4000/upload'))

Make Upload work

When you pipe the file contents to the POST function in upload.js, the multipart form data is lost. You need to create a new POST request and pass the original file contents.

Do the following in app.js:

 form.on('file', function(name, file) {

    var formData = {
      file: {
        value:  fs.createReadStream(file.path),
        options: {
          filename: file.originalFilename
        }
      }
    };

    // Post the file to the upload server
    request.post({url: 'http://localhost:4000/upload', formData: formData});
}

This will also pass the original filename. For more information see: https://github.com/request/request#multipartform-data-multipart-form-uploads

kmandov
  • 3,130
  • 16
  • 15
  • thats removes the Invalid protocol, but the upload.js server prints no output. Maybe the request.post is no multipart/form-data. I will test this http://stackoverflow.com/a/13821169/1584115 – svenhornberg Aug 21 '15 at 12:14
  • I updated your code into https://github.com/svenhornberg/pipeupload/, but still no output in upload.js. Test it with https://chrome.google.com/webstore/detail/postman/fhbjgbiflinjbdggehcddcbncdddomop (http://postimg.org/image/wq7plmjyf/) – svenhornberg Aug 21 '15 at 12:43
  • It works for me. When I go to http://localhost:3000/static/index.html and upload a file I can see that upload.js has received it: CORS-enabled web server listening on port 4000 { fieldName: 'file', originalFilename: 'Screen Shot 2015-08-12 at 6.24.23 PM.png', path: '/var/folders/r7/w49fh_652tv89r9qqwf2hg_40000gn/T/b4Z4AEZSB95INiXy1cUH_u4n.png', headers: { 'content-disposition': 'form-data; name="file"; filename="Screen Shot 2015-08-12 at 6.24.23 PM.png"', 'content-type': 'image/png' }, size: 464485 } file – kmandov Aug 21 '15 at 12:51
  • okay, any idea whats different between request.post and the way the postman extension posts to localhost:4000/upload ? because if I do it with postmen I get the output as expected : { fieldName: 'file', originalFilename: 'hello world.txt', path: '/tmp/g6Jd0O-pPXi2tPOcfeP7cPHd.txt', headers: { 'content-disposition': 'form-data; name="file"; filename="hello world.txt"', 'content-type': 'text/plain' }, size: 13 } file **lines which should output from uploadjs:(from this lines form.on('file', function(name,file) { console.log(file); console.log(name);** });) – svenhornberg Aug 21 '15 at 13:01
  • The difference should be in the way the POST requests are made, however when you use the latest code above, the POST request should work. Is the latest code working on the server? I mean.. if it works for me, it should work for you as well. Maybe the changes are not deployed? – kmandov Aug 21 '15 at 13:09
  • Frist thank you very much, but is your postet output really from this line https://github.com/svenhornberg/pipeupload/blob/master/upload.js#L25 ? Because I wrote a python server, which also not get any files. with postman it gets a file. – svenhornberg Aug 21 '15 at 13:14
  • yes, it is. I start the app.js and the upload.js servers. Then I upload a sample text file. App.js saves the file under '/var/folders/r7/w49fh_652tv89r9qqwf2hg_40000gn/T/3mAvM3mraBIv8etf4jb13cOt.txt; Then in the second console(upload.js) I see: { fieldName: 'file', originalFilename: 'testfile.txt', path: '/var/folders/r7/w49fh_652tv89r9qqwf2hg_40000gn/T/xDU83ZuLOpkJCZkQer4t25q8.txt', headers: { 'content-disposition': 'form-data; name="file"; filename="testfile.txt"', 'content-type': 'text/plain' }, size: 6 } file; i.e. the file has been re-uploaded to the upload.js server – kmandov Aug 21 '15 at 13:19
  • okay, i mark your answer as right, but just for info which node version are you using ? – svenhornberg Aug 21 '15 at 13:20
  • Thanks. I would be happy to make it work for you as well. I'm using node v0.12.5 on mac os – kmandov Aug 21 '15 at 13:27
  • btw. I noticed that the request package is missing from the project dependencies. I had to add it with: npm install request --save – kmandov Aug 21 '15 at 13:42
8

I had a similar problem but I wanted to stream the file directly to the remote server instead of saving it locally first. Here's my modifications to get streaming to work:

app.post('/upload', function(request, response, next) {
    var form = new multiparty.Form();

    form.on('part', function(formPart) {
        var contentType = formPart.headers['content-type'];

        var formData = {
            file: {
                value: formPart,
                options: {
                    filename: formPart.filename,
                    contentType: contentType,
                    knownLength: formPart.byteCount
                }
            }
        };

        requestJs.post({
            url: 'http://localhost:4000/upload',
            formData: formData,

            // These may or may not be necessary for your server:
            preambleCRLF: true,
            postambleCRLF: true
        });
    });

    form.on('error', function(error) {
        next(error);
    });

    form.on('close', function() {
       response.send('received upload');
    });

    form.parse(request);
});
jameslk
  • 4,298
  • 4
  • 21
  • 19
1

You actually can do stream.pipe(request.post(url));, and then on the another server your req variable will have a stream-like behavior, so you should be able to req.pipe(fs.createWriteStream(path));.

Use req.params on your route to preserve the filename.

Example code for server one (use some hashing like md5 to validate your request):

const stream = fs.createReadStream(path);

const md5 = crypto.createHash('md5')
    .update(`${filename}${secret}`)
    .digest('hex').toUpperCase();

const url = `http://${host}/upload/${md5}/${filename}`;

const r = request.post(url, (err, resp, body) => {});

stream.pipe(r);

Example code for server two:

router.post('/upload/:md5/:filename', function(req, res, next) {
    const { md5, filename } = req.params;

    const check = crypto.createHash('md5')
        .update(`${filename}${secret}`)
        .digest('hex').toUpperCase();

    if (md5 !== check) {
        res.writeHead(403);
        res.end('403');
        return;
    }

    req.pipe(fs.createWriteStream(filename));

    req.on('end', next);
});
HydraOrc
  • 657
  • 7
  • 15