Is there a way to upload to S3 from a url using node.js?

Question

I found this question, but it doesn't seem to answer my question as I think it's still talking about local files.

I want to take, say, and imgur.com link and upload it to S3 using node. Is knox capable of this or do I need to use something else?

Not sure where to get started.

many answers indicate that the "accepted" answer might not be the correct one, this might need updating and upvoting other relevant answers. There is, of course, no obligation to do so though. — Bassem, May 18 '20 at 12:06

micmia · Accepted Answer · 2014-08-29T09:01:16.437

33

I’m not using knox but the official AWS SDK for JavaScript in Node.js. I issue a request to a URL with {encoding: null} in order to retrieve the data in buffer which can be passed directly to the Body parameter for s3.putObject(). Here below is an example of putting a remote image in a bucket with aws-sdk and request.

var AWS = require('aws-sdk');
var request = require('request');

AWS.config.loadFromPath('./config.json');
var s3 = new AWS.S3();

function put_from_url(url, bucket, key, callback) {
    request({
        url: url,
        encoding: null
    }, function(err, res, body) {
        if (err)
            return callback(err, res);

        s3.putObject({
            Bucket: bucket,
            Key: key,
            ContentType: res.headers['content-type'],
            ContentLength: res.headers['content-length'],
            Body: body // buffer
        }, callback);
    })
}

put_from_url('http://a0.awsstatic.com/main/images/logos/aws_logo.png', 'your_bucket', 'media/aws_logo.png', function(err, res) {
    if (err)
        throw err;

    console.log('Uploaded data successfully!');
});

edited Aug 29 '14 at 09:01

answered Aug 29 '14 at 08:45

micmia

1,371
1
14
29

2

Exactly same approach I took. My image files get corrupted on S3 :( – Vyacheslav Cotruta Apr 02 '15 at 13:07
1

+1 for the example. My approach was also similar to this but I was having corrupt upload issues. This code seems to be working for me though. – minimalpop Mar 12 '16 at 23:53
2

If you use axios - set header responseType: 'stream' – Alexander Jul 09 '19 at 13:19
This worked for me. To avoid the corrupt image issue, I had to ensure the `encoding: null` was set in the request options – kevgathuku Nov 06 '19 at 08:29
@Alexander, your comment really helped me a lot! Thank you! – alberto vielma Feb 19 '20 at 10:50

score 14 · Answer 2 · answered Aug 20 '18 at 18:46

For those that are looking for a solution that doesn't involves callbacks, and prefeers promises, based on @micmia code here is an alternative:

var AWS = require('aws-sdk'),
request = require('request');


const bucketName='yourBucketName';
const bucketOptions = {...Your options};
var s3 = new AWS.S3(options);

function UploadFromUrlToS3(url,destPath){
    return new Promise((resolve,reject)=> {            
        request({
            url: url,
            encoding: null
        }, function(err, res, body) {        
            if (err){
                reject(err);
            }
            var objectParams = {
                ContentType: res.headers['content-type'],
                ContentLength: res.headers['content-length'],
                Key: destPath,
                Body: body 
            };
            resolve(s3.putObject(objectParams).promise());
        });
    });
}

UploadFromUrlToS3(
    'http://a0.awsstatic.com/main/images/logos/aws_logo.png',
    'your/s3/path/aws_logo.png' )
    .then(function() {
        console.log('image was saved...');
    }).catch(function(err) {
        console.log('image was not saved!',err);
    });

score 8 · Answer 3 · answered May 18 '20 at 12:04

Building on @Yuri's post, for those who would like to use axios instead of request & ES6 syntax for a more modern approach + added the required Bucket property to params (and it downloads any file, not only images):

const uploadFileToS3 = (url, bucket, key) => {
  return axios.get(url, { responseType: "arraybuffer", responseEncoding: "binary" }).then((response) => {
    const params = {
      ContentType: response.headers["content-type"],
      ContentLength: response.data.length.toString(), // or response.header["content-length"] if available for the type of file downloaded
      Bucket: bucket,
      Body: response.data,
      Key: key,
    };
    return s3.putObject(params).promise();
  });
}

uploadFileToS3(<your_file_url>, <your_s3_path>, <your_s3_bucket>)
   .then(() => console.log("File saved!"))
   .catch(error) => console.log(error));

jclyons52 · Answer 4 · 2021-02-02T04:55:53.173

Same thing as the above answer but with fetch:

async function upload(url: string, key: string, bucket: string) {
  const response = await fetch(url)
  const contentType = response.headers.get("content-type") ?? undefined;
  const contentLength =
    response.headers.get("content-length") != null
      ? Number(response.headers.get("content-length"))
      : undefined;

  return s3
    .putObject({
      Bucket: bucket,
      Key: key,
      ContentType: contentType,
      ContentLength: contentLength,
      Body: response.body, // buffer
    })
    .promise();
}

Peter Lyons · Answer 5 · 2018-09-27T20:53:27.123

-1

Yes. There's an example of doing this in the knox README

http.get('http://google.com/doodle.png', function(res){
  var headers = {
      'Content-Length': res.headers['content-length']
    , 'Content-Type': res.headers['content-type']
  };
  client.putStream(res, '/doodle.png', headers, function(err, res){
    // Logic
  });
});

edited Sep 27 '18 at 20:53

answered May 29 '13 at 03:21

Peter Lyons

142,938
30
279
274

3

There's no reason to be sarcastic and rude. – Michał Miszczyszyn Sep 27 '18 at 20:11

Is there a way to upload to S3 from a url using node.js?

5 Answers5