Amazon S3 Remote File Upload with Axios

Question

I am trying to write a function that would:

1. Take a remote URL as a parameter,
2. Get the file using axios
3. Upload the stream to amazon s3
4. And finally, return the uploaded url

I found help here on stackoverflow. So far, I have this:

/* 
 * Method to pipe the stream 
 */
const uploadFromStream = (file_name, content_type) => {
  const pass = new stream.PassThrough();

  const obj_key = generateObjKey(file_name);
  const params = { Bucket: config.bucket, ACL: config.acl, Key: obj_key, ContentType: content_type, Body: pass };

  s3.upload(params, function(err, data) {
    if(!err){
        return data.Location;
    } else {
        console.log(err, data);
    }
  });

  return pass;
}


/*
 * Method to upload remote file to s3
 */
const uploadRemoteFileToS3 = async (remoteAddr) => {
    axios({
        method: 'get',
        url: remoteAddr,
        responseType: 'stream'
    }).then( (response) => {
        if(response.status===200){
            const file_name = remoteAddr.substring(remoteAddr.lastIndexOf('/')+1);
            const content_type = response.headers['content-type'];
            response.data.pipe(uploadFromStream(file_name, content_type));
        }
    });
}

But uploadRemoteFileToS3 does not return anything (because it's a asynchronous function). How can I get the uploaded url?

UPDATE

I have further improved upon the code and wrote a class. Here is what I have now:

const config = require('../config.json');

const stream = require('stream');
const axios = require('axios');
const AWS = require('aws-sdk');

class S3RemoteUploader {
    constructor(remoteAddr){
        this.remoteAddr = remoteAddr;
        this.stream = stream;
        this.axios = axios;
        this.config = config;
        this.AWS = AWS;
        this.AWS.config.update({
            accessKeyId: this.config.api_key,
            secretAccessKey: this.config.api_secret
        });
        this.spacesEndpoint = new this.AWS.Endpoint(this.config.endpoint);
        this.s3 = new this.AWS.S3({endpoint: this.spacesEndpoint});

        this.file_name = this.remoteAddr.substring(this.remoteAddr.lastIndexOf('/')+1);
        this.obj_key = this.config.subfolder+'/'+this.file_name;
        this.content_type = 'application/octet-stream';

        this.uploadStream();
    }

    uploadStream(){
        const pass = new this.stream.PassThrough();
        this.promise = this.s3.upload({
            Bucket: this.config.bucket,
            Key: this.obj_key,
            ACL: this.config.acl,
            Body: pass,
            ContentType: this.content_type
        }).promise();
        return pass;
    }

    initiateAxiosCall() {
        axios({
            method: 'get',
            url: this.remoteAddr,
            responseType: 'stream'
        }).then( (response) => {
            if(response.status===200){
                this.content_type = response.headers['content-type'];
                response.data.pipe(this.uploadStream());
            }
        });
    }

    dispatch() {
        this.initiateAxiosCall();
    }

    async finish(){
        //console.log(this.promise); /* return Promise { Pending } */
        return this.promise.then( (r) => {
            console.log(r.Location);
            return r.Location;
        }).catch( (e)=>{
            console.log(e);
        });
    }

    run() {
        this.dispatch();
        this.finish();
    }
}

But still have no clue how to catch the result when the promise is resolved. So far, I tried these:

testUpload = new S3RemoteUploader('https://avatars2.githubusercontent.com/u/41177');
testUpload.run();
//console.log(testUpload.promise); /* Returns Promise { Pending } */
testUpload.promise.then(r => console.log); // does nothing

But none of the above works. I have a feeling I am missing something very subtle. Any clue, anyone?

Answer 1

After an upload you can call the getsignedurl function in s3 sdk to get the url where you can also specify the expiry of the url as well. You need to pass the key for that function. Now travelling will update with example later.

To generate a simple pre-signed URL that allows any user to view the contents of a private object in a bucket you own, you can use the following call to getSignedUrl():

 var s3 = new AWS.S3(); 
 var params = {Bucket: 'myBucket', Key: 'myKey'}; 
 s3.getSignedUrl('getObject', params, function (err, url) {  
   console.log("The URL is", url); 
 });

Official documentation link http://docs.amazonaws.cn/en_us/AWSJavaScriptSDK/guide/node-examples.html

Code must be something like this

function uploadFileToS3AndGenerateUrl(cb) {
const pass = new stream.PassThrough();//I have generated streams from file. Using this since this is what you have used. Must be a valid one.
var params = {
            Bucket: "your-bucket", // required
            Key: key , // required
            Body: pass,
            ContentType: 'your content type',

        };
s3.upload(params, function(s3Err, data) {
    if (s3Err) {
        cb(s3Err)
    }
    console.log(`File uploaded successfully at ${data.Location}`)

    const params = {
        Bucket: 'your-bucket',
        Key: data.key,
        Expires: 180
    };
    s3.getSignedUrl('getObject', params, (urlErr, urlData) => {
        if (urlErr) {

            console.log('There was an error getting your files: ' + urlErr);
            cb(urlErr);

        } else {
            console.log(`url: ${urlData}`);
            cb(null, urlData);

        }
    })
})
}

Answer 2

Please check i have update your code might its help you.

    /*
         * Method to upload remote file to s3
         */
        const uploadRemoteFileToS3 = async (remoteAddr) => {
            const response = await axios({
                method: 'get',
                url: remoteAddr,
                responseType: 'stream'
            })
               if(response.status===200){
                    const file_name = remoteAddr.substring(remoteAddr.lastIndexOf('/')+1);
                    const content_type = response.headers['content-type'];
                    response.data.pipe(uploadFromStream(file_name, content_type));
                }
                return new Promise((resolve, reject) => {
                    response.data.on('end', (response) => {
                      console.log(response)
                      resolve(response)
                    })

                    response.data.on('error', () => {
                      console.log(response);
                      reject(response)
                    })
              })
        };

       * 
     * Method to pipe the stream 
     */
    const uploadFromStream = (file_name, content_type) => {
       return new Promise((resolve, reject) => {
          const pass = new stream.PassThrough();
          const obj_key = generateObjKey(file_name);
          const params = { Bucket: config.bucket, ACL: config.acl, Key: obj_key, ContentType: content_type, Body: pass };
          s3.upload(params, function(err, data) {
            if(!err){
                console.log(data)
                return resolve(data.Location);
            } else {
                console.log(err)
                return reject(err);
            }
          });
       });
    }

//call uploadRemoteFileToS3
    uploadRemoteFileToS3(remoteAddr)
      .then((finalResponse) => {
            console.log(finalResponse)
       })
       .catch((err) => {
         console.log(err);
    });