Can't connect to S3, using Cloudcube Addon on Heroku Express/Node App

Question

I am in the middle of testing out my Heroku setup and connections, and added CloudCube to my Express/Node app to use it for storage.

I wanted to connect to enable file storage through my live Heroku app, I attached an addon to my Heroku app called Cloudcube, which grants me a cube inside their S3 bucket with environment access credentials. Pretty sure I used credentials properly.

I have been trying to get this connection to work for hours.

What I tried so far:

Following this post here: Uploading image to amazon s3 using multer-s3 nodejs

I did npm install on aws-sdk, multer, and multer-s3, required them, then used it as advised, but switching out my credentials, and added a "dest" key in the upload function setup to point to my sub-cube & desired directory in the Cloudcube bucket, this is my best guess of how to do it:

aws.config.update({
  secretAccessKey: process.env.CLOUDCUBE_SECRET_ACCESS_KEY,
  accessKeyId: process.env.CLOUDCUBE_ACCESS_KEY_ID,
  region: "eu-west-1"
});

const s3 = new aws.S3();

const upload = multer({
  dest: "yutqk2v0mx6h/public/",
  storage: multerS3({
    s3: s3,
    bucket: "cloud-cube-eu",
    key: function(req, file, cb) {
      console.log(file);
      cb(null, file.originalname); //use Date.now() for unique file keys
    }
  })
});

I am testing this using the test routes explained in the previously shared stackoverflow post, as follows:

//open in browser to see upload form
app.get('/', function (req, res) {
  res.sendFile(__dirname + '/index.html');
});

//use by upload form
app.post('/upload', upload.array('upl',1), function (req, res, next) {
  res.send("Uploaded!");
});

I tried many different variations of this with no success.

You can see my full backend testing repo here: https://github.com/malkhuzayyim/hasad-backend

It has some extra code from previous testing on setup, everything else works just the static file uploads fail.

Upon attempting to upload, i get these logs on my heroku logs dashboard:

2019-01-05T01:10:33.693468+00:00 heroku[router]: at=info method=GET path="/" host=hasad-backend.herokuapp.com request_id=782646c6-e150-44a5-bfb6-ff6a4b54c147 fwd="95.218.80.131" dyno=web.1 connect=0ms service=9ms status=200 bytes=899 protocol=https
2019-01-05T01:10:47.797075+00:00 app[web.1]: { fieldname: 'upl',
2019-01-05T01:10:47.797094+00:00 app[web.1]:   originalname: '1f227fe3048fdf82d4dc52ca577c381d.jpg',
2019-01-05T01:10:47.797096+00:00 app[web.1]:   encoding: '7bit',
2019-01-05T01:10:47.797098+00:00 app[web.1]:   mimetype: 'image/jpeg' }
2019-01-05T01:10:47.906948+00:00 app[web.1]: AccessDenied: Access Denied
2019-01-05T01:10:47.906951+00:00 app[web.1]:     at Request.extractError (/app/node_modules/aws-sdk/lib/services/s3.js:585:35)
2019-01-05T01:10:47.906953+00:00 app[web.1]:     at Request.callListeners (/app/node_modules/aws-sdk/lib/sequential_executor.js:106:20)
2019-01-05T01:10:47.906955+00:00 app[web.1]:     at Request.emit (/app/node_modules/aws-sdk/lib/sequential_executor.js:78:10)
2019-01-05T01:10:47.906956+00:00 app[web.1]:     at Request.emit (/app/node_modules/aws-sdk/lib/request.js:683:14)
2019-01-05T01:10:47.906958+00:00 app[web.1]:     at Request.transition (/app/node_modules/aws-sdk/lib/request.js:22:10)
2019-01-05T01:10:47.906959+00:00 app[web.1]:     at AcceptorStateMachine.runTo (/app/node_modules/aws-sdk/lib/state_machine.js:14:12)
2019-01-05T01:10:47.906961+00:00 app[web.1]:     at /app/node_modules/aws-sdk/lib/state_machine.js:26:10
2019-01-05T01:10:47.906963+00:00 app[web.1]:     at Request.<anonymous> (/app/node_modules/aws-sdk/lib/request.js:38:9)
2019-01-05T01:10:47.906964+00:00 app[web.1]:     at Request.<anonymous> (/app/node_modules/aws-sdk/lib/request.js:685:12)
2019-01-05T01:10:47.906966+00:00 app[web.1]:     at Request.callListeners (/app/node_modules/aws-sdk/lib/sequential_executor.js:116:18)
2019-01-05T01:10:47.906967+00:00 app[web.1]:     at Request.emit (/app/node_modules/aws-sdk/lib/sequential_executor.js:78:10)
2019-01-05T01:10:47.906968+00:00 app[web.1]:     at Request.emit (/app/node_modules/aws-sdk/lib/request.js:683:14)
2019-01-05T01:10:47.906969+00:00 app[web.1]:     at Request.transition (/app/node_modules/aws-sdk/lib/request.js:22:10)
2019-01-05T01:10:47.906971+00:00 app[web.1]:     at AcceptorStateMachine.runTo (/app/node_modules/aws-sdk/lib/state_machine.js:14:12)
2019-01-05T01:10:47.906973+00:00 app[web.1]:     at /app/node_modules/aws-sdk/lib/state_machine.js:26:10
2019-01-05T01:10:47.906974+00:00 app[web.1]:     at Request.<anonymous> (/app/node_modules/aws-sdk/lib/request.js:38:9)
2019-01-05T01:10:47.906976+00:00 app[web.1]:     at Request.<anonymous> (/app/node_modules/aws-sdk/lib/request.js:685:12)
2019-01-05T01:10:47.906977+00:00 app[web.1]:     at Request.callListeners (/app/node_modules/aws-sdk/lib/sequential_executor.js:116:18)
2019-01-05T01:10:47.906979+00:00 app[web.1]:     at callNextListener (/app/node_modules/aws-sdk/lib/sequential_executor.js:96:12)
2019-01-05T01:10:47.906980+00:00 app[web.1]:     at IncomingMessage.onEnd (/app/node_modules/aws-sdk/lib/event_listeners.js:299:13)
2019-01-05T01:10:47.906982+00:00 app[web.1]:     at IncomingMessage.emit (events.js:187:15)
2019-01-05T01:10:47.906984+00:00 app[web.1]:     at IncomingMessage.EventEmitter.emit (domain.js:441:20)
2019-01-05T01:10:47.906764+00:00 heroku[router]: at=info method=POST path="/upload" host=hasad-backend.herokuapp.com request_id=d025c4df-2a09-4a5b-bad8-f4a349cf3aa4 fwd="95.218.80.131" dyno=web.1 connect=1ms service=121ms status=403 bytes=560 protocol=https

Multer S3 Documentation can be seen here: https://www.npmjs.com/package/multer-s3

You can also see the Cloudcube documentation here: https://devcenter.heroku.com/articles/cloudcube

My api is live on https://hasad-backend.herokuapp.com/

I really need to get this to work, as I plan on using this setup for future projects and am using this course as a proxy to get myself familiar with this setup.

Any advice on best approaches to link Heroku Express API to Cloudcube?

I would really appreciate the help.

Thank you very much.

Best,

MK

score 3 · Accepted Answer · answered Jan 05 '19 at 15:25

I figured it out, everything was in place, i was just parsing the destination of the path incorrectly, which made cloudcube reject my upload, since my "storage cube" is technically a sub-directory in their S3 bucket.

Mistake was:

const upload = multer({
  dest: "yutqk2v0mx6h/public/",
  storage: multerS3({
    s3: s3,
    bucket: "cloud-cube-eu",
    key: function(req, file, cb) {
      console.log(file);
      cb(null, file.originalname);
        }
    })
});

Where as the correct way to parse my file storage directory was to pre-pend it to my file-name inside the key function of multerS3:

const upload = multer({
  storage: multerS3({
    s3: s3,
    bucket: "cloud-cube-eu",
    key: function(req, file, cb) {
      console.log(file);
      const myFileName = "yutqk2v0mx6h/public/" + Date.now();
      console.log(myFileName);
      cb(null, myFileName);
    }
  })
});

A response from cloudcube's email support hinted at the solution, as soon as I tried this it worked right away.

Thanks for taking a look Scott, I'll leave this here in case anyone else goes through a Heroku/S3 setup.

Best,

MK

Crashhopper · Answer 2 · 2023-03-16T15:05:59.913

I am using aws-sdk in a sveltekit project, whith heroku and cloudcube.

//file s3.js
//install aws-sdw package
import AWS from 'aws-sdk';

AWS.config.update( {
    secretAccessKey: '<your-secret-key>',
    accessKeyId: '<your-access-key-id>'
} );

export const s3 = new AWS.S3( { region: 'eu-west-1' } );
export const bucketName = 'cloud-cube-eu';
export const directory = '<your-cube>/public/';

And as I am using sveltekit, on +server.js you can use this:

//file +server.js
import { s3, bucketName, directory } from '$lib/s3'; //import of the s3.js file

export async function POST( { request } ) {

 const body = await request.json();
 const buffer = Buffer.from( body.image, 'base64' );
 let image;

 s3.upload(
  {
    Bucket: bucketName,
    Key: directory + 'image/xxxxx.png',
    Body: buffer,
    ContentType: 'image/jpeg',
  },
  ( err, data ) => {
    if ( err ) 
    {
      console.error( err );
    } 
    else
    {
      console.log( `File uploaded successfully to ${data.Location}` );
      image = data.Location;
    }
  }
 );

 return new Response( JSON.stringify( image ), { status: 200 } )
}

Note that my body.image is a base64 image

This is working form me Hope this help someone

Can't connect to S3, using Cloudcube Addon on Heroku Express/Node App

2 Answers2