How to access S3 bucket through JavaScript?

Question

I've looked at some other posts and tried to replicate what they've done, but none of them seem to be running into the same issue I am. Basically, I'm trying to store the list of keys from a S3 bucket so I can iterate through them in my Vue app. I've got the code below, and you can see I have 3 console.log statements where I'm trying to print the value of files. The first one prints exactly what I expect it to, while the 2nd one prints [], and the 3rd doesn't print at all. So for some reason it's not persisting the value of files outside of the s3.listObjectsV2() function, which means I can't access the actual files themselves in the app.

let AWS = require("aws-sdk");
AWS.config.update({
  accessKeyId: process.env.VUE_APP_ACCESS_KEY,
  secretAccessKey: process.env.VUE_APP_ACCESS_KEY,
  region: "us-east-1",
});
let s3 = new AWS.S3();

let params = {
  Bucket: "my-bucket",
  Delimiter: "",
};

let getS3Files = () => {
  let files = [];
  s3.listObjectsV2(params, function (err, data) {
    if (data) {
      data.Contents.forEach((file) => {
        files.push({
          fileName: file.Key,
          fileDate: file.LastModified,
        });
      });
      console.log(files);
    }
  });
  console.log(files);
  if (files.length > 0) {
    files = files.sort((a, b) => b.fileDate - a.fileDate);
    console.log(files);
  }
  return files;
};

getS3Files();

I think the behavior is given by the way you check if data has values. Please instead of `if (data) {...}` try `if (Object.keys(data).length > 0) {...}` — Mario, Jun 27 '20 at 05:18

score 1 · Answer 1 · answered Jun 27 '20 at 05:48

It is because you are not waiting until the s3.listObjectsV2 completes fetching its data. Although s3.listObjectsV2 does not seem like a function you should wait until it completes, it needs time to send a request to s3 and get the data you requested.

In the code you provided, the second console.log will most probably execute before the first console.log and by that time you would not have any data pushed to files. Hence it will print []. And before the third log, it is checking whether you have any element in your files which would again equal to false since files = [].

So, you need to wrap this around a Promise and wait until it completes.

let getS3Files = async () => {
  let files = [];

  await new Promise(function (resolve, reject) {
    s3.listObjectsV2(params, function (err, data) {
      if (data) {
        data.Contents.forEach((file) => {
          files.push({
            fileName: file.Key,
            fileDate: file.LastModified,
          });
        });
        console.log(files);
        resolve();
      } else {
        reject(err);
      }
    });
  });
  console.log(files);
  if (files.length > 0) {
    files = files.sort((a, b) => b.fileDate - a.fileDate);
    console.log(files);
  }
  return files;
};

await getS3Files();

Mario · Answer 2 · 2020-06-27T05:50:26.707

0

I am not familiar with the s3 api, but I understand listObjectsV2 is an asynchronous operation, in consequence files will be empty because it happens outside the asynchronous call and is executed before having values

Please try

let AWS = require("aws-sdk");

AWS.config.update({
  accessKeyId: process.env.VUE_APP_ACCESS_KEY,
  secretAccessKey: process.env.VUE_APP_ACCESS_KEY,
  region: "us-east-1",
});

let s3 = new AWS.S3();
let params = {
  Bucket: "my-bucket",
  Delimiter: "",
};

let getS3Files = (callback) => {
  s3.listObjectsV2(params, function (err, data) {
    callback(data);
  });
};

getS3Files((data) => {
  const files = data.Contents.map((file) => ({
    fileName: file.key,
    fileDate: file.LastModified,
  })).sort((a, b) => b.fileDate - a.fileDate);

  return files;
});

In this case, the getS3Files receives a callback that will contain the data, which you will process in the function call.

Take a look to this question How do I return the response from an asynchronous call?

edited Jun 27 '20 at 05:50

answered Jun 27 '20 at 05:30

Mario

4,784
3
34
50

It seems like this works perfectly fine if I put a `console.log` above the `return files` and call the function from the terminal, but if I try to import and reference the function from inside the Vue app, I get `Uncaught TypeError: Cannot read property 'Contents' of null`. – huskermdl Jun 27 '20 at 14:10
Please add `if (err) { console.log(err, err.stack); return; }` before `callback(data);` in order to identify the error reason – Mario Jun 27 '20 at 18:19
Actually getting a mix of different errors now. The first is `[Vue warn]: Error in render: "TypeError: _s3Data__WEBPACK_IMPORTED_MODULE_1___default(...) is not a function"` and the second is `SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your key and signing method.` which looks like an issue with grabbing the data from S3. – huskermdl Jun 27 '20 at 20:21
About `SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your key and signing method.` take a look to this answer and comments https://stackoverflow.com/a/30519762/615274 could be relevant – Mario Jun 28 '20 at 01:39
I came across that earlier and I wish it was that easy. I only have 1 file in the bucket right now and it's `MOVIE.mp4`. – huskermdl Jun 28 '20 at 11:17

How to access S3 bucket through JavaScript?

2 Answers2