Node.js AWS Lambda putObject doesn't seem to finish

Question

I have a Lambda function that is meant to download a directory of files from s3, convert them, delete the old files, and upload the new output files back to s3. The output for each file will be at least one file and a folder.

Everything seems to be working as intended, except for the upload. No errors are thrown, it just ends without putting.

I'm a novice, so feel free to point out I've done it all wrong.

exports.handler = async ({ dirName }) => {
    // const jsonIn = JSON.parse(event.body);
    // const dirName = jsonIn.dirName;
    
    const localDir = `/tmp/${dirName}`;
    const params = {
        Bucket: 'to-pdf-test',
        Delimiter: '/',
        Prefix: dirName + '/',
        StartAfter: dirName + '/'
    };
    var s3List;
    var localList = [];

    execSync(`mkdir ${localDir}`);

    try {
        s3List = await s3.listObjectsV2(params).promise();
    } catch (e) {
        throw e;
    }

    await Promise.all(
        s3List.Contents.map(async (file) => {
            let f = await getFiles(file);
            localList.push(f);
        })
    ).then(res => {console.log('Get Successful' + res) } )
        .catch(err => {console.log('error' + err) } );
        
    await Promise.all(
        localList.map(async (file) => {
            convertFile(file);
        })
    ).then(res => {console.log('Convert Successful' + res) } )
        .catch(err => {console.log('error' + err) } );
    

    dirSync(localDir, async (filePath, stat) => {
        let bucketPath = filePath.substring(5);
        let uploadParams = { Bucket: 'to-pdf-test', 
        Key: `${bucketPath}`, 
        Body: fs.readFileSync(filePath) };

        console.log('DS fPath ' + filePath);
        console.log('DS bPath ' + bucketPath);
        console.log(uploadParams.Body);
        
        try {
            let res = await s3.putObject(uploadParams).promise();
            console.log('Upload Complete', res);
        } catch (e) {
            console.log('Error', e);
        }
    });

};

async function getFiles(file) {
    let filePath = `/tmp/${file.Key}`;
    let fileParams = {
        Bucket: 'to-pdf-test',
        Key: file.Key
    };
    try {
        const { Body: inputFileBuffer } = await s3.getObject(fileParams).promise();
        fs.writeFileSync(filePath, inputFileBuffer);
    } catch (e) {
        throw (e);
    }
    return filePath;
}


function convertFile(file) {
    const noPath = getFilename(file);
    const fPath = getFilePath(file);

    if (path.extname(noPath) === '.msg') {
        execSync(`cd ${fPath} && ${command} ${noPath}`);
    } else {
        console.log(`${noPath} not run. Not .msg`);
    }

    fs.unlinkSync(file);
    
}
function getFilename(fullPath) {
    return fullPath.replace(/^.*[\\\/]/, '');
}

function getFilePath(fullPath) {
    return fullPath.substring(fullPath.lastIndexOf('/'), 0);
}

function dirSync(dirPath, callback) {
    fs.readdirSync(dirPath).forEach((name) => {
        var filePath = path.join(dirPath, name);
        var stat = fs.statSync(filePath);
        if (stat.isDirectory()) {
            dirSync(filePath, callback);
        } else {
            callback(filePath, stat);
        }
    });
}

I had the upload working in a previous version of this function, so thanks to this post for when it was working.

score 0 · Accepted Answer · answered Oct 21 '22 at 04:33

My solution for the moment - Read the local directory separately, push the paths of the files to localList then .map the array with all the paths to upload them.

localList = [];
//read dir and push to localList array 
await dirSync(localDir, (filePath, stat) => {
    localList.push(filePath);
});

console.log(localList);

await Promise.all(
    localList.map( async (file) => {
        let bucketPath = file.substring(5);
        let uploadParams = { 
            Bucket: 'to-pdf-test',
            Key: bucketPath,
            Body: fs.readFileSync(file) };
        console.log('Uploading', file);
        await s3.putObject(uploadParams).promise()
        .then((res) => {console.log('Upload Successful', bucketPath) } )
        .catch((err) => {console.log('error' + err) } );
    })
);

If there is better (or proper) way to do this, someone let me know :)

Node.js AWS Lambda putObject doesn't seem to finish

1 Answers1