Node JS async/await with multiple fs.writeFile using through2 (Gulp/Vinyl)

Question

I'm using through2 to generate multiple files from a Gulp stream. I'm using NodeJS 10.6.0 so thought I'd make full use of async/await, but am not fully understanding the mechanics yet. Currently the through2 done() callback is being fired before all files have been written.

Here's what I have (simplified) - note that I'm not returning the stream at the end as there is no need to.

async function createDirectory(pathDir) {
    return new Promise((resolve, reject) => {
        mkdirp(pathDir, (err) => {
            if (err) reject(err);
            else resolve();
        });
    });
}

async function writeFile(outputFilePath, outputFileContent) {
    return new Promise((resolve, reject) => {
        fs.writeFile(outputFilePath, outputFileContent, (err) => {
            if (err) reject(err);
            else resolve();
        });
    });
}

async function doWriteFile(outputFolderPath, outputFilePath, outputContent) {
    await createDirectory(outputFolderPath);
    await writeFile(outputFilePath, outputContent, outputContent);
}

async function doGenerateVariant(data, variantArr) {
    for (const variant of variantArr) {

        /* Do a load of stuff */

        const variantOutputFolderPath = blah;
        const variantOutputFilePath = blah;
        const variantOutputContent = blah;

        await doWriteFile(variantOutputFolderPath, variantOutputFilePath, variantOutputContent);
    }
}

const generateVariant = () => {
    return through.obj((file, enc, done) => {
        const data = JSON.parse(file.contents.toString());

        */ Do a load of stuff */

        const { variant } = data;
        const variantArr = Object.values(variant);
        doGenerateVariant(data, variantArr);
        return done();
    });
};

This doesn't work as done() gets returned before all files have been written. I'm guessing I'm missing a return or two but nothing I do seems to be working.

If I pass done() into doGenerateVariant and call it after doWriteFile everything works as expected but I know this isn't correct.

score 1 · Accepted Answer · answered Jul 09 '18 at 12:31

1

You need to wait for doGenerateVariant to do its job before calling done. Remember async function always returns a Promise. So you could do it this way

const generateVariant = () => {
    return through.obj((file, enc, done) => {
        const data = JSON.parse(file.contents.toString());

        */ Do a load of stuff */

        const { variant } = data;
        const variantArr = Object.values(variant);
        doGenerateVariant(data, variantArr).then(() => done());
    });
};

or using async/await

const generateVariant = () => {
    return through.obj(async (file, enc, done) => {
        const data = JSON.parse(file.contents.toString());

        */ Do a load of stuff */

        const { variant } = data;
        const variantArr = Object.values(variant);
        await doGenerateVariant(data, variantArr);
        done();
    });
};

answered Jul 09 '18 at 12:31

Yury Tarabanko

44,270
9
84
98

Perfect! Just what I needed. Thank you! :) – Gareth James Jul 09 '18 at 12:50
One question though... I've got `await doWriteFile` in `doGenerateVariant` and eslint states "Performing an operation on each element of an iterable is a common task. However, performing an await as part of each operation is an indication that the program is not taking full advantage of the parallelization benefits of async/await." Does this mean that my functions aren't fully taking advantage of async/await? – Gareth James Jul 09 '18 at 12:52
@GarethJames It says you are writing one file at a time. `await` stops function execution until the awaited promise got resolved. If you want to write files in parallel you could convert all `variantArr` to promises and wait for all `await Promise.all(variantArr.map(variant => { your logic; return doWriteFile(...)}))` – Yury Tarabanko Jul 09 '18 at 12:55
@GarethJames No problem. Glad I could help :) – Yury Tarabanko Jul 09 '18 at 13:03

Node JS async/await with multiple fs.writeFile using through2 (Gulp/Vinyl)

1 Answers1