Sync a forEach with an event inside (Firebase & node 6.11.5)

Question

I'm reading, bit per bit, all the PNG files inside a directory and I've to summarize some data in a json. The problem is that, if I understand, the PNG reader send an async event "parsed" when finished. That cause that the function exits beforse json is populated...

I'm using node 6.11.5, so I cannot use sync/await.

var fs = require('fs'),
PNG = require('pngjs').PNG;

exports.movie = functions.https.onRequest((req, res) => {
    console.log('********** START FUNCTION ************');

    var movieFolder = 1;
    if (req.query.id) movieFolder = '../movies/' + req.query.id + '/png/';

    var exitJson = [];

    fs.readdir(movieFolder, (err, files) => {
        files.forEach((file) => {
          fs.createReadStream(movieFolder + file)
            .pipe(new PNG({
                filterType: 1
             }))
            .on('parsed', function () {
                console.log('Parsing: ' + movieFolder + file);
                exitJson.push({
                    width: this.width,
                    height: this.height,
                    data: []
                });
            });
        });
    });
    console.log('************* FINISHED *************');
    res.status(200).json(exitJson);
});

Possible duplicate of [Resolve promises one after another (i.e. in sequence)?](https://stackoverflow.com/questions/24586110/resolve-promises-one-after-another-i-e-in-sequence) — Callam, Jan 10 '19 at 12:06

score 1 · Accepted Answer · answered Jan 10 '19 at 12:14

You can use a simple itemsProcessed counter to detect if all of your callbacks were resolved.

var movieFolder = 1;
if (req.query.id) movieFolder = '../movies/' + req.query.id + '/png/';

var exitJson = [];

var itemsProcessed = 0;

fs.readdir(movieFolder, (err, files) => {
    files.forEach((file) => {
      fs.createReadStream(movieFolder + file)
        .pipe(new PNG({
            filterType: 1
         }))
        .on('parsed', function () {
            console.log('Parsing: ' + movieFolder + file);
            exitJson.push({
                width: this.width,
                height: this.height,
                data: []
            });
            itemsProcessed++;
            if (itemsProcessed === files.length) {
              console.log('************* FINISHED *************');
              res.status(200).json(exitJson);
            }
        });
    });
});

Callam · Answer 2 · 2019-01-10T15:04:49.593

const exports={};const sizes={'foo.png':[100,200],'bar.png':[200,200],'baz.png':[300,200]};Promise.delay = (t) => new Promise(r => setTimeout(r, t)); const randomTime = (a = 500, b = 1500) => Math.floor(Math.random() * b) + a;
const require=src=>({'fs':{readdir:(d,c)=>{Promise.delay(randomTime()).then(() => c(null,['foo.png','bar.png','baz.png']))},createReadStream:(path)=>({pipe:(f)=>({on:(e,c)=>{const s=sizes[path.split('/').slice(-1)[0]];const a={width:s[0],height:s[1]};a.c=c;Promise.delay(randomTime()).then(() => a.c())}})})},'pngjs':{PNG:class PNG{constructor(a){}}},'firebase-functions':{https:{onRequest:(handler)=>{handler({query:({id:2})},{status:(s)=>({json:(a) => document.getElementById('res').innerHTML = `<pre><code>${JSON.stringify(a, null, 4)}</code></pre>`})})}}}})[src];

// ------------------- ignore the above

const fs = require('fs');
const PNG = require('pngjs').PNG;
const functions = require('firebase-functions');

/**
 * Using a new Promise, we can perform multiple async tasks all contained 
 * within that one Promise which can be resolved or rejected. We read the 
 * folder directory for its files and pass it on to our Promised 'readFiles'.
 */
function readMovieFiles(folder) { console.log('readMovieFiles', folder)
  return new Promise((res, rej) => {
    fs.readdir(folder, (err, files) => {
      readFiles(files, folder).then(res).catch(rej)
    });
  });
}

/**
 * Given an array of file names within a folder, we can chain together the 
 * file promises using the reduce method. Starting at an initial value of
 * Promise<[]>, each file in the array will be read sequentially.
 */
function readFiles(files, folder) { console.log('readFiles', folder, files)
  return Promise.all(files.map(name => readFile(folder + name)));
}

/**
 * We read a file and in the parsed callback, we call the res() and pass it 
 * the newly constructed array containing the newest file to be parsed.
 */
function readFile(path) { console.log('readFile', path)
  return new Promise((res, rej) => {
    fs.createReadStream(path)
      .pipe(new PNG({ filterType: 1 }))
      .on('parsed', function() {
        console.log('parsedFile', path)
        res({
          data: [],
          width: this.width,
          height: this.height
        });
      });
  });
}

exports.movie = functions.https.onRequest((req, res) => {

  console.log('********** START FUNCTION ************');
  
  if (!req.query.id) req.query.id = 1;
    
  readMovieFiles(`../movies/${req.query.id}/png/`).then(exitJson => {
    res.status(200).json(exitJson);
  }).catch(error => {
    res.status(500).json(error);
  });
  
  console.log('************* FINISHED *************');
});

<pre><code id="res"></code></pre>

Thanks! Can you explain what you did? You put every functions in a promise, right? Was my idea... but too complex for me... — Alberto Boz, Jan 10 '19 at 14:19
Sure, I've added some comments. The `readFiles` function is a little confusing but, it's essentially converting an array of file names to an array of promises, and executing them sequentially. For example, if you wanted them to run at this same time, you could just say `Promise.all(files.map(name => readFile(folder + name, arr)))`. — Callam, Jan 10 '19 at 14:37

score 0 · Answer 3 · answered Jan 10 '19 at 14:25

You can load files one by one through recursion calls. Don't forget to check errors.

exports.movie = functions.https.onRequest((req, res) => {
    var movieFolder = 1;
    if (req.query.id) 
        movieFolder = '../movies/' + req.query.id + '/png/';

    var exitJson = [];
    fs.readdir(movieFolder, function (err, files) {
        var sendError = (err) => res.status(500).send(err.message);

        if (err)
            return sendError(err);

        function loadFile (i) {
            if (i == files.length) 
                return res.status(200).json(exitJson); // !!!DONE!!!

            var file = files[i];
            fs.createReadStream(movieFolder + file)
                .pipe(new PNG({filterType: 1}))
                .on('parsed', function () {
                    console.log('Parsing: ' + movieFolder + file);
                    exitJson.push({width: this.width, height: this.height, data: []});
                    loadFile (i + 1); // go to next file
                })
                .on('error', sendError);
        }

        loadFile(0); // start recursion
    }); 
});

Sync a forEach with an event inside (Firebase & node 6.11.5)

3 Answers3