Node.js Fetch Remote Files and Concatenate and Save Them Locally

Question

Below works but it looks like I'm missing little chunks of data. What's the best way to achieve what I'm trying to do here. I run node updateFiles.js 5.5.9 and it's lightning fast but like I said, when I eyeball the result file, it appears I miss some chunks...?

Below is the contents of updateFiles.js.

const fs = require('fs');
const fetch = require('node-fetch');
var version = process.argv[2];
var filePath = 'source/js/firebase.js';
var sdks = [
'https://www.gstatic.com/firebasejs/' + version + '/firebase-app.js',
'https://www.gstatic.com/firebasejs/' + version + '/firebase-auth.js',
'https://www.gstatic.com/firebasejs/' + version + '/firebase-database.js',
'https://www.gstatic.com/firebasejs/' + version + '/firebase-firestore.js',
'https://www.gstatic.com/firebasejs/' + version + '/firebase-messaging.js',
'https://www.gstatic.com/firebasejs/' + version + '/firebase-functions.js'
]; 

//fetch each firebase file and write it to source/js/firebase.js
for (var sdk of sdks){
    fetch(sdk).then(res => {
        const dest = fs.createWriteStream(filePath,{flags:'a'});
        res.body.pipe(dest);
    });    
}

UPDATE: I ended up doing this and it seems to work, but, if there's a guru out there that can indicate a better way please let me know. Thank you in advance.

const fs = require('fs');
const fetch = require('node-fetch');
var version = process.argv[2];
var promisesArray = [];
var sdks = [
// Firebase App is always required and must be first
'https://www.gstatic.com/firebasejs/' + version + '/firebase-app.js',
// Add additional services that you want to use
'https://www.gstatic.com/firebasejs/' + version + '/firebase-auth.js',
'https://www.gstatic.com/firebasejs/' + version + '/firebase-database.js',
'https://www.gstatic.com/firebasejs/' + version + '/firebase-firestore.js',
'https://www.gstatic.com/firebasejs/' + version + '/firebase-messaging.js',
'https://www.gstatic.com/firebasejs/' + version + '/firebase-functions.js'
];

var filePath = 'source/js/firebase.js'; 

for (var sdk of sdks){
    promisesArray.push(fetch(sdk));
}

Promise.all(promisesArray).then(sdkFiles => {
    for (var file of sdkFiles){
        const dest = fs.createWriteStream(filePath,{flags:'a'});
        file.body.pipe(dest);
    }
});

score 0 · Answer 1 · answered Nov 29 '18 at 21:50

0

Maybe you cannot write all files at once but wait for each file to be processed and then continue with another one? The promise for loop is taken from: JavaScript ES6 promise for loop

for (let i = 0, p = Promise.resolve(); i < sdks.length; i++) {
  p = p.then(_ => new Promise(resolve =>
    fetch(sdks[i]).then(res => {
      const dest = fs.createWriteStream(filePath, { flags: 'a' });
      res.body.pipe(dest);
      resolve()
    })
  ));
}

answered Nov 29 '18 at 21:50

Jihlavanka

370
2
10

True. ...I'm just trying to do it the best possible way, or the "Node way", or the advanced way as shown [here](https://www.npmjs.com/package/node-fetch#streams). Realistically and practically, I could do that, although Promise.all might be slightly faster (parallel) – Ronnie Royston Nov 29 '18 at 21:57

Node.js Fetch Remote Files and Concatenate and Save Them Locally

1 Answers1