Gulp handle an array of files with gulp-rev

Question

Context

I have a gulpfile.js. It basicaly concats two folders with js files to two files to export to a dist folder. It also adds a cache buster and a uglified version for production.

//gulpfile.js
const gulp = require('gulp');
const sass = require('gulp-sass');
const concat = require('gulp-concat');
const rename = require('gulp-rename');
const maps = require('gulp-sourcemaps');
const uglify = require('gulp-uglify');
const gulpif = require('gulp-if');
const inlineCss = require('gulp-inline-css');
const jsValidate = require('gulp-jsvalidate');
const notify = require('gulp-notify');
const rev = require('gulp-rev');
const fs = require("fs");
const del = require('del');

const distFolder = './myapp/dist/';

/* Concat all scripts and add cache buster */
gulp.task('app_concatScripts', (done) => {
    /*first remove old app.js files */
    return del([distFolder + "app-*.{js,map}"])
        .then(paths => {
            /*Concat files with sourcemaps, hash them and write to manifest*/
            return gulp.src("./myapp/js/source-app/*.js")
                .pipe(maps.init())
                .pipe(concat({
                    path: "app.js",
                    cwd: '',
                    newLine: '; \n'
                }))
                .pipe(gulp.dest(distFolder))
                .pipe(rev())
                .pipe(maps.write('./'))
                .pipe(gulp.dest(distFolder))
                .pipe(rev.manifest(distFolder + 'rev-manifest.json', {
                    base: distFolder,
                    merge: true
                }))
                .pipe(gulp.dest(distFolder));
        });

});

/* Minify scripts and add cache buster */
gulp.task('app_minifyScripts', (done) => {
    return gulp.src(distFolder + "app.js", {
            allowEmpty: true
        })
        .pipe(uglify())
        .on('error', notify.onError((error) => {
            console.log(error);
            return error.message + ' in ' + error.fileName;
        }))
        .pipe(rename("app.min.js"))
        .pipe(rev())
        .pipe(gulp.dest(distFolder))
        .pipe(rev.manifest(distFolder + 'rev-manifest.json', {
            base: distFolder,
            merge: true
        }))
        .pipe(gulp.dest(distFolder));
});

/* Concat all scripts and add cache buster, this function is exactly the same as app_concatScripts */
gulp.task('main_concatScripts', (done) => {
     /*Concat files with sourcemaps, hash them and write to manifest*/
    return del(["./myapp/dist/main-*.{js,map}"])
        .then(paths => {
            return gulp.src("./myapp/js/source-main/*.js")
                .pipe(maps.init())
                .pipe(concat({
                    path: "main.js",
                    cwd: '',
                    newLine: '; \n'
                }))
                .pipe(gulp.dest(distFolder))
                .pipe(rev())
                .pipe(maps.write('./'))
                .pipe(gulp.dest(distFolder))
                .pipe(rev.manifest(distFolder + 'rev-manifest.json', {
                    base: distFolder,
                    merge: true
                }))
                .pipe(gulp.dest(distFolder));
        });
});

/* Minify scripts and add cache buster, this function is the same as app_minifyScripts */
gulp.task('main_minifyScripts', (done) => {

    return gulp.src("./myapp/dist/main.js")
        .pipe(uglify())
        .on('error', notify.onError((error) => {
            console.log(error);
            return error.message + ' in ' + error.fileName;
        }))
        .pipe(rename("main.min.js"))
        .pipe(rev())
        .pipe(gulp.dest(distFolder))
        .pipe(rev.manifest(distFolder + 'rev-manifest.json', {
            base: distFolder,
            merge: true
        }))
        .pipe(gulp.dest(distFolder));
});


/* watch task */
gulp.task('watch', (done) => {
    gulp.watch('./myapp/js/source-app/*.js', gulp.series('app_concatScripts', 'app_minifyScripts'));
    gulp.watch('./myapp/js/source-main/*.js', gulp.series('main_concatScripts', 'main_minifyScripts'));
    done();
});
gulp.task('default', gulp.series('main_concatScripts', 'main_minifyScripts', 'app_concatScripts', 'app_minifyScripts', 'watch'));

Output

The gulpfile.js works. It builds a rev-manifest.json like this:

{
  "app.js": "app-234318a58d.js",
  "app.min.js": "app-7788fee7f3.min.js",
  "main.js": "main-60788c863c.js",
  "main.min.js": "main-1e92517890.min.js",
  "style.css": "style-296f22c598.css",
  "admin.css": "admin-a3742ed2f6.css",
}

Problem

To get this working i copied the _concatScripts and _minifyScripts task for each js file: app.js and main.js. Making the gulpfile unnecessary large and hard to maintain.

What I have tried

I tried using an array to define scripts source files and output names. This does not work because the code in the loop block does not wait for the previous block to be ready. So the minify task fails sometimes because the original js file does not exists yet and sometimes the rev-manifest.json get overwritten by a previous code block. So it is not really a gulp series anymore. What is the best practice to do something like this?

const scripts = [{
    name: 'app',
    source: './myapp/js/source-app/*.js'
}, {
    name: 'main',
    source: './myapp/js/source-main/*.js'
}];

/* Concat all scripts and add cache buster */
gulp.task('concatScripts', (done) => {
    for (var i = 0; i < scripts.length; i++) {
        var script = scripts[i];
        del([distFolder + script.name + "-*.{js,map}"])
            .then(paths => {
                /*Concat files with sourcemaps, hash them and write to manifest*/
                gulp.src(script.source)
                    .pipe(maps.init())
                    .pipe(concat({
                        path: script.name + ".js",
                        cwd: '',
                        newLine: '; \n'
                    }))
                    .pipe(gulp.dest(distFolder))
                    .pipe(rev())
                    .pipe(maps.write('./'))
                    .pipe(gulp.dest(distFolder))
                    .pipe(rev.manifest(distFolder + 'rev-manifest.json', {
                        base: distFolder,
                        merge: true
                    }))
                    .pipe(gulp.dest(distFolder));
            });
    }

});

/* Minify scripts and add cache buster */
gulp.task('minifyScripts', (done) => {
    for (var i = 0; i < scripts.length; i++) {
        var script = scripts[i];
        gulp.src(distFolder + script.name + ".js", {
                allowEmpty: true
            })
            .pipe(uglify())
            .on('error', notify.onError((error) => {
                console.log(error);
                return error.message + ' in ' + error.fileName;
            }))
            .pipe(rename(script.name + ".min.js"))
            .pipe(rev())
            .pipe(gulp.dest(distFolder))
            .pipe(rev.manifest(distFolder + 'rev-manifest.json', {
                base: distFolder,
                merge: true
            }))
            .pipe(gulp.dest(distFolder));
    }
});

/* watch task */
gulp.task('watch', (done) => {
    gulp.watch(['./myapp/js/source-app/*.js', './myapp/js/source-main/*.js'], gulp.series('concatScripts', 'minifyScripts'));
    done();
});

gulp.task('default', gulp.series('concatScripts', 'minifyScripts', 'watch'));

update

My gulp file now looks like this:

//gulpfile.js
const gulp = require('gulp');
const sass = require('gulp-sass');
const concat = require('gulp-concat');
const rename = require('gulp-rename');
const maps = require('gulp-sourcemaps');
const uglify = require('gulp-uglify');
const gulpif = require('gulp-if');
const inlineCss = require('gulp-inline-css');
const jsValidate = require('gulp-jsvalidate');
const notify = require('gulp-notify');
const rev = require('gulp-rev');
const fs = require("fs");
const del = require('del');
/* Concat all scripts and add cache buster */

const distFolder = './myapp/dist/';

function concatTask(opts) {
    const taskName = opts.name + '_concatScripts';
    gulp.task(taskName, (done) => {
        /*Concat files with sourcemaps, hash them and write to manifest*/
        return del(opts.del)
            .then(paths => {
                return gulp.src(opts.source)
                    .pipe(maps.init())
                    .pipe(concat({
                        'path': opts.name + '.js',
                        'cwd': '',
                        'newLine': '; \n'
                    }))
                    .pipe(gulp.dest(distFolder))
                    .pipe(rev())
                    .pipe(maps.write('./'))
                    .pipe(gulp.dest(distFolder))
                    .pipe(rev.manifest(distFolder + 'rev-manifest.json', {
                        'base': distFolder,
                        'merge': true
                    }))
                    .pipe(gulp.dest(distFolder));
            });
    });
    return taskName;
}

function minifyTask(opts) {
    const taskName = opts.name + '_minifyScripts';
    gulp.task(taskName, (done) => {
        return gulp.src(distFolder + opts.name + '.js', opts.sourceParams)
            .pipe(uglify())
            .on('error', notify.onError((error) => {
                console.log(error);
                return error.message + ' in ' + error.fileName;
            }))
            .pipe(rename(opts.name + '.min.js'))
            .pipe(rev())
            .pipe(gulp.dest(distFolder))
            .pipe(rev.manifest(distFolder + 'rev-manifest.json', {
                'base': distFolder,
                'merge': true
            }))
            .pipe(gulp.dest(distFolder));
    });
    return taskName;
}

const taskOptions = [{
    'name': 'main',
    'source': './myapp/js/source-main/*.js',
    'sourceParams': {
        'allowEmpty': false
    },
    'del': ['./myapp/dist/main-*.{js,map}']
}, {
    'name': 'app',
    'source': './myapp/js/source-app/*.js',
    'sourceParams': {
        'allowEmpty': false
    },
    'del': ['./myapp/dist/app-*.{js,map}']
},

 ];

const taskNames = taskOptions.map(opts => {
    return [concatTask(opts), minifyTask(opts)];
});
/* equivalent to defining tasks and assigning:
 *  var taskNames = [
 *    ['main_concatScripts', 'main_minifyScripts'],
 *    ['app_concatScripts', 'app_minifyScripts']
 *  ];
 */

gulp.task('watch', (done) => {
    taskOptions.forEach((opts, i) => {
        gulp.watch(opts.source, gulp.series.apply(gulp, taskNames[i]));
    });
    done();
});
/* equivalent to:
 *  gulp.task('watch', (done) => {
 *    gulp.watch('./myapp/js/source-main/*.js', gulp.series('main_concatScripts', 'main_minifyScripts'));
 *    gulp.watch('./myapp/js/source-app/*.js', gulp.series('app_concatScripts', 'app_minifyScripts'));
 *    done();
 *  });
 */

const allTaskNames = taskNames.reduce((arr, names) => arr.concat(names), []); // flatmap the task names into a single array
// console.log(allTaskNames.concat(['watch']));
/* equivalent to:
 *  let allTaskNames = ['main_concatScripts', 'main_minifyScripts', 'app_concatScripts', 'app_minifyScripts'];
 */

gulp.task('default', gulp.series.apply(gulp, allTaskNames.concat(['styles', 'watch'])));
/* equivalent to:
 *  gulp.task('default', gulp.series('main_concatScripts', 'main_minifyScripts', 'app_concatScripts', 'app_minifyScripts', 'watch'));
 */

The dynamic task creators work. Unfortunately the minify tasks are starting before the concatenation task are finished. I am getting the following error, adding the allowEmpty option does work, but then the wrong concatenated file gets minified:

[12:57:19] Using gulpfile /repo/myapp/gulpfile.js
[12:57:19] Starting 'default'...
[12:57:19] Starting 'main_concatScripts'...
[12:57:19] Finished 'main_concatScripts' after 47 ms
[12:57:19] Starting 'main_minifyScripts'...
[12:57:19] 'main_minifyScripts' errored after 11 ms
[12:57:19] Error: File not found with singular glob: /repo/myapp/dist/main.js (if this was purposeful, use `allowEmpty` option)

Answer 1

If it's just a question of keeping the code DRY, you can write two functions containing generalisations of the WET code :

• concatTask()
• minifyTask()

Each of these funtions should define a gulp task and return an internally-generated task name.

function concatTask(opts) {
    const taskName = opts.name + '_concatScripts';
    gulp.task(taskName, (done) => {
        /*Concat files with sourcemaps, hash them and write to manifest*/
        return del(opts.del)
        .then(paths => {
            return gulp.src(opts.source)
            .pipe(maps.init())
            .pipe(concat({
                'path': opts.name + '.js',
                'cwd': '',
                'newLine': '; \n'
            }))
            .pipe(gulp.dest(distFolder))
            .pipe(rev())
            .pipe(maps.write('./'))
            .pipe(gulp.dest(distFolder))
            .pipe(rev.manifest(distFolder + 'rev-manifest.json', {
                'base': distFolder,
                'merge': true
            }))
            .pipe(gulp.dest(distFolder));
        });
    });
    return taskName;
}

function minifyTask(opts) {
    const taskName = opts.name + '_minifyScripts';
    gulp.task(taskName, (done) => {
        return gulp.src(opts.source, opts.sourceParams)
        .pipe(uglify())
        .on('error', notify.onError((error) => {
            console.log(error);
            return error.message + ' in ' + error.fileName;
        }))
        .pipe(rename(opts.name + '.min.js'))
        .pipe(rev())
        .pipe(gulp.dest(distFolder))
        .pipe(rev.manifest(distFolder + 'rev-manifest.json', {
            'base': distFolder,
            'merge': true
        }))
        .pipe(gulp.dest(distFolder));
    });
    return taskName;
}

Task definitions, watches and series executions can then be proceduralized as follows :

const taskOptions = [
    {
        'name': 'main',
        'source': './myapp/js/source-main/*.js',
        'sourceParams': {},
        'del': ['./myapp/dist/main-*.{js,map}']
    },
    {
        'name': 'app',
        'source': './myapp/js/source-app/*.js',
        'sourceParams': { 'allowEmpty': true },
        'del': ['./myapp/dist/app-*.{js,map}']
    }
];

const taskNames = taskOptions.map(opts => {
    return [concatTask(opts), minifyTask(opts)];
});
/* equivalent to defining tasks and assigning:
 *  var taskNames = [
 *    ['main_concatScripts', 'main_minifyScripts'],
 *    ['app_concatScripts', 'app_minifyScripts']
 *  ];
 */

gulp.task('watch', (done) => {
    taskOptions.forEach((opts, i) => {
        gulp.watch(opts.source, gulp.series.apply(gulp, taskNames[i]));
    });
    done();
});
/* equivalent to:
 *  gulp.task('watch', (done) => {
 *    gulp.watch('./myapp/js/source-main/*.js', gulp.series('main_concatScripts', 'main_minifyScripts'));
 *    gulp.watch('./myapp/js/source-app/*.js', gulp.series('app_concatScripts', 'app_minifyScripts'));
 *    done();
 *  });
 */

const allTaskNames = taskNames.reduce((arr, names) => arr.concat(names), []); // flatmap the task names into a single array
/* equivalent to:
 *  let allTaskNames = ['main_concatScripts', 'main_minifyScripts', 'app_concatScripts', 'app_minifyScripts'];
 */

gulp.task('default', gulp.series.apply(gulp, allTaskNames.concat('watch')));
/* equivalent to:
 *  gulp.task('default', gulp.series('main_concatScripts', 'main_minifyScripts', 'app_concatScripts', 'app_minifyScripts', 'watch'));
 */

Barring mistakes, everything will happen in the same order as in the original code.

What you end up with is not particularly self-explanatory. An understanding of Function.prototype.apply() will certainly help.

Personally, I would leave the "equivalent to:..." comments in the deployed code as a reminder to oneself (and an explanation to others) of what is actually going on.

Answer 2

So thanks to Roamer if figured to add another task for the deleting of js files. So one task can wait for another to be finished. Here is the gulp file:

//gulpfile.js
const gulp = require('gulp');
const sass = require('gulp-sass');
const concat = require('gulp-concat');
const rename = require('gulp-rename');
const maps = require('gulp-sourcemaps');
const uglify = require('gulp-uglify');
const gulpif = require('gulp-if');
const inlineCss = require('gulp-inline-css');
const jsValidate = require('gulp-jsvalidate');
const notify = require('gulp-notify');
const rev = require('gulp-rev');
const fs = require("fs");
const del = require('del');
/* Concat all scripts and add cache buster */

const distFolder = './myapp/dist/';

function delTask(opts) {
    const taskName = opts.name + '_delScripts';
    gulp.task(taskName, (done) => {
        /*Delete old files*/
        return del(opts.del)

    });
    return taskName;
}

function concatTask(opts) {
    const taskName = opts.name + '_concatScripts';
    gulp.task(taskName, (done) => {
        /*Concat files with sourcemaps, hash them and write to manifest*/
        return gulp.src(opts.source)
            .pipe(maps.init())
            .pipe(concat({
                'path': opts.name + '.js',
                'cwd': '',
                'newLine': '; \n'
            }))
            .pipe(gulp.dest(distFolder))
            .pipe(rev())
            .pipe(maps.write('./'))
            .pipe(gulp.dest(distFolder))
            .pipe(rev.manifest(distFolder + 'rev-manifest.json', {
                'base': distFolder,
                'merge': true
            }))
            .pipe(gulp.dest(distFolder));
    });
    return taskName;
}

function minifyTask(opts) {
    const taskName = opts.name + '_minifyScripts';
    gulp.task(taskName, (done) => {
        return gulp.src(distFolder + opts.name + '.js', opts.sourceParams)
            .pipe(uglify())
            .on('error', notify.onError((error) => {
                console.log(error);
                return error.message + ' in ' + error.fileName;
            }))
            .pipe(rename(opts.name + '.min.js'))
            .pipe(rev())
            .pipe(gulp.dest(distFolder))
            .pipe(rev.manifest(distFolder + 'rev-manifest.json', {
                'base': distFolder,
                'merge': true
            }))
            .pipe(gulp.dest(distFolder));
    });
    return taskName;
}

const taskOptions = [{
    'name': 'main',
    'source': './myapp/js/source-main/*.js',
    'sourceParams': {
        'allowEmpty': false
    },
    'del': ['./myapp/dist/main-*.{js,map}']
}, {
    'name': 'app',
    'source': './myapp/js/source-app/*.js',
    'sourceParams': {
        'allowEmpty': false
    },
    'del': ['./myapp/dist/app-*.{js,map}']
}];

const taskNames = taskOptions.map(opts => {
    return [delTask(opts), concatTask(opts), minifyTask(opts)];
});
/* equivalent to defining tasks and assigning:
 *  var taskNames = [
 *    ['main_delScripts', 'main_concatScripts', 'main_minifyScripts'],
 *    ['app_delScripts', 'app_concatScripts', 'app_minifyScripts']
 *  ];
 */

gulp.task('watch', (done) => {
    taskOptions.forEach((opts, i) => {
        gulp.watch(opts.source, gulp.series.apply(gulp, taskNames[i]));
    });
    done();
});
/* equivalent to:
 *  gulp.task('watch', (done) => {
 *    gulp.watch('./myapp/js/source-main/*.js', gulp.series('main_delScripts', 'main_concatScripts', 'main_minifyScripts'));
 *    gulp.watch('./myapp/js/source-app/*.js', gulp.series('app_delScripts', 'app_concatScripts', 'app_minifyScripts'));
 *    done();
 *  });
 */

const allTaskNames = taskNames.reduce((arr, names) => arr.concat(names), []); // flatmap the task names into a single array
// console.log(allTaskNames.concat(['watch']));
/* equivalent to:
 *  let allTaskNames = ['main_delScripts', 'main_concatScripts', 'main_minifyScripts', 'app_delScripts', 'app_concatScripts', 'app_minifyScripts'];
 */

gulp.task('default', gulp.series.apply(gulp, allTaskNames.concat(['styles', 'watch'])));
/* equivalent to:
 *  gulp.task('default', gulp.series('main_delScripts', 'main_concatScripts', 'main_minifyScripts', 'app_delScripts', 'app_concatScripts', 'app_minifyScripts', 'watch'));
 */

And the output:

[16:40:31] Using gulpfile gulpfile.js
[16:40:31] Starting 'default'...
[16:40:31] Starting 'main_delScripts'...
[16:40:31] Finished 'main_delScripts' after 29 ms
[16:40:31] Starting 'main_concatScripts'...
[16:40:31] Finished 'main_concatScripts' after 84 ms
[16:40:31] Starting 'main_minifyScripts'...
[16:40:34] Finished 'main_minifyScripts' after 2.45 s
[16:40:34] Starting 'app_delScripts'...
[16:40:34] Finished 'app_delScripts' after 941 μs
[16:40:34] Starting 'app_concatScripts'...
[16:40:34] Finished 'app_concatScripts' after 58 ms
[16:40:34] Starting 'app_minifyScripts'...
[16:40:37] Finished 'app_minifyScripts' after 3.12 s
[16:40:37] Starting 'styles'...
[16:40:37] Finished 'styles' after 215 ms
[16:40:37] Starting 'watch'...
[16:40:37] Finished 'watch' after 19 ms
[16:40:37] Finished 'default' after 6.04 s