With Gulp 4+, when using gulp.parallel(), is there a way of using multiple CPUs for the concurrent tasks?

Question

I am considering using gulp.parallel() to concurrently run build tasks. When I tried a toy example, I found that only 1 CPU core (1 gulp process seen when I run top) is running all the tasks. Here is my toy example:

const { parallel } = require('gulp');

function javascript(cb) {
  setTimeout(function() {
   console.log("Javascript task!")
   cb();
 }, 20000);
}

function css(cb) {
  setTimeout(function() {
   console.log("CSS task!")
   cb();
 }, 10000);
}

function prettify(cb) {
  setTimeout(function() {
   console.log("Prettify task!")
   cb();
 }, 15000);
}

function colorize(cb) {
  setTimeout(function() {
   console.log("Colorize task!")
   cb();
 }, 5000);
}


function defaultTask(cb) {
  // place code for your default task here
  cb();
}

exports.default = defaultTask
exports.build = parallel(javascript, css, prettify,colorize);

When I do a "gulp build" I see the tasks running concurrently (total run time of the above example is 20s for all 4 tasks), but I see only one gulp process executing the 4 tasks.

I am hoping to utilize a multi-core CPU to speed up my gulp builds. Hence the question - are there any ways of utilizing multiple cores for a gulp task? I have looked at "gulp-multi-process" and this does spawn multiple processes when "gulp multi" is run against the gulpfile.js shown below:

const { parallel } = require('gulp');
var gulp = require('gulp');
var gulpMultiProcess = require('gulp-multi-process');

function javascript(cb) {
  setTimeout(function() {
   console.log("Javascript task!")
   cb();
 }, 20000);
}

function css(cb) {
  setTimeout(function() {
   console.log("CSS task!")
   cb();
 }, 10000);
}

function prettify(cb) {
  setTimeout(function() {
   console.log("Prettify task!")
   cb();
 }, 15000);
}

function colorize(cb) {
  setTimeout(function() {
   console.log("Colorize task!")
   cb();
 }, 5000);
}


gulp.task("javascript",javascript);
gulp.task("css",css);
gulp.task("prettify",prettify);
gulp.task("colorize",colorize);

function defaultTask(cb) {
  // place code for your default task here
  cb();
}

gulp.task('multi', function(cb) {
  // task1 and task2 will run in different processes
  return gulpMultiProcess(["javascript","css","prettify","colorize"], cb);
});

exports.default = defaultTask
exports.build = parallel(javascript, css, prettify,colorize);

...but is there any other option for gulp to spin up multiple processes as well?

score 0 · Answer 1 · answered Apr 23 '20 at 02:23

I finally found a way to achieve this. Inspired by this question which cexplains that the problem lies in node not in gulp and this question which suggests a way to make node run on multicores.

So basically all you need to do is to use PM2 and set it up to use clusters with the maximum (or your choice of) number of cores to run and watch the server.

//install pm2
apt install pm2

//start the server
pm2 start server.js -i [NUMBER_OF_INSTANCE|max]

Zardoz89 · Answer 2 · 2020-07-03T06:55:05.083

Actually, should be possible to fork and rewrite golp-multi-process to use WorkerThreads. This should be more optimal, as not spawns multiple process (that is slower).

Perhaps I would do my self. Because, like you, I have a slow task that could be run on another thread to speedup the whole compilation.

Edit: I make a fork where I updated it to gulp 4 and verify that for slow task (ie, takes more that 250 ms...) this really makes a difference : https://github.com/Zardoz89/gulp-multi-process

$../node_modules/gulp/bin/gulp.js single
[08:53:54] Using gulpfile ~/repos/gulp-multi-process/test/gulpfile.js
[08:53:54] Starting 'single'...
[08:53:54] Starting 'task1'...
[08:53:54] Finished 'task1' after 252 ms
[08:53:54] Starting 'task2'...
[08:53:54] Finished 'task2' after 251 ms
[08:53:54] Starting 'task3'...
[08:53:55] Finished 'task3' after 252 ms
[08:53:55] Finished 'single' after 756 ms

$../node_modules/gulp/bin/gulp.js multi
[08:54:00] Using gulpfile ~/repos/gulp-multi-process/test/gulpfile.js
[08:54:00] Starting 'multi'...
[08:54:00] Using gulpfile ~/repos/gulp-multi-process/test/gulpfile.js
[08:54:00] Starting 'task3'...
[08:54:00] Using gulpfile ~/repos/gulp-multi-process/test/gulpfile.js
[08:54:00] [08:54:00] Starting 'task2'...
Using gulpfile ~/repos/gulp-multi-process/test/gulpfile.js
[08:54:00] Starting 'task1'...
[08:54:00] Finished 'task3' after 252 ms
[08:54:00] Finished 'task1' after 251 ms
[08:54:00] Finished 'task2' after 252 ms
[08:54:00] Finished 'multi' after 400 ms

With Gulp 4+, when using gulp.parallel(), is there a way of using multiple CPUs for the concurrent tasks?

2 Answers2