3

Associated GitHub issue: https://github.com/sgentle/phantomjs-node/issues/280

I have a simple app that does the following:

var
    phantom = require('phantom'),
    express = require('express'),
    serve = express();

serve.get('/foo', function (req, res) {
    try {
        phantom.create(function (ph) {
            console.log('Phantom browser created w/ pid: ', ph.process.pid);
            ph.onError = function (msg, trace) {
                var msgStack = ['PHANTOM ERROR: ' + msg];
                if (trace && trace.length) {
                    msgStack.push('TRACE:');
                    trace.forEach(function (t) {
                        msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line + (t.function ? ' (in function ' + t.function + ')' : ''));
                    });
                }
                console.log(msgStack.join('\n'));
            };
            ph.createPage(function(page){
                page.open('http://www.stackoverflow.com', function(status){
                    res.json({pageStatus: status});
                    page.close();
                    ph.exit();
                });
            });
        });
    } catch(e){
        console.log(e);
        throw e;
    }

});

As long as I run this in WebStorm, it runs just fine and I can hit the /foo endpoint w/ as many concurrent requests as I want and everything works as expected.

But as soon as I try to scale it behind PM2 w/ pm2 start -i 0 phantomapp.js, everything still works fine as long as I don't throw too many requests at it. The second I open up two browser windows and hit refresh in both, I get the following in pm2 logs. At which point, ALL 8 processes managed by PM2 just "disappear"! What am I doing wrong?? :

PM2: 2015-05-27 18:53:17: [PM2] Error caught by domain:
PM2: AssertionError: false == true
PM2:     at RoundRobinHandle.add (cluster.js:140:3)
PM2:     at queryServer (cluster.js:480:12)
PM2:     at Worker.onmessage (cluster.js:438:7)
PM2:     at ChildProcess.<anonymous> (cluster.js:692:8)
PM2:     at ChildProcess.emit (events.js:129:20)
PM2:     at handleMessage (child_process.js:324:10)
PM2:     at Pipe.channel.onread (child_process.js:352:11)
PM2: 2015-05-27 18:53:18: [PM2] Automatic `pm2 update` failed. Killing PM2 daemon and its processes...
PM2: 2015-05-27 18:53:18: pm2 has been killed by signal, dumping process list before exit...
PM2: 2015-05-27 18:53:18: Stopping app:phantomapp id:0
PM2: assert.js:86
PM2:   throw new assert.AssertionError({
PM2:         ^
PM2: AssertionError: false == true
PM2:     at RoundRobinHandle.add (cluster.js:140:3)
PM2:     at queryServer (cluster.js:480:12)
PM2:     at Worker.onmessage (cluster.js:438:7)
PM2:     at ChildProcess.<anonymous> (cluster.js:692:8)
PM2:     at ChildProcess.emit (events.js:129:20)
PM2:     at handleMessage (child_process.js:324:10)
PM2:     at Pipe.channel.onread (child_process.js:352:11)
phantomapp-0 (err): Process disconnected from parent !
PM2: [PM2] Spawning PM2 daemon
PM2: 2015-05-27 18:53:18: [PM2][WORKER] Started with refreshing interval: 30000
PM2: 2015-05-27 18:53:18: [[[[ PM2/God daemon launched ]]]]
PM2: 2015-05-27 18:53:18: BUS system [READY] on port /Users/matthewmarcus/.pm2/pub.sock
PM2: 2015-05-27 18:53:18: RPC interface [READY] on port /Users/matthewmarcus/.pm2/rpc.sock
PM2: [PM2] PM2 Successfully daemonized
PM2: Be sure to have the latest version by doing `npm install pm2@latest -g` before doing this procedure.
PM2: [PM2] Stopping PM2...
PM2: [PM2][WARN] No process found
PM2: [PM2] All processes have been stopped and deleted
PM2: 2015-05-27 18:53:18: PM2 is being killed via kill method
PM2: 2015-05-27 18:53:18: RPC socket closed
PM2: 2015-05-27 18:53:18: PUB socket closed
PM2: [PM2] PM2 stopped
PM2: 2015-05-27 18:53:19: [PM2][WORKER] Started with refreshing interval: 30000
PM2: 2015-05-27 18:53:19: [[[[ PM2/God daemon launched ]]]]
PM2: 2015-05-27 18:53:19: BUS system [READY] on port /Users/matthewmarcus/.pm2/pub.sock
PM2: 2015-05-27 18:53:19: RPC interface [READY] on port /Users/matthewmarcus/.pm2/rpc.sock
PM2: >>>>>>>>>> PM2 updated
PM2: ┌──────────┬────┬──────┬─────┬────────┬─────────┬────────┬────────┬──────────┐
PM2: │ App name │ id │ mode │ pid │ status │ restart │ uptime │ memory │ watching │
PM2: └──────────┴────┴──────┴─────┴────────┴─────────┴────────┴────────┴──────────┘
PM2:  Use `pm2 show <id|name>` to get more details about an app

You can reproduce the same issue using node-native cluster:

var cluster = require('cluster');

if(cluster.isMaster){
    var cpuCount = require('os').cpus().length;

    for (var i = 0; i < cpuCount; i++){
        cluster.fork();
    }

    cluster.on('exit', function(worker){
        console.log('Worker ' + woker.id + ' died. Forking...');
        cluster.fork();
    });

} else {
    var
        phantom = require('phantom'),
        express = require('express'),
        serve = express();

    serve.get('/foo', function (req, res) {
        try {
            phantom.create(function (ph) {
                console.log('Phantom browser created w/ pid: ', ph.process.pid);
                ph.onError = function (msg, trace) {
                    var msgStack = ['PHANTOM ERROR: ' + msg];
                    if (trace && trace.length) {
                        msgStack.push('TRACE:');
                        trace.forEach(function (t) {
                            msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line + (t.function ? ' (in function ' + t.function + ')' : ''));
                        });
                    }
                    console.log(msgStack.join('\n'));
                };
                ph.createPage(function(page){
                    page.open('http://www.stackoverflow.com', function(status){
                        res.json({pageStatus: status});
                        page.close();
                        ph.exit();
                    });
                });
            });
        } catch(e){
            console.log(e);
            throw e;
        }

    });
}

Which produces the following error in the console when attempting to handle > 1 request concurrently:

assert.js:86
  throw new assert.AssertionError({
        ^
AssertionError: false == true
    at RoundRobinHandle.add (cluster.js:140:3)
    at queryServer (cluster.js:480:12)
    at Worker.onmessage (cluster.js:438:7)
    at ChildProcess.<anonymous> (cluster.js:692:8)
    at ChildProcess.emit (events.js:129:20)
    at handleMessage (child_process.js:324:10)
    at Pipe.channel.onread (child_process.js:352:11)
RavenHursT
  • 2,336
  • 1
  • 25
  • 46

1 Answers1

0

I tried with this gist and I'm able to create the phantom browser and to get a response.

  • If I run two tabs on localhost:3000/foo, the second will be spawned only when the first has ended.
  • If I run more than two concurrent requests in my browser (chrome), I get the error too.
    • If I try to run more requests (from a bash) than the number of clusters it crashes.
  • If I run requests (less than or equal to the number of cluster) from a bash script it doesn't crash at all.

    Also, I'm noticing that with the bash script I'm running everything in concurrence:

    http GET localhost:3000/foo &
    http GET localhost:3000/foo &
    http GET localhost:3000/foo &
    http GET localhost:3000/foo &
    

The error you're seeing comes from this assertion:

  assert(worker.id in this.all === false);

Meaning that the worker is not in the round robin anymore.

I don't think that this is direclty related to PM2 but there is a bug. PM2 should not crash. I suggest your report an issue by quoting this stackoverflow. Not sure that it can be fixed if it's a node bug though.

Sadly there are no fix for this at the moment but you can take a look at a nodejs issue. I've read that iojs fixed this, maybe you can give it a try ;).

Another related stackoverflow: NodeJS Cluster unexpected assert.AssertionError.

Community
  • 1
  • 1
soyuka
  • 8,839
  • 3
  • 39
  • 54
  • 1
    Yeah.. I don't think this is a PM2 issue at all. I updated the question w/ a simpler example that handles cluster forks directly and the same error is thrown there. "Meaning that the worker is not in the round robin anymore." Since that assertion is in the ```RoundRobinHandle.add``` wouldn't that mean the inverse? That the worker it's trying to add is already in the the rr? I'm about 100% sure it's because of how phantomjs-node is using node's child_process to communicate w/ phantomjs processes. I'll update the answer w/ the github issue that I created for phantomjs-node – RavenHursT May 28 '15 at 17:17
  • To me it doesn't seem to be a phatomjs error but a nodejs issue, see my last 2 links. May you try with iojs? – soyuka May 28 '15 at 17:31
  • The error stack in the nodejs issue you referenced is pointing to ```SharedHandle.add```, not ```RoundRobinHandle.add```, not sure if both cases are the same? – RavenHursT May 28 '15 at 18:18
  • My bad what is your current nodejs/pm2 version? Note [he](https://github.com/joyent/node/issues/9296#issuecomment-76573830) made the same mistake.. – soyuka May 28 '15 at 18:21
  • Yeah.. guess it could be the same thing: https://github.com/joyent/node/blob/master/lib/cluster.js#L99 Looks like in both cases, it's just verifying that the worker hasn't already been added. But that sounds correct, right? You wouldn't wan the same worker being added twice, so it sounds like phantomjs-node's use of ```spawn()`` here: https://github.com/sgentle/phantomjs-node/blob/master/phantom.js#L17 could be the culprit, which in turn uses require(child_process).spawn() (https://github.com/ForbesLindesay/win-spawn/blob/master/index.js#L1) – RavenHursT May 28 '15 at 18:23
  • Node is v0.12.2, and PM2 is 0.12.14. But I don't think the PM2 version matters much since I can reproduce this w/o PM2. – RavenHursT May 28 '15 at 18:39