All callbacks are executing at the same time but why?

Question

I'm currently wrapping an executable I've made with NodeJS. The executable can save strings for use in other processes within the executable. Each time the executable 'saves' a string it sends a pointer back to the server via stdout. The NodeJS server saves strings by sending them to stdin of the executable.

Originally I was writing code like this:

CLRProcess.stdout.once('data',function(strptr){
    CLRProcess.stdout.once('data', function(str){
         console.log(str.toString())
    })
    CLRProcess.stdin.write("StringReturn " + strptr.toString())
})
CLRProcess.stdin.write("StringInject __CrLf__ Mary had a__CrLf__little lamb.")

The above code injects a string

Mary had a
little lamb.

receives a pointer to the string, and then requests the string in the next step, by sending the pointer back to the host application.

To make coding algorithms easier I wanted a system like this:

strPtr = Exec("StringInject __CrLf__ Mary had a__CrLf__little lamb.")
str = Exec("StringReturn " + strPtr)
// do stuff with str

This is the code I made:

class Pointer {
    constructor(){
        this.value = undefined
        this.type = "ptr"
    }
}


class CLR_Events extends Array {
    constructor(CLR){
        super()
        this.CLR = CLR
    }
    runAll(){
        if(this.length>0){
            //Contribution by le_m: https://stackoverflow.com/a/44447739/6302131. See Contrib#1
            this.shift().run(this.runAll.bind(this))
        }
    }
    new(cmd,args,ret){
        var requireRun = !(this.length>0)   //If events array is initially empty, a run is required
        var e = new CLR_Event(cmd,args,ret,this.CLR)
        this.push(e)
        if(requireRun){
            this.runAll()
        }
    }
}

class CLR_Event {
    constructor(cmd,args,ret,CLR){
        this.command = cmd;
        this.args = args
        this.CLR = CLR
        this.proc = CLR.CLRProcess;
        this.ptr = ret
    }

    run(callback){
        //Implementing event to execute callback after some other events have been created.
        if(this.command == "Finally"){
            this.args[0]()
            console.log("Running Finally")
            return callback(null)
        }

        //Implementation for all CLR events.
        var thisEvent = this
        this.proc.stdout.once('data',function(data){
            this.read()
            data = JSON.parse(data.toString())
            thisEvent.ptr.value = data
            callback(data);
        })
        this.proc.stdin.write(this.command + " " + this._getArgValues(this.args).join(" ") + "\n");
    }
    _getArgValues(args){
        var newArgs = []
        this.args.forEach(
            function(arg){
                if(arg.type=='ptr'){
                    if(typeof arg.value == "object"){
                        newArgs.push(JSON.stringify(arg.value))
                    } else {
                        newArgs.push(arg.value)
                    }
                } else if(typeof arg == "object"){
                    newArgs.push(JSON.stringify(arg))
                } else {
                    newArgs.push(arg)
                }
            }
        )
        return newArgs  
    }
}

var CLR = {}
CLR.CLRProcess = require('child_process').spawn('DynaCLR.exe')
CLR.CLRProcess.stdout.once('data',function(data){
    if(data!="Ready for input."){
        CLR.CLRProcess.kill()
        CLR = undefined
        throw new Error("Cannot create CLR process")
    } else {
        console.log('CLR is ready for input...')
    }
})
CLR.Events = new CLR_Events(CLR)

//UDFs

CLR.StringInject = function(str,CrLf="__CLR-CrLf__"){
    var ptr = new Pointer
    this.Events.new("StringInject",[CrLf,str.replace(/\n/g,CrLf)],ptr) //Note CLR.exe requires arguments to be the other way round -- easier command line passing
    return ptr
}
CLR.StringReturn = function(ptr){
    var sRet = new Pointer
    this.Events.new("StringReturn",[ptr],sRet)
    return sRet
}

CLR.Finally = function(callback){
    this.Events.new("Finally",[callback])
}

I intended this to do the following:

Functions StringInject, StringReturn and Finally create events and append them to the Events array.
The runAll() function of the Events object, removes the first 'event' from its array and runs the run() function of the array, passing itself as a callback.
The run functions writes to stdin of the executable, waits for a response in stdout, appends the data to the passed in pointer and then executes the runAll() function passed to it.

This is what I don't understand... When executing the multiple string injections:

S_ptr_1 = CLR.StringInject("Hello world!")
S_ptr_2 = CLR.StringInject("Hello world!__CLR-CrLf__My name is Sancarn!")
S_ptr_3 = CLR.StringInject("Mary had a little lamb;And it's name was Doug!",";")

I get the following data:

S_ptr_1 = {value:123,type:'ptr'}
S_ptr_2 = {value:123,type:'ptr'}
S_ptr_3 = {value:123,type:'ptr'}

Where as the data should be:

S_ptr_1 = {value:1,type:'ptr'}
S_ptr_2 = {value:2,type:'ptr'}
S_ptr_3 = {value:3,type:'ptr'}

The only scenario I can think this would happen is if, in pseudocode, the following happenned:

CLRProcess.stdin.write("StringInject Val1")
CLRProcess.stdin.write("StringInject Val2")
CLRProcess.stdin.write("StringInject Val3")
CLRProcess.stdout.once('data') ==> S_ptr_1
CLRProcess.stdout.once('data') ==> S_ptr_2
CLRProcess.stdout.once('data') ==> S_ptr_3

But why? Am I overlooking something or is there something fundamentally wrong with this algorithm?

Sounds like you're overlooking the fact that `write` is asynchronous. Also, `new Pointer` does not create an object. You need to invoke the constructor: `new Pointer()` — slebetman, Jun 10 '17 at 21:40
You only have one CLR Object, and so only one DynaCLR.exe, thus only one each stdin and stdout stream. If you were to create a new DynaCLR for each call, you may have more success. — GregHNZ, Jun 10 '17 at 21:50
@GregHNZ Yes. It is required that there be only 1 stdin/stdout stream. Everything should be held by the same DynaCLR executable, since I will be using this for compiling VB and C# source code. Storing them in multiple different processes will not allow that possibility (without storing physical files, which isn't a possibility for me.) Using stdin/out is just a means of IPC. — Sancarn, Jun 11 '17 at 10:45
@slebetman but that confuses me... Surely I call the next `CLR_Event` when I receive stdout. I should only receive stdout after I have written to stdin... So I should be waiting for both the stdin's and stdout's asynchronous time...? Interesting about `new Pointer()` though... That's not what I've seen in the REPL. Which is odd — Sancarn, Jun 11 '17 at 10:48
@slebetman At the pointer stuff see https://puu.sh/wgV6Q/70c0852d69.png - Might just be missing something of course. — Sancarn, Jun 11 '17 at 11:00

Sancarn · Answer 1 · 2017-06-11T15:36:33.813

I found out the problem and I have managed to create a solution.

THE PROBLEM

CLR.StringInject is called which calls CLR_Events.new() CLR_Events.new(), first checks the array to see whether it is empty or if it has events, creates a new event and pushes the event to the events array. If the array was initially empty CLR_Events.runAll() is called.

CLR_Events.runAll() then removes the first element of the CLR_Events.runAll() array and executes it setting up the listener on the STDOUT and writes data to STDIN.

Then the next line of code runs:

CLR.StringInject is called which calls CLR_Events.new() CLR_Events.new(), first checks the array to see whether it is empty or if it has events, it sees that the array is empty and so calls runAll()

This is the issue.

runAll() will call itself to call the next event in the events array. However the Events array is always empty, because the CLR_Events.new() doesn't know how to check whether an event is currently being executed. It only checks whether there are events in the array. So now we have written to STDIN twice and set up 2 listeners on the STDOUT. This chains onto the third command ultimately meaning that all returned objects contain the same data.

THE SOLUTION

To fix this I had to create a this.isRunning variable.

RunAll() should only be called If isRunning == false

isRunning should only be false when both of the following are true:

There are no currently executing event calls
Events.length == 0

I.E. isRunning should be set to false after the callback.

This ensures all events trigger within the same callback loop.

I then had a few other problems with my callbacks because this was undefined in:

function(){
    this.runAll.bind(this)()
    if (!(this.length>0))  this.isRunning = false
}

To solve this, I had to add a CLREvents variable before the callback definition to store this, and replace this with CLREvents within the callback. And now it finally works as intended.

Full working code:

class Pointer {
    constructor(){
        this.value = undefined
        this.type = "ptr"
    }
}

class CLR_Events extends Array {
    constructor(CLR){
        super()
        this.CLR = CLR
        this.isRunning = false
    }
    runAll(){
        console.log('RunAll')
        this.isRunning = true
        if(this.length>0){
            //Contribution by le_m: https://stackoverflow.com/a/44447739/6302131. See Contrib#1
            var CLREvents = this
            this.shift().run(function(){
                CLREvents.runAll.bind(CLREvents)()
                if (!(CLREvents.length>0))  CLREvents.isRunning = false
            })
        }
    }
    new(cmd,args,ret){
        console.log("New Event: " + JSON.stringify([cmd,args,ret]) + " - requireRun:" + (!(this.length>0)).toString())
        //If events array is initially empty, a run is required
        var requireRun = !(this.length>0)

        var e = new CLR_Event(cmd,args,ret,this.CLR)
        this.push(e)
        if(!this.isRunning){
            this.runAll()
        }
    }
}

class CLR_Event {
    constructor(cmd,args,ret,CLR){
        this.command = cmd;
        this.args = args
        this.CLR = CLR
        this.proc = CLR.CLRProcess;
        this.ptr = ret
    }

    run(callback){
        console.log("RunOne")
        //Implementing event to execute callback after some other events have been created.
        if(this.command == "Finally"){
            this.args[0]()
            console.log("Running Finally")
            return callback(null)
        }

        //Implementation for all CLR events.
        var thisEvent = this
        this.proc.stdout.once('data',function(data){
            console.log('Callback')
            this.read()
            data = JSON.parse(data.toString())
            thisEvent.ptr.value = data
            callback(data);
        })
        this.proc.stdin.write(this.command + " " + this._getArgValues(this.args).join(" ") + "\n");
    }
    _getArgValues(args){
        var newArgs = []
        this.args.forEach(
            function(arg){
                if(arg.type=='ptr'){
                    if(typeof arg.value == "object"){
                        newArgs.push(JSON.stringify(arg.value))
                    } else {
                        newArgs.push(arg.value)
                    }
                } else if(typeof arg == "object"){
                    newArgs.push(JSON.stringify(arg))
                } else {
                    newArgs.push(arg)
                }
            }
        )
        return newArgs  
    }
}

var CLR = {}
CLR.CLRProcess = require('child_process').spawn('DynaCLR.exe')
CLR.CLRProcess.stdout.once('data',function(data){
    if(data!="Ready for input."){
        CLR.CLRProcess.kill()
        CLR = undefined
        throw new Error("Cannot create CLR process")
    } else {
        console.log('CLR is ready for input...\n')
        /* Example 1 - Using String Inject */
        S_ptr_1 = CLR.StringInject("Hello world!")
        S_ptr_2 = CLR.StringInject("Hello world!__CLR-CrLf__My name is Sancarn!")
        S_ptr_3 = CLR.StringInject("Mary had a little lamb;And it's name was Doug!",";")
        console.log(S_ptr_1)
        console.log(S_ptr_2)
        console.log(S_ptr_3)
    }
})
CLR.Events = new CLR_Events(CLR)

//UDFs

CLR.StringInject = function(str,CrLf="__CLR-CrLf__"){
    var ptr = new Pointer
    this.Events.new("StringInject",[CrLf,str.replace(/\n/g,CrLf)],ptr) //Note CLR.exe requires arguments to be the other way round -- easier command line passing
    return ptr
}
CLR.StringReturn = function(ptr){
    var sRet = new Pointer
    this.Events.new("StringReturn",[ptr],sRet)
    return sRet
}

CLR.Finally = function(callback){
    this.Events.new("Finally",[callback])
}

All callbacks are executing at the same time but why?

1 Answers1

THE PROBLEM

THE SOLUTION