Async in for loop, Nodejs

Question

I am wanting the for loop to run sequentially, finishing one loop completely before moving to the next. The loop is putting a JSON message into another JSON message then sending it to a function that begins posting to an api. I need that function to complete before moving on to the next item in the JSON. p is the item name that is being used to post back whether it was successfully posted to the db via the api service.

Here is the bit of code simplified for this question.

let processJson = function(items) {
    for (const p in items) {
        let newObj = {
            "key1": items[p].key1,
            "key2": items[p].key2,
            "keySpecial": items[p].key3 + items[p].key4
        };
        await validateJson(p, newObj);
    }
};

I need the validateJson to finish its chain of asynchronous work before moving on to the next p in the loop.

How can I do this?

Here is the validateJson function as requested.

const validateJson = function (id, jsonObj) {
    const processItemSchema = {
        "properties": {
            "key1": {
                "type": "string"
            },
            "key2": {
                "type": "string",
                "minLength": 3,
                "maxLength": 3
            },
            "keySpecial": {
                "type": "string",
                "minLength": 4,
                "maxLength": 4
            }
        }
    };
    const ajv = new Ajv();
    let validate = ajv.compile(processItemSchema);
    let valid = validate(jsonObj);
    if (!valid){
        resCallback(id + ": invalid JSON");
    }
    else{
        // Generate Special Flag(s) value, Comma Separated Value
        let specialFlag = "";
        specialFlag += specialCheck1(jsonObj.keySpecial);
        if(specialFlag.length > 0) {
            let temp = specialCheck2(jsonObj.keySpecial);
            if (temp.length > 0) {
                specialCheck += "," + temp;
                maintenanceCall(id, jsonObj, specialFlag);
            }
            else {
                mainenanceCall(id, jsonObj, specialFlag);
            }
        }
        else {
            specialFlag += specialCheck1(jsonObj.keySpecial);
            maintenanceCall(id, jsonObj, specialFlag);
        }
    }
};

More Code as requested

const maintenanceCall= function (id, jsonObj, specialFlag) {
        request.post({
            url: 'https://url.poster/something',
            auth: {
                'user': 'user',
                'pass': 'pass',
                'sendImmediately': true
            },
            json: true,
            body: {
                "Input": {
                    "InputParameters": {
                        "KEY": jsonObj.key1,
                        "Hole": jsonObj.Key2,
                        "SomeWhere": jsonObj.keySpecial
                    }
                }
            }
        }
        , function (error, response, body) {
            if (body.OutputParameters.X_MSG_DATA !== null) {
                resCallback(id + , Message: "
                    + body.OutputParameters.DATA);
            }
            else {
                const sampCheck = function(smsFlag){
                    if(flag=== "Y")
                        return ".X";
                    else if(flag=== "N")
                        return "";
                    else
                        resCallback(id + ": this item can not be processed");
                    processItem(id, jsonObj, stats);
                }
            }
        });
};

Answer 1

To "pause" your for loop using await, you have to be awaiting a promise. So you have to make validateJson() return a promise that resolves when any asynchronous operations inside that function are done. That's how async/await works in Javascript.

It is not clear exactly what is or isn't asynchronous in validateJson(). If nothing is asynchronous, then it's just serial execution and you don't need await or promises at all. Javascript is single threaded so it will just run validateJson() until it's done and the for loop will be blocked until validateJson() returns.

If validateJson() does have some asynchronous operations in it, then you have to make sure that validateJson() returns a promise that resolves only when all those asynchronous operations are done. Then, and only then, can you use await to "pause" your for loop while your asynchronous operations run. To help you fix validateJson(), we would have to understand more about what is and isn't asynchronous and what interface the asynchronous operations have for knowing when they are done. Then, we could help you make validateJson() return a promise that resolves at the right time to make your await work properly.

Also, you can only use await inside a function that is declared async so you would also have to add that to the processJson() definition.

let processJson = async function(items) {

---

To illustrate the concept, here's a simple example that you can run right here in the snippet to see how it pauses the for loop:

function delay(t) {
    return new Promise(function(resolve) {
        setTimeout(resolve, t);
    });
}

async function run() {
    console.log("starting...");
    for (let i = 0; i < 10; i++) {
        await delay(1000);
        console.log("Timer " + i + " fired");
    }
    console.log("done");
}

run();

---

Now that you've added more code, we can talk about your real code (though it is still unclear what resCallback() or processItem() do so this may not yet be the end of changes.

First change maintenanceCall() to return a promise which I will do mostly by switching to the request-promise module and returning that promise:

const rp = require('request-promise');

const maintenanceCall= function (id, jsonObj, specialFlag) {
        return rp.post({
            url: 'https://url.poster/something',
            auth: {
                'user': 'user',
                'pass': 'pass',
                'sendImmediately': true
            },
            json: true,
            body: {
                "Input": {
                    "InputParameters": {
                        "KEY": jsonObj.key1,
                        "Hole": jsonObj.Key2,
                        "SomeWhere": jsonObj.keySpecial
                    }
                }
            }
        }).then(function(body) {
            if (body.OutputParameters.X_MSG_DATA !== null) {
                resCallback(id + , Message: "
                    + body.OutputParameters.DATA);
            }
            else {
                // FIX THIS: You define this function here, but never use it, that's odd
                const sampCheck = function(smsFlag){
                    if(flag=== "Y")
                        return ".X";
                    else if(flag=== "N")
                        return "";
                    else
                        resCallback(id + ": this item can not be processed");
                    processItem(id, jsonObj, stats);
                }
            }
        });
};

Now that maintenanceCall() returns a promise, you can use that in validateJson() like this so it always returns a promise:

const validateJson = function (id, jsonObj) {
    const processItemSchema = {
        "properties": {
            "key1": {
                "type": "string"
            },
            "key2": {
                "type": "string",
                "minLength": 3,
                "maxLength": 3
            },
            "keySpecial": {
                "type": "string",
                "minLength": 4,
                "maxLength": 4
            }
        }
    };
    const ajv = new Ajv();
    let validate = ajv.compile(processItemSchema);
    let valid = validate(jsonObj);
    if (!valid) {
        resCallback(id + ": invalid JSON");
        return Promise.reject(new Error(id + ": invalid JSON"));
    } else {
        // Generate Special Flag(s) value, Comma Separated Value
        let specialFlag = "";
        specialFlag += specialCheck1(jsonObj.keySpecial);
        if(specialFlag.length > 0) {
            let temp = specialCheck2(jsonObj.keySpecial);
            if (temp.length > 0) {
                specialCheck += "," + temp;
            }
        } else {
            specialFlag += specialCheck1(jsonObj.keySpecial);
        }
        return maintenanceCall(id, jsonObj, specialFlag);
    }
};

And, then you can go back to your processJson() function and make your for loop run serially like this:

let processJson = async function(items) {
    for (const p in items) {
        let newObj = {
            "key1": items[p].key1,
            "key2": items[p].key2,
            "keySpecial": items[p].key3 + items[p].key4
        };
        await validateJson(p, newObj);
    }
};

And, the caller can use processJson()like this:

processJson(someItems).then(() => {
   console.log("all done here");
}).catch(err => {
   console.log(err);
});

Answer 2

You can use node-async-loop https://www.npmjs.com/package/node-async-loop

var asyncLoop = require('node-async-loop');
 
var array = ['item0', 'item1', 'item2'];
asyncLoop(array, function (item, next)
{
    do.some.action(item, function (err)
    {
        if (err)
        {
            next(err);
            return;
        }
 
        next();
    });
}, function (err)
{
    if (err)
    {
        console.error('Error: ' + err.message);
        return;
    }
 
    console.log('Finished!');
});

Answer 3

For those who don't want to rewrite and restructure all of their code. For those who don't want unnecessary complexity. For those who want an item in their for loop to finish posting before the next item is looped through. For those who like to keep it simple. For these.. Here it is.

/**
 * Clever way to do asynchronous sleep. 
 * Check this: https://stackoverflow.com/a/46720712/778272
 *
 * @param {Number} millis - how long to sleep in milliseconds
 * @return {Promise<void>}
 */
async function sleep(millis) {
    return new Promise(resolve => setTimeout(resolve, millis));
}

async function run() {
    const urls = await fetchUrls(INITIAL_URL);
    for (const url of urls) {
        await sleep(10000);
        const $ = await fetchPage(url);
        // do stuff with cheerio-processed page
    }
}

Answer 4

If you want your code-block to run sync, use this JavaScript function:

// Your loop..
{
  (function (p) 
  {
      // Your code...
  })(p);
}