Nodejs forEach function doesn't wait for async functions

Question

I'm trying to fetch json, check with mongodb if duplicate exist and if not, then to insert data into mongodb.

The problem is loops goes very fast, without waiting for duplicate check and insert.

What am I doing wrong here?

const fetch = require('node-fetch');
var MongoClient = require('mongodb').MongoClient;


async function fetchPhotos() {
    MongoClient.connect("mongodb://localhost:27017", async function (err, dbo) {
        const db = dbo.db('mydb')
        if (err) throw err;
        for (var i = 1; i < 100; i++) {
            await fetch(`domain.com/?page=${i}`)
               .then(res => res.json())
                .then((response) => {
                    response.forEach(photo => {
                        console.log("checking if there is duplicate: " + photo.id);

                        var exists =  db.collection("photos").countDocuments({"id": photo.id}, {limit: 1});

                        if (exists === 1) {
                            console.log("dupl found, next!");

                        } else {
                            db.collection("photos").insertOne(photo, function (err, res) {
                                if (err) throw err;
                                console.log("1 document inserted");
                            });
                        }
                    });
                });

        }

    });
}

module.exports.fetchPhotos = fetchPhotos;

Does this answer your question? [Using async/await with a forEach loop](https://stackoverflow.com/questions/37576685/using-async-await-with-a-foreach-loop) — goto, Feb 25 '20 at 23:55
Don't ever write `if (err) throw err;` inside a plain asynchronous callback. It's completely useless and can't be caught by any of your code. Write real error handling. — jfriend00, Feb 26 '20 at 00:56

jfriend00 · Accepted Answer · 2020-02-26T01:07:32.110

There are a few problems with your asynchronous code inside a loop.

a .forEach() loop does not wait for await. You have to use a for loop or while loop if you want it to wait for await.
You aren't using await on db.collection("photos").countDocuments({"id": photo.id}, {limit: 1});.
You aren't using await on db.collection("photos").insertOne(photo).
Do not mix plain callbacks and promises in the same flow of control. It makes it very difficult to code safely and with proper error handling.
You are missing appropriate error handling in a number of places.

You can restructure the whole thing to use the promise interface to your database and then sequence the iteration through the photos and simplify things like this:

const fetch = require('node-fetch');
var MongoClient = require('mongodb').MongoClient;

async function fetchPhotos() {
    const dbo = await MongoClient.connect("mongodb://localhost:27017");
    const db = dbo.db('mydb');
    for (let i = 1; i < 100; i++) {
        let response = await fetch(`http://example.com/?page=${i}`);
        let data = await response.json();
        for (let photo of data) {
            console.log("checking if there is duplicate: " + photo.id);
            let cnt = await db.collection("photos").countDocuments({"id": photo.id}, {limit: 1});
            if (cnt > 0) {
                console.log("dupl found, next!");
            } else {
                await db.collection("photos").insertOne(photo);
                console.log("photo inserted");
            }
        }
    }
}

// caller of fetchPhotos() gets a promise that tells you if the 
// operation completed or had an error
module.exports.fetchPhotos = fetchPhotos;

Thank you very much. Lets say I have following lines inside for loop: `const analyzeData = await recognize.analyze(photo.urls.regular);` The function recognize.analyze is an async axios post request. Why console.log(analyzeData); returns undefined? — demonoid, Feb 26 '20 at 02:32
@demonoid - You would have to show the actual code. It seems likely that `recognize.analyze` is not returning the right value/promise. If you want further help with that, please post a new question that shows the code for that situation. — jfriend00, Feb 26 '20 at 03:00
please take a look at this question: https://stackoverflow.com/questions/60406120/nodejs-triggers-function-before-previous-function-finish — demonoid, Feb 26 '20 at 03:01

Nodejs forEach function doesn't wait for async functions

1 Answers1

Linked