What's the correct way to use async/await and Promises with Javascript (and therefore Node.js)

Question

I'm actually building a script to extract data from a MySQL database and then populating a MongoDB. In the process there are some asynchronous stuff like establishing a connection to MySQL (through Sequelize library) and MongoDB (through Mongoose library), and some synchronous stuff like fetching and converting data.

I read a lot about async/await and Promises and my script is globally doing what I want want, but still have some issues.

Here's the code :

Migration.class.mjs

import MigrationBase from './Base/MigrationBase.class.mjs';

export default class Migration extends MigrationBase
{
    constructor(config) {
        super(config);

        this.mysqlData = {};
        this.mongoData = {};
    }

    async run() {
        await this.selectMySQLData();
        let docs = await this.convertMySQLToMongo();
        await this.checkConvertedData(docs);
        await this.insertMongoData();
    }

    async selectMySQLData() {
        return new Promise(async resolve => {
            await this.runSequelize();
            console.log('B - Grabbing MySQL data\n');

            for(var key in this.mysqlModels) {
                if (this.mysqlModels.hasOwnProperty(key)) {
                    let search = { raw: true };
                    this.mysqlData[key] = await this.mysqlModels[key].findAll(search);
                }
            }

            await this.closeSequelize();
            resolve();
        });
    };

    convertMySQLToMongo() {
        return new Promise(async resolve => {
            console.log('D - Convert MySQL data to MongoDB\n');

            let customersDocument = this.defaultDocuments.customers;
            let personalInfosDocument = this.defaultDocuments.personal_infos;
            let billingInfosDocument = this.defaultDocuments.billing_infos;
            // ... etc ...

            await Object.entries(this.mysqlData.customer).forEach(async keyRow => {
                let [key, row] = keyRow;

                await Object.entries(row).forEach(async keyValue => {
                    customersDocument = await this._processCustomersFields(customersDocument, 'Customer', keyValue);
                    personalInfosDocument = await this._processPersonalInfosFields(personalInfosDocument, 'PersonalInfo', keyValue);
                    billingInfosDocument = await this._processBillingInfosFields(billingInfosDocument, 'BillingInfo', keyValue);
                    // ... etc ...
                    
            });

            resolve([
                customersDocument,
                personalInfosDocument,
                billingInfosDocument,
                // ... etc ...
            ]);
        });
    };

    checkConvertedData([
        customersDocument,
        personalInfosDocument,
        billingInfosDocument,
        // ... etc ...
    ]) {
        return new Promise(resolve => {
            console.log('E - Checking converted data');

            if (! this._isNull(customersDocument, 'Customers')) {
                this.mongoData.customers = customersDocument;
            }
            
            if (! this._isNull(personalInfosDocument, 'PersonalInfos')) {
                this.mongoData.personal_infos = personalInfosDocument;
            }
            
            if (! this._isNull(billingInfosDocument, 'BillingInfos')) {
            
            }   this.mongoData.billing_infos = billingInfosDocument;
            // ... etc ...
            
            resolve();
        });
    }

    async insertMongoData() {
        return new Promise(async resolve => {
            await this.runMongoose();
            console.log('G - Insert MongoDB data.');
            
            await this.mongoModels.customers.create(this.mongoData.customers);
            await this.mongoModels.personal_infos.create(this.mongoData.personal_infos);
            await this.mongoModels.billing_infos.create(this.mongoData.billing_infos);
            // ... etc ...

            await this.closeMongoose();
            resolve();
        });
    };

    _processCustomersFields(defaultDoc, docName, [colName, val]) {
        return new Promise(resolve => {
            switch (colName) {
                case 'id_customer':
                    console.log(`${docName}: ${colName} => ${val}`);
                    defaultDoc.id = val;
                    break;
                case 'email_customer':
                    console.log(`${docName}: ${colName} => ${val}`);
                    defaultDoc.email = val;
                    break;
                case 'password_customer':
                    console.log(`${docName}: ${colName} => ${val}`);
                    defaultDoc.password = val;
                    break;
                // ... etc ...
            }
    
            resolve(defaultDoc);
        });
    }

    _processPersonalInfosFields(defaultDoc, docName, [colName, val]) {
        return new Promise(resolve => {
            switch (colName) {
                // ... Same kind of code as in _processCustomersFields() ...
            }
            
            resolve(defaultDoc);
        });
    }

    _processBillingInfosFields(defaultDoc, docName, [colName, val]) {
        return new Promise(resolve => {
            switch (colName) {
                // ... Same kind of code as in _processCustomersFields() ...
            }
            
            resolve(defaultDoc);
        });
    }
    
    _isNull(document, mongoName) {
        if (document !== null) {
            console.log(`\n${mongoName}:\n`, JSON.stringify(document));
            return false;
        } else {
            console.log(`Error processing \`${mongoName}\` data!`);
            return true;
        }
    }
    
    _valueExists(val) {
        return (val !== null && val !== "" && typeof val !== "undefined")
            ? true
            : false
        ;
    }
}

MigrationBase.class.mjs

import Sequelize from 'sequelize';
import DataTypes from 'sequelize';
import Mongoose from 'mongoose';
import Crypto from 'crypto';
import Models from '../../../models.mjs';
import Schemas from '../../../schemas.mjs';

export default class MigrationBase
{
    constructor(config) {
        this.config = config;
        this.sequelize = this.createSequelize();
        this.mongoose = Mongoose;
        this.defaultDocuments = this.initDefaultDocuments();
        this.mysqlModels = this.initMysqlModels();
        this.mongoModels = this.initMongoSchemas();
        this.mysqlData = {};
        this.mongoData = {};
    }

    createSequelize() {
        return new Sequelize(
            this.config.mysql.dbName,
            this.config.mysql.dbUser,
            this.config.mysql.dbPass,
            this.config.sequelize
        );
    }

    initDefaultDocuments() {
        const defaultDocument = {
            "deleted_at": 0 // Thu Jan 01 1970 01:00:00 GMT+0100
        };

        let defaultDocuments = {
            "customers": Object.assign({}, defaultDocument),
            "personal_infos": Object.assign({}, defaultDocument),
            "billing_infos": Object.assign({}, defaultDocument)
            // ... etc ...
        };

        return defaultDocuments;
    }

    initMysqlModels() {
        return {
            "customer": Models.Customer(this.sequelize, DataTypes),
            "billing_address": Models.BillingAddress(this.sequelize, DataTypes),
            // ... etc ...
        };
    }

    initMongoSchemas() {
        return {
            "customers": this.mongoose.model('Customer', Schemas.Customers),
            "personal_infos": this.mongoose.model('PersonalInfo', Schemas.PersonalInfos),
            "billing_infos": this.mongoose.model('BillingInfo', Schemas.BillingInfos),
            // ... etc ...
        }
    }

    async runSequelize() {
        console.log('A - Connection to MySQL');

        try {
            await this.sequelize.authenticate();
            console.log('Connection to MySQL has been established successfully.\n');
        } catch (err) {
            console.error("Unable to connect to the MySQL database:", err + '\n');
        }
    }

    async closeSequelize() {
        console.log('C - Closing MySQL connection.\n');

        await this.sequelize.close();
    };

    runMongoose() {
        return new Promise(async resolve => {
            console.log('F - Connection to MongoDB');

            try {
                await this.mongoose.connect(
                    `mongodb://${this.config.mongo.dbHost}:${this.config.mongo.dbPort}/${this.config.mongo.dbName}`,
                    { useNewUrlParser: true, useUnifiedTopology: true }
                );

                console.log('Connection to MongoDB has been established successfully.');
            } catch (err) {
                console.error('Unable to connect to the MongoDB database: ', err);
            }

            resolve();
        });
    }

    async closeMongoose() {
        console.log('H - Closing MongoDB connection.');
        await this.mongoose.connection.close();
    };
}

And here is the Logs output:

A - Connection to MySQL
Connection to MySQL has been established successfully.

B - Grabbing MySQL data

C - Closing MySQL connection.

D - Convert MySQL data to MongoDB

Customer: id_customer => 1
Customer: email_customer => contact@example.com
Customer: password_customer => 0a1b2c3d4e5f0a1b2c3d4e5f0a1b2c3d
// ... etc ...
PersonalInfo: id_customer => 1
PersonalInfo: lastname_customer => Doe
PersonalInfo: firstname_customer => John
// ... etc ...
E - Checking converted data

Customers:
 {"deleted_at":0,"id":"000000000000000000000001","email":"contact@example.com","password":"0a1b2c3d4e5f0a1b2c3d4e5f0a1b2c3d", ... etc ... }

PersonalInfos:
 {"deleted_at":0,"customer_id":"000000000000000000000001","last_name":"Doe","first_name":"John", ... etc ... }

BillingInfos:
 {"deleted_at":0}
BillingInfos: id_customer => 1
BillingInfo: company => ExampleCompany
F - Connection to MongoDB
BillingInfos: lastname => Doe
BillingInfo: firstname => John
Connection to MongoDB has been established successfully.
G - Insert MongoDB data.
/home/user/Workspaces/namespace/project-name/node_modules/mongoose/lib/document.js:2757
    this.$__.validationError = new ValidationError(this);
                               ^

ValidationError: BillingInfos validation failed: id_customer: Cast to ObjectId failed for value "1" (type number) at path "customer_id", values: Path `values` is required., id: Path `id` is required.

Here we can see in the right order:

A - Connection to MySQL

B - Grabbing MySQL data

C - Closing MySQL connection

D - Convert MySQL data to MongoDB

Then we can see E - Checking converted data but the conversion process is not finished, despite the await statement and the fact that it return a Promise.

After that we can also see BillingInfos: id_customer => 1 and BillingInfo: company => ExampleCompany meaning that the convertion process is still doing stuff in the loop.

Then F - Connection to MongoDB

Then another convertion logs BillingInfos: lastname => Doe and BillingInfo: firstname => John (convertion process is still doing stuff in the loop).

Then G - Insert MongoDB data.

And finally a Validation Error because some Mongo Documents are incomplete and so the rules are not fullfiled.

Question?

So the question is what am I doing wrong here ?

As I said I read a lot about async/await and Promises but still struglle to understand why it's not working.

Thanks in advance and let me know if you need more info.

score 1 · Accepted Answer · answered Sep 22 '21 at 12:38

That's because await will not work inside forEach(), which you are trying to do in your convertMySQLToMongo() function.

There are many ways in which you can solve it, one of the ways is using for ... of instead of forEach()

for (const keyRow of Object.entries(this.mysqlData.customer)) {
    let [key, row] = keyRow;
  
    for (const keyValue of Object.entries(row)) {
        customersDocument = await this._processCustomersFields(customersDocument, 'Customer', keyValue);
        personalInfosDocument = await this._processPersonalInfosFields(personalInfosDocument, 'PersonalInfo', keyValue);
        billingInfosDocument = await this._processBillingInfosFields(billingInfosDocument, 'BillingInfo', keyValue);
    }
}

Yay it works! And I understand my mistake now. It's becoming way more clearer for me what's going on under the hood. Also I should have read more about the "forEach" documentation and especially the blue "Notes" here : [MDN Reference](https://developer.mozilla.org/fr/docs/Web/JavaScript/Reference/Global_Objects/Array/forEach#description) I think from now I will definitely ban "forEach" and only use its non-async alternatives like `for`, `for ... in` or `for ... of` as you suggested. Thanks a lot! — TwystO, Sep 22 '21 at 13:55

What's the correct way to use async/await and Promises with Javascript (and therefore Node.js)

1 Answers1