Mongoose how to be connected to many database

Question

For a project, I need to switch between some MongoDB databases for doing some queries.

I found some exemple like that one : Mongoose and multiple database in single node.js project

His answer is perfectly working but now I'm trying to do it in a loop and unfortunally it's not work, I get this error:

events.js:72
        throw er; // Unhandled 'error' event
              ^
Error: failed to connect to [localhost:27017]
    at null.<anonymous> (/home/user/test_many_db_mongodb/node_modules/mongoose/node_modules/mongodb/lib/mongodb/connection/server.js:549:74)
    at EventEmitter.emit (events.js:106:17)
    at null.<anonymous> (/home/user/test_many_db_mongodb/node_modules/mongoose/node_modules/mongodb/lib/mongodb/connection/connection_pool.js:150:15)
    at EventEmitter.emit (events.js:98:17)
    at Socket.<anonymous> (/home/user/test_many_db_mongodb/node_modules/mongoose/node_modules/mongodb/lib/mongodb/connection/connection.js:533:10)
    at Socket.EventEmitter.emit (events.js:95:17)
    at net.js:440:14
    at process._tickCallback (node.js:419:13)

And I don't understand why I get this error, if the example in the link above work my code should too:

var mongoose = require('mongoose');

for (var i = 0; i != 1000; i++) {
    var conn = mongoose.createConnection('mongodb://localhost/test' + i);

    conn.on('connected', function() {
        console.log('Mongoose connected to database');

        var Model    = conn.model('Model', new mongoose.Schema({
            title : { type : String, default : 'model in test database' }
        }));

        var newModelA = new Model();
        newModelA.save(function(err) {
            if (err)
                console.log(err);
            console.log('save A');
        });
    });
}

Thanks for your help.

Answer 1

I think there is a problem with for loop. Because the code inside the loop is asynchronous in nature. So I have used forEach instead of for. Please take a look at the following code. It works fine for me.

var mongoose = require('mongoose'),
    arr = [0, 1, 2];

arr.forEach(function(i) {
    var conn = mongoose.createConnection('mongodb://localhost/test' + i);

    conn.on('connected', function() {
        console.log('Mongoose connected to database', i);

        var Model = conn.model('Model', new mongoose.Schema({
            title: {
                type: String,
                default: 'model in test database'
            }
        }));

        var newModelA = new Model();
        newModelA.save({
            name: 'a'
        }, function(err) {
            if (err)
                console.log(err);
            console.log('save A');
        });
    });
});

I have does this for three times. You can do it for 1000 times also.

Answer 2

You're getting the failed to connect error because you're creating all 1000 database connection pools in a synchronous loop which is exhausting the supply of available MongoDB connections.

So you need to introduce some asynchronous flow control to your approach using something like the async library's eachLimit method:

var mongoose = require('mongoose');
var async = require('async');

var iterations = [];
for (var i = 0; i != 1000; i++) {
    iterations.push(i);
}

// Iterate over iterations, allowing no more than 5 async iterations to be
// outstanding at any one time.
async.eachLimit(iterations, 5, function(i, callback) {
    var conn = mongoose.createConnection('mongodb://localhost/test' + i);

    conn.on('connected', function(err) {
        console.log('Mongoose connected to database');

        var Model = conn.model('Model', new mongoose.Schema({
            title : { type : String, default : 'model in test database' }
        }));

        var newModelA = new Model();
        newModelA.save(function(err) {
            if (err)
                console.log(err);
            console.log('save A');

            // Close the connection and tell eachLimit this iteration is
            // complete by having the close method call the eachLimit callback
            // when the close completes.
            conn.close(callback);
        });    
    });
}, function() {
    console.log('All done!');
});

Answer 3

Thanks for your help.

I tried both way but it didn't work on my project, probably because I didn't show you all the code. I found a way using useDb():

var amqp = require('amqp');

var MailParser = require('mailparser').MailParser;
var mailparser = new MailParser();

var mongoose = require('mongoose');
var conn = mongoose.createConnection('mongodb://localhost');

var count = 1;

var connection = amqp.createConnection({host: 'localhost'});

    connection.on('ready', function() {
        connection.queue('task_queue', {autoDelete: false,
                                        durable: true}, function(queue) {

            console.log('Waiting for emails. To exit press CTRL+C');

            queue.subscribe({ack: true, prefetchCount: 1}, function(message, headers, deliveryInfo, ack) {
                mailparser.write(message.data.toString('utf-8'));
                mailparser.end();

                ack.acknowledge();
            });
        });
    });

mailparser.on("end", function(email_object){
    var d = new Date(); // just for be sure the db name is unique for the test
    var db = conn.useDb('test_'+d.getDate()+'-'+d.getMonth()+'-'+d.getYear()+'_'+d.getHours()+'-'+d.getMinutes()+'-'+d.getSeconds()+'-'+d.getMilliseconds());

    var Model    = conn.model('Model', new mongoose.Schema({
            subject : { type : String },
            body : { type : String }
        }));
    var newEmail = new Model();
    newEmail.subject = email_object.subject;
    newEmail.body = email_object.body;
    newEmail.save(function(err) {
        if (err) console.error(err);
        console.log(count++);
    });
});