Stack:
- Mongo native driver 2.2
- NodeJS v 6.11.3
- Windows 10
If I launch more task for driver than his pool size in parallel, I can not get any response from my NodeJS http server.
Example:
I use async's lib async.each to call 50 000 inserts using mongo driver with pool size set to 1 While this tasks are running (slowly, cause we can have only one query running at the moment), I can not fetch any page from my server. I can't even see this request anywhere. The question is why?
FAQ
- I can increase pool size for driver or use async.eachSeries, if I want just to solve current situation.
- I have new Node so max sockets for agents are set to Infinity by default, so this brilliant article seems to be too old.
- My RAM and CPU are feeling fine all times. Disk I/O is OK too.
- I will gladly use read articles on this topic, but I really need someone to chew this for me
Here is some code to "try this at home"
const mongoUrl = "mongodb://******";
var db = require('./db'); //basic database workflows
const http = require("http");
const async = require('async');
db.connect(mongoUrl, (err, database) => {
if (err) {
console.log(err);
process.exit(1);
} else {
server = http.createServer(function (request, response) {
response.writeHead(200, { "Content-Type": "text/plain" });
response.write("Hello World");
response.end();
}).listen(8888);
console.log("running on 8888");
var t = setTimeout(function () {
//here we can see that after 30s node is still working alright
console.log("Timeout fired in 30 seconds - I'm ok, I can fire timeouts!");
}, 30 * 1000);
//let's create a giant array for async.each
var testArray = [];
for (let i = 0; i < 60000; i++) {
testArray[i] = Math.floor(Math.random() * 16777215).toString(16);
}
//clear the collection
db.get().collection('someCollection').drop();
async.each(testArray, function (artist, callback) {
//insert to mongo
db.get().collection('someCollection').
insert({ "unimportant": artist },
function (err, result) {
if (err) return callback(err);
return callback(null, "OK");
});
}, function (err, results) {
if (err) return console.log(err);
return console.log("somehow finished");
});
console.log("No blocking operations before this point - hooray!");
}
});
db.js
var mongodb = require('mongodb');
var MongoClient = mongodb.MongoClient;
var ObjectID = mongodb.ObjectID;
var state = {
db: null,
};
exports.connect = function(url, done) {
if (state.db) return done();
MongoClient.connect(url, {
poolSize:1, //pool size
}, function(err, db) {
if (err) return done(err);
state.db = db;
state.discogsDB = db.db("discogs");
done();
});
};
exports._id = function(id) {
return new ObjectID(id);
};
exports.get = function() {
return state.db;
};
exports.getD = function() {
return state.discogsDB;
};
exports.close = function(done) {
if (state.db) {
state.db.close(function(err, result) {
state.db = null;
state.mode = null;
done(err);
});
}
};
