Connection pool is closed using Mongoose

Question

I have a little web app which serves data from a Mongo database. I have configured 2 cron jobs (through Heroku scheduler) to run everyday and manipulate a remote db. The problem is that I need those jobs to conclude and not keep running after finishing, that means closing my connection, otherwhise my function keeps running.

When I call mongoose.disconnect() in one of my files I get Mongo error: Connection pool closed.

This is the problematic file:

require('dotenv').config();
const malagaCulturaScrapper = require("../scrappers/malagaCultura");
const Event = require("../schemas/eventSchema");
const mongoose = require("mongoose");

(async () => {
    try {
        await mongoose.connect((process.env.DEV_MODE === "true") ? process.env.SAMPLEDB_URL:process.env.DATABASE_URL, {
            useNewUrlParser: true,
            useUnifiedTopology: true
        })
    } catch (err) {
        console.log('Error: ' + err)
    }
})()

const scrapeAndSave = async() =>{
    await Event.deleteMany({}).exec();
    await malagaCulturaScrapper.malagaCulturaScrapper();
    mongoose.disconnect();
}

scrapeAndSave();

Which calls this function:

const malagaCulturaScrapper = async() =>{
        await scrapeIt("https://malagadecultura.com/agenda/", {
            events: {
                listItem: "article",
                data: {
                    title: "h4",
                    time: ".mec-event-time",
                    date: {
                        closest: "li",
                        attr: "id"                        
                    },
                    event_img: {
                        selector: ".attachment-thumbnail",
                        attr: "src"
                    },
                    event_link:{
                        selector: "h4 a",
                        attr: "href",
                    },
                    location: ".mec-event-loc-place",
                }
        }
        }).then(({ data, response }) => {

            for(const index in data.events){
                let newEvent = new Event({
                    _id: new mongoose.Types.ObjectId(),
                    title: data.events[index].title,
                    dateTime: createDateObject(data.events[index].date.substring(59), data.events[index].time),
                    event_img: data.events[index].event_img,
                    event_link: data.events[index].event_link,
                    location: data.events[index].location,
                })
                Event.addEvents(newEvent);
            }
        })

}

And this is my adding function on the schema:

Event.addEvents = async(newEvent) => {
    await Event.findOne({
        location: newEvent.location || {
            '$regex': newEvent.location, $options: 'i',
        },
        dateTime: { $in: newEvent.dateTime }
    }, (err, event) => {
        if(err) throw err;
        if (!event && !helpers.isPastEvent(newEvent.dateTime[0], Date.now()) && new Date(newEvent.dateTime[0]).getFullYear() === new Date(Date.now()).getFullYear()) {
            newEvent.save((err) => {
                if (err) throw err;
            })
        } 
    })
}

Lastly, the stacktrace I get when running the file:

/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongoose/lib/helpers/promiseOrCallback.js:19
            throw error;
            ^

MongoError: connection pool closed
    at ConnectionPool.close (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/cmap/connection_pool.js:300:34)
    at Server.destroy (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/sdam/server.js:213:17)
    at destroyServer (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/sdam/topology.js:806:10)
    at eachAsync (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/sdam/topology.js:353:25)
    at eachAsync (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/utils.js:131:5)
    at s.sessionPool.endAllPooledSessions (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/sdam/topology.js:351:7)
    at topology.endSessions (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/sessions.js:592:13)
    at command (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/sdam/topology.js:501:45)
    at cb (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/sdam/topology.js:681:26)
    at fn (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/cmap/connection_pool.js:350:13)
    at Object.handleOperationResult [as cb] (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/sdam/server.js:558:5)
    at Connection.write (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/cmap/connection.js:378:26)
    at _command (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/wireprotocol/command.js:120:10)
    at Object.command (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/wireprotocol/command.js:28:5)
    at Connection.command (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/cmap/connection.js:171:8)
    at s.pool.withConnection (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/sdam/server.js:285:12)
Emitted 'error' event at:
    at /home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongoose/lib/model.js:4844:13
    at /home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongoose/lib/helpers/promiseOrCallback.js:16:11
    at /home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongoose/lib/model.js:4865:21
    at (anonymous function).call (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongoose/lib/query.js:4411:18)
    at Immediate.Query.base.findOne.call (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongoose/lib/query.js:2145:7)
    at Immediate.<anonymous> (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mquery/lib/utils.js:116:16)
    at runCallback (timers.js:705:18)
    at tryOnImmediate (timers.js:676:5)
    at processImmediate (timers.js:658:5)

I would really appreciate any guidance or help on the subject. Thanks in advance.

The example code has `mongoose.connection.close()`. Have you tried `await mongoose.disconnect()` as well? — Matt, Nov 18 '20 at 00:17
Just tried, same result, weird thing since I have another file which is for sending daily mails which uses a connection to the db and closes fine. My guess is something is happening in my scrapping or event saving function. Because leaving deleteMany() alone won't cause the error. — Carlos Molero, Nov 18 '20 at 00:23
This is likely caused by what appears to be a long running process. In time, TCP times out and the connection is closed. Use the `keepAlive` option in connection settings object to fix this. — Randy Casburn, Nov 18 '20 at 00:47
Oh yeah, you're mixing callbacks and async in addEvents, and also not waiting for it in malagaCulturaScrapper — Matt, Nov 18 '20 at 09:21

score 4 · Accepted Answer · answered Nov 18 '20 at 09:32

The code is trying to disconnect before the queries have completed.

Use the Promise API to mongoose by not supplying any callback functions and make sure you await promises so the disconnect run's after everything has completed.

const malagaCulturaScrapper = async() =>{
    const { data, response } = await scrapeIt("https://malagadecultura.com/agenda/", {
         ...eventsInfo
    })
    for(const index in data.events){
        let newEvent = new Event({
            _id: new mongoose.Types.ObjectId(),
            title: data.events[index].title,
            dateTime: createDateObject(data.events[index].date.substring(59), data.events[index].time),
            event_img: data.events[index].event_img,
            event_link: data.events[index].event_link,
            location: data.events[index].location,
        })
        await Event.addEvents(newEvent);
    }
}

Event.addEvents = async(newEvent) => {
    const event = await Event.findOne({
        location: newEvent.location || {
            '$regex': newEvent.location, $options: 'i',
        },
        dateTime: { $in: newEvent.dateTime }
    })

    if (!event && !helpers.isPastEvent(newEvent.dateTime[0], Date.now()) && new Date(newEvent.dateTime[0]).getFullYear() === new Date(Date.now()).getFullYear()) {
        await newEvent.save()
    })
}

Thanks a lot, now is working! But, a question, why callbacks kind of interferes with promises? I usually get things quickly but I'm having a hard time understanding asynchronous javascript. Thanks again — Carlos Molero, Nov 18 '20 at 12:06
They don't interfere as such, but mixing the two will make code more confusing than just using one or the other, and nested callbacks are already confusing enough. The reason `async`/`await` was introduced was to allow async/promise code to be laid out more like people write synchronous code. Promises also provide much more reliable error handling. — Matt, Nov 18 '20 at 22:37
If you hang around here you'll notice there's various forms [of this question](https://stackoverflow.com/questions/14220321/how-do-i-return-the-response-from-an-asynchronous-call) every day. Even this question is essentially the same thing, how to wait for async code to complete. Promises with async/await made this a lot easier code. — Matt, Nov 18 '20 at 22:42

Connection pool is closed using Mongoose

1 Answers1