Application database connection state detection and management in single and replica set environments.

gistfile1.js

// initiate the connection
app._connectDatabase();


MyAppServer.prototype._reconnectDb = function() {

    if (!this.stopping) {
        if (this.dbConnected) {
            this._recon.firstTry = null;
            return;
        }
        if (this._recon.firstTry && (this._recon.firstTry+this._recon.maxTimeout < (new Date().getTime()) )) {
            logger.error('too many mongodb reconnection attempt failures!');
            this.emit('error', faults.create('kError', 'MongoDB could not reconnect!'));
        } else {
            if (!this._recon.timer) {

                this._recon.timer = setTimeout(function() {
                    self._recon.timer = null;
                    self._reconnectDb();
                }, this._recon.interval);

                this._recon.firstTry = this._recon.firstTry || new Date().getTime();

                metrics.incrementMetric('Database', 'connectionErrors', 1);

                var self = this;
                var Account = this.model('Account');
                if (Account) {
                    logger.info('retrying mongodb connection...');

                    if (!self.dbHasConnected && mongoose.connection.readyState == 0) {

                        // if we haven't already connected, try a new connection. it seems an initial connection has to be established or
                        // queries made during this period will not recover.
                        self._connectDatabase();

                    } else if (self.dbHasConnected) {
                        
                        self._doDbPing(function(err) {
                            self.dbConnected = !err;
                            if (!err) {
                                logger.info('mongodb reconnected');
                            }
                        });
                    }

                }

            }
        }
    }

};

MyAppServer.prototype._doDbPing = function(callback) {

    // because we're auto-reconnecting, this command will be stored. force it to run without storage.
    var autoReconnect = mongoose.connection.db.serverConfig.isAutoReconnect;
    mongoose.connection.db.serverConfig.isAutoReconnect = function(){return false;};
    mongoose.connection.db.executeDbCommand({ping:1}, {failFast: true}, callback);
    mongoose.connection.db.serverConfig.isAutoReconnect = autoReconnect;

};

MyAppServer.prototype._connectDatabase = function() {

    // on the first go, we attempt to reconnect until we have a connection of we've timed out.
    if (!this._dbEventHandlers) {
        var self = this;
        var triggeredFromDisconnect = false;
        this._dbEventHandlers = {
            connecting: function() {
                logger.info('mongodb connecting...');
            },
            connected: function() {
                logger.info('mongodb connected...');
                self.dbConnected = true;
                self.dbHasConnected = true;
                triggeredFromDisconnect = false;
            },
            open: function() {
                logger.info('mongodb opened...');
            },
            disconnecting:  function() {
                logger.info('mongodb disconnecting...');
            },
            close: function() {
                logger.info('mongodb closed...');
            },
            reconnected: function() {
                self.dbConnected = true;
                logger.info('mongodb reconnected...');
            },
            fullsetup: function() {

                logger.info('mongodb replica-set connected...');

                // mongo is not firing disconnection errors for replica sets, so we will have to poll.
                if (!self.dbReplicaPollInterval) {
                    self.dbReplicaPollInterval = setInterval(function() {
                        if (self.dbConnected) {
                            self._doDbPing(function(err) {
                                if (err) {
                                    logger.error("MongoDB replica set disconnection detected");
                                    self.dbConnected = false;
                                    self._reconnectDb();
                                }
                            });
                        }
                    }, self.config.mongo.replPingInterval);
                }

            },
            disconnected: function() {
                logger.info('mongodb disconnected...');
                if (!self.dbHasConnected) {
                    triggeredFromDisconnect = true;
                    self.dbConnected = false;
                    self._reconnectDb();
                }
            },
            error: function(err) {
                logger.error("MongoDB "+ err);
                if (!self.dbHasConnected) {
                    if (!triggeredFromDisconnect) {
    				    // this will fire 'disconnected' and kick off the re-connection loop for the initial connection.
                        mongoose.connection.close();
                    }
                    triggeredFromDisconnect = false;
                } else {
                    self.dbConnected = false;
                    self._reconnectDb();
                }
            }
        };
    }

    var mongoOpts = this.config.mongo.auth ? {user: this.config.mongo.user, pass: this.config.mongo.pass}: {};
    mongoOpts.replset = this.config.mongo.replOpts;
    mongoOpts.server  = {auto_reconnect: true};

    mongoose.connect(this.config.mongo.url, mongoOpts);

    Object.keys(this._dbEventHandlers).forEach(function(event) {
        mongoose.connection.removeListener(event, this._dbEventHandlers[event]).on(event, this._dbEventHandlers[event]);
    }.bind(this));

};