Demonstrate a failure to reconnect to a MongoDB replicaset when it loses a primary

repeatCounts.js

// Run `npm i mongo` and run this test program after filling in the variables
// below. Run it against a replicaset with a sharded collection, with your
// MongoS process running on the same host as the script.
// You should // see start and finish messages from fxCountWithLog.

// Now do an rs.stepDown() on the replicaset primary. You should see an error
// message and the script should continue. So that's fine.

// Wait until the Replicaset settles down. Now on the new
// replicaset primary, do:
// > sudo ifconfig eth0 down
// ... or kill the network on the primary in some similarly brutal way.
// Now you should see that the test script hangs indefinitely rather than
// err'ing out, even though the replicaset elects a new primary and MongoS
// fails over just fine. With some experimentation, it seems that you can
// get one or more bad connections in the pool. and they will just never
// recover until the process is killed or they are killed programmatically.

// Note that the stepDown is not necessary to the repro, it is just meant to
// illustrate that in that case, everyting behaves properly.

var port = "YOUR LOCAL MONGOS"
var db = "YOUR DB NAME";
var shardedCollection = "YOUR SHARDED COLLECTION NAME";

var MongoClient = require('mongodb').MongoClient;
var assert = require('assert');

var host = "localhost";
var conn = "mongodb://" + host + ":" + port + "/" + db;

MongoClient.connect(conn, function(err, db) {
  assert.equal(null, err);
  assert.ok(db != null);
  var counter = 0;
  var fxCountWithLog = function() {
    var i = ++counter;
    console.log("start", i);
    db.collection(shardedCollection).count(function(err, count) {
      console.log("finish", i, err, count);
      setTimeout(fxCountWithLog, 1000);
    });
  };

  var counter2 = 0;
  var fxCount = function() {
    var i = ++counter2;
    db.collection('applications').count(function(err, count) {
      if (err) console.log("count error", i, err, count);
      setTimeout(fxCount, 1);
    });
  };

  fxCountWithLog();
  fxCount()
});