Failpoint benchmarking scripts

bench.patch

diff --git a/src/mongo/SConscript b/src/mongo/SConscript
index e23047f..95465ca 100644
--- a/src/mongo/SConscript
+++ b/src/mongo/SConscript
@@ -700,3 +700,9 @@ env.Alias("clientBuild", ['#buildscripts/build_and_test_client.py',
           '$PYTHON ${SOURCES[0]} ${SOURCES[2]} ${EXTRAPATH and "--extrapath"} $EXTRAPATH'
           )
 env.AlwaysBuild("clientBuild")
+
+bench_fp = env.Install('#/', env.Program('bench_fp',
+                                         'util/fail_point_bench.cpp',
+                                         LIBDEPS=["alltools"]))
+env.Alias('bench_fp', bench_fp)
+
diff --git a/src/mongo/db/dbcommands_generic.cpp b/src/mongo/db/dbcommands_generic.cpp
index 5804859..eab9131 100644
--- a/src/mongo/db/dbcommands_generic.cpp
+++ b/src/mongo/db/dbcommands_generic.cpp
@@ -42,6 +42,7 @@
 #include "../util/ramlog.h"
 #include "repl/multicmd.h"
 #include "server.h"
+#include "mongo/util/fail_point_service.h"
 
 namespace mongo {
 
@@ -272,6 +273,8 @@ namespace mongo {
         }
     } cmdSet;
 
+    MONGO_FP_DECLARE(pingFP);
+
     class PingCommand : public Command {
     public:
         PingCommand() : Command( "ping" ) {}
@@ -281,6 +284,7 @@ namespace mongo {
         virtual bool requiresAuth() { return false; }
         virtual bool run(const string& badns, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) {
             // IMPORTANT: Don't put anything in here that might lock db - including authentication
+            if (MONGO_FAIL_POINT(pingFP)) return false;
             return true;
         }
     } pingCmd;
diff --git a/src/mongo/util/fail_point.cpp b/src/mongo/util/fail_point.cpp
index eca5fd8..8e35f6d 100644
--- a/src/mongo/util/fail_point.cpp
+++ b/src/mongo/util/fail_point.cpp
@@ -23,7 +23,7 @@ using mongoutils::str::stream;
 
 namespace mongo {
     FailPoint::FailPoint():
-            _fpInfo(0),
+            _fpInfo(1),
             _mode(off),
             _timesOrPeriod(0),
             _modMutex("failPointMutex") {

bench_suite.rb

#! /usr/bin/ruby

# Runs the entire benchmark suite. Should be run on the root source directory.
# Note: all requires the patch file to work.

PATCH_FILE = "bench.patch"

if ARGV.size != 1 then
  puts "usage: #{__FILE__} <none|off|slowoff|all>"
  exit -1
end

def run_test(mode)
  case mode
  when "none"
    puts "#{Time.now} Running no fail points"
    `mkdir nofp_1`
    `./fp_bench.rb 27017 10 500 > nofp_1/stats`
    `mv *.svg nofp_1`

    `mkdir nofp_10`
    `./fp_bench.rb 27017 10 500 > nofp_10/stats`
    `mv *.svg nofp_10`

    `mkdir nofp_20`
    `./fp_bench.rb 27017 20 500 > nofp_20/stats`
    `mv *.svg nofp_20`

  when "off"
    puts "#{Time.now} Running fail points off"
    `mkdir fpoff_1`
    `./fp_bench.rb 27017 10 500 > fpoff_1/stats`
    `mv *.svg fpoff_1`

    `mkdir fpoff_10`
    `./fp_bench.rb 27017 10 500 > fpoff_10/stats`
    `mv *.svg fpoff_10`

    `mkdir fpoff_20`
    `./fp_bench.rb 27017 20 500 > fpoff_20/stats`
    `mv *.svg fpoff_20`

  when "slowoff"
    puts "#{Time.now} Running fail points slow off"
    `mkdir fpslowoff_1`
    `./fp_bench.rb 27017 10 500 > fpslowoff_1/stats`
    `mv *.svg fpslowoff_1`

    `mkdir fpslowoff_10`
    `./fp_bench.rb 27017 10 500 > fpslowoff_10/stats`
    `mv *.svg fpslowoff_10`

    `mkdir fpslowoff_20`
    `./fp_bench.rb 27017 20 500 > fpslowoff_20/stats`
    `mv *.svg fpslowoff_20`
  end
end

MODE = ARGV[0]
puts `patch -N -p1 < #{PATCH_FILE}`

if MODE == "slowoff" || MODE == "all" then
  puts `scons -j4 mongod`
  run_test("slowoff")
end

puts `git checkout src/mongo/util/fail_point.cpp`
if MODE == "off" || MODE == "all" then
  puts `scons -j4 mongod`
  run_test("off")
end

puts `git checkout src/mongo/db/dbcommands_generic.cpp`
if MODE == "none" || MODE == "all" then
  puts `scons -j4 mongod`
  run_test("none")
end

fail_point_bench.cpp

/**
 *    Copyright (C) 2012 10gen Inc.
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

/**
 * Sconscript:

bench_fp = env.Install('#/', env.Program('bench_fp',
                                         'util/fail_point_bench.cpp',
                                         LIBDEPS=["alltools"]))
env.Alias('bench_fp', bench_fp)

 */

#include <boost/thread/thread.hpp>
#include <exception>
#include <iostream>
#include <vector>

#include "mongo/base/parse_number.h"
#include "mongo/client/dbclientinterface.h"
#include "mongo/util/fail_point.h"
#include "mongo/util/timer.h"
#include "mongo/util/stacktrace.h"

using mongo::DBClientConnection;

using std::cerr;
using std::cout;
using std::endl;
using std::vector;

class Pinger {
public:
    Pinger(const char* hostName, size_t pingCount):
            _pingCount(pingCount),
            _conn(true, NULL, 5 /* sec socket timeout */),
            _hostName(hostName) {
    }

    bool init(unsigned long long timeoutMicroSec) {
        mongo::Timer timer;

        // Keep on retrying until it succeeds. The most common error is EADDRNOTAVAIL (cannot
        // assign requested address), which can be resolved after a period of time.
        while (timer.micros() < timeoutMicroSec) {
            try {
                _conn.connect(_hostName);
                return true;
            }
            catch (mongo::ConnectException& ex) {
                // try again
            }
        }

        return false;
    }

    void launchPingerThread() {
        _thread = boost::thread(pingServer, &_conn, _pingCount);
    }

    void joinPingerThread() {
        _thread.join();
    }

private:
    static void pingServer(DBClientConnection* conn, size_t pingCount) {
        for (size_t x = 0; x < pingCount; x++) {
            mongo::BSONObj reply;
            verify(conn->runCommand("test", BSON("ping" << 1), reply));
        }
    }

    const size_t _pingCount;
    DBClientConnection _conn;
    const char* const _hostName;
    boost::thread _thread;
};

// this will be called in certain c++ error cases, for example if there are two active
// exceptions
void myterminate() {
    cout << "terminate() called, printing stack (if implemented for platform):" << endl;
    mongo::printStackTrace();
    ::_exit(-1);
}

unsigned long long runTest(const char* targetServer, size_t pingCount, size_t maxConns) {
    vector<Pinger*> allPingers;

    for (size_t connCount = 0; connCount < maxConns; connCount++) {
        Pinger* pinger = new Pinger(targetServer, pingCount);
        verify(pinger->init(10 * 1000 * 1000));
        allPingers.push_back(pinger);
    }

    mongo::Timer timer;
    for (vector<Pinger*>::iterator iter = allPingers.begin(); iter != allPingers.end(); ++iter) {
        (*iter)->launchPingerThread();
    }

    for (vector<Pinger*>::iterator iter = allPingers.begin(); iter != allPingers.end(); ++iter) {
        (*iter)->joinPingerThread();
        delete *iter;
    }

    return timer.micros();
}

int main(int argc, char* argv[]) {
    if (argc != 4) {
        cerr << "args: <host:port> <num connections> <ping per conn>" << endl;
        return -1;
    }

    std::set_terminate( myterminate );

    mongo::logLevel = -1; // make log quiet
    const size_t HOST_PORT_ARG = 1;
    const size_t NUM_CONN_ARG = 2;
    const size_t PING_PER_CONN_ARG = 3;

    size_t maxConns = 0;
    mongo::parseNumberFromStringWithBase(argv[NUM_CONN_ARG], 10, &maxConns);

    size_t pingsPerConn = 0;
    mongo::parseNumberFromStringWithBase(argv[PING_PER_CONN_ARG], 10, &pingsPerConn);

    cout << runTest(argv[HOST_PORT_ARG], pingsPerConn, maxConns) << endl;

    return 0;
}

fp_bench.rb

#! /usr/bin/ruby

# Before running, install dependencies with:
#   sudo gem install statsample mongo
#
# Do not forget to set ulimit for file descriptors to be at least 4000
# on both the server and this environment before running the tests.
#
# This script also assumes that you have a mongod binary on the current
# working directory

require 'statsample'
require 'mongo'

def report_stats(stat_vector, verbose)
  print "total samples: " if verbose
  puts stat_vector.size
  print "median: " if verbose
  puts stat_vector.median
  print "mean: " if verbose
  puts stat_vector.mean
  print "sample sd: " if verbose
  puts stat_vector.standard_deviation_sample
  print "95th percentile: " if verbose
  puts stat_vector.percentil(95)
  print "99th percentile: " if verbose
  puts stat_vector.percentil(99)
  print "99.99th percentile: " if verbose
  puts stat_vector.percentil(99.99)
end

def create_histogram(file_name, stat_vector)
  open(file_name) do |file|
    histogram = Statsample::Graph::Histogram.new(stat_vector)
    histogram.name = file_name
    histogram.line_normal_distribution = true
    file.puts(histogram.to_svg)
  end
end

if ARGV.size != 3 then
  puts "usage: #{__FILE__} <port> <pings per conn> <trials>"
  exit -1
end

PORT = ARGV[0]
PINGS_PER_CONN = ARGV[1].to_i
TRIALS = ARGV[2].to_i

NUM_CONNS = [1, 10, 100, 1000]
#NUM_CONNS = [1]

CMD = "./bench_fp"
DB_PATH = "/data/db"
MONGOD = "./mongod --noprealloc --nojournal --nohttpinterface --port #{PORT}" +
  " --dbpath #{DB_PATH} --fork --logpath fp_mongod.log 2> /dev/null"
VERBOSE = false
MAX_CONN_RETRY = 5

`#{MONGOD}`
mongod_pid = File.open(File.join(DB_PATH, 'mongod.lock')).read.strip.to_i

retries = MAX_CONN_RETRY
while retries >= 0 do
  begin
    Mongo::Connection.new("localhost", PORT.to_i)
    break
  rescue Mongo::OperationFailure, Mongo::ConnectionFailure => ex
    sleep(1)
    retries -= 1
  end
end

if retries < 0 then
  puts "Unable to start mongod"
  Process.kill('INT', mongod_pid)
  exit(-1)
end

NUM_CONNS.each do |conns|
  puts "Running test for #{conns} connections"
  data_points = []

  TRIALS.times do |trial_num|
    data_points << `#{CMD} localhost:#{PORT} #{conns} #{PINGS_PER_CONN}`.to_i
    # For debugging
    # puts `#{CMD} localhost:#{PORT} #{conns} #{PINGS_PER_CONN}`
    # puts "trial##{trial_num}"
  end

  data_points.reject! do |sample|
    sample.nil? or sample.zero?
  end

  stat_vector = data_points.to_scale

  puts "#"*50
  puts "Stats for #{conns} concurrent connections"
  report_stats(stat_vector, VERBOSE)
  puts

  open("histogram_#{conns}.svg", "w") do |file|
    histogram = Statsample::Graph::Histogram.new(stat_vector)
    histogram.width = 1000 # pixels
    histogram.bins = 50
    file.puts histogram.to_svg
  end
end

Process.kill('INT', mongod_pid)