Failpoint microbenchmark scripts

fail_point_micro_bench.cpp

/**
 *    Copyright (C) 2012 10gen Inc.
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

/**
 * Sconscript:

microbench_fp = env.Install('#/', env.Program('microbench_fp',
                                         'util/fail_point_micro_bench.cpp',
                                         LIBDEPS=["alltools"]))
env.Alias('microbench_fp', microbench_fp)

 */

#include <boost/thread/thread.hpp>
#include <exception>
#include <iostream>

#include "mongo/base/parse_number.h"
#include "mongo/client/dbclientinterface.h"
#include "mongo/util/fail_point.h"
#include "mongo/util/timer.h"
#include "mongo/util/stacktrace.h"

using mongo::DBClientConnection;
using mongo::FailPoint;

using std::cerr;
using std::cout;
using std::endl;

int normalLoop(size_t count) {
    int val = 0;

    for (size_t x = 0; x < count; x++) {
        val++;
    }

    return val;
}

int fpOffLoop(size_t count, FailPoint* failPoint) {
    int val = 0;

    for (size_t x = 0; x < count; x++) {
        if (MONGO_FAIL_POINT((*failPoint))) {
            val--;
        }
        else {
            val++;
        }
    }

    return val;
}

// this will be called in certain c++ error cases, for example if there are two active
// exceptions
void myterminate() {
    cout << "terminate() called, printing stack (if implemented for platform):" << endl;
    mongo::printStackTrace();
    ::_exit(-1);
}

unsigned long long runNormalTest(size_t numThreads, size_t loops) {
    boost::thread_group threads;

    mongo::Timer timer;
    for (size_t x = 0; x < numThreads; x++) {
        threads.add_thread(new boost::thread(normalLoop, loops));
    }

    threads.join_all();
    return timer.micros();
}

unsigned long long runFailPointOffTest(size_t numThreads, size_t loops) {
    boost::thread_group threads;
    FailPoint fp;

    mongo::Timer timer;
    for (size_t x = 0; x < numThreads; x++) {
        threads.add_thread(new boost::thread(fpOffLoop, loops, &fp));
    }

    threads.join_all();
    return timer.micros();
}

int main(int argc, char* argv[]) {
    if (argc != 4) {
        cerr << "args: <num threads> <loop> <mode>" << endl;
        return -1;
    }

    std::set_terminate(myterminate);

    mongo::logLevel = -1; // make log quiet
    const size_t NUM_THREADS_ARG = 1;
    const size_t LOOP_COUNT_ARG = 2;
    const size_t MODE_ARG = 3;

    size_t numThreads = 0;
    mongo::parseNumberFromStringWithBase(argv[NUM_THREADS_ARG], 10, &numThreads);

    size_t loopCount = 0;
    mongo::parseNumberFromStringWithBase(argv[LOOP_COUNT_ARG], 10, &loopCount);

    size_t mode = 0;
    mongo::parseNumberFromStringWithBase(argv[MODE_ARG], 10, &mode);

    if (mode == 0) {
        cout << runNormalTest(numThreads, loopCount) << endl;
    }
    else {
        cout << runFailPointOffTest(numThreads, loopCount) << endl;
    }

    return 0;
}

fastload.patch

diff --git a/src/mongo/util/fail_point.h b/src/mongo/util/fail_point.h
index 1dd42d0..054eb01 100644
--- a/src/mongo/util/fail_point.h
+++ b/src/mongo/util/fail_point.h
@@ -86,7 +86,7 @@ namespace mongo {
          */
         inline RetCode shouldFailOpenBlock() {
             // TODO: optimization - use unordered load once available
-            if (MONGO_likely((_fpInfo.load() & ACTIVE_BIT) == 0)) {
+            if (MONGO_likely((_fpInfo.loadRelaxed() & ACTIVE_BIT) == 0)) {
                 return fastOff;
             }

fp_microbench.rb

#! /usr/bin/ruby

# Before running, install dependencies with:
#   sudo gem install statsample

require 'statsample'

def report_stats(stat_vector, verbose)
  print "total samples: " if verbose
  puts stat_vector.size
  print "median: " if verbose
  puts stat_vector.median
  print "mean: " if verbose
  puts stat_vector.mean
  print "sample sd: " if verbose
  puts stat_vector.standard_deviation_sample
  print "95th percentile: " if verbose
  puts stat_vector.percentil(95)
  print "99th percentile: " if verbose
  puts stat_vector.percentil(99)
  print "99.99th percentile: " if verbose
  puts stat_vector.percentil(99.99)
end

def create_histogram(file_name, stat_vector)
  open(file_name) do |file|
    histogram = Statsample::Graph::Histogram.new(stat_vector)
    histogram.name = file_name
    histogram.line_normal_distribution = true
    file.puts(histogram.to_svg)
  end
end

if ARGV.size != 2 then
  puts "usage: #{__FILE__} <trials> <mode>"
  exit -1
end

NUM_THREADS = [1, 4, 10, 100]
NUM_LOOPS = [100, 1000, 10000, 100000]
TRIALS = ARGV[0].to_i
MODE = ARGV[1]

CMD = "./microbench_fp"
VERBOSE = false

NUM_THREADS.each do |threads|
  NUM_LOOPS.each do |loops|
    data_points = []

    TRIALS.times do |trial_num|
      data_points << `#{CMD} #{threads} #{loops} #{MODE}`.to_i
      # For debugging
      # puts `#{CMD} #{threads} #{loops} #{MODE}`
      # puts "trial##{trial_num}"
    end

    data_points.reject! do |sample|
      sample.nil? or sample.zero?
    end

    stat_vector = data_points.to_scale

    puts "#"*50
    puts "Stats for #{threads} threads, #{loops} loops"
    report_stats(stat_vector, VERBOSE)
    puts

    open("histogram_#{threads}_#{loops}.svg", "w") do |file|
      histogram = Statsample::Graph::Histogram.new(stat_vector)
      histogram.width = 1000 # pixels
      histogram.bins = 50
      file.puts histogram.to_svg
    end
  end
end

micro_bench_scons.patch

diff --git a/src/mongo/SConscript b/src/mongo/SConscript
index 382747a..94a1a1b 100644
--- a/src/mongo/SConscript
+++ b/src/mongo/SConscript
@@ -787,3 +787,9 @@ env.Alias("clientBuild", ['#buildscripts/build_and_test_client.py',
           '$PYTHON ${SOURCES[0]} ${SOURCES[2]} ${EXTRAPATH and "--extrapath"} $EXTRAPATH'
           )
 env.AlwaysBuild("clientBuild")
+
+microbench_fp = env.Install('#/', env.Program('microbench_fp',
+                                         'util/fail_point_micro_bench.cpp',
+                                         LIBDEPS=["alltools"]))
+env.Alias('microbench_fp', microbench_fp)
+

microbench_suite.rb

#! /usr/bin/ruby

# Runs the entire benchmark suite. Should be run on the root source directory.
# Note: all requires the patch file to work.

FAST_LOAD_PATCH_FILE = "fastload.patch"
SCONS_PATCH_FILE = "micro_bench_scons.patch"
SLOW_FP_PATCH_FILE = "slowfp.patch"

if ARGV.size != 1 then
  puts "usage: #{__FILE__} <none|off|fastoff|slowoff|all>"
  exit -1
end

def run_test(mode)
  case mode
  when "none"
    `mkdir nofp`
    `./fp_microbench.rb 1000 0 > nofp/stats`
    `mv *.svg nofp`

  when "off"
    `mkdir fpoff`
    `./fp_microbench.rb 1000 1 > fpoff/stats`
  end
end

MODE = ARGV[0]
puts `patch -N -p1 < #{SCONS_PATCH_FILE}`

if MODE == "none" || MODE == "all" then
  puts `scons -j4 microbench_fp`
  `mkdir nofp`
  `./fp_microbench.rb 1000 0 > nofp/stats`
  `mv *.svg nofp`
end

if MODE == "off" || MODE == "all" then
  `mkdir fpoff`
  `./fp_microbench.rb 1000 1 > fpoff/stats`
  `mv *.svg fpoff`
end

if MODE == "fastoff" || MODE == "all" then
  puts `patch -N -p1 < #{FAST_LOAD_PATCH_FILE}`
  puts `scons -j4 microbench_fp`

  `mkdir fastfpoff`
  `./fp_microbench.rb 1000 1 > fastfpoff/stats`
  `mv *.svg fastfpoff`

  puts `git checkout src/mongo/util/fail_point.h`
end

if MODE == "fastoff" || MODE == "all" then
  puts `patch -N -p1 < #{SLOW_FP_PATCH_FILE}`
  puts `scons -j4 microbench_fp`

  `mkdir fpslow`
  `./fp_microbench.rb 1000 1 > fpslow/stats`
  `mv *.svg fpslow`

  puts `git checkout src/mongo/util/fail_point.cpp`
end

slowfp.patch

diff --git a/src/mongo/util/fail_point.cpp b/src/mongo/util/fail_point.cpp
index eca5fd8..8e35f6d 100644
--- a/src/mongo/util/fail_point.cpp
+++ b/src/mongo/util/fail_point.cpp
@@ -23,7 +23,7 @@ using mongoutils::str::stream;
 
 namespace mongo {
     FailPoint::FailPoint():
-            _fpInfo(0),
+            _fpInfo(1),
             _mode(off),
             _timesOrPeriod(0),
             _modMutex("failPointMutex") {