Skip to content

Instantly share code, notes, and snippets.

@flisboac
Last active September 27, 2016 15:56
Show Gist options
  • Save flisboac/ec3ccb67ebeb6fdd9562a6c0d9d876db to your computer and use it in GitHub Desktop.
Save flisboac/ec3ccb67ebeb6fdd9562a6c0d9d876db to your computer and use it in GitHub Desktop.
A quick and dirty method dispatch benchmark intended to measure method execution time in some scenarios (e.g. inheritance vs method pointers, things like that). Portable C++11, no compiler extensions.
#include <iostream>
#include <chrono>
#include <thread>
#include <type_traits>
#include <vector>
#include <random>
#include <iomanip>
#include <array>
#include <algorithm>
#include <sstream>
#include <thread>
#include <future>
#include <cctype>
#include <set>
#define NELEMS 1000
#define NCALLS 100000
// The timer library
#include <iosfwd>
#include <type_traits>
#include <chrono>
class Timer {
public:
using Clock = std::conditional<
std::chrono::high_resolution_clock::is_steady,
std::chrono::high_resolution_clock,
std::chrono::steady_clock
>::type;
using TimePoint = Clock::time_point;
using Duration = Clock::duration;
enum class EScale : char {
Unscaled = '\0',
Minute = 'M',
Hour = 'h',
Day ='d',
Mili = 'm',
Micro = 'u',
Nano = 'n',
Pico = 'p'
};
template <EScale Value = EScale::Unscaled>
struct Scale {};
template <typename T>
struct Value {
T value = 0;
EScale scale = EScale::Unscaled;
Value(T value_, EScale scale_ = EScale::Unscaled) : value(value_), scale(scale_) {}
Value() = default;
Value(const Value& rhs) = default;
Value(Value&& rhs) = default;
Value& operator=(const Value& rhs) = default;
Value& operator=(Value&& rhs) = default;
inline Value<T>& rescale() {
if (scale == EScale::Unscaled) {
T abs_value = value < 0 ? -value : value;
if (abs_value > 1800) {
value = value / 60;
scale = EScale::Minute;
abs_value = value < 0 ? -value : value;
if (abs_value > 60) {
value = value / 60;
scale = EScale::Hour;
abs_value = value < 0 ? -value : value;
if (abs_value > 96) {
value = value / 24;
scale = EScale::Day;
}
}
} else {
while (abs_value < 1 && scale != EScale::Pico) {
switch(scale) {
case EScale::Unscaled: scale = EScale::Mili; break;
case EScale::Mili: scale = EScale::Micro; break;
case EScale::Micro: scale = EScale::Nano; break;
case EScale::Nano: scale = EScale::Pico; break;
}
value = value * 1000;
abs_value = value < 0 ? -value : value;
}
}
}
return *this;
}
};
template <typename Callable>
inline Timer& execute(Callable fn)
{ start(); fn(); return stop(); }
template <typename T, typename Callable>
inline Timer& execute(T context, Callable fn)
{ start(); fn(context); return stop(); }
inline void start()
{ _start = Clock::now(); }
inline Timer& stop() {
TimePoint stop = Clock::now();
Duration _diff = stop - _start;
_count++;
if (_count > 1) {
_avg = (_avg * (_count - 1) + _diff) / _count;
if (_diff > _max) {
_max = _diff;
_slowest = _count;
}
if (_diff < _min) {
_min = _diff;
_fastest = _count;
}
} else {
_fastest = _slowest = _count;
_min = _max = _avg = _diff;
}
}
inline Timer& reset() {
_count = _fastest = _slowest = 0;
}
inline long count() const
{ return _count; }
inline bool empty() const
{ return _count <= 0; }
template <typename V = double>
inline V min() const
{ return !empty() ? std::chrono::duration<V>(_min).count() : _zero<V>(); }
template <typename V = double>
inline Value<V> min_value() const
{ return Timer::value(min<V>()); }
template <typename V = double>
inline V max() const
{ return !empty() ? std::chrono::duration<V>(_max).count() : _zero<V>(); }
template <typename V = double>
inline Value<V> max_value() const
{ return Timer::value(max<V>()); }
template <typename V = double>
inline V avg() const
{ return !empty() ? std::chrono::duration<V>(_avg).count() : _zero<V>(); }
template <typename V = double>
inline Value<V> avg_value() const
{ return Timer::value(avg<V>()); }
inline bool is_fastest() const
{ return _count > 0 && _count == _fastest; }
inline bool is_slowest() const
{ return _count > 0 && _count == _slowest; }
inline bool operator<(const Timer& rhs) const
{ return _avg < rhs._avg; }
template <typename V>
static inline Value<V> value(V sec) {
return Value<V>(sec).rescale();
}
private:
template <typename V = double>
inline V _zero() const
{ return std::chrono::duration<V>().count(); }
long _count = 0;
long _fastest = 0;
long _slowest = 0;
TimePoint _start;
Duration _min;
Duration _max;
Duration _avg;
};
template <> struct Timer::Scale<Timer::EScale::Minute> {
constexpr static const Timer::EScale id = Timer::EScale::Minute;
constexpr static const char *const unit_name = "min";
constexpr static const char *const units_name = "mins";
};
template <> struct Timer::Scale<Timer::EScale::Hour> {
constexpr static const Timer::EScale id = Timer::EScale::Hour;
constexpr static const char *const unit_name = "hour";
constexpr static const char *const units_name = "hours";
};
template <> struct Timer::Scale<Timer::EScale::Day> {
constexpr static const Timer::EScale id = Timer::EScale::Day;
constexpr static const char *const unit_name = "day";
constexpr static const char *const units_name = "days";
};
template <> struct Timer::Scale<Timer::EScale::Mili> {
constexpr static const Timer::EScale id = Timer::EScale::Mili;
constexpr static const char *const unit_name = "msec";
constexpr static const char *const units_name = "msecs";
};
template <> struct Timer::Scale<Timer::EScale::Micro> {
constexpr static const Timer::EScale id = Timer::EScale::Micro;
constexpr static const char *const unit_name = "usec";
constexpr static const char *const units_name = "usec";
};
template <> struct Timer::Scale<Timer::EScale::Nano> {
constexpr static const Timer::EScale id = Timer::EScale::Nano;
constexpr static const char *const unit_name = "nsec";
constexpr static const char *const units_name = "nsecs";
};
template <> struct Timer::Scale<Timer::EScale::Pico> {
constexpr static const Timer::EScale id = Timer::EScale::Pico;
constexpr static const char *const unit_name = "psec";
constexpr static const char *const units_name = "psecs";
};
template <> struct Timer::Scale<Timer::EScale::Unscaled> {
constexpr static const Timer::EScale id = Timer::EScale::Unscaled;
constexpr static const char *const unit_name = "sec";
constexpr static const char *const units_name = "secs";
template <typename V>
static const char *name(const Timer::Value<V>& value) {
return name(value.scale, value.value);
}
template <typename V>
static const char *name(Timer::EScale scale, V value = V()) {
switch(scale) {
case EScale::Unscaled:
return value <= 1 && value >= -1
? Scale<EScale::Unscaled>::unit_name
: Scale<EScale::Unscaled>::units_name;
case EScale::Minute:
return value <= 1 && value >= -1
? Scale<EScale::Minute>::unit_name
: Scale<EScale::Minute>::units_name;
case EScale::Hour:
return value <= 1 && value >= -1
? Scale<EScale::Hour>::unit_name
: Scale<EScale::Hour>::units_name;
case EScale::Day:
return value <= 1 && value >= -1
? Scale<EScale::Day>::unit_name
: Scale<EScale::Day>::units_name;
case EScale::Mili:
return value <= 1 && value >= -1
? Scale<EScale::Mili>::unit_name
: Scale<EScale::Mili>::units_name;
case EScale::Micro:
return value <= 1 && value >= -1
? Scale<EScale::Micro>::unit_name
: Scale<EScale::Micro>::units_name;
case EScale::Nano:
return value <= 1 && value >= -1
? Scale<EScale::Nano>::unit_name
: Scale<EScale::Nano>::units_name;
case EScale::Pico:
return value <= 1 && value >= -1
? Scale<EScale::Pico>::unit_name
: Scale<EScale::Pico>::units_name;
}
}
};
template <>
inline Timer::Duration Timer::_zero<Timer::Duration>() const
{ return Timer::Duration(); }
template <>
inline Timer::Duration Timer::min<Timer::Duration>() const
{ return !empty() ? _min : _zero<Timer::Duration>(); }
template <>
inline Timer::Duration Timer::max<Timer::Duration>() const
{ return !empty() ? _max : _zero<Timer::Duration>(); }
template <>
inline Timer::Duration Timer::avg<Timer::Duration>() const
{ return !empty() ? _avg : _zero<Timer::Duration>(); }
template <typename T>
inline std::ostream& operator<<(std::ostream& os, const Timer::Value<T>& sec) {
os << sec.value << " " << Timer::Scale<>::name(sec);
return os;
}
// ---------------
// Test structures
// pure virtual member call on subclass
struct IInterfaceVector {
virtual ~IInterfaceVector() {}
virtual double method() const = 0;
};
struct InterfaceVector : public IInterfaceVector {
public:
InterfaceVector(float x, float y, float z, float w)
: _x(x), _y(y), _z(z), _w(w) {}
double method() const { return _x * _y + _z / _w; }
private:
double _x = 1, _y = 1, _z = 1, _w = 1;
};
// virtual-only member call on subclass
struct IConcreteVector {
virtual ~IConcreteVector() {}
virtual double method() const { return 0; }
};
struct ConcreteVector : public IConcreteVector {
public:
ConcreteVector(float x, float y, float z, float w)
: _x(x), _y(y), _z(z), _w(w) {}
double method() const { return _x * _y + _z / _w; }
private:
double _x = 1, _y = 1, _z = 1, _w = 1;
};
// call to non-member method passing a reference
struct StructVector {
StructVector(float x, float y, float z, float w)
: _x(x), _y(y), _z(z), _w(w) {}
double _x = 1, _y = 1, _z = 1, _w = 1;
static double method(const StructVector& v) {
return v._x * v._y + v._z / v._w;
}
};
// call to pointer to non-member method passing a reference
struct PointerVector {
typedef double (*method_f)(const PointerVector&);
PointerVector(float x, float y, float z, float w)
: _x(x), _y(y), _z(z), _w(w), method(PointerVector::static_method), instance(this) {}
double _x = 1, _y = 1, _z = 1, _w = 1;
method_f method;
const PointerVector* instance;
static double static_method(const PointerVector& v) {
return v._x * v._y + v._z / v._w;
}
};
// no inheritance, no virtual, member call
struct PlainVector {
public:
PlainVector(float x, float y, float z, float w)
: _x(x), _y(y), _z(z), _w(w) {}
double method() const { return _x * _y + _z / _w; }
private:
double _x = 1, _y = 1, _z = 1, _w = 1;
};
// Call to pointer-to-member function on virtual-method superclass
struct VirtualMemberVector : public ConcreteVector {
VirtualMemberVector(float x, float y, float z, float w)
: ConcreteVector::ConcreteVector(x, y, z, w), method(&ConcreteVector::method) {}
double (VirtualMemberVector::*method)() const;
};
// Call to pointer-to-member function on non-virtual-method superclass
struct NonVirtualMemberVector : public PlainVector {
NonVirtualMemberVector(float x, float y, float z, float w)
: PlainVector::PlainVector(x, y, z, w), method(&PlainVector::method) {}
double (PlainVector::*method)() const;
};
// ------
// main.c
template <typename T>
std::vector<T> prepareElems(size_t nelems) {
std::vector<T> elems;
std::random_device randomd;
std::default_random_engine r(randomd());
std::uniform_real_distribution<double> random(-1000, 1000);
elems.reserve(nelems);
for (size_t i = 0; i < nelems; ++i) {
elems.emplace_back(T(random(r), random(r), random(r), random(r)));
}
return elems;
}
void testCost() {}
double testByMethod(const IInterfaceVector& elem) { return elem.method(); }
double testByMethod(const IConcreteVector& elem) { return elem.method(); }
double testByMethod(const PlainVector& elem) { return elem.method(); }
double testByMethod(const StructVector& elem) { return StructVector::method(elem); }
double testByMethod(const PointerVector& elem) { return elem.method(*elem.instance); }
double testByMethod(const VirtualMemberVector& elem) { return (elem.*(elem.method))(); }
double testByMethod(const NonVirtualMemberVector& elem) { return (elem.*(elem.method))(); }
Timer executeCostTest(long ncalls, size_t nelems) {
Timer timer;
for (long icall = 0; icall < ncalls; ++icall) {
timer.start();
for (size_t ielems = 0; ielems < nelems; ++ielems) testCost();
timer.stop();
}
return timer;
}
template <typename T>
Timer executeTest(long ncalls, size_t nelems) {
Timer timer;
auto elems = prepareElems<T>(nelems);
for (long long icall = 0; icall < ncalls * nelems; ++icall) {
timer.start();
for (auto& elem : elems) testByMethod(elem);
timer.stop();
}
return timer;
}
std::ostream& operator<<(std::ostream& os, const Timer& timer) {
os << timer.avg_value()
<< ", avg " << timer.avg<Timer::Duration>().count() << " ticks"
<< ", min " << timer.min<Timer::Duration>().count() << " ticks"
<< ", max " << timer.max<Timer::Duration>().count() << " ticks"
;
return os;
}
struct Result {
const char* name;
long ncalls;
size_t nelems;
Timer total_timer;
Timer call_timer;
};
std::ostream& operator<<(std::ostream& os, const Result& result) {
return os
<< "\t" << result.name << ":"
<< std::endl << "\t\t- total test execution time: " << result.total_timer.avg_value()
<< std::endl << "\t\t- effective call time: " << Timer::value(result.call_timer.avg() / result.nelems)
<< std::endl << "\t\t- average call time: " << result.call_timer.avg<Timer::Duration>().count() << " ticks"
<< std::endl << "\t\t- minimum call time: " << result.call_timer.min<Timer::Duration>().count() << " ticks"
<< std::endl << "\t\t- maximum call time: " << result.call_timer.max<Timer::Duration>().count() << " ticks"
<< std::endl
;
}
template <typename T>
std::function<Result(void)> makeExecFunction(const char* name, long ncalls, size_t nelems) {
std::string begin = std::string("*** Executing test ") + name + "...\n";
std::string end = std::string("*** Finished test ") + name + ".\n";
return [=]() -> Result {
std::cerr << begin;
Result result;
result.ncalls = ncalls;
result.nelems = nelems;
result.name = name;
result.total_timer.start();
result.call_timer = executeTest<T>(ncalls, nelems);
result.total_timer.stop();
std::cerr << end;
return result;
};
}
#define TEST(dc) (tests.empty() || tests.find(dc) != tests.end()) // || !tests.find(std::toupper(dc)))
#define PAIRS() \
if (TEST("struct")) PAIR(StructVector);\
if (TEST("plain")) PAIR(PlainVector);\
if (TEST("interface")) PAIR(InterfaceVector);\
if (TEST("concrete")) PAIR(ConcreteVector);\
if (TEST("virtual")) PAIR(VirtualMemberVector);\
if (TEST("nonvirtual")) PAIR(NonVirtualMemberVector);\
if (TEST("pointer")) PAIR(PointerVector);
std::vector<Result> executeAsync(long ncalls, size_t nelems, const std::set<std::string>& tests) {
# define PAIR(T) \
future_results.emplace_back( \
std::async( \
std::launch::async, \
makeExecFunction<T>(#T, ncalls, nelems) \
) \
)
std::vector<std::future<Result>> future_results;
PAIRS();
for (auto& future_result : future_results) {
future_result.wait_for(std::chrono::microseconds(1));
}
std::vector<Result> results;
for (auto& future_result : future_results) {
results.push_back(future_result.get());
}
return results;
# undef PAIR
}
std::vector<Result> executeSync(long ncalls, size_t nelems, const std::set<std::string>& tests) {
# define PAIR(T) \
results.emplace_back(makeExecFunction<T>(#T, ncalls, nelems)())
std::vector<Result> results;
PAIRS();
return results;
# undef PAIR
}
#undef TEST
#undef PAIRS
int main(int argc, char** argv) {
long ncalls = NCALLS;
size_t nelems = NELEMS;
bool async = false;
std::set<std::string> tests;
for (int iargc = 1; iargc < argc; ++iargc) {
std::string arg = argv[iargc];
if (arg[0] == '-' && arg.size() > 1) {
arg = &arg[1];
switch(arg[0]) {
case 't': async = true; break;
case 'T': async = false; break;
case 'c': arg = &arg[1]; ncalls = std::stol(arg); break;
case 'e': arg = &arg[1]; nelems = std::stoi(arg); break;
case 'h':
std::cerr << "Usage: " << argv[0] << " [-cNCALLS] [-eNELEMS] [-t|-T] [TEST...]"
<< std::endl << " where TEST = {struct|plain|interface|concrete|virtual|nonvirtual|pointer}"
<< std::endl << " NCALLS specifies the number of batch call rounds"
<< std::endl << " NELEMS specifies the number of batch calls"
<< std::endl << " -t activates asynchronous (threaded) mode"
<< std::endl << " -T deactivates asynchronous (threaded) mode"
<< std::endl << " The total number of calls are NCALLS * NELEMS, times the number of tests."
<< std::endl << " By default, all tests are executed."
<< std::endl << " By default, test execution is " << (async ? "asynchronous" : "synchronous") << "."
<< std::endl;
return 1;
}
} else {
tests.insert(arg);
}
}
std::cerr << "Hello. :)" << std::endl;
//Timer costTimer = executeCostTest(ncalls, nelems);
std::cerr << "Executing " << (async ? "asynchronous" : "synchronous") << " tests"
<< " (" << ncalls << " batch calls to methods on " << nelems << " objects)"
<< "..." << std::endl;
std::vector<Result> results = async
? executeAsync(ncalls, nelems, tests)
: executeSync(ncalls, nelems, tests);
std::sort(results.begin(), results.end(), [](Result& a, Result& b) {
return a.call_timer < b.call_timer;
});
std::cerr << std::fixed << std::setprecision(6);
std::cerr << "Finished. Results are shown ordered by average call time (in ticks)." << std::endl;
for (auto& result : results) {
std::cerr << result << std::endl;
}
return 0;
}
#if 0
[flisboac@sonic ~]$ grep "model name" /proc/cpuinfo && uname -a && head -n1 /etc/issue \
&& g++ --std=c++11 -O3 -s -pthread -o test-method-dispatch-benchmark test-method-dispatch-benchmark.cpp \
&& time ./test-method-dispatch-benchmark -c1000000 -e500 -t
model name : Intel(R) Core(TM) i5-6300HQ CPU @ 2.30GHz
model name : Intel(R) Core(TM) i5-6300HQ CPU @ 2.30GHz
model name : Intel(R) Core(TM) i5-6300HQ CPU @ 2.30GHz
model name : Intel(R) Core(TM) i5-6300HQ CPU @ 2.30GHz
Linux sonic 4.7.4-1-ARCH #1 SMP PREEMPT Thu Sep 15 15:24:29 CEST 2016 x86_64 GNU/Linux
Arch Linux \r (\l)
Hello. :)
Executing tests (1000000 batch calls to methods on 500 objects)...
*** Executing test StructVector...
*** Executing test InterfaceVector...
*** Executing test ConcreteVector...
*** Executing test PlainVector...
*** Executing test VirtualMemberVector...
*** Executing test NonVirtualMemberVector...
*** Executing test PointerVector...
*** Finished test StructVector.
*** Finished test PlainVector.
*** Finished test InterfaceVector.
*** Finished test ConcreteVector.
*** Finished test PointerVector.
*** Finished test VirtualMemberVector.
*** Finished test NonVirtualMemberVector.
Finished. Results are shown ordered by average call time (in ticks).
StructVector:
- total test execution time: 53.723240 secs
- effective call time: 44.000000 psecs
- average call time: 22 ticks
- minimum call time: 22 ticks
- maximum call time: 26670344 ticks
PlainVector:
- total test execution time: 62.199295 secs
- effective call time: 44.000000 psecs
- average call time: 22 ticks
- minimum call time: 22 ticks
- maximum call time: 23336146 ticks
InterfaceVector:
- total test execution time: 214.925010 secs
- effective call time: 504.000000 psecs
- average call time: 252 ticks
- minimum call time: 251 ticks
- maximum call time: 36489831 ticks
ConcreteVector:
- total test execution time: 218.173779 secs
- effective call time: 506.000000 psecs
- average call time: 253 ticks
- minimum call time: 251 ticks
- maximum call time: 29935182 ticks
PointerVector:
- total test execution time: 571.687202 secs
- effective call time: 1.834000 nsecs
- average call time: 917 ticks
- minimum call time: 917 ticks
- maximum call time: 42646643 ticks
VirtualMemberVector:
- total test execution time: 654.880021 secs
- effective call time: 2.078000 nsecs
- average call time: 1039 ticks
- minimum call time: 1039 ticks
- maximum call time: 33457194 ticks
NonVirtualMemberVector:
- total test execution time: 720.059822 secs
- effective call time: 2.224000 nsecs
- average call time: 1112 ticks
- minimum call time: 1112 ticks
- maximum call time: 31136479 ticks
real 12m0.063s
user 35m35.153s
sys 0m0.453s
[flisboac@sonic ~]$ grep "model name" /proc/cpuinfo && uname -a && head -n1 /etc/issue \
&& g++ --std=c++11 -O3 -s -pthread -o test-method-dispatch-benchmark test-method-dispatch-benchmark.cpp \
&& time ./test-method-dispatch-benchmark -c1000000 -e500 -T
model name : Intel(R) Core(TM) i5-6300HQ CPU @ 2.30GHz
model name : Intel(R) Core(TM) i5-6300HQ CPU @ 2.30GHz
model name : Intel(R) Core(TM) i5-6300HQ CPU @ 2.30GHz
model name : Intel(R) Core(TM) i5-6300HQ CPU @ 2.30GHz
Linux sonic 4.7.4-1-ARCH #1 SMP PREEMPT Thu Sep 15 15:24:29 CEST 2016 x86_64 GNU/Linux
Arch Linux \r (\l)
Hello. :)
Executing synchronous tests (1000000 batch calls to methods on 500 objects)...
*** Executing test StructVector...
*** Finished test StructVector.
*** Executing test PlainVector...
*** Finished test PlainVector.
*** Executing test InterfaceVector...
*** Finished test InterfaceVector.
*** Executing test ConcreteVector...
*** Finished test ConcreteVector.
*** Executing test VirtualMemberVector...
*** Finished test VirtualMemberVector.
*** Executing test NonVirtualMemberVector...
*** Finished test NonVirtualMemberVector.
*** Executing test PointerVector...
*** Finished test PointerVector.
Finished. Results are shown ordered by average call time (in ticks).
StructVector:
- total test execution time: 30.570722 secs
- effective call time: 38.000000 psecs
- average call time: 19 ticks
- minimum call time: 19 ticks
- maximum call time: 397317 ticks
PlainVector:
- total test execution time: 31.339930 secs
- effective call time: 38.000000 psecs
- average call time: 19 ticks
- minimum call time: 19 ticks
- maximum call time: 63780 ticks
InterfaceVector:
- total test execution time: 161.599458 secs
- effective call time: 462.000000 psecs
- average call time: 231 ticks
- minimum call time: 231 ticks
- maximum call time: 6559300 ticks
ConcreteVector:
- total test execution time: 144.793157 secs
- effective call time: 466.000000 psecs
- average call time: 233 ticks
- minimum call time: 233 ticks
- maximum call time: 571174 ticks
VirtualMemberVector:
- total test execution time: 582.550321 secs
- effective call time: 1.836000 nsecs
- average call time: 918 ticks
- minimum call time: 917 ticks
- maximum call time: 1099146 ticks
PointerVector:
- total test execution time: 540.824537 secs
- effective call time: 1.924000 nsecs
- average call time: 962 ticks
- minimum call time: 962 ticks
- maximum call time: 2753155 ticks
NonVirtualMemberVector:
- total test execution time: 652.211828 secs
- effective call time: 2.224000 nsecs
- average call time: 1112 ticks
- minimum call time: 1112 ticks
- maximum call time: 20039886 ticks
real 35m43.892s
user 35m42.720s
sys 0m0.480s
#endif
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment