Benchmark for https://upscaledb.com

benchmark.cc

#include <vector>
#include <string>
#include <algorithm>
#include <unistd.h>

#include <db.h>

#include <leveldb/slice.h>
#include <leveldb/cache.h>
#include <leveldb/db.h>

#include <ups/upscaledb.h>
#include <ups/upscaledb_uqi.h>

#include "timer.h"

// stub for including upscaledb w/o compression
#ifndef UPS_PARAM_KEY_COMPRESSION
#  define UPS_PARAM_KEY_COMPRESSION 0
#endif

enum {
  // default cache size is 4 mb for all databases
  kCachesize = 4 * 1024 * 1024,

  // Number of keys for the uint32 test
  kNumberOfKeys = 50 * 1000 * 1000,

  // Number of keys for the bin16 test
  kThreeMillion = 3 * 1000 * 1000,

  // Number of runs per test
  kRunsPerTest = 1,

  // Record size for the bin16 test
  kRecordSize = 16,
};

static int
compare_uint32(DB *db, const DBT *dbt1, const DBT *dbt2)
{
  uint32_t l = *(uint32_t *)dbt1->data;
  uint32_t r = *(uint32_t *)dbt2->data;
  if (l < r) return (-1);
  if (r < l) return (+1);
  return 0;
}

typedef std::vector<uint32_t> ivec;

static uint64_t
ups_write_int32(const ivec &kvec, const ivec &rvec)
{
  ups_env_t *env;
  ups_db_t *db;
  ups_parameter_t env_params[] = {
    {UPS_PARAM_CACHE_SIZE, kCachesize},
    {0, 0}
  };
  ups_parameter_t db_params[] = {
    {UPS_PARAM_KEY_TYPE, UPS_TYPE_UINT32},
    {UPS_PARAM_RECORD_TYPE, UPS_TYPE_UINT32},
    {0, 0}
  };

  const char *filename = "benchmark_int32.hdb";

  ups_status_t st = ups_env_create(&env, filename, 0, 0, env_params);
  if (st) {
    printf("ups_env_create failed: %s\n", ups_strerror(st));
    exit(-1);
  }

  st = ups_env_create_db(env, &db, 1, 0, db_params);
  if (st) {
    printf("ups_env_create_db failed: %s\n", ups_strerror(st));
    exit(-1);
  }

  WallClockTimer timer;

  ivec::const_iterator kit = kvec.begin();
  ivec::const_iterator rit = rvec.begin();
  for (; kit != kvec.end(); kit++, rit++) {
    uint32_t kv = *kit;
    ups_key_t key = ups_make_key(&kv, sizeof(kv));
    uint32_t rv = *rit;
    ups_record_t rec = ups_make_record(&rv, sizeof(rv));

    st = ups_db_insert(db, 0, &key, &rec, 0);
    if (st) {
      printf("ups_db_insert failed: %s\n", ups_strerror(st));
      exit(-1);
    }
  }

  uint64_t elapsed = timer.elapsed();

  st = ups_env_close(env, UPS_AUTO_CLEANUP);
  if (st) {
    printf("ups_env_close failed: %s\n", ups_strerror(st));
    exit(-1);
  }

  return elapsed;
}

static uint64_t
ups_read_int32(const ivec &kvec, const ivec &rvec)
{
  ups_env_t *env;
  ups_db_t *db;
  ups_parameter_t env_params[] = {
    {UPS_PARAM_CACHE_SIZE, kCachesize},
    {0, 0}
  };

  const char *filename = "benchmark_int32.hdb";

  ups_status_t st = ups_env_open(&env, filename, 0, env_params);
  if (st) {
    printf("ups_env_open failed: %s\n", ups_strerror(st));
    exit(-1);
  }

  st = ups_env_open_db(env, &db, 1, 0, 0);
  if (st) {
    printf("ups_env_open_db failed: %s\n", ups_strerror(st));
    exit(-1);
  }

  WallClockTimer timer;

  ivec::const_iterator kit = kvec.begin();
  ivec::const_iterator rit = rvec.begin();
  for (; kit != kvec.end(); kit++, rit++) {
    uint32_t kv = *kit;
    ups_key_t key = ups_make_key(&kv, sizeof(kv));
    uint32_t rv = *rit;
    ups_record_t rec = {0};

    st = ups_db_find(db, 0, &key, &rec, 0);
    if (st) {
      printf("ups_db_find failed: %s\n", ups_strerror(st));
      exit(-1);
    }

    if (rv != *(uint32_t *)rec.data) {
      printf("ups_db_find returned bad value\n");
      exit(-1);
    }
  }

  uint64_t elapsed = timer.elapsed();

  st = ups_env_close(env, UPS_AUTO_CLEANUP);
  if (st) {
    printf("ups_env_close failed: %s\n", ups_strerror(st));
    exit(-1);
  }

  return elapsed;
}

static uint64_t
ups_max_int32()
{
  ups_env_t *env;
  ups_db_t *db;
  ups_parameter_t env_params[] = {
    {UPS_PARAM_CACHE_SIZE, kCachesize},
    {0, 0}
  };

  const char *filename = "benchmark_int32.hdb";

  ups_status_t st = ups_env_open(&env, filename, 0, env_params);
  if (st) {
    printf("ups_env_open failed: %s\n", ups_strerror(st));
    exit(-1);
  }

  st = ups_env_open_db(env, &db, 1, 0, 0);
  if (st) {
    printf("ups_env_open_db failed: %s\n", ups_strerror(st));
    exit(-1);
  }

  WallClockTimer timer;

  uqi_result_t *result = 0;
  st = uqi_select(env, "MAX($record) FROM DATABASE 1", &result);
  if (st) {
    printf("uqi_select failed: %s\n", ups_strerror(st));
    exit(-1);
  }

  ups_record_t record = {0};
  uqi_result_get_record(result, 0, &record);
  if (*(uint32_t *)record.data != kNumberOfKeys - 1) {
    printf("uqi_select unexpected result %u\n", *(uint32_t *)record.data);
    //exit(-1);
  }

  uqi_result_close(result);

  uint64_t elapsed = timer.elapsed();

  st = ups_env_close(env, UPS_AUTO_CLEANUP);
  if (st) {
    printf("ups_env_close failed: %s\n", ups_strerror(st));
    exit(-1);
  }

  return elapsed;
}


static uint64_t
bdb_write_int32(const ivec &kvec, const ivec &rvec)
{
  DB *db;

  int st = db_create(&db, 0, 0);
  if (st) {
    printf("db_create failed: %d\n", st);
    exit(-1);
  }

  st = db->set_cachesize(db, 0, kCachesize, 1);
  if (st) {
    printf("db->set_cachesize failed: %d\n", st);
    exit(-1);
  }

  st = db->set_bt_compare(db, compare_uint32);
  if (st) {
    printf("db->set_bt_compare failed: %d\n", st);
    exit(-1);
  }

  unlink("benchmark_int32.bdb");
  st = db->open(db, 0, "benchmark_int32.bdb", 0, DB_BTREE, DB_CREATE, 0644);
  if (st) {
    printf("db->open failed: %d\n", st);
    exit(-1);
  }

  WallClockTimer timer;

  ivec::const_iterator kit = kvec.begin();
  ivec::const_iterator rit = rvec.begin();
  for (; kit != kvec.end(); kit++, rit++) {
    uint32_t kv = *kit;
    DBT key, rec;
    memset(&key, 0, sizeof(key));
    memset(&rec, 0, sizeof(rec));
    key.data = &kv;
    key.size = sizeof(kv);
    uint32_t rv = *rit;
    rec.data = &rv;
    rec.size = sizeof(rv);

    st = db->put(db, 0, &key, &rec, 0);
    if (st) {
      printf("db->put failed: %d\n", st);
      exit(-1);
    }
  }

  uint64_t elapsed = timer.elapsed();

  st = db->close(db, 0);
  if (st) {
    printf("db->close failed: %d\n", st);
    exit(-1);
  }

  return elapsed;
}

static uint64_t
bdb_read_int32(const ivec &kvec, const ivec &rvec)
{
  DB *db;

  int st = db_create(&db, 0, 0);
  if (st) {
    printf("db_create failed: %d\n", st);
    exit(-1);
  }

  st = db->set_cachesize(db, 0, kCachesize, 1);
  if (st) {
    printf("db->set_cachesize failed: %d\n", st);
    exit(-1);
  }

  st = db->set_bt_compare(db, compare_uint32);
  if (st) {
    printf("db->set_bt_compare failed: %d\n", st);
    exit(-1);
  }

  st = db->open(db, 0, "benchmark_int32.bdb", 0, DB_BTREE, 0, 0);
  if (st) {
    printf("db->open failed: %d\n", st);
    exit(-1);
  }

  WallClockTimer timer;

  ivec::const_iterator kit = kvec.begin();
  ivec::const_iterator rit = rvec.begin();
  for (; kit != kvec.end(); kit++, rit++) {
    uint32_t kv = *kit;
    DBT key, rec;
    memset(&key, 0, sizeof(key));
    memset(&rec, 0, sizeof(rec));
    key.data = &kv;
    key.size = sizeof(kv);
    uint32_t rv = *rit;

    st = db->get(db, 0, &key, &rec, 0);
    if (st) {
      printf("db->get failed: %d\n", st);
      exit(-1);
    }
    if (rv != *(uint32_t *)rec.data) {
      printf("db->get returned bad value\n");
      exit(-1);
    }
  }

  uint64_t elapsed = timer.elapsed();

  st = db->close(db, 0);
  if (st) {
    printf("db->close failed: %d\n", st);
    exit(-1);
  }

  return elapsed;
}

static uint64_t
bdb_max_int32()
{
  DB *db;
  DBC *dbc;

  int st = db_create(&db, 0, 0);
  if (st) {
    printf("db_create failed: %d\n", st);
    exit(-1);
  }

  st = db->set_cachesize(db, 0, kCachesize, 1);
  if (st) {
    printf("db->set_cachesize failed: %d\n", st);
    exit(-1);
  }

  st = db->set_bt_compare(db, compare_uint32);
  if (st) {
    printf("db->set_bt_compare failed: %d\n", st);
    exit(-1);
  }

  st = db->open(db, 0, "benchmark_int32.bdb", 0, DB_BTREE, 0, 0);
  if (st) {
    printf("db->open failed: %d\n", st);
    exit(-1);
  }

  st = db->cursor(db, 0, &dbc, 0);
  if (st) {
    printf("db->cursor failed: %d\n", st);
    exit(-1);
  }

  WallClockTimer timer;

  uint64_t max = 0;
  while (true) {
    DBT key, rec;
    memset(&key, 0, sizeof(key));
    memset(&rec, 0, sizeof(rec));

    st = dbc->c_get(dbc, &key, &rec, DB_NEXT);
    if (st)
      break;

    uint32_t v = *(uint32_t *)rec.data;
    if (v > max)
      max = v;
  }

  if (max != kNumberOfKeys - 1) {
    printf("bdb_max unexpected result %lu\n", max);
    //exit(-1);
  }

  uint64_t elapsed = timer.elapsed();

  st = db->close(db, 0);
  if (st) {
    printf("db->close failed: %d\n", st);
    exit(-1);
  }

  return elapsed;
}


static uint64_t
lev_write_int32(const ivec &kvec, const ivec &rvec)
{
  leveldb::DB *db;
  leveldb::Options options;
  options.create_if_missing = true;
  options.block_cache = leveldb::NewLRUCache(kCachesize);
  options.compression = leveldb::kNoCompression;

  system("rm -rf benchmark_int32.ldb");

  leveldb::Status st = leveldb::DB::Open(options, "benchmark_int32.ldb", &db);
  if (!st.ok()) {
    printf("leveldb::DB::Open failed: %s\n", st.ToString().c_str());
    exit(-1);
  }
    
  WallClockTimer timer;

  leveldb::WriteOptions writeOptions;

  ivec::const_iterator kit = kvec.begin();
  ivec::const_iterator rit = rvec.begin();
  for (; kit != kvec.end(); kit++, rit++) {
    uint32_t kv = *kit;
    uint32_t rv = *rit;

    st = db->Put(writeOptions,
                leveldb::Slice((const char *)&kv, sizeof(kv)),
                leveldb::Slice((const char *)&rv, sizeof(rv)));
    if (!st.ok()) {
      printf("db->Put failed: %s\n", st.ToString().c_str());
      exit(-1);
    }
  }

  uint64_t elapsed = timer.elapsed();

  // Compact the file to get an exact file size
  db->CompactRange(NULL, NULL);

  delete db;
  delete options.block_cache;

  return elapsed;
}

static uint64_t
lev_read_int32(const ivec &kvec, const ivec &rvec)
{
  leveldb::DB *db;
  leveldb::Options options;
  options.create_if_missing = false;
  options.block_cache = leveldb::NewLRUCache(kCachesize);
  options.compression = leveldb::kNoCompression;

  leveldb::Status st = leveldb::DB::Open(options, "benchmark_int32.ldb", &db);
  if (!st.ok()) {
    printf("leveldb::DB::Open failed: %s\n", st.ToString().c_str());
    exit(-1);
  }
    
  WallClockTimer timer;

  leveldb::ReadOptions readOptions;

  ivec::const_iterator kit = kvec.begin();
  ivec::const_iterator rit = rvec.begin();
  for (; kit != kvec.end(); kit++, rit++) {
    uint32_t kv = *kit;
    uint32_t rv = *rit;

    std::string value;
    st = db->Get(readOptions,
                leveldb::Slice((const char *)&kv, sizeof(kv)),
                &value);
    if (!st.ok()) {
      printf("db->Get failed: %s\n", st.ToString().c_str());
      exit(-1);
    }
    if (*(uint32_t *)value.data() != rv) {
      printf("db->Get returned bad value\n");
      exit(-1);
    }
  }

  uint64_t elapsed = timer.elapsed();

  delete db;
  delete options.block_cache;

  return elapsed;
}

static uint64_t
lev_max_int32()
{
  leveldb::DB *db;
  leveldb::Options options;
  options.create_if_missing = false;
  options.block_cache = leveldb::NewLRUCache(kCachesize);
  options.compression = leveldb::kNoCompression;

  leveldb::Status st = leveldb::DB::Open(options, "benchmark_int32.ldb", &db);
  if (!st.ok()) {
    printf("leveldb::DB::Open failed: %s\n", st.ToString().c_str());
    exit(-1);
  }
    
  leveldb::Iterator* it = db->NewIterator(leveldb::ReadOptions());

  WallClockTimer timer;

  uint64_t max = 0;
  for (it->SeekToFirst(); it->Valid(); it->Next()) {
    uint32_t v = *(uint32_t *)it->value().data();
    if (v > max)
      max = v;
  }

  if (max != kNumberOfKeys - 1) {
    printf("ldb_max unexpected result: %lu\n", max);
    //exit(-1);
  }

  uint64_t elapsed = timer.elapsed();

  delete it;
  delete db;
  delete options.block_cache;

  return elapsed;
}


int
main()
{
  uint64_t elapsed;

  std::vector<uint32_t> uint32;
  for (uint32_t i = 0; i < kNumberOfKeys; i++)
    uint32.push_back(i);

  std::vector<uint32_t> randuint32 = uint32;
  std::random_shuffle(randuint32.begin(), randuint32.end());

  printf("upscaledb write (%u keys)\n", kNumberOfKeys);
  for (int r = 0; r < kRunsPerTest; r++) {
    elapsed = ups_write_int32(uint32, randuint32);
    printf("  %d: %lu microsec\n", r + 1, elapsed);
  }

  printf("upscaledb read\n");
  for (int r = 0; r < kRunsPerTest; r++) {
    elapsed = ups_read_int32(uint32, randuint32);
    printf("  %d: %lu microsec\n", r + 1, elapsed);
  }

  printf("upscaledb max\n");
  for (int r = 0; r < kRunsPerTest; r++) {
    elapsed = ups_max_int32();
    printf("  %d: %lu microsec\n", r + 1, elapsed);
  }

  printf("berkeleydb write (%u keys)\n", kNumberOfKeys);
  for (int r = 0; r < kRunsPerTest; r++) {
    elapsed = bdb_write_int32(uint32, randuint32);
    printf("  %d: %lu microsec\n", r + 1, elapsed);
  }

  printf("berkeleydb read\n");
  for (int r = 0; r < kRunsPerTest; r++) {
    elapsed = bdb_read_int32(uint32, randuint32);
    printf("  %d: %lu microsec\n", r + 1, elapsed);
  }

  printf("berkeleydb max\n");
  for (int r = 0; r < kRunsPerTest; r++) {
    elapsed = bdb_max_int32();
    printf("  %d: %lu microsec\n", r + 1, elapsed);
  }

  printf("leveldb write (%u keys)\n", kNumberOfKeys);
  for (int r = 0; r < kRunsPerTest; r++) {
    elapsed = lev_write_int32(uint32, randuint32);
    printf("  %d: %lu microsec\n", r + 1, elapsed);
  }

  printf("leveldb read\n");
  for (int r = 0; r < kRunsPerTest; r++) {
    elapsed = lev_read_int32(uint32, randuint32);
    printf("  %d: %lu microsec\n", r + 1, elapsed);
  }

  printf("leveldb max\n");
  for (int r = 0; r < kRunsPerTest; r++) {
    elapsed = lev_max_int32();
    printf("  %d: %lu microsec\n", r + 1, elapsed);
  }

  return 0;
}

Makefile

UPSCALEDB=/home/ruppc/prj/upscaledb
INCDIR=-I$(UPSCALEDB)/include
LIBDIR=-L$(UPSCALEDB)/src/.libs
LIBS=-lleveldb -ldb -lboost_system -lboost_thread -lcrypto -lpthread -lsnappy -lz -ltcmalloc_minimal -lprotobuf -ldl
CC=g++ -std=c++11 -g -O0

all:
	$(CC) benchmark.cc -o benchmark \
        $(UPSCALEDB)/src/.libs/libupscaledb.a  \
        $(INCDIR) $(LIBDIR) $(LIBS)

clean:
	rm -rf *.o benchmark *.hdb *.bdb
	rm -rf *.ldb

timer.h

#ifndef TIMER_H
#define TIMER_H

#include <stdint.h>
#include <chrono>

struct WallClockTimer {
  typedef std::chrono::high_resolution_clock clock;

  std::chrono::time_point<clock> t1;

  WallClockTimer() {
    reset();
  }

  void reset() {
    t1 = clock::now();
  }

  uint64_t elapsed() {
    std::chrono::microseconds delta = std::chrono::duration_cast<
                        std::chrono::microseconds>(clock::now() - t1);
    return delta.count();
  }
};

#endif /* TIMER_H */