db-bench-plot.sh

#!/bin/sh

gnuplot <<EOF
  reset
  set terminal pngcairo truecolor enhanced font "Ubuntu Mono,13" size 1920, 1080
  set output "/tmp/5mbench.png"
  set datafile separator ','

  set logscale y
  set nologscale y2
  unset log y2
  set autoscale y
  set autoscale y2
  set ytics nomirror
  set y2tics
  set tics out

  set xlabel "Minutes" tc rgb "#777777"
  set ylabel "Milliseconds per write" tc rgb "#777777"
  set y2label "Throughput MB/s" tc rgb "#777777"

  set title "Node.js LevelDB (LevelDOWN): 100,000,000 random writes, 64M write buffer, HDD RAID1" tc rgb "#777777"
  set key left tc rgb "#777777"
  set border lc rgb "#777777"

  set style line 1 lt 7 ps 1.2 lc rgb "#55019FD7"
  set style line 2 lt 7 ps 0.1 lc rgb "#55019FD7"
  set style line 3 lt 1 lw 2   lc rgb "#55019FD7"

  set style line 4 lt 7 ps 1.2 lc rgb "#559ECC3C"
  set style line 5 lt 7 ps 0.1 lc rgb "#559ECC3C"
  set style line 6 lt 1 lw 2   lc rgb "#559ECC3C"

  set style line 7 lt 7 ps 1.2 lc rgb "#55CC3C3C"
  set style line 8 lt 7 ps 0.1 lc rgb "#55CC3C3C"
  set style line 9 lt 1 lw 2   lc rgb "#55CC3C3C"

  set style line 10 lt 7 ps 1.2 lc rgb "#553C3C3C"
  set style line 11 lt 7 ps 0.1 lc rgb "#553C3C3C"
  set style line 12 lt 1 lw 2   lc rgb "#553C3C3C"

  plot \
      1/0 with points title "Google LevelDB (master)" ls 1 \
    , 1/0 with points title "Google LevelDB (napi)"  ls 4 \
    , "5m_google_master.csv" using (\$1/1000/60):(\$4/1000000) notitle         ls 2 axes x1y1 \
    , "5m_google_napi.csv"  using (\$1/1000/60):(\$4/1000000) notitle         ls 5 axes x1y1 \
    , "5m_google_master.csv" using (\$1/1000/60):(\$5)         w lines notitle ls 3 axes x1y2 \
    , "5m_google_napi.csv"  using (\$1/1000/60):(\$5)         w lines notitle ls 6 axes x1y2 \

EOF

db-bench.js

#!/usr/bin/env node

const leveldown = require('../')
const fs = require('fs')
const du = require('du')
const path = require('path')
const argv = require('optimist').argv

const options = {
  benchmark: argv.benchmark,
  useExisting: argv.use_existing,
  db: argv.db || path.join(__dirname, 'db'),
  num: argv.num || 1000000,
  concurrency: argv.concurrency || 4,
  cacheSize: argv.cacheSize || 8,
  writeBufferSize: argv.writeBufferSize || 4,
  valueSize: argv.valueSize || 100,
  timingOutput: argv.timingOutput || path.join(__dirname, 'timingOutput'),
  throughputOutput: argv.throughputOutput
}

const randomString = require('slump').string
const keyTmpl = '0000000000000000'

if (!options.useExisting) {
  leveldown.destroy(options.db, function () {})
}

const db = leveldown(options.db)
const timesStream = fs.createWriteStream(options.timingOutput, 'utf8')

function make16CharPaddedKey () {
  const r = Math.floor(Math.random() * options.num)
  const k = keyTmpl + r
  return k.substr(k.length - 16)
}

timesStream.write('Elapsed (ms), Entries, Bytes, Last 1000 Avg Time, MB/s\n')

function start () {
  let inProgress = 0
  let totalWrites = 0
  let totalBytes = 0
  const startTime = Date.now()
  let timesAccum = 0
  let elapsed

  function report () {
    console.log(
      'Wrote', options.num, 'entries in',
      Math.floor((Date.now() - startTime) / 1000) + 's,',
      (Math.floor((totalBytes / 1048576) * 100) / 100) + 'MB'
    )
    timesStream.end()

    du(options.db, function (err, size) {
      if (err) throw err
      console.log('Database size:', Math.floor(size / 1024 / 1024) + 'M')
    })
  }

  function write () {
    if (totalWrites++ === options.num) {
      db.close(function () {
        report(Date.now() - startTime)
      })
    }

    if (inProgress >= options.concurrency || totalWrites > options.num) return

    inProgress++

    if (totalWrites % 100000 === 0) {
      console.log('' + inProgress, totalWrites,
        Math.round(totalWrites / options.num * 100) + '%')
    }

    if (totalWrites % 1000 === 0) {
      elapsed = Date.now() - startTime
      timesStream.write(
        elapsed +
        ',' + totalWrites +
        ',' + totalBytes +
        ',' + Math.floor(timesAccum / 1000) +
        ',' + (Math.floor(((totalBytes / 1048576) / (elapsed / 1000)) * 100) / 100) +
        '\n')
      timesAccum = 0
    }

    var key make16CharPaddedKey()
    var value = randomString({ length: options.valueSize })
    var time = process.hrtime()

    db.put(key, value, function (err) {
      if (err) throw err

      timesAccum += process.hrtime(time)[1]
      totalBytes += key.length + value.length
      inProgress--
      process.nextTick(write)
    })
  }

  for (var i = 0; i < options.concurrency; i++) write()
}

setTimeout(function () {
  db.open({
    errorIfExists: false,
    createIfMissing: true,
    cacheSize: options.cacheSize << 20,
    writeBufferSize: options.writeBufferSize << 20
  }, function (err) {
    if (err) throw err
    start()
  })
}, 500)