LMDB Put Performance Results

lmdb_put_perf.md

Batch size = 400

Keys are sorted.

The results are below.

10s_lmdb_put_large_8MB.txt

$ make write-large-hurry
go test -bench PutLarge -benchtime 10s
testing: warning: no tests to run
PASS
BenchmarkPutLarge_LMDB-4	write_test.go:51: bench put LARGE (8MB), take 1 (n=1)
write_test.go:96: last put took: 18.978851ms
write_test.go:97: last key: 00000000152fdfc07218265
write_test.go:147: depth: 1, branch pg: 0, leaf pg: 1, entries: 1
write_test.go:153: gc: 184.289µs
write_test.go:156: mem: heap 8MB/8MB, alloc total 24MB
write_test.go:48: take 1 (n=1) done in 20.869385ms

write_test.go:51: bench put LARGE (8MB), take 2 (n=100)
write_test.go:96: last put took: 1.559197151s
write_test.go:97: last key: 00000010179ea3dfe4136ab
write_test.go:147: depth: 2, branch pg: 1, leaf pg: 2, entries: 100
write_test.go:153: gc: 171.23µs
write_test.go:156: mem: heap 8MB/8MB, alloc total 24MB
write_test.go:48: take 2 (n=100) done in 2.127909016s

write_test.go:51: bench put LARGE (8MB), take 3 (n=1000)
write_test.go:96: last put took: 3.155108132s
write_test.go:97: last key: 000001101e1ff3733982c8c
write_test.go:147: depth: 2, branch pg: 1, leaf pg: 11, entries: 1000
write_test.go:153: gc: 544.892µs
write_test.go:156: mem: heap 8MB/8MB, alloc total 24MB
write_test.go:48: take 3 (n=1000) done in 22.81609776s

    1000	  22815616 ns/op	 367.67 MB/s	     160 B/op	       7 allocs/op
ok  	github.com/zenhotels/lmdb-go/lmdb/bench	25.010s

10s_lmdb_put_small_8KB.txt

$ make write-hurry
go test -bench PutSmall -benchtime 10s
testing: warning: no tests to run
PASS
BenchmarkPutSmall_LMDB-4	write_test.go:38: bench put SMALL (8KB), take 1 (n=1)
write_test.go:96: last put took: 301.578µs
write_test.go:97: last key: 00000000152fdfc07218265
write_test.go:147: depth: 1, branch pg: 0, leaf pg: 1, entries: 1
write_test.go:153: gc: 143.364µs
write_test.go:156: mem: heap 8MB/8MB, alloc total 24MB
write_test.go:35: take 1 (n=1) done in 1.508596ms

write_test.go:38: bench put SMALL (8KB), take 2 (n=100)
write_test.go:96: last put took: 2.287069ms
write_test.go:97: last key: 00000010179ea3dfe4136ab
write_test.go:147: depth: 2, branch pg: 1, leaf pg: 2, entries: 100
write_test.go:153: gc: 194.344µs
write_test.go:156: mem: heap 8MB/8MB, alloc total 24MB
write_test.go:35: take 2 (n=100) done in 5.483406ms

write_test.go:38: bench put SMALL (8KB), take 3 (n=10000)
write_test.go:96: last put took: 8.260064ms
write_test.go:97: last key: 0000101010ecd732fac1b32
write_test.go:147: depth: 2, branch pg: 1, leaf pg: 105, entries: 10000
write_test.go:153: gc: 382.646µs
write_test.go:156: mem: heap 8MB/8MB, alloc total 25MB
write_test.go:35: take 3 (n=10000) done in 324.691326ms

write_test.go:38: bench put SMALL (8KB), take 4 (n=500000)
write_test.go:96: last put took: 53.81874ms
write_test.go:97: last key: 0005101013d76ffb6040ec8
write_test.go:147: depth: 3, branch pg: 45, leaf pg: 5209, entries: 500000
write_test.go:153: gc: 2.083416ms
write_test.go:156: mem: heap 8MB/8MB, alloc total 98MB
write_test.go:35: take 4 (n=500000) done in 28.616411957s

  500000	     57231 ns/op	 143.14 MB/s	     152 B/op	       7 allocs/op
ok  	github.com/zenhotels/lmdb-go/lmdb/bench	28.977s

20s_lmdb_put_large_8MB.txt

$ make write-large
go test -bench PutLarge -benchtime 20s
testing: warning: no tests to run
PASS
BenchmarkPutLarge_LMDB-4	write_test.go:51: bench put LARGE (8MB), take 1 (n=1)
write_test.go:96: last put took: 19.75205ms
write_test.go:97: last key: 00000000152fdfc07218265
write_test.go:147: depth: 1, branch pg: 0, leaf pg: 1, entries: 1
write_test.go:153: gc: 187.799µs
write_test.go:156: mem: heap 8MB/8MB, alloc total 24MB
write_test.go:48: take 1 (n=1) done in 21.552316ms

write_test.go:51: bench put LARGE (8MB), take 2 (n=100)
write_test.go:96: last put took: 1.778199557s
write_test.go:97: last key: 00000010179ea3dfe4136ab
write_test.go:147: depth: 2, branch pg: 1, leaf pg: 2, entries: 100
write_test.go:153: gc: 182.877µs
write_test.go:156: mem: heap 8MB/8MB, alloc total 24MB
write_test.go:48: take 2 (n=100) done in 2.327995135s

write_test.go:51: bench put LARGE (8MB), take 3 (n=2000)
write_test.go:96: last put took: 7.498721507s
write_test.go:97: last key: 000002101e53a42c34a1968
write_test.go:147: depth: 2, branch pg: 1, leaf pg: 21, entries: 2000
write_test.go:153: gc: 586.254µs
write_test.go:156: mem: heap 8MB/8MB, alloc total 24MB
write_test.go:48: take 3 (n=2000) done in 46.341166336s

    2000	  23170289 ns/op	 362.04 MB/s	     156 B/op	       7 allocs/op
ok  	github.com/zenhotels/lmdb-go/lmdb/bench	48.730s

20s_lmdb_put_small_8KB.txt

$ make write
go test -bench PutSmall -benchtime 20s
testing: warning: no tests to run
PASS
BenchmarkPutSmall_LMDB-4	write_test.go:38: bench put SMALL (8KB), take 1 (n=1)
write_test.go:96: last put took: 301.775µs
write_test.go:97: last key: 00000000152fdfc07218265
write_test.go:147: depth: 1, branch pg: 0, leaf pg: 1, entries: 1
write_test.go:153: gc: 146.24µs
write_test.go:156: mem: heap 8MB/8MB, alloc total 24MB
write_test.go:35: take 1 (n=1) done in 1.47649ms

write_test.go:38: bench put SMALL (8KB), take 2 (n=100)
write_test.go:96: last put took: 2.356666ms
write_test.go:97: last key: 00000010179ea3dfe4136ab
write_test.go:147: depth: 2, branch pg: 1, leaf pg: 2, entries: 100
write_test.go:153: gc: 193.454µs
write_test.go:156: mem: heap 8MB/8MB, alloc total 24MB
write_test.go:35: take 2 (n=100) done in 5.527084ms

write_test.go:38: bench put SMALL (8KB), take 3 (n=10000)
write_test.go:96: last put took: 13.846333ms
write_test.go:97: last key: 0000101010ecd732fac1b32
write_test.go:147: depth: 2, branch pg: 1, leaf pg: 105, entries: 10000
write_test.go:153: gc: 599.585µs
write_test.go:156: mem: heap 8MB/8MB, alloc total 25MB
write_test.go:35: take 3 (n=10000) done in 268.923477ms

write_test.go:38: bench put SMALL (8KB), take 4 (n=1000000)
write_test.go:96: last put took: 47.42336ms
write_test.go:97: last key: 001010101a4db5511fb13ea
write_test.go:147: depth: 3, branch pg: 89, leaf pg: 10417, entries: 1000000
write_test.go:153: gc: 2.834288ms
write_test.go:156: mem: heap 8MB/8MB, alloc total 170MB
write_test.go:35: take 4 (n=1000000) done in 1m22.323152958s

 1000000	     82322 ns/op	  99.51 MB/s	     152 B/op	       7 allocs/op
ok  	github.com/zenhotels/lmdb-go/lmdb/bench	82.640s

5s_lmdb_put_large_8MB.txt

$ make write-large-quick
go test -bench PutLarge -benchtime 5s
testing: warning: no tests to run
PASS
BenchmarkPutLarge_LMDB-4	write_test.go:51: bench put LARGE (8MB), take 1 (n=1)
write_test.go:96: last put took: 18.986991ms
write_test.go:97: last key: 00000000152fdfc07218265
write_test.go:147: depth: 1, branch pg: 0, leaf pg: 1, entries: 1
write_test.go:153: gc: 195.429µs
write_test.go:156: mem: heap 8MB/8MB, alloc total 24MB
write_test.go:48: take 1 (n=1) done in 20.79676ms

write_test.go:51: bench put LARGE (8MB), take 2 (n=100)
write_test.go:96: last put took: 1.595430655s
write_test.go:97: last key: 00000010179ea3dfe4136ab
write_test.go:147: depth: 2, branch pg: 1, leaf pg: 2, entries: 100
write_test.go:153: gc: 210.741µs
write_test.go:156: mem: heap 8MB/8MB, alloc total 24MB
write_test.go:48: take 2 (n=100) done in 2.113833661s

write_test.go:51: bench put LARGE (8MB), take 3 (n=300)
write_test.go:96: last put took: 4.540079777s
write_test.go:97: last key: 0000004013f64434abae060
write_test.go:147: depth: 2, branch pg: 1, leaf pg: 4, entries: 300
write_test.go:153: gc: 220.927µs
write_test.go:156: mem: heap 8MB/8MB, alloc total 24MB
write_test.go:48: take 3 (n=300) done in 6.136362697s

     300	  20453282 ns/op	 410.14 MB/s	     180 B/op	       7 allocs/op
ok  	github.com/zenhotels/lmdb-go/lmdb/bench	8.314s

5s_lmdb_put_small_8KB.txt

$ make write-quick
go test -bench PutSmall -benchtime 5s
testing: warning: no tests to run
PASS
BenchmarkPutSmall_LMDB-4	write_test.go:38: bench put SMALL (8KB), take 1 (n=1)
write_test.go:96: last put took: 304.234µs
write_test.go:97: last key: 00000000152fdfc07218265
write_test.go:147: depth: 1, branch pg: 0, leaf pg: 1, entries: 1
write_test.go:153: gc: 145.611µs
write_test.go:156: mem: heap 8MB/8MB, alloc total 24MB
write_test.go:35: take 1 (n=1) done in 1.455508ms

write_test.go:38: bench put SMALL (8KB), take 2 (n=100)
write_test.go:96: last put took: 2.346986ms
write_test.go:97: last key: 00000010179ea3dfe4136ab
write_test.go:147: depth: 2, branch pg: 1, leaf pg: 2, entries: 100
write_test.go:153: gc: 178.743µs
write_test.go:156: mem: heap 8MB/8MB, alloc total 24MB
write_test.go:35: take 2 (n=100) done in 5.730579ms

write_test.go:38: bench put SMALL (8KB), take 3 (n=10000)
write_test.go:96: last put took: 8.728329ms
write_test.go:97: last key: 0000101010ecd732fac1b32
write_test.go:147: depth: 2, branch pg: 1, leaf pg: 105, entries: 10000
write_test.go:153: gc: 396.966µs
write_test.go:156: mem: heap 8MB/8MB, alloc total 25MB
write_test.go:35: take 3 (n=10000) done in 267.64682ms

write_test.go:38: bench put SMALL (8KB), take 4 (n=300000)
write_test.go:96: last put took: 8.511175ms
write_test.go:97: last key: 00031010104adb724664ce4
write_test.go:147: depth: 3, branch pg: 28, leaf pg: 3125, entries: 300000
write_test.go:153: gc: 2.118246ms
write_test.go:156: mem: heap 8MB/8MB, alloc total 69MB
write_test.go:35: take 4 (n=300000) done in 8.280420858s

  300000	     27599 ns/op	 296.82 MB/s	     152 B/op	       7 allocs/op
ok  	github.com/zenhotels/lmdb-go/lmdb/bench	8.581s