- 4 KB random reads and writes with one fio instance and two job number
- single OSD instance
- deployed on purley server by vstart.sh
- crimson-seastore vs ceph-bluestore
write: 24,493,276,912 / 19073 = 1,284,185
>>>
read: 189,978,771,710 / 1655133 = 114,781
>>>
write: 839,463,751,717 / 73053 = 11,491,160
>>>
read: 844,600,463,654 / 1507616 = 560,222
>>>
write: 25161.73 (us)
read: 288.84 (us)
write: 6569.57 (us)
read: 317.26 (us)
./do_cmake.sh -DWITH_SEASTAR=ON -DWITH_MGR_DASHBOARD_FRONTEND=OFF -DCMAKE_BUILD_TYPE=RelWithDebInfo
[build]$ MDS=0 MGR=1 OSD=1 MON=1 ../src/vstart.sh -n --without-dashboard --seastore -X --crimson --seastore-devs /dev/nvme1n1
[build]$ bin/ceph osd pool create rbd 128 128 && bin/ceph osd pool set --yes-i-really-mean-it rbd size 1 && bin/ceph osd pool --yes-i-really-mean-it set rbd min_size 1
[build]$ bin/rbd create fio_test --size 2G --image-format=2 --rbd_default_features=3
...
[build]$ perf stat -p `pgrep -u ${UID} crimson-osd` & fio ../rbd_write.fio; sleep 1; killall -INT perf
[1] 57309
rbd_iodepth32: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 409 6B-4096B, ioengine=rbd, iodepth=2
fio-3.26-46-g30be
Starting 1 process
Jobs: 1 (f=1): [w(1)][100.0%][w=552KiB/s][w=138 IOPS][eta 00m:00s]
rbd_iodepth32: (groupid=0, jobs=1): err= 0: pid=57401: Fri Jul 2 15:56:19 2021
write: IOPS=304, BW=1217KiB/s (1246kB/s)(285MiB/240061msec); 0 zone resets
slat (nsec): min=2387, max=68398, avg=9642.74, stdev=5437.33
clat (usec): min=235, max=1386.2k, avg=6559.92, stdev=22222.70
lat (usec): min=242, max=1386.2k, avg=6569.57, stdev=22222.81
clat percentiles (usec):
| 1.00th=[ 265], 5.00th=[ 277], 10.00th=[ 285], 20.00th=[ 302],
| 30.00th=[ 318], 40.00th=[ 351], 50.00th=[ 416], 60.00th=[ 469],
| 70.00th=[ 529], 80.00th=[ 725], 90.00th=[ 19006], 95.00th=[ 50594],
| 99.00th=[105382], 99.50th=[130548], 99.90th=[179307], 99.95th=[193987],
| 99.99th=[283116]
bw ( KiB/s): min= 88, max=15200, per=100.00%, avg=1224.82, stdev=1578.16, samples=477
iops : min= 22, max= 3800, avg=306.16, stdev=394.55, samples=477
lat (usec) : 250=0.02%, 500=66.05%, 750=14.43%, 1000=2.10%
lat (msec) : 2=1.96%, 4=1.95%, 10=1.90%, 20=1.75%, 50=4.75%
lat (msec) : 100=3.97%, 250=1.11%, 500=0.01%, 750=0.01%, 2000=0.01%
cpu : usr=0.49%, sys=0.38%, ctx=72981, majf=0, minf=702
IO depths : 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,73053,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=2
Run status group 0 (all jobs):
WRITE: bw=1217KiB/s (1246kB/s), 1217KiB/s-1217KiB/s (1246kB/s-1246kB/s), io=28 5MiB (299MB), run=240061-240061msec
Disk stats (read/write):
dm-0: ios=0/4062, merge=0/0, ticks=0/4720, in_queue=4720, util=1.15%, aggrio s=3/2789, aggrmerge=0/1880, aggrticks=0/2768, aggrin_queue=2764, aggrutil=1.15%
sda: ios=3/2789, merge=0/1880, ticks=0/2768, in_queue=2764, util=1.15%
Performance counter stats for process id '57077':
236135.485984 task-clock (msec) # 0.979 CPUs utilized
797,682 context-switches # 0.003 M/sec
0 cpu-migrations # 0.000 K/sec
3,493 page-faults # 0.015 K/sec
839,463,751,717 cycles # 3.555 GHz
620,920,122,183 instructions # 0.74 insn per cycle
125,743,849,074 branches # 532.507 M/sec
1,529,786,548 branch-misses # 1.22% of all branches
241.162522168 seconds time elapsed
...
[build]$ perf stat -p `pgrep -u ${UID} crimson-osd` & fio ../rbd_read.fio; sleep 1; killall -INT perf
[1] 57408
rbd_iodepth32: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=rbd, iodepth=2
fio-3.26-46-g30be
Starting 1 process
Jobs: 1 (f=1): [r(1)][100.0%][r=24.2MiB/s][r=6189 IOPS][eta 00m:00s]
rbd_iodepth32: (groupid=0, jobs=1): err= 0: pid=57500: Fri Jul 2 16:01:19 2021
read: IOPS=6280, BW=24.5MiB/s (25.7MB/s)(5889MiB/240043msec)
slat (nsec): min=875, max=661664, avg=5432.42, stdev=1329.77
clat (usec): min=123, max=59979, avg=311.83, stdev=437.74
lat (usec): min=129, max=59984, avg=317.26, stdev=437.75
clat percentiles (usec):
| 1.00th=[ 157], 5.00th=[ 165], 10.00th=[ 178], 20.00th=[ 233],
| 30.00th=[ 243], 40.00th=[ 247], 50.00th=[ 251], 60.00th=[ 258],
| 70.00th=[ 269], 80.00th=[ 347], 90.00th=[ 424], 95.00th=[ 461],
| 99.00th=[ 914], 99.50th=[ 2704], 99.90th=[ 5211], 99.95th=[ 5342],
| 99.99th=[ 5604]
bw ( KiB/s): min=17832, max=28840, per=100.00%, avg=25157.30, stdev=1634.58, samples=479
iops : min= 4458, max= 7210, avg=6289.22, stdev=408.63, samples=479
lat (usec) : 250=49.89%, 500=46.39%, 750=2.65%, 1000=0.08%
lat (msec) : 2=0.08%, 4=0.48%, 10=0.43%, 20=0.01%, 50=0.01%
lat (msec) : 100=0.01%
cpu : usr=6.01%, sys=6.26%, ctx=1502947, majf=0, minf=735
IO depths : 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=1507616,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=2
Run status group 0 (all jobs):
READ: bw=24.5MiB/s (25.7MB/s), 24.5MiB/s-24.5MiB/s (25.7MB/s-25.7MB/s), io=5889MiB (6175MB), run=240043-240043msec
Disk stats (read/write):
dm-0: ios=1/4417, merge=0/0, ticks=4/6068, in_queue=6072, util=1.22%, aggrios=1/2947, aggrmerge=0/2098, aggrticks=4/4092, aggrin_queue=4092, aggrutil=1.22%
sda: ios=1/2947, merge=0/2098, ticks=4/4092, in_queue=4092, util=1.22%
root@otccldstore05:/home/chunmei/ceph/build#
Performance counter stats for process id '57077':
235910.639453 task-clock (msec) # 0.978 CPUs utilized
909,496 context-switches # 0.004 M/sec
0 cpu-migrations # 0.000 K/sec
1,860 page-faults # 0.008 K/sec
844,600,463,654 cycles # 3.580 GHz
591,109,251,207 instructions # 0.70 insn per cycle
119,452,898,576 branches # 506.348 M/sec
1,185,049,695 branch-misses # 0.99% of all branches
241.118039219 seconds time elapsed
...
ms_async_op_threads = 1
osd_op_num_threads_per_shard = 1
osd_op_num_shards =1
./do_cmake.sh -DWITH_MGR_DASHBOARD_FRONTEND=OFF -DCMAKE_BUILD_TYPE=RelWithDebInfo
[build]$ MDS=0 MGR=1 OSD=1 MON=1 ../src/vstart.sh -n --without-dashboard --bluestore -X --bluestore-devs /dev/nvme1n1
[build]$ bin/ceph osd pool create rbd 128 128 && bin/ceph osd pool set --yes-i-really-mean-it rbd size 1 && bin/ceph osd pool --yes-i-really-mean-it set rbd min_size 1
[build]$ bin/rbd create fio_test --size 2G --image-format=2 --rbd_default_features=3
[build]$ taskset -cp 0 <ceph_osd Pid>
...
[build]$ perf stat -p `pgrep -u ${UID} ceph-osd` & fio ../rbd_write.fio; sleep 1; killall -INT perf
[1] 5441
rbd_iodepth32: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=rbd, iodepth=2
fio-3.26-46-g30be
Starting 1 process
Jobs: 1 (f=1): [w(1)][100.0%][w=312KiB/s][w=78 IOPS][eta 00m:00s]
rbd_iodepth32: (groupid=0, jobs=1): err= 0: pid=5533: Fri Jul 2 21:20:03 2021
write: IOPS=79, BW=318KiB/s (325kB/s)(74.5MiB/240026msec); 0 zone resets
slat (nsec): min=8216, max=84288, avg=23516.17, stdev=3407.43
clat (usec): min=9430, max=57272, avg=25138.21, stdev=3054.29
lat (usec): min=9442, max=57295, avg=25161.73, stdev=3054.30
clat percentiles (usec):
| 1.00th=[17957], 5.00th=[17957], 10.00th=[18220], 20.00th=[25560],
| 30.00th=[25822], 40.00th=[25822], 50.00th=[25822], 60.00th=[25822],
| 70.00th=[25822], 80.00th=[26084], 90.00th=[26084], 95.00th=[26608],
| 99.00th=[33817], 99.50th=[33817], 99.90th=[35914], 99.95th=[42730],
| 99.99th=[56361]
bw ( KiB/s): min= 296, max= 360, per=99.73%, avg=318.00, stdev=12.79, samples=479
iops : min= 74, max= 90, avg=79.50, stdev= 3.20, samples=479
lat (msec) : 10=0.01%, 20=12.33%, 50=87.65%, 100=0.02%
cpu : usr=0.28%, sys=0.21%, ctx=19101, majf=0, minf=264
IO depths : 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,19073,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=2
Run status group 0 (all jobs):
WRITE: bw=318KiB/s (325kB/s), 318KiB/s-318KiB/s (325kB/s-325kB/s), io=74.5MiB (78.1MB), run=240026-240026msec
Disk stats (read/write):
dm-0: ios=63/226026, merge=0/0, ticks=272/297768, in_queue=298144, util=93.59%, aggrios=61/132749, aggrmerge=2/121114, aggrticks=260/227172, aggrin_queue=227352, aggrutil=93.30%
sda: ios=61/132749, merge=2/121114, ticks=260/227172, in_queue=227352, util=93.30%
root@otccldstore05:/home/chunmei/ceph/build#
Performance counter stats for process id '4797':
25636.833893 task-clock (msec) # 0.106 CPUs utilized
265,556 context-switches # 0.010 M/sec
38,693 cpu-migrations # 0.002 M/sec
102,927 page-faults # 0.004 M/sec
24,493,276,912 cycles # 0.955 GHz
9,754,591,520 instructions # 0.40 insn per cycle
1,925,325,487 branches # 75.100 M/sec
227,869,898 branch-misses # 11.84% of all branches
241.129338302 seconds time elapsed
...
[build]$ perf stat -p `pgrep -u ${UID} ceph-osd` & fio ../rbd_read.fio; sleep 1; killall -INT perf
[1] 5542
rbd_iodepth32: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=rbd, iodepth=2
fio-3.26-46-g30be
Starting 1 process
Jobs: 1 (f=1): [r(1)][100.0%][r=27.4MiB/s][r=7004 IOPS][eta 00m:00s]
rbd_iodepth32: (groupid=0, jobs=1): err= 0: pid=5634: Fri Jul 2 21:24:49 2021
read: IOPS=6896, BW=26.9MiB/s (28.2MB/s)(6465MiB/240001msec)
slat (nsec): min=1096, max=47766, avg=5959.94, stdev=950.37
clat (usec): min=188, max=8303, avg=282.88, stdev=48.56
lat (usec): min=194, max=8310, avg=288.84, stdev=48.60
clat percentiles (usec):
| 1.00th=[ 239], 5.00th=[ 258], 10.00th=[ 269], 20.00th=[ 273],
| 30.00th=[ 273], 40.00th=[ 277], 50.00th=[ 277], 60.00th=[ 281],
| 70.00th=[ 281], 80.00th=[ 285], 90.00th=[ 297], 95.00th=[ 306],
| 99.00th=[ 644], 99.50th=[ 668], 99.90th=[ 701], 99.95th=[ 725],
| 99.99th=[ 898]
bw ( KiB/s): min=11400, max=29992, per=100.00%, avg=27612.59, stdev=2610.06, samples=479
iops : min= 2850, max= 7498, avg=6903.02, stdev=652.50, samples=479
lat (usec) : 250=3.00%, 500=95.83%, 750=1.14%, 1000=0.02%
lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%
cpu : usr=6.84%, sys=7.59%, ctx=1616604, majf=0, minf=807
IO depths : 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=1655133,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=2
Run status group 0 (all jobs):
READ: bw=26.9MiB/s (28.2MB/s), 26.9MiB/s-26.9MiB/s (28.2MB/s-28.2MB/s), io=6465MiB (6779MB), run=240001-240001msec
Disk stats (read/write):
dm-0: ios=2/4057, merge=0/0, ticks=4/6184, in_queue=6204, util=1.25%, aggrios=2/2710, aggrmerge=0/1981, aggrticks=4/4116, aggrin_queue=4120, aggrutil=1.25%
sda: ios=2/2710, merge=0/1981, ticks=4/4116, in_queue=4120, util=1.25%
root@otccldstore05:/home/chunmei/ceph/build#
Performance counter stats for process id '4797':
197541.727415 task-clock (msec) # 0.819 CPUs utilized
2,890,451 context-switches # 0.015 M/sec
38,637 cpu-migrations # 0.196 K/sec
167,770 page-faults # 0.849 K/sec
189,978,771,710 cycles # 0.962 GHz
152,676,096,895 instructions # 0.80 insn per cycle
31,260,044,639 branches # 158.245 M/sec
447,794,313 branch-misses # 1.43% of all branches
241.101676967 seconds time elapsed
...
[global] ioengine=rbd clientname=admin pool=rbd rbdname=fio_test rw=randwrite bs=4K runtime=240 numjobs=1 direct=1 group_reporting
[rbd_iodepth32] iodepth=2 ...
[global] ioengine=rbd clientname=admin pool=rbd rbdname=fio_test rw=randread bs=4K time_based=1 runtime=240 group_reporting
[rbd_iodepth32] iodepth=2 numjobs=1