- 4 KB random reads and writes with one fio instance and two job number
- single OSD instance
- deployed on purley server by vstart.sh
- with minimal content stored in
memstore/cyanstore.
write: 135,556,125,646 /317739 = 426627.2810
>>>
read: 180,596,074,788 /1231875 = 146602.5974
>>>
write: 92,465,570,052 /304513 = 303650.6489
>>>
read: 100,894,866,318 /1517528 = 66486.3292
>>>
write: 12.09(ms)
read: 3117.05 (us)
write: 12627.25(us)
read: 2533.58(us)
./do_cmake.sh -DWITH_SEASTAR=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_CXX_FLAGS="-fno-omit-frame-pointer" -DWITH_TESTS=OFF
[build]$ MDS=0 MGR=0 OSD=1 MON=1 ../src/vstart.sh -n --without-dashboard --memstore -X -o "memstore_device_bytes=2516582400" --nolockdep --crimson --nodaemon --redirect-output
[build]$ bin/ceph osd pool create rbd 128 128 && bin/ceph osd pool set --yes-i-really-mean-it rbd size 1 && bin/ceph osd pool --yes-i-really-mean-it set rbd min_size 1
[build]$ bin/rbd create fio_test --size 2G --image-format=2 --rbd_default_features=3
...
[build]$ perf stat -p `pgrep -u ${UID} crimson-osd` & fio ../rbd_write.fio; sleep 1; killall -INT perf
[1] 38206
rbd_iodepth32: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=rbd, iodepth=64
...
fio-3.1
Starting 2 processes
Jobs: 2 (f=2): [w(2)][100.0%][r=0KiB/s,w=23.8MiB/s][r=0,w=6102 IOPS][eta 00m:00s]
rbd_iodepth32: (groupid=0, jobs=2): err= 0: pid=38319: Wed Feb 3 00:32:24 2021
write: IOPS=10.1k, BW=39.6MiB/s (41.5MB/s)(1190MiB/30044msec)
slat (nsec): min=842, max=114432, avg=5931.96, stdev=5059.90
clat (usec): min=1028, max=169210, avg=12621.32, stdev=12856.93
lat (usec): min=1034, max=169222, avg=12627.25, stdev=12857.12
clat percentiles (msec):
| 1.00th=[ 3], 5.00th=[ 3], 10.00th=[ 4], 20.00th=[ 5],
| 30.00th=[ 6], 40.00th=[ 7], 50.00th=[ 9], 60.00th=[ 11],
| 70.00th=[ 13], 80.00th=[ 16], 90.00th=[ 29], 95.00th=[ 48],
| 99.00th=[ 56], 99.50th=[ 59], 99.90th=[ 88], 99.95th=[ 99],
| 99.99th=[ 126]
bw ( KiB/s): min= 4745, max=76288, per=50.06%, avg=20294.77, stdev=10959.40, samples=120
iops : min= 1186, max=19072, avg=5073.65, stdev=2739.87, samples=120
lat (msec) : 2=0.18%, 4=14.95%, 10=44.01%, 20=28.57%, 50=8.97%
lat (msec) : 100=3.28%, 250=0.04%
cpu : usr=50.75%, sys=49.02%, ctx=29182, majf=0, minf=1221
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwt: total=0,304513,0, short=0,0,0, dropped=0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
WRITE: bw=39.6MiB/s (41.5MB/s), 39.6MiB/s-39.6MiB/s (41.5MB/s-41.5MB/s), io=1190MiB (1247MB), run=30044-30044msec
Disk stats (read/write):
dm-1: ios=0/115, merge=0/0, ticks=0/60, in_queue=60, util=0.19%, aggrios=0/71, aggrmerge=0/56, aggrticks=0/56, aggrin_queue=56, aggrutil=0.19%
sda: ios=0/71, merge=0/56, ticks=0/56, in_queue=56, util=0.19%
chunmei@otccldstore05:~/ceph/build$
Performance counter stats for process id '38096':
28052.782758 task-clock (msec) # 0.896 CPUs utilized
114,012 context-switches # 0.004 M/sec
0 cpu-migrations # 0.000 K/sec
237,692 page-faults # 0.008 M/sec
92,465,570,052 cycles # 3.296 GHz
101,483,088,881 instructions # 1.10 insn per cycle
20,126,489,495 branches # 717.451 M/sec
137,456,674 branch-misses # 0.68% of all branches
31.305689522 seconds time elapsed
...
[build]$ perf stat -p `pgrep -u ${UID} crimson-osd` & fio ../rbd_read.fio; sleep 1; killall -INT perf
[1] 38323
rbd_iodepth32: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=rbd, iodepth=64
...
fio-3.1
Starting 2 processes
Jobs: 2 (f=2): [r(2)][100.0%][r=201MiB/s,w=0KiB/s][r=51.4k,w=0 IOPS][eta 00m:00s]
rbd_iodepth32: (groupid=0, jobs=2): err= 0: pid=38436: Wed Feb 3 00:36:52 2021
read: IOPS=50.5k, BW=197MiB/s (207MB/s)(5928MiB/30044msec)
slat (nsec): min=196, max=807057, avg=2071.82, stdev=2064.28
clat (usec): min=455, max=46061, avg=2531.51, stdev=663.56
lat (usec): min=459, max=46061, avg=2533.58, stdev=663.56
clat percentiles (usec):
| 1.00th=[ 1532], 5.00th=[ 1844], 10.00th=[ 2024], 20.00th=[ 2278],
| 30.00th=[ 2409], 40.00th=[ 2442], 50.00th=[ 2507], 60.00th=[ 2540],
| 70.00th=[ 2606], 80.00th=[ 2704], 90.00th=[ 2999], 95.00th=[ 3326],
| 99.00th=[ 3916], 99.50th=[ 4621], 99.90th=[ 8455], 99.95th=[11207],
| 99.99th=[19268]
bw ( KiB/s): min=81088, max=116888, per=50.08%, avg=101175.18, stdev=3626.79, samples=120
iops : min=20272, max=29222, avg=25293.76, stdev=906.71, samples=120
lat (usec) : 500=0.01%, 750=0.01%, 1000=0.03%
lat (msec) : 2=9.16%, 4=89.93%, 10=0.81%, 20=0.06%, 50=0.01%
cpu : usr=53.70%, sys=46.11%, ctx=26674, majf=0, minf=462
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwt: total=1517528,0,0, short=0,0,0, dropped=0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=197MiB/s (207MB/s), 197MiB/s-197MiB/s (207MB/s-207MB/s), io=5928MiB (6216MB), run=30044-30044msec
Disk stats (read/write):
dm-1: ios=0/268, merge=0/0, ticks=0/256, in_queue=256, util=0.65%, aggrios=0/177, aggrmerge=0/127, aggrticks=0/192, aggrin_queue=192, aggrutil=0.64%
sda: ios=0/177, merge=0/127, ticks=0/192, in_queue=192, util=0.64%
chunmei@otccldstore05:~/ceph/build$
Performance counter stats for process id '38096':
29642.606495 task-clock (msec) # 0.947 CPUs utilized
120,225 context-switches # 0.004 M/sec
0 cpu-migrations # 0.000 K/sec
523 page-faults # 0.018 K/sec
100,894,866,318 cycles # 3.404 GHz
76,282,985,821 instructions # 0.76 insn per cycle
15,146,255,994 branches # 510.962 M/sec
55,043,674 branch-misses # 0.36% of all branches
31.317012681 seconds time elapsed
...
ms_async_op_threads = 1
osd_op_num_threads_per_shard = 1
osd_op_num_shards =1
./do_cmake.sh -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_CXX_FLAGS="-fno-omit-frame-pointer" -DWITH_TESTS=OFF
[build]$ MDS=0 MGR=0 OSD=1 MON=1 ../src/vstart.sh -n --without-dashboard --memstore -X -o "memstore_device_bytes=2516582400" --nolockdep --nodaemon --redirect-output
[build]$ bin/ceph osd pool create rbd 128 128 && bin/ceph osd pool set --yes-i-really-mean-it rbd size 1 && bin/ceph osd pool --yes-i-really-mean-it set rbd min_size 1
[build]$ bin/rbd create fio_test --size 2G --image-format=2 --rbd_default_features=3
[build]$ taskset -cp 0 <ceph_osd Pid>
...
[build]$ perf stat -p `pgrep -u ${UID} ceph-osd` & fio ../rbd_write.fio; sleep 1; killall -INT perf
[1] 37526
rbd_iodepth32: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=rbd, iodepth=64
...
fio-3.1
Starting 2 processes
Jobs: 2 (f=2): [w(2)][100.0%][r=0KiB/s,w=25.4MiB/s][r=0,w=6506 IOPS][eta 00m:00s]
rbd_iodepth32: (groupid=0, jobs=2): err= 0: pid=37639: Wed Feb 3 00:22:20 2021
write: IOPS=10.6k, BW=41.3MiB/s (43.4MB/s)(1241MiB/30021msec)
slat (nsec): min=1296, max=131535, avg=5758.49, stdev=1934.07
clat (msec): min=3, max=234, avg=12.09, stdev= 7.13
lat (msec): min=3, max=234, avg=12.09, stdev= 7.13
clat percentiles (msec):
| 1.00th=[ 5], 5.00th=[ 5], 10.00th=[ 6], 20.00th=[ 7],
| 30.00th=[ 9], 40.00th=[ 10], 50.00th=[ 12], 60.00th=[ 13],
| 70.00th=[ 15], 80.00th=[ 17], 90.00th=[ 18], 95.00th=[ 20],
| 99.00th=[ 31], 99.50th=[ 36], 99.90th=[ 103], 99.95th=[ 129],
| 99.99th=[ 163]
bw ( KiB/s): min= 2104, max=49960, per=50.03%, avg=21178.27, stdev=9811.25, samples=120
iops : min= 526, max=12490, avg=5294.51, stdev=2452.83, samples=120
lat (msec) : 4=0.01%, 10=41.92%, 20=53.28%, 50=4.49%, 100=0.20%
lat (msec) : 250=0.11%
cpu : usr=50.76%, sys=49.18%, ctx=3249, majf=0, minf=1422
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwt: total=0,317739,0, short=0,0,0, dropped=0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
WRITE: bw=41.3MiB/s (43.4MB/s), 41.3MiB/s-41.3MiB/s (43.4MB/s-43.4MB/s), io=1241MiB (1301MB), run=30021-30021msec
Disk stats (read/write):
dm-1: ios=0/129, merge=0/0, ticks=0/60, in_queue=60, util=0.20%, aggrios=0/77, aggrmerge=0/65, aggrticks=0/60, aggrin_queue=60, aggrutil=0.20%
sda: ios=0/77, merge=0/65, ticks=0/60, in_queue=60, util=0.20%
Performance counter stats for process id '37380':
53161.843754 task-clock (msec) # 1.699 CPUs utilized
720,762 context-switches # 0.014 M/sec
18 cpu-migrations # 0.000 K/sec
1,805,282 page-faults # 0.034 M/sec
135,556,125,646 cycles # 2.550 GHz
90,259,333,420 instructions # 0.67 insn per cycle
17,410,244,625 branches # 327.495 M/sec
100,216,022 branch-misses # 0.58% of all branches
31.285592179 seconds time elapsed
...
[build]$ perf stat -p `pgrep -u ${UID} ceph-osd` & fio ../rbd_read.fio; sleep 1; killall -INT perf
[1] 37644
rbd_iodepth32: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=rbd, iodepth=64
...
fio-3.1
Starting 2 processes
Jobs: 2 (f=2): [r(2)][100.0%][r=180MiB/s,w=0KiB/s][r=45.0k,w=0 IOPS][eta 00m:00s]
rbd_iodepth32: (groupid=0, jobs=2): err= 0: pid=37757: Wed Feb 3 00:26:27 2021
read: IOPS=41.1k, BW=160MiB/s (168MB/s)(4812MiB/30003msec)
slat (nsec): min=238, max=737432, avg=3485.81, stdev=1930.53
clat (usec): min=1238, max=15937, avg=3113.57, stdev=464.04
lat (usec): min=1239, max=15956, avg=3117.05, stdev=464.12
clat percentiles (usec):
| 1.00th=[ 2212], 5.00th=[ 2606], 10.00th=[ 2671], 20.00th=[ 2769],
| 30.00th=[ 2868], 40.00th=[ 2999], 50.00th=[ 3195], 60.00th=[ 3261],
| 70.00th=[ 3294], 80.00th=[ 3359], 90.00th=[ 3425], 95.00th=[ 3490],
| 99.00th=[ 5211], 99.50th=[ 5735], 99.90th=[ 6980], 99.95th=[ 8455],
| 99.99th=[11469]
bw ( KiB/s): min=67144, max=114864, per=50.01%, avg=82137.21, stdev=8147.17, samples=120
iops : min=16786, max=28716, avg=20534.26, stdev=2036.81, samples=120
lat (msec) : 2=0.64%, 4=97.85%, 10=1.49%, 20=0.02%
cpu : usr=52.61%, sys=47.33%, ctx=10007, majf=0, minf=465
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwt: total=1231875,0,0, short=0,0,0, dropped=0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=160MiB/s (168MB/s), 160MiB/s-160MiB/s (168MB/s-168MB/s), io=4812MiB (5046MB), run=30003-30003msec
Disk stats (read/write):
dm-1: ios=0/271, merge=0/0, ticks=0/288, in_queue=288, util=0.68%, aggrios=0/181, aggrmerge=0/126, aggrticks=0/204, aggrin_queue=204, aggrutil=0.68%
sda: ios=0/181, merge=0/126, ticks=0/204, in_queue=204, util=0.68%
Performance counter stats for process id '37380':
55431.466486 task-clock (msec) # 1.772 CPUs utilized
645,319 context-switches # 0.012 M/sec
18 cpu-migrations # 0.000 K/sec
1,132,945 page-faults # 0.020 M/sec
180,596,074,788 cycles # 3.258 GHz
90,476,210,908 instructions # 0.50 insn per cycle
17,630,567,178 branches # 318.061 M/sec
147,929,888 branch-misses # 0.84% of all branches
31.279219246 seconds time elapsed
...
[global] ioengine=rbd clientname=admin pool=rbd rbdname=fio_test rw=randwrite bs=4K runtime=30 numjobs=2 direct=1 group_reporting
[rbd_iodepth32] iodepth=64 ...
[global] ioengine=rbd clientname=admin pool=rbd rbdname=fio_test rw=randread bs=4K time_based=1 runtime=30 group_reporting
[rbd_iodepth32] iodepth=64 numjobs=2