fio: Rough perf testing of crimson-osd
4 KB random reads and writes with one fio instance and two job number
single OSD instance
deployed on purley server by vstart.sh
crimson-seastore
master (commit ID 8de72972d77784b39224689326c17a809e204f60 ) + omap missing hint PRcrimson-osd(new) with interruptable future and new lba treeiodepth=2, numjobs=1 iodepth=16, numjobs=1
write:884,264,592,469/524288 = 1,686,600 write: 820,465,174,060/524288 = 1,564,913
>>>
read: 1,977,709,087,682/1353967= 1,460,677 read: 2,070,438,533,537/7830300 =264,413
>>>
iodepth=32, numjobs=2 iodepth=64, numjobs4
write: write:
>>>
read: read:
>>>
iodepth=2, numjobs=1 iodepth=16, numjobs=1
write: 371,934,645,258/382934= 971,276 write: 252,452,130,069 /524288 = 481,514
>>>
read: 524,354,983,424/3082082 = 170,130 read: 800,683,282,134/4659116 =171,853
>>>
iodepth=32, numjobs=2 iodepth=64, numjobs4
write: write: =
>>>
read: read:
>>>
crimson-osd(new) with interruptable future and new lba treeiodepth=2, numjobs=1 iodepth=16, numjobs=1 iodepth=32, numjobs=2 iodepth=64, numjobs=4
write:1086.67(us) write: 13788.85(us) write: write:
read: 884.05(us) read: 1224.91(us) read: read:
iodepth=2, numjobs=1 iodepth=16, numjobs=1 iodepth=32, numjobs=2 iodepth=64, numjobs=4
write:3128.93(us) write: 5101.48(us) write: write:
read: 387.52(us) read: 2058.99(us) read: read:
crimson Build configuration ./do_cmake.sh -DWITH_SEASTAR=ON -DWITH_MGR_DASHBOARD_FRONTEND=OFF -DCMAKE_BUILD_TYPE=RelWithDebInfo
[build]$ MDS=0 MGR=1 OSD=1 MON=1 ../src/vstart.sh -n --without-dashboard --seastore -X --crimson --nodaemon --redirect-output --seastore-devs /dev/nvme1n1
[build]$ bin/ceph osd pool create rbd 128 128 && bin/ceph osd pool set --yes-i-really-mean-it rbd size 1 && bin/ceph osd pool --yes-i-really-mean-it set rbd min_size 1
[build]$ bin/rbd create fio_test --size 2G --image-format=2 --rbd_default_features=3
...
crimson 4 KB random write, iodepth=2, numjobs=1 [build]$ perf stat -p `pgrep -u ${UID} crimson-osd` & fio ../rbd_write.fio; sleep 1; killall -INT perf
[1] 1393613
rbd_iodepth32: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=rbd, iodepth=2
fio-3.26-46-g30be
Starting 1 process
Jobs: 1 (f=1): [w(1)][100.0%][w=556KiB/s][w=139 IOPS][eta 00m:00s]
rbd_iodepth32: (groupid=0, jobs=1): err= 0: pid=1393700: Tue Sep 21 16:20:48 2021
write: IOPS=1836, BW=7345KiB/s (7521kB/s)(2048MiB/285517msec); 0 zone resets
slat (usec): min=3, max=783, avg=12.25, stdev= 4.81
clat (usec): min=277, max=183546, avg=1074.42, stdev=2169.88
lat (usec): min=287, max=183557, avg=1086.67, stdev=2170.45
clat percentiles (usec):
| 1.00th=[ 338], 5.00th=[ 379], 10.00th=[ 424], 20.00th=[ 502],
| 30.00th=[ 570], 40.00th=[ 644], 50.00th=[ 725], 60.00th=[ 816],
| 70.00th=[ 938], 80.00th=[ 1139], 90.00th=[ 1582], 95.00th=[ 2474],
| 99.00th=[ 6194], 99.50th=[10552], 99.90th=[36439], 99.95th=[44303],
| 99.99th=[64226]
bw ( KiB/s): min= 408, max=13336, per=100.00%, avg=7362.65, stdev=1818.35, samples=570
iops : min= 102, max= 3334, avg=1840.52, stdev=454.61, samples=570
lat (usec) : 500=19.67%, 750=33.13%, 1000=21.28%
lat (msec) : 2=19.28%, 4=4.27%, 10=1.84%, 20=0.27%, 50=0.24%
lat (msec) : 100=0.02%, 250=0.01%
cpu : usr=3.42%, sys=2.59%, ctx=519090, majf=0, minf=1258
IO depths : 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,524288,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=2
Run status group 0 (all jobs):
WRITE: bw=7345KiB/s (7521kB/s), 7345KiB/s-7345KiB/s (7521kB/s-7521kB/s), io=2048MiB (2147MB), run=285517-285517msec
Disk stats (read/write):
dm-0: ios=0/8193, merge=0/0, ticks=0/1428, in_queue=1428, util=0.97%, aggrios=0/4149, aggrmerge=0/4042, aggrticks=0/1177, aggrin_queue=16, aggrutil=0.97%
sda: ios=0/4149, merge=0/4042, ticks=0/1177, in_queue=16, util=0.97%
root@otccldstore05:/home/chunmei/ceph/build#
Performance counter stats for process id '1393424':
269,200.25 msec task-clock # 0.940 CPUs utilized
17,583 context-switches # 0.065 K/sec
0 cpu-migrations # 0.000 K/sec
18,973 page-faults # 0.070 K/sec
884,264,592,469 cycles # 3.285 GHz
1,133,874,527,348 instructions # 1.28 insn per cycle
208,564,017,093 branches # 774.754 M/sec
3,408,945,732 branch-misses # 1.63% of all branches
286.511761129 seconds time elapsed
...
crimson 4 KB random read, iodepth=2, numjobs=1 [build]$ perf stat -p `pgrep -u ${UID} crimson-osd` & fio ../rbd_read.fio; sleep 1; killall -INT perf
[1] 1393708
rbd_iodepth32: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=rbd, iodepth=2
fio-3.26-46-g30be
Starting 1 process
Jobs: 1 (f=1): [r(1)][100.0%][r=6236KiB/s][r=1559 IOPS][eta 00m:00s]
rbd_iodepth32: (groupid=0, jobs=1): err= 0: pid=1393796: Tue Sep 21 16:33:26 2021
read: IOPS=2256, BW=9026KiB/s (9242kB/s)(5289MiB/600046msec)
slat (nsec): min=830, max=852697, avg=10624.87, stdev=4175.89
clat (usec): min=195, max=49866, avg=873.43, stdev=810.09
lat (usec): min=204, max=49877, avg=884.05, stdev=811.20
clat percentiles (usec):
| 1.00th=[ 258], 5.00th=[ 334], 10.00th=[ 371], 20.00th=[ 445],
| 30.00th=[ 498], 40.00th=[ 553], 50.00th=[ 652], 60.00th=[ 807],
| 70.00th=[ 1045], 80.00th=[ 1270], 90.00th=[ 1582], 95.00th=[ 1811],
| 99.00th=[ 2769], 99.50th=[ 4686], 99.90th=[ 6390], 99.95th=[ 6652],
| 99.99th=[44303]
bw ( KiB/s): min= 4112, max=21298, per=100.00%, avg=9035.82, stdev=4279.78, samples=1199
iops : min= 1028, max= 5324, avg=2258.80, stdev=1069.97, samples=1199
lat (usec) : 250=0.52%, 500=29.94%, 750=26.35%, 1000=11.48%
lat (msec) : 2=29.05%, 4=2.09%, 10=0.55%, 20=0.01%, 50=0.01%
cpu : usr=3.83%, sys=3.28%, ctx=1347378, majf=0, minf=1016
IO depths : 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=1353967,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=2
Run status group 0 (all jobs):
READ: bw=9026KiB/s (9242kB/s), 9026KiB/s-9026KiB/s (9242kB/s-9242kB/s), io=5289MiB (5546MB), run=600046-600046msec
Disk stats (read/write):
dm-0: ios=0/17713, merge=0/0, ticks=0/3068, in_queue=3068, util=0.98%, aggrios=0/8645, aggrmerge=0/9083, aggrticks=0/2505, aggrin_queue=36, aggrutil=0.98%
sda: ios=0/8645, merge=0/9083, ticks=0/2505, in_queue=36, util=0.98%
root@otccldstore05:/home/chunmei/ceph/build#
Performance counter stats for process id '1393424':
584,724.43 msec task-clock # 0.973 CPUs utilized
83,111 context-switches # 0.142 K/sec
0 cpu-migrations # 0.000 K/sec
16,042 page-faults # 0.027 K/sec
1,977,709,087,682 cycles # 3.382 GHz
3,029,427,566,958 instructions # 1.53 insn per cycle
557,583,545,801 branches # 953.583 M/sec
5,516,745,928 branch-misses # 0.99% of all branches
601.056389558 seconds time elapsed
...
crimson 4 KB random write, iodepth=16, numjobs=1 [build]$ perf stat -p `pgrep -u ${UID} crimson-osd` & fio ../rbd_write.fio; sleep 1; killall -INT perf
[1] 1394461
rbd_iodepth32: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=rbd, iodepth=16
fio-3.26-46-g30be
Starting 1 process
Jobs: 1 (f=1): [w(1)][100.0%][w=1772KiB/s][w=443 IOPS][eta 00m:00s]
rbd_iodepth32: (groupid=0, jobs=1): err= 0: pid=1394548: Tue Sep 21 16:48:10 2021
write: IOPS=1160, BW=4640KiB/s (4752kB/s)(2048MiB/451949msec); 0 zone resets
slat (usec): min=2, max=599, avg=17.19, stdev=10.45
clat (usec): min=375, max=339578, avg=13771.66, stdev=17507.72
lat (usec): min=388, max=339583, avg=13788.85, stdev=17507.51
clat percentiles (usec):
| 1.00th=[ 1303], 5.00th=[ 1745], 10.00th=[ 2114], 20.00th=[ 2769],
| 30.00th=[ 3523], 40.00th=[ 4621], 50.00th=[ 6325], 60.00th=[ 8586],
| 70.00th=[ 12387], 80.00th=[ 20841], 90.00th=[ 43254], 95.00th=[ 51119],
| 99.00th=[ 72877], 99.50th=[ 86508], 99.90th=[121111], 99.95th=[143655],
| 99.99th=[208667]
bw ( KiB/s): min= 1616, max=10176, per=100.00%, avg=4646.84, stdev=1289.13, samples=903
iops : min= 404, max= 2544, avg=1161.56, stdev=322.32, samples=903
lat (usec) : 500=0.01%, 750=0.02%, 1000=0.14%
lat (msec) : 2=8.27%, 4=26.38%, 10=29.65%, 20=14.94%, 50=15.03%
lat (msec) : 100=5.28%, 250=0.27%, 500=0.01%
cpu : usr=3.52%, sys=2.43%, ctx=319569, majf=0, minf=1361
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,524288,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=16
Run status group 0 (all jobs):
WRITE: bw=4640KiB/s (4752kB/s), 4640KiB/s-4640KiB/s (4752kB/s-4752kB/s), io=2048MiB (2147MB), run=451949-451949msec
Disk stats (read/write):
dm-0: ios=4/13075, merge=0/0, ticks=0/2348, in_queue=2348, util=0.98%, aggrios=4/6486, aggrmerge=0/6636, aggrticks=3/1874, aggrin_queue=16, aggrutil=0.99%
sda: ios=4/6486, merge=0/6636, ticks=3/1874, in_queue=16, util=0.99%
root@otccldstore05:/home/chunmei/ceph/build#
Performance counter stats for process id '1394272':
320,213.17 msec task-clock # 0.707 CPUs utilized
22,073 context-switches # 0.069 K/sec
0 cpu-migrations # 0.000 K/sec
17,658 page-faults # 0.055 K/sec
820,465,174,060 cycles # 2.562 GHz
890,364,882,190 instructions # 1.09 insn per cycle
162,792,385,847 branches # 508.388 M/sec
3,434,767,714 branch-misses # 2.11% of all branches
452.951978201 seconds time elapsed
...
crimson 4 KB random read, iodepth=16, numjobs=1 [build]$ perf stat -p `pgrep -u ${UID} crimson-osd` & fio ../rbd_read.fio; sleep 1; killall -INT perf
[1] 1394565
rbd_iodepth32: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=rbd, iodepth=16
fio-3.26-46-g30be
Starting 1 process
Jobs: 1 (f=1): [r(1)][100.0%][r=51.2MiB/s][r=13.1k IOPS][eta 00m:00s]
rbd_iodepth32: (groupid=0, jobs=1): err= 0: pid=1394652: Tue Sep 21 17:01:25 2021
read: IOPS=13.0k, BW=51.0MiB/s (53.5MB/s)(29.9GiB/600041msec)
slat (nsec): min=1234, max=1048.7k, avg=6559.11, stdev=3270.27
clat (usec): min=205, max=41597, avg=1218.35, stdev=311.36
lat (usec): min=214, max=41607, avg=1224.91, stdev=311.25
clat percentiles (usec):
| 1.00th=[ 644], 5.00th=[ 791], 10.00th=[ 881], 20.00th=[ 1012],
| 30.00th=[ 1090], 40.00th=[ 1156], 50.00th=[ 1205], 60.00th=[ 1254],
| 70.00th=[ 1319], 80.00th=[ 1385], 90.00th=[ 1500], 95.00th=[ 1663],
| 99.00th=[ 2114], 99.50th=[ 2343], 99.90th=[ 4080], 99.95th=[ 5080],
| 99.99th=[ 6259]
bw ( KiB/s): min=38917, max=56216, per=100.00%, avg=52249.34, stdev=587.47, samples=1199
iops : min= 9729, max=14054, avg=13062.29, stdev=146.88, samples=1199
lat (usec) : 250=0.01%, 500=0.11%, 750=3.45%, 1000=15.37%
lat (msec) : 2=79.62%, 4=1.34%, 10=0.11%, 20=0.01%, 50=0.01%
cpu : usr=13.03%, sys=10.22%, ctx=4846193, majf=0, minf=910
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=7830300,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=16
Run status group 0 (all jobs):
READ: bw=51.0MiB/s (53.5MB/s), 51.0MiB/s-51.0MiB/s (53.5MB/s-53.5MB/s), io=29.9GiB (32.1GB), run=600041-600041msec
Disk stats (read/write):
dm-0: ios=0/17025, merge=0/0, ticks=0/2996, in_queue=2996, util=0.95%, aggrios=0/8530, aggrmerge=0/8504, aggrticks=0/2370, aggrin_queue=24, aggrutil=0.95%
sda: ios=0/8530, merge=0/8504, ticks=0/2370, in_queue=24, util=0.95%
root@otccldstore05:/home/chunmei/ceph/build#
Performance counter stats for process id '1394272':
599,697.02 msec task-clock # 0.998 CPUs utilized
5,819 context-switches # 0.010 K/sec
0 cpu-migrations # 0.000 K/sec
21,631 page-faults # 0.036 K/sec
2,070,438,533,537 cycles # 3.452 GHz
2,017,732,757,266 instructions # 0.97 insn per cycle
367,572,970,437 branches # 612.931 M/sec
4,736,661,933 branch-misses # 1.29% of all branches
601.045628346 seconds time elapsed
...
crimson 4 KB random write, iodepth=32, numjobs=2 [build]$ perf stat -p `pgrep -u ${UID} crimson-osd` & fio ../rbd_write.fio; sleep 1; killall -INT perf
...
crimson 4 KB random read, iodepth=32, numjobs=2 [build]$ perf stat -p `pgrep -u ${UID} crimson-osd` & fio ../rbd_read.fio; sleep 1; killall -INT perf
...
crimson 4 KB random write, iodepth=64, numjobs=4 [build]$ perf stat -p `pgrep -u ${UID} crimson-osd` & fio ../rbd_write.fio; sleep 1; killall -INT perf
...
crimson 4 KB random read, iodepth=64, numjobs=4 [build]$ perf stat -p `pgrep -u ${UID} crimson-osd` & fio ../rbd_read.fio; sleep 1; killall -INT perf
...
ceph options configuration ms_async_op_threads = 1
osd_op_num_threads_per_shard = 1
osd_op_num_shards =1
./do_cmake.sh -DWITH_MGR_DASHBOARD_FRONTEND=OFF -DCMAKE_BUILD_TYPE=RelWithDebInfo
[build]$ MDS=0 MGR=1 OSD=1 MON=1 ../src/vstart.sh -n --without-dashboard --bluestore -X --bluestore-devs /dev/nvme1n1
[build]$ bin/ceph osd pool create rbd 128 128 && bin/ceph osd pool set --yes-i-really-mean-it rbd size 1 && bin/ceph osd pool --yes-i-really-mean-it set rbd min_size 1
[build]$ bin/rbd create fio_test --size 2G --image-format=2 --rbd_default_features=3
[build]$ taskset -cp 0 <ceph_osd Pid>
...
ceph 4 KB random write, iodepth=2, numjobs=1 [build]$ perf stat -p `pgrep -u ${UID} ceph-osd` & fio ../rbd_write.fio; sleep 1; killall -INT perf
[1] 1392035
rbd_iodepth32: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, =rbd, iodepth=2
fio-3.26-46-g30be
Starting 1 process
Jobs: 1 (f=1): [w(1)][100.0%][w=4156KiB/s][w=1039 IOPS][eta 00m:00s]
rbd_iodepth32: (groupid=0, jobs=1): err= 0: pid=1392122: Tue Sep 21 15:54:09 2021
write: IOPS=638, BW=2553KiB/s (2614kB/s)(1496MiB/600002msec); 0 zone resets
slat (usec): min=3, max=170, avg=22.29, stdev= 8.59
clat (usec): min=1222, max=205554, avg=3106.64, stdev=2274.90
lat (usec): min=1251, max=205585, avg=3128.93, stdev=2278.49
clat percentiles (usec):
| 1.00th=[ 1401], 5.00th=[ 1450], 10.00th=[ 1516], 20.00th=[ 1631],
| 30.00th=[ 1745], 40.00th=[ 2008], 50.00th=[ 2737], 60.00th=[ 2999],
| 70.00th=[ 3261], 80.00th=[ 3621], 90.00th=[ 5342], 95.00th=[ 7767],
| 99.00th=[11469], 99.50th=[12256], 99.90th=[16581], 99.95th=[30016],
| 99.99th=[39584]
bw ( KiB/s): min= 648, max= 4752, per=100.00%, avg=2554.48, stdev=1200.51, samples=1199
iops : min= 162, max= 1188, avg=638.54, stdev=300.14, samples=1199
lat (msec) : 2=39.85%, 4=44.92%, 10=12.50%, 20=2.66%, 50=0.07%
lat (msec) : 100=0.01%, 250=0.01%
cpu : usr=2.68%, sys=1.79%, ctx=382941, majf=0, minf=2013
IO depths : 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,382934,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=2
Run status group 0 (all jobs):
WRITE: bw=2553KiB/s (2614kB/s), 2553KiB/s-2553KiB/s (2614kB/s-2614kB/s), io=1496MiB (1568MB), run=600002-600002msec
Disk stats (read/write):
dm-0: ios=0/3148570, merge=0/0, ticks=0/542908, in_queue=542908, util=99.77%, aggrios=0/2045471, aggrmerge=0/1103691, aggrticks=0/475556, aggrin_queue=23432, aggrutil=99.77%
sda: ios=0/2045471, merge=0/1103691, ticks=0/475556, in_queue=23432, util=99.77%
root@otccldstore05:/home/chunmei/ceph/build#
Performance counter stats for process id '1391463':
328,996.95 msec task-clock # 0.547 CPUs utilized
5,099,353 context-switches # 0.015 M/sec
1,311 cpu-migrations # 0.004 K/sec
5,395,120 page-faults # 0.016 M/sec
371,934,645,258 cycles # 1.131 GHz
218,314,177,666 instructions # 0.59 insn per cycle
42,369,967,425 branches # 128.785 M/sec
2,331,674,402 branch-misses # 5.50% of all branches
601.010226134 seconds time elapsed
[1]+ Interrupt perf stat -p `pgrep -u ${UID} ceph-osd`
...
ceph 4 KB random read, iodepth=2, numjobs=1 [build]$ perf stat -p `pgrep -u ${UID} ceph-osd` & fio ../rbd_read.fio; sleep 1; killall -INT perf
[1] 1392130
rbd_iodepth32: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, i oengine=rbd, iodepth=2
fio-3.26-46-g30be
Starting 1 process
Jobs: 1 (f=1): [r(1)][100.0%][r=18.3MiB/s][r=4695 IOPS][eta 00m:00s]
rbd_iodepth32: (groupid=0, jobs=1): err= 0: pid=1392217: Tue Sep 21 16:07:32 2021
read: IOPS=5136, BW=20.1MiB/s (21.0MB/s)(11.8GiB/600001msec)
slat (nsec): min=1162, max=1035.4k, avg=11057.91, stdev=6038.83
clat (usec): min=94, max=19998, avg=376.46, stdev=196.80
lat (usec): min=198, max=20006, avg=387.52, stdev=196.76
clat percentiles (usec):
| 1.00th=[ 253], 5.00th=[ 277], 10.00th=[ 285], 20.00th=[ 293],
| 30.00th=[ 302], 40.00th=[ 310], 50.00th=[ 318], 60.00th=[ 334],
| 70.00th=[ 363], 80.00th=[ 408], 90.00th=[ 594], 95.00th=[ 685],
| 99.00th=[ 816], 99.50th=[ 873], 99.90th=[ 988], 99.95th=[ 1057],
| 99.99th=[ 8455]
bw ( KiB/s): min=14813, max=27360, per=100.00%, avg=20568.89, stdev=1661.03, samples=1199
iops : min= 3703, max= 6840, avg=5142.07, stdev=415.28, samples=1199
lat (usec) : 100=0.01%, 250=0.80%, 500=83.02%, 750=13.08%, 1000=3.01%
lat (msec) : 2=0.05%, 4=0.01%, 10=0.02%, 20=0.01%
cpu : usr=7.00%, sys=7.10%, ctx=2521604, majf=0, minf=1545
IO depths : 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=3082082,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=2
Run status group 0 (all jobs):
READ: bw=20.1MiB/s (21.0MB/s), 20.1MiB/s-20.1MiB/s (21.0MB/s-21.0MB/s), io=11.8GiB (12.6GB), run=600001-600001msec
Disk stats (read/write):
dm-0: ios=0/18083, merge=0/0, ticks=0/3092, in_queue=3092, util=1.01%, aggrios=0/8640, aggrmerge=0/9458, aggrticks=0/2562, aggrin_queue=12, aggrutil=1.01%
sda: ios=0/8640, merge=0/9458, ticks=0/2562, in_queue=12, util=1.01%
root@otccldstore05:/home/chunmei/ceph/build#
Performance counter stats for process id '1391463':
527,346.91 msec task-clock # 0.877 CPUs utilized
7,696,298 context-switches # 0.015 M/sec
726 cpu-migrations # 0.001 K/sec
8,424,355 page-faults # 0.016 M/sec
524,354,983,424 cycles # 0.994 GHz
367,301,940,139 instructions # 0.70 insn per cycle
73,485,387,821 branches # 139.349 M/sec
1,164,983,332 branch-misses # 1.59% of all branches
600.989253669 seconds time elapsed
...
ceph 4 KB random write, iodepth=16, numjobs=1 [build]$ perf stat -p `pgrep -u ${UID} ceph-osd` & fio ../rbd_write.fio; sleep 1; killall -INT perf
[1] 1396588
rbd_iodepth32: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=rbd, iodepth=16
fio-3.26-46-g30be
Starting 1 process
Jobs: 1 (f=1): [w(1)][100.0%][w=17.9MiB/s][w=4589 IOPS][eta 00m:00s]
rbd_iodepth32: (groupid=0, jobs=1): err= 0: pid=1396675: Tue Sep 21 17:08:30 2021
write: IOPS=3135, BW=12.2MiB/s (12.8MB/s)(2048MiB/167222msec); 0 zone resets
slat (nsec): min=1194, max=1092.3k, avg=9304.50, stdev=9112.91
clat (usec): min=1222, max=54692, avg=5092.17, stdev=3010.02
lat (usec): min=1243, max=54700, avg=5101.48, stdev=3009.47
clat percentiles (usec):
| 1.00th=[ 1663], 5.00th=[ 2245], 10.00th=[ 2474], 20.00th=[ 3425],
| 30.00th=[ 3687], 40.00th=[ 3884], 50.00th=[ 4146], 60.00th=[ 4817],
| 70.00th=[ 5407], 80.00th=[ 6128], 90.00th=[ 8225], 95.00th=[11994],
| 99.00th=[15401], 99.50th=[17433], 99.90th=[33424], 99.95th=[38536],
| 99.99th=[42730]
bw ( KiB/s): min= 4128, max=25963, per=100.00%, avg=12560.13, stdev=4867.31, samples=334
iops : min= 1032, max= 6490, avg=3139.94, stdev=1216.82, samples=334
lat (msec) : 2=3.17%, 4=41.80%, 10=48.50%, 20=6.26%, 50=0.27%
lat (msec) : 100=0.01%
cpu : usr=5.03%, sys=2.01%, ctx=119663, majf=0, minf=1080
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,524288,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=16
Run status group 0 (all jobs):
WRITE: bw=12.2MiB/s (12.8MB/s), 12.2MiB/s-12.2MiB/s (12.8MB/s-12.8MB/s), io=2048MiB (2147MB), run=167222-167222msec
Disk stats (read/write):
dm-0: ios=0/1110622, merge=0/0, ticks=0/167740, in_queue=167740, util=99.96%, aggrios=0/565383, aggrmerge=0/545411, aggrticks=0/133710, aggrin_queue=6808, aggrutil=99.95%
sda: ios=0/565383, merge=0/545411, ticks=0/133710, in_queue=6808, util=99.95%
root@otccldstore05:/home/chunmei/ceph/build#
Performance counter stats for process id '1396015':
175,277.57 msec task-clock # 1.042 CPUs utilized
1,612,461 context-switches # 0.009 M/sec
419 cpu-migrations # 0.002 K/sec
3,334,853 page-faults # 0.019 M/sec
252,452,130,069 cycles # 1.440 GHz
181,846,032,532 instructions # 0.72 insn per cycle
36,128,394,246 branches # 206.121 M/sec
1,035,125,205 branch-misses # 2.87% of all branches
168.214044168 seconds time elapsed
...
ceph 4 KB random read, iodepth=16, numjobs=1 [build]$ perf stat -p `pgrep -u ${UID} ceph-osd` & fio ../rbd_read.fio; sleep 1; killall -INT perf
[1] 1396680
rbd_iodepth32: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=rbd, iodepth=16
fio-3.26-46-g30be
Starting 1 process
Jobs: 1 (f=1): [r(1)][100.0%][r=19.8MiB/s][r=5069 IOPS][eta 00m:00s]
rbd_iodepth32: (groupid=0, jobs=1): err= 0: pid=1396767: Tue Sep 21 17:22:39 2021
read: IOPS=7765, BW=30.3MiB/s (31.8MB/s)(17.8GiB/600003msec)
slat (nsec): min=936, max=1550.6k, avg=9551.39, stdev=4437.08
clat (usec): min=313, max=14394, avg=2049.44, stdev=784.61
lat (usec): min=321, max=14405, avg=2058.99, stdev=785.16
clat percentiles (usec):
| 1.00th=[ 848], 5.00th=[ 1057], 10.00th=[ 1172], 20.00th=[ 1369],
| 30.00th=[ 1549], 40.00th=[ 1745], 50.00th=[ 1942], 60.00th=[ 2180],
| 70.00th=[ 2409], 80.00th=[ 2704], 90.00th=[ 3064], 95.00th=[ 3294],
| 99.00th=[ 3785], 99.50th=[ 4015], 99.90th=[ 8848], 99.95th=[10028],
| 99.99th=[11863]
bw ( KiB/s): min=18148, max=63584, per=100.00%, avg=31099.93, stdev=8999.74, samples=1199
iops : min= 4537, max=15896, avg=7774.82, stdev=2249.96, samples=1199
lat (usec) : 500=0.01%, 750=0.42%, 1000=2.81%
lat (msec) : 2=49.39%, 4=46.87%, 10=0.46%, 20=0.05%
cpu : usr=9.56%, sys=8.69%, ctx=3841142, majf=0, minf=1515
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=4659116,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=16
Run status group 0 (all jobs):
READ: bw=30.3MiB/s (31.8MB/s), 30.3MiB/s-30.3MiB/s (31.8MB/s-31.8MB/s), io=17.8GiB (19.1GB), run=600003-600003msec
Disk stats (read/write):
dm-0: ios=0/17848, merge=0/0, ticks=0/5480, in_queue=5480, util=1.42%, aggrios=0/8694, aggrmerge=0/9170, aggrticks=0/4960, aggrin_queue=76, aggrutil=1.42%
sda: ios=0/8694, merge=0/9170, ticks=0/4960, in_queue=76, util=1.42%
root@otccldstore05:/home/chunmei/ceph/build#
Performance counter stats for process id '1396015':
685,253.96 msec task-clock # 1.140 CPUs utilized
10,086,665 context-switches # 0.015 M/sec
1,475 cpu-migrations # 0.002 K/sec
18,056,709 page-faults # 0.026 M/sec
800,683,282,134 cycles # 1.168 GHz
541,340,409,507 instructions # 0.68 insn per cycle
108,515,600,861 branches # 158.358 M/sec
1,515,750,808 branch-misses # 1.40% of all branches
601.007259885 seconds time elapsed
...
ceph 4 KB random write, iodepth=32, numjobs=2 [build]$ perf stat -p `pgrep -u ${UID} ceph-osd` & fio ../rbd_write.fio; sleep 1; killall -INT perf
...
ceph 4 KB random read, iodepth=32, numjobs=2 [build]$ perf stat -p `pgrep -u ${UID} ceph-osd` & fio ../rbd_read.fio; sleep 1; killall -INT perf
...
ceph 4 KB random write, iodepth=64, numjobs=4 [build]$ perf stat -p `pgrep -u ${UID} ceph-osd` & fio ../rbd_write.fio; sleep 1; killall -INT perf
...
ceph 4 KB random read, iodepth=64, numjobs=4 [build]$ perf stat -p `pgrep -u ${UID} ceph-osd` & fio ../rbd_read.fio; sleep 1; killall -INT perf
...
