Hypervisor: VMware ESXi, 7.0.3, 19482537
Model: PowerEdge R730xd (Maximum Performance Profile)
HBA: Dell HBA330 Mini (Embedded, Firmware: 16.17.01.00) - Passed Through to VM
Enclosure: BP13G+EXP 0:1 (No "Split Mode" Backplane)
Processor Type: Intel(R) Xeon(R) CPU E5-2699 v4 @ 2.20GHz
Logical Processors: 88
Drives: 24x SAMSUNG MZILS1T9HEJH0D3
Drive Size: 1788.50 GB
Block Size: 512 bytes
Bus Protocol: SAS
Media Type: SSD
Negotiated Speed: 12 Gbps
VM: TrueNAS Scale
vCPU: 8, then 22 to test if write LZ4 compression was an issue
RAM: 64 GB (Reserved) at 2400 MHz
capacity operations bandwidth
pool alloc free read write read write
---------------------------------------- ----- ----- ----- ----- ----- -----
tank 8.03G 20.8T 0 0 0 0
mirror 626M 1.73T 0 0 0 0
c30f6195-dc79-4a63-bfa6-ef4b04d4801f - - 0 0 0 0
8bf44580-62db-4f5b-8e18-3d41c152719a - - 0 0 0 0
mirror 847M 1.73T 0 0 0 0
1d1476bf-934d-4291-8a61-dd35a3cfe4c4 - - 0 0 0 0
5e83bcb3-afa6-449f-9c1c-4d34a4151f67 - - 0 0 0 0
mirror 683M 1.73T 0 0 0 0
820d6d88-42aa-4d53-a4b2-0f432ac3ecf3 - - 0 0 0 0
51fc2719-47c0-473d-a667-ecadc7451b91 - - 0 0 0 0
mirror 670M 1.73T 0 0 0 0
72650930-b318-444a-b2b8-e832b40b5470 - - 0 0 0 0
6951e676-d608-480e-acfa-26f7d2f9c23e - - 0 0 0 0
mirror 668M 1.73T 0 0 0 0
e3368f18-9759-42fa-895c-014a7cac0646 - - 0 0 0 0
8269ddcb-4271-4768-8dd7-4ada28b37391 - - 0 0 0 0
mirror 636M 1.73T 0 0 0 0
832715fe-4d36-454f-9615-a75ef056e01a - - 0 0 0 0
2c57aa1d-0b91-4957-866c-8e363c4553fb - - 0 0 0 0
mirror 570M 1.73T 0 0 0 0
afabd65c-eda0-4965-a02d-cfa3b051d573 - - 0 0 0 0
c446daca-a974-4bc8-9b6e-3b90338715de - - 0 0 0 0
mirror 755M 1.73T 0 0 0 0
26f8c599-c31d-450b-ae6f-1d38f4bae889 - - 0 0 0 0
6eaee91e-a18f-441a-a479-f4ec6e5df691 - - 0 0 0 0
mirror 575M 1.73T 0 0 0 0
7da3d637-3f27-4553-b6ab-9438136d2c38 - - 0 0 0 0
8a3bbc3e-b102-474d-b5f6-24331c05b406 - - 0 0 0 0
mirror 794M 1.73T 0 0 0 0
b3d45273-24e6-4043-abf6-765bbb686003 - - 0 0 0 0
685c1623-6eb7-459f-87be-8260c12402ed - - 0 0 0 0
mirror 667M 1.73T 0 0 0 0
ae1f5b25-45b6-479a-9729-82eda6595839 - - 0 0 0 0
d9bfa827-5669-4e5a-80df-f38f343666b8 - - 0 0 0 0
mirror 731M 1.73T 0 0 0 0
0a636a7f-f432-4f38-bf86-f3ffb938cc19 - - 0 0 0 0
93543f55-b1d3-43f7-a147-2e505ca4ce59 - - 0 0 0 0
---------------------------------------- ----- ----- ----- ----- ----- -----
rw: IO direction
alias: readwrite
type: string (opt=bla)
default: read
valid values: read Sequential read
: write Sequential write
: trim Sequential trim
: randread Random read
: randwrite Random write
: randtrim Random trim
: rw Sequential read and write mix
: readwrite Sequential read and write mix
: randrw Random read and write mix
: trimwrite Trim and write mix, trims preceding writes
Best of three: 2814MB/s
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=read --ramp_time=4
test: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=0): [f(1)][-.-%][r=2521MiB/s][r=2521 IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=149670: Fri May 6 12:07:59 2022
read: IOPS=2638, BW=2638MiB/s (2766MB/s)(8192MiB/3105msec)
cpu : usr=0.87%, sys=99.03%, ctx=8, majf=0, minf=264
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=8192,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
READ: bw=2638MiB/s (2766MB/s), 2638MiB/s-2638MiB/s (2766MB/s-2766MB/s), io=8192MiB (8590MB), run=3105-3105msec
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=read --ramp_time=4
test: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=0): [/(1)][-.-%][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=149794: Fri May 6 12:08:09 2022
read: IOPS=2683, BW=2683MiB/s (2814MB/s)(8192MiB/3053msec)
cpu : usr=0.00%, sys=100.00%, ctx=4, majf=0, minf=263
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=8192,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
READ: bw=2683MiB/s (2814MB/s), 2683MiB/s-2683MiB/s (2814MB/s-2814MB/s), io=8192MiB (8590MB), run=3053-3053msec
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=read --ramp_time=4
test: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=0): [f(1)][-.-%][r=2521MiB/s][r=2521 IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=149869: Fri May 6 12:08:14 2022
read: IOPS=2664, BW=2664MiB/s (2793MB/s)(8192MiB/3075msec)
cpu : usr=0.49%, sys=99.45%, ctx=5, majf=0, minf=264
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=8192,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
READ: bw=2664MiB/s (2793MB/s), 2664MiB/s-2664MiB/s (2793MB/s-2793MB/s), io=8192MiB (8590MB), run=3075-3075msec
Best of three: 1458MB/s
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=write --ramp_time=4
test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=1): [W(1)][-.-%][w=247KiB/s][w=0 IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=150523: Fri May 6 12:09:17 2022
write: IOPS=1387, BW=1390MiB/s (1458MB/s)(506MiB/364msec); 0 zone resets
slat (usec): min=258, max=10074, avg=715.26, stdev=586.09
clat (nsec): min=1083, max=18716, avg=1854.29, stdev=1152.15
lat (usec): min=259, max=10081, avg=717.64, stdev=586.99
clat percentiles (nsec):
| 1.00th=[ 1208], 5.00th=[ 1304], 10.00th=[ 1416], 20.00th=[ 1560],
| 30.00th=[ 1624], 40.00th=[ 1672], 50.00th=[ 1720], 60.00th=[ 1768],
| 70.00th=[ 1816], 80.00th=[ 1880], 90.00th=[ 2064], 95.00th=[ 2256],
| 99.00th=[ 5792], 99.50th=[ 9536], 99.90th=[18816], 99.95th=[18816],
| 99.99th=[18816]
lat (usec) : 2=88.51%, 4=9.31%, 10=1.78%, 20=0.40%
cpu : usr=1.93%, sys=41.60%, ctx=1735, majf=0, minf=58
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,505,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=1390MiB/s (1458MB/s), 1390MiB/s-1390MiB/s (1458MB/s-1458MB/s), io=506MiB (531MB), run=364-364msec
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=write --ramp_time=4
test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=1): [W(1)][-.-%][w=247KiB/s][w=0 IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=156804: Fri May 6 12:09:27 2022
write: IOPS=1338, BW=1341MiB/s (1406MB/s)(594MiB/443msec); 0 zone resets
slat (usec): min=258, max=6836, avg=743.01, stdev=586.11
clat (nsec): min=1048, max=11740, avg=1739.04, stdev=902.10
lat (usec): min=259, max=6843, avg=745.09, stdev=586.93
clat percentiles (nsec):
| 1.00th=[ 1064], 5.00th=[ 1144], 10.00th=[ 1272], 20.00th=[ 1448],
| 30.00th=[ 1496], 40.00th=[ 1544], 50.00th=[ 1592], 60.00th=[ 1640],
| 70.00th=[ 1704], 80.00th=[ 1784], 90.00th=[ 2128], 95.00th=[ 2384],
| 99.00th=[ 6304], 99.50th=[ 9280], 99.90th=[11712], 99.95th=[11712],
| 99.99th=[11712]
lat (usec) : 2=88.87%, 4=8.94%, 10=1.85%, 20=0.34%
cpu : usr=1.13%, sys=41.63%, ctx=2029, majf=0, minf=60
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,593,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=1341MiB/s (1406MB/s), 1341MiB/s-1341MiB/s (1406MB/s-1406MB/s), io=594MiB (623MB), run=443-443msec
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=write --ramp_time=4
test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=1): [W(1)][18.5%][w=78.2MiB/s][w=78 IOPS][eta 00m:22s]
test: (groupid=0, jobs=1): err= 0: pid=163509: Fri May 6 12:09:37 2022
write: IOPS=1364, BW=1368MiB/s (1434MB/s)(409MiB/299msec); 0 zone resets
slat (usec): min=259, max=6849, avg=726.75, stdev=494.63
clat (nsec): min=1106, max=5170, avg=1678.61, stdev=304.13
lat (usec): min=261, max=6852, avg=728.88, stdev=495.38
clat percentiles (nsec):
| 1.00th=[ 1144], 5.00th=[ 1256], 10.00th=[ 1384], 20.00th=[ 1480],
| 30.00th=[ 1560], 40.00th=[ 1608], 50.00th=[ 1656], 60.00th=[ 1704],
| 70.00th=[ 1752], 80.00th=[ 1800], 90.00th=[ 1944], 95.00th=[ 2224],
| 99.00th=[ 2416], 99.50th=[ 2448], 99.90th=[ 5152], 99.95th=[ 5152],
| 99.99th=[ 5152]
lat (usec) : 2=91.18%, 4=8.58%, 10=0.25%
cpu : usr=1.68%, sys=41.61%, ctx=1411, majf=0, minf=59
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,408,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=1368MiB/s (1434MB/s), 1368MiB/s-1368MiB/s (1434MB/s-1434MB/s), io=409MiB (429MB), run=299-299msec
Best of three: 2820MB/s
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=read --ramp_time=4
test: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=0): [/(1)][-.-%][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=125973: Fri May 6 12:02:13 2022
read: IOPS=2689, BW=2689MiB/s (2820MB/s)(8192MiB/3046msec)
cpu : usr=0.62%, sys=99.28%, ctx=8, majf=0, minf=263
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=8192,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
READ: bw=2689MiB/s (2820MB/s), 2689MiB/s-2689MiB/s (2820MB/s-2820MB/s), io=8192MiB (8590MB), run=3046-3046msec
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=read --ramp_time=4
test: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=0): [f(1)][-.-%][r=2521MiB/s][r=2521 IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=126019: Fri May 6 12:02:18 2022
read: IOPS=2650, BW=2650MiB/s (2779MB/s)(8192MiB/3091msec)
cpu : usr=0.23%, sys=99.64%, ctx=68, majf=0, minf=263
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=8192,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
READ: bw=2650MiB/s (2779MB/s), 2650MiB/s-2650MiB/s (2779MB/s-2779MB/s), io=8192MiB (8590MB), run=3091-3091msec
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=read --ramp_time=4
test: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=0): [f(1)][-.-%][r=2521MiB/s][r=2521 IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=126075: Fri May 6 12:02:23 2022
read: IOPS=2657, BW=2657MiB/s (2786MB/s)(8192MiB/3083msec)
cpu : usr=0.23%, sys=99.71%, ctx=9, majf=0, minf=264
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=8192,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
READ: bw=2657MiB/s (2786MB/s), 2657MiB/s-2657MiB/s (2786MB/s-2786MB/s), io=8192MiB (8590MB), run=3083-3083msec
Best of three: 2800MB/s
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=write --ramp_time=4
test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=0): [f(1)][-.-%][w=120MiB/s][w=119 IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=126857: Fri May 6 12:03:54 2022
write: IOPS=2664, BW=2670MiB/s (2800MB/s)(526MiB/197msec); 0 zone resets
slat (usec): min=194, max=24500, avg=366.73, stdev=1267.42
clat (nsec): min=1048, max=9697, avg=1338.78, stdev=621.31
lat (usec): min=196, max=24508, avg=368.63, stdev=1269.03
clat percentiles (nsec):
| 1.00th=[ 1080], 5.00th=[ 1112], 10.00th=[ 1128], 20.00th=[ 1160],
| 30.00th=[ 1208], 40.00th=[ 1240], 50.00th=[ 1272], 60.00th=[ 1304],
| 70.00th=[ 1336], 80.00th=[ 1400], 90.00th=[ 1464], 95.00th=[ 1528],
| 99.00th=[ 1944], 99.50th=[ 8256], 99.90th=[ 9664], 99.95th=[ 9664],
| 99.99th=[ 9664]
lat (usec) : 2=99.05%, 4=0.19%, 10=0.76%
cpu : usr=7.65%, sys=62.24%, ctx=91, majf=0, minf=59
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,525,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=2670MiB/s (2800MB/s), 2670MiB/s-2670MiB/s (2800MB/s-2800MB/s), io=526MiB (552MB), run=197-197msec
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=write --ramp_time=4
test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=0): [/(1)][-.-%][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=130880: Fri May 6 12:04:06 2022
write: IOPS=2057, BW=2057MiB/s (2157MB/s)(8192MiB/3982msec); 0 zone resets
cpu : usr=4.02%, sys=51.70%, ctx=8343, majf=0, minf=8
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,8192,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=2057MiB/s (2157MB/s), 2057MiB/s-2057MiB/s (2157MB/s-2157MB/s), io=8192MiB (8590MB), run=3982-3982msec
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=write --ramp_time=4
test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=0): [f(1)][-.-%][w=65.6MiB/s][w=65 IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=135570: Fri May 6 12:04:20 2022
write: IOPS=1347, BW=1352MiB/s (1418MB/s)(288MiB/213msec); 0 zone resets
slat (usec): min=254, max=7432, avg=733.98, stdev=508.90
clat (nsec): min=1122, max=5484, avg=1747.74, stdev=355.75
lat (usec): min=256, max=7435, avg=736.44, stdev=509.92
clat percentiles (nsec):
| 1.00th=[ 1128], 5.00th=[ 1320], 10.00th=[ 1432], 20.00th=[ 1528],
| 30.00th=[ 1624], 40.00th=[ 1656], 50.00th=[ 1720], 60.00th=[ 1768],
| 70.00th=[ 1816], 80.00th=[ 1880], 90.00th=[ 2096], 95.00th=[ 2352],
| 99.00th=[ 2704], 99.50th=[ 2704], 99.90th=[ 5472], 99.95th=[ 5472],
| 99.99th=[ 5472]
lat (usec) : 2=87.80%, 4=11.85%, 10=0.35%
cpu : usr=1.42%, sys=40.57%, ctx=1078, majf=0, minf=60
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,287,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=1352MiB/s (1418MB/s), 1352MiB/s-1352MiB/s (1418MB/s-1418MB/s), io=288MiB (302MB), run=213-213msec
Best of three: 2806MB/s
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=read --ramp_time=4
test: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=0): [f(1)][-.-%][r=2521MiB/s][r=2521 IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=80921: Fri May 6 11:52:14 2022
read: IOPS=2658, BW=2658MiB/s (2787MB/s)(8192MiB/3082msec)
cpu : usr=0.62%, sys=99.25%, ctx=7, majf=0, minf=264
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=8192,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
READ: bw=2658MiB/s (2787MB/s), 2658MiB/s-2658MiB/s (2787MB/s-2787MB/s), io=8192MiB (8590MB), run=3082-3082msec
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=read --ramp_time=4
test: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=0): [/(1)][-.-%][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=80934: Fri May 6 11:52:19 2022
read: IOPS=2676, BW=2676MiB/s (2806MB/s)(8192MiB/3061msec)
cpu : usr=1.14%, sys=98.73%, ctx=7, majf=0, minf=263
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=8192,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
READ: bw=2676MiB/s (2806MB/s), 2676MiB/s-2676MiB/s (2806MB/s-2806MB/s), io=8192MiB (8590MB), run=3061-3061msec
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=read --ramp_time=4
test: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=0): [f(1)][-.-%][r=2521MiB/s][r=2521 IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=80999: Fri May 6 11:52:25 2022
read: IOPS=2657, BW=2657MiB/s (2786MB/s)(8192MiB/3083msec)
cpu : usr=0.49%, sys=99.35%, ctx=14, majf=0, minf=264
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=8192,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
READ: bw=2657MiB/s (2786MB/s), 2657MiB/s-2657MiB/s (2786MB/s-2786MB/s), io=8192MiB (8590MB), run=3083-3083msec
Best of three: 456MB/s
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=write --ramp_time=4
test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=1): [W(1)][82.6%][w=419MiB/s][w=419 IOPS][eta 00m:04s]
test: (groupid=0, jobs=1): err= 0: pid=83991: Fri May 6 11:59:03 2022
write: IOPS=434, BW=435MiB/s (456MB/s)(6471MiB/14874msec); 0 zone resets
slat (usec): min=1758, max=6490, avg=2294.59, stdev=422.95
clat (nsec): min=1443, max=12435, avg=1902.45, stdev=255.82
lat (usec): min=1761, max=6493, avg=2297.09, stdev=422.97
clat percentiles (nsec):
| 1.00th=[ 1608], 5.00th=[ 1672], 10.00th=[ 1720], 20.00th=[ 1768],
| 30.00th=[ 1816], 40.00th=[ 1848], 50.00th=[ 1880], 60.00th=[ 1912],
| 70.00th=[ 1944], 80.00th=[ 1992], 90.00th=[ 2064], 95.00th=[ 2256],
| 99.00th=[ 2544], 99.50th=[ 2608], 99.90th=[ 2736], 99.95th=[ 7456],
| 99.99th=[12480]
bw ( KiB/s): min=409600, max=476088, per=100.00%, avg=446557.93, stdev=18432.47, samples=29
iops : min= 400, max= 464, avg=436.00, stdev=17.96, samples=29
lat (usec) : 2=80.68%, 4=19.26%, 10=0.05%, 20=0.02%
cpu : usr=0.95%, sys=14.50%, ctx=13849, majf=0, minf=58
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,6470,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=435MiB/s (456MB/s), 435MiB/s-435MiB/s (456MB/s-456MB/s), io=6471MiB (6785MB), run=14874-14874msec
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=write --ramp_time=4
test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=1): [W(1)][84.0%][w=446MiB/s][w=446 IOPS][eta 00m:04s]
test: (groupid=0, jobs=1): err= 0: pid=97788: Fri May 6 11:59:33 2022
write: IOPS=403, BW=404MiB/s (424MB/s)(6736MiB/16678msec); 0 zone resets
slat (usec): min=1811, max=7159, avg=2471.57, stdev=394.34
clat (nsec): min=1518, max=21227, avg=2073.76, stdev=419.56
lat (usec): min=1814, max=7162, avg=2474.37, stdev=394.36
clat percentiles (nsec):
| 1.00th=[ 1704], 5.00th=[ 1816], 10.00th=[ 1864], 20.00th=[ 1928],
| 30.00th=[ 1976], 40.00th=[ 2008], 50.00th=[ 2040], 60.00th=[ 2064],
| 70.00th=[ 2128], 80.00th=[ 2160], 90.00th=[ 2288], 95.00th=[ 2448],
| 99.00th=[ 2640], 99.50th=[ 2704], 99.90th=[ 8096], 99.95th=[ 9920],
| 99.99th=[21120]
bw ( KiB/s): min=365274, max=462848, per=99.93%, avg=413298.94, stdev=30686.72, samples=33
iops : min= 356, max= 452, avg=403.52, stdev=29.97, samples=33
lat (usec) : 2=37.77%, 4=62.11%, 10=0.07%, 20=0.03%, 50=0.01%
cpu : usr=1.00%, sys=13.65%, ctx=14084, majf=0, minf=59
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,6735,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=404MiB/s (424MB/s), 404MiB/s-404MiB/s (424MB/s-424MB/s), io=6736MiB (7063MB), run=16678-16678msec
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=write --ramp_time=4
test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=1): [W(1)][83.3%][w=366MiB/s][w=366 IOPS][eta 00m:04s]
test: (groupid=0, jobs=1): err= 0: pid=111267: Fri May 6 11:59:55 2022
write: IOPS=417, BW=417MiB/s (437MB/s)(6416MiB/15381msec); 0 zone resets
slat (usec): min=1771, max=5926, avg=2393.03, stdev=420.57
clat (nsec): min=1560, max=18328, avg=2014.58, stdev=291.81
lat (usec): min=1774, max=5929, avg=2395.77, stdev=420.61
clat percentiles (nsec):
| 1.00th=[ 1704], 5.00th=[ 1784], 10.00th=[ 1832], 20.00th=[ 1880],
| 30.00th=[ 1928], 40.00th=[ 1960], 50.00th=[ 1992], 60.00th=[ 2024],
| 70.00th=[ 2064], 80.00th=[ 2096], 90.00th=[ 2192], 95.00th=[ 2320],
| 99.00th=[ 2608], 99.50th=[ 2672], 99.90th=[ 2864], 99.95th=[ 6112],
| 99.99th=[18304]
bw ( KiB/s): min=370688, max=487424, per=100.00%, avg=428623.10, stdev=34784.37, samples=30
iops : min= 362, max= 476, avg=418.47, stdev=33.97, samples=30
lat (usec) : 2=51.22%, 4=48.71%, 10=0.05%, 20=0.02%
cpu : usr=1.02%, sys=14.07%, ctx=13281, majf=0, minf=59
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,6415,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=417MiB/s (437MB/s), 417MiB/s-417MiB/s (437MB/s-437MB/s), io=6416MiB (6728MB), run=15381-15381msec
Write Performance (Sync: Always, Compression: LZ4, Atime: Off, Dedup: Off, Record Size: 128 KB, vCPU: 22)
Best of three: 449MB/s
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=write --ramp_time=4
test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=0): [f(1)][100.0%][w=379MiB/s][w=379 IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=14464: Fri May 6 12:20:34 2022
write: IOPS=428, BW=429MiB/s (449MB/s)(6534MiB/15244msec); 0 zone resets
slat (usec): min=1756, max=6337, avg=2329.34, stdev=423.70
clat (nsec): min=1538, max=12907, avg=1855.81, stdev=222.37
lat (usec): min=1759, max=6340, avg=2331.71, stdev=423.76
clat percentiles (nsec):
| 1.00th=[ 1592], 5.00th=[ 1640], 10.00th=[ 1672], 20.00th=[ 1720],
| 30.00th=[ 1768], 40.00th=[ 1816], 50.00th=[ 1848], 60.00th=[ 1896],
| 70.00th=[ 1928], 80.00th=[ 1960], 90.00th=[ 2024], 95.00th=[ 2096],
| 99.00th=[ 2256], 99.50th=[ 2352], 99.90th=[ 2864], 99.95th=[ 3440],
| 99.99th=[12864]
bw ( KiB/s): min=385795, max=493568, per=100.00%, avg=439755.87, stdev=29344.74, samples=30
iops : min= 376, max= 482, avg=429.33, stdev=28.63, samples=30
lat (usec) : 2=86.99%, 4=12.96%, 10=0.03%, 20=0.02%
cpu : usr=0.72%, sys=12.98%, ctx=16910, majf=1, minf=58
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,6533,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=429MiB/s (449MB/s), 429MiB/s-429MiB/s (449MB/s-449MB/s), io=6534MiB (6851MB), run=15244-15244msec
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=write --ramp_time=4
test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=1): [W(1)][85.2%][w=344MiB/s][w=344 IOPS][eta 00m:04s]
test: (groupid=0, jobs=1): err= 0: pid=28357: Fri May 6 12:20:59 2022
write: IOPS=360, BW=361MiB/s (378MB/s)(6627MiB/18373msec); 0 zone resets
slat (usec): min=2049, max=6916, avg=2768.50, stdev=441.99
clat (nsec): min=1563, max=12597, avg=1896.78, stdev=251.20
lat (usec): min=2052, max=6918, avg=2771.08, stdev=441.99
clat percentiles (nsec):
| 1.00th=[ 1672], 5.00th=[ 1720], 10.00th=[ 1752], 20.00th=[ 1784],
| 30.00th=[ 1816], 40.00th=[ 1848], 50.00th=[ 1880], 60.00th=[ 1912],
| 70.00th=[ 1944], 80.00th=[ 1976], 90.00th=[ 2040], 95.00th=[ 2096],
| 99.00th=[ 2224], 99.50th=[ 2352], 99.90th=[ 2864], 99.95th=[ 7776],
| 99.99th=[12608]
bw ( KiB/s): min=332440, max=413696, per=100.00%, avg=369744.89, stdev=23645.29, samples=36
iops : min= 324, max= 404, avg=361.00, stdev=23.10, samples=36
lat (usec) : 2=83.46%, 4=16.47%, 10=0.05%, 20=0.03%
cpu : usr=0.69%, sys=11.25%, ctx=16031, majf=0, minf=58
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,6626,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=361MiB/s (378MB/s), 361MiB/s-361MiB/s (378MB/s-378MB/s), io=6627MiB (6949MB), run=18373-18373msec
root@truenas[/mnt/tank]# sync;fio --randrepeat=1 --ioengine=libaio --direct=1 --name=test --filename=test --bs=1M --size=8G --readwrite=write --ramp_time=4
test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=1
fio-3.25
Starting 1 process
Jobs: 1 (f=1): [W(1)][85.2%][w=334MiB/s][w=334 IOPS][eta 00m:04s]
test: (groupid=0, jobs=1): err= 0: pid=39230: Fri May 6 12:21:25 2022
write: IOPS=357, BW=357MiB/s (375MB/s)(6847MiB/19161msec); 0 zone resets
slat (usec): min=2178, max=5919, avg=2794.75, stdev=429.26
clat (nsec): min=1600, max=11995, avg=2022.19, stdev=213.30
lat (usec): min=2180, max=5922, avg=2797.18, stdev=429.03
clat percentiles (nsec):
| 1.00th=[ 1752], 5.00th=[ 1816], 10.00th=[ 1848], 20.00th=[ 1896],
| 30.00th=[ 1944], 40.00th=[ 1976], 50.00th=[ 2008], 60.00th=[ 2040],
| 70.00th=[ 2096], 80.00th=[ 2128], 90.00th=[ 2192], 95.00th=[ 2256],
| 99.00th=[ 2384], 99.50th=[ 2448], 99.90th=[ 2928], 99.95th=[ 3152],
| 99.99th=[11968]
bw ( KiB/s): min=331776, max=399360, per=100.00%, avg=365968.87, stdev=18814.20, samples=38
iops : min= 324, max= 390, avg=357.32, stdev=18.36, samples=38
lat (usec) : 2=47.72%, 4=52.23%, 10=0.03%, 20=0.01%
cpu : usr=0.67%, sys=11.45%, ctx=16222, majf=0, minf=58
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,6846,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
Run status group 0 (all jobs):
WRITE: bw=357MiB/s (375MB/s), 357MiB/s-357MiB/s (375MB/s-375MB/s), io=6847MiB (7180MB), run=19161-19161msec