gistfile1.txt

root@ubuntu:/mnt/test# fio --name=randwrite --ioengine=libaio --iodepth=1 --rw=randwrite --bs=4k --direct=0 --size=512M --numjobs=8 --runtime=240 --group_reporting
randwrite: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1
...
fio-3.1
Starting 8 processes
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
Jobs: 8 (f=8): [w(8)][27.8%][r=0KiB/s,w=89.6MiB/s][r=0,w=22.9k IOPS][eta 00m:26sJobs: 8 (f=8): [w(8)][28.9%][r=0KiB/s,w=69.7MiB/s][r=0,w=17.8k IOPS][eta 00m:27sJobs: 8 (f=8): [w(8)][30.0%][r=0KiB/s,w=43.6MiB/s][r=0,w=11.2k IOPS][eta 00m:28sJobs: 8 (f=8): [w(8)][31.0%][r=0KiB/s,w=34.6MiB/s][r=0,w=8865 IOPS][eta 00m:29s]Jobs: 8 (f=8): [w(8)][32.1%][r=0KiB/s,w=6818^Cbs: 8 (f=8): [w(8)][35.4%][r=0KiB/s,w=4060KiB/s][r=0,w=1015 IOPS][eta 01m:55s]
fio: terminating on signal 2

randwrite: (groupid=0, jobs=8): err= 0: pid=9350: Sat Jan 19 03:26:53 2019
  write: IOPS=6083, BW=23.8MiB/s (24.9MB/s)(1480MiB/62269msec)
    slat (usec): min=5, max=139185, avg=1307.52, stdev=2922.16
    clat (nsec): min=399, max=81665k, avg=3473.66, stdev=201307.61
     lat (usec): min=5, max=139187, avg=1311.95, stdev=2936.57
    clat percentiles (nsec):
     |  1.00th=[    442],  5.00th=[    482], 10.00th=[    532],
     | 20.00th=[    564], 30.00th=[    596], 40.00th=[    644],
     | 50.00th=[    700], 60.00th=[    772], 70.00th=[    884],
     | 80.00th=[   1816], 90.00th=[   5024], 95.00th=[   5216],
     | 99.00th=[   5728], 99.50th=[   9152], 99.90th=[  96768],
     | 99.95th=[ 329728], 99.99th=[4292608]
   bw (  KiB/s): min=  264, max=39128, per=12.54%, avg=3051.83, stdev=6464.61, samples=992
   iops        : min=   66, max= 9782, avg=762.87, stdev=1616.12, samples=992
  lat (nsec)   : 500=6.40%, 750=51.02%, 1000=16.99%
  lat (usec)   : 2=6.04%, 4=5.19%, 10=13.90%, 20=0.23%, 50=0.09%
  lat (usec)   : 100=0.05%, 250=0.04%, 500=0.02%, 750=0.01%, 1000=0.01%
  lat (msec)   : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
  lat (msec)   : 100=0.01%
  cpu          : usr=0.33%, sys=3.91%, ctx=246509, majf=0, minf=70
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwt: total=0,378841,0, short=0,0,0, dropped=0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
  WRITE: bw=23.8MiB/s (24.9MB/s), 23.8MiB/s-23.8MiB/s (24.9MB/s-24.9MB/s), io=1480MiB (1552MB), run=62269-62269msec
root@ubuntu:/mnt/test# cd
root@ubuntu:~# zpool export rpool
root@ubuntu:~# zpool create -f -o ashift=12 -O atime=off -O canmount=off -O compression=lz4 -O normalization=formD       -O xattr=sa -O mountpoint=/ -R /mnt       rpool /dev/disk/by-id/nvme-LENSE20256GMSP34MEAT2TA_1197069906090-part1
root@ubuntu:~# zfs create rpool/test
root@ubuntu:~# cd /mnt/test/
root@ubuntu:/mnt/test# fio --name=randwrite --ioengine=libaio --iodepth=1 --rw=randwrite --bs=4k --direct=0 --size=512M --numjobs=8 --runtime=240 --group_reporting
randwrite: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1
...
fio-3.1
Starting 8 processes
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
Jobs: 8 (f=8): [w(8)][100.0%][r=0KiB/s,w=4216KiB/s][r=0,w=1054 IOPS][eta 00m:00s]
randwrite: (groupid=0, jobs=8): err= 0: pid=28206: Sat Jan 19 03:32:04 2019
  write: IOPS=2299, BW=9200KiB/s (9421kB/s)(2156MiB/240008msec)
    slat (usec): min=4, max=82250, avg=3467.81, stdev=3155.54
    clat (nsec): min=393, max=38411k, avg=3919.46, stdev=83858.28
     lat (usec): min=5, max=82252, avg=3472.97, stdev=3158.43
    clat percentiles (nsec):
     |  1.00th=[   454],  5.00th=[   532], 10.00th=[   596], 20.00th=[   708],
     | 30.00th=[  1192], 40.00th=[  2992], 50.00th=[  3568], 60.00th=[  4896],
     | 70.00th=[  5024], 80.00th=[  5088], 90.00th=[  5216], 95.00th=[  5344],
     | 99.00th=[  5728], 99.50th=[  6688], 99.90th=[ 28800], 99.95th=[ 68096],
     | 99.99th=[937984]
   bw (  KiB/s): min=  320, max=32336, per=12.50%, avg=1150.21, stdev=2619.81, samples=3840
   iops        : min=   80, max= 8084, avg=287.48, stdev=654.94, samples=3840
  lat (nsec)   : 500=3.45%, 750=18.68%, 1000=6.25%
  lat (usec)   : 2=5.48%, 4=18.85%, 10=46.97%, 20=0.16%, 50=0.11%
  lat (usec)   : 100=0.02%, 250=0.02%, 500=0.01%, 750=0.01%, 1000=0.01%
  lat (msec)   : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
  cpu          : usr=0.30%, sys=3.02%, ctx=469095, majf=0, minf=77
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwt: total=0,552014,0, short=0,0,0, dropped=0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
  WRITE: bw=9200KiB/s (9421kB/s), 9200KiB/s-9200KiB/s (9421kB/s-9421kB/s), io=2156MiB (2261MB), run=240008-240008msec
root@ubuntu:/mnt/test# fio --name=randwrite --ioengine=sync --iodepth=1 --rw=randwrite --bs=4k --direct=0 --size=512M --numjobs=8 --runtime=240 --group_reporting
randwrite: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=sync, iodepth=1
...
fio-3.1
Starting 8 processes
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
randwrite: Laying out IO file (1 file / 512MiB)
fio: native_fallocate call failed: Operation not supported
Jobs: 8 (f=8): [w(8)][100.0%][r=0KiB/s,w=3288KiB/s][r=0,w=822 IOPS][eta 00m:00s] 
randwrite: (groupid=0, jobs=8): err= 0: pid=32326: Sat Jan 19 05:03:26 2019
  write: IOPS=1694, BW=6780KiB/s (6943kB/s)(1589MiB/240012msec)
    clat (usec): min=4, max=112394, avg=4711.37, stdev=4873.88
     lat (usec): min=4, max=112394, avg=4712.18, stdev=4874.28
    clat percentiles (usec):
     |  1.00th=[    7],  5.00th=[   23], 10.00th=[   55], 20.00th=[  139],
     | 30.00th=[  437], 40.00th=[ 1811], 50.00th=[ 2999], 60.00th=[ 5342],
     | 70.00th=[ 6456], 80.00th=[ 8848], 90.00th=[12649], 95.00th=[14091],
     | 99.00th=[17171], 99.50th=[18220], 99.90th=[19268], 99.95th=[20055],
     | 99.99th=[33817]
   bw (  KiB/s): min=  200, max=53251, per=12.49%, avg=846.91, stdev=2197.45, samples=3840
   iops        : min=   50, max=13312, avg=211.68, stdev=549.33, samples=3840
  lat (usec)   : 10=3.39%, 20=0.39%, 50=5.26%, 100=9.03%, 250=7.34%
  lat (usec)   : 500=5.23%, 750=1.59%, 1000=1.09%
  lat (msec)   : 2=8.38%, 4=14.16%, 10=25.84%, 20=18.24%, 50=0.04%
  lat (msec)   : 100=0.01%, 250=0.01%
  cpu          : usr=0.20%, sys=2.30%, ctx=353874, majf=0, minf=68
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwt: total=0,406819,0, short=0,0,0, dropped=0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
  WRITE: bw=6780KiB/s (6943kB/s), 6780KiB/s-6780KiB/s (6943kB/s-6943kB/s), io=1589MiB (1666MB), run=240012-240012msec
root@ubuntu:/mnt/test#