4krw.txt

[global]
name=4k random rw 4ios in 32 queues
filename=fio.bin
ioengine=libaio
direct=1
bs=4k
rw=randrw
iodepth=32
numjobs=8
buffered=0
size=128GB
randrepeat=0
norandommap
refill_buffers
group_reporting
[job1]

fioout.txt

pnorman@legolas:~$ fio tilerw.txt
job1: (g=0): rw=randrw, bs=4K-4K/4K-4K/4K-4K, ioengine=libaio, iodepth=4
...
fio-2.10
Starting 8 processes
job1: Laying out IO file(s) (1 file(s) / 8192MB)
Jobs: 8 (f=8): [m(8)] [100.0% done] [79904KB/79648KB/0KB /s] [19.1K/19.1K/0 iops] [eta 00m:00s]
job1: (groupid=0, jobs=8): err= 0: pid=22330: Mon Jan 30 11:33:59 2017
  read : io=32783MB, bw=135129KB/s, iops=33782, runt=248429msec
    slat (usec): min=1, max=12137, avg=18.95, stdev=18.75
    clat (usec): min=1, max=305991, avg=573.96, stdev=1509.96
     lat (usec): min=21, max=306018, avg=593.09, stdev=1509.56
    clat percentiles (usec):
     |  1.00th=[   78],  5.00th=[   85], 10.00th=[   94], 20.00th=[  114],
     | 30.00th=[  155], 40.00th=[  215], 50.00th=[  290], 60.00th=[  382],
     | 70.00th=[  524], 80.00th=[  756], 90.00th=[ 1192], 95.00th=[ 1464],
     | 99.00th=[ 5088], 99.50th=[ 8256], 99.90th=[13376], 99.95th=[15936],
     | 99.99th=[47872]
    bw (KB  /s): min= 2248, max=30510, per=12.53%, avg=16930.93, stdev=6023.26
  write: io=32753MB, bw=135003KB/s, iops=33750, runt=248429msec
    slat (usec): min=1, max=5963, avg=19.86, stdev=18.60
    clat (usec): min=0, max=305871, avg=328.67, stdev=3704.16
     lat (usec): min=12, max=305941, avg=348.71, stdev=3703.87
    clat percentiles (usec):
     |  1.00th=[   13],  5.00th=[   16], 10.00th=[   18], 20.00th=[   20],
     | 30.00th=[   21], 40.00th=[   23], 50.00th=[   27], 60.00th=[   34],
     | 70.00th=[   45], 80.00th=[   63], 90.00th=[  105], 95.00th=[  438],
     | 99.00th=[ 6752], 99.50th=[10944], 99.90th=[31104], 99.95th=[58112],
     | 99.99th=[189440]
    bw (KB  /s): min= 2144, max=31125, per=12.53%, avg=16914.81, stdev=6035.46
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=9.61%, 50=27.06%
    lat (usec) : 100=15.03%, 250=17.48%, 500=12.73%, 750=6.11%, 1000=3.32%
    lat (msec) : 2=5.93%, 4=1.05%, 10=1.23%, 20=0.26%, 50=0.13%
    lat (msec) : 100=0.02%, 250=0.02%, 500=0.01%
  cpu          : usr=3.52%, sys=18.91%, ctx=9331221, majf=0, minf=103
  IO depths    : 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=8392520/w=8384696/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: io=32783MB, aggrb=135129KB/s, minb=135129KB/s, maxb=135129KB/s, mint=248429msec, maxt=248429msec
  WRITE: io=32753MB, aggrb=135003KB/s, minb=135003KB/s, maxb=135003KB/s, mint=248429msec, maxt=248429msec

Disk stats (read/write):
    md2: ios=8387393/8379953, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=4196295/4192508, aggrmerge=2/117, aggrticks=2308730/1298878, aggrin_queue=3649510, aggrutil=84.76%
  nvme0n1: ios=4195169/4192184, merge=5/184, ticks=1737448/1381604, in_queue=3153460, util=79.52%
  nvme1n1: ios=4197421/4192832, merge=0/51, ticks=2880012/1216152, in_queue=4145560, util=84.76%

tilerw.txt

[global]
name=4k random rw 4ios in 32 queues
filename=fio.bin
ioengine=libaio
direct=1
bs=4k
rw=randrw
iodepth=4
numjobs=8
buffered=0
size=8GB
randrepeat=0
norandommap
refill_buffers
group_reporting
[job1]