fio - ZFS 0.7.0rc4 - 4kb zvol with zvol_request_sync=1

gistfile1.txt

# FIRST RUN
  write: io=29141MB, bw=99467KB/s, iops=24866, runt=300008msec
    slat (usec): min=1, max=40195, avg=38.91, stdev=587.62
    clat (usec): min=7, max=40697, avg=1206.90, stdev=3075.50
     lat (usec): min=130, max=40702, avg=1245.92, stdev=3120.81
    clat percentiles (usec):
     |  1.00th=[  143],  5.00th=[  147], 10.00th=[  153], 20.00th=[  161],
     | 30.00th=[  165], 40.00th=[  171], 50.00th=[  177], 60.00th=[  183],
     | 70.00th=[  195], 80.00th=[  217], 90.00th=[ 6752], 95.00th=[ 9024],
     | 99.00th=[12736], 99.50th=[14656], 99.90th=[20352], 99.95th=[24448],
     | 99.99th=[39168]
    bw (KB  /s): min=13905, max=821224, per=100.00%, avg=99860.64, stdev=104071.41
    lat (usec) : 10=0.01%, 250=84.11%, 500=4.22%, 750=0.49%, 1000=0.08%
    lat (msec) : 2=0.04%, 4=0.01%, 10=7.65%, 20=3.29%, 50=0.11%
  cpu          : usr=3.20%, sys=11.93%, ctx=58192, majf=0, minf=24
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=7460203/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
#
# SECOND RUN
  write: io=29273MB, bw=99918KB/s, iops=24979, runt=300001msec
    slat (usec): min=1, max=40308, avg=38.72, stdev=577.40
    clat (usec): min=6, max=40685, avg=1201.45, stdev=3020.71
     lat (usec): min=130, max=40691, avg=1240.29, stdev=3065.16
    clat percentiles (usec):
     |  1.00th=[  145],  5.00th=[  151], 10.00th=[  155], 20.00th=[  163],
     | 30.00th=[  169], 40.00th=[  175], 50.00th=[  181], 60.00th=[  189],
     | 70.00th=[  199], 80.00th=[  223], 90.00th=[ 6752], 95.00th=[ 9024],
     | 99.00th=[12992], 99.50th=[14144], 99.90th=[17024], 99.95th=[22400],
     | 99.99th=[39168]
    bw (KB  /s): min=15256, max=799344, per=100.00%, avg=100257.70, stdev=101599.29
    lat (usec) : 10=0.01%, 250=83.49%, 500=4.74%, 750=0.61%, 1000=0.09%
    lat (msec) : 2=0.04%, 4=0.01%, 10=7.56%, 20=3.40%, 50=0.06%
  cpu          : usr=3.21%, sys=12.28%, ctx=60861, majf=0, minf=24
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=7493853/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
#
# THIRD RUN
  write: io=27804MB, bw=94906KB/s, iops=23726, runt=300000msec
    slat (usec): min=1, max=40450, avg=40.88, stdev=645.76
    clat (usec): min=2, max=40712, avg=1264.87, stdev=3389.50
     lat (usec): min=6, max=40717, avg=1305.86, stdev=3439.90
    clat percentiles (usec):
     |  1.00th=[  145],  5.00th=[  151], 10.00th=[  155], 20.00th=[  159],
     | 30.00th=[  165], 40.00th=[  171], 50.00th=[  177], 60.00th=[  183],
     | 70.00th=[  193], 80.00th=[  217], 90.00th=[ 6880], 95.00th=[ 8896],
     | 99.00th=[15552], 99.50th=[16768], 99.90th=[36096], 99.95th=[39168],
     | 99.99th=[39680]
    bw (KB  /s): min= 7913, max=787424, per=100.00%, avg=95336.97, stdev=104351.01
    lat (usec) : 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=84.22%, 500=4.24%, 750=0.50%, 1000=0.08%
    lat (msec) : 2=0.04%, 4=0.01%, 10=7.35%, 20=3.28%, 50=0.28%
  cpu          : usr=2.95%, sys=11.44%, ctx=56171, majf=0, minf=24
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=7117917/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
#
# FOURTH RUN
  write: io=26466MB, bw=90336KB/s, iops=22583, runt=300007msec
    slat (usec): min=1, max=39922, avg=43.01, stdev=663.45
    clat (usec): min=7, max=40668, avg=1328.83, stdev=3469.80
     lat (usec): min=133, max=40675, avg=1371.95, stdev=3520.93
    clat percentiles (usec):
     |  1.00th=[  143],  5.00th=[  149], 10.00th=[  153], 20.00th=[  159],
     | 30.00th=[  165], 40.00th=[  169], 50.00th=[  175], 60.00th=[  183],
     | 70.00th=[  193], 80.00th=[  215], 90.00th=[ 6944], 95.00th=[ 9664],
     | 99.00th=[15040], 99.50th=[16512], 99.90th=[31104], 99.95th=[35072],
     | 99.99th=[39168]
    bw (KB  /s): min=12762, max=811432, per=100.00%, avg=90861.40, stdev=107463.53
    lat (usec) : 10=0.01%, 250=84.15%, 500=3.96%, 750=0.45%, 1000=0.07%
    lat (msec) : 2=0.04%, 4=0.01%, 10=6.95%, 20=4.17%, 50=0.21%
  cpu          : usr=2.85%, sys=10.76%, ctx=51965, majf=0, minf=24
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=6775338/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
#
# FIFTH RUN
  write: io=25671MB, bw=87620KB/s, iops=21905, runt=300007msec
    slat (usec): min=1, max=55835, avg=44.38, stdev=796.23
    clat (usec): min=5, max=56045, avg=1370.00, stdev=4213.29
     lat (usec): min=131, max=56049, avg=1414.49, stdev=4277.41
    clat percentiles (usec):
     |  1.00th=[  145],  5.00th=[  151], 10.00th=[  153], 20.00th=[  159],
     | 30.00th=[  165], 40.00th=[  171], 50.00th=[  177], 60.00th=[  183],
     | 70.00th=[  193], 80.00th=[  217], 90.00th=[ 6752], 95.00th=[ 8896],
     | 99.00th=[15040], 99.50th=[38144], 99.90th=[46848], 99.95th=[51968],
     | 99.99th=[56064]
    bw (KB  /s): min= 2353, max=802888, per=100.00%, avg=88250.11, stdev=109554.35
    lat (usec) : 10=0.01%, 250=84.05%, 500=4.22%, 750=0.51%, 1000=0.08%
    lat (msec) : 2=0.03%, 4=0.01%, 10=7.73%, 20=2.73%, 50=0.56%
    lat (msec) : 100=0.07%
  cpu          : usr=2.74%, sys=10.51%, ctx=51486, majf=0, minf=24
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=6571675/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
     
     Average write IOPS: 23612