fio - ZFS 0.6.4.2 - 4kb zvol

gistfile1.txt

# FIRST RUN
  write: io=49474MB, bw=168858KB/s, iops=42214, runt=300021msec
    slat (usec): min=2, max=100676, avg=22.43, stdev=504.87
    clat (usec): min=15, max=100974, avg=711.14, stdev=3180.29
     lat (usec): min=139, max=100980, avg=733.67, stdev=3248.00
    clat percentiles (usec):
     |  1.00th=[  147],  5.00th=[  155], 10.00th=[  165], 20.00th=[  175],
     | 30.00th=[  183], 40.00th=[  191], 50.00th=[  199], 60.00th=[  207],
     | 70.00th=[  215], 80.00th=[  225], 90.00th=[  253], 95.00th=[ 5536],
     | 99.00th=[10048], 99.50th=[12096], 99.90th=[51968], 99.95th=[68096],
     | 99.99th=[86528]
    bw (KB  /s): min= 1524, max=807840, per=100.00%, avg=170087.31, stdev=127242.78
    lat (usec) : 20=0.01%, 250=89.58%, 500=3.94%, 750=0.22%, 1000=0.20%
    lat (msec) : 2=0.71%, 4=0.02%, 10=4.33%, 20=0.72%, 50=0.18%
    lat (msec) : 100=0.11%, 250=0.01%
  cpu          : usr=5.23%, sys=25.05%, ctx=27353, majf=0, minf=24
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=12665274/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
#
# SECOND RUN
  write: io=49356MB, bw=168463KB/s, iops=42115, runt=300010msec
    slat (usec): min=1, max=94565, avg=22.56, stdev=483.98
    clat (usec): min=5, max=94851, avg=712.75, stdev=2937.73
     lat (usec): min=136, max=94856, avg=735.41, stdev=2997.30
    clat percentiles (usec):
     |  1.00th=[  143],  5.00th=[  151], 10.00th=[  161], 20.00th=[  171],
     | 30.00th=[  181], 40.00th=[  189], 50.00th=[  197], 60.00th=[  205],
     | 70.00th=[  213], 80.00th=[  225], 90.00th=[  262], 95.00th=[ 5600],
     | 99.00th=[10176], 99.50th=[11968], 99.90th=[45824], 99.95th=[58112],
     | 99.99th=[80384]
    bw (KB  /s): min= 1676, max=831952, per=100.00%, avg=169747.55, stdev=125706.91
    lat (usec) : 10=0.01%, 250=88.47%, 500=4.56%, 750=0.29%, 1000=0.22%
    lat (msec) : 2=0.89%, 4=0.03%, 10=4.43%, 20=0.86%, 50=0.18%
    lat (msec) : 100=0.08%
  cpu          : usr=5.24%, sys=25.16%, ctx=27948, majf=0, minf=24
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=12635182/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
#
# THIRD RUN
  write: io=52987MB, bw=180860KB/s, iops=45215, runt=300003msec
    slat (usec): min=1, max=102560, avg=20.92, stdev=393.92
    clat (usec): min=15, max=110971, avg=663.93, stdev=2658.84
     lat (usec): min=138, max=110976, avg=684.96, stdev=2720.65
    clat percentiles (usec):
     |  1.00th=[  149],  5.00th=[  157], 10.00th=[  163], 20.00th=[  171],
     | 30.00th=[  181], 40.00th=[  189], 50.00th=[  195], 60.00th=[  203],
     | 70.00th=[  211], 80.00th=[  223], 90.00th=[  255], 95.00th=[ 5536],
     | 99.00th=[ 9920], 99.50th=[11328], 99.90th=[27008], 99.95th=[54016],
     | 99.99th=[93696]
    bw (KB  /s): min= 1200, max=788912, per=100.00%, avg=181476.78, stdev=115599.43
    lat (usec) : 20=0.01%, 250=89.20%, 500=4.09%, 750=0.24%, 1000=0.19%
    lat (msec) : 2=0.83%, 4=0.03%, 10=4.45%, 20=0.84%, 50=0.08%
    lat (msec) : 100=0.05%, 250=0.01%
  cpu          : usr=5.65%, sys=26.73%, ctx=28756, majf=0, minf=24
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=13564669/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
#
# FORTH RUN
  write: io=49338MB, bw=168400KB/s, iops=42100, runt=300009msec
    slat (usec): min=1, max=94581, avg=22.51, stdev=481.59
    clat (usec): min=4, max=94820, avg=713.07, stdev=2838.48
     lat (usec): min=141, max=94831, avg=735.68, stdev=2893.12
    clat percentiles (usec):
     |  1.00th=[  147],  5.00th=[  155], 10.00th=[  163], 20.00th=[  173],
     | 30.00th=[  183], 40.00th=[  191], 50.00th=[  199], 60.00th=[  207],
     | 70.00th=[  217], 80.00th=[  229], 90.00th=[  258], 95.00th=[ 5728],
     | 99.00th=[10048], 99.50th=[11968], 99.90th=[43776], 99.95th=[51968],
     | 99.99th=[80384]
    bw (KB  /s): min= 2453, max=808344, per=100.00%, avg=169424.80, stdev=122442.66
    lat (usec) : 10=0.01%, 250=88.89%, 500=4.29%, 750=0.20%, 1000=0.18%
    lat (msec) : 2=0.73%, 4=0.02%, 10=4.60%, 20=0.83%, 50=0.18%
    lat (msec) : 100=0.06%
  cpu          : usr=5.15%, sys=25.05%, ctx=29038, majf=0, minf=25
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=12630404/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
#
# FIFTH RUN
  write: io=59357MB, bw=202604KB/s, iops=50650, runt=300004msec
    slat (usec): min=1, max=88701, avg=18.53, stdev=323.84
    clat (usec): min=5, max=88955, avg=592.73, stdev=1904.31
     lat (usec): min=137, max=88968, avg=611.37, stdev=1940.53
    clat percentiles (usec):
     |  1.00th=[  145],  5.00th=[  153], 10.00th=[  161], 20.00th=[  173],
     | 30.00th=[  183], 40.00th=[  193], 50.00th=[  201], 60.00th=[  209],
     | 70.00th=[  219], 80.00th=[  233], 90.00th=[  266], 95.00th=[ 5344],
     | 99.00th=[ 8768], 99.50th=[10176], 99.90th=[16512], 99.95th=[25984],
     | 99.99th=[51968]
    bw (KB  /s): min= 2227, max=823712, per=100.00%, avg=203210.13, stdev=99851.01
    lat (usec) : 10=0.01%, 250=87.44%, 500=5.40%, 750=0.45%, 1000=0.37%
    lat (msec) : 2=1.28%, 4=0.01%, 10=4.47%, 20=0.51%, 50=0.06%
    lat (msec) : 100=0.01%
  cpu          : usr=6.31%, sys=31.40%, ctx=28547, majf=0, minf=28
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=15195489/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
     
     Average write IOPS: 44459