fio - ZFS 0.7.0rc4 - zfs file system

gistfile1.txt

# FIRST RUN
  write: io=66674MB, bw=227580KB/s, iops=56894, runt=300000msec
    slat (usec): min=8, max=110578, avg=15.98, stdev=128.26
    clat (usec): min=1, max=111181, avg=527.73, stdev=934.56
     lat (usec): min=12, max=111194, avg=543.86, stdev=955.93
    clat percentiles (usec):
     |  1.00th=[  370],  5.00th=[  378], 10.00th=[  382], 20.00th=[  390],
     | 30.00th=[  398], 40.00th=[  406], 50.00th=[  410], 60.00th=[  418],
     | 70.00th=[  426], 80.00th=[  442], 90.00th=[  628], 95.00th=[ 1112],
     | 99.00th=[ 2416], 99.50th=[ 2736], 99.90th=[ 7392], 99.95th=[13248],
     | 99.99th=[41216]
    bw (KB  /s): min=41904, max=318920, per=99.84%, avg=227214.89, stdev=69463.30
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=85.80%, 750=6.15%, 1000=2.33%
    lat (msec) : 2=3.88%, 4=1.60%, 10=0.16%, 20=0.03%, 50=0.02%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=9.61%, sys=87.20%, ctx=165208, majf=0, minf=24
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=17068466/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
#
# SECOND RUN
  write: io=62693MB, bw=213992KB/s, iops=53498, runt=300000msec
    slat (usec): min=9, max=114319, avg=17.07, stdev=167.99
    clat (usec): min=1, max=115315, avg=561.23, stdev=1258.56
     lat (usec): min=13, max=115329, avg=578.44, stdev=1287.18
    clat percentiles (usec):
     |  1.00th=[  374],  5.00th=[  382], 10.00th=[  386], 20.00th=[  394],
     | 30.00th=[  402], 40.00th=[  410], 50.00th=[  418], 60.00th=[  426],
     | 70.00th=[  434], 80.00th=[  450], 90.00th=[  684], 95.00th=[ 1240],
     | 99.00th=[ 2672], 99.50th=[ 3088], 99.90th=[ 9920], 99.95th=[20864],
     | 99.99th=[63744]
    bw (KB  /s): min=28048, max=312896, per=97.91%, avg=209519.53, stdev=75299.23
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=84.17%, 750=6.87%, 1000=2.49%
    lat (msec) : 2=4.12%, 4=2.09%, 10=0.15%, 20=0.04%, 50=0.04%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=8.68%, sys=86.81%, ctx=178756, majf=0, minf=24
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=16049421/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
#
# THIRD RUN
  write: io=67418MB, bw=230120KB/s, iops=57530, runt=300001msec
    slat (usec): min=8, max=112072, avg=15.84, stdev=122.46
    clat (usec): min=1, max=114723, avg=521.90, stdev=891.96
     lat (usec): min=12, max=114827, avg=537.89, stdev=912.19
    clat percentiles (usec):
     |  1.00th=[  370],  5.00th=[  374], 10.00th=[  378], 20.00th=[  386],
     | 30.00th=[  394], 40.00th=[  402], 50.00th=[  406], 60.00th=[  414],
     | 70.00th=[  422], 80.00th=[  438], 90.00th=[  612], 95.00th=[ 1096],
     | 99.00th=[ 2416], 99.50th=[ 2736], 99.90th=[ 6752], 99.95th=[12096],
     | 99.99th=[37632]
    bw (KB  /s): min=48192, max=319776, per=100.00%, avg=231289.28, stdev=71720.32
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=85.99%, 750=6.10%, 1000=2.29%
    lat (msec) : 2=3.78%, 4=1.59%, 10=0.18%, 20=0.03%, 50=0.02%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=9.54%, sys=87.46%, ctx=166134, majf=0, minf=24
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=17259063/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
#
# FOURTH RUN
  write: io=65929MB, bw=225039KB/s, iops=56259, runt=300000msec
    slat (usec): min=8, max=112245, avg=16.24, stdev=142.82
    clat (usec): min=1, max=119494, avg=533.66, stdev=1041.57
     lat (usec): min=12, max=119724, avg=550.05, stdev=1064.93
    clat percentiles (usec):
     |  1.00th=[  366],  5.00th=[  374], 10.00th=[  378], 20.00th=[  386],
     | 30.00th=[  394], 40.00th=[  402], 50.00th=[  406], 60.00th=[  414],
     | 70.00th=[  422], 80.00th=[  438], 90.00th=[  644], 95.00th=[ 1160],
     | 99.00th=[ 2512], 99.50th=[ 2864], 99.90th=[ 7456], 99.95th=[14528],
     | 99.99th=[50432]
    bw (KB  /s): min=41664, max=321288, per=100.00%, avg=225058.48, stdev=75673.61
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=85.37%, 750=6.29%, 1000=2.35%
    lat (msec) : 2=3.95%, 4=1.81%, 10=0.15%, 20=0.03%, 50=0.03%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=9.27%, sys=87.13%, ctx=172041, majf=0, minf=24
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=16877894/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
#
# FIFTH RUN
  write: io=63520MB, bw=216815KB/s, iops=54203, runt=300000msec
    slat (usec): min=8, max=109387, avg=16.87, stdev=148.46
    clat (usec): min=1, max=112551, avg=553.91, stdev=1151.61
     lat (usec): min=12, max=112590, avg=570.93, stdev=1178.58
    clat percentiles (usec):
     |  1.00th=[  370],  5.00th=[  378], 10.00th=[  382], 20.00th=[  390],
     | 30.00th=[  398], 40.00th=[  406], 50.00th=[  414], 60.00th=[  418],
     | 70.00th=[  430], 80.00th=[  446], 90.00th=[  668], 95.00th=[ 1224],
     | 99.00th=[ 2736], 99.50th=[ 3120], 99.90th=[ 9920], 99.95th=[20608],
     | 99.99th=[55552]
    bw (KB  /s): min=38944, max=319936, per=98.15%, avg=212797.81, stdev=78274.94
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=84.67%, 750=6.56%, 1000=2.38%
    lat (msec) : 2=3.98%, 4=2.18%, 10=0.12%, 20=0.05%, 50=0.04%
    lat (msec) : 100=0.01%, 250=0.01%
  cpu          : usr=8.93%, sys=86.86%, ctx=179388, majf=0, minf=24
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=16261145/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
     
     Average write IOPS: 55677