fio - ZFS 0.6.4.2 - zfs file system

gistfile1.txt

# FIRST RUN
  write: io=74507MB, bw=254319KB/s, iops=63579, runt=300000msec
    slat (usec): min=11, max=6063, avg=14.14, stdev= 7.56
    clat (usec): min=1, max=13242, avg=472.25, stdev=166.23
     lat (usec): min=15, max=13319, avg=486.54, stdev=171.60
    clat percentiles (usec):
     |  1.00th=[  442],  5.00th=[  446], 10.00th=[  446], 20.00th=[  450],
     | 30.00th=[  454], 40.00th=[  454], 50.00th=[  458], 60.00th=[  458],
     | 70.00th=[  462], 80.00th=[  462], 90.00th=[  466], 95.00th=[  474],
     | 99.00th=[  940], 99.50th=[ 1448], 99.90th=[ 2608], 99.95th=[ 4512],
     | 99.99th=[ 5280]
    bw (KB  /s): min=99528, max=268712, per=99.56%, avg=253189.00, stdev=33090.74
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=97.30%, 750=1.33%, 1000=0.45%
    lat (msec) : 2=0.68%, 4=0.17%, 10=0.07%, 20=0.01%
  cpu          : usr=11.00%, sys=88.99%, ctx=2436, majf=0, minf=23
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=19073892/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
#
# SECOND RUN
  write: io=76676MB, bw=261722KB/s, iops=65430, runt=300000msec
    slat (usec): min=10, max=6427, avg=13.67, stdev= 7.29
    clat (usec): min=1, max=14227, avg=458.92, stdev=161.67
     lat (usec): min=13, max=14334, avg=472.74, stdev=166.90
    clat percentiles (usec):
     |  1.00th=[  430],  5.00th=[  434], 10.00th=[  434], 20.00th=[  438],
     | 30.00th=[  438], 40.00th=[  442], 50.00th=[  442], 60.00th=[  446],
     | 70.00th=[  446], 80.00th=[  450], 90.00th=[  454], 95.00th=[  458],
     | 99.00th=[  900], 99.50th=[ 1400], 99.90th=[ 2480], 99.95th=[ 4448],
     | 99.99th=[ 5216]
    bw (KB  /s): min=104080, max=275792, per=99.34%, avg=259993.79, stdev=33544.40
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=97.36%, 750=1.34%, 1000=0.43%
    lat (msec) : 2=0.64%, 4=0.15%, 10=0.07%, 20=0.01%
  cpu          : usr=11.45%, sys=88.53%, ctx=2594, majf=0, minf=23
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=19629145/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
#
# THIRD RUN
  write: io=76527MB, bw=261211KB/s, iops=65302, runt=300000msec
    slat (usec): min=11, max=611, avg=13.70, stdev= 7.75
    clat (usec): min=1, max=8755, avg=459.81, stdev=187.50
     lat (usec): min=13, max=8916, avg=473.66, stdev=193.58
    clat percentiles (usec):
     |  1.00th=[  430],  5.00th=[  434], 10.00th=[  434], 20.00th=[  438],
     | 30.00th=[  442], 40.00th=[  442], 50.00th=[  442], 60.00th=[  446],
     | 70.00th=[  446], 80.00th=[  450], 90.00th=[  454], 95.00th=[  458],
     | 99.00th=[  884], 99.50th=[ 1368], 99.90th=[ 2576], 99.95th=[ 5536],
     | 99.99th=[ 6752]
    bw (KB  /s): min=93696, max=275584, per=99.27%, avg=259304.05, stdev=36589.14
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=97.36%, 750=1.36%, 1000=0.43%
    lat (msec) : 2=0.62%, 4=0.15%, 10=0.08%
  cpu          : usr=11.54%, sys=88.46%, ctx=2292, majf=0, minf=23
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=19590806/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
#
# FOURTH RUN
  write: io=74578MB, bw=254560KB/s, iops=63639, runt=300000msec
    slat (usec): min=11, max=542, avg=14.11, stdev= 7.97
    clat (usec): min=1, max=8796, avg=471.81, stdev=183.68
     lat (usec): min=14, max=9031, avg=486.07, stdev=189.65
    clat percentiles (usec):
     |  1.00th=[  442],  5.00th=[  442], 10.00th=[  446], 20.00th=[  450],
     | 30.00th=[  450], 40.00th=[  454], 50.00th=[  454], 60.00th=[  458],
     | 70.00th=[  458], 80.00th=[  462], 90.00th=[  466], 95.00th=[  470],
     | 99.00th=[  980], 99.50th=[ 1544], 99.90th=[ 2640], 99.95th=[ 5152],
     | 99.99th=[ 6112]
    bw (KB  /s): min=84776, max=270144, per=99.46%, avg=253185.86, stdev=35350.56
    lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
    lat (usec) : 100=0.01%, 250=0.01%, 500=97.31%, 750=1.25%, 1000=0.44%
    lat (msec) : 2=0.71%, 4=0.20%, 10=0.07%
  cpu          : usr=11.08%, sys=88.90%, ctx=2420, majf=0, minf=23
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=19091970/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
#
# FIFTH RUN
  write: io=74886MB, bw=255611KB/s, iops=63902, runt=300000msec
    slat (usec): min=10, max=3278, avg=14.07, stdev= 8.39
    clat (usec): min=1, max=11914, avg=469.84, stdev=194.26
     lat (usec): min=13, max=12169, avg=484.07, stdev=200.57
    clat percentiles (usec):
     |  1.00th=[  434],  5.00th=[  438], 10.00th=[  438], 20.00th=[  442],
     | 30.00th=[  446], 40.00th=[  446], 50.00th=[  450], 60.00th=[  454],
     | 70.00th=[  458], 80.00th=[  462], 90.00th=[  470], 95.00th=[  474],
     | 99.00th=[  988], 99.50th=[ 1576], 99.90th=[ 2672], 99.95th=[ 5536],
     | 99.99th=[ 6688]
    bw (KB  /s): min=82880, max=274472, per=99.31%, avg=253852.73, stdev=37419.72
    lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01%
    lat (usec) : 250=0.01%, 500=97.30%, 750=1.25%, 1000=0.44%
    lat (msec) : 2=0.71%, 4=0.21%, 10=0.07%, 20=0.01%
  cpu          : usr=11.18%, sys=88.80%, ctx=2468, majf=0, minf=23
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=19170855/d=0, short=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=31
     
     Average write IOPS: 64370