[fio profiles] fio profiles for load generation, device or system stress and IO simulation/profiling #tags: fio, load, profiling, fio-load, load-gen, load generation, io, io profiling, filesystem, stress, io-load

basic-sanity-validation-profile.fio

; fio profile for basic sanity validation of configured system.

[global]
ioengine=solarisaio
; ioengine=linuxaio
; ioengine=libaio
; fsync=16
; rw=read
iodepth=16
; rw=randrw ; Random IO
; rw=readwrite ; Sequential IO
bs=1m
; blockalign=16k
direct=0 ; Set to 1 if doing RAW device IO, i.e. /dev/*
fallocate=none
; Limit to 1 job per definition below
; Jobs are serialized, so each reporting period should
; only have data for a single job group.
numjobs=8
nrfiles=8
; Raw device IO filepath
; filename=/dev/mapper/mpathl
; Names and location(s) for data
filename_format=data$filenum
directory=/storage/${POOL}/fio0
group_reporting ; jobs are clumped into single report line
; we do not want to be able to compress data
buffer_compress_percentage=0
refill_buffers
; limit rate to a known value of 10MB/sec reads and writes
; rate=10m,10m ; when we just generate data this won't matter
; 50/50 read/write
rwmixread=50
; we may want to constrain the job before we processed all bytes
; runtime is 60 min., i.e. 60 * 60 sec.
runtime=60
size=5t
; io_size=100g
; create threads instead of forking-off instances
thread
; loop over the workload XX-times
; loops=2

; -- start job desc --
[1m-s]
bs=1m
rw=rw
stonewall

[4k-s]
bs=4k
rw=rw
stonewall

[8k-s]
bs=8k
rw=rw
stonewall

[16k-s]
bs=16k
rw=rw
stonewall

[32k-s]
bs=32k
rw=rw
stonewall

[64k-s]
bs=64k
rw=rw
stonewall

[128k-s]
bs=128k
rw=rw
stonewall

[1m-r]
bs=1m
rw=randrw
stonewall

[4k-r]
bs=4k
rw=randrw
stonewall

[8k-r]
bs=8k
rw=randrw
stonewall

[16k-r]
bs=16k
rw=randrw
stonewall

[32k-r]
bs=32k
rw=randrw
stonewall

[64k-r]
bs=64k
rw=randrw
stonewall

[128k-r]
bs=128k
rw=randrw
stonewall
; -- end job --

compare-block-alignments-more-jobs.fio

# This job file tries to mimic the Intel IOMeter File Server Access Pattern
[global]
description=Compare different block alignments with 32+64K IO
direct=0
directory=/storage/p01/test32K
ioengine=solarisaio
time_based
# runtime=360
# group_reporting
# write_bw_log=bandw
# write_iops_log=iops
# write_lat_log=lat

[aligned-512]
blockalign=512
bssplit=32k/50:64k/50
rw=randrw
rwmixread=50
size=8g
iodepth=16
nrfiles=4
numjobs=2
runtime=60
stonewall

[aligned-512]
blockalign=512
bssplit=32k/50:64k/50
rw=randrw
rwmixread=50
size=8g
iodepth=16
nrfiles=4
numjobs=2
runtime=60
stonewall

[aligned-4096]
blockalign=4096
bssplit=32k/50:64k/50
rw=randrw
rwmixread=50
size=8g
iodepth=16
nrfiles=4
numjobs=2
runtime=60
stonewall

[aligned-4096]
blockalign=4096
bssplit=32k/50:64k/50
rw=randrw
rwmixread=50
size=8g
iodepth=16
nrfiles=4
numjobs=2
runtime=60
stonewall

[aligned-8192]
blockalign=8192
bssplit=32k/50:64k/50
rw=randrw
rwmixread=50
size=8g
iodepth=16
nrfiles=4
numjobs=2
runtime=60
stonewall

[aligned-8192]
blockalign=8192
bssplit=32k/50:64k/50
rw=randrw
rwmixread=50
size=8g
iodepth=16
nrfiles=4
numjobs=2
runtime=60
stonewall

[aligned-577]
blockalign=577
bssplit=32k/50:64k/50
rw=randrw
rwmixread=50
size=8g
iodepth=16
nrfiles=4
numjobs=2
runtime=60
stonewall

[aligned-577]
blockalign=577
bssplit=32k/50:64k/50
rw=randrw
rwmixread=50
size=8g
iodepth=16
nrfiles=4
numjobs=2
runtime=60
stonewall

[aligned-1117]
blockalign=1117
bssplit=32k/50:64k/50
rw=randrw
rwmixread=50
size=8g
iodepth=16
nrfiles=4
numjobs=2
runtime=60
stonewall

[aligned-1117]
blockalign=1117
bssplit=32k/50:64k/50
rw=randrw
rwmixread=50
size=8g
iodepth=16
nrfiles=4
numjobs=2
runtime=60
stonewall

compare-diff-block-alignments-more-jobs-1.fio

# This job file tries to mimic the Intel IOMeter File Server Access Pattern
[global]
description=Compare different block alignments with 32+64K IO
direct=0
directory=/storage/p01/test32K
ioengine=solarisaio
time_based
# runtime=360
# group_reporting
# write_bw_log=bandw
# write_iops_log=iops
# write_lat_log=lat
bssplit=32k/50:64k/50
rw=randrw
rwmixread=50
size=4g
iodepth=16
nrfiles=4
numjobs=2

[aligned-512]
blockalign=512
stonewall
runtime=30

[aligned-512]
blockalign=512
stonewall
runtime=30

[aligned-512]
blockalign=512
stonewall
runtime=30

[aligned-512]
blockalign=512
stonewall
runtime=30

[aligned-4096]
blockalign=4096
stonewall
runtime=30

[aligned-4096]
blockalign=4096
stonewall
runtime=30

[aligned-4096]
blockalign=4096
stonewall
runtime=30

[aligned-4096]
blockalign=4096
stonewall
runtime=30

[aligned-8192]
blockalign=8192
stonewall
runtime=30

[aligned-8192]
blockalign=8192
stonewall
runtime=30

[aligned-8192]
blockalign=8192
stonewall
runtime=30

[aligned-8192]
blockalign=8192
stonewall
runtime=30

[aligned-16384]
blockalign=16384
stonewall
runtime=30

[aligned-16384]
blockalign=16384
stonewall
runtime=30

[aligned-16384]
blockalign=16384
stonewall
runtime=30

[aligned-16384]
blockalign=16384
stonewall
runtime=30

[aligned-577]
blockalign=577
stonewall
runtime=30

[aligned-577]
blockalign=577
stonewall
runtime=30

[aligned-577]
blockalign=577
stonewall
runtime=30

[aligned-577]
blockalign=577
stonewall
runtime=30

[aligned-1117]
blockalign=1117
stonewall
runtime=30

[aligned-1117]
blockalign=1117
stonewall
runtime=30

[aligned-1117]
blockalign=1117
stonewall
runtime=30

[aligned-1117]
blockalign=1117
stonewall
runtime=30

compare-diff-block-alignments.fio

# This job file tries to mimic the Intel IOMeter File Server Access Pattern
[global]
description=Compare different block alignments with 32+64K IO
direct=0
directory=/storage/p01/test32K
ioengine=solarisaio
time_based
# runtime=360
# group_reporting
# write_bw_log=bandw
# write_iops_log=iops
# write_lat_log=lat

[aligned-512]
blockalign=512
bssplit=32k/50:64k/50
rw=randrw
rwmixread=50
size=8g
iodepth=16
nrfiles=4
numjobs=2
runtime=120
stonewall

[aligned-4096]
blockalign=4096
bssplit=32k/50:64k/50
rw=randrw
rwmixread=50
size=8g
iodepth=16
nrfiles=4
numjobs=2
runtime=120
stonewall

[aligned-8192]
blockalign=8192
bssplit=32k/50:64k/50
rw=randrw
rwmixread=50
size=8g
iodepth=16
nrfiles=4
numjobs=2
runtime=120
stonewall

[aligned-1234]
blockalign=1234
bssplit=32k/50:64k/50
rw=randrw
rwmixread=50
size=8g
iodepth=16
nrfiles=4
numjobs=2
runtime=120
stonewall

fio-seed-datasets-for-ld.txt

; fio profile fills datasets with data for further testing with LoadDynamix.

[global]
; ioengine=libaio
; fsync=16
; rw=read
iodepth=16
rw=write
bs=1m
direct=0
; limit to 1 job per definition below, 
; for a total of 11 parallel jobs, and 100 file(s) per definition
numjobs=1
nrfiles=100
filename_format=data$filenum
group_reporting ; jobs are clumped into single report line
; we do not want to be able to compress data
buffer_compress_percentage=0
refill_buffers
; limit rate to a known value of 10MB/sec reads and writes
; rate=10m,10m ; when we just generate data this won't matter
; 50/50 read/write
; rwmixread=50
; we may want to constrain the job before we processed all bytes
; runtime is 60 min., i.e. 60 * 60 sec.
; runtime=3600
size=5t
; io_size=100g
;
; Don't forget to create datasets prior to generating the data,
; otherwise this will fail immediately due to missing mountpoints.
; To build datasets in one command run:
; # for i in `seq 0 9` ; do zfs create -p p01/ld/nfs/ds128-${i} ; done
; This does assume that datasets will be 128K, but if any other recordsize
; is used, just replace 128 in command with relevant recordsize. Assuming
; recordsize is 32K, we can issue the following loop:
; # for i in `seq 0 9` ; do zfs set recordsize=32K -p p01/ld/nfs/ds32-${i} ; done
;

; -- start job desc --
[d0]
directory=/storage/p01/ld/nfs/ds128-0

[d1]
directory=/storage/p01/ld/nfs/ds128-1

[d2]
directory=/storage/p01/ld/nfs/ds128-2

[d3]
directory=/storage/p01/ld/nfs/ds128-3

[d4]
directory=/storage/p01/ld/nfs/ds128-4

[d5]
directory=/storage/p01/ld/nfs/ds128-5

[d6]
directory=/storage/p01/ld/nfs/ds128-6

[d7]
directory=/storage/p01/ld/nfs/ds128-7

[d8]
directory=/storage/p01/ld/nfs/ds128-8

[d9]
directory=/storage/p01/ld/nfs/ds128-9
; -- end job --

fio-vers3-header.csv

first-attempt-benchmark-profile.fio

# Concurrently run 4 jobs each with identical parameters.
# This is used to generate a profile for a given device/devices.
[global]
description=Random and Sequential IO in discrete tests to establish random and sequential capabilities of devices.
direct=0
directory=/storage/p03/test32K01
ioengine=solarisaio
time_based
# runtime=360
# group_reporting
# write_bw_log=bandw
# write_iops_log=iops
# write_lat_log=lat
bssplit=32k/50:64k/50
# rw=randrw
# rwmixread=50
size=1g
iodepth=16
nrfiles=4
# numjobs=1

[random70pct_4k_70w30r]
blockalign=4096
bs=4096
percentage_random=70
rw=randrw
rwmixread=30
rwmixwrite=70
runtime=30
numjobs=4
stonewall

[rand100pct_70w30r]
blockalign=4096
rw=randrw
rwmixread=30
rwmixwrite=70
runtime=30
numjobs=4
stonewall

[rand70pct_100w]
blockalign=4096
percentage_random=70
rw=randrw
rwmixread=0
rwmixwrite=100
runtime=30
numjobs=4
stonewall

[rand30pct_70w30r]
blockalign=4096
percentage_random=30
rw=randrw
rwmixread=30
rwmixwrite=70
runtime=30
numjobs=4
stonewall

[seq100pct_70w30r]
blockalign=4096
rw=readwrite
rwmixread=30
rwmixwrite=70
runtime=30
numjobs=4
stonewall

[seq100pct_100w]
blockalign=4096
percentage_random=0
rw=randrw
rwmixread=0
rwmixwrite=100
runtime=30
numjobs=4
stonewall

[seq100pct_50w50r]
blockalign=4096
percentage_random=0
rw=readwrite
rwmixread=50
rwmixwrite=50
runtime=30
numjobs=4
stonewall

[seq100pct_100r]
blockalign=4096
percentage_random=0
rw=read
# rwmixread=100
# rwmixwrite=0
runtime=30
numjobs=4
stonewall

fsstressmac.sh

#!/bin/bash

# Create a bunch of directories first
for (( i=0 ; i<1000 ; i++ )); do
    mkdir -p testdir$i
done

for (( i=0 ; i<1000 ; i++ )); do
    cd testdir$i
    # Do stressful stuff here
    /usr/local/bin/fio \
        -j 10 \
        --name=global \
        --thread \
        --ioengine=posixaio \
        --filename_format=stress$filenum \
        --directory=. \
        --group_reporting \
        --name=thrash1 \
        --iodepth=16 \
        --nrfiles=10 \
        --rwmixread=50 \
        --rw=randrw \
        --percentage_random=0 \
        --size=2G \
        --bs=128K
    cd ..
    sleep 2 ; 
    rm -rf testdir$i/* && rmdir testdir$i
done

gauss-exploratory.fio

; profile.name=exploratory
; filename=gauss-exploratory.fio
; profile.id=exp1
;
; Environment Variables
; POOL      ==> required name of pool
; NUMJOBS   ==> number of clones of specified job
; SIZE      ==> bytes of IO roughly
;
; To run this profile, issue the following from the command line.
; Notice that variables are being passed inline, which makes them local to
; this command and they are parsed into this config, replacing variable
; names with values from those passed via command line.
; POOL=p01 NUMJOBS=1 SIZE=100G fio ./gauss-exploratory.fio
;
[global]
; ioengine=solarisaio
ioengine=linuxaio
; ioengine=libaio
rw=randrw ; Random IO
direct=0 ; Set to 1 if doing RAW device IO, i.e. /dev/*
fallocate=none
; Limit to 1 job per definition below
; Jobs are serialized, so each reporting period should
; only have data for a single job group.
numjobs=${NUMJOBS}
nrfiles=${NUMJOBS}
; Names and location(s) for data
filename_format=data$filenum
; This should be our mountpoint from storage connected via NFS
; ${POOL} is a variable here, just an example of how we make things flexible
directory=/mnt/${POOL}/nfs/0/fio:/mnt/${POOL}/nfs/1/fio
; create threads instead of forking-off instances
thread
; loop over the workload XX-times
; loops=2
; Logging Configuration
per_job_logs=0
write_bw_log=exp1
write_hist_log=exp1
write_iops_log=exp1
write_lat_log=exp1
log_hist_coarseness=6
log_avg_msec=1000
log_hist_msec=1000
group_reporting

[r=gauss-50%-write-4k] ; Random 50% read at 4K
rwmixread=50 ; read % of mixed workload, where write = 100 - rwmixread
random_distribution=normal:10
percentage_random=100
bs=4k
buffer_compress_percentage=0
sync=0 ; If enabled likely will mean O_SYNC is used
size=${SIZE}
io_size=${SIZE}
iodepth=16 ; How many IOs to keep in flight?
stonewall

[r=gauss-50%-write-128k] ; Random 50% read at 128K
rwmixread=50 ; read % of mixed workload, where write = 100 - rwmixread
random_distribution=normal:10
percentage_random=100
bs=128k
buffer_compress_percentage=0
sync=0 ; If enabled likely will mean O_SYNC is used
size=${SIZE}
io_size=${SIZE}
iodepth=16 ; How many IOs to keep in flight?
stonewall

[r=gauss-0%-write-4k] ; Random 100% read at 4K
rwmixread=100 ; read % of mixed workload, where write = 100 - rwmixread
random_distribution=normal:10
percentage_random=100
bs=4k
buffer_compress_percentage=0
sync=0 ; If enabled likely will mean O_SYNC is used
size=${SIZE}
io_size=${SIZE}
iodepth=16 ; How many IOs to keep in flight?
stonewall

[r=gauss-0%-write-128k] ; Random 100% read at 128K
rwmixread=100 ; read % of mixed workload, where write = 100 - rwmixread
random_distribution=normal:10
percentage_random=100
bs=128k
buffer_compress_percentage=0
sync=0 ; If enabled likely will mean O_SYNC is used
size=${SIZE}
io_size=${SIZE}
iodepth=16 ; How many IOs to keep in flight?
stonewall

[r=gauss-100%-write-4k] ; Random 100% write at 4K
rwmixread=0 ; read % of mixed workload, where write = 100 - rwmixread
random_distribution=normal:10
percentage_random=100
bs=4k
buffer_compress_percentage=0
sync=0 ; If enabled likely will mean O_SYNC is used
size=${SIZE}
io_size=${SIZE}
iodepth=16 ; How many IOs to keep in flight?
stonewall

[r=gauss-100%-write-128k] ; Random 100% write at 128K
rwmixread=0 ; read % of mixed workload, where write = 100 - rwmixread
random_distribution=normal:10
percentage_random=100
bs=128k
buffer_compress_percentage=0
sync=0 ; If enabled likely will mean O_SYNC is used
size=${SIZE}
io_size=${SIZE}
iodepth=16 ; How many IOs to keep in flight?
stonewall

[s=gauss-100%-write-4k] ; Sequential 100% write at 4K
rwmixread=0 ; read % of mixed workload, where write = 100 - rwmixread
random_distribution=normal:10
percentage_random=0
bs=4k
buffer_compress_percentage=0
sync=0 ; If enabled likely will mean O_SYNC is used
size=${SIZE}
io_size=${SIZE}
iodepth=16 ; How many IOs to keep in flight?
stonewall

[s=gauss-100%-write-128k] ; Sequential 100% write at 128K
rwmixread=0 ; read % of mixed workload, where write = 100 - rwmixread
random_distribution=normal:10
percentage_random=0
bs=128k
buffer_compress_percentage=0
sync=0 ; If enabled likely will mean O_SYNC is used
size=${SIZE}
io_size=${SIZE}
iodepth=16 ; How many IOs to keep in flight?
stonewall

gauss11-rand100-mixedblock.fio

; profile.name=gauss11-rand100-mixedblock
; profile.id=001a
;
; Environment Variables
; POOL      ==> required name of pool
; NUMJOBS   ==> number of clones of specified job
;
[global]
ioengine=solarisaio
; ioengine=linuxaio
; ioengine=libaio
rw=randrw ; Random IO
direct=0 ; Set to 1 if doing RAW device IO, i.e. /dev/*
fallocate=none
; Limit to 1 job per definition below
; Jobs are serialized, so each reporting period should
; only have data for a single job group.
numjobs=${NUMJOBS}
nrfiles=8
; Names and location(s) for data
filename_format=data$filenum
directory=/storage/${POOL}/fio
; create threads instead of forking-off instances
thread
; loop over the workload XX-times
; loops=2
; Logging Configuration
per_job_logs=0
write_bw_log=001a
write_hist_log=001a
write_iops_log=001a
write_lat_log=001a
log_hist_coarseness=6
log_avg_msec=1000
log_hist_msec=1000
group_reporting

[r=gauss-100%-write]
rwmixread=0 ; read % of mixed workload, where write = 100 - rwmixread
random_distribution=normal:10
percentage_random=70
bssplit=4k/20:8k/10:16k/30:32k/10:64k/10:128k/20
buffer_pattern=0x0638f227
sync=1 ; If enabled likely will mean O_SYNC is used
size=100G
io_size=10G
iodepth=16 ; How many IOs to keep in flight?

parse-fio-terse.py

#!/usr/bin/env python
import sys, os

lines = []  # all lines extracted from raw fio terse output
tmp_list = []  # intermediate list for partially-processed results
output_list = []  # final list
header=""  # Header line, if any for csv

# We may want to pass a header file which will print prior to data.
# Because fio's format is somewhat flexible and has changed over the
# years, we are assuming here that number of fields this utilities
# produces may also be variable. As such, we don't bake a header in,
# and instead allow for header to be fed from another file.
if len(sys.argv[:]) > 2:
    try:
        arg, fp = sys.argv[1], sys.argv[2]
        if arg == "-h" or arg == "--header":
            with open(fp, "rb") as f:
                header=f.readline().rstrip("\n")
    except ValueError:
        pass

while True:
    line = sys.stdin.readline()
    if not line:
        break
    lines.append(line.strip("\n"))

if len(lines) == 0:
    sys.exit(1)

if header:
    print(header)

for line in lines:
    skip=True
    for idx, elem in enumerate(line.split(";")):
        if len(elem.split("=")) > 1:
            tmp_list.append(elem.split("=")[1])
        else:
            if idx == 8 or idx == 49:
                if int(elem) > 0:
                    skip=False
            tmp_list.append(elem.rstrip("%"))
    if not skip:
        output_list.append(",".join(tmp_list))
    tmp_list = []

for line in output_list:
    print line

rand-mixed-bs-zone-dist-80read.fio

; fio profile used to simulate relatively random, read-biased IO for the
; purposes of stressing the ARC.

[global]
; ioengine=libaio
; fsync=16
; rw=read
iodepth=16
rw=randrw
; bs=1m
bssplit=4k/5:16k/25:32k/35:64k/20:128k/15
direct=0
; limit to 1 job per definition below, 
; for a total of 11 parallel jobs, and 100 file(s) per definition
numjobs=2
nrfiles=100
filename_format=data$filenum
group_reporting ; jobs are clumped into single report line
; we do not want to be able to compress data
buffer_compress_percentage=0
refill_buffers
; limit rate to a known value of 10MB/sec reads and writes
; rate=10m,10m ; when we just generate data this won't matter
; 50/50 read/write
rwmixread=80
;
; How random do we want this IO to be? In some cases we may want
; to do some mix of random and some sequential IO, in particular
; when we are trying to exercise the ARC.
percentage_random=80
;
; Zoned distribution of accesses across the entire range of offsets
; in each file. We are breaking a file into zones and for each zone
; we ascribe some fraction of total, with 1 being full range of a file.
random_distribution=zoned:30/20:20/20:15/10:30/20:5/30
;
; we may want to constrain the job before we processed all bytes
; runtime is 60 min., i.e. 60 * 60 sec.
; runtime=86400
size=500g
; io_size=100g
;
; Don't forget to create datasets prior to generating the data,
; otherwise this will fail immediately due to missing mountpoints.
; To build datasets in one command run:
; # for i in `seq 0 9` ; do zfs create -p p01/ld/nfs/ds128-${i} ; done
; This does assume that datasets will be 128K, but if any other recordsize
; is used, just replace 128 in command with relevant recordsize. Assuming
; recordsize is 32K, we can issue the following loop:
; # for i in `seq 0 9` ; do zfs set recordsize=32K -p p01/ld/nfs/ds32-${i} ; done
;

; -- start job desc --
[d0]
directory=/storage/p01/ld/nfs/ds128-0

[d1]
directory=/storage/p01/ld/nfs/ds128-1

[d2]
directory=/storage/p01/ld/nfs/ds128-2

[d3]
directory=/storage/p01/ld/nfs/ds128-3

[d4]
directory=/storage/p01/ld/nfs/ds128-4

[d5]
directory=/storage/p01/ld/nfs/ds128-5

[d6]
directory=/storage/p01/ld/nfs/ds128-6

[d7]
directory=/storage/p01/ld/nfs/ds128-7

[d8]
directory=/storage/p01/ld/nfs/ds128-8

[d9]
directory=/storage/p01/ld/nfs/ds128-9
; -- end job --

random-writers.fio

; Name :: random-writers.fio
;
; Run this script as follows:
; # ENV_SIZE=`mdb -k < <(echo physmem/D)|awk 'NR > 1 {printf "%.0fg", 8 * (($2*4096) / (1024 * 1024 * 1024))}'` ENV_NUMJOBS=4 ENV_LOOPS=4 /path/to/fio --output -f /path/to/this/file
;
; This tells us what the size of ARC (in gigabytes) is now: 
; # kstat -p zfs:0:arcstats:size \
; | awk '{printf "%.2fGB\n", $2/(1024 * 1024 * 1024)}'
;
[global]
description=Profile for exercising of the ARC.
direct=0
directory=/storage/p01/test/01
filename_format=scratch.$filenum ; We share files between jobs.
ioengine=solarisaio
fallocate=none ; Do not preallocate space. ZFS does not support `fallocate`.
bs=128k ; Default, if not specificed by job explicitly.
size=${ENV_SIZE}
iodepth=32
nrfiles=${ENV_NUMJOBS}
;
; Setup work area and wait to run other jobs until after this is ready.
;
[prepare-workarea]
create_only=1 ; Only create the files, we'll do reading next.
bs=1M
rw=write
numjobs=${ENV_NUMJOBS}
stonewall ; wait to finish before we start others.
;
; Try to get drives to the point where they are no longer fresh, 
; and the pool has some level of fragmentation.
;
[condition]
rw=write
numjobs=${ENV_NUMJOBS}
loops=${ENV_LOOPS} ; Repeat each job n-times.
stonewall ; wait to finish before we start others.
;
; These two should be able to run in parallel.
;
[rand4k-RW]
bs=4k
rw=randrw
rwmixread=30
percentage_random=100
numjobs=${ENV_NUMJOBS}
loops=${ENV_LOOPS} ; Repeat each job n-times.
;
[rand4k-W]
bs=4k
rw=randwrite
percentage_random=100
numjobs=${ENV_NUMJOBS}
loops=${ENV_LOOPS} ; Repeat each job n-times.

randomwr.fio

[global]
rw=randread

; -- start job desc --
[random-writers]
directory=/test
ioengine=solarisaio
fsync=16
iodepth=16
rw=randwrite
bs=4k
direct=0
numjobs=4
; runtime is 60 min., i.e. 60 * 60 sec.
runtime=3600
size=800g
; -- end job --

raw-device-and-zfs-seq-and-random-benchmark.fio

; fio profile for basic validation of hardware or ZFS pool configuration

[global]
; ioengine=solarisaio
ioengine=posixaio
; ioengine=linuxaio
; ioengine=libaio
; fsync=16
; rw=read
; Keep 16 IOs in-flight at all times, this is equivalent to QD=16
; We normally have 2 jobs, which means maximum possible QD is 16*2=32
iodepth=16
; All of these parameters are set below, these are here for reference
; rw=randrw ; Random Read/Write IOs
; rw=readwrite ; Sequential IOs
; rw=read ; Sequential Read IOs
; rw=write ; Sequential Write IOs
; rwmixread=0 ; Percentage of Read IOs
; rwmixwrite=0 ; Percentage of Write IOs
; bs=4k ; Size of IOs to issue
; blockalign=16k ; Boundary on which to align IOs
direct=0 ; Set to 0 if doing RAW device IO, i.e. /dev/rdsk/*, 1 otherwise
fallocate=none
; Limit to 2 job(s) per definition below
; Jobs are serialized, so each reporting period should
; only have data for a single job group.
numjobs=2
; Create some distance between parallel jobs doing IO to prevent more than
; one job reading from or to same offset.
; This results in offset being: offset + offset_increment * thread_number,
; where thread_number starts at 0, and increments by 1.
offset_increment=4
; nrfiles=100 ; If doing raw IO, irrelevant
; Raw device IO filepath
filename=/dev/rdsk/c0t5000CCA04F258F08d0p0
; Names and location(s) for data if doing IO to ZFS filesystem
; filename_format=data$filenum
; directory=/storage/p01/ld/nfs/ds128-0
group_reporting ; jobs are clumped into single report line
; we do not want to be able to compress data
buffer_compress_percentage=0
refill_buffers
; we may want to constrain the job before we processed all bytes
; runtime is 5 min., i.e. 5 * 60 sec.
runtime=60
time_based
; This size number is arbitrary, we are constrained by wall-clock
size=5t
; io_size=100g
; create threads instead of forking-off instances
thread

; -- start job desc --

; stonewall is a barrier between jobs, without which all jobs would run
; in parallel. We break each job group out, so multiple jobs run, but only
; jobs from one of the groups in the job definitions below.

; -- Sequential IO Throughput Tests --

; 128K Sequential 0/100 read/write split
[128k-seq-100%w]
bs=128k
rw=write
rwmixread=0 ; 100% write
stonewall

; 128K Sequential 100/0 read/write split
[128k-seq-100%r]
bs=128k
rw=read
rwmixwrite=0 ; 100% read
stonewall

; 128K Sequential 30/70 read/write split
[128k-seq-30%r70%w]
bs=128k
rw=readwrite
rwmixread=30 ; 30% write
stonewall

; 128K Sequential 70/30 read/write split
[128k-seq-70%r30%w]
bs=128k
rw=readwrite
rwmixwrite=30 ; 70% read
stonewall

[128k-seq-50%r50%w]
bs=128k
rw=readwrite
rwmixread=50 ; 50% write
stonewall

; 64K Sequential 0/100 read/write split
[64k-seq-100%w]
bs=64k
rw=write
rwmixread=0 ; 100% write
stonewall

; 64K Sequential 100/0 read/write split
[64k-seq-100%r]
bs=64k
rw=read
rwmixwrite=0 ; 100% read
stonewall

; 64K Sequential 30/70 read/write split
[64k-seq-30%r70%w]
bs=64k
rw=readwrite
rwmixread=30 ; 70% write
stonewall

; 64K Sequential 70/30 read/write split
[64k-seq-70%r30%w]
bs=64k
rw=readwrite
rwmixwrite=30 ; 70% read
stonewall

[64k-seq-50%r50%w]
bs=64k
rw=readwrite
rwmixread=50 ; 50% write
stonewall

; -- End Sequential IO Throughput Tests --

; -- Random IO Throughput Tests --

; 128K Random 0/100 read/write split
[128k-rand-100%w]
bs=128k
rw=randwrite
rwmixread=0 ; 100% write
stonewall

; 128K Random 100/0 read/write split
[128k-rand-100%r]
bs=128k
rw=randread
rwmixwrite=0 ; 100% read
stonewall

; 128K Random 30/70 read/write split
[128k-rand-30%r70%w]
bs=128k
rw=randrw
rwmixread=30 ; 70% write
stonewall

; 128K Random 70/30 read/write split
[128k-rand-70%r30%w]
bs=128k
rw=randrw
rwmixwrite=30 ; 70% read
stonewall

; 64K Random 0/100 read/write split
[64k-rand-100%w]
bs=64k
rw=randwrite
rwmixread=0 ; 100% write
stonewall

; 64K Random 100/0 read/write split
[64k-rand-100%r]
bs=64k
rw=randread
rwmixwrite=0 ; 100% read
stonewall

; 64K Random 30/70 read/write split
[64k-rand-30%r70%w]
bs=64k
rw=randrw
rwmixread=30 ; 70% write
stonewall

; 64K Random 70/30 read/write split
[64k-rand-70%r30%w]
bs=64k
rw=randrw
rwmixwrite=30 ; 70% read
stonewall

; 64K Random 50/50 read/write split
[64k-rand-50%r50%w]
bs=64k
rw=randrw
rwmixread=50 ; 50% write
stonewall

; -- End Random IO Throughput Tests --

; -- Random IO IOPS Tests --

; 4K Random 0/100 read/write split
[4k-rand-100%w]
bs=4k
rw=randwrite
rwmixread=0 ; 100% write
stonewall

; 4K Random 100/0 read/write split
[4k-rand-100%r]
bs=4k
rw=randread
rwmixwrite=0 ; 100% read
stonewall

; 4K Random 30/70 read/write split
[4k-rand-30%r70%w]
bs=4k
rw=randrw
rwmixread=30 ; 70% write
stonewall

; 4K Random 70/30 read/write split
[4k-rand-70%r30%w]
bs=4k
rw=randrw
rwmixwrite=30 ; 70% read
stonewall

; 4K Random 50/50 read/write split
[4k-rand-50%r50%w]
bs=4k
rw=randrw
rwmixread=50 ; 50% write
stonewall

; 8K Random 0/100 read/write split
[8k-rand-100%w]
bs=8k
rw=randwrite
rwmixread=0 ; 100% write
stonewall

; 8K Random 100/0 read/write split
[8k-rand-100%r]
bs=8k
rw=randread
rwmixwrite=0 ; 100% read
stonewall

; 8K Random 30/70 read/write split
[8k-rand-30%r70%w]
bs=8k
rw=randrw
rwmixread=30 ; 70% write
stonewall

; 8K Random 70/30 read/write split
[8k-rand-70%r30%w]
bs=8k
rw=randrw
rwmixwrite=30 ; 70% read
stonewall

; 8K Random 50/50 read/write split
[8k-rand-50%r50%w]
bs=8k
rw=randrw
rwmixread=50 ; 50% write
stonewall

; 16K Random 0/100 read/write split
[16k-rand-100%w]
bs=16k
rw=randwrite
rwmixread=0 ; 100% write
stonewall

; 16K Random 100/0 read/write split
[16k-rand-100%r]
bs=16k
rw=randread
rwmixwrite=0 ; 100% read
stonewall

; 16K Random 30/70 read/write split
[16k-rand-30%r70%w]
bs=16k
rw=randrw
rwmixread=30 ; 70% write
stonewall

; 16K Random 70/30 read/write split
[16k-rand-70%r30%w]
bs=16k
rw=randrw
rwmixwrite=30 ; 70% read
stonewall

; 16K Random 50/50 read/write split
[16k-rand-50%r50%w]
bs=16k
rw=randrw
rwmixread=50 ; 50% write
stonewall

; -- End Random IO IOPS Tests --
; -- end job --

run-random-writers.sh

#!/bin/sh
# Use this script in conjunction with the random-writers.fio
# profile. Device IDs are volatile, so should be configured
# for the given system at the time the hardware exists/known.
POOL=p01
DS="${POOL}/test/01"

[[ `zpool list "${POOL}" 2>/dev/null` ]] \
    || zpool create \
        -ocachefile=/etc/.zpools/spa_info \
        -m /storage/"${POOL}" "${POOL}" \
        c0t5000CCA04F03DB9Cd0 \
        c0t5000CCA04F03DBA0d0 \
        c0t5000CCA04F03DF80d0 \
        c0t5000CCA04F03D760d0

[[ `zfs list p01/test/01 2>/dev/null` ]] \
    || zfs create -orecordsize=32K -p "${DS}"

ENV_SIZE=`mdb -k < <(echo physmem/D)|awk 'NR > 1 {printf "%.0fg", 8 * (($2*4096) / (1024 * 1024 * 1024))}'` ENV_NUMJOBS=4 ENV_LOOPS=4 /bp/fio/fio --output /bp/perf/fio.log -f /bp/perf/fio.jobs/random-writers.fio

runbench.sh

#!/bin/sh
#
# Setup Instructions:
#
# Create a dataset on bp, called perf with directory fio.jobs.
# zfs create -p bp/perf ; mkdir /bp/perf/fio.jobs
#
# Copy profile from this URL there:
# https://goo.gl/Z7VHOU
#
# Make sure to follow instructions in that file for setting up the
# dataset with which to test.
#
set -o xtrace
FIO=/bp/fio/fio
JOBS_DIR=/bp/perf/fio.jobs
PROFILE_NAME=a1.fio
PROFILE_PATH="${JOBS_DIR}/${PROFILE_NAME}"
LOGS_DIR="${JOBS_DIR}/logs"

[[ ! -d "${LOGS_DIR}" ]] && mkdir "${LOGS_DIR}"

function run_test {
    set -o xtrace
    number=$1
    if [[ -z "${number}" ]]; then return; fi
    #
    # Each job in the profile will run for 5 * 5 times,
    # meaning there should be 25 results * 4 jobs concurrently,
    # for a total of 100 samples for each unique job.
    #  
    ENV_LOOPS=1 ENV_SIZE=100g ENV_NUM_JOBS=4 "${FIO}" \
        --output-format=json \
        --output="${LOGS_DIR}/bench.${number}.log" \
        "${PROFILE_PATH}" \
        "${PROFILE_PATH}" \
        "${PROFILE_PATH}" \
        "${PROFILE_PATH}" \
        "${PROFILE_PATH}" 2>/dev/null
}

for i in `seq 1 4`; do run_test "${i}" || exit 1; done
#
# Compress all logs into one file to ease retrieval.
#
{ 
    cd "${LOGS_DIR}" && \
    tar czvf \
        "${PROFILE_NAME}-logs-`date +%Y-%m-%d_%H:%M:%S`.tgz"  \
        `find . -name *.log` ; 
}

second-attempt-benchmark-profile.fio

# Benchmak Profile for Evaluation of Devices
#
# Steps:
# 1) Create pool and then create dataset.
# zpool create -ocachefile=/etc/.zpools/spa_info -m /storage/p01 p01 mirror c0t5000CCA04F03D908d0 c0t5000CCA04F041338d0
# 2) Create dataset where IO will happen.
# zfs create -p -orecordsize=64K p01/test/01
# 
# This assumes fio command is in /bp/fio.
# To Run this set of jobs, cd into directory where files are to be made, after
# completing above steps.
#
# Then run command as follows:
# ENV_LOOPS=8 ENV_SIZE=100g ENV_NUM_JOBS=4 /bp/fio/fio -f /path/to/this/file
# 
[global]
description=Random and Sequential IO in discrete tests to establish random and sequential capabilities of devices.
direct=0
directory=/storage/p01/test/01
filename_format=benchmark.$jobnum.$filenum
ioengine=solarisaio
# Do not preallocate space. ZFS does not support `fallocate`.
fallocate=none
size=${ENV_SIZE}
iodepth=16
nrfiles=${ENV_NUM_JOBS}
# Seed the random generator to make tests 100% repeatable.
randseed=1277
# randrepeat=true We may want to play with this later.

[warmup_noop]
stonewall
bs=128k
rw=write
loops=1
time_based=0

# This global section applies to all jobs that follow it.

[global]
# Repeat each job n-times.
loops=${ENV_LOOPS}
# Assume time-based, to limit number of seconds each job takes.
time_based
runtime=30
bssplit=32k/50:64k/50

[rand100pct_100w]
blockalign=4096
percentage_random=100
rw=randrw
rwmixread=0
rwmixwrite=100
numjobs=${ENV_NUM_JOBS}
stonewall

[rand100pct_4k_100w]
blockalign=4096
bs=4096
percentage_random=100
rw=randrw
rwmixread=0
rwmixwrite=100
numjobs=${ENV_NUM_JOBS}
stonewall

[rand100pct_50w50r]
blockalign=4096
percentage_random=100
rw=randrw
rwmixread=50
rwmixwrite=50
numjobs=${ENV_NUM_JOBS}
stonewall

[rand100pct_4k_50w50r]
blockalign=4096
bs=4096
percentage_random=100
rw=randrw
rwmixread=50
rwmixwrite=50
numjobs=${ENV_NUM_JOBS}
stonewall

[rand70pct_4k_50w50r]
blockalign=4096
bs=4096
percentage_random=70
rw=randrw
rwmixread=50
rwmixwrite=50
numjobs=${ENV_NUM_JOBS}
stonewall

[rand70pct_100w]
blockalign=4096
percentage_random=70
rw=randrw
rwmixread=0
rwmixwrite=100
numjobs=${ENV_NUM_JOBS}
stonewall

[rand50pct_50w50r]
blockalign=4096
percentage_random=50
rw=randrw
rwmixread=50
rwmixwrite=50
numjobs=${ENV_NUM_JOBS}
stonewall

[rand50pct_4k_100w]
blockalign=4096
bs=4096
percentage_random=50
rw=randrw
rwmixread=0
rwmixwrite=100
numjobs=${ENV_NUM_JOBS}
stonewall

[seq100pct_100w]
blockalign=4096
percentage_random=0
rw=write
rwmixread=0
rwmixwrite=100
numjobs=${ENV_NUM_JOBS}
stonewall

[seq100pct_4k_100w]
blockalign=4096
bs=4096
percentage_random=0
rw=write
rwmixread=0
rwmixwrite=100
numjobs=${ENV_NUM_JOBS}
stonewall

[seq100pct_50w50r]
blockalign=4096
percentage_random=0
rw=readwrite
rwmixread=50
rwmixwrite=50
numjobs=${ENV_NUM_JOBS}
stonewall

simulate-mixed-rw-esx.fio

# This job file tries to mimic the Intel IOMeter File Server Access Pattern
[global]
description=Mixed blocksizes between jobs simulating ESX clients
direct=0
directory=/storage/p01/test/02
ioengine=solarisaio
time_based
runtime=3600
; group_reporting
; write_bw_log=bandw
; write_iops_log=iops
; write_lat_log=lat

[small-IO-50pct-read]
bssplit=4k/60:8k/20:16k/20
rw=randrw
rwmixread=50
size=50g
iodepth=32
nrfiles=4
numjobs=2

[avg-IO-50pct-read]
bssplit=32k/10:64k/90
rw=randrw
rwmixread=50
size=50g
iodepth=32
nrfiles=4
numjobs=2

[above-avg-IO-30pct-read]
bssplit=64k/80:128k/20
rw=randrw
rwmixread=30
size=50g
iodepth=32
nrfiles=4
numjobs=2

stress-zfs-arc.fio

; Run this script as follows:
; # ENV_SIZE=`mdb -k < <(echo physmem/D)|awk 'NR > 1 {printf "%.0fg", 8 * (($2*4096) / (1024 * 1024 * 1024))}'` ENV_NUMJOBS=4 ENV_LOOPS=4 /path/to/fio -f /path/to/this/file
;
; This tells us what the size of ARC (in gigabytes) is now: 
; # kstat -p zfs:0:arcstats:size \
; | awk '{printf "%.2fGB\n", $2/(1024 * 1024 * 1024)}'
;
[global]
description=Profile for exercising of the ARC.
direct=0
directory=/storage/p01/test/01
filename_format=fillarc.$filenum ; We share files between jobs.
ioengine=solarisaio
fallocate=none ; Do not preallocate space. ZFS does not support `fallocate`.
bs=128k
size=${ENV_SIZE}
iodepth=32
nrfiles=${ENV_NUMJOBS}

[j0]
create_only=1 ; Only create the files, we'll do reading next.
rw=write
numjobs=${ENV_NUMJOBS}
stonewall ; wait to finish before we start others.

[j1]
bs=1M
rw=read
pre_read=1
numjobs=${ENV_NUMJOBS}
loops=${ENV_LOOPS} ; Repeat each job n-times.

[j2]
bs=128k
rw=randread
numjobs=${ENV_NUMJOBS}
loops=${ENV_LOOPS}

stress-zfs-pool.fio

; fio profile for stress-testing zfs pool

[global]
ioengine=solarisaio
; ioengine=linuxaio
; ioengine=libaio
; fsync=16
; rw=read
iodepth=32
; rw=randrw ; Random IO
rw=readwrite ; Sequential IO
bs=4k
; blockalign=16k
direct=0 ; Set to 1 if doing RAW device IO, i.e. /dev/*
fallocate=none
; Limit to 1 job per definition below
; Jobs are serialized, so each reporting period should
; only have data for a single job group.
numjobs=4
nrfiles=8
; Raw device IO filepath
; filename=/dev/mapper/mpathl
; Names and location(s) for data
filename_format=data$filenum
directory=/storage/p02/load
group_reporting ; jobs are clumped into single report line
; we do not want to be able to compress data
buffer_compress_percentage=0
refill_buffers
; limit rate to a known value of 10MB/sec reads and writes
; rate=10m,10m ; when we just generate data this won't matter
; 50/50 read/write
rwmixread=50
; we may want to constrain the job before we processed all bytes
; runtime is 60 min., i.e. 60 * 60 sec.
runtime=86400
size=5t
; io_size=100g
; create threads instead of forking-off instances
thread
; loop over the workload XX-times
; loops=2

; -- start job desc --
[128k]
bs=128k

[16k]
bs=16k

; -- end job --