Created
February 26, 2020 20:46
-
-
Save sneak/f28bb6eb18d46af3eed72391f89ba04b to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Script started on Wed Feb 26 01:18:46 2020 | |
| command: bash -c make run | |
| rsync -avP ./disktest.sh root@fsn1.datavibe.net:/root/disktest.sh | |
| sending incremental file list | |
| disktest.sh | |
| 700 23% 0.00kB/s 0:00:00 2,977 100% 2.17MB/s 0:00:00 (xfr#1, to-chk=0/1) | |
| sent 1,003 bytes received 65 bytes 712.00 bytes/sec | |
| total size is 2,977 speedup is 2.79 | |
| ssh root@fsn1.datavibe.net "bash /root/disktest.sh 2>&1" | |
| ++ cd /dev/disk/by-id | |
| ++ grep -v part | |
| ++ ls -1 nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| + RAWDISKS='nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 | |
| nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 | |
| nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 | |
| nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 | |
| nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 | |
| nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110' | |
| + FULLPATHS= | |
| + for RAWDISK in $RAWDISKS | |
| + FULLPATHS+=' /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527' | |
| + for RAWDISK in $RAWDISKS | |
| + FULLPATHS+=' /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435' | |
| + for RAWDISK in $RAWDISKS | |
| + FULLPATHS+=' /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437' | |
| + for RAWDISK in $RAWDISKS | |
| + FULLPATHS+=' /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438' | |
| + for RAWDISK in $RAWDISKS | |
| + FULLPATHS+=' /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442' | |
| + for RAWDISK in $RAWDISKS | |
| + FULLPATHS+=' /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110' | |
| + POOLNAME=tank | |
| + MPT=/srv/z | |
| + JOBFILE=/tmp/jobfile.fio | |
| + KEYLOC=/tmp/testkey.raw | |
| + main | |
| + show_system_info | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + date -u | |
| Wed 26 Feb 2020 09:18:48 AM UTC | |
| + lsb_release -a | |
| No LSB modules are available. | |
| Distributor ID: Ubuntu | |
| Description: Ubuntu Focal Fossa (development branch) | |
| Release: 20.04 | |
| Codename: focal | |
| + uname -a | |
| Linux fsn1 5.4.0-14-generic #17-Ubuntu SMP Thu Feb 6 22:47:59 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux | |
| + lsblk -t | |
| NAME ALIGNMENT MIN-IO OPT-IO PHY-SEC LOG-SEC ROTA SCHED RQ-SIZE RA WSAME | |
| sda 0 4096 0 4096 512 0 mq-deadline 64 128 0B | |
| ├─sda1 0 4096 0 4096 512 0 mq-deadline 64 128 0B | |
| │ └─md0 0 4096 0 4096 512 0 128 128 0B | |
| ├─sda2 0 4096 0 4096 512 0 mq-deadline 64 128 0B | |
| │ └─md1 0 4096 0 4096 512 0 128 128 0B | |
| │ └─vg0-root 0 4096 0 4096 512 0 128 128 0B | |
| └─sda3 0 4096 0 4096 512 0 mq-deadline 64 128 0B | |
| sdb 0 4096 0 4096 512 0 mq-deadline 64 128 0B | |
| ├─sdb1 0 4096 0 4096 512 0 mq-deadline 64 128 0B | |
| │ └─md0 0 4096 0 4096 512 0 128 128 0B | |
| ├─sdb2 0 4096 0 4096 512 0 mq-deadline 64 128 0B | |
| │ └─md1 0 4096 0 4096 512 0 128 128 0B | |
| │ └─vg0-root 0 4096 0 4096 512 0 128 128 0B | |
| └─sdb3 0 4096 0 4096 512 0 mq-deadline 64 128 0B | |
| nvme5n1 0 512 512 512 512 0 none 1023 128 0B | |
| nvme4n1 0 512 512 512 512 0 none 1023 128 0B | |
| nvme3n1 0 512 512 512 512 0 none 1023 128 0B | |
| nvme1n1 0 512 512 512 512 0 none 1023 128 0B | |
| nvme2n1 0 512 512 512 512 0 none 1023 128 0B | |
| nvme0n1 0 512 512 512 512 0 none 1023 128 0B | |
| + nvme list | |
| Node SN Model Namespace Usage Format FW Rev | |
| ---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- -------- | |
| /dev/nvme0n1 S437NE0M804438 SAMSUNG MZQLB960HAJR-00007 1 0.00 B / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme1n1 S437NE0M804437 SAMSUNG MZQLB960HAJR-00007 1 0.00 B / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme2n1 S437NE0M804442 SAMSUNG MZQLB960HAJR-00007 1 0.00 B / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme3n1 S437NE0M804435 SAMSUNG MZQLB960HAJR-00007 1 10.49 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme4n1 S437NE0M802527 SAMSUNG MZQLB960HAJR-00007 1 10.49 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme5n1 S437NX0M402110 SAMSUNG MZQLB960HAJR-00007 1 0.00 B / 960.20 GB 512 B + 0 B EDA5202Q | |
| + modinfo zfs | |
| + grep -iw version | |
| version: 0.8.3-1ubuntu3 | |
| + modinfo spl | |
| + grep -iw version | |
| version: 0.8.3-1ubuntu3 | |
| + zfs version | |
| zfs-0.8.3-1ubuntu3 | |
| zfs-kmod-0.8.3-1ubuntu3 | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + prep_system | |
| + killall fio | |
| + sleep 1 | |
| + [[ -e /srv/z/.zfs ]] | |
| + do_direct_tests | |
| + trim_disks | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| Success formatting namespace:1 | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '################## waiting 60s for nvme format to complete...' | |
| ################## waiting 60s for nvme format to complete... | |
| + sleep 60 | |
| + nvme list | |
| Node SN Model Namespace Usage Format FW Rev | |
| ---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- -------- | |
| /dev/nvme0n1 S437NE0M804438 SAMSUNG MZQLB960HAJR-00007 1 0.00 B / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme1n1 S437NE0M804437 SAMSUNG MZQLB960HAJR-00007 1 0.00 B / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme2n1 S437NE0M804442 SAMSUNG MZQLB960HAJR-00007 1 0.00 B / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme3n1 S437NE0M804435 SAMSUNG MZQLB960HAJR-00007 1 0.00 B / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme4n1 S437NE0M802527 SAMSUNG MZQLB960HAJR-00007 1 0.00 B / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme5n1 S437NX0M402110 SAMSUNG MZQLB960HAJR-00007 1 0.00 B / 960.20 GB 512 B + 0 B EDA5202Q | |
| + for DISK in $FULLPATHS | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo 'testing /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527' | |
| testing /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + cat | |
| ++ expr 10000 / 1 | |
| + cat /tmp/jobfile.fio | |
| [test-64kchunk-1thread] | |
| bs=64k | |
| filename=/dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| size=10000M | |
| runtime=60 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio /tmp/jobfile.fio | |
| test-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=14932: Wed Feb 26 09:21:01 2020 | |
| read: IOPS=11.1k, BW=693MiB/s (727MB/s)(40.6GiB/60001msec) | |
| slat (usec): min=4, max=536, avg=17.71, stdev=20.27 | |
| clat (usec): min=19, max=8688.4k, avg=2854.52, stdev=119842.31 | |
| lat (usec): min=78, max=8688.4k, avg=2872.30, stdev=119842.29 | |
| clat percentiles (usec): | |
| | 1.00th=[ 922], 5.00th=[ 996], 10.00th=[ 1057], | |
| | 20.00th=[ 1106], 30.00th=[ 1123], 40.00th=[ 1123], | |
| | 50.00th=[ 1139], 60.00th=[ 1156], 70.00th=[ 1172], | |
| | 80.00th=[ 1221], 90.00th=[ 1352], 95.00th=[ 1745], | |
| | 99.00th=[ 2089], 99.50th=[ 2114], 99.90th=[ 2343], | |
| | 99.95th=[ 2507], 99.99th=[8657044] | |
| bw ( MiB/s): min= 46, max= 1876, per=100.00%, avg=1510.69, stdev=390.78, samples=54 | |
| iops : min= 746, max=30018, avg=24171.02, stdev=6252.54, samples=54 | |
| write: IOPS=11.1k, BW=693MiB/s (726MB/s)(40.6GiB/60001msec); 0 zone resets | |
| slat (usec): min=12, max=245, avg=18.37, stdev= 5.06 | |
| clat (usec): min=2, max=8688.4k, avg=2826.85, stdev=118945.83 | |
| lat (usec): min=18, max=8688.4k, avg=2845.32, stdev=118945.82 | |
| clat percentiles (usec): | |
| | 1.00th=[ 922], 5.00th=[ 996], 10.00th=[ 1057], | |
| | 20.00th=[ 1106], 30.00th=[ 1123], 40.00th=[ 1123], | |
| | 50.00th=[ 1139], 60.00th=[ 1156], 70.00th=[ 1172], | |
| | 80.00th=[ 1205], 90.00th=[ 1319], 95.00th=[ 1729], | |
| | 99.00th=[ 2089], 99.50th=[ 2114], 99.90th=[ 2376], | |
| | 99.95th=[ 2507], 99.99th=[8657044] | |
| bw ( MiB/s): min= 47, max= 1885, per=100.00%, avg=1509.47, stdev=393.71, samples=54 | |
| iops : min= 752, max=30168, avg=24151.46, stdev=6299.42, samples=54 | |
| lat (usec) : 4=0.01%, 20=0.01%, 100=0.01%, 250=0.01%, 500=0.01% | |
| lat (usec) : 750=0.01%, 1000=5.29% | |
| lat (msec) : 2=91.18%, 4=3.48%, 10=0.01%, 20=0.01%, 50=0.01% | |
| lat (msec) : >=2000=0.02% | |
| cpu : usr=3.02%, sys=62.53%, ctx=445675, majf=0, minf=23 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=665734,665087,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=693MiB/s (727MB/s), 693MiB/s-693MiB/s (727MB/s-727MB/s), io=40.6GiB (43.6GB), run=60001-60001msec | |
| WRITE: bw=693MiB/s (726MB/s), 693MiB/s-693MiB/s (726MB/s-726MB/s), io=40.6GiB (43.6GB), run=60001-60001msec | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + cat | |
| ++ expr 10000 / 16 | |
| + cat /tmp/jobfile.fio | |
| [test-64kchunk-16thread] | |
| bs=64k | |
| filename=/dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| size=625M | |
| runtime=60 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio /tmp/jobfile.fio | |
| test-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=16047: Wed Feb 26 09:22:02 2020 | |
| read: IOPS=49.8k, BW=3111MiB/s (3263MB/s)(185GiB/60935msec) | |
| slat (usec): min=4, max=4109, avg=41.00, stdev=47.24 | |
| clat (usec): min=5, max=2386.2k, avg=10093.57, stdev=99978.30 | |
| lat (usec): min=34, max=2386.2k, avg=10134.66, stdev=99978.21 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1188], 5.00th=[ 1532], 10.00th=[ 1795], | |
| | 20.00th=[ 2180], 30.00th=[ 2474], 40.00th=[ 2704], | |
| | 50.00th=[ 2933], 60.00th=[ 3195], 70.00th=[ 3523], | |
| | 80.00th=[ 4047], 90.00th=[ 5276], 95.00th=[ 6652], | |
| | 99.00th=[ 10421], 99.50th=[ 471860], 99.90th=[1769997], | |
| | 99.95th=[1803551], 99.99th=[2055209] | |
| bw ( MiB/s): min= 38, max=12442, per=100.00%, avg=5074.35, stdev=247.87, samples=1196 | |
| iops : min= 614, max=199072, avg=81189.07, stdev=3965.96, samples=1196 | |
| write: IOPS=49.8k, BW=3116MiB/s (3267MB/s)(185GiB/60935msec); 0 zone resets | |
| slat (usec): min=10, max=4021, avg=61.32, stdev=49.86 | |
| clat (usec): min=3, max=2386.2k, avg=10132.06, stdev=100133.98 | |
| lat (usec): min=50, max=2386.3k, avg=10193.49, stdev=100136.53 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1188], 5.00th=[ 1532], 10.00th=[ 1795], | |
| | 20.00th=[ 2180], 30.00th=[ 2474], 40.00th=[ 2737], | |
| | 50.00th=[ 2966], 60.00th=[ 3195], 70.00th=[ 3523], | |
| | 80.00th=[ 4047], 90.00th=[ 5276], 95.00th=[ 6652], | |
| | 99.00th=[ 10421], 99.50th=[ 471860], 99.90th=[1769997], | |
| | 99.95th=[1803551], 99.99th=[2055209] | |
| bw ( MiB/s): min= 38, max=12536, per=100.00%, avg=5071.99, stdev=248.70, samples=1198 | |
| iops : min= 619, max=200588, avg=81151.16, stdev=3979.18, samples=1198 | |
| lat (usec) : 4=0.01%, 10=0.01%, 750=0.01%, 1000=0.29% | |
| lat (msec) : 2=14.32%, 4=64.71%, 10=19.55%, 20=0.52%, 250=0.01% | |
| lat (msec) : 500=0.14%, 750=0.14%, 2000=0.30%, >=2000=0.01% | |
| cpu : usr=1.75%, sys=81.71%, ctx=7762297, majf=0, minf=210281 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=3033555,3037570,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=3111MiB/s (3263MB/s), 3111MiB/s-3111MiB/s (3263MB/s-3263MB/s), io=185GiB (199GB), run=60935-60935msec | |
| WRITE: bw=3116MiB/s (3267MB/s), 3116MiB/s-3116MiB/s (3267MB/s-3267MB/s), io=185GiB (199GB), run=60935-60935msec | |
| + for DISK in $FULLPATHS | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo 'testing /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435' | |
| testing /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + cat | |
| ++ expr 10000 / 1 | |
| + cat /tmp/jobfile.fio | |
| [test-64kchunk-1thread] | |
| bs=64k | |
| filename=/dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| size=10000M | |
| runtime=60 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio /tmp/jobfile.fio | |
| test-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=17197: Wed Feb 26 09:23:03 2020 | |
| read: IOPS=10.8k, BW=677MiB/s (710MB/s)(39.6GiB/60001msec) | |
| slat (usec): min=4, max=25719, avg=19.08, stdev=103.10 | |
| clat (usec): min=17, max=8673.7k, avg=2923.72, stdev=117888.32 | |
| lat (usec): min=26, max=8673.8k, avg=2942.88, stdev=117888.76 | |
| clat percentiles (usec): | |
| | 1.00th=[ 938], 5.00th=[ 1020], 10.00th=[ 1074], | |
| | 20.00th=[ 1123], 30.00th=[ 1156], 40.00th=[ 1172], | |
| | 50.00th=[ 1205], 60.00th=[ 1270], 70.00th=[ 1287], | |
| | 80.00th=[ 1303], 90.00th=[ 1352], 95.00th=[ 1696], | |
| | 99.00th=[ 2933], 99.50th=[ 3425], 99.90th=[ 4015], | |
| | 99.95th=[ 5211], 99.99th=[8556381] | |
| bw ( MiB/s): min= 266, max= 1838, per=100.00%, avg=1454.27, stdev=368.74, samples=55 | |
| iops : min= 4264, max=29414, avg=23268.38, stdev=5899.81, samples=55 | |
| write: IOPS=10.8k, BW=676MiB/s (709MB/s)(39.6GiB/60001msec); 0 zone resets | |
| slat (usec): min=12, max=350, avg=19.34, stdev= 5.08 | |
| clat (nsec): min=1610, max=8673.7M, avg=2898393.47, stdev=117115955.57 | |
| lat (usec): min=16, max=8673.8k, avg=2917.83, stdev=117115.99 | |
| clat percentiles (usec): | |
| | 1.00th=[ 938], 5.00th=[ 1020], 10.00th=[ 1074], | |
| | 20.00th=[ 1123], 30.00th=[ 1139], 40.00th=[ 1172], | |
| | 50.00th=[ 1188], 60.00th=[ 1254], 70.00th=[ 1287], | |
| | 80.00th=[ 1303], 90.00th=[ 1352], 95.00th=[ 1680], | |
| | 99.00th=[ 2900], 99.50th=[ 3425], 99.90th=[ 4015], | |
| | 99.95th=[ 15926], 99.99th=[8657044] | |
| bw ( MiB/s): min= 268, max= 1832, per=100.00%, avg=1453.40, stdev=371.05, samples=55 | |
| iops : min= 4290, max=29324, avg=23254.38, stdev=5936.75, samples=55 | |
| lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01% | |
| lat (usec) : 500=0.01%, 750=0.01%, 1000=3.63% | |
| lat (msec) : 2=93.06%, 4=3.19%, 10=0.05%, 20=0.01%, 50=0.01% | |
| lat (msec) : 100=0.01%, 250=0.01%, >=2000=0.02% | |
| cpu : usr=3.08%, sys=64.86%, ctx=630381, majf=0, minf=26 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=649593,649060,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=677MiB/s (710MB/s), 677MiB/s-677MiB/s (710MB/s-710MB/s), io=39.6GiB (42.6GB), run=60001-60001msec | |
| WRITE: bw=676MiB/s (709MB/s), 676MiB/s-676MiB/s (709MB/s-709MB/s), io=39.6GiB (42.5GB), run=60001-60001msec | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + cat | |
| ++ expr 10000 / 16 | |
| + cat /tmp/jobfile.fio | |
| [test-64kchunk-16thread] | |
| bs=64k | |
| filename=/dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| size=625M | |
| runtime=60 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio /tmp/jobfile.fio | |
| test-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=18300: Wed Feb 26 09:24:04 2020 | |
| read: IOPS=50.2k, BW=3138MiB/s (3290MB/s)(185GiB/60420msec) | |
| slat (usec): min=5, max=4226, avg=41.17, stdev=47.06 | |
| clat (usec): min=6, max=2387.2k, avg=10012.19, stdev=98577.33 | |
| lat (usec): min=39, max=2387.2k, avg=10053.44, stdev=98577.18 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1270], 5.00th=[ 1614], 10.00th=[ 1844], | |
| | 20.00th=[ 2212], 30.00th=[ 2474], 40.00th=[ 2704], | |
| | 50.00th=[ 2933], 60.00th=[ 3195], 70.00th=[ 3523], | |
| | 80.00th=[ 4047], 90.00th=[ 5276], 95.00th=[ 6521], | |
| | 99.00th=[ 11076], 99.50th=[ 484443], 99.90th=[1753220], | |
| | 99.95th=[1803551], 99.99th=[1904215] | |
| bw ( MiB/s): min= 70, max=13496, per=100.00%, avg=5014.40, stdev=249.20, samples=1210 | |
| iops : min= 1123, max=215948, avg=80230.08, stdev=3987.24, samples=1210 | |
| write: IOPS=50.3k, BW=3142MiB/s (3295MB/s)(185GiB/60420msec); 0 zone resets | |
| slat (usec): min=10, max=3908, avg=61.70, stdev=51.31 | |
| clat (usec): min=4, max=2387.3k, avg=10047.83, stdev=98643.32 | |
| lat (usec): min=57, max=2387.4k, avg=10109.64, stdev=98645.60 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1270], 5.00th=[ 1631], 10.00th=[ 1860], | |
| | 20.00th=[ 2212], 30.00th=[ 2474], 40.00th=[ 2704], | |
| | 50.00th=[ 2933], 60.00th=[ 3195], 70.00th=[ 3523], | |
| | 80.00th=[ 4047], 90.00th=[ 5276], 95.00th=[ 6521], | |
| | 99.00th=[ 11076], 99.50th=[ 484443], 99.90th=[1753220], | |
| | 99.95th=[1803551], 99.99th=[1971323] | |
| bw ( MiB/s): min= 66, max=13512, per=100.00%, avg=5012.88, stdev=249.77, samples=1212 | |
| iops : min= 1067, max=216198, avg=80205.79, stdev=3996.31, samples=1212 | |
| lat (usec) : 10=0.01%, 20=0.01%, 1000=0.14% | |
| lat (msec) : 2=13.64%, 4=65.51%, 10=19.43%, 20=0.68%, 250=0.01% | |
| lat (msec) : 500=0.11%, 750=0.17%, 2000=0.31%, >=2000=0.01% | |
| cpu : usr=1.73%, sys=81.88%, ctx=7704572, majf=0, minf=208261 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=3033555,3037570,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=3138MiB/s (3290MB/s), 3138MiB/s-3138MiB/s (3290MB/s-3290MB/s), io=185GiB (199GB), run=60420-60420msec | |
| WRITE: bw=3142MiB/s (3295MB/s), 3142MiB/s-3142MiB/s (3295MB/s-3295MB/s), io=185GiB (199GB), run=60420-60420msec | |
| + for DISK in $FULLPATHS | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo 'testing /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437' | |
| testing /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + cat | |
| ++ expr 10000 / 1 | |
| + cat /tmp/jobfile.fio | |
| [test-64kchunk-1thread] | |
| bs=64k | |
| filename=/dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| size=10000M | |
| runtime=60 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio /tmp/jobfile.fio | |
| test-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=19412: Wed Feb 26 09:25:06 2020 | |
| read: IOPS=10.9k, BW=684MiB/s (717MB/s)(40.1GiB/60001msec) | |
| slat (usec): min=4, max=516, avg=17.77, stdev=20.28 | |
| clat (usec): min=20, max=8772.3k, avg=2895.99, stdev=120693.11 | |
| lat (usec): min=28, max=8772.3k, avg=2913.84, stdev=120693.09 | |
| clat percentiles (usec): | |
| | 1.00th=[ 938], 5.00th=[ 1020], 10.00th=[ 1090], | |
| | 20.00th=[ 1123], 30.00th=[ 1123], 40.00th=[ 1139], | |
| | 50.00th=[ 1156], 60.00th=[ 1172], 70.00th=[ 1188], | |
| | 80.00th=[ 1237], 90.00th=[ 1369], 95.00th=[ 1844], | |
| | 99.00th=[ 2114], 99.50th=[ 2147], 99.90th=[ 2409], | |
| | 99.95th=[ 2671], 99.99th=[8657044] | |
| bw ( MiB/s): min= 166, max= 1866, per=100.00%, avg=1488.37, stdev=382.98, samples=54 | |
| iops : min= 2660, max=29866, avg=23813.89, stdev=6127.68, samples=54 | |
| write: IOPS=10.9k, BW=683MiB/s (716MB/s)(40.0GiB/60001msec); 0 zone resets | |
| slat (usec): min=12, max=347, avg=18.81, stdev= 5.47 | |
| clat (nsec): min=1790, max=8772.3M, avg=2867426.07, stdev=119755425.27 | |
| lat (usec): min=19, max=8772.3k, avg=2886.34, stdev=119755.42 | |
| clat percentiles (usec): | |
| | 1.00th=[ 938], 5.00th=[ 1020], 10.00th=[ 1090], | |
| | 20.00th=[ 1106], 30.00th=[ 1123], 40.00th=[ 1139], | |
| | 50.00th=[ 1156], 60.00th=[ 1172], 70.00th=[ 1188], | |
| | 80.00th=[ 1221], 90.00th=[ 1352], 95.00th=[ 1827], | |
| | 99.00th=[ 2114], 99.50th=[ 2147], 99.90th=[ 2409], | |
| | 99.95th=[ 2737], 99.99th=[8657044] | |
| bw ( MiB/s): min= 167, max= 1845, per=100.00%, avg=1487.57, stdev=383.67, samples=54 | |
| iops : min= 2676, max=29522, avg=23801.09, stdev=6138.75, samples=54 | |
| lat (usec) : 2=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01% | |
| lat (usec) : 750=0.01%, 1000=3.70% | |
| lat (msec) : 2=92.54%, 4=3.72%, 10=0.01%, 20=0.01%, 50=0.01% | |
| lat (msec) : >=2000=0.02% | |
| cpu : usr=2.86%, sys=65.02%, ctx=637533, majf=0, minf=23 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=656271,655605,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=684MiB/s (717MB/s), 684MiB/s-684MiB/s (717MB/s-717MB/s), io=40.1GiB (43.0GB), run=60001-60001msec | |
| WRITE: bw=683MiB/s (716MB/s), 683MiB/s-683MiB/s (716MB/s-716MB/s), io=40.0GiB (42.0GB), run=60001-60001msec | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + cat | |
| ++ expr 10000 / 16 | |
| + cat /tmp/jobfile.fio | |
| [test-64kchunk-16thread] | |
| bs=64k | |
| filename=/dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| size=625M | |
| runtime=60 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio /tmp/jobfile.fio | |
| test-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=20519: Wed Feb 26 09:26:07 2020 | |
| read: IOPS=51.4k, BW=3214MiB/s (3370MB/s)(188GiB/60002msec) | |
| slat (usec): min=5, max=11527, avg=40.98, stdev=46.95 | |
| clat (nsec): min=1640, max=2457.3M, avg=9765413.70, stdev=95411969.51 | |
| lat (usec): min=19, max=2457.3k, avg=9806.48, stdev=95412.02 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1237], 5.00th=[ 1598], 10.00th=[ 1827], | |
| | 20.00th=[ 2147], 30.00th=[ 2376], 40.00th=[ 2606], | |
| | 50.00th=[ 2835], 60.00th=[ 3064], 70.00th=[ 3392], | |
| | 80.00th=[ 3916], 90.00th=[ 5604], 95.00th=[ 6980], | |
| | 99.00th=[ 10814], 99.50th=[ 459277], 99.90th=[1702888], | |
| | 99.95th=[1753220], 99.99th=[1988101] | |
| bw ( MiB/s): min= 22, max=12782, per=100.00%, avg=5024.51, stdev=258.23, samples=1215 | |
| iops : min= 361, max=204512, avg=80391.88, stdev=4131.65, samples=1215 | |
| write: IOPS=51.5k, BW=3218MiB/s (3375MB/s)(189GiB/60002msec); 0 zone resets | |
| slat (usec): min=10, max=4463, avg=60.98, stdev=50.25 | |
| clat (nsec): min=1890, max=2456.4M, avg=9811268.90, stdev=95578988.10 | |
| lat (usec): min=25, max=2456.5k, avg=9872.35, stdev=95580.57 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1237], 5.00th=[ 1598], 10.00th=[ 1844], | |
| | 20.00th=[ 2147], 30.00th=[ 2409], 40.00th=[ 2638], | |
| | 50.00th=[ 2868], 60.00th=[ 3097], 70.00th=[ 3392], | |
| | 80.00th=[ 3949], 90.00th=[ 5604], 95.00th=[ 7046], | |
| | 99.00th=[ 10945], 99.50th=[ 459277], 99.90th=[1702888], | |
| | 99.95th=[1753220], 99.99th=[2021655] | |
| bw ( MiB/s): min= 23, max=12836, per=100.00%, avg=5023.58, stdev=258.57, samples=1217 | |
| iops : min= 369, max=205384, avg=80377.04, stdev=4137.09, samples=1217 | |
| lat (usec) : 2=0.01%, 4=0.01%, 20=0.01%, 50=0.01%, 100=0.01% | |
| lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.11% | |
| lat (msec) : 2=15.13%, 4=65.54%, 10=17.95%, 20=0.65%, 250=0.01% | |
| lat (msec) : 500=0.17%, 750=0.12%, 2000=0.30%, >=2000=0.01% | |
| cpu : usr=1.69%, sys=81.87%, ctx=7763019, majf=0, minf=185829 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=3085811,3089780,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=3214MiB/s (3370MB/s), 3214MiB/s-3214MiB/s (3370MB/s-3370MB/s), io=188GiB (202GB), run=60002-60002msec | |
| WRITE: bw=3218MiB/s (3375MB/s), 3218MiB/s-3218MiB/s (3375MB/s-3375MB/s), io=189GiB (202GB), run=60002-60002msec | |
| + for DISK in $FULLPATHS | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo 'testing /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438' | |
| testing /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + cat | |
| ++ expr 10000 / 1 | |
| + cat /tmp/jobfile.fio | |
| [test-64kchunk-1thread] | |
| bs=64k | |
| filename=/dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| size=10000M | |
| runtime=60 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio /tmp/jobfile.fio | |
| test-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=21636: Wed Feb 26 09:27:09 2020 | |
| read: IOPS=11.4k, BW=710MiB/s (744MB/s)(41.6GiB/60001msec) | |
| slat (usec): min=4, max=547, avg=17.60, stdev=20.01 | |
| clat (nsec): min=1570, max=8585.3M, avg=2788272.03, stdev=116568477.32 | |
| lat (usec): min=9, max=8585.3k, avg=2805.96, stdev=116568.45 | |
| clat percentiles (usec): | |
| | 1.00th=[ 922], 5.00th=[ 996], 10.00th=[ 1057], | |
| | 20.00th=[ 1106], 30.00th=[ 1123], 40.00th=[ 1123], | |
| | 50.00th=[ 1139], 60.00th=[ 1156], 70.00th=[ 1172], | |
| | 80.00th=[ 1205], 90.00th=[ 1352], 95.00th=[ 1762], | |
| | 99.00th=[ 2114], 99.50th=[ 2147], 99.90th=[ 2376], | |
| | 99.95th=[ 2638], 99.99th=[8556381] | |
| bw ( MiB/s): min= 26, max= 1879, per=100.00%, avg=1518.66, stdev=404.11, samples=55 | |
| iops : min= 424, max=30071, avg=24298.47, stdev=6465.77, samples=55 | |
| write: IOPS=11.3k, BW=709MiB/s (743MB/s)(41.5GiB/60001msec); 0 zone resets | |
| slat (usec): min=12, max=279, avg=18.35, stdev= 5.25 | |
| clat (usec): min=10, max=8585.3k, avg=2762.59, stdev=115783.39 | |
| lat (usec): min=24, max=8585.3k, avg=2781.04, stdev=115783.38 | |
| clat percentiles (usec): | |
| | 1.00th=[ 922], 5.00th=[ 996], 10.00th=[ 1057], | |
| | 20.00th=[ 1106], 30.00th=[ 1106], 40.00th=[ 1123], | |
| | 50.00th=[ 1139], 60.00th=[ 1156], 70.00th=[ 1172], | |
| | 80.00th=[ 1205], 90.00th=[ 1319], 95.00th=[ 1745], | |
| | 99.00th=[ 2114], 99.50th=[ 2114], 99.90th=[ 2409], | |
| | 99.95th=[ 2704], 99.99th=[8556381] | |
| bw ( MiB/s): min= 27, max= 1872, per=100.00%, avg=1517.08, stdev=404.59, samples=55 | |
| iops : min= 446, max=29958, avg=24273.24, stdev=6473.42, samples=55 | |
| lat (usec) : 2=0.01%, 20=0.01%, 50=0.01%, 100=0.01%, 250=0.01% | |
| lat (usec) : 500=0.01%, 750=0.01%, 1000=5.06% | |
| lat (msec) : 2=91.72%, 4=3.18%, 10=0.01%, 20=0.01%, 50=0.01% | |
| lat (msec) : >=2000=0.02% | |
| cpu : usr=3.28%, sys=65.35%, ctx=615099, majf=0, minf=24 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=681467,680656,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=710MiB/s (744MB/s), 710MiB/s-710MiB/s (744MB/s-744MB/s), io=41.6GiB (44.7GB), run=60001-60001msec | |
| WRITE: bw=709MiB/s (743MB/s), 709MiB/s-709MiB/s (743MB/s-743MB/s), io=41.5GiB (44.6GB), run=60001-60001msec | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + cat | |
| ++ expr 10000 / 16 | |
| + cat /tmp/jobfile.fio | |
| [test-64kchunk-16thread] | |
| bs=64k | |
| filename=/dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| size=625M | |
| runtime=60 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio /tmp/jobfile.fio | |
| test-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=22772: Wed Feb 26 09:28:11 2020 | |
| read: IOPS=49.6k, BW=3097MiB/s (3247MB/s)(185GiB/61221msec) | |
| slat (usec): min=5, max=8409, avg=42.38, stdev=49.27 | |
| clat (usec): min=4, max=2372.2k, avg=10138.54, stdev=99611.93 | |
| lat (usec): min=32, max=2372.2k, avg=10181.01, stdev=99611.83 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1205], 5.00th=[ 1516], 10.00th=[ 1827], | |
| | 20.00th=[ 2212], 30.00th=[ 2474], 40.00th=[ 2737], | |
| | 50.00th=[ 2966], 60.00th=[ 3261], 70.00th=[ 3621], | |
| | 80.00th=[ 4178], 90.00th=[ 5473], 95.00th=[ 6849], | |
| | 99.00th=[ 10945], 99.50th=[ 471860], 99.90th=[1753220], | |
| | 99.95th=[1803551], 99.99th=[2071987] | |
| bw ( MiB/s): min= 53, max=12784, per=100.00%, avg=5020.07, stdev=246.44, samples=1209 | |
| iops : min= 852, max=204548, avg=80320.74, stdev=3943.01, samples=1209 | |
| write: IOPS=49.6k, BW=3101MiB/s (3252MB/s)(185GiB/61221msec); 0 zone resets | |
| slat (usec): min=9, max=4239, avg=62.55, stdev=50.85 | |
| clat (usec): min=4, max=2370.4k, avg=10177.00, stdev=99732.31 | |
| lat (usec): min=105, max=2370.5k, avg=10239.66, stdev=99734.17 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1205], 5.00th=[ 1532], 10.00th=[ 1827], | |
| | 20.00th=[ 2212], 30.00th=[ 2474], 40.00th=[ 2737], | |
| | 50.00th=[ 2999], 60.00th=[ 3294], 70.00th=[ 3654], | |
| | 80.00th=[ 4228], 90.00th=[ 5473], 95.00th=[ 6849], | |
| | 99.00th=[ 10945], 99.50th=[ 476054], 99.90th=[1753220], | |
| | 99.95th=[1803551], 99.99th=[2071987] | |
| bw ( MiB/s): min= 65, max=12877, per=100.00%, avg=5026.39, stdev=246.61, samples=1209 | |
| iops : min= 1050, max=206032, avg=80421.85, stdev=3945.70, samples=1209 | |
| lat (usec) : 10=0.01%, 20=0.01%, 750=0.01%, 1000=0.16% | |
| lat (msec) : 2=13.56%, 4=63.47%, 10=21.51%, 20=0.70%, 50=0.01% | |
| lat (msec) : 250=0.01%, 500=0.14%, 750=0.15%, 2000=0.31%, >=2000=0.01% | |
| cpu : usr=1.75%, sys=81.57%, ctx=7656226, majf=0, minf=191138 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=3033555,3037570,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=3097MiB/s (3247MB/s), 3097MiB/s-3097MiB/s (3247MB/s-3247MB/s), io=185GiB (199GB), run=61221-61221msec | |
| WRITE: bw=3101MiB/s (3252MB/s), 3101MiB/s-3101MiB/s (3252MB/s-3252MB/s), io=185GiB (199GB), run=61221-61221msec | |
| + for DISK in $FULLPATHS | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo 'testing /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442' | |
| testing /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + cat | |
| ++ expr 10000 / 1 | |
| + cat /tmp/jobfile.fio | |
| [test-64kchunk-1thread] | |
| bs=64k | |
| filename=/dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| size=10000M | |
| runtime=60 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio /tmp/jobfile.fio | |
| test-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=23916: Wed Feb 26 09:29:13 2020 | |
| read: IOPS=11.1k, BW=692MiB/s (725MB/s)(40.5GiB/60001msec) | |
| slat (usec): min=4, max=510, avg=17.59, stdev=20.08 | |
| clat (usec): min=2, max=8766.2k, avg=2862.01, stdev=120619.44 | |
| lat (usec): min=48, max=8766.3k, avg=2879.68, stdev=120619.43 | |
| clat percentiles (usec): | |
| | 1.00th=[ 922], 5.00th=[ 996], 10.00th=[ 1057], | |
| | 20.00th=[ 1106], 30.00th=[ 1123], 40.00th=[ 1139], | |
| | 50.00th=[ 1139], 60.00th=[ 1156], 70.00th=[ 1172], | |
| | 80.00th=[ 1221], 90.00th=[ 1319], 95.00th=[ 1565], | |
| | 99.00th=[ 2089], 99.50th=[ 2114], 99.90th=[ 2442], | |
| | 99.95th=[ 2802], 99.99th=[8791262] | |
| bw ( MiB/s): min= 81, max= 1876, per=100.00%, avg=1535.56, stdev=349.79, samples=53 | |
| iops : min= 1296, max=30028, avg=24569.02, stdev=5596.60, samples=53 | |
| write: IOPS=11.1k, BW=691MiB/s (724MB/s)(40.5GiB/60001msec); 0 zone resets | |
| slat (usec): min=12, max=306, avg=18.31, stdev= 5.09 | |
| clat (usec): min=49, max=8766.3k, avg=2834.62, stdev=119801.24 | |
| lat (usec): min=64, max=8766.3k, avg=2853.03, stdev=119801.24 | |
| clat percentiles (usec): | |
| | 1.00th=[ 922], 5.00th=[ 996], 10.00th=[ 1057], | |
| | 20.00th=[ 1106], 30.00th=[ 1123], 40.00th=[ 1123], | |
| | 50.00th=[ 1139], 60.00th=[ 1156], 70.00th=[ 1172], | |
| | 80.00th=[ 1205], 90.00th=[ 1303], 95.00th=[ 1516], | |
| | 99.00th=[ 2089], 99.50th=[ 2114], 99.90th=[ 2540], | |
| | 99.95th=[ 2868], 99.99th=[8791262] | |
| bw ( MiB/s): min= 79, max= 1872, per=100.00%, avg=1534.27, stdev=352.82, samples=53 | |
| iops : min= 1278, max=29964, avg=24548.28, stdev=5645.18, samples=53 | |
| lat (usec) : 4=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01% | |
| lat (usec) : 750=0.01%, 1000=5.22% | |
| lat (msec) : 2=92.11%, 4=2.63%, 10=0.01%, 20=0.01%, 50=0.01% | |
| lat (msec) : >=2000=0.02% | |
| cpu : usr=2.99%, sys=64.64%, ctx=629456, majf=0, minf=24 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=663958,663295,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=692MiB/s (725MB/s), 692MiB/s-692MiB/s (725MB/s-725MB/s), io=40.5GiB (43.5GB), run=60001-60001msec | |
| WRITE: bw=691MiB/s (724MB/s), 691MiB/s-691MiB/s (724MB/s-724MB/s), io=40.5GiB (43.5GB), run=60001-60001msec | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + cat | |
| ++ expr 10000 / 16 | |
| + cat /tmp/jobfile.fio | |
| [test-64kchunk-16thread] | |
| bs=64k | |
| filename=/dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| size=625M | |
| runtime=60 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio /tmp/jobfile.fio | |
| test-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=24998: Wed Feb 26 09:30:15 2020 | |
| read: IOPS=49.7k, BW=3108MiB/s (3259MB/s)(185GiB/61000msec) | |
| slat (usec): min=5, max=25016, avg=42.90, stdev=110.43 | |
| clat (usec): min=5, max=2358.9k, avg=10105.31, stdev=98662.06 | |
| lat (usec): min=42, max=2358.9k, avg=10148.29, stdev=98662.38 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1237], 5.00th=[ 1582], 10.00th=[ 1876], | |
| | 20.00th=[ 2245], 30.00th=[ 2540], 40.00th=[ 2769], | |
| | 50.00th=[ 2999], 60.00th=[ 3261], 70.00th=[ 3589], | |
| | 80.00th=[ 4113], 90.00th=[ 5538], 95.00th=[ 7177], | |
| | 99.00th=[ 11207], 99.50th=[ 476054], 99.90th=[1769997], | |
| | 99.95th=[1803551], 99.99th=[2164261] | |
| bw ( MiB/s): min= 34, max=12730, per=100.00%, avg=5103.38, stdev=244.50, samples=1189 | |
| iops : min= 556, max=203680, avg=81653.70, stdev=3912.09, samples=1189 | |
| write: IOPS=49.8k, BW=3112MiB/s (3263MB/s)(185GiB/61000msec); 0 zone resets | |
| slat (usec): min=10, max=24082, avg=63.85, stdev=111.52 | |
| clat (usec): min=3, max=2358.9k, avg=10143.67, stdev=98705.39 | |
| lat (usec): min=78, max=2359.0k, avg=10207.64, stdev=98707.68 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1254], 5.00th=[ 1582], 10.00th=[ 1876], | |
| | 20.00th=[ 2278], 30.00th=[ 2540], 40.00th=[ 2769], | |
| | 50.00th=[ 3032], 60.00th=[ 3261], 70.00th=[ 3589], | |
| | 80.00th=[ 4113], 90.00th=[ 5538], 95.00th=[ 7177], | |
| | 99.00th=[ 11207], 99.50th=[ 480248], 99.90th=[1769997], | |
| | 99.95th=[1803551], 99.99th=[2197816] | |
| bw ( MiB/s): min= 31, max=12749, per=100.00%, avg=5101.27, stdev=245.19, samples=1191 | |
| iops : min= 504, max=203998, avg=81619.85, stdev=3923.01, samples=1191 | |
| lat (usec) : 4=0.01%, 10=0.01%, 1000=0.12% | |
| lat (msec) : 2=12.45%, 4=65.85%, 10=20.16%, 20=0.80%, 50=0.01% | |
| lat (msec) : 100=0.01%, 250=0.02%, 500=0.13%, 750=0.15%, 2000=0.30% | |
| lat (msec) : >=2000=0.01% | |
| cpu : usr=1.64%, sys=81.86%, ctx=7812219, majf=0, minf=178052 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=3033555,3037570,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=3108MiB/s (3259MB/s), 3108MiB/s-3108MiB/s (3259MB/s-3259MB/s), io=185GiB (199GB), run=61000-61000msec | |
| WRITE: bw=3112MiB/s (3263MB/s), 3112MiB/s-3112MiB/s (3263MB/s-3263MB/s), io=185GiB (199GB), run=61000-61000msec | |
| + for DISK in $FULLPATHS | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo 'testing /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110' | |
| testing /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + cat | |
| ++ expr 10000 / 1 | |
| + cat /tmp/jobfile.fio | |
| [test-64kchunk-1thread] | |
| bs=64k | |
| filename=/dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| size=10000M | |
| runtime=60 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio /tmp/jobfile.fio | |
| test-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=26136: Wed Feb 26 09:31:17 2020 | |
| read: IOPS=11.3k, BW=706MiB/s (740MB/s)(41.4GiB/60001msec) | |
| slat (usec): min=4, max=537, avg=17.67, stdev=20.39 | |
| clat (usec): min=2, max=8634.6k, avg=2803.62, stdev=117440.44 | |
| lat (usec): min=62, max=8634.6k, avg=2821.36, stdev=117440.43 | |
| clat percentiles (usec): | |
| | 1.00th=[ 914], 5.00th=[ 996], 10.00th=[ 1057], | |
| | 20.00th=[ 1090], 30.00th=[ 1106], 40.00th=[ 1123], | |
| | 50.00th=[ 1139], 60.00th=[ 1156], 70.00th=[ 1188], | |
| | 80.00th=[ 1254], 90.00th=[ 1336], 95.00th=[ 1582], | |
| | 99.00th=[ 2089], 99.50th=[ 2114], 99.90th=[ 2343], | |
| | 99.95th=[ 2638], 99.99th=[8657044] | |
| bw ( MiB/s): min= 392, max= 1904, per=100.00%, avg=1566.95, stdev=291.00, samples=53 | |
| iops : min= 6276, max=30470, avg=25071.23, stdev=4656.11, samples=53 | |
| write: IOPS=11.3k, BW=705MiB/s (739MB/s)(41.3GiB/60001msec); 0 zone resets | |
| slat (usec): min=12, max=304, avg=18.35, stdev= 5.19 | |
| clat (usec): min=65, max=8634.6k, avg=2777.23, stdev=116662.53 | |
| lat (usec): min=92, max=8634.6k, avg=2795.67, stdev=116662.52 | |
| clat percentiles (usec): | |
| | 1.00th=[ 914], 5.00th=[ 996], 10.00th=[ 1057], | |
| | 20.00th=[ 1090], 30.00th=[ 1106], 40.00th=[ 1123], | |
| | 50.00th=[ 1139], 60.00th=[ 1156], 70.00th=[ 1172], | |
| | 80.00th=[ 1237], 90.00th=[ 1319], 95.00th=[ 1549], | |
| | 99.00th=[ 2089], 99.50th=[ 2114], 99.90th=[ 2376], | |
| | 99.95th=[ 2704], 99.99th=[8657044] | |
| bw ( MiB/s): min= 399, max= 1885, per=100.00%, avg=1565.46, stdev=292.05, samples=53 | |
| iops : min= 6390, max=30168, avg=25047.30, stdev=4672.88, samples=53 | |
| lat (usec) : 4=0.01%, 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01% | |
| lat (usec) : 1000=5.48% | |
| lat (msec) : 2=91.62%, 4=2.85%, 10=0.01%, 20=0.01%, 50=0.01% | |
| lat (msec) : >=2000=0.02% | |
| cpu : usr=3.10%, sys=64.84%, ctx=612499, majf=0, minf=23 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=677834,676953,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=706MiB/s (740MB/s), 706MiB/s-706MiB/s (740MB/s-740MB/s), io=41.4GiB (44.4GB), run=60001-60001msec | |
| WRITE: bw=705MiB/s (739MB/s), 705MiB/s-705MiB/s (739MB/s-739MB/s), io=41.3GiB (44.4GB), run=60001-60001msec | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + cat | |
| ++ expr 10000 / 16 | |
| + cat /tmp/jobfile.fio | |
| [test-64kchunk-16thread] | |
| bs=64k | |
| filename=/dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| size=625M | |
| runtime=60 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio /tmp/jobfile.fio | |
| test-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=27274: Wed Feb 26 09:32:19 2020 | |
| read: IOPS=49.8k, BW=3115MiB/s (3266MB/s)(185GiB/60864msec) | |
| slat (usec): min=4, max=4167, avg=43.40, stdev=47.56 | |
| clat (usec): min=4, max=2332.0k, avg=10072.07, stdev=97858.53 | |
| lat (usec): min=42, max=2332.0k, avg=10115.56, stdev=97858.42 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1270], 5.00th=[ 1598], 10.00th=[ 1844], | |
| | 20.00th=[ 2212], 30.00th=[ 2540], 40.00th=[ 2769], | |
| | 50.00th=[ 3032], 60.00th=[ 3294], 70.00th=[ 3621], | |
| | 80.00th=[ 4178], 90.00th=[ 5473], 95.00th=[ 6849], | |
| | 99.00th=[ 10945], 99.50th=[ 484443], 99.90th=[1736442], | |
| | 99.95th=[1769997], 99.99th=[2071987] | |
| bw ( MiB/s): min= 36, max=12187, per=100.00%, avg=5047.50, stdev=243.59, samples=1202 | |
| iops : min= 580, max=194996, avg=80759.58, stdev=3897.42, samples=1202 | |
| write: IOPS=49.9k, BW=3119MiB/s (3271MB/s)(185GiB/60864msec); 0 zone resets | |
| slat (usec): min=10, max=4151, avg=62.00, stdev=50.73 | |
| clat (usec): min=4, max=2331.6k, avg=10119.60, stdev=98085.61 | |
| lat (usec): min=52, max=2331.7k, avg=10181.70, stdev=98086.99 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1270], 5.00th=[ 1598], 10.00th=[ 1860], | |
| | 20.00th=[ 2245], 30.00th=[ 2540], 40.00th=[ 2802], | |
| | 50.00th=[ 3032], 60.00th=[ 3294], 70.00th=[ 3621], | |
| | 80.00th=[ 4178], 90.00th=[ 5473], 95.00th=[ 6849], | |
| | 99.00th=[ 10945], 99.50th=[ 484443], 99.90th=[1736442], | |
| | 99.95th=[1769997], 99.99th=[2088764] | |
| bw ( MiB/s): min= 34, max=12198, per=100.00%, avg=5045.86, stdev=244.26, samples=1204 | |
| iops : min= 554, max=195176, avg=80733.39, stdev=3908.16, samples=1204 | |
| lat (usec) : 10=0.01%, 750=0.01%, 1000=0.13% | |
| lat (msec) : 2=13.45%, 4=63.94%, 10=21.23%, 20=0.64%, 250=0.01% | |
| lat (msec) : 500=0.11%, 750=0.17%, 2000=0.30%, >=2000=0.01% | |
| cpu : usr=1.82%, sys=82.04%, ctx=7411939, majf=0, minf=195639 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=3033555,3037570,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=3115MiB/s (3266MB/s), 3115MiB/s-3115MiB/s (3266MB/s-3266MB/s), io=185GiB (199GB), run=60864-60864msec | |
| WRITE: bw=3119MiB/s (3271MB/s), 3119MiB/s-3119MiB/s (3271MB/s-3271MB/s), io=185GiB (199GB), run=60864-60864msec | |
| + do_zfs_tests | |
| + for ASHIFTVAL in 12 13 14 15 16 | |
| + trim_disks | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| Success formatting namespace:1 | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '################## waiting 60s for nvme format to complete...' | |
| ################## waiting 60s for nvme format to complete... | |
| + sleep 60 | |
| + nvme list | |
| Node SN Model Namespace Usage Format FW Rev | |
| ---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- -------- | |
| /dev/nvme0n1 S437NE0M804438 SAMSUNG MZQLB960HAJR-00007 1 0.00 B / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme1n1 S437NE0M804437 SAMSUNG MZQLB960HAJR-00007 1 0.00 B / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme2n1 S437NE0M804442 SAMSUNG MZQLB960HAJR-00007 1 0.00 B / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme3n1 S437NE0M804435 SAMSUNG MZQLB960HAJR-00007 1 0.00 B / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme4n1 S437NE0M802527 SAMSUNG MZQLB960HAJR-00007 1 0.00 B / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme5n1 S437NX0M402110 SAMSUNG MZQLB960HAJR-00007 1 0.00 B / 960.20 GB 512 B + 0 B EDA5202Q | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + zpool create tank -o ashift=12 -O mountpoint=/srv/z raidz1 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| + dd if=/dev/urandom of=/tmp/testkey.raw bs=32 count=1 | |
| 1+0 records in | |
| 1+0 records out | |
| 32 bytes copied, 0.000220101 s, 145 kB/s | |
| + COMP=' | |
| -o compression=lz4 | |
| ' | |
| + ENC=' | |
| -o encryption=aes-256-gcm | |
| -o keyformat=raw | |
| -o keylocation=file:///tmp/testkey.raw | |
| ' | |
| + zfs create tank/none | |
| + zfs create -o compression=lz4 tank/comp | |
| + zfs create -o encryption=aes-256-gcm -o keyformat=raw -o keylocation=file:///tmp/testkey.raw tank/enc | |
| + zfs create -o compression=lz4 -o encryption=aes-256-gcm -o keyformat=raw -o keylocation=file:///tmp/testkey.raw tank/both | |
| + FO='--eta-newline=10 --eta-interval=10 --eta=always' | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-none-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/none/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-none-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-none-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=1086MiB/s,w=1086MiB/s][r=17.4k,w=17.4k IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.7%][r=1076MiB/s,w=1074MiB/s][r=17.2k,w=17.2k IOPS][eta 01m:40s] Jobs: 1 (f=1): [M(1)][24.2%][r=1062MiB/s,w=1060MiB/s][r=16.0k,w=16.0k IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.8%][r=1035MiB/s,w=1036MiB/s][r=16.6k,w=16.6k IOPS][eta 01m:20s] Jobs: 1 (f=1): [M(1)][40.8%][r=1049MiB/s,w=1047MiB/s][r=16.8k,w=16.8k IOPS][eta 01m:11s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=1079MiB/s,w=1078MiB/s][r=17.3k,w=17.2k IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=1085MiB/s,w=1090MiB/s][r=17.4k,w=17.4k IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][65.5%][r=1054MiB/s,w=1048MiB/s][r=16.9k,w=16.8k IOPS][eta 00m:41s] Jobs: 1 (f=1): [M(1)][72.5%][r=1045MiB/s,w=1044MiB/s][r=16.7k,w=16.7k IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=1080MiB/s,w=1081MiB/s][r=17.3k,w=17.3k IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][89.9%][r=1044MiB/s,w=1042MiB/s][r=16.7k,w=16.7k IOPS][eta 00m:12s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=1016MiB/s,w=1014MiB/s][r=16.3k,w=16.2k IOPS][eta 00m:04s] | |
| test-none-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=45480: Wed Feb 26 09:35:40 2020 | |
| read: IOPS=16.0k, BW=1061MiB/s (1112MB/s)(124GiB/120001msec) | |
| slat (usec): min=5, max=45006, avg=27.86, stdev=153.59 | |
| clat (usec): min=270, max=76365, avg=1857.82, stdev=1904.16 | |
| lat (usec): min=280, max=76411, avg=1885.83, stdev=1927.52 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1172], 5.00th=[ 1270], 10.00th=[ 1287], 20.00th=[ 1319], | |
| | 30.00th=[ 1385], 40.00th=[ 1450], 50.00th=[ 1467], 60.00th=[ 1500], | |
| | 70.00th=[ 1549], 80.00th=[ 1663], 90.00th=[ 2638], 95.00th=[ 3589], | |
| | 99.00th=[10028], 99.50th=[14746], 99.90th=[26084], 99.95th=[30540], | |
| | 99.99th=[43254] | |
| bw ( MiB/s): min= 767, max= 1517, per=100.00%, avg=1061.16, stdev=168.12, samples=239 | |
| iops : min=12284, max=24274, avg=16978.59, stdev=2689.83, samples=239 | |
| write: IOPS=16.9k, BW=1059MiB/s (1111MB/s)(124GiB/120001msec); 0 zone resets | |
| slat (usec): min=11, max=35270, avg=28.36, stdev=147.03 | |
| clat (nsec): min=1700, max=76349k, avg=1857943.22, stdev=1912268.19 | |
| lat (usec): min=21, max=76406, avg=1886.46, stdev=1935.27 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1172], 5.00th=[ 1270], 10.00th=[ 1287], 20.00th=[ 1319], | |
| | 30.00th=[ 1385], 40.00th=[ 1450], 50.00th=[ 1467], 60.00th=[ 1500], | |
| | 70.00th=[ 1549], 80.00th=[ 1663], 90.00th=[ 2638], 95.00th=[ 3589], | |
| | 99.00th=[10028], 99.50th=[14877], 99.90th=[26084], 99.95th=[30802], | |
| | 99.99th=[43779] | |
| bw ( MiB/s): min= 773, max= 1512, per=100.00%, avg=1059.88, stdev=168.67, samples=239 | |
| iops : min=12374, max=24202, avg=16958.00, stdev=2698.73, samples=239 | |
| lat (usec) : 2=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01% | |
| lat (usec) : 750=0.01%, 1000=0.02% | |
| lat (msec) : 2=87.60%, 4=8.52%, 10=2.86%, 20=0.75%, 50=0.24% | |
| lat (msec) : 100=0.01% | |
| cpu : usr=6.96%, sys=81.53%, ctx=26531, majf=0, minf=53401 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=2036380,2033954,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=1061MiB/s (1112MB/s), 1061MiB/s-1061MiB/s (1112MB/s-1112MB/s), io=124GiB (133GB), run=120001-120001msec | |
| WRITE: bw=1059MiB/s (1111MB/s), 1059MiB/s-1059MiB/s (1111MB/s-1111MB/s), io=124GiB (133GB), run=120001-120001msec | |
| + rm -rf /srv/z/none/test-none-64kchunk-1thread.0.0 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-comp-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/comp/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-comp-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-comp-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=1066MiB/s,w=1066MiB/s][r=17.1k,w=17.0k IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.7%][r=1080MiB/s,w=1078MiB/s][r=17.3k,w=17.2k IOPS][eta 01m:40s] Jobs: 1 (f=1): [M(1)][24.2%][r=1059MiB/s,w=1057MiB/s][r=16.9k,w=16.9k IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.8%][r=1057MiB/s,w=1057MiB/s][r=16.9k,w=16.9k IOPS][eta 01m:20s] Jobs: 1 (f=1): [M(1)][40.5%][r=1065MiB/s,w=1063MiB/s][r=17.0k,w=17.0k IOPS][eta 01m:12s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=1049MiB/s,w=1048MiB/s][r=16.8k,w=16.8k IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=1054MiB/s,w=1058MiB/s][r=16.9k,w=16.9k IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][65.0%][r=1051MiB/s,w=1044MiB/s][r=16.8k,w=16.7k IOPS][eta 00m:42s] Jobs: 1 (f=1): [M(1)][72.5%][r=1029MiB/s,w=1029MiB/s][r=16.5k,w=16.5k IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=1067MiB/s,w=1069MiB/s][r=17.1k,w=17.1k IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][89.9%][r=1053MiB/s,w=1049MiB/s][r=16.9k,w=16.8k IOPS][eta 00m:12s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=1035MiB/s,w=1035MiB/s][r=16.6k,w=16.6k IOPS][eta 00m:04s] | |
| test-comp-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=329592: Wed Feb 26 09:37:52 2020 | |
| read: IOPS=16.9k, BW=1057MiB/s (1108MB/s)(124GiB/120001msec) | |
| slat (usec): min=5, max=45386, avg=28.13, stdev=184.98 | |
| clat (usec): min=171, max=58725, avg=1869.21, stdev=2170.95 | |
| lat (usec): min=180, max=58785, avg=1897.50, stdev=2195.89 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1188], 5.00th=[ 1254], 10.00th=[ 1270], 20.00th=[ 1319], | |
| | 30.00th=[ 1401], 40.00th=[ 1418], 50.00th=[ 1434], 60.00th=[ 1450], | |
| | 70.00th=[ 1516], 80.00th=[ 1614], 90.00th=[ 2147], 95.00th=[ 3425], | |
| | 99.00th=[12256], 99.50th=[16712], 99.90th=[29230], 99.95th=[34866], | |
| | 99.99th=[45351] | |
| bw ( MiB/s): min= 766, max= 1508, per=99.80%, avg=1054.58, stdev=180.94, samples=239 | |
| iops : min=12264, max=24134, avg=16873.17, stdev=2895.05, samples=239 | |
| write: IOPS=16.9k, BW=1055MiB/s (1107MB/s)(124GiB/120001msec); 0 zone resets | |
| slat (usec): min=11, max=36326, avg=28.23, stdev=181.65 | |
| clat (nsec): min=1890, max=57017k, avg=1860438.52, stdev=2134277.93 | |
| lat (usec): min=19, max=58647, avg=1888.83, stdev=2158.11 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1188], 5.00th=[ 1254], 10.00th=[ 1270], 20.00th=[ 1319], | |
| | 30.00th=[ 1401], 40.00th=[ 1418], 50.00th=[ 1434], 60.00th=[ 1450], | |
| | 70.00th=[ 1516], 80.00th=[ 1614], 90.00th=[ 2147], 95.00th=[ 3392], | |
| | 99.00th=[11994], 99.50th=[16450], 99.90th=[28705], 99.95th=[34341], | |
| | 99.99th=[45351] | |
| bw ( MiB/s): min= 784, max= 1505, per=99.81%, avg=1053.39, stdev=182.57, samples=239 | |
| iops : min=12556, max=24086, avg=16854.10, stdev=2921.09, samples=239 | |
| lat (usec) : 2=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01% | |
| lat (usec) : 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=89.34%, 4=6.64%, 10=2.62%, 20=1.07%, 50=0.32% | |
| lat (msec) : 100=0.01% | |
| cpu : usr=7.29%, sys=78.09%, ctx=26221, majf=0, minf=39787 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=2028860,2026368,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=1057MiB/s (1108MB/s), 1057MiB/s-1057MiB/s (1108MB/s-1108MB/s), io=124GiB (133GB), run=120001-120001msec | |
| WRITE: bw=1055MiB/s (1107MB/s), 1055MiB/s-1055MiB/s (1107MB/s-1107MB/s), io=124GiB (133GB), run=120001-120001msec | |
| + rm -rf /srv/z/comp/test-comp-64kchunk-1thread.0.0 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-enc-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/enc/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-enc-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-enc-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=34.5MiB/s,w=34.7MiB/s][r=552,w=555 IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.7%][r=35.2MiB/s,w=34.8MiB/s][r=563,w=556 IOPS][eta 01m:40s] Jobs: 1 (f=1): [M(1)][24.2%][r=34.6MiB/s,w=34.4MiB/s][r=552,w=550 IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.5%][r=34.9MiB/s,w=35.0MiB/s][r=558,w=560 IOPS][eta 01m:21s] Jobs: 1 (f=1): [M(1)][40.8%][r=35.0MiB/s,w=35.1MiB/s][r=560,w=562 IOPS][eta 01m:11s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=34.6MiB/s,w=35.1MiB/s][r=553,w=561 IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=35.2MiB/s,w=34.0MiB/s][r=563,w=559 IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][65.0%][r=34.0MiB/s,w=32.7MiB/s][r=559,w=523 IOPS][eta 00m:42s] Jobs: 1 (f=1): [M(1)][72.5%][r=35.6MiB/s,w=35.7MiB/s][r=569,w=570 IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=34.9MiB/s,w=35.5MiB/s][r=558,w=568 IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][89.2%][r=34.4MiB/s,w=34.7MiB/s][r=549,w=554 IOPS][eta 00m:13s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=34.8MiB/s,w=33.6MiB/s][r=556,w=538 IOPS][eta 00m:04s] | |
| test-enc-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=549276: Wed Feb 26 09:40:05 2020 | |
| read: IOPS=558, BW=34.9MiB/s (36.6MB/s)(4191MiB/120002msec) | |
| slat (usec): min=6, max=84039, avg=1142.14, stdev=1724.94 | |
| clat (usec): min=3, max=190055, avg=56516.69, stdev=11018.76 | |
| lat (msec): min=3, max=193, avg=57.66, stdev=11.17 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 45], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 53], 50.00th=[ 55], 60.00th=[ 58], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 66], 95.00th=[ 75], | |
| | 99.00th=[ 101], 99.50th=[ 108], 99.90th=[ 153], 99.95th=[ 161], | |
| | 99.99th=[ 186] | |
| bw ( KiB/s): min=23808, max=42624, per=99.94%, avg=35739.38, stdev=3392.07, samples=240 | |
| iops : min= 372, max= 666, avg=558.35, stdev=52.98, samples=240 | |
| write: IOPS=556, BW=34.8MiB/s (36.5MB/s)(4172MiB/120002msec); 0 zone resets | |
| slat (usec): min=12, max=97834, avg=647.42, stdev=1410.33 | |
| clat (msec): min=3, max=189, avg=56.45, stdev=10.94 | |
| lat (msec): min=3, max=190, avg=57.10, stdev=11.01 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 45], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 53], 50.00th=[ 55], 60.00th=[ 58], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 66], 95.00th=[ 74], | |
| | 99.00th=[ 100], 99.50th=[ 107], 99.90th=[ 155], 99.95th=[ 176], | |
| | 99.99th=[ 188] | |
| bw ( KiB/s): min=19584, max=43904, per=99.94%, avg=35575.20, stdev=3898.13, samples=240 | |
| iops : min= 306, max= 686, avg=555.79, stdev=60.89, samples=240 | |
| lat (usec) : 4=0.01% | |
| lat (msec) : 4=0.01%, 10=0.01%, 20=0.01%, 50=23.06%, 100=75.92% | |
| lat (msec) : 250=1.01% | |
| cpu : usr=0.23%, sys=85.30%, ctx=12847, majf=0, minf=23064 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=67055,66749,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=34.9MiB/s (36.6MB/s), 34.9MiB/s-34.9MiB/s (36.6MB/s-36.6MB/s), io=4191MiB (4395MB), run=120002-120002msec | |
| WRITE: bw=34.8MiB/s (36.5MB/s), 34.8MiB/s-34.8MiB/s (36.5MB/s-36.5MB/s), io=4172MiB (4374MB), run=120002-120002msec | |
| + rm -rf /srv/z/enc/test-enc-64kchunk-1thread.0.0 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-both-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/both/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-both-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-both-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=33.0MiB/s,w=34.3MiB/s][r=543,w=548 IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.7%][r=35.1MiB/s,w=34.7MiB/s][r=561,w=554 IOPS][eta 01m:40s] Jobs: 1 (f=1): [M(1)][24.2%][r=34.6MiB/s,w=34.4MiB/s][r=553,w=550 IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.5%][r=35.1MiB/s,w=35.2MiB/s][r=561,w=563 IOPS][eta 01m:21s] Jobs: 1 (f=1): [M(1)][40.8%][r=34.6MiB/s,w=34.6MiB/s][r=553,w=552 IOPS][eta 01m:11s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=34.5MiB/s,w=34.0MiB/s][r=551,w=559 IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=35.4MiB/s,w=35.1MiB/s][r=565,w=562 IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][65.0%][r=35.1MiB/s,w=32.9MiB/s][r=562,w=525 IOPS][eta 00m:42s] Jobs: 1 (f=1): [M(1)][72.5%][r=35.8MiB/s,w=36.0MiB/s][r=572,w=576 IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=34.8MiB/s,w=35.5MiB/s][r=556,w=567 IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][89.2%][r=34.4MiB/s,w=34.4MiB/s][r=550,w=550 IOPS][eta 00m:13s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=34.6MiB/s,w=33.7MiB/s][r=553,w=538 IOPS][eta 00m:04s] | |
| test-both-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=660255: Wed Feb 26 09:42:18 2020 | |
| read: IOPS=558, BW=34.9MiB/s (36.6MB/s)(4187MiB/120004msec) | |
| slat (usec): min=6, max=101076, avg=1140.26, stdev=1721.99 | |
| clat (usec): min=3, max=193080, avg=56574.80, stdev=10885.48 | |
| lat (msec): min=3, max=196, avg=57.72, stdev=11.02 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 45], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 53], 50.00th=[ 55], 60.00th=[ 58], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 67], 95.00th=[ 75], | |
| | 99.00th=[ 99], 99.50th=[ 108], 99.90th=[ 144], 99.95th=[ 171], | |
| | 99.99th=[ 190] | |
| bw ( KiB/s): min=20864, max=43904, per=99.92%, avg=35696.89, stdev=3583.94, samples=240 | |
| iops : min= 326, max= 686, avg=557.68, stdev=55.99, samples=240 | |
| write: IOPS=555, BW=34.7MiB/s (36.4MB/s)(4167MiB/120004msec); 0 zone resets | |
| slat (usec): min=13, max=64594, avg=651.02, stdev=1400.21 | |
| clat (msec): min=3, max=193, avg=56.51, stdev=10.73 | |
| lat (msec): min=3, max=197, avg=57.16, stdev=10.82 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 45], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 53], 50.00th=[ 55], 60.00th=[ 58], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 66], 95.00th=[ 74], | |
| | 99.00th=[ 97], 99.50th=[ 107], 99.90th=[ 138], 99.95th=[ 174], | |
| | 99.99th=[ 190] | |
| bw ( KiB/s): min=19840, max=44928, per=99.92%, avg=35529.23, stdev=3883.82, samples=240 | |
| iops : min= 310, max= 702, avg=555.06, stdev=60.68, samples=240 | |
| lat (usec) : 4=0.01% | |
| lat (msec) : 4=0.01%, 10=0.01%, 20=0.01%, 50=22.78%, 100=76.36% | |
| lat (msec) : 250=0.84% | |
| cpu : usr=0.24%, sys=85.05%, ctx=13015, majf=0, minf=23151 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=66987,66675,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=34.9MiB/s (36.6MB/s), 34.9MiB/s-34.9MiB/s (36.6MB/s-36.6MB/s), io=4187MiB (4390MB), run=120004-120004msec | |
| WRITE: bw=34.7MiB/s (36.4MB/s), 34.7MiB/s-34.7MiB/s (36.4MB/s-36.4MB/s), io=4167MiB (4370MB), run=120004-120004msec | |
| + rm -rf /srv/z/both/test-both-64kchunk-1thread.0.0 | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-none-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/none/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-none-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=2791MiB/s,w=2789MiB/s][r=44.7k,w=44.6k IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.5%][r=2560MiB/s,w=2566MiB/s][r=40.0k,w=41.1k IOPS][eta 01m:41s] Jobs: 16 (f=256): [M(16)][24.2%][r=2510MiB/s,w=2518MiB/s][r=40.2k,w=40.3k IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.2%][r=2551MiB/s,w=2560MiB/s][r=40.8k,w=40.0k IOPS][eta 01m:22s] Jobs: 16 (f=256): [M(16)][40.0%][r=2705MiB/s,w=2701MiB/s][r=43.3k,w=43.2k IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][48.3%][r=2678MiB/s,w=2674MiB/s][r=42.9k,w=42.8k IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][56.2%][r=2726MiB/s,w=2729MiB/s][r=43.6k,w=43.7k IOPS][eta 00m:53s] | |
| Jobs: 16 (f=256): [M(16)][64.7%][r=2669MiB/s,w=2667MiB/s][r=42.7k,w=42.7k IOPS][eta 00m:42s] Jobs: 16 (f=256): [M(16)][71.9%][r=2682MiB/s,w=2681MiB/s][r=42.9k,w=42.9k IOPS][eta 00m:34s] | |
| Jobs: 16 (f=256): [M(16)][80.0%][r=2491MiB/s,w=2494MiB/s][r=39.9k,w=39.9k IOPS][eta 00m:24s] Jobs: 16 (f=256): [M(16)][88.3%][r=2506MiB/s,w=2509MiB/s][r=40.1k,w=40.1k IOPS][eta 00m:14s] | |
| Jobs: 16 (f=256): [M(16)][95.9%][r=2503MiB/s,w=2503MiB/s][r=40.0k,w=40.0k IOPS][eta 00m:05s] | |
| test-none-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=951511: Wed Feb 26 09:47:03 2020 | |
| read: IOPS=41.8k, BW=2612MiB/s (2739MB/s)(306GiB/120006msec) | |
| slat (usec): min=8, max=106912, avg=172.59, stdev=843.02 | |
| clat (usec): min=3, max=253849, avg=12052.84, stdev=11554.85 | |
| lat (usec): min=28, max=254083, avg=12225.88, stdev=11726.35 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 5], 10.00th=[ 5], 20.00th=[ 6], | |
| | 30.00th=[ 6], 40.00th=[ 7], 50.00th=[ 8], 60.00th=[ 10], | |
| | 70.00th=[ 12], 80.00th=[ 17], 90.00th=[ 26], 95.00th=[ 35], | |
| | 99.00th=[ 60], 99.50th=[ 71], 99.90th=[ 97], 99.95th=[ 108], | |
| | 99.99th=[ 133] | |
| bw ( MiB/s): min= 1151, max= 5079, per=99.88%, avg=2608.92, stdev=49.55, samples=3833 | |
| iops : min=18425, max=81276, avg=41741.01, stdev=792.73, samples=3833 | |
| write: IOPS=41.8k, BW=2614MiB/s (2741MB/s)(306GiB/120006msec); 0 zone resets | |
| slat (usec): min=17, max=109582, avg=201.16, stdev=888.14 | |
| clat (usec): min=3, max=244378, avg=12053.70, stdev=11534.66 | |
| lat (usec): min=54, max=244425, avg=12255.35, stdev=11651.11 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 5], 10.00th=[ 5], 20.00th=[ 6], | |
| | 30.00th=[ 6], 40.00th=[ 7], 50.00th=[ 8], 60.00th=[ 10], | |
| | 70.00th=[ 12], 80.00th=[ 17], 90.00th=[ 26], 95.00th=[ 35], | |
| | 99.00th=[ 60], 99.50th=[ 71], 99.90th=[ 96], 99.95th=[ 107], | |
| | 99.99th=[ 131] | |
| bw ( MiB/s): min= 1143, max= 5107, per=99.88%, avg=2610.61, stdev=49.33, samples=3833 | |
| iops : min=18290, max=81716, avg=41768.10, stdev=789.21, samples=3833 | |
| lat (usec) : 4=0.01%, 10=0.01%, 50=0.01%, 100=0.01%, 250=0.01% | |
| lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=0.01%, 4=3.75%, 10=59.89%, 20=21.21%, 50=13.29% | |
| lat (msec) : 100=1.78%, 250=0.08%, 500=0.01% | |
| cpu : usr=3.73%, sys=48.67%, ctx=1305558, majf=0, minf=473389 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=5015611,5018721,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=2612MiB/s (2739MB/s), 2612MiB/s-2612MiB/s (2739MB/s-2739MB/s), io=306GiB (329GB), run=120006-120006msec | |
| WRITE: bw=2614MiB/s (2741MB/s), 2614MiB/s-2614MiB/s (2741MB/s-2741MB/s), io=306GiB (329GB), run=120006-120006msec | |
| + rm -rf /srv/z/none/test-none-64kchunk-16thread.0.0 /srv/z/none/test-none-64kchunk-16thread.0.1 /srv/z/none/test-none-64kchunk-16thread.0.10 /srv/z/none/test-none-64kchunk-16thread.0.11 /srv/z/none/test-none-64kchunk-16thread.0.12 /srv/z/none/test-none-64kchunk-16thread.0.13 /srv/z/none/test-none-64kchunk-16thread.0.14 /srv/z/none/test-none-64kchunk-16thread.0.15 /srv/z/none/test-none-64kchunk-16thread.0.2 /srv/z/none/test-none-64kchunk-16thread.0.3 /srv/z/none/test-none-64kchunk-16thread.0.4 /srv/z/none/test-none-64kchunk-16thread.0.5 /srv/z/none/test-none-64kchunk-16thread.0.6 /srv/z/none/test-none-64kchunk-16thread.0.7 /srv/z/none/test-none-64kchunk-16thread.0.8 /srv/z/none/test-none-64kchunk-16thread.0.9 /srv/z/none/test-none-64kchunk-16thread.1.0 /srv/z/none/test-none-64kchunk-16thread.10.0 /srv/z/none/test-none-64kchunk-16thread.10.1 /srv/z/none/test-none-64kchunk-16thread.10.10 /srv/z/none/test-none-64kchunk-16thread.10.11 /srv/z/none/test-none-64kchunk-16thread.10.12 /srv/z/none/test-none-64kchunk-16thread.10.13 /srv/z/none/test-none-64kchunk-16thread.10.14 /srv/z/none/test-none-64kchunk-16thread.10.15 /srv/z/none/test-none-64kchunk-16thread.10.2 /srv/z/none/test-none-64kchunk-16thread.10.3 /srv/z/none/test-none-64kchunk-16thread.10.4 /srv/z/none/test-none-64kchunk-16thread.10.5 /srv/z/none/test-none-64kchunk-16thread.10.6 /srv/z/none/test-none-64kchunk-16thread.10.7 /srv/z/none/test-none-64kchunk-16thread.10.8 /srv/z/none/test-none-64kchunk-16thread.10.9 /srv/z/none/test-none-64kchunk-16thread.1.1 /srv/z/none/test-none-64kchunk-16thread.1.10 /srv/z/none/test-none-64kchunk-16thread.11.0 /srv/z/none/test-none-64kchunk-16thread.1.11 /srv/z/none/test-none-64kchunk-16thread.11.1 /srv/z/none/test-none-64kchunk-16thread.11.10 /srv/z/none/test-none-64kchunk-16thread.11.11 /srv/z/none/test-none-64kchunk-16thread.11.12 /srv/z/none/test-none-64kchunk-16thread.11.13 /srv/z/none/test-none-64kchunk-16thread.11.14 /srv/z/none/test-none-64kchunk-16thread.11.15 /srv/z/none/test-none-64kchunk-16thread.1.12 /srv/z/none/test-none-64kchunk-16thread.11.2 /srv/z/none/test-none-64kchunk-16thread.1.13 /srv/z/none/test-none-64kchunk-16thread.11.3 /srv/z/none/test-none-64kchunk-16thread.1.14 /srv/z/none/test-none-64kchunk-16thread.11.4 /srv/z/none/test-none-64kchunk-16thread.1.15 /srv/z/none/test-none-64kchunk-16thread.11.5 /srv/z/none/test-none-64kchunk-16thread.11.6 /srv/z/none/test-none-64kchunk-16thread.11.7 /srv/z/none/test-none-64kchunk-16thread.11.8 /srv/z/none/test-none-64kchunk-16thread.11.9 /srv/z/none/test-none-64kchunk-16thread.1.2 /srv/z/none/test-none-64kchunk-16thread.12.0 /srv/z/none/test-none-64kchunk-16thread.12.1 /srv/z/none/test-none-64kchunk-16thread.12.10 /srv/z/none/test-none-64kchunk-16thread.12.11 /srv/z/none/test-none-64kchunk-16thread.12.12 /srv/z/none/test-none-64kchunk-16thread.12.13 /srv/z/none/test-none-64kchunk-16thread.12.14 /srv/z/none/test-none-64kchunk-16thread.12.15 /srv/z/none/test-none-64kchunk-16thread.12.2 /srv/z/none/test-none-64kchunk-16thread.12.3 /srv/z/none/test-none-64kchunk-16thread.12.4 /srv/z/none/test-none-64kchunk-16thread.12.5 /srv/z/none/test-none-64kchunk-16thread.12.6 /srv/z/none/test-none-64kchunk-16thread.12.7 /srv/z/none/test-none-64kchunk-16thread.12.8 /srv/z/none/test-none-64kchunk-16thread.12.9 /srv/z/none/test-none-64kchunk-16thread.1.3 /srv/z/none/test-none-64kchunk-16thread.13.0 /srv/z/none/test-none-64kchunk-16thread.13.1 /srv/z/none/test-none-64kchunk-16thread.13.10 /srv/z/none/test-none-64kchunk-16thread.13.11 /srv/z/none/test-none-64kchunk-16thread.13.12 /srv/z/none/test-none-64kchunk-16thread.13.13 /srv/z/none/test-none-64kchunk-16thread.13.14 /srv/z/none/test-none-64kchunk-16thread.13.15 /srv/z/none/test-none-64kchunk-16thread.13.2 /srv/z/none/test-none-64kchunk-16thread.13.3 /srv/z/none/test-none-64kchunk-16thread.13.4 /srv/z/none/test-none-64kchunk-16thread.13.5 /srv/z/none/test-none-64kchunk-16thread.13.6 /srv/z/none/test-none-64kchunk-16thread.13.7 /srv/z/none/test-none-64kchunk-16thread.13.8 /srv/z/none/test-none-64kchunk-16thread.13.9 /srv/z/none/test-none-64kchunk-16thread.1.4 /srv/z/none/test-none-64kchunk-16thread.14.0 /srv/z/none/test-none-64kchunk-16thread.14.1 /srv/z/none/test-none-64kchunk-16thread.14.10 /srv/z/none/test-none-64kchunk-16thread.14.11 /srv/z/none/test-none-64kchunk-16thread.14.12 /srv/z/none/test-none-64kchunk-16thread.14.13 /srv/z/none/test-none-64kchunk-16thread.14.14 /srv/z/none/test-none-64kchunk-16thread.14.15 /srv/z/none/test-none-64kchunk-16thread.14.2 /srv/z/none/test-none-64kchunk-16thread.14.3 /srv/z/none/test-none-64kchunk-16thread.14.4 /srv/z/none/test-none-64kchunk-16thread.14.5 /srv/z/none/test-none-64kchunk-16thread.14.6 /srv/z/none/test-none-64kchunk-16thread.14.7 /srv/z/none/test-none-64kchunk-16thread.14.8 /srv/z/none/test-none-64kchunk-16thread.14.9 /srv/z/none/test-none-64kchunk-16thread.1.5 /srv/z/none/test-none-64kchunk-16thread.15.0 /srv/z/none/test-none-64kchunk-16thread.15.1 /srv/z/none/test-none-64kchunk-16thread.15.10 /srv/z/none/test-none-64kchunk-16thread.15.11 /srv/z/none/test-none-64kchunk-16thread.15.12 /srv/z/none/test-none-64kchunk-16thread.15.13 /srv/z/none/test-none-64kchunk-16thread.15.14 /srv/z/none/test-none-64kchunk-16thread.15.15 /srv/z/none/test-none-64kchunk-16thread.15.2 /srv/z/none/test-none-64kchunk-16thread.15.3 /srv/z/none/test-none-64kchunk-16thread.15.4 /srv/z/none/test-none-64kchunk-16thread.15.5 /srv/z/none/test-none-64kchunk-16thread.15.6 /srv/z/none/test-none-64kchunk-16thread.15.7 /srv/z/none/test-none-64kchunk-16thread.15.8 /srv/z/none/test-none-64kchunk-16thread.15.9 /srv/z/none/test-none-64kchunk-16thread.1.6 /srv/z/none/test-none-64kchunk-16thread.1.7 /srv/z/none/test-none-64kchunk-16thread.1.8 /srv/z/none/test-none-64kchunk-16thread.1.9 /srv/z/none/test-none-64kchunk-16thread.2.0 /srv/z/none/test-none-64kchunk-16thread.2.1 /srv/z/none/test-none-64kchunk-16thread.2.10 /srv/z/none/test-none-64kchunk-16thread.2.11 /srv/z/none/test-none-64kchunk-16thread.2.12 /srv/z/none/test-none-64kchunk-16thread.2.13 /srv/z/none/test-none-64kchunk-16thread.2.14 /srv/z/none/test-none-64kchunk-16thread.2.15 /srv/z/none/test-none-64kchunk-16thread.2.2 /srv/z/none/test-none-64kchunk-16thread.2.3 /srv/z/none/test-none-64kchunk-16thread.2.4 /srv/z/none/test-none-64kchunk-16thread.2.5 /srv/z/none/test-none-64kchunk-16thread.2.6 /srv/z/none/test-none-64kchunk-16thread.2.7 /srv/z/none/test-none-64kchunk-16thread.2.8 /srv/z/none/test-none-64kchunk-16thread.2.9 /srv/z/none/test-none-64kchunk-16thread.3.0 /srv/z/none/test-none-64kchunk-16thread.3.1 /srv/z/none/test-none-64kchunk-16thread.3.10 /srv/z/none/test-none-64kchunk-16thread.3.11 /srv/z/none/test-none-64kchunk-16thread.3.12 /srv/z/none/test-none-64kchunk-16thread.3.13 /srv/z/none/test-none-64kchunk-16thread.3.14 /srv/z/none/test-none-64kchunk-16thread.3.15 /srv/z/none/test-none-64kchunk-16thread.3.2 /srv/z/none/test-none-64kchunk-16thread.3.3 /srv/z/none/test-none-64kchunk-16thread.3.4 /srv/z/none/test-none-64kchunk-16thread.3.5 /srv/z/none/test-none-64kchunk-16thread.3.6 /srv/z/none/test-none-64kchunk-16thread.3.7 /srv/z/none/test-none-64kchunk-16thread.3.8 /srv/z/none/test-none-64kchunk-16thread.3.9 /srv/z/none/test-none-64kchunk-16thread.4.0 /srv/z/none/test-none-64kchunk-16thread.4.1 /srv/z/none/test-none-64kchunk-16thread.4.10 /srv/z/none/test-none-64kchunk-16thread.4.11 /srv/z/none/test-none-64kchunk-16thread.4.12 /srv/z/none/test-none-64kchunk-16thread.4.13 /srv/z/none/test-none-64kchunk-16thread.4.14 /srv/z/none/test-none-64kchunk-16thread.4.15 /srv/z/none/test-none-64kchunk-16thread.4.2 /srv/z/none/test-none-64kchunk-16thread.4.3 /srv/z/none/test-none-64kchunk-16thread.4.4 /srv/z/none/test-none-64kchunk-16thread.4.5 /srv/z/none/test-none-64kchunk-16thread.4.6 /srv/z/none/test-none-64kchunk-16thread.4.7 /srv/z/none/test-none-64kchunk-16thread.4.8 /srv/z/none/test-none-64kchunk-16thread.4.9 /srv/z/none/test-none-64kchunk-16thread.5.0 /srv/z/none/test-none-64kchunk-16thread.5.1 /srv/z/none/test-none-64kchunk-16thread.5.10 /srv/z/none/test-none-64kchunk-16thread.5.11 /srv/z/none/test-none-64kchunk-16thread.5.12 /srv/z/none/test-none-64kchunk-16thread.5.13 /srv/z/none/test-none-64kchunk-16thread.5.14 /srv/z/none/test-none-64kchunk-16thread.5.15 /srv/z/none/test-none-64kchunk-16thread.5.2 /srv/z/none/test-none-64kchunk-16thread.5.3 /srv/z/none/test-none-64kchunk-16thread.5.4 /srv/z/none/test-none-64kchunk-16thread.5.5 /srv/z/none/test-none-64kchunk-16thread.5.6 /srv/z/none/test-none-64kchunk-16thread.5.7 /srv/z/none/test-none-64kchunk-16thread.5.8 /srv/z/none/test-none-64kchunk-16thread.5.9 /srv/z/none/test-none-64kchunk-16thread.6.0 /srv/z/none/test-none-64kchunk-16thread.6.1 /srv/z/none/test-none-64kchunk-16thread.6.10 /srv/z/none/test-none-64kchunk-16thread.6.11 /srv/z/none/test-none-64kchunk-16thread.6.12 /srv/z/none/test-none-64kchunk-16thread.6.13 /srv/z/none/test-none-64kchunk-16thread.6.14 /srv/z/none/test-none-64kchunk-16thread.6.15 /srv/z/none/test-none-64kchunk-16thread.6.2 /srv/z/none/test-none-64kchunk-16thread.6.3 /srv/z/none/test-none-64kchunk-16thread.6.4 /srv/z/none/test-none-64kchunk-16thread.6.5 /srv/z/none/test-none-64kchunk-16thread.6.6 /srv/z/none/test-none-64kchunk-16thread.6.7 /srv/z/none/test-none-64kchunk-16thread.6.8 /srv/z/none/test-none-64kchunk-16thread.6.9 /srv/z/none/test-none-64kchunk-16thread.7.0 /srv/z/none/test-none-64kchunk-16thread.7.1 /srv/z/none/test-none-64kchunk-16thread.7.10 /srv/z/none/test-none-64kchunk-16thread.7.11 /srv/z/none/test-none-64kchunk-16thread.7.12 /srv/z/none/test-none-64kchunk-16thread.7.13 /srv/z/none/test-none-64kchunk-16thread.7.14 /srv/z/none/test-none-64kchunk-16thread.7.15 /srv/z/none/test-none-64kchunk-16thread.7.2 /srv/z/none/test-none-64kchunk-16thread.7.3 /srv/z/none/test-none-64kchunk-16thread.7.4 /srv/z/none/test-none-64kchunk-16thread.7.5 /srv/z/none/test-none-64kchunk-16thread.7.6 /srv/z/none/test-none-64kchunk-16thread.7.7 /srv/z/none/test-none-64kchunk-16thread.7.8 /srv/z/none/test-none-64kchunk-16thread.7.9 /srv/z/none/test-none-64kchunk-16thread.8.0 /srv/z/none/test-none-64kchunk-16thread.8.1 /srv/z/none/test-none-64kchunk-16thread.8.10 /srv/z/none/test-none-64kchunk-16thread.8.11 /srv/z/none/test-none-64kchunk-16thread.8.12 /srv/z/none/test-none-64kchunk-16thread.8.13 /srv/z/none/test-none-64kchunk-16thread.8.14 /srv/z/none/test-none-64kchunk-16thread.8.15 /srv/z/none/test-none-64kchunk-16thread.8.2 /srv/z/none/test-none-64kchunk-16thread.8.3 /srv/z/none/test-none-64kchunk-16thread.8.4 /srv/z/none/test-none-64kchunk-16thread.8.5 /srv/z/none/test-none-64kchunk-16thread.8.6 /srv/z/none/test-none-64kchunk-16thread.8.7 /srv/z/none/test-none-64kchunk-16thread.8.8 /srv/z/none/test-none-64kchunk-16thread.8.9 /srv/z/none/test-none-64kchunk-16thread.9.0 /srv/z/none/test-none-64kchunk-16thread.9.1 /srv/z/none/test-none-64kchunk-16thread.9.10 /srv/z/none/test-none-64kchunk-16thread.9.11 /srv/z/none/test-none-64kchunk-16thread.9.12 /srv/z/none/test-none-64kchunk-16thread.9.13 /srv/z/none/test-none-64kchunk-16thread.9.14 /srv/z/none/test-none-64kchunk-16thread.9.15 /srv/z/none/test-none-64kchunk-16thread.9.2 /srv/z/none/test-none-64kchunk-16thread.9.3 /srv/z/none/test-none-64kchunk-16thread.9.4 /srv/z/none/test-none-64kchunk-16thread.9.5 /srv/z/none/test-none-64kchunk-16thread.9.6 /srv/z/none/test-none-64kchunk-16thread.9.7 /srv/z/none/test-none-64kchunk-16thread.9.8 /srv/z/none/test-none-64kchunk-16thread.9.9 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-comp-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/comp/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-comp-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=2838MiB/s,w=2837MiB/s][r=45.4k,w=45.4k IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.5%][r=2494MiB/s,w=2503MiB/s][r=39.9k,w=40.1k IOPS][eta 01m:41s] Jobs: 16 (f=256): [M(16)][24.2%][r=2540MiB/s,w=2543MiB/s][r=40.6k,w=40.7k IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.8%][r=2513MiB/s,w=2518MiB/s][r=40.2k,w=40.3k IOPS][eta 01m:20s] Jobs: 16 (f=256): [M(16)][40.5%][r=2652MiB/s,w=2654MiB/s][r=42.4k,w=42.5k IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][48.3%][r=2686MiB/s,w=2681MiB/s][r=42.0k,w=42.9k IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][56.2%][r=2711MiB/s,w=2707MiB/s][r=43.4k,w=43.3k IOPS][eta 00m:53s] | |
| Jobs: 16 (f=256): [M(16)][64.7%][r=2597MiB/s,w=2600MiB/s][r=41.6k,w=41.6k IOPS][eta 00m:42s] Jobs: 16 (f=256): [M(16)][72.5%][r=2581MiB/s,w=2581MiB/s][r=41.3k,w=41.3k IOPS][eta 00m:33s] | |
| Jobs: 16 (f=256): [M(16)][80.2%][r=2591MiB/s,w=2595MiB/s][r=41.4k,w=41.5k IOPS][eta 00m:24s] Jobs: 16 (f=256): [M(16)][88.3%][r=2494MiB/s,w=2497MiB/s][r=39.9k,w=39.9k IOPS][eta 00m:14s] | |
| Jobs: 16 (f=256): [M(16)][96.7%][r=2450MiB/s,w=2454MiB/s][r=39.2k,w=39.3k IOPS][eta 00m:04s] | |
| test-comp-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=1584172: Wed Feb 26 09:51:53 2020 | |
| read: IOPS=41.5k, BW=2592MiB/s (2718MB/s)(304GiB/120005msec) | |
| slat (usec): min=10, max=156916, avg=168.76, stdev=931.43 | |
| clat (usec): min=3, max=236059, avg=12142.24, stdev=12178.70 | |
| lat (usec): min=44, max=236252, avg=12311.42, stdev=12332.15 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 4], 10.00th=[ 5], 20.00th=[ 5], | |
| | 30.00th=[ 6], 40.00th=[ 6], 50.00th=[ 8], 60.00th=[ 10], | |
| | 70.00th=[ 13], 80.00th=[ 18], 90.00th=[ 27], 95.00th=[ 37], | |
| | 99.00th=[ 61], 99.50th=[ 71], 99.90th=[ 106], 99.95th=[ 118], | |
| | 99.99th=[ 144] | |
| bw ( MiB/s): min= 1163, max= 5113, per=99.78%, avg=2586.04, stdev=49.20, samples=3830 | |
| iops : min=18618, max=81813, avg=41374.58, stdev=787.27, samples=3830 | |
| write: IOPS=41.5k, BW=2594MiB/s (2720MB/s)(304GiB/120005msec); 0 zone resets | |
| slat (usec): min=18, max=155660, avg=208.14, stdev=1096.05 | |
| clat (usec): min=5, max=236019, avg=12155.90, stdev=12176.86 | |
| lat (usec): min=139, max=236115, avg=12364.49, stdev=12306.54 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 4], 10.00th=[ 5], 20.00th=[ 5], | |
| | 30.00th=[ 6], 40.00th=[ 6], 50.00th=[ 8], 60.00th=[ 10], | |
| | 70.00th=[ 13], 80.00th=[ 18], 90.00th=[ 27], 95.00th=[ 37], | |
| | 99.00th=[ 61], 99.50th=[ 71], 99.90th=[ 106], 99.95th=[ 120], | |
| | 99.99th=[ 144] | |
| bw ( MiB/s): min= 1188, max= 5070, per=99.79%, avg=2588.11, stdev=48.88, samples=3830 | |
| iops : min=19019, max=81119, avg=41407.79, stdev=782.07, samples=3830 | |
| lat (usec) : 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01% | |
| lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=0.01%, 4=8.57%, 10=54.03%, 20=21.12%, 50=14.25% | |
| lat (msec) : 100=1.89%, 250=0.12% | |
| cpu : usr=2.88%, sys=39.81%, ctx=1839832, majf=0, minf=500765 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=4976417,4979938,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=2592MiB/s (2718MB/s), 2592MiB/s-2592MiB/s (2718MB/s-2718MB/s), io=304GiB (326GB), run=120005-120005msec | |
| WRITE: bw=2594MiB/s (2720MB/s), 2594MiB/s-2594MiB/s (2720MB/s-2720MB/s), io=304GiB (326GB), run=120005-120005msec | |
| + rm -rf /srv/z/comp/test-comp-64kchunk-16thread.0.0 /srv/z/comp/test-comp-64kchunk-16thread.0.1 /srv/z/comp/test-comp-64kchunk-16thread.0.10 /srv/z/comp/test-comp-64kchunk-16thread.0.11 /srv/z/comp/test-comp-64kchunk-16thread.0.12 /srv/z/comp/test-comp-64kchunk-16thread.0.13 /srv/z/comp/test-comp-64kchunk-16thread.0.14 /srv/z/comp/test-comp-64kchunk-16thread.0.15 /srv/z/comp/test-comp-64kchunk-16thread.0.2 /srv/z/comp/test-comp-64kchunk-16thread.0.3 /srv/z/comp/test-comp-64kchunk-16thread.0.4 /srv/z/comp/test-comp-64kchunk-16thread.0.5 /srv/z/comp/test-comp-64kchunk-16thread.0.6 /srv/z/comp/test-comp-64kchunk-16thread.0.7 /srv/z/comp/test-comp-64kchunk-16thread.0.8 /srv/z/comp/test-comp-64kchunk-16thread.0.9 /srv/z/comp/test-comp-64kchunk-16thread.1.0 /srv/z/comp/test-comp-64kchunk-16thread.10.0 /srv/z/comp/test-comp-64kchunk-16thread.10.1 /srv/z/comp/test-comp-64kchunk-16thread.10.10 /srv/z/comp/test-comp-64kchunk-16thread.10.11 /srv/z/comp/test-comp-64kchunk-16thread.10.12 /srv/z/comp/test-comp-64kchunk-16thread.10.13 /srv/z/comp/test-comp-64kchunk-16thread.10.14 /srv/z/comp/test-comp-64kchunk-16thread.10.15 /srv/z/comp/test-comp-64kchunk-16thread.10.2 /srv/z/comp/test-comp-64kchunk-16thread.10.3 /srv/z/comp/test-comp-64kchunk-16thread.10.4 /srv/z/comp/test-comp-64kchunk-16thread.10.5 /srv/z/comp/test-comp-64kchunk-16thread.10.6 /srv/z/comp/test-comp-64kchunk-16thread.10.7 /srv/z/comp/test-comp-64kchunk-16thread.10.8 /srv/z/comp/test-comp-64kchunk-16thread.10.9 /srv/z/comp/test-comp-64kchunk-16thread.1.1 /srv/z/comp/test-comp-64kchunk-16thread.1.10 /srv/z/comp/test-comp-64kchunk-16thread.11.0 /srv/z/comp/test-comp-64kchunk-16thread.1.11 /srv/z/comp/test-comp-64kchunk-16thread.11.1 /srv/z/comp/test-comp-64kchunk-16thread.11.10 /srv/z/comp/test-comp-64kchunk-16thread.11.11 /srv/z/comp/test-comp-64kchunk-16thread.11.12 /srv/z/comp/test-comp-64kchunk-16thread.11.13 /srv/z/comp/test-comp-64kchunk-16thread.11.14 /srv/z/comp/test-comp-64kchunk-16thread.11.15 /srv/z/comp/test-comp-64kchunk-16thread.1.12 /srv/z/comp/test-comp-64kchunk-16thread.11.2 /srv/z/comp/test-comp-64kchunk-16thread.1.13 /srv/z/comp/test-comp-64kchunk-16thread.11.3 /srv/z/comp/test-comp-64kchunk-16thread.1.14 /srv/z/comp/test-comp-64kchunk-16thread.11.4 /srv/z/comp/test-comp-64kchunk-16thread.1.15 /srv/z/comp/test-comp-64kchunk-16thread.11.5 /srv/z/comp/test-comp-64kchunk-16thread.11.6 /srv/z/comp/test-comp-64kchunk-16thread.11.7 /srv/z/comp/test-comp-64kchunk-16thread.11.8 /srv/z/comp/test-comp-64kchunk-16thread.11.9 /srv/z/comp/test-comp-64kchunk-16thread.1.2 /srv/z/comp/test-comp-64kchunk-16thread.12.0 /srv/z/comp/test-comp-64kchunk-16thread.12.1 /srv/z/comp/test-comp-64kchunk-16thread.12.10 /srv/z/comp/test-comp-64kchunk-16thread.12.11 /srv/z/comp/test-comp-64kchunk-16thread.12.12 /srv/z/comp/test-comp-64kchunk-16thread.12.13 /srv/z/comp/test-comp-64kchunk-16thread.12.14 /srv/z/comp/test-comp-64kchunk-16thread.12.15 /srv/z/comp/test-comp-64kchunk-16thread.12.2 /srv/z/comp/test-comp-64kchunk-16thread.12.3 /srv/z/comp/test-comp-64kchunk-16thread.12.4 /srv/z/comp/test-comp-64kchunk-16thread.12.5 /srv/z/comp/test-comp-64kchunk-16thread.12.6 /srv/z/comp/test-comp-64kchunk-16thread.12.7 /srv/z/comp/test-comp-64kchunk-16thread.12.8 /srv/z/comp/test-comp-64kchunk-16thread.12.9 /srv/z/comp/test-comp-64kchunk-16thread.1.3 /srv/z/comp/test-comp-64kchunk-16thread.13.0 /srv/z/comp/test-comp-64kchunk-16thread.13.1 /srv/z/comp/test-comp-64kchunk-16thread.13.10 /srv/z/comp/test-comp-64kchunk-16thread.13.11 /srv/z/comp/test-comp-64kchunk-16thread.13.12 /srv/z/comp/test-comp-64kchunk-16thread.13.13 /srv/z/comp/test-comp-64kchunk-16thread.13.14 /srv/z/comp/test-comp-64kchunk-16thread.13.15 /srv/z/comp/test-comp-64kchunk-16thread.13.2 /srv/z/comp/test-comp-64kchunk-16thread.13.3 /srv/z/comp/test-comp-64kchunk-16thread.13.4 /srv/z/comp/test-comp-64kchunk-16thread.13.5 /srv/z/comp/test-comp-64kchunk-16thread.13.6 /srv/z/comp/test-comp-64kchunk-16thread.13.7 /srv/z/comp/test-comp-64kchunk-16thread.13.8 /srv/z/comp/test-comp-64kchunk-16thread.13.9 /srv/z/comp/test-comp-64kchunk-16thread.1.4 /srv/z/comp/test-comp-64kchunk-16thread.14.0 /srv/z/comp/test-comp-64kchunk-16thread.14.1 /srv/z/comp/test-comp-64kchunk-16thread.14.10 /srv/z/comp/test-comp-64kchunk-16thread.14.11 /srv/z/comp/test-comp-64kchunk-16thread.14.12 /srv/z/comp/test-comp-64kchunk-16thread.14.13 /srv/z/comp/test-comp-64kchunk-16thread.14.14 /srv/z/comp/test-comp-64kchunk-16thread.14.15 /srv/z/comp/test-comp-64kchunk-16thread.14.2 /srv/z/comp/test-comp-64kchunk-16thread.14.3 /srv/z/comp/test-comp-64kchunk-16thread.14.4 /srv/z/comp/test-comp-64kchunk-16thread.14.5 /srv/z/comp/test-comp-64kchunk-16thread.14.6 /srv/z/comp/test-comp-64kchunk-16thread.14.7 /srv/z/comp/test-comp-64kchunk-16thread.14.8 /srv/z/comp/test-comp-64kchunk-16thread.14.9 /srv/z/comp/test-comp-64kchunk-16thread.1.5 /srv/z/comp/test-comp-64kchunk-16thread.15.0 /srv/z/comp/test-comp-64kchunk-16thread.15.1 /srv/z/comp/test-comp-64kchunk-16thread.15.10 /srv/z/comp/test-comp-64kchunk-16thread.15.11 /srv/z/comp/test-comp-64kchunk-16thread.15.12 /srv/z/comp/test-comp-64kchunk-16thread.15.13 /srv/z/comp/test-comp-64kchunk-16thread.15.14 /srv/z/comp/test-comp-64kchunk-16thread.15.15 /srv/z/comp/test-comp-64kchunk-16thread.15.2 /srv/z/comp/test-comp-64kchunk-16thread.15.3 /srv/z/comp/test-comp-64kchunk-16thread.15.4 /srv/z/comp/test-comp-64kchunk-16thread.15.5 /srv/z/comp/test-comp-64kchunk-16thread.15.6 /srv/z/comp/test-comp-64kchunk-16thread.15.7 /srv/z/comp/test-comp-64kchunk-16thread.15.8 /srv/z/comp/test-comp-64kchunk-16thread.15.9 /srv/z/comp/test-comp-64kchunk-16thread.1.6 /srv/z/comp/test-comp-64kchunk-16thread.1.7 /srv/z/comp/test-comp-64kchunk-16thread.1.8 /srv/z/comp/test-comp-64kchunk-16thread.1.9 /srv/z/comp/test-comp-64kchunk-16thread.2.0 /srv/z/comp/test-comp-64kchunk-16thread.2.1 /srv/z/comp/test-comp-64kchunk-16thread.2.10 /srv/z/comp/test-comp-64kchunk-16thread.2.11 /srv/z/comp/test-comp-64kchunk-16thread.2.12 /srv/z/comp/test-comp-64kchunk-16thread.2.13 /srv/z/comp/test-comp-64kchunk-16thread.2.14 /srv/z/comp/test-comp-64kchunk-16thread.2.15 /srv/z/comp/test-comp-64kchunk-16thread.2.2 /srv/z/comp/test-comp-64kchunk-16thread.2.3 /srv/z/comp/test-comp-64kchunk-16thread.2.4 /srv/z/comp/test-comp-64kchunk-16thread.2.5 /srv/z/comp/test-comp-64kchunk-16thread.2.6 /srv/z/comp/test-comp-64kchunk-16thread.2.7 /srv/z/comp/test-comp-64kchunk-16thread.2.8 /srv/z/comp/test-comp-64kchunk-16thread.2.9 /srv/z/comp/test-comp-64kchunk-16thread.3.0 /srv/z/comp/test-comp-64kchunk-16thread.3.1 /srv/z/comp/test-comp-64kchunk-16thread.3.10 /srv/z/comp/test-comp-64kchunk-16thread.3.11 /srv/z/comp/test-comp-64kchunk-16thread.3.12 /srv/z/comp/test-comp-64kchunk-16thread.3.13 /srv/z/comp/test-comp-64kchunk-16thread.3.14 /srv/z/comp/test-comp-64kchunk-16thread.3.15 /srv/z/comp/test-comp-64kchunk-16thread.3.2 /srv/z/comp/test-comp-64kchunk-16thread.3.3 /srv/z/comp/test-comp-64kchunk-16thread.3.4 /srv/z/comp/test-comp-64kchunk-16thread.3.5 /srv/z/comp/test-comp-64kchunk-16thread.3.6 /srv/z/comp/test-comp-64kchunk-16thread.3.7 /srv/z/comp/test-comp-64kchunk-16thread.3.8 /srv/z/comp/test-comp-64kchunk-16thread.3.9 /srv/z/comp/test-comp-64kchunk-16thread.4.0 /srv/z/comp/test-comp-64kchunk-16thread.4.1 /srv/z/comp/test-comp-64kchunk-16thread.4.10 /srv/z/comp/test-comp-64kchunk-16thread.4.11 /srv/z/comp/test-comp-64kchunk-16thread.4.12 /srv/z/comp/test-comp-64kchunk-16thread.4.13 /srv/z/comp/test-comp-64kchunk-16thread.4.14 /srv/z/comp/test-comp-64kchunk-16thread.4.15 /srv/z/comp/test-comp-64kchunk-16thread.4.2 /srv/z/comp/test-comp-64kchunk-16thread.4.3 /srv/z/comp/test-comp-64kchunk-16thread.4.4 /srv/z/comp/test-comp-64kchunk-16thread.4.5 /srv/z/comp/test-comp-64kchunk-16thread.4.6 /srv/z/comp/test-comp-64kchunk-16thread.4.7 /srv/z/comp/test-comp-64kchunk-16thread.4.8 /srv/z/comp/test-comp-64kchunk-16thread.4.9 /srv/z/comp/test-comp-64kchunk-16thread.5.0 /srv/z/comp/test-comp-64kchunk-16thread.5.1 /srv/z/comp/test-comp-64kchunk-16thread.5.10 /srv/z/comp/test-comp-64kchunk-16thread.5.11 /srv/z/comp/test-comp-64kchunk-16thread.5.12 /srv/z/comp/test-comp-64kchunk-16thread.5.13 /srv/z/comp/test-comp-64kchunk-16thread.5.14 /srv/z/comp/test-comp-64kchunk-16thread.5.15 /srv/z/comp/test-comp-64kchunk-16thread.5.2 /srv/z/comp/test-comp-64kchunk-16thread.5.3 /srv/z/comp/test-comp-64kchunk-16thread.5.4 /srv/z/comp/test-comp-64kchunk-16thread.5.5 /srv/z/comp/test-comp-64kchunk-16thread.5.6 /srv/z/comp/test-comp-64kchunk-16thread.5.7 /srv/z/comp/test-comp-64kchunk-16thread.5.8 /srv/z/comp/test-comp-64kchunk-16thread.5.9 /srv/z/comp/test-comp-64kchunk-16thread.6.0 /srv/z/comp/test-comp-64kchunk-16thread.6.1 /srv/z/comp/test-comp-64kchunk-16thread.6.10 /srv/z/comp/test-comp-64kchunk-16thread.6.11 /srv/z/comp/test-comp-64kchunk-16thread.6.12 /srv/z/comp/test-comp-64kchunk-16thread.6.13 /srv/z/comp/test-comp-64kchunk-16thread.6.14 /srv/z/comp/test-comp-64kchunk-16thread.6.15 /srv/z/comp/test-comp-64kchunk-16thread.6.2 /srv/z/comp/test-comp-64kchunk-16thread.6.3 /srv/z/comp/test-comp-64kchunk-16thread.6.4 /srv/z/comp/test-comp-64kchunk-16thread.6.5 /srv/z/comp/test-comp-64kchunk-16thread.6.6 /srv/z/comp/test-comp-64kchunk-16thread.6.7 /srv/z/comp/test-comp-64kchunk-16thread.6.8 /srv/z/comp/test-comp-64kchunk-16thread.6.9 /srv/z/comp/test-comp-64kchunk-16thread.7.0 /srv/z/comp/test-comp-64kchunk-16thread.7.1 /srv/z/comp/test-comp-64kchunk-16thread.7.10 /srv/z/comp/test-comp-64kchunk-16thread.7.11 /srv/z/comp/test-comp-64kchunk-16thread.7.12 /srv/z/comp/test-comp-64kchunk-16thread.7.13 /srv/z/comp/test-comp-64kchunk-16thread.7.14 /srv/z/comp/test-comp-64kchunk-16thread.7.15 /srv/z/comp/test-comp-64kchunk-16thread.7.2 /srv/z/comp/test-comp-64kchunk-16thread.7.3 /srv/z/comp/test-comp-64kchunk-16thread.7.4 /srv/z/comp/test-comp-64kchunk-16thread.7.5 /srv/z/comp/test-comp-64kchunk-16thread.7.6 /srv/z/comp/test-comp-64kchunk-16thread.7.7 /srv/z/comp/test-comp-64kchunk-16thread.7.8 /srv/z/comp/test-comp-64kchunk-16thread.7.9 /srv/z/comp/test-comp-64kchunk-16thread.8.0 /srv/z/comp/test-comp-64kchunk-16thread.8.1 /srv/z/comp/test-comp-64kchunk-16thread.8.10 /srv/z/comp/test-comp-64kchunk-16thread.8.11 /srv/z/comp/test-comp-64kchunk-16thread.8.12 /srv/z/comp/test-comp-64kchunk-16thread.8.13 /srv/z/comp/test-comp-64kchunk-16thread.8.14 /srv/z/comp/test-comp-64kchunk-16thread.8.15 /srv/z/comp/test-comp-64kchunk-16thread.8.2 /srv/z/comp/test-comp-64kchunk-16thread.8.3 /srv/z/comp/test-comp-64kchunk-16thread.8.4 /srv/z/comp/test-comp-64kchunk-16thread.8.5 /srv/z/comp/test-comp-64kchunk-16thread.8.6 /srv/z/comp/test-comp-64kchunk-16thread.8.7 /srv/z/comp/test-comp-64kchunk-16thread.8.8 /srv/z/comp/test-comp-64kchunk-16thread.8.9 /srv/z/comp/test-comp-64kchunk-16thread.9.0 /srv/z/comp/test-comp-64kchunk-16thread.9.1 /srv/z/comp/test-comp-64kchunk-16thread.9.10 /srv/z/comp/test-comp-64kchunk-16thread.9.11 /srv/z/comp/test-comp-64kchunk-16thread.9.12 /srv/z/comp/test-comp-64kchunk-16thread.9.13 /srv/z/comp/test-comp-64kchunk-16thread.9.14 /srv/z/comp/test-comp-64kchunk-16thread.9.15 /srv/z/comp/test-comp-64kchunk-16thread.9.2 /srv/z/comp/test-comp-64kchunk-16thread.9.3 /srv/z/comp/test-comp-64kchunk-16thread.9.4 /srv/z/comp/test-comp-64kchunk-16thread.9.5 /srv/z/comp/test-comp-64kchunk-16thread.9.6 /srv/z/comp/test-comp-64kchunk-16thread.9.7 /srv/z/comp/test-comp-64kchunk-16thread.9.8 /srv/z/comp/test-comp-64kchunk-16thread.9.9 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-enc-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/enc/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-enc-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=421MiB/s,w=421MiB/s][r=6735,w=6743 IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.5%][r=373MiB/s,w=377MiB/s][r=5972,w=6034 IOPS][eta 01m:41s] Jobs: 16 (f=256): [M(16)][24.2%][r=335MiB/s,w=335MiB/s][r=5354,w=5365 IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.2%][r=331MiB/s,w=330MiB/s][r=5291,w=5283 IOPS][eta 01m:22s] Jobs: 16 (f=256): [M(16)][40.0%][r=331MiB/s,w=326MiB/s][r=5293,w=5223 IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][47.9%][r=317MiB/s,w=317MiB/s][r=5075,w=5079 IOPS][eta 01m:03s] Jobs: 16 (f=256): [M(16)][55.8%][r=313MiB/s,w=316MiB/s][r=5006,w=5050 IOPS][eta 00m:53s] | |
| Jobs: 16 (f=256): [M(16)][64.2%][r=319MiB/s,w=318MiB/s][r=5101,w=5087 IOPS][eta 00m:43s] Jobs: 16 (f=256): [M(16)][71.7%][r=357MiB/s,w=355MiB/s][r=5709,w=5673 IOPS][eta 00m:34s] | |
| Jobs: 16 (f=256): [M(16)][80.0%][r=353MiB/s,w=352MiB/s][r=5655,w=5639 IOPS][eta 00m:24s] Jobs: 16 (f=256): [M(16)][87.6%][r=367MiB/s,w=371MiB/s][r=5877,w=5929 IOPS][eta 00m:15s] | |
| Jobs: 16 (f=256): [M(16)][95.8%][r=362MiB/s,w=361MiB/s][r=5798,w=5775 IOPS][eta 00m:05s] | |
| test-enc-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=3698924: Wed Feb 26 10:14:54 2020 | |
| read: IOPS=5560, BW=348MiB/s (364MB/s)(40.7GiB/120008msec) | |
| slat (usec): min=9, max=129130, avg=1430.43, stdev=3634.61 | |
| clat (usec): min=3, max=562542, avg=90503.77, stdev=53215.62 | |
| lat (usec): min=70, max=562609, avg=91934.69, stdev=53961.51 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 14], 10.00th=[ 24], 20.00th=[ 43], | |
| | 30.00th=[ 64], 40.00th=[ 77], 50.00th=[ 87], 60.00th=[ 96], | |
| | 70.00th=[ 108], 80.00th=[ 129], 90.00th=[ 165], 95.00th=[ 192], | |
| | 99.00th=[ 241], 99.50th=[ 257], 99.90th=[ 305], 99.95th=[ 330], | |
| | 99.99th=[ 388] | |
| bw ( KiB/s): min=126065, max=890096, per=99.82%, avg=355242.95, stdev=13621.32, samples=3835 | |
| iops : min= 1967, max=13907, avg=5548.39, stdev=212.87, samples=3835 | |
| write: IOPS=5559, BW=347MiB/s (364MB/s)(40.7GiB/120008msec); 0 zone resets | |
| slat (usec): min=15, max=137873, avg=1439.17, stdev=3712.73 | |
| clat (usec): min=2, max=567430, avg=90587.80, stdev=53361.51 | |
| lat (usec): min=44, max=567615, avg=92027.35, stdev=54064.16 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 14], 10.00th=[ 24], 20.00th=[ 43], | |
| | 30.00th=[ 64], 40.00th=[ 77], 50.00th=[ 87], 60.00th=[ 96], | |
| | 70.00th=[ 108], 80.00th=[ 130], 90.00th=[ 167], 95.00th=[ 192], | |
| | 99.00th=[ 241], 99.50th=[ 259], 99.90th=[ 305], 99.95th=[ 330], | |
| | 99.99th=[ 422] | |
| bw ( KiB/s): min=122996, max=882953, per=99.82%, avg=355137.85, stdev=13642.13, samples=3835 | |
| iops : min= 1919, max=13795, avg=5546.68, stdev=213.20, samples=3835 | |
| lat (usec) : 4=0.01%, 10=0.01%, 50=0.01%, 100=0.01% | |
| lat (msec) : 4=1.69%, 10=1.78%, 20=4.61%, 50=15.76%, 100=40.32% | |
| lat (msec) : 250=35.17%, 500=0.67%, 750=0.01% | |
| cpu : usr=0.31%, sys=38.60%, ctx=874117, majf=0, minf=476005 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=667354,667156,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=348MiB/s (364MB/s), 348MiB/s-348MiB/s (364MB/s-364MB/s), io=40.7GiB (43.7GB), run=120008-120008msec | |
| WRITE: bw=347MiB/s (364MB/s), 347MiB/s-347MiB/s (364MB/s-364MB/s), io=40.7GiB (43.7GB), run=120008-120008msec | |
| + rm -rf /srv/z/enc/test-enc-64kchunk-16thread.0.0 /srv/z/enc/test-enc-64kchunk-16thread.0.1 /srv/z/enc/test-enc-64kchunk-16thread.0.10 /srv/z/enc/test-enc-64kchunk-16thread.0.11 /srv/z/enc/test-enc-64kchunk-16thread.0.12 /srv/z/enc/test-enc-64kchunk-16thread.0.13 /srv/z/enc/test-enc-64kchunk-16thread.0.14 /srv/z/enc/test-enc-64kchunk-16thread.0.15 /srv/z/enc/test-enc-64kchunk-16thread.0.2 /srv/z/enc/test-enc-64kchunk-16thread.0.3 /srv/z/enc/test-enc-64kchunk-16thread.0.4 /srv/z/enc/test-enc-64kchunk-16thread.0.5 /srv/z/enc/test-enc-64kchunk-16thread.0.6 /srv/z/enc/test-enc-64kchunk-16thread.0.7 /srv/z/enc/test-enc-64kchunk-16thread.0.8 /srv/z/enc/test-enc-64kchunk-16thread.0.9 /srv/z/enc/test-enc-64kchunk-16thread.1.0 /srv/z/enc/test-enc-64kchunk-16thread.10.0 /srv/z/enc/test-enc-64kchunk-16thread.10.1 /srv/z/enc/test-enc-64kchunk-16thread.10.10 /srv/z/enc/test-enc-64kchunk-16thread.10.11 /srv/z/enc/test-enc-64kchunk-16thread.10.12 /srv/z/enc/test-enc-64kchunk-16thread.10.13 /srv/z/enc/test-enc-64kchunk-16thread.10.14 /srv/z/enc/test-enc-64kchunk-16thread.10.15 /srv/z/enc/test-enc-64kchunk-16thread.10.2 /srv/z/enc/test-enc-64kchunk-16thread.10.3 /srv/z/enc/test-enc-64kchunk-16thread.10.4 /srv/z/enc/test-enc-64kchunk-16thread.10.5 /srv/z/enc/test-enc-64kchunk-16thread.10.6 /srv/z/enc/test-enc-64kchunk-16thread.10.7 /srv/z/enc/test-enc-64kchunk-16thread.10.8 /srv/z/enc/test-enc-64kchunk-16thread.10.9 /srv/z/enc/test-enc-64kchunk-16thread.1.1 /srv/z/enc/test-enc-64kchunk-16thread.1.10 /srv/z/enc/test-enc-64kchunk-16thread.11.0 /srv/z/enc/test-enc-64kchunk-16thread.1.11 /srv/z/enc/test-enc-64kchunk-16thread.11.1 /srv/z/enc/test-enc-64kchunk-16thread.11.10 /srv/z/enc/test-enc-64kchunk-16thread.11.11 /srv/z/enc/test-enc-64kchunk-16thread.11.12 /srv/z/enc/test-enc-64kchunk-16thread.11.13 /srv/z/enc/test-enc-64kchunk-16thread.11.14 /srv/z/enc/test-enc-64kchunk-16thread.11.15 /srv/z/enc/test-enc-64kchunk-16thread.1.12 /srv/z/enc/test-enc-64kchunk-16thread.11.2 /srv/z/enc/test-enc-64kchunk-16thread.1.13 /srv/z/enc/test-enc-64kchunk-16thread.11.3 /srv/z/enc/test-enc-64kchunk-16thread.1.14 /srv/z/enc/test-enc-64kchunk-16thread.11.4 /srv/z/enc/test-enc-64kchunk-16thread.1.15 /srv/z/enc/test-enc-64kchunk-16thread.11.5 /srv/z/enc/test-enc-64kchunk-16thread.11.6 /srv/z/enc/test-enc-64kchunk-16thread.11.7 /srv/z/enc/test-enc-64kchunk-16thread.11.8 /srv/z/enc/test-enc-64kchunk-16thread.11.9 /srv/z/enc/test-enc-64kchunk-16thread.1.2 /srv/z/enc/test-enc-64kchunk-16thread.12.0 /srv/z/enc/test-enc-64kchunk-16thread.12.1 /srv/z/enc/test-enc-64kchunk-16thread.12.10 /srv/z/enc/test-enc-64kchunk-16thread.12.11 /srv/z/enc/test-enc-64kchunk-16thread.12.12 /srv/z/enc/test-enc-64kchunk-16thread.12.13 /srv/z/enc/test-enc-64kchunk-16thread.12.14 /srv/z/enc/test-enc-64kchunk-16thread.12.15 /srv/z/enc/test-enc-64kchunk-16thread.12.2 /srv/z/enc/test-enc-64kchunk-16thread.12.3 /srv/z/enc/test-enc-64kchunk-16thread.12.4 /srv/z/enc/test-enc-64kchunk-16thread.12.5 /srv/z/enc/test-enc-64kchunk-16thread.12.6 /srv/z/enc/test-enc-64kchunk-16thread.12.7 /srv/z/enc/test-enc-64kchunk-16thread.12.8 /srv/z/enc/test-enc-64kchunk-16thread.12.9 /srv/z/enc/test-enc-64kchunk-16thread.1.3 /srv/z/enc/test-enc-64kchunk-16thread.13.0 /srv/z/enc/test-enc-64kchunk-16thread.13.1 /srv/z/enc/test-enc-64kchunk-16thread.13.10 /srv/z/enc/test-enc-64kchunk-16thread.13.11 /srv/z/enc/test-enc-64kchunk-16thread.13.12 /srv/z/enc/test-enc-64kchunk-16thread.13.13 /srv/z/enc/test-enc-64kchunk-16thread.13.14 /srv/z/enc/test-enc-64kchunk-16thread.13.15 /srv/z/enc/test-enc-64kchunk-16thread.13.2 /srv/z/enc/test-enc-64kchunk-16thread.13.3 /srv/z/enc/test-enc-64kchunk-16thread.13.4 /srv/z/enc/test-enc-64kchunk-16thread.13.5 /srv/z/enc/test-enc-64kchunk-16thread.13.6 /srv/z/enc/test-enc-64kchunk-16thread.13.7 /srv/z/enc/test-enc-64kchunk-16thread.13.8 /srv/z/enc/test-enc-64kchunk-16thread.13.9 /srv/z/enc/test-enc-64kchunk-16thread.1.4 /srv/z/enc/test-enc-64kchunk-16thread.14.0 /srv/z/enc/test-enc-64kchunk-16thread.14.1 /srv/z/enc/test-enc-64kchunk-16thread.14.10 /srv/z/enc/test-enc-64kchunk-16thread.14.11 /srv/z/enc/test-enc-64kchunk-16thread.14.12 /srv/z/enc/test-enc-64kchunk-16thread.14.13 /srv/z/enc/test-enc-64kchunk-16thread.14.14 /srv/z/enc/test-enc-64kchunk-16thread.14.15 /srv/z/enc/test-enc-64kchunk-16thread.14.2 /srv/z/enc/test-enc-64kchunk-16thread.14.3 /srv/z/enc/test-enc-64kchunk-16thread.14.4 /srv/z/enc/test-enc-64kchunk-16thread.14.5 /srv/z/enc/test-enc-64kchunk-16thread.14.6 /srv/z/enc/test-enc-64kchunk-16thread.14.7 /srv/z/enc/test-enc-64kchunk-16thread.14.8 /srv/z/enc/test-enc-64kchunk-16thread.14.9 /srv/z/enc/test-enc-64kchunk-16thread.1.5 /srv/z/enc/test-enc-64kchunk-16thread.15.0 /srv/z/enc/test-enc-64kchunk-16thread.15.1 /srv/z/enc/test-enc-64kchunk-16thread.15.10 /srv/z/enc/test-enc-64kchunk-16thread.15.11 /srv/z/enc/test-enc-64kchunk-16thread.15.12 /srv/z/enc/test-enc-64kchunk-16thread.15.13 /srv/z/enc/test-enc-64kchunk-16thread.15.14 /srv/z/enc/test-enc-64kchunk-16thread.15.15 /srv/z/enc/test-enc-64kchunk-16thread.15.2 /srv/z/enc/test-enc-64kchunk-16thread.15.3 /srv/z/enc/test-enc-64kchunk-16thread.15.4 /srv/z/enc/test-enc-64kchunk-16thread.15.5 /srv/z/enc/test-enc-64kchunk-16thread.15.6 /srv/z/enc/test-enc-64kchunk-16thread.15.7 /srv/z/enc/test-enc-64kchunk-16thread.15.8 /srv/z/enc/test-enc-64kchunk-16thread.15.9 /srv/z/enc/test-enc-64kchunk-16thread.1.6 /srv/z/enc/test-enc-64kchunk-16thread.1.7 /srv/z/enc/test-enc-64kchunk-16thread.1.8 /srv/z/enc/test-enc-64kchunk-16thread.1.9 /srv/z/enc/test-enc-64kchunk-16thread.2.0 /srv/z/enc/test-enc-64kchunk-16thread.2.1 /srv/z/enc/test-enc-64kchunk-16thread.2.10 /srv/z/enc/test-enc-64kchunk-16thread.2.11 /srv/z/enc/test-enc-64kchunk-16thread.2.12 /srv/z/enc/test-enc-64kchunk-16thread.2.13 /srv/z/enc/test-enc-64kchunk-16thread.2.14 /srv/z/enc/test-enc-64kchunk-16thread.2.15 /srv/z/enc/test-enc-64kchunk-16thread.2.2 /srv/z/enc/test-enc-64kchunk-16thread.2.3 /srv/z/enc/test-enc-64kchunk-16thread.2.4 /srv/z/enc/test-enc-64kchunk-16thread.2.5 /srv/z/enc/test-enc-64kchunk-16thread.2.6 /srv/z/enc/test-enc-64kchunk-16thread.2.7 /srv/z/enc/test-enc-64kchunk-16thread.2.8 /srv/z/enc/test-enc-64kchunk-16thread.2.9 /srv/z/enc/test-enc-64kchunk-16thread.3.0 /srv/z/enc/test-enc-64kchunk-16thread.3.1 /srv/z/enc/test-enc-64kchunk-16thread.3.10 /srv/z/enc/test-enc-64kchunk-16thread.3.11 /srv/z/enc/test-enc-64kchunk-16thread.3.12 /srv/z/enc/test-enc-64kchunk-16thread.3.13 /srv/z/enc/test-enc-64kchunk-16thread.3.14 /srv/z/enc/test-enc-64kchunk-16thread.3.15 /srv/z/enc/test-enc-64kchunk-16thread.3.2 /srv/z/enc/test-enc-64kchunk-16thread.3.3 /srv/z/enc/test-enc-64kchunk-16thread.3.4 /srv/z/enc/test-enc-64kchunk-16thread.3.5 /srv/z/enc/test-enc-64kchunk-16thread.3.6 /srv/z/enc/test-enc-64kchunk-16thread.3.7 /srv/z/enc/test-enc-64kchunk-16thread.3.8 /srv/z/enc/test-enc-64kchunk-16thread.3.9 /srv/z/enc/test-enc-64kchunk-16thread.4.0 /srv/z/enc/test-enc-64kchunk-16thread.4.1 /srv/z/enc/test-enc-64kchunk-16thread.4.10 /srv/z/enc/test-enc-64kchunk-16thread.4.11 /srv/z/enc/test-enc-64kchunk-16thread.4.12 /srv/z/enc/test-enc-64kchunk-16thread.4.13 /srv/z/enc/test-enc-64kchunk-16thread.4.14 /srv/z/enc/test-enc-64kchunk-16thread.4.15 /srv/z/enc/test-enc-64kchunk-16thread.4.2 /srv/z/enc/test-enc-64kchunk-16thread.4.3 /srv/z/enc/test-enc-64kchunk-16thread.4.4 /srv/z/enc/test-enc-64kchunk-16thread.4.5 /srv/z/enc/test-enc-64kchunk-16thread.4.6 /srv/z/enc/test-enc-64kchunk-16thread.4.7 /srv/z/enc/test-enc-64kchunk-16thread.4.8 /srv/z/enc/test-enc-64kchunk-16thread.4.9 /srv/z/enc/test-enc-64kchunk-16thread.5.0 /srv/z/enc/test-enc-64kchunk-16thread.5.1 /srv/z/enc/test-enc-64kchunk-16thread.5.10 /srv/z/enc/test-enc-64kchunk-16thread.5.11 /srv/z/enc/test-enc-64kchunk-16thread.5.12 /srv/z/enc/test-enc-64kchunk-16thread.5.13 /srv/z/enc/test-enc-64kchunk-16thread.5.14 /srv/z/enc/test-enc-64kchunk-16thread.5.15 /srv/z/enc/test-enc-64kchunk-16thread.5.2 /srv/z/enc/test-enc-64kchunk-16thread.5.3 /srv/z/enc/test-enc-64kchunk-16thread.5.4 /srv/z/enc/test-enc-64kchunk-16thread.5.5 /srv/z/enc/test-enc-64kchunk-16thread.5.6 /srv/z/enc/test-enc-64kchunk-16thread.5.7 /srv/z/enc/test-enc-64kchunk-16thread.5.8 /srv/z/enc/test-enc-64kchunk-16thread.5.9 /srv/z/enc/test-enc-64kchunk-16thread.6.0 /srv/z/enc/test-enc-64kchunk-16thread.6.1 /srv/z/enc/test-enc-64kchunk-16thread.6.10 /srv/z/enc/test-enc-64kchunk-16thread.6.11 /srv/z/enc/test-enc-64kchunk-16thread.6.12 /srv/z/enc/test-enc-64kchunk-16thread.6.13 /srv/z/enc/test-enc-64kchunk-16thread.6.14 /srv/z/enc/test-enc-64kchunk-16thread.6.15 /srv/z/enc/test-enc-64kchunk-16thread.6.2 /srv/z/enc/test-enc-64kchunk-16thread.6.3 /srv/z/enc/test-enc-64kchunk-16thread.6.4 /srv/z/enc/test-enc-64kchunk-16thread.6.5 /srv/z/enc/test-enc-64kchunk-16thread.6.6 /srv/z/enc/test-enc-64kchunk-16thread.6.7 /srv/z/enc/test-enc-64kchunk-16thread.6.8 /srv/z/enc/test-enc-64kchunk-16thread.6.9 /srv/z/enc/test-enc-64kchunk-16thread.7.0 /srv/z/enc/test-enc-64kchunk-16thread.7.1 /srv/z/enc/test-enc-64kchunk-16thread.7.10 /srv/z/enc/test-enc-64kchunk-16thread.7.11 /srv/z/enc/test-enc-64kchunk-16thread.7.12 /srv/z/enc/test-enc-64kchunk-16thread.7.13 /srv/z/enc/test-enc-64kchunk-16thread.7.14 /srv/z/enc/test-enc-64kchunk-16thread.7.15 /srv/z/enc/test-enc-64kchunk-16thread.7.2 /srv/z/enc/test-enc-64kchunk-16thread.7.3 /srv/z/enc/test-enc-64kchunk-16thread.7.4 /srv/z/enc/test-enc-64kchunk-16thread.7.5 /srv/z/enc/test-enc-64kchunk-16thread.7.6 /srv/z/enc/test-enc-64kchunk-16thread.7.7 /srv/z/enc/test-enc-64kchunk-16thread.7.8 /srv/z/enc/test-enc-64kchunk-16thread.7.9 /srv/z/enc/test-enc-64kchunk-16thread.8.0 /srv/z/enc/test-enc-64kchunk-16thread.8.1 /srv/z/enc/test-enc-64kchunk-16thread.8.10 /srv/z/enc/test-enc-64kchunk-16thread.8.11 /srv/z/enc/test-enc-64kchunk-16thread.8.12 /srv/z/enc/test-enc-64kchunk-16thread.8.13 /srv/z/enc/test-enc-64kchunk-16thread.8.14 /srv/z/enc/test-enc-64kchunk-16thread.8.15 /srv/z/enc/test-enc-64kchunk-16thread.8.2 /srv/z/enc/test-enc-64kchunk-16thread.8.3 /srv/z/enc/test-enc-64kchunk-16thread.8.4 /srv/z/enc/test-enc-64kchunk-16thread.8.5 /srv/z/enc/test-enc-64kchunk-16thread.8.6 /srv/z/enc/test-enc-64kchunk-16thread.8.7 /srv/z/enc/test-enc-64kchunk-16thread.8.8 /srv/z/enc/test-enc-64kchunk-16thread.8.9 /srv/z/enc/test-enc-64kchunk-16thread.9.0 /srv/z/enc/test-enc-64kchunk-16thread.9.1 /srv/z/enc/test-enc-64kchunk-16thread.9.10 /srv/z/enc/test-enc-64kchunk-16thread.9.11 /srv/z/enc/test-enc-64kchunk-16thread.9.12 /srv/z/enc/test-enc-64kchunk-16thread.9.13 /srv/z/enc/test-enc-64kchunk-16thread.9.14 /srv/z/enc/test-enc-64kchunk-16thread.9.15 /srv/z/enc/test-enc-64kchunk-16thread.9.2 /srv/z/enc/test-enc-64kchunk-16thread.9.3 /srv/z/enc/test-enc-64kchunk-16thread.9.4 /srv/z/enc/test-enc-64kchunk-16thread.9.5 /srv/z/enc/test-enc-64kchunk-16thread.9.6 /srv/z/enc/test-enc-64kchunk-16thread.9.7 /srv/z/enc/test-enc-64kchunk-16thread.9.8 /srv/z/enc/test-enc-64kchunk-16thread.9.9 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-both-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/both/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-both-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=324MiB/s,w=326MiB/s][r=5182,w=5212 IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.5%][r=367MiB/s,w=369MiB/s][r=5872,w=5907 IOPS][eta 01m:41s] Jobs: 16 (f=256): [M(16)][24.2%][r=322MiB/s,w=323MiB/s][r=5144,w=5173 IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.5%][r=326MiB/s,w=323MiB/s][r=5220,w=5174 IOPS][eta 01m:21s] Jobs: 16 (f=256): [M(16)][40.5%][r=320MiB/s,w=320MiB/s][r=5127,w=5121 IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][48.3%][r=315MiB/s,w=314MiB/s][r=5037,w=5029 IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][56.2%][r=320MiB/s,w=322MiB/s][r=5115,w=5147 IOPS][eta 00m:53s] | |
| Jobs: 16 (f=256): [M(16)][64.2%][r=321MiB/s,w=317MiB/s][r=5132,w=5067 IOPS][eta 00m:43s] Jobs: 16 (f=256): [M(16)][72.5%][r=311MiB/s,w=314MiB/s][r=4973,w=5025 IOPS][eta 00m:33s] | |
| Jobs: 16 (f=256): [M(16)][80.2%][r=318MiB/s,w=314MiB/s][r=5088,w=5027 IOPS][eta 00m:24s] Jobs: 16 (f=256): [M(16)][88.3%][r=339MiB/s,w=340MiB/s][r=5419,w=5434 IOPS][eta 00m:14s] | |
| Jobs: 16 (f=256): [M(16)][95.9%][r=359MiB/s,w=358MiB/s][r=5744,w=5720 IOPS][eta 00m:05s] | |
| test-both-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=1492932: Wed Feb 26 10:37:54 2020 | |
| read: IOPS=5293, BW=331MiB/s (347MB/s)(38.8GiB/120008msec) | |
| slat (usec): min=9, max=121215, avg=1534.66, stdev=3754.80 | |
| clat (usec): min=3, max=447444, avg=95150.33, stdev=50764.67 | |
| lat (usec): min=691, max=447485, avg=96685.46, stdev=51435.92 | |
| clat percentiles (msec): | |
| | 1.00th=[ 11], 5.00th=[ 24], 10.00th=[ 34], 20.00th=[ 52], | |
| | 30.00th=[ 70], 40.00th=[ 81], 50.00th=[ 89], 60.00th=[ 99], | |
| | 70.00th=[ 110], 80.00th=[ 132], 90.00th=[ 167], 95.00th=[ 194], | |
| | 99.00th=[ 241], 99.50th=[ 259], 99.90th=[ 305], 99.95th=[ 330], | |
| | 99.99th=[ 393] | |
| bw ( KiB/s): min=123784, max=735357, per=99.72%, avg=337815.73, stdev=9936.48, samples=3828 | |
| iops : min= 1931, max=11489, avg=5276.28, stdev=155.30, samples=3828 | |
| write: IOPS=5292, BW=331MiB/s (347MB/s)(38.8GiB/120008msec); 0 zone resets | |
| slat (usec): min=15, max=145464, avg=1480.03, stdev=3751.08 | |
| clat (usec): min=3, max=447416, avg=95094.68, stdev=50738.86 | |
| lat (usec): min=714, max=447482, avg=96575.09, stdev=51414.95 | |
| clat percentiles (msec): | |
| | 1.00th=[ 11], 5.00th=[ 24], 10.00th=[ 34], 20.00th=[ 52], | |
| | 30.00th=[ 70], 40.00th=[ 81], 50.00th=[ 89], 60.00th=[ 99], | |
| | 70.00th=[ 110], 80.00th=[ 132], 90.00th=[ 167], 95.00th=[ 194], | |
| | 99.00th=[ 241], 99.50th=[ 262], 99.90th=[ 309], 99.95th=[ 334], | |
| | 99.99th=[ 388] | |
| bw ( KiB/s): min=128615, max=741277, per=99.73%, avg=337806.95, stdev=9898.20, samples=3828 | |
| iops : min= 2008, max=11582, avg=5276.16, stdev=154.70, samples=3828 | |
| lat (usec) : 4=0.01%, 10=0.01%, 20=0.01%, 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=0.01%, 4=0.14%, 10=0.66%, 20=2.86%, 50=15.51% | |
| lat (msec) : 100=42.95%, 250=37.17%, 500=0.70% | |
| cpu : usr=0.29%, sys=39.60%, ctx=864841, majf=0, minf=479355 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=635212,635163,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=331MiB/s (347MB/s), 331MiB/s-331MiB/s (347MB/s-347MB/s), io=38.8GiB (41.6GB), run=120008-120008msec | |
| WRITE: bw=331MiB/s (347MB/s), 331MiB/s-331MiB/s (347MB/s-347MB/s), io=38.8GiB (41.6GB), run=120008-120008msec | |
| + rm -rf /srv/z/both/test-both-64kchunk-16thread.0.0 /srv/z/both/test-both-64kchunk-16thread.0.1 /srv/z/both/test-both-64kchunk-16thread.0.10 /srv/z/both/test-both-64kchunk-16thread.0.11 /srv/z/both/test-both-64kchunk-16thread.0.12 /srv/z/both/test-both-64kchunk-16thread.0.13 /srv/z/both/test-both-64kchunk-16thread.0.14 /srv/z/both/test-both-64kchunk-16thread.0.15 /srv/z/both/test-both-64kchunk-16thread.0.2 /srv/z/both/test-both-64kchunk-16thread.0.3 /srv/z/both/test-both-64kchunk-16thread.0.4 /srv/z/both/test-both-64kchunk-16thread.0.5 /srv/z/both/test-both-64kchunk-16thread.0.6 /srv/z/both/test-both-64kchunk-16thread.0.7 /srv/z/both/test-both-64kchunk-16thread.0.8 /srv/z/both/test-both-64kchunk-16thread.0.9 /srv/z/both/test-both-64kchunk-16thread.1.0 /srv/z/both/test-both-64kchunk-16thread.10.0 /srv/z/both/test-both-64kchunk-16thread.10.1 /srv/z/both/test-both-64kchunk-16thread.10.10 /srv/z/both/test-both-64kchunk-16thread.10.11 /srv/z/both/test-both-64kchunk-16thread.10.12 /srv/z/both/test-both-64kchunk-16thread.10.13 /srv/z/both/test-both-64kchunk-16thread.10.14 /srv/z/both/test-both-64kchunk-16thread.10.15 /srv/z/both/test-both-64kchunk-16thread.10.2 /srv/z/both/test-both-64kchunk-16thread.10.3 /srv/z/both/test-both-64kchunk-16thread.10.4 /srv/z/both/test-both-64kchunk-16thread.10.5 /srv/z/both/test-both-64kchunk-16thread.10.6 /srv/z/both/test-both-64kchunk-16thread.10.7 /srv/z/both/test-both-64kchunk-16thread.10.8 /srv/z/both/test-both-64kchunk-16thread.10.9 /srv/z/both/test-both-64kchunk-16thread.1.1 /srv/z/both/test-both-64kchunk-16thread.1.10 /srv/z/both/test-both-64kchunk-16thread.11.0 /srv/z/both/test-both-64kchunk-16thread.1.11 /srv/z/both/test-both-64kchunk-16thread.11.1 /srv/z/both/test-both-64kchunk-16thread.11.10 /srv/z/both/test-both-64kchunk-16thread.11.11 /srv/z/both/test-both-64kchunk-16thread.11.12 /srv/z/both/test-both-64kchunk-16thread.11.13 /srv/z/both/test-both-64kchunk-16thread.11.14 /srv/z/both/test-both-64kchunk-16thread.11.15 /srv/z/both/test-both-64kchunk-16thread.1.12 /srv/z/both/test-both-64kchunk-16thread.11.2 /srv/z/both/test-both-64kchunk-16thread.1.13 /srv/z/both/test-both-64kchunk-16thread.11.3 /srv/z/both/test-both-64kchunk-16thread.1.14 /srv/z/both/test-both-64kchunk-16thread.11.4 /srv/z/both/test-both-64kchunk-16thread.1.15 /srv/z/both/test-both-64kchunk-16thread.11.5 /srv/z/both/test-both-64kchunk-16thread.11.6 /srv/z/both/test-both-64kchunk-16thread.11.7 /srv/z/both/test-both-64kchunk-16thread.11.8 /srv/z/both/test-both-64kchunk-16thread.11.9 /srv/z/both/test-both-64kchunk-16thread.1.2 /srv/z/both/test-both-64kchunk-16thread.12.0 /srv/z/both/test-both-64kchunk-16thread.12.1 /srv/z/both/test-both-64kchunk-16thread.12.10 /srv/z/both/test-both-64kchunk-16thread.12.11 /srv/z/both/test-both-64kchunk-16thread.12.12 /srv/z/both/test-both-64kchunk-16thread.12.13 /srv/z/both/test-both-64kchunk-16thread.12.14 /srv/z/both/test-both-64kchunk-16thread.12.15 /srv/z/both/test-both-64kchunk-16thread.12.2 /srv/z/both/test-both-64kchunk-16thread.12.3 /srv/z/both/test-both-64kchunk-16thread.12.4 /srv/z/both/test-both-64kchunk-16thread.12.5 /srv/z/both/test-both-64kchunk-16thread.12.6 /srv/z/both/test-both-64kchunk-16thread.12.7 /srv/z/both/test-both-64kchunk-16thread.12.8 /srv/z/both/test-both-64kchunk-16thread.12.9 /srv/z/both/test-both-64kchunk-16thread.1.3 /srv/z/both/test-both-64kchunk-16thread.13.0 /srv/z/both/test-both-64kchunk-16thread.13.1 /srv/z/both/test-both-64kchunk-16thread.13.10 /srv/z/both/test-both-64kchunk-16thread.13.11 /srv/z/both/test-both-64kchunk-16thread.13.12 /srv/z/both/test-both-64kchunk-16thread.13.13 /srv/z/both/test-both-64kchunk-16thread.13.14 /srv/z/both/test-both-64kchunk-16thread.13.15 /srv/z/both/test-both-64kchunk-16thread.13.2 /srv/z/both/test-both-64kchunk-16thread.13.3 /srv/z/both/test-both-64kchunk-16thread.13.4 /srv/z/both/test-both-64kchunk-16thread.13.5 /srv/z/both/test-both-64kchunk-16thread.13.6 /srv/z/both/test-both-64kchunk-16thread.13.7 /srv/z/both/test-both-64kchunk-16thread.13.8 /srv/z/both/test-both-64kchunk-16thread.13.9 /srv/z/both/test-both-64kchunk-16thread.1.4 /srv/z/both/test-both-64kchunk-16thread.14.0 /srv/z/both/test-both-64kchunk-16thread.14.1 /srv/z/both/test-both-64kchunk-16thread.14.10 /srv/z/both/test-both-64kchunk-16thread.14.11 /srv/z/both/test-both-64kchunk-16thread.14.12 /srv/z/both/test-both-64kchunk-16thread.14.13 /srv/z/both/test-both-64kchunk-16thread.14.14 /srv/z/both/test-both-64kchunk-16thread.14.15 /srv/z/both/test-both-64kchunk-16thread.14.2 /srv/z/both/test-both-64kchunk-16thread.14.3 /srv/z/both/test-both-64kchunk-16thread.14.4 /srv/z/both/test-both-64kchunk-16thread.14.5 /srv/z/both/test-both-64kchunk-16thread.14.6 /srv/z/both/test-both-64kchunk-16thread.14.7 /srv/z/both/test-both-64kchunk-16thread.14.8 /srv/z/both/test-both-64kchunk-16thread.14.9 /srv/z/both/test-both-64kchunk-16thread.1.5 /srv/z/both/test-both-64kchunk-16thread.15.0 /srv/z/both/test-both-64kchunk-16thread.15.1 /srv/z/both/test-both-64kchunk-16thread.15.10 /srv/z/both/test-both-64kchunk-16thread.15.11 /srv/z/both/test-both-64kchunk-16thread.15.12 /srv/z/both/test-both-64kchunk-16thread.15.13 /srv/z/both/test-both-64kchunk-16thread.15.14 /srv/z/both/test-both-64kchunk-16thread.15.15 /srv/z/both/test-both-64kchunk-16thread.15.2 /srv/z/both/test-both-64kchunk-16thread.15.3 /srv/z/both/test-both-64kchunk-16thread.15.4 /srv/z/both/test-both-64kchunk-16thread.15.5 /srv/z/both/test-both-64kchunk-16thread.15.6 /srv/z/both/test-both-64kchunk-16thread.15.7 /srv/z/both/test-both-64kchunk-16thread.15.8 /srv/z/both/test-both-64kchunk-16thread.15.9 /srv/z/both/test-both-64kchunk-16thread.1.6 /srv/z/both/test-both-64kchunk-16thread.1.7 /srv/z/both/test-both-64kchunk-16thread.1.8 /srv/z/both/test-both-64kchunk-16thread.1.9 /srv/z/both/test-both-64kchunk-16thread.2.0 /srv/z/both/test-both-64kchunk-16thread.2.1 /srv/z/both/test-both-64kchunk-16thread.2.10 /srv/z/both/test-both-64kchunk-16thread.2.11 /srv/z/both/test-both-64kchunk-16thread.2.12 /srv/z/both/test-both-64kchunk-16thread.2.13 /srv/z/both/test-both-64kchunk-16thread.2.14 /srv/z/both/test-both-64kchunk-16thread.2.15 /srv/z/both/test-both-64kchunk-16thread.2.2 /srv/z/both/test-both-64kchunk-16thread.2.3 /srv/z/both/test-both-64kchunk-16thread.2.4 /srv/z/both/test-both-64kchunk-16thread.2.5 /srv/z/both/test-both-64kchunk-16thread.2.6 /srv/z/both/test-both-64kchunk-16thread.2.7 /srv/z/both/test-both-64kchunk-16thread.2.8 /srv/z/both/test-both-64kchunk-16thread.2.9 /srv/z/both/test-both-64kchunk-16thread.3.0 /srv/z/both/test-both-64kchunk-16thread.3.1 /srv/z/both/test-both-64kchunk-16thread.3.10 /srv/z/both/test-both-64kchunk-16thread.3.11 /srv/z/both/test-both-64kchunk-16thread.3.12 /srv/z/both/test-both-64kchunk-16thread.3.13 /srv/z/both/test-both-64kchunk-16thread.3.14 /srv/z/both/test-both-64kchunk-16thread.3.15 /srv/z/both/test-both-64kchunk-16thread.3.2 /srv/z/both/test-both-64kchunk-16thread.3.3 /srv/z/both/test-both-64kchunk-16thread.3.4 /srv/z/both/test-both-64kchunk-16thread.3.5 /srv/z/both/test-both-64kchunk-16thread.3.6 /srv/z/both/test-both-64kchunk-16thread.3.7 /srv/z/both/test-both-64kchunk-16thread.3.8 /srv/z/both/test-both-64kchunk-16thread.3.9 /srv/z/both/test-both-64kchunk-16thread.4.0 /srv/z/both/test-both-64kchunk-16thread.4.1 /srv/z/both/test-both-64kchunk-16thread.4.10 /srv/z/both/test-both-64kchunk-16thread.4.11 /srv/z/both/test-both-64kchunk-16thread.4.12 /srv/z/both/test-both-64kchunk-16thread.4.13 /srv/z/both/test-both-64kchunk-16thread.4.14 /srv/z/both/test-both-64kchunk-16thread.4.15 /srv/z/both/test-both-64kchunk-16thread.4.2 /srv/z/both/test-both-64kchunk-16thread.4.3 /srv/z/both/test-both-64kchunk-16thread.4.4 /srv/z/both/test-both-64kchunk-16thread.4.5 /srv/z/both/test-both-64kchunk-16thread.4.6 /srv/z/both/test-both-64kchunk-16thread.4.7 /srv/z/both/test-both-64kchunk-16thread.4.8 /srv/z/both/test-both-64kchunk-16thread.4.9 /srv/z/both/test-both-64kchunk-16thread.5.0 /srv/z/both/test-both-64kchunk-16thread.5.1 /srv/z/both/test-both-64kchunk-16thread.5.10 /srv/z/both/test-both-64kchunk-16thread.5.11 /srv/z/both/test-both-64kchunk-16thread.5.12 /srv/z/both/test-both-64kchunk-16thread.5.13 /srv/z/both/test-both-64kchunk-16thread.5.14 /srv/z/both/test-both-64kchunk-16thread.5.15 /srv/z/both/test-both-64kchunk-16thread.5.2 /srv/z/both/test-both-64kchunk-16thread.5.3 /srv/z/both/test-both-64kchunk-16thread.5.4 /srv/z/both/test-both-64kchunk-16thread.5.5 /srv/z/both/test-both-64kchunk-16thread.5.6 /srv/z/both/test-both-64kchunk-16thread.5.7 /srv/z/both/test-both-64kchunk-16thread.5.8 /srv/z/both/test-both-64kchunk-16thread.5.9 /srv/z/both/test-both-64kchunk-16thread.6.0 /srv/z/both/test-both-64kchunk-16thread.6.1 /srv/z/both/test-both-64kchunk-16thread.6.10 /srv/z/both/test-both-64kchunk-16thread.6.11 /srv/z/both/test-both-64kchunk-16thread.6.12 /srv/z/both/test-both-64kchunk-16thread.6.13 /srv/z/both/test-both-64kchunk-16thread.6.14 /srv/z/both/test-both-64kchunk-16thread.6.15 /srv/z/both/test-both-64kchunk-16thread.6.2 /srv/z/both/test-both-64kchunk-16thread.6.3 /srv/z/both/test-both-64kchunk-16thread.6.4 /srv/z/both/test-both-64kchunk-16thread.6.5 /srv/z/both/test-both-64kchunk-16thread.6.6 /srv/z/both/test-both-64kchunk-16thread.6.7 /srv/z/both/test-both-64kchunk-16thread.6.8 /srv/z/both/test-both-64kchunk-16thread.6.9 /srv/z/both/test-both-64kchunk-16thread.7.0 /srv/z/both/test-both-64kchunk-16thread.7.1 /srv/z/both/test-both-64kchunk-16thread.7.10 /srv/z/both/test-both-64kchunk-16thread.7.11 /srv/z/both/test-both-64kchunk-16thread.7.12 /srv/z/both/test-both-64kchunk-16thread.7.13 /srv/z/both/test-both-64kchunk-16thread.7.14 /srv/z/both/test-both-64kchunk-16thread.7.15 /srv/z/both/test-both-64kchunk-16thread.7.2 /srv/z/both/test-both-64kchunk-16thread.7.3 /srv/z/both/test-both-64kchunk-16thread.7.4 /srv/z/both/test-both-64kchunk-16thread.7.5 /srv/z/both/test-both-64kchunk-16thread.7.6 /srv/z/both/test-both-64kchunk-16thread.7.7 /srv/z/both/test-both-64kchunk-16thread.7.8 /srv/z/both/test-both-64kchunk-16thread.7.9 /srv/z/both/test-both-64kchunk-16thread.8.0 /srv/z/both/test-both-64kchunk-16thread.8.1 /srv/z/both/test-both-64kchunk-16thread.8.10 /srv/z/both/test-both-64kchunk-16thread.8.11 /srv/z/both/test-both-64kchunk-16thread.8.12 /srv/z/both/test-both-64kchunk-16thread.8.13 /srv/z/both/test-both-64kchunk-16thread.8.14 /srv/z/both/test-both-64kchunk-16thread.8.15 /srv/z/both/test-both-64kchunk-16thread.8.2 /srv/z/both/test-both-64kchunk-16thread.8.3 /srv/z/both/test-both-64kchunk-16thread.8.4 /srv/z/both/test-both-64kchunk-16thread.8.5 /srv/z/both/test-both-64kchunk-16thread.8.6 /srv/z/both/test-both-64kchunk-16thread.8.7 /srv/z/both/test-both-64kchunk-16thread.8.8 /srv/z/both/test-both-64kchunk-16thread.8.9 /srv/z/both/test-both-64kchunk-16thread.9.0 /srv/z/both/test-both-64kchunk-16thread.9.1 /srv/z/both/test-both-64kchunk-16thread.9.10 /srv/z/both/test-both-64kchunk-16thread.9.11 /srv/z/both/test-both-64kchunk-16thread.9.12 /srv/z/both/test-both-64kchunk-16thread.9.13 /srv/z/both/test-both-64kchunk-16thread.9.14 /srv/z/both/test-both-64kchunk-16thread.9.15 /srv/z/both/test-both-64kchunk-16thread.9.2 /srv/z/both/test-both-64kchunk-16thread.9.3 /srv/z/both/test-both-64kchunk-16thread.9.4 /srv/z/both/test-both-64kchunk-16thread.9.5 /srv/z/both/test-both-64kchunk-16thread.9.6 /srv/z/both/test-both-64kchunk-16thread.9.7 /srv/z/both/test-both-64kchunk-16thread.9.8 /srv/z/both/test-both-64kchunk-16thread.9.9 | |
| + zpool destroy tank | |
| + for ASHIFTVAL in 12 13 14 15 16 | |
| + trim_disks | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| Success formatting namespace:1 | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '################## waiting 60s for nvme format to complete...' | |
| ################## waiting 60s for nvme format to complete... | |
| + sleep 60 | |
| + nvme list | |
| Node SN Model Namespace Usage Format FW Rev | |
| ---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- -------- | |
| /dev/nvme0n1 S437NE0M804438 SAMSUNG MZQLB960HAJR-00007 1 70.51 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme1n1 S437NE0M804437 SAMSUNG MZQLB960HAJR-00007 1 69.72 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme2n1 S437NE0M804442 SAMSUNG MZQLB960HAJR-00007 1 80.72 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme3n1 S437NE0M804435 SAMSUNG MZQLB960HAJR-00007 1 59.83 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme4n1 S437NE0M802527 SAMSUNG MZQLB960HAJR-00007 1 58.23 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme5n1 S437NX0M402110 SAMSUNG MZQLB960HAJR-00007 1 80.95 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + zpool create tank -o ashift=13 -O mountpoint=/srv/z raidz1 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| + dd if=/dev/urandom of=/tmp/testkey.raw bs=32 count=1 | |
| 1+0 records in | |
| 1+0 records out | |
| 32 bytes copied, 0.000303632 s, 105 kB/s | |
| + COMP=' | |
| -o compression=lz4 | |
| ' | |
| + ENC=' | |
| -o encryption=aes-256-gcm | |
| -o keyformat=raw | |
| -o keylocation=file:///tmp/testkey.raw | |
| ' | |
| + zfs create tank/none | |
| + zfs create -o compression=lz4 tank/comp | |
| + zfs create -o encryption=aes-256-gcm -o keyformat=raw -o keylocation=file:///tmp/testkey.raw tank/enc | |
| + zfs create -o compression=lz4 -o encryption=aes-256-gcm -o keyformat=raw -o keylocation=file:///tmp/testkey.raw tank/both | |
| + FO='--eta-newline=10 --eta-interval=10 --eta=always' | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-none-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/none/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-none-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-none-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=1118MiB/s,w=1117MiB/s][r=17.9k,w=17.9k IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.7%][r=1111MiB/s,w=1108MiB/s][r=17.8k,w=17.7k IOPS][eta 01m:40s] Jobs: 1 (f=1): [M(1)][24.2%][r=1110MiB/s,w=1109MiB/s][r=17.8k,w=17.7k IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.8%][r=1097MiB/s,w=1096MiB/s][r=17.5k,w=17.5k IOPS][eta 01m:20s] Jobs: 1 (f=1): [M(1)][40.5%][r=1110MiB/s,w=1109MiB/s][r=17.8k,w=17.7k IOPS][eta 01m:12s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=1102MiB/s,w=1100MiB/s][r=17.6k,w=17.6k IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=1094MiB/s,w=1099MiB/s][r=17.5k,w=17.6k IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][65.0%][r=1090MiB/s,w=1085MiB/s][r=17.4k,w=17.4k IOPS][eta 00m:42s] Jobs: 1 (f=1): [M(1)][72.5%][r=1095MiB/s,w=1093MiB/s][r=17.5k,w=17.5k IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=1091MiB/s,w=1095MiB/s][r=17.5k,w=17.5k IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][88.4%][r=1113MiB/s,w=1109MiB/s][r=17.8k,w=17.7k IOPS][eta 00m:14s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=1105MiB/s,w=1102MiB/s][r=17.7k,w=17.6k IOPS][eta 00m:04s] | |
| test-none-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=1722082: Wed Feb 26 10:41:19 2020 | |
| read: IOPS=17.7k, BW=1105MiB/s (1159MB/s)(130GiB/120001msec) | |
| slat (usec): min=5, max=44802, avg=26.73, stdev=142.34 | |
| clat (usec): min=5, max=64404, avg=1787.04, stdev=1885.00 | |
| lat (usec): min=98, max=64455, avg=1813.91, stdev=1905.80 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1172], 5.00th=[ 1270], 10.00th=[ 1287], 20.00th=[ 1319], | |
| | 30.00th=[ 1352], 40.00th=[ 1385], 50.00th=[ 1418], 60.00th=[ 1450], | |
| | 70.00th=[ 1500], 80.00th=[ 1598], 90.00th=[ 2540], 95.00th=[ 3392], | |
| | 99.00th=[ 9110], 99.50th=[14353], 99.90th=[27657], 99.95th=[33162], | |
| | 99.99th=[45876] | |
| bw ( MiB/s): min= 802, max= 1564, per=99.88%, avg=1103.74, stdev=153.68, samples=239 | |
| iops : min=12832, max=25027, avg=17659.74, stdev=2458.86, samples=239 | |
| write: IOPS=17.7k, BW=1104MiB/s (1157MB/s)(129GiB/120001msec); 0 zone resets | |
| slat (usec): min=10, max=38429, avg=27.16, stdev=158.20 | |
| clat (usec): min=139, max=64433, avg=1779.21, stdev=1849.50 | |
| lat (usec): min=202, max=64485, avg=1806.51, stdev=1873.80 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1172], 5.00th=[ 1270], 10.00th=[ 1287], 20.00th=[ 1319], | |
| | 30.00th=[ 1352], 40.00th=[ 1385], 50.00th=[ 1418], 60.00th=[ 1434], | |
| | 70.00th=[ 1500], 80.00th=[ 1598], 90.00th=[ 2507], 95.00th=[ 3359], | |
| | 99.00th=[ 8848], 99.50th=[14091], 99.90th=[27657], 99.95th=[32637], | |
| | 99.99th=[44827] | |
| bw ( MiB/s): min= 809, max= 1562, per=99.88%, avg=1102.46, stdev=154.45, samples=239 | |
| iops : min=12950, max=24994, avg=17639.27, stdev=2471.14, samples=239 | |
| lat (usec) : 10=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=87.89%, 4=8.96%, 10=2.27%, 20=0.61%, 50=0.25% | |
| lat (msec) : 100=0.01% | |
| cpu : usr=7.40%, sys=81.63%, ctx=27316, majf=0, minf=35608 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=2121779,2119247,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=1105MiB/s (1159MB/s), 1105MiB/s-1105MiB/s (1159MB/s-1159MB/s), io=130GiB (139GB), run=120001-120001msec | |
| WRITE: bw=1104MiB/s (1157MB/s), 1104MiB/s-1104MiB/s (1157MB/s-1157MB/s), io=129GiB (139GB), run=120001-120001msec | |
| + rm -rf /srv/z/none/test-none-64kchunk-1thread.0.0 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-comp-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/comp/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-comp-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-comp-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=1107MiB/s,w=1106MiB/s][r=17.7k,w=17.7k IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.5%][r=1091MiB/s,w=1089MiB/s][r=17.5k,w=17.4k IOPS][eta 01m:41s] Jobs: 1 (f=1): [M(1)][24.2%][r=1058MiB/s,w=1056MiB/s][r=16.9k,w=16.9k IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.8%][r=1081MiB/s,w=1081MiB/s][r=17.3k,w=17.3k IOPS][eta 01m:20s] Jobs: 1 (f=1): [M(1)][40.5%][r=1070MiB/s,w=1068MiB/s][r=17.1k,w=17.1k IOPS][eta 01m:12s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=1069MiB/s,w=1067MiB/s][r=17.1k,w=17.1k IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=1061MiB/s,w=1065MiB/s][r=16.0k,w=17.0k IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][65.0%][r=1071MiB/s,w=1065MiB/s][r=17.1k,w=17.0k IOPS][eta 00m:42s] Jobs: 1 (f=1): [M(1)][72.5%][r=1080MiB/s,w=1078MiB/s][r=17.3k,w=17.3k IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=1062MiB/s,w=1066MiB/s][r=16.0k,w=17.1k IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][88.4%][r=1063MiB/s,w=1058MiB/s][r=17.0k,w=16.9k IOPS][eta 00m:14s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=1071MiB/s,w=1071MiB/s][r=17.1k,w=17.1k IOPS][eta 00m:04s] | |
| test-comp-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=2023255: Wed Feb 26 10:43:30 2020 | |
| read: IOPS=17.2k, BW=1075MiB/s (1127MB/s)(126GiB/120001msec) | |
| slat (usec): min=5, max=39620, avg=27.62, stdev=196.71 | |
| clat (usec): min=21, max=91052, avg=1835.88, stdev=2362.27 | |
| lat (usec): min=58, max=91081, avg=1863.64, stdev=2386.63 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1188], 5.00th=[ 1270], 10.00th=[ 1287], 20.00th=[ 1319], | |
| | 30.00th=[ 1352], 40.00th=[ 1385], 50.00th=[ 1401], 60.00th=[ 1434], | |
| | 70.00th=[ 1516], 80.00th=[ 1565], 90.00th=[ 1762], 95.00th=[ 3032], | |
| | 99.00th=[13173], 99.50th=[18744], 99.90th=[31065], 99.95th=[36963], | |
| | 99.99th=[50070] | |
| bw ( MiB/s): min= 798, max= 1486, per=99.98%, avg=1074.98, stdev=173.68, samples=239 | |
| iops : min=12774, max=23786, avg=17199.64, stdev=2778.85, samples=239 | |
| write: IOPS=17.2k, BW=1074MiB/s (1126MB/s)(126GiB/120001msec); 0 zone resets | |
| slat (usec): min=10, max=87308, avg=27.63, stdev=211.48 | |
| clat (usec): min=2, max=91022, avg=1829.66, stdev=2328.13 | |
| lat (usec): min=19, max=91061, avg=1857.45, stdev=2355.63 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1188], 5.00th=[ 1270], 10.00th=[ 1287], 20.00th=[ 1319], | |
| | 30.00th=[ 1352], 40.00th=[ 1385], 50.00th=[ 1401], 60.00th=[ 1434], | |
| | 70.00th=[ 1516], 80.00th=[ 1565], 90.00th=[ 1762], 95.00th=[ 3032], | |
| | 99.00th=[12911], 99.50th=[18482], 99.90th=[31065], 99.95th=[36963], | |
| | 99.99th=[49546] | |
| bw ( MiB/s): min= 791, max= 1485, per=99.97%, avg=1073.58, stdev=174.54, samples=239 | |
| iops : min=12666, max=23770, avg=17177.28, stdev=2792.61, samples=239 | |
| lat (usec) : 4=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01% | |
| lat (usec) : 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=91.31%, 4=5.12%, 10=2.15%, 20=1.01%, 50=0.40% | |
| lat (msec) : 100=0.01% | |
| cpu : usr=7.20%, sys=76.94%, ctx=25297, majf=0, minf=42914 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=2064383,2061842,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=1075MiB/s (1127MB/s), 1075MiB/s-1075MiB/s (1127MB/s-1127MB/s), io=126GiB (135GB), run=120001-120001msec | |
| WRITE: bw=1074MiB/s (1126MB/s), 1074MiB/s-1074MiB/s (1126MB/s-1126MB/s), io=126GiB (135GB), run=120001-120001msec | |
| + rm -rf /srv/z/comp/test-comp-64kchunk-1thread.0.0 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-enc-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/enc/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-enc-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-enc-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=34.4MiB/s,w=34.7MiB/s][r=550,w=555 IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.5%][r=35.1MiB/s,w=34.6MiB/s][r=561,w=553 IOPS][eta 01m:41s] Jobs: 1 (f=1): [M(1)][24.2%][r=34.0MiB/s,w=34.8MiB/s][r=559,w=557 IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.5%][r=35.3MiB/s,w=35.4MiB/s][r=565,w=566 IOPS][eta 01m:21s] Jobs: 1 (f=1): [M(1)][40.5%][r=34.7MiB/s,w=34.8MiB/s][r=555,w=557 IOPS][eta 01m:12s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=34.2MiB/s,w=34.7MiB/s][r=546,w=554 IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=35.2MiB/s,w=34.9MiB/s][r=562,w=558 IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][64.5%][r=35.1MiB/s,w=32.9MiB/s][r=562,w=525 IOPS][eta 00m:43s] Jobs: 1 (f=1): [M(1)][72.5%][r=34.9MiB/s,w=35.0MiB/s][r=558,w=560 IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=34.6MiB/s,w=35.5MiB/s][r=554,w=567 IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][88.4%][r=34.0MiB/s,w=35.1MiB/s][r=559,w=560 IOPS][eta 00m:14s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=34.9MiB/s,w=33.9MiB/s][r=558,w=542 IOPS][eta 00m:04s] | |
| test-enc-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=2232693: Wed Feb 26 10:45:44 2020 | |
| read: IOPS=558, BW=34.9MiB/s (36.6MB/s)(4190MiB/120003msec) | |
| slat (usec): min=6, max=10042, avg=1143.25, stdev=1675.61 | |
| clat (usec): min=4, max=161475, avg=56509.13, stdev=10450.46 | |
| lat (msec): min=3, max=161, avg=57.65, stdev=10.61 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 45], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 53], 50.00th=[ 55], 60.00th=[ 58], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 67], 95.00th=[ 77], | |
| | 99.00th=[ 95], 99.50th=[ 102], 99.90th=[ 131], 99.95th=[ 148], | |
| | 99.99th=[ 159] | |
| bw ( KiB/s): min=22483, max=44416, per=99.93%, avg=35729.39, stdev=3282.79, samples=240 | |
| iops : min= 351, max= 694, avg=558.18, stdev=51.29, samples=240 | |
| write: IOPS=556, BW=34.8MiB/s (36.4MB/s)(4171MiB/120003msec); 0 zone resets | |
| slat (usec): min=12, max=83180, avg=646.73, stdev=1419.20 | |
| clat (msec): min=3, max=161, avg=56.48, stdev=10.42 | |
| lat (msec): min=3, max=161, avg=57.12, stdev=10.48 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 45], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 53], 50.00th=[ 55], 60.00th=[ 58], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 67], 95.00th=[ 75], | |
| | 99.00th=[ 95], 99.50th=[ 102], 99.90th=[ 128], 99.95th=[ 146], | |
| | 99.99th=[ 157] | |
| bw ( KiB/s): min=20608, max=43520, per=99.92%, avg=35567.14, stdev=3813.51, samples=240 | |
| iops : min= 322, max= 680, avg=555.65, stdev=59.57, samples=240 | |
| lat (usec) : 10=0.01% | |
| lat (msec) : 4=0.01%, 10=0.01%, 20=0.01%, 50=22.92%, 100=76.50% | |
| lat (msec) : 250=0.56% | |
| cpu : usr=0.21%, sys=85.56%, ctx=12867, majf=0, minf=20964 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=67039,66741,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=34.9MiB/s (36.6MB/s), 34.9MiB/s-34.9MiB/s (36.6MB/s-36.6MB/s), io=4190MiB (4393MB), run=120003-120003msec | |
| WRITE: bw=34.8MiB/s (36.4MB/s), 34.8MiB/s-34.8MiB/s (36.4MB/s-36.4MB/s), io=4171MiB (4374MB), run=120003-120003msec | |
| + rm -rf /srv/z/enc/test-enc-64kchunk-1thread.0.0 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-both-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/both/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-both-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-both-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=34.2MiB/s,w=34.6MiB/s][r=547,w=553 IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.5%][r=35.0MiB/s,w=34.6MiB/s][r=560,w=553 IOPS][eta 01m:41s] Jobs: 1 (f=1): [M(1)][24.2%][r=34.9MiB/s,w=34.7MiB/s][r=557,w=554 IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.5%][r=35.1MiB/s,w=35.1MiB/s][r=560,w=561 IOPS][eta 01m:21s] Jobs: 1 (f=1): [M(1)][40.5%][r=34.8MiB/s,w=34.9MiB/s][r=556,w=558 IOPS][eta 01m:12s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=34.5MiB/s,w=35.0MiB/s][r=551,w=560 IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=34.9MiB/s,w=34.7MiB/s][r=558,w=554 IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][64.5%][r=34.9MiB/s,w=32.7MiB/s][r=559,w=523 IOPS][eta 00m:43s] Jobs: 1 (f=1): [M(1)][72.5%][r=35.4MiB/s,w=35.4MiB/s][r=566,w=566 IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=34.9MiB/s,w=35.8MiB/s][r=559,w=572 IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][88.4%][r=34.9MiB/s,w=35.1MiB/s][r=558,w=560 IOPS][eta 00m:14s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=34.6MiB/s,w=33.6MiB/s][r=553,w=537 IOPS][eta 00m:04s] | |
| test-both-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=2344499: Wed Feb 26 10:47:57 2020 | |
| read: IOPS=558, BW=34.9MiB/s (36.6MB/s)(4187MiB/120003msec) | |
| slat (usec): min=6, max=22726, avg=1138.62, stdev=1664.36 | |
| clat (usec): min=3, max=212348, avg=56538.39, stdev=10737.55 | |
| lat (msec): min=3, max=215, avg=57.68, stdev=10.87 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 45], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 54], 50.00th=[ 55], 60.00th=[ 58], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 67], 95.00th=[ 75], | |
| | 99.00th=[ 97], 99.50th=[ 105], 99.90th=[ 124], 99.95th=[ 140], | |
| | 99.99th=[ 209] | |
| bw ( KiB/s): min=23168, max=42240, per=99.94%, avg=35707.55, stdev=3426.36, samples=240 | |
| iops : min= 362, max= 660, avg=557.87, stdev=53.52, samples=240 | |
| write: IOPS=555, BW=34.7MiB/s (36.4MB/s)(4168MiB/120003msec); 0 zone resets | |
| slat (usec): min=13, max=136971, avg=652.45, stdev=1477.26 | |
| clat (msec): min=3, max=215, avg=56.53, stdev=10.70 | |
| lat (msec): min=3, max=215, avg=57.18, stdev=10.79 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 45], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 54], 50.00th=[ 55], 60.00th=[ 58], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 67], 95.00th=[ 75], | |
| | 99.00th=[ 96], 99.50th=[ 105], 99.90th=[ 125], 99.95th=[ 142], | |
| | 99.99th=[ 211] | |
| bw ( KiB/s): min=21120, max=44672, per=99.92%, avg=35539.97, stdev=3786.00, samples=240 | |
| iops : min= 330, max= 698, avg=555.25, stdev=59.16, samples=240 | |
| lat (usec) : 4=0.01% | |
| lat (msec) : 4=0.01%, 10=0.01%, 20=0.01%, 50=22.76%, 100=76.44% | |
| lat (msec) : 250=0.78% | |
| cpu : usr=0.17%, sys=85.29%, ctx=13079, majf=0, minf=16789 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=66997,66690,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=34.9MiB/s (36.6MB/s), 34.9MiB/s-34.9MiB/s (36.6MB/s-36.6MB/s), io=4187MiB (4391MB), run=120003-120003msec | |
| WRITE: bw=34.7MiB/s (36.4MB/s), 34.7MiB/s-34.7MiB/s (36.4MB/s-36.4MB/s), io=4168MiB (4371MB), run=120003-120003msec | |
| + rm -rf /srv/z/both/test-both-64kchunk-1thread.0.0 | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-none-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/none/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-none-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=2895MiB/s,w=2895MiB/s][r=46.3k,w=46.3k IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.5%][r=2765MiB/s,w=2773MiB/s][r=44.2k,w=44.4k IOPS][eta 01m:41s] Jobs: 16 (f=256): [M(16)][24.2%][r=3046MiB/s,w=3050MiB/s][r=48.7k,w=48.8k IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.8%][r=3201MiB/s,w=3209MiB/s][r=51.2k,w=51.3k IOPS][eta 01m:20s] Jobs: 16 (f=256): [M(16)][40.5%][r=3204MiB/s,w=3204MiB/s][r=51.3k,w=51.3k IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][48.3%][r=3104MiB/s,w=3097MiB/s][r=49.7k,w=49.5k IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][56.2%][r=2706MiB/s,w=2709MiB/s][r=43.3k,w=43.3k IOPS][eta 00m:53s] | |
| Jobs: 16 (f=256): [M(16)][64.7%][r=2803MiB/s,w=2798MiB/s][r=44.9k,w=44.8k IOPS][eta 00m:42s] Jobs: 16 (f=256): [M(16)][72.5%][r=2774MiB/s,w=2783MiB/s][r=44.4k,w=44.5k IOPS][eta 00m:33s] | |
| Jobs: 16 (f=256): [M(16)][80.2%][r=2711MiB/s,w=2713MiB/s][r=43.4k,w=43.4k IOPS][eta 00m:24s] Jobs: 16 (f=256): [M(16)][88.3%][r=2631MiB/s,w=2633MiB/s][r=42.1k,w=42.1k IOPS][eta 00m:14s] | |
| Jobs: 16 (f=256): [M(16)][96.7%][r=2545MiB/s,w=2546MiB/s][r=40.7k,w=40.7k IOPS][eta 00m:04s] | |
| test-none-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=2594427: Wed Feb 26 10:52:39 2020 | |
| read: IOPS=45.7k, BW=2856MiB/s (2994MB/s)(335GiB/120002msec) | |
| slat (usec): min=9, max=102681, avg=146.05, stdev=731.58 | |
| clat (usec): min=2, max=239656, avg=11026.39, stdev=9787.06 | |
| lat (usec): min=31, max=239711, avg=11172.81, stdev=9913.36 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 5], 10.00th=[ 5], 20.00th=[ 6], | |
| | 30.00th=[ 6], 40.00th=[ 7], 50.00th=[ 7], 60.00th=[ 9], | |
| | 70.00th=[ 12], 80.00th=[ 16], 90.00th=[ 23], 95.00th=[ 32], | |
| | 99.00th=[ 50], 99.50th=[ 58], 99.90th=[ 81], 99.95th=[ 91], | |
| | 99.99th=[ 114] | |
| bw ( MiB/s): min= 1473, max= 5189, per=99.91%, avg=2853.13, stdev=47.15, samples=3825 | |
| iops : min=23568, max=83023, avg=45648.03, stdev=754.46, samples=3825 | |
| write: IOPS=45.7k, BW=2858MiB/s (2997MB/s)(335GiB/120002msec); 0 zone resets | |
| slat (usec): min=16, max=106214, avg=196.16, stdev=860.43 | |
| clat (usec): min=2, max=239676, avg=11026.26, stdev=9784.02 | |
| lat (usec): min=40, max=239727, avg=11222.84, stdev=9889.98 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 5], 10.00th=[ 5], 20.00th=[ 6], | |
| | 30.00th=[ 6], 40.00th=[ 7], 50.00th=[ 7], 60.00th=[ 9], | |
| | 70.00th=[ 12], 80.00th=[ 16], 90.00th=[ 23], 95.00th=[ 31], | |
| | 99.00th=[ 50], 99.50th=[ 59], 99.90th=[ 81], 99.95th=[ 91], | |
| | 99.99th=[ 114] | |
| bw ( MiB/s): min= 1487, max= 5203, per=99.91%, avg=2855.02, stdev=47.07, samples=3825 | |
| iops : min=23806, max=83259, avg=45678.34, stdev=753.08, samples=3825 | |
| lat (usec) : 4=0.01%, 10=0.01%, 50=0.01%, 100=0.01%, 250=0.01% | |
| lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=0.01%, 4=3.68%, 10=62.81%, 20=20.38%, 50=12.16% | |
| lat (msec) : 100=0.95%, 250=0.03% | |
| cpu : usr=3.22%, sys=50.18%, ctx=1155714, majf=0, minf=489903 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=5483144,5486905,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=2856MiB/s (2994MB/s), 2856MiB/s-2856MiB/s (2994MB/s-2994MB/s), io=335GiB (359GB), run=120002-120002msec | |
| WRITE: bw=2858MiB/s (2997MB/s), 2858MiB/s-2858MiB/s (2997MB/s-2997MB/s), io=335GiB (360GB), run=120002-120002msec | |
| + rm -rf /srv/z/none/test-none-64kchunk-16thread.0.0 /srv/z/none/test-none-64kchunk-16thread.0.1 /srv/z/none/test-none-64kchunk-16thread.0.10 /srv/z/none/test-none-64kchunk-16thread.0.11 /srv/z/none/test-none-64kchunk-16thread.0.12 /srv/z/none/test-none-64kchunk-16thread.0.13 /srv/z/none/test-none-64kchunk-16thread.0.14 /srv/z/none/test-none-64kchunk-16thread.0.15 /srv/z/none/test-none-64kchunk-16thread.0.2 /srv/z/none/test-none-64kchunk-16thread.0.3 /srv/z/none/test-none-64kchunk-16thread.0.4 /srv/z/none/test-none-64kchunk-16thread.0.5 /srv/z/none/test-none-64kchunk-16thread.0.6 /srv/z/none/test-none-64kchunk-16thread.0.7 /srv/z/none/test-none-64kchunk-16thread.0.8 /srv/z/none/test-none-64kchunk-16thread.0.9 /srv/z/none/test-none-64kchunk-16thread.1.0 /srv/z/none/test-none-64kchunk-16thread.10.0 /srv/z/none/test-none-64kchunk-16thread.10.1 /srv/z/none/test-none-64kchunk-16thread.10.10 /srv/z/none/test-none-64kchunk-16thread.10.11 /srv/z/none/test-none-64kchunk-16thread.10.12 /srv/z/none/test-none-64kchunk-16thread.10.13 /srv/z/none/test-none-64kchunk-16thread.10.14 /srv/z/none/test-none-64kchunk-16thread.10.15 /srv/z/none/test-none-64kchunk-16thread.10.2 /srv/z/none/test-none-64kchunk-16thread.10.3 /srv/z/none/test-none-64kchunk-16thread.10.4 /srv/z/none/test-none-64kchunk-16thread.10.5 /srv/z/none/test-none-64kchunk-16thread.10.6 /srv/z/none/test-none-64kchunk-16thread.10.7 /srv/z/none/test-none-64kchunk-16thread.10.8 /srv/z/none/test-none-64kchunk-16thread.10.9 /srv/z/none/test-none-64kchunk-16thread.1.1 /srv/z/none/test-none-64kchunk-16thread.1.10 /srv/z/none/test-none-64kchunk-16thread.11.0 /srv/z/none/test-none-64kchunk-16thread.1.11 /srv/z/none/test-none-64kchunk-16thread.11.1 /srv/z/none/test-none-64kchunk-16thread.11.10 /srv/z/none/test-none-64kchunk-16thread.11.11 /srv/z/none/test-none-64kchunk-16thread.11.12 /srv/z/none/test-none-64kchunk-16thread.11.13 /srv/z/none/test-none-64kchunk-16thread.11.14 /srv/z/none/test-none-64kchunk-16thread.11.15 /srv/z/none/test-none-64kchunk-16thread.1.12 /srv/z/none/test-none-64kchunk-16thread.11.2 /srv/z/none/test-none-64kchunk-16thread.1.13 /srv/z/none/test-none-64kchunk-16thread.11.3 /srv/z/none/test-none-64kchunk-16thread.1.14 /srv/z/none/test-none-64kchunk-16thread.11.4 /srv/z/none/test-none-64kchunk-16thread.1.15 /srv/z/none/test-none-64kchunk-16thread.11.5 /srv/z/none/test-none-64kchunk-16thread.11.6 /srv/z/none/test-none-64kchunk-16thread.11.7 /srv/z/none/test-none-64kchunk-16thread.11.8 /srv/z/none/test-none-64kchunk-16thread.11.9 /srv/z/none/test-none-64kchunk-16thread.1.2 /srv/z/none/test-none-64kchunk-16thread.12.0 /srv/z/none/test-none-64kchunk-16thread.12.1 /srv/z/none/test-none-64kchunk-16thread.12.10 /srv/z/none/test-none-64kchunk-16thread.12.11 /srv/z/none/test-none-64kchunk-16thread.12.12 /srv/z/none/test-none-64kchunk-16thread.12.13 /srv/z/none/test-none-64kchunk-16thread.12.14 /srv/z/none/test-none-64kchunk-16thread.12.15 /srv/z/none/test-none-64kchunk-16thread.12.2 /srv/z/none/test-none-64kchunk-16thread.12.3 /srv/z/none/test-none-64kchunk-16thread.12.4 /srv/z/none/test-none-64kchunk-16thread.12.5 /srv/z/none/test-none-64kchunk-16thread.12.6 /srv/z/none/test-none-64kchunk-16thread.12.7 /srv/z/none/test-none-64kchunk-16thread.12.8 /srv/z/none/test-none-64kchunk-16thread.12.9 /srv/z/none/test-none-64kchunk-16thread.1.3 /srv/z/none/test-none-64kchunk-16thread.13.0 /srv/z/none/test-none-64kchunk-16thread.13.1 /srv/z/none/test-none-64kchunk-16thread.13.10 /srv/z/none/test-none-64kchunk-16thread.13.11 /srv/z/none/test-none-64kchunk-16thread.13.12 /srv/z/none/test-none-64kchunk-16thread.13.13 /srv/z/none/test-none-64kchunk-16thread.13.14 /srv/z/none/test-none-64kchunk-16thread.13.15 /srv/z/none/test-none-64kchunk-16thread.13.2 /srv/z/none/test-none-64kchunk-16thread.13.3 /srv/z/none/test-none-64kchunk-16thread.13.4 /srv/z/none/test-none-64kchunk-16thread.13.5 /srv/z/none/test-none-64kchunk-16thread.13.6 /srv/z/none/test-none-64kchunk-16thread.13.7 /srv/z/none/test-none-64kchunk-16thread.13.8 /srv/z/none/test-none-64kchunk-16thread.13.9 /srv/z/none/test-none-64kchunk-16thread.1.4 /srv/z/none/test-none-64kchunk-16thread.14.0 /srv/z/none/test-none-64kchunk-16thread.14.1 /srv/z/none/test-none-64kchunk-16thread.14.10 /srv/z/none/test-none-64kchunk-16thread.14.11 /srv/z/none/test-none-64kchunk-16thread.14.12 /srv/z/none/test-none-64kchunk-16thread.14.13 /srv/z/none/test-none-64kchunk-16thread.14.14 /srv/z/none/test-none-64kchunk-16thread.14.15 /srv/z/none/test-none-64kchunk-16thread.14.2 /srv/z/none/test-none-64kchunk-16thread.14.3 /srv/z/none/test-none-64kchunk-16thread.14.4 /srv/z/none/test-none-64kchunk-16thread.14.5 /srv/z/none/test-none-64kchunk-16thread.14.6 /srv/z/none/test-none-64kchunk-16thread.14.7 /srv/z/none/test-none-64kchunk-16thread.14.8 /srv/z/none/test-none-64kchunk-16thread.14.9 /srv/z/none/test-none-64kchunk-16thread.1.5 /srv/z/none/test-none-64kchunk-16thread.15.0 /srv/z/none/test-none-64kchunk-16thread.15.1 /srv/z/none/test-none-64kchunk-16thread.15.10 /srv/z/none/test-none-64kchunk-16thread.15.11 /srv/z/none/test-none-64kchunk-16thread.15.12 /srv/z/none/test-none-64kchunk-16thread.15.13 /srv/z/none/test-none-64kchunk-16thread.15.14 /srv/z/none/test-none-64kchunk-16thread.15.15 /srv/z/none/test-none-64kchunk-16thread.15.2 /srv/z/none/test-none-64kchunk-16thread.15.3 /srv/z/none/test-none-64kchunk-16thread.15.4 /srv/z/none/test-none-64kchunk-16thread.15.5 /srv/z/none/test-none-64kchunk-16thread.15.6 /srv/z/none/test-none-64kchunk-16thread.15.7 /srv/z/none/test-none-64kchunk-16thread.15.8 /srv/z/none/test-none-64kchunk-16thread.15.9 /srv/z/none/test-none-64kchunk-16thread.1.6 /srv/z/none/test-none-64kchunk-16thread.1.7 /srv/z/none/test-none-64kchunk-16thread.1.8 /srv/z/none/test-none-64kchunk-16thread.1.9 /srv/z/none/test-none-64kchunk-16thread.2.0 /srv/z/none/test-none-64kchunk-16thread.2.1 /srv/z/none/test-none-64kchunk-16thread.2.10 /srv/z/none/test-none-64kchunk-16thread.2.11 /srv/z/none/test-none-64kchunk-16thread.2.12 /srv/z/none/test-none-64kchunk-16thread.2.13 /srv/z/none/test-none-64kchunk-16thread.2.14 /srv/z/none/test-none-64kchunk-16thread.2.15 /srv/z/none/test-none-64kchunk-16thread.2.2 /srv/z/none/test-none-64kchunk-16thread.2.3 /srv/z/none/test-none-64kchunk-16thread.2.4 /srv/z/none/test-none-64kchunk-16thread.2.5 /srv/z/none/test-none-64kchunk-16thread.2.6 /srv/z/none/test-none-64kchunk-16thread.2.7 /srv/z/none/test-none-64kchunk-16thread.2.8 /srv/z/none/test-none-64kchunk-16thread.2.9 /srv/z/none/test-none-64kchunk-16thread.3.0 /srv/z/none/test-none-64kchunk-16thread.3.1 /srv/z/none/test-none-64kchunk-16thread.3.10 /srv/z/none/test-none-64kchunk-16thread.3.11 /srv/z/none/test-none-64kchunk-16thread.3.12 /srv/z/none/test-none-64kchunk-16thread.3.13 /srv/z/none/test-none-64kchunk-16thread.3.14 /srv/z/none/test-none-64kchunk-16thread.3.15 /srv/z/none/test-none-64kchunk-16thread.3.2 /srv/z/none/test-none-64kchunk-16thread.3.3 /srv/z/none/test-none-64kchunk-16thread.3.4 /srv/z/none/test-none-64kchunk-16thread.3.5 /srv/z/none/test-none-64kchunk-16thread.3.6 /srv/z/none/test-none-64kchunk-16thread.3.7 /srv/z/none/test-none-64kchunk-16thread.3.8 /srv/z/none/test-none-64kchunk-16thread.3.9 /srv/z/none/test-none-64kchunk-16thread.4.0 /srv/z/none/test-none-64kchunk-16thread.4.1 /srv/z/none/test-none-64kchunk-16thread.4.10 /srv/z/none/test-none-64kchunk-16thread.4.11 /srv/z/none/test-none-64kchunk-16thread.4.12 /srv/z/none/test-none-64kchunk-16thread.4.13 /srv/z/none/test-none-64kchunk-16thread.4.14 /srv/z/none/test-none-64kchunk-16thread.4.15 /srv/z/none/test-none-64kchunk-16thread.4.2 /srv/z/none/test-none-64kchunk-16thread.4.3 /srv/z/none/test-none-64kchunk-16thread.4.4 /srv/z/none/test-none-64kchunk-16thread.4.5 /srv/z/none/test-none-64kchunk-16thread.4.6 /srv/z/none/test-none-64kchunk-16thread.4.7 /srv/z/none/test-none-64kchunk-16thread.4.8 /srv/z/none/test-none-64kchunk-16thread.4.9 /srv/z/none/test-none-64kchunk-16thread.5.0 /srv/z/none/test-none-64kchunk-16thread.5.1 /srv/z/none/test-none-64kchunk-16thread.5.10 /srv/z/none/test-none-64kchunk-16thread.5.11 /srv/z/none/test-none-64kchunk-16thread.5.12 /srv/z/none/test-none-64kchunk-16thread.5.13 /srv/z/none/test-none-64kchunk-16thread.5.14 /srv/z/none/test-none-64kchunk-16thread.5.15 /srv/z/none/test-none-64kchunk-16thread.5.2 /srv/z/none/test-none-64kchunk-16thread.5.3 /srv/z/none/test-none-64kchunk-16thread.5.4 /srv/z/none/test-none-64kchunk-16thread.5.5 /srv/z/none/test-none-64kchunk-16thread.5.6 /srv/z/none/test-none-64kchunk-16thread.5.7 /srv/z/none/test-none-64kchunk-16thread.5.8 /srv/z/none/test-none-64kchunk-16thread.5.9 /srv/z/none/test-none-64kchunk-16thread.6.0 /srv/z/none/test-none-64kchunk-16thread.6.1 /srv/z/none/test-none-64kchunk-16thread.6.10 /srv/z/none/test-none-64kchunk-16thread.6.11 /srv/z/none/test-none-64kchunk-16thread.6.12 /srv/z/none/test-none-64kchunk-16thread.6.13 /srv/z/none/test-none-64kchunk-16thread.6.14 /srv/z/none/test-none-64kchunk-16thread.6.15 /srv/z/none/test-none-64kchunk-16thread.6.2 /srv/z/none/test-none-64kchunk-16thread.6.3 /srv/z/none/test-none-64kchunk-16thread.6.4 /srv/z/none/test-none-64kchunk-16thread.6.5 /srv/z/none/test-none-64kchunk-16thread.6.6 /srv/z/none/test-none-64kchunk-16thread.6.7 /srv/z/none/test-none-64kchunk-16thread.6.8 /srv/z/none/test-none-64kchunk-16thread.6.9 /srv/z/none/test-none-64kchunk-16thread.7.0 /srv/z/none/test-none-64kchunk-16thread.7.1 /srv/z/none/test-none-64kchunk-16thread.7.10 /srv/z/none/test-none-64kchunk-16thread.7.11 /srv/z/none/test-none-64kchunk-16thread.7.12 /srv/z/none/test-none-64kchunk-16thread.7.13 /srv/z/none/test-none-64kchunk-16thread.7.14 /srv/z/none/test-none-64kchunk-16thread.7.15 /srv/z/none/test-none-64kchunk-16thread.7.2 /srv/z/none/test-none-64kchunk-16thread.7.3 /srv/z/none/test-none-64kchunk-16thread.7.4 /srv/z/none/test-none-64kchunk-16thread.7.5 /srv/z/none/test-none-64kchunk-16thread.7.6 /srv/z/none/test-none-64kchunk-16thread.7.7 /srv/z/none/test-none-64kchunk-16thread.7.8 /srv/z/none/test-none-64kchunk-16thread.7.9 /srv/z/none/test-none-64kchunk-16thread.8.0 /srv/z/none/test-none-64kchunk-16thread.8.1 /srv/z/none/test-none-64kchunk-16thread.8.10 /srv/z/none/test-none-64kchunk-16thread.8.11 /srv/z/none/test-none-64kchunk-16thread.8.12 /srv/z/none/test-none-64kchunk-16thread.8.13 /srv/z/none/test-none-64kchunk-16thread.8.14 /srv/z/none/test-none-64kchunk-16thread.8.15 /srv/z/none/test-none-64kchunk-16thread.8.2 /srv/z/none/test-none-64kchunk-16thread.8.3 /srv/z/none/test-none-64kchunk-16thread.8.4 /srv/z/none/test-none-64kchunk-16thread.8.5 /srv/z/none/test-none-64kchunk-16thread.8.6 /srv/z/none/test-none-64kchunk-16thread.8.7 /srv/z/none/test-none-64kchunk-16thread.8.8 /srv/z/none/test-none-64kchunk-16thread.8.9 /srv/z/none/test-none-64kchunk-16thread.9.0 /srv/z/none/test-none-64kchunk-16thread.9.1 /srv/z/none/test-none-64kchunk-16thread.9.10 /srv/z/none/test-none-64kchunk-16thread.9.11 /srv/z/none/test-none-64kchunk-16thread.9.12 /srv/z/none/test-none-64kchunk-16thread.9.13 /srv/z/none/test-none-64kchunk-16thread.9.14 /srv/z/none/test-none-64kchunk-16thread.9.15 /srv/z/none/test-none-64kchunk-16thread.9.2 /srv/z/none/test-none-64kchunk-16thread.9.3 /srv/z/none/test-none-64kchunk-16thread.9.4 /srv/z/none/test-none-64kchunk-16thread.9.5 /srv/z/none/test-none-64kchunk-16thread.9.6 /srv/z/none/test-none-64kchunk-16thread.9.7 /srv/z/none/test-none-64kchunk-16thread.9.8 /srv/z/none/test-none-64kchunk-16thread.9.9 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-comp-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/comp/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-comp-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=2674MiB/s,w=2675MiB/s][r=42.8k,w=42.8k IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.7%][r=2334MiB/s,w=2338MiB/s][r=37.3k,w=37.4k IOPS][eta 01m:40s] Jobs: 16 (f=256): [M(16)][24.2%][r=2325MiB/s,w=2335MiB/s][r=37.2k,w=37.4k IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.8%][r=2427MiB/s,w=2428MiB/s][r=38.8k,w=38.8k IOPS][eta 01m:20s] Jobs: 16 (f=256): [M(16)][40.5%][r=2523MiB/s,w=2522MiB/s][r=40.4k,w=40.4k IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][48.3%][r=2581MiB/s,w=2585MiB/s][r=41.3k,w=41.4k IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][56.7%][r=2539MiB/s,w=2533MiB/s][r=40.6k,w=40.5k IOPS][eta 00m:52s] | |
| Jobs: 16 (f=256): [M(16)][64.7%][r=2427MiB/s,w=2428MiB/s][r=38.8k,w=38.9k IOPS][eta 00m:42s] Jobs: 16 (f=256): [M(16)][72.5%][r=2383MiB/s,w=2379MiB/s][r=38.1k,w=38.1k IOPS][eta 00m:33s] | |
| Jobs: 16 (f=256): [M(16)][80.2%][r=2392MiB/s,w=2400MiB/s][r=38.3k,w=38.4k IOPS][eta 00m:24s] Jobs: 16 (f=256): [M(16)][88.3%][r=2386MiB/s,w=2387MiB/s][r=38.2k,w=38.2k IOPS][eta 00m:14s] | |
| Jobs: 16 (f=256): [M(16)][96.7%][r=2357MiB/s,w=2363MiB/s][r=37.7k,w=37.8k IOPS][eta 00m:04s] | |
| test-comp-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=3252014: Wed Feb 26 10:57:26 2020 | |
| read: IOPS=39.1k, BW=2446MiB/s (2565MB/s)(287GiB/120005msec) | |
| slat (usec): min=10, max=165527, avg=179.95, stdev=980.25 | |
| clat (usec): min=2, max=231969, avg=12871.59, stdev=12054.07 | |
| lat (usec): min=36, max=232103, avg=13052.07, stdev=12204.57 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 5], 10.00th=[ 5], 20.00th=[ 6], | |
| | 30.00th=[ 7], 40.00th=[ 8], 50.00th=[ 9], 60.00th=[ 11], | |
| | 70.00th=[ 13], 80.00th=[ 18], 90.00th=[ 27], 95.00th=[ 37], | |
| | 99.00th=[ 62], 99.50th=[ 73], 99.90th=[ 104], 99.95th=[ 118], | |
| | 99.99th=[ 163] | |
| bw ( MiB/s): min= 1143, max= 4581, per=99.72%, avg=2439.09, stdev=40.04, samples=3833 | |
| iops : min=18298, max=73308, avg=39023.22, stdev=640.56, samples=3833 | |
| write: IOPS=39.2k, BW=2448MiB/s (2567MB/s)(287GiB/120005msec); 0 zone resets | |
| slat (usec): min=18, max=160608, avg=218.70, stdev=1098.63 | |
| clat (usec): min=3, max=232072, avg=12873.90, stdev=12039.28 | |
| lat (usec): min=106, max=232155, avg=13093.16, stdev=12162.43 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 5], 10.00th=[ 5], 20.00th=[ 6], | |
| | 30.00th=[ 7], 40.00th=[ 8], 50.00th=[ 9], 60.00th=[ 11], | |
| | 70.00th=[ 13], 80.00th=[ 18], 90.00th=[ 27], 95.00th=[ 37], | |
| | 99.00th=[ 62], 99.50th=[ 73], 99.90th=[ 105], 99.95th=[ 118], | |
| | 99.99th=[ 163] | |
| bw ( MiB/s): min= 1147, max= 4590, per=99.72%, avg=2441.09, stdev=40.05, samples=3833 | |
| iops : min=18356, max=73447, avg=39055.25, stdev=640.81, samples=3833 | |
| lat (usec) : 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 250=0.01% | |
| lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=0.01%, 4=2.89%, 10=56.65%, 20=23.83%, 50=14.56% | |
| lat (msec) : 100=1.93%, 250=0.12% | |
| cpu : usr=3.58%, sys=46.86%, ctx=1338223, majf=0, minf=438653 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=4696183,4700026,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=2446MiB/s (2565MB/s), 2446MiB/s-2446MiB/s (2565MB/s-2565MB/s), io=287GiB (308GB), run=120005-120005msec | |
| WRITE: bw=2448MiB/s (2567MB/s), 2448MiB/s-2448MiB/s (2567MB/s-2567MB/s), io=287GiB (308GB), run=120005-120005msec | |
| + rm -rf /srv/z/comp/test-comp-64kchunk-16thread.0.0 /srv/z/comp/test-comp-64kchunk-16thread.0.1 /srv/z/comp/test-comp-64kchunk-16thread.0.10 /srv/z/comp/test-comp-64kchunk-16thread.0.11 /srv/z/comp/test-comp-64kchunk-16thread.0.12 /srv/z/comp/test-comp-64kchunk-16thread.0.13 /srv/z/comp/test-comp-64kchunk-16thread.0.14 /srv/z/comp/test-comp-64kchunk-16thread.0.15 /srv/z/comp/test-comp-64kchunk-16thread.0.2 /srv/z/comp/test-comp-64kchunk-16thread.0.3 /srv/z/comp/test-comp-64kchunk-16thread.0.4 /srv/z/comp/test-comp-64kchunk-16thread.0.5 /srv/z/comp/test-comp-64kchunk-16thread.0.6 /srv/z/comp/test-comp-64kchunk-16thread.0.7 /srv/z/comp/test-comp-64kchunk-16thread.0.8 /srv/z/comp/test-comp-64kchunk-16thread.0.9 /srv/z/comp/test-comp-64kchunk-16thread.1.0 /srv/z/comp/test-comp-64kchunk-16thread.10.0 /srv/z/comp/test-comp-64kchunk-16thread.10.1 /srv/z/comp/test-comp-64kchunk-16thread.10.10 /srv/z/comp/test-comp-64kchunk-16thread.10.11 /srv/z/comp/test-comp-64kchunk-16thread.10.12 /srv/z/comp/test-comp-64kchunk-16thread.10.13 /srv/z/comp/test-comp-64kchunk-16thread.10.14 /srv/z/comp/test-comp-64kchunk-16thread.10.15 /srv/z/comp/test-comp-64kchunk-16thread.10.2 /srv/z/comp/test-comp-64kchunk-16thread.10.3 /srv/z/comp/test-comp-64kchunk-16thread.10.4 /srv/z/comp/test-comp-64kchunk-16thread.10.5 /srv/z/comp/test-comp-64kchunk-16thread.10.6 /srv/z/comp/test-comp-64kchunk-16thread.10.7 /srv/z/comp/test-comp-64kchunk-16thread.10.8 /srv/z/comp/test-comp-64kchunk-16thread.10.9 /srv/z/comp/test-comp-64kchunk-16thread.1.1 /srv/z/comp/test-comp-64kchunk-16thread.1.10 /srv/z/comp/test-comp-64kchunk-16thread.11.0 /srv/z/comp/test-comp-64kchunk-16thread.1.11 /srv/z/comp/test-comp-64kchunk-16thread.11.1 /srv/z/comp/test-comp-64kchunk-16thread.11.10 /srv/z/comp/test-comp-64kchunk-16thread.11.11 /srv/z/comp/test-comp-64kchunk-16thread.11.12 /srv/z/comp/test-comp-64kchunk-16thread.11.13 /srv/z/comp/test-comp-64kchunk-16thread.11.14 /srv/z/comp/test-comp-64kchunk-16thread.11.15 /srv/z/comp/test-comp-64kchunk-16thread.1.12 /srv/z/comp/test-comp-64kchunk-16thread.11.2 /srv/z/comp/test-comp-64kchunk-16thread.1.13 /srv/z/comp/test-comp-64kchunk-16thread.11.3 /srv/z/comp/test-comp-64kchunk-16thread.1.14 /srv/z/comp/test-comp-64kchunk-16thread.11.4 /srv/z/comp/test-comp-64kchunk-16thread.1.15 /srv/z/comp/test-comp-64kchunk-16thread.11.5 /srv/z/comp/test-comp-64kchunk-16thread.11.6 /srv/z/comp/test-comp-64kchunk-16thread.11.7 /srv/z/comp/test-comp-64kchunk-16thread.11.8 /srv/z/comp/test-comp-64kchunk-16thread.11.9 /srv/z/comp/test-comp-64kchunk-16thread.1.2 /srv/z/comp/test-comp-64kchunk-16thread.12.0 /srv/z/comp/test-comp-64kchunk-16thread.12.1 /srv/z/comp/test-comp-64kchunk-16thread.12.10 /srv/z/comp/test-comp-64kchunk-16thread.12.11 /srv/z/comp/test-comp-64kchunk-16thread.12.12 /srv/z/comp/test-comp-64kchunk-16thread.12.13 /srv/z/comp/test-comp-64kchunk-16thread.12.14 /srv/z/comp/test-comp-64kchunk-16thread.12.15 /srv/z/comp/test-comp-64kchunk-16thread.12.2 /srv/z/comp/test-comp-64kchunk-16thread.12.3 /srv/z/comp/test-comp-64kchunk-16thread.12.4 /srv/z/comp/test-comp-64kchunk-16thread.12.5 /srv/z/comp/test-comp-64kchunk-16thread.12.6 /srv/z/comp/test-comp-64kchunk-16thread.12.7 /srv/z/comp/test-comp-64kchunk-16thread.12.8 /srv/z/comp/test-comp-64kchunk-16thread.12.9 /srv/z/comp/test-comp-64kchunk-16thread.1.3 /srv/z/comp/test-comp-64kchunk-16thread.13.0 /srv/z/comp/test-comp-64kchunk-16thread.13.1 /srv/z/comp/test-comp-64kchunk-16thread.13.10 /srv/z/comp/test-comp-64kchunk-16thread.13.11 /srv/z/comp/test-comp-64kchunk-16thread.13.12 /srv/z/comp/test-comp-64kchunk-16thread.13.13 /srv/z/comp/test-comp-64kchunk-16thread.13.14 /srv/z/comp/test-comp-64kchunk-16thread.13.15 /srv/z/comp/test-comp-64kchunk-16thread.13.2 /srv/z/comp/test-comp-64kchunk-16thread.13.3 /srv/z/comp/test-comp-64kchunk-16thread.13.4 /srv/z/comp/test-comp-64kchunk-16thread.13.5 /srv/z/comp/test-comp-64kchunk-16thread.13.6 /srv/z/comp/test-comp-64kchunk-16thread.13.7 /srv/z/comp/test-comp-64kchunk-16thread.13.8 /srv/z/comp/test-comp-64kchunk-16thread.13.9 /srv/z/comp/test-comp-64kchunk-16thread.1.4 /srv/z/comp/test-comp-64kchunk-16thread.14.0 /srv/z/comp/test-comp-64kchunk-16thread.14.1 /srv/z/comp/test-comp-64kchunk-16thread.14.10 /srv/z/comp/test-comp-64kchunk-16thread.14.11 /srv/z/comp/test-comp-64kchunk-16thread.14.12 /srv/z/comp/test-comp-64kchunk-16thread.14.13 /srv/z/comp/test-comp-64kchunk-16thread.14.14 /srv/z/comp/test-comp-64kchunk-16thread.14.15 /srv/z/comp/test-comp-64kchunk-16thread.14.2 /srv/z/comp/test-comp-64kchunk-16thread.14.3 /srv/z/comp/test-comp-64kchunk-16thread.14.4 /srv/z/comp/test-comp-64kchunk-16thread.14.5 /srv/z/comp/test-comp-64kchunk-16thread.14.6 /srv/z/comp/test-comp-64kchunk-16thread.14.7 /srv/z/comp/test-comp-64kchunk-16thread.14.8 /srv/z/comp/test-comp-64kchunk-16thread.14.9 /srv/z/comp/test-comp-64kchunk-16thread.1.5 /srv/z/comp/test-comp-64kchunk-16thread.15.0 /srv/z/comp/test-comp-64kchunk-16thread.15.1 /srv/z/comp/test-comp-64kchunk-16thread.15.10 /srv/z/comp/test-comp-64kchunk-16thread.15.11 /srv/z/comp/test-comp-64kchunk-16thread.15.12 /srv/z/comp/test-comp-64kchunk-16thread.15.13 /srv/z/comp/test-comp-64kchunk-16thread.15.14 /srv/z/comp/test-comp-64kchunk-16thread.15.15 /srv/z/comp/test-comp-64kchunk-16thread.15.2 /srv/z/comp/test-comp-64kchunk-16thread.15.3 /srv/z/comp/test-comp-64kchunk-16thread.15.4 /srv/z/comp/test-comp-64kchunk-16thread.15.5 /srv/z/comp/test-comp-64kchunk-16thread.15.6 /srv/z/comp/test-comp-64kchunk-16thread.15.7 /srv/z/comp/test-comp-64kchunk-16thread.15.8 /srv/z/comp/test-comp-64kchunk-16thread.15.9 /srv/z/comp/test-comp-64kchunk-16thread.1.6 /srv/z/comp/test-comp-64kchunk-16thread.1.7 /srv/z/comp/test-comp-64kchunk-16thread.1.8 /srv/z/comp/test-comp-64kchunk-16thread.1.9 /srv/z/comp/test-comp-64kchunk-16thread.2.0 /srv/z/comp/test-comp-64kchunk-16thread.2.1 /srv/z/comp/test-comp-64kchunk-16thread.2.10 /srv/z/comp/test-comp-64kchunk-16thread.2.11 /srv/z/comp/test-comp-64kchunk-16thread.2.12 /srv/z/comp/test-comp-64kchunk-16thread.2.13 /srv/z/comp/test-comp-64kchunk-16thread.2.14 /srv/z/comp/test-comp-64kchunk-16thread.2.15 /srv/z/comp/test-comp-64kchunk-16thread.2.2 /srv/z/comp/test-comp-64kchunk-16thread.2.3 /srv/z/comp/test-comp-64kchunk-16thread.2.4 /srv/z/comp/test-comp-64kchunk-16thread.2.5 /srv/z/comp/test-comp-64kchunk-16thread.2.6 /srv/z/comp/test-comp-64kchunk-16thread.2.7 /srv/z/comp/test-comp-64kchunk-16thread.2.8 /srv/z/comp/test-comp-64kchunk-16thread.2.9 /srv/z/comp/test-comp-64kchunk-16thread.3.0 /srv/z/comp/test-comp-64kchunk-16thread.3.1 /srv/z/comp/test-comp-64kchunk-16thread.3.10 /srv/z/comp/test-comp-64kchunk-16thread.3.11 /srv/z/comp/test-comp-64kchunk-16thread.3.12 /srv/z/comp/test-comp-64kchunk-16thread.3.13 /srv/z/comp/test-comp-64kchunk-16thread.3.14 /srv/z/comp/test-comp-64kchunk-16thread.3.15 /srv/z/comp/test-comp-64kchunk-16thread.3.2 /srv/z/comp/test-comp-64kchunk-16thread.3.3 /srv/z/comp/test-comp-64kchunk-16thread.3.4 /srv/z/comp/test-comp-64kchunk-16thread.3.5 /srv/z/comp/test-comp-64kchunk-16thread.3.6 /srv/z/comp/test-comp-64kchunk-16thread.3.7 /srv/z/comp/test-comp-64kchunk-16thread.3.8 /srv/z/comp/test-comp-64kchunk-16thread.3.9 /srv/z/comp/test-comp-64kchunk-16thread.4.0 /srv/z/comp/test-comp-64kchunk-16thread.4.1 /srv/z/comp/test-comp-64kchunk-16thread.4.10 /srv/z/comp/test-comp-64kchunk-16thread.4.11 /srv/z/comp/test-comp-64kchunk-16thread.4.12 /srv/z/comp/test-comp-64kchunk-16thread.4.13 /srv/z/comp/test-comp-64kchunk-16thread.4.14 /srv/z/comp/test-comp-64kchunk-16thread.4.15 /srv/z/comp/test-comp-64kchunk-16thread.4.2 /srv/z/comp/test-comp-64kchunk-16thread.4.3 /srv/z/comp/test-comp-64kchunk-16thread.4.4 /srv/z/comp/test-comp-64kchunk-16thread.4.5 /srv/z/comp/test-comp-64kchunk-16thread.4.6 /srv/z/comp/test-comp-64kchunk-16thread.4.7 /srv/z/comp/test-comp-64kchunk-16thread.4.8 /srv/z/comp/test-comp-64kchunk-16thread.4.9 /srv/z/comp/test-comp-64kchunk-16thread.5.0 /srv/z/comp/test-comp-64kchunk-16thread.5.1 /srv/z/comp/test-comp-64kchunk-16thread.5.10 /srv/z/comp/test-comp-64kchunk-16thread.5.11 /srv/z/comp/test-comp-64kchunk-16thread.5.12 /srv/z/comp/test-comp-64kchunk-16thread.5.13 /srv/z/comp/test-comp-64kchunk-16thread.5.14 /srv/z/comp/test-comp-64kchunk-16thread.5.15 /srv/z/comp/test-comp-64kchunk-16thread.5.2 /srv/z/comp/test-comp-64kchunk-16thread.5.3 /srv/z/comp/test-comp-64kchunk-16thread.5.4 /srv/z/comp/test-comp-64kchunk-16thread.5.5 /srv/z/comp/test-comp-64kchunk-16thread.5.6 /srv/z/comp/test-comp-64kchunk-16thread.5.7 /srv/z/comp/test-comp-64kchunk-16thread.5.8 /srv/z/comp/test-comp-64kchunk-16thread.5.9 /srv/z/comp/test-comp-64kchunk-16thread.6.0 /srv/z/comp/test-comp-64kchunk-16thread.6.1 /srv/z/comp/test-comp-64kchunk-16thread.6.10 /srv/z/comp/test-comp-64kchunk-16thread.6.11 /srv/z/comp/test-comp-64kchunk-16thread.6.12 /srv/z/comp/test-comp-64kchunk-16thread.6.13 /srv/z/comp/test-comp-64kchunk-16thread.6.14 /srv/z/comp/test-comp-64kchunk-16thread.6.15 /srv/z/comp/test-comp-64kchunk-16thread.6.2 /srv/z/comp/test-comp-64kchunk-16thread.6.3 /srv/z/comp/test-comp-64kchunk-16thread.6.4 /srv/z/comp/test-comp-64kchunk-16thread.6.5 /srv/z/comp/test-comp-64kchunk-16thread.6.6 /srv/z/comp/test-comp-64kchunk-16thread.6.7 /srv/z/comp/test-comp-64kchunk-16thread.6.8 /srv/z/comp/test-comp-64kchunk-16thread.6.9 /srv/z/comp/test-comp-64kchunk-16thread.7.0 /srv/z/comp/test-comp-64kchunk-16thread.7.1 /srv/z/comp/test-comp-64kchunk-16thread.7.10 /srv/z/comp/test-comp-64kchunk-16thread.7.11 /srv/z/comp/test-comp-64kchunk-16thread.7.12 /srv/z/comp/test-comp-64kchunk-16thread.7.13 /srv/z/comp/test-comp-64kchunk-16thread.7.14 /srv/z/comp/test-comp-64kchunk-16thread.7.15 /srv/z/comp/test-comp-64kchunk-16thread.7.2 /srv/z/comp/test-comp-64kchunk-16thread.7.3 /srv/z/comp/test-comp-64kchunk-16thread.7.4 /srv/z/comp/test-comp-64kchunk-16thread.7.5 /srv/z/comp/test-comp-64kchunk-16thread.7.6 /srv/z/comp/test-comp-64kchunk-16thread.7.7 /srv/z/comp/test-comp-64kchunk-16thread.7.8 /srv/z/comp/test-comp-64kchunk-16thread.7.9 /srv/z/comp/test-comp-64kchunk-16thread.8.0 /srv/z/comp/test-comp-64kchunk-16thread.8.1 /srv/z/comp/test-comp-64kchunk-16thread.8.10 /srv/z/comp/test-comp-64kchunk-16thread.8.11 /srv/z/comp/test-comp-64kchunk-16thread.8.12 /srv/z/comp/test-comp-64kchunk-16thread.8.13 /srv/z/comp/test-comp-64kchunk-16thread.8.14 /srv/z/comp/test-comp-64kchunk-16thread.8.15 /srv/z/comp/test-comp-64kchunk-16thread.8.2 /srv/z/comp/test-comp-64kchunk-16thread.8.3 /srv/z/comp/test-comp-64kchunk-16thread.8.4 /srv/z/comp/test-comp-64kchunk-16thread.8.5 /srv/z/comp/test-comp-64kchunk-16thread.8.6 /srv/z/comp/test-comp-64kchunk-16thread.8.7 /srv/z/comp/test-comp-64kchunk-16thread.8.8 /srv/z/comp/test-comp-64kchunk-16thread.8.9 /srv/z/comp/test-comp-64kchunk-16thread.9.0 /srv/z/comp/test-comp-64kchunk-16thread.9.1 /srv/z/comp/test-comp-64kchunk-16thread.9.10 /srv/z/comp/test-comp-64kchunk-16thread.9.11 /srv/z/comp/test-comp-64kchunk-16thread.9.12 /srv/z/comp/test-comp-64kchunk-16thread.9.13 /srv/z/comp/test-comp-64kchunk-16thread.9.14 /srv/z/comp/test-comp-64kchunk-16thread.9.15 /srv/z/comp/test-comp-64kchunk-16thread.9.2 /srv/z/comp/test-comp-64kchunk-16thread.9.3 /srv/z/comp/test-comp-64kchunk-16thread.9.4 /srv/z/comp/test-comp-64kchunk-16thread.9.5 /srv/z/comp/test-comp-64kchunk-16thread.9.6 /srv/z/comp/test-comp-64kchunk-16thread.9.7 /srv/z/comp/test-comp-64kchunk-16thread.9.8 /srv/z/comp/test-comp-64kchunk-16thread.9.9 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-enc-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/enc/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-enc-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=342MiB/s,w=344MiB/s][r=5475,w=5506 IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.7%][r=357MiB/s,w=359MiB/s][r=5707,w=5742 IOPS][eta 01m:40s] Jobs: 16 (f=256): [M(16)][24.2%][r=346MiB/s,w=348MiB/s][r=5541,w=5566 IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.5%][r=327MiB/s,w=326MiB/s][r=5227,w=5213 IOPS][eta 01m:21s] Jobs: 16 (f=256): [M(16)][40.8%][r=319MiB/s,w=316MiB/s][r=5111,w=5048 IOPS][eta 01m:11s] | |
| Jobs: 16 (f=256): [M(16)][48.8%][r=316MiB/s,w=315MiB/s][r=5050,w=5038 IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][56.7%][r=334MiB/s,w=336MiB/s][r=5346,w=5381 IOPS][eta 00m:52s] | |
| Jobs: 16 (f=256): [M(16)][65.0%][r=336MiB/s,w=334MiB/s][r=5371,w=5337 IOPS][eta 00m:42s] Jobs: 16 (f=256): [M(16)][72.7%][r=367MiB/s,w=367MiB/s][r=5874,w=5877 IOPS][eta 00m:33s] | |
| Jobs: 16 (f=256): [M(16)][80.8%][r=359MiB/s,w=357MiB/s][r=5744,w=5716 IOPS][eta 00m:23s] Jobs: 16 (f=256): [M(16)][89.2%][r=340MiB/s,w=343MiB/s][r=5447,w=5485 IOPS][eta 00m:13s] | |
| Jobs: 16 (f=256): [M(16)][96.7%][r=324MiB/s,w=322MiB/s][r=5178,w=5152 IOPS][eta 00m:04s] | |
| test-enc-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=1064712: Wed Feb 26 11:20:24 2020 | |
| read: IOPS=5430, BW=339MiB/s (356MB/s)(39.8GiB/120017msec) | |
| slat (usec): min=10, max=132452, avg=1471.41, stdev=3674.91 | |
| clat (usec): min=4, max=514833, avg=92700.89, stdev=52702.18 | |
| lat (msec): min=2, max=521, avg=94.17, stdev=53.41 | |
| clat percentiles (msec): | |
| | 1.00th=[ 8], 5.00th=[ 17], 10.00th=[ 26], 20.00th=[ 47], | |
| | 30.00th=[ 67], 40.00th=[ 79], 50.00th=[ 88], 60.00th=[ 97], | |
| | 70.00th=[ 109], 80.00th=[ 130], 90.00th=[ 167], 95.00th=[ 194], | |
| | 99.00th=[ 243], 99.50th=[ 259], 99.90th=[ 309], 99.95th=[ 330], | |
| | 99.99th=[ 401] | |
| bw ( KiB/s): min=121158, max=795964, per=99.80%, avg=346902.55, stdev=11251.91, samples=3840 | |
| iops : min= 1891, max=12436, avg=5418.26, stdev=175.85, samples=3840 | |
| write: IOPS=5428, BW=339MiB/s (356MB/s)(39.8GiB/120017msec); 0 zone resets | |
| slat (usec): min=15, max=158704, avg=1467.49, stdev=3724.44 | |
| clat (usec): min=6, max=514829, avg=92751.51, stdev=52689.66 | |
| lat (msec): min=2, max=514, avg=94.22, stdev=53.40 | |
| clat percentiles (msec): | |
| | 1.00th=[ 8], 5.00th=[ 17], 10.00th=[ 26], 20.00th=[ 47], | |
| | 30.00th=[ 67], 40.00th=[ 79], 50.00th=[ 88], 60.00th=[ 97], | |
| | 70.00th=[ 109], 80.00th=[ 130], 90.00th=[ 167], 95.00th=[ 194], | |
| | 99.00th=[ 243], 99.50th=[ 262], 99.90th=[ 309], 99.95th=[ 334], | |
| | 99.99th=[ 405] | |
| bw ( KiB/s): min=115650, max=805742, per=99.80%, avg=346728.60, stdev=11193.16, samples=3840 | |
| iops : min= 1803, max=12589, avg=5415.53, stdev=174.93, samples=3840 | |
| lat (usec) : 10=0.01%, 20=0.01% | |
| lat (msec) : 4=0.54%, 10=1.29%, 20=5.14%, 50=14.31%, 100=41.94% | |
| lat (msec) : 250=36.03%, 500=0.74%, 750=0.01% | |
| cpu : usr=0.29%, sys=39.24%, ctx=889783, majf=0, minf=469603 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=651810,651535,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=339MiB/s (356MB/s), 339MiB/s-339MiB/s (356MB/s-356MB/s), io=39.8GiB (42.7GB), run=120017-120017msec | |
| WRITE: bw=339MiB/s (356MB/s), 339MiB/s-339MiB/s (356MB/s-356MB/s), io=39.8GiB (42.7GB), run=120017-120017msec | |
| + rm -rf /srv/z/enc/test-enc-64kchunk-16thread.0.0 /srv/z/enc/test-enc-64kchunk-16thread.0.1 /srv/z/enc/test-enc-64kchunk-16thread.0.10 /srv/z/enc/test-enc-64kchunk-16thread.0.11 /srv/z/enc/test-enc-64kchunk-16thread.0.12 /srv/z/enc/test-enc-64kchunk-16thread.0.13 /srv/z/enc/test-enc-64kchunk-16thread.0.14 /srv/z/enc/test-enc-64kchunk-16thread.0.15 /srv/z/enc/test-enc-64kchunk-16thread.0.2 /srv/z/enc/test-enc-64kchunk-16thread.0.3 /srv/z/enc/test-enc-64kchunk-16thread.0.4 /srv/z/enc/test-enc-64kchunk-16thread.0.5 /srv/z/enc/test-enc-64kchunk-16thread.0.6 /srv/z/enc/test-enc-64kchunk-16thread.0.7 /srv/z/enc/test-enc-64kchunk-16thread.0.8 /srv/z/enc/test-enc-64kchunk-16thread.0.9 /srv/z/enc/test-enc-64kchunk-16thread.1.0 /srv/z/enc/test-enc-64kchunk-16thread.10.0 /srv/z/enc/test-enc-64kchunk-16thread.10.1 /srv/z/enc/test-enc-64kchunk-16thread.10.10 /srv/z/enc/test-enc-64kchunk-16thread.10.11 /srv/z/enc/test-enc-64kchunk-16thread.10.12 /srv/z/enc/test-enc-64kchunk-16thread.10.13 /srv/z/enc/test-enc-64kchunk-16thread.10.14 /srv/z/enc/test-enc-64kchunk-16thread.10.15 /srv/z/enc/test-enc-64kchunk-16thread.10.2 /srv/z/enc/test-enc-64kchunk-16thread.10.3 /srv/z/enc/test-enc-64kchunk-16thread.10.4 /srv/z/enc/test-enc-64kchunk-16thread.10.5 /srv/z/enc/test-enc-64kchunk-16thread.10.6 /srv/z/enc/test-enc-64kchunk-16thread.10.7 /srv/z/enc/test-enc-64kchunk-16thread.10.8 /srv/z/enc/test-enc-64kchunk-16thread.10.9 /srv/z/enc/test-enc-64kchunk-16thread.1.1 /srv/z/enc/test-enc-64kchunk-16thread.1.10 /srv/z/enc/test-enc-64kchunk-16thread.11.0 /srv/z/enc/test-enc-64kchunk-16thread.1.11 /srv/z/enc/test-enc-64kchunk-16thread.11.1 /srv/z/enc/test-enc-64kchunk-16thread.11.10 /srv/z/enc/test-enc-64kchunk-16thread.11.11 /srv/z/enc/test-enc-64kchunk-16thread.11.12 /srv/z/enc/test-enc-64kchunk-16thread.11.13 /srv/z/enc/test-enc-64kchunk-16thread.11.14 /srv/z/enc/test-enc-64kchunk-16thread.11.15 /srv/z/enc/test-enc-64kchunk-16thread.1.12 /srv/z/enc/test-enc-64kchunk-16thread.11.2 /srv/z/enc/test-enc-64kchunk-16thread.1.13 /srv/z/enc/test-enc-64kchunk-16thread.11.3 /srv/z/enc/test-enc-64kchunk-16thread.1.14 /srv/z/enc/test-enc-64kchunk-16thread.11.4 /srv/z/enc/test-enc-64kchunk-16thread.1.15 /srv/z/enc/test-enc-64kchunk-16thread.11.5 /srv/z/enc/test-enc-64kchunk-16thread.11.6 /srv/z/enc/test-enc-64kchunk-16thread.11.7 /srv/z/enc/test-enc-64kchunk-16thread.11.8 /srv/z/enc/test-enc-64kchunk-16thread.11.9 /srv/z/enc/test-enc-64kchunk-16thread.1.2 /srv/z/enc/test-enc-64kchunk-16thread.12.0 /srv/z/enc/test-enc-64kchunk-16thread.12.1 /srv/z/enc/test-enc-64kchunk-16thread.12.10 /srv/z/enc/test-enc-64kchunk-16thread.12.11 /srv/z/enc/test-enc-64kchunk-16thread.12.12 /srv/z/enc/test-enc-64kchunk-16thread.12.13 /srv/z/enc/test-enc-64kchunk-16thread.12.14 /srv/z/enc/test-enc-64kchunk-16thread.12.15 /srv/z/enc/test-enc-64kchunk-16thread.12.2 /srv/z/enc/test-enc-64kchunk-16thread.12.3 /srv/z/enc/test-enc-64kchunk-16thread.12.4 /srv/z/enc/test-enc-64kchunk-16thread.12.5 /srv/z/enc/test-enc-64kchunk-16thread.12.6 /srv/z/enc/test-enc-64kchunk-16thread.12.7 /srv/z/enc/test-enc-64kchunk-16thread.12.8 /srv/z/enc/test-enc-64kchunk-16thread.12.9 /srv/z/enc/test-enc-64kchunk-16thread.1.3 /srv/z/enc/test-enc-64kchunk-16thread.13.0 /srv/z/enc/test-enc-64kchunk-16thread.13.1 /srv/z/enc/test-enc-64kchunk-16thread.13.10 /srv/z/enc/test-enc-64kchunk-16thread.13.11 /srv/z/enc/test-enc-64kchunk-16thread.13.12 /srv/z/enc/test-enc-64kchunk-16thread.13.13 /srv/z/enc/test-enc-64kchunk-16thread.13.14 /srv/z/enc/test-enc-64kchunk-16thread.13.15 /srv/z/enc/test-enc-64kchunk-16thread.13.2 /srv/z/enc/test-enc-64kchunk-16thread.13.3 /srv/z/enc/test-enc-64kchunk-16thread.13.4 /srv/z/enc/test-enc-64kchunk-16thread.13.5 /srv/z/enc/test-enc-64kchunk-16thread.13.6 /srv/z/enc/test-enc-64kchunk-16thread.13.7 /srv/z/enc/test-enc-64kchunk-16thread.13.8 /srv/z/enc/test-enc-64kchunk-16thread.13.9 /srv/z/enc/test-enc-64kchunk-16thread.1.4 /srv/z/enc/test-enc-64kchunk-16thread.14.0 /srv/z/enc/test-enc-64kchunk-16thread.14.1 /srv/z/enc/test-enc-64kchunk-16thread.14.10 /srv/z/enc/test-enc-64kchunk-16thread.14.11 /srv/z/enc/test-enc-64kchunk-16thread.14.12 /srv/z/enc/test-enc-64kchunk-16thread.14.13 /srv/z/enc/test-enc-64kchunk-16thread.14.14 /srv/z/enc/test-enc-64kchunk-16thread.14.15 /srv/z/enc/test-enc-64kchunk-16thread.14.2 /srv/z/enc/test-enc-64kchunk-16thread.14.3 /srv/z/enc/test-enc-64kchunk-16thread.14.4 /srv/z/enc/test-enc-64kchunk-16thread.14.5 /srv/z/enc/test-enc-64kchunk-16thread.14.6 /srv/z/enc/test-enc-64kchunk-16thread.14.7 /srv/z/enc/test-enc-64kchunk-16thread.14.8 /srv/z/enc/test-enc-64kchunk-16thread.14.9 /srv/z/enc/test-enc-64kchunk-16thread.1.5 /srv/z/enc/test-enc-64kchunk-16thread.15.0 /srv/z/enc/test-enc-64kchunk-16thread.15.1 /srv/z/enc/test-enc-64kchunk-16thread.15.10 /srv/z/enc/test-enc-64kchunk-16thread.15.11 /srv/z/enc/test-enc-64kchunk-16thread.15.12 /srv/z/enc/test-enc-64kchunk-16thread.15.13 /srv/z/enc/test-enc-64kchunk-16thread.15.14 /srv/z/enc/test-enc-64kchunk-16thread.15.15 /srv/z/enc/test-enc-64kchunk-16thread.15.2 /srv/z/enc/test-enc-64kchunk-16thread.15.3 /srv/z/enc/test-enc-64kchunk-16thread.15.4 /srv/z/enc/test-enc-64kchunk-16thread.15.5 /srv/z/enc/test-enc-64kchunk-16thread.15.6 /srv/z/enc/test-enc-64kchunk-16thread.15.7 /srv/z/enc/test-enc-64kchunk-16thread.15.8 /srv/z/enc/test-enc-64kchunk-16thread.15.9 /srv/z/enc/test-enc-64kchunk-16thread.1.6 /srv/z/enc/test-enc-64kchunk-16thread.1.7 /srv/z/enc/test-enc-64kchunk-16thread.1.8 /srv/z/enc/test-enc-64kchunk-16thread.1.9 /srv/z/enc/test-enc-64kchunk-16thread.2.0 /srv/z/enc/test-enc-64kchunk-16thread.2.1 /srv/z/enc/test-enc-64kchunk-16thread.2.10 /srv/z/enc/test-enc-64kchunk-16thread.2.11 /srv/z/enc/test-enc-64kchunk-16thread.2.12 /srv/z/enc/test-enc-64kchunk-16thread.2.13 /srv/z/enc/test-enc-64kchunk-16thread.2.14 /srv/z/enc/test-enc-64kchunk-16thread.2.15 /srv/z/enc/test-enc-64kchunk-16thread.2.2 /srv/z/enc/test-enc-64kchunk-16thread.2.3 /srv/z/enc/test-enc-64kchunk-16thread.2.4 /srv/z/enc/test-enc-64kchunk-16thread.2.5 /srv/z/enc/test-enc-64kchunk-16thread.2.6 /srv/z/enc/test-enc-64kchunk-16thread.2.7 /srv/z/enc/test-enc-64kchunk-16thread.2.8 /srv/z/enc/test-enc-64kchunk-16thread.2.9 /srv/z/enc/test-enc-64kchunk-16thread.3.0 /srv/z/enc/test-enc-64kchunk-16thread.3.1 /srv/z/enc/test-enc-64kchunk-16thread.3.10 /srv/z/enc/test-enc-64kchunk-16thread.3.11 /srv/z/enc/test-enc-64kchunk-16thread.3.12 /srv/z/enc/test-enc-64kchunk-16thread.3.13 /srv/z/enc/test-enc-64kchunk-16thread.3.14 /srv/z/enc/test-enc-64kchunk-16thread.3.15 /srv/z/enc/test-enc-64kchunk-16thread.3.2 /srv/z/enc/test-enc-64kchunk-16thread.3.3 /srv/z/enc/test-enc-64kchunk-16thread.3.4 /srv/z/enc/test-enc-64kchunk-16thread.3.5 /srv/z/enc/test-enc-64kchunk-16thread.3.6 /srv/z/enc/test-enc-64kchunk-16thread.3.7 /srv/z/enc/test-enc-64kchunk-16thread.3.8 /srv/z/enc/test-enc-64kchunk-16thread.3.9 /srv/z/enc/test-enc-64kchunk-16thread.4.0 /srv/z/enc/test-enc-64kchunk-16thread.4.1 /srv/z/enc/test-enc-64kchunk-16thread.4.10 /srv/z/enc/test-enc-64kchunk-16thread.4.11 /srv/z/enc/test-enc-64kchunk-16thread.4.12 /srv/z/enc/test-enc-64kchunk-16thread.4.13 /srv/z/enc/test-enc-64kchunk-16thread.4.14 /srv/z/enc/test-enc-64kchunk-16thread.4.15 /srv/z/enc/test-enc-64kchunk-16thread.4.2 /srv/z/enc/test-enc-64kchunk-16thread.4.3 /srv/z/enc/test-enc-64kchunk-16thread.4.4 /srv/z/enc/test-enc-64kchunk-16thread.4.5 /srv/z/enc/test-enc-64kchunk-16thread.4.6 /srv/z/enc/test-enc-64kchunk-16thread.4.7 /srv/z/enc/test-enc-64kchunk-16thread.4.8 /srv/z/enc/test-enc-64kchunk-16thread.4.9 /srv/z/enc/test-enc-64kchunk-16thread.5.0 /srv/z/enc/test-enc-64kchunk-16thread.5.1 /srv/z/enc/test-enc-64kchunk-16thread.5.10 /srv/z/enc/test-enc-64kchunk-16thread.5.11 /srv/z/enc/test-enc-64kchunk-16thread.5.12 /srv/z/enc/test-enc-64kchunk-16thread.5.13 /srv/z/enc/test-enc-64kchunk-16thread.5.14 /srv/z/enc/test-enc-64kchunk-16thread.5.15 /srv/z/enc/test-enc-64kchunk-16thread.5.2 /srv/z/enc/test-enc-64kchunk-16thread.5.3 /srv/z/enc/test-enc-64kchunk-16thread.5.4 /srv/z/enc/test-enc-64kchunk-16thread.5.5 /srv/z/enc/test-enc-64kchunk-16thread.5.6 /srv/z/enc/test-enc-64kchunk-16thread.5.7 /srv/z/enc/test-enc-64kchunk-16thread.5.8 /srv/z/enc/test-enc-64kchunk-16thread.5.9 /srv/z/enc/test-enc-64kchunk-16thread.6.0 /srv/z/enc/test-enc-64kchunk-16thread.6.1 /srv/z/enc/test-enc-64kchunk-16thread.6.10 /srv/z/enc/test-enc-64kchunk-16thread.6.11 /srv/z/enc/test-enc-64kchunk-16thread.6.12 /srv/z/enc/test-enc-64kchunk-16thread.6.13 /srv/z/enc/test-enc-64kchunk-16thread.6.14 /srv/z/enc/test-enc-64kchunk-16thread.6.15 /srv/z/enc/test-enc-64kchunk-16thread.6.2 /srv/z/enc/test-enc-64kchunk-16thread.6.3 /srv/z/enc/test-enc-64kchunk-16thread.6.4 /srv/z/enc/test-enc-64kchunk-16thread.6.5 /srv/z/enc/test-enc-64kchunk-16thread.6.6 /srv/z/enc/test-enc-64kchunk-16thread.6.7 /srv/z/enc/test-enc-64kchunk-16thread.6.8 /srv/z/enc/test-enc-64kchunk-16thread.6.9 /srv/z/enc/test-enc-64kchunk-16thread.7.0 /srv/z/enc/test-enc-64kchunk-16thread.7.1 /srv/z/enc/test-enc-64kchunk-16thread.7.10 /srv/z/enc/test-enc-64kchunk-16thread.7.11 /srv/z/enc/test-enc-64kchunk-16thread.7.12 /srv/z/enc/test-enc-64kchunk-16thread.7.13 /srv/z/enc/test-enc-64kchunk-16thread.7.14 /srv/z/enc/test-enc-64kchunk-16thread.7.15 /srv/z/enc/test-enc-64kchunk-16thread.7.2 /srv/z/enc/test-enc-64kchunk-16thread.7.3 /srv/z/enc/test-enc-64kchunk-16thread.7.4 /srv/z/enc/test-enc-64kchunk-16thread.7.5 /srv/z/enc/test-enc-64kchunk-16thread.7.6 /srv/z/enc/test-enc-64kchunk-16thread.7.7 /srv/z/enc/test-enc-64kchunk-16thread.7.8 /srv/z/enc/test-enc-64kchunk-16thread.7.9 /srv/z/enc/test-enc-64kchunk-16thread.8.0 /srv/z/enc/test-enc-64kchunk-16thread.8.1 /srv/z/enc/test-enc-64kchunk-16thread.8.10 /srv/z/enc/test-enc-64kchunk-16thread.8.11 /srv/z/enc/test-enc-64kchunk-16thread.8.12 /srv/z/enc/test-enc-64kchunk-16thread.8.13 /srv/z/enc/test-enc-64kchunk-16thread.8.14 /srv/z/enc/test-enc-64kchunk-16thread.8.15 /srv/z/enc/test-enc-64kchunk-16thread.8.2 /srv/z/enc/test-enc-64kchunk-16thread.8.3 /srv/z/enc/test-enc-64kchunk-16thread.8.4 /srv/z/enc/test-enc-64kchunk-16thread.8.5 /srv/z/enc/test-enc-64kchunk-16thread.8.6 /srv/z/enc/test-enc-64kchunk-16thread.8.7 /srv/z/enc/test-enc-64kchunk-16thread.8.8 /srv/z/enc/test-enc-64kchunk-16thread.8.9 /srv/z/enc/test-enc-64kchunk-16thread.9.0 /srv/z/enc/test-enc-64kchunk-16thread.9.1 /srv/z/enc/test-enc-64kchunk-16thread.9.10 /srv/z/enc/test-enc-64kchunk-16thread.9.11 /srv/z/enc/test-enc-64kchunk-16thread.9.12 /srv/z/enc/test-enc-64kchunk-16thread.9.13 /srv/z/enc/test-enc-64kchunk-16thread.9.14 /srv/z/enc/test-enc-64kchunk-16thread.9.15 /srv/z/enc/test-enc-64kchunk-16thread.9.2 /srv/z/enc/test-enc-64kchunk-16thread.9.3 /srv/z/enc/test-enc-64kchunk-16thread.9.4 /srv/z/enc/test-enc-64kchunk-16thread.9.5 /srv/z/enc/test-enc-64kchunk-16thread.9.6 /srv/z/enc/test-enc-64kchunk-16thread.9.7 /srv/z/enc/test-enc-64kchunk-16thread.9.8 /srv/z/enc/test-enc-64kchunk-16thread.9.9 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-both-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/both/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-both-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=355MiB/s,w=355MiB/s][r=5672,w=5684 IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.5%][r=395MiB/s,w=397MiB/s][r=6319,w=6352 IOPS][eta 01m:41s] Jobs: 16 (f=256): [M(16)][24.2%][r=347MiB/s,w=350MiB/s][r=5548,w=5601 IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.5%][r=337MiB/s,w=334MiB/s][r=5390,w=5339 IOPS][eta 01m:21s] Jobs: 16 (f=256): [M(16)][40.0%][r=327MiB/s,w=325MiB/s][r=5231,w=5206 IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][48.3%][r=322MiB/s,w=322MiB/s][r=5155,w=5157 IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][56.7%][r=324MiB/s,w=324MiB/s][r=5182,w=5177 IOPS][eta 00m:52s] | |
| Jobs: 16 (f=256): [M(16)][64.2%][r=355MiB/s,w=352MiB/s][r=5673,w=5637 IOPS][eta 00m:43s] Jobs: 16 (f=256): [M(16)][72.5%][r=338MiB/s,w=340MiB/s][r=5403,w=5438 IOPS][eta 00m:33s] | |
| Jobs: 16 (f=256): [M(16)][80.2%][r=356MiB/s,w=358MiB/s][r=5695,w=5720 IOPS][eta 00m:24s] Jobs: 16 (f=256): [M(16)][88.3%][r=332MiB/s,w=330MiB/s][r=5316,w=5278 IOPS][eta 00m:14s] | |
| Jobs: 16 (f=256): [M(16)][96.7%][r=355MiB/s,w=356MiB/s][r=5679,w=5696 IOPS][eta 00m:04s] | |
| test-both-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=3025466: Wed Feb 26 11:43:26 2020 | |
| read: IOPS=5518, BW=345MiB/s (362MB/s)(40.4GiB/120013msec) | |
| slat (usec): min=9, max=155732, avg=1451.31, stdev=3714.06 | |
| clat (usec): min=4, max=479285, avg=91205.87, stdev=54571.23 | |
| lat (msec): min=2, max=482, avg=92.66, stdev=55.36 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 15], 10.00th=[ 22], 20.00th=[ 39], | |
| | 30.00th=[ 63], 40.00th=[ 78], 50.00th=[ 88], 60.00th=[ 97], | |
| | 70.00th=[ 110], 80.00th=[ 132], 90.00th=[ 169], 95.00th=[ 194], | |
| | 99.00th=[ 243], 99.50th=[ 259], 99.90th=[ 300], 99.95th=[ 326], | |
| | 99.99th=[ 380] | |
| bw ( KiB/s): min=121024, max=864898, per=99.78%, avg=352416.00, stdev=13013.86, samples=3837 | |
| iops : min= 1886, max=13513, avg=5504.12, stdev=203.37, samples=3837 | |
| write: IOPS=5520, BW=345MiB/s (362MB/s)(40.4GiB/120013msec); 0 zone resets | |
| slat (usec): min=15, max=127487, avg=1439.34, stdev=3676.86 | |
| clat (usec): min=5, max=482232, avg=91223.02, stdev=54626.85 | |
| lat (msec): min=2, max=488, avg=92.66, stdev=55.34 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 15], 10.00th=[ 22], 20.00th=[ 40], | |
| | 30.00th=[ 63], 40.00th=[ 78], 50.00th=[ 88], 60.00th=[ 97], | |
| | 70.00th=[ 110], 80.00th=[ 132], 90.00th=[ 169], 95.00th=[ 194], | |
| | 99.00th=[ 243], 99.50th=[ 262], 99.90th=[ 305], 99.95th=[ 326], | |
| | 99.99th=[ 393] | |
| bw ( KiB/s): min=128451, max=866688, per=99.79%, avg=352572.78, stdev=13066.69, samples=3837 | |
| iops : min= 2004, max=13540, avg=5506.59, stdev=204.19, samples=3837 | |
| lat (usec) : 10=0.01% | |
| lat (msec) : 4=1.46%, 10=1.50%, 20=5.89%, 50=15.76%, 100=37.97% | |
| lat (msec) : 250=36.67%, 500=0.74% | |
| cpu : usr=0.31%, sys=37.86%, ctx=890339, majf=0, minf=484996 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=662307,662565,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=345MiB/s (362MB/s), 345MiB/s-345MiB/s (362MB/s-362MB/s), io=40.4GiB (43.4GB), run=120013-120013msec | |
| WRITE: bw=345MiB/s (362MB/s), 345MiB/s-345MiB/s (362MB/s-362MB/s), io=40.4GiB (43.4GB), run=120013-120013msec | |
| + rm -rf /srv/z/both/test-both-64kchunk-16thread.0.0 /srv/z/both/test-both-64kchunk-16thread.0.1 /srv/z/both/test-both-64kchunk-16thread.0.10 /srv/z/both/test-both-64kchunk-16thread.0.11 /srv/z/both/test-both-64kchunk-16thread.0.12 /srv/z/both/test-both-64kchunk-16thread.0.13 /srv/z/both/test-both-64kchunk-16thread.0.14 /srv/z/both/test-both-64kchunk-16thread.0.15 /srv/z/both/test-both-64kchunk-16thread.0.2 /srv/z/both/test-both-64kchunk-16thread.0.3 /srv/z/both/test-both-64kchunk-16thread.0.4 /srv/z/both/test-both-64kchunk-16thread.0.5 /srv/z/both/test-both-64kchunk-16thread.0.6 /srv/z/both/test-both-64kchunk-16thread.0.7 /srv/z/both/test-both-64kchunk-16thread.0.8 /srv/z/both/test-both-64kchunk-16thread.0.9 /srv/z/both/test-both-64kchunk-16thread.1.0 /srv/z/both/test-both-64kchunk-16thread.10.0 /srv/z/both/test-both-64kchunk-16thread.10.1 /srv/z/both/test-both-64kchunk-16thread.10.10 /srv/z/both/test-both-64kchunk-16thread.10.11 /srv/z/both/test-both-64kchunk-16thread.10.12 /srv/z/both/test-both-64kchunk-16thread.10.13 /srv/z/both/test-both-64kchunk-16thread.10.14 /srv/z/both/test-both-64kchunk-16thread.10.15 /srv/z/both/test-both-64kchunk-16thread.10.2 /srv/z/both/test-both-64kchunk-16thread.10.3 /srv/z/both/test-both-64kchunk-16thread.10.4 /srv/z/both/test-both-64kchunk-16thread.10.5 /srv/z/both/test-both-64kchunk-16thread.10.6 /srv/z/both/test-both-64kchunk-16thread.10.7 /srv/z/both/test-both-64kchunk-16thread.10.8 /srv/z/both/test-both-64kchunk-16thread.10.9 /srv/z/both/test-both-64kchunk-16thread.1.1 /srv/z/both/test-both-64kchunk-16thread.1.10 /srv/z/both/test-both-64kchunk-16thread.11.0 /srv/z/both/test-both-64kchunk-16thread.1.11 /srv/z/both/test-both-64kchunk-16thread.11.1 /srv/z/both/test-both-64kchunk-16thread.11.10 /srv/z/both/test-both-64kchunk-16thread.11.11 /srv/z/both/test-both-64kchunk-16thread.11.12 /srv/z/both/test-both-64kchunk-16thread.11.13 /srv/z/both/test-both-64kchunk-16thread.11.14 /srv/z/both/test-both-64kchunk-16thread.11.15 /srv/z/both/test-both-64kchunk-16thread.1.12 /srv/z/both/test-both-64kchunk-16thread.11.2 /srv/z/both/test-both-64kchunk-16thread.1.13 /srv/z/both/test-both-64kchunk-16thread.11.3 /srv/z/both/test-both-64kchunk-16thread.1.14 /srv/z/both/test-both-64kchunk-16thread.11.4 /srv/z/both/test-both-64kchunk-16thread.1.15 /srv/z/both/test-both-64kchunk-16thread.11.5 /srv/z/both/test-both-64kchunk-16thread.11.6 /srv/z/both/test-both-64kchunk-16thread.11.7 /srv/z/both/test-both-64kchunk-16thread.11.8 /srv/z/both/test-both-64kchunk-16thread.11.9 /srv/z/both/test-both-64kchunk-16thread.1.2 /srv/z/both/test-both-64kchunk-16thread.12.0 /srv/z/both/test-both-64kchunk-16thread.12.1 /srv/z/both/test-both-64kchunk-16thread.12.10 /srv/z/both/test-both-64kchunk-16thread.12.11 /srv/z/both/test-both-64kchunk-16thread.12.12 /srv/z/both/test-both-64kchunk-16thread.12.13 /srv/z/both/test-both-64kchunk-16thread.12.14 /srv/z/both/test-both-64kchunk-16thread.12.15 /srv/z/both/test-both-64kchunk-16thread.12.2 /srv/z/both/test-both-64kchunk-16thread.12.3 /srv/z/both/test-both-64kchunk-16thread.12.4 /srv/z/both/test-both-64kchunk-16thread.12.5 /srv/z/both/test-both-64kchunk-16thread.12.6 /srv/z/both/test-both-64kchunk-16thread.12.7 /srv/z/both/test-both-64kchunk-16thread.12.8 /srv/z/both/test-both-64kchunk-16thread.12.9 /srv/z/both/test-both-64kchunk-16thread.1.3 /srv/z/both/test-both-64kchunk-16thread.13.0 /srv/z/both/test-both-64kchunk-16thread.13.1 /srv/z/both/test-both-64kchunk-16thread.13.10 /srv/z/both/test-both-64kchunk-16thread.13.11 /srv/z/both/test-both-64kchunk-16thread.13.12 /srv/z/both/test-both-64kchunk-16thread.13.13 /srv/z/both/test-both-64kchunk-16thread.13.14 /srv/z/both/test-both-64kchunk-16thread.13.15 /srv/z/both/test-both-64kchunk-16thread.13.2 /srv/z/both/test-both-64kchunk-16thread.13.3 /srv/z/both/test-both-64kchunk-16thread.13.4 /srv/z/both/test-both-64kchunk-16thread.13.5 /srv/z/both/test-both-64kchunk-16thread.13.6 /srv/z/both/test-both-64kchunk-16thread.13.7 /srv/z/both/test-both-64kchunk-16thread.13.8 /srv/z/both/test-both-64kchunk-16thread.13.9 /srv/z/both/test-both-64kchunk-16thread.1.4 /srv/z/both/test-both-64kchunk-16thread.14.0 /srv/z/both/test-both-64kchunk-16thread.14.1 /srv/z/both/test-both-64kchunk-16thread.14.10 /srv/z/both/test-both-64kchunk-16thread.14.11 /srv/z/both/test-both-64kchunk-16thread.14.12 /srv/z/both/test-both-64kchunk-16thread.14.13 /srv/z/both/test-both-64kchunk-16thread.14.14 /srv/z/both/test-both-64kchunk-16thread.14.15 /srv/z/both/test-both-64kchunk-16thread.14.2 /srv/z/both/test-both-64kchunk-16thread.14.3 /srv/z/both/test-both-64kchunk-16thread.14.4 /srv/z/both/test-both-64kchunk-16thread.14.5 /srv/z/both/test-both-64kchunk-16thread.14.6 /srv/z/both/test-both-64kchunk-16thread.14.7 /srv/z/both/test-both-64kchunk-16thread.14.8 /srv/z/both/test-both-64kchunk-16thread.14.9 /srv/z/both/test-both-64kchunk-16thread.1.5 /srv/z/both/test-both-64kchunk-16thread.15.0 /srv/z/both/test-both-64kchunk-16thread.15.1 /srv/z/both/test-both-64kchunk-16thread.15.10 /srv/z/both/test-both-64kchunk-16thread.15.11 /srv/z/both/test-both-64kchunk-16thread.15.12 /srv/z/both/test-both-64kchunk-16thread.15.13 /srv/z/both/test-both-64kchunk-16thread.15.14 /srv/z/both/test-both-64kchunk-16thread.15.15 /srv/z/both/test-both-64kchunk-16thread.15.2 /srv/z/both/test-both-64kchunk-16thread.15.3 /srv/z/both/test-both-64kchunk-16thread.15.4 /srv/z/both/test-both-64kchunk-16thread.15.5 /srv/z/both/test-both-64kchunk-16thread.15.6 /srv/z/both/test-both-64kchunk-16thread.15.7 /srv/z/both/test-both-64kchunk-16thread.15.8 /srv/z/both/test-both-64kchunk-16thread.15.9 /srv/z/both/test-both-64kchunk-16thread.1.6 /srv/z/both/test-both-64kchunk-16thread.1.7 /srv/z/both/test-both-64kchunk-16thread.1.8 /srv/z/both/test-both-64kchunk-16thread.1.9 /srv/z/both/test-both-64kchunk-16thread.2.0 /srv/z/both/test-both-64kchunk-16thread.2.1 /srv/z/both/test-both-64kchunk-16thread.2.10 /srv/z/both/test-both-64kchunk-16thread.2.11 /srv/z/both/test-both-64kchunk-16thread.2.12 /srv/z/both/test-both-64kchunk-16thread.2.13 /srv/z/both/test-both-64kchunk-16thread.2.14 /srv/z/both/test-both-64kchunk-16thread.2.15 /srv/z/both/test-both-64kchunk-16thread.2.2 /srv/z/both/test-both-64kchunk-16thread.2.3 /srv/z/both/test-both-64kchunk-16thread.2.4 /srv/z/both/test-both-64kchunk-16thread.2.5 /srv/z/both/test-both-64kchunk-16thread.2.6 /srv/z/both/test-both-64kchunk-16thread.2.7 /srv/z/both/test-both-64kchunk-16thread.2.8 /srv/z/both/test-both-64kchunk-16thread.2.9 /srv/z/both/test-both-64kchunk-16thread.3.0 /srv/z/both/test-both-64kchunk-16thread.3.1 /srv/z/both/test-both-64kchunk-16thread.3.10 /srv/z/both/test-both-64kchunk-16thread.3.11 /srv/z/both/test-both-64kchunk-16thread.3.12 /srv/z/both/test-both-64kchunk-16thread.3.13 /srv/z/both/test-both-64kchunk-16thread.3.14 /srv/z/both/test-both-64kchunk-16thread.3.15 /srv/z/both/test-both-64kchunk-16thread.3.2 /srv/z/both/test-both-64kchunk-16thread.3.3 /srv/z/both/test-both-64kchunk-16thread.3.4 /srv/z/both/test-both-64kchunk-16thread.3.5 /srv/z/both/test-both-64kchunk-16thread.3.6 /srv/z/both/test-both-64kchunk-16thread.3.7 /srv/z/both/test-both-64kchunk-16thread.3.8 /srv/z/both/test-both-64kchunk-16thread.3.9 /srv/z/both/test-both-64kchunk-16thread.4.0 /srv/z/both/test-both-64kchunk-16thread.4.1 /srv/z/both/test-both-64kchunk-16thread.4.10 /srv/z/both/test-both-64kchunk-16thread.4.11 /srv/z/both/test-both-64kchunk-16thread.4.12 /srv/z/both/test-both-64kchunk-16thread.4.13 /srv/z/both/test-both-64kchunk-16thread.4.14 /srv/z/both/test-both-64kchunk-16thread.4.15 /srv/z/both/test-both-64kchunk-16thread.4.2 /srv/z/both/test-both-64kchunk-16thread.4.3 /srv/z/both/test-both-64kchunk-16thread.4.4 /srv/z/both/test-both-64kchunk-16thread.4.5 /srv/z/both/test-both-64kchunk-16thread.4.6 /srv/z/both/test-both-64kchunk-16thread.4.7 /srv/z/both/test-both-64kchunk-16thread.4.8 /srv/z/both/test-both-64kchunk-16thread.4.9 /srv/z/both/test-both-64kchunk-16thread.5.0 /srv/z/both/test-both-64kchunk-16thread.5.1 /srv/z/both/test-both-64kchunk-16thread.5.10 /srv/z/both/test-both-64kchunk-16thread.5.11 /srv/z/both/test-both-64kchunk-16thread.5.12 /srv/z/both/test-both-64kchunk-16thread.5.13 /srv/z/both/test-both-64kchunk-16thread.5.14 /srv/z/both/test-both-64kchunk-16thread.5.15 /srv/z/both/test-both-64kchunk-16thread.5.2 /srv/z/both/test-both-64kchunk-16thread.5.3 /srv/z/both/test-both-64kchunk-16thread.5.4 /srv/z/both/test-both-64kchunk-16thread.5.5 /srv/z/both/test-both-64kchunk-16thread.5.6 /srv/z/both/test-both-64kchunk-16thread.5.7 /srv/z/both/test-both-64kchunk-16thread.5.8 /srv/z/both/test-both-64kchunk-16thread.5.9 /srv/z/both/test-both-64kchunk-16thread.6.0 /srv/z/both/test-both-64kchunk-16thread.6.1 /srv/z/both/test-both-64kchunk-16thread.6.10 /srv/z/both/test-both-64kchunk-16thread.6.11 /srv/z/both/test-both-64kchunk-16thread.6.12 /srv/z/both/test-both-64kchunk-16thread.6.13 /srv/z/both/test-both-64kchunk-16thread.6.14 /srv/z/both/test-both-64kchunk-16thread.6.15 /srv/z/both/test-both-64kchunk-16thread.6.2 /srv/z/both/test-both-64kchunk-16thread.6.3 /srv/z/both/test-both-64kchunk-16thread.6.4 /srv/z/both/test-both-64kchunk-16thread.6.5 /srv/z/both/test-both-64kchunk-16thread.6.6 /srv/z/both/test-both-64kchunk-16thread.6.7 /srv/z/both/test-both-64kchunk-16thread.6.8 /srv/z/both/test-both-64kchunk-16thread.6.9 /srv/z/both/test-both-64kchunk-16thread.7.0 /srv/z/both/test-both-64kchunk-16thread.7.1 /srv/z/both/test-both-64kchunk-16thread.7.10 /srv/z/both/test-both-64kchunk-16thread.7.11 /srv/z/both/test-both-64kchunk-16thread.7.12 /srv/z/both/test-both-64kchunk-16thread.7.13 /srv/z/both/test-both-64kchunk-16thread.7.14 /srv/z/both/test-both-64kchunk-16thread.7.15 /srv/z/both/test-both-64kchunk-16thread.7.2 /srv/z/both/test-both-64kchunk-16thread.7.3 /srv/z/both/test-both-64kchunk-16thread.7.4 /srv/z/both/test-both-64kchunk-16thread.7.5 /srv/z/both/test-both-64kchunk-16thread.7.6 /srv/z/both/test-both-64kchunk-16thread.7.7 /srv/z/both/test-both-64kchunk-16thread.7.8 /srv/z/both/test-both-64kchunk-16thread.7.9 /srv/z/both/test-both-64kchunk-16thread.8.0 /srv/z/both/test-both-64kchunk-16thread.8.1 /srv/z/both/test-both-64kchunk-16thread.8.10 /srv/z/both/test-both-64kchunk-16thread.8.11 /srv/z/both/test-both-64kchunk-16thread.8.12 /srv/z/both/test-both-64kchunk-16thread.8.13 /srv/z/both/test-both-64kchunk-16thread.8.14 /srv/z/both/test-both-64kchunk-16thread.8.15 /srv/z/both/test-both-64kchunk-16thread.8.2 /srv/z/both/test-both-64kchunk-16thread.8.3 /srv/z/both/test-both-64kchunk-16thread.8.4 /srv/z/both/test-both-64kchunk-16thread.8.5 /srv/z/both/test-both-64kchunk-16thread.8.6 /srv/z/both/test-both-64kchunk-16thread.8.7 /srv/z/both/test-both-64kchunk-16thread.8.8 /srv/z/both/test-both-64kchunk-16thread.8.9 /srv/z/both/test-both-64kchunk-16thread.9.0 /srv/z/both/test-both-64kchunk-16thread.9.1 /srv/z/both/test-both-64kchunk-16thread.9.10 /srv/z/both/test-both-64kchunk-16thread.9.11 /srv/z/both/test-both-64kchunk-16thread.9.12 /srv/z/both/test-both-64kchunk-16thread.9.13 /srv/z/both/test-both-64kchunk-16thread.9.14 /srv/z/both/test-both-64kchunk-16thread.9.15 /srv/z/both/test-both-64kchunk-16thread.9.2 /srv/z/both/test-both-64kchunk-16thread.9.3 /srv/z/both/test-both-64kchunk-16thread.9.4 /srv/z/both/test-both-64kchunk-16thread.9.5 /srv/z/both/test-both-64kchunk-16thread.9.6 /srv/z/both/test-both-64kchunk-16thread.9.7 /srv/z/both/test-both-64kchunk-16thread.9.8 /srv/z/both/test-both-64kchunk-16thread.9.9 | |
| + zpool destroy tank | |
| + for ASHIFTVAL in 12 13 14 15 16 | |
| + trim_disks | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| Success formatting namespace:1 | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '################## waiting 60s for nvme format to complete...' | |
| ################## waiting 60s for nvme format to complete... | |
| + sleep 60 | |
| + nvme list | |
| Node SN Model Namespace Usage Format FW Rev | |
| ---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- -------- | |
| /dev/nvme0n1 S437NE0M804438 SAMSUNG MZQLB960HAJR-00007 1 68.59 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme1n1 S437NE0M804437 SAMSUNG MZQLB960HAJR-00007 1 63.00 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme2n1 S437NE0M804442 SAMSUNG MZQLB960HAJR-00007 1 73.07 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme3n1 S437NE0M804435 SAMSUNG MZQLB960HAJR-00007 1 58.73 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme4n1 S437NE0M802527 SAMSUNG MZQLB960HAJR-00007 1 52.11 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme5n1 S437NX0M402110 SAMSUNG MZQLB960HAJR-00007 1 77.36 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + zpool create tank -o ashift=14 -O mountpoint=/srv/z raidz1 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| + dd if=/dev/urandom of=/tmp/testkey.raw bs=32 count=1 | |
| 1+0 records in | |
| 1+0 records out | |
| 32 bytes copied, 0.000307782 s, 104 kB/s | |
| + COMP=' | |
| -o compression=lz4 | |
| ' | |
| + ENC=' | |
| -o encryption=aes-256-gcm | |
| -o keyformat=raw | |
| -o keylocation=file:///tmp/testkey.raw | |
| ' | |
| + zfs create tank/none | |
| + zfs create -o compression=lz4 tank/comp | |
| + zfs create -o encryption=aes-256-gcm -o keyformat=raw -o keylocation=file:///tmp/testkey.raw tank/enc | |
| + zfs create -o compression=lz4 -o encryption=aes-256-gcm -o keyformat=raw -o keylocation=file:///tmp/testkey.raw tank/both | |
| + FO='--eta-newline=10 --eta-interval=10 --eta=always' | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-none-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/none/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-none-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-none-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=1104MiB/s,w=1103MiB/s][r=17.7k,w=17.7k IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.7%][r=1115MiB/s,w=1112MiB/s][r=17.8k,w=17.8k IOPS][eta 01m:40s] Jobs: 1 (f=1): [M(1)][24.2%][r=1068MiB/s,w=1067MiB/s][r=17.1k,w=17.1k IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.8%][r=1082MiB/s,w=1081MiB/s][r=17.3k,w=17.3k IOPS][eta 01m:20s] Jobs: 1 (f=1): [M(1)][40.8%][r=1077MiB/s,w=1076MiB/s][r=17.2k,w=17.2k IOPS][eta 01m:11s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=1094MiB/s,w=1091MiB/s][r=17.5k,w=17.5k IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=1084MiB/s,w=1089MiB/s][r=17.3k,w=17.4k IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][65.5%][r=1109MiB/s,w=1101MiB/s][r=17.7k,w=17.6k IOPS][eta 00m:41s] Jobs: 1 (f=1): [M(1)][72.5%][r=1071MiB/s,w=1070MiB/s][r=17.1k,w=17.1k IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=1082MiB/s,w=1087MiB/s][r=17.3k,w=17.4k IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][89.9%][r=1067MiB/s,w=1064MiB/s][r=17.1k,w=17.0k IOPS][eta 00m:12s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=1063MiB/s,w=1059MiB/s][r=17.0k,w=16.9k IOPS][eta 00m:04s] | |
| test-none-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=3266552: Wed Feb 26 11:46:52 2020 | |
| read: IOPS=17.4k, BW=1086MiB/s (1138MB/s)(127GiB/120001msec) | |
| slat (usec): min=5, max=100378, avg=27.21, stdev=155.62 | |
| clat (usec): min=2, max=102900, avg=1818.53, stdev=1932.37 | |
| lat (usec): min=38, max=102982, avg=1845.89, stdev=1954.04 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1188], 5.00th=[ 1270], 10.00th=[ 1287], 20.00th=[ 1319], | |
| | 30.00th=[ 1385], 40.00th=[ 1434], 50.00th=[ 1450], 60.00th=[ 1483], | |
| | 70.00th=[ 1516], 80.00th=[ 1631], 90.00th=[ 2474], 95.00th=[ 3392], | |
| | 99.00th=[ 9503], 99.50th=[14746], 99.90th=[27657], 99.95th=[32900], | |
| | 99.99th=[46924] | |
| bw ( MiB/s): min= 751, max= 1512, per=100.00%, avg=1085.67, stdev=169.46, samples=239 | |
| iops : min=12028, max=24200, avg=17370.73, stdev=2711.33, samples=239 | |
| write: IOPS=17.3k, BW=1084MiB/s (1137MB/s)(127GiB/120001msec); 0 zone resets | |
| slat (usec): min=11, max=43759, avg=27.66, stdev=156.17 | |
| clat (usec): min=52, max=102939, avg=1811.84, stdev=1896.91 | |
| lat (usec): min=79, max=102961, avg=1839.66, stdev=1920.60 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1188], 5.00th=[ 1270], 10.00th=[ 1287], 20.00th=[ 1319], | |
| | 30.00th=[ 1385], 40.00th=[ 1434], 50.00th=[ 1450], 60.00th=[ 1467], | |
| | 70.00th=[ 1516], 80.00th=[ 1631], 90.00th=[ 2474], 95.00th=[ 3359], | |
| | 99.00th=[ 9372], 99.50th=[14222], 99.90th=[26870], 99.95th=[32375], | |
| | 99.99th=[46924] | |
| bw ( MiB/s): min= 755, max= 1508, per=100.00%, avg=1084.24, stdev=170.56, samples=239 | |
| iops : min=12086, max=24128, avg=17347.74, stdev=2729.00, samples=239 | |
| lat (usec) : 4=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01% | |
| lat (usec) : 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=88.35%, 4=8.29%, 10=2.43%, 20=0.67%, 50=0.25% | |
| lat (msec) : 100=0.01%, 250=0.01% | |
| cpu : usr=7.57%, sys=81.48%, ctx=25707, majf=0, minf=48142 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=2084301,2081855,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=1086MiB/s (1138MB/s), 1086MiB/s-1086MiB/s (1138MB/s-1138MB/s), io=127GiB (137GB), run=120001-120001msec | |
| WRITE: bw=1084MiB/s (1137MB/s), 1084MiB/s-1084MiB/s (1137MB/s-1137MB/s), io=127GiB (136GB), run=120001-120001msec | |
| + rm -rf /srv/z/none/test-none-64kchunk-1thread.0.0 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-comp-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/comp/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-comp-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-comp-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=1061MiB/s,w=1061MiB/s][r=16.0k,w=16.0k IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.5%][r=1085MiB/s,w=1082MiB/s][r=17.4k,w=17.3k IOPS][eta 01m:41s] Jobs: 1 (f=1): [M(1)][24.2%][r=1070MiB/s,w=1068MiB/s][r=17.1k,w=17.1k IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.8%][r=1057MiB/s,w=1058MiB/s][r=16.9k,w=16.9k IOPS][eta 01m:20s] Jobs: 1 (f=1): [M(1)][40.5%][r=1078MiB/s,w=1075MiB/s][r=17.2k,w=17.2k IOPS][eta 01m:12s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=1052MiB/s,w=1051MiB/s][r=16.8k,w=16.8k IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=1060MiB/s,w=1064MiB/s][r=16.0k,w=17.0k IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][65.0%][r=1031MiB/s,w=1024MiB/s][r=16.5k,w=16.4k IOPS][eta 00m:42s] Jobs: 1 (f=1): [M(1)][72.5%][r=1034MiB/s,w=1033MiB/s][r=16.5k,w=16.5k IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=1046MiB/s,w=1048MiB/s][r=16.7k,w=16.8k IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][88.4%][r=1070MiB/s,w=1067MiB/s][r=17.1k,w=17.1k IOPS][eta 00m:14s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=1050MiB/s,w=1049MiB/s][r=16.8k,w=16.8k IOPS][eta 00m:04s] | |
| test-comp-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=3532800: Wed Feb 26 11:49:03 2020 | |
| read: IOPS=16.9k, BW=1059MiB/s (1110MB/s)(124GiB/120001msec) | |
| slat (usec): min=5, max=47303, avg=28.10, stdev=202.68 | |
| clat (nsec): min=1690, max=72992k, avg=1864903.20, stdev=2364921.43 | |
| lat (usec): min=37, max=73054, avg=1893.14, stdev=2390.69 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1188], 5.00th=[ 1287], 10.00th=[ 1303], 20.00th=[ 1336], | |
| | 30.00th=[ 1385], 40.00th=[ 1418], 50.00th=[ 1434], 60.00th=[ 1467], | |
| | 70.00th=[ 1516], 80.00th=[ 1598], 90.00th=[ 1795], 95.00th=[ 3163], | |
| | 99.00th=[12911], 99.50th=[18744], 99.90th=[32637], 99.95th=[38011], | |
| | 99.99th=[50070] | |
| bw ( MiB/s): min= 748, max= 1534, per=100.00%, avg=1058.94, stdev=185.92, samples=239 | |
| iops : min=11974, max=24550, avg=16942.97, stdev=2974.65, samples=239 | |
| write: IOPS=16.9k, BW=1058MiB/s (1109MB/s)(124GiB/120001msec); 0 zone resets | |
| slat (usec): min=11, max=49868, avg=28.07, stdev=203.12 | |
| clat (usec): min=88, max=73012, avg=1857.36, stdev=2341.03 | |
| lat (usec): min=109, max=73051, avg=1885.62, stdev=2367.20 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1188], 5.00th=[ 1287], 10.00th=[ 1303], 20.00th=[ 1336], | |
| | 30.00th=[ 1385], 40.00th=[ 1418], 50.00th=[ 1434], 60.00th=[ 1467], | |
| | 70.00th=[ 1516], 80.00th=[ 1598], 90.00th=[ 1795], 95.00th=[ 3163], | |
| | 99.00th=[12649], 99.50th=[18482], 99.90th=[32637], 99.95th=[38011], | |
| | 99.99th=[50070] | |
| bw ( MiB/s): min= 738, max= 1529, per=100.00%, avg=1057.64, stdev=185.53, samples=239 | |
| iops : min=11808, max=24470, avg=16922.21, stdev=2968.48, samples=239 | |
| lat (usec) : 2=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01% | |
| lat (usec) : 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=91.25%, 4=5.10%, 10=2.20%, 20=1.02%, 50=0.42% | |
| lat (msec) : 100=0.01% | |
| cpu : usr=7.01%, sys=77.49%, ctx=24960, majf=0, minf=40837 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=2032867,2030479,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=1059MiB/s (1110MB/s), 1059MiB/s-1059MiB/s (1110MB/s-1110MB/s), io=124GiB (133GB), run=120001-120001msec | |
| WRITE: bw=1058MiB/s (1109MB/s), 1058MiB/s-1058MiB/s (1109MB/s-1109MB/s), io=124GiB (133GB), run=120001-120001msec | |
| + rm -rf /srv/z/comp/test-comp-64kchunk-1thread.0.0 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-enc-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/enc/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-enc-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-enc-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=34.6MiB/s,w=34.0MiB/s][r=553,w=559 IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.5%][r=35.4MiB/s,w=34.9MiB/s][r=566,w=557 IOPS][eta 01m:41s] Jobs: 1 (f=1): [M(1)][24.2%][r=34.8MiB/s,w=34.7MiB/s][r=557,w=554 IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.5%][r=34.9MiB/s,w=34.0MiB/s][r=558,w=559 IOPS][eta 01m:21s] Jobs: 1 (f=1): [M(1)][40.5%][r=34.5MiB/s,w=34.7MiB/s][r=552,w=554 IOPS][eta 01m:12s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=34.6MiB/s,w=35.1MiB/s][r=552,w=560 IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=35.1MiB/s,w=34.8MiB/s][r=560,w=556 IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][65.0%][r=35.1MiB/s,w=32.0MiB/s][r=561,w=527 IOPS][eta 00m:42s] Jobs: 1 (f=1): [M(1)][72.5%][r=35.4MiB/s,w=35.4MiB/s][r=566,w=565 IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=34.8MiB/s,w=35.6MiB/s][r=556,w=569 IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][88.4%][r=34.9MiB/s,w=35.0MiB/s][r=557,w=560 IOPS][eta 00m:14s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=35.1MiB/s,w=33.9MiB/s][r=561,w=541 IOPS][eta 00m:04s] | |
| test-enc-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=3746836: Wed Feb 26 11:51:17 2020 | |
| read: IOPS=559, BW=34.0MiB/s (36.7MB/s)(4199MiB/120002msec) | |
| slat (usec): min=6, max=97514, avg=1139.91, stdev=1698.95 | |
| clat (usec): min=4, max=178722, avg=56395.91, stdev=10383.75 | |
| lat (msec): min=3, max=182, avg=57.54, stdev=10.54 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 45], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 53], 50.00th=[ 55], 60.00th=[ 58], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 66], 95.00th=[ 74], | |
| | 99.00th=[ 97], 99.50th=[ 104], 99.90th=[ 117], 99.95th=[ 140], | |
| | 99.99th=[ 178] | |
| bw ( KiB/s): min=21632, max=41472, per=99.92%, avg=35799.84, stdev=3258.02, samples=240 | |
| iops : min= 338, max= 648, avg=559.29, stdev=50.89, samples=240 | |
| write: IOPS=557, BW=34.8MiB/s (36.5MB/s)(4180MiB/120002msec); 0 zone resets | |
| slat (usec): min=13, max=35752, avg=646.09, stdev=1368.57 | |
| clat (msec): min=3, max=182, avg=56.35, stdev=10.39 | |
| lat (msec): min=3, max=182, avg=57.00, stdev=10.45 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 45], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 53], 50.00th=[ 55], 60.00th=[ 58], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 66], 95.00th=[ 74], | |
| | 99.00th=[ 97], 99.50th=[ 104], 99.90th=[ 121], 99.95th=[ 157], | |
| | 99.99th=[ 178] | |
| bw ( KiB/s): min=20992, max=43264, per=99.93%, avg=35645.75, stdev=3805.26, samples=240 | |
| iops : min= 328, max= 676, avg=556.89, stdev=59.44, samples=240 | |
| lat (usec) : 10=0.01% | |
| lat (msec) : 4=0.01%, 10=0.01%, 20=0.01%, 50=22.91%, 100=76.34% | |
| lat (msec) : 250=0.74% | |
| cpu : usr=0.20%, sys=85.12%, ctx=12974, majf=0, minf=21249 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=67177,66882,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=34.0MiB/s (36.7MB/s), 34.0MiB/s-34.0MiB/s (36.7MB/s-36.7MB/s), io=4199MiB (4403MB), run=120002-120002msec | |
| WRITE: bw=34.8MiB/s (36.5MB/s), 34.8MiB/s-34.8MiB/s (36.5MB/s-36.5MB/s), io=4180MiB (4383MB), run=120002-120002msec | |
| + rm -rf /srv/z/enc/test-enc-64kchunk-1thread.0.0 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-both-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/both/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-both-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-both-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=34.3MiB/s,w=34.6MiB/s][r=549,w=554 IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.5%][r=35.6MiB/s,w=35.1MiB/s][r=569,w=561 IOPS][eta 01m:41s] Jobs: 1 (f=1): [M(1)][24.2%][r=35.0MiB/s,w=34.9MiB/s][r=560,w=558 IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.5%][r=34.8MiB/s,w=34.8MiB/s][r=556,w=557 IOPS][eta 01m:21s] Jobs: 1 (f=1): [M(1)][40.5%][r=34.9MiB/s,w=34.0MiB/s][r=557,w=559 IOPS][eta 01m:12s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=34.5MiB/s,w=35.0MiB/s][r=551,w=560 IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=35.2MiB/s,w=34.9MiB/s][r=562,w=558 IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][64.5%][r=34.7MiB/s,w=32.5MiB/s][r=555,w=519 IOPS][eta 00m:43s] Jobs: 1 (f=1): [M(1)][72.5%][r=34.9MiB/s,w=34.8MiB/s][r=558,w=556 IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=35.1MiB/s,w=35.0MiB/s][r=561,w=575 IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][89.2%][r=35.0MiB/s,w=35.2MiB/s][r=560,w=562 IOPS][eta 00m:13s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=34.5MiB/s,w=33.5MiB/s][r=552,w=535 IOPS][eta 00m:04s] | |
| test-both-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=3857017: Wed Feb 26 11:53:30 2020 | |
| read: IOPS=558, BW=34.9MiB/s (36.6MB/s)(4190MiB/120002msec) | |
| slat (usec): min=6, max=175630, avg=1136.60, stdev=1783.99 | |
| clat (usec): min=3, max=263797, avg=56512.28, stdev=11950.63 | |
| lat (msec): min=3, max=267, avg=57.65, stdev=12.08 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 46], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 52], 50.00th=[ 55], 60.00th=[ 56], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 66], 95.00th=[ 74], | |
| | 99.00th=[ 102], 99.50th=[ 110], 99.90th=[ 186], 99.95th=[ 243], | |
| | 99.99th=[ 259] | |
| bw ( KiB/s): min=23424, max=41984, per=99.94%, avg=35730.75, stdev=3291.19, samples=240 | |
| iops : min= 366, max= 656, avg=558.23, stdev=51.41, samples=240 | |
| write: IOPS=556, BW=34.8MiB/s (36.4MB/s)(4171MiB/120002msec); 0 zone resets | |
| slat (usec): min=12, max=125728, avg=653.35, stdev=1566.84 | |
| clat (msec): min=10, max=263, avg=56.48, stdev=11.93 | |
| lat (msec): min=10, max=263, avg=57.13, stdev=12.05 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 46], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 53], 50.00th=[ 55], 60.00th=[ 57], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 66], 95.00th=[ 74], | |
| | 99.00th=[ 101], 99.50th=[ 111], 99.90th=[ 180], 99.95th=[ 203], | |
| | 99.99th=[ 259] | |
| bw ( KiB/s): min=21248, max=43392, per=99.93%, avg=35567.55, stdev=3944.43, samples=240 | |
| iops : min= 332, max= 678, avg=555.68, stdev=61.62, samples=240 | |
| lat (usec) : 4=0.01% | |
| lat (msec) : 4=0.01%, 10=0.01%, 20=0.02%, 50=23.35%, 100=75.61% | |
| lat (msec) : 250=0.99%, 500=0.03% | |
| cpu : usr=0.20%, sys=85.17%, ctx=12883, majf=0, minf=24100 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=67035,66735,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=34.9MiB/s (36.6MB/s), 34.9MiB/s-34.9MiB/s (36.6MB/s-36.6MB/s), io=4190MiB (4393MB), run=120002-120002msec | |
| WRITE: bw=34.8MiB/s (36.4MB/s), 34.8MiB/s-34.8MiB/s (36.4MB/s-36.4MB/s), io=4171MiB (4374MB), run=120002-120002msec | |
| + rm -rf /srv/z/both/test-both-64kchunk-1thread.0.0 | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-none-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/none/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-none-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=2858MiB/s,w=2858MiB/s][r=45.7k,w=45.7k IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.5%][r=2760MiB/s,w=2762MiB/s][r=44.2k,w=44.2k IOPS][eta 01m:41s] Jobs: 16 (f=256): [M(16)][24.2%][r=2949MiB/s,w=2964MiB/s][r=47.2k,w=47.4k IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.8%][r=2988MiB/s,w=2989MiB/s][r=47.8k,w=47.8k IOPS][eta 01m:20s] Jobs: 16 (f=256): [M(16)][40.5%][r=3033MiB/s,w=3035MiB/s][r=48.5k,w=48.6k IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][48.3%][r=2901MiB/s,w=2896MiB/s][r=46.4k,w=46.3k IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][56.2%][r=2695MiB/s,w=2694MiB/s][r=43.1k,w=43.1k IOPS][eta 00m:53s] | |
| Jobs: 16 (f=256): [M(16)][64.7%][r=2700MiB/s,w=2699MiB/s][r=43.2k,w=43.2k IOPS][eta 00m:42s] Jobs: 16 (f=256): [M(16)][72.5%][r=2694MiB/s,w=2696MiB/s][r=43.1k,w=43.1k IOPS][eta 00m:33s] | |
| Jobs: 16 (f=256): [M(16)][80.2%][r=2624MiB/s,w=2626MiB/s][r=41.0k,w=42.0k IOPS][eta 00m:24s] Jobs: 16 (f=256): [M(16)][88.3%][r=2535MiB/s,w=2546MiB/s][r=40.6k,w=40.7k IOPS][eta 00m:14s] | |
| Jobs: 16 (f=256): [M(16)][95.9%][r=2430MiB/s,w=2431MiB/s][r=38.9k,w=38.9k IOPS][eta 00m:05s] | |
| test-none-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=4100481: Wed Feb 26 11:58:12 2020 | |
| read: IOPS=44.1k, BW=2756MiB/s (2890MB/s)(323GiB/120014msec) | |
| slat (usec): min=10, max=106477, avg=153.17, stdev=737.49 | |
| clat (usec): min=2, max=171959, avg=11421.81, stdev=9856.91 | |
| lat (usec): min=36, max=172017, avg=11575.46, stdev=9983.50 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 5], 10.00th=[ 5], 20.00th=[ 6], | |
| | 30.00th=[ 6], 40.00th=[ 7], 50.00th=[ 8], 60.00th=[ 9], | |
| | 70.00th=[ 12], 80.00th=[ 16], 90.00th=[ 24], 95.00th=[ 32], | |
| | 99.00th=[ 51], 99.50th=[ 59], 99.90th=[ 81], 99.95th=[ 89], | |
| | 99.99th=[ 118] | |
| bw ( MiB/s): min= 1450, max= 5092, per=99.95%, avg=2755.03, stdev=44.70, samples=3832 | |
| iops : min=23209, max=81475, avg=44079.16, stdev=715.24, samples=3832 | |
| write: IOPS=44.1k, BW=2758MiB/s (2892MB/s)(323GiB/120014msec); 0 zone resets | |
| slat (usec): min=19, max=112806, avg=200.54, stdev=856.27 | |
| clat (usec): min=2, max=171770, avg=11424.16, stdev=9842.11 | |
| lat (usec): min=64, max=171899, avg=11625.22, stdev=9949.02 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 5], 10.00th=[ 5], 20.00th=[ 6], | |
| | 30.00th=[ 6], 40.00th=[ 7], 50.00th=[ 8], 60.00th=[ 9], | |
| | 70.00th=[ 12], 80.00th=[ 16], 90.00th=[ 24], 95.00th=[ 32], | |
| | 99.00th=[ 51], 99.50th=[ 59], 99.90th=[ 81], 99.95th=[ 89], | |
| | 99.99th=[ 118] | |
| bw ( MiB/s): min= 1478, max= 5103, per=99.96%, avg=2757.21, stdev=44.64, samples=3832 | |
| iops : min=23651, max=81653, avg=44113.98, stdev=714.30, samples=3832 | |
| lat (usec) : 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01% | |
| lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=0.01%, 4=2.52%, 10=62.60%, 20=21.30%, 50=12.51% | |
| lat (msec) : 100=1.04%, 250=0.03% | |
| cpu : usr=3.83%, sys=51.92%, ctx=1075326, majf=0, minf=569461 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=5292766,5296829,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=2756MiB/s (2890MB/s), 2756MiB/s-2756MiB/s (2890MB/s-2890MB/s), io=323GiB (347GB), run=120014-120014msec | |
| WRITE: bw=2758MiB/s (2892MB/s), 2758MiB/s-2758MiB/s (2892MB/s-2892MB/s), io=323GiB (347GB), run=120014-120014msec | |
| + rm -rf /srv/z/none/test-none-64kchunk-16thread.0.0 /srv/z/none/test-none-64kchunk-16thread.0.1 /srv/z/none/test-none-64kchunk-16thread.0.10 /srv/z/none/test-none-64kchunk-16thread.0.11 /srv/z/none/test-none-64kchunk-16thread.0.12 /srv/z/none/test-none-64kchunk-16thread.0.13 /srv/z/none/test-none-64kchunk-16thread.0.14 /srv/z/none/test-none-64kchunk-16thread.0.15 /srv/z/none/test-none-64kchunk-16thread.0.2 /srv/z/none/test-none-64kchunk-16thread.0.3 /srv/z/none/test-none-64kchunk-16thread.0.4 /srv/z/none/test-none-64kchunk-16thread.0.5 /srv/z/none/test-none-64kchunk-16thread.0.6 /srv/z/none/test-none-64kchunk-16thread.0.7 /srv/z/none/test-none-64kchunk-16thread.0.8 /srv/z/none/test-none-64kchunk-16thread.0.9 /srv/z/none/test-none-64kchunk-16thread.1.0 /srv/z/none/test-none-64kchunk-16thread.10.0 /srv/z/none/test-none-64kchunk-16thread.10.1 /srv/z/none/test-none-64kchunk-16thread.10.10 /srv/z/none/test-none-64kchunk-16thread.10.11 /srv/z/none/test-none-64kchunk-16thread.10.12 /srv/z/none/test-none-64kchunk-16thread.10.13 /srv/z/none/test-none-64kchunk-16thread.10.14 /srv/z/none/test-none-64kchunk-16thread.10.15 /srv/z/none/test-none-64kchunk-16thread.10.2 /srv/z/none/test-none-64kchunk-16thread.10.3 /srv/z/none/test-none-64kchunk-16thread.10.4 /srv/z/none/test-none-64kchunk-16thread.10.5 /srv/z/none/test-none-64kchunk-16thread.10.6 /srv/z/none/test-none-64kchunk-16thread.10.7 /srv/z/none/test-none-64kchunk-16thread.10.8 /srv/z/none/test-none-64kchunk-16thread.10.9 /srv/z/none/test-none-64kchunk-16thread.1.1 /srv/z/none/test-none-64kchunk-16thread.1.10 /srv/z/none/test-none-64kchunk-16thread.11.0 /srv/z/none/test-none-64kchunk-16thread.1.11 /srv/z/none/test-none-64kchunk-16thread.11.1 /srv/z/none/test-none-64kchunk-16thread.11.10 /srv/z/none/test-none-64kchunk-16thread.11.11 /srv/z/none/test-none-64kchunk-16thread.11.12 /srv/z/none/test-none-64kchunk-16thread.11.13 /srv/z/none/test-none-64kchunk-16thread.11.14 /srv/z/none/test-none-64kchunk-16thread.11.15 /srv/z/none/test-none-64kchunk-16thread.1.12 /srv/z/none/test-none-64kchunk-16thread.11.2 /srv/z/none/test-none-64kchunk-16thread.1.13 /srv/z/none/test-none-64kchunk-16thread.11.3 /srv/z/none/test-none-64kchunk-16thread.1.14 /srv/z/none/test-none-64kchunk-16thread.11.4 /srv/z/none/test-none-64kchunk-16thread.1.15 /srv/z/none/test-none-64kchunk-16thread.11.5 /srv/z/none/test-none-64kchunk-16thread.11.6 /srv/z/none/test-none-64kchunk-16thread.11.7 /srv/z/none/test-none-64kchunk-16thread.11.8 /srv/z/none/test-none-64kchunk-16thread.11.9 /srv/z/none/test-none-64kchunk-16thread.1.2 /srv/z/none/test-none-64kchunk-16thread.12.0 /srv/z/none/test-none-64kchunk-16thread.12.1 /srv/z/none/test-none-64kchunk-16thread.12.10 /srv/z/none/test-none-64kchunk-16thread.12.11 /srv/z/none/test-none-64kchunk-16thread.12.12 /srv/z/none/test-none-64kchunk-16thread.12.13 /srv/z/none/test-none-64kchunk-16thread.12.14 /srv/z/none/test-none-64kchunk-16thread.12.15 /srv/z/none/test-none-64kchunk-16thread.12.2 /srv/z/none/test-none-64kchunk-16thread.12.3 /srv/z/none/test-none-64kchunk-16thread.12.4 /srv/z/none/test-none-64kchunk-16thread.12.5 /srv/z/none/test-none-64kchunk-16thread.12.6 /srv/z/none/test-none-64kchunk-16thread.12.7 /srv/z/none/test-none-64kchunk-16thread.12.8 /srv/z/none/test-none-64kchunk-16thread.12.9 /srv/z/none/test-none-64kchunk-16thread.1.3 /srv/z/none/test-none-64kchunk-16thread.13.0 /srv/z/none/test-none-64kchunk-16thread.13.1 /srv/z/none/test-none-64kchunk-16thread.13.10 /srv/z/none/test-none-64kchunk-16thread.13.11 /srv/z/none/test-none-64kchunk-16thread.13.12 /srv/z/none/test-none-64kchunk-16thread.13.13 /srv/z/none/test-none-64kchunk-16thread.13.14 /srv/z/none/test-none-64kchunk-16thread.13.15 /srv/z/none/test-none-64kchunk-16thread.13.2 /srv/z/none/test-none-64kchunk-16thread.13.3 /srv/z/none/test-none-64kchunk-16thread.13.4 /srv/z/none/test-none-64kchunk-16thread.13.5 /srv/z/none/test-none-64kchunk-16thread.13.6 /srv/z/none/test-none-64kchunk-16thread.13.7 /srv/z/none/test-none-64kchunk-16thread.13.8 /srv/z/none/test-none-64kchunk-16thread.13.9 /srv/z/none/test-none-64kchunk-16thread.1.4 /srv/z/none/test-none-64kchunk-16thread.14.0 /srv/z/none/test-none-64kchunk-16thread.14.1 /srv/z/none/test-none-64kchunk-16thread.14.10 /srv/z/none/test-none-64kchunk-16thread.14.11 /srv/z/none/test-none-64kchunk-16thread.14.12 /srv/z/none/test-none-64kchunk-16thread.14.13 /srv/z/none/test-none-64kchunk-16thread.14.14 /srv/z/none/test-none-64kchunk-16thread.14.15 /srv/z/none/test-none-64kchunk-16thread.14.2 /srv/z/none/test-none-64kchunk-16thread.14.3 /srv/z/none/test-none-64kchunk-16thread.14.4 /srv/z/none/test-none-64kchunk-16thread.14.5 /srv/z/none/test-none-64kchunk-16thread.14.6 /srv/z/none/test-none-64kchunk-16thread.14.7 /srv/z/none/test-none-64kchunk-16thread.14.8 /srv/z/none/test-none-64kchunk-16thread.14.9 /srv/z/none/test-none-64kchunk-16thread.1.5 /srv/z/none/test-none-64kchunk-16thread.15.0 /srv/z/none/test-none-64kchunk-16thread.15.1 /srv/z/none/test-none-64kchunk-16thread.15.10 /srv/z/none/test-none-64kchunk-16thread.15.11 /srv/z/none/test-none-64kchunk-16thread.15.12 /srv/z/none/test-none-64kchunk-16thread.15.13 /srv/z/none/test-none-64kchunk-16thread.15.14 /srv/z/none/test-none-64kchunk-16thread.15.15 /srv/z/none/test-none-64kchunk-16thread.15.2 /srv/z/none/test-none-64kchunk-16thread.15.3 /srv/z/none/test-none-64kchunk-16thread.15.4 /srv/z/none/test-none-64kchunk-16thread.15.5 /srv/z/none/test-none-64kchunk-16thread.15.6 /srv/z/none/test-none-64kchunk-16thread.15.7 /srv/z/none/test-none-64kchunk-16thread.15.8 /srv/z/none/test-none-64kchunk-16thread.15.9 /srv/z/none/test-none-64kchunk-16thread.1.6 /srv/z/none/test-none-64kchunk-16thread.1.7 /srv/z/none/test-none-64kchunk-16thread.1.8 /srv/z/none/test-none-64kchunk-16thread.1.9 /srv/z/none/test-none-64kchunk-16thread.2.0 /srv/z/none/test-none-64kchunk-16thread.2.1 /srv/z/none/test-none-64kchunk-16thread.2.10 /srv/z/none/test-none-64kchunk-16thread.2.11 /srv/z/none/test-none-64kchunk-16thread.2.12 /srv/z/none/test-none-64kchunk-16thread.2.13 /srv/z/none/test-none-64kchunk-16thread.2.14 /srv/z/none/test-none-64kchunk-16thread.2.15 /srv/z/none/test-none-64kchunk-16thread.2.2 /srv/z/none/test-none-64kchunk-16thread.2.3 /srv/z/none/test-none-64kchunk-16thread.2.4 /srv/z/none/test-none-64kchunk-16thread.2.5 /srv/z/none/test-none-64kchunk-16thread.2.6 /srv/z/none/test-none-64kchunk-16thread.2.7 /srv/z/none/test-none-64kchunk-16thread.2.8 /srv/z/none/test-none-64kchunk-16thread.2.9 /srv/z/none/test-none-64kchunk-16thread.3.0 /srv/z/none/test-none-64kchunk-16thread.3.1 /srv/z/none/test-none-64kchunk-16thread.3.10 /srv/z/none/test-none-64kchunk-16thread.3.11 /srv/z/none/test-none-64kchunk-16thread.3.12 /srv/z/none/test-none-64kchunk-16thread.3.13 /srv/z/none/test-none-64kchunk-16thread.3.14 /srv/z/none/test-none-64kchunk-16thread.3.15 /srv/z/none/test-none-64kchunk-16thread.3.2 /srv/z/none/test-none-64kchunk-16thread.3.3 /srv/z/none/test-none-64kchunk-16thread.3.4 /srv/z/none/test-none-64kchunk-16thread.3.5 /srv/z/none/test-none-64kchunk-16thread.3.6 /srv/z/none/test-none-64kchunk-16thread.3.7 /srv/z/none/test-none-64kchunk-16thread.3.8 /srv/z/none/test-none-64kchunk-16thread.3.9 /srv/z/none/test-none-64kchunk-16thread.4.0 /srv/z/none/test-none-64kchunk-16thread.4.1 /srv/z/none/test-none-64kchunk-16thread.4.10 /srv/z/none/test-none-64kchunk-16thread.4.11 /srv/z/none/test-none-64kchunk-16thread.4.12 /srv/z/none/test-none-64kchunk-16thread.4.13 /srv/z/none/test-none-64kchunk-16thread.4.14 /srv/z/none/test-none-64kchunk-16thread.4.15 /srv/z/none/test-none-64kchunk-16thread.4.2 /srv/z/none/test-none-64kchunk-16thread.4.3 /srv/z/none/test-none-64kchunk-16thread.4.4 /srv/z/none/test-none-64kchunk-16thread.4.5 /srv/z/none/test-none-64kchunk-16thread.4.6 /srv/z/none/test-none-64kchunk-16thread.4.7 /srv/z/none/test-none-64kchunk-16thread.4.8 /srv/z/none/test-none-64kchunk-16thread.4.9 /srv/z/none/test-none-64kchunk-16thread.5.0 /srv/z/none/test-none-64kchunk-16thread.5.1 /srv/z/none/test-none-64kchunk-16thread.5.10 /srv/z/none/test-none-64kchunk-16thread.5.11 /srv/z/none/test-none-64kchunk-16thread.5.12 /srv/z/none/test-none-64kchunk-16thread.5.13 /srv/z/none/test-none-64kchunk-16thread.5.14 /srv/z/none/test-none-64kchunk-16thread.5.15 /srv/z/none/test-none-64kchunk-16thread.5.2 /srv/z/none/test-none-64kchunk-16thread.5.3 /srv/z/none/test-none-64kchunk-16thread.5.4 /srv/z/none/test-none-64kchunk-16thread.5.5 /srv/z/none/test-none-64kchunk-16thread.5.6 /srv/z/none/test-none-64kchunk-16thread.5.7 /srv/z/none/test-none-64kchunk-16thread.5.8 /srv/z/none/test-none-64kchunk-16thread.5.9 /srv/z/none/test-none-64kchunk-16thread.6.0 /srv/z/none/test-none-64kchunk-16thread.6.1 /srv/z/none/test-none-64kchunk-16thread.6.10 /srv/z/none/test-none-64kchunk-16thread.6.11 /srv/z/none/test-none-64kchunk-16thread.6.12 /srv/z/none/test-none-64kchunk-16thread.6.13 /srv/z/none/test-none-64kchunk-16thread.6.14 /srv/z/none/test-none-64kchunk-16thread.6.15 /srv/z/none/test-none-64kchunk-16thread.6.2 /srv/z/none/test-none-64kchunk-16thread.6.3 /srv/z/none/test-none-64kchunk-16thread.6.4 /srv/z/none/test-none-64kchunk-16thread.6.5 /srv/z/none/test-none-64kchunk-16thread.6.6 /srv/z/none/test-none-64kchunk-16thread.6.7 /srv/z/none/test-none-64kchunk-16thread.6.8 /srv/z/none/test-none-64kchunk-16thread.6.9 /srv/z/none/test-none-64kchunk-16thread.7.0 /srv/z/none/test-none-64kchunk-16thread.7.1 /srv/z/none/test-none-64kchunk-16thread.7.10 /srv/z/none/test-none-64kchunk-16thread.7.11 /srv/z/none/test-none-64kchunk-16thread.7.12 /srv/z/none/test-none-64kchunk-16thread.7.13 /srv/z/none/test-none-64kchunk-16thread.7.14 /srv/z/none/test-none-64kchunk-16thread.7.15 /srv/z/none/test-none-64kchunk-16thread.7.2 /srv/z/none/test-none-64kchunk-16thread.7.3 /srv/z/none/test-none-64kchunk-16thread.7.4 /srv/z/none/test-none-64kchunk-16thread.7.5 /srv/z/none/test-none-64kchunk-16thread.7.6 /srv/z/none/test-none-64kchunk-16thread.7.7 /srv/z/none/test-none-64kchunk-16thread.7.8 /srv/z/none/test-none-64kchunk-16thread.7.9 /srv/z/none/test-none-64kchunk-16thread.8.0 /srv/z/none/test-none-64kchunk-16thread.8.1 /srv/z/none/test-none-64kchunk-16thread.8.10 /srv/z/none/test-none-64kchunk-16thread.8.11 /srv/z/none/test-none-64kchunk-16thread.8.12 /srv/z/none/test-none-64kchunk-16thread.8.13 /srv/z/none/test-none-64kchunk-16thread.8.14 /srv/z/none/test-none-64kchunk-16thread.8.15 /srv/z/none/test-none-64kchunk-16thread.8.2 /srv/z/none/test-none-64kchunk-16thread.8.3 /srv/z/none/test-none-64kchunk-16thread.8.4 /srv/z/none/test-none-64kchunk-16thread.8.5 /srv/z/none/test-none-64kchunk-16thread.8.6 /srv/z/none/test-none-64kchunk-16thread.8.7 /srv/z/none/test-none-64kchunk-16thread.8.8 /srv/z/none/test-none-64kchunk-16thread.8.9 /srv/z/none/test-none-64kchunk-16thread.9.0 /srv/z/none/test-none-64kchunk-16thread.9.1 /srv/z/none/test-none-64kchunk-16thread.9.10 /srv/z/none/test-none-64kchunk-16thread.9.11 /srv/z/none/test-none-64kchunk-16thread.9.12 /srv/z/none/test-none-64kchunk-16thread.9.13 /srv/z/none/test-none-64kchunk-16thread.9.14 /srv/z/none/test-none-64kchunk-16thread.9.15 /srv/z/none/test-none-64kchunk-16thread.9.2 /srv/z/none/test-none-64kchunk-16thread.9.3 /srv/z/none/test-none-64kchunk-16thread.9.4 /srv/z/none/test-none-64kchunk-16thread.9.5 /srv/z/none/test-none-64kchunk-16thread.9.6 /srv/z/none/test-none-64kchunk-16thread.9.7 /srv/z/none/test-none-64kchunk-16thread.9.8 /srv/z/none/test-none-64kchunk-16thread.9.9 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-comp-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/comp/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-comp-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=2985MiB/s,w=2982MiB/s][r=47.8k,w=47.7k IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.5%][r=2625MiB/s,w=2636MiB/s][r=41.0k,w=42.2k IOPS][eta 01m:41s] Jobs: 16 (f=256): [M(16)][24.2%][r=2656MiB/s,w=2664MiB/s][r=42.5k,w=42.6k IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.8%][r=2822MiB/s,w=2824MiB/s][r=45.1k,w=45.2k IOPS][eta 01m:20s] Jobs: 16 (f=256): [M(16)][40.0%][r=2894MiB/s,w=2894MiB/s][r=46.3k,w=46.3k IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][48.3%][r=2926MiB/s,w=2916MiB/s][r=46.8k,w=46.7k IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][56.2%][r=2881MiB/s,w=2885MiB/s][r=46.1k,w=46.2k IOPS][eta 00m:53s] | |
| Jobs: 16 (f=256): [M(16)][64.7%][r=2717MiB/s,w=2718MiB/s][r=43.5k,w=43.5k IOPS][eta 00m:42s] Jobs: 16 (f=256): [M(16)][71.9%][r=2781MiB/s,w=2786MiB/s][r=44.5k,w=44.6k IOPS][eta 00m:34s] | |
| Jobs: 16 (f=256): [M(16)][80.0%][r=2561MiB/s,w=2562MiB/s][r=40.0k,w=40.0k IOPS][eta 00m:24s] Jobs: 16 (f=256): [M(16)][88.3%][r=2524MiB/s,w=2532MiB/s][r=40.4k,w=40.5k IOPS][eta 00m:14s] | |
| Jobs: 16 (f=256): [M(16)][95.8%][r=2536MiB/s,w=2536MiB/s][r=40.6k,w=40.6k IOPS][eta 00m:05s] | |
| test-comp-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=518136: Wed Feb 26 12:02:56 2020 | |
| read: IOPS=43.7k, BW=2733MiB/s (2866MB/s)(320GiB/120001msec) | |
| slat (usec): min=10, max=177343, avg=158.19, stdev=969.01 | |
| clat (usec): min=2, max=249073, avg=11513.55, stdev=11825.49 | |
| lat (usec): min=19, max=250116, avg=11672.15, stdev=11969.74 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 4], 10.00th=[ 4], 20.00th=[ 5], | |
| | 30.00th=[ 6], 40.00th=[ 6], 50.00th=[ 7], 60.00th=[ 9], | |
| | 70.00th=[ 12], 80.00th=[ 17], 90.00th=[ 26], 95.00th=[ 35], | |
| | 99.00th=[ 59], 99.50th=[ 71], 99.90th=[ 107], 99.95th=[ 124], | |
| | 99.99th=[ 155] | |
| bw ( MiB/s): min= 1223, max= 5292, per=99.84%, avg=2728.94, stdev=49.01, samples=3826 | |
| iops : min=19570, max=84676, avg=43660.87, stdev=784.19, samples=3826 | |
| write: IOPS=43.8k, BW=2735MiB/s (2868MB/s)(321GiB/120001msec); 0 zone resets | |
| slat (usec): min=18, max=175384, avg=198.98, stdev=1100.77 | |
| clat (usec): min=3, max=250031, avg=11526.85, stdev=11824.28 | |
| lat (usec): min=58, max=250136, avg=11726.30, stdev=11947.97 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 4], 10.00th=[ 4], 20.00th=[ 5], | |
| | 30.00th=[ 6], 40.00th=[ 6], 50.00th=[ 7], 60.00th=[ 9], | |
| | 70.00th=[ 12], 80.00th=[ 17], 90.00th=[ 26], 95.00th=[ 35], | |
| | 99.00th=[ 59], 99.50th=[ 71], 99.90th=[ 107], 99.95th=[ 124], | |
| | 99.99th=[ 157] | |
| bw ( MiB/s): min= 1227, max= 5303, per=99.84%, avg=2730.96, stdev=48.91, samples=3826 | |
| iops : min=19632, max=84853, avg=43693.20, stdev=782.55, samples=3826 | |
| lat (usec) : 4=0.01%, 10=0.01%, 50=0.01%, 100=0.01%, 250=0.01% | |
| lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=0.01%, 4=10.89%, 10=53.95%, 20=20.42%, 50=13.00% | |
| lat (msec) : 100=1.59%, 250=0.13%, 500=0.01% | |
| cpu : usr=2.90%, sys=40.76%, ctx=1760282, majf=0, minf=435631 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=5248177,5251819,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=2733MiB/s (2866MB/s), 2733MiB/s-2733MiB/s (2866MB/s-2866MB/s), io=320GiB (344GB), run=120001-120001msec | |
| WRITE: bw=2735MiB/s (2868MB/s), 2735MiB/s-2735MiB/s (2868MB/s-2868MB/s), io=321GiB (344GB), run=120001-120001msec | |
| + rm -rf /srv/z/comp/test-comp-64kchunk-16thread.0.0 /srv/z/comp/test-comp-64kchunk-16thread.0.1 /srv/z/comp/test-comp-64kchunk-16thread.0.10 /srv/z/comp/test-comp-64kchunk-16thread.0.11 /srv/z/comp/test-comp-64kchunk-16thread.0.12 /srv/z/comp/test-comp-64kchunk-16thread.0.13 /srv/z/comp/test-comp-64kchunk-16thread.0.14 /srv/z/comp/test-comp-64kchunk-16thread.0.15 /srv/z/comp/test-comp-64kchunk-16thread.0.2 /srv/z/comp/test-comp-64kchunk-16thread.0.3 /srv/z/comp/test-comp-64kchunk-16thread.0.4 /srv/z/comp/test-comp-64kchunk-16thread.0.5 /srv/z/comp/test-comp-64kchunk-16thread.0.6 /srv/z/comp/test-comp-64kchunk-16thread.0.7 /srv/z/comp/test-comp-64kchunk-16thread.0.8 /srv/z/comp/test-comp-64kchunk-16thread.0.9 /srv/z/comp/test-comp-64kchunk-16thread.1.0 /srv/z/comp/test-comp-64kchunk-16thread.10.0 /srv/z/comp/test-comp-64kchunk-16thread.10.1 /srv/z/comp/test-comp-64kchunk-16thread.10.10 /srv/z/comp/test-comp-64kchunk-16thread.10.11 /srv/z/comp/test-comp-64kchunk-16thread.10.12 /srv/z/comp/test-comp-64kchunk-16thread.10.13 /srv/z/comp/test-comp-64kchunk-16thread.10.14 /srv/z/comp/test-comp-64kchunk-16thread.10.15 /srv/z/comp/test-comp-64kchunk-16thread.10.2 /srv/z/comp/test-comp-64kchunk-16thread.10.3 /srv/z/comp/test-comp-64kchunk-16thread.10.4 /srv/z/comp/test-comp-64kchunk-16thread.10.5 /srv/z/comp/test-comp-64kchunk-16thread.10.6 /srv/z/comp/test-comp-64kchunk-16thread.10.7 /srv/z/comp/test-comp-64kchunk-16thread.10.8 /srv/z/comp/test-comp-64kchunk-16thread.10.9 /srv/z/comp/test-comp-64kchunk-16thread.1.1 /srv/z/comp/test-comp-64kchunk-16thread.1.10 /srv/z/comp/test-comp-64kchunk-16thread.11.0 /srv/z/comp/test-comp-64kchunk-16thread.1.11 /srv/z/comp/test-comp-64kchunk-16thread.11.1 /srv/z/comp/test-comp-64kchunk-16thread.11.10 /srv/z/comp/test-comp-64kchunk-16thread.11.11 /srv/z/comp/test-comp-64kchunk-16thread.11.12 /srv/z/comp/test-comp-64kchunk-16thread.11.13 /srv/z/comp/test-comp-64kchunk-16thread.11.14 /srv/z/comp/test-comp-64kchunk-16thread.11.15 /srv/z/comp/test-comp-64kchunk-16thread.1.12 /srv/z/comp/test-comp-64kchunk-16thread.11.2 /srv/z/comp/test-comp-64kchunk-16thread.1.13 /srv/z/comp/test-comp-64kchunk-16thread.11.3 /srv/z/comp/test-comp-64kchunk-16thread.1.14 /srv/z/comp/test-comp-64kchunk-16thread.11.4 /srv/z/comp/test-comp-64kchunk-16thread.1.15 /srv/z/comp/test-comp-64kchunk-16thread.11.5 /srv/z/comp/test-comp-64kchunk-16thread.11.6 /srv/z/comp/test-comp-64kchunk-16thread.11.7 /srv/z/comp/test-comp-64kchunk-16thread.11.8 /srv/z/comp/test-comp-64kchunk-16thread.11.9 /srv/z/comp/test-comp-64kchunk-16thread.1.2 /srv/z/comp/test-comp-64kchunk-16thread.12.0 /srv/z/comp/test-comp-64kchunk-16thread.12.1 /srv/z/comp/test-comp-64kchunk-16thread.12.10 /srv/z/comp/test-comp-64kchunk-16thread.12.11 /srv/z/comp/test-comp-64kchunk-16thread.12.12 /srv/z/comp/test-comp-64kchunk-16thread.12.13 /srv/z/comp/test-comp-64kchunk-16thread.12.14 /srv/z/comp/test-comp-64kchunk-16thread.12.15 /srv/z/comp/test-comp-64kchunk-16thread.12.2 /srv/z/comp/test-comp-64kchunk-16thread.12.3 /srv/z/comp/test-comp-64kchunk-16thread.12.4 /srv/z/comp/test-comp-64kchunk-16thread.12.5 /srv/z/comp/test-comp-64kchunk-16thread.12.6 /srv/z/comp/test-comp-64kchunk-16thread.12.7 /srv/z/comp/test-comp-64kchunk-16thread.12.8 /srv/z/comp/test-comp-64kchunk-16thread.12.9 /srv/z/comp/test-comp-64kchunk-16thread.1.3 /srv/z/comp/test-comp-64kchunk-16thread.13.0 /srv/z/comp/test-comp-64kchunk-16thread.13.1 /srv/z/comp/test-comp-64kchunk-16thread.13.10 /srv/z/comp/test-comp-64kchunk-16thread.13.11 /srv/z/comp/test-comp-64kchunk-16thread.13.12 /srv/z/comp/test-comp-64kchunk-16thread.13.13 /srv/z/comp/test-comp-64kchunk-16thread.13.14 /srv/z/comp/test-comp-64kchunk-16thread.13.15 /srv/z/comp/test-comp-64kchunk-16thread.13.2 /srv/z/comp/test-comp-64kchunk-16thread.13.3 /srv/z/comp/test-comp-64kchunk-16thread.13.4 /srv/z/comp/test-comp-64kchunk-16thread.13.5 /srv/z/comp/test-comp-64kchunk-16thread.13.6 /srv/z/comp/test-comp-64kchunk-16thread.13.7 /srv/z/comp/test-comp-64kchunk-16thread.13.8 /srv/z/comp/test-comp-64kchunk-16thread.13.9 /srv/z/comp/test-comp-64kchunk-16thread.1.4 /srv/z/comp/test-comp-64kchunk-16thread.14.0 /srv/z/comp/test-comp-64kchunk-16thread.14.1 /srv/z/comp/test-comp-64kchunk-16thread.14.10 /srv/z/comp/test-comp-64kchunk-16thread.14.11 /srv/z/comp/test-comp-64kchunk-16thread.14.12 /srv/z/comp/test-comp-64kchunk-16thread.14.13 /srv/z/comp/test-comp-64kchunk-16thread.14.14 /srv/z/comp/test-comp-64kchunk-16thread.14.15 /srv/z/comp/test-comp-64kchunk-16thread.14.2 /srv/z/comp/test-comp-64kchunk-16thread.14.3 /srv/z/comp/test-comp-64kchunk-16thread.14.4 /srv/z/comp/test-comp-64kchunk-16thread.14.5 /srv/z/comp/test-comp-64kchunk-16thread.14.6 /srv/z/comp/test-comp-64kchunk-16thread.14.7 /srv/z/comp/test-comp-64kchunk-16thread.14.8 /srv/z/comp/test-comp-64kchunk-16thread.14.9 /srv/z/comp/test-comp-64kchunk-16thread.1.5 /srv/z/comp/test-comp-64kchunk-16thread.15.0 /srv/z/comp/test-comp-64kchunk-16thread.15.1 /srv/z/comp/test-comp-64kchunk-16thread.15.10 /srv/z/comp/test-comp-64kchunk-16thread.15.11 /srv/z/comp/test-comp-64kchunk-16thread.15.12 /srv/z/comp/test-comp-64kchunk-16thread.15.13 /srv/z/comp/test-comp-64kchunk-16thread.15.14 /srv/z/comp/test-comp-64kchunk-16thread.15.15 /srv/z/comp/test-comp-64kchunk-16thread.15.2 /srv/z/comp/test-comp-64kchunk-16thread.15.3 /srv/z/comp/test-comp-64kchunk-16thread.15.4 /srv/z/comp/test-comp-64kchunk-16thread.15.5 /srv/z/comp/test-comp-64kchunk-16thread.15.6 /srv/z/comp/test-comp-64kchunk-16thread.15.7 /srv/z/comp/test-comp-64kchunk-16thread.15.8 /srv/z/comp/test-comp-64kchunk-16thread.15.9 /srv/z/comp/test-comp-64kchunk-16thread.1.6 /srv/z/comp/test-comp-64kchunk-16thread.1.7 /srv/z/comp/test-comp-64kchunk-16thread.1.8 /srv/z/comp/test-comp-64kchunk-16thread.1.9 /srv/z/comp/test-comp-64kchunk-16thread.2.0 /srv/z/comp/test-comp-64kchunk-16thread.2.1 /srv/z/comp/test-comp-64kchunk-16thread.2.10 /srv/z/comp/test-comp-64kchunk-16thread.2.11 /srv/z/comp/test-comp-64kchunk-16thread.2.12 /srv/z/comp/test-comp-64kchunk-16thread.2.13 /srv/z/comp/test-comp-64kchunk-16thread.2.14 /srv/z/comp/test-comp-64kchunk-16thread.2.15 /srv/z/comp/test-comp-64kchunk-16thread.2.2 /srv/z/comp/test-comp-64kchunk-16thread.2.3 /srv/z/comp/test-comp-64kchunk-16thread.2.4 /srv/z/comp/test-comp-64kchunk-16thread.2.5 /srv/z/comp/test-comp-64kchunk-16thread.2.6 /srv/z/comp/test-comp-64kchunk-16thread.2.7 /srv/z/comp/test-comp-64kchunk-16thread.2.8 /srv/z/comp/test-comp-64kchunk-16thread.2.9 /srv/z/comp/test-comp-64kchunk-16thread.3.0 /srv/z/comp/test-comp-64kchunk-16thread.3.1 /srv/z/comp/test-comp-64kchunk-16thread.3.10 /srv/z/comp/test-comp-64kchunk-16thread.3.11 /srv/z/comp/test-comp-64kchunk-16thread.3.12 /srv/z/comp/test-comp-64kchunk-16thread.3.13 /srv/z/comp/test-comp-64kchunk-16thread.3.14 /srv/z/comp/test-comp-64kchunk-16thread.3.15 /srv/z/comp/test-comp-64kchunk-16thread.3.2 /srv/z/comp/test-comp-64kchunk-16thread.3.3 /srv/z/comp/test-comp-64kchunk-16thread.3.4 /srv/z/comp/test-comp-64kchunk-16thread.3.5 /srv/z/comp/test-comp-64kchunk-16thread.3.6 /srv/z/comp/test-comp-64kchunk-16thread.3.7 /srv/z/comp/test-comp-64kchunk-16thread.3.8 /srv/z/comp/test-comp-64kchunk-16thread.3.9 /srv/z/comp/test-comp-64kchunk-16thread.4.0 /srv/z/comp/test-comp-64kchunk-16thread.4.1 /srv/z/comp/test-comp-64kchunk-16thread.4.10 /srv/z/comp/test-comp-64kchunk-16thread.4.11 /srv/z/comp/test-comp-64kchunk-16thread.4.12 /srv/z/comp/test-comp-64kchunk-16thread.4.13 /srv/z/comp/test-comp-64kchunk-16thread.4.14 /srv/z/comp/test-comp-64kchunk-16thread.4.15 /srv/z/comp/test-comp-64kchunk-16thread.4.2 /srv/z/comp/test-comp-64kchunk-16thread.4.3 /srv/z/comp/test-comp-64kchunk-16thread.4.4 /srv/z/comp/test-comp-64kchunk-16thread.4.5 /srv/z/comp/test-comp-64kchunk-16thread.4.6 /srv/z/comp/test-comp-64kchunk-16thread.4.7 /srv/z/comp/test-comp-64kchunk-16thread.4.8 /srv/z/comp/test-comp-64kchunk-16thread.4.9 /srv/z/comp/test-comp-64kchunk-16thread.5.0 /srv/z/comp/test-comp-64kchunk-16thread.5.1 /srv/z/comp/test-comp-64kchunk-16thread.5.10 /srv/z/comp/test-comp-64kchunk-16thread.5.11 /srv/z/comp/test-comp-64kchunk-16thread.5.12 /srv/z/comp/test-comp-64kchunk-16thread.5.13 /srv/z/comp/test-comp-64kchunk-16thread.5.14 /srv/z/comp/test-comp-64kchunk-16thread.5.15 /srv/z/comp/test-comp-64kchunk-16thread.5.2 /srv/z/comp/test-comp-64kchunk-16thread.5.3 /srv/z/comp/test-comp-64kchunk-16thread.5.4 /srv/z/comp/test-comp-64kchunk-16thread.5.5 /srv/z/comp/test-comp-64kchunk-16thread.5.6 /srv/z/comp/test-comp-64kchunk-16thread.5.7 /srv/z/comp/test-comp-64kchunk-16thread.5.8 /srv/z/comp/test-comp-64kchunk-16thread.5.9 /srv/z/comp/test-comp-64kchunk-16thread.6.0 /srv/z/comp/test-comp-64kchunk-16thread.6.1 /srv/z/comp/test-comp-64kchunk-16thread.6.10 /srv/z/comp/test-comp-64kchunk-16thread.6.11 /srv/z/comp/test-comp-64kchunk-16thread.6.12 /srv/z/comp/test-comp-64kchunk-16thread.6.13 /srv/z/comp/test-comp-64kchunk-16thread.6.14 /srv/z/comp/test-comp-64kchunk-16thread.6.15 /srv/z/comp/test-comp-64kchunk-16thread.6.2 /srv/z/comp/test-comp-64kchunk-16thread.6.3 /srv/z/comp/test-comp-64kchunk-16thread.6.4 /srv/z/comp/test-comp-64kchunk-16thread.6.5 /srv/z/comp/test-comp-64kchunk-16thread.6.6 /srv/z/comp/test-comp-64kchunk-16thread.6.7 /srv/z/comp/test-comp-64kchunk-16thread.6.8 /srv/z/comp/test-comp-64kchunk-16thread.6.9 /srv/z/comp/test-comp-64kchunk-16thread.7.0 /srv/z/comp/test-comp-64kchunk-16thread.7.1 /srv/z/comp/test-comp-64kchunk-16thread.7.10 /srv/z/comp/test-comp-64kchunk-16thread.7.11 /srv/z/comp/test-comp-64kchunk-16thread.7.12 /srv/z/comp/test-comp-64kchunk-16thread.7.13 /srv/z/comp/test-comp-64kchunk-16thread.7.14 /srv/z/comp/test-comp-64kchunk-16thread.7.15 /srv/z/comp/test-comp-64kchunk-16thread.7.2 /srv/z/comp/test-comp-64kchunk-16thread.7.3 /srv/z/comp/test-comp-64kchunk-16thread.7.4 /srv/z/comp/test-comp-64kchunk-16thread.7.5 /srv/z/comp/test-comp-64kchunk-16thread.7.6 /srv/z/comp/test-comp-64kchunk-16thread.7.7 /srv/z/comp/test-comp-64kchunk-16thread.7.8 /srv/z/comp/test-comp-64kchunk-16thread.7.9 /srv/z/comp/test-comp-64kchunk-16thread.8.0 /srv/z/comp/test-comp-64kchunk-16thread.8.1 /srv/z/comp/test-comp-64kchunk-16thread.8.10 /srv/z/comp/test-comp-64kchunk-16thread.8.11 /srv/z/comp/test-comp-64kchunk-16thread.8.12 /srv/z/comp/test-comp-64kchunk-16thread.8.13 /srv/z/comp/test-comp-64kchunk-16thread.8.14 /srv/z/comp/test-comp-64kchunk-16thread.8.15 /srv/z/comp/test-comp-64kchunk-16thread.8.2 /srv/z/comp/test-comp-64kchunk-16thread.8.3 /srv/z/comp/test-comp-64kchunk-16thread.8.4 /srv/z/comp/test-comp-64kchunk-16thread.8.5 /srv/z/comp/test-comp-64kchunk-16thread.8.6 /srv/z/comp/test-comp-64kchunk-16thread.8.7 /srv/z/comp/test-comp-64kchunk-16thread.8.8 /srv/z/comp/test-comp-64kchunk-16thread.8.9 /srv/z/comp/test-comp-64kchunk-16thread.9.0 /srv/z/comp/test-comp-64kchunk-16thread.9.1 /srv/z/comp/test-comp-64kchunk-16thread.9.10 /srv/z/comp/test-comp-64kchunk-16thread.9.11 /srv/z/comp/test-comp-64kchunk-16thread.9.12 /srv/z/comp/test-comp-64kchunk-16thread.9.13 /srv/z/comp/test-comp-64kchunk-16thread.9.14 /srv/z/comp/test-comp-64kchunk-16thread.9.15 /srv/z/comp/test-comp-64kchunk-16thread.9.2 /srv/z/comp/test-comp-64kchunk-16thread.9.3 /srv/z/comp/test-comp-64kchunk-16thread.9.4 /srv/z/comp/test-comp-64kchunk-16thread.9.5 /srv/z/comp/test-comp-64kchunk-16thread.9.6 /srv/z/comp/test-comp-64kchunk-16thread.9.7 /srv/z/comp/test-comp-64kchunk-16thread.9.8 /srv/z/comp/test-comp-64kchunk-16thread.9.9 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-enc-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/enc/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-enc-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=421MiB/s,w=422MiB/s][r=6733,w=6754 IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.7%][r=375MiB/s,w=378MiB/s][r=5999,w=6054 IOPS][eta 01m:40s] Jobs: 16 (f=256): [M(16)][24.2%][r=328MiB/s,w=329MiB/s][r=5244,w=5266 IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.5%][r=328MiB/s,w=323MiB/s][r=5253,w=5170 IOPS][eta 01m:21s] Jobs: 16 (f=256): [M(16)][40.5%][r=339MiB/s,w=339MiB/s][r=5422,w=5418 IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][48.3%][r=331MiB/s,w=332MiB/s][r=5296,w=5307 IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][56.2%][r=355MiB/s,w=355MiB/s][r=5677,w=5686 IOPS][eta 00m:53s] | |
| Jobs: 16 (f=256): [M(16)][64.2%][r=352MiB/s,w=350MiB/s][r=5631,w=5607 IOPS][eta 00m:43s] Jobs: 16 (f=256): [M(16)][72.5%][r=371MiB/s,w=372MiB/s][r=5936,w=5947 IOPS][eta 00m:33s] | |
| Jobs: 16 (f=256): [M(16)][80.0%][r=333MiB/s,w=330MiB/s][r=5327,w=5287 IOPS][eta 00m:24s] Jobs: 16 (f=256): [M(16)][88.3%][r=343MiB/s,w=346MiB/s][r=5481,w=5532 IOPS][eta 00m:14s] | |
| Jobs: 16 (f=256): [M(16)][95.9%][r=330MiB/s,w=328MiB/s][r=5286,w=5243 IOPS][eta 00m:05s] | |
| test-enc-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=2570477: Wed Feb 26 12:25:43 2020 | |
| read: IOPS=5615, BW=351MiB/s (368MB/s)(41.1GiB/120010msec) | |
| slat (usec): min=9, max=173739, avg=1395.80, stdev=3507.39 | |
| clat (usec): min=4, max=495364, avg=89620.01, stdev=52890.57 | |
| lat (msec): min=2, max=495, avg=91.02, stdev=53.58 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 14], 10.00th=[ 23], 20.00th=[ 43], | |
| | 30.00th=[ 63], 40.00th=[ 75], 50.00th=[ 86], 60.00th=[ 95], | |
| | 70.00th=[ 107], 80.00th=[ 128], 90.00th=[ 165], 95.00th=[ 190], | |
| | 99.00th=[ 239], 99.50th=[ 257], 99.90th=[ 300], 99.95th=[ 321], | |
| | 99.99th=[ 409] | |
| bw ( KiB/s): min=124096, max=995581, per=99.77%, avg=358577.09, stdev=14160.81, samples=3839 | |
| iops : min= 1934, max=15555, avg=5600.56, stdev=221.29, samples=3839 | |
| write: IOPS=5616, BW=351MiB/s (368MB/s)(41.1GiB/120010msec); 0 zone resets | |
| slat (usec): min=15, max=153168, avg=1444.89, stdev=3793.71 | |
| clat (usec): min=3, max=495323, avg=89681.04, stdev=52953.16 | |
| lat (msec): min=2, max=495, avg=91.13, stdev=53.70 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 14], 10.00th=[ 23], 20.00th=[ 43], | |
| | 30.00th=[ 63], 40.00th=[ 75], 50.00th=[ 86], 60.00th=[ 95], | |
| | 70.00th=[ 107], 80.00th=[ 128], 90.00th=[ 165], 95.00th=[ 192], | |
| | 99.00th=[ 239], 99.50th=[ 255], 99.90th=[ 300], 99.95th=[ 321], | |
| | 99.99th=[ 397] | |
| bw ( KiB/s): min=129120, max=992319, per=99.77%, avg=358634.80, stdev=14123.83, samples=3839 | |
| iops : min= 2013, max=15504, avg=5601.51, stdev=220.72, samples=3839 | |
| lat (usec) : 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% | |
| lat (msec) : 4=1.64%, 10=1.83%, 20=4.99%, 50=15.15%, 100=41.26% | |
| lat (msec) : 250=34.50%, 500=0.63% | |
| cpu : usr=0.32%, sys=39.30%, ctx=878527, majf=0, minf=446983 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=673932,674045,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=351MiB/s (368MB/s), 351MiB/s-351MiB/s (368MB/s-368MB/s), io=41.1GiB (44.2GB), run=120010-120010msec | |
| WRITE: bw=351MiB/s (368MB/s), 351MiB/s-351MiB/s (368MB/s-368MB/s), io=41.1GiB (44.2GB), run=120010-120010msec | |
| + rm -rf /srv/z/enc/test-enc-64kchunk-16thread.0.0 /srv/z/enc/test-enc-64kchunk-16thread.0.1 /srv/z/enc/test-enc-64kchunk-16thread.0.10 /srv/z/enc/test-enc-64kchunk-16thread.0.11 /srv/z/enc/test-enc-64kchunk-16thread.0.12 /srv/z/enc/test-enc-64kchunk-16thread.0.13 /srv/z/enc/test-enc-64kchunk-16thread.0.14 /srv/z/enc/test-enc-64kchunk-16thread.0.15 /srv/z/enc/test-enc-64kchunk-16thread.0.2 /srv/z/enc/test-enc-64kchunk-16thread.0.3 /srv/z/enc/test-enc-64kchunk-16thread.0.4 /srv/z/enc/test-enc-64kchunk-16thread.0.5 /srv/z/enc/test-enc-64kchunk-16thread.0.6 /srv/z/enc/test-enc-64kchunk-16thread.0.7 /srv/z/enc/test-enc-64kchunk-16thread.0.8 /srv/z/enc/test-enc-64kchunk-16thread.0.9 /srv/z/enc/test-enc-64kchunk-16thread.1.0 /srv/z/enc/test-enc-64kchunk-16thread.10.0 /srv/z/enc/test-enc-64kchunk-16thread.10.1 /srv/z/enc/test-enc-64kchunk-16thread.10.10 /srv/z/enc/test-enc-64kchunk-16thread.10.11 /srv/z/enc/test-enc-64kchunk-16thread.10.12 /srv/z/enc/test-enc-64kchunk-16thread.10.13 /srv/z/enc/test-enc-64kchunk-16thread.10.14 /srv/z/enc/test-enc-64kchunk-16thread.10.15 /srv/z/enc/test-enc-64kchunk-16thread.10.2 /srv/z/enc/test-enc-64kchunk-16thread.10.3 /srv/z/enc/test-enc-64kchunk-16thread.10.4 /srv/z/enc/test-enc-64kchunk-16thread.10.5 /srv/z/enc/test-enc-64kchunk-16thread.10.6 /srv/z/enc/test-enc-64kchunk-16thread.10.7 /srv/z/enc/test-enc-64kchunk-16thread.10.8 /srv/z/enc/test-enc-64kchunk-16thread.10.9 /srv/z/enc/test-enc-64kchunk-16thread.1.1 /srv/z/enc/test-enc-64kchunk-16thread.1.10 /srv/z/enc/test-enc-64kchunk-16thread.11.0 /srv/z/enc/test-enc-64kchunk-16thread.1.11 /srv/z/enc/test-enc-64kchunk-16thread.11.1 /srv/z/enc/test-enc-64kchunk-16thread.11.10 /srv/z/enc/test-enc-64kchunk-16thread.11.11 /srv/z/enc/test-enc-64kchunk-16thread.11.12 /srv/z/enc/test-enc-64kchunk-16thread.11.13 /srv/z/enc/test-enc-64kchunk-16thread.11.14 /srv/z/enc/test-enc-64kchunk-16thread.11.15 /srv/z/enc/test-enc-64kchunk-16thread.1.12 /srv/z/enc/test-enc-64kchunk-16thread.11.2 /srv/z/enc/test-enc-64kchunk-16thread.1.13 /srv/z/enc/test-enc-64kchunk-16thread.11.3 /srv/z/enc/test-enc-64kchunk-16thread.1.14 /srv/z/enc/test-enc-64kchunk-16thread.11.4 /srv/z/enc/test-enc-64kchunk-16thread.1.15 /srv/z/enc/test-enc-64kchunk-16thread.11.5 /srv/z/enc/test-enc-64kchunk-16thread.11.6 /srv/z/enc/test-enc-64kchunk-16thread.11.7 /srv/z/enc/test-enc-64kchunk-16thread.11.8 /srv/z/enc/test-enc-64kchunk-16thread.11.9 /srv/z/enc/test-enc-64kchunk-16thread.1.2 /srv/z/enc/test-enc-64kchunk-16thread.12.0 /srv/z/enc/test-enc-64kchunk-16thread.12.1 /srv/z/enc/test-enc-64kchunk-16thread.12.10 /srv/z/enc/test-enc-64kchunk-16thread.12.11 /srv/z/enc/test-enc-64kchunk-16thread.12.12 /srv/z/enc/test-enc-64kchunk-16thread.12.13 /srv/z/enc/test-enc-64kchunk-16thread.12.14 /srv/z/enc/test-enc-64kchunk-16thread.12.15 /srv/z/enc/test-enc-64kchunk-16thread.12.2 /srv/z/enc/test-enc-64kchunk-16thread.12.3 /srv/z/enc/test-enc-64kchunk-16thread.12.4 /srv/z/enc/test-enc-64kchunk-16thread.12.5 /srv/z/enc/test-enc-64kchunk-16thread.12.6 /srv/z/enc/test-enc-64kchunk-16thread.12.7 /srv/z/enc/test-enc-64kchunk-16thread.12.8 /srv/z/enc/test-enc-64kchunk-16thread.12.9 /srv/z/enc/test-enc-64kchunk-16thread.1.3 /srv/z/enc/test-enc-64kchunk-16thread.13.0 /srv/z/enc/test-enc-64kchunk-16thread.13.1 /srv/z/enc/test-enc-64kchunk-16thread.13.10 /srv/z/enc/test-enc-64kchunk-16thread.13.11 /srv/z/enc/test-enc-64kchunk-16thread.13.12 /srv/z/enc/test-enc-64kchunk-16thread.13.13 /srv/z/enc/test-enc-64kchunk-16thread.13.14 /srv/z/enc/test-enc-64kchunk-16thread.13.15 /srv/z/enc/test-enc-64kchunk-16thread.13.2 /srv/z/enc/test-enc-64kchunk-16thread.13.3 /srv/z/enc/test-enc-64kchunk-16thread.13.4 /srv/z/enc/test-enc-64kchunk-16thread.13.5 /srv/z/enc/test-enc-64kchunk-16thread.13.6 /srv/z/enc/test-enc-64kchunk-16thread.13.7 /srv/z/enc/test-enc-64kchunk-16thread.13.8 /srv/z/enc/test-enc-64kchunk-16thread.13.9 /srv/z/enc/test-enc-64kchunk-16thread.1.4 /srv/z/enc/test-enc-64kchunk-16thread.14.0 /srv/z/enc/test-enc-64kchunk-16thread.14.1 /srv/z/enc/test-enc-64kchunk-16thread.14.10 /srv/z/enc/test-enc-64kchunk-16thread.14.11 /srv/z/enc/test-enc-64kchunk-16thread.14.12 /srv/z/enc/test-enc-64kchunk-16thread.14.13 /srv/z/enc/test-enc-64kchunk-16thread.14.14 /srv/z/enc/test-enc-64kchunk-16thread.14.15 /srv/z/enc/test-enc-64kchunk-16thread.14.2 /srv/z/enc/test-enc-64kchunk-16thread.14.3 /srv/z/enc/test-enc-64kchunk-16thread.14.4 /srv/z/enc/test-enc-64kchunk-16thread.14.5 /srv/z/enc/test-enc-64kchunk-16thread.14.6 /srv/z/enc/test-enc-64kchunk-16thread.14.7 /srv/z/enc/test-enc-64kchunk-16thread.14.8 /srv/z/enc/test-enc-64kchunk-16thread.14.9 /srv/z/enc/test-enc-64kchunk-16thread.1.5 /srv/z/enc/test-enc-64kchunk-16thread.15.0 /srv/z/enc/test-enc-64kchunk-16thread.15.1 /srv/z/enc/test-enc-64kchunk-16thread.15.10 /srv/z/enc/test-enc-64kchunk-16thread.15.11 /srv/z/enc/test-enc-64kchunk-16thread.15.12 /srv/z/enc/test-enc-64kchunk-16thread.15.13 /srv/z/enc/test-enc-64kchunk-16thread.15.14 /srv/z/enc/test-enc-64kchunk-16thread.15.15 /srv/z/enc/test-enc-64kchunk-16thread.15.2 /srv/z/enc/test-enc-64kchunk-16thread.15.3 /srv/z/enc/test-enc-64kchunk-16thread.15.4 /srv/z/enc/test-enc-64kchunk-16thread.15.5 /srv/z/enc/test-enc-64kchunk-16thread.15.6 /srv/z/enc/test-enc-64kchunk-16thread.15.7 /srv/z/enc/test-enc-64kchunk-16thread.15.8 /srv/z/enc/test-enc-64kchunk-16thread.15.9 /srv/z/enc/test-enc-64kchunk-16thread.1.6 /srv/z/enc/test-enc-64kchunk-16thread.1.7 /srv/z/enc/test-enc-64kchunk-16thread.1.8 /srv/z/enc/test-enc-64kchunk-16thread.1.9 /srv/z/enc/test-enc-64kchunk-16thread.2.0 /srv/z/enc/test-enc-64kchunk-16thread.2.1 /srv/z/enc/test-enc-64kchunk-16thread.2.10 /srv/z/enc/test-enc-64kchunk-16thread.2.11 /srv/z/enc/test-enc-64kchunk-16thread.2.12 /srv/z/enc/test-enc-64kchunk-16thread.2.13 /srv/z/enc/test-enc-64kchunk-16thread.2.14 /srv/z/enc/test-enc-64kchunk-16thread.2.15 /srv/z/enc/test-enc-64kchunk-16thread.2.2 /srv/z/enc/test-enc-64kchunk-16thread.2.3 /srv/z/enc/test-enc-64kchunk-16thread.2.4 /srv/z/enc/test-enc-64kchunk-16thread.2.5 /srv/z/enc/test-enc-64kchunk-16thread.2.6 /srv/z/enc/test-enc-64kchunk-16thread.2.7 /srv/z/enc/test-enc-64kchunk-16thread.2.8 /srv/z/enc/test-enc-64kchunk-16thread.2.9 /srv/z/enc/test-enc-64kchunk-16thread.3.0 /srv/z/enc/test-enc-64kchunk-16thread.3.1 /srv/z/enc/test-enc-64kchunk-16thread.3.10 /srv/z/enc/test-enc-64kchunk-16thread.3.11 /srv/z/enc/test-enc-64kchunk-16thread.3.12 /srv/z/enc/test-enc-64kchunk-16thread.3.13 /srv/z/enc/test-enc-64kchunk-16thread.3.14 /srv/z/enc/test-enc-64kchunk-16thread.3.15 /srv/z/enc/test-enc-64kchunk-16thread.3.2 /srv/z/enc/test-enc-64kchunk-16thread.3.3 /srv/z/enc/test-enc-64kchunk-16thread.3.4 /srv/z/enc/test-enc-64kchunk-16thread.3.5 /srv/z/enc/test-enc-64kchunk-16thread.3.6 /srv/z/enc/test-enc-64kchunk-16thread.3.7 /srv/z/enc/test-enc-64kchunk-16thread.3.8 /srv/z/enc/test-enc-64kchunk-16thread.3.9 /srv/z/enc/test-enc-64kchunk-16thread.4.0 /srv/z/enc/test-enc-64kchunk-16thread.4.1 /srv/z/enc/test-enc-64kchunk-16thread.4.10 /srv/z/enc/test-enc-64kchunk-16thread.4.11 /srv/z/enc/test-enc-64kchunk-16thread.4.12 /srv/z/enc/test-enc-64kchunk-16thread.4.13 /srv/z/enc/test-enc-64kchunk-16thread.4.14 /srv/z/enc/test-enc-64kchunk-16thread.4.15 /srv/z/enc/test-enc-64kchunk-16thread.4.2 /srv/z/enc/test-enc-64kchunk-16thread.4.3 /srv/z/enc/test-enc-64kchunk-16thread.4.4 /srv/z/enc/test-enc-64kchunk-16thread.4.5 /srv/z/enc/test-enc-64kchunk-16thread.4.6 /srv/z/enc/test-enc-64kchunk-16thread.4.7 /srv/z/enc/test-enc-64kchunk-16thread.4.8 /srv/z/enc/test-enc-64kchunk-16thread.4.9 /srv/z/enc/test-enc-64kchunk-16thread.5.0 /srv/z/enc/test-enc-64kchunk-16thread.5.1 /srv/z/enc/test-enc-64kchunk-16thread.5.10 /srv/z/enc/test-enc-64kchunk-16thread.5.11 /srv/z/enc/test-enc-64kchunk-16thread.5.12 /srv/z/enc/test-enc-64kchunk-16thread.5.13 /srv/z/enc/test-enc-64kchunk-16thread.5.14 /srv/z/enc/test-enc-64kchunk-16thread.5.15 /srv/z/enc/test-enc-64kchunk-16thread.5.2 /srv/z/enc/test-enc-64kchunk-16thread.5.3 /srv/z/enc/test-enc-64kchunk-16thread.5.4 /srv/z/enc/test-enc-64kchunk-16thread.5.5 /srv/z/enc/test-enc-64kchunk-16thread.5.6 /srv/z/enc/test-enc-64kchunk-16thread.5.7 /srv/z/enc/test-enc-64kchunk-16thread.5.8 /srv/z/enc/test-enc-64kchunk-16thread.5.9 /srv/z/enc/test-enc-64kchunk-16thread.6.0 /srv/z/enc/test-enc-64kchunk-16thread.6.1 /srv/z/enc/test-enc-64kchunk-16thread.6.10 /srv/z/enc/test-enc-64kchunk-16thread.6.11 /srv/z/enc/test-enc-64kchunk-16thread.6.12 /srv/z/enc/test-enc-64kchunk-16thread.6.13 /srv/z/enc/test-enc-64kchunk-16thread.6.14 /srv/z/enc/test-enc-64kchunk-16thread.6.15 /srv/z/enc/test-enc-64kchunk-16thread.6.2 /srv/z/enc/test-enc-64kchunk-16thread.6.3 /srv/z/enc/test-enc-64kchunk-16thread.6.4 /srv/z/enc/test-enc-64kchunk-16thread.6.5 /srv/z/enc/test-enc-64kchunk-16thread.6.6 /srv/z/enc/test-enc-64kchunk-16thread.6.7 /srv/z/enc/test-enc-64kchunk-16thread.6.8 /srv/z/enc/test-enc-64kchunk-16thread.6.9 /srv/z/enc/test-enc-64kchunk-16thread.7.0 /srv/z/enc/test-enc-64kchunk-16thread.7.1 /srv/z/enc/test-enc-64kchunk-16thread.7.10 /srv/z/enc/test-enc-64kchunk-16thread.7.11 /srv/z/enc/test-enc-64kchunk-16thread.7.12 /srv/z/enc/test-enc-64kchunk-16thread.7.13 /srv/z/enc/test-enc-64kchunk-16thread.7.14 /srv/z/enc/test-enc-64kchunk-16thread.7.15 /srv/z/enc/test-enc-64kchunk-16thread.7.2 /srv/z/enc/test-enc-64kchunk-16thread.7.3 /srv/z/enc/test-enc-64kchunk-16thread.7.4 /srv/z/enc/test-enc-64kchunk-16thread.7.5 /srv/z/enc/test-enc-64kchunk-16thread.7.6 /srv/z/enc/test-enc-64kchunk-16thread.7.7 /srv/z/enc/test-enc-64kchunk-16thread.7.8 /srv/z/enc/test-enc-64kchunk-16thread.7.9 /srv/z/enc/test-enc-64kchunk-16thread.8.0 /srv/z/enc/test-enc-64kchunk-16thread.8.1 /srv/z/enc/test-enc-64kchunk-16thread.8.10 /srv/z/enc/test-enc-64kchunk-16thread.8.11 /srv/z/enc/test-enc-64kchunk-16thread.8.12 /srv/z/enc/test-enc-64kchunk-16thread.8.13 /srv/z/enc/test-enc-64kchunk-16thread.8.14 /srv/z/enc/test-enc-64kchunk-16thread.8.15 /srv/z/enc/test-enc-64kchunk-16thread.8.2 /srv/z/enc/test-enc-64kchunk-16thread.8.3 /srv/z/enc/test-enc-64kchunk-16thread.8.4 /srv/z/enc/test-enc-64kchunk-16thread.8.5 /srv/z/enc/test-enc-64kchunk-16thread.8.6 /srv/z/enc/test-enc-64kchunk-16thread.8.7 /srv/z/enc/test-enc-64kchunk-16thread.8.8 /srv/z/enc/test-enc-64kchunk-16thread.8.9 /srv/z/enc/test-enc-64kchunk-16thread.9.0 /srv/z/enc/test-enc-64kchunk-16thread.9.1 /srv/z/enc/test-enc-64kchunk-16thread.9.10 /srv/z/enc/test-enc-64kchunk-16thread.9.11 /srv/z/enc/test-enc-64kchunk-16thread.9.12 /srv/z/enc/test-enc-64kchunk-16thread.9.13 /srv/z/enc/test-enc-64kchunk-16thread.9.14 /srv/z/enc/test-enc-64kchunk-16thread.9.15 /srv/z/enc/test-enc-64kchunk-16thread.9.2 /srv/z/enc/test-enc-64kchunk-16thread.9.3 /srv/z/enc/test-enc-64kchunk-16thread.9.4 /srv/z/enc/test-enc-64kchunk-16thread.9.5 /srv/z/enc/test-enc-64kchunk-16thread.9.6 /srv/z/enc/test-enc-64kchunk-16thread.9.7 /srv/z/enc/test-enc-64kchunk-16thread.9.8 /srv/z/enc/test-enc-64kchunk-16thread.9.9 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-both-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/both/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-both-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=328MiB/s,w=330MiB/s][r=5250,w=5278 IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.5%][r=366MiB/s,w=369MiB/s][r=5859,w=5897 IOPS][eta 01m:41s] Jobs: 16 (f=256): [M(16)][24.2%][r=318MiB/s,w=319MiB/s][r=5084,w=5109 IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.5%][r=315MiB/s,w=311MiB/s][r=5032,w=4981 IOPS][eta 01m:21s] Jobs: 16 (f=256): [M(16)][40.0%][r=314MiB/s,w=315MiB/s][r=5026,w=5042 IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][48.3%][r=324MiB/s,w=323MiB/s][r=5186,w=5171 IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][56.2%][r=332MiB/s,w=334MiB/s][r=5319,w=5345 IOPS][eta 00m:53s] | |
| Jobs: 16 (f=256): [M(16)][64.2%][r=318MiB/s,w=315MiB/s][r=5090,w=5034 IOPS][eta 00m:43s] Jobs: 16 (f=256): [M(16)][71.9%][r=310MiB/s,w=309MiB/s][r=4952,w=4949 IOPS][eta 00m:34s] | |
| Jobs: 16 (f=256): [M(16)][80.0%][r=323MiB/s,w=321MiB/s][r=5170,w=5143 IOPS][eta 00m:24s] Jobs: 16 (f=256): [M(16)][88.3%][r=341MiB/s,w=342MiB/s][r=5463,w=5473 IOPS][eta 00m:14s] | |
| Jobs: 16 (f=256): [M(16)][95.9%][r=357MiB/s,w=357MiB/s][r=5713,w=5710 IOPS][eta 00m:05s] | |
| test-both-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=312314: Wed Feb 26 12:48:47 2020 | |
| read: IOPS=5311, BW=332MiB/s (348MB/s)(38.9GiB/120009msec) | |
| slat (usec): min=9, max=159606, avg=1510.97, stdev=3683.23 | |
| clat (usec): min=3, max=498860, avg=94798.78, stdev=51681.45 | |
| lat (usec): min=90, max=498874, avg=96310.20, stdev=52344.03 | |
| clat percentiles (msec): | |
| | 1.00th=[ 11], 5.00th=[ 23], 10.00th=[ 33], 20.00th=[ 51], | |
| | 30.00th=[ 69], 40.00th=[ 80], 50.00th=[ 89], 60.00th=[ 99], | |
| | 70.00th=[ 110], 80.00th=[ 132], 90.00th=[ 169], 95.00th=[ 194], | |
| | 99.00th=[ 243], 99.50th=[ 264], 99.90th=[ 309], 99.95th=[ 338], | |
| | 99.99th=[ 405] | |
| bw ( KiB/s): min=117366, max=735616, per=99.81%, avg=339248.10, stdev=10332.13, samples=3840 | |
| iops : min= 1830, max=11494, avg=5298.69, stdev=161.48, samples=3840 | |
| write: IOPS=5310, BW=332MiB/s (348MB/s)(38.9GiB/120009msec); 0 zone resets | |
| slat (usec): min=15, max=175253, avg=1493.64, stdev=3883.33 | |
| clat (usec): min=3, max=498879, avg=94742.57, stdev=51644.27 | |
| lat (usec): min=51, max=504563, avg=96236.58, stdev=52358.72 | |
| clat percentiles (msec): | |
| | 1.00th=[ 11], 5.00th=[ 23], 10.00th=[ 33], 20.00th=[ 51], | |
| | 30.00th=[ 69], 40.00th=[ 80], 50.00th=[ 89], 60.00th=[ 97], | |
| | 70.00th=[ 110], 80.00th=[ 132], 90.00th=[ 169], 95.00th=[ 194], | |
| | 99.00th=[ 243], 99.50th=[ 264], 99.90th=[ 309], 99.95th=[ 342], | |
| | 99.99th=[ 405] | |
| bw ( KiB/s): min=119396, max=744258, per=99.81%, avg=339231.34, stdev=10371.59, samples=3840 | |
| iops : min= 1862, max=11629, avg=5298.41, stdev=162.09, samples=3840 | |
| lat (usec) : 4=0.01%, 10=0.01%, 100=0.01%, 500=0.01%, 750=0.01% | |
| lat (msec) : 2=0.01%, 4=0.10%, 10=0.73%, 20=3.25%, 50=15.83% | |
| lat (msec) : 100=42.52%, 250=36.79%, 500=0.78% | |
| cpu : usr=0.29%, sys=39.02%, ctx=870155, majf=0, minf=462397 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=637378,637336,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=332MiB/s (348MB/s), 332MiB/s-332MiB/s (348MB/s-348MB/s), io=38.9GiB (41.8GB), run=120009-120009msec | |
| WRITE: bw=332MiB/s (348MB/s), 332MiB/s-332MiB/s (348MB/s-348MB/s), io=38.9GiB (41.8GB), run=120009-120009msec | |
| + rm -rf /srv/z/both/test-both-64kchunk-16thread.0.0 /srv/z/both/test-both-64kchunk-16thread.0.1 /srv/z/both/test-both-64kchunk-16thread.0.10 /srv/z/both/test-both-64kchunk-16thread.0.11 /srv/z/both/test-both-64kchunk-16thread.0.12 /srv/z/both/test-both-64kchunk-16thread.0.13 /srv/z/both/test-both-64kchunk-16thread.0.14 /srv/z/both/test-both-64kchunk-16thread.0.15 /srv/z/both/test-both-64kchunk-16thread.0.2 /srv/z/both/test-both-64kchunk-16thread.0.3 /srv/z/both/test-both-64kchunk-16thread.0.4 /srv/z/both/test-both-64kchunk-16thread.0.5 /srv/z/both/test-both-64kchunk-16thread.0.6 /srv/z/both/test-both-64kchunk-16thread.0.7 /srv/z/both/test-both-64kchunk-16thread.0.8 /srv/z/both/test-both-64kchunk-16thread.0.9 /srv/z/both/test-both-64kchunk-16thread.1.0 /srv/z/both/test-both-64kchunk-16thread.10.0 /srv/z/both/test-both-64kchunk-16thread.10.1 /srv/z/both/test-both-64kchunk-16thread.10.10 /srv/z/both/test-both-64kchunk-16thread.10.11 /srv/z/both/test-both-64kchunk-16thread.10.12 /srv/z/both/test-both-64kchunk-16thread.10.13 /srv/z/both/test-both-64kchunk-16thread.10.14 /srv/z/both/test-both-64kchunk-16thread.10.15 /srv/z/both/test-both-64kchunk-16thread.10.2 /srv/z/both/test-both-64kchunk-16thread.10.3 /srv/z/both/test-both-64kchunk-16thread.10.4 /srv/z/both/test-both-64kchunk-16thread.10.5 /srv/z/both/test-both-64kchunk-16thread.10.6 /srv/z/both/test-both-64kchunk-16thread.10.7 /srv/z/both/test-both-64kchunk-16thread.10.8 /srv/z/both/test-both-64kchunk-16thread.10.9 /srv/z/both/test-both-64kchunk-16thread.1.1 /srv/z/both/test-both-64kchunk-16thread.1.10 /srv/z/both/test-both-64kchunk-16thread.11.0 /srv/z/both/test-both-64kchunk-16thread.1.11 /srv/z/both/test-both-64kchunk-16thread.11.1 /srv/z/both/test-both-64kchunk-16thread.11.10 /srv/z/both/test-both-64kchunk-16thread.11.11 /srv/z/both/test-both-64kchunk-16thread.11.12 /srv/z/both/test-both-64kchunk-16thread.11.13 /srv/z/both/test-both-64kchunk-16thread.11.14 /srv/z/both/test-both-64kchunk-16thread.11.15 /srv/z/both/test-both-64kchunk-16thread.1.12 /srv/z/both/test-both-64kchunk-16thread.11.2 /srv/z/both/test-both-64kchunk-16thread.1.13 /srv/z/both/test-both-64kchunk-16thread.11.3 /srv/z/both/test-both-64kchunk-16thread.1.14 /srv/z/both/test-both-64kchunk-16thread.11.4 /srv/z/both/test-both-64kchunk-16thread.1.15 /srv/z/both/test-both-64kchunk-16thread.11.5 /srv/z/both/test-both-64kchunk-16thread.11.6 /srv/z/both/test-both-64kchunk-16thread.11.7 /srv/z/both/test-both-64kchunk-16thread.11.8 /srv/z/both/test-both-64kchunk-16thread.11.9 /srv/z/both/test-both-64kchunk-16thread.1.2 /srv/z/both/test-both-64kchunk-16thread.12.0 /srv/z/both/test-both-64kchunk-16thread.12.1 /srv/z/both/test-both-64kchunk-16thread.12.10 /srv/z/both/test-both-64kchunk-16thread.12.11 /srv/z/both/test-both-64kchunk-16thread.12.12 /srv/z/both/test-both-64kchunk-16thread.12.13 /srv/z/both/test-both-64kchunk-16thread.12.14 /srv/z/both/test-both-64kchunk-16thread.12.15 /srv/z/both/test-both-64kchunk-16thread.12.2 /srv/z/both/test-both-64kchunk-16thread.12.3 /srv/z/both/test-both-64kchunk-16thread.12.4 /srv/z/both/test-both-64kchunk-16thread.12.5 /srv/z/both/test-both-64kchunk-16thread.12.6 /srv/z/both/test-both-64kchunk-16thread.12.7 /srv/z/both/test-both-64kchunk-16thread.12.8 /srv/z/both/test-both-64kchunk-16thread.12.9 /srv/z/both/test-both-64kchunk-16thread.1.3 /srv/z/both/test-both-64kchunk-16thread.13.0 /srv/z/both/test-both-64kchunk-16thread.13.1 /srv/z/both/test-both-64kchunk-16thread.13.10 /srv/z/both/test-both-64kchunk-16thread.13.11 /srv/z/both/test-both-64kchunk-16thread.13.12 /srv/z/both/test-both-64kchunk-16thread.13.13 /srv/z/both/test-both-64kchunk-16thread.13.14 /srv/z/both/test-both-64kchunk-16thread.13.15 /srv/z/both/test-both-64kchunk-16thread.13.2 /srv/z/both/test-both-64kchunk-16thread.13.3 /srv/z/both/test-both-64kchunk-16thread.13.4 /srv/z/both/test-both-64kchunk-16thread.13.5 /srv/z/both/test-both-64kchunk-16thread.13.6 /srv/z/both/test-both-64kchunk-16thread.13.7 /srv/z/both/test-both-64kchunk-16thread.13.8 /srv/z/both/test-both-64kchunk-16thread.13.9 /srv/z/both/test-both-64kchunk-16thread.1.4 /srv/z/both/test-both-64kchunk-16thread.14.0 /srv/z/both/test-both-64kchunk-16thread.14.1 /srv/z/both/test-both-64kchunk-16thread.14.10 /srv/z/both/test-both-64kchunk-16thread.14.11 /srv/z/both/test-both-64kchunk-16thread.14.12 /srv/z/both/test-both-64kchunk-16thread.14.13 /srv/z/both/test-both-64kchunk-16thread.14.14 /srv/z/both/test-both-64kchunk-16thread.14.15 /srv/z/both/test-both-64kchunk-16thread.14.2 /srv/z/both/test-both-64kchunk-16thread.14.3 /srv/z/both/test-both-64kchunk-16thread.14.4 /srv/z/both/test-both-64kchunk-16thread.14.5 /srv/z/both/test-both-64kchunk-16thread.14.6 /srv/z/both/test-both-64kchunk-16thread.14.7 /srv/z/both/test-both-64kchunk-16thread.14.8 /srv/z/both/test-both-64kchunk-16thread.14.9 /srv/z/both/test-both-64kchunk-16thread.1.5 /srv/z/both/test-both-64kchunk-16thread.15.0 /srv/z/both/test-both-64kchunk-16thread.15.1 /srv/z/both/test-both-64kchunk-16thread.15.10 /srv/z/both/test-both-64kchunk-16thread.15.11 /srv/z/both/test-both-64kchunk-16thread.15.12 /srv/z/both/test-both-64kchunk-16thread.15.13 /srv/z/both/test-both-64kchunk-16thread.15.14 /srv/z/both/test-both-64kchunk-16thread.15.15 /srv/z/both/test-both-64kchunk-16thread.15.2 /srv/z/both/test-both-64kchunk-16thread.15.3 /srv/z/both/test-both-64kchunk-16thread.15.4 /srv/z/both/test-both-64kchunk-16thread.15.5 /srv/z/both/test-both-64kchunk-16thread.15.6 /srv/z/both/test-both-64kchunk-16thread.15.7 /srv/z/both/test-both-64kchunk-16thread.15.8 /srv/z/both/test-both-64kchunk-16thread.15.9 /srv/z/both/test-both-64kchunk-16thread.1.6 /srv/z/both/test-both-64kchunk-16thread.1.7 /srv/z/both/test-both-64kchunk-16thread.1.8 /srv/z/both/test-both-64kchunk-16thread.1.9 /srv/z/both/test-both-64kchunk-16thread.2.0 /srv/z/both/test-both-64kchunk-16thread.2.1 /srv/z/both/test-both-64kchunk-16thread.2.10 /srv/z/both/test-both-64kchunk-16thread.2.11 /srv/z/both/test-both-64kchunk-16thread.2.12 /srv/z/both/test-both-64kchunk-16thread.2.13 /srv/z/both/test-both-64kchunk-16thread.2.14 /srv/z/both/test-both-64kchunk-16thread.2.15 /srv/z/both/test-both-64kchunk-16thread.2.2 /srv/z/both/test-both-64kchunk-16thread.2.3 /srv/z/both/test-both-64kchunk-16thread.2.4 /srv/z/both/test-both-64kchunk-16thread.2.5 /srv/z/both/test-both-64kchunk-16thread.2.6 /srv/z/both/test-both-64kchunk-16thread.2.7 /srv/z/both/test-both-64kchunk-16thread.2.8 /srv/z/both/test-both-64kchunk-16thread.2.9 /srv/z/both/test-both-64kchunk-16thread.3.0 /srv/z/both/test-both-64kchunk-16thread.3.1 /srv/z/both/test-both-64kchunk-16thread.3.10 /srv/z/both/test-both-64kchunk-16thread.3.11 /srv/z/both/test-both-64kchunk-16thread.3.12 /srv/z/both/test-both-64kchunk-16thread.3.13 /srv/z/both/test-both-64kchunk-16thread.3.14 /srv/z/both/test-both-64kchunk-16thread.3.15 /srv/z/both/test-both-64kchunk-16thread.3.2 /srv/z/both/test-both-64kchunk-16thread.3.3 /srv/z/both/test-both-64kchunk-16thread.3.4 /srv/z/both/test-both-64kchunk-16thread.3.5 /srv/z/both/test-both-64kchunk-16thread.3.6 /srv/z/both/test-both-64kchunk-16thread.3.7 /srv/z/both/test-both-64kchunk-16thread.3.8 /srv/z/both/test-both-64kchunk-16thread.3.9 /srv/z/both/test-both-64kchunk-16thread.4.0 /srv/z/both/test-both-64kchunk-16thread.4.1 /srv/z/both/test-both-64kchunk-16thread.4.10 /srv/z/both/test-both-64kchunk-16thread.4.11 /srv/z/both/test-both-64kchunk-16thread.4.12 /srv/z/both/test-both-64kchunk-16thread.4.13 /srv/z/both/test-both-64kchunk-16thread.4.14 /srv/z/both/test-both-64kchunk-16thread.4.15 /srv/z/both/test-both-64kchunk-16thread.4.2 /srv/z/both/test-both-64kchunk-16thread.4.3 /srv/z/both/test-both-64kchunk-16thread.4.4 /srv/z/both/test-both-64kchunk-16thread.4.5 /srv/z/both/test-both-64kchunk-16thread.4.6 /srv/z/both/test-both-64kchunk-16thread.4.7 /srv/z/both/test-both-64kchunk-16thread.4.8 /srv/z/both/test-both-64kchunk-16thread.4.9 /srv/z/both/test-both-64kchunk-16thread.5.0 /srv/z/both/test-both-64kchunk-16thread.5.1 /srv/z/both/test-both-64kchunk-16thread.5.10 /srv/z/both/test-both-64kchunk-16thread.5.11 /srv/z/both/test-both-64kchunk-16thread.5.12 /srv/z/both/test-both-64kchunk-16thread.5.13 /srv/z/both/test-both-64kchunk-16thread.5.14 /srv/z/both/test-both-64kchunk-16thread.5.15 /srv/z/both/test-both-64kchunk-16thread.5.2 /srv/z/both/test-both-64kchunk-16thread.5.3 /srv/z/both/test-both-64kchunk-16thread.5.4 /srv/z/both/test-both-64kchunk-16thread.5.5 /srv/z/both/test-both-64kchunk-16thread.5.6 /srv/z/both/test-both-64kchunk-16thread.5.7 /srv/z/both/test-both-64kchunk-16thread.5.8 /srv/z/both/test-both-64kchunk-16thread.5.9 /srv/z/both/test-both-64kchunk-16thread.6.0 /srv/z/both/test-both-64kchunk-16thread.6.1 /srv/z/both/test-both-64kchunk-16thread.6.10 /srv/z/both/test-both-64kchunk-16thread.6.11 /srv/z/both/test-both-64kchunk-16thread.6.12 /srv/z/both/test-both-64kchunk-16thread.6.13 /srv/z/both/test-both-64kchunk-16thread.6.14 /srv/z/both/test-both-64kchunk-16thread.6.15 /srv/z/both/test-both-64kchunk-16thread.6.2 /srv/z/both/test-both-64kchunk-16thread.6.3 /srv/z/both/test-both-64kchunk-16thread.6.4 /srv/z/both/test-both-64kchunk-16thread.6.5 /srv/z/both/test-both-64kchunk-16thread.6.6 /srv/z/both/test-both-64kchunk-16thread.6.7 /srv/z/both/test-both-64kchunk-16thread.6.8 /srv/z/both/test-both-64kchunk-16thread.6.9 /srv/z/both/test-both-64kchunk-16thread.7.0 /srv/z/both/test-both-64kchunk-16thread.7.1 /srv/z/both/test-both-64kchunk-16thread.7.10 /srv/z/both/test-both-64kchunk-16thread.7.11 /srv/z/both/test-both-64kchunk-16thread.7.12 /srv/z/both/test-both-64kchunk-16thread.7.13 /srv/z/both/test-both-64kchunk-16thread.7.14 /srv/z/both/test-both-64kchunk-16thread.7.15 /srv/z/both/test-both-64kchunk-16thread.7.2 /srv/z/both/test-both-64kchunk-16thread.7.3 /srv/z/both/test-both-64kchunk-16thread.7.4 /srv/z/both/test-both-64kchunk-16thread.7.5 /srv/z/both/test-both-64kchunk-16thread.7.6 /srv/z/both/test-both-64kchunk-16thread.7.7 /srv/z/both/test-both-64kchunk-16thread.7.8 /srv/z/both/test-both-64kchunk-16thread.7.9 /srv/z/both/test-both-64kchunk-16thread.8.0 /srv/z/both/test-both-64kchunk-16thread.8.1 /srv/z/both/test-both-64kchunk-16thread.8.10 /srv/z/both/test-both-64kchunk-16thread.8.11 /srv/z/both/test-both-64kchunk-16thread.8.12 /srv/z/both/test-both-64kchunk-16thread.8.13 /srv/z/both/test-both-64kchunk-16thread.8.14 /srv/z/both/test-both-64kchunk-16thread.8.15 /srv/z/both/test-both-64kchunk-16thread.8.2 /srv/z/both/test-both-64kchunk-16thread.8.3 /srv/z/both/test-both-64kchunk-16thread.8.4 /srv/z/both/test-both-64kchunk-16thread.8.5 /srv/z/both/test-both-64kchunk-16thread.8.6 /srv/z/both/test-both-64kchunk-16thread.8.7 /srv/z/both/test-both-64kchunk-16thread.8.8 /srv/z/both/test-both-64kchunk-16thread.8.9 /srv/z/both/test-both-64kchunk-16thread.9.0 /srv/z/both/test-both-64kchunk-16thread.9.1 /srv/z/both/test-both-64kchunk-16thread.9.10 /srv/z/both/test-both-64kchunk-16thread.9.11 /srv/z/both/test-both-64kchunk-16thread.9.12 /srv/z/both/test-both-64kchunk-16thread.9.13 /srv/z/both/test-both-64kchunk-16thread.9.14 /srv/z/both/test-both-64kchunk-16thread.9.15 /srv/z/both/test-both-64kchunk-16thread.9.2 /srv/z/both/test-both-64kchunk-16thread.9.3 /srv/z/both/test-both-64kchunk-16thread.9.4 /srv/z/both/test-both-64kchunk-16thread.9.5 /srv/z/both/test-both-64kchunk-16thread.9.6 /srv/z/both/test-both-64kchunk-16thread.9.7 /srv/z/both/test-both-64kchunk-16thread.9.8 /srv/z/both/test-both-64kchunk-16thread.9.9 | |
| + zpool destroy tank | |
| + for ASHIFTVAL in 12 13 14 15 16 | |
| + trim_disks | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| Success formatting namespace:1 | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '################## waiting 60s for nvme format to complete...' | |
| ################## waiting 60s for nvme format to complete... | |
| + sleep 60 | |
| + nvme list | |
| Node SN Model Namespace Usage Format FW Rev | |
| ---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- -------- | |
| /dev/nvme0n1 S437NE0M804438 SAMSUNG MZQLB960HAJR-00007 1 70.14 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme1n1 S437NE0M804437 SAMSUNG MZQLB960HAJR-00007 1 64.40 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme2n1 S437NE0M804442 SAMSUNG MZQLB960HAJR-00007 1 75.93 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme3n1 S437NE0M804435 SAMSUNG MZQLB960HAJR-00007 1 61.06 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme4n1 S437NE0M802527 SAMSUNG MZQLB960HAJR-00007 1 55.92 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme5n1 S437NX0M402110 SAMSUNG MZQLB960HAJR-00007 1 80.25 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + zpool create tank -o ashift=15 -O mountpoint=/srv/z raidz1 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| + dd if=/dev/urandom of=/tmp/testkey.raw bs=32 count=1 | |
| 1+0 records in | |
| 1+0 records out | |
| 32 bytes copied, 0.000328883 s, 97.3 kB/s | |
| + COMP=' | |
| -o compression=lz4 | |
| ' | |
| + ENC=' | |
| -o encryption=aes-256-gcm | |
| -o keyformat=raw | |
| -o keylocation=file:///tmp/testkey.raw | |
| ' | |
| + zfs create tank/none | |
| + zfs create -o compression=lz4 tank/comp | |
| + zfs create -o encryption=aes-256-gcm -o keyformat=raw -o keylocation=file:///tmp/testkey.raw tank/enc | |
| + zfs create -o compression=lz4 -o encryption=aes-256-gcm -o keyformat=raw -o keylocation=file:///tmp/testkey.raw tank/both | |
| + FO='--eta-newline=10 --eta-interval=10 --eta=always' | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-none-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/none/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-none-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-none-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=1092MiB/s,w=1092MiB/s][r=17.5k,w=17.5k IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.5%][r=1138MiB/s,w=1135MiB/s][r=18.2k,w=18.2k IOPS][eta 01m:41s] Jobs: 1 (f=1): [M(1)][24.2%][r=1105MiB/s,w=1104MiB/s][r=17.7k,w=17.7k IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.8%][r=1110MiB/s,w=1110MiB/s][r=17.8k,w=17.8k IOPS][eta 01m:20s] Jobs: 1 (f=1): [M(1)][40.5%][r=1116MiB/s,w=1114MiB/s][r=17.9k,w=17.8k IOPS][eta 01m:12s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=1120MiB/s,w=1119MiB/s][r=17.9k,w=17.9k IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=1075MiB/s,w=1079MiB/s][r=17.2k,w=17.3k IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][65.0%][r=1094MiB/s,w=1088MiB/s][r=17.5k,w=17.4k IOPS][eta 00m:42s] Jobs: 1 (f=1): [M(1)][72.5%][r=1093MiB/s,w=1091MiB/s][r=17.5k,w=17.4k IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=1081MiB/s,w=1084MiB/s][r=17.3k,w=17.3k IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][88.4%][r=1089MiB/s,w=1085MiB/s][r=17.4k,w=17.4k IOPS][eta 00m:14s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=1116MiB/s,w=1114MiB/s][r=17.9k,w=17.8k IOPS][eta 00m:04s] | |
| test-none-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=541817: Wed Feb 26 12:52:13 2020 | |
| read: IOPS=17.6k, BW=1103MiB/s (1156MB/s)(129GiB/120001msec) | |
| slat (usec): min=5, max=44556, avg=26.78, stdev=160.24 | |
| clat (usec): min=91, max=79769, avg=1788.59, stdev=1933.91 | |
| lat (usec): min=151, max=79885, avg=1815.50, stdev=1955.74 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1221], 5.00th=[ 1287], 10.00th=[ 1303], 20.00th=[ 1336], | |
| | 30.00th=[ 1369], 40.00th=[ 1385], 50.00th=[ 1418], 60.00th=[ 1450], | |
| | 70.00th=[ 1549], 80.00th=[ 1598], 90.00th=[ 2278], 95.00th=[ 3228], | |
| | 99.00th=[ 9241], 99.50th=[14615], 99.90th=[28705], 99.95th=[34341], | |
| | 99.99th=[47449] | |
| bw ( MiB/s): min= 809, max= 1473, per=100.00%, avg=1102.96, stdev=149.87, samples=239 | |
| iops : min=12948, max=23578, avg=17647.31, stdev=2397.88, samples=239 | |
| write: IOPS=17.6k, BW=1101MiB/s (1155MB/s)(129GiB/120001msec); 0 zone resets | |
| slat (usec): min=11, max=41583, avg=27.11, stdev=161.96 | |
| clat (usec): min=5, max=79525, avg=1785.11, stdev=1918.20 | |
| lat (usec): min=49, max=79602, avg=1812.38, stdev=1941.47 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1221], 5.00th=[ 1287], 10.00th=[ 1303], 20.00th=[ 1336], | |
| | 30.00th=[ 1369], 40.00th=[ 1385], 50.00th=[ 1418], 60.00th=[ 1450], | |
| | 70.00th=[ 1549], 80.00th=[ 1598], 90.00th=[ 2278], 95.00th=[ 3228], | |
| | 99.00th=[ 9110], 99.50th=[14484], 99.90th=[28705], 99.95th=[34341], | |
| | 99.99th=[47449] | |
| bw ( MiB/s): min= 775, max= 1448, per=100.00%, avg=1101.65, stdev=147.93, samples=239 | |
| iops : min=12410, max=23174, avg=17626.42, stdev=2366.89, samples=239 | |
| lat (usec) : 10=0.01%, 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01% | |
| lat (usec) : 1000=0.01% | |
| lat (msec) : 2=89.09%, 4=7.87%, 10=2.13%, 20=0.63%, 50=0.26% | |
| lat (msec) : 100=0.01% | |
| cpu : usr=7.44%, sys=81.26%, ctx=24837, majf=0, minf=42921 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=2117395,2114887,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=1103MiB/s (1156MB/s), 1103MiB/s-1103MiB/s (1156MB/s-1156MB/s), io=129GiB (139GB), run=120001-120001msec | |
| WRITE: bw=1101MiB/s (1155MB/s), 1101MiB/s-1101MiB/s (1155MB/s-1155MB/s), io=129GiB (139GB), run=120001-120001msec | |
| + rm -rf /srv/z/none/test-none-64kchunk-1thread.0.0 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-comp-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/comp/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-comp-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-comp-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=1058MiB/s,w=1058MiB/s][r=16.9k,w=16.9k IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.5%][r=1092MiB/s,w=1089MiB/s][r=17.5k,w=17.4k IOPS][eta 01m:41s] Jobs: 1 (f=1): [M(1)][24.2%][r=1096MiB/s,w=1094MiB/s][r=17.5k,w=17.5k IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.8%][r=1076MiB/s,w=1077MiB/s][r=17.2k,w=17.2k IOPS][eta 01m:20s] Jobs: 1 (f=1): [M(1)][40.5%][r=1072MiB/s,w=1069MiB/s][r=17.2k,w=17.1k IOPS][eta 01m:12s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=1073MiB/s,w=1072MiB/s][r=17.2k,w=17.2k IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=1082MiB/s,w=1086MiB/s][r=17.3k,w=17.4k IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][65.0%][r=1079MiB/s,w=1073MiB/s][r=17.3k,w=17.2k IOPS][eta 00m:42s] Jobs: 1 (f=1): [M(1)][72.5%][r=1080MiB/s,w=1078MiB/s][r=17.3k,w=17.2k IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=1062MiB/s,w=1066MiB/s][r=16.0k,w=17.1k IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][88.4%][r=1098MiB/s,w=1092MiB/s][r=17.6k,w=17.5k IOPS][eta 00m:14s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=1071MiB/s,w=1070MiB/s][r=17.1k,w=17.1k IOPS][eta 00m:04s] | |
| test-comp-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=797574: Wed Feb 26 12:54:25 2020 | |
| read: IOPS=17.3k, BW=1078MiB/s (1131MB/s)(126GiB/120001msec) | |
| slat (usec): min=5, max=66723, avg=27.47, stdev=204.44 | |
| clat (usec): min=85, max=75268, avg=1832.14, stdev=2377.26 | |
| lat (usec): min=97, max=75294, avg=1859.75, stdev=2403.48 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1205], 5.00th=[ 1287], 10.00th=[ 1303], 20.00th=[ 1319], | |
| | 30.00th=[ 1352], 40.00th=[ 1369], 50.00th=[ 1401], 60.00th=[ 1450], | |
| | 70.00th=[ 1516], 80.00th=[ 1565], 90.00th=[ 1745], 95.00th=[ 3130], | |
| | 99.00th=[12780], 99.50th=[19006], 99.90th=[32900], 99.95th=[38011], | |
| | 99.99th=[50594] | |
| bw ( MiB/s): min= 820, max= 1522, per=100.00%, avg=1078.63, stdev=191.67, samples=239 | |
| iops : min=13132, max=24364, avg=17258.09, stdev=3066.73, samples=239 | |
| write: IOPS=17.2k, BW=1077MiB/s (1129MB/s)(126GiB/120001msec); 0 zone resets | |
| slat (usec): min=10, max=37577, avg=27.68, stdev=205.58 | |
| clat (usec): min=2, max=75256, avg=1822.49, stdev=2342.30 | |
| lat (usec): min=29, max=75296, avg=1850.31, stdev=2368.84 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1205], 5.00th=[ 1287], 10.00th=[ 1303], 20.00th=[ 1319], | |
| | 30.00th=[ 1352], 40.00th=[ 1369], 50.00th=[ 1401], 60.00th=[ 1450], | |
| | 70.00th=[ 1516], 80.00th=[ 1565], 90.00th=[ 1745], 95.00th=[ 3130], | |
| | 99.00th=[12387], 99.50th=[18482], 99.90th=[32375], 99.95th=[38011], | |
| | 99.99th=[53216] | |
| bw ( MiB/s): min= 794, max= 1530, per=100.00%, avg=1077.28, stdev=191.34, samples=239 | |
| iops : min=12706, max=24482, avg=17236.38, stdev=3061.37, samples=239 | |
| lat (usec) : 4=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01% | |
| lat (usec) : 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=91.17%, 4=5.37%, 10=2.08%, 20=0.93%, 50=0.43% | |
| lat (msec) : 100=0.01% | |
| cpu : usr=6.88%, sys=77.86%, ctx=25343, majf=0, minf=50249 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=2070546,2067995,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=1078MiB/s (1131MB/s), 1078MiB/s-1078MiB/s (1131MB/s-1131MB/s), io=126GiB (136GB), run=120001-120001msec | |
| WRITE: bw=1077MiB/s (1129MB/s), 1077MiB/s-1077MiB/s (1129MB/s-1129MB/s), io=126GiB (136GB), run=120001-120001msec | |
| + rm -rf /srv/z/comp/test-comp-64kchunk-1thread.0.0 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-enc-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/enc/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-enc-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-enc-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=34.7MiB/s,w=34.9MiB/s][r=555,w=558 IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.7%][r=34.9MiB/s,w=34.4MiB/s][r=557,w=551 IOPS][eta 01m:40s] Jobs: 1 (f=1): [M(1)][24.2%][r=34.6MiB/s,w=34.5MiB/s][r=554,w=551 IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.5%][r=35.0MiB/s,w=35.2MiB/s][r=560,w=563 IOPS][eta 01m:21s] Jobs: 1 (f=1): [M(1)][40.8%][r=35.2MiB/s,w=35.3MiB/s][r=563,w=565 IOPS][eta 01m:11s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=34.9MiB/s,w=35.3MiB/s][r=557,w=564 IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=34.9MiB/s,w=34.7MiB/s][r=558,w=555 IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][65.0%][r=34.9MiB/s,w=32.7MiB/s][r=559,w=523 IOPS][eta 00m:42s] Jobs: 1 (f=1): [M(1)][72.5%][r=35.5MiB/s,w=35.5MiB/s][r=567,w=568 IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=35.1MiB/s,w=35.7MiB/s][r=561,w=571 IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][89.2%][r=34.8MiB/s,w=35.1MiB/s][r=556,w=562 IOPS][eta 00m:13s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=34.0MiB/s,w=33.8MiB/s][r=559,w=540 IOPS][eta 00m:04s] | |
| test-enc-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=1005711: Wed Feb 26 12:56:38 2020 | |
| read: IOPS=560, BW=35.0MiB/s (36.7MB/s)(4200MiB/120004msec) | |
| slat (usec): min=6, max=27099, avg=1137.75, stdev=1658.87 | |
| clat (usec): min=3, max=147547, avg=56358.06, stdev=10043.03 | |
| lat (msec): min=3, max=147, avg=57.50, stdev=10.16 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 45], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 53], 50.00th=[ 55], 60.00th=[ 58], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 66], 95.00th=[ 74], | |
| | 99.00th=[ 96], 99.50th=[ 103], 99.90th=[ 112], 99.95th=[ 126], | |
| | 99.99th=[ 146] | |
| bw ( KiB/s): min=26112, max=41472, per=99.94%, avg=35817.36, stdev=3033.69, samples=240 | |
| iops : min= 408, max= 648, avg=559.59, stdev=47.40, samples=240 | |
| write: IOPS=557, BW=34.8MiB/s (36.5MB/s)(4182MiB/120004msec); 0 zone resets | |
| slat (usec): min=12, max=59450, avg=647.70, stdev=1389.09 | |
| clat (msec): min=6, max=147, avg=56.35, stdev= 9.99 | |
| lat (msec): min=6, max=150, avg=57.00, stdev=10.09 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 45], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 53], 50.00th=[ 55], 60.00th=[ 58], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 66], 95.00th=[ 74], | |
| | 99.00th=[ 95], 99.50th=[ 102], 99.90th=[ 112], 99.95th=[ 118], | |
| | 99.99th=[ 142] | |
| bw ( KiB/s): min=22912, max=43904, per=99.94%, avg=35660.50, stdev=3587.84, samples=240 | |
| iops : min= 358, max= 686, avg=557.14, stdev=56.06, samples=240 | |
| lat (usec) : 4=0.01% | |
| lat (msec) : 4=0.01%, 10=0.01%, 20=0.01%, 50=22.88%, 100=76.55% | |
| lat (msec) : 250=0.56% | |
| cpu : usr=0.21%, sys=85.83%, ctx=12817, majf=0, minf=18905 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=67203,66907,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=35.0MiB/s (36.7MB/s), 35.0MiB/s-35.0MiB/s (36.7MB/s-36.7MB/s), io=4200MiB (4404MB), run=120004-120004msec | |
| WRITE: bw=34.8MiB/s (36.5MB/s), 34.8MiB/s-34.8MiB/s (36.5MB/s-36.5MB/s), io=4182MiB (4385MB), run=120004-120004msec | |
| + rm -rf /srv/z/enc/test-enc-64kchunk-1thread.0.0 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-both-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/both/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-both-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-both-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=33.9MiB/s,w=34.3MiB/s][r=542,w=548 IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.7%][r=35.8MiB/s,w=35.3MiB/s][r=572,w=564 IOPS][eta 01m:40s] Jobs: 1 (f=1): [M(1)][24.2%][r=35.2MiB/s,w=35.4MiB/s][r=564,w=567 IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.5%][r=34.7MiB/s,w=34.5MiB/s][r=555,w=551 IOPS][eta 01m:21s] Jobs: 1 (f=1): [M(1)][40.8%][r=34.9MiB/s,w=35.0MiB/s][r=558,w=560 IOPS][eta 01m:11s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=34.8MiB/s,w=35.2MiB/s][r=557,w=563 IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=35.1MiB/s,w=34.0MiB/s][r=562,w=559 IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][65.0%][r=34.8MiB/s,w=32.6MiB/s][r=556,w=521 IOPS][eta 00m:42s] Jobs: 1 (f=1): [M(1)][72.5%][r=34.7MiB/s,w=34.9MiB/s][r=555,w=558 IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=34.0MiB/s,w=35.6MiB/s][r=559,w=570 IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][89.2%][r=35.3MiB/s,w=35.5MiB/s][r=565,w=568 IOPS][eta 00m:13s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=35.0MiB/s,w=33.9MiB/s][r=560,w=541 IOPS][eta 00m:04s] | |
| test-both-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=1104483: Wed Feb 26 12:58:51 2020 | |
| read: IOPS=559, BW=34.0MiB/s (36.7MB/s)(4199MiB/120002msec) | |
| slat (usec): min=6, max=54630, avg=1135.71, stdev=1665.07 | |
| clat (usec): min=3, max=196702, avg=56387.91, stdev=10669.30 | |
| lat (msec): min=3, max=200, avg=57.52, stdev=10.78 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 46], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 53], 50.00th=[ 55], 60.00th=[ 58], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 66], 95.00th=[ 73], | |
| | 99.00th=[ 100], 99.50th=[ 107], 99.90th=[ 122], 99.95th=[ 176], | |
| | 99.99th=[ 197] | |
| bw ( KiB/s): min=23377, max=41472, per=99.94%, avg=35811.13, stdev=3193.76, samples=240 | |
| iops : min= 365, max= 648, avg=559.49, stdev=49.90, samples=240 | |
| write: IOPS=557, BW=34.8MiB/s (36.5MB/s)(4181MiB/120002msec); 0 zone resets | |
| slat (usec): min=12, max=92457, avg=650.08, stdev=1425.77 | |
| clat (msec): min=3, max=199, avg=56.34, stdev=10.45 | |
| lat (msec): min=3, max=199, avg=57.00, stdev=10.56 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 45], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 53], 50.00th=[ 55], 60.00th=[ 58], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 66], 95.00th=[ 73], | |
| | 99.00th=[ 100], 99.50th=[ 106], 99.90th=[ 122], 99.95th=[ 125], | |
| | 99.99th=[ 184] | |
| bw ( KiB/s): min=22483, max=43520, per=99.94%, avg=35655.02, stdev=3898.63, samples=240 | |
| iops : min= 351, max= 680, avg=557.06, stdev=60.91, samples=240 | |
| lat (usec) : 4=0.01% | |
| lat (msec) : 4=0.01%, 10=0.01%, 20=0.01%, 50=23.09%, 100=76.01% | |
| lat (msec) : 250=0.88% | |
| cpu : usr=0.21%, sys=85.19%, ctx=13039, majf=0, minf=21586 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=67191,66894,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=34.0MiB/s (36.7MB/s), 34.0MiB/s-34.0MiB/s (36.7MB/s-36.7MB/s), io=4199MiB (4403MB), run=120002-120002msec | |
| WRITE: bw=34.8MiB/s (36.5MB/s), 34.8MiB/s-34.8MiB/s (36.5MB/s-36.5MB/s), io=4181MiB (4384MB), run=120002-120002msec | |
| + rm -rf /srv/z/both/test-both-64kchunk-1thread.0.0 | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-none-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/none/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-none-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=2630MiB/s,w=2631MiB/s][r=42.1k,w=42.1k IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.5%][r=2595MiB/s,w=2601MiB/s][r=41.5k,w=41.6k IOPS][eta 01m:41s] Jobs: 16 (f=256): [M(16)][24.2%][r=2832MiB/s,w=2840MiB/s][r=45.3k,w=45.4k IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.8%][r=3090MiB/s,w=3091MiB/s][r=49.4k,w=49.4k IOPS][eta 01m:20s] Jobs: 16 (f=256): [M(16)][40.5%][r=3144MiB/s,w=3148MiB/s][r=50.3k,w=50.4k IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][48.3%][r=3074MiB/s,w=3070MiB/s][r=49.2k,w=49.1k IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][56.7%][r=2689MiB/s,w=2689MiB/s][r=43.0k,w=43.0k IOPS][eta 00m:52s] | |
| Jobs: 16 (f=256): [M(16)][64.7%][r=2744MiB/s,w=2741MiB/s][r=43.9k,w=43.8k IOPS][eta 00m:42s] Jobs: 16 (f=256): [M(16)][72.5%][r=2719MiB/s,w=2723MiB/s][r=43.5k,w=43.6k IOPS][eta 00m:33s] | |
| Jobs: 16 (f=256): [M(16)][80.2%][r=2709MiB/s,w=2717MiB/s][r=43.3k,w=43.5k IOPS][eta 00m:24s] Jobs: 16 (f=256): [M(16)][88.3%][r=2689MiB/s,w=2690MiB/s][r=43.0k,w=43.0k IOPS][eta 00m:14s] | |
| Jobs: 16 (f=256): [M(16)][95.9%][r=2621MiB/s,w=2622MiB/s][r=41.9k,w=41.0k IOPS][eta 00m:05s] | |
| test-none-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=1346872: Wed Feb 26 13:03:30 2020 | |
| read: IOPS=44.7k, BW=2792MiB/s (2927MB/s)(327GiB/120002msec) | |
| slat (usec): min=10, max=112621, avg=149.28, stdev=691.71 | |
| clat (usec): min=3, max=212412, avg=11272.28, stdev=9202.12 | |
| lat (usec): min=72, max=214538, avg=11421.99, stdev=9313.25 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 5], 10.00th=[ 6], 20.00th=[ 6], | |
| | 30.00th=[ 7], 40.00th=[ 7], 50.00th=[ 8], 60.00th=[ 10], | |
| | 70.00th=[ 12], 80.00th=[ 16], 90.00th=[ 23], 95.00th=[ 30], | |
| | 99.00th=[ 48], 99.50th=[ 57], 99.90th=[ 80], 99.95th=[ 90], | |
| | 99.99th=[ 121] | |
| bw ( MiB/s): min= 1510, max= 5055, per=99.95%, avg=2790.14, stdev=42.42, samples=3828 | |
| iops : min=24159, max=80884, avg=44640.58, stdev=678.65, samples=3828 | |
| write: IOPS=44.7k, BW=2794MiB/s (2929MB/s)(327GiB/120002msec); 0 zone resets | |
| slat (usec): min=18, max=111031, avg=200.38, stdev=833.11 | |
| clat (usec): min=2, max=169879, avg=11283.76, stdev=9199.99 | |
| lat (usec): min=41, max=169971, avg=11484.61, stdev=9301.08 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 5], 10.00th=[ 6], 20.00th=[ 6], | |
| | 30.00th=[ 7], 40.00th=[ 7], 50.00th=[ 8], 60.00th=[ 10], | |
| | 70.00th=[ 12], 80.00th=[ 16], 90.00th=[ 23], 95.00th=[ 30], | |
| | 99.00th=[ 48], 99.50th=[ 57], 99.90th=[ 79], 99.95th=[ 90], | |
| | 99.99th=[ 120] | |
| bw ( MiB/s): min= 1503, max= 5069, per=99.95%, avg=2792.35, stdev=42.35, samples=3828 | |
| iops : min=24056, max=81106, avg=44676.05, stdev=677.54, samples=3828 | |
| lat (usec) : 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01% | |
| lat (usec) : 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=0.01%, 4=1.49%, 10=62.71%, 20=23.47%, 50=11.49% | |
| lat (msec) : 100=0.81%, 250=0.03% | |
| cpu : usr=3.42%, sys=54.26%, ctx=1068204, majf=0, minf=484771 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=5359897,5364076,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=2792MiB/s (2927MB/s), 2792MiB/s-2792MiB/s (2927MB/s-2927MB/s), io=327GiB (351GB), run=120002-120002msec | |
| WRITE: bw=2794MiB/s (2929MB/s), 2794MiB/s-2794MiB/s (2929MB/s-2929MB/s), io=327GiB (352GB), run=120002-120002msec | |
| + rm -rf /srv/z/none/test-none-64kchunk-16thread.0.0 /srv/z/none/test-none-64kchunk-16thread.0.1 /srv/z/none/test-none-64kchunk-16thread.0.10 /srv/z/none/test-none-64kchunk-16thread.0.11 /srv/z/none/test-none-64kchunk-16thread.0.12 /srv/z/none/test-none-64kchunk-16thread.0.13 /srv/z/none/test-none-64kchunk-16thread.0.14 /srv/z/none/test-none-64kchunk-16thread.0.15 /srv/z/none/test-none-64kchunk-16thread.0.2 /srv/z/none/test-none-64kchunk-16thread.0.3 /srv/z/none/test-none-64kchunk-16thread.0.4 /srv/z/none/test-none-64kchunk-16thread.0.5 /srv/z/none/test-none-64kchunk-16thread.0.6 /srv/z/none/test-none-64kchunk-16thread.0.7 /srv/z/none/test-none-64kchunk-16thread.0.8 /srv/z/none/test-none-64kchunk-16thread.0.9 /srv/z/none/test-none-64kchunk-16thread.1.0 /srv/z/none/test-none-64kchunk-16thread.10.0 /srv/z/none/test-none-64kchunk-16thread.10.1 /srv/z/none/test-none-64kchunk-16thread.10.10 /srv/z/none/test-none-64kchunk-16thread.10.11 /srv/z/none/test-none-64kchunk-16thread.10.12 /srv/z/none/test-none-64kchunk-16thread.10.13 /srv/z/none/test-none-64kchunk-16thread.10.14 /srv/z/none/test-none-64kchunk-16thread.10.15 /srv/z/none/test-none-64kchunk-16thread.10.2 /srv/z/none/test-none-64kchunk-16thread.10.3 /srv/z/none/test-none-64kchunk-16thread.10.4 /srv/z/none/test-none-64kchunk-16thread.10.5 /srv/z/none/test-none-64kchunk-16thread.10.6 /srv/z/none/test-none-64kchunk-16thread.10.7 /srv/z/none/test-none-64kchunk-16thread.10.8 /srv/z/none/test-none-64kchunk-16thread.10.9 /srv/z/none/test-none-64kchunk-16thread.1.1 /srv/z/none/test-none-64kchunk-16thread.1.10 /srv/z/none/test-none-64kchunk-16thread.11.0 /srv/z/none/test-none-64kchunk-16thread.1.11 /srv/z/none/test-none-64kchunk-16thread.11.1 /srv/z/none/test-none-64kchunk-16thread.11.10 /srv/z/none/test-none-64kchunk-16thread.11.11 /srv/z/none/test-none-64kchunk-16thread.11.12 /srv/z/none/test-none-64kchunk-16thread.11.13 /srv/z/none/test-none-64kchunk-16thread.11.14 /srv/z/none/test-none-64kchunk-16thread.11.15 /srv/z/none/test-none-64kchunk-16thread.1.12 /srv/z/none/test-none-64kchunk-16thread.11.2 /srv/z/none/test-none-64kchunk-16thread.1.13 /srv/z/none/test-none-64kchunk-16thread.11.3 /srv/z/none/test-none-64kchunk-16thread.1.14 /srv/z/none/test-none-64kchunk-16thread.11.4 /srv/z/none/test-none-64kchunk-16thread.1.15 /srv/z/none/test-none-64kchunk-16thread.11.5 /srv/z/none/test-none-64kchunk-16thread.11.6 /srv/z/none/test-none-64kchunk-16thread.11.7 /srv/z/none/test-none-64kchunk-16thread.11.8 /srv/z/none/test-none-64kchunk-16thread.11.9 /srv/z/none/test-none-64kchunk-16thread.1.2 /srv/z/none/test-none-64kchunk-16thread.12.0 /srv/z/none/test-none-64kchunk-16thread.12.1 /srv/z/none/test-none-64kchunk-16thread.12.10 /srv/z/none/test-none-64kchunk-16thread.12.11 /srv/z/none/test-none-64kchunk-16thread.12.12 /srv/z/none/test-none-64kchunk-16thread.12.13 /srv/z/none/test-none-64kchunk-16thread.12.14 /srv/z/none/test-none-64kchunk-16thread.12.15 /srv/z/none/test-none-64kchunk-16thread.12.2 /srv/z/none/test-none-64kchunk-16thread.12.3 /srv/z/none/test-none-64kchunk-16thread.12.4 /srv/z/none/test-none-64kchunk-16thread.12.5 /srv/z/none/test-none-64kchunk-16thread.12.6 /srv/z/none/test-none-64kchunk-16thread.12.7 /srv/z/none/test-none-64kchunk-16thread.12.8 /srv/z/none/test-none-64kchunk-16thread.12.9 /srv/z/none/test-none-64kchunk-16thread.1.3 /srv/z/none/test-none-64kchunk-16thread.13.0 /srv/z/none/test-none-64kchunk-16thread.13.1 /srv/z/none/test-none-64kchunk-16thread.13.10 /srv/z/none/test-none-64kchunk-16thread.13.11 /srv/z/none/test-none-64kchunk-16thread.13.12 /srv/z/none/test-none-64kchunk-16thread.13.13 /srv/z/none/test-none-64kchunk-16thread.13.14 /srv/z/none/test-none-64kchunk-16thread.13.15 /srv/z/none/test-none-64kchunk-16thread.13.2 /srv/z/none/test-none-64kchunk-16thread.13.3 /srv/z/none/test-none-64kchunk-16thread.13.4 /srv/z/none/test-none-64kchunk-16thread.13.5 /srv/z/none/test-none-64kchunk-16thread.13.6 /srv/z/none/test-none-64kchunk-16thread.13.7 /srv/z/none/test-none-64kchunk-16thread.13.8 /srv/z/none/test-none-64kchunk-16thread.13.9 /srv/z/none/test-none-64kchunk-16thread.1.4 /srv/z/none/test-none-64kchunk-16thread.14.0 /srv/z/none/test-none-64kchunk-16thread.14.1 /srv/z/none/test-none-64kchunk-16thread.14.10 /srv/z/none/test-none-64kchunk-16thread.14.11 /srv/z/none/test-none-64kchunk-16thread.14.12 /srv/z/none/test-none-64kchunk-16thread.14.13 /srv/z/none/test-none-64kchunk-16thread.14.14 /srv/z/none/test-none-64kchunk-16thread.14.15 /srv/z/none/test-none-64kchunk-16thread.14.2 /srv/z/none/test-none-64kchunk-16thread.14.3 /srv/z/none/test-none-64kchunk-16thread.14.4 /srv/z/none/test-none-64kchunk-16thread.14.5 /srv/z/none/test-none-64kchunk-16thread.14.6 /srv/z/none/test-none-64kchunk-16thread.14.7 /srv/z/none/test-none-64kchunk-16thread.14.8 /srv/z/none/test-none-64kchunk-16thread.14.9 /srv/z/none/test-none-64kchunk-16thread.1.5 /srv/z/none/test-none-64kchunk-16thread.15.0 /srv/z/none/test-none-64kchunk-16thread.15.1 /srv/z/none/test-none-64kchunk-16thread.15.10 /srv/z/none/test-none-64kchunk-16thread.15.11 /srv/z/none/test-none-64kchunk-16thread.15.12 /srv/z/none/test-none-64kchunk-16thread.15.13 /srv/z/none/test-none-64kchunk-16thread.15.14 /srv/z/none/test-none-64kchunk-16thread.15.15 /srv/z/none/test-none-64kchunk-16thread.15.2 /srv/z/none/test-none-64kchunk-16thread.15.3 /srv/z/none/test-none-64kchunk-16thread.15.4 /srv/z/none/test-none-64kchunk-16thread.15.5 /srv/z/none/test-none-64kchunk-16thread.15.6 /srv/z/none/test-none-64kchunk-16thread.15.7 /srv/z/none/test-none-64kchunk-16thread.15.8 /srv/z/none/test-none-64kchunk-16thread.15.9 /srv/z/none/test-none-64kchunk-16thread.1.6 /srv/z/none/test-none-64kchunk-16thread.1.7 /srv/z/none/test-none-64kchunk-16thread.1.8 /srv/z/none/test-none-64kchunk-16thread.1.9 /srv/z/none/test-none-64kchunk-16thread.2.0 /srv/z/none/test-none-64kchunk-16thread.2.1 /srv/z/none/test-none-64kchunk-16thread.2.10 /srv/z/none/test-none-64kchunk-16thread.2.11 /srv/z/none/test-none-64kchunk-16thread.2.12 /srv/z/none/test-none-64kchunk-16thread.2.13 /srv/z/none/test-none-64kchunk-16thread.2.14 /srv/z/none/test-none-64kchunk-16thread.2.15 /srv/z/none/test-none-64kchunk-16thread.2.2 /srv/z/none/test-none-64kchunk-16thread.2.3 /srv/z/none/test-none-64kchunk-16thread.2.4 /srv/z/none/test-none-64kchunk-16thread.2.5 /srv/z/none/test-none-64kchunk-16thread.2.6 /srv/z/none/test-none-64kchunk-16thread.2.7 /srv/z/none/test-none-64kchunk-16thread.2.8 /srv/z/none/test-none-64kchunk-16thread.2.9 /srv/z/none/test-none-64kchunk-16thread.3.0 /srv/z/none/test-none-64kchunk-16thread.3.1 /srv/z/none/test-none-64kchunk-16thread.3.10 /srv/z/none/test-none-64kchunk-16thread.3.11 /srv/z/none/test-none-64kchunk-16thread.3.12 /srv/z/none/test-none-64kchunk-16thread.3.13 /srv/z/none/test-none-64kchunk-16thread.3.14 /srv/z/none/test-none-64kchunk-16thread.3.15 /srv/z/none/test-none-64kchunk-16thread.3.2 /srv/z/none/test-none-64kchunk-16thread.3.3 /srv/z/none/test-none-64kchunk-16thread.3.4 /srv/z/none/test-none-64kchunk-16thread.3.5 /srv/z/none/test-none-64kchunk-16thread.3.6 /srv/z/none/test-none-64kchunk-16thread.3.7 /srv/z/none/test-none-64kchunk-16thread.3.8 /srv/z/none/test-none-64kchunk-16thread.3.9 /srv/z/none/test-none-64kchunk-16thread.4.0 /srv/z/none/test-none-64kchunk-16thread.4.1 /srv/z/none/test-none-64kchunk-16thread.4.10 /srv/z/none/test-none-64kchunk-16thread.4.11 /srv/z/none/test-none-64kchunk-16thread.4.12 /srv/z/none/test-none-64kchunk-16thread.4.13 /srv/z/none/test-none-64kchunk-16thread.4.14 /srv/z/none/test-none-64kchunk-16thread.4.15 /srv/z/none/test-none-64kchunk-16thread.4.2 /srv/z/none/test-none-64kchunk-16thread.4.3 /srv/z/none/test-none-64kchunk-16thread.4.4 /srv/z/none/test-none-64kchunk-16thread.4.5 /srv/z/none/test-none-64kchunk-16thread.4.6 /srv/z/none/test-none-64kchunk-16thread.4.7 /srv/z/none/test-none-64kchunk-16thread.4.8 /srv/z/none/test-none-64kchunk-16thread.4.9 /srv/z/none/test-none-64kchunk-16thread.5.0 /srv/z/none/test-none-64kchunk-16thread.5.1 /srv/z/none/test-none-64kchunk-16thread.5.10 /srv/z/none/test-none-64kchunk-16thread.5.11 /srv/z/none/test-none-64kchunk-16thread.5.12 /srv/z/none/test-none-64kchunk-16thread.5.13 /srv/z/none/test-none-64kchunk-16thread.5.14 /srv/z/none/test-none-64kchunk-16thread.5.15 /srv/z/none/test-none-64kchunk-16thread.5.2 /srv/z/none/test-none-64kchunk-16thread.5.3 /srv/z/none/test-none-64kchunk-16thread.5.4 /srv/z/none/test-none-64kchunk-16thread.5.5 /srv/z/none/test-none-64kchunk-16thread.5.6 /srv/z/none/test-none-64kchunk-16thread.5.7 /srv/z/none/test-none-64kchunk-16thread.5.8 /srv/z/none/test-none-64kchunk-16thread.5.9 /srv/z/none/test-none-64kchunk-16thread.6.0 /srv/z/none/test-none-64kchunk-16thread.6.1 /srv/z/none/test-none-64kchunk-16thread.6.10 /srv/z/none/test-none-64kchunk-16thread.6.11 /srv/z/none/test-none-64kchunk-16thread.6.12 /srv/z/none/test-none-64kchunk-16thread.6.13 /srv/z/none/test-none-64kchunk-16thread.6.14 /srv/z/none/test-none-64kchunk-16thread.6.15 /srv/z/none/test-none-64kchunk-16thread.6.2 /srv/z/none/test-none-64kchunk-16thread.6.3 /srv/z/none/test-none-64kchunk-16thread.6.4 /srv/z/none/test-none-64kchunk-16thread.6.5 /srv/z/none/test-none-64kchunk-16thread.6.6 /srv/z/none/test-none-64kchunk-16thread.6.7 /srv/z/none/test-none-64kchunk-16thread.6.8 /srv/z/none/test-none-64kchunk-16thread.6.9 /srv/z/none/test-none-64kchunk-16thread.7.0 /srv/z/none/test-none-64kchunk-16thread.7.1 /srv/z/none/test-none-64kchunk-16thread.7.10 /srv/z/none/test-none-64kchunk-16thread.7.11 /srv/z/none/test-none-64kchunk-16thread.7.12 /srv/z/none/test-none-64kchunk-16thread.7.13 /srv/z/none/test-none-64kchunk-16thread.7.14 /srv/z/none/test-none-64kchunk-16thread.7.15 /srv/z/none/test-none-64kchunk-16thread.7.2 /srv/z/none/test-none-64kchunk-16thread.7.3 /srv/z/none/test-none-64kchunk-16thread.7.4 /srv/z/none/test-none-64kchunk-16thread.7.5 /srv/z/none/test-none-64kchunk-16thread.7.6 /srv/z/none/test-none-64kchunk-16thread.7.7 /srv/z/none/test-none-64kchunk-16thread.7.8 /srv/z/none/test-none-64kchunk-16thread.7.9 /srv/z/none/test-none-64kchunk-16thread.8.0 /srv/z/none/test-none-64kchunk-16thread.8.1 /srv/z/none/test-none-64kchunk-16thread.8.10 /srv/z/none/test-none-64kchunk-16thread.8.11 /srv/z/none/test-none-64kchunk-16thread.8.12 /srv/z/none/test-none-64kchunk-16thread.8.13 /srv/z/none/test-none-64kchunk-16thread.8.14 /srv/z/none/test-none-64kchunk-16thread.8.15 /srv/z/none/test-none-64kchunk-16thread.8.2 /srv/z/none/test-none-64kchunk-16thread.8.3 /srv/z/none/test-none-64kchunk-16thread.8.4 /srv/z/none/test-none-64kchunk-16thread.8.5 /srv/z/none/test-none-64kchunk-16thread.8.6 /srv/z/none/test-none-64kchunk-16thread.8.7 /srv/z/none/test-none-64kchunk-16thread.8.8 /srv/z/none/test-none-64kchunk-16thread.8.9 /srv/z/none/test-none-64kchunk-16thread.9.0 /srv/z/none/test-none-64kchunk-16thread.9.1 /srv/z/none/test-none-64kchunk-16thread.9.10 /srv/z/none/test-none-64kchunk-16thread.9.11 /srv/z/none/test-none-64kchunk-16thread.9.12 /srv/z/none/test-none-64kchunk-16thread.9.13 /srv/z/none/test-none-64kchunk-16thread.9.14 /srv/z/none/test-none-64kchunk-16thread.9.15 /srv/z/none/test-none-64kchunk-16thread.9.2 /srv/z/none/test-none-64kchunk-16thread.9.3 /srv/z/none/test-none-64kchunk-16thread.9.4 /srv/z/none/test-none-64kchunk-16thread.9.5 /srv/z/none/test-none-64kchunk-16thread.9.6 /srv/z/none/test-none-64kchunk-16thread.9.7 /srv/z/none/test-none-64kchunk-16thread.9.8 /srv/z/none/test-none-64kchunk-16thread.9.9 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-comp-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/comp/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-comp-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=2931MiB/s,w=2928MiB/s][r=46.9k,w=46.9k IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.5%][r=2715MiB/s,w=2727MiB/s][r=43.4k,w=43.6k IOPS][eta 01m:41s] Jobs: 16 (f=256): [M(16)][24.2%][r=2660MiB/s,w=2664MiB/s][r=42.6k,w=42.6k IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.8%][r=2843MiB/s,w=2847MiB/s][r=45.5k,w=45.5k IOPS][eta 01m:20s] Jobs: 16 (f=256): [M(16)][40.0%][r=2922MiB/s,w=2924MiB/s][r=46.8k,w=46.8k IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][48.3%][r=2892MiB/s,w=2885MiB/s][r=46.3k,w=46.2k IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][56.2%][r=2878MiB/s,w=2879MiB/s][r=46.0k,w=46.1k IOPS][eta 00m:53s] | |
| Jobs: 16 (f=256): [M(16)][64.7%][r=2645MiB/s,w=2643MiB/s][r=42.3k,w=42.3k IOPS][eta 00m:42s] Jobs: 16 (f=256): [M(16)][72.5%][r=2718MiB/s,w=2724MiB/s][r=43.5k,w=43.6k IOPS][eta 00m:33s] | |
| Jobs: 16 (f=256): [M(16)][80.0%][r=2664MiB/s,w=2669MiB/s][r=42.6k,w=42.7k IOPS][eta 00m:24s] Jobs: 16 (f=256): [M(16)][88.3%][r=2540MiB/s,w=2542MiB/s][r=40.6k,w=40.7k IOPS][eta 00m:14s] | |
| Jobs: 16 (f=256): [M(16)][95.9%][r=2586MiB/s,w=2586MiB/s][r=41.4k,w=41.4k IOPS][eta 00m:05s] | |
| test-comp-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=1944480: Wed Feb 26 13:08:13 2020 | |
| read: IOPS=43.9k, BW=2745MiB/s (2879MB/s)(322GiB/120006msec) | |
| slat (usec): min=10, max=157229, avg=151.41, stdev=869.67 | |
| clat (usec): min=7, max=235692, avg=11465.11, stdev=11050.56 | |
| lat (usec): min=86, max=235737, avg=11616.93, stdev=11171.91 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 4], 10.00th=[ 5], 20.00th=[ 5], | |
| | 30.00th=[ 6], 40.00th=[ 6], 50.00th=[ 8], 60.00th=[ 9], | |
| | 70.00th=[ 12], 80.00th=[ 17], 90.00th=[ 25], 95.00th=[ 34], | |
| | 99.00th=[ 56], 99.50th=[ 67], 99.90th=[ 92], 99.95th=[ 107], | |
| | 99.99th=[ 144] | |
| bw ( MiB/s): min= 1277, max= 5621, per=99.79%, avg=2739.54, stdev=48.83, samples=3825 | |
| iops : min=20441, max=89945, avg=43830.33, stdev=781.30, samples=3825 | |
| write: IOPS=43.0k, BW=2747MiB/s (2881MB/s)(322GiB/120006msec); 0 zone resets | |
| slat (usec): min=19, max=143402, avg=203.67, stdev=1072.21 | |
| clat (usec): min=4, max=235686, avg=11473.79, stdev=11026.74 | |
| lat (usec): min=56, max=235761, avg=11677.92, stdev=11151.54 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 4], 10.00th=[ 5], 20.00th=[ 5], | |
| | 30.00th=[ 6], 40.00th=[ 6], 50.00th=[ 8], 60.00th=[ 9], | |
| | 70.00th=[ 12], 80.00th=[ 17], 90.00th=[ 25], 95.00th=[ 34], | |
| | 99.00th=[ 56], 99.50th=[ 66], 99.90th=[ 93], 99.95th=[ 107], | |
| | 99.99th=[ 144] | |
| bw ( MiB/s): min= 1270, max= 5621, per=99.79%, avg=2741.40, stdev=48.61, samples=3825 | |
| iops : min=20318, max=89940, avg=43860.10, stdev=777.74, samples=3825 | |
| lat (usec) : 10=0.01%, 20=0.01%, 100=0.01%, 250=0.01%, 500=0.01% | |
| lat (usec) : 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=0.01%, 4=8.53%, 10=55.52%, 20=21.46%, 50=13.04% | |
| lat (msec) : 100=1.38%, 250=0.07% | |
| cpu : usr=3.08%, sys=42.03%, ctx=1583989, majf=0, minf=528063 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=5271285,5274997,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=2745MiB/s (2879MB/s), 2745MiB/s-2745MiB/s (2879MB/s-2879MB/s), io=322GiB (345GB), run=120006-120006msec | |
| WRITE: bw=2747MiB/s (2881MB/s), 2747MiB/s-2747MiB/s (2881MB/s-2881MB/s), io=322GiB (346GB), run=120006-120006msec | |
| + rm -rf /srv/z/comp/test-comp-64kchunk-16thread.0.0 /srv/z/comp/test-comp-64kchunk-16thread.0.1 /srv/z/comp/test-comp-64kchunk-16thread.0.10 /srv/z/comp/test-comp-64kchunk-16thread.0.11 /srv/z/comp/test-comp-64kchunk-16thread.0.12 /srv/z/comp/test-comp-64kchunk-16thread.0.13 /srv/z/comp/test-comp-64kchunk-16thread.0.14 /srv/z/comp/test-comp-64kchunk-16thread.0.15 /srv/z/comp/test-comp-64kchunk-16thread.0.2 /srv/z/comp/test-comp-64kchunk-16thread.0.3 /srv/z/comp/test-comp-64kchunk-16thread.0.4 /srv/z/comp/test-comp-64kchunk-16thread.0.5 /srv/z/comp/test-comp-64kchunk-16thread.0.6 /srv/z/comp/test-comp-64kchunk-16thread.0.7 /srv/z/comp/test-comp-64kchunk-16thread.0.8 /srv/z/comp/test-comp-64kchunk-16thread.0.9 /srv/z/comp/test-comp-64kchunk-16thread.1.0 /srv/z/comp/test-comp-64kchunk-16thread.10.0 /srv/z/comp/test-comp-64kchunk-16thread.10.1 /srv/z/comp/test-comp-64kchunk-16thread.10.10 /srv/z/comp/test-comp-64kchunk-16thread.10.11 /srv/z/comp/test-comp-64kchunk-16thread.10.12 /srv/z/comp/test-comp-64kchunk-16thread.10.13 /srv/z/comp/test-comp-64kchunk-16thread.10.14 /srv/z/comp/test-comp-64kchunk-16thread.10.15 /srv/z/comp/test-comp-64kchunk-16thread.10.2 /srv/z/comp/test-comp-64kchunk-16thread.10.3 /srv/z/comp/test-comp-64kchunk-16thread.10.4 /srv/z/comp/test-comp-64kchunk-16thread.10.5 /srv/z/comp/test-comp-64kchunk-16thread.10.6 /srv/z/comp/test-comp-64kchunk-16thread.10.7 /srv/z/comp/test-comp-64kchunk-16thread.10.8 /srv/z/comp/test-comp-64kchunk-16thread.10.9 /srv/z/comp/test-comp-64kchunk-16thread.1.1 /srv/z/comp/test-comp-64kchunk-16thread.1.10 /srv/z/comp/test-comp-64kchunk-16thread.11.0 /srv/z/comp/test-comp-64kchunk-16thread.1.11 /srv/z/comp/test-comp-64kchunk-16thread.11.1 /srv/z/comp/test-comp-64kchunk-16thread.11.10 /srv/z/comp/test-comp-64kchunk-16thread.11.11 /srv/z/comp/test-comp-64kchunk-16thread.11.12 /srv/z/comp/test-comp-64kchunk-16thread.11.13 /srv/z/comp/test-comp-64kchunk-16thread.11.14 /srv/z/comp/test-comp-64kchunk-16thread.11.15 /srv/z/comp/test-comp-64kchunk-16thread.1.12 /srv/z/comp/test-comp-64kchunk-16thread.11.2 /srv/z/comp/test-comp-64kchunk-16thread.1.13 /srv/z/comp/test-comp-64kchunk-16thread.11.3 /srv/z/comp/test-comp-64kchunk-16thread.1.14 /srv/z/comp/test-comp-64kchunk-16thread.11.4 /srv/z/comp/test-comp-64kchunk-16thread.1.15 /srv/z/comp/test-comp-64kchunk-16thread.11.5 /srv/z/comp/test-comp-64kchunk-16thread.11.6 /srv/z/comp/test-comp-64kchunk-16thread.11.7 /srv/z/comp/test-comp-64kchunk-16thread.11.8 /srv/z/comp/test-comp-64kchunk-16thread.11.9 /srv/z/comp/test-comp-64kchunk-16thread.1.2 /srv/z/comp/test-comp-64kchunk-16thread.12.0 /srv/z/comp/test-comp-64kchunk-16thread.12.1 /srv/z/comp/test-comp-64kchunk-16thread.12.10 /srv/z/comp/test-comp-64kchunk-16thread.12.11 /srv/z/comp/test-comp-64kchunk-16thread.12.12 /srv/z/comp/test-comp-64kchunk-16thread.12.13 /srv/z/comp/test-comp-64kchunk-16thread.12.14 /srv/z/comp/test-comp-64kchunk-16thread.12.15 /srv/z/comp/test-comp-64kchunk-16thread.12.2 /srv/z/comp/test-comp-64kchunk-16thread.12.3 /srv/z/comp/test-comp-64kchunk-16thread.12.4 /srv/z/comp/test-comp-64kchunk-16thread.12.5 /srv/z/comp/test-comp-64kchunk-16thread.12.6 /srv/z/comp/test-comp-64kchunk-16thread.12.7 /srv/z/comp/test-comp-64kchunk-16thread.12.8 /srv/z/comp/test-comp-64kchunk-16thread.12.9 /srv/z/comp/test-comp-64kchunk-16thread.1.3 /srv/z/comp/test-comp-64kchunk-16thread.13.0 /srv/z/comp/test-comp-64kchunk-16thread.13.1 /srv/z/comp/test-comp-64kchunk-16thread.13.10 /srv/z/comp/test-comp-64kchunk-16thread.13.11 /srv/z/comp/test-comp-64kchunk-16thread.13.12 /srv/z/comp/test-comp-64kchunk-16thread.13.13 /srv/z/comp/test-comp-64kchunk-16thread.13.14 /srv/z/comp/test-comp-64kchunk-16thread.13.15 /srv/z/comp/test-comp-64kchunk-16thread.13.2 /srv/z/comp/test-comp-64kchunk-16thread.13.3 /srv/z/comp/test-comp-64kchunk-16thread.13.4 /srv/z/comp/test-comp-64kchunk-16thread.13.5 /srv/z/comp/test-comp-64kchunk-16thread.13.6 /srv/z/comp/test-comp-64kchunk-16thread.13.7 /srv/z/comp/test-comp-64kchunk-16thread.13.8 /srv/z/comp/test-comp-64kchunk-16thread.13.9 /srv/z/comp/test-comp-64kchunk-16thread.1.4 /srv/z/comp/test-comp-64kchunk-16thread.14.0 /srv/z/comp/test-comp-64kchunk-16thread.14.1 /srv/z/comp/test-comp-64kchunk-16thread.14.10 /srv/z/comp/test-comp-64kchunk-16thread.14.11 /srv/z/comp/test-comp-64kchunk-16thread.14.12 /srv/z/comp/test-comp-64kchunk-16thread.14.13 /srv/z/comp/test-comp-64kchunk-16thread.14.14 /srv/z/comp/test-comp-64kchunk-16thread.14.15 /srv/z/comp/test-comp-64kchunk-16thread.14.2 /srv/z/comp/test-comp-64kchunk-16thread.14.3 /srv/z/comp/test-comp-64kchunk-16thread.14.4 /srv/z/comp/test-comp-64kchunk-16thread.14.5 /srv/z/comp/test-comp-64kchunk-16thread.14.6 /srv/z/comp/test-comp-64kchunk-16thread.14.7 /srv/z/comp/test-comp-64kchunk-16thread.14.8 /srv/z/comp/test-comp-64kchunk-16thread.14.9 /srv/z/comp/test-comp-64kchunk-16thread.1.5 /srv/z/comp/test-comp-64kchunk-16thread.15.0 /srv/z/comp/test-comp-64kchunk-16thread.15.1 /srv/z/comp/test-comp-64kchunk-16thread.15.10 /srv/z/comp/test-comp-64kchunk-16thread.15.11 /srv/z/comp/test-comp-64kchunk-16thread.15.12 /srv/z/comp/test-comp-64kchunk-16thread.15.13 /srv/z/comp/test-comp-64kchunk-16thread.15.14 /srv/z/comp/test-comp-64kchunk-16thread.15.15 /srv/z/comp/test-comp-64kchunk-16thread.15.2 /srv/z/comp/test-comp-64kchunk-16thread.15.3 /srv/z/comp/test-comp-64kchunk-16thread.15.4 /srv/z/comp/test-comp-64kchunk-16thread.15.5 /srv/z/comp/test-comp-64kchunk-16thread.15.6 /srv/z/comp/test-comp-64kchunk-16thread.15.7 /srv/z/comp/test-comp-64kchunk-16thread.15.8 /srv/z/comp/test-comp-64kchunk-16thread.15.9 /srv/z/comp/test-comp-64kchunk-16thread.1.6 /srv/z/comp/test-comp-64kchunk-16thread.1.7 /srv/z/comp/test-comp-64kchunk-16thread.1.8 /srv/z/comp/test-comp-64kchunk-16thread.1.9 /srv/z/comp/test-comp-64kchunk-16thread.2.0 /srv/z/comp/test-comp-64kchunk-16thread.2.1 /srv/z/comp/test-comp-64kchunk-16thread.2.10 /srv/z/comp/test-comp-64kchunk-16thread.2.11 /srv/z/comp/test-comp-64kchunk-16thread.2.12 /srv/z/comp/test-comp-64kchunk-16thread.2.13 /srv/z/comp/test-comp-64kchunk-16thread.2.14 /srv/z/comp/test-comp-64kchunk-16thread.2.15 /srv/z/comp/test-comp-64kchunk-16thread.2.2 /srv/z/comp/test-comp-64kchunk-16thread.2.3 /srv/z/comp/test-comp-64kchunk-16thread.2.4 /srv/z/comp/test-comp-64kchunk-16thread.2.5 /srv/z/comp/test-comp-64kchunk-16thread.2.6 /srv/z/comp/test-comp-64kchunk-16thread.2.7 /srv/z/comp/test-comp-64kchunk-16thread.2.8 /srv/z/comp/test-comp-64kchunk-16thread.2.9 /srv/z/comp/test-comp-64kchunk-16thread.3.0 /srv/z/comp/test-comp-64kchunk-16thread.3.1 /srv/z/comp/test-comp-64kchunk-16thread.3.10 /srv/z/comp/test-comp-64kchunk-16thread.3.11 /srv/z/comp/test-comp-64kchunk-16thread.3.12 /srv/z/comp/test-comp-64kchunk-16thread.3.13 /srv/z/comp/test-comp-64kchunk-16thread.3.14 /srv/z/comp/test-comp-64kchunk-16thread.3.15 /srv/z/comp/test-comp-64kchunk-16thread.3.2 /srv/z/comp/test-comp-64kchunk-16thread.3.3 /srv/z/comp/test-comp-64kchunk-16thread.3.4 /srv/z/comp/test-comp-64kchunk-16thread.3.5 /srv/z/comp/test-comp-64kchunk-16thread.3.6 /srv/z/comp/test-comp-64kchunk-16thread.3.7 /srv/z/comp/test-comp-64kchunk-16thread.3.8 /srv/z/comp/test-comp-64kchunk-16thread.3.9 /srv/z/comp/test-comp-64kchunk-16thread.4.0 /srv/z/comp/test-comp-64kchunk-16thread.4.1 /srv/z/comp/test-comp-64kchunk-16thread.4.10 /srv/z/comp/test-comp-64kchunk-16thread.4.11 /srv/z/comp/test-comp-64kchunk-16thread.4.12 /srv/z/comp/test-comp-64kchunk-16thread.4.13 /srv/z/comp/test-comp-64kchunk-16thread.4.14 /srv/z/comp/test-comp-64kchunk-16thread.4.15 /srv/z/comp/test-comp-64kchunk-16thread.4.2 /srv/z/comp/test-comp-64kchunk-16thread.4.3 /srv/z/comp/test-comp-64kchunk-16thread.4.4 /srv/z/comp/test-comp-64kchunk-16thread.4.5 /srv/z/comp/test-comp-64kchunk-16thread.4.6 /srv/z/comp/test-comp-64kchunk-16thread.4.7 /srv/z/comp/test-comp-64kchunk-16thread.4.8 /srv/z/comp/test-comp-64kchunk-16thread.4.9 /srv/z/comp/test-comp-64kchunk-16thread.5.0 /srv/z/comp/test-comp-64kchunk-16thread.5.1 /srv/z/comp/test-comp-64kchunk-16thread.5.10 /srv/z/comp/test-comp-64kchunk-16thread.5.11 /srv/z/comp/test-comp-64kchunk-16thread.5.12 /srv/z/comp/test-comp-64kchunk-16thread.5.13 /srv/z/comp/test-comp-64kchunk-16thread.5.14 /srv/z/comp/test-comp-64kchunk-16thread.5.15 /srv/z/comp/test-comp-64kchunk-16thread.5.2 /srv/z/comp/test-comp-64kchunk-16thread.5.3 /srv/z/comp/test-comp-64kchunk-16thread.5.4 /srv/z/comp/test-comp-64kchunk-16thread.5.5 /srv/z/comp/test-comp-64kchunk-16thread.5.6 /srv/z/comp/test-comp-64kchunk-16thread.5.7 /srv/z/comp/test-comp-64kchunk-16thread.5.8 /srv/z/comp/test-comp-64kchunk-16thread.5.9 /srv/z/comp/test-comp-64kchunk-16thread.6.0 /srv/z/comp/test-comp-64kchunk-16thread.6.1 /srv/z/comp/test-comp-64kchunk-16thread.6.10 /srv/z/comp/test-comp-64kchunk-16thread.6.11 /srv/z/comp/test-comp-64kchunk-16thread.6.12 /srv/z/comp/test-comp-64kchunk-16thread.6.13 /srv/z/comp/test-comp-64kchunk-16thread.6.14 /srv/z/comp/test-comp-64kchunk-16thread.6.15 /srv/z/comp/test-comp-64kchunk-16thread.6.2 /srv/z/comp/test-comp-64kchunk-16thread.6.3 /srv/z/comp/test-comp-64kchunk-16thread.6.4 /srv/z/comp/test-comp-64kchunk-16thread.6.5 /srv/z/comp/test-comp-64kchunk-16thread.6.6 /srv/z/comp/test-comp-64kchunk-16thread.6.7 /srv/z/comp/test-comp-64kchunk-16thread.6.8 /srv/z/comp/test-comp-64kchunk-16thread.6.9 /srv/z/comp/test-comp-64kchunk-16thread.7.0 /srv/z/comp/test-comp-64kchunk-16thread.7.1 /srv/z/comp/test-comp-64kchunk-16thread.7.10 /srv/z/comp/test-comp-64kchunk-16thread.7.11 /srv/z/comp/test-comp-64kchunk-16thread.7.12 /srv/z/comp/test-comp-64kchunk-16thread.7.13 /srv/z/comp/test-comp-64kchunk-16thread.7.14 /srv/z/comp/test-comp-64kchunk-16thread.7.15 /srv/z/comp/test-comp-64kchunk-16thread.7.2 /srv/z/comp/test-comp-64kchunk-16thread.7.3 /srv/z/comp/test-comp-64kchunk-16thread.7.4 /srv/z/comp/test-comp-64kchunk-16thread.7.5 /srv/z/comp/test-comp-64kchunk-16thread.7.6 /srv/z/comp/test-comp-64kchunk-16thread.7.7 /srv/z/comp/test-comp-64kchunk-16thread.7.8 /srv/z/comp/test-comp-64kchunk-16thread.7.9 /srv/z/comp/test-comp-64kchunk-16thread.8.0 /srv/z/comp/test-comp-64kchunk-16thread.8.1 /srv/z/comp/test-comp-64kchunk-16thread.8.10 /srv/z/comp/test-comp-64kchunk-16thread.8.11 /srv/z/comp/test-comp-64kchunk-16thread.8.12 /srv/z/comp/test-comp-64kchunk-16thread.8.13 /srv/z/comp/test-comp-64kchunk-16thread.8.14 /srv/z/comp/test-comp-64kchunk-16thread.8.15 /srv/z/comp/test-comp-64kchunk-16thread.8.2 /srv/z/comp/test-comp-64kchunk-16thread.8.3 /srv/z/comp/test-comp-64kchunk-16thread.8.4 /srv/z/comp/test-comp-64kchunk-16thread.8.5 /srv/z/comp/test-comp-64kchunk-16thread.8.6 /srv/z/comp/test-comp-64kchunk-16thread.8.7 /srv/z/comp/test-comp-64kchunk-16thread.8.8 /srv/z/comp/test-comp-64kchunk-16thread.8.9 /srv/z/comp/test-comp-64kchunk-16thread.9.0 /srv/z/comp/test-comp-64kchunk-16thread.9.1 /srv/z/comp/test-comp-64kchunk-16thread.9.10 /srv/z/comp/test-comp-64kchunk-16thread.9.11 /srv/z/comp/test-comp-64kchunk-16thread.9.12 /srv/z/comp/test-comp-64kchunk-16thread.9.13 /srv/z/comp/test-comp-64kchunk-16thread.9.14 /srv/z/comp/test-comp-64kchunk-16thread.9.15 /srv/z/comp/test-comp-64kchunk-16thread.9.2 /srv/z/comp/test-comp-64kchunk-16thread.9.3 /srv/z/comp/test-comp-64kchunk-16thread.9.4 /srv/z/comp/test-comp-64kchunk-16thread.9.5 /srv/z/comp/test-comp-64kchunk-16thread.9.6 /srv/z/comp/test-comp-64kchunk-16thread.9.7 /srv/z/comp/test-comp-64kchunk-16thread.9.8 /srv/z/comp/test-comp-64kchunk-16thread.9.9 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-enc-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/enc/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-enc-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=430MiB/s,w=432MiB/s][r=6881,w=6906 IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.5%][r=373MiB/s,w=376MiB/s][r=5967,w=6008 IOPS][eta 01m:41s] Jobs: 16 (f=256): [M(16)][24.2%][r=325MiB/s,w=326MiB/s][r=5204,w=5220 IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.5%][r=339MiB/s,w=339MiB/s][r=5429,w=5419 IOPS][eta 01m:21s] Jobs: 16 (f=256): [M(16)][40.0%][r=333MiB/s,w=331MiB/s][r=5332,w=5297 IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][48.3%][r=318MiB/s,w=318MiB/s][r=5088,w=5095 IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][56.2%][r=327MiB/s,w=327MiB/s][r=5230,w=5233 IOPS][eta 00m:53s] | |
| Jobs: 16 (f=256): [M(16)][64.2%][r=340MiB/s,w=338MiB/s][r=5439,w=5404 IOPS][eta 00m:43s] Jobs: 16 (f=256): [M(16)][71.9%][r=337MiB/s,w=339MiB/s][r=5388,w=5416 IOPS][eta 00m:34s] | |
| Jobs: 16 (f=256): [M(16)][80.0%][r=346MiB/s,w=345MiB/s][r=5539,w=5518 IOPS][eta 00m:24s] Jobs: 16 (f=256): [M(16)][88.3%][r=354MiB/s,w=354MiB/s][r=5660,w=5663 IOPS][eta 00m:14s] | |
| Jobs: 16 (f=256): [M(16)][95.8%][r=374MiB/s,w=373MiB/s][r=5982,w=5972 IOPS][eta 00m:05s] | |
| test-enc-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=3808807: Wed Feb 26 13:31:06 2020 | |
| read: IOPS=5629, BW=352MiB/s (369MB/s)(41.2GiB/120007msec) | |
| slat (usec): min=8, max=180163, avg=1408.29, stdev=3490.27 | |
| clat (usec): min=4, max=492606, avg=89402.15, stdev=51657.24 | |
| lat (usec): min=200, max=500879, avg=90810.82, stdev=52344.17 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 15], 10.00th=[ 26], 20.00th=[ 43], | |
| | 30.00th=[ 63], 40.00th=[ 77], 50.00th=[ 86], 60.00th=[ 95], | |
| | 70.00th=[ 107], 80.00th=[ 126], 90.00th=[ 161], 95.00th=[ 188], | |
| | 99.00th=[ 236], 99.50th=[ 255], 99.90th=[ 305], 99.95th=[ 326], | |
| | 99.99th=[ 401] | |
| bw ( KiB/s): min=120976, max=970108, per=99.68%, avg=359164.94, stdev=14097.81, samples=3838 | |
| iops : min= 1887, max=15157, avg=5610.09, stdev=220.30, samples=3838 | |
| write: IOPS=5628, BW=352MiB/s (369MB/s)(41.2GiB/120007msec); 0 zone resets | |
| slat (usec): min=15, max=131387, avg=1426.15, stdev=3647.07 | |
| clat (usec): min=4, max=468741, avg=89478.51, stdev=51701.44 | |
| lat (usec): min=263, max=468788, avg=90905.07, stdev=52416.34 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 15], 10.00th=[ 26], 20.00th=[ 43], | |
| | 30.00th=[ 63], 40.00th=[ 77], 50.00th=[ 86], 60.00th=[ 95], | |
| | 70.00th=[ 107], 80.00th=[ 127], 90.00th=[ 161], 95.00th=[ 188], | |
| | 99.00th=[ 236], 99.50th=[ 257], 99.90th=[ 309], 99.95th=[ 326], | |
| | 99.99th=[ 397] | |
| bw ( KiB/s): min=119697, max=972199, per=99.67%, avg=359067.51, stdev=14030.77, samples=3838 | |
| iops : min= 1867, max=15190, avg=5608.52, stdev=219.26, samples=3838 | |
| lat (usec) : 10=0.01%, 250=0.01%, 500=0.01%, 750=0.01% | |
| lat (msec) : 2=0.01%, 4=1.20%, 10=1.68%, 20=4.54%, 50=16.82% | |
| lat (msec) : 100=40.65%, 250=34.51%, 500=0.60% | |
| cpu : usr=0.30%, sys=39.25%, ctx=866651, majf=0, minf=458125 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=675603,675517,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=352MiB/s (369MB/s), 352MiB/s-352MiB/s (369MB/s-369MB/s), io=41.2GiB (44.3GB), run=120007-120007msec | |
| WRITE: bw=352MiB/s (369MB/s), 352MiB/s-352MiB/s (369MB/s-369MB/s), io=41.2GiB (44.3GB), run=120007-120007msec | |
| + rm -rf /srv/z/enc/test-enc-64kchunk-16thread.0.0 /srv/z/enc/test-enc-64kchunk-16thread.0.1 /srv/z/enc/test-enc-64kchunk-16thread.0.10 /srv/z/enc/test-enc-64kchunk-16thread.0.11 /srv/z/enc/test-enc-64kchunk-16thread.0.12 /srv/z/enc/test-enc-64kchunk-16thread.0.13 /srv/z/enc/test-enc-64kchunk-16thread.0.14 /srv/z/enc/test-enc-64kchunk-16thread.0.15 /srv/z/enc/test-enc-64kchunk-16thread.0.2 /srv/z/enc/test-enc-64kchunk-16thread.0.3 /srv/z/enc/test-enc-64kchunk-16thread.0.4 /srv/z/enc/test-enc-64kchunk-16thread.0.5 /srv/z/enc/test-enc-64kchunk-16thread.0.6 /srv/z/enc/test-enc-64kchunk-16thread.0.7 /srv/z/enc/test-enc-64kchunk-16thread.0.8 /srv/z/enc/test-enc-64kchunk-16thread.0.9 /srv/z/enc/test-enc-64kchunk-16thread.1.0 /srv/z/enc/test-enc-64kchunk-16thread.10.0 /srv/z/enc/test-enc-64kchunk-16thread.10.1 /srv/z/enc/test-enc-64kchunk-16thread.10.10 /srv/z/enc/test-enc-64kchunk-16thread.10.11 /srv/z/enc/test-enc-64kchunk-16thread.10.12 /srv/z/enc/test-enc-64kchunk-16thread.10.13 /srv/z/enc/test-enc-64kchunk-16thread.10.14 /srv/z/enc/test-enc-64kchunk-16thread.10.15 /srv/z/enc/test-enc-64kchunk-16thread.10.2 /srv/z/enc/test-enc-64kchunk-16thread.10.3 /srv/z/enc/test-enc-64kchunk-16thread.10.4 /srv/z/enc/test-enc-64kchunk-16thread.10.5 /srv/z/enc/test-enc-64kchunk-16thread.10.6 /srv/z/enc/test-enc-64kchunk-16thread.10.7 /srv/z/enc/test-enc-64kchunk-16thread.10.8 /srv/z/enc/test-enc-64kchunk-16thread.10.9 /srv/z/enc/test-enc-64kchunk-16thread.1.1 /srv/z/enc/test-enc-64kchunk-16thread.1.10 /srv/z/enc/test-enc-64kchunk-16thread.11.0 /srv/z/enc/test-enc-64kchunk-16thread.1.11 /srv/z/enc/test-enc-64kchunk-16thread.11.1 /srv/z/enc/test-enc-64kchunk-16thread.11.10 /srv/z/enc/test-enc-64kchunk-16thread.11.11 /srv/z/enc/test-enc-64kchunk-16thread.11.12 /srv/z/enc/test-enc-64kchunk-16thread.11.13 /srv/z/enc/test-enc-64kchunk-16thread.11.14 /srv/z/enc/test-enc-64kchunk-16thread.11.15 /srv/z/enc/test-enc-64kchunk-16thread.1.12 /srv/z/enc/test-enc-64kchunk-16thread.11.2 /srv/z/enc/test-enc-64kchunk-16thread.1.13 /srv/z/enc/test-enc-64kchunk-16thread.11.3 /srv/z/enc/test-enc-64kchunk-16thread.1.14 /srv/z/enc/test-enc-64kchunk-16thread.11.4 /srv/z/enc/test-enc-64kchunk-16thread.1.15 /srv/z/enc/test-enc-64kchunk-16thread.11.5 /srv/z/enc/test-enc-64kchunk-16thread.11.6 /srv/z/enc/test-enc-64kchunk-16thread.11.7 /srv/z/enc/test-enc-64kchunk-16thread.11.8 /srv/z/enc/test-enc-64kchunk-16thread.11.9 /srv/z/enc/test-enc-64kchunk-16thread.1.2 /srv/z/enc/test-enc-64kchunk-16thread.12.0 /srv/z/enc/test-enc-64kchunk-16thread.12.1 /srv/z/enc/test-enc-64kchunk-16thread.12.10 /srv/z/enc/test-enc-64kchunk-16thread.12.11 /srv/z/enc/test-enc-64kchunk-16thread.12.12 /srv/z/enc/test-enc-64kchunk-16thread.12.13 /srv/z/enc/test-enc-64kchunk-16thread.12.14 /srv/z/enc/test-enc-64kchunk-16thread.12.15 /srv/z/enc/test-enc-64kchunk-16thread.12.2 /srv/z/enc/test-enc-64kchunk-16thread.12.3 /srv/z/enc/test-enc-64kchunk-16thread.12.4 /srv/z/enc/test-enc-64kchunk-16thread.12.5 /srv/z/enc/test-enc-64kchunk-16thread.12.6 /srv/z/enc/test-enc-64kchunk-16thread.12.7 /srv/z/enc/test-enc-64kchunk-16thread.12.8 /srv/z/enc/test-enc-64kchunk-16thread.12.9 /srv/z/enc/test-enc-64kchunk-16thread.1.3 /srv/z/enc/test-enc-64kchunk-16thread.13.0 /srv/z/enc/test-enc-64kchunk-16thread.13.1 /srv/z/enc/test-enc-64kchunk-16thread.13.10 /srv/z/enc/test-enc-64kchunk-16thread.13.11 /srv/z/enc/test-enc-64kchunk-16thread.13.12 /srv/z/enc/test-enc-64kchunk-16thread.13.13 /srv/z/enc/test-enc-64kchunk-16thread.13.14 /srv/z/enc/test-enc-64kchunk-16thread.13.15 /srv/z/enc/test-enc-64kchunk-16thread.13.2 /srv/z/enc/test-enc-64kchunk-16thread.13.3 /srv/z/enc/test-enc-64kchunk-16thread.13.4 /srv/z/enc/test-enc-64kchunk-16thread.13.5 /srv/z/enc/test-enc-64kchunk-16thread.13.6 /srv/z/enc/test-enc-64kchunk-16thread.13.7 /srv/z/enc/test-enc-64kchunk-16thread.13.8 /srv/z/enc/test-enc-64kchunk-16thread.13.9 /srv/z/enc/test-enc-64kchunk-16thread.1.4 /srv/z/enc/test-enc-64kchunk-16thread.14.0 /srv/z/enc/test-enc-64kchunk-16thread.14.1 /srv/z/enc/test-enc-64kchunk-16thread.14.10 /srv/z/enc/test-enc-64kchunk-16thread.14.11 /srv/z/enc/test-enc-64kchunk-16thread.14.12 /srv/z/enc/test-enc-64kchunk-16thread.14.13 /srv/z/enc/test-enc-64kchunk-16thread.14.14 /srv/z/enc/test-enc-64kchunk-16thread.14.15 /srv/z/enc/test-enc-64kchunk-16thread.14.2 /srv/z/enc/test-enc-64kchunk-16thread.14.3 /srv/z/enc/test-enc-64kchunk-16thread.14.4 /srv/z/enc/test-enc-64kchunk-16thread.14.5 /srv/z/enc/test-enc-64kchunk-16thread.14.6 /srv/z/enc/test-enc-64kchunk-16thread.14.7 /srv/z/enc/test-enc-64kchunk-16thread.14.8 /srv/z/enc/test-enc-64kchunk-16thread.14.9 /srv/z/enc/test-enc-64kchunk-16thread.1.5 /srv/z/enc/test-enc-64kchunk-16thread.15.0 /srv/z/enc/test-enc-64kchunk-16thread.15.1 /srv/z/enc/test-enc-64kchunk-16thread.15.10 /srv/z/enc/test-enc-64kchunk-16thread.15.11 /srv/z/enc/test-enc-64kchunk-16thread.15.12 /srv/z/enc/test-enc-64kchunk-16thread.15.13 /srv/z/enc/test-enc-64kchunk-16thread.15.14 /srv/z/enc/test-enc-64kchunk-16thread.15.15 /srv/z/enc/test-enc-64kchunk-16thread.15.2 /srv/z/enc/test-enc-64kchunk-16thread.15.3 /srv/z/enc/test-enc-64kchunk-16thread.15.4 /srv/z/enc/test-enc-64kchunk-16thread.15.5 /srv/z/enc/test-enc-64kchunk-16thread.15.6 /srv/z/enc/test-enc-64kchunk-16thread.15.7 /srv/z/enc/test-enc-64kchunk-16thread.15.8 /srv/z/enc/test-enc-64kchunk-16thread.15.9 /srv/z/enc/test-enc-64kchunk-16thread.1.6 /srv/z/enc/test-enc-64kchunk-16thread.1.7 /srv/z/enc/test-enc-64kchunk-16thread.1.8 /srv/z/enc/test-enc-64kchunk-16thread.1.9 /srv/z/enc/test-enc-64kchunk-16thread.2.0 /srv/z/enc/test-enc-64kchunk-16thread.2.1 /srv/z/enc/test-enc-64kchunk-16thread.2.10 /srv/z/enc/test-enc-64kchunk-16thread.2.11 /srv/z/enc/test-enc-64kchunk-16thread.2.12 /srv/z/enc/test-enc-64kchunk-16thread.2.13 /srv/z/enc/test-enc-64kchunk-16thread.2.14 /srv/z/enc/test-enc-64kchunk-16thread.2.15 /srv/z/enc/test-enc-64kchunk-16thread.2.2 /srv/z/enc/test-enc-64kchunk-16thread.2.3 /srv/z/enc/test-enc-64kchunk-16thread.2.4 /srv/z/enc/test-enc-64kchunk-16thread.2.5 /srv/z/enc/test-enc-64kchunk-16thread.2.6 /srv/z/enc/test-enc-64kchunk-16thread.2.7 /srv/z/enc/test-enc-64kchunk-16thread.2.8 /srv/z/enc/test-enc-64kchunk-16thread.2.9 /srv/z/enc/test-enc-64kchunk-16thread.3.0 /srv/z/enc/test-enc-64kchunk-16thread.3.1 /srv/z/enc/test-enc-64kchunk-16thread.3.10 /srv/z/enc/test-enc-64kchunk-16thread.3.11 /srv/z/enc/test-enc-64kchunk-16thread.3.12 /srv/z/enc/test-enc-64kchunk-16thread.3.13 /srv/z/enc/test-enc-64kchunk-16thread.3.14 /srv/z/enc/test-enc-64kchunk-16thread.3.15 /srv/z/enc/test-enc-64kchunk-16thread.3.2 /srv/z/enc/test-enc-64kchunk-16thread.3.3 /srv/z/enc/test-enc-64kchunk-16thread.3.4 /srv/z/enc/test-enc-64kchunk-16thread.3.5 /srv/z/enc/test-enc-64kchunk-16thread.3.6 /srv/z/enc/test-enc-64kchunk-16thread.3.7 /srv/z/enc/test-enc-64kchunk-16thread.3.8 /srv/z/enc/test-enc-64kchunk-16thread.3.9 /srv/z/enc/test-enc-64kchunk-16thread.4.0 /srv/z/enc/test-enc-64kchunk-16thread.4.1 /srv/z/enc/test-enc-64kchunk-16thread.4.10 /srv/z/enc/test-enc-64kchunk-16thread.4.11 /srv/z/enc/test-enc-64kchunk-16thread.4.12 /srv/z/enc/test-enc-64kchunk-16thread.4.13 /srv/z/enc/test-enc-64kchunk-16thread.4.14 /srv/z/enc/test-enc-64kchunk-16thread.4.15 /srv/z/enc/test-enc-64kchunk-16thread.4.2 /srv/z/enc/test-enc-64kchunk-16thread.4.3 /srv/z/enc/test-enc-64kchunk-16thread.4.4 /srv/z/enc/test-enc-64kchunk-16thread.4.5 /srv/z/enc/test-enc-64kchunk-16thread.4.6 /srv/z/enc/test-enc-64kchunk-16thread.4.7 /srv/z/enc/test-enc-64kchunk-16thread.4.8 /srv/z/enc/test-enc-64kchunk-16thread.4.9 /srv/z/enc/test-enc-64kchunk-16thread.5.0 /srv/z/enc/test-enc-64kchunk-16thread.5.1 /srv/z/enc/test-enc-64kchunk-16thread.5.10 /srv/z/enc/test-enc-64kchunk-16thread.5.11 /srv/z/enc/test-enc-64kchunk-16thread.5.12 /srv/z/enc/test-enc-64kchunk-16thread.5.13 /srv/z/enc/test-enc-64kchunk-16thread.5.14 /srv/z/enc/test-enc-64kchunk-16thread.5.15 /srv/z/enc/test-enc-64kchunk-16thread.5.2 /srv/z/enc/test-enc-64kchunk-16thread.5.3 /srv/z/enc/test-enc-64kchunk-16thread.5.4 /srv/z/enc/test-enc-64kchunk-16thread.5.5 /srv/z/enc/test-enc-64kchunk-16thread.5.6 /srv/z/enc/test-enc-64kchunk-16thread.5.7 /srv/z/enc/test-enc-64kchunk-16thread.5.8 /srv/z/enc/test-enc-64kchunk-16thread.5.9 /srv/z/enc/test-enc-64kchunk-16thread.6.0 /srv/z/enc/test-enc-64kchunk-16thread.6.1 /srv/z/enc/test-enc-64kchunk-16thread.6.10 /srv/z/enc/test-enc-64kchunk-16thread.6.11 /srv/z/enc/test-enc-64kchunk-16thread.6.12 /srv/z/enc/test-enc-64kchunk-16thread.6.13 /srv/z/enc/test-enc-64kchunk-16thread.6.14 /srv/z/enc/test-enc-64kchunk-16thread.6.15 /srv/z/enc/test-enc-64kchunk-16thread.6.2 /srv/z/enc/test-enc-64kchunk-16thread.6.3 /srv/z/enc/test-enc-64kchunk-16thread.6.4 /srv/z/enc/test-enc-64kchunk-16thread.6.5 /srv/z/enc/test-enc-64kchunk-16thread.6.6 /srv/z/enc/test-enc-64kchunk-16thread.6.7 /srv/z/enc/test-enc-64kchunk-16thread.6.8 /srv/z/enc/test-enc-64kchunk-16thread.6.9 /srv/z/enc/test-enc-64kchunk-16thread.7.0 /srv/z/enc/test-enc-64kchunk-16thread.7.1 /srv/z/enc/test-enc-64kchunk-16thread.7.10 /srv/z/enc/test-enc-64kchunk-16thread.7.11 /srv/z/enc/test-enc-64kchunk-16thread.7.12 /srv/z/enc/test-enc-64kchunk-16thread.7.13 /srv/z/enc/test-enc-64kchunk-16thread.7.14 /srv/z/enc/test-enc-64kchunk-16thread.7.15 /srv/z/enc/test-enc-64kchunk-16thread.7.2 /srv/z/enc/test-enc-64kchunk-16thread.7.3 /srv/z/enc/test-enc-64kchunk-16thread.7.4 /srv/z/enc/test-enc-64kchunk-16thread.7.5 /srv/z/enc/test-enc-64kchunk-16thread.7.6 /srv/z/enc/test-enc-64kchunk-16thread.7.7 /srv/z/enc/test-enc-64kchunk-16thread.7.8 /srv/z/enc/test-enc-64kchunk-16thread.7.9 /srv/z/enc/test-enc-64kchunk-16thread.8.0 /srv/z/enc/test-enc-64kchunk-16thread.8.1 /srv/z/enc/test-enc-64kchunk-16thread.8.10 /srv/z/enc/test-enc-64kchunk-16thread.8.11 /srv/z/enc/test-enc-64kchunk-16thread.8.12 /srv/z/enc/test-enc-64kchunk-16thread.8.13 /srv/z/enc/test-enc-64kchunk-16thread.8.14 /srv/z/enc/test-enc-64kchunk-16thread.8.15 /srv/z/enc/test-enc-64kchunk-16thread.8.2 /srv/z/enc/test-enc-64kchunk-16thread.8.3 /srv/z/enc/test-enc-64kchunk-16thread.8.4 /srv/z/enc/test-enc-64kchunk-16thread.8.5 /srv/z/enc/test-enc-64kchunk-16thread.8.6 /srv/z/enc/test-enc-64kchunk-16thread.8.7 /srv/z/enc/test-enc-64kchunk-16thread.8.8 /srv/z/enc/test-enc-64kchunk-16thread.8.9 /srv/z/enc/test-enc-64kchunk-16thread.9.0 /srv/z/enc/test-enc-64kchunk-16thread.9.1 /srv/z/enc/test-enc-64kchunk-16thread.9.10 /srv/z/enc/test-enc-64kchunk-16thread.9.11 /srv/z/enc/test-enc-64kchunk-16thread.9.12 /srv/z/enc/test-enc-64kchunk-16thread.9.13 /srv/z/enc/test-enc-64kchunk-16thread.9.14 /srv/z/enc/test-enc-64kchunk-16thread.9.15 /srv/z/enc/test-enc-64kchunk-16thread.9.2 /srv/z/enc/test-enc-64kchunk-16thread.9.3 /srv/z/enc/test-enc-64kchunk-16thread.9.4 /srv/z/enc/test-enc-64kchunk-16thread.9.5 /srv/z/enc/test-enc-64kchunk-16thread.9.6 /srv/z/enc/test-enc-64kchunk-16thread.9.7 /srv/z/enc/test-enc-64kchunk-16thread.9.8 /srv/z/enc/test-enc-64kchunk-16thread.9.9 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-both-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/both/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-both-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=333MiB/s,w=334MiB/s][r=5334,w=5351 IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.5%][r=366MiB/s,w=368MiB/s][r=5851,w=5885 IOPS][eta 01m:41s] Jobs: 16 (f=256): [M(16)][24.2%][r=324MiB/s,w=327MiB/s][r=5177,w=5231 IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.5%][r=327MiB/s,w=324MiB/s][r=5228,w=5178 IOPS][eta 01m:21s] Jobs: 16 (f=256): [M(16)][40.0%][r=322MiB/s,w=322MiB/s][r=5153,w=5154 IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][48.3%][r=327MiB/s,w=326MiB/s][r=5239,w=5213 IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][55.8%][r=324MiB/s,w=324MiB/s][r=5176,w=5187 IOPS][eta 00m:53s] | |
| Jobs: 16 (f=256): [M(16)][64.2%][r=316MiB/s,w=315MiB/s][r=5063,w=5037 IOPS][eta 00m:43s] Jobs: 16 (f=256): [M(16)][71.9%][r=326MiB/s,w=327MiB/s][r=5222,w=5226 IOPS][eta 00m:34s] | |
| Jobs: 16 (f=256): [M(16)][80.0%][r=350MiB/s,w=350MiB/s][r=5606,w=5599 IOPS][eta 00m:24s] Jobs: 16 (f=256): [M(16)][88.3%][r=352MiB/s,w=352MiB/s][r=5632,w=5629 IOPS][eta 00m:14s] | |
| Jobs: 16 (f=256): [M(16)][95.8%][r=371MiB/s,w=369MiB/s][r=5929,w=5910 IOPS][eta 00m:05s] | |
| test-both-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=1382283: Wed Feb 26 13:54:05 2020 | |
| read: IOPS=5421, BW=339MiB/s (355MB/s)(39.7GiB/120021msec) | |
| slat (usec): min=9, max=222257, avg=1493.63, stdev=3667.77 | |
| clat (usec): min=4, max=515413, avg=92853.34, stdev=51235.58 | |
| lat (msec): min=2, max=515, avg=94.35, stdev=51.91 | |
| clat percentiles (msec): | |
| | 1.00th=[ 9], 5.00th=[ 19], 10.00th=[ 30], 20.00th=[ 47], | |
| | 30.00th=[ 67], 40.00th=[ 79], 50.00th=[ 88], 60.00th=[ 97], | |
| | 70.00th=[ 110], 80.00th=[ 131], 90.00th=[ 165], 95.00th=[ 190], | |
| | 99.00th=[ 236], 99.50th=[ 255], 99.90th=[ 309], 99.95th=[ 338], | |
| | 99.99th=[ 418] | |
| bw ( KiB/s): min=133432, max=752707, per=99.83%, avg=346420.76, stdev=10972.56, samples=3832 | |
| iops : min= 2082, max=11761, avg=5410.75, stdev=171.48, samples=3832 | |
| write: IOPS=5420, BW=339MiB/s (355MB/s)(39.7GiB/120021msec); 0 zone resets | |
| slat (usec): min=15, max=166400, avg=1449.57, stdev=3665.23 | |
| clat (usec): min=5, max=511152, avg=92872.94, stdev=51195.91 | |
| lat (msec): min=2, max=515, avg=94.32, stdev=51.89 | |
| clat percentiles (msec): | |
| | 1.00th=[ 9], 5.00th=[ 19], 10.00th=[ 30], 20.00th=[ 47], | |
| | 30.00th=[ 67], 40.00th=[ 79], 50.00th=[ 88], 60.00th=[ 97], | |
| | 70.00th=[ 110], 80.00th=[ 131], 90.00th=[ 165], 95.00th=[ 190], | |
| | 99.00th=[ 236], 99.50th=[ 255], 99.90th=[ 309], 99.95th=[ 338], | |
| | 99.99th=[ 414] | |
| bw ( KiB/s): min=135831, max=757644, per=99.84%, avg=346380.75, stdev=10916.25, samples=3832 | |
| iops : min= 2119, max=11838, avg=5410.12, stdev=170.59, samples=3832 | |
| lat (usec) : 10=0.01%, 20=0.01% | |
| lat (msec) : 4=0.27%, 10=1.28%, 20=4.05%, 50=15.84%, 100=41.25% | |
| lat (msec) : 250=36.71%, 500=0.60%, 750=0.01% | |
| cpu : usr=0.28%, sys=39.18%, ctx=870913, majf=0, minf=461014 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=650739,650625,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=339MiB/s (355MB/s), 339MiB/s-339MiB/s (355MB/s-355MB/s), io=39.7GiB (42.6GB), run=120021-120021msec | |
| WRITE: bw=339MiB/s (355MB/s), 339MiB/s-339MiB/s (355MB/s-355MB/s), io=39.7GiB (42.6GB), run=120021-120021msec | |
| + rm -rf /srv/z/both/test-both-64kchunk-16thread.0.0 /srv/z/both/test-both-64kchunk-16thread.0.1 /srv/z/both/test-both-64kchunk-16thread.0.10 /srv/z/both/test-both-64kchunk-16thread.0.11 /srv/z/both/test-both-64kchunk-16thread.0.12 /srv/z/both/test-both-64kchunk-16thread.0.13 /srv/z/both/test-both-64kchunk-16thread.0.14 /srv/z/both/test-both-64kchunk-16thread.0.15 /srv/z/both/test-both-64kchunk-16thread.0.2 /srv/z/both/test-both-64kchunk-16thread.0.3 /srv/z/both/test-both-64kchunk-16thread.0.4 /srv/z/both/test-both-64kchunk-16thread.0.5 /srv/z/both/test-both-64kchunk-16thread.0.6 /srv/z/both/test-both-64kchunk-16thread.0.7 /srv/z/both/test-both-64kchunk-16thread.0.8 /srv/z/both/test-both-64kchunk-16thread.0.9 /srv/z/both/test-both-64kchunk-16thread.1.0 /srv/z/both/test-both-64kchunk-16thread.10.0 /srv/z/both/test-both-64kchunk-16thread.10.1 /srv/z/both/test-both-64kchunk-16thread.10.10 /srv/z/both/test-both-64kchunk-16thread.10.11 /srv/z/both/test-both-64kchunk-16thread.10.12 /srv/z/both/test-both-64kchunk-16thread.10.13 /srv/z/both/test-both-64kchunk-16thread.10.14 /srv/z/both/test-both-64kchunk-16thread.10.15 /srv/z/both/test-both-64kchunk-16thread.10.2 /srv/z/both/test-both-64kchunk-16thread.10.3 /srv/z/both/test-both-64kchunk-16thread.10.4 /srv/z/both/test-both-64kchunk-16thread.10.5 /srv/z/both/test-both-64kchunk-16thread.10.6 /srv/z/both/test-both-64kchunk-16thread.10.7 /srv/z/both/test-both-64kchunk-16thread.10.8 /srv/z/both/test-both-64kchunk-16thread.10.9 /srv/z/both/test-both-64kchunk-16thread.1.1 /srv/z/both/test-both-64kchunk-16thread.1.10 /srv/z/both/test-both-64kchunk-16thread.11.0 /srv/z/both/test-both-64kchunk-16thread.1.11 /srv/z/both/test-both-64kchunk-16thread.11.1 /srv/z/both/test-both-64kchunk-16thread.11.10 /srv/z/both/test-both-64kchunk-16thread.11.11 /srv/z/both/test-both-64kchunk-16thread.11.12 /srv/z/both/test-both-64kchunk-16thread.11.13 /srv/z/both/test-both-64kchunk-16thread.11.14 /srv/z/both/test-both-64kchunk-16thread.11.15 /srv/z/both/test-both-64kchunk-16thread.1.12 /srv/z/both/test-both-64kchunk-16thread.11.2 /srv/z/both/test-both-64kchunk-16thread.1.13 /srv/z/both/test-both-64kchunk-16thread.11.3 /srv/z/both/test-both-64kchunk-16thread.1.14 /srv/z/both/test-both-64kchunk-16thread.11.4 /srv/z/both/test-both-64kchunk-16thread.1.15 /srv/z/both/test-both-64kchunk-16thread.11.5 /srv/z/both/test-both-64kchunk-16thread.11.6 /srv/z/both/test-both-64kchunk-16thread.11.7 /srv/z/both/test-both-64kchunk-16thread.11.8 /srv/z/both/test-both-64kchunk-16thread.11.9 /srv/z/both/test-both-64kchunk-16thread.1.2 /srv/z/both/test-both-64kchunk-16thread.12.0 /srv/z/both/test-both-64kchunk-16thread.12.1 /srv/z/both/test-both-64kchunk-16thread.12.10 /srv/z/both/test-both-64kchunk-16thread.12.11 /srv/z/both/test-both-64kchunk-16thread.12.12 /srv/z/both/test-both-64kchunk-16thread.12.13 /srv/z/both/test-both-64kchunk-16thread.12.14 /srv/z/both/test-both-64kchunk-16thread.12.15 /srv/z/both/test-both-64kchunk-16thread.12.2 /srv/z/both/test-both-64kchunk-16thread.12.3 /srv/z/both/test-both-64kchunk-16thread.12.4 /srv/z/both/test-both-64kchunk-16thread.12.5 /srv/z/both/test-both-64kchunk-16thread.12.6 /srv/z/both/test-both-64kchunk-16thread.12.7 /srv/z/both/test-both-64kchunk-16thread.12.8 /srv/z/both/test-both-64kchunk-16thread.12.9 /srv/z/both/test-both-64kchunk-16thread.1.3 /srv/z/both/test-both-64kchunk-16thread.13.0 /srv/z/both/test-both-64kchunk-16thread.13.1 /srv/z/both/test-both-64kchunk-16thread.13.10 /srv/z/both/test-both-64kchunk-16thread.13.11 /srv/z/both/test-both-64kchunk-16thread.13.12 /srv/z/both/test-both-64kchunk-16thread.13.13 /srv/z/both/test-both-64kchunk-16thread.13.14 /srv/z/both/test-both-64kchunk-16thread.13.15 /srv/z/both/test-both-64kchunk-16thread.13.2 /srv/z/both/test-both-64kchunk-16thread.13.3 /srv/z/both/test-both-64kchunk-16thread.13.4 /srv/z/both/test-both-64kchunk-16thread.13.5 /srv/z/both/test-both-64kchunk-16thread.13.6 /srv/z/both/test-both-64kchunk-16thread.13.7 /srv/z/both/test-both-64kchunk-16thread.13.8 /srv/z/both/test-both-64kchunk-16thread.13.9 /srv/z/both/test-both-64kchunk-16thread.1.4 /srv/z/both/test-both-64kchunk-16thread.14.0 /srv/z/both/test-both-64kchunk-16thread.14.1 /srv/z/both/test-both-64kchunk-16thread.14.10 /srv/z/both/test-both-64kchunk-16thread.14.11 /srv/z/both/test-both-64kchunk-16thread.14.12 /srv/z/both/test-both-64kchunk-16thread.14.13 /srv/z/both/test-both-64kchunk-16thread.14.14 /srv/z/both/test-both-64kchunk-16thread.14.15 /srv/z/both/test-both-64kchunk-16thread.14.2 /srv/z/both/test-both-64kchunk-16thread.14.3 /srv/z/both/test-both-64kchunk-16thread.14.4 /srv/z/both/test-both-64kchunk-16thread.14.5 /srv/z/both/test-both-64kchunk-16thread.14.6 /srv/z/both/test-both-64kchunk-16thread.14.7 /srv/z/both/test-both-64kchunk-16thread.14.8 /srv/z/both/test-both-64kchunk-16thread.14.9 /srv/z/both/test-both-64kchunk-16thread.1.5 /srv/z/both/test-both-64kchunk-16thread.15.0 /srv/z/both/test-both-64kchunk-16thread.15.1 /srv/z/both/test-both-64kchunk-16thread.15.10 /srv/z/both/test-both-64kchunk-16thread.15.11 /srv/z/both/test-both-64kchunk-16thread.15.12 /srv/z/both/test-both-64kchunk-16thread.15.13 /srv/z/both/test-both-64kchunk-16thread.15.14 /srv/z/both/test-both-64kchunk-16thread.15.15 /srv/z/both/test-both-64kchunk-16thread.15.2 /srv/z/both/test-both-64kchunk-16thread.15.3 /srv/z/both/test-both-64kchunk-16thread.15.4 /srv/z/both/test-both-64kchunk-16thread.15.5 /srv/z/both/test-both-64kchunk-16thread.15.6 /srv/z/both/test-both-64kchunk-16thread.15.7 /srv/z/both/test-both-64kchunk-16thread.15.8 /srv/z/both/test-both-64kchunk-16thread.15.9 /srv/z/both/test-both-64kchunk-16thread.1.6 /srv/z/both/test-both-64kchunk-16thread.1.7 /srv/z/both/test-both-64kchunk-16thread.1.8 /srv/z/both/test-both-64kchunk-16thread.1.9 /srv/z/both/test-both-64kchunk-16thread.2.0 /srv/z/both/test-both-64kchunk-16thread.2.1 /srv/z/both/test-both-64kchunk-16thread.2.10 /srv/z/both/test-both-64kchunk-16thread.2.11 /srv/z/both/test-both-64kchunk-16thread.2.12 /srv/z/both/test-both-64kchunk-16thread.2.13 /srv/z/both/test-both-64kchunk-16thread.2.14 /srv/z/both/test-both-64kchunk-16thread.2.15 /srv/z/both/test-both-64kchunk-16thread.2.2 /srv/z/both/test-both-64kchunk-16thread.2.3 /srv/z/both/test-both-64kchunk-16thread.2.4 /srv/z/both/test-both-64kchunk-16thread.2.5 /srv/z/both/test-both-64kchunk-16thread.2.6 /srv/z/both/test-both-64kchunk-16thread.2.7 /srv/z/both/test-both-64kchunk-16thread.2.8 /srv/z/both/test-both-64kchunk-16thread.2.9 /srv/z/both/test-both-64kchunk-16thread.3.0 /srv/z/both/test-both-64kchunk-16thread.3.1 /srv/z/both/test-both-64kchunk-16thread.3.10 /srv/z/both/test-both-64kchunk-16thread.3.11 /srv/z/both/test-both-64kchunk-16thread.3.12 /srv/z/both/test-both-64kchunk-16thread.3.13 /srv/z/both/test-both-64kchunk-16thread.3.14 /srv/z/both/test-both-64kchunk-16thread.3.15 /srv/z/both/test-both-64kchunk-16thread.3.2 /srv/z/both/test-both-64kchunk-16thread.3.3 /srv/z/both/test-both-64kchunk-16thread.3.4 /srv/z/both/test-both-64kchunk-16thread.3.5 /srv/z/both/test-both-64kchunk-16thread.3.6 /srv/z/both/test-both-64kchunk-16thread.3.7 /srv/z/both/test-both-64kchunk-16thread.3.8 /srv/z/both/test-both-64kchunk-16thread.3.9 /srv/z/both/test-both-64kchunk-16thread.4.0 /srv/z/both/test-both-64kchunk-16thread.4.1 /srv/z/both/test-both-64kchunk-16thread.4.10 /srv/z/both/test-both-64kchunk-16thread.4.11 /srv/z/both/test-both-64kchunk-16thread.4.12 /srv/z/both/test-both-64kchunk-16thread.4.13 /srv/z/both/test-both-64kchunk-16thread.4.14 /srv/z/both/test-both-64kchunk-16thread.4.15 /srv/z/both/test-both-64kchunk-16thread.4.2 /srv/z/both/test-both-64kchunk-16thread.4.3 /srv/z/both/test-both-64kchunk-16thread.4.4 /srv/z/both/test-both-64kchunk-16thread.4.5 /srv/z/both/test-both-64kchunk-16thread.4.6 /srv/z/both/test-both-64kchunk-16thread.4.7 /srv/z/both/test-both-64kchunk-16thread.4.8 /srv/z/both/test-both-64kchunk-16thread.4.9 /srv/z/both/test-both-64kchunk-16thread.5.0 /srv/z/both/test-both-64kchunk-16thread.5.1 /srv/z/both/test-both-64kchunk-16thread.5.10 /srv/z/both/test-both-64kchunk-16thread.5.11 /srv/z/both/test-both-64kchunk-16thread.5.12 /srv/z/both/test-both-64kchunk-16thread.5.13 /srv/z/both/test-both-64kchunk-16thread.5.14 /srv/z/both/test-both-64kchunk-16thread.5.15 /srv/z/both/test-both-64kchunk-16thread.5.2 /srv/z/both/test-both-64kchunk-16thread.5.3 /srv/z/both/test-both-64kchunk-16thread.5.4 /srv/z/both/test-both-64kchunk-16thread.5.5 /srv/z/both/test-both-64kchunk-16thread.5.6 /srv/z/both/test-both-64kchunk-16thread.5.7 /srv/z/both/test-both-64kchunk-16thread.5.8 /srv/z/both/test-both-64kchunk-16thread.5.9 /srv/z/both/test-both-64kchunk-16thread.6.0 /srv/z/both/test-both-64kchunk-16thread.6.1 /srv/z/both/test-both-64kchunk-16thread.6.10 /srv/z/both/test-both-64kchunk-16thread.6.11 /srv/z/both/test-both-64kchunk-16thread.6.12 /srv/z/both/test-both-64kchunk-16thread.6.13 /srv/z/both/test-both-64kchunk-16thread.6.14 /srv/z/both/test-both-64kchunk-16thread.6.15 /srv/z/both/test-both-64kchunk-16thread.6.2 /srv/z/both/test-both-64kchunk-16thread.6.3 /srv/z/both/test-both-64kchunk-16thread.6.4 /srv/z/both/test-both-64kchunk-16thread.6.5 /srv/z/both/test-both-64kchunk-16thread.6.6 /srv/z/both/test-both-64kchunk-16thread.6.7 /srv/z/both/test-both-64kchunk-16thread.6.8 /srv/z/both/test-both-64kchunk-16thread.6.9 /srv/z/both/test-both-64kchunk-16thread.7.0 /srv/z/both/test-both-64kchunk-16thread.7.1 /srv/z/both/test-both-64kchunk-16thread.7.10 /srv/z/both/test-both-64kchunk-16thread.7.11 /srv/z/both/test-both-64kchunk-16thread.7.12 /srv/z/both/test-both-64kchunk-16thread.7.13 /srv/z/both/test-both-64kchunk-16thread.7.14 /srv/z/both/test-both-64kchunk-16thread.7.15 /srv/z/both/test-both-64kchunk-16thread.7.2 /srv/z/both/test-both-64kchunk-16thread.7.3 /srv/z/both/test-both-64kchunk-16thread.7.4 /srv/z/both/test-both-64kchunk-16thread.7.5 /srv/z/both/test-both-64kchunk-16thread.7.6 /srv/z/both/test-both-64kchunk-16thread.7.7 /srv/z/both/test-both-64kchunk-16thread.7.8 /srv/z/both/test-both-64kchunk-16thread.7.9 /srv/z/both/test-both-64kchunk-16thread.8.0 /srv/z/both/test-both-64kchunk-16thread.8.1 /srv/z/both/test-both-64kchunk-16thread.8.10 /srv/z/both/test-both-64kchunk-16thread.8.11 /srv/z/both/test-both-64kchunk-16thread.8.12 /srv/z/both/test-both-64kchunk-16thread.8.13 /srv/z/both/test-both-64kchunk-16thread.8.14 /srv/z/both/test-both-64kchunk-16thread.8.15 /srv/z/both/test-both-64kchunk-16thread.8.2 /srv/z/both/test-both-64kchunk-16thread.8.3 /srv/z/both/test-both-64kchunk-16thread.8.4 /srv/z/both/test-both-64kchunk-16thread.8.5 /srv/z/both/test-both-64kchunk-16thread.8.6 /srv/z/both/test-both-64kchunk-16thread.8.7 /srv/z/both/test-both-64kchunk-16thread.8.8 /srv/z/both/test-both-64kchunk-16thread.8.9 /srv/z/both/test-both-64kchunk-16thread.9.0 /srv/z/both/test-both-64kchunk-16thread.9.1 /srv/z/both/test-both-64kchunk-16thread.9.10 /srv/z/both/test-both-64kchunk-16thread.9.11 /srv/z/both/test-both-64kchunk-16thread.9.12 /srv/z/both/test-both-64kchunk-16thread.9.13 /srv/z/both/test-both-64kchunk-16thread.9.14 /srv/z/both/test-both-64kchunk-16thread.9.15 /srv/z/both/test-both-64kchunk-16thread.9.2 /srv/z/both/test-both-64kchunk-16thread.9.3 /srv/z/both/test-both-64kchunk-16thread.9.4 /srv/z/both/test-both-64kchunk-16thread.9.5 /srv/z/both/test-both-64kchunk-16thread.9.6 /srv/z/both/test-both-64kchunk-16thread.9.7 /srv/z/both/test-both-64kchunk-16thread.9.8 /srv/z/both/test-both-64kchunk-16thread.9.9 | |
| + zpool destroy tank | |
| + for ASHIFTVAL in 12 13 14 15 16 | |
| + trim_disks | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 | |
| Success formatting namespace:1 | |
| + for DISK in $FULLPATHS | |
| + blkdiscard /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| + nvme format -r /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| Success formatting namespace:1 | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '################## waiting 60s for nvme format to complete...' | |
| ################## waiting 60s for nvme format to complete... | |
| + sleep 60 | |
| + nvme list | |
| Node SN Model Namespace Usage Format FW Rev | |
| ---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- -------- | |
| /dev/nvme0n1 S437NE0M804438 SAMSUNG MZQLB960HAJR-00007 1 45.19 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme1n1 S437NE0M804437 SAMSUNG MZQLB960HAJR-00007 1 89.69 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme2n1 S437NE0M804442 SAMSUNG MZQLB960HAJR-00007 1 100.88 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme3n1 S437NE0M804435 SAMSUNG MZQLB960HAJR-00007 1 36.62 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme4n1 S437NE0M802527 SAMSUNG MZQLB960HAJR-00007 1 78.68 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| /dev/nvme5n1 S437NX0M402110 SAMSUNG MZQLB960HAJR-00007 1 50.47 GB / 960.20 GB 512 B + 0 B EDA5202Q | |
| + print_divider | |
| + echo '##############################################################' | |
| ############################################################## | |
| + echo '##############################################################' | |
| ############################################################## | |
| + zpool create tank -o ashift=16 -O mountpoint=/srv/z raidz1 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M802527 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804435 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804437 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804438 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NE0M804442 /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M402110 | |
| + dd if=/dev/urandom of=/tmp/testkey.raw bs=32 count=1 | |
| 1+0 records in | |
| 1+0 records out | |
| 32 bytes copied, 0.000431353 s, 74.2 kB/s | |
| + COMP=' | |
| -o compression=lz4 | |
| ' | |
| + ENC=' | |
| -o encryption=aes-256-gcm | |
| -o keyformat=raw | |
| -o keylocation=file:///tmp/testkey.raw | |
| ' | |
| + zfs create tank/none | |
| + zfs create -o compression=lz4 tank/comp | |
| + zfs create -o encryption=aes-256-gcm -o keyformat=raw -o keylocation=file:///tmp/testkey.raw tank/enc | |
| + zfs create -o compression=lz4 -o encryption=aes-256-gcm -o keyformat=raw -o keylocation=file:///tmp/testkey.raw tank/both | |
| + FO='--eta-newline=10 --eta-interval=10 --eta=always' | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-none-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/none/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-none-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-none-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=1142MiB/s,w=1140MiB/s][r=18.3k,w=18.2k IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.7%][r=1176MiB/s,w=1175MiB/s][r=18.8k,w=18.8k IOPS][eta 01m:40s] Jobs: 1 (f=1): [M(1)][24.2%][r=1153MiB/s,w=1152MiB/s][r=18.4k,w=18.4k IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.8%][r=1158MiB/s,w=1155MiB/s][r=18.5k,w=18.5k IOPS][eta 01m:20s] Jobs: 1 (f=1): [M(1)][40.8%][r=1133MiB/s,w=1132MiB/s][r=18.1k,w=18.1k IOPS][eta 01m:11s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=1114MiB/s,w=1116MiB/s][r=17.8k,w=17.8k IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=1130MiB/s,w=1129MiB/s][r=18.1k,w=18.1k IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][65.5%][r=1151MiB/s,w=1147MiB/s][r=18.4k,w=18.3k IOPS][eta 00m:41s] Jobs: 1 (f=1): [M(1)][72.5%][r=1147MiB/s,w=1148MiB/s][r=18.4k,w=18.4k IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=1137MiB/s,w=1136MiB/s][r=18.2k,w=18.2k IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][89.9%][r=1145MiB/s,w=1143MiB/s][r=18.3k,w=18.3k IOPS][eta 00m:12s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=1144MiB/s,w=1141MiB/s][r=18.3k,w=18.3k IOPS][eta 00m:04s] | |
| test-none-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=1612261: Wed Feb 26 13:57:29 2020 | |
| read: IOPS=18.3k, BW=1145MiB/s (1201MB/s)(134GiB/120001msec) | |
| slat (usec): min=5, max=28870, avg=25.73, stdev=69.59 | |
| clat (usec): min=2, max=36727, avg=1721.74, stdev=933.72 | |
| lat (usec): min=38, max=36767, avg=1747.61, stdev=945.40 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1237], 5.00th=[ 1287], 10.00th=[ 1319], 20.00th=[ 1336], | |
| | 30.00th=[ 1385], 40.00th=[ 1418], 50.00th=[ 1450], 60.00th=[ 1500], | |
| | 70.00th=[ 1582], 80.00th=[ 1680], 90.00th=[ 2606], 95.00th=[ 3064], | |
| | 99.00th=[ 4490], 99.50th=[ 5407], 99.90th=[13829], 99.95th=[17695], | |
| | 99.99th=[27657] | |
| bw ( MiB/s): min= 952, max= 1474, per=99.93%, avg=1144.29, stdev=111.76, samples=239 | |
| iops : min=15240, max=23594, avg=18308.64, stdev=1788.11, samples=239 | |
| write: IOPS=18.3k, BW=1144MiB/s (1200MB/s)(134GiB/120001msec); 0 zone resets | |
| slat (usec): min=11, max=22200, avg=26.01, stdev=62.13 | |
| clat (usec): min=39, max=36722, avg=1719.64, stdev=927.04 | |
| lat (usec): min=63, max=36754, avg=1745.80, stdev=938.88 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1237], 5.00th=[ 1287], 10.00th=[ 1319], 20.00th=[ 1336], | |
| | 30.00th=[ 1385], 40.00th=[ 1418], 50.00th=[ 1450], 60.00th=[ 1500], | |
| | 70.00th=[ 1582], 80.00th=[ 1680], 90.00th=[ 2606], 95.00th=[ 3064], | |
| | 99.00th=[ 4424], 99.50th=[ 5342], 99.90th=[13698], 99.95th=[17695], | |
| | 99.99th=[28181] | |
| bw ( MiB/s): min= 960, max= 1472, per=99.93%, avg=1143.18, stdev=111.76, samples=239 | |
| iops : min=15374, max=23558, avg=18290.84, stdev=1788.15, samples=239 | |
| lat (usec) : 4=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01% | |
| lat (usec) : 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=82.80%, 4=15.60%, 10=1.43%, 20=0.13%, 50=0.03% | |
| cpu : usr=7.92%, sys=88.87%, ctx=22727, majf=0, minf=49248 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=2198637,2196451,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=1145MiB/s (1201MB/s), 1145MiB/s-1145MiB/s (1201MB/s-1201MB/s), io=134GiB (144GB), run=120001-120001msec | |
| WRITE: bw=1144MiB/s (1200MB/s), 1144MiB/s-1144MiB/s (1200MB/s-1200MB/s), io=134GiB (144GB), run=120001-120001msec | |
| + rm -rf /srv/z/none/test-none-64kchunk-1thread.0.0 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-comp-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/comp/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-comp-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-comp-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=1113MiB/s,w=1112MiB/s][r=17.8k,w=17.8k IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.5%][r=1110MiB/s,w=1108MiB/s][r=17.8k,w=17.7k IOPS][eta 01m:41s] Jobs: 1 (f=1): [M(1)][24.2%][r=1094MiB/s,w=1092MiB/s][r=17.5k,w=17.5k IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.8%][r=1079MiB/s,w=1078MiB/s][r=17.3k,w=17.2k IOPS][eta 01m:20s] Jobs: 1 (f=1): [M(1)][40.5%][r=1101MiB/s,w=1099MiB/s][r=17.6k,w=17.6k IOPS][eta 01m:12s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=1105MiB/s,w=1102MiB/s][r=17.7k,w=17.6k IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=1099MiB/s,w=1105MiB/s][r=17.6k,w=17.7k IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][65.0%][r=1101MiB/s,w=1093MiB/s][r=17.6k,w=17.5k IOPS][eta 00m:42s] Jobs: 1 (f=1): [M(1)][72.5%][r=1078MiB/s,w=1077MiB/s][r=17.3k,w=17.2k IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=1101MiB/s,w=1104MiB/s][r=17.6k,w=17.7k IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][88.4%][r=1084MiB/s,w=1079MiB/s][r=17.3k,w=17.3k IOPS][eta 00m:14s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=1109MiB/s,w=1107MiB/s][r=17.7k,w=17.7k IOPS][eta 00m:04s] | |
| test-comp-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=2006008: Wed Feb 26 13:59:41 2020 | |
| read: IOPS=17.6k, BW=1099MiB/s (1152MB/s)(129GiB/120001msec) | |
| slat (usec): min=5, max=100406, avg=26.96, stdev=221.50 | |
| clat (usec): min=191, max=102679, avg=1797.63, stdev=2308.58 | |
| lat (usec): min=211, max=102690, avg=1824.72, stdev=2332.52 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1205], 5.00th=[ 1287], 10.00th=[ 1303], 20.00th=[ 1336], | |
| | 30.00th=[ 1369], 40.00th=[ 1385], 50.00th=[ 1418], 60.00th=[ 1450], | |
| | 70.00th=[ 1516], 80.00th=[ 1598], 90.00th=[ 2442], 95.00th=[ 2933], | |
| | 99.00th=[ 8717], 99.50th=[15795], 99.90th=[34866], 99.95th=[44827], | |
| | 99.99th=[70779] | |
| bw ( MiB/s): min= 661, max= 1467, per=99.85%, avg=1097.08, stdev=168.33, samples=239 | |
| iops : min=10578, max=23474, avg=17553.14, stdev=2693.18, samples=239 | |
| write: IOPS=17.6k, BW=1097MiB/s (1151MB/s)(129GiB/120001msec); 0 zone resets | |
| slat (usec): min=11, max=80549, avg=26.93, stdev=200.16 | |
| clat (usec): min=2, max=102670, avg=1789.38, stdev=2253.96 | |
| lat (usec): min=33, max=102687, avg=1816.48, stdev=2275.40 | |
| clat percentiles (usec): | |
| | 1.00th=[ 1205], 5.00th=[ 1287], 10.00th=[ 1303], 20.00th=[ 1336], | |
| | 30.00th=[ 1369], 40.00th=[ 1385], 50.00th=[ 1418], 60.00th=[ 1450], | |
| | 70.00th=[ 1516], 80.00th=[ 1598], 90.00th=[ 2442], 95.00th=[ 2933], | |
| | 99.00th=[ 8455], 99.50th=[15270], 99.90th=[34341], 99.95th=[43254], | |
| | 99.99th=[70779] | |
| bw ( MiB/s): min= 657, max= 1451, per=99.86%, avg=1095.89, stdev=168.67, samples=239 | |
| iops : min=10520, max=23218, avg=17534.15, stdev=2698.67, samples=239 | |
| lat (usec) : 4=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=0.01% | |
| lat (usec) : 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=86.99%, 4=10.85%, 10=1.29%, 20=0.52%, 50=0.30% | |
| lat (msec) : 100=0.03%, 250=0.01% | |
| cpu : usr=7.31%, sys=81.69%, ctx=25619, majf=0, minf=62805 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=2109602,2107098,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=1099MiB/s (1152MB/s), 1099MiB/s-1099MiB/s (1152MB/s-1152MB/s), io=129GiB (138GB), run=120001-120001msec | |
| WRITE: bw=1097MiB/s (1151MB/s), 1097MiB/s-1097MiB/s (1151MB/s-1151MB/s), io=129GiB (138GB), run=120001-120001msec | |
| + rm -rf /srv/z/comp/test-comp-64kchunk-1thread.0.0 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-enc-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/enc/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-enc-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-enc-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=34.6MiB/s,w=34.9MiB/s][r=553,w=559 IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.5%][r=35.8MiB/s,w=35.1MiB/s][r=572,w=561 IOPS][eta 01m:41s] Jobs: 1 (f=1): [M(1)][24.2%][r=34.0MiB/s,w=35.0MiB/s][r=559,w=560 IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.5%][r=34.6MiB/s,w=34.6MiB/s][r=552,w=552 IOPS][eta 01m:21s] Jobs: 1 (f=1): [M(1)][40.5%][r=35.0MiB/s,w=35.1MiB/s][r=560,w=562 IOPS][eta 01m:12s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=34.9MiB/s,w=35.4MiB/s][r=558,w=566 IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=35.3MiB/s,w=35.1MiB/s][r=564,w=560 IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][64.5%][r=35.0MiB/s,w=32.8MiB/s][r=560,w=524 IOPS][eta 00m:43s] Jobs: 1 (f=1): [M(1)][72.5%][r=35.1MiB/s,w=35.2MiB/s][r=562,w=563 IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=35.1MiB/s,w=35.6MiB/s][r=561,w=570 IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][88.4%][r=34.9MiB/s,w=35.4MiB/s][r=559,w=566 IOPS][eta 00m:14s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=35.1MiB/s,w=33.0MiB/s][r=562,w=543 IOPS][eta 00m:04s] | |
| test-enc-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=2261607: Wed Feb 26 14:01:54 2020 | |
| read: IOPS=561, BW=35.1MiB/s (36.8MB/s)(4210MiB/120002msec) | |
| slat (usec): min=6, max=9638, avg=1137.65, stdev=1653.92 | |
| clat (usec): min=4, max=125934, avg=56237.21, stdev=9573.80 | |
| lat (msec): min=3, max=125, avg=57.38, stdev= 9.70 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 46], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 53], 50.00th=[ 55], 60.00th=[ 58], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 66], 95.00th=[ 73], | |
| | 99.00th=[ 93], 99.50th=[ 99], 99.90th=[ 111], 99.95th=[ 116], | |
| | 99.99th=[ 123] | |
| bw ( KiB/s): min=25216, max=42752, per=99.93%, avg=35898.24, stdev=2983.15, samples=240 | |
| iops : min= 394, max= 668, avg=560.84, stdev=46.60, samples=240 | |
| write: IOPS=558, BW=34.9MiB/s (36.6MB/s)(4190MiB/120002msec); 0 zone resets | |
| slat (usec): min=13, max=12603, avg=643.80, stdev=1356.17 | |
| clat (msec): min=3, max=125, avg=56.21, stdev= 9.49 | |
| lat (msec): min=3, max=125, avg=56.85, stdev= 9.56 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 45], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 53], 50.00th=[ 55], 60.00th=[ 58], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 66], 95.00th=[ 73], | |
| | 99.00th=[ 92], 99.50th=[ 97], 99.90th=[ 109], 99.95th=[ 113], | |
| | 99.99th=[ 118] | |
| bw ( KiB/s): min=22528, max=44160, per=99.94%, avg=35733.35, stdev=3591.32, samples=240 | |
| iops : min= 352, max= 690, avg=558.26, stdev=56.10, samples=240 | |
| lat (usec) : 10=0.01% | |
| lat (msec) : 4=0.01%, 10=0.01%, 20=0.01%, 50=22.97%, 100=76.67% | |
| lat (msec) : 250=0.34% | |
| cpu : usr=0.21%, sys=85.68%, ctx=12797, majf=0, minf=26458 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=67357,67042,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=35.1MiB/s (36.8MB/s), 35.1MiB/s-35.1MiB/s (36.8MB/s-36.8MB/s), io=4210MiB (4414MB), run=120002-120002msec | |
| WRITE: bw=34.9MiB/s (36.6MB/s), 34.9MiB/s-34.9MiB/s (36.6MB/s-36.6MB/s), io=4190MiB (4394MB), run=120002-120002msec | |
| + rm -rf /srv/z/enc/test-enc-64kchunk-1thread.0.0 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-both-64kchunk-1thread] | |
| bs=64k | |
| directory=/srv/z/both/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=1 | |
| nrfiles=1 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-both-64kchunk-1thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| fio-3.16 | |
| Starting 1 process | |
| test-both-64kchunk-1thread: Laying out IO file (1 file / 10000MiB) | |
| Jobs: 1 (f=1): [M(1)][8.3%][r=34.6MiB/s,w=34.8MiB/s][r=553,w=556 IOPS][eta 01m:50s] Jobs: 1 (f=1): [M(1)][16.7%][r=35.7MiB/s,w=35.2MiB/s][r=571,w=562 IOPS][eta 01m:40s] Jobs: 1 (f=1): [M(1)][24.2%][r=35.3MiB/s,w=35.6MiB/s][r=564,w=569 IOPS][eta 01m:31s] | |
| Jobs: 1 (f=1): [M(1)][32.5%][r=34.9MiB/s,w=34.6MiB/s][r=558,w=552 IOPS][eta 01m:21s] Jobs: 1 (f=1): [M(1)][40.8%][r=34.8MiB/s,w=34.0MiB/s][r=557,w=559 IOPS][eta 01m:11s] | |
| Jobs: 1 (f=1): [M(1)][48.3%][r=34.7MiB/s,w=35.4MiB/s][r=555,w=566 IOPS][eta 01m:02s] Jobs: 1 (f=1): [M(1)][56.7%][r=35.9MiB/s,w=35.4MiB/s][r=573,w=566 IOPS][eta 00m:52s] | |
| Jobs: 1 (f=1): [M(1)][65.0%][r=35.1MiB/s,w=32.9MiB/s][r=561,w=526 IOPS][eta 00m:42s] Jobs: 1 (f=1): [M(1)][72.5%][r=35.1MiB/s,w=35.1MiB/s][r=561,w=561 IOPS][eta 00m:33s] | |
| Jobs: 1 (f=1): [M(1)][80.8%][r=35.4MiB/s,w=35.9MiB/s][r=565,w=574 IOPS][eta 00m:23s] Jobs: 1 (f=1): [M(1)][89.2%][r=35.4MiB/s,w=36.0MiB/s][r=565,w=576 IOPS][eta 00m:13s] | |
| Jobs: 1 (f=1): [M(1)][96.7%][r=35.1MiB/s,w=33.0MiB/s][r=562,w=543 IOPS][eta 00m:04s] | |
| test-both-64kchunk-1thread: (groupid=0, jobs=1): err= 0: pid=2326082: Wed Feb 26 14:04:07 2020 | |
| read: IOPS=563, BW=35.2MiB/s (36.9MB/s)(4225MiB/120002msec) | |
| slat (usec): min=6, max=10828, avg=1134.34, stdev=1641.55 | |
| clat (usec): min=3, max=137797, avg=56047.62, stdev=9463.27 | |
| lat (msec): min=3, max=138, avg=57.18, stdev= 9.57 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 45], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 52], 50.00th=[ 55], 60.00th=[ 58], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 66], 95.00th=[ 72], | |
| | 99.00th=[ 93], 99.50th=[ 99], 99.90th=[ 111], 99.95th=[ 118], | |
| | 99.99th=[ 131] | |
| bw ( KiB/s): min=23296, max=44416, per=99.94%, avg=36028.65, stdev=3077.91, samples=240 | |
| iops : min= 364, max= 694, avg=562.88, stdev=48.08, samples=240 | |
| write: IOPS=560, BW=35.0MiB/s (36.7MB/s)(4204MiB/120002msec); 0 zone resets | |
| slat (usec): min=12, max=38447, avg=640.96, stdev=1360.84 | |
| clat (msec): min=6, max=137, avg=56.03, stdev= 9.42 | |
| lat (msec): min=6, max=137, avg=56.67, stdev= 9.50 | |
| clat percentiles (msec): | |
| | 1.00th=[ 39], 5.00th=[ 45], 10.00th=[ 45], 20.00th=[ 49], | |
| | 30.00th=[ 52], 40.00th=[ 53], 50.00th=[ 55], 60.00th=[ 58], | |
| | 70.00th=[ 59], 80.00th=[ 62], 90.00th=[ 66], 95.00th=[ 72], | |
| | 99.00th=[ 92], 99.50th=[ 99], 99.90th=[ 111], 99.95th=[ 120], | |
| | 99.99th=[ 129] | |
| bw ( KiB/s): min=21376, max=45477, per=99.94%, avg=35854.05, stdev=3614.21, samples=240 | |
| iops : min= 334, max= 710, avg=560.15, stdev=56.45, samples=240 | |
| lat (usec) : 4=0.01% | |
| lat (msec) : 4=0.01%, 10=0.01%, 20=0.01%, 50=23.05%, 100=76.53% | |
| lat (msec) : 250=0.41% | |
| cpu : usr=0.20%, sys=85.87%, ctx=12902, majf=0, minf=18882 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=67599,67268,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=35.2MiB/s (36.9MB/s), 35.2MiB/s-35.2MiB/s (36.9MB/s-36.9MB/s), io=4225MiB (4430MB), run=120002-120002msec | |
| WRITE: bw=35.0MiB/s (36.7MB/s), 35.0MiB/s-35.0MiB/s (36.7MB/s-36.7MB/s), io=4204MiB (4408MB), run=120002-120002msec | |
| + rm -rf /srv/z/both/test-both-64kchunk-1thread.0.0 | |
| + for NJ in 1 16 | |
| + for BS in 64k | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-none-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/none/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-none-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-none-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=2938MiB/s,w=2940MiB/s][r=47.0k,w=47.0k IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.7%][r=2527MiB/s,w=2533MiB/s][r=40.4k,w=40.5k IOPS][eta 01m:40s] Jobs: 16 (f=256): [M(16)][24.2%][r=2708MiB/s,w=2712MiB/s][r=43.3k,w=43.4k IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.8%][r=2849MiB/s,w=2854MiB/s][r=45.6k,w=45.7k IOPS][eta 01m:20s] Jobs: 16 (f=256): [M(16)][40.5%][r=3017MiB/s,w=3017MiB/s][r=48.3k,w=48.3k IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][48.3%][r=3006MiB/s,w=3002MiB/s][r=48.1k,w=48.0k IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][56.7%][r=2691MiB/s,w=2691MiB/s][r=43.0k,w=43.1k IOPS][eta 00m:52s] | |
| Jobs: 16 (f=256): [M(16)][65.0%][r=2325MiB/s,w=2333MiB/s][r=37.2k,w=37.3k IOPS][eta 00m:42s] Jobs: 16 (f=256): [M(16)][72.5%][r=2398MiB/s,w=2399MiB/s][r=38.4k,w=38.4k IOPS][eta 00m:33s] | |
| Jobs: 16 (f=256): [M(16)][80.8%][r=2719MiB/s,w=2716MiB/s][r=43.5k,w=43.5k IOPS][eta 00m:23s] Jobs: 16 (f=256): [M(16)][88.4%][r=2881MiB/s,w=2898MiB/s][r=46.1k,w=46.4k IOPS][eta 00m:14s] | |
| Jobs: 16 (f=256): [M(16)][96.7%][r=3127MiB/s,w=3124MiB/s][r=50.0k,w=49.0k IOPS][eta 00m:04s] | |
| test-none-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=2595459: Wed Feb 26 14:08:48 2020 | |
| read: IOPS=44.5k, BW=2780MiB/s (2915MB/s)(326GiB/120002msec) | |
| slat (usec): min=9, max=107636, avg=146.04, stdev=486.04 | |
| clat (usec): min=3, max=151452, avg=11326.99, stdev=8011.32 | |
| lat (usec): min=163, max=151690, avg=11473.48, stdev=8101.92 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 5], 10.00th=[ 6], 20.00th=[ 6], | |
| | 30.00th=[ 7], 40.00th=[ 8], 50.00th=[ 9], 60.00th=[ 11], | |
| | 70.00th=[ 12], 80.00th=[ 15], 90.00th=[ 21], 95.00th=[ 27], | |
| | 99.00th=[ 44], 99.50th=[ 51], 99.90th=[ 71], 99.95th=[ 80], | |
| | 99.99th=[ 103] | |
| bw ( MiB/s): min= 1424, max= 5628, per=99.90%, avg=2776.96, stdev=49.19, samples=3832 | |
| iops : min=22789, max=90057, avg=44430.45, stdev=787.03, samples=3832 | |
| write: IOPS=44.5k, BW=2782MiB/s (2917MB/s)(326GiB/120002msec); 0 zone resets | |
| slat (usec): min=19, max=111600, avg=205.13, stdev=637.80 | |
| clat (usec): min=5, max=151429, avg=11326.94, stdev=7978.98 | |
| lat (usec): min=141, max=151509, avg=11532.55, stdev=8083.77 | |
| clat percentiles (usec): | |
| | 1.00th=[ 3785], 5.00th=[ 4555], 10.00th=[ 5080], 20.00th=[ 5932], | |
| | 30.00th=[ 6849], 40.00th=[ 7832], 50.00th=[ 8979], 60.00th=[10290], | |
| | 70.00th=[11994], 80.00th=[14746], 90.00th=[20317], 95.00th=[26870], | |
| | 99.00th=[43254], 99.50th=[50594], 99.90th=[69731], 99.95th=[78119], | |
| | 99.99th=[99091] | |
| bw ( MiB/s): min= 1418, max= 5621, per=99.90%, avg=2779.25, stdev=49.05, samples=3832 | |
| iops : min=22702, max=89942, avg=44466.96, stdev=784.87, samples=3832 | |
| lat (usec) : 4=0.01%, 10=0.01%, 250=0.01%, 500=0.01%, 750=0.01% | |
| lat (usec) : 1000=0.01% | |
| lat (msec) : 2=0.01%, 4=1.69%, 10=56.47%, 20=31.45%, 50=9.86% | |
| lat (msec) : 100=0.52%, 250=0.01% | |
| cpu : usr=3.74%, sys=50.37%, ctx=1943989, majf=0, minf=441797 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=5337226,5341633,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=2780MiB/s (2915MB/s), 2780MiB/s-2780MiB/s (2915MB/s-2915MB/s), io=326GiB (350GB), run=120002-120002msec | |
| WRITE: bw=2782MiB/s (2917MB/s), 2782MiB/s-2782MiB/s (2917MB/s-2917MB/s), io=326GiB (350GB), run=120002-120002msec | |
| + rm -rf /srv/z/none/test-none-64kchunk-16thread.0.0 /srv/z/none/test-none-64kchunk-16thread.0.1 /srv/z/none/test-none-64kchunk-16thread.0.10 /srv/z/none/test-none-64kchunk-16thread.0.11 /srv/z/none/test-none-64kchunk-16thread.0.12 /srv/z/none/test-none-64kchunk-16thread.0.13 /srv/z/none/test-none-64kchunk-16thread.0.14 /srv/z/none/test-none-64kchunk-16thread.0.15 /srv/z/none/test-none-64kchunk-16thread.0.2 /srv/z/none/test-none-64kchunk-16thread.0.3 /srv/z/none/test-none-64kchunk-16thread.0.4 /srv/z/none/test-none-64kchunk-16thread.0.5 /srv/z/none/test-none-64kchunk-16thread.0.6 /srv/z/none/test-none-64kchunk-16thread.0.7 /srv/z/none/test-none-64kchunk-16thread.0.8 /srv/z/none/test-none-64kchunk-16thread.0.9 /srv/z/none/test-none-64kchunk-16thread.1.0 /srv/z/none/test-none-64kchunk-16thread.10.0 /srv/z/none/test-none-64kchunk-16thread.10.1 /srv/z/none/test-none-64kchunk-16thread.10.10 /srv/z/none/test-none-64kchunk-16thread.10.11 /srv/z/none/test-none-64kchunk-16thread.10.12 /srv/z/none/test-none-64kchunk-16thread.10.13 /srv/z/none/test-none-64kchunk-16thread.10.14 /srv/z/none/test-none-64kchunk-16thread.10.15 /srv/z/none/test-none-64kchunk-16thread.10.2 /srv/z/none/test-none-64kchunk-16thread.10.3 /srv/z/none/test-none-64kchunk-16thread.10.4 /srv/z/none/test-none-64kchunk-16thread.10.5 /srv/z/none/test-none-64kchunk-16thread.10.6 /srv/z/none/test-none-64kchunk-16thread.10.7 /srv/z/none/test-none-64kchunk-16thread.10.8 /srv/z/none/test-none-64kchunk-16thread.10.9 /srv/z/none/test-none-64kchunk-16thread.1.1 /srv/z/none/test-none-64kchunk-16thread.1.10 /srv/z/none/test-none-64kchunk-16thread.11.0 /srv/z/none/test-none-64kchunk-16thread.1.11 /srv/z/none/test-none-64kchunk-16thread.11.1 /srv/z/none/test-none-64kchunk-16thread.11.10 /srv/z/none/test-none-64kchunk-16thread.11.11 /srv/z/none/test-none-64kchunk-16thread.11.12 /srv/z/none/test-none-64kchunk-16thread.11.13 /srv/z/none/test-none-64kchunk-16thread.11.14 /srv/z/none/test-none-64kchunk-16thread.11.15 /srv/z/none/test-none-64kchunk-16thread.1.12 /srv/z/none/test-none-64kchunk-16thread.11.2 /srv/z/none/test-none-64kchunk-16thread.1.13 /srv/z/none/test-none-64kchunk-16thread.11.3 /srv/z/none/test-none-64kchunk-16thread.1.14 /srv/z/none/test-none-64kchunk-16thread.11.4 /srv/z/none/test-none-64kchunk-16thread.1.15 /srv/z/none/test-none-64kchunk-16thread.11.5 /srv/z/none/test-none-64kchunk-16thread.11.6 /srv/z/none/test-none-64kchunk-16thread.11.7 /srv/z/none/test-none-64kchunk-16thread.11.8 /srv/z/none/test-none-64kchunk-16thread.11.9 /srv/z/none/test-none-64kchunk-16thread.1.2 /srv/z/none/test-none-64kchunk-16thread.12.0 /srv/z/none/test-none-64kchunk-16thread.12.1 /srv/z/none/test-none-64kchunk-16thread.12.10 /srv/z/none/test-none-64kchunk-16thread.12.11 /srv/z/none/test-none-64kchunk-16thread.12.12 /srv/z/none/test-none-64kchunk-16thread.12.13 /srv/z/none/test-none-64kchunk-16thread.12.14 /srv/z/none/test-none-64kchunk-16thread.12.15 /srv/z/none/test-none-64kchunk-16thread.12.2 /srv/z/none/test-none-64kchunk-16thread.12.3 /srv/z/none/test-none-64kchunk-16thread.12.4 /srv/z/none/test-none-64kchunk-16thread.12.5 /srv/z/none/test-none-64kchunk-16thread.12.6 /srv/z/none/test-none-64kchunk-16thread.12.7 /srv/z/none/test-none-64kchunk-16thread.12.8 /srv/z/none/test-none-64kchunk-16thread.12.9 /srv/z/none/test-none-64kchunk-16thread.1.3 /srv/z/none/test-none-64kchunk-16thread.13.0 /srv/z/none/test-none-64kchunk-16thread.13.1 /srv/z/none/test-none-64kchunk-16thread.13.10 /srv/z/none/test-none-64kchunk-16thread.13.11 /srv/z/none/test-none-64kchunk-16thread.13.12 /srv/z/none/test-none-64kchunk-16thread.13.13 /srv/z/none/test-none-64kchunk-16thread.13.14 /srv/z/none/test-none-64kchunk-16thread.13.15 /srv/z/none/test-none-64kchunk-16thread.13.2 /srv/z/none/test-none-64kchunk-16thread.13.3 /srv/z/none/test-none-64kchunk-16thread.13.4 /srv/z/none/test-none-64kchunk-16thread.13.5 /srv/z/none/test-none-64kchunk-16thread.13.6 /srv/z/none/test-none-64kchunk-16thread.13.7 /srv/z/none/test-none-64kchunk-16thread.13.8 /srv/z/none/test-none-64kchunk-16thread.13.9 /srv/z/none/test-none-64kchunk-16thread.1.4 /srv/z/none/test-none-64kchunk-16thread.14.0 /srv/z/none/test-none-64kchunk-16thread.14.1 /srv/z/none/test-none-64kchunk-16thread.14.10 /srv/z/none/test-none-64kchunk-16thread.14.11 /srv/z/none/test-none-64kchunk-16thread.14.12 /srv/z/none/test-none-64kchunk-16thread.14.13 /srv/z/none/test-none-64kchunk-16thread.14.14 /srv/z/none/test-none-64kchunk-16thread.14.15 /srv/z/none/test-none-64kchunk-16thread.14.2 /srv/z/none/test-none-64kchunk-16thread.14.3 /srv/z/none/test-none-64kchunk-16thread.14.4 /srv/z/none/test-none-64kchunk-16thread.14.5 /srv/z/none/test-none-64kchunk-16thread.14.6 /srv/z/none/test-none-64kchunk-16thread.14.7 /srv/z/none/test-none-64kchunk-16thread.14.8 /srv/z/none/test-none-64kchunk-16thread.14.9 /srv/z/none/test-none-64kchunk-16thread.1.5 /srv/z/none/test-none-64kchunk-16thread.15.0 /srv/z/none/test-none-64kchunk-16thread.15.1 /srv/z/none/test-none-64kchunk-16thread.15.10 /srv/z/none/test-none-64kchunk-16thread.15.11 /srv/z/none/test-none-64kchunk-16thread.15.12 /srv/z/none/test-none-64kchunk-16thread.15.13 /srv/z/none/test-none-64kchunk-16thread.15.14 /srv/z/none/test-none-64kchunk-16thread.15.15 /srv/z/none/test-none-64kchunk-16thread.15.2 /srv/z/none/test-none-64kchunk-16thread.15.3 /srv/z/none/test-none-64kchunk-16thread.15.4 /srv/z/none/test-none-64kchunk-16thread.15.5 /srv/z/none/test-none-64kchunk-16thread.15.6 /srv/z/none/test-none-64kchunk-16thread.15.7 /srv/z/none/test-none-64kchunk-16thread.15.8 /srv/z/none/test-none-64kchunk-16thread.15.9 /srv/z/none/test-none-64kchunk-16thread.1.6 /srv/z/none/test-none-64kchunk-16thread.1.7 /srv/z/none/test-none-64kchunk-16thread.1.8 /srv/z/none/test-none-64kchunk-16thread.1.9 /srv/z/none/test-none-64kchunk-16thread.2.0 /srv/z/none/test-none-64kchunk-16thread.2.1 /srv/z/none/test-none-64kchunk-16thread.2.10 /srv/z/none/test-none-64kchunk-16thread.2.11 /srv/z/none/test-none-64kchunk-16thread.2.12 /srv/z/none/test-none-64kchunk-16thread.2.13 /srv/z/none/test-none-64kchunk-16thread.2.14 /srv/z/none/test-none-64kchunk-16thread.2.15 /srv/z/none/test-none-64kchunk-16thread.2.2 /srv/z/none/test-none-64kchunk-16thread.2.3 /srv/z/none/test-none-64kchunk-16thread.2.4 /srv/z/none/test-none-64kchunk-16thread.2.5 /srv/z/none/test-none-64kchunk-16thread.2.6 /srv/z/none/test-none-64kchunk-16thread.2.7 /srv/z/none/test-none-64kchunk-16thread.2.8 /srv/z/none/test-none-64kchunk-16thread.2.9 /srv/z/none/test-none-64kchunk-16thread.3.0 /srv/z/none/test-none-64kchunk-16thread.3.1 /srv/z/none/test-none-64kchunk-16thread.3.10 /srv/z/none/test-none-64kchunk-16thread.3.11 /srv/z/none/test-none-64kchunk-16thread.3.12 /srv/z/none/test-none-64kchunk-16thread.3.13 /srv/z/none/test-none-64kchunk-16thread.3.14 /srv/z/none/test-none-64kchunk-16thread.3.15 /srv/z/none/test-none-64kchunk-16thread.3.2 /srv/z/none/test-none-64kchunk-16thread.3.3 /srv/z/none/test-none-64kchunk-16thread.3.4 /srv/z/none/test-none-64kchunk-16thread.3.5 /srv/z/none/test-none-64kchunk-16thread.3.6 /srv/z/none/test-none-64kchunk-16thread.3.7 /srv/z/none/test-none-64kchunk-16thread.3.8 /srv/z/none/test-none-64kchunk-16thread.3.9 /srv/z/none/test-none-64kchunk-16thread.4.0 /srv/z/none/test-none-64kchunk-16thread.4.1 /srv/z/none/test-none-64kchunk-16thread.4.10 /srv/z/none/test-none-64kchunk-16thread.4.11 /srv/z/none/test-none-64kchunk-16thread.4.12 /srv/z/none/test-none-64kchunk-16thread.4.13 /srv/z/none/test-none-64kchunk-16thread.4.14 /srv/z/none/test-none-64kchunk-16thread.4.15 /srv/z/none/test-none-64kchunk-16thread.4.2 /srv/z/none/test-none-64kchunk-16thread.4.3 /srv/z/none/test-none-64kchunk-16thread.4.4 /srv/z/none/test-none-64kchunk-16thread.4.5 /srv/z/none/test-none-64kchunk-16thread.4.6 /srv/z/none/test-none-64kchunk-16thread.4.7 /srv/z/none/test-none-64kchunk-16thread.4.8 /srv/z/none/test-none-64kchunk-16thread.4.9 /srv/z/none/test-none-64kchunk-16thread.5.0 /srv/z/none/test-none-64kchunk-16thread.5.1 /srv/z/none/test-none-64kchunk-16thread.5.10 /srv/z/none/test-none-64kchunk-16thread.5.11 /srv/z/none/test-none-64kchunk-16thread.5.12 /srv/z/none/test-none-64kchunk-16thread.5.13 /srv/z/none/test-none-64kchunk-16thread.5.14 /srv/z/none/test-none-64kchunk-16thread.5.15 /srv/z/none/test-none-64kchunk-16thread.5.2 /srv/z/none/test-none-64kchunk-16thread.5.3 /srv/z/none/test-none-64kchunk-16thread.5.4 /srv/z/none/test-none-64kchunk-16thread.5.5 /srv/z/none/test-none-64kchunk-16thread.5.6 /srv/z/none/test-none-64kchunk-16thread.5.7 /srv/z/none/test-none-64kchunk-16thread.5.8 /srv/z/none/test-none-64kchunk-16thread.5.9 /srv/z/none/test-none-64kchunk-16thread.6.0 /srv/z/none/test-none-64kchunk-16thread.6.1 /srv/z/none/test-none-64kchunk-16thread.6.10 /srv/z/none/test-none-64kchunk-16thread.6.11 /srv/z/none/test-none-64kchunk-16thread.6.12 /srv/z/none/test-none-64kchunk-16thread.6.13 /srv/z/none/test-none-64kchunk-16thread.6.14 /srv/z/none/test-none-64kchunk-16thread.6.15 /srv/z/none/test-none-64kchunk-16thread.6.2 /srv/z/none/test-none-64kchunk-16thread.6.3 /srv/z/none/test-none-64kchunk-16thread.6.4 /srv/z/none/test-none-64kchunk-16thread.6.5 /srv/z/none/test-none-64kchunk-16thread.6.6 /srv/z/none/test-none-64kchunk-16thread.6.7 /srv/z/none/test-none-64kchunk-16thread.6.8 /srv/z/none/test-none-64kchunk-16thread.6.9 /srv/z/none/test-none-64kchunk-16thread.7.0 /srv/z/none/test-none-64kchunk-16thread.7.1 /srv/z/none/test-none-64kchunk-16thread.7.10 /srv/z/none/test-none-64kchunk-16thread.7.11 /srv/z/none/test-none-64kchunk-16thread.7.12 /srv/z/none/test-none-64kchunk-16thread.7.13 /srv/z/none/test-none-64kchunk-16thread.7.14 /srv/z/none/test-none-64kchunk-16thread.7.15 /srv/z/none/test-none-64kchunk-16thread.7.2 /srv/z/none/test-none-64kchunk-16thread.7.3 /srv/z/none/test-none-64kchunk-16thread.7.4 /srv/z/none/test-none-64kchunk-16thread.7.5 /srv/z/none/test-none-64kchunk-16thread.7.6 /srv/z/none/test-none-64kchunk-16thread.7.7 /srv/z/none/test-none-64kchunk-16thread.7.8 /srv/z/none/test-none-64kchunk-16thread.7.9 /srv/z/none/test-none-64kchunk-16thread.8.0 /srv/z/none/test-none-64kchunk-16thread.8.1 /srv/z/none/test-none-64kchunk-16thread.8.10 /srv/z/none/test-none-64kchunk-16thread.8.11 /srv/z/none/test-none-64kchunk-16thread.8.12 /srv/z/none/test-none-64kchunk-16thread.8.13 /srv/z/none/test-none-64kchunk-16thread.8.14 /srv/z/none/test-none-64kchunk-16thread.8.15 /srv/z/none/test-none-64kchunk-16thread.8.2 /srv/z/none/test-none-64kchunk-16thread.8.3 /srv/z/none/test-none-64kchunk-16thread.8.4 /srv/z/none/test-none-64kchunk-16thread.8.5 /srv/z/none/test-none-64kchunk-16thread.8.6 /srv/z/none/test-none-64kchunk-16thread.8.7 /srv/z/none/test-none-64kchunk-16thread.8.8 /srv/z/none/test-none-64kchunk-16thread.8.9 /srv/z/none/test-none-64kchunk-16thread.9.0 /srv/z/none/test-none-64kchunk-16thread.9.1 /srv/z/none/test-none-64kchunk-16thread.9.10 /srv/z/none/test-none-64kchunk-16thread.9.11 /srv/z/none/test-none-64kchunk-16thread.9.12 /srv/z/none/test-none-64kchunk-16thread.9.13 /srv/z/none/test-none-64kchunk-16thread.9.14 /srv/z/none/test-none-64kchunk-16thread.9.15 /srv/z/none/test-none-64kchunk-16thread.9.2 /srv/z/none/test-none-64kchunk-16thread.9.3 /srv/z/none/test-none-64kchunk-16thread.9.4 /srv/z/none/test-none-64kchunk-16thread.9.5 /srv/z/none/test-none-64kchunk-16thread.9.6 /srv/z/none/test-none-64kchunk-16thread.9.7 /srv/z/none/test-none-64kchunk-16thread.9.8 /srv/z/none/test-none-64kchunk-16thread.9.9 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-comp-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/comp/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-comp-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-comp-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=2911MiB/s,w=2912MiB/s][r=46.6k,w=46.6k IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.5%][r=2467MiB/s,w=2476MiB/s][r=39.5k,w=39.6k IOPS][eta 01m:41s] Jobs: 16 (f=256): [M(16)][24.2%][r=2636MiB/s,w=2636MiB/s][r=42.2k,w=42.2k IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.8%][r=2736MiB/s,w=2739MiB/s][r=43.8k,w=43.8k IOPS][eta 01m:20s] Jobs: 16 (f=256): [M(16)][40.0%][r=2859MiB/s,w=2864MiB/s][r=45.8k,w=45.8k IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][48.3%][r=3018MiB/s,w=3018MiB/s][r=48.3k,w=48.3k IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][56.2%][r=2901MiB/s,w=2901MiB/s][r=46.4k,w=46.4k IOPS][eta 00m:53s] | |
| Jobs: 16 (f=256): [M(16)][64.7%][r=2578MiB/s,w=2575MiB/s][r=41.2k,w=41.2k IOPS][eta 00m:42s] Jobs: 16 (f=256): [M(16)][72.5%][r=2484MiB/s,w=2493MiB/s][r=39.7k,w=39.9k IOPS][eta 00m:33s] | |
| Jobs: 16 (f=256): [M(16)][80.0%][r=2720MiB/s,w=2710MiB/s][r=43.5k,w=43.4k IOPS][eta 00m:24s] Jobs: 16 (f=256): [M(16)][88.3%][r=2863MiB/s,w=2878MiB/s][r=45.8k,w=46.0k IOPS][eta 00m:14s] | |
| Jobs: 16 (f=256): [M(16)][95.9%][r=2859MiB/s,w=2860MiB/s][r=45.7k,w=45.8k IOPS][eta 00m:05s] | |
| test-comp-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=3476111: Wed Feb 26 14:13:38 2020 | |
| read: IOPS=44.1k, BW=2757MiB/s (2891MB/s)(323GiB/120001msec) | |
| slat (usec): min=9, max=123484, avg=137.84, stdev=760.93 | |
| clat (usec): min=3, max=182813, avg=11421.54, stdev=10099.94 | |
| lat (usec): min=68, max=182838, avg=11559.85, stdev=10189.64 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 4], 10.00th=[ 5], 20.00th=[ 6], | |
| | 30.00th=[ 6], 40.00th=[ 7], 50.00th=[ 9], 60.00th=[ 10], | |
| | 70.00th=[ 12], 80.00th=[ 16], 90.00th=[ 23], 95.00th=[ 32], | |
| | 99.00th=[ 53], 99.50th=[ 63], 99.90th=[ 90], 99.95th=[ 107], | |
| | 99.99th=[ 130] | |
| bw ( MiB/s): min= 1304, max= 5841, per=99.78%, avg=2751.28, stdev=49.55, samples=3827 | |
| iops : min=20868, max=93460, avg=44018.24, stdev=792.81, samples=3827 | |
| write: IOPS=44.2k, BW=2760MiB/s (2894MB/s)(323GiB/120001msec); 0 zone resets | |
| slat (usec): min=16, max=137033, avg=215.09, stdev=1039.13 | |
| clat (usec): min=2, max=180575, avg=11417.14, stdev=10061.00 | |
| lat (usec): min=36, max=182894, avg=11632.80, stdev=10188.56 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 4], 10.00th=[ 5], 20.00th=[ 6], | |
| | 30.00th=[ 6], 40.00th=[ 7], 50.00th=[ 9], 60.00th=[ 10], | |
| | 70.00th=[ 12], 80.00th=[ 16], 90.00th=[ 23], 95.00th=[ 32], | |
| | 99.00th=[ 53], 99.50th=[ 63], 99.90th=[ 90], 99.95th=[ 106], | |
| | 99.99th=[ 130] | |
| bw ( MiB/s): min= 1321, max= 5768, per=99.79%, avg=2753.87, stdev=49.39, samples=3827 | |
| iops : min=21137, max=92287, avg=44059.61, stdev=790.20, samples=3827 | |
| lat (usec) : 4=0.01%, 10=0.01%, 50=0.01%, 100=0.01%, 250=0.01% | |
| lat (usec) : 500=0.01%, 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=0.01%, 4=5.70%, 10=56.39%, 20=25.54%, 50=11.14% | |
| lat (msec) : 100=1.16%, 250=0.06% | |
| cpu : usr=3.35%, sys=43.38%, ctx=1478493, majf=0, minf=477871 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=5293928,5298682,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=2757MiB/s (2891MB/s), 2757MiB/s-2757MiB/s (2891MB/s-2891MB/s), io=323GiB (347GB), run=120001-120001msec | |
| WRITE: bw=2760MiB/s (2894MB/s), 2760MiB/s-2760MiB/s (2894MB/s-2894MB/s), io=323GiB (347GB), run=120001-120001msec | |
| + rm -rf /srv/z/comp/test-comp-64kchunk-16thread.0.0 /srv/z/comp/test-comp-64kchunk-16thread.0.1 /srv/z/comp/test-comp-64kchunk-16thread.0.10 /srv/z/comp/test-comp-64kchunk-16thread.0.11 /srv/z/comp/test-comp-64kchunk-16thread.0.12 /srv/z/comp/test-comp-64kchunk-16thread.0.13 /srv/z/comp/test-comp-64kchunk-16thread.0.14 /srv/z/comp/test-comp-64kchunk-16thread.0.15 /srv/z/comp/test-comp-64kchunk-16thread.0.2 /srv/z/comp/test-comp-64kchunk-16thread.0.3 /srv/z/comp/test-comp-64kchunk-16thread.0.4 /srv/z/comp/test-comp-64kchunk-16thread.0.5 /srv/z/comp/test-comp-64kchunk-16thread.0.6 /srv/z/comp/test-comp-64kchunk-16thread.0.7 /srv/z/comp/test-comp-64kchunk-16thread.0.8 /srv/z/comp/test-comp-64kchunk-16thread.0.9 /srv/z/comp/test-comp-64kchunk-16thread.1.0 /srv/z/comp/test-comp-64kchunk-16thread.10.0 /srv/z/comp/test-comp-64kchunk-16thread.10.1 /srv/z/comp/test-comp-64kchunk-16thread.10.10 /srv/z/comp/test-comp-64kchunk-16thread.10.11 /srv/z/comp/test-comp-64kchunk-16thread.10.12 /srv/z/comp/test-comp-64kchunk-16thread.10.13 /srv/z/comp/test-comp-64kchunk-16thread.10.14 /srv/z/comp/test-comp-64kchunk-16thread.10.15 /srv/z/comp/test-comp-64kchunk-16thread.10.2 /srv/z/comp/test-comp-64kchunk-16thread.10.3 /srv/z/comp/test-comp-64kchunk-16thread.10.4 /srv/z/comp/test-comp-64kchunk-16thread.10.5 /srv/z/comp/test-comp-64kchunk-16thread.10.6 /srv/z/comp/test-comp-64kchunk-16thread.10.7 /srv/z/comp/test-comp-64kchunk-16thread.10.8 /srv/z/comp/test-comp-64kchunk-16thread.10.9 /srv/z/comp/test-comp-64kchunk-16thread.1.1 /srv/z/comp/test-comp-64kchunk-16thread.1.10 /srv/z/comp/test-comp-64kchunk-16thread.11.0 /srv/z/comp/test-comp-64kchunk-16thread.1.11 /srv/z/comp/test-comp-64kchunk-16thread.11.1 /srv/z/comp/test-comp-64kchunk-16thread.11.10 /srv/z/comp/test-comp-64kchunk-16thread.11.11 /srv/z/comp/test-comp-64kchunk-16thread.11.12 /srv/z/comp/test-comp-64kchunk-16thread.11.13 /srv/z/comp/test-comp-64kchunk-16thread.11.14 /srv/z/comp/test-comp-64kchunk-16thread.11.15 /srv/z/comp/test-comp-64kchunk-16thread.1.12 /srv/z/comp/test-comp-64kchunk-16thread.11.2 /srv/z/comp/test-comp-64kchunk-16thread.1.13 /srv/z/comp/test-comp-64kchunk-16thread.11.3 /srv/z/comp/test-comp-64kchunk-16thread.1.14 /srv/z/comp/test-comp-64kchunk-16thread.11.4 /srv/z/comp/test-comp-64kchunk-16thread.1.15 /srv/z/comp/test-comp-64kchunk-16thread.11.5 /srv/z/comp/test-comp-64kchunk-16thread.11.6 /srv/z/comp/test-comp-64kchunk-16thread.11.7 /srv/z/comp/test-comp-64kchunk-16thread.11.8 /srv/z/comp/test-comp-64kchunk-16thread.11.9 /srv/z/comp/test-comp-64kchunk-16thread.1.2 /srv/z/comp/test-comp-64kchunk-16thread.12.0 /srv/z/comp/test-comp-64kchunk-16thread.12.1 /srv/z/comp/test-comp-64kchunk-16thread.12.10 /srv/z/comp/test-comp-64kchunk-16thread.12.11 /srv/z/comp/test-comp-64kchunk-16thread.12.12 /srv/z/comp/test-comp-64kchunk-16thread.12.13 /srv/z/comp/test-comp-64kchunk-16thread.12.14 /srv/z/comp/test-comp-64kchunk-16thread.12.15 /srv/z/comp/test-comp-64kchunk-16thread.12.2 /srv/z/comp/test-comp-64kchunk-16thread.12.3 /srv/z/comp/test-comp-64kchunk-16thread.12.4 /srv/z/comp/test-comp-64kchunk-16thread.12.5 /srv/z/comp/test-comp-64kchunk-16thread.12.6 /srv/z/comp/test-comp-64kchunk-16thread.12.7 /srv/z/comp/test-comp-64kchunk-16thread.12.8 /srv/z/comp/test-comp-64kchunk-16thread.12.9 /srv/z/comp/test-comp-64kchunk-16thread.1.3 /srv/z/comp/test-comp-64kchunk-16thread.13.0 /srv/z/comp/test-comp-64kchunk-16thread.13.1 /srv/z/comp/test-comp-64kchunk-16thread.13.10 /srv/z/comp/test-comp-64kchunk-16thread.13.11 /srv/z/comp/test-comp-64kchunk-16thread.13.12 /srv/z/comp/test-comp-64kchunk-16thread.13.13 /srv/z/comp/test-comp-64kchunk-16thread.13.14 /srv/z/comp/test-comp-64kchunk-16thread.13.15 /srv/z/comp/test-comp-64kchunk-16thread.13.2 /srv/z/comp/test-comp-64kchunk-16thread.13.3 /srv/z/comp/test-comp-64kchunk-16thread.13.4 /srv/z/comp/test-comp-64kchunk-16thread.13.5 /srv/z/comp/test-comp-64kchunk-16thread.13.6 /srv/z/comp/test-comp-64kchunk-16thread.13.7 /srv/z/comp/test-comp-64kchunk-16thread.13.8 /srv/z/comp/test-comp-64kchunk-16thread.13.9 /srv/z/comp/test-comp-64kchunk-16thread.1.4 /srv/z/comp/test-comp-64kchunk-16thread.14.0 /srv/z/comp/test-comp-64kchunk-16thread.14.1 /srv/z/comp/test-comp-64kchunk-16thread.14.10 /srv/z/comp/test-comp-64kchunk-16thread.14.11 /srv/z/comp/test-comp-64kchunk-16thread.14.12 /srv/z/comp/test-comp-64kchunk-16thread.14.13 /srv/z/comp/test-comp-64kchunk-16thread.14.14 /srv/z/comp/test-comp-64kchunk-16thread.14.15 /srv/z/comp/test-comp-64kchunk-16thread.14.2 /srv/z/comp/test-comp-64kchunk-16thread.14.3 /srv/z/comp/test-comp-64kchunk-16thread.14.4 /srv/z/comp/test-comp-64kchunk-16thread.14.5 /srv/z/comp/test-comp-64kchunk-16thread.14.6 /srv/z/comp/test-comp-64kchunk-16thread.14.7 /srv/z/comp/test-comp-64kchunk-16thread.14.8 /srv/z/comp/test-comp-64kchunk-16thread.14.9 /srv/z/comp/test-comp-64kchunk-16thread.1.5 /srv/z/comp/test-comp-64kchunk-16thread.15.0 /srv/z/comp/test-comp-64kchunk-16thread.15.1 /srv/z/comp/test-comp-64kchunk-16thread.15.10 /srv/z/comp/test-comp-64kchunk-16thread.15.11 /srv/z/comp/test-comp-64kchunk-16thread.15.12 /srv/z/comp/test-comp-64kchunk-16thread.15.13 /srv/z/comp/test-comp-64kchunk-16thread.15.14 /srv/z/comp/test-comp-64kchunk-16thread.15.15 /srv/z/comp/test-comp-64kchunk-16thread.15.2 /srv/z/comp/test-comp-64kchunk-16thread.15.3 /srv/z/comp/test-comp-64kchunk-16thread.15.4 /srv/z/comp/test-comp-64kchunk-16thread.15.5 /srv/z/comp/test-comp-64kchunk-16thread.15.6 /srv/z/comp/test-comp-64kchunk-16thread.15.7 /srv/z/comp/test-comp-64kchunk-16thread.15.8 /srv/z/comp/test-comp-64kchunk-16thread.15.9 /srv/z/comp/test-comp-64kchunk-16thread.1.6 /srv/z/comp/test-comp-64kchunk-16thread.1.7 /srv/z/comp/test-comp-64kchunk-16thread.1.8 /srv/z/comp/test-comp-64kchunk-16thread.1.9 /srv/z/comp/test-comp-64kchunk-16thread.2.0 /srv/z/comp/test-comp-64kchunk-16thread.2.1 /srv/z/comp/test-comp-64kchunk-16thread.2.10 /srv/z/comp/test-comp-64kchunk-16thread.2.11 /srv/z/comp/test-comp-64kchunk-16thread.2.12 /srv/z/comp/test-comp-64kchunk-16thread.2.13 /srv/z/comp/test-comp-64kchunk-16thread.2.14 /srv/z/comp/test-comp-64kchunk-16thread.2.15 /srv/z/comp/test-comp-64kchunk-16thread.2.2 /srv/z/comp/test-comp-64kchunk-16thread.2.3 /srv/z/comp/test-comp-64kchunk-16thread.2.4 /srv/z/comp/test-comp-64kchunk-16thread.2.5 /srv/z/comp/test-comp-64kchunk-16thread.2.6 /srv/z/comp/test-comp-64kchunk-16thread.2.7 /srv/z/comp/test-comp-64kchunk-16thread.2.8 /srv/z/comp/test-comp-64kchunk-16thread.2.9 /srv/z/comp/test-comp-64kchunk-16thread.3.0 /srv/z/comp/test-comp-64kchunk-16thread.3.1 /srv/z/comp/test-comp-64kchunk-16thread.3.10 /srv/z/comp/test-comp-64kchunk-16thread.3.11 /srv/z/comp/test-comp-64kchunk-16thread.3.12 /srv/z/comp/test-comp-64kchunk-16thread.3.13 /srv/z/comp/test-comp-64kchunk-16thread.3.14 /srv/z/comp/test-comp-64kchunk-16thread.3.15 /srv/z/comp/test-comp-64kchunk-16thread.3.2 /srv/z/comp/test-comp-64kchunk-16thread.3.3 /srv/z/comp/test-comp-64kchunk-16thread.3.4 /srv/z/comp/test-comp-64kchunk-16thread.3.5 /srv/z/comp/test-comp-64kchunk-16thread.3.6 /srv/z/comp/test-comp-64kchunk-16thread.3.7 /srv/z/comp/test-comp-64kchunk-16thread.3.8 /srv/z/comp/test-comp-64kchunk-16thread.3.9 /srv/z/comp/test-comp-64kchunk-16thread.4.0 /srv/z/comp/test-comp-64kchunk-16thread.4.1 /srv/z/comp/test-comp-64kchunk-16thread.4.10 /srv/z/comp/test-comp-64kchunk-16thread.4.11 /srv/z/comp/test-comp-64kchunk-16thread.4.12 /srv/z/comp/test-comp-64kchunk-16thread.4.13 /srv/z/comp/test-comp-64kchunk-16thread.4.14 /srv/z/comp/test-comp-64kchunk-16thread.4.15 /srv/z/comp/test-comp-64kchunk-16thread.4.2 /srv/z/comp/test-comp-64kchunk-16thread.4.3 /srv/z/comp/test-comp-64kchunk-16thread.4.4 /srv/z/comp/test-comp-64kchunk-16thread.4.5 /srv/z/comp/test-comp-64kchunk-16thread.4.6 /srv/z/comp/test-comp-64kchunk-16thread.4.7 /srv/z/comp/test-comp-64kchunk-16thread.4.8 /srv/z/comp/test-comp-64kchunk-16thread.4.9 /srv/z/comp/test-comp-64kchunk-16thread.5.0 /srv/z/comp/test-comp-64kchunk-16thread.5.1 /srv/z/comp/test-comp-64kchunk-16thread.5.10 /srv/z/comp/test-comp-64kchunk-16thread.5.11 /srv/z/comp/test-comp-64kchunk-16thread.5.12 /srv/z/comp/test-comp-64kchunk-16thread.5.13 /srv/z/comp/test-comp-64kchunk-16thread.5.14 /srv/z/comp/test-comp-64kchunk-16thread.5.15 /srv/z/comp/test-comp-64kchunk-16thread.5.2 /srv/z/comp/test-comp-64kchunk-16thread.5.3 /srv/z/comp/test-comp-64kchunk-16thread.5.4 /srv/z/comp/test-comp-64kchunk-16thread.5.5 /srv/z/comp/test-comp-64kchunk-16thread.5.6 /srv/z/comp/test-comp-64kchunk-16thread.5.7 /srv/z/comp/test-comp-64kchunk-16thread.5.8 /srv/z/comp/test-comp-64kchunk-16thread.5.9 /srv/z/comp/test-comp-64kchunk-16thread.6.0 /srv/z/comp/test-comp-64kchunk-16thread.6.1 /srv/z/comp/test-comp-64kchunk-16thread.6.10 /srv/z/comp/test-comp-64kchunk-16thread.6.11 /srv/z/comp/test-comp-64kchunk-16thread.6.12 /srv/z/comp/test-comp-64kchunk-16thread.6.13 /srv/z/comp/test-comp-64kchunk-16thread.6.14 /srv/z/comp/test-comp-64kchunk-16thread.6.15 /srv/z/comp/test-comp-64kchunk-16thread.6.2 /srv/z/comp/test-comp-64kchunk-16thread.6.3 /srv/z/comp/test-comp-64kchunk-16thread.6.4 /srv/z/comp/test-comp-64kchunk-16thread.6.5 /srv/z/comp/test-comp-64kchunk-16thread.6.6 /srv/z/comp/test-comp-64kchunk-16thread.6.7 /srv/z/comp/test-comp-64kchunk-16thread.6.8 /srv/z/comp/test-comp-64kchunk-16thread.6.9 /srv/z/comp/test-comp-64kchunk-16thread.7.0 /srv/z/comp/test-comp-64kchunk-16thread.7.1 /srv/z/comp/test-comp-64kchunk-16thread.7.10 /srv/z/comp/test-comp-64kchunk-16thread.7.11 /srv/z/comp/test-comp-64kchunk-16thread.7.12 /srv/z/comp/test-comp-64kchunk-16thread.7.13 /srv/z/comp/test-comp-64kchunk-16thread.7.14 /srv/z/comp/test-comp-64kchunk-16thread.7.15 /srv/z/comp/test-comp-64kchunk-16thread.7.2 /srv/z/comp/test-comp-64kchunk-16thread.7.3 /srv/z/comp/test-comp-64kchunk-16thread.7.4 /srv/z/comp/test-comp-64kchunk-16thread.7.5 /srv/z/comp/test-comp-64kchunk-16thread.7.6 /srv/z/comp/test-comp-64kchunk-16thread.7.7 /srv/z/comp/test-comp-64kchunk-16thread.7.8 /srv/z/comp/test-comp-64kchunk-16thread.7.9 /srv/z/comp/test-comp-64kchunk-16thread.8.0 /srv/z/comp/test-comp-64kchunk-16thread.8.1 /srv/z/comp/test-comp-64kchunk-16thread.8.10 /srv/z/comp/test-comp-64kchunk-16thread.8.11 /srv/z/comp/test-comp-64kchunk-16thread.8.12 /srv/z/comp/test-comp-64kchunk-16thread.8.13 /srv/z/comp/test-comp-64kchunk-16thread.8.14 /srv/z/comp/test-comp-64kchunk-16thread.8.15 /srv/z/comp/test-comp-64kchunk-16thread.8.2 /srv/z/comp/test-comp-64kchunk-16thread.8.3 /srv/z/comp/test-comp-64kchunk-16thread.8.4 /srv/z/comp/test-comp-64kchunk-16thread.8.5 /srv/z/comp/test-comp-64kchunk-16thread.8.6 /srv/z/comp/test-comp-64kchunk-16thread.8.7 /srv/z/comp/test-comp-64kchunk-16thread.8.8 /srv/z/comp/test-comp-64kchunk-16thread.8.9 /srv/z/comp/test-comp-64kchunk-16thread.9.0 /srv/z/comp/test-comp-64kchunk-16thread.9.1 /srv/z/comp/test-comp-64kchunk-16thread.9.10 /srv/z/comp/test-comp-64kchunk-16thread.9.11 /srv/z/comp/test-comp-64kchunk-16thread.9.12 /srv/z/comp/test-comp-64kchunk-16thread.9.13 /srv/z/comp/test-comp-64kchunk-16thread.9.14 /srv/z/comp/test-comp-64kchunk-16thread.9.15 /srv/z/comp/test-comp-64kchunk-16thread.9.2 /srv/z/comp/test-comp-64kchunk-16thread.9.3 /srv/z/comp/test-comp-64kchunk-16thread.9.4 /srv/z/comp/test-comp-64kchunk-16thread.9.5 /srv/z/comp/test-comp-64kchunk-16thread.9.6 /srv/z/comp/test-comp-64kchunk-16thread.9.7 /srv/z/comp/test-comp-64kchunk-16thread.9.8 /srv/z/comp/test-comp-64kchunk-16thread.9.9 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-enc-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/enc/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-enc-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-enc-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=366MiB/s,w=367MiB/s][r=5850,w=5871 IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.5%][r=384MiB/s,w=386MiB/s][r=6149,w=6183 IOPS][eta 01m:41s] Jobs: 16 (f=256): [M(16)][24.2%][r=344MiB/s,w=344MiB/s][r=5504,w=5511 IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.5%][r=325MiB/s,w=326MiB/s][r=5199,w=5208 IOPS][eta 01m:21s] Jobs: 16 (f=256): [M(16)][40.0%][r=342MiB/s,w=340MiB/s][r=5468,w=5436 IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][48.3%][r=337MiB/s,w=337MiB/s][r=5396,w=5397 IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][56.2%][r=335MiB/s,w=336MiB/s][r=5364,w=5371 IOPS][eta 00m:53s] | |
| Jobs: 16 (f=256): [M(16)][64.2%][r=355MiB/s,w=353MiB/s][r=5679,w=5652 IOPS][eta 00m:43s] Jobs: 16 (f=256): [M(16)][71.9%][r=366MiB/s,w=368MiB/s][r=5856,w=5895 IOPS][eta 00m:34s] | |
| Jobs: 16 (f=256): [M(16)][80.0%][r=385MiB/s,w=380MiB/s][r=6163,w=6082 IOPS][eta 00m:24s] Jobs: 16 (f=256): [M(16)][88.3%][r=398MiB/s,w=402MiB/s][r=6370,w=6430 IOPS][eta 00m:14s] | |
| Jobs: 16 (f=256): [M(16)][95.9%][r=394MiB/s,w=392MiB/s][r=6300,w=6269 IOPS][eta 00m:05s] | |
| test-enc-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=643126: Wed Feb 26 14:36:19 2020 | |
| read: IOPS=5805, BW=363MiB/s (380MB/s)(42.5GiB/120006msec) | |
| slat (usec): min=9, max=212161, avg=1407.06, stdev=3384.68 | |
| clat (usec): min=3, max=460345, avg=86706.70, stdev=49224.40 | |
| lat (usec): min=414, max=460378, avg=88114.25, stdev=49910.16 | |
| clat percentiles (msec): | |
| | 1.00th=[ 8], 5.00th=[ 15], 10.00th=[ 23], 20.00th=[ 40], | |
| | 30.00th=[ 61], 40.00th=[ 74], 50.00th=[ 85], 60.00th=[ 94], | |
| | 70.00th=[ 106], 80.00th=[ 125], 90.00th=[ 155], 95.00th=[ 178], | |
| | 99.00th=[ 220], 99.50th=[ 239], 99.90th=[ 292], 99.95th=[ 330], | |
| | 99.99th=[ 380] | |
| bw ( KiB/s): min=130817, max=947065, per=99.76%, avg=370675.18, stdev=12805.22, samples=3833 | |
| iops : min= 2038, max=14797, avg=5789.57, stdev=200.12, samples=3833 | |
| write: IOPS=5805, BW=363MiB/s (380MB/s)(42.5GiB/120006msec); 0 zone resets | |
| slat (usec): min=15, max=189092, avg=1340.90, stdev=3332.86 | |
| clat (usec): min=3, max=460352, avg=86743.54, stdev=49230.35 | |
| lat (usec): min=819, max=460384, avg=88084.85, stdev=49889.17 | |
| clat percentiles (msec): | |
| | 1.00th=[ 8], 5.00th=[ 15], 10.00th=[ 23], 20.00th=[ 40], | |
| | 30.00th=[ 61], 40.00th=[ 74], 50.00th=[ 85], 60.00th=[ 94], | |
| | 70.00th=[ 106], 80.00th=[ 125], 90.00th=[ 155], 95.00th=[ 178], | |
| | 99.00th=[ 220], 99.50th=[ 239], 99.90th=[ 296], 99.95th=[ 326], | |
| | 99.99th=[ 380] | |
| bw ( KiB/s): min=132276, max=947453, per=99.76%, avg=370691.25, stdev=12744.34, samples=3833 | |
| iops : min= 2062, max=14803, avg=5789.77, stdev=199.16, samples=3833 | |
| lat (usec) : 4=0.01%, 10=0.01%, 500=0.01%, 750=0.01%, 1000=0.01% | |
| lat (msec) : 2=0.01%, 4=0.44%, 10=1.59%, 20=6.30%, 50=16.87% | |
| lat (msec) : 100=40.39%, 250=34.10%, 500=0.32% | |
| cpu : usr=0.32%, sys=39.52%, ctx=898402, majf=0, minf=485669 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=696694,696736,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=363MiB/s (380MB/s), 363MiB/s-363MiB/s (380MB/s-380MB/s), io=42.5GiB (45.7GB), run=120006-120006msec | |
| WRITE: bw=363MiB/s (380MB/s), 363MiB/s-363MiB/s (380MB/s-380MB/s), io=42.5GiB (45.7GB), run=120006-120006msec | |
| + rm -rf /srv/z/enc/test-enc-64kchunk-16thread.0.0 /srv/z/enc/test-enc-64kchunk-16thread.0.1 /srv/z/enc/test-enc-64kchunk-16thread.0.10 /srv/z/enc/test-enc-64kchunk-16thread.0.11 /srv/z/enc/test-enc-64kchunk-16thread.0.12 /srv/z/enc/test-enc-64kchunk-16thread.0.13 /srv/z/enc/test-enc-64kchunk-16thread.0.14 /srv/z/enc/test-enc-64kchunk-16thread.0.15 /srv/z/enc/test-enc-64kchunk-16thread.0.2 /srv/z/enc/test-enc-64kchunk-16thread.0.3 /srv/z/enc/test-enc-64kchunk-16thread.0.4 /srv/z/enc/test-enc-64kchunk-16thread.0.5 /srv/z/enc/test-enc-64kchunk-16thread.0.6 /srv/z/enc/test-enc-64kchunk-16thread.0.7 /srv/z/enc/test-enc-64kchunk-16thread.0.8 /srv/z/enc/test-enc-64kchunk-16thread.0.9 /srv/z/enc/test-enc-64kchunk-16thread.1.0 /srv/z/enc/test-enc-64kchunk-16thread.10.0 /srv/z/enc/test-enc-64kchunk-16thread.10.1 /srv/z/enc/test-enc-64kchunk-16thread.10.10 /srv/z/enc/test-enc-64kchunk-16thread.10.11 /srv/z/enc/test-enc-64kchunk-16thread.10.12 /srv/z/enc/test-enc-64kchunk-16thread.10.13 /srv/z/enc/test-enc-64kchunk-16thread.10.14 /srv/z/enc/test-enc-64kchunk-16thread.10.15 /srv/z/enc/test-enc-64kchunk-16thread.10.2 /srv/z/enc/test-enc-64kchunk-16thread.10.3 /srv/z/enc/test-enc-64kchunk-16thread.10.4 /srv/z/enc/test-enc-64kchunk-16thread.10.5 /srv/z/enc/test-enc-64kchunk-16thread.10.6 /srv/z/enc/test-enc-64kchunk-16thread.10.7 /srv/z/enc/test-enc-64kchunk-16thread.10.8 /srv/z/enc/test-enc-64kchunk-16thread.10.9 /srv/z/enc/test-enc-64kchunk-16thread.1.1 /srv/z/enc/test-enc-64kchunk-16thread.1.10 /srv/z/enc/test-enc-64kchunk-16thread.11.0 /srv/z/enc/test-enc-64kchunk-16thread.1.11 /srv/z/enc/test-enc-64kchunk-16thread.11.1 /srv/z/enc/test-enc-64kchunk-16thread.11.10 /srv/z/enc/test-enc-64kchunk-16thread.11.11 /srv/z/enc/test-enc-64kchunk-16thread.11.12 /srv/z/enc/test-enc-64kchunk-16thread.11.13 /srv/z/enc/test-enc-64kchunk-16thread.11.14 /srv/z/enc/test-enc-64kchunk-16thread.11.15 /srv/z/enc/test-enc-64kchunk-16thread.1.12 /srv/z/enc/test-enc-64kchunk-16thread.11.2 /srv/z/enc/test-enc-64kchunk-16thread.1.13 /srv/z/enc/test-enc-64kchunk-16thread.11.3 /srv/z/enc/test-enc-64kchunk-16thread.1.14 /srv/z/enc/test-enc-64kchunk-16thread.11.4 /srv/z/enc/test-enc-64kchunk-16thread.1.15 /srv/z/enc/test-enc-64kchunk-16thread.11.5 /srv/z/enc/test-enc-64kchunk-16thread.11.6 /srv/z/enc/test-enc-64kchunk-16thread.11.7 /srv/z/enc/test-enc-64kchunk-16thread.11.8 /srv/z/enc/test-enc-64kchunk-16thread.11.9 /srv/z/enc/test-enc-64kchunk-16thread.1.2 /srv/z/enc/test-enc-64kchunk-16thread.12.0 /srv/z/enc/test-enc-64kchunk-16thread.12.1 /srv/z/enc/test-enc-64kchunk-16thread.12.10 /srv/z/enc/test-enc-64kchunk-16thread.12.11 /srv/z/enc/test-enc-64kchunk-16thread.12.12 /srv/z/enc/test-enc-64kchunk-16thread.12.13 /srv/z/enc/test-enc-64kchunk-16thread.12.14 /srv/z/enc/test-enc-64kchunk-16thread.12.15 /srv/z/enc/test-enc-64kchunk-16thread.12.2 /srv/z/enc/test-enc-64kchunk-16thread.12.3 /srv/z/enc/test-enc-64kchunk-16thread.12.4 /srv/z/enc/test-enc-64kchunk-16thread.12.5 /srv/z/enc/test-enc-64kchunk-16thread.12.6 /srv/z/enc/test-enc-64kchunk-16thread.12.7 /srv/z/enc/test-enc-64kchunk-16thread.12.8 /srv/z/enc/test-enc-64kchunk-16thread.12.9 /srv/z/enc/test-enc-64kchunk-16thread.1.3 /srv/z/enc/test-enc-64kchunk-16thread.13.0 /srv/z/enc/test-enc-64kchunk-16thread.13.1 /srv/z/enc/test-enc-64kchunk-16thread.13.10 /srv/z/enc/test-enc-64kchunk-16thread.13.11 /srv/z/enc/test-enc-64kchunk-16thread.13.12 /srv/z/enc/test-enc-64kchunk-16thread.13.13 /srv/z/enc/test-enc-64kchunk-16thread.13.14 /srv/z/enc/test-enc-64kchunk-16thread.13.15 /srv/z/enc/test-enc-64kchunk-16thread.13.2 /srv/z/enc/test-enc-64kchunk-16thread.13.3 /srv/z/enc/test-enc-64kchunk-16thread.13.4 /srv/z/enc/test-enc-64kchunk-16thread.13.5 /srv/z/enc/test-enc-64kchunk-16thread.13.6 /srv/z/enc/test-enc-64kchunk-16thread.13.7 /srv/z/enc/test-enc-64kchunk-16thread.13.8 /srv/z/enc/test-enc-64kchunk-16thread.13.9 /srv/z/enc/test-enc-64kchunk-16thread.1.4 /srv/z/enc/test-enc-64kchunk-16thread.14.0 /srv/z/enc/test-enc-64kchunk-16thread.14.1 /srv/z/enc/test-enc-64kchunk-16thread.14.10 /srv/z/enc/test-enc-64kchunk-16thread.14.11 /srv/z/enc/test-enc-64kchunk-16thread.14.12 /srv/z/enc/test-enc-64kchunk-16thread.14.13 /srv/z/enc/test-enc-64kchunk-16thread.14.14 /srv/z/enc/test-enc-64kchunk-16thread.14.15 /srv/z/enc/test-enc-64kchunk-16thread.14.2 /srv/z/enc/test-enc-64kchunk-16thread.14.3 /srv/z/enc/test-enc-64kchunk-16thread.14.4 /srv/z/enc/test-enc-64kchunk-16thread.14.5 /srv/z/enc/test-enc-64kchunk-16thread.14.6 /srv/z/enc/test-enc-64kchunk-16thread.14.7 /srv/z/enc/test-enc-64kchunk-16thread.14.8 /srv/z/enc/test-enc-64kchunk-16thread.14.9 /srv/z/enc/test-enc-64kchunk-16thread.1.5 /srv/z/enc/test-enc-64kchunk-16thread.15.0 /srv/z/enc/test-enc-64kchunk-16thread.15.1 /srv/z/enc/test-enc-64kchunk-16thread.15.10 /srv/z/enc/test-enc-64kchunk-16thread.15.11 /srv/z/enc/test-enc-64kchunk-16thread.15.12 /srv/z/enc/test-enc-64kchunk-16thread.15.13 /srv/z/enc/test-enc-64kchunk-16thread.15.14 /srv/z/enc/test-enc-64kchunk-16thread.15.15 /srv/z/enc/test-enc-64kchunk-16thread.15.2 /srv/z/enc/test-enc-64kchunk-16thread.15.3 /srv/z/enc/test-enc-64kchunk-16thread.15.4 /srv/z/enc/test-enc-64kchunk-16thread.15.5 /srv/z/enc/test-enc-64kchunk-16thread.15.6 /srv/z/enc/test-enc-64kchunk-16thread.15.7 /srv/z/enc/test-enc-64kchunk-16thread.15.8 /srv/z/enc/test-enc-64kchunk-16thread.15.9 /srv/z/enc/test-enc-64kchunk-16thread.1.6 /srv/z/enc/test-enc-64kchunk-16thread.1.7 /srv/z/enc/test-enc-64kchunk-16thread.1.8 /srv/z/enc/test-enc-64kchunk-16thread.1.9 /srv/z/enc/test-enc-64kchunk-16thread.2.0 /srv/z/enc/test-enc-64kchunk-16thread.2.1 /srv/z/enc/test-enc-64kchunk-16thread.2.10 /srv/z/enc/test-enc-64kchunk-16thread.2.11 /srv/z/enc/test-enc-64kchunk-16thread.2.12 /srv/z/enc/test-enc-64kchunk-16thread.2.13 /srv/z/enc/test-enc-64kchunk-16thread.2.14 /srv/z/enc/test-enc-64kchunk-16thread.2.15 /srv/z/enc/test-enc-64kchunk-16thread.2.2 /srv/z/enc/test-enc-64kchunk-16thread.2.3 /srv/z/enc/test-enc-64kchunk-16thread.2.4 /srv/z/enc/test-enc-64kchunk-16thread.2.5 /srv/z/enc/test-enc-64kchunk-16thread.2.6 /srv/z/enc/test-enc-64kchunk-16thread.2.7 /srv/z/enc/test-enc-64kchunk-16thread.2.8 /srv/z/enc/test-enc-64kchunk-16thread.2.9 /srv/z/enc/test-enc-64kchunk-16thread.3.0 /srv/z/enc/test-enc-64kchunk-16thread.3.1 /srv/z/enc/test-enc-64kchunk-16thread.3.10 /srv/z/enc/test-enc-64kchunk-16thread.3.11 /srv/z/enc/test-enc-64kchunk-16thread.3.12 /srv/z/enc/test-enc-64kchunk-16thread.3.13 /srv/z/enc/test-enc-64kchunk-16thread.3.14 /srv/z/enc/test-enc-64kchunk-16thread.3.15 /srv/z/enc/test-enc-64kchunk-16thread.3.2 /srv/z/enc/test-enc-64kchunk-16thread.3.3 /srv/z/enc/test-enc-64kchunk-16thread.3.4 /srv/z/enc/test-enc-64kchunk-16thread.3.5 /srv/z/enc/test-enc-64kchunk-16thread.3.6 /srv/z/enc/test-enc-64kchunk-16thread.3.7 /srv/z/enc/test-enc-64kchunk-16thread.3.8 /srv/z/enc/test-enc-64kchunk-16thread.3.9 /srv/z/enc/test-enc-64kchunk-16thread.4.0 /srv/z/enc/test-enc-64kchunk-16thread.4.1 /srv/z/enc/test-enc-64kchunk-16thread.4.10 /srv/z/enc/test-enc-64kchunk-16thread.4.11 /srv/z/enc/test-enc-64kchunk-16thread.4.12 /srv/z/enc/test-enc-64kchunk-16thread.4.13 /srv/z/enc/test-enc-64kchunk-16thread.4.14 /srv/z/enc/test-enc-64kchunk-16thread.4.15 /srv/z/enc/test-enc-64kchunk-16thread.4.2 /srv/z/enc/test-enc-64kchunk-16thread.4.3 /srv/z/enc/test-enc-64kchunk-16thread.4.4 /srv/z/enc/test-enc-64kchunk-16thread.4.5 /srv/z/enc/test-enc-64kchunk-16thread.4.6 /srv/z/enc/test-enc-64kchunk-16thread.4.7 /srv/z/enc/test-enc-64kchunk-16thread.4.8 /srv/z/enc/test-enc-64kchunk-16thread.4.9 /srv/z/enc/test-enc-64kchunk-16thread.5.0 /srv/z/enc/test-enc-64kchunk-16thread.5.1 /srv/z/enc/test-enc-64kchunk-16thread.5.10 /srv/z/enc/test-enc-64kchunk-16thread.5.11 /srv/z/enc/test-enc-64kchunk-16thread.5.12 /srv/z/enc/test-enc-64kchunk-16thread.5.13 /srv/z/enc/test-enc-64kchunk-16thread.5.14 /srv/z/enc/test-enc-64kchunk-16thread.5.15 /srv/z/enc/test-enc-64kchunk-16thread.5.2 /srv/z/enc/test-enc-64kchunk-16thread.5.3 /srv/z/enc/test-enc-64kchunk-16thread.5.4 /srv/z/enc/test-enc-64kchunk-16thread.5.5 /srv/z/enc/test-enc-64kchunk-16thread.5.6 /srv/z/enc/test-enc-64kchunk-16thread.5.7 /srv/z/enc/test-enc-64kchunk-16thread.5.8 /srv/z/enc/test-enc-64kchunk-16thread.5.9 /srv/z/enc/test-enc-64kchunk-16thread.6.0 /srv/z/enc/test-enc-64kchunk-16thread.6.1 /srv/z/enc/test-enc-64kchunk-16thread.6.10 /srv/z/enc/test-enc-64kchunk-16thread.6.11 /srv/z/enc/test-enc-64kchunk-16thread.6.12 /srv/z/enc/test-enc-64kchunk-16thread.6.13 /srv/z/enc/test-enc-64kchunk-16thread.6.14 /srv/z/enc/test-enc-64kchunk-16thread.6.15 /srv/z/enc/test-enc-64kchunk-16thread.6.2 /srv/z/enc/test-enc-64kchunk-16thread.6.3 /srv/z/enc/test-enc-64kchunk-16thread.6.4 /srv/z/enc/test-enc-64kchunk-16thread.6.5 /srv/z/enc/test-enc-64kchunk-16thread.6.6 /srv/z/enc/test-enc-64kchunk-16thread.6.7 /srv/z/enc/test-enc-64kchunk-16thread.6.8 /srv/z/enc/test-enc-64kchunk-16thread.6.9 /srv/z/enc/test-enc-64kchunk-16thread.7.0 /srv/z/enc/test-enc-64kchunk-16thread.7.1 /srv/z/enc/test-enc-64kchunk-16thread.7.10 /srv/z/enc/test-enc-64kchunk-16thread.7.11 /srv/z/enc/test-enc-64kchunk-16thread.7.12 /srv/z/enc/test-enc-64kchunk-16thread.7.13 /srv/z/enc/test-enc-64kchunk-16thread.7.14 /srv/z/enc/test-enc-64kchunk-16thread.7.15 /srv/z/enc/test-enc-64kchunk-16thread.7.2 /srv/z/enc/test-enc-64kchunk-16thread.7.3 /srv/z/enc/test-enc-64kchunk-16thread.7.4 /srv/z/enc/test-enc-64kchunk-16thread.7.5 /srv/z/enc/test-enc-64kchunk-16thread.7.6 /srv/z/enc/test-enc-64kchunk-16thread.7.7 /srv/z/enc/test-enc-64kchunk-16thread.7.8 /srv/z/enc/test-enc-64kchunk-16thread.7.9 /srv/z/enc/test-enc-64kchunk-16thread.8.0 /srv/z/enc/test-enc-64kchunk-16thread.8.1 /srv/z/enc/test-enc-64kchunk-16thread.8.10 /srv/z/enc/test-enc-64kchunk-16thread.8.11 /srv/z/enc/test-enc-64kchunk-16thread.8.12 /srv/z/enc/test-enc-64kchunk-16thread.8.13 /srv/z/enc/test-enc-64kchunk-16thread.8.14 /srv/z/enc/test-enc-64kchunk-16thread.8.15 /srv/z/enc/test-enc-64kchunk-16thread.8.2 /srv/z/enc/test-enc-64kchunk-16thread.8.3 /srv/z/enc/test-enc-64kchunk-16thread.8.4 /srv/z/enc/test-enc-64kchunk-16thread.8.5 /srv/z/enc/test-enc-64kchunk-16thread.8.6 /srv/z/enc/test-enc-64kchunk-16thread.8.7 /srv/z/enc/test-enc-64kchunk-16thread.8.8 /srv/z/enc/test-enc-64kchunk-16thread.8.9 /srv/z/enc/test-enc-64kchunk-16thread.9.0 /srv/z/enc/test-enc-64kchunk-16thread.9.1 /srv/z/enc/test-enc-64kchunk-16thread.9.10 /srv/z/enc/test-enc-64kchunk-16thread.9.11 /srv/z/enc/test-enc-64kchunk-16thread.9.12 /srv/z/enc/test-enc-64kchunk-16thread.9.13 /srv/z/enc/test-enc-64kchunk-16thread.9.14 /srv/z/enc/test-enc-64kchunk-16thread.9.15 /srv/z/enc/test-enc-64kchunk-16thread.9.2 /srv/z/enc/test-enc-64kchunk-16thread.9.3 /srv/z/enc/test-enc-64kchunk-16thread.9.4 /srv/z/enc/test-enc-64kchunk-16thread.9.5 /srv/z/enc/test-enc-64kchunk-16thread.9.6 /srv/z/enc/test-enc-64kchunk-16thread.9.7 /srv/z/enc/test-enc-64kchunk-16thread.9.8 /srv/z/enc/test-enc-64kchunk-16thread.9.9 | |
| + for DIRNAME in none comp enc both | |
| + cat | |
| + cat /tmp/jobfile.fio | |
| [test-both-64kchunk-16thread] | |
| bs=64k | |
| directory=/srv/z/both/ | |
| iodepth=64 | |
| ioengine=libaio | |
| readwrite=readwrite | |
| numjobs=16 | |
| nrfiles=16 | |
| size=10000M | |
| runtime=120 | |
| time_based=1 | |
| group_reporting=1 | |
| + fio --eta-newline=10 --eta-interval=10 --eta=always /tmp/jobfile.fio | |
| test-both-64kchunk-16thread: (g=0): rw=rw, bs=(R) 64.0KiB-64.0KiB, (W) 64.0KiB-64.0KiB, (T) 64.0KiB-64.0KiB, ioengine=libaio, iodepth=64 | |
| ... | |
| fio-3.16 | |
| Starting 16 processes | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| test-both-64kchunk-16thread: Laying out IO files (16 files / total 10000MiB) | |
| Jobs: 16 (f=256): [M(16)][8.3%][r=396MiB/s,w=397MiB/s][r=6330,w=6357 IOPS][eta 01m:50s] Jobs: 16 (f=256): [M(16)][16.7%][r=385MiB/s,w=387MiB/s][r=6154,w=6192 IOPS][eta 01m:40s] Jobs: 16 (f=256): [M(16)][24.2%][r=347MiB/s,w=348MiB/s][r=5547,w=5563 IOPS][eta 01m:31s] | |
| Jobs: 16 (f=256): [M(16)][32.5%][r=328MiB/s,w=326MiB/s][r=5241,w=5222 IOPS][eta 01m:21s] Jobs: 16 (f=256): [M(16)][40.5%][r=325MiB/s,w=321MiB/s][r=5199,w=5138 IOPS][eta 01m:12s] | |
| Jobs: 16 (f=256): [M(16)][48.3%][r=343MiB/s,w=343MiB/s][r=5492,w=5487 IOPS][eta 01m:02s] Jobs: 16 (f=256): [M(16)][56.7%][r=345MiB/s,w=347MiB/s][r=5518,w=5550 IOPS][eta 00m:52s] | |
| Jobs: 16 (f=256): [M(16)][64.5%][r=362MiB/s,w=361MiB/s][r=5790,w=5769 IOPS][eta 00m:43s] Jobs: 16 (f=256): [M(16)][72.5%][r=369MiB/s,w=370MiB/s][r=5903,w=5924 IOPS][eta 00m:33s] | |
| Jobs: 16 (f=256): [M(16)][80.2%][r=381MiB/s,w=379MiB/s][r=6088,w=6066 IOPS][eta 00m:24s] Jobs: 16 (f=256): [M(16)][88.3%][r=358MiB/s,w=358MiB/s][r=5724,w=5734 IOPS][eta 00m:14s] | |
| Jobs: 16 (f=256): [M(16)][96.7%][r=366MiB/s,w=365MiB/s][r=5850,w=5838 IOPS][eta 00m:04s] | |
| test-both-64kchunk-16thread: (groupid=0, jobs=16): err= 0: pid=1765030: Wed Feb 26 14:59:08 2020 | |
| read: IOPS=5735, BW=358MiB/s (376MB/s)(42.0GiB/120011msec) | |
| slat (usec): min=9, max=140759, avg=1441.54, stdev=3519.76 | |
| clat (usec): min=4, max=440884, avg=87845.33, stdev=50593.21 | |
| lat (msec): min=2, max=440, avg=89.29, stdev=51.31 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 14], 10.00th=[ 22], 20.00th=[ 40], | |
| | 30.00th=[ 62], 40.00th=[ 75], 50.00th=[ 86], 60.00th=[ 96], | |
| | 70.00th=[ 108], 80.00th=[ 128], 90.00th=[ 157], 95.00th=[ 180], | |
| | 99.00th=[ 222], 99.50th=[ 241], 99.90th=[ 296], 99.95th=[ 326], | |
| | 99.99th=[ 384] | |
| bw ( KiB/s): min=138785, max=903118, per=99.81%, avg=366362.41, stdev=13838.29, samples=3838 | |
| iops : min= 2162, max=14110, avg=5721.65, stdev=216.25, samples=3838 | |
| write: IOPS=5735, BW=358MiB/s (376MB/s)(42.0GiB/120011msec); 0 zone resets | |
| slat (usec): min=15, max=196669, avg=1340.57, stdev=3351.45 | |
| clat (usec): min=4, max=440869, avg=87714.16, stdev=50483.79 | |
| lat (msec): min=2, max=440, avg=89.06, stdev=51.15 | |
| clat percentiles (msec): | |
| | 1.00th=[ 4], 5.00th=[ 14], 10.00th=[ 22], 20.00th=[ 40], | |
| | 30.00th=[ 62], 40.00th=[ 75], 50.00th=[ 86], 60.00th=[ 95], | |
| | 70.00th=[ 108], 80.00th=[ 128], 90.00th=[ 157], 95.00th=[ 180], | |
| | 99.00th=[ 222], 99.50th=[ 241], 99.90th=[ 296], 99.95th=[ 326], | |
| | 99.99th=[ 388] | |
| bw ( KiB/s): min=138896, max=900156, per=99.80%, avg=366371.84, stdev=13803.54, samples=3838 | |
| iops : min= 2167, max=14064, avg=5721.86, stdev=215.71, samples=3838 | |
| lat (usec) : 10=0.01%, 20=0.01% | |
| lat (msec) : 4=1.59%, 10=1.90%, 20=5.39%, 50=16.29%, 100=38.94% | |
| lat (msec) : 250=35.53%, 500=0.35% | |
| cpu : usr=0.31%, sys=38.11%, ctx=877277, majf=0, minf=440809 | |
| IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9% | |
| submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% | |
| complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0% | |
| issued rwts: total=688325,688361,0,0 short=0,0,0,0 dropped=0,0,0,0 | |
| latency : target=0, window=0, percentile=100.00%, depth=64 | |
| Run status group 0 (all jobs): | |
| READ: bw=358MiB/s (376MB/s), 358MiB/s-358MiB/s (376MB/s-376MB/s), io=42.0GiB (45.1GB), run=120011-120011msec | |
| WRITE: bw=358MiB/s (376MB/s), 358MiB/s-358MiB/s (376MB/s-376MB/s), io=42.0GiB (45.1GB), run=120011-120011msec | |
| + rm -rf /srv/z/both/test-both-64kchunk-16thread.0.0 /srv/z/both/test-both-64kchunk-16thread.0.1 /srv/z/both/test-both-64kchunk-16thread.0.10 /srv/z/both/test-both-64kchunk-16thread.0.11 /srv/z/both/test-both-64kchunk-16thread.0.12 /srv/z/both/test-both-64kchunk-16thread.0.13 /srv/z/both/test-both-64kchunk-16thread.0.14 /srv/z/both/test-both-64kchunk-16thread.0.15 /srv/z/both/test-both-64kchunk-16thread.0.2 /srv/z/both/test-both-64kchunk-16thread.0.3 /srv/z/both/test-both-64kchunk-16thread.0.4 /srv/z/both/test-both-64kchunk-16thread.0.5 /srv/z/both/test-both-64kchunk-16thread.0.6 /srv/z/both/test-both-64kchunk-16thread.0.7 /srv/z/both/test-both-64kchunk-16thread.0.8 /srv/z/both/test-both-64kchunk-16thread.0.9 /srv/z/both/test-both-64kchunk-16thread.1.0 /srv/z/both/test-both-64kchunk-16thread.10.0 /srv/z/both/test-both-64kchunk-16thread.10.1 /srv/z/both/test-both-64kchunk-16thread.10.10 /srv/z/both/test-both-64kchunk-16thread.10.11 /srv/z/both/test-both-64kchunk-16thread.10.12 /srv/z/both/test-both-64kchunk-16thread.10.13 /srv/z/both/test-both-64kchunk-16thread.10.14 /srv/z/both/test-both-64kchunk-16thread.10.15 /srv/z/both/test-both-64kchunk-16thread.10.2 /srv/z/both/test-both-64kchunk-16thread.10.3 /srv/z/both/test-both-64kchunk-16thread.10.4 /srv/z/both/test-both-64kchunk-16thread.10.5 /srv/z/both/test-both-64kchunk-16thread.10.6 /srv/z/both/test-both-64kchunk-16thread.10.7 /srv/z/both/test-both-64kchunk-16thread.10.8 /srv/z/both/test-both-64kchunk-16thread.10.9 /srv/z/both/test-both-64kchunk-16thread.1.1 /srv/z/both/test-both-64kchunk-16thread.1.10 /srv/z/both/test-both-64kchunk-16thread.11.0 /srv/z/both/test-both-64kchunk-16thread.1.11 /srv/z/both/test-both-64kchunk-16thread.11.1 /srv/z/both/test-both-64kchunk-16thread.11.10 /srv/z/both/test-both-64kchunk-16thread.11.11 /srv/z/both/test-both-64kchunk-16thread.11.12 /srv/z/both/test-both-64kchunk-16thread.11.13 /srv/z/both/test-both-64kchunk-16thread.11.14 /srv/z/both/test-both-64kchunk-16thread.11.15 /srv/z/both/test-both-64kchunk-16thread.1.12 /srv/z/both/test-both-64kchunk-16thread.11.2 /srv/z/both/test-both-64kchunk-16thread.1.13 /srv/z/both/test-both-64kchunk-16thread.11.3 /srv/z/both/test-both-64kchunk-16thread.1.14 /srv/z/both/test-both-64kchunk-16thread.11.4 /srv/z/both/test-both-64kchunk-16thread.1.15 /srv/z/both/test-both-64kchunk-16thread.11.5 /srv/z/both/test-both-64kchunk-16thread.11.6 /srv/z/both/test-both-64kchunk-16thread.11.7 /srv/z/both/test-both-64kchunk-16thread.11.8 /srv/z/both/test-both-64kchunk-16thread.11.9 /srv/z/both/test-both-64kchunk-16thread.1.2 /srv/z/both/test-both-64kchunk-16thread.12.0 /srv/z/both/test-both-64kchunk-16thread.12.1 /srv/z/both/test-both-64kchunk-16thread.12.10 /srv/z/both/test-both-64kchunk-16thread.12.11 /srv/z/both/test-both-64kchunk-16thread.12.12 /srv/z/both/test-both-64kchunk-16thread.12.13 /srv/z/both/test-both-64kchunk-16thread.12.14 /srv/z/both/test-both-64kchunk-16thread.12.15 /srv/z/both/test-both-64kchunk-16thread.12.2 /srv/z/both/test-both-64kchunk-16thread.12.3 /srv/z/both/test-both-64kchunk-16thread.12.4 /srv/z/both/test-both-64kchunk-16thread.12.5 /srv/z/both/test-both-64kchunk-16thread.12.6 /srv/z/both/test-both-64kchunk-16thread.12.7 /srv/z/both/test-both-64kchunk-16thread.12.8 /srv/z/both/test-both-64kchunk-16thread.12.9 /srv/z/both/test-both-64kchunk-16thread.1.3 /srv/z/both/test-both-64kchunk-16thread.13.0 /srv/z/both/test-both-64kchunk-16thread.13.1 /srv/z/both/test-both-64kchunk-16thread.13.10 /srv/z/both/test-both-64kchunk-16thread.13.11 /srv/z/both/test-both-64kchunk-16thread.13.12 /srv/z/both/test-both-64kchunk-16thread.13.13 /srv/z/both/test-both-64kchunk-16thread.13.14 /srv/z/both/test-both-64kchunk-16thread.13.15 /srv/z/both/test-both-64kchunk-16thread.13.2 /srv/z/both/test-both-64kchunk-16thread.13.3 /srv/z/both/test-both-64kchunk-16thread.13.4 /srv/z/both/test-both-64kchunk-16thread.13.5 /srv/z/both/test-both-64kchunk-16thread.13.6 /srv/z/both/test-both-64kchunk-16thread.13.7 /srv/z/both/test-both-64kchunk-16thread.13.8 /srv/z/both/test-both-64kchunk-16thread.13.9 /srv/z/both/test-both-64kchunk-16thread.1.4 /srv/z/both/test-both-64kchunk-16thread.14.0 /srv/z/both/test-both-64kchunk-16thread.14.1 /srv/z/both/test-both-64kchunk-16thread.14.10 /srv/z/both/test-both-64kchunk-16thread.14.11 /srv/z/both/test-both-64kchunk-16thread.14.12 /srv/z/both/test-both-64kchunk-16thread.14.13 /srv/z/both/test-both-64kchunk-16thread.14.14 /srv/z/both/test-both-64kchunk-16thread.14.15 /srv/z/both/test-both-64kchunk-16thread.14.2 /srv/z/both/test-both-64kchunk-16thread.14.3 /srv/z/both/test-both-64kchunk-16thread.14.4 /srv/z/both/test-both-64kchunk-16thread.14.5 /srv/z/both/test-both-64kchunk-16thread.14.6 /srv/z/both/test-both-64kchunk-16thread.14.7 /srv/z/both/test-both-64kchunk-16thread.14.8 /srv/z/both/test-both-64kchunk-16thread.14.9 /srv/z/both/test-both-64kchunk-16thread.1.5 /srv/z/both/test-both-64kchunk-16thread.15.0 /srv/z/both/test-both-64kchunk-16thread.15.1 /srv/z/both/test-both-64kchunk-16thread.15.10 /srv/z/both/test-both-64kchunk-16thread.15.11 /srv/z/both/test-both-64kchunk-16thread.15.12 /srv/z/both/test-both-64kchunk-16thread.15.13 /srv/z/both/test-both-64kchunk-16thread.15.14 /srv/z/both/test-both-64kchunk-16thread.15.15 /srv/z/both/test-both-64kchunk-16thread.15.2 /srv/z/both/test-both-64kchunk-16thread.15.3 /srv/z/both/test-both-64kchunk-16thread.15.4 /srv/z/both/test-both-64kchunk-16thread.15.5 /srv/z/both/test-both-64kchunk-16thread.15.6 /srv/z/both/test-both-64kchunk-16thread.15.7 /srv/z/both/test-both-64kchunk-16thread.15.8 /srv/z/both/test-both-64kchunk-16thread.15.9 /srv/z/both/test-both-64kchunk-16thread.1.6 /srv/z/both/test-both-64kchunk-16thread.1.7 /srv/z/both/test-both-64kchunk-16thread.1.8 /srv/z/both/test-both-64kchunk-16thread.1.9 /srv/z/both/test-both-64kchunk-16thread.2.0 /srv/z/both/test-both-64kchunk-16thread.2.1 /srv/z/both/test-both-64kchunk-16thread.2.10 /srv/z/both/test-both-64kchunk-16thread.2.11 /srv/z/both/test-both-64kchunk-16thread.2.12 /srv/z/both/test-both-64kchunk-16thread.2.13 /srv/z/both/test-both-64kchunk-16thread.2.14 /srv/z/both/test-both-64kchunk-16thread.2.15 /srv/z/both/test-both-64kchunk-16thread.2.2 /srv/z/both/test-both-64kchunk-16thread.2.3 /srv/z/both/test-both-64kchunk-16thread.2.4 /srv/z/both/test-both-64kchunk-16thread.2.5 /srv/z/both/test-both-64kchunk-16thread.2.6 /srv/z/both/test-both-64kchunk-16thread.2.7 /srv/z/both/test-both-64kchunk-16thread.2.8 /srv/z/both/test-both-64kchunk-16thread.2.9 /srv/z/both/test-both-64kchunk-16thread.3.0 /srv/z/both/test-both-64kchunk-16thread.3.1 /srv/z/both/test-both-64kchunk-16thread.3.10 /srv/z/both/test-both-64kchunk-16thread.3.11 /srv/z/both/test-both-64kchunk-16thread.3.12 /srv/z/both/test-both-64kchunk-16thread.3.13 /srv/z/both/test-both-64kchunk-16thread.3.14 /srv/z/both/test-both-64kchunk-16thread.3.15 /srv/z/both/test-both-64kchunk-16thread.3.2 /srv/z/both/test-both-64kchunk-16thread.3.3 /srv/z/both/test-both-64kchunk-16thread.3.4 /srv/z/both/test-both-64kchunk-16thread.3.5 /srv/z/both/test-both-64kchunk-16thread.3.6 /srv/z/both/test-both-64kchunk-16thread.3.7 /srv/z/both/test-both-64kchunk-16thread.3.8 /srv/z/both/test-both-64kchunk-16thread.3.9 /srv/z/both/test-both-64kchunk-16thread.4.0 /srv/z/both/test-both-64kchunk-16thread.4.1 /srv/z/both/test-both-64kchunk-16thread.4.10 /srv/z/both/test-both-64kchunk-16thread.4.11 /srv/z/both/test-both-64kchunk-16thread.4.12 /srv/z/both/test-both-64kchunk-16thread.4.13 /srv/z/both/test-both-64kchunk-16thread.4.14 /srv/z/both/test-both-64kchunk-16thread.4.15 /srv/z/both/test-both-64kchunk-16thread.4.2 /srv/z/both/test-both-64kchunk-16thread.4.3 /srv/z/both/test-both-64kchunk-16thread.4.4 /srv/z/both/test-both-64kchunk-16thread.4.5 /srv/z/both/test-both-64kchunk-16thread.4.6 /srv/z/both/test-both-64kchunk-16thread.4.7 /srv/z/both/test-both-64kchunk-16thread.4.8 /srv/z/both/test-both-64kchunk-16thread.4.9 /srv/z/both/test-both-64kchunk-16thread.5.0 /srv/z/both/test-both-64kchunk-16thread.5.1 /srv/z/both/test-both-64kchunk-16thread.5.10 /srv/z/both/test-both-64kchunk-16thread.5.11 /srv/z/both/test-both-64kchunk-16thread.5.12 /srv/z/both/test-both-64kchunk-16thread.5.13 /srv/z/both/test-both-64kchunk-16thread.5.14 /srv/z/both/test-both-64kchunk-16thread.5.15 /srv/z/both/test-both-64kchunk-16thread.5.2 /srv/z/both/test-both-64kchunk-16thread.5.3 /srv/z/both/test-both-64kchunk-16thread.5.4 /srv/z/both/test-both-64kchunk-16thread.5.5 /srv/z/both/test-both-64kchunk-16thread.5.6 /srv/z/both/test-both-64kchunk-16thread.5.7 /srv/z/both/test-both-64kchunk-16thread.5.8 /srv/z/both/test-both-64kchunk-16thread.5.9 /srv/z/both/test-both-64kchunk-16thread.6.0 /srv/z/both/test-both-64kchunk-16thread.6.1 /srv/z/both/test-both-64kchunk-16thread.6.10 /srv/z/both/test-both-64kchunk-16thread.6.11 /srv/z/both/test-both-64kchunk-16thread.6.12 /srv/z/both/test-both-64kchunk-16thread.6.13 /srv/z/both/test-both-64kchunk-16thread.6.14 /srv/z/both/test-both-64kchunk-16thread.6.15 /srv/z/both/test-both-64kchunk-16thread.6.2 /srv/z/both/test-both-64kchunk-16thread.6.3 /srv/z/both/test-both-64kchunk-16thread.6.4 /srv/z/both/test-both-64kchunk-16thread.6.5 /srv/z/both/test-both-64kchunk-16thread.6.6 /srv/z/both/test-both-64kchunk-16thread.6.7 /srv/z/both/test-both-64kchunk-16thread.6.8 /srv/z/both/test-both-64kchunk-16thread.6.9 /srv/z/both/test-both-64kchunk-16thread.7.0 /srv/z/both/test-both-64kchunk-16thread.7.1 /srv/z/both/test-both-64kchunk-16thread.7.10 /srv/z/both/test-both-64kchunk-16thread.7.11 /srv/z/both/test-both-64kchunk-16thread.7.12 /srv/z/both/test-both-64kchunk-16thread.7.13 /srv/z/both/test-both-64kchunk-16thread.7.14 /srv/z/both/test-both-64kchunk-16thread.7.15 /srv/z/both/test-both-64kchunk-16thread.7.2 /srv/z/both/test-both-64kchunk-16thread.7.3 /srv/z/both/test-both-64kchunk-16thread.7.4 /srv/z/both/test-both-64kchunk-16thread.7.5 /srv/z/both/test-both-64kchunk-16thread.7.6 /srv/z/both/test-both-64kchunk-16thread.7.7 /srv/z/both/test-both-64kchunk-16thread.7.8 /srv/z/both/test-both-64kchunk-16thread.7.9 /srv/z/both/test-both-64kchunk-16thread.8.0 /srv/z/both/test-both-64kchunk-16thread.8.1 /srv/z/both/test-both-64kchunk-16thread.8.10 /srv/z/both/test-both-64kchunk-16thread.8.11 /srv/z/both/test-both-64kchunk-16thread.8.12 /srv/z/both/test-both-64kchunk-16thread.8.13 /srv/z/both/test-both-64kchunk-16thread.8.14 /srv/z/both/test-both-64kchunk-16thread.8.15 /srv/z/both/test-both-64kchunk-16thread.8.2 /srv/z/both/test-both-64kchunk-16thread.8.3 /srv/z/both/test-both-64kchunk-16thread.8.4 /srv/z/both/test-both-64kchunk-16thread.8.5 /srv/z/both/test-both-64kchunk-16thread.8.6 /srv/z/both/test-both-64kchunk-16thread.8.7 /srv/z/both/test-both-64kchunk-16thread.8.8 /srv/z/both/test-both-64kchunk-16thread.8.9 /srv/z/both/test-both-64kchunk-16thread.9.0 /srv/z/both/test-both-64kchunk-16thread.9.1 /srv/z/both/test-both-64kchunk-16thread.9.10 /srv/z/both/test-both-64kchunk-16thread.9.11 /srv/z/both/test-both-64kchunk-16thread.9.12 /srv/z/both/test-both-64kchunk-16thread.9.13 /srv/z/both/test-both-64kchunk-16thread.9.14 /srv/z/both/test-both-64kchunk-16thread.9.15 /srv/z/both/test-both-64kchunk-16thread.9.2 /srv/z/both/test-both-64kchunk-16thread.9.3 /srv/z/both/test-both-64kchunk-16thread.9.4 /srv/z/both/test-both-64kchunk-16thread.9.5 /srv/z/both/test-both-64kchunk-16thread.9.6 /srv/z/both/test-both-64kchunk-16thread.9.7 /srv/z/both/test-both-64kchunk-16thread.9.8 /srv/z/both/test-both-64kchunk-16thread.9.9 | |
| + zpool destroy tank | |
| Script done on Wed Feb 26 06:59:11 2020 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment