zfs_stats.log

root@pve1:~# dmesg | grep -i sata | grep 'link up'
[    1.383085] ata3: SATA link up 3.0 Gbps (SStatus 123 SControl 300)
[    1.383112] ata2: SATA link up 3.0 Gbps (SStatus 123 SControl 300)
[    1.383134] ata4: SATA link up 3.0 Gbps (SStatus 123 SControl 300)
root@pve1:~# zpool list rpool
NAME    SIZE  ALLOC   FREE  CKPOINT  EXPANDSZ   FRAG    CAP  DEDUP    HEALTH  ALTROOT
rpool  2.72T   357G  2.37T        -         -     4%    12%  1.00x    ONLINE  -
root@pve1:~# zpool status rpool
  pool: rpool
 state: ONLINE
  scan: scrub repaired 0B in 01:39:44 with 0 errors on Sun Jun 11 02:03:50 2023
config:

	NAME                                                    STATE     READ WRITE CKSUM
	rpool                                                   ONLINE       0     0     0
	  raidz1-0                                              ONLINE       0     0     0
	    ata-APPLE_HDD_HTS541010A9E632_JD8008D8GSSR8S-part3  ONLINE       0     0     0
	    ata-APPLE_HDD_HTS541010A9E632_JD8002D8GRS7GD-part3  ONLINE       0     0     0
	    ata-APPLE_HDD_HTS541010A9E632_J88000D8GDDUHD-part3  ONLINE       0     0     0

errors: No known data errors
root@pve1:~# curl https://gist.githubusercontent.com/mergwyn/7be8dde50e21ed0e72a6e74a911a8dce/raw/78dfc33e31de5d5c46201df690cae7904153f749/zfsbench | bash -
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   739  100   739    0     0   1955      0 --:--:-- --:--:-- --:--:--  1955
Sat 24 Jun 2023 11:37:18 AM CEST:Random read
test: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=psync, iodepth=4
fio-3.25
Starting 1 process
test: Laying out IO file (1 file / 10240MiB)
Jobs: 1 (f=1): [r(1)][100.0%][r=152KiB/s][r=38 IOPS][eta 00m:00s]s]
test: (groupid=0, jobs=1): err= 0: pid=3499457: Sat Jun 24 11:45:59 2023
  read: IOPS=50, BW=203KiB/s (208kB/s)(59.4MiB/300006msec)
    clat (usec): min=3, max=3730.8k, avg=19720.51, stdev=42075.26
     lat (usec): min=3, max=3730.8k, avg=19720.94, stdev=42075.26
    clat percentiles (usec):
     |  1.00th=[    26],  5.00th=[    31], 10.00th=[  5932], 20.00th=[  8848],
     | 30.00th=[ 10552], 40.00th=[ 11863], 50.00th=[ 12911], 60.00th=[ 13960],
     | 70.00th=[ 15533], 80.00th=[ 21627], 90.00th=[ 36963], 95.00th=[ 50070],
     | 99.00th=[164627], 99.50th=[235930], 99.90th=[312476], 99.95th=[379585],
     | 99.99th=[624952]
   bw (  KiB/s): min=    8, max=  424, per=100.00%, avg=205.60, stdev=119.59, samples=591
   iops        : min=    2, max=  106, avg=51.40, stdev=29.90, samples=591
  lat (usec)   : 4=0.01%, 10=0.05%, 20=0.43%, 50=6.84%, 100=0.22%
  lat (usec)   : 250=0.03%, 500=0.03%, 750=0.07%
  lat (msec)   : 4=0.16%, 10=18.69%, 20=52.13%, 50=16.26%, 100=3.24%
  lat (msec)   : 250=1.45%, 500=0.36%, 750=0.02%, >=2000=0.01%
  cpu          : usr=0.05%, sys=0.38%, ctx=14344, majf=7, minf=13
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=15206,0,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: bw=203KiB/s (208kB/s), 203KiB/s-203KiB/s (208kB/s-208kB/s), io=59.4MiB (62.3MB), run=300006-300006msec
Sat 24 Jun 2023 11:46:04 AM CEST:Random write
test: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=psync, iodepth=4
fio-3.25
Starting 1 process
test: Laying out IO file (1 file / 10240MiB)
Jobs: 1 (f=1): [w(1)][100.0%][w=48KiB/s][w=12 IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=3516539: Sat Jun 24 11:51:07 2023
  write: IOPS=49, BW=200KiB/s (205kB/s)(58.6MiB/300014msec); 0 zone resets
    clat (msec): min=5, max=504, avg=20.00, stdev=33.18
     lat (msec): min=5, max=504, avg=20.00, stdev=33.18
    clat percentiles (msec):
     |  1.00th=[   11],  5.00th=[   12], 10.00th=[   12], 20.00th=[   12],
     | 30.00th=[   12], 40.00th=[   12], 50.00th=[   12], 60.00th=[   12],
     | 70.00th=[   12], 80.00th=[   12], 90.00th=[   31], 95.00th=[   65],
     | 99.00th=[  203], 99.50th=[  249], 99.90th=[  359], 99.95th=[  401],
     | 99.99th=[  489]
   bw (  KiB/s): min=    8, max=  360, per=100.00%, avg=200.25, stdev=121.10, samples=599
   iops        : min=    2, max=   90, avg=50.06, stdev=30.27, samples=599
  lat (msec)   : 10=0.06%, 20=84.72%, 50=8.45%, 100=3.94%, 250=2.35%
  lat (msec)   : 500=0.47%, 750=0.01%
  cpu          : usr=0.07%, sys=0.52%, ctx=30058, majf=0, minf=13
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=0,14999,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
  WRITE: bw=200KiB/s (205kB/s), 200KiB/s-200KiB/s (205kB/s-205kB/s), io=58.6MiB (61.4MB), run=300014-300014msec
Sat 24 Jun 2023 11:51:07 AM CEST:Sequential read
test: (g=0): rw=read, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=psync, iodepth=4
fio-3.25
Starting 1 process
test: Laying out IO file (1 file / 10240MiB)
Jobs: 1 (f=1): [R(1)][100.0%][r=52.3MiB/s][r=13.4k IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=3560097: Sat Jun 24 11:58:14 2023
  read: IOPS=12.7k, BW=49.5MiB/s (51.9MB/s)(10.0GiB/207029msec)
    clat (nsec): min=1469, max=522169k, avg=78740.22, stdev=2978143.72
     lat (nsec): min=1489, max=522169k, avg=78769.62, stdev=2978143.68
    clat percentiles (nsec):
     |  1.00th=[     1512],  5.00th=[     1528], 10.00th=[     1544],
     | 20.00th=[     1544], 30.00th=[     1560], 40.00th=[     1560],
     | 50.00th=[     1576], 60.00th=[     1592], 70.00th=[     1608],
     | 80.00th=[     1976], 90.00th=[     2320], 95.00th=[     2576],
     | 99.00th=[    35584], 99.50th=[    43264], 99.90th=[ 11730944],
     | 99.95th=[ 39583744], 99.99th=[152043520]
   bw (  KiB/s): min=  768, max=164096, per=100.00%, avg=50673.74, stdev=32090.28, samples=413
   iops        : min=  192, max=41024, avg=12668.44, stdev=8022.57, samples=413
  lat (usec)   : 2=80.47%, 4=15.99%, 10=0.30%, 20=0.13%, 50=2.73%
  lat (usec)   : 100=0.11%, 250=0.02%, 500=0.01%, 750=0.01%, 1000=0.01%
  lat (msec)   : 2=0.01%, 4=0.02%, 10=0.09%, 20=0.05%, 50=0.03%
  lat (msec)   : 100=0.02%, 250=0.02%, 500=0.01%, 750=0.01%
  cpu          : usr=0.52%, sys=3.23%, ctx=7673, majf=0, minf=16
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=2621440,0,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
   READ: bw=49.5MiB/s (51.9MB/s), 49.5MiB/s-49.5MiB/s (51.9MB/s-51.9MB/s), io=10.0GiB (10.7GB), run=207029-207029msec
Sat 24 Jun 2023 11:58:14 AM CEST:Sequential write
test: (g=0): rw=write, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=psync, iodepth=4
fio-3.25
Starting 1 process
test: Laying out IO file (1 file / 10240MiB)
Jobs: 1 (f=1): [W(1)][100.0%][w=320KiB/s][w=80 IOPS][eta 00m:00s]
test: (groupid=0, jobs=1): err= 0: pid=3595926: Sat Jun 24 12:03:15 2023
  write: IOPS=44, BW=179KiB/s (183kB/s)(52.4MiB/300011msec); 0 zone resets
    clat (msec): min=8, max=472, avg=22.38, stdev=35.54
     lat (msec): min=8, max=472, avg=22.38, stdev=35.54
    clat percentiles (msec):
     |  1.00th=[   11],  5.00th=[   12], 10.00th=[   12], 20.00th=[   12],
     | 30.00th=[   12], 40.00th=[   12], 50.00th=[   12], 60.00th=[   12],
     | 70.00th=[   12], 80.00th=[   23], 90.00th=[   45], 95.00th=[   69],
     | 99.00th=[  220], 99.50th=[  271], 99.90th=[  347], 99.95th=[  388],
     | 99.99th=[  456]
   bw (  KiB/s): min=    8, max=  360, per=99.62%, avg=178.58, stdev=121.00, samples=599
   iops        : min=    2, max=   90, avg=44.64, stdev=30.25, samples=599
  lat (msec)   : 10=0.01%, 20=79.40%, 50=11.51%, 100=6.15%, 250=2.33%
  lat (msec)   : 500=0.60%
  cpu          : usr=0.04%, sys=0.37%, ctx=26841, majf=0, minf=12
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=0,13402,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=4

Run status group 0 (all jobs):
  WRITE: bw=179KiB/s (183kB/s), 179KiB/s-179KiB/s (183kB/s-183kB/s), io=52.4MiB (54.9MB), run=300011-300011msec