root@R2:/root # uname -a
FreeBSD R2 9.1-RC1 FreeBSD 9.1-RC1 #0: Tue Aug 14 04:25:06 UTC 2012 root@farrell.cse.buffalo.edu:/usr/obj/usr/src/sys/GENERIC amd64
Four 3 TB Western digital RED's
root@R2:/root # diskinfo -t /dev/ada0
/dev/ada0
512 # sectorsize
3000592982016 # mediasize in bytes (2.7T)
5860533168 # mediasize in sectors
4096 # stripesize
0 # stripeoffset
5814021 # Cylinders according to firmware.
16 # Heads according to firmware.
63 # Sectors according to firmware.
WD-WMC1T0125395 # Disk ident.
Seek times:
Full stroke: 250 iter in 8.721419 sec = 34.886 msec
Half stroke: 250 iter in 6.793017 sec = 27.172 msec
Quarter stroke: 500 iter in 9.502784 sec = 19.006 msec
Short forward: 400 iter in 3.173772 sec = 7.934 msec
Short backward: 400 iter in 1.703673 sec = 4.259 msec
Seq outer: 2048 iter in 0.147995 sec = 0.072 msec
Seq inner: 2048 iter in 0.136383 sec = 0.067 msec
Transfer rates:
outside: 102400 kbytes in 0.748601 sec = 136788 kbytes/sec
middle: 102400 kbytes in 0.805569 sec = 127115 kbytes/sec
inside: 102400 kbytes in 1.406525 sec = 72804 kbytes/sec
root@R2:/root # diskinfo -t /dev/ada1
/dev/ada1
512 # sectorsize
3000592982016 # mediasize in bytes (2.7T)
5860533168 # mediasize in sectors
4096 # stripesize
0 # stripeoffset
5814021 # Cylinders according to firmware.
16 # Heads according to firmware.
63 # Sectors according to firmware.
WD-WMC1T0122050 # Disk ident.
Seek times:
Full stroke: 250 iter in 7.977339 sec = 31.909 msec
Half stroke: 250 iter in 6.962494 sec = 27.850 msec
Quarter stroke: 500 iter in 9.440825 sec = 18.882 msec
Short forward: 400 iter in 1.954362 sec = 4.886 msec
Short backward: 400 iter in 2.524959 sec = 6.312 msec
Seq outer: 2048 iter in 0.157460 sec = 0.077 msec
Seq inner: 2048 iter in 0.130703 sec = 0.064 msec
Transfer rates:
outside: 102400 kbytes in 0.763129 sec = 134184 kbytes/sec
middle: 102400 kbytes in 0.893443 sec = 114613 kbytes/sec
inside: 102400 kbytes in 1.564621 sec = 65447 kbytes/sec
root@R2:/root # diskinfo -t /dev/ada2
/dev/ada2
512 # sectorsize
3000592982016 # mediasize in bytes (2.7T)
5860533168 # mediasize in sectors
4096 # stripesize
0 # stripeoffset
5814021 # Cylinders according to firmware.
16 # Heads according to firmware.
63 # Sectors according to firmware.
WD-WMC1T0124803 # Disk ident.
Seek times:
Full stroke: 250 iter in 8.655306 sec = 34.621 msec
Half stroke: 250 iter in 6.805282 sec = 27.221 msec
Quarter stroke: 500 iter in 9.381524 sec = 18.763 msec
Short forward: 400 iter in 1.670955 sec = 4.177 msec
Short backward: 400 iter in 2.782484 sec = 6.956 msec
Seq outer: 2048 iter in 0.147525 sec = 0.072 msec
Seq inner: 2048 iter in 0.133447 sec = 0.065 msec
Transfer rates:
outside: 102400 kbytes in 0.776709 sec = 131838 kbytes/sec
middle: 102400 kbytes in 0.888563 sec = 115242 kbytes/sec
inside: 102400 kbytes in 1.564208 sec = 65464 kbytes/sec
root@R2:/root # diskinfo -t /dev/ada3
/dev/ada3
512 # sectorsize
3000592982016 # mediasize in bytes (2.7T)
5860533168 # mediasize in sectors
4096 # stripesize
0 # stripeoffset
5814021 # Cylinders according to firmware.
16 # Heads according to firmware.
63 # Sectors according to firmware.
WD-WMC1T0091404 # Disk ident.
Seek times:
Full stroke: 250 iter in 8.647653 sec = 34.591 msec
Half stroke: 250 iter in 6.700303 sec = 26.801 msec
Quarter stroke: 500 iter in 9.385778 sec = 18.772 msec
Short forward: 400 iter in 2.224320 sec = 5.561 msec
Short backward: 400 iter in 2.846229 sec = 7.116 msec
Seq outer: 2048 iter in 0.151230 sec = 0.074 msec
Seq inner: 2048 iter in 0.130517 sec = 0.064 msec
Transfer rates:
outside: 102400 kbytes in 0.693033 sec = 147756 kbytes/sec
middle: 102400 kbytes in 0.834100 sec = 122767 kbytes/sec
inside: 102400 kbytes in 1.595687 sec = 64173 kbytes/sec
Kingston 32G usb3 flash memory (OS disk) Unfortunately limited to usb2 because EFI doesn't recognize it on usb3 bus
root@R2:/root # diskinfo -t /dev/da0
/dev/da0
512 # sectorsize
31440502784 # mediasize in bytes (29G)
61407232 # mediasize in sectors
0 # stripesize
0 # stripeoffset
3822 # Cylinders according to firmware.
255 # Heads according to firmware.
63 # Sectors according to firmware.
00018F3D97D02AC60500000B0 # Disk ident.
Seek times:
Full stroke: 250 iter in 0.154182 sec = 0.617 msec
Half stroke: 250 iter in 0.151905 sec = 0.608 msec
Quarter stroke: 500 iter in 0.292242 sec = 0.584 msec
Short forward: 400 iter in 0.150083 sec = 0.375 msec
Short backward: 400 iter in 0.150300 sec = 0.376 msec
Seq outer: 2048 iter in 0.767999 sec = 0.375 msec
Seq inner: 2048 iter in 0.768323 sec = 0.375 msec
Transfer rates:
outside: 102400 kbytes in 2.862591 sec = 35772 kbytes/sec
middle: 102400 kbytes in 2.833482 sec = 36139 kbytes/sec
inside: 102400 kbytes in 2.807747 sec = 36471 kbytes/sec
root@R2:/ # newfs -U /dev/ada0
/dev/ada0: 2861588.5MB (5860533168 sectors) block size 32768, fragment size 4096
using 4571 cylinder groups of 626.09MB, 20035 blks, 80256 inodes.
with soft updates
super-block backups (for fsck_ffs -b #) at:
192, 1282432, 2564672, 3846912, 5129152, 6411392, 7693632, 8975872, 10258112,
SNAP
root@R2:/mnt/tmp # bonnie++ -u root
Version 1.96 ------Sequential Output------ --Sequential Input- --Random-
Concurrency 1 -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks--
Machine Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP /sec %CP
R2 16G 752 99 144591 13 65424 8 1540 99 147107 10 207.4 2
Latency 11371us 114ms 2110ms 17032us 104ms 422ms
Version 1.96 ------Sequential Create------ --------Random Create--------
R2 -Create-- --Read--- -Delete-- -Create-- --Read--- -Delete--
files /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP
16 +++++ +++ +++++ +++ +++++ +++ 16164 18 +++++ +++ +++++ +++
Latency 78982us 24us 24us 738ms 26us 113ms
root@R2:/mnt # zpool create test /dev/ada0
root@R2:/test # bonnie++ -u root
Version 1.96 ------Sequential Output------ --Sequential Input- --Random-
Concurrency 1 -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks--
Machine Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP /sec %CP
R2 16G 126 99 69729 10 35898 6 335 95 144350 8 151.7 3
Latency 64748us 16739ms 11253ms 210ms 491ms 896ms
Version 1.96 ------Sequential Create------ --------Random Create--------
R2 -Create-- --Read--- -Delete-- -Create-- --Read--- -Delete--
files /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP
16 26777 96 +++++ +++ 25068 96 23928 96 +++++ +++ 24780 97
Latency 14865us 113us 331us 30074us 36us 74us
Why so low write? Perhaps sectors are misaligned, because this is emulated 512b sector drive.
root@R2:/mnt # zdb test | grep ashift
ashift: 9
Yep. Lets try to fix that:
root@R2:/mnt # zpool destroy test
root@R2:/mnt # gnop create -S 4096 /dev/ada0
root@R2:/mnt # zpool create test /dev/ada0.nop
root@R2:/mnt # zdb test | grep ashift
ashift: 12
root@R2:/test # bonnie++ -u root
Version 1.96 ------Sequential Output------ --Sequential Input- --Random-
Concurrency 1 -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks--
Machine Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP /sec %CP
R2 16G 126 99 104425 16 56852 10 322 93 159982 9 173.3 1
Latency 81387us 3230ms 3315ms 565ms 449ms 759ms
Version 1.96 ------Sequential Create------ --------Random Create--------
R2 -Create-- --Read--- -Delete-- -Create-- --Read--- -Delete--
files /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP
16 26735 96 +++++ +++ 25516 97 24358 96 +++++ +++ 25117 97
Latency 14853us 111us 146us 30305us 36us 72us
There I fixed it. Interestingly read speed is much larger now, probably ARC doing it's job. Note: you can delete the gnop drive and re-import the pool after. Only one gnop drive is sufficient for vdev to set ashift=12:
root@R2:/root # zpool create test raidz /dev/ada0.nop /dev/ada1 /dev/ada2 /dev/ada3
root@R2:/root # zdb test | grep ashift
ashift: 12
Version 1.96 ------Sequential Output------ --Sequential Input- --Random-
Concurrency 1 -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks--
Machine Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP /sec %CP
R2 16G 132 99 273438 49 92328 17 335 97 233657 15 200.6 2
Latency 64409us 230ms 1825ms 132ms 567ms 606ms
Version 1.96 ------Sequential Create------ --------Random Create--------
R2 -Create-- --Read--- -Delete-- -Create-- --Read--- -Delete--
files /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP
16 27481 97 +++++ +++ 25592 98 25907 92 +++++ +++ 25503 97
Latency 14829us 110us 150us 18764us 37us 80us
Version 1.96 ------Sequential Output------ --Sequential Input- --Random-
Concurrency 1 -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks--
Machine Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP /sec %CP
R2 16G 123 99 173276 32 87668 17 317 93 218519 15 231.3 3
Latency 68048us 735ms 2022ms 354ms 290ms 558ms
Version 1.96 ------Sequential Create------ --------Random Create--------
R2 -Create-- --Read--- -Delete-- -Create-- --Read--- -Delete--
files /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP
16 26819 96 +++++ +++ 25468 97 25150 90 +++++ +++ 24682 94
Latency 14884us 115us 147us 19351us 39us 79us
root@R2:/root # zpool create test mirror /dev/ada0.nop /dev/ada1 mirror /dev/ada2.nop /dev/ada3
Version 1.96 ------Sequential Output------ --Sequential Input- --Random-
Concurrency 1 -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks--
Machine Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP /sec %CP
R2 16G 128 99 202158 33 124471 22 340 98 359587 21 313.2 7
Latency 65528us 95026us 1376ms 108ms 147ms 572ms
Version 1.96 ------Sequential Create------ --------Random Create--------
R2 -Create-- --Read--- -Delete-- -Create-- --Read--- -Delete--
files /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP
16 27317 95 +++++ +++ 25917 98 25051 97 +++++ +++ 25705 96
Latency 14853us 108us 137us 29771us 305us 82us