ZFS on Linux with LUKS encrypted disks
$ lsblk -io KNAME,TYPE,SIZE,MODEL
symbol:~# lsblk -io KNAME,TYPE,SIZE,MODEL
KNAME TYPE SIZE MODEL
sda disk 3.7T ST...
sdb disk 3.7T WDC WD40...
dm-1 crypt 3.7T
dm-2 lvm 3.7T
sdc disk 3.7T WDC WD....
sdd disk 931.5G WDC WD....-0
dm-5 crypt 931.5G
sde disk 1.8T WDC WD....
dm-4 crypt 1.8T
sdf disk 14.7G DT Micro
sdf1 part 478M
sdf2 part 14.2G
dm-0 crypt 14.2G$ cat /sys/class/block/sda/queue/physical_block_size
4096
$ cat /sys/class/block/sdb/queue/physical_block_size
4096
$ cat /sys/class/block/sdc/queue/physical_block_size
4096cryptsetup luksFormat --cipher aes-xts-plain64 --key-size 512 --iter-time=10000 --use-random /dev/sda
cryptsetup luksFormat --cipher aes-xts-plain64 --key-size 512 --iter-time=10000 --use-random /dev/sdb
cryptsetup luksFormat --cipher aes-xts-plain64 --key-size 512 --iter-time=10000 --use-random /dev/sdc
cryptsetup luksFormat --cipher aes-xts-plain64 --key-size 512 --iter-time=10000 --use-random /dev/sdf5 # ssd
zpool create -f -o ashift=12 -m /mnt/storage storage raidz /dev/mapper/crypt-{a,b,c}
zfs set acltype=posixacl storage
zfs set xattr=sa storage
zfs set compression=on storage
zfs get all
After reinstall of operating system
zpool import storage
Rearrange mountpoint
zfs set mountpoint=/srv/storage storage
Add cache
zpool add storage cache /dev/mapper/crypt-cache-a
replace disk in pool
cryptsetup luksFormat --cipher aes-xts-plain64 --key-size 512 --iter-time=10000 --use-random /dev/sdd
# wrong
# zpool add -o ashift=12 storage spare /dev/mapper/crypt-d
$ zpool replace storage crypt-a crypt-d
cannot open '/dev/mapper/crypt-d': Device or resource busy
^C
root@jalapeno:~# zp
zpios zpool
root@jalapeno:~# zpool status
pool: storage
state: ONLINE
status: One or more devices is currently being resilvered. The pool will
continue to function, possibly in a degraded state.
action: Wait for the resilver to complete.
scan: resilver in progress since Tue Jun 6 21:47:13 2017
665M scanned out of 2.78T at 83.1M/s, 9h45m to go
220M resilvered, 0.02% done
config:
NAME STATE READ WRITE CKSUM
storage ONLINE 0 0 0
raidz1-0 ONLINE 0 0 0
spare-0 ONLINE 0 0 0
crypt-a ONLINE 0 0 0
crypt-d ONLINE 0 0 0 (resilvering)
crypt-b ONLINE 0 0 0
crypt-c ONLINE 0 0 0
cache
crypt-cache-a ONLINE 0 0 0
spares
crypt-d INUSE currently in use
$ zpool detach storage crypt-d
$ zpool status
pool: storage
state: ONLINE
status: One or more devices is currently being resilvered. The pool will
continue to function, possibly in a degraded state.
action: Wait for the resilver to complete.
scan: resilver in progress since Tue Jun 6 21:47:13 2017
87.6G scanned out of 2.78T at 557M/s, 1h24m to go
11.0G resilvered, 3.07% done
config:
NAME STATE READ WRITE CKSUM
storage ONLINE 0 0 0
raidz1-0 ONLINE 0 0 0
crypt-a ONLINE 0 0 0
crypt-b ONLINE 0 0 0
crypt-c ONLINE 0 0 0
cache
crypt-cache-a ONLINE 0 0 0
spares
crypt-d AVAIL
errors: No known data errors
$ zpool detach storage crypt-d
$ zpool status
pool: storage
state: ONLINE
status: One or more devices is currently being resilvered. The pool will
continue to function, possibly in a degraded state.
action: Wait for the resilver to complete.
scan: resilver in progress since Tue Jun 6 21:47:13 2017
87.6G scanned out of 2.78T at 557M/s, 1h24m to go
11.0G resilvered, 3.07% done
config:
NAME STATE READ WRITE CKSUM
storage ONLINE 0 0 0
raidz1-0 ONLINE 0 0 0
crypt-a ONLINE 0 0 0
crypt-b ONLINE 0 0 0
crypt-c ONLINE 0 0 0
cache
crypt-cache-a ONLINE 0 0 0
spares
crypt-d AVAIL
errors: No known data errors