zfs encryption speed test

zfs-native-encryption-test.sh

#!/bin/bash

# From https://www.medo64.com/2022/10/native-zfs-encryption-speed-ubuntu-22-10/
# https://docs.google.com/spreadsheets/d/1spTTlEJESnVFSEsB98iVviA-l_unf6Hemx9Nh8KrBBI/edit#gid=1839759404
#
# Tweaks for faster test: only do 2 tests, only sleep 2 seconds, only use aes-256-gcm for zfs
# Tweaks to LUKS config
# don't use sync=always
# use only 1 vdev

ANSI_RESET="$(tput sgr0)"
ANSI_RED="`[ $(tput colors) -ge 16 ] && tput setaf 9 || tput setaf 1 bold`"
ANSI_CYAN="`[ $(tput colors) -ge 16 ] && tput setaf 14 || tput setaf 6 bold`"

ntests=1
tsleep=2

# Checks

if [ "$EUID" -ne 0 ]; then
    echo "${ANSI_RED}Must run as root!${ANSI_RESET}"
    exit 1
fi

if [[ -e "/ramdisk" ]]; then
    echo "${ANSI_RED}/ramdisk already exists!${ANSI_RESET}"
    exit 1
fi


# Setup RAM disk

mkdir /ramdisk
mount -t tmpfs -o size=42G tmpfs /ramdisk
echo "1073741824" > /sys/module/zfs/parameters/zfs_arc_max


# Create test data
swapoff --all
dd if=/dev/urandom of=/ramdisk/data.bin bs=1M count=4096 &> /dev/null || exit


# Test raw

echo "${ANSI_CYAN}raw${ANSI_RESET}"
for ((I=1; I<=ntests; I++)); do
    sleep ${tsleep}
    echo -n "  write $I: "
    dd if=/ramdisk/data.bin of=/ramdisk/raw$I.bin bs=1M |& grep "copied" || exit
done

for ((I=1; I<=ntests; I++)); do
    sleep ${tsleep}
    echo -n "  read $I: "
    dd if=/ramdisk/raw$I.bin of=/dev/null bs=1M |& grep "copied" || exit
done

rm /ramdisk/raw*.bin


# Test ZFS

for ENCRYPTION in \
                  "none" \
                  "aes-256-gcm" \
                  "luks" \
                  ; do

    rm /ramdisk/disk*.bin 2>/dev/null
    dd if=/dev/zero of=/ramdisk/disk1.bin bs=1MB count=6144 &> /dev/null || exit

    echo "${ANSI_CYAN}$ENCRYPTION${ANSI_RESET}"
    if [[ "$ENCRYPTION" == "none" ]]; then
        sudo zpool create -o ashift=12 -O normalization=formD \
            -O acltype=posixacl -O xattr=sa -O dnodesize=auto -O atime=off \
            -O compression=off -O mountpoint=/zfs TestPool /ramdisk/disk1.bin  || exit
    elif [[ "$ENCRYPTION" == "luks" ]]; then
        losetup -f /ramdisk/disk1.bin || exit
        DEVS=""
        DISKS=`losetup -a | grep "/ramdisk/" | cut -d: -f1`
        for DISK in $DISKS; do
            echo "12345678" | cryptsetup luksFormat -q --cipher aes-xts-plain64 --key-size 512 --use-random --sector-size 4096 --pbkdf argon2id $DISK
            echo "12345678" | cryptsetup luksOpen $DISK --allow-discards --perf-no_read_workqueue --perf-no_write_workqueue `basename $DISK`
            DEVS="$DEVS /dev/mapper/`basename $DISK` "
        done
        zpool create -o ashift=12 -O normalization=formD \
            -O acltype=posixacl -O xattr=sa -O dnodesize=auto -O atime=off \
            -O compression=off -O mountpoint=/zfs TestPool $DEVS || exit
    else
        echo "12345678" | zpool create -o ashift=12 -O normalization=formD \
            -O acltype=posixacl -O xattr=sa -O dnodesize=auto -O atime=off \
            -O encryption=$ENCRYPTION -O keylocation=prompt -O keyformat=passphrase \
            -O compression=off -O mountpoint=/zfs TestPool /ramdisk/disk1.bin || exit
    fi

    for ((I=1; I<=ntests; I++)); do
        sleep ${tsleep}
        echo -n "  Write $I: "
        dd if=/ramdisk/data.bin of=/zfs/data$I.bin bs=1M |& grep "copied" || exit
    done

    for ((I=1; I<=ntests; I++)); do
        sleep ${tsleep}
        echo -n "  read $I: "
        dd if=/zfs/data$I.bin of=/dev/null bs=1M |& grep "copied" || exit
    done

    zpool destroy TestPool || exit

done

adding back sync=always, same test in VM

raw
  write 1: 4294967296 bytes (4.3 GB, 4.0 GiB) copied, 2.09128 s, 2.1 GB/s
  read 1: 4294967296 bytes (4.3 GB, 4.0 GiB) copied, 0.58907 s, 7.3 GB/s
none
  Write 1: 4294967296 bytes (4.3 GB, 4.0 GiB) copied, 2.49589 s, 1.7 GB/s
  read 1: 4294967296 bytes (4.3 GB, 4.0 GiB) copied, 0.670201 s, 6.4 GB/s
aes-256-gcm
  Write 1: 4294967296 bytes (4.3 GB, 4.0 GiB) copied, 2.87812 s, 1.5 GB/s
  read 1: 4294967296 bytes (4.3 GB, 4.0 GiB) copied, 2.5671 s, 1.7 GB/s
luks
  Write 1: 4294967296 bytes (4.3 GB, 4.0 GiB) copied, 3.38855 s, 1.3 GB/s
  read 1: 4294967296 bytes (4.3 GB, 4.0 GiB) copied, 0.679988 s, 6.3 GB/s

So raidz2 with 6 disks vs no raid 1 disk, the latter consistently shows ZFS with no encryption or LUKS + ZFS with no encryption get 6GB/sec reads