Ubuntu 18.04.1 root-on-ZFS-on-LUKS (HP ProLiant DL380 G7)

01-pre-install.sh

#!/bin/sh

#### Run as user `ubuntu` from the Ubuntu Desktop installer's live environment.

passwd ubuntu
sudo apt install --yes openssh-server
ip addr

02-install.sh

#!/bin/sh

#### Run as user `root` from the Ubuntu Desktop installer's live environment.

apt-add-repository universe
apt update
apt install --yes debootstrap gdisk zfs-initramfs mdadm

## These IDs correspond to the LU, not the underlying physical disk!
disks=$(ls -l /dev/disk/by-id/scsi* \
            | fgrep sd \
            | grep -v part \
            | sed -e 's/^.*\/dev/\/dev/' \
            | awk '{print $3 " " $1}' \
            | sort \
            | awk '{print $2}')
for disk in $disks; do ls -l $disk; done
(for disk in $disks; do ls -l $disk; done) | wc -l

## Explicitly force Linux to re-read the partition table as there
## seems to be a short delay between sgdisk exiting and the entries in
## /dev updating.
for disk in $disks; do
    sgdisk --zap-all                 $disk
    partprobe
    sgdisk -a1 -n2:34:2047  -t2:EF02 $disk
    # sgdisk     -n3:1M:+512M -t3:EF00 $disk
    sgdisk     -n4:0:+512M  -t4:8300 $disk
    sgdisk     -n1:0:0      -t1:8300 $disk
done
partprobe
sleep 60

ctr=1
for disk in $disks; do
    cryptsetup luksFormat -q -y -c aes-xts-plain64 -s 256 -h sha256 $disk-part1
    cryptsetup luksOpen $disk-part1 luks$ctr
    ctr=$(expr $ctr + 1)
done

zpool create -f -o ashift=12 \
      -O atime=off -O canmount=off -O compression=lz4 -O normalization=formD \
      -O xattr=sa -O mountpoint=/ -R /mnt \
      rpool raidz2 /dev/mapper/luks*

zfs create -o canmount=off -o mountpoint=none rpool/ROOT
zfs create -o canmount=noauto -o mountpoint=/ rpool/ROOT/ubuntu
zfs mount rpool/ROOT/ubuntu
zfs create                 -o setuid=off              rpool/home
zfs create -o mountpoint=/root                        rpool/home/root
zfs create -o canmount=off -o setuid=off  -o exec=off rpool/var
zfs create -o com.sun:auto-snapshot=false             rpool/var/cache
zfs create -o acltype=posixacl -o xattr=sa            rpool/var/log
zfs create                                            rpool/var/spool
zfs create -o com.sun:auto-snapshot=false -o exec=on  rpool/var/tmp
zfs create                                            rpool/srv
zfs create                                            rpool/var/games
zfs create                                            rpool/var/mail
zfs create -o exec=on rpool/var/spool/postfix
zfs create -o com.sun:auto-snapshot=false \
           -o mountpoint=/var/lib/nfs                 rpool/var/nfs
zfs create -o com.sun:auto-snapshot=false \
           -o setuid=off                              rpool/tmp
chmod 1777 /mnt/tmp

bootparts=""
for disk in $disks; do bootparts="$bootparts $disk-part4"; done
zpool create -f -o ashift=12 -o feature@large_dnode=disabled \
      -O atime=off -O canmount=off -O compression=lz4 -O normalization=formD \
      -O xattr=sa -O mountpoint=/boot -R /mnt \
      bootpool mirror $bootparts
zfs create -o canmount=off -o mountpoint=none bootpool/BOOT
zfs create -o canmount=noauto -o mountpoint=/boot bootpool/BOOT/ubuntu
zfs mount bootpool/BOOT/ubuntu

### TODO: mirrored /boot/efi

chmod 1777 /mnt/var/tmp
debootstrap bionic /mnt
zfs set devices=off rpool
zfs set devices=off bootpool

echo myhost > /mnt/etc/hostname
echo 127.0.1.1 myhost.example.com myhost >> /mnt/etc/hostname

### TODO: NIC teaming/bonding, static IP

cat > /mnt/etc/netplan/enp3s0f0.yaml <<EOF
network:
  version: 2
  ethernets:
    enp3s0f0:
      dhcp4: true
EOF

cat > /mnt/etc/apt/sources.list <<EOF
deb http://us.archive.ubuntu.com/ubuntu/ bionic main universe restricted multiverse
deb-src http://us.archive.ubuntu.com/ubuntu/ bionic main universe restricted multiverse

deb http://us.archive.ubuntu.com/ubuntu/ bionic-security main universe restricted multiverse
deb-src http://us.archive.ubuntu.com/ubuntu/ bionic-security main universe restricted multiverse

deb http://us.archive.ubuntu.com/ubuntu/ bionic-updates main universe restricted multiverse
deb-src http://us.archive.ubuntu.com/ubuntu/ bionic-updates main universe restricted multiverse
deb http://us.archive.ubuntu.com/ubuntu/ bionic-backports universe main restricted multiverse
deb http://us.archive.ubuntu.com/ubuntu/ bionic-proposed universe multiverse main restricted
EOF

mount --rbind /dev  /mnt/dev
mount --rbind /proc /mnt/proc
mount --rbind /sys  /mnt/sys

03-post-install.sh

#!/bin/sh

#### Run as user `root` from a chroot shell in the installed system.

ln -s /proc/self/mounts /etc/mtab
apt update

## TODO: generate all locales and use en_us.utf-8 as the default
dpkg-reconfigure locales

## TODO: pick America/New_York
dpkg-reconfigure tzdata

apt install --yes --no-install-recommends linux-image-generic
apt install --yes zfs-initramfs

### TODO: add bootpool to ZFS_POOL_IMPORT in /etc/default/zfs

apt install --yes cryptsetup
disks=$(ls -l /dev/disk/by-id/scsi* \
            | fgrep sd \
            | grep -v part \
            | sed -e 's/^.*\/dev/\/dev/' \
            | awk '{print $3 " " $1}' \
            | sort \
            | awk '{print $2}')
for disk in $disks; do ls -l $disk; done
(for disk in $disks; do ls -l $disk; done) | wc -l
ctr=1
for disk in $disks; do
    echo luks$ctr UUID=$(blkid -s UUID -o value $disk-part1) none luks,discard,initramfs >> /etc/crypttab
    ctr=$(expr $ctr + 1)
done
cat /etc/crypttab

## TODO: pick all disks
apt install --yes grub-pc

### TODO: install Grub for EFI

addgroup --system lpadmin
addgroup --system sambashare

## TODO: set root password
passwd

zfs set mountpoint=legacy rpool/var/log
zfs set mountpoint=legacy rpool/var/tmp
zfs set mountpoint=legacy rpool/tmp
cat >> /etc/fstab << EOF
rpool/var/log /var/log zfs noatime,nodev,noexec,nosuid 0 0
rpool/var/tmp /var/tmp zfs noatime,nodev,nosuid 0 0
rpool/tmp /tmp zfs noatime,nodev,nosuid 0 0
EOF

grub-probe /
update-initramfs -u -k all
## TODO: edit /etc/default/grub
for disk in $disks; do grub-install $disk; done
ls /boot/grub/*/zfs.mod

zfs snapshot rpool/ROOT/ubuntu@install

04-firstboot.sh

#!/bin/sh

#### Run as `root` after booting for the first time.

zfs create rpool/home/admin
adduser admin
cp -a /etc/skel/.[!.]* /home/admin
chown -R admin:admin /home/admin
usermod -a -G adm,cdrom,dip,lpadmin,plugdev,sambashare,sudo critical

zfs create -V 4G -b $(getconf PAGESIZE) -o compression=zle \
      -o logbias=throughput -o sync=always \
      -o primarycache=metadata -o secondarycache=none \
      -o com.sun:auto-snapshot=false rpool/swap
mkswap -f /dev/zvol/rpool/swap
echo /dev/zvol/rpool/swap none swap defaults 0 0 >> /etc/fstab
echo RESUME=none > /etc/initramfs-tools/conf.d/resume
swapon -av

apt update
apt dist-upgrade --yes

for file in /etc/logrotate.d/* ; do
    if grep -Eq "(^|[^#y])compress" "$file" ; then
        sed -i -r "s/(^|[^#y])(compress)/\1#\2/" "$file"
    fi
done

apt install --yes gnupg2
wget -O - https://repo.saltstack.com/apt/ubuntu/18.04/amd64/latest/SALTSTACK-GPG-KEY.pub | apt-key add -
cat > /etc/apt/sources.list.d/saltstack.list <<EOF
deb http://repo.saltstack.com/apt/ubuntu/18.04/amd64/latest bionic main
EOF
apt-get update
apt-get install --yes salt-minion

## This installs `ubuntu-standard` or `ubuntu-desktop` as appropriate.
salt-call state.apply

zfs destroy rpool/ROOT/ubuntu@install