A small HOWTO about installing Ubuntu 16.04 LTS on ZFS

xenial-on-zfs.sh

# Boot Ubuntu 16.04 LTS from a ZFS rpool
#
# 
# We boot the remote server to its (Debian or Ubuntu based) rescue mode
# then ssh into it with forwarded ports for the VM's SSH and VNC servers
#
# Shell here documents that are not passed to a command are used for
# documentation and copy and paste sources.
# "<<'LOCAL'" denotes commands to be run on the local machine, 'RESCUE'
# and 'VM' should be run in the rescue system or the VM, respectively.
#
# This is for a single-disk pool on /dev/sda, rpools on raidz{1,2,3} or
# mirror vdevs are known to work, too. Just pass through the additional
# disks to the VM and 

<<'LOCAL'
ssh \
  -L 5900:localhost:5900 \
  -L 5901:localhost:5901 \
  -o "UserKnownHostsFile /dev/null" \
  -o "StrictHostKeyChecking no" \
  root@91.121.135.62
LOCAL

<<'RESCUE'
# Get Xubuntu iso
wget http://cdimage.ubuntu.com/xubuntu/releases/16.04/release/xubuntu-16.04-desktop-amd64.iso

# Start a VM, boot the Xubuntu iso and pass through our local disks
# TODO: Redirect the disks you want to use, here: /dev/sda
qemu-system-x86_64 \
  -cdrom xubuntu-16.04-desktop-amd64.iso \
  -boot d \
  -drive file=/dev/sda,media=disk \
  -nographic \
  -vnc 127.0.0.1:0 \
  -m 12G \
  -cpu host \
  -smp 4 \
  -enable-kvm \
  -redir tcp:5901::22

# For testing the installation it's possible to run qemu without the iso
# attached, to start the VM from qemu's console: Ctrl+a c, "cont"
# TODO: same as above, you may want to add more disks
qemu-system-x86_64 \
  -drive file=/dev/sda,media=disk \
  -nographic \
  -vnc 127.0.0.1:0 \
  -m 12G \
  -cpu host \
  -smp 4 \
  -enable-kvm \
  -redir tcp:5901::22 \
  -S
RESCUE

<<'VM'
# Add password to 'xubuntu' user
echo "xubuntu:xubuntu" | chpasswd

# Install openssh-server
apt-get --quiet update
apt-get --yes --quiet install openssh-server
VM

<<'LOCAL'
# copy file over
scp \
  -P 5901 \
  -o "UserKnownHostsFile /dev/null" \
  -o "StrictHostKeyChecking no" \
  xenial-on-zfs.sh \
  xubuntu@localhost:
LOCAL

<<'VM'
# now run this script on the virtual machine
sudo bash -x /home/xubuntu/xenial-on-zfs.sh
VM

# Disable automatic installation of recommended or suggested packages
cat > /etc/apt/apt.conf.d/01norecommend <<EOF
APT::Install-Recommends "0";
APT::Install-Suggests "0";
EOF

# Zap our disks' GPT and MBR structures so we don't accidentally
# import an outdated pool during installation of zfsutils-linux
# TODO: 
for disk in /dev/sda
do
  sgdisk -Z "$disk"
done

# Get required tools, i.e. debootstrap and ZFS userspace tools
apt-get --quiet update
apt-get --yes --quiet install debootstrap zfsutils-linux

# We follow partman-auto's partition size for ESP:
# https://bugs.launchpad.net/curtin/+bug/1306164
#
# Our disks will get partitioned in this way:
# (note the out-of-order partition numbering)
# 
# Protective MBR
#   LBA 0
# Primary GPT
#   LBA 1 → 33
# Partition 2
#   LBA 34 → 2047
#   BIOS boot partition, used for grub
# Partition 3
#   1 MiB → 539 MiB
#   EFI system partition, unused as of now
# Partition 1
#   539 MiB → -9 MiB
#   ZFS pool
# Partition 9
#   -9 MiB → -1 MiB (= LBA -34)
#   Reserved partition, here used for ZFS label padding
# Secondary GPT
#   LBA -33 → -1
#
# We wipe stale, overlapping ZFS labels from partition 1 if they exist

for disk in /dev/sda
do
  sgdisk -a1  -n2:34:2047  -t2:EF02 -c2:grub \
         "$disk"
  sgdisk      -n3:1M:+538M -t3:EF00 -c3:EFI \
              -n9:-8M:0    -t9:BF07 -c9:reserved \
              -n1:0:0      -t1:BF01 -c1:zfs \
         "$disk"
  zpool labelclear -f "$disk"1
done

# I am using ashift=9 as my sample server uses an HUS724020ALA640:
#  HUS724020ALE640 = 2TB, SATA 6Gb/s, 512n, 64MB buffer
#  H = HGST
#  U = Ultrastar
#  S = Standard
#  72 = 7200 RPM
#  40 = Full capacity, 4TB
#  20 = This model's Capacity, 40 = 4TB, 30 = 3TB, 20 = 2TB
#  A = Generation code
#  L = 26.1mm z-height
#  A6 = Interface, SATA 6Gb/s, 512n (E6 = SATA 512e, S6 = SAS 512n)
#  4 = 64MB buffer
#  0 = No encryption (1 = encryption)
#
# TODO: If using more than one disk, replace "/dev/sda1" with your own
# pool layout, e.g. "mirror /dev/sda1 /dev/sdb1"

zpool create \
  -o ashift=9 \
  -O normalization=formD \
  -O atime=off \
  -O canmount=off \
  -O compression=lz4 \
  -O mountpoint=/ \
  -O recordsize=1M \
  -R /mnt \
  rpool \
  /dev/sda1

zfs create -o canmount=off -o mountpoint=none rpool/ROOT
zfs create -o canmount=noauto -o mountpoint=/ rpool/ROOT/xenial

# Turn synchronous requests into async ones for faster installation
zfs set sync=disabled rpool

zfs mount rpool/ROOT/xenial

# TODO: Replace with your closest mirror
debootstrap xenial /mnt http://de.archive.ubuntu.com/ubuntu

zfs snap rpool/ROOT/xenial@inst-debootstrap

# TODO: Replace with real host name
echo 'xenial' > /mnt/etc/hostname
sed -i '1 a 127.0.1.1       xenial' /mnt/etc/hosts 

# fix "predictable" device naming, see
# https://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/
ln -s /dev/null /mnt/etc/udev/rules.d/80-net-setup-link.rules

# TODO: This is my network configuration, use your own!
cat > /mnt/etc/network/interfaces.d/eth0 <<'EOF'
auto eth0
iface eth0 inet static
  address 91.121.135.62
  netmask 255.255.255.0
  gateway 91.121.135.254
  dns-nameservers 8.8.8.8 8.8.4.4

iface eth0 inet6 static
  address 2001:41d0:1:b63e::1
  netmask 64
  gateway 2001:41d0:1:b6ff:ff:ff:ff:ff
  dns-nameservers 2001:4860:4860::8888 2001:4860:4860::8844
EOF

# Install a sane sources.list
cat > /mnt/etc/apt/sources.list <<'EOF'
deb http://de.archive.ubuntu.com/ubuntu xenial           main restricted universe multiverse
deb http://de.archive.ubuntu.com/ubuntu xenial-backports main restricted universe multiverse
deb http://de.archive.ubuntu.com/ubuntu xenial-updates   main restricted universe multiverse
deb http://de.archive.ubuntu.com/ubuntu xenial-security  main restricted universe multiverse
EOF

# Disable automatic installation of recommended or suggested packages
cat > /mnt/etc/apt/apt.conf.d/01norecommend <<'EOF'
APT::Install-Recommends "0";
APT::Install-Suggests "0";
EOF

# This is a remote system, so we disable plymouth, enable quiet boot
# and do not wait for user input
cat > /mnt/etc/default/grub <<'EOF'
# If you change this file, run 'update-grub' afterwards to update
# /boot/grub/grub.cfg.
# For full documentation of the options in this file, see:
#   info -f grub -n 'Simple configuration'

GRUB_DEFAULT=0
#GRUB_HIDDEN_TIMEOUT=0
GRUB_HIDDEN_TIMEOUT_QUIET=true
GRUB_TIMEOUT=0
GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian`
GRUB_CMDLINE_LINUX_DEFAULT="noplymouth quiet"
GRUB_CMDLINE_LINUX=""

# Uncomment to enable BadRAM filtering, modify to suit your needs
# This works with Linux (no patch required) and with any kernel that obtains
# the memory map information from GRUB (GNU Mach, kernel of FreeBSD ...)
#GRUB_BADRAM="0x01234567,0xfefefefe,0x89abcdef,0xefefefef"

# Uncomment to disable graphical terminal (grub-pc only)
GRUB_TERMINAL=console

# The resolution used on graphical terminal
# note that you can use only modes which your graphic card supports via VBE
# you can see them in real GRUB with the command `vbeinfo'
#GRUB_GFXMODE=640x480

# Uncomment if you don't want GRUB to pass "root=UUID=xxx" parameter to Linux
#GRUB_DISABLE_LINUX_UUID=true

# Uncomment to disable generation of recovery mode menu entries
#GRUB_DISABLE_RECOVERY="true"

# Uncomment to get a beep at grub start
#GRUB_INIT_TUNE="480 440 1"
EOF

echo 'LANG="en_US.UTF-8"' > /mnt/etc/default/locale

# Fix the chroot's mtab
ln -s /proc/mounts /mnt/etc/mtab

mount -t devtmpfs /dev /mnt/dev
mount -t devpts /dev/pts /mnt/dev/pts 
mount -t sysfs /sys /mnt/sys
mount -t proc /proc /mnt/proc 
mount -t tmpfs /run /mnt/run

# Create temporary resolv.conf by manually setting it up
# TODO: feel free to use a different name server, this is Google's
mkdir /mnt/run/resolvconf/
echo "nameserver 8.8.8.8" > /mnt/run/resolvconf/resolv.conf

# Finish installation in a chroot
chroot /mnt /bin/bash -l -- <<'EOF'
# Enable bash's debug mode, to show executed commands
set -x

# TODO: replace with your locale needs
for locale in en_US.UTF-8 de_DE.UTF-8; do locale-gen "$locale"; done

# TODO: edit timezone
# This is a workaround for https://bugs.launchpad.net/ubuntu/+source/tzdata/+bug/1554806
ln -fs /usr/share/zoneinfo/Europe/Berlin /etc/localtime
dpkg-reconfigure -f noninteractive tzdata

# Update package cache and upgrade everything
apt-get --quiet update
apt-get --yes --quiet dist-upgrade

# Install a minimal but sane set of packages
# TODO: edit according your needs
apt-get --yes --quiet install \
  linux-image-generic \
  zfsutils-linux \
  zfs-initramfs \
  openssh-server \
  ssh-import-id \
  tmux \
  aptitude \
  vim

# Install grub, force usage of our custom /etc/default/grub
DEBIAN_FRONTEND=noninteractive apt-get --yes --quiet \
  -o Dpkg::Options::="--force-confdef" \
  -o Dpkg::Options::="--force-confold" \
  install grub-pc

# Actually install grub to disk
# TODO: 
for disk in /dev/sda
  do grub-install "$disk"
done

update-grub

# Add local user with appropriate groups, import SSH key from Launchpad
# TODO: Edit username and password to your own
adduser --disabled-password --gecos "Hajo Moeller,,," dasjoe
usermod -a -G adm,sudo dasjoe
echo "dasjoe:dasjoe" | chpasswd
sudo -Hu dasjoe ssh-import-id dasjoe

# Disable password login via SSH
sed -i 's/^#\{0,1\}\(PasswordAuthentication\) .*/\1 no/g' /etc/ssh/sshd_config

exit
EOF

# Prepare zpool for export, unmount /run, /proc, /sys, /dev/pts and /dev
umount /mnt/run
umount /mnt/proc
umount /mnt/sys
umount /mnt/dev/pts
umount /mnt/dev
umount /mnt

# Reset synchronous behaviour to ZFS defaults
zfs inherit sync rpool
zfs snap rpool/ROOT/xenial@inst-done

zpool export rpool

<<'FIXME'
Stuff left TODO:
  - encryption (+ dropbear?)
  - cpu ondemand
  - EFI
  - grub-mkpasswd-pbkdf2
FIXME