FreeBSD ZFS-on-Root (GCP)

mk_image.sh

#!/bin/sh
set -e
set -x

# Adapted from https://www.daemonology.net/blog/2019-02-16-FreeBSD-ZFS-AMIs-now-available.html
# Note: You need to run this on an instance with read/write access to Google Compute
# Alternatively, you can run the gcloud commands from somewhere with privileges

disk=da1

instance_name=$(curl http://metadata.google.internal/computeMetadata/v1/instance/name -H Metadata-Flavor:Google)
zone=$(gcloud compute instances list --filter="name=${instance_name}" --format="value[no-heading](zone)")
family=freebsd-131-zfs-base
image_name=${family}-$(date +"%Y%m%d-%H%M%S")

gcloud compute disks create $image_name --guest-os-features=UEFI_COMPATIBLE --zone=$zone --size=10GB --type=pd-balanced
gcloud compute instances attach-disk $instance_name --disk=$image_name --zone=$zone

# boot
gpart create -s gpt $disk
gpart add -a 4k -s 40M -t efi $disk
newfs_msdos -F 32 -c 1 /dev/${disk}p1
mount -t msdosfs -o longnames /dev/${disk}p1 /mnt
mkdir -p /mnt/EFI/BOOT
cp /boot/loader.efi /mnt/EFI/BOOT/BOOTX64.efi
umount /mnt

# root
gpart add -a 1m -t freebsd-zfs -l disk0 $disk
zpool create -o altroot=/mnt -o autoexpand=on -O compress=lz4 -O atime=off -m none -f zroot ${disk}p2
zfs create -o mountpoint=none zroot/ROOT
zfs create -o mountpoint=/ -o canmount=noauto zroot/ROOT/default
mount -t zfs zroot/ROOT/default /mnt
zpool set bootfs=zroot/ROOT/default zroot

# data
zfs create -o mountpoint=none zroot/DATA
zfs create -o mountpoint=/tmp -o exec=on -o setuid=off zroot/DATA/tmp
zfs create -o mountpoint=/usr -o canmount=off zroot/DATA/usr
zfs create zroot/DATA/usr/home
zfs create -o mountpoint=/var zroot/DATA/var
zfs create -o exec=off -o setuid=off zroot/DATA/var/audit
zfs create -o exec=off -o setuid=off zroot/DATA/var/crash
zfs create -o exec=off -o setuid=off zroot/DATA/var/log
zfs create -o atime=on zroot/DATA/var/mail
zfs create -o setuid=off zroot/DATA/var/tmp
zfs create -o canmount=off zroot/DATA/var/db

# configure
fetch -o /tmp/base.txz https://download.freebsd.org/ftp/releases/amd64/13.1-RELEASE/base.txz
tar -xf /tmp/base.txz -C /mnt

fetch -o /tmp/kernel.txz https://download.freebsd.org/ftp/releases/amd64/13.1-RELEASE/kernel.txz
tar -xf /tmp/kernel.txz -C /mnt

: > /mnt/etc/fstab
echo 'zfs_load="YES"' >> /mnt/boot/loader.conf
echo 'kern.geom.label.disk_ident.enable="0"' >> /mnt/boot/loader.conf
echo 'kern.geom.label.gptid.enable="0"' >> /mnt/boot/loader.conf
echo 'vfs.zfs.min_auto_ashift=12' >> /mnt/etc/sysctl.conf

## copied from default freebsd image
sysrc -f /mnt/etc/rc.conf ntpd_enable="YES"
sysrc -f /mnt/etc/rc.conf sshd_enable="YES"
sysrc -f /mnt/etc/rc.conf growfs_enable="YES"
sysrc -f /mnt/etc/rc.conf google_startup_enable="YES"
sysrc -f /mnt/etc/rc.conf google_accounts_daemon_enable="YES"
sysrc -f /mnt/etc/rc.conf google_clock_skew_daemon_enable="YES"
sysrc -f /mnt/etc/rc.conf google_instance_setup_enable="YES"
sysrc -f /mnt/etc/rc.conf google_network_daemon_enable="YES"
sysrc -f /mnt/etc/rc.conf dumpdev="AUTO"
sysrc -f /mnt/etc/rc.conf ifconfig_DEFAULT="SYNCDHCP mtu 1460"
sysrc -f /mnt/etc/rc.conf ntpd_sync_on_start="YES"

## new for this image
sysrc -f /mnt/etc/rc.conf zfs_enable="YES"

## tailscale
zfs create zroot/DATA/var/db/tailscale
sysrc -f /mnt/etc/rc.conf gateway_enable=yes
sysrc -f /mnt/etc/rc.conf pf_enable=yes
sysrc -f /mnt/etc/rc.conf tailscaled_enable=yes

## pf
cat >/mnt/etc/pf.conf <<EOF
ext_if="vtnet0"

set skip on lo
scrub in

# nat enables networking for jails and tailscale
nat on \$ext_if inet from !(\$ext_if) -> (\$ext_if:0)

block in
pass out

pass in proto tcp to port {22}
pass in inet proto icmp icmp-type { echoreq }
EOF

# packages
## get latest packages locally
mkdir -p /usr/local/etc/pkg/repos/
cat /etc/pkg/FreeBSD.conf | sed -e 's/pkg+http:/https:/' | sed -e 's/quarterly/latest/' | > /usr/local/etc/pkg/repos/FreeBSD.conf
pkg install -y pkg
pkg update
pkg upgrade -y

## image package config
mkdir -p /mnt/usr/local/etc/pkg/repos/
cp /usr/local/etc/pkg/repos/FreeBSD.conf /mnt/usr/local/etc/pkg/repos/FreeBSD.conf
pkg -r /mnt install -y pkg
pkg -r /mnt update
pkg -r /mnt upgrade -y
pkg -r /mnt install -y google-cloud-sdk py39-google-compute-engine

## my favorite packages
pkg -r /mnt install -y tailscale emacs-nox htop tmux sudo git jq

# snapshot
zfs snapshot -r zroot@init
zpool export zroot

gcloud compute instances detach-disk $instance_name --disk=$image_name --zone=$zone
gcloud compute images create $image_name --guest-os-features=UEFI_COMPATIBLE --source-disk-zone=$zone --source-disk=$image_name --family=$family
echo 'y' | gcloud compute disks delete $image_name --zone=$zone

# cleanup
rm -f /tmp/base.txz /tmp/kernel.txz