Last active
July 20, 2024 12:23
-
-
Save lukebfox/1518cb16758023ff8db22a555a02adc6 to your computer and use it in GitHub Desktop.
Instructions to bootstrap nixos with root zfs on hetzner dedicated AX41-NVMe server
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
set -eo pipefail | |
################################################################################ | |
## Setup Helpers | |
################################################################################ | |
export COLOR_RESET="\033[0m" | |
export RED_BG="\033[41m" | |
export BLUE_BG="\033[44m" | |
function err { | |
echo -e "${RED_BG}$1${COLOR_RESET}" | |
} | |
function info { | |
echo -e "${BLUE_BG}$1${COLOR_RESET}" | |
} | |
export AUTHORISED_SSH_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDgFoF1gx50bc273ZtaxisKq/GO8Y+awYhZenlwB6YzF robot@lukebentleyfox.net" | |
################################################################################ | |
## 1. Generate NixOS kernel and kexec into it from hetzner rescue live system | |
################################################################################ | |
install_nix() { | |
info "Installing sudo for the installer" | |
apt install -y sudo | |
info "Installing Nix in multi-user mode" | |
sh <(curl -L https://nixos.org/nix/install) --daemon | |
. $HOME/.nix-profile/etc/profile.d/nix-daemon.sh | |
info "Installing nixos-generators" | |
nix-env -f https://github.com/nix-community/nixos-generators/archive/master.tar.gz -i -v | |
} | |
generate_kernel () { | |
info "Creating an initial nix configuration to kexec into" | |
cat <<EOF > /root/config.nix | |
{ | |
services.openssh.enable = true; | |
users.users.root.openssh.authorizedKeys.keys = [ | |
# Replace with your public key | |
"$AUTHORISED_SSH_KEY" | |
]; | |
} | |
EOF | |
info "Generating the kexec script" | |
nixos-generate -o /root/result -f kexec-bundle -c /root/config.nix | |
info "Switching to the new NixOS live system" | |
/root/result | |
# This shell is now dead. To continue with stage 2, SSH from different terminal. | |
# Note: the host key will have changed, and you need to rerun the setup helpers. | |
} | |
################################################################################ | |
## 2. Format our 2 SSD drives for using ZFS as root file system for NixOS | |
################################################################################ | |
format_drives () { | |
# list available disks | |
#ls /dev/disk/by-id/* | |
# create disk array | |
DISK='/dev/disk/by-id/nvme-eui.002538b521b6f7cc /dev/disk/by-id/nvme-eui.002538b521b6f7cf' | |
INST_PARTSIZE_SWAP=64 # set swap size. minimum should be no less than RAM size. | |
INST_PARTSIZE_RPOOL= # root pool size. use all remaining disk space if not set | |
# boot and root pools | |
export ZFS_BOOT_POOL="bpool" | |
export ZFS_ROOT_POOL="rpool" | |
# system and boot logical containers | |
export ZFS_SYSTEM_CONTAINER="${ZFS_ROOT_POOL}/nixos" | |
export ZFS_BOOT_CONTAINER="${ZFS_BOOT_POOL}/nixos" | |
# boot dataset | |
export ZFS_DS_BOOT="${ZFS_BOOT_CONTAINER}/root" | |
# ephemeral datasets | |
export ZFS_LOCAL="${ZFS_SYSTEM_CONTAINER}/local" | |
export ZFS_DS_ROOT="${ZFS_LOCAL}/root" | |
export ZFS_DS_NIX="${ZFS_LOCAL}/nix" | |
# persistent datasets | |
export ZFS_SAFE="${ZFS_SYSTEM_CONTAINER}/safe" | |
export ZFS_DS_HOME="${ZFS_SAFE}/home" | |
export ZFS_DS_PERSIST="${ZFS_SAFE}/persist" | |
# initial blank root snapshot | |
export ZFS_BLANK_SNAPSHOT="${ZFS_DS_ROOT}@blank" | |
################## | |
## PARTITIONING ## | |
################## | |
info "Partitioning disks" | |
for i in ${DISK}; do | |
sgdisk --zap-all $i | |
sgdisk -n1:1M:+1G -t1:EF00 $i | |
sgdisk -n2:0:+4G -t2:BE00 $i | |
test -z $INST_PARTSIZE_SWAP || sgdisk -n4:0:+${INST_PARTSIZE_SWAP}G -t4:8200 $i | |
if test -z $INST_PARTSIZE_RPOOL; then | |
sgdisk -n3:0:0 -t3:BF00 $i | |
else | |
sgdisk -n3:0:+${INST_PARTSIZE_RPOOL}G -t3:BF00 $i | |
fi | |
sgdisk -a1 -n5:24K:+1000K -t5:EF02 $i | |
done | |
############### | |
## ZFS POOLS ## | |
############### | |
info "Creating '$ZFS_BOOT_POOL' ZFS boot pool on each drive (mirror)" | |
zpool create -f \ | |
-o compatibility=grub2 \ | |
-o ashift=12 \ | |
-o autotrim=on \ | |
-O acltype=posixacl \ | |
-O canmount=off \ | |
-O compression=lz4 \ | |
-O devices=off \ | |
-O normalization=formD \ | |
-O relatime=on \ | |
-O xattr=sa \ | |
-O mountpoint=/boot \ | |
-R /mnt \ | |
$ZFS_BOOT_POOL \ | |
mirror \ | |
$(for i in ${DISK}; do | |
printf "$i-part2 "; | |
done) | |
info "Creating '$ZFS_ROOT_POOL' ZFS root pool on each drive (mirror)" | |
zpool create -f \ | |
-o ashift=12 \ | |
-o autotrim=on \ | |
-R /mnt \ | |
-O acltype=posixacl \ | |
-O canmount=off \ | |
-O compression=zstd \ | |
-O dnodesize=auto \ | |
-O normalization=formD \ | |
-O relatime=on \ | |
-O xattr=sa \ | |
-O mountpoint=/ \ | |
$ZFS_ROOT_POOL \ | |
mirror \ | |
$(for i in ${DISK}; do | |
printf "$i-part3 "; | |
done) | |
########################## | |
## ZFS SYSTEM CONTAINER ## | |
########################## | |
info "Creating '$ZFS_SYSTEM_CONTAINER' unencrypted root system container" | |
zfs create -o canmount=off -o mountpoint=none $ZFS_SYSTEM_CONTAINER | |
######################### | |
## ZFS LOCAL CONTAINER ## | |
######################### | |
info "Creating 'local' container" | |
zfs create -o canmount=off -o mountpoint=none "$ZFS_LOCAL" | |
################## | |
## ZFS '/' DATASET | |
info "Creating '$ZFS_DS_ROOT' ZFS root dataset" | |
zfs create -o canmount=off -o mountpoint=/ "$ZFS_DS_ROOT" | |
info "Configuring extended attributes setting for '$ZFS_DS_ROOT' ZFS dataset ..." | |
zfs set xattr=sa "$ZFS_DS_ROOT" | |
info "Configuring access control list setting for '$ZFS_DS_ROOT' ZFS dataset ..." | |
zfs set acltype=posixacl "$ZFS_DS_ROOT" | |
info "Creating '$ZFS_BLANK_SNAPSHOT' ZFS snapshot ..." | |
zfs snapshot "$ZFS_BLANK_SNAPSHOT" | |
info "Mounting '$ZFS_DS_ROOT' to /mnt ..." | |
zfs set canmount=on "$ZFS_DS_ROOT" | |
zfs mount "$ZFS_DS_ROOT" | |
#################### | |
# ZFS '/nix' dataset | |
info "Creating '$ZFS_DS_NIX' ZFS nix dataset ..." | |
zfs create -o canmount=on -o mountpoint=/nix "$ZFS_DS_NIX" | |
info "Configuring extended attributes setting for '$ZFS_DS_NIX' ZFS dataset ..." | |
zfs set xattr=sa "$ZFS_DS_NIX" | |
info "Disabling access time setting for '$ZFS_DS_NIX' ZFS dataset ..." | |
zfs set atime=off "$ZFS_DS_NIX" | |
#################### | |
## SAFE CONTAINER ## | |
#################### | |
info "Creating 'safe' container" | |
zfs create -o canmount=off -o mountpoint=none "$ZFS_SAFE" | |
################################### | |
## ZFS '/home', '/persist' datasets | |
info "Creating '$ZFS_SAFE' ZFS datasets {home, persist}" | |
zfs create -o canmount=on -o mountpoint=/home "$ZFS_DS_HOME" | |
zfs create -o canmount=on -o mountpoint=/persist "$ZFS_DS_PERSIST" | |
info "Permit ZFS auto-snapshots on '${ZFS_SAFE}/*' datasets ..." | |
zfs set com.sun:auto-snapshot=true "$ZFS_DS_HOME" | |
zfs set com.sun:auto-snapshot=true "$ZFS_DS_PERSIST" | |
#################### | |
## BOOT CONTAINER ## | |
#################### | |
info "Creating '$ZFS_BOOT_CONTAINER' unencrypted boot system container" | |
zfs create -o canmount=off -o mountpoint=none $ZFS_BOOT_CONTAINER | |
###################### | |
## ZFS '/boot' dataset | |
info "Creating '$ZFS_DS_BOOT' ZFS boot dataset" | |
zfs create -o canmount=on -o mountpoint=/boot "$ZFS_DS_BOOT" | |
info "Formatting and mounting ESP" | |
for i in ${DISK}; do | |
mkfs.vfat -n EFI ${i}-part1 | |
mkdir -p /mnt/boot/efis/${i##*/}-part1 | |
mount -t vfat ${i}-part1 /mnt/boot/efis/${i##*/}-part1 | |
done | |
mkdir -p /mnt/boot/efi | |
mount -t vfat $(echo $DISK | cut -f1 -d\ )-part1 /mnt/boot/efi | |
} | |
################################################################################ | |
## 3. Build and install NixOS onto the local drives. | |
################################################################################ | |
prepare_nixos () { | |
INIT_USER_NAME="robot" | |
info "Disabling cache, stale cache will prevent system from booting" | |
mkdir -p /mnt/etc/zfs/ | |
rm -f /mnt/etc/zfs/zpool.cache | |
touch /mnt/etc/zfs/zpool.cache | |
chmod a-w /mnt/etc/zfs/zpool.cache | |
chattr +i /mnt/etc/zfs/zpool.cache | |
######################### | |
## NixOS Configuration ## | |
######################### | |
info "Generating initial system configuration" | |
nixos-generate-config --root /mnt/ | |
info "Fixing up configuration.nix" | |
sed -i "s|./hardware-configuration.nix|./hardware-configuration.nix ./zfs.nix|g" /mnt/etc/nixos/configuration.nix | |
sed -i '/boot.loader/d' /mnt/etc/nixos/configuration.nix | |
sed -i '/services.xserver/d' /mnt/etc/nixos/configuration.nix | |
info "Configuring hostid" | |
tee -a /mnt/etc/nixos/zfs.nix <<EOF | |
{ config, pkgs, lib, ... }: | |
{ networking.hostId = "$(head -c 8 /etc/machine-id)"; | |
boot = { | |
supportedFilesystems = [ "zfs" ] | |
initrd.postDeviceCommands = lib.mkAfter '' | |
zfs rollback -r ${ZFS_BLANK_SNAPSHOT} | |
''; | |
kernelPackages = config.boot.zfs.package.latestCompatibleLinuxPackages; | |
loader = { | |
efi = { | |
efiSysMountPoint = "/boot/efi"; | |
canTouchEfiVariables = false; | |
}; | |
generationsDir.copyKernels = true; | |
grub = { | |
efiInstallAsRemovable = true; | |
enable = true; | |
version = 2; | |
copyKernels = true; | |
efiSupport = true; | |
zfsSupport = true; | |
extraPrepareConfig = '' | |
mkdir -p /boot/efis | |
for i in /boot/efis/*; do mount $i ; done | |
mkdir -p /boot/efi | |
mount /boot/efi | |
''; | |
extraInstallCommands = '' | |
ESP_MIRROR=$(mktemp -d) | |
cp -r /boot/efi/EFI $ESP_MIRROR | |
for i in /boot/efis/*; do | |
cp -r $ESP_MIRROR/EFI $i | |
done | |
rm -rf $ESP_MIRROR | |
''; | |
devices = [ | |
EOF | |
for i in $DISK; do | |
printf " \"$i\"\n" >>/mnt/etc/nixos/zfs.nix | |
done | |
tee -a /mnt/etc/nixos/zfs.nix <<EOF | |
]; | |
}; | |
}; | |
}; | |
services.zfs = { | |
autoScrub.enable = true; | |
autoSnapshot.enable = true; | |
}; | |
services.udev.extraRules = '' | |
KERNEL=="sd[a-z]*[0-9]*|mmcblk[0-9]*p[0-9]*|nvme[0-9]*n[0-9]*p[0-9]*", ENV{ID_FS_TYPE}=="zfs_member", ATTR{../queue/scheduler}="none" | |
''; | |
EOF | |
info "Moving configuration to '/persist'" | |
mkdir -p /mnt/persist/etc/nixos | |
mv /mnt/etc/nixos/*.nix /mnt/persist/etc/nixos/ | |
rmdir /mnt/etc/nixos | |
ln -s /mnt/persist/etc/nixos /mnt/etc/nixos | |
info "Mounting datasets with zfsutil option" | |
sed -i 's|fsType = "zfs";|fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];|g' \ | |
/mnt/etc/nixos/hardware-configuration.nix | |
# set root password | |
ROOT_PASSWORD_HASH=$(mkpasswd -m SHA-512 -s) | |
USER_PASSWORD_HASH=$(mkpasswd -m SHA-512 -s) | |
info "Declaring root password/SSH access in configuration" | |
tee -a /mnt/etc/nixos/zfs.nix <<EOF | |
users.users = { | |
root = { | |
initialHashedPassword = "${ROOT_PASSWORD_HASH}"; | |
openssh.authorizedKeys.keys = [ "${AUTHORISED_SSH_KEY}" ]; | |
}; | |
${INIT_USER_NAME} = { | |
isNormalUser = true; | |
initialHashedPassword = "${USER_PASSWORD_HASH}"; | |
extraGroups = [ "wheel" ]; | |
uid = 1002; | |
openssh.authorizedKeys.keys = [ "${AUTHORISED_SSH_KEY}" ]; | |
}; | |
}; | |
EOF | |
mkdir -p /mnt/persist/etc/ssh | |
tee -a /mnt/etc/nixos/zfs.nix <<EOF | |
services.openssh = { | |
enable = true; | |
passwordAuthentication = false; | |
hostKeys = [ | |
{ | |
path = "/persist/etc/ssh/ssh_host_ed25519_key"; | |
type = "ed25519"; | |
} | |
{ | |
path = "/persist/etc/ssh/ssh_host_rsa_key"; | |
type = "rsa"; | |
bits = 4096; | |
} | |
]; | |
}; | |
} | |
EOF | |
# CHECK CONFIG BEFORE CONTINUING | |
} | |
################### | |
## Install NixOS ## | |
################### | |
install_nixos () { | |
info "Installing NixOS system and applying configuration" | |
nixos-install -v --show-trace --no-root-passwd --root /mnt | |
info "Unmounting filesystems" | |
umount -Rl /mnt | |
zpool export -a | |
info "Rebooting your system 😎" | |
reboot | |
} | |
install_nix | |
#generate_kernel | |
format_drives | |
prepare_nixos | |
install_nixos |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment