Skip to content

Instantly share code, notes, and snippets.

@lukebfox
Last active July 20, 2024 12:23
Show Gist options
  • Save lukebfox/1518cb16758023ff8db22a555a02adc6 to your computer and use it in GitHub Desktop.
Save lukebfox/1518cb16758023ff8db22a555a02adc6 to your computer and use it in GitHub Desktop.
Instructions to bootstrap nixos with root zfs on hetzner dedicated AX41-NVMe server
#!/usr/bin/env bash
set -eo pipefail
################################################################################
## Setup Helpers
################################################################################
export COLOR_RESET="\033[0m"
export RED_BG="\033[41m"
export BLUE_BG="\033[44m"
function err {
echo -e "${RED_BG}$1${COLOR_RESET}"
}
function info {
echo -e "${BLUE_BG}$1${COLOR_RESET}"
}
export AUTHORISED_SSH_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDgFoF1gx50bc273ZtaxisKq/GO8Y+awYhZenlwB6YzF robot@lukebentleyfox.net"
################################################################################
## 1. Generate NixOS kernel and kexec into it from hetzner rescue live system
################################################################################
install_nix() {
info "Installing sudo for the installer"
apt install -y sudo
info "Installing Nix in multi-user mode"
sh <(curl -L https://nixos.org/nix/install) --daemon
. $HOME/.nix-profile/etc/profile.d/nix-daemon.sh
info "Installing nixos-generators"
nix-env -f https://github.com/nix-community/nixos-generators/archive/master.tar.gz -i -v
}
generate_kernel () {
info "Creating an initial nix configuration to kexec into"
cat <<EOF > /root/config.nix
{
services.openssh.enable = true;
users.users.root.openssh.authorizedKeys.keys = [
# Replace with your public key
"$AUTHORISED_SSH_KEY"
];
}
EOF
info "Generating the kexec script"
nixos-generate -o /root/result -f kexec-bundle -c /root/config.nix
info "Switching to the new NixOS live system"
/root/result
# This shell is now dead. To continue with stage 2, SSH from different terminal.
# Note: the host key will have changed, and you need to rerun the setup helpers.
}
################################################################################
## 2. Format our 2 SSD drives for using ZFS as root file system for NixOS
################################################################################
format_drives () {
# list available disks
#ls /dev/disk/by-id/*
# create disk array
DISK='/dev/disk/by-id/nvme-eui.002538b521b6f7cc /dev/disk/by-id/nvme-eui.002538b521b6f7cf'
INST_PARTSIZE_SWAP=64 # set swap size. minimum should be no less than RAM size.
INST_PARTSIZE_RPOOL= # root pool size. use all remaining disk space if not set
# boot and root pools
export ZFS_BOOT_POOL="bpool"
export ZFS_ROOT_POOL="rpool"
# system and boot logical containers
export ZFS_SYSTEM_CONTAINER="${ZFS_ROOT_POOL}/nixos"
export ZFS_BOOT_CONTAINER="${ZFS_BOOT_POOL}/nixos"
# boot dataset
export ZFS_DS_BOOT="${ZFS_BOOT_CONTAINER}/root"
# ephemeral datasets
export ZFS_LOCAL="${ZFS_SYSTEM_CONTAINER}/local"
export ZFS_DS_ROOT="${ZFS_LOCAL}/root"
export ZFS_DS_NIX="${ZFS_LOCAL}/nix"
# persistent datasets
export ZFS_SAFE="${ZFS_SYSTEM_CONTAINER}/safe"
export ZFS_DS_HOME="${ZFS_SAFE}/home"
export ZFS_DS_PERSIST="${ZFS_SAFE}/persist"
# initial blank root snapshot
export ZFS_BLANK_SNAPSHOT="${ZFS_DS_ROOT}@blank"
##################
## PARTITIONING ##
##################
info "Partitioning disks"
for i in ${DISK}; do
sgdisk --zap-all $i
sgdisk -n1:1M:+1G -t1:EF00 $i
sgdisk -n2:0:+4G -t2:BE00 $i
test -z $INST_PARTSIZE_SWAP || sgdisk -n4:0:+${INST_PARTSIZE_SWAP}G -t4:8200 $i
if test -z $INST_PARTSIZE_RPOOL; then
sgdisk -n3:0:0 -t3:BF00 $i
else
sgdisk -n3:0:+${INST_PARTSIZE_RPOOL}G -t3:BF00 $i
fi
sgdisk -a1 -n5:24K:+1000K -t5:EF02 $i
done
###############
## ZFS POOLS ##
###############
info "Creating '$ZFS_BOOT_POOL' ZFS boot pool on each drive (mirror)"
zpool create -f \
-o compatibility=grub2 \
-o ashift=12 \
-o autotrim=on \
-O acltype=posixacl \
-O canmount=off \
-O compression=lz4 \
-O devices=off \
-O normalization=formD \
-O relatime=on \
-O xattr=sa \
-O mountpoint=/boot \
-R /mnt \
$ZFS_BOOT_POOL \
mirror \
$(for i in ${DISK}; do
printf "$i-part2 ";
done)
info "Creating '$ZFS_ROOT_POOL' ZFS root pool on each drive (mirror)"
zpool create -f \
-o ashift=12 \
-o autotrim=on \
-R /mnt \
-O acltype=posixacl \
-O canmount=off \
-O compression=zstd \
-O dnodesize=auto \
-O normalization=formD \
-O relatime=on \
-O xattr=sa \
-O mountpoint=/ \
$ZFS_ROOT_POOL \
mirror \
$(for i in ${DISK}; do
printf "$i-part3 ";
done)
##########################
## ZFS SYSTEM CONTAINER ##
##########################
info "Creating '$ZFS_SYSTEM_CONTAINER' unencrypted root system container"
zfs create -o canmount=off -o mountpoint=none $ZFS_SYSTEM_CONTAINER
#########################
## ZFS LOCAL CONTAINER ##
#########################
info "Creating 'local' container"
zfs create -o canmount=off -o mountpoint=none "$ZFS_LOCAL"
##################
## ZFS '/' DATASET
info "Creating '$ZFS_DS_ROOT' ZFS root dataset"
zfs create -o canmount=off -o mountpoint=/ "$ZFS_DS_ROOT"
info "Configuring extended attributes setting for '$ZFS_DS_ROOT' ZFS dataset ..."
zfs set xattr=sa "$ZFS_DS_ROOT"
info "Configuring access control list setting for '$ZFS_DS_ROOT' ZFS dataset ..."
zfs set acltype=posixacl "$ZFS_DS_ROOT"
info "Creating '$ZFS_BLANK_SNAPSHOT' ZFS snapshot ..."
zfs snapshot "$ZFS_BLANK_SNAPSHOT"
info "Mounting '$ZFS_DS_ROOT' to /mnt ..."
zfs set canmount=on "$ZFS_DS_ROOT"
zfs mount "$ZFS_DS_ROOT"
####################
# ZFS '/nix' dataset
info "Creating '$ZFS_DS_NIX' ZFS nix dataset ..."
zfs create -o canmount=on -o mountpoint=/nix "$ZFS_DS_NIX"
info "Configuring extended attributes setting for '$ZFS_DS_NIX' ZFS dataset ..."
zfs set xattr=sa "$ZFS_DS_NIX"
info "Disabling access time setting for '$ZFS_DS_NIX' ZFS dataset ..."
zfs set atime=off "$ZFS_DS_NIX"
####################
## SAFE CONTAINER ##
####################
info "Creating 'safe' container"
zfs create -o canmount=off -o mountpoint=none "$ZFS_SAFE"
###################################
## ZFS '/home', '/persist' datasets
info "Creating '$ZFS_SAFE' ZFS datasets {home, persist}"
zfs create -o canmount=on -o mountpoint=/home "$ZFS_DS_HOME"
zfs create -o canmount=on -o mountpoint=/persist "$ZFS_DS_PERSIST"
info "Permit ZFS auto-snapshots on '${ZFS_SAFE}/*' datasets ..."
zfs set com.sun:auto-snapshot=true "$ZFS_DS_HOME"
zfs set com.sun:auto-snapshot=true "$ZFS_DS_PERSIST"
####################
## BOOT CONTAINER ##
####################
info "Creating '$ZFS_BOOT_CONTAINER' unencrypted boot system container"
zfs create -o canmount=off -o mountpoint=none $ZFS_BOOT_CONTAINER
######################
## ZFS '/boot' dataset
info "Creating '$ZFS_DS_BOOT' ZFS boot dataset"
zfs create -o canmount=on -o mountpoint=/boot "$ZFS_DS_BOOT"
info "Formatting and mounting ESP"
for i in ${DISK}; do
mkfs.vfat -n EFI ${i}-part1
mkdir -p /mnt/boot/efis/${i##*/}-part1
mount -t vfat ${i}-part1 /mnt/boot/efis/${i##*/}-part1
done
mkdir -p /mnt/boot/efi
mount -t vfat $(echo $DISK | cut -f1 -d\ )-part1 /mnt/boot/efi
}
################################################################################
## 3. Build and install NixOS onto the local drives.
################################################################################
prepare_nixos () {
INIT_USER_NAME="robot"
info "Disabling cache, stale cache will prevent system from booting"
mkdir -p /mnt/etc/zfs/
rm -f /mnt/etc/zfs/zpool.cache
touch /mnt/etc/zfs/zpool.cache
chmod a-w /mnt/etc/zfs/zpool.cache
chattr +i /mnt/etc/zfs/zpool.cache
#########################
## NixOS Configuration ##
#########################
info "Generating initial system configuration"
nixos-generate-config --root /mnt/
info "Fixing up configuration.nix"
sed -i "s|./hardware-configuration.nix|./hardware-configuration.nix ./zfs.nix|g" /mnt/etc/nixos/configuration.nix
sed -i '/boot.loader/d' /mnt/etc/nixos/configuration.nix
sed -i '/services.xserver/d' /mnt/etc/nixos/configuration.nix
info "Configuring hostid"
tee -a /mnt/etc/nixos/zfs.nix <<EOF
{ config, pkgs, lib, ... }:
{ networking.hostId = "$(head -c 8 /etc/machine-id)";
boot = {
supportedFilesystems = [ "zfs" ]
initrd.postDeviceCommands = lib.mkAfter ''
zfs rollback -r ${ZFS_BLANK_SNAPSHOT}
'';
kernelPackages = config.boot.zfs.package.latestCompatibleLinuxPackages;
loader = {
efi = {
efiSysMountPoint = "/boot/efi";
canTouchEfiVariables = false;
};
generationsDir.copyKernels = true;
grub = {
efiInstallAsRemovable = true;
enable = true;
version = 2;
copyKernels = true;
efiSupport = true;
zfsSupport = true;
extraPrepareConfig = ''
mkdir -p /boot/efis
for i in /boot/efis/*; do mount $i ; done
mkdir -p /boot/efi
mount /boot/efi
'';
extraInstallCommands = ''
ESP_MIRROR=$(mktemp -d)
cp -r /boot/efi/EFI $ESP_MIRROR
for i in /boot/efis/*; do
cp -r $ESP_MIRROR/EFI $i
done
rm -rf $ESP_MIRROR
'';
devices = [
EOF
for i in $DISK; do
printf " \"$i\"\n" >>/mnt/etc/nixos/zfs.nix
done
tee -a /mnt/etc/nixos/zfs.nix <<EOF
];
};
};
};
services.zfs = {
autoScrub.enable = true;
autoSnapshot.enable = true;
};
services.udev.extraRules = ''
KERNEL=="sd[a-z]*[0-9]*|mmcblk[0-9]*p[0-9]*|nvme[0-9]*n[0-9]*p[0-9]*", ENV{ID_FS_TYPE}=="zfs_member", ATTR{../queue/scheduler}="none"
'';
EOF
info "Moving configuration to '/persist'"
mkdir -p /mnt/persist/etc/nixos
mv /mnt/etc/nixos/*.nix /mnt/persist/etc/nixos/
rmdir /mnt/etc/nixos
ln -s /mnt/persist/etc/nixos /mnt/etc/nixos
info "Mounting datasets with zfsutil option"
sed -i 's|fsType = "zfs";|fsType = "zfs"; options = [ "zfsutil" "X-mount.mkdir" ];|g' \
/mnt/etc/nixos/hardware-configuration.nix
# set root password
ROOT_PASSWORD_HASH=$(mkpasswd -m SHA-512 -s)
USER_PASSWORD_HASH=$(mkpasswd -m SHA-512 -s)
info "Declaring root password/SSH access in configuration"
tee -a /mnt/etc/nixos/zfs.nix <<EOF
users.users = {
root = {
initialHashedPassword = "${ROOT_PASSWORD_HASH}";
openssh.authorizedKeys.keys = [ "${AUTHORISED_SSH_KEY}" ];
};
${INIT_USER_NAME} = {
isNormalUser = true;
initialHashedPassword = "${USER_PASSWORD_HASH}";
extraGroups = [ "wheel" ];
uid = 1002;
openssh.authorizedKeys.keys = [ "${AUTHORISED_SSH_KEY}" ];
};
};
EOF
mkdir -p /mnt/persist/etc/ssh
tee -a /mnt/etc/nixos/zfs.nix <<EOF
services.openssh = {
enable = true;
passwordAuthentication = false;
hostKeys = [
{
path = "/persist/etc/ssh/ssh_host_ed25519_key";
type = "ed25519";
}
{
path = "/persist/etc/ssh/ssh_host_rsa_key";
type = "rsa";
bits = 4096;
}
];
};
}
EOF
# CHECK CONFIG BEFORE CONTINUING
}
###################
## Install NixOS ##
###################
install_nixos () {
info "Installing NixOS system and applying configuration"
nixos-install -v --show-trace --no-root-passwd --root /mnt
info "Unmounting filesystems"
umount -Rl /mnt
zpool export -a
info "Rebooting your system 😎"
reboot
}
install_nix
#generate_kernel
format_drives
prepare_nixos
install_nixos
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment