Raspberry Pi 4 NixOS installation with ZFS on SSD

01-prepare-sd.sh

#!/usr/bin/env bash

# Run this as root from an USB stick with an aarch64 NixOS image.
# I.e. make use of the auto login to the nixos account and `sudo su`.
# All required commands are available in the installer image.

# This script will prepare the partitions and file systems on the device located at $DEV.

set -Eeuo pipefail

device="${1:-}"

if [ -z "${device}" ]; then
    echo "No device specified (first argument), aborting"
    exit 1
fi

echo "Current zpool status"
zpool import || true
echo "Current partitioning"
sfdisk -d "${device}" || true

read -p "Do you want to continue (y/n)?" -n 1 -r
echo
if [[ ! "${REPLY}" =~ ^[Yy]$ ]]; then
    echo "Aborting"
    exit 0
fi

echo "Disabling swap"
swapoff -a

echo "Clearing zpool labels"

zpool labelclear -f "${device}p3" || true

echo "Wiping existing filesystem"
wipefs -a "${device}"

echo "Partitions before:"
sfdisk -d "${device}" || true

echo "Creating partitions"
parted --script "${device}" \
    mklabel msdos \
    mkpart -a optimal primary fat32 0% 1GiB \
    set 1 boot on \
    mkpart primary linux-swap 1GiB 9GiB \
    mkpart primary 9GiB 100%

# Detect the disk as disk by-id
disk_by_id=
for dsk in /dev/disk/by-id/*; do
    if [ "$(readlink -f "${dsk}")" = "${device}" ]; then
        disk_by_id="${dsk}"
    fi
done

if [ -z "${disk_by_id}" ]; then
    echo "No disk found by id"
    exit 1
fi
echo "Drive is at '${disk_by_id}'"

echo "Partitions after:"
sfdisk -d "${device}"

# If we move too fast some path doesn't exist yet
sleep 2

boot_part="${disk_by_id}-part1"
swap_part="${disk_by_id}-part2"
zfs_part="${disk_by_id}-part3"

echo "Setting up boot partition"

mkfs.vfat "${boot_part}"
fatlabel "${boot_part}" BOOT

echo "Setting up swap partition"

mkswap -L swap "${swap_part}"

echo "Creating ZFS pool"

zpool create -O mountpoint=none -O atime=off -O xattr=sa -O acltype=posixacl rpool "${zfs_part}"

echo "Creating ZFS datasets"

zfs create -p -o mountpoint=legacy rpool/local/root
zfs snapshot rpool/local/root@blank
zfs create -p -o mountpoint=legacy rpool/local/nix
zfs create -p -o mountpoint=legacy rpool/safe/home
zfs create -p -o mountpoint=legacy rpool/safe/log
zfs create -p -o mountpoint=legacy rpool/safe/persist

echo "DONE"

02-install-nix-sh

#!/usr/bin/env bash

# Run this as root from an USB with an aarch64 NixOS image;
# all required commands are then available.

# This will install NixOS on a device located at $DEV
# Run 01-prepare-sd.sh first to create partitions and the file system.

set -Eeuo pipefail

root_dir="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")"

device="${1:-}"
server_name="${2:-}"

if [ -z "${device}" ]; then
    echo "No device specified (first argument), aborting"
    exit 1
fi

if [ -z "${server_name}" ]; then
    echo "No servername specified (second argument), aborting"
    exit 1
fi

if [ ! -d "${root_dir}/${server_name}" ]; then
    echo "No config available for servername ${server_name}, aborting"
    exit 1
fi

# Detect the disk as disk by-id
disk_by_id=
for dsk in /dev/disk/by-id/*; do
    if [ "$(readlink -f "${dsk}")" = "${device}" ]; then
        disk_by_id="${dsk}"
    fi
done

if [ -z "${disk_by_id}" ]; then
    echo "No disk found by id"
    exit 1
fi
echo "Disk is at '${disk_by_id}'"

boot_part="${disk_by_id}-part1"
swap_part="${disk_by_id}-part2"

if [ ! -f /root/.config/nix/nix.conf ]; then
    echo "Enabling flakes"
    mkdir -p /root/.config/nix
    echo "experimental-features = nix-command flakes" > /root/.config/nix/nix.conf
fi

echo "Nix config:"

cat /root/.config/nix/nix.conf

echo "Mounting disks"

mount -t zfs rpool/local/root /mnt

mkdir -p /mnt/boot
mount "${boot_part}" /mnt/boot

mkdir -p /mnt/nix
mount -t zfs rpool/local/nix /mnt/nix

mkdir -p /mnt/home
mount -t zfs rpool/safe/home /mnt/home

mkdir -p /mnt/persist
mount -t zfs rpool/safe/persist /mnt/persist

echo "Setting up swap partition"

swapoff -a
swapon "${swap_part}"

echo "Generating NixOS config (detecting hardware)"

nixos-generate-config --root /mnt

echo "Overriding configuration.nix"

mkdir -p /mnt/persist/etc/nixos
cp -t /mnt/persist/etc/nixos/ "${root_dir}/${server_name}/*"
cp -t /mnt/persist/etc/nixos/ /mnt/etc/nixos/hardware-configuration.nix
chmod -R a+rX /mnt/persist/etc/nixos

echo "Creating persistent directory for host SSH keys"
mkdir -p /mnt/persist/etc/ssh

echo "Installing NixOS"

nixos-install --root /mnt --flake "/mnt/persist/etc/nixos#${server_name}" --no-root-password

echo "Copying machine-id to persistent path"

cp -t /mnt/persist/etc /mnt/etc/machine-id

echo "DONE"

README.md

Installing NixOS with ZFS on an SD card for Raspberry Pi

Put default SD image on USB stick

https://hydra.nixos.org/job/nixos/release-22.05/nixos.sd_image.aarch64-linux

Download a recent image and extract it:

$ nix-shell -p zstd --run "unzstd nixos-sd-image-22.05.3389.9cac4585028-aarch64-linux.img.zst"

Write it to a USB stick, e.g.:

$ sudo dd if=./nixos-sd-image-22.05.3389.9cac4585028-aarch64-linux.img of=/dev/sda bs=1M status=progress

Put this repo on the USB stick

Just copy it to /home/nixos. It's also a good idea to clone personalise and copy some contents for .ssh into /home/nixos.

Boot into the USB stick on the Raspberry Pi

After boot you will automatically be logged in as user nixos. Become root via sudo su.

Install NixOS to SD card

Attach the device to install NixOS onto.

Run

# cd nixos-servers
# ./01-prepare-sd.sh <dev>
# ./02-install-nixos.sh <dev> <servername>

where <dev> is the device to install onto (existing data will be wiped) and <server> is the name of the server config to be used.

Shut down and remove the USB stick. Power the Pi again. We will now boot into the NixOS system on the ZFS pool on <dev>. You can log in as a normal user now.

configuration.diff

diff --git a/configuration.nix b/configuration.nix
index 97512ec..15f77f0 100644
--- a/configuration.nix
+++ b/configuration.nix
@@ -117,53 +117,142 @@ in
     ./vim-minimal.nix
   ];
 
+  # From nixos-hardware module
+  hardware.raspberry-pi."4".apply-overlays-dtmerge.enable = true;
+

@@ -227,25 +316,6 @@ in
   };
 
   boot = {
-    loader = {
-      grub.enable = false;
-      raspberryPi = {
-        enable = true;
-        version = 4;
-        uboot.enable = false;
-        # Disable LEDs
-        firmwareConfig = ''
-          dtparam=act_led_trigger=none
-          dtparam=act_led_activelow=off
-          dtparam=pwr_led_trigger=default-on
-          dtparam=pwr_led_activelow=off
-          dtparam=eth_led0=4
-          dtparam=eth_led1=4
-        '';
-      };
-    };
-
-    kernelPackages = pkgs.linuxPackages_rpi4;
     # To allow VPN to forward, already enabled by default
     kernel.sysctl."net.ipv4.ip_forward" = 1;
 
@@ -267,7 +337,6 @@ in
     device = "debelain:/tank/backups/master";
   };
 

diff --git a/flake.nix b/flake.nix
index 458a6a3..3046e8a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -3,12 +3,14 @@
   # To update the lock to some particular revision, e.g.: sudo nix flake update --override-input nixpkgs github:NixOS/nixpkgs/<rev>
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    nixos-hardware.url = "github:NixOS/nixos-hardware/master";
   };
 
-  outputs = { nixpkgs, ... }@inputs: {
+  outputs = { nixpkgs, nixos-hardware, ... }@inputs: {
     nixosConfigurations.<redacted> = nixpkgs.lib.nixosSystem {
       system = "aarch64-linux";
       modules = [
+        nixos-hardware.nixosModules.raspberry-pi-4
         (import ./logwatch.nix)
         (import ./configuration.nix)
       ];

configuration.nix

# Reduced version of the Raspberry Pi configuration

{ inputs, pkgs, ... }:

let
  lib = pkgs.lib;
in
{
  imports = [
    ./hardware-configuration.nix
  ];

  hardware.deviceTree = {
    enable = true;

    overlays = [
      # Elided
    ];
  };

  boot = {
    loader = {
      grub.enable = false;
      raspberryPi = {
        enable = true;
        version = 4;
        uboot.enable = false;
        # Disable LEDs
        firmwareConfig = ''
          dtparam=act_led_trigger=none
          dtparam=act_led_activelow=off
          dtparam=pwr_led_trigger=default-on
          dtparam=pwr_led_activelow=off
          dtparam=eth_led0=4
          dtparam=eth_led1=4
        '';
      };
    };

    kernelPackages = pkgs.linuxPackages_rpi4;

    supportedFilesystems = [ "zfs" ];

    # https://grahamc.com/blog/erase-your-darlings
    initrd.postDeviceCommands = lib.mkAfter ''
      echo "Starting rollback rpool/local/root@blank"
      zfs rollback -r rpool/local/root@blank
      echo "Finished rollback"
    '';
  };

  # This value determines the NixOS release from which the default
  # settings for stateful data, like file locations and database versions
  # on your system were taken. It‘s perfectly fine and recommended to leave
  # this value at the release version of the first install of this system.
  # Before changing this value read the documentation for this option
  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
  system.stateVersion = "22.05"; # Did you read the comment?

}

hardware-configuration.nix

# Do not modify this file!  It was generated by ‘nixos-generate-config’
# and may be overwritten by future invocations.  Please make changes
# to /etc/nixos/configuration.nix instead.
{ config, lib, pkgs, modulesPath, ... }:

{
  imports =
    [ (modulesPath + "/installer/scan/not-detected.nix")
    ];

  boot.initrd.availableKernelModules = [ "xhci_pci" "usbhid" "usb_storage" ];
  boot.initrd.kernelModules = [ ];
  boot.kernelModules = [ ];
  boot.extraModulePackages = [ ];

  fileSystems."/" =
    { device = "rpool/local/root";
      fsType = "zfs";
    };

  fileSystems."/boot" =
    { device = "/dev/disk/by-uuid/D48A-BFBC";
      fsType = "vfat";
    };

  fileSystems."/nix" =
    { device = "rpool/local/nix";
      fsType = "zfs";
    };

  fileSystems."/home" =
    { device = "rpool/safe/home";
      fsType = "zfs";
    };

  fileSystems."/persist" =
    { device = "rpool/safe/persist";
      fsType = "zfs";
      neededForBoot = true;
    };

  fileSystems."/var/log" =
    { device = "rpool/safe/log";
      fsType = "zfs";
      neededForBoot = true;
    };

  fileSystems."/share" =
    { device = "rpool/safe/share";
      fsType = "zfs";
    };

  swapDevices = [
    { device = "/dev/disk/by-uuid/1243f757-5bd9-4423-83a9-7c98645a4ecf";
      priority = 1;
    }
  ];

  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
  # (the default) this is the recommended approach. When using systemd-networkd it's
  # still possible to use this option, but it's recommended to use it in conjunction
  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
  # networking.useDHCP = lib.mkDefault true;
  # networking.interfaces.eth0.useDHCP = lib.mkDefault true;

  powerManagement.cpuFreqGovernor = lib.mkDefault "ondemand";
}