netbooting a raspberry pi running alpine with nix

netboot.nix

# dhcp server settings:
# dhcp-mac=set:rpi,b8:27:eb:*:*:*
# dhcp-boot=tag:rpi,boot,10.69.69.179

{ lib, stdenv, config, pkgs, ... }:
let
  alpineRelease = "3.11";
  alpineRevision = "2";
  alpineVersion = "${alpineRelease}.${alpineRevision}";
  
  fetchAlpinePi = {arch, sha256}: builtins.fetchTarball {
    url = "http://dl-cdn.alpinelinux.org/alpine/v${alpineRelease}/releases/${arch}/alpine-rpi-${alpineVersion}-${arch}.tar.gz";
    sha256 = sha256;
  };

  alpinePiArmhf   = fetchAlpinePi { arch = "armhf";   sha256 = "0rkv6d0hpqfxmzn2fv6i4scn9hf5azg32kdbcxafyk2s92yrvlgr"; };
  alpinePiArmv7   = fetchAlpinePi { arch = "armv7";   sha256 = "1k83wyfb186hv14ajvqw9qi5j8jz2g033idpdvxz6ycm2gla3gk5"; };
  alpinePiAarch64 = fetchAlpinePi { arch = "aarch64"; sha256 = "1qj0ipg80k5sv4bj1h0sd6z3nj7l4frwznbs31yw3m04gc300vmp"; };

  configTxt = pkgs.writeTextFile {
    name = "config.txt";
    text = ''#
[pi0]
kernel=vmlinuz-rpi
initramfs initramfs-rpi
[pi0w]
kernel=vmlinuz-rpi
initramfs initramfs-rpi
enable_uart=1
[pi1]
kernel=vmlinuz-rpi
initramfs initramfs-rpi
[pi2]
kernel=vmlinuz-rpi2
initramfs initramfs-rpi2
[pi3]
kernel=vmlinuz-rpi2
initramfs initramfs-rpi2
enable_uart=1
[pi3+]
kernel=vmlinuz-rpi2
initramfs initramfs-rpi2
enable_uart=1
[pi4]
enable_gic=1
kernel=vmlinuz-rpi4
initramfs initramfs-rpi4
arm_64bit=1'';
  };

  cmdlineTxt = pkgs.writeTextFile {
    name = "cmdline.txt";
    text = "modules=loop,squashfs,sd-mod,usb-storage console=ttyS0,115200 ip=dhcp alpine_repo=http://dl-cdn.alpinelinux.org/alpine/v${alpineRelease}/main/ apkovl=http://netboot.service.consul/apkovl/{MAC}.tar.gz";
  };

  tftpRoot = pkgs.stdenv.mkDerivation { 
    name = "tftproot";
    preferLocalBuild = true; 
    allowSubstitutes = false;
    buildInputs = [ pkgs.squashfsTools pkgs.cpio ];

    buildCommand = ''
      mkdir -p $out

      # Hack: add af_packet.ko to the initramfs
      mkdir modloop-rpi modloop-rpi2 modloop-rpi4
      unsquashfs -d modloop-rpi/lib  ${alpinePiArmhf}/boot/modloop-rpi     'modules/*/modules.*' 'modules/*/kernel/net/packet/af_packet.ko'
      unsquashfs -d modloop-rpi2/lib ${alpinePiArmv7}/boot/modloop-rpi2    'modules/*/modules.*' 'modules/*/kernel/net/packet/af_packet.ko'
      unsquashfs -d modloop-rpi4/lib ${alpinePiAarch64}/boot/modloop-rpi4  'modules/*/modules.*' 'modules/*/kernel/net/packet/af_packet.ko'
      
      (cd modloop-rpi  && find . | cpio -H newc -ov | gzip) > initramfs-ext-rpi
      (cd modloop-rpi2 && find . | cpio -H newc -ov | gzip) > initramfs-ext-rpi2
      (cd modloop-rpi4 && find . | cpio -H newc -ov | gzip) > initramfs-ext-rpi4

      cat ${alpinePiArmhf}/boot/initramfs-rpi    initramfs-ext-rpi  > $out/initramfs-rpi
      cat ${alpinePiArmv7}/boot/initramfs-rpi2   initramfs-ext-rpi2 > $out/initramfs-rpi2
      cat ${alpinePiAarch64}/boot/initramfs-rpi4 initramfs-ext-rpi4 > $out/initramfs-rpi4

      cd $out

      # Pi Classic initial bootloader (VC4)
      sed -e "s/BOOT_UART=0/BOOT_UART=1/" bootcode.bin ${alpinePiArmhf}/bootcode.bin >bootcode.bin

      # Primary bootloader (ARM)
      ln -s ${alpinePiArmhf}/start{,4}.elf .

      # SDRAM setup
      ln -s ${alpinePiArmhf}/fixup{,4}.dat .

      # Configuration
      ln -s ${configTxt} ./config.txt

      # Kernels
      ln -s ${alpinePiArmhf}/boot/vmlinuz-rpi ./
      ln -s ${alpinePiArmv7}/boot/vmlinuz-rpi2 ./
      ln -s ${alpinePiAarch64}/boot/vmlinuz-rpi4 ./

      # Device Tree Blobs
      ln -s ${alpinePiArmhf}/*.dtb .

      # Commandlines
      ln -s ${cmdlineTxt} ./cmdline.txt
    '';
  };
in
{

  services.atftpd = {
    enable = true;
    root = tftpRoot;
    extraOptions = [ "--verbose=7" ];
  };

  consulServices = [
    {
      name = "tftp";
      port = 69;
      tags = [ "udp" ];
    }
  ];

  networking.firewall = {
    allowedUDPPorts = [ 69 ];
    connectionTrackingModules = [ "tftp" ];
  };
}

Hello @erincandescent! If you wouldn't mind helping me.. I'm am trying to netboot my Raspberry Pi 4s and I have "hacked in" af_packet.ko into the initramfs by following what you have done. Now I am stuck trying to get the RPi to mount the root filesystem from my NFS server. After it has gotten everything from the TFTP server and booted the kernel it just sits with a blinking underscore (_) with no network activity afterwards (monitored by tcpdump) and no other output to the screen. Might you have any advice on how to get it to continue?

My cmdline.txt:

modules=loop,squashfs,sd-mod,usb-storage quiet console=ttyS0 ip=dhcp alpine_dev=nfs:10.0.0.64:/nfs/node01 rootwait elevator=deadline

I have also tried:

modules=loop,squashfs,sd-mod,usb-storage quiet console=tty0 ip=dhcp alpine_dev=nfs:10.0.0.64:/nfs/node01 alpine_repo=http://dl-cdn.alpinelinux.org/alpine/v3.11/main/ rootwait elevator=deadline

While trying to get af_packet.ko to work it would fail into an sh environment where I could do some troubleshooting. Might there be a way to activate sh in its current state? Is there a different console that might show it?

Thank you for your help!

I have no idea with regards to NFS, I'm afraid. My setup uses APKOVL downloads from a HTTP server in order to net-boot individual devices

This is nothing short of magic! I hope you don't mind I've adapted the rpi4 part of this gist to a bash script (which keeps all the packages and tarballs on the local network), since I don't intend to use nix (looks cool though). Script lives here: https://github.com/biemster/rpi4_alpine_netboot. Thanks a lot for sharing this!