VFIO Passthrough

libvirt.nix

{ lib, pkgs, config, ... }:
with lib;
let
  cfg = config.virtualisation.libvirtd;

  boolToZeroOne = x: if x then "1" else "0";

  aclString = with lib.strings;
              concatMapStringsSep ",\n  " escapeNixString cfg.deviceACL;
in {
  options.virtualisation.libvirtd = {
    deviceACL = mkOption {
      type = types.listOf types.str;
      default = [];
    };
    clearEmulationCapabilities = mkOption {
      type = types.bool;
      default = true;
    };
  };
  
  config.virtualisation.libvirtd.qemuVerbatimConfig = ''
  clear_emulation_capabilities = ${boolToZeroOne cfg.clearEmulationCapabilities}
  cgroup_device_acl = [
    ${aclString}
  ]
  '';

 
  #config.services.udev.extraRules = ''
  #  SUBSYSTEM=="usb", ATTR{"DEVPATH"}=="/dev/input/by-id/usb-04d9_USB_Keyboard-event-kbd", GROUP="wheel"
  #''; 
}

section configuration.nix

  virtualisation = {
    sharedMemoryFiles = {
      looking-glass = { size = 32; user = "root"; group = "root"; mode = "666"; };
    };
    libvirtd = {
      enable = true;
      qemuOvmf = true;
      clearEmulationCapabilities = false;
      deviceACL = ["/dev/input/by-id/usb-04d9_USB_Keyboard-if01-event-kbd"
                   "/dev/input/by-id/usb-04d9_USB_Keyboard-event-kbd"
                   "/dev/input/by-id/usb-Primax_Kensington_Eagle_Trackball-event-mouse"
                   "/dev/input/by-id/usb-Primax_Kensington_Eagle_Trackball-mouse"
                   "/dev/vfio/vfio"
                   "/dev/vfio/13"
                   "/dev/vfio/14"
                   "/dev/vfio/15"
                   "/dev/kvm"
                   "/dev/shm/looking-glass"
                   ];
    };
    vfio = {
      enable = true;
      IOMMUType = "intel";
      devices = [ "10de:1b80" "10de:10f0" ];
      blacklistNvidia = true;
      disableEFIfb = true;
      ignoreMSRs = true;
      applyACSpatch = true;
    };
  };

vfio.nix

{ lib, pkgs, config, ... }:
with lib;
let
  cfg = config.virtualisation.vfio;
in {
  options.virtualisation.vfio = {
    enable = mkEnableOption "VFIO Configuration";
    IOMMUType = mkOption {
      type = types.enum [ "intel" "amd" ];
      example = "intel";
      description = "Type of the IOMMU used";
    };
    devices = mkOption {
      type = types.listOf (types.strMatching "[0-9a-f]{4}:[0-9a-f]{4}");
      default = [];
      example = ["10de:1b80" "10de:10f0"];
      description = "PCI IDs of devices to bind to vfio-pci";
    };
    disableEFIfb = mkOption {
      type = types.bool;
      default = false;
      example = true;
      description = "Disables the usage of the EFI framebuffer on boot.";
    };
    blacklistNvidia = mkOption {
      type = types.bool;
      default = false;
      description = "Add Nvidia GPU modules to blacklist";
    };
    ignoreMSRs = mkOption {
      type = types.bool;
      default = false;
      example = true;
      description = "Enables or disables kvm guest access to model-specific registers";
    };
    applyACSpatch = mkOption {
      type = types.bool;
      default = false;
      description = ''
        If set, the following things will happen:
          - The ACS override patch is applied
          - Applies the i915-vga-arbiter patch
          - Adds pcie_acs_override=downstream to the command line
      '';
    };
  };

  config = lib.mkIf cfg.enable {
    boot.kernelParams = ( if cfg.IOMMUType == "intel" then
                            [ "intel_iommu=on" "intel_iommu=igfx_off" ]
                          else
                            [ "amd_iommu=on" ]
                        ) ++ (optional
                            (builtins.length cfg.devices > 0)
                            ("vfio-pci.ids=" +
                              builtins.concatStringsSep
                              ","
                              cfg.devices)
                         ) ++ (optional
                              cfg.applyACSpatch
                              "pcie_acs_override=downstream,multifunction"
                         ) ++ (optional
                              cfg.disableEFIfb
                              "video=efifb:off"
                         ) ++ (optional
                              cfg.ignoreMSRs
                              "kvm.ignore_msrs=1"
                         );
                            
    boot.kernelModules = [
      "vfio_virqfd"
      "vfio_pci"
      "vfio_iommu_type1"
      "vfio"
    ];
                           
    boot.initrd.kernelModules = [
      "vfio_virqfd"
      "vfio_pci"
      "vfio_iommu_type1"
      "vfio"
    ];
    boot.blacklistedKernelModules = optionals
                                      cfg.blacklistNvidia
                                      [ "nvidia" "nouveau" ];
    
    boot.kernelPatches = optionals cfg.applyACSpatch [
      { name = "add-acs-overrides";
        patch = pkgs.fetchurl {
          name = "add-acs-overrides.patch";
          url = "https://aur.archlinux.org/cgit/aur.git/plain/add-acs-overrides.patch?h=linux-vfio&id=6f5c5ff2e42abf6606564383d5cb3c56b13d895e";
          sha256 = "1qd68s9r0ppynksbffqn2qbp1whqpbfp93dpccp9griwhx5srx6v";
        };
      }
      { name = "i915-vga-arbiter";
        patch = pkgs.fetchurl {
          name = "i915-vga-arbiter.patch";
          url = "https://aur.archlinux.org/cgit/aur.git/plain/i915-vga-arbiter.patch?h=linux-vfio&id=6f5c5ff2e42abf6606564383d5cb3c56b13d895e";
          sha256 = "1mg06dmlsdzf9w6jy73izjpa8ma7yh80k48rjj6iq30qs4jw1d5g";
        };
      }
    ];
  };
}

virtualisation.nix

{ lib, pkgs, config, ... }:
with lib;
let
  cfg = config.virtualisation;
  functionBlock = name: f: ''
      fallocate -l ${toString f.size}M   /dev/shm/${name};
      chown ${f.user}:${f.group}         /dev/shm/${name};
      chmod ${f.mode}                    /dev/shm/${name};
    '';
in {
  options.virtualisation.sharedMemoryFiles = mkOption {
    type = types.attrsOf (
      types.submodule (
        { name, ...}: {
          options = {
            name = mkOption {
              visible = false;
              default = name;
              type = types.str;
            };
            size = mkOption {
              type = types.int;
              default = 0;
              description = "Size in MB.";
            };
            user = mkOption {
              type = types.str;
              default = "root";
              description = "Owner of the memory file";
            };
            group = mkOption {
              type = types.str;
              default = "root";
              description = "Group of the memory file";
            };
            mode = mkOption {
              type = types.str;
              default = "0600";
              description = "Group of the memory file";
            };
          };
        }
      )
    );
    default = {};
  };

  config.system.activationScripts.sharedMemoryFiles = {
  text = concatStringsSep "\n" (mapAttrsToList (functionBlock) cfg.sharedMemoryFiles);
    deps = [ ];
  };
}