Skip to content

Instantly share code, notes, and snippets.

@LGUG2Z
Created March 5, 2023 23:19
Show Gist options
  • Save LGUG2Z/0495a36b2c0afecb9bf82ae94fed18e4 to your computer and use it in GitHub Desktop.
Save LGUG2Z/0495a36b2c0afecb9bf82ae94fed18e4 to your computer and use it in GitHub Desktop.
My VFIO setup for a Windows 11 VM in NixOS on a Ryzen 3700x with an Nvidia 3070ti passed through to the VM
let
iommuIds = [
"10de:2482" # 3070 Ti Graphics
"10de:228b" # 3070 Ti Audio
];
in
{
pkgs,
lib,
config,
username,
home-manager,
...
}: {
options.vfio.enable = with lib;
mkEnableOption "Configure the machine for VFIO";
config = let
cfg = config.vfio;
in {
boot = {
initrd.kernelModules = [
"vfio_pci"
"vfio"
"vfio_iommu_type1"
"vfio_virqfd"
"nvidia"
"nvidia_modeset"
"nvidia_uvm"
"nvidia_drm"
];
kernelParams =
[
# enable IOMMU
"amd_iommu=on"
]
++ lib.optional cfg.enable
# isolate the GPU
("vfio-pci.ids=" + lib.concatStringsSep "," iommuIds);
};
hardware.opengl.enable = true;
virtualisation = with pkgs; {
spiceUSBRedirection.enable = true;
libvirtd.enable = true;
libvirtd.extraConfig = ''
uri_default = "qemu:///system"
'';
libvirtd.qemu = {
package = unstable.qemu_kvm;
runAsRoot = true;
swtpm.enable = true;
ovmf.enable = true;
ovmf.packages = [
((unstable.OVMFFull.override
{
secureBoot = true;
tpmSupport = true;
csmSupport = true;
httpSupport = true;
})
.fd)
];
};
};
systemd.tmpfiles.rules = [
"f /dev/shm/looking-glass 0660 ${username} qemu-libvirtd -"
"f /dev/shm/scream 0660 ${username} qemu-libvirtd -"
];
environment.systemPackages = with pkgs; [
virt-manager
pciutils
looking-glass-client
];
systemd.user.services.scream = {
enable = true;
description = "Scream IVSHMEM";
serviceConfig = {
ExecStartPre = "/run/current-system/sw/bin/sleep 5";
ExecStart = "${pkgs.scream}/bin/scream -m /dev/shm/scream";
Restart = "always";
RestartSec = 2;
};
wantedBy = ["default.target"];
requires = ["pipewire-pulse.service"];
};
home-manager.users.${username} = {
xdg.desktopEntries.looking-glass-client = {
exec = "looking-glass-client";
icon = "lg-logo";
name = "Looking Glass Client";
genericName = "View Virtual Machine Desktops";
type = "Application";
terminal = false;
};
};
};
}
<domain type='kvm' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'>
<name>win11</name>
<uuid>191944a0-7cc4-43ba-a097-8ec64a2bca23</uuid>
<metadata>
<libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
<libosinfo:os id="http://microsoft.com/win/10"/>
</libosinfo:libosinfo>
</metadata>
<memory unit='KiB'>16777216</memory>
<currentMemory unit='KiB'>16777216</currentMemory>
<vcpu placement='static'>8</vcpu>
<cputune>
<vcpupin vcpu='0' cpuset='2'/>
<vcpupin vcpu='1' cpuset='8'/>
<vcpupin vcpu='2' cpuset='3'/>
<vcpupin vcpu='3' cpuset='9'/>
<vcpupin vcpu='4' cpuset='4'/>
<vcpupin vcpu='5' cpuset='10'/>
<vcpupin vcpu='6' cpuset='5'/>
<vcpupin vcpu='7' cpuset='11'/>
<emulatorpin cpuset='0-1,6-7'/>
</cputune>
<os>
<type arch='x86_64' machine='pc-i440fx-7.2'>hvm</type>
<loader readonly='yes' type='pflash'>/run/libvirt/nix-ovmf/OVMF_CODE.fd</loader>
<nvram>/var/lib/libvirt/qemu/nvram/win11_VARS.fd</nvram>
<bootmenu enable='no'/>
</os>
<features>
<acpi/>
<apic/>
<hyperv mode='custom'>
<relaxed state='on'/>
<vapic state='on'/>
<spinlocks state='on' retries='8191'/>
<vpindex state='on'/>
<synic state='on'/>
<stimer state='on'>
<direct state='on'/>
</stimer>
<reset state='on'/>
<frequencies state='on'/>
<reenlightenment state='on'/>
<tlbflush state='on'/>
<ipi state='on'/>
</hyperv>
<vmport state='off'/>
</features>
<cpu mode='host-passthrough' check='none' migratable='on'>
<topology sockets='1' dies='1' cores='4' threads='2'/>
</cpu>
<clock offset='localtime'>
<timer name='rtc' present='no' tickpolicy='catchup'/>
<timer name='pit' present='no' tickpolicy='delay'/>
<timer name='hpet' present='no'/>
<timer name='kvmclock' present='no'/>
<timer name='hypervclock' present='yes'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<pm>
<suspend-to-mem enabled='no'/>
<suspend-to-disk enabled='no'/>
</pm>
<devices>
<emulator>/run/libvirt/nix-emulators/qemu-system-x86_64</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2' discard='unmap'/>
<source file='/var/lib/libvirt/images/win11.qcow2'/>
<target dev='vda' bus='virtio'/>
<boot order='1'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native' discard='unmap'/>
<source dev='/dev/disk/by-id/ata-Samsung_SSD_870_QVO_2TB_S6R4NJ0R614116X'/>
<backingStore/>
<target dev='sdd' bus='sata'/>
<address type='drive' controller='0' bus='0' target='0' unit='3'/>
</disk>
<controller type='usb' index='0' model='qemu-xhci' ports='15'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
</controller>
<controller type='pci' index='0' model='pci-root'/>
<controller type='pci' index='1' model='pci-bridge'>
<model name='pci-bridge'/>
<target chassisNr='1'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x0f' function='0x0'/>
</controller>
<controller type='pci' index='2' model='pci-bridge'>
<model name='pci-bridge'/>
<target chassisNr='2'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x10' function='0x0'/>
</controller>
<controller type='pci' index='3' model='pci-bridge'>
<model name='pci-bridge'/>
<target chassisNr='3'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x11' function='0x0'/>
</controller>
<controller type='pci' index='4' model='pci-bridge'>
<model name='pci-bridge'/>
<target chassisNr='4'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x12' function='0x0'/>
</controller>
<controller type='pci' index='5' model='pci-bridge'>
<model name='pci-bridge'/>
<target chassisNr='5'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x13' function='0x0'/>
</controller>
<controller type='pci' index='6' model='pci-bridge'>
<model name='pci-bridge'/>
<target chassisNr='6'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x14' function='0x0'/>
</controller>
<controller type='pci' index='7' model='pci-bridge'>
<model name='pci-bridge'/>
<target chassisNr='7'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x15' function='0x0'/>
</controller>
<controller type='pci' index='8' model='pci-bridge'>
<model name='pci-bridge'/>
<target chassisNr='8'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x16' function='0x0'/>
</controller>
<controller type='pci' index='9' model='pci-bridge'>
<model name='pci-bridge'/>
<target chassisNr='9'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x17' function='0x0'/>
</controller>
<controller type='pci' index='10' model='pci-bridge'>
<model name='pci-bridge'/>
<target chassisNr='10'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x18' function='0x0'/>
</controller>
<controller type='pci' index='11' model='pci-bridge'>
<model name='pci-bridge'/>
<target chassisNr='11'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x19' function='0x0'/>
</controller>
<controller type='sata' index='0'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
</controller>
<controller type='virtio-serial' index='0'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
</controller>
<controller type='scsi' index='0' model='lsilogic'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x0c' function='0x0'/>
</controller>
<interface type='network'>
<mac address='52:54:00:f2:fc:9a'/>
<source network='default'/>
<model type='e1000e'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</interface>
<serial type='pty'>
<target type='isa-serial' port='0'>
<model name='isa-serial'/>
</target>
</serial>
<console type='pty'>
<target type='serial' port='0'/>
</console>
<channel type='spicevmc'>
<target type='virtio' name='com.redhat.spice.0'/>
<address type='virtio-serial' controller='0' bus='0' port='1'/>
</channel>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<tpm model='tpm-crb'>
<backend type='emulator' version='2.0'/>
</tpm>
<graphics type='spice' autoport='yes'>
<listen type='address'/>
<image compression='off'/>
</graphics>
<sound model='ich9'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
</sound>
<audio id='1' type='spice'/>
<video>
<model type='vga' vram='16384' heads='1' primary='yes'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
</video>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x33' slot='0x00' function='0x0'/>
</source>
<address type='pci' domain='0x0000' bus='0x00' slot='0x09' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x33' slot='0x00' function='0x1'/>
</source>
<address type='pci' domain='0x0000' bus='0x00' slot='0x0b' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
</source>
<address type='pci' domain='0x0000' bus='0x00' slot='0x0d' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x32' slot='0x00' function='0x0'/>
</source>
<address type='pci' domain='0x0000' bus='0x00' slot='0x0e' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='usb' managed='yes'>
<source>
<vendor id='0x8087'/>
<product id='0x0029'/>
</source>
<address type='usb' bus='0' port='1'/>
</hostdev>
<hostdev mode='subsystem' type='usb' managed='yes'>
<source>
<vendor id='0x0fd9'/>
<product id='0x0080'/>
</source>
<address type='usb' bus='0' port='5'/>
</hostdev>
<hostdev mode='subsystem' type='usb' managed='yes'>
<source>
<vendor id='0x0fd9'/>
<product id='0x0070'/>
</source>
<address type='usb' bus='0' port='6'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x08' slot='0x00' function='0x0'/>
</source>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1a' function='0x0'/>
</hostdev>
<redirdev bus='usb' type='spicevmc'>
<address type='usb' bus='0' port='2'/>
</redirdev>
<redirdev bus='usb' type='spicevmc'>
<address type='usb' bus='0' port='3'/>
</redirdev>
<memballoon model='none'/>
<shmem name='looking-glass'>
<model type='ivshmem-plain'/>
<size unit='M'>128</size>
<address type='pci' domain='0x0000' bus='0x00' slot='0x0a' function='0x0'/>
</shmem>
<shmem name='scream'>
<model type='ivshmem-plain'/>
<size unit='M'>2</size>
<address type='pci' domain='0x0000' bus='0x0b' slot='0x02' function='0x0'/>
</shmem>
</devices>
@LGUG2Z
Copy link
Author

LGUG2Z commented Mar 5, 2023

Note that I have a bunch of other PCI devices (hard drives, webcam etc) passed through on the VM configuration that you'll probably want to remove from the VM configuration.

You'll need to install the drivers for Looking Glass and Scream on the VM while running QXL video (the slow, laggy one that doesn't use your GPU) before you can get Looking Glass working, and also IDDSampleDriver from scoop so that you can use your GPU with Looking Glass without needing to buy and use a dummy HDMI plug.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment