Arch QEMU KVM Libvirt Single GPU PCI

start.sh

#
#
# /etc/libvirt/hooks/qemu.d/winnvme/prepare/begin/start.sh
#
#

#!/bin/bash
set -x

# Stop display manager
systemctl stop display-manager
# rc-service xdm stop

# Unbind VTconsoles: might not be needed
echo 0 > /sys/class/vtconsole/vtcon0/bind
echo 0 > /sys/class/vtconsole/vtcon1/bind

# Unbind EFI Framebuffer
echo efi-framebuffer.0 > /sys/bus/platform/drivers/efi-framebuffer/unbind

# Unload NVIDIA kernel modules
modprobe -r nvidia_drm nvidia_modeset nvidia_uvm nvidia

# Unload AMD kernel module
# modprobe -r amdgpu

# Load vfio module
modprobe vfio-pci

# Detach GPU devices from host
# Use your GPU and HDMI Audio PCI host device
virsh nodedev-detach pci_0000_09_00_0
#virsh nodedev-detach pci_0000_01_00_1

stop.sh

#
#
# /etc/libvirt/hooks/qemu.d/winnvme/release/end/stop.sh
#
#

#!/bin/bash
set -x

# Attach GPU devices to host
# Use your GPU and HDMI Audio PCI host device
virsh nodedev-reattach pci_0000_09_00_0
#virsh nodedev-reattach pci_0000_01_00_1

# Unload vfio module
modprobe -r vfio-pci

# Rebind framebuffer to host
echo "efi-framebuffer.0" > /sys/bus/platform/drivers/efi-framebuffer/bind

# Load NVIDIA kernel modules
modprobe nvidia_drm
modprobe nvidia_modeset
modprobe nvidia_uvm
modprobe nvidia

# Load AMD kernel module
# modprobe amdgpu

# Bind VTconsoles: might not be needed
echo 1 > /sys/class/vtconsole/vtcon0/bind
echo 1 > /sys/class/vtconsole/vtcon1/bind

# Restart Display Manager
systemctl start display-manager

winnvme.log

2022-03-02 11:46:37.741+0000: starting up libvirt version: 8.0.0, qemu version: 6.2.0, kernel: 5.16.11-arch1-1, hostname: archlinux
LC_ALL=C \
PATH=/usr/local/sbin:/usr/local/bin:/usr/bin \
HOME=/var/lib/libvirt/qemu/domain-1-winnvme \
XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-1-winnvme/.local/share \
XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-1-winnvme/.cache \
XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-1-winnvme/.config \
/usr/bin/qemu-system-x86_64 \
-name guest=winnvme,debug-threads=on \
-S \
-object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-1-winnvme/master-key.aes"}' \
-blockdev '{"driver":"file","filename":"/usr/share/edk2-ovmf/x64/OVMF_CODE.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"}' \
-blockdev '{"driver":"file","filename":"/var/lib/libvirt/qemu/nvram/winnvme_VARS.fd","node-name":"libvirt-pflash1-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-pflash1-format","read-only":false,"driver":"raw","file":"libvirt-pflash1-storage"}' \
-machine pc-q35-6.2,usb=off,vmport=off,dump-guest-core=off,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format,memory-backend=pc.ram \
-accel kvm \
-cpu EPYC-Rome,x2apic=on,tsc-deadline=on,hypervisor=on,tsc-adjust=on,stibp=on,arch-capabilities=on,ssbd=on,xsaves=on,cmp-legacy=on,amd-ssbd=on,virt-ssbd=on,tsc-scale=on,svme-addr-chk=on,rdctl-no=on,skip-l1dfl-vmentry=on,mds-no=on,pschange-mc-no=on,hv-time=on,hv-relaxed=on,hv-vapic=on,hv-spinlocks=0x1fff,hv-vendor-id=whatever \
-m 16042 \
-object '{"qom-type":"memory-backend-ram","id":"pc.ram","size":16821256192}' \
-overcommit mem-lock=off \
-smp 6,sockets=6,cores=1,threads=1 \
-uuid 5dc4bfaf-1d36-4dca-8cc9-11af6258d96a \
-display none \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=31,server=on,wait=off \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=localtime,driftfix=slew \
-global kvm-pit.lost_tick_policy=delay \
-no-hpet \
-no-shutdown \
-global ICH9-LPC.disable_s3=1 \
-global ICH9-LPC.disable_s4=1 \
-boot strict=on \
-device pcie-root-port,port=8,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x1 \
-device pcie-root-port,port=9,chassis=2,id=pci.2,bus=pcie.0,addr=0x1.0x1 \
-device pcie-root-port,port=10,chassis=3,id=pci.3,bus=pcie.0,addr=0x1.0x2 \
-device pcie-root-port,port=11,chassis=4,id=pci.4,bus=pcie.0,addr=0x1.0x3 \
-device qemu-xhci,p2=15,p3=15,id=usb,bus=pci.1,addr=0x0 \
-chardev pty,id=charserial0 \
-device isa-serial,chardev=charserial0,id=serial0 \
-device usb-tablet,id=input0,bus=usb.0,port=1 \
-audiodev '{"id":"audio1","driver":"none"}' \
-device ich9-intel-hda,id=sound0,bus=pcie.0,addr=0x1b \
-device hda-duplex,id=sound0-codec0,bus=sound0.0,cad=0,audiodev=audio1 \
-device vfio-pci,host=0000:0a:00.0,id=hostdev0,bootindex=1,bus=pci.2,addr=0x0 \
-device virtio-balloon-pci,id=balloon0,bus=pci.3,addr=0x0 \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on
char device redirected to /dev/pts/0 (label charserial0)
2022-03-02T11:46:37.783486Z qemu-system-x86_64: warning: This feature depends on other features that were not requested: CPUID.8000000AH:EDX.tsc-scale [bit 4]
2022-03-02T11:46:37.783509Z qemu-system-x86_64: warning: This feature depends on other features that were not requested: CPUID.8000000AH:EDX.svme-addr-chk [bit 28]
2022-03-02T11:46:37.783815Z qemu-system-x86_64: warning: This feature depends on other features that were not requested: CPUID.8000000AH:EDX.tsc-scale [bit 4]
2022-03-02T11:46:37.783819Z qemu-system-x86_64: warning: This feature depends on other features that were not requested: CPUID.8000000AH:EDX.svme-addr-chk [bit 28]
2022-03-02T11:46:37.783967Z qemu-system-x86_64: warning: This feature depends on other features that were not requested: CPUID.8000000AH:EDX.tsc-scale [bit 4]
2022-03-02T11:46:37.783970Z qemu-system-x86_64: warning: This feature depends on other features that were not requested: CPUID.8000000AH:EDX.svme-addr-chk [bit 28]
2022-03-02T11:46:37.784117Z qemu-system-x86_64: warning: This feature depends on other features that were not requested: CPUID.8000000AH:EDX.tsc-scale [bit 4]
2022-03-02T11:46:37.784119Z qemu-system-x86_64: warning: This feature depends on other features that were not requested: CPUID.8000000AH:EDX.svme-addr-chk [bit 28]
2022-03-02T11:46:37.784288Z qemu-system-x86_64: warning: This feature depends on other features that were not requested: CPUID.8000000AH:EDX.tsc-scale [bit 4]
2022-03-02T11:46:37.784291Z qemu-system-x86_64: warning: This feature depends on other features that were not requested: CPUID.8000000AH:EDX.svme-addr-chk [bit 28]
2022-03-02T11:46:37.784447Z qemu-system-x86_64: warning: This feature depends on other features that were not requested: CPUID.8000000AH:EDX.tsc-scale [bit 4]
2022-03-02T11:46:37.784449Z qemu-system-x86_64: warning: This feature depends on other features that were not requested: CPUID.8000000AH:EDX.svme-addr-chk [bit 28]

winnvme.xml

<domain type="kvm">
  <name>winnvme</name>
  <uuid>5dc4bfaf-1d36-4dca-8cc9-11af6258d96a</uuid>
  <metadata>
    <libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
      <libosinfo:os id="http://microsoft.com/win/10"/>
    </libosinfo:libosinfo>
  </metadata>
  <memory unit="KiB">16427008</memory>
  <currentMemory unit="KiB">16427008</currentMemory>
  <vcpu placement="static">6</vcpu>
  <os>
    <type arch="x86_64" machine="pc-q35-6.2">hvm</type>
    <loader readonly="yes" type="pflash">/usr/share/edk2-ovmf/x64/OVMF_CODE.fd</loader>
    <nvram>/var/lib/libvirt/qemu/nvram/winnvme_VARS.fd</nvram>
  </os>
  <features>
    <acpi/>
    <apic/>
    <hyperv mode="custom">
      <relaxed state="on"/>
      <vapic state="on"/>
      <spinlocks state="on" retries="8191"/>
      <vendor_id state="on" value="whatever"/>
    </hyperv>
    <vmport state="off"/>
  </features>
  <cpu mode="host-model" check="partial"/>
  <clock offset="localtime">
    <timer name="rtc" tickpolicy="catchup"/>
    <timer name="pit" tickpolicy="delay"/>
    <timer name="hpet" present="no"/>
    <timer name="hypervclock" present="yes"/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <pm>
    <suspend-to-mem enabled="no"/>
    <suspend-to-disk enabled="no"/>
  </pm>
  <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <controller type="usb" index="0" model="qemu-xhci" ports="15">
      <address type="pci" domain="0x0000" bus="0x01" slot="0x00" function="0x0"/>
    </controller>
    <controller type="sata" index="0">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x1f" function="0x2"/>
    </controller>
    <controller type="pci" index="0" model="pcie-root"/>
    <controller type="pci" index="1" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="1" port="0x8"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x0" multifunction="on"/>
    </controller>
    <controller type="pci" index="2" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="2" port="0x9"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x1"/>
    </controller>
    <controller type="pci" index="3" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="3" port="0xa"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x2"/>
    </controller>
    <controller type="pci" index="4" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="4" port="0xb"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x3"/>
    </controller>
    <serial type="pty">
      <target type="isa-serial" port="0">
        <model name="isa-serial"/>
      </target>
    </serial>
    <console type="pty">
      <target type="serial" port="0"/>
    </console>
    <input type="tablet" bus="usb">
      <address type="usb" bus="0" port="1"/>
    </input>
    <input type="mouse" bus="ps2"/>
    <input type="keyboard" bus="ps2"/>
    <sound model="ich9">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x1b" function="0x0"/>
    </sound>
    <audio id="1" type="none"/>
    <hostdev mode="subsystem" type="pci" managed="yes">
      <source>
        <address domain="0x0000" bus="0x0a" slot="0x00" function="0x0"/>
      </source>
      <boot order="1"/>
      <address type="pci" domain="0x0000" bus="0x02" slot="0x00" function="0x0"/>
    </hostdev>
    <memballoon model="virtio">
      <address type="pci" domain="0x0000" bus="0x03" slot="0x00" function="0x0"/>
    </memballoon>
  </devices>
</domain>