Some bash helper functions for launching QEMU guests directly

example-usage.sh

#!/bin/bash

source vm-helpers.sh

NODE=1
RAM=16

allocateRAM $NODE $RAM
unbindConsole
takeDevByName "\[AMD/ATI\] Navi 10 (rev c1)"    1
takeDevByName "\[AMD/ATI\] Navi 10 HDMI Audio"  1
takeDevByBus  0000:47:00.0
 
qemu-system-x86_64 \
  -m ${RAM}G \
  -object memory-backend-file,id=mem,policy=bind,host-nodes=${NODE},size=${RAM}G,mem-path=/mnt/hugepages1G,share=on,prealloc=yes \
  -numa node,memdev=mem \
  ... your configuration arguments \
  $PT

vm-helpers.sh

#!/bin/bash
# This file is intended to be sourced by your launch script

NL=$' \\\n'
RPORT=0
HDEV=0
DEVLIST=""
PT=""

# Unbind all vitconsoles to allow GPU passthrough
# I am yet to find a way to identify the vtcon based on GPU so for now this unbinds all consoles.
function unbindConsole
{
  for F in /sys/class/vtconsole/vtcon*; do
    echo 0 > $F/bind
  done
}

# Ensures there is N 1GB huge pages available on the specified NUMA node, if not tries to allocate them
# usage: allocateRAM 1 16
function allocateRAM
{
  local NODE=$1
  local MEM=$2

  local FREE=$(cat /sys/devices/system/node/node$NODE/hugepages/hugepages-1048576kB/free_hugepages)
  if [ $FREE -lt $MEM ]; then
    local NEED=$(($MEM - $FREE))
    local HAVE=$(cat /sys/devices/system/node/node$NODE/hugepages/hugepages-1048576kB/nr_hugepages)
    echo $(($HAVE + $NEED)) > /sys/devices/system/node/node$NODE/hugepages/hugepages-1048576kB/nr_hugepages
    FREE=$(cat /sys/devices/system/node/node$NODE/hugepages/hugepages-1048576kB/free_hugepages)
    NEED=$(($MEM - $FREE))
    if [ $FREE -lt $MEM ]; then
      echo "Unable to allocate $MEM hugepages on node $NODE, have $FREE and need $NEED more"
      exit 1
    fi
  fi
}

# Readies a PCI device and all devices in it's IOMMU group for VFIO Passthrough usage
# It also accepts an argument to specify a ROMFILE for QEMU for devices that require it.
#
# This method can be dangerous as if the device you intend to pass through is grouped with
# your HDD controller or some other critical device it WILL unbind it and re-bind it to
# vfio-pci
#
# Usage: takeDevByBus 0000:41:00.0
#        takeDevByBus 0000:44:00.0 /path/to/vbios.rom
#
# $PT is updated with the arguments to give to QEMU
# Each device is added to $DEVLIST, including other devices in the IOMMU group.
function takeDevByBus
{
  local BUS="$1"
  local ROM="$2"

  PT="$PT -device pcie-root-port,id=root_port$RPORT,chassis=0,slot=$RPORT,bus=pcie.0$NL"
  PT="$PT -set device.root_port$RPORT.x-speed=8$NL"
  PT="$PT -set device.root_port$RPORT.x-width=16$NL"

  local LIST=(/sys/bus/pci/devices/$BUS/iommu_group/devices/*)
  local COUNT=${#LIST[@]}
  for ((i = 0; i < $COUNT; i++)); do
    local D=$(basename ${LIST[$i]})
    DEVLIST="$DEVLIST $D"

    local MODULE=$(readlink /sys/bus/pci/devices/$D/driver/module)
    local UNBIND=0
    local OVERRIDE=0
    if [ -z "$MODULE" ]; then
      OVERRIDE=1
    else
      MODULE=$(basename $MODULE)
      if [ "$MODULE" != "vfio_pci" ]; then
        UNBIND=1
        OVERRIDE=1
      fi
    fi

    if [ $UNBIND -eq 1 ]; then
      echo $D > /sys/bus/pci/devices/$D/driver/unbind
    fi

    if [ $OVERRIDE -eq 1 ]; then
      echo vfio-pci > /sys/bus/pci/devices/$D/driver_override
      echo $D > /sys/bus/pci/drivers_probe
    fi

    if [ $i == 0 ] && [ $COUNT -gt 1 ]; then
      MF=",multifunction=on"
    else
      MF=""
    fi

    if [ $i == 0 ] && [ ! -z "$ROM" ]; then
      RF=",romfile=$ROM"
    else
      RF=""
    fi

    PT="$PT -device $(printf 'vfio-pci,host=%s,id=hostdev%d,bus=root_port%d,addr=0x00.%d%s' "$D" "$HDEV" "$RPORT" "$i" "$MF$RF")$NL"
    HDEV=$(($HDEV+1))
  done

  RPORT=$(($RPORT+1))
}

# Same as takeDevByBus except searches for the device by the name listed in lspci
# Be sure to read the details on takeDevByBus, this method is dangerous if misused.
#
# Takes three arguments:
#  1) The grep search argument
#  2) How many devices to match (blank for all)
#  3) The ROM file to use, see takeDevByBus
#
# Usage:
#  takeDevByName "GeForce GTX 1080 Ti" 1 "/path/to/XOC.rom"
#  takeDevByName "USB 3.0" 1
#  takeDevByName "Samsung Electronics Co Ltd NVMe SSD Controller" 2
function takeDevByName
{
  local SEARCH="$1"
  local LIMIT=$2
  local ROM="$3"

  local DEVS=($(lspci -D | grep "$SEARCH" | cut -d' ' -f1))
  test ${#DEVS[@]} -eq 0 && return -1

  for ((n = 0; n < ${#DEVS[@]}; n++)); do
    if [ ! -z "$LIMIT" ] && [ $n -eq $LIMIT ]; then
      break
    fi
    takeDevByBus "${DEVS[$n]}" "$ROM"
  done
}

Note, this assumes your devices have 8x16 PCIe lanes, you should either fix this to detect the slot details, or adjust it accordingly.

Sample PT output:

-device pcie-root-port,id=root_port0,chassis=0,slot=0,bus=pcie.0 \
-set device.root_port0.x-speed=8 \
-set device.root_port0.x-width=16 \
-device vfio-pci,host=0000:45:00.0,id=hostdev0,bus=root_port0,addr=0x00.0 \
-device pcie-root-port,id=root_port1,chassis=0,slot=1,bus=pcie.0 \
-set device.root_port1.x-speed=8 \
-set device.root_port1.x-width=16 \
-device vfio-pci,host=0000:45:00.1,id=hostdev1,bus=root_port1,addr=0x00.0 \
-device pcie-root-port,id=root_port2,chassis=0,slot=2,bus=pcie.0 \
-set device.root_port2.x-speed=8 \
-set device.root_port2.x-width=16 \
-device vfio-pci,host=0000:47:00.0,id=hostdev2,bus=root_port2,addr=0x00.0,multifunction=on \
-device vfio-pci,host=0000:47:00.2,id=hostdev3,bus=root_port2,addr=0x00.1 \
-device vfio-pci,host=0000:47:00.3,id=hostdev4,bus=root_port2,addr=0x00.2 \