command: apt-get install qemu-kvm libvirt-bin bridge-utils virt-manager ovmf qemu-utils hugepages
command: for iommu_group in $(find /sys/kernel/iommu_groups/ -maxdepth 1 -mindepth 1 -type d); do echo "IOMMU group $(basename "$iommu_group")"; for device in $(ls -1 "$iommu_group"/devices/); do echo -n $'\t'; lspci -nns "$device"; done; done
outputs:
IOMMU group 17
0a:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP106 [GeForce GTX 1060 6GB] [10de:1c03] (rev a1)
0a:00.1 Audio device [0403]: NVIDIA Corporation GP106 High Definition Audio Controller [10de:10f1] (rev a1)
IOMMU group 16
09:00.0 VGA compatible controller [0300]: NVIDIA Corporation GK208 [GeForce GT 710B] [10de:128b] (rev a1)
09:00.1 Audio device [0403]: NVIDIA Corporation GK208 HDMI/DP Audio Controller [10de:0e0f] (rev a1)
Will use IOMMU group 17 (geforce 1060):
export VGAPT_VGA_ID='10de:1c03'
export VGAPT_VGA_AUDIO_ID='10de:10f1'
export VGAPT_VGA_BUS=0a:00.0
export VGAPT_VGA_AUDIO_BUS=0a:00.1
file: /etc/modprobe.d/vfio-pci-nvidia.conf
softdep nvidia pre: vfio-pci
file: /etc/modprobe.d/vfio-pci-override.conf
install vfio-pci /sbin/vfio-pci-override.sh
file: /sbin/vfio-pci-override.sh
# PCI_ROOT & PCI_BRIDGE can be resolved by looking at the output of: lspci -nnv
# and then comparing with the symlink on the /sys/bus/pci/devices
#
#!/bin/sh
PCI_ROOT="pci0000:00"
PCI_BRIDGE="0000:00:03.2"
PCI_DEVS="0000:0b:00.0 0000:0b:00.1 ."
if [ ! -z "$(ls -A /sys/class/iommu)" ]; then
for PCI_DEV in $PCI_DEVS; do
echo "vfio-pci" "/sys/devices/$PCI_ROOT/$PCI_BRIDGE/$PCI_DEV/driver_override"
done
fi
modprobe -i vfio-pci
file: /etc/initramfs-tools/hooks/vfio-pci-override
#!/bin/sh
PREREQ=""
prereqs()
{
echo "$PREREQ"
}
case $1 in
prereqs)
prereqs
exit 0
;;
esac
. /usr/share/initramfs-tools/hook-functions
mkdir -p ${DESTDIR}/sbin || true
cp -pnL /sbin/vfio-pci-override.sh ${DESTDIR}/sbin/vfio-pci-override.sh
chmod +x ${DESTDIR}/sbin/vfio-pci-override.sh
file: /etc/modules-load.d/vfio.conf
vfio
vfio_iommu_type1
vfio_pci
vfio_virqfd
vhost-net
command: update-initramfs -u
or mkinitcpio -P
virtio windows driver: https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/stable-virtio/virtio-win.iso
network bridge:
file /etc/netplan/01-network-manager-all.yaml
(ubuntu 18.04 desktop)
# Let NetworkManager manage all devices on this system
network:
version: 2
renderer: NetworkManager
ethernets:
enp4s0:
dhcp4: true
dhcp6: false
vlans:
vlan.101:
id: 101
link: enp4s0
vlan.102:
id: 102
link: enp4s0
vlan.103:
id: 103
link: enp4s0
vlan.104:
id: 104
link: enp4s0
bridges:
br101:
interfaces: [vlan.101]
dhcp4: true
parameters:
stp: false
br102:
interfaces: [vlan.102]
dhcp4: true
parameters:
stp: false
br103:
interfaces: [vlan.103]
dhcp4: true
parameters:
stp: false
br104:
interfaces: [vlan.104]
dhcp4: true
parameters:
stp: false
network with network-manager bridge:
nmcli con add type bridge ifname br-enp5s0
nmcli con add type bridge-slave ifname enp5s0 master br-enp5s0
nmcli con up br-enp5s0
nmcli con up bridge-slave-enp5s0
iptables -I FORWARD -m physdev --physdev-is-bridged -j ACCEPT
network with OpenvSwitch:
- to start openvswitch on arch:
- systemctl enable ovs-vswitchd.service
- systemctl start ovs-vswitchd.service
- create new bridge
br-enp4s0-vlan
:ovs-vsctl add-br br-enp4s0-vlan
- add
enp4s0
tobr-enp4s0-vlan
bridge:ovs-vsctl add-port br-enp4s0-vlan enp4s0
- reconfigure network:
- remove ip from
enp4s0
:ip addr del 192.168.254.184/24 dev enp4s0
- obtain dhcp client for the
br-enp4s0-vlan
bridge:dhclient br-enp4s0-vlan
- remove ip from
- configure systemd-networkd to use dhcp on
br-enp4s0-vlan
:printf "[Match]\nName=br-enp4s0-vlan\n\n[Network]\nDHCP=ipv4\n" > /etc/systemd/network/1000-br-enp4s0.network
- define a new network in libvirt:
- create xml file
ovs-enp4s0-vlan.xml
with the following contents:
- create xml file
<?xml version="1.0"?>
<network>
<name>ovs-enp4s0-vlan</name>
<forward mode="bridge"/>
<bridge name="br-enp4s0-vlan"/>
<virtualport type="openvswitch"/>
<portgroup name="vlan-101">
<vlan>
<tag id="101"/>
</vlan>
</portgroup>
<portgroup name="vlan-102">
<vlan>
<tag id="102"/>
</vlan>
</portgroup>
<portgroup name="vlan-103">
<vlan>
<tag id="103"/>
</vlan>
</portgroup>
<portgroup name="vlan-104">
<vlan>
<tag id="104"/>
</vlan>
</portgroup>
<portgroup name="vlan-all" default="yes">
<vlan trunk="yes">
<tag id="1" nativeMode="untagged"/>
<tag id="101"/>
<tag id="102"/>
<tag id="103"/>
<tag id="104"/>
</vlan>
</portgroup>
</network>
- import the xml file:
virsh net-define --file ovs-enp4s0-vlan.xml
- make network auto-start:
virsh net-autostart ovs-enp4s0-vlan
- start the new network:
virsh net-start ovs-enp4s0-vlan
hugepages:
file: /etc/sysctl.conf
vm.min_free_kbytes=112640
# Set hugetables / hugepages for KVM single guest using 6GB RAM
vm.nr_hugepages=3380
kernel.shmmax=7088373760
command: sysctl -p
ignore MSRS
command: echo "options kvm ignore_msrs=1" >> /etc/modprobe.d/kvm.conf
enable nested page table on ryzen cpu
file: grub config file
GRUB_CMDLINE_LINUX_DEFAULT="quiet amd_iommu=on iommu=pt kvm_amd.npt=1"
interrupt remapping (not required on ryzen) echo "options vfio_iommu_type1 allow_unsafe_interrupts=1" > /etc/modprobe.d/vfio_iommu_type1.conf
manjaro linux file: /etc/libvirt/qemu.conf
nvram = [
"/usr/share/ovmf/x64/OVMF_CODE.fd:/usr/share/ovmf/x64/OVMF_VARS.fd"
]
libvirt domain xml:
- virtio network bridge using br103 (game vlan)
- usb controller passthrough (0b:00.3), using physical usb switch
<domain type='kvm'>
<name>dijedodol-game-vm</name>
<uuid>ea6d541b-5cd4-4ab8-b864-8605d46c3f9e</uuid>
<memory unit='KiB'>12582912</memory>
<currentMemory unit='KiB'>12582912</currentMemory>
<memoryBacking>
<hugepages/>
</memoryBacking>
<vcpu placement='static'>12</vcpu>
<os>
<type arch='x86_64' machine='pc-q35-2.12'>hvm</type>
<loader readonly='yes' type='pflash'>/usr/share/ovmf/x64/OVMF_CODE.fd</loader>
<nvram>/var/lib/libvirt/qemu/nvram/dijedodol-game-vm_VARS.fd</nvram>
<bootmenu enable='no'/>
</os>
<features>
<acpi/>
<apic/>
<hyperv>
<relaxed state='on'/>
<vapic state='on'/>
<spinlocks state='on' retries='8191'/>
<vendor_id state='on' value='durex'/>
</hyperv>
<kvm>
<hidden state='on'/>
</kvm>
<vmport state='off'/>
</features>
<cpu mode='custom' match='exact' check='partial'>
<model fallback='allow'>EPYC-IBPB</model>
<topology sockets='1' cores='12' threads='1'/>
</cpu>
<clock offset='localtime'>
<timer name='rtc' tickpolicy='catchup'/>
<timer name='pit' tickpolicy='delay'/>
<timer name='hpet' present='no'/>
<timer name='hypervclock' present='yes'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<pm>
<suspend-to-mem enabled='no'/>
<suspend-to-disk enabled='no'/>
</pm>
<devices>
<emulator>/usr/bin/qemu-system-x86_64</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='raw'/>
<source file='/var/lib/libvirt/images/dijedodol-game-vm.img'/>
<target dev='vda' bus='virtio'/>
<boot order='1'/>
<address type='pci' domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
</disk>
<disk type='file' device='cdrom'>
<driver name='qemu' type='raw'/>
<source file='/home/gde/Downloads/iso/Win10_1803_EnglishInternational_x64.iso'/>
<target dev='sda' bus='sata'/>
<readonly/>
<boot order='2'/>
<address type='drive' controller='0' bus='0' target='0' unit='0'/>
</disk>
<disk type='file' device='cdrom'>
<driver name='qemu' type='raw'/>
<source file='/home/gde/Downloads/iso/virtio-win.iso'/>
<target dev='sdb' bus='sata'/>
<readonly/>
<address type='drive' controller='0' bus='0' target='0' unit='1'/>
</disk>
<controller type='usb' index='0' model='ich9-ehci1'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1d' function='0x7'/>
</controller>
<controller type='usb' index='0' model='ich9-uhci1'>
<master startport='0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1d' function='0x0' multifunction='on'/>
</controller>
<controller type='usb' index='0' model='ich9-uhci2'>
<master startport='2'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1d' function='0x1'/>
</controller>
<controller type='usb' index='0' model='ich9-uhci3'>
<master startport='4'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1d' function='0x2'/>
</controller>
<controller type='sata' index='0'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
</controller>
<controller type='pci' index='0' model='pcie-root'/>
<controller type='pci' index='1' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='1' port='0x8'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='2' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='2' port='0x9'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
</controller>
<controller type='pci' index='3' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='3' port='0xa'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
</controller>
<controller type='pci' index='4' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='4' port='0xb'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x3'/>
</controller>
<controller type='pci' index='5' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='5' port='0xc'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x4'/>
</controller>
<controller type='pci' index='6' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='6' port='0xd'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x5'/>
</controller>
<controller type='pci' index='7' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='7' port='0xe'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x6'/>
</controller>
<controller type='pci' index='8' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='8' port='0xf'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x7'/>
</controller>
<controller type='pci' index='9' model='pcie-to-pci-bridge'>
<model name='pcie-pci-bridge'/>
<address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
</controller>
<controller type='scsi' index='0' model='lsilogic'>
<address type='pci' domain='0x0000' bus='0x09' slot='0x01' function='0x0'/>
</controller>
<interface type='network'>
<mac address='52:54:00:93:a6:e8'/>
<source network='ovs-enp4s0-vlan' portgroup='vlan-103'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
</interface>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<sound model='ich9'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1b' function='0x0'/>
</sound>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x0a' slot='0x00' function='0x0'/>
</source>
<address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x0a' slot='0x00' function='0x1'/>
</source>
<address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x0b' slot='0x00' function='0x3'/>
</source>
<address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
</hostdev>
<memballoon model='none'/>
</devices>
</domain>