Enable SR-IOV VF on Proxmox 7+

pve-trick-sriov-vf-configuration

# The following is a step-by-step example for enabling SR-IOV VF on Promxox 7+ with BCM57810 10G NIC

---
References

https://forum.proxmox.com/threads/enabling-sr-iov-for-intel-nic-x550-t2-on-proxmox-6.56677/
https://zhuanlan.zhihu.com/p/91197211
https://zhuanlan.zhihu.com/p/356437308
https://zhiliao.h3c.com/Theme/details/24770
https://blog.csdn.net/Jackykxy/article/details/120585563

---
Update grub to enable SR-IOV

# update the following line in /etc/default/grub
vim /etc/default/grub
Intel：
GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on pci=realloc iommu=pt pci=assign-busses pcie_acs_override=downstream,multifunction nofb textonly nomodeset initcall_blacklist=sysfb_init"

AMD：
GRUB_CMDLINE_LINUX_DEFAULT="quiet amd_iommu=on pci=realloc iommu=pt pci=assign-busses pcie_acs_override=downstream,multifunction nofb textonly nomodeset initcall_blacklist=sysfb_init"

update-grub

# Local modules
vim /etc/modules

# add the following modules
vfio
vfio_iommu_type1
vfio_pci
vfio_virqfd

# IOMMU interrupt remapping
echo "options vfio_iommu_type1 allow_unsafe_interrupts=1" > /etc/modprobe.d/iommu_unsafe_interrupts.conf
echo "options kvm ignore_msrs=1" > /etc/modprobe.d/kvm.conf

# Update kernel
update-initramfs -u

# Reboot
reboot

---
Inspect SR-IOV Capabilities

# After enabling SR-IOV in BIOS, you should be able to see the SR-IOV capabilities
lspci -s "xx:xx" -vvv

---
Instantiate VFs

echo 8 > /sys/class/net/enp67s0f0/device/sriov_numvfs

# verify result
lspci | grep –i ethernet
ip addr

---
Create SR-IOV daemon service

# /etc/systemd/system/sriov.service
[Unit]
Description=Script to enable SR-IOV on boot

[Service]
Type=simple
# set PF UP
ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0 up'
ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f1 up'
# start SR-IOV
ExecStartPre=/usr/bin/bash -c '/usr/bin/echo 8 > /sys/class/net/enp67s0f0/device/sriov_numvfs'
# ExecStartPre=/usr/bin/bash -c '/usr/bin/echo 8 > /sys/class/net/enp67s0f1/device/sriov_numvfs'
# set VF MAC
ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v0 addr 24:8a:07:b7:a8:01'
ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v1 addr 24:8a:07:b7:a8:02'
ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v2 addr 24:8a:07:b7:a8:03'
ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v3 addr 24:8a:07:b7:a8:04'
ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v4 addr 24:8a:07:b7:a8:05'
ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v5 addr 24:8a:07:b7:a8:06'
ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v6 addr 24:8a:07:b7:a8:07'
ExecStartPre=/usr/bin/bash -c 'ip link set enp67s0f0v7 addr 24:8a:07:b7:a8:08'
# set VF UP
ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0v0 up'
ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0v1 up'
ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0v2 up'
ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0v3 up'
ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0v4 up'
ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0v5 up'
ExecStartPre=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0v6 up'
ExecStart=/usr/bin/bash -c '/usr/bin/ip link set enp67s0f0v7 up'
Restart=on-failure

[Install]
WantedBy=multi-user.target

systemctl daemon-reload
systemctl enable --now sriov.service
reboot
reboot

---
Inspect system logs

dmesg | grep enp*s*f*

---
/etc/netplan example

# /etc/netplan/main.yaml
network:
  ethernets:
    ens16:
      macaddress: 24:8a:07:b7:a8:08
      addresses:
      - 10.178.0.56/24
      gateway4: 10.178.0.5
      dhcp: false
      dhcp6: false
      link-local: []
      nameservers:
        addresses:
        - 10.178.0.5

  vlans:
    vlan10:
      dhcp4: true
      dhcp4-overrides:
          route-metric: 200
      id: 10
      link: eno1
      macaddress: 00:00:00:00:00:10

    vlan20:
      dhcp4: true
      dhcp4-overrides:
          route-metric: 500