Note: virtio-serial does cannot be connected to a pcie bus. It does work when connected directly to the host bridge as a transistional virtio device. This needs to be investigated
#!/bin/bash
# -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
# ex: ts=8 sw=4 sts=4 et filetype=sh
VMN=${VMN:=1}
NEMU=qemu-system-x86_64
sudo $NEMU \
-bios ~/OVMF.fd.virt \
-kernel ~/linux/arch/x86_64/boot/bzImage -append 'console=hvc0 iommu=false root=/dev/vda3 rw rootfstype=ext4 data=ordered rcupdate.rcu_expedited=1 tsc=reliable no_timer_check reboot=t noapictimer acpi.debug_layer=0xffffffff acpi.debug_level=0x2 loglevel=2' \
-nographic \
-nodefaults \
-L . \
-machine virt,accel=kvm,kernel_irqchip \
-smp sockets=1,cpus=4,cores=2,maxcpus=8 -cpu host \
-m 512 \
-monitor telnet:127.0.0.1:55555,server,nowait \
-device virtio-serial-pci,id=virtio-serial0 -device virtconsole,chardev=charconsole0,id=console0 -chardev stdio,id=charconsole0 \
-device pcie-root-port,id=rp20,bus=pcie.0,chassis=1,addr=2.0,multifunction=on,pref64-reserve=32M \
-device pcie-root-port,id=rp21,bus=pcie.0,chassis=2,addr=2.1 \
-device pcie-root-port,id=rp22,bus=pcie.0,chassis=3,addr=2.2 \
-device pcie-root-port,id=rp23,bus=pcie.0,chassis=4,addr=2.3 \
-device pcie-root-port,id=rp24,bus=pcie.0,chassis=5,addr=2.4 \
-device pcie-root-port,id=rp25,bus=pcie.0,chassis=6,addr=2.5 \
-device pcie-root-port,id=rp26,bus=pcie.0,chassis=7,addr=2.6 \
-device pcie-root-port,id=rp27,bus=pcie.0,chassis=8,addr=2.7 \
-device virtio-blk-pci,drive=clear,bus=rp27 -drive if=none,id=clear,file=~/clear.img,format=raw \
-device pcie-root-port,id=rp30,bus=pcie.0,chassis=31,addr=3.0,multifunction=on,pref64-reserve=32M \
-device pcie-root-port,id=rp31,bus=pcie.0,chassis=32,addr=3.1 \
-device pcie-root-port,id=rp32,bus=pcie.0,chassis=33,addr=3.2 \
-device pcie-root-port,id=rp33,bus=pcie.0,chassis=34,addr=3.3 \
-device pcie-root-port,id=rp34,bus=pcie.0,chassis=35,addr=3.4 \
-device pcie-root-port,id=rp35,bus=pcie.0,chassis=36,addr=3.5 \
-device pcie-root-port,id=rp36,bus=pcie.0,chassis=37,addr=3.6 \
-device pcie-root-port,id=rp37,bus=pcie.0,chassis=38,addr=3.7 \
-device virtio-net-pci,netdev=mynet0,romfile="",bus=rp31 -netdev user,id=mynet0,hostfwd=tcp::${VMN}0022-:22 \
-device pcie-root-port,id=rp50,bus=pcie.0,chassis=49,addr=5.0,multifunction=on,pref64-reserve=32M \
-device x3130-upstream,id=upstream_port1,bus=rp50 \
-device xio3130-downstream,id=downstream_port1,bus=upstream_port1,chassis=0,slot=1 \
-device xio3130-downstream,id=downstream_port2,bus=upstream_port1,chassis=0,slot=2 \
-device xio3130-downstream,id=downstream_port3,bus=upstream_port1,chassis=0,slot=3 \
-device xio3130-downstream,id=downstream_port4,bus=upstream_port1,chassis=0,slot=4 \
-device xio3130-downstream,id=downstream_port5,bus=upstream_port1,chassis=0,slot=5 \
-device xio3130-downstream,id=downstream_port6,bus=upstream_port1,chassis=0,slot=6 \
-device xio3130-downstream,id=downstream_port7,bus=upstream_port1,chassis=0,slot=7 \
-device virtio-blk-pci,drive=test,bus=downstream_port1 -drive if=none,id=test,file=~test.img,format=raw \
-device pcie-root-port,id=rp60,bus=pcie.0,chassis=50,addr=6.0,multifunction=on,pref64-reserve=32M \
-device x3130-upstream,id=upstream_port61,bus=rp60 \
-device xio3130-downstream,id=downstream_port61,bus=upstream_port61,chassis=1,slot=1 \
lspci -v shows that
Note:
- Each root port is also pcie-pcie bridge and consumes a whole bus even though it has a single device connected to it. Leading to the 256 devices limitation of q35. This is also true for upstream and downstream ports. The upstream port consumes a bus without adding a device. So just multi function root ports are a better option.
- We also explicitly reserve a larger aperture per pcie root port. If we do not do this QEMU defaults to 6MB. And 6MB is not enough to support all the root ports/downstream ports attached to the root port. Reserving 16 or 32 MB ensures that we do not have any failures.
lspci -t -vv
-[0000:00]-+-00.0 Red Hat, Inc. QEMU PCIe Host bridge
+-01.0 Red Hat, Inc. Virtio console
+-02.0-[01]--
+-02.1-[02]--
+-02.2-[03]--
+-02.3-[04]--
+-02.4-[05]--
+-02.5-[06]--
+-02.6-[07]--
+-02.7-[08]----00.0 Red Hat, Inc. Virtio block device
+-03.0-[09]--
+-03.1-[0a]----00.0 Red Hat, Inc. Virtio network device
+-03.2-[0b]--
+-03.3-[0c]--
+-03.4-[0d]--
+-03.5-[0e]--
+-03.6-[0f]--
+-03.7-[10]--
\-05.0-[11-19]----00.0-[12-19]--+-00.0-[13]----00.0 Red Hat, Inc. Virtio block device
+-01.0-[14]--
+-02.0-[15]--
+-03.0-[16]--
+-04.0-[17]--
+-05.0-[18]--
\-06.0-[19]--
lspci -v
00:00.0 Host bridge: Red Hat, Inc. QEMU PCIe Host bridge
Subsystem: Red Hat, Inc. Device 1100
Flags: bus master, fast devsel, latency 0
00:01.0 Communication controller: Red Hat, Inc. Virtio console
Subsystem: Red Hat, Inc. Device 0003
Physical Slot: 1
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
I/O ports at 1000 [size=64]
Memory at 90911000 (32-bit, non-prefetchable) [size=4K]
Memory at 800300000 (64-bit, prefetchable) [size=16K]
Capabilities: [98] MSI-X: Enable+ Count=2 Masked-
Capabilities: [84] Vendor Specific Information: VirtIO: <unknown>
Capabilities: [70] Vendor Specific Information: VirtIO: Notify
Capabilities: [60] Vendor Specific Information: VirtIO: DeviceCfg
Capabilities: [50] Vendor Specific Information: VirtIO: ISR
Capabilities: [40] Vendor Specific Information: VirtIO: CommonCfg
Kernel driver in use: virtio-pci
00:02.0 PCI bridge: Red Hat, Inc. QEMU PCIe Root port (prog-if 00 [Normal decode])
Physical Slot: 2
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 90910000 (32-bit, non-prefetchable) [size=4K]
Bus: primary=00, secondary=01, subordinate=01, sec-latency=0
I/O behind bridge: None
Memory behind bridge: 90a00000-90bfffff [size=2M]
Prefetchable memory behind bridge: 0000000800400000-00000008005fffff [size=2M]
Capabilities: [90] Vendor Specific Information: Len=20 <?>
Capabilities: [54] Express Root Port (Slot+), MSI 00
Capabilities: [48] MSI-X: Enable- Count=1 Masked-
Capabilities: [40] Subsystem: Red Hat, Inc. Device 0000
Kernel driver in use: pcieport
00:02.1 PCI bridge: Red Hat, Inc. QEMU PCIe Root port (prog-if 00 [Normal decode])
Physical Slot: 2
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 9090f000 (32-bit, non-prefetchable) [size=4K]
Bus: primary=00, secondary=02, subordinate=02, sec-latency=0
I/O behind bridge: None
Memory behind bridge: 90c00000-90dfffff [size=2M]
Prefetchable memory behind bridge: 0000000800600000-00000008007fffff [size=2M]
Capabilities: [54] Express Root Port (Slot+), MSI 00
Capabilities: [48] MSI-X: Enable- Count=1 Masked-
Capabilities: [40] Subsystem: Red Hat, Inc. Device 0000
Kernel driver in use: pcieport
00:02.2 PCI bridge: Red Hat, Inc. QEMU PCIe Root port (prog-if 00 [Normal decode])
Physical Slot: 2
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 9090e000 (32-bit, non-prefetchable) [size=4K]
Bus: primary=00, secondary=03, subordinate=03, sec-latency=0
I/O behind bridge: None
Memory behind bridge: 90e00000-90ffffff [size=2M]
Prefetchable memory behind bridge: 0000000800800000-00000008009fffff [size=2M]
Capabilities: [54] Express Root Port (Slot+), MSI 00
Capabilities: [48] MSI-X: Enable- Count=1 Masked-
Capabilities: [40] Subsystem: Red Hat, Inc. Device 0000
Kernel driver in use: pcieport
00:02.3 PCI bridge: Red Hat, Inc. QEMU PCIe Root port (prog-if 00 [Normal decode])
Physical Slot: 2
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 9090d000 (32-bit, non-prefetchable) [size=4K]
Bus: primary=00, secondary=04, subordinate=04, sec-latency=0
I/O behind bridge: None
Memory behind bridge: 91000000-911fffff [size=2M]
Prefetchable memory behind bridge: 0000000800a00000-0000000800bfffff [size=2M]
Capabilities: [54] Express Root Port (Slot+), MSI 00
Capabilities: [48] MSI-X: Enable- Count=1 Masked-
Capabilities: [40] Subsystem: Red Hat, Inc. Device 0000
Kernel driver in use: pcieport
00:02.4 PCI bridge: Red Hat, Inc. QEMU PCIe Root port (prog-if 00 [Normal decode])
Physical Slot: 2
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 9090c000 (32-bit, non-prefetchable) [size=4K]
Bus: primary=00, secondary=05, subordinate=05, sec-latency=0
I/O behind bridge: None
Memory behind bridge: 91200000-913fffff [size=2M]
Prefetchable memory behind bridge: 0000000800c00000-0000000800dfffff [size=2M]
Capabilities: [54] Express Root Port (Slot+), MSI 00
Capabilities: [48] MSI-X: Enable- Count=1 Masked-
Capabilities: [40] Subsystem: Red Hat, Inc. Device 0000
Kernel driver in use: pcieport
00:02.5 PCI bridge: Red Hat, Inc. QEMU PCIe Root port (prog-if 00 [Normal decode])
Physical Slot: 2
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 9090b000 (32-bit, non-prefetchable) [size=4K]
Bus: primary=00, secondary=06, subordinate=06, sec-latency=0
I/O behind bridge: None
Memory behind bridge: 91400000-915fffff [size=2M]
Prefetchable memory behind bridge: 0000000800e00000-0000000800ffffff [size=2M]
Capabilities: [54] Express Root Port (Slot+), MSI 00
Capabilities: [48] MSI-X: Enable- Count=1 Masked-
Capabilities: [40] Subsystem: Red Hat, Inc. Device 0000
Kernel driver in use: pcieport
00:02.6 PCI bridge: Red Hat, Inc. QEMU PCIe Root port (prog-if 00 [Normal decode])
Physical Slot: 2
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 9090a000 (32-bit, non-prefetchable) [size=4K]
Bus: primary=00, secondary=07, subordinate=07, sec-latency=0
I/O behind bridge: None
Memory behind bridge: 91600000-917fffff [size=2M]
Prefetchable memory behind bridge: 0000000801000000-00000008011fffff [size=2M]
Capabilities: [54] Express Root Port (Slot+), MSI 00
Capabilities: [48] MSI-X: Enable- Count=1 Masked-
Capabilities: [40] Subsystem: Red Hat, Inc. Device 0000
Kernel driver in use: pcieport
00:02.7 PCI bridge: Red Hat, Inc. QEMU PCIe Root port (prog-if 00 [Normal decode])
Physical Slot: 2
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 90909000 (32-bit, non-prefetchable) [size=4K]
Bus: primary=00, secondary=08, subordinate=08, sec-latency=0
I/O behind bridge: None
Memory behind bridge: 90800000-908fffff [size=1M]
Prefetchable memory behind bridge: 0000000800000000-00000008000fffff [size=1M]
Capabilities: [54] Express Root Port (Slot+), MSI 00
Capabilities: [48] MSI-X: Enable- Count=1 Masked-
Capabilities: [40] Subsystem: Red Hat, Inc. Device 0000
Kernel driver in use: pcieport
00:03.0 PCI bridge: Red Hat, Inc. QEMU PCIe Root port (prog-if 00 [Normal decode])
Physical Slot: 3
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 90908000 (32-bit, non-prefetchable) [size=4K]
Bus: primary=00, secondary=09, subordinate=09, sec-latency=0
I/O behind bridge: None
Memory behind bridge: 91800000-919fffff [size=2M]
Prefetchable memory behind bridge: 0000000801200000-00000008013fffff [size=2M]
Capabilities: [90] Vendor Specific Information: Len=20 <?>
Capabilities: [54] Express Root Port (Slot+), MSI 00
Capabilities: [48] MSI-X: Enable- Count=1 Masked-
Capabilities: [40] Subsystem: Red Hat, Inc. Device 0000
Kernel driver in use: pcieport
00:03.1 PCI bridge: Red Hat, Inc. QEMU PCIe Root port (prog-if 00 [Normal decode])
Physical Slot: 3
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 90907000 (32-bit, non-prefetchable) [size=4K]
Bus: primary=00, secondary=0a, subordinate=0a, sec-latency=0
I/O behind bridge: None
Memory behind bridge: 90700000-907fffff [size=1M]
Prefetchable memory behind bridge: 0000000800100000-00000008001fffff [size=1M]
Capabilities: [54] Express Root Port (Slot+), MSI 00
Capabilities: [48] MSI-X: Enable- Count=1 Masked-
Capabilities: [40] Subsystem: Red Hat, Inc. Device 0000
Kernel driver in use: pcieport
00:03.2 PCI bridge: Red Hat, Inc. QEMU PCIe Root port (prog-if 00 [Normal decode])
Physical Slot: 3
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 90906000 (32-bit, non-prefetchable) [size=4K]
Bus: primary=00, secondary=0b, subordinate=0b, sec-latency=0
I/O behind bridge: None
Memory behind bridge: 91a00000-91bfffff [size=2M]
Prefetchable memory behind bridge: 0000000801400000-00000008015fffff [size=2M]
Capabilities: [54] Express Root Port (Slot+), MSI 00
Capabilities: [48] MSI-X: Enable- Count=1 Masked-
Capabilities: [40] Subsystem: Red Hat, Inc. Device 0000
Kernel driver in use: pcieport
00:03.3 PCI bridge: Red Hat, Inc. QEMU PCIe Root port (prog-if 00 [Normal decode])
Physical Slot: 3
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 90905000 (32-bit, non-prefetchable) [size=4K]
Bus: primary=00, secondary=0c, subordinate=0c, sec-latency=0
I/O behind bridge: None
Memory behind bridge: 91c00000-91dfffff [size=2M]
Prefetchable memory behind bridge: 0000000801600000-00000008017fffff [size=2M]
Capabilities: [54] Express Root Port (Slot+), MSI 00
Capabilities: [48] MSI-X: Enable- Count=1 Masked-
Capabilities: [40] Subsystem: Red Hat, Inc. Device 0000
Kernel driver in use: pcieport
00:03.4 PCI bridge: Red Hat, Inc. QEMU PCIe Root port (prog-if 00 [Normal decode])
Physical Slot: 3
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 90904000 (32-bit, non-prefetchable) [size=4K]
Bus: primary=00, secondary=0d, subordinate=0d, sec-latency=0
I/O behind bridge: None
Memory behind bridge: 91e00000-91ffffff [size=2M]
Prefetchable memory behind bridge: 0000000801800000-00000008019fffff [size=2M]
Capabilities: [54] Express Root Port (Slot+), MSI 00
Capabilities: [48] MSI-X: Enable- Count=1 Masked-
Capabilities: [40] Subsystem: Red Hat, Inc. Device 0000
Kernel driver in use: pcieport
00:03.5 PCI bridge: Red Hat, Inc. QEMU PCIe Root port (prog-if 00 [Normal decode])
Physical Slot: 3
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 90903000 (32-bit, non-prefetchable) [size=4K]
Bus: primary=00, secondary=0e, subordinate=0e, sec-latency=0
I/O behind bridge: None
Memory behind bridge: 92000000-921fffff [size=2M]
Prefetchable memory behind bridge: 0000000801a00000-0000000801bfffff [size=2M]
Capabilities: [54] Express Root Port (Slot+), MSI 00
Capabilities: [48] MSI-X: Enable- Count=1 Masked-
Capabilities: [40] Subsystem: Red Hat, Inc. Device 0000
Kernel driver in use: pcieport
00:03.6 PCI bridge: Red Hat, Inc. QEMU PCIe Root port (prog-if 00 [Normal decode])
Physical Slot: 3
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 90902000 (32-bit, non-prefetchable) [size=4K]
Bus: primary=00, secondary=0f, subordinate=0f, sec-latency=0
I/O behind bridge: None
Memory behind bridge: 92200000-923fffff [size=2M]
Prefetchable memory behind bridge: 0000000801c00000-0000000801dfffff [size=2M]
Capabilities: [54] Express Root Port (Slot+), MSI 00
Capabilities: [48] MSI-X: Enable- Count=1 Masked-
Capabilities: [40] Subsystem: Red Hat, Inc. Device 0000
Kernel driver in use: pcieport
00:03.7 PCI bridge: Red Hat, Inc. QEMU PCIe Root port (prog-if 00 [Normal decode])
Physical Slot: 3
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 90901000 (32-bit, non-prefetchable) [size=4K]
Bus: primary=00, secondary=10, subordinate=10, sec-latency=0
I/O behind bridge: 00009000-00009fff [size=4K]
Memory behind bridge: 92400000-925fffff [size=2M]
Prefetchable memory behind bridge: 0000000801e00000-0000000801ffffff [size=2M]
Capabilities: [54] Express Root Port (Slot+), MSI 00
Capabilities: [48] MSI-X: Enable- Count=1 Masked-
Capabilities: [40] Subsystem: Red Hat, Inc. Device 0000
Kernel driver in use: pcieport
00:05.0 PCI bridge: Red Hat, Inc. QEMU PCIe Root port (prog-if 00 [Normal decode])
Physical Slot: 5
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 90900000 (32-bit, non-prefetchable) [size=4K]
Bus: primary=00, secondary=11, subordinate=19, sec-latency=0
I/O behind bridge: 00002000-00008fff [size=28K]
Memory behind bridge: 90000000-906fffff [size=7M]
Prefetchable memory behind bridge: 0000000800200000-00000008002fffff [size=1M]
Capabilities: [90] Vendor Specific Information: Len=20 <?>
Capabilities: [54] Express Root Port (Slot+), MSI 00
Capabilities: [48] MSI-X: Enable- Count=1 Masked-
Capabilities: [40] Subsystem: Red Hat, Inc. Device 0000
Kernel driver in use: pcieport
08:00.0 SCSI storage controller: Red Hat, Inc. Virtio block device (rev 01)
Subsystem: Red Hat, Inc. Device 1100
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 90800000 (32-bit, non-prefetchable) [size=4K]
[virtual] Memory at 800000000 (64-bit, prefetchable) [size=16K]
Capabilities: [dc] MSI-X: Enable+ Count=2 Masked-
Capabilities: [c8] Vendor Specific Information: VirtIO: <unknown>
Capabilities: [b4] Vendor Specific Information: VirtIO: Notify
Capabilities: [a4] Vendor Specific Information: VirtIO: DeviceCfg
Capabilities: [94] Vendor Specific Information: VirtIO: ISR
Capabilities: [84] Vendor Specific Information: VirtIO: CommonCfg
Capabilities: [7c] Power Management version 3
Capabilities: [40] Express Endpoint, MSI 00
Kernel driver in use: virtio-pci
0a:00.0 Ethernet controller: Red Hat, Inc. Virtio network device (rev 01)
Subsystem: Red Hat, Inc. Device 1100
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 90700000 (32-bit, non-prefetchable) [size=4K]
Memory at 800100000 (64-bit, prefetchable) [size=16K]
Capabilities: [dc] MSI-X: Enable+ Count=3 Masked-
Capabilities: [c8] Vendor Specific Information: VirtIO: <unknown>
Capabilities: [b4] Vendor Specific Information: VirtIO: Notify
Capabilities: [a4] Vendor Specific Information: VirtIO: DeviceCfg
Capabilities: [94] Vendor Specific Information: VirtIO: ISR
Capabilities: [84] Vendor Specific Information: VirtIO: CommonCfg
Capabilities: [7c] Power Management version 3
Capabilities: [40] Express Endpoint, MSI 00
Kernel driver in use: virtio-pci
11:00.0 PCI bridge: Texas Instruments XIO3130 PCI Express Switch (Upstream) (rev 02) (prog-if 00 [Normal decode])
Flags: bus master, fast devsel, latency 0
Bus: primary=11, secondary=12, subordinate=19, sec-latency=0
I/O behind bridge: 00002000-00008fff [size=28K]
Memory behind bridge: 90000000-906fffff [size=7M]
Prefetchable memory behind bridge: 0000000800200000-00000008002fffff [size=1M]
Capabilities: [90] Express Upstream Port, MSI 00
Capabilities: [80] Subsystem: Device 0000:0000
Capabilities: [70] MSI: Enable- Count=1/1 Maskable- 64bit+
Kernel driver in use: pcieport
12:00.0 PCI bridge: Texas Instruments XIO3130 PCI Express Switch (Downstream) (rev 01) (prog-if 00 [Normal decode])
Flags: bus master, fast devsel, latency 0
Bus: primary=12, secondary=13, subordinate=13, sec-latency=0
I/O behind bridge: 00002000-00002fff [size=4K]
Memory behind bridge: 90600000-906fffff [size=1M]
Prefetchable memory behind bridge: 0000000800200000-00000008002fffff [size=1M]
Capabilities: [90] Express Downstream Port (Slot+), MSI 00
Capabilities: [80] Subsystem: Device 0000:0000
Capabilities: [70] MSI: Enable- Count=1/1 Maskable- 64bit+
Kernel driver in use: pcieport
12:01.0 PCI bridge: Texas Instruments XIO3130 PCI Express Switch (Downstream) (rev 01) (prog-if 00 [Normal decode])
Flags: bus master, fast devsel, latency 0
Bus: primary=12, secondary=14, subordinate=14, sec-latency=0
I/O behind bridge: 00003000-00003fff [size=4K]
Memory behind bridge: None
Prefetchable memory behind bridge: None
Capabilities: [90] Express Downstream Port (Slot+), MSI 00
Capabilities: [80] Subsystem: Device 0000:0000
Capabilities: [70] MSI: Enable- Count=1/1 Maskable- 64bit+
Kernel driver in use: pcieport
12:02.0 PCI bridge: Texas Instruments XIO3130 PCI Express Switch (Downstream) (rev 01) (prog-if 00 [Normal decode])
Flags: bus master, fast devsel, latency 0
Bus: primary=12, secondary=15, subordinate=15, sec-latency=0
I/O behind bridge: 00004000-00004fff [size=4K]
Memory behind bridge: None
Prefetchable memory behind bridge: None
Capabilities: [90] Express Downstream Port (Slot+), MSI 00
Capabilities: [80] Subsystem: Device 0000:0000
Capabilities: [70] MSI: Enable- Count=1/1 Maskable- 64bit+
Kernel driver in use: pcieport
12:03.0 PCI bridge: Texas Instruments XIO3130 PCI Express Switch (Downstream) (rev 01) (prog-if 00 [Normal decode])
Flags: bus master, fast devsel, latency 0
Bus: primary=12, secondary=16, subordinate=16, sec-latency=0
I/O behind bridge: 00005000-00005fff [size=4K]
Memory behind bridge: None
Prefetchable memory behind bridge: None
Capabilities: [90] Express Downstream Port (Slot+), MSI 00
Capabilities: [80] Subsystem: Device 0000:0000
Capabilities: [70] MSI: Enable- Count=1/1 Maskable- 64bit+
Kernel driver in use: pcieport
12:04.0 PCI bridge: Texas Instruments XIO3130 PCI Express Switch (Downstream) (rev 01) (prog-if 00 [Normal decode])
Flags: bus master, fast devsel, latency 0
Bus: primary=12, secondary=17, subordinate=17, sec-latency=0
I/O behind bridge: 00006000-00006fff [size=4K]
Memory behind bridge: 90400000-905fffff [size=2M]
Prefetchable memory behind bridge: None
Capabilities: [90] Express Downstream Port (Slot+), MSI 00
Capabilities: [80] Subsystem: Device 0000:0000
Capabilities: [70] MSI: Enable- Count=1/1 Maskable- 64bit+
Kernel driver in use: pcieport
12:05.0 PCI bridge: Texas Instruments XIO3130 PCI Express Switch (Downstream) (rev 01) (prog-if 00 [Normal decode])
Flags: bus master, fast devsel, latency 0
Bus: primary=12, secondary=18, subordinate=18, sec-latency=0
I/O behind bridge: 00007000-00007fff [size=4K]
Memory behind bridge: 90200000-903fffff [size=2M]
Prefetchable memory behind bridge: None
Capabilities: [90] Express Downstream Port (Slot+), MSI 00
Capabilities: [80] Subsystem: Device 0000:0000
Capabilities: [70] MSI: Enable- Count=1/1 Maskable- 64bit+
Kernel driver in use: pcieport
12:06.0 PCI bridge: Texas Instruments XIO3130 PCI Express Switch (Downstream) (rev 01) (prog-if 00 [Normal decode])
Flags: bus master, fast devsel, latency 0
Bus: primary=12, secondary=19, subordinate=19, sec-latency=0
I/O behind bridge: 00008000-00008fff [size=4K]
Memory behind bridge: 90000000-901fffff [size=2M]
Prefetchable memory behind bridge: None
Capabilities: [90] Express Downstream Port (Slot+), MSI 00
Capabilities: [80] Subsystem: Device 0000:0000
Capabilities: [70] MSI: Enable- Count=1/1 Maskable- 64bit+
Kernel driver in use: pcieport
13:00.0 SCSI storage controller: Red Hat, Inc. Virtio block device (rev 01)
Subsystem: Red Hat, Inc. Device 1100
Flags: bus master, fast devsel, latency 0, IRQ -2147483648
Memory at 90600000 (32-bit, non-prefetchable) [size=4K]
Memory at 800200000 (64-bit, prefetchable) [size=16K]
Capabilities: [dc] MSI-X: Enable+ Count=2 Masked-
Capabilities: [c8] Vendor Specific Information: VirtIO: <unknown>
Capabilities: [b4] Vendor Specific Information: VirtIO: Notify
Capabilities: [a4] Vendor Specific Information: VirtIO: DeviceCfg
Capabilities: [94] Vendor Specific Information: VirtIO: ISR
Capabilities: [84] Vendor Specific Information: VirtIO: CommonCfg
Capabilities: [7c] Power Management version 3
Capabilities: [40] Express Endpoint, MSI 00
Kernel driver in use: virtio-pci
nc -N 127.0.0.1 55555
drive_add 0 if=none,file=./test.img,format=raw,id=test
device_add virtio-blk-pci,drive=test,bus=downstream_port3
or
device_add virtio-blk-pci,drive=test,bus=rp46