fedora 30 virt server setup to be used with macvlan - virtualbmc is also installed and setup as a daemon.

fedora-virt-setup.sh

#hostname setup
hostnamectl --static --pretty --transient set-hostname compute2.rk-home


# Storage for my nvme drive
parted --script --align=optimal /dev/nvme0n1 mklabel gpt -- mkpart libvirt XFS  0% 100%
parted --script /dev/nvme0n1 print

mkfs.xfs -f -i size=2048 -L libvirt /dev/nvme0n1


# Storage for my sata drive
parted --script --align=optimal /dev/sdb mklabel gpt -- mkpart storage XFS  0% 100%
parted --script /dev/sdb print

mkfs.xfs -f -i size=2048 -L storage /dev/sdb1


# Install and enable libvirt
dnf -y install python3-virtualenv nfs-utils ipmitool OpenIPMI @virtualization

systemctl start libvirtd
systemctl enable libvirtd


# Create mount to install storage server
mkdir /mnt/rhv
cat > /etc/systemd/system/mnt-rhv.mount <<EOF
[Unit]
Description=nfs devices
After=network.target

[Mount]
What=172.16.24.92:/mnt/flix/media/rhv
Where=/mnt/rhv
Type=nfs
Options=_netdev,auto

[Install]
WantedBy=multi-user.target
EOF


# Create mount for local drives
cat > /etc/systemd/system/var-lib-libvirt-images.mount <<EOF
[Unit]
Description=libvirt devices
After=network.target

[Mount]
What=/dev/disk/by-uuid/$(blkid | awk '/libvirt/ {print $3}' | awk -F'"' '{print $2}')
Where=/var/lib/libvirt/images
Type=xfs
Options=noatime,nodiratime

[Install]
WantedBy=multi-user.target
EOF

cat > /etc/systemd/system/srv.mount <<EOF
[Unit]
Description=srv devices
After=network.target

[Mount]
What=/dev/disk/by-uuid/$(blkid | awk '/storage/ {print $3}' | awk -F'"' '{print $2}')
Where=/srv
Type=xfs
Options=noatime,nodiratime

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload

systemctl enable mnt-rhv.mount
systemctl enable srv.mount
systemctl enable var-lib-libvirt-images.mount


systemctl start mnt-rhv.mount
systemctl start srv.mount
systemctl start var-lib-libvirt-images.mount


# Create module config options
cat > /etc/modules-load.d/qemu-server.conf <<EOF
vhost_net
EOF

cat > /etc/modules-load.d/virtio.conf <<EOF
virtio_balloon
virtio_blk
virtio_net
virtio_ring
virtio_rng
virtio_scsi
virtio
EOF

cat > /etc/modules-load.d/8021q.conf <<EOF
8021q
EOF

cat > /etc/modprobe.d/kvm.conf <<EOF
options kvm-amd nested=1
EOF


# Host network setup
nmcli con add type vlan ifname BOND0-VLAN10 dev bond0 id 10 ip4 172.17.27.224/22
nmcli con up vlan-BOND0-VLAN10


# fix resolvers
ln -sf /var/run/systemd/resolve/resolv.conf /etc/resolv.conf
systemctl enable systemd-resolved
systemctl start systemd-resolved


# libvirt setup
cat > /etc/libvirt/qemu/networks/external.xml <<EOF
<network>
  <name>external</name>
  <forward mode="bridge">
    <interface dev="bond0"/>
  </forward>
</network>
EOF

cat > /etc/libvirt/qemu/networks/provisioning.xml <<EOF
<network>
  <name>provisioning</name>
  <forward mode="bridge">
    <interface dev="BOND0-VLAN10"/>
  </forward>
</network>
EOF

virsh net-destroy default
virsh net-undefine default
virsh net-define /etc/libvirt/qemu/networks/external.xml
virsh net-autostart external
virsh net-start external
virsh net-define /etc/libvirt/qemu/networks/provisioning.xml
virsh net-autostart provisioning
virsh net-start provisioning
virsh net-list


# Setup vbmc service
python3 -m virtualenv --system-site-packages --download /opt/vbmc
/opt/vbmc/bin/pip install virtualbmc


cat > /etc/systemd/system/vbmcd.service <<EOF
[Install]
WantedBy = multi-user.target
[Service]
BlockIOAccounting = True
CPUAccounting = True
ExecReload = /bin/kill -HUP $MAINPID
ExecStart = /opt/vbmc/bin/vbmcd --foreground
Group = root
MemoryAccounting = True
PrivateDevices = False
PrivateNetwork = False
PrivateTmp = False
PrivateUsers = False
Restart = on-failure
RestartSec = 2
Slice = vbmc.slice
TasksAccounting = True
TimeoutSec = 120
Type = simple
User = root
[Unit]
After = libvirtd.service
After = syslog.target
After = network.target
Description = vbmc service
EOF

systemctl daemon-reload
systemctl enable vbmcd.service
systemctl start vbmcd.service

echo -e "\n\nReboot now for the changes to take effect\n"

virt-lab-create.sh

# Lab virt create

for i in {1..3}; do
  virsh destroy controller-node$i || true
  virsh undefine controller-node$i || true
  
  qemu-img create -f qcow2 /var/lib/libvirt/images/controller-node$i.qcow2 96G

  virt-install --ram 8192 \
               --vcpus=8,sockets=1,cores=8 \
               --os-variant rhel7 \
               --disk path=/var/lib/libvirt/images/controller-node$i.qcow2,device=disk,bus=virtio,format=qcow2,discard=unmap \
               --graphics vnc \
               --network network:provisioning,model=virtio,mac=52:54:00:e6:db:0$i \
               --network network:external,model=virtio,mac=52:54:00:e6:b4:0$i \
               --name controller-node$i \
               --hvm \
               --virt-type kvm \
               --memballoon virtio \
               --cpu host \
               --dry-run \
               --rng /dev/urandom \
               --print-xml > /etc/libvirt/qemu/controller-node$i.xml

  virsh define --file /etc/libvirt/qemu/controller-node$i.xml
  
  /opt/vbmc/bin/vbmc delete controller-node$i || true
  /opt/vbmc/bin/vbmc add controller-node$i --port 1600$i \
                                           --username admin \
                                           --password secrete

  /opt/vbmc/bin/vbmc start controller-node$i

  ipmitool -I lanplus -U admin -P secrete -H 127.0.0.1 -p 1600$i power status
done

for i in {1..2}; do
  virsh destroy compute-node$i || true
  virsh undefine compute-node$i || true

  qemu-img create -f qcow2 /var/lib/libvirt/images/compute-node$i.qcow2 96G

  virt-install --ram 8192 \
               --vcpus=8,sockets=1,cores=8 \
               --os-variant rhel7 \
               --disk path=/var/lib/libvirt/images/compute-node$i.qcow2,device=disk,bus=virtio,format=qcow2,discard=unmap \
               --graphics vnc \
               --network network:provisioning,model=virtio,mac=52:54:00:66:db:0$i \
               --network network:external,model=virtio,mac=52:54:00:66:b4:0$i \
               --name compute-node$i \
               --hvm \
               --virt-type kvm \
               --memballoon virtio \
               --cpu host \
               --dry-run \
               --rng /dev/urandom \
               --print-xml > /etc/libvirt/qemu/compute-node$i.xml

  virsh define --file /etc/libvirt/qemu/compute-node$i.xml
  
  /opt/vbmc/bin/vbmc delete compute-node$i || true
  /opt/vbmc/bin/vbmc add compute-node$i --port 1601$i \
                                        --username admin \
                                        --password secrete

  /opt/vbmc/bin/vbmc start compute-node$i

  ipmitool -I lanplus -U admin -P secrete -H 127.0.0.1 -p 1601$i power status
done