windows.sh

#!/bin/bash
MEMORY_MB_VLOW=8192
MEMORY_MB_LOW=32768
MEMORY_MB_NORMAL=65536
MEMORY_MB_HIGH=122880

SINGLE_SOCKET_CPU_CORES=18
TOTAL_CORES_MASK=ff,ffffffff,ffffffff
HOST_CORES_MASK=3ffff0,3ffff # 0b1111111111111111110000,0b00000000000000111111111111111111

PIN_CPU_CORES=10
PIN_CPU_THREADS=1
PIN_CPU_OFFSET=18

SMP_ALL="sockets=2,cores=16,threads=2"
SMP_PIN="sockets=1,cores=${PIN_CPU_CORES},threads=${PIN_CPU_THREADS}"

ALL_AFFINITY="2-17,20-35,38-53,56-71"
PIN_AFFINITY="${SINGLE_SOCKET_CPU_CORES}-$((${SINGLE_SOCKET_CPU_CORES} + ${PIN_CPU_CORES} - 1)),$((${SINGLE_SOCKET_CPU_CORES} * 3))-$((${SINGLE_SOCKET_CPU_CORES} * 3 + ${PIN_CPU_CORES} - 1))"

# UDP 27031 and 27036 (incoming, for In-Home Streaming)
# TCP 27036 and 27037 (incoming, for In-Home Streaming)

# QEMU 4.0?
# AUDIO_OPTS='QEMU_AUDIO_TIMER_PERIOD=100 QEMU_PA_SAMPLES=256'

GUEST_IP=192.168.122.30
TCP_EXPOSE=47984,47989,48010,65000,27031,27036
UDP_EXPOSE=5353,47998,47999,48000,48002,48010,65000,27036,27037

NORMAL_DRIVE_OPT='if=virtio,id=drive0,file=/media/a80b94ea-e789-4a40-8f7a-3813cddf7390/vm-images/windows-10/windows-10.qcow2,cache=writeback,l2-cache-size=32M,aio=threads'

YES_LP_OPT='-device usb-host,vendorid=0x1d50,productid=0x6022,id=lightpack'
NO_LP_OPT=''

# GPU_ROMFILE_OPT=',rombar=1,romfile=/media/3fc1e28a-25ad-4dfe-8f7d-af33716ec324/Downloads/XFX.RXVega64.8176.170730_1.rom'

echo "========== Windows 10 VM launcher (GPU Passthrough) =========="
echo -n "CPU Pinning? ${PIN_CPU_CORES}c$((${PIN_CPU_CORES} * ${PIN_CPU_THREADS}))t(pin)/32c64t. (1)/2: "
read -n 1 cpu
if [ "$cpu" == "2" ]; then
  smp=${SMP_ALL}
  aff=${ALL_AFFINITY}
  PIN_CSET_AFFINITY=${ALL_AFFINITY}
else
  smp=${SMP_PIN}
  aff=${PIN_AFFINITY}
  if [ "$PIN_CPU_THREADS" == "2" ]; then
    PIN_QEMU_SPARE_CORES=0
    PIN_CSET_AFFINITY="${SINGLE_SOCKET_CPU_CORES}-$((${SINGLE_SOCKET_CPU_CORES} + ${PIN_CPU_CORES} + ${PIN_QEMU_SPARE_CORES} - 1)),$((${SINGLE_SOCKET_CPU_CORES} * 3))-$((${SINGLE_SOCKET_CPU_CORES} * 3 + ${PIN_CPU_CORES} + ${PIN_QEMU_SPARE_CORES} - 1))"
    # echo PIN_CSET_AFFINITY=$PIN_CSET_AFFINITY
    # PIN_CSET_AFFINITY="${SINGLE_SOCKET_CPU_CORES}-$((${SINGLE_SOCKET_CPU_CORES} + ${PIN_CPU_CORES} - 1)),$((${SINGLE_SOCKET_CPU_CORES} * 3))-$((${SINGLE_SOCKET_CPU_CORES} * 3 + ${PIN_CPU_CORES} - 1))"
  else
    PIN_QEMU_SPARE_CORES=0
    PIN_CSET_AFFINITY=${PIN_AFFINITY}
  fi
fi
echo

echo -n "Memory size in MB? ${MEMORY_MB_VLOW}/${MEMORY_MB_LOW}/${MEMORY_MB_NORMAL}/${MEMORY_MB_HIGH}. 0/(1)/2/3: "
read -n 1 size
if [ "$size" == "2" ]; then
  mem=${MEMORY_MB_NORMAL}
elif [ "$size" == "3" ]; then
  mem=${MEMORY_MB_HIGH}
elif [ "$size" == "0" ]; then
  mem=${MEMORY_MB_VLOW}
else
  mem=${MEMORY_MB_LOW}
fi
echo

echo -n "Use hugepages? (y)/n: "
read -n 1 opt
if [ "$opt" == "n" ]; then
  hugepages_opt=''
else
  hugepages_opt='-mem-path /dev/hugepages'
fi
echo

drive=${NORMAL_DRIVE_OPT}

echo -n "Passthrough lightpack? (y)/n: "
read -n 1 lp
if [ "$lp" == "n" ]; then
  lightpack=${NO_LP_OPT}
else
  lightpack=${YES_LP_OPT}
fi
echo

echo "Configuring network"
sudo virsh net-start default > /dev/null 2>&1
sudo ip tuntap add dev tap0 mode tap > /dev/null 2>&1
sudo ip link set tap0 up promisc on > /dev/null 2>&1
sudo brctl addif virbr0 tap0 > /dev/null 2>&1
sudo iptables -C FORWARD -m state -d 192.168.122.0/24 --state NEW,RELATED,ESTABLISHED -j ACCEPT > /dev/null 2>&1 || sudo iptables -I FORWARD -m state -d 192.168.122.0/24 --state NEW,RELATED,ESTABLISHED -j ACCEPT
sudo iptables -t nat -C PREROUTING -p tcp -m multiport --dports ${TCP_EXPOSE} -j DNAT --to ${GUEST_IP} > /dev/null 2>&1 || sudo iptables -t nat -A PREROUTING -p tcp -m multiport --dports ${TCP_EXPOSE} -j DNAT --to ${GUEST_IP}
sudo iptables -t nat -C PREROUTING -p udp -m multiport --dports ${UDP_EXPOSE} -j DNAT --to ${GUEST_IP} > /dev/null 2>&1 || sudo iptables -t nat -A PREROUTING -p udp -m multiport --dports ${UDP_EXPOSE} -j DNAT --to ${GUEST_IP}

if [ "$lp" != "n" ]; then
  echo "Stopping prismatik"
  pkill prismatik
fi;

echo "Starting Synergy"
(while true; do if [ $(xrandr | grep current | awk '{print $8}') -eq "1720" ]; then config="-pip"; else config=""; fi; synergys -f -n host --config /etc/synergy${config}.conf > /dev/null 2>&1; done)&

echo "Starting samba"
sudo systemctl start smb

if [ "$hugepages_opt" != "" ]; then
  cur=$(cat /proc/sys/vm/nr_hugepages)
  hugepages=$(((${mem} / 2) + ${cur}))
  syms='|/—\'
  idx=0
  while true; do
    if [ "$cpu" == "2" ]; then
      echo -e -n "\e[0K\rAllocating hugepages value of ${hugepages} ${syms:${idx}:1}"
      sudo bash -c "echo ${hugepages} > /proc/sys/vm/nr_hugepages"
      if [[ "$(cat /proc/sys/vm/nr_hugepages)" == "${hugepages}" ]]; then break; fi
    else
      echo -e -n "\e[0K\rAllocating hugepages value of ${hugepages} (node1) ${syms:${idx}:1}"
      sudo bash -c "echo ${hugepages} > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages"
      if [[ "$(cat /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages)" == "${hugepages}" ]]; then break; fi
    fi
    idx=$(($idx + 1))
    if [[ ${idx} == "4" ]]; then idx=0; fi
    sudo bash -c "echo 3 > /proc/sys/vm/drop_caches"
    sudo bash -c "echo 1 > /proc/sys/vm/compact_memory"
  done
  echo
fi

# Make CPU threads exclusive to VM
echo "Shielding CPU cores with aff: ${PIN_CSET_AFFINITY}"
sudo cset shield --reset
sudo cset set --destroy --set=system
sudo cset shield --cpu ${PIN_CSET_AFFINITY}
sudo cset shield --kthread on
if [ "$smp" == "${SMP_PIN}" ]; then
  sudo cset set --mem=0 --set=system
  sudo cset set --mem=1 --set=user
else
  sudo cset set --mem=0,1 --set=system
  sudo cset set --mem=0,1 --set=user
fi

all_params=`python3 - <<END
if __name__ == '__main__':
    aff = '${ALL_AFFINITY}'
    sets = aff.split(',')
    args = []
    nodes_count = int(len(sets) / 2)
    idx = 0
    for i, s in enumerate(sets[:nodes_count]):
        cores, threads = s, sets[i + nodes_count]
        c_start, c_end = map(lambda s: int(s), cores.split('-'))
        t_start, t_end = map(lambda s: int(s), threads.split('-'))
        for a in zip(range(c_start, c_end + 1), range(t_start, t_end + 1)):
            args.append(f'-vcpu vcpunum={idx},affinity={a[0]}')
            idx += 1
            args.append(f'-vcpu vcpunum={idx},affinity={a[1]}')
            idx += 1
    print(' '.join(args))

END`

if [ "$smp" == "${SMP_PIN}" ]; then
  # Make CPU0 running at highest freq if needed
  echo "Turning off extra CPUs..."
  for x in $(seq $((${PIN_CPU_CORES} + ${PIN_QEMU_SPARE_CORES})) $((${SINGLE_SOCKET_CPU_CORES} - 1))); do
    cpu=$((${x} + ${PIN_CPU_OFFSET}))
    sudo bash -c "echo 0 > /sys/devices/system/cpu/cpu${cpu}/online"
    echo "Turned off CPU ${cpu}"
    cpu=$((${x} + ${PIN_CPU_OFFSET} + ${SINGLE_SOCKET_CPU_CORES} + ${SINGLE_SOCKET_CPU_CORES}))
    sudo bash -c "echo 0 > /sys/devices/system/cpu/cpu${cpu}/online"
    echo "Turned off CPU ${cpu}"
  done

  # Pin CPU cores
  offset_end=$(((PIN_CPU_CORES) + PIN_CPU_OFFSET - 1)) 
  # echo ${PIN_OFFSET} ${offset_end}
  for core_number in $(seq ${PIN_CPU_OFFSET} ${offset_end}); do # 18-27
    vcpu=$((core_number - ${PIN_CPU_OFFSET}))
    if [[ ${PIN_CPU_THREADS} -gt "1" ]]; then
      VGAPT_PINNING_PARAMS=" $VGAPT_PINNING_PARAMS -vcpu vcpunum=$((vcpu * 2)),affinity=$((vcpu + ${PIN_CPU_OFFSET}))"
      VGAPT_PINNING_PARAMS=" $VGAPT_PINNING_PARAMS -vcpu vcpunum=$((vcpu * 2 + 1)),affinity=$((vcpu + ${PIN_CPU_OFFSET} + ${SINGLE_SOCKET_CPU_CORES} + ${SINGLE_SOCKET_CPU_CORES}))"
    else
      VGAPT_PINNING_PARAMS=" $VGAPT_PINNING_PARAMS -vcpu vcpunum=$((vcpu)),affinity=$((vcpu + ${PIN_CPU_OFFSET}))"
    fi
  done
else 
  uname -a | grep '\-ck'
  if [ $? -eq 0 ]; then
    VGAPT_PINNING_PARAMS='' # linux-ck (MuQSS) does not like threads getting pinned
  else
    VGAPT_PINNING_PARAMS=${all_params}
  fi
fi

touch /dev/shm/looking-glass
chmod 660 /dev/shm/looking-glass
touch /dev/shm/scream
chmod 660 /dev/shm/scream

echo "Extra optimizations"
sudo bash -c "echo 0 > /sys/bus/workqueue/devices/writeback/numa"
sudo bash -c "echo $HOST_CORES_MASK > /sys/bus/workqueue/devices/writeback/cpumask"
sudo sysctl vm.stat_interval=120
sudo sysctl -w kernel.watchdog=0
sudo bash -c "echo never > /sys/kernel/mm/transparent_hugepage/enabled"
sudo bash -c "echo 1 > /sys/module/kvm/parameters/ignore_msrs"

# echo "Applying CPU microcode..."
# microcode-update

vive_ids=`python3 - <<END

from subprocess import check_output
ids = {'28de:2101', '0bb4:0306', '0bb4:2c87', '28de:2000'} # OG Vive
# Pimax, '0424:2137' is the controller
ids |= {'28de:2101', '0d8c:0013', '0483:0101', '28de:2300'}
special_id = '0bb4:2c87'
special_added = False

if __name__ == '__main__':
  busdevs = []
  usbs = check_output('lsusb').decode().strip().split('\n')
  special_bd = None
  for usb in usbs:
    usb = usb.split()
    usb_id = usb[5]
    if usb_id not in ids:
      continue
    bd = int(usb[1]), int(usb[3].replace(':', ''))
    if usb_id == special_id:
      if special_added:
        special_bd = bd
      special_added = True
    busdevs.append(bd)
  buf = ''
  for bd in busdevs:
    if bd == special_bd:
      buf += '-device usb-host,hostbus={},hostaddr={},bus=ehci.0 '.format(bd[0], bd[1])
    else:
      buf += '-device usb-host,hostbus={},hostaddr={},bus=xhci.0 '.format(bd[0], bd[1])
  print(buf)

END`

# echo $VGAPT_PINNING_PARAMS
# echo $vive_ids ${vive_ids}

echo "Starting VM"
echo "load-module module-native-protocol-unix  auth-anonymous=1 socket=/tmp/win10" | pacmd
echo "qemu-system-x86_64 \
  -name Windows \
  -enable-kvm \
  -machine type=pc,accel=kvm,kernel_irqchip=on \
  -cpu host,kvm=off,hv_time,hv_relaxed,hv_spinlocks=0x1fff,hv_vpindex,hv_reset,hv_runtime,hv_crash,hv_vendor_id=saren-pc \
  -smp ${smp} \
  $VGAPT_PINNING_PARAMS \
  -m ${mem} \
  ${hugepages_opt} \
  -audiodev pa,id=pa1,server=/tmp/win10 \
  -soundhw hda \
  -device virtio-scsi-pci,id=scsi \
  -object memory-backend-file,id=ivshmem1,share=on,mem-path=/dev/shm/looking-glass,size=256M \
  -device ivshmem-plain,memdev=ivshmem1 \
  -object memory-backend-file,id=ivshmem2,share=on,mem-path=/dev/shm/scream,size=2M \
  -device ivshmem-plain,memdev=ivshmem2 \
  -netdev tap,id=t0,ifname=tap0,script=no,downscript=no,vhost=on \
  -net nic,model=virtio,netdev=t0,id=nic0,macaddr=52:54:00:00:00:04 \
  -rtc base=localtime \
  -boot order=c \
  -drive ${drive} \
  -drive if=pflash,format=raw,file=/media/3fc1e28a-25ad-4dfe-8f7d-af33716ec324/Images/Windows_ovmf_vars_x64.bin \
  -drive if=virtio,id=drive1,file=/dev/vg0/win10-storage,format=raw \
  -usb \
  -device ich9-usb-uhci3,id=uhci \
  -device usb-ehci,id=ehci \
  -device nec-usb-xhci,id=xhci \
  ${lightpack} \
  -device usb-host,vendorid=0x05ac,productid=0x12ab \
  -device usb-host,vendorid=0x04a9,productid=0x1747 \
  -device usb-host,vendorid=0x1004,productid=0x633e \
  -device usb-host,vendorid=0x045e,productid=0x028e \
  -vga none \
  -monitor stdio \
  -device vfio-pci,host=83:00.0,id=vega1,multifunction=on${GPU_ROMFILE_OPT} \
  -device vfio-pci,host=83:00.1,id=vega2" > /tmp/windows-qemu.sh

#   -device vfio-pci,host=81:00.0,id=rtx2080ti1,multifunction=on${GPU_ROMFILE_OPT} \
#   -device vfio-pci,host=81:00.1,id=rtx2080ti2 \
#   -device vfio-pci,host=81:00.2,id=rtx2080ti3 \
#   -device vfio-pci,host=81:00.3,id=rtx2080ti4" > /tmp/windows-qemu.sh
#  -device vfio-pci,host=83:00.0,id=vega1,multifunction=on${GPU_ROMFILE_OPT} \
#  -device vfio-pci,host=83:00.1,id=vega2" > /tmp/windows-qemu.sh


sudo ${AUDIO_OPTS} cset proc --set=user --exec bash /tmp/windows-qemu.sh

echo "VM closed"
sudo rm /dev/shm/looking-glass
sudo rm /dev/shm/scream

if [ "$cpu" == "2" ]; then
  sudo bash -c "echo 0 > /proc/sys/vm/nr_hugepages"
else
  sudo bash -c "echo 0 > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages"
fi

echo "Reseting CPU cores shield"
sudo cset shield --reset

if [ "$smp" == "${SMP_PIN}" ]; then
  echo "Restore CPUs..."
  for x in $(seq $((${PIN_CPU_CORES} + ${PIN_QEMU_SPARE_CORES})) $((${SINGLE_SOCKET_CPU_CORES} - 1))); do
    cpu=$((${x} + ${PIN_CPU_OFFSET}))
    sudo bash -c "echo 1 > /sys/devices/system/cpu/cpu${cpu}/online"
    echo "Turned on CPU ${cpu}"
    cpu=$((${x} + ${PIN_CPU_OFFSET} + ${SINGLE_SOCKET_CPU_CORES} + ${SINGLE_SOCKET_CPU_CORES}))
    sudo bash -c "echo 1 > /sys/devices/system/cpu/cpu${cpu}/online"
    echo "Turned on CPU ${cpu}"
  done
fi

echo "Undo extra optimizations"
sudo bash -c "echo always > /sys/kernel/mm/transparent_hugepage/enabled"
sudo sysctl vm.stat_interval=1
sudo sysctl -w kernel.watchdog=1
sudo bash -c "echo 1 > /sys/bus/workqueue/devices/writeback/numa"
sudo bash -c "echo $TOTAL_CORES_MASK > /sys/bus/workqueue/devices/writeback/cpumask"

echo "Stopping samba"
sudo systemctl stop smb

echo "Stopping Synergy"
pkill synergys

if [ "$lp" != "n" ]; then
  echo "Respawning prismatik"
  nohup prismatik > /dev/null 2>&1 &
fi

echo "Exiting..."
sleep 10