run-qemu.sh script to run qemu for kernel development

run-qemu.sh

#!/bin/bash
# Author: Joel Fernandes <joel@joelfernandes.org>
#
# Usage, run-qemu		: Runs with busybox chrooting into a debian qemu.img 
# run-qemu --rt			: Set all Qemu vCPU threads as RT and pin them to different CPUs.
# run-qemu --rcu		: Run qemu with rcutorture boot parameters
#
# run-qemu --cpio 		: Runs only with busy-box initramfs (no chrooting)
# run-qemu --debian		: Boot into a debian rootfs (see end of this file for how to build debian images)
# If neither --cpio nor --debian is passed, then it will try to chroot into a qemu.img file.
# To construct a qemu.img file, see the end of this file.
#
#
# run-qemu --cpio --cmd "hackbench"	: Run hackbench as soon as we boot into the CPIO.

spath="$(dirname "$(readlink -f "$0")")"
export DISPLAY=:0

BBOX=$spath
KIMG=./arch/x86/boot/bzImage
CHROOT_HDA=$spath/qemu.img
TRACE_RCU=0
TRACE_SCHED=0
GDB=0
QMP=0

CPUS=4

# Default to verbose output
VERBOSE=1

while (( "$#" )); do
  case "$1" in
--trace-sched)
      TRACE_SCHED=1; shift; continue ;;
--trace-rcu)
      TRACE_RCU=1; shift; continue ;;
--cpio)
      CPIO=1
      shift
      continue
      ;;
--debug)
      set -x
      shift
      continue
      ;;
--rt)
      RT=1
      shift; continue    ;;
--cpus)
      CPUS=$2
      shift 2; continue    ;;
--rcu)
      RCU=1
      shift; continue    ;;
--debian)
      DEB=1
      shift; continue    ;;
--boot-args)
      BOOT_ARGS="$2"
      shift 2; continue  ;;
--verbose)
      VERBOSE=1
      shift; continue    ;;
--gdb)
      GDB=1
      shift; continue    ;;
--qmp)
      QMP=1
      shift; continue    ;;
-v)
      VERBOSE=1
      shift; continue    ;;
-q)
      VERBOSE=0
      shift; continue    ;;
--) # end argument parsing
      shift
      break
      ;;
--*=) # unsupported flags
      echo "Error: Unsupported flag $1" >&2
      exit 1
      ;;
    *) # preserve positional arguments
      PARAMS="$PARAMS $1"
      shift
      ;;
  esac
done

eval set -- "$PARAMS"

# Trace related stuff
if [ $TRACE_SCHED -eq 1 ]; then
	BOOT_ARGS="$BOOT_ARGS ftrace_dump_on_oops"
	TRACE_ARGS="$TRACE_ARGS sched:sched_switch"
fi
if [ $TRACE_RCU -eq 1 ]; then
	TRACE_ARGS="$TRACE_ARGS rcu:rcu_grace_period rcu:rcu_callback rcu:rcu_invoke_callback rcu:rcu_kvfree_callback"
	BOOT_ARGS="$BOOT_ARGS ftrace_dump_on_oops"
fi

if [ ! -z "$TRACE_ARGS" ]; then
  te=""
  for t in $TRACE_ARGS; do
    te="$te$t,"
  done
  BOOT_ARGS="$BOOT_ARGS trace_event=$te"
fi

GDB_ARGS=""
if [ $GDB -eq 1 ]; then
	GDB_ARGS="-s -S"
	BOOT_ARGS="$BOOT_ARGS"
fi

QMP_ARGS=""
if [ $QMP -eq 1 ]; then
	QMP_ARGS="-qmp tcp:localhost:4444,server,nowait"
fi

# Helper to pin all vCPU threads to different CPUs so that RT works as expected.
function pin_vcpu_threads() {
	echo "Pinning all vCPU threads to different CPUs for RT, waiting for 5 secs"
	sleep 5

	set -x
	# Get all qemu-system-x86 threads and store them in an array
	threads=($(sudo ps -eT | grep qemu-system-x86 | awk '{print $2}'))
	cpu=4 # Set the starting CPU
	# Loop through the threads and pin each to a different CPU
	for tid in "${threads[@]}"; do
		echo "Pinning thread $tid to CPU $cpu for RT"
		sudo taskset -cp ${cpu}-$((${cpu} + 7)) $tid
		cpu=$((cpu + 8))
	done
	set +x
}

# For rt-testing, pass -rt to run hypervisor as highest prio FF
if [ "x$RT" == "x1" ]; then
	sudo chrt -f -p 99 $$
	pin_vcpu_threads & # Run in bg to give time for qemu to start
fi

# 3 modes to run this script in
# --debian: in this case don't pass initrd
# --cpio: pass initrd, but only run from it
# if none of above passed, try to run a chroot image,
# the cpio is passed as first stage which chroots into image.

# Setup params for chroot
if [ -f $CHROOT_HDA ] && [ "x$CPIO" != "x1" ]; then
	HDA="-drive file=$CHROOT_HDA,format=raw "
fi

# Setup final params
if [ "x$DEB" == "x1" ]; then  # For debian
	DISK_ARGS="-hda $spath/debian.qcow"
	ROOT_ARGS="/dev/sda1"
else                          # For chroot, or cpio
	DISK_ARGS="$HDA -initrd $BBOX/cpio.gz"
	ROOT_ARGS="/dev/sda"
	INIT_ARGS="init=/init"
fi

# Configure the kernel commandline
TESTCMD=""
if [ "x$RCU" == "x1" ]; then
	# rcutorture when built-in runs at boot and is configurable
	BOOT_ARGS="$BOOT_ARGS rcutorture.shutdown_secs=60 rcutorture.n_barrier_cbs=4 rcutree.kthread_prio=2"
else
	# Pass the first param as a test command to exec (useful for running hackbench etc)
	# Works only when --cpio is requested (does not work from chroot)
	TESTCMD=$1
	if [ "x$TESTCMD" != "x" ] && [ "x$CPIO" != "x1" ]; then
		echo "TESTCMD ($1) will work only if CPIO=1"
		exit 1
	fi
fi
if [ "x$VERBOSE" != "x1" ]; then BOOT_ARGS="$BOOT_ARGS quiet"; fi

RAM_SIZE=2048
NVDIMM_SIZE=128
MAX_SIZE=$(( RAM_SIZE + NVDIMM_SIZE ))
RAMOOPS_ARGS="ramoops.mem_address=0x140000000 ramoops.mem_size=204800 ramoops.ecc=1"

echo "Running qemu with boot args: ${BOOT_ARGS}"

# Uncomment to print the final command and exit
# cat <<EOFF
set -x

# Seabios clears the screen on boot. Nothing anyone can do about it.
# So I ignore the stdout for 3 seconds, also so it does not mess up my terminal (ctrl L doesn't work anymore).
sudo qemu-system-x86_64 \
-kernel $KIMG \
-append "console=ttyS0 nokaslr $BOOT_ARGS testcmd=\"$TESTCMD\" root=$ROOT_ARGS $INIT_ARGS $RAMOOPS_ARGS" \
-m ${RAM_SIZE}M,slots=2,maxmem=${MAX_SIZE}M \
-enable-kvm \
-smp cpus=$CPUS,threads=2,sockets=1 \
-cpu host \
$DISK_ARGS \
$GDB_ARGS \
$QMP_ARGS \
-nographic \
-no-reboot | ignore-stdin-for-secs.sh 3

# -bios /usr/share/ovmf/OVMF.fd
# EOFF

# For selinux, set selinux=1, and also mount selinuxfs manually on to /sys/fs/selinux,
# then getenforce returns Enforcing

# For ftrace tracing, consider the following boot cmdline:
#	ftrace_dump_on_oops trace_buf_size=1K \
#	trace_event=sched:sched_switch,sched_switch,sched:sched_waking,sched:sched_wakeup threadirqs threadedirqs \

# For debian images passed to HDA above:
#
# (1) SLirp networking is enabled in qemu by default without passing of any netdev options
# Just add the following 3 commands to your qemu HDA image's bashrc:
# ifconfig eth0 10.0.2.15  (for debian pass enp0s3 instead of eth0)
# route add default gw 10.0.2.2
# echo "nameserver 10.0.2.3" > /etc/resolv.conf 
#
# (2) To fix the row/col height of the terminal, add to .bashrc
# stty rows 50   # This is mostly useful for vim, the shell really doesn't stop at rows and doesn't care.
# stty cols 132  # This is mostly useful for the shell, without it, the line badly wraps and overwrites.

# For nvdimm to work (which I use for pstore), the kernel needs CONFIG_ACPI_NFIT=y

# How to install debian images (note you can skip these steps if you want,
#				as the final qcow can be downloaded from my corp google
#				drive's root and placed in $spath)
# Download iso from https://cdimage.debian.org/cdimage/release/current/amd64/iso-cd/
#  Then, qemu-img create -f qcow2 debian.qcow 2G
#  Then, qemu-system-x86_64 -hda debian.qcow -cdrom debian-X.YY-netinst.iso -boot c -m 1024
#  Hit CTRL+C immediate and enter on boot: install console=ttyS0,9600,n8
#  Finish the install and don't install grub. Also during install, don't chose swap.
#  To boot, just run this script but remove "-initrd", add "-hda debian.qcow" and pass "root=/dev/sda1" on cmdline.
#								(these already done with --debian)
# Note: To install additional packages, you can do:
#           sudo guestmount -a /s/qemu//debian.qcow  -m /dev/sda1 /tmp/qc , followed by 'chroot /tmp/qc'

# To change the cpio init, read the build-cpio script. It might be copying busybox-related init from different place.

# To add a better prompt, add following to .bashrc
# # Fix up hostname via a hack
# export HOST_NAME=qemubox
# export PS1="\e[92m[\u@$HOST_NAME \W]\$ \e[0m"

# To build a qemu.img file to chroot into
# 1. dd a sparse 4GB qemu.img file.
# 2. mkfs.ext4 qemu.img and mount it.
# 3. download debian tarball (example: debian-12-generic-amd64-20230802-1460.tar.xz)
# 4. tar -xvf it into the mounted qemu.img.
# Note that you still to build a cpio archive and chroot into it from the initramfs's init.