Multikernel vs. KVM lmbench benchmark scripts (multikernel.io: Measuring the Virtualization Tax with lmbench)

lmbench-full-daxfs.sh

#!/bin/bash
# Boot one spawn kernel that runs the full lmbench suite built from the
# autubrew/lmbench image (Ubuntu 20.04 + lmbench source, no prebuilt
# binaries). kerf pulls --image through skopeo from the registry, so a
# locally built image is invisible to it; instead the built container is
# exported once to a rootfs directory and loaded with --rootfs-dir.
#
# One-time setup (already done on this machine):
#   docker run --name lmbench-build --entrypoint /bin/bash \
#       autubrew/lmbench -c 'cd /root/lmbench/src && make -j8'
#   mkdir -p /root/lmbench-rootfs
#   docker export lmbench-build | tar -x -C /root/lmbench-rootfs
#   dd if=/dev/zero of=/root/lmbench-rootfs/scratch.dat bs=1M count=64
#   cp run-lmbench.sh /root/lmbench-rootfs/run-lmbench.sh
#
# Usage:
#   lmbench-full-daxfs.sh          # set up and boot the instance
#   lmbench-full-daxfs.sh cleanup  # kill, unload and delete the instance
#
# After boot, watch the run with:
#   kerf console lmbench   (Ctrl+] . to detach)
# Results stream into the daxfs overlay, visible from the host at
# /var/lib/kerf/daxfs/lmbench/lmbench-result.txt (last line is DONE).
# The whole suite takes several minutes; lat_mem_rd dominates.

set -euo pipefail

# /usr/local/bin/kerf is installed under python3.8 which cannot parse the
# newer annotations, so run kerf from the source tree under python3.9
kerf() {
    PYTHONPATH=/root/kerf/src python3.9 -c 'from kerf.cli import main; main()' "$@"
}

KERNEL=/root/linux/vmlinux
INITRD=/root/daxfs-initrd.img
ROOTFS=/root/lmbench-rootfs

POOL_CPUS=2,4,6,8
POOL_MEM=1024MB

INST=lmbench
CPUS=2,4
INST_MEM=1024MB

CMDLINE_BASE="pci=noaer,lastbus=0 lpj=999999 clocksource=tsc tsc=reliable pci=realloc=off random.trust_cpu=on pci=nobar"

cleanup() {
    kerf kill -f "${INST}" 2>/dev/null || true
    kerf unload "${INST}" 2>/dev/null || true
    kerf delete "${INST}" 2>/dev/null || true
    kerf show
}

if [ "${1:-}" = "cleanup" ]; then
    cleanup
    exit 0
fi

if [ ! -x "${ROOTFS}/run-lmbench.sh" ]; then
    echo "Error: ${ROOTFS}/run-lmbench.sh missing, redo the one-time setup" >&2
    exit 1
fi

# Initialize the multikernel pool unless it already exists
if [ ! -d /sys/fs/multikernel/instances ]; then
    kerf init --cpus="${POOL_CPUS}" --memory="${POOL_MEM}"
fi

kerf create "${INST}" --cpus="${CPUS}" --memory="${INST_MEM}" 2>/dev/null || true

kerf load "${INST}" \
    --kernel="${KERNEL}" \
    --initrd="${INITRD}" \
    --rootfs-dir="${ROOTFS}" \
    --entrypoint="/bin/bash /run-lmbench.sh" \
    --cmdline="${CMDLINE_BASE}" \
    --console=mktty0 \
    --verbose

kerf exec "${INST}"

kerf show
echo
echo "Instance booted. Attach with:"
echo "  kerf console ${INST}"
echo "Result visible from the host at: /var/lib/kerf/daxfs/${INST}/lmbench-result.txt"

lmbench-vm.sh

#!/bin/bash
# Run the same full lmbench suite as lmbench-full-daxfs.sh, but inside a
# QEMU/KVM guest booting the same kernel, for multikernel vs VM overhead
# numbers.
#
# The kernel has no virtio-blk/9p/pmem drivers, so the shared rootfs at
# /root/lmbench-rootfs is packed into an initramfs each run (picking up
# any run-lmbench.sh edits) and the guest runs entirely from RAM. That is
# also why the guest gets more RAM than the multikernel instance: the
# rootfs (~360MB unpacked) lives inside guest RAM here, while the
# multikernel instance kept it in pool memory outside its 512MB, so 1GB
# leaves the benchmarks a comparable amount of free memory. None of the
# tests are memory-capacity-bound.
#
# The guest replays the result file over the serial console between
# markers; this script extracts it to /root/lmbench-vm-result.txt.
# Full console log: /root/lmbench-vm-console.log
#
# Usage: lmbench-vm.sh

set -euo pipefail

KERNEL=/root/linux/arch/x86/boot/bzImage
ROOTFS=/root/lmbench-rootfs
INITRD=/root/lmbench-vm-initrd.gz
CONSOLE_LOG=/root/lmbench-vm-console.log
RESULT=/root/lmbench-vm-result.txt

VCPUS=2
MEM=1024
# Host CPUs to pin the guest to. CPUs 2,4 belong to the multikernel pool
# and are offline on the host, so use two other cores.
HOST_CPUS=10,12

CMDLINE="console=ttyS0 lpj=999999 clocksource=tsc tsc=reliable random.trust_cpu=on panic=-1 rdinit=/vm-init.sh"

if [ ! -x "${ROOTFS}/run-lmbench.sh" ] || [ ! -x "${ROOTFS}/vm-init.sh" ]; then
    echo "Error: ${ROOTFS} is missing run-lmbench.sh or vm-init.sh" >&2
    exit 1
fi

echo "Packing ${ROOTFS} into ${INITRD}..."
(cd "${ROOTFS}" && find . | cpio -o -H newc --quiet | gzip -1) > "${INITRD}"

echo "Booting guest (${VCPUS} vCPUs pinned to host CPUs ${HOST_CPUS}, ${MEM}MB)..."
taskset -c "${HOST_CPUS}" timeout 1800 qemu-system-x86_64 \
    -enable-kvm \
    -cpu host \
    -smp "${VCPUS}" \
    -m "${MEM}" \
    -kernel "${KERNEL}" \
    -initrd "${INITRD}" \
    -append "${CMDLINE}" \
    -nographic \
    -no-reboot \
    > "${CONSOLE_LOG}" 2>&1 || true

sed -n '/===LMBENCH-RESULT-BEGIN===/,/===LMBENCH-RESULT-END===/p' \
    "${CONSOLE_LOG}" | sed '1d;$d' > "${RESULT}"

if ! grep -q '^DONE' "${RESULT}"; then
    echo "Error: run did not complete, check ${CONSOLE_LOG}" >&2
    exit 1
fi

echo "Done. Result: ${RESULT}"

run-lmbench.sh

#!/bin/bash
# Runs inside the spawn kernel as the kerf entrypoint. lmbench prints
# results on stderr, so everything is redirected into the daxfs overlay
# where the host can read it live.

BIN=/root/lmbench/bin/x86_64-linux-gnu
RESULT=/lmbench-result.txt

exec > "${RESULT}" 2>&1

mount -t proc proc /proc 2>/dev/null
mount -t sysfs sys /sys 2>/dev/null
mount -t devtmpfs dev /dev 2>/dev/null

echo "==== uname: $(uname -a)"
echo "==== nproc: $(nproc)"

run() {
    echo
    echo "==== $*"
    "$@"
}

# lat_proc shell spawns "/bin/sh -c /tmp/hello" and needs the binary there
cp ${BIN}/hello /tmp/hello

run ${BIN}/lat_syscall -N 10 null
run ${BIN}/lat_syscall -N 10 read
run ${BIN}/lat_syscall -N 10 write
run ${BIN}/lat_syscall -N 10 stat /bin/sh
run ${BIN}/lat_syscall -N 10 open /bin/sh
run ${BIN}/lat_sig -N 10 install
run ${BIN}/lat_sig -N 10 catch
run ${BIN}/lat_proc -N 5 fork
run ${BIN}/lat_proc -N 5 exec
run ${BIN}/lat_proc -N 5 shell
run ${BIN}/lat_ctx -N 5 -s 0 2 4 8 16
run ${BIN}/lat_pipe -N 5
run ${BIN}/lat_unix -N 5

for op in rd wr rdwr cp bzero bcopy; do
    run ${BIN}/bw_mem -P 1 -W 2 -N 5 64m ${op}
done

run ${BIN}/lat_mem_rd 128 128

# daxfs-backed mmap tests last: they exercise the newest code paths and
# must not take the rest of the suite down with them if they misbehave
run ${BIN}/lat_mmap -N 5 64m /scratch.dat
run ${BIN}/lat_pagefault -N 5 /scratch.dat

echo
echo DONE

run-matrix.sh

#!/bin/bash
set -u
R=/root/results; mkdir -p $R; rm -f $R/ALL-DONE $R/mk-*.txt $R/vm-*.txt
mk_run() {
  local i=$1
  bash /root/lmbench-full-daxfs.sh cleanup >/dev/null 2>&1
  bash /root/lmbench-full-daxfs.sh >/dev/null 2>&1
  for t in $(seq 1 60); do
    sleep 10
    grep -q "^DONE" /var/lib/kerf/daxfs/lmbench/lmbench-result.txt 2>/dev/null && break
  done
  cp /var/lib/kerf/daxfs/lmbench/lmbench-result.txt $R/mk-$i.txt 2>/dev/null
  [ -s $R/mk-$i.txt ] && echo "PROGRESS mk-$i done" || echo "PROGRESS WARNING mk-$i empty"
  bash /root/lmbench-full-daxfs.sh cleanup >/dev/null 2>&1
}
vm_run() {
  local cfg=$1 i=$2 xq=$3 xc=$4
  taskset -c 10,12 timeout 900 qemu-system-x86_64 -enable-kvm -cpu host $xq -smp 2 -m 1024 \
    -kernel /root/linux/arch/x86/boot/bzImage -initrd /root/lmbench-vm-initrd.gz \
    -append "console=ttyS0 lpj=999999 clocksource=tsc tsc=reliable random.trust_cpu=on panic=-1 $xc rdinit=/vm-init.sh" \
    -nographic -no-reboot > /root/vm-run.log 2>&1 || true
  sed -n "/===LMBENCH-RESULT-BEGIN===/,/===LMBENCH-RESULT-END===/p" /root/vm-run.log | sed "1d;\$d" > $R/vm-$cfg-$i.txt
  grep -q "^DONE" $R/vm-$cfg-$i.txt && echo "PROGRESS vm-$cfg-$i done" || echo "PROGRESS WARNING vm-$cfg-$i incomplete"
}
for i in 1 2 3; do mk_run $i; done
for i in 1 2 3; do vm_run default $i "" ""; done
for i in 1 2 3; do vm_run idlepoll $i "" "idle=poll"; done
for i in 1 2 3; do vm_run cpupm $i "-overcommit cpu-pm=on" ""; done
touch $R/ALL-DONE
echo "PROGRESS ALL DONE"

vm-init.sh

#!/bin/bash
# PID 1 inside the QEMU guest. Runs the exact same run-lmbench.sh as the
# multikernel instance, then replays the result file over the serial
# console between markers so the host can extract it, and powers off.

/run-lmbench.sh

# Quiesce kernel console printk so it cannot interleave with the markers
dmesg -n 1

echo "===LMBENCH-RESULT-BEGIN==="
cat /lmbench-result.txt
echo "===LMBENCH-RESULT-END==="

echo 1 > /proc/sys/kernel/sysrq
echo o > /proc/sysrq-trigger

# PID 1 must not exit while the poweroff is in flight
sleep 60