Skip to content

Instantly share code, notes, and snippets.

@stewdk
Last active February 27, 2023 12:17
Show Gist options
  • Save stewdk/110f43e0cc1d905fc6ed4c7e10d8d35e to your computer and use it in GitHub Desktop.
Save stewdk/110f43e0cc1d905fc6ed4c7e10d8d35e to your computer and use it in GitHub Desktop.
All-in-one script to run Xen on qemu-system-aarch64
#!/bin/bash -ex
# Copyright (c) 2018, DornerWorks, Ltd.
# Author: Stewart Hildebrand
# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
# following conditions are met:
# 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following
# disclaimer.
# THIS SOFTWARE IS PROVIDED BY DORNERWORKS ''AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
# DORNERWORKS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
WRKDIR=$(pwd)/
NPROC=$(nproc)
USERNAME=dornerworks
PASSWORD=dornerworks
SALT=dw
HASHED_PASSWORD=$(perl -e "print crypt(\"${PASSWORD}\",\"${SALT}\");")
sudo apt install zlib1g-dev libglib2.0-dev libpixman-1-dev libssl-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu pkg-config-aarch64-linux-gnu iasl curl qemu-utils qemu-user-static git build-essential kpartx flex bison device-tree-compiler u-boot-tools automake autoconf
if [ ! -d qemu ]; then
git clone git://git.qemu.org/qemu.git
fi
if [ ! -d xen ]; then
git clone git://xenbits.xen.org/xen.git
fi
if [ ! -d linux ]; then
git clone --depth 1 --branch linux-4.14.y git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git linux
fi
# Download Ubuntu Base file system (https://wiki.ubuntu.com/Base)
ROOTFSURL=http://cdimage.ubuntu.com/ubuntu-base/releases/18.04.1/release/
ROOTFS=ubuntu-base-18.04.1-base-arm64.tar.gz
if [ ! -s ${ROOTFS} ]; then
curl -OLf ${ROOTFSURL}${ROOTFS}
fi
# Build qemu
cd qemu
mkdir -p build
cd build
if [ ! -f config.status ]; then
../configure --target-list=aarch64-softmmu
fi
if [ ! -s aarch64-softmmu/qemu-system-aarch64 ]; then
make -j ${NPROC}
fi
cd ${WRKDIR}
# Build Xen
if [ ! -s ${WRKDIR}xen/xen/xen ]; then
cd ${WRKDIR}xen
if [ ! -s xen/.config ]; then
make -C xen XEN_TARGET_ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- CONFIG_EARLY_PRINTK=pl011,0x09000000,115200 defconfig
#make -C xen XEN_TARGET_ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- CONFIG_EARLY_PRINTK=pl011,0x09000000,115200 menuconfig
#sed -i 's/# CONFIG_DEVICE_TREE_DEBUG is not set/CONFIG_DEVICE_TREE_DEBUG=y/' xen/.config
fi
make XEN_TARGET_ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- CONFIG_EARLY_PRINTK=pl011,0x09000000,115200 dist-xen -j ${NPROC}
cd ${WRKDIR}
fi
# Build Linux
if [ ! -s ${WRKDIR}linux/arch/arm64/boot/Image ]; then
cd ${WRKDIR}linux
cp arch/arm64/configs/defconfig arch/arm64/configs/xenback_defconfig
echo "CONFIG_XEN_BLKDEV_BACKEND=y" >> arch/arm64/configs/xenback_defconfig
echo "CONFIG_XEN_NETDEV_BACKEND=y" >> arch/arm64/configs/xenback_defconfig
make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- xenback_defconfig
# Uncomment the following line if you wish to change the kernel configuration
#make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- menuconfig
echo "Building kernel. This takes a while. To monitor progress, open a new terminal and use \"tail -f buildoutput.log\""
make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- -j ${NPROC} > ${WRKDIR}buildoutput.log 2> ${WRKDIR}buildoutput2.log
cd ${WRKDIR}
fi
MNTRAMDISK=/mnt/ramdisk/
MNTROOTFS=/mnt/qemu-arm64-rootfs/
MNTBOOT=${MNTROOTFS}boot/
if [ ! -s qemuxen.img ]; then
IMGFILE=${MNTRAMDISK}qemuxen.img
else
IMGFILE=qemuxen.img
IMG_EXISTS=yes
fi
unmountstuff () {
sudo umount ${MNTROOTFS}proc || true
sudo umount ${MNTROOTFS}dev/pts || true
sudo umount ${MNTROOTFS}dev || true
sudo umount ${MNTROOTFS}sys || true
sudo umount ${MNTROOTFS}tmp || true
sudo umount ${MNTBOOT} || true
sudo umount ${MNTROOTFS} || true
}
mountstuff () {
sudo mkdir -p ${MNTROOTFS}
sudo mount ${LOOPDEVROOTFS} ${MNTROOTFS}
sudo mkdir -p ${MNTBOOT}
sudo mount ${LOOPDEVBOOT} ${MNTBOOT}
sudo mount -o bind /proc ${MNTROOTFS}proc
sudo mount -o bind /dev ${MNTROOTFS}dev
sudo mount -o bind /dev/pts ${MNTROOTFS}dev/pts
sudo mount -o bind /sys ${MNTROOTFS}sys
sudo mount -o bind /tmp ${MNTROOTFS}tmp
}
finish () {
cd ${WRKDIR}
sudo sync
unmountstuff
sudo kpartx -dvs ${IMGFILE} || true
sudo rmdir ${MNTROOTFS} || true
mv ${IMGFILE} . || true
sudo umount ${MNTRAMDISK} || true
sudo rmdir ${MNTRAMDISK} || true
}
trap finish EXIT
sudo mkdir -p ${MNTRAMDISK}
sudo mount -t tmpfs -o size=3g tmpfs ${MNTRAMDISK}
if [ "${IMG_EXISTS}" != "yes" ]; then
qemu-img create ${IMGFILE} 2G
parted ${IMGFILE} --script -- mklabel msdos
parted ${IMGFILE} --script -- mkpart primary fat32 2048s 264191s
parted ${IMGFILE} --script -- mkpart primary ext4 264192s -1s
fi
LOOPDEVS=$(sudo kpartx -avs ${IMGFILE} | awk '{print $3}')
LOOPDEVBOOT=/dev/mapper/$(echo ${LOOPDEVS} | awk '{print $1}')
LOOPDEVROOTFS=/dev/mapper/$(echo ${LOOPDEVS} | awk '{print $2}')
if [ "${IMG_EXISTS}" != "yes" ]; then
sudo mkfs.vfat ${LOOPDEVBOOT}
sudo mkfs.ext4 ${LOOPDEVROOTFS}
sudo fatlabel ${LOOPDEVBOOT} BOOT
sudo e2label ${LOOPDEVROOTFS} QemuUbuntu
sudo mkdir -p ${MNTROOTFS}
sudo mount ${LOOPDEVROOTFS} ${MNTROOTFS}
sudo tar -C ${MNTROOTFS} -xf ${ROOTFS}
sudo umount ${MNTROOTFS}
fi
mountstuff
sudo cp `which qemu-aarch64-static` ${MNTROOTFS}usr/bin/
sudo cp ${WRKDIR}linux/arch/arm64/boot/Image ${MNTBOOT}
cd ${WRKDIR}linux
sudo make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- INSTALL_MOD_PATH=${MNTROOTFS} modules_install > ${WRKDIR}modules_install.log
cd ${WRKDIR}
# /etc/resolv.conf is required for internet connectivity in chroot. It will get overwritten by dhcp, so don't get too attached to it.
sudo chroot ${MNTROOTFS} bash -c 'echo "nameserver 8.8.8.8" > /etc/resolv.conf'
sudo chroot ${MNTROOTFS} bash -c 'echo "nameserver 2001:4860:4860::8888" >> /etc/resolv.conf'
if [ "${IMG_EXISTS}" != "yes" ]; then
sudo sed -i -e "s/# deb /deb /" ${MNTROOTFS}etc/apt/sources.list
sudo chroot ${MNTROOTFS} apt-get update
fi
# Install the dialog package and others first to squelch some warnings
sudo chroot ${MNTROOTFS} apt-get -y install dialog apt-utils
sudo chroot ${MNTROOTFS} apt-get -y upgrade
sudo chroot ${MNTROOTFS} apt-get -y install systemd systemd-sysv sysvinit-utils sudo udev rsyslog kmod util-linux sed netbase dnsutils ifupdown isc-dhcp-client isc-dhcp-common less vim net-tools iproute2 iputils-ping libnss-mdns iw software-properties-common ethtool dmsetup hostname iptables logrotate lsb-base lsb-release plymouth psmisc tar tcpd libsystemd-dev symlinks uuid-dev libc6-dev libncurses-dev libglib2.0-dev build-essential bridge-utils zlib1g-dev patch libpixman-1-dev libyajl-dev libfdt-dev libaio-dev
# Change the shared library symlinks to relative instead of absolute so they play nice with cross-compiling
sudo chroot ${MNTROOTFS} symlinks -c /usr/lib/aarch64-linux-gnu/
if [ "${IMG_EXISTS}" != "yes" ]; then
cd ${WRKDIR}xen
# TODO: --with-xenstored=oxenstored
# Ask the native compiler what system include directories it searches through.
SYSINCDIRS=$(echo $(sudo chroot ${MNTROOTFS} bash -c "echo | gcc -E -Wp,-v -o /dev/null - 2>&1" | grep "^ " | sed "s|^ /| -isystem${MNTROOTFS}|"))
SYSINCDIRSCXX=$(echo $(sudo chroot ${MNTROOTFS} bash -c "echo | g++ -x c++ -E -Wp,-v -o /dev/null - 2>&1" | grep "^ " | sed "s|^ /| -isystem${MNTROOTFS}|"))
LDFLAGS="-Wl,-rpath-link=${MNTROOTFS}lib/aarch64-linux-gnu -Wl,-rpath-link=${MNTROOTFS}usr/lib/aarch64-linux-gnu" \
./configure \
PYTHON_PREFIX_ARG=--install-layout=deb \
--enable-systemd \
--disable-xen \
--enable-tools \
--disable-docs \
--disable-stubdom \
--prefix=/usr \
--with-xenstored=xenstored \
--build=x86_64-linux-gnu \
--host=aarch64-linux-gnu \
CC="aarch64-linux-gnu-gcc --sysroot=${MNTROOTFS} -nostdinc ${SYSINCDIRS} -B${MNTROOTFS}lib/aarch64-linux-gnu -B${MNTROOTFS}usr/lib/aarch64-linux-gnu" \
CXX="aarch64-linux-gnu-g++ --sysroot=${MNTROOTFS} -nostdinc ${SYSINCDIRSCXX} -B${MNTROOTFS}lib/aarch64-linux-gnu -B${MNTROOTFS}usr/lib/aarch64-linux-gnu" \
PKG_CONFIG_PATH=${MNTROOTFS}usr/lib/aarch64-linux-gnu/pkgconfig:${MNTROOTFS}usr/share/pkgconfig
LDFLAGS="-Wl,-rpath-link=${MNTROOTFS}lib/aarch64-linux-gnu -Wl,-rpath-link=${MNTROOTFS}usr/lib/aarch64-linux-gnu" \
make dist-tools \
CROSS_COMPILE=aarch64-linux-gnu- XEN_TARGET_ARCH=arm64 \
CC="aarch64-linux-gnu-gcc --sysroot=${MNTROOTFS} -nostdinc ${SYSINCDIRS} -B${MNTROOTFS}lib/aarch64-linux-gnu -B${MNTROOTFS}usr/lib/aarch64-linux-gnu" \
CXX="aarch64-linux-gnu-g++ --sysroot=${MNTROOTFS} -nostdinc ${SYSINCDIRSCXX} -B${MNTROOTFS}lib/aarch64-linux-gnu -B${MNTROOTFS}usr/lib/aarch64-linux-gnu" \
PKG_CONFIG_PATH=${MNTROOTFS}usr/lib/aarch64-linux-gnu/pkgconfig:${MNTROOTFS}usr/share/pkgconfig \
QEMU_PKG_CONFIG_FLAGS=--define-variable=prefix=${MNTROOTFS}usr \
-j $(nproc)
sudo --preserve-env PATH=${PATH} \
LDFLAGS="-Wl,-rpath-link=${MNTROOTFS}lib/aarch64-linux-gnu -Wl,-rpath-link=${MNTROOTFS}usr/lib/aarch64-linux-gnu" \
make install-tools \
CROSS_COMPILE=aarch64-linux-gnu- XEN_TARGET_ARCH=arm64 \
CC="aarch64-linux-gnu-gcc --sysroot=${MNTROOTFS} -nostdinc ${SYSINCDIRS} -B${MNTROOTFS}lib/aarch64-linux-gnu -B${MNTROOTFS}usr/lib/aarch64-linux-gnu" \
CXX="aarch64-linux-gnu-g++ --sysroot=${MNTROOTFS} -nostdinc ${SYSINCDIRSCXX} -B${MNTROOTFS}lib/aarch64-linux-gnu -B${MNTROOTFS}usr/lib/aarch64-linux-gnu" \
PKG_CONFIG_PATH=${MNTROOTFS}usr/lib/aarch64-linux-gnu/pkgconfig:${MNTROOTFS}usr/share/pkgconfig \
QEMU_PKG_CONFIG_FLAGS=--define-variable=prefix=${MNTROOTFS}usr \
DESTDIR=${MNTROOTFS}
sudo chroot ${MNTROOTFS} systemctl enable xen-qemu-dom0-disk-backend.service
sudo chroot ${MNTROOTFS} systemctl enable xen-init-dom0.service
sudo chroot ${MNTROOTFS} systemctl enable xenconsoled.service
sudo chroot ${MNTROOTFS} systemctl enable xendomains.service
sudo chroot ${MNTROOTFS} systemctl enable xen-watchdog.service
cd ${WRKDIR}
fi
# It seems like the xen tools configure script selects a few too many of these backend driver modules, so we override it with a simpler list.
# /usr/lib/modules-load.d/xen.conf
cat > tmp-qemuxen-script-generated-xen.conf <<EOF
xen-evtchn
xen-gntdev
xen-gntalloc
xen-blkback
xen-netback
EOF
sudo cp tmp-qemuxen-script-generated-xen.conf ${MNTROOTFS}usr/lib/modules-load.d/xen.conf
rm tmp-qemuxen-script-generated-xen.conf
# /etc/hostname
HOSTNAME=ubuntu
sudo bash -c "echo ${HOSTNAME} > ${MNTROOTFS}etc/hostname"
# /etc/hosts
cat > tmp-qemuxen-script-generated-hosts <<EOF
127.0.0.1 localhost
127.0.1.1 ${HOSTNAME}
# The following lines are desirable for IPv6 capable hosts
::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
EOF
sudo cp tmp-qemuxen-script-generated-hosts ${MNTROOTFS}etc/hosts
rm tmp-qemuxen-script-generated-hosts
# /etc/fstab
cat > tmp-qemuxen-script-generated-fstab <<EOF
proc /proc proc defaults 0 0
/dev/vda1 /boot vfat defaults 0 2
/dev/vda2 / ext4 defaults,noatime 0 1
EOF
sudo cp tmp-qemuxen-script-generated-fstab ${MNTROOTFS}etc/fstab
rm tmp-qemuxen-script-generated-fstab
# /etc/network/interfaces.d/eth0br0
cat > tmp-qemuxen-script-generated-interfaces <<EOF
auto eth0
iface eth0 inet manual
auto br0
iface br0 inet dhcp
bridge_ports eth0
EOF
sudo cp tmp-qemuxen-script-generated-interfaces ${MNTROOTFS}etc/network/interfaces.d/eth0br0
rm tmp-qemuxen-script-generated-interfaces
sudo chmod 0600 ${MNTROOTFS}etc/network/interfaces.d/eth0br0
# User account setup
if [ "${IMG_EXISTS}" != "yes" ]; then
sudo chroot ${MNTROOTFS} useradd -s /bin/bash -G adm,sudo -m -p ${HASHED_PASSWORD} ${USERNAME}
fi
LINUX_ADDR=0x40600000
KERNEL_SIZE=$(printf '0x%x\n' $(ls -l linux/arch/arm64/boot/Image | awk '{print $5}'))
DTBFILE=virt.dtb
QEMUOPTS="-gdb tcp::3333 \
-machine virt,gic_version=3 -machine virtualization=true \
-cpu cortex-a53 -smp 4 -m 2G -nographic \
-netdev user,id=net0,hostfwd=tcp::2222-:22 -device virtio-net-device,netdev=net0 \
-kernel ${WRKDIR}xen/xen/xen \
-device loader,file=${WRKDIR}linux/arch/arm64/boot/Image,addr=${LINUX_ADDR} \
-drive if=virtio,file=${IMGFILE},format=raw"
unmountstuff
${WRKDIR}qemu/build/aarch64-softmmu/qemu-system-aarch64 ${QEMUOPTS} -machine dumpdtb=${DTBFILE}
dtc -I dtb -O dts -o virt.dts ${DTBFILE}
patch -p1 <<EOF
diff --git a/virt.dts b/virt.dts
--- a/virt.dts
+++ b/virt.dts
@@ -385,5 +385,13 @@
chosen {
stdout-path = "/pl011@9000000";
+ xen,xen-bootargs = "console=dtuart dtuart=/pl011@9000000 dom0_mem=512M";
+ xen,dom0-bootargs = "console=hvc0 ignore_loglevel clk_ignore_unused root=/dev/vda2";
+ #address-cells = <1>;
+ #size-cells = <1>;
+ module@0 {
+ compatible = "multiboot,module", "multiboot,kernel";
+ reg = <${LINUX_ADDR} ${KERNEL_SIZE}>;
+ };
};
};
EOF
dtc -I dts -O dtb -o ${DTBFILE} virt.dts
${WRKDIR}qemu/build/aarch64-softmmu/qemu-system-aarch64 ${QEMUOPTS} -dtb ${DTBFILE}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment