prepare host (docker host) export EALAB=~/Projects/qemu-arm-lab/
mkdir -p ${EALAB} && cd "$_"
# This's for rootfs booted by NFS
# install nfs server. For ubuntu apt install -y nfs-kernel-server
echo "${EALAB}/rootfs *(rw,sync,no_root_squash,no_subtree_check)" | sudo tee --append /etc/exports
sudo systemctl restart nfs-kernel-server.service
docker run -it --rm -v $(pwd):/mnt --privileged -w /mnt --net host -h qemu-arm-lab --name qemu-arm-lab ubuntu:18.10
export EALAB=/mnt
export EALABSRC=${EALAB}/src
export EALABDWN=${EALAB}/downloads
export EALABBOOT=${EALAB}/boot
export EALABBLD=${EALAB}/build
export EALABRFS=${EALAB}/rootfs
mkdir -p ${EALABSRC} && cd "$_"
apt update && apt upgrade -y
apt install -y gcc wget xz-utils python libglib2.0-dev pkg-config libpixman-1-dev libncurses-dev bison flex git bc gcc-arm-linux-gnueabihf iproute2 dnsmasq
cd ${EALAB}
mkdir -p {downloads,boot,build,rootfs}
Export package version and vars export QEMUVER=4.0.0
export UBOOTVER=2019.04
export LINUXVER=5.0.10
export BUSYBOXVER=1.30.1
export ARCH=arm
export CROSS_COMPILE=arm-linux-gnueabihf-
Download and compile qemu wget https://download.qemu.org/qemu-${QEMUVER}.tar.xz -P ${EALABDWN}
tar xf ${EALABDWN}/qemu-${QEMUVER}.tar.xz -C ${EALABBLD}
cd ${EALABBLD}/qemu-${QEMUVER}
./configure \
--target-list=arm-softmmu,arm-linux-user \
--enable-debug \
--extra-cflags="-g3 -pg -fno-omit-frame-pointer" \
--extra-cxxflags="-g3 -pg -fno-omit-frame-pointer" \
--extra-ldflags="-g3" \
--disable-strip --disable-pie
make -j $(nproc) && make install
Download and compile u-boot wget https://github.com/u-boot/u-boot/archive/v${UBOOTVER}.tar.gz -P ${EALABDWN}
tar xf ${EALABDWN}/v${UBOOTVER}.tar.gz -C ${EALABBLD}
cd ${EALABBLD}/u-boot-${UBOOTVER}
ARCH=${ARCH} CROSS_COMPILE=${CROSS_COMPILE} make vexpress_ca9x4_defconfig O=${EALABBLD}/u-boot-${UBOOTVER}-build
ARCH=${ARCH} CROSS_COMPILE=${CROSS_COMPILE} make -j $(nproc) O=${EALABBLD}/u-boot-${UBOOTVER}-build
cp ${EALABBLD}/u-boot-${UBOOTVER}-build/{u-boot,u-boot.bin} ${EALABBOOT}
Download and compile Linux Kernel export LNXMAJ=$(echo ${LINUXVER} | cut -c1)
wget https://mirror.yandex.ru/pub/linux/kernel/v${LNXMAJ}.x/linux-${LINUXVER}.tar.xz -P ${EALABDWN}
tar xf ${EALABDWN}/linux-${LINUXVER}.tar.xz -C ${EALABBLD}
cd ${EALABBLD}/linux-${LINUXVER}
ARCH=${ARCH} CROSS_COMPILE=${CROSS_COMPILE} make vexpress_defconfig O=${EALABBLD}/linux-${LINUXVER}-build
ARCH=${ARCH} CROSS_COMPILE=${CROSS_COMPILE} make -j $(nproc) O=${EALABBLD}/linux-${LINUXVER}-build zImage dtbs
ARCH=${ARCH} CROSS_COMPILE=${CROSS_COMPILE} make -j $(nproc) O=${EALABBLD}/linux-${LINUXVER}-build INSTALL_MOD_PATH=${EALABRFS} modules modules_install
cp ${EALABBLD}/linux-${LINUXVER}-build/arch/arm/boot/{Image,zImage} ${EALABBOOT}
cp ${EALABBLD}/linux-${LINUXVER}-build/arch/arm/boot/dts/*.dtb ${EALABBOOT}
Download and compile Busybox wget https://www.busybox.net/downloads/busybox-${BUSYBOXVER}.tar.bz2 -P ${EALABDWN}
tar xf ${EALABDWN}/busybox-${BUSYBOXVER}.tar.bz2 -C ${EALABBLD}
mkdir -p ${EALABBLD}/busybox-${BUSYBOXVER}-build
cd ${EALABBLD}/busybox-${BUSYBOXVER}
ARCH=${ARCH} CROSS_COMPILE=${CROSS_COMPILE} make defconfig O=${EALABBLD}/busybox-${BUSYBOXVER}-build
cd ${EALABBLD}/busybox-${BUSYBOXVER}-build
ARCH=${ARCH} CROSS_COMPILE=${CROSS_COMPILE} make -j $(nproc) install CONFIG_STATIC=y CONFIG_PREFIX=${EALABRFS} O=${EALABBLD}/busybox-${BUSYBOXVER}-build
qemu-system-arm -M vexpress-a9 -m 256 -dtb ${EALABBOOT}/vexpress-v2p-ca9.dtb \
-kernel ${EALABBOOT}/zImage -nographic
Linux kernel (Cortex A15) qemu-system-arm -M vexpress-a15 -m 256 -dtb ${EALABBOOT}/vexpress-v2p-ca15-tc1.dtb \
-kernel ${EALABBOOT}/zImage -nographic
qemu-system-arm -M vexpress-a9 -m 256 -kernel ${EALABBOOT}/u-boot -nographic -dtb ${EALABBOOT}/vexpress-v2p-ca9.dtb
ip link add qemu-arm-lab type bridge
ip link set dev qemu-arm-lab up
ip a a 192.168.1.250/24 dev qemu-arm-lab
dnsmasq -p 1053 -F 192.168.1.100,192.168.1.200,12h --enable-tftp --tftp-root=${EALABBOOT}
mkdir -p /usr/local/etc/qemu
echo 'allow all' > /usr/local/etc/qemu/bridge.conf
qemu-system-arm -M vexpress-a9 -m 256 -kernel ${EALABBOOT}/u-boot -nographic -dtb ${EALABBOOT}/vexpress-v2p-ca9.dtb -net nic -net bridge,br=qemu-arm-lab
setenv ipaddr 192.168.1.100
setenv serverip 192.168.1.250
setenv netmask 255.255.255.0
tftpboot 0x61000000 vexpress-v2p-ca9.dtb
tftpboot 0x62000000 zImage
bootz 0x62000000 - 0x61000000
mkdir -p ${EALABRFS}/dev
setenv ipaddr 192.168.1.100
setenv serverip 192.168.1.250
setenv netmask 255.255.255.0
setenv bootargs 'root=/dev/nfs rw nfsroot=192.168.1.250:<path>/Projects/qemu-arm-lab/rootfs,v3,tcp console=ttyAMA0 ip=dhcp rw'
tftpboot 0x61000000 vexpress-v2p-ca9.dtb
tftpboot 0x62000000 zImage
bootz 0x62000000 - 0x61000000
qemu-system-arm -M vexpress-a9 -m 256 -kernel ${EALABBOOT}/zImage -nographic -dtb ${EALABBOOT}/vexpress-v2p-ca9.dtb -append "loglevel=8 root=/dev/nfs rw nfsroot=192.168.1.250:/<path>/Projects/qemu-arm-lab/rootfs/,v3,tcp nfsrootdebug console=ttyAMA0 ip=dhcp rw rootwait" -net nic -net bridge,br=qemu-arm-lab
cd ${EALABBLD}/linux-${LINUXVER}
ARCH=${ARCH} CROSS_COMPILE=${CROSS_COMPILE} make menuconfig O=${EALABBLD}/linux-${LINUXVER}-build
Kernel hacking ---> Compile-time checks and compiler options ---> Compile the kernel with debug info
Kernel hacking ---> KGDB: kernel debugger ---> KGDB: use kgdb over the serial console
Kernel hacking ---> KGDB: kernel debugger ---> KGDB_KDB: include kdb frontend for kgdb
ARCH=${ARCH} CROSS_COMPILE=${CROSS_COMPILE} make -j $(nproc) O=${EALABBLD}/linux-${LINUXVER}-build zImage dtbs
ARCH=${ARCH} CROSS_COMPILE=${CROSS_COMPILE} make -j $(nproc) O=${EALABBLD}/linux-${LINUXVER}-build INSTALL_MOD_PATH=${EALABRFS} modules modules_install
cp ${EALABBLD}/linux-${LINUXVER}-build/arch/arm/boot/{Image,zImage} ${EALABBOOT}
cp ${EALABBLD}/linux-${LINUXVER}-build/arch/arm/boot/dts/*.dtb ${EALABBOOT}
cp ${EALABBLD}/linux-${LINUXVER}-build/vmlinux ${EALABBOOT}
qemu-system-arm -M vexpress-a9 -m 256 -kernel ${EALABBOOT}/zImage -nographic -dtb ${EALABBOOT}/vexpress-v2p-ca9.dtb -append "loglevel=8 root=/dev/nfs rw nfsroot=192.168.1.250:/home/olegsl/Projects/qemu-arm-lab/rootfs/,v3,tcp nfsrootdebug console=ttyAMA0 init=/bin/busybox ip=dhcp rw rootwait" -net nic -net bridge,br=qemu-arm-lab -s -S
docker run -it --rm --privileged --name ealab-gdb --net host -w /mnt -v ${EALAB}:/mnt ubuntu:18.10 bash -c 'apt update && apt install -y python gdb-multiarch wget && wget -O /etc/gdb/gdbinit git.io/.gdbinit && gdb-multiarch -q /mnt/boot/vmlinux'
set architecture armv5tej
target extended-remote :1234
b do_mount_root
c
Breakpoint 1, do_mount_root (name=0x80a58910 <nfs_root_device> "192.168.1.250:/home/olegsl/Projects/qemu-arm-lab/rootfs/", fs=0x808d6b68 "nfs", flags=32768, data=0x80a58408 <nfs_root_options>) at /mnt/build/linux-5.0.10/init/do_mounts.c:388
388 int err = ksys_mount(name, "/root", fs, flags, data);
─── Assembly ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
0x80a012a8 do_mount_root+0 push {r4, r5, r6, lr}
0x80a012ac do_mount_root+4 sub sp, sp, #8
0x80a012b0 do_mount_root+8 str r3, [sp]
0x80a012b4 do_mount_root+12 mov r3, r2
─── Expressions ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
─── History ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
─── Memory ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
─── Registers ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
r0 0x80a58910 r1 0x808d6b68 r2 0x00008000 r3 0x80a58408 r4 0x80b08c08
r5 0x00000005 r6 0x80b0ba00 r7 0x80b69530 r8 0x00000006 r9 0x808d6b68
r10 0x000003e8 r11 0x00000000 r12 0x8efc7a00 sp 0x8e83ff60 lr 0x80a01700
pc 0x80a012a8 cpsr 0x60000013 fpscr 0x00000000 fpsid 0x41033090 fpexc 0x00000000
PAR 0x00000000 PAR_S 0x00000000 ID_PFR0_S 0x00001031 ID_AFR0_S 0x00000000 ID_MMFR0_S 0x00100103
ID_ISAR2 0x21232041 ID_ISAR3 0x11112131 ID_PFR0 0x00001031 ID_MMFR0 0x00100103 ID_ISAR5 0x00000000
ID_ISAR1_S 0x13112111 ID_DFR0 0x00000000 ID_AFR0 0x00000000 ID_MMFR1 0x20000000 ID_MMFR2 0x01230000
ID_MMFR3 0x00002111 ID_ISAR4 0x00111142 ID_ISAR0_S 0x00101111 ID_ISAR6 0x00000000 ID_ISAR2_S 0x21232041
ID_ISAR3_S 0x11112131 ID_DFR0_S 0x00000000 ID_ISAR5_S 0x00000000 ID_MMFR4_S 0x00000000 ID_MMFR1_S 0x20000000
ID_ISAR1 0x13112111 TPIDRPRW 0x00000000 WFAR_S 0x00000000 IFAR_S 0x00000000 ID_ISAR0 0x00101111
ID_ISAR4_S 0x00111142 ID_ISAR6_S 0x00000000 ID_MMFR4 0x00000000 CLIDR 0x09000003 WFAR 0x00000000
ID_MMFR2_S 0x01230000 ID_MMFR3_S 0x00002111 MAIR0_S 0xff0a81a8 AIDR 0x00000000 CSSELR 0x00000000
PMCR_S 0x41002000 PMCR 0x41002000 PMSELR 0x00000000 CLIDR_S 0x09000003 MAIR1 0x00000000
DBGBVR_S 0x00000000 PMUSERENR_S 0x00000000 IFAR 0x00000000 PMINTENCLR_S 0x00000000 CPACR 0x00f00000
MAIR1_S 0x40e040e0 DBGBCR 0x00000000 PMCCNTR_S 0x00000000 OSLSR_EL1 0x0000000a DBGBVR 0x00000000
PMCNTENSET_S 0x00000000 PMCNTENCLR_S 0x00000000 PMOVSR_S 0x00000000 TPIDRURW 0x00000000 TPIDRURO 0x00000000
DBGBVR 0x00000000 DBGBCR 0x00000000 DBGBVR 0x00000000 MDCCSR_EL0 0x00000000 DBGBCR_S 0x00000000
DBGDIDR_S 0x35141000 MDCCSR_EL0_S 0x00000000 DBGDSAR_S 0x00000000 MDSCR_EL1 0x00000000 PMUSERENR 0x00000000
PMCCNTR 0x00000000 DBGBVR_S 0x00000000 DBGBCR_S 0x00000000 AIDR_S 0x00000000 CSSELR_S 0x00000000
PMCNTENSET 0x00000000 PMCNTENCLR 0x00000000 PMOVSR 0x00000000 VTTBR_S 0x0000000000000000 FAR_EL2 0x00000000
DBGBCR 0x00000000 AFSR0_EL1 0x00000000 AFSR1_EL1 0x00000000 DBGBVR 0x00000000 DBGBCR 0x00000000
PMINTENSET_S 0x00000000 MAIR0 0x00000000 DBGWCR 0x00000000 PMSELR_S 0x00000000 DBGDIDR 0x35141000
DBGBVR_S 0x00000000 DBGDSAR 0x00000000 DBGBVR 0x00000000 DBGWVR_S 0x00000000 PMINTENSET 0x00000000
PMINTENCLR 0x00000000 VBAR_EL2_S 0x00000000 DBGWVR_S 0x00000000 DBGWCR_S 0x00000000 CPTR_EL2 0x00000000
AFSR0_EL2 0x00000000 HACR_EL2 0x00000000 DBGBVR 0x00000000 DBGBCR 0x00000000 DBGWVR 0x00000000
DBGBVR_S 0x00000000 DBGWVR_S 0x00000000 DBGBVR_S 0x00000000 VPIDR_EL2_S 0x410fc090 DBGBCR 0x00000000
DBGWCR 0x00000000 DBGWVR 0x00000000 DBGBCR_S 0x00000000 DBGWVR_S 0x00000000 MDSCR_EL1_S 0x00000000
DBGBVR_S 0x00000000 DBGBCR_S 0x00000000 DBGWCR_S 0x00000000 HAMAIR1 0x00000000 DBGBCR_S 0x00000000
DBGWVR 0x00000000 HPFAR_EL2 0x00000000 CNTHCTL_EL2_S 0x00000000 CNTHP_TVAL_EL2 0x00000000 DBGWCR 0x00000000
MAIR_EL2_S 0x00000000 DBGBCR_S 0x00000000 DBGWVR 0x00000000 DBGWCR_S 0x00000000 MVBAR 0x00000000
HTTBR_S 0x0000000000000000 ESR_EL2 0x00000000 VPIDR_EL2 0x410fc090 MAIR_EL2 0x00000000 DBGWCR 0x00000000
HSTR_EL2 0x00000000 DBGWCR_S 0x00000000 AFSR1_EL2 0x00000000 DFAR 0x00000000 PMEVCNTR3_S 0x00000000
TEECR 0x00000001 CTR 0x80038003 TCMTR 0x00000000 PMEVCNTR0_S 0x00000000 PMEVCNTR1_S 0x00000000
PMEVCNTR2_S 0x00000000 VBAR_EL2 0x00000000 HMAIR1 0x00000000 SCTLR_EL2 0x00000000 AFSR0_EL3 0x00000000
HMAIR1_S 0x00000000 AMAIR_EL2 0x00000000 CNTHP_CTL_EL2 0x00000000 HTTBR 0x0000000000000000 CNTHP_TVAL_EL2_S 0x00000000
TTBR0_EL1_S 0x60004059 TTBR1_EL1_S 0x60004059 AMAIR_EL2_S 0x00000000 HAMAIR1_S 0x00000000 DFAR_S 0x00000000
DUMMY 0x00000000 TPIDR_EL2 0x00000000 SCR 0x00000000 VTTBR 0x0000000000000000 TCR_EL2 0x00000000
HIFAR 0x00000000 TTBR0_EL1 0x00000000 TTBR1_EL1 0x00000000 TTBCR 0x00000000 VTCR_EL2_S 0x00000000
VTCR_EL2 0x00000000 TCR_EL2_S 0x00000000 TEEHBR 0x00000000 DUMMY 0x00000000 FAR_EL2_S 0x00000000
CNTHP_CTL_EL2_S 0x00000000 ESR_EL2_S 0x00000000 HPFAR_EL2_S 0x00000000 HCR_EL2 0x00000000 MDCR_EL2 0x00000000
CTR_S 0x80038003 TCMTR_S 0x00000000 TLBTR_S 0x00000000 TTBCR_S 0x00000000 DUMMY 0x00000000
MIDR 0x410fc090 DUMMY 0x00000000 TLBTR 0x00000000 MVBAR_S 0x00000000 DUMMY_S 0x00000000
DUMMY_S 0x00000000 DUMMY 0x00000000 CNTHCTL_EL2 0x00000000 DUMMY_S 0x00000000 HIFAR_S 0x00000000
NSACR 0x00000000 DUMMY_S 0x00000000 TEECR_S 0x00000001 DUMMY_S 0x00000000 DFSR 0x00000000
IFSR 0x00000000 AFSR1_EL3 0x00000000 MIDR_S 0x410fc090 ACTLR_EL2 0x00000000 VBAR_S 0x00000000
SDER 0x00000000 CBAR 0x1e000000 SCTLR 0x00c50078 ACTLR_EL1 0x00000000 PMEVCNTR3 0x00000000
PMEVCNTR0 0x00000000 PMEVCNTR1 0x00000000 PMEVCNTR2 0x00000000 VBAR 0x00000000 PMEVTYPER0_S 0x00000000
PMEVTYPER1_S 0x00000000 PMEVTYPER2_S 0x00000000 PMEVTYPER3_S 0x00000000 PMEVTYPER0 0x00000000 PMEVTYPER1 0x00000000
PMEVTYPER2 0x00000000 PMEVTYPER3 0x00000000 CNTVOFF_S 0x0000000000000000 CNTHP_CVAL_S 0x0000000000000000 PMCCFILTR_S 0x00000000
DBGDRAR_S 0x00000000 TEEHBR_S 0x00000000 ACTLR_EL1_S 0x00000000 CPACR_S 0x00f00000 DACR_S 0x00000051
A9_DIAG_S 0x00000000 A9_PWRDIAG_S 0x00000000 DFSR_S 0x00000005 FCSEIDR_S 0x00000000 CONTEXTIDR_S 0x00000000
A9_PWRCTL_S 0x00000000 TPIDR_EL2_S 0x00000000 CBAR_S 0x1e000000 OSLSR_EL1_S 0x0000000a IFSR_S 0x00000000
TPIDRURW_S 0x00000000 TPIDRURO_S 0x00000000 CNTVOFF 0x0000000000000000 SCTLR_S 0x10c5387d SCTLR_EL2_S 0x00000000
ACTLR_EL2_S 0x00000000 TPIDRPRW_S 0x0e340000 CNTHP_CVAL 0x0000000000000000 SCR_S 0x00000000 SDER_S 0x00000000
AFSR0_EL1_S 0x00000000 AFSR1_EL1_S 0x00000000 AFSR0_EL2_S 0x00000000 NEONBUSY_S 0x00000000 CPTR_EL2_S 0x00000000
HSTR_EL2_S 0x00000000 PMCCFILTR 0x00000000 AFSR0_EL3_S 0x00000000 AFSR1_EL3_S 0x00000000 HACR_EL2_S 0x00000000
HCR_EL2_S 0x00000000 NSACR_S 0x00000000 AFSR1_EL2_S 0x00000000 MDCR_EL2_S 0x00000000 DBGDRAR 0x00000000
DACR 0x00000000 A9_PWRCTL 0x00000000 A9_DIAG 0x00000000 FCSEIDR 0x00000000 CONTEXTIDR_EL1 0x00000000
A9_PWRDIAG 0x00000000 NEONBUSY 0x00000000 TLB_VA_S 0x00000000 TLB_PA_S 0x00000000 TLB_VA 0x00000000
TLB_ATTR_S 0x00000000 TLB_PA 0x00000000 TLB_ATTR 0x00000000
─── Source ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
383 }
384
385 static int __init do_mount_root(char *name, char *fs, int flags, void *data)
386 {
387 struct super_block *s;
388 int err = ksys_mount(name, "/root", fs, flags, data);
389 if (err)
390 return err;
391
392 ksys_chdir("/root");
393 s = current->fs->pwd.dentry->d_sb;
─── Stack ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
[0] from 0x80a012a8 in do_mount_root+0 at /mnt/build/linux-5.0.10/init/do_mounts.c:388
arg name = 0x80a58910 <nfs_root_device> "192.168.1.250:/home/olegsl/Projects/qemu-arm-lab/rootfs/"
arg fs = 0x808d6b68 "nfs"
arg flags = 32768
arg data = 0x80a58408 <nfs_root_options>
[1] from 0x80a01700 in mount_nfs_root at /mnt/build/linux-5.0.10/init/do_mounts.c:486
(no arguments)
[+]
─── Threads ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
[1] id 1 from 0x80a012a8 in do_mount_root+0 at /mnt/build/linux-5.0.10/init/do_mounts.c:388
─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
>>>
