Skip to content

Instantly share code, notes, and snippets.

@tonosaman
Last active February 26, 2017 10:03
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save tonosaman/8607802de06d5bb74e9ee5fb35d46ed4 to your computer and use it in GitHub Desktop.
Save tonosaman/8607802de06d5bb74e9ee5fb35d46ed4 to your computer and use it in GitHub Desktop.
RPi3 aarch64 u-boot with jtag

u-boot with jtag enabler

based on ports/sysutils/u-boot-rpi3/@FreeBSD11.0-RELEASE-p1

diff --git a/arch/arm/cpu/armv8/start.S b/arch/arm/cpu/armv8/start.S
index 19c771d..26e2266 100644
--- a/arch/arm/cpu/armv8/start.S
+++ b/arch/arm/cpu/armv8/start.S
@@ -57,6 +57,7 @@ reset:
 	b	save_boot_params
 .globl	save_boot_params_ret
 save_boot_params_ret:
+	bl bcm2835_enable_jtag_early_boot
 
 #ifdef CONFIG_SYS_RESET_SCTRL
 	bl reset_sctrl
diff --git a/drivers/gpio/bcm2835_gpio.c b/drivers/gpio/bcm2835_gpio.c
index 8dd7a28..f99bcbe 100644
--- a/drivers/gpio/bcm2835_gpio.c
+++ b/drivers/gpio/bcm2835_gpio.c
@@ -126,3 +126,47 @@ U_BOOT_DRIVER(gpio_bcm2835) = {
 	.flags	= DM_FLAG_PRE_RELOC,
 	.priv_auto_alloc_size = sizeof(struct bcm2835_gpios),
 };
+
+/*
+ * Derived from code: https://github.com/dwelch67/raspberrypi/tree/master/armjtag
+ * ARM_TRST      22 GPIO_GEN3 P1-15 IN  (22 ALT4)
+ * ARM_TDO     5/24 GPIO_GEN5 P1-18 OUT (24 ALT4)
+ * ARM_TCK    13/25 GPIO_GEN6 P1-22 OUT (25 ALT4)
+ * ARM_TDI     4/26 GPIO_GCLK P1-7   IN ( 4 ALT5)
+ * ARM_TMS    12/27 CAM_GPIO  S5-11 OUT (27 ALT4)
+ */
+static int bcm2835_enable_jtag(struct udevice *dev) {
+    struct bcm2835_gpios _stub = { .reg = (void*)BCM2835_GPIO_BASE };
+    struct bcm2835_gpios *gpios = dev ? dev_get_priv(dev) : &_stub;
+    {
+        writel(0, &gpios->reg->gppud);
+        for (int i = 0; i < 150; i++) { __asm__ __volatile__ ("nop"); }
+        writel(BIT(4) | BIT(22) | BIT(24) | BIT(25) | BIT(27), &gpios->reg->gppudclk[0]);
+        for (int i = 0; i < 150; i++) { __asm__ __volatile__ ("nop"); }
+        writel(0, &gpios->reg->gppudclk[0]);
+    }
+    {
+        uint32_t val = readl(&gpios->reg->gpfsel[BCM2835_GPIO_FSEL_BANK(4)]);
+        val &= ~(BCM2835_GPIO_FSEL_MASK << BCM2835_GPIO_FSEL_SHIFT(4)); // Gpio4
+        val |= (BCM2835_GPIO_ALT5 << BCM2835_GPIO_FSEL_SHIFT(4)); // Alt5: ARM_TDI
+        writel(val, &gpios->reg->gpfsel[BCM2835_GPIO_FSEL_BANK(4)]);
+    }
+    {
+        uint32_t val = readl(&gpios->reg->gpfsel[BCM2835_GPIO_FSEL_BANK(20)]);
+        val &= ~(BCM2835_GPIO_FSEL_MASK << BCM2835_GPIO_FSEL_SHIFT(22)); // Gpio22
+        val |= (BCM2835_GPIO_ALT4 << BCM2835_GPIO_FSEL_SHIFT(22)); // Alt4: ARM_TRST
+        val &= ~(BCM2835_GPIO_FSEL_MASK << BCM2835_GPIO_FSEL_SHIFT(24)); // Gpio24
+        val |= (BCM2835_GPIO_ALT4 << BCM2835_GPIO_FSEL_SHIFT(24)); // Alt4: ARM_TDO
+        val &= ~(BCM2835_GPIO_FSEL_MASK << BCM2835_GPIO_FSEL_SHIFT(25)); // Gpio25
+        val |= (BCM2835_GPIO_ALT4 << BCM2835_GPIO_FSEL_SHIFT(25)); // Alt4: ARM_TCK
+        val &= ~(BCM2835_GPIO_FSEL_MASK << BCM2835_GPIO_FSEL_SHIFT(27)); // Gpio27
+        val |= (BCM2835_GPIO_ALT4 << BCM2835_GPIO_FSEL_SHIFT(27)); // Alt4: ARM_TMS
+        writel(val, &gpios->reg->gpfsel[BCM2835_GPIO_FSEL_BANK(20)]);
+    }
+	return 0;
+}
+
+extern void bcm2835_enable_jtag_early_boot(void);
+void bcm2835_enable_jtag_early_boot() {
+    bcm2835_enable_jtag(NULL);
+}
$ CROSS_COMPILE=/opt/gcc-linaro-5.3.1-2016.05-rc2-x86_64_aarch64-linux-gnu/bin/aarch64-linux-gnu- make drivers/gpio/bcm2835_gpio.o

disassemble

#!/bin/bash
less -f <(${CROSS_COMPILE}objdump -D -b binary -m aarch64 u-boot.bin) <(${CROSS_COMPILE}objdump -D -m aarch64 u-boot) arch/arm/cpu/armv8/start.S

u-boot works

U-Boot> dm tree
U-Boot> gpio status -a
U-Boot> fdt addr 0x100
U-Boot> fdt print gpio
U-Boot> usb reset
scanning bus 0 for devices... 3 USB Device(s) found
       scanning usb for storage devices... 0 Storage Device(s) found
       scanning usb for ethernet devices... 1 Ethernet Device(s) found
Hit any key to stop autoboot:  0
U-Boot> ping 192.168.1.1

extract raw initrd image which is guset boot media

% md5sum $HOME/rpi-aarch64-trial/xvisor/udisk.img
ba94adec40e08d380a099485b2db1aa3
% tail -c+65 < /home/tono/rpi-aarch64-trial/xvisor/udisk.img > /home/tono/rpi-aarch64-trial/xvisor/initrd.img
  • initrd.img will be loaded by u-boot nfs command.
  • xvisor will be assigned initrd(=rbd device) in accordance with the description in one_guest_virt-v8.dtb.

Edit .dts

one_guest_virt-v8.dtb generated from xvisor-next/arch/arm/board/generic/dts/bcm2837/*.dts.

  • To take over u-boot serial output, need to arrange dts files.
    [ARM] DTS files for raspi3 take over uart0 from u-boot (BCM2837)
    
    Disable uart_mu entry from bcm2837.dtsi, to avoid infinite wait at
    bcm283x_mu_driver_probe().
    
    Add skip_baudrate_config to uart0@bcm2837.dtsi, to prevent pl011_lowlevel_init()
    will disconnect a serial line which opened by u-boot.

diff --git a/arch/arm/board/generic/dts/bcm2837/bcm2837.dtsi b/arch/arm/board/generic/dts/bcm2837/bcm2837.dtsi
index 3d9c3b1..9be71ad 100644
--- a/arch/arm/board/generic/dts/bcm2837/bcm2837.dtsi
+++ b/arch/arm/board/generic/dts/bcm2837/bcm2837.dtsi
@@ -103,19 +103,20 @@
                        compatible = "brcm,bcm2835-poweroff";
                        reg = <0x3f100000 0x28>;
                };
-
+/*
                SERIAL_MU: uart_mu {
                        compatible = "brcm,bcm283x-mu";
                        reg = <0x3f215040 0x100>;
                        clock-frequency = <250000000>;
                        interrupts = <1 29>;
                };
-
+*/
                SERIAL0: uart0 {
                        compatible = "brcm,bcm2835-pl011", "arm,pl011";
                        reg = <0x3f201000 0x1000>;
                        clock-frequency = <3000000>;
                        interrupts = <2 25>;
+                       skip-baudrate-config;
                };
 
                i2c0: i2c@3f205000 {
  • xvisor-next/arch/arm/board/generic/dts/bcm2837/one_guest_virt-v8.dts

    • change console SERIAL_MU->SERIAL0
    • initrd -> raw block device
diff --git a/arch/arm/board/generic/dts/bcm2837/one_guest_virt-v8.dts b/arch/arm/board/generic/dts/bcm2837/one_guest_virt-v8.dts
index fa565f8..af25a9f 100644
--- a/arch/arm/board/generic/dts/bcm2837/one_guest_virt-v8.dts
+++ b/arch/arm/board/generic/dts/bcm2837/one_guest_virt-v8.dts
@@ -14,9 +14,10 @@
        };

        chosen {
-               console = &SERIAL_MU;
+               console = &SERIAL0;
                bootcmd = /* Mount initrd device */
-                         "vfs mount initrd /",
+                         "rbd create rbd0 0x2000000 0x1000000", // allocated by u-boot
+                         "vfs mount rbd0 /",

                          /* Load guest0 device tree from file */
                          "vfs fdt_load /guests guest0 /images/arm64/virt-v8x2.dtb mem0,physical_size,physsize,0x06000000 net0,switch,string,br0",
@@ -36,4 +37,3 @@
                          "vfs cat /system/banner.txt";
        };
 };
-
  • Build virt-v8.dtb
% cd xvisor-next/
% mkdir -p build/arch/arm/board/generic/dts/bcm2837/
% tools/dtc/dtc -o build/arch/arm/board/generic/dts/bcm2837/one_guest_virt-v8.dtb -I dts -O dtb -i arch/arm/board/generic/dts/bcm2837/ arch/arm/board/generic/dts/bcm2837/one_guest_virt-v8.dts
  • dump dtb xvisor-next/tools/dtc/fdtdump one_guest_virt-v8.dtb

boot/startup.nsh

setenv ipaddr 192.168.1.2
setenv serverip 192.168.1.1
setenv boot_targets xvisor
setenv bootcmd_xvisor 'run boot_net_usb_start; nfs 0x10000000 ${serverip}:/home/tono/rpi-aarch64-trial/xvisor/vmm.bin; nfs 0x800000 ${serverip}:/home/tono/rpi-aarch64-trial/xvisor/one_guest_virt-v8.dtb; nfs 0x2000000 ${serverip}:/home/tono/rpi-aarch64-trial/xvisor/initrd.img'

/etx/exports @ 192.168.1.1

"/home/tono/rpi-aarch64-trial" 192.168.1.2(rw,sync,no_root_squash,no_subtree_check)
file
$HOME/rpi-aarch64-trial/xvisor/vmm.bin
$HOME/rpi-aarch64-trial/xvisor/one_guest_virt-v8.dtb
$HOME/rpi-aarch64-trial/xvisor/initrd.img

Build OpenOCD

$ git clone https://github.com/daniel-k/openocd.git
$ cd openocd
$ git checkout armv8
$ ./bootstrap
$ ./configure --enable-ftdi --enable-bcm2835gpio
$ (cd jimtcl; make libjim.a; make all)
$ make install

openocd installed to /usr/local/bin/openocd

setup openocd

$ wget "https://gist.githubusercontent.com/benpye/f081f5dc82f9bc64be72f376ce66025d/raw/dc04c0aee6c0b89b8bcebecff122569e1a553b39/bcm2837.cfg"
$ sudo cp ../benpye/bcm2837.cfg /usr/local/share/openocd/scripts/target/
$ echo 'ATTRS{idProduct}=="002b", ATTRS{idVendor}=="15ba", MODE="666", GROUP="plugdev"' | sudo tee /etc/udev/rules.d/10-olimex-arm-usb-ocd-h.rules
$ sudo udevadm trigger
$ sudo usermod -a -G plugdev ${USER}

run openocd

$ openocd -s /usr/local/share/openocd/scripts -f interface/ftdi/olimex-arm-usb-ocd-h.cfg -f target/bcm2837.cfg

run gdb

Open emacs and exec M-x gdb command, then input below to mini-buffer:

~/github/gcc-linaro-5.3.1-2016.05-rc2-x86_64_aarch64-linux-gnu/bin/aarch64-linux-gnu-gdb -i=mi -x rpi3-xvisor.gdb
  • gud-rpi3-xvisor.gdb
target remote localhost:3333
monitor targets rpi3.cpu
monitor halt
monitor targets

symbol ./xvisor/build/vmm.elf -readnow
# If vmm.bin have not been loaded by u-boot, then use load command to transfer via jtag.
# (gdb) file ./xvisor/build/vmm.elf -readnow
# (gdb) load

set $pc = 0x10000000
set $x0 = 0x00800000

define enable_irq
  set $cpsr = ($cpsr & ~0x80)
end

define disable_irq
  set $cpsr = ($cpsr | 0x80)
end

# thb *((void()*) _bss_zero+12)
# thb *((void()*) align_4k_boundary)
# thb *((void()*) _start_mmu_init)

# record
  • start xvisor
(gdb) continue

Xvisor bootlog

Xvisor v0.2.8 (Jan 29 2017 15:11:59)

init: host address space
init: heap management
init: exception table
init: per-CPU areas
init: device tree
init: host irq subsystem
init: CPU early
init: board early
init: standard I/O
init: clocksource manager
init: clockchip manager
init: hypervisor timer
init: soft delay
init: hypervisor manager
init: hypervisor scheduler
init: hypervisor threads
init: inter-processor interrupts
init: workqueue framework
init: wallclock subsystem
init: secondary CPUs
init: failed to start CPU3 (error -15)
init: command manager
init: device driver framework
init: device emulation framework
init: character device framework
init: iommu framework
init: hypervisor modules
vgic: GIC node not found
vgic: emulator not available
init: block device framework
init: network switch framework
init: network port framework
initrd: linux,initrd-start/initrd-start attribute not found
init: CPU final
init: board final
INFO: mailbox@3f00b800: mailbox enabled
sdhci@3f300000: SDHCI controller v3 at 0x3f300000 irq 94 [PIO]
init: CPU0 online
init: CPU1 online
init: CPU2 online
init: CPU3 possible
init: brought-up 3 CPUs
init: freeing init memory 104K
init: change stdio device to uart0
XVisor#

entering Xvisor console

XVisor# version    
Xvisor v0.2.8 (Jan 29 2017 13:31:34)
XVisor# rbd create rbd0 0x2000000 0x1000000
XVisor# rbd list
--------------------------------------------------------------------------------
 Name                             Physical Address       Physical Size         
--------------------------------------------------------------------------------
 rbd0                             0x0000000002000000     0x0000000001000000    
--------------------------------------------------------------------------------
XVisor# vfs mount rbd0 /
Trying: ext4
Mounted rbd0 using ext4 at /
XVisor# vfs mplist
--------------------------------------------------------------------------------
 BlockDev        Filesystem  Mode        Path                                   
--------------------------------------------------------------------------------
 rbd0            ext4        read-write  /                                      
--------------------------------------------------------------------------------
XVisor# vfs fdt_load /guests guest0 /images/arm64/virt-v8/virt-v8.dtb
XVisor# devtree node dump /guests/guest0
<<<device_type = "guest" not found>>>
XVisor# guest create guest0
guest0: Failed to create
Error: command guest failed (code -1)
Failed to find guest guest0

Error occurred at vmm_manager_guest_create()@vmm_manager.c, because device_tree = "guest" not found.

build latest initrd.img for guest(linux4.9)

% tests/common/scripts/build-arm-images.sh -a v8 -g virt-v8 -p . -o build/ -l 4.9 -i
=== Build configuration ===
arm_family = v8
guest_type = virt-v8
guest_output_path = /home/tono/github/xvisor-next/build-guest
num_threads = 1
tarball_path = /home/tono/github/xvisor-next/build-targz
xvisor_arch = arm
xvisor_cross_compile = aarch64-linux-gnu-
xvisor_defconfig = generic-v8-defconfig
xvisor_source_path = .
xvisor_output_path = build/
xvisor_guest_dts_basename = virt-v8x2
xvisor_disk_path = build//disk-virt-v8
xvisor_disk_ext2_path = build//disk-virt-v8.ext2
xvisor_basic_firmware_source_path = ./tests/arm64/virt-v8/basic
xvisor_only = no
linux_version = 4.9
linux_cpatch = no
linux_arch = arm64
linux_cross_compile = aarch64-linux-gnu-
linux_tarball = linux-4.9.tar.gz
linux_tarball_path = /home/tono/github/xvisor-next/build-targz/linux-4.9.tar.gz
linux_tarball_url = https://www.kernel.org/pub/linux/kernel/v4.x/linux-4.9.tar.gz
linux_oldconfig_path = ./tests/arm64/virt-v8/linux/linux-4.9_defconfig
linux_source_path = /home/tono/github/xvisor-next/build-guest/linux-4.9
linux_output_path = /home/tono/github/xvisor-next/build-guest/linux-virt-v8
linux_dtb_name = virt-v8.dtb
linux_dts_path = ./tests/arm64/virt-v8/linux/virt-v8.dts
busybox_version = 1.21.1
busybox_cross_compile = aarch64-linux-gnu-
busybox_tarball = busybox-1.21.1.tar.bz2
busybox_tarball_path = /home/tono/github/xvisor-next/build-targz/busybox-1.21.1.tar.bz2
busybox_tarball_url = http://busybox.net/downloads/busybox-1.21.1.tar.bz2
busybox_oldconfig_path = ./tests/common/busybox/busybox-1.21.1_defconfig
busybox_output_path = /home/tono/github/xvisor-next/build-guest/busybox-virt-v8
busybox_rootfs_cpio_path = /home/tono/github/xvisor-next/build-guest/rootfs-virt-v8.cpio
busybox_rootfs_ext2_path = /home/tono/github/xvisor-next/build-guest/rootfs-virt-v8.ext2
sed -i -- 's/SERIAL_MU/SERIAL0/g' arch/arm/board/generic/dts/bcm2837/*.dts
export PATH=/home/tono/github/gcc-linaro-5.3.1-2016.05-rc2-x86_64_aarch64-linux-gnu/bin/:$PATH
mkdir -p build/tests/arm64/virt-v8/basic/
tests/common/scripts/build-arm-images.sh -a v8 -g virt-v8 -p . -o build/ -l 4.9
  • (dtc) arch/arm/board/generic/dts/bcm2837/zero_guest.dtb
  • (dtc) arch/arm/board/generic/dts/bcm2837/one_guest_pb-a8.dtb
  • (dtc) arch/arm/board/generic/dts/bcm2837/one_guest_vexpress-a9.dtb
  • (dtc) arch/arm/board/generic/dts/bcm2837/one_guest_vexpress-a15.dtb
  • (dtc) arch/arm/board/generic/dts/bcm2837/one_guest_virt-v7.dtb
  • (dtc) arch/arm/board/generic/dts/bcm2837/two_guest_pb-a8.dtb
  • (dtc) arch/arm/board/generic/dts/bcm2837/two_guest_vexpress-a15.dtb

initrd.img for virt-v8 guest

xvisor-next/tests/arm64/virt-v8/linux/README

$ sudo mount-o loop xvisor/initrd.img ./mnt
├── images
│   └── arm64 <- build/disk-virt-v8/images/arm64/
|       ├── rootfs.img <- build-guest/rootfs-virt-v8.cpio
│       │   │             └- build-guest/rootfs-virt-v8.ext2 (busybox user land)
│       ├── virt-v8
│       │   ├── Image <- build-guest/linux-virt-v8/arch/arm64/boot/Image
│       │   │            └- build-guest/linux-virt-v8/vmlinux
│       │   │               └- tests/arm64/virt-v8/linux/linux-4.9_defconfig
│       │   ├── cmdlist <- tests/arm64/virt-v8/linux/cmdlist
│       │   ├── firmware.bin <- build/tests/arm64/virt-v8/basic/firmware.bin
│       │   ├── nor_flash.list <- tests/arm64/virt-v8/linux/nor_flash.list
│       │   └── virt-v8.dtb <- tests/arm64/virt-v8/linux/virt-v8.dts
│       └── virt-v8x2.dtb <- tests/arm64/virt-v8/virt-v8x2.dts
├── system
│   ├── banner.txt
│   └── logo.ppm
├── tmp
└── lost+found
  • images/arm64/virt-v8/Image ... Linux version 4.9.0 (gcc version 5.3.1 20160412 (Linaro GCC 5.3-2016.05-rc2) )

md5sum-disk.img.sh

#!/bin/bash
XVISOR_ROOT=${HOME}/rpi-aarch64-trial/xvisor
function dtcmd5() {
  md5sum <(${XVISOR_ROOT}/build/tools/dtc/bin/dtc -I dts -O dtb -o - $1 2>/dev/null) | awk "{ x=\$1 \"  $1\"; sub(\"dts$\", \"dts -> dtb\", x); print x; }"
}
md5sum ${XVISOR_ROOT}/initrd.img
md5sum ${XVISOR_ROOT}/vmlinux      ${XVISOR_ROOT}/build-guest/linux-virt-v8/vmlinux
md5sum ${XVISOR_ROOT}/firmware.elf ${XVISOR_ROOT}/build/tests/arm64/virt-v8/basic/firmware.elf
md5sum ${XVISOR_ROOT}/build/tests/arm64/virt-v8/basic/firmware.bin
md5sum ${XVISOR_ROOT}/tests/arm64/virt-v8/linux/cmdlist
md5sum ${XVISOR_ROOT}/build-guest/linux-virt-v8/arch/arm64/boot/Image
dtcmd5 ${XVISOR_ROOT}/tests/arm64/virt-v8/linux/virt-v8.dts
md5sum ${XVISOR_ROOT}/tests/arm64/virt-v8/linux/nor_flash.list
dtcmd5 ${XVISOR_ROOT}/tests/arm64/virt-v8/virt-v8x2.dts
md5sum ${XVISOR_ROOT}/build-guest/rootfs-virt-v8.cpio
mkdir -p ./mnt
sudo mount -oloop ${XVISOR_ROOT}/initrd.img ./mnt
find ./mnt/images/arm64 -type f -exec md5sum \{\} \;
# diff <(${XVISOR_ROOT}/build/tools/dtc/bin/fdtdump <(${XVISOR_ROOT}/build/tools/dtc/bin/dtc -I dts -O dtb -o - ${XVISOR_ROOT}/tests/arm64/virt-v8/linux/virt-v8.dts)) <(${XVISOR_ROOT}/build/tools/dtc/bin/fdtdump mnt/images/arm64/virt-v8/virt-v8.dtb)
sudo umount ./mnt; rmdir ./mnt

# strings ${XVISOR_ROOT}/build/disk-virt-v8/images/arm64/virt-v8/Image | grep "Linux version" | head -1
# strings mnt/images/arm64/virt-v8/Image | grep "Linux version" | head -1
f425031ce5293283f7092e5f9537ab4b  xvisor/build/vmm.elf
83a3eb699d08db80c4e5027af3e10313  xvisor/initrd.img
5eb94db67f735f62545d8f8bec1d9f8f  xvisor/one_guest_virt-v8.dtb
---
4d69f58a0ca915ea9faf32725ae0e270  ./mnt/images/arm64/virt-v8x2.dtb
4d69f58a0ca915ea9faf32725ae0e270  xvisor/tests/arm64/virt-v8/virt-v8x2.dts -> dtb
5f17159836909763745b3b23209c8480  ./mnt/images/arm64/rootfs.img
5f17159836909763745b3b23209c8480  /xvisor/build-guest/rootfs-virt-v8.cpio
733ff9869994447dae43f5a0a3a8ed23  ./mnt/images/arm64/virt-v8/nor_flash.list
733ff9869994447dae43f5a0a3a8ed23  xvisor/tests/arm64/virt-v8/linux/nor_flash.list
7b1a743b5dbcb9563c8cb3fb6ce75985  xvisor/build/tests/arm64/virt-v8/basic/firmware.elf
7b1a743b5dbcb9563c8cb3fb6ce75985  xvisor/firmware.elf
85005cbd275631d9e477bed19c581e9d  ./mnt/images/arm64/virt-v8/Image
85005cbd275631d9e477bed19c581e9d  xvisor/build-guest/linux-virt-v8/arch/arm64/boot/Image
8dfc37716a398c5ce82d958b644ec673  ./mnt/images/arm64/virt-v8/cmdlist
8dfc37716a398c5ce82d958b644ec673  xvisor/tests/arm64/virt-v8/linux/cmdlist
a55d81a0d73a38ce10d14adc39ac90b1  xvisor/build-guest/linux-virt-v8/vmlinux
a55d81a0d73a38ce10d14adc39ac90b1  xvisor/vmlinux
aa796cbf9b2cd2c5d7a6fd99f5824931  ./mnt/images/arm64/virt-v8/firmware.bin
aa796cbf9b2cd2c5d7a6fd99f5824931  xvisor/build/tests/arm64/virt-v8/basic/firmware.bin
c9da07f85d63c66dc9494ffd40e03941  ./mnt/images/arm64/virt-v8/virt-v8.dtb
c9da07f85d63c66dc9494ffd40e03941  xvisor/tests/arm64/virt-v8/linux/virt-v8.dts -> dtb

boot sequence

3 USB Device(s) found
       scanning usb for storage devices... 0 Storage Device(s) found
       scanning usb for ethernet devices... 1 Ethernet Device(s) found
Hit any key to stop autoboot:  0
Unknown command 'boot_net_usb_start' - try 'help'
Waiting for Ethernet connection... done.
#################################################################
         #################################################################
         #################################################################
         #####################################
done
Bytes transferred = 1183432 (120ec8 hex)
Waiting for Ethernet connection... done.
#
done
Bytes transferred = 3667 (e53 hex)
Waiting for Ethernet connection... done.
#################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         #################################################################
         ###########################
done
Bytes transferred = 16777216 (1000000 hex)
U-Boot>
Xvisor v0.2.8 (Jan 29 2017 15:11:59)

init: host address space
init: heap management
init: exception table
init: per-CPU areas
init: device tree
init: host irq subsystem
init: CPU early
init: board early
init: standard I/O
init: clocksource manager
init: clockchip manager
init: hypervisor timer
init: soft delay
init: hypervisor manager
init: hypervisor scheduler
init: hypervisor threads
init: inter-processor interrupts
init: workqueue framework
init: wallclock subsystem
init: secondary CPUs
init: failed to start CPU3 (error -15)
init: command manager
init: device driver framework
init: device emulation framework
init: character device framework
init: iommu framework
init: hypervisor modules
vgic: GIC node not found
vgic: emulator not available
init: block device framework
init: network switch framework
init: network port framework
initrd: linux,initrd-start/initrd-start attribute not found
init: CPU final
init: board final
INFO: mailbox@3f00b800: mailbox enabled
sdhci@3f300000: SDHCI controller v3 at 0x3f300000 irq 94 [PIO]
init: CPU0 online
init: CPU1 online
init: CPU2 online
init: CPU3 possible
init: brought-up 3 CPUs
init: freeing init memory 104K
init: change stdio device to uart0
Created rbd0 RBD instance
Trying: ext4
Mounted rbd0 using ext4 at /
guest0: Created
guest0: Loading 0x0000000000000000 with file /images/arm64/virt-v8/firmware.bin
guest0: Loaded 0x0000000000000000 with 74024 bytes
guest0: Loading 0x00000000000FF000 with file /images/arm64/virt-v8/cmdlist
guest0: Loaded 0x00000000000FF000 with 163 bytes
guest0: Loading 0x0000000000100000 with file /images/arm64/virt-v8/Image
guest0: Loaded 0x0000000000100000 with 8024576 bytes
guest0: Loading 0x0000000000FF0000 with file /images/arm64/virt-v8/virt-v8.dtb
guest0: Loaded 0x0000000000FF0000 with 2254 bytes
guest0: Loading 0x0000000001000000 with file /images/arm64/rootfs.img
guest0: Loaded 0x0000000001000000 with 1670656 bytes

ooooooo  ooooo oooooo     oooo ooooo  .oooooo..o   .oooooo.   ooooooooo.
 `8888    d8'   `888.     .8'  `888' d8P'    `Y8  d8P'  `Y8b  `888   `Y88.
   Y888..8P      `888.   .8'    888  Y88bo.      888      888  888   .d88'
    `8888'        `888. .8'     888    `Y8888o.  888      888  888ooo88P'
   .8PY888.        `888.8'      888        `Y88b 888      888  888`88b.
  d8'  `888b        `888'       888  oo     .d8P `88b    d88'  888  `88b.
o888o  o88888o       `8'       o888o 8''88888P'   `Y8bood8P'  o888o  o888o

XVisor#
XVisor# guest kick guest0
guest0: Kicked
XVisor# vserial bind guest0/uart0
[guest0/uart0] Virt-v8 Basic Firmware
[guest0/uart0]
[guest0/uart0] autoboot: disabled
[guest0/uart0]
[guest0/uart0] basic# hi
[guest0/uart0] hello
[guest0/uart0] basic# hello
[guest0/uart0] hi
[guest0/uart0] basic# help
[guest0/uart0] help        - List commands and their usage
[guest0/uart0]
[guest0/uart0] hi          - Say hi to ARM test code
[guest0/uart0]
[guest0/uart0] hello       - Say hello to ARM test code
[guest0/uart0]
[guest0/uart0] wfi_test    - Run wait for irq instruction test for ARM test code
[guest0/uart0]             Usage: wfi_test [<msecs>]
[guest0/uart0]             <msecs>  = delay in milliseconds to wait for
[guest0/uart0]
[guest0/uart0] mmu_setup   - Setup MMU for ARM test code
[guest0/uart0]
[guest0/uart0] mmu_state   - MMU is enabled/disabled for ARM test code
[guest0/uart0]
[guest0/uart0] mmu_test    - Run MMU test suite for ARM test code
[guest0/uart0]
[guest0/uart0] mmu_cleanup - Cleanup MMU for ARM test code
[guest0/uart0]
[guest0/uart0] timer       - Display timer information
[guest0/uart0]
[guest0/uart0] dhrystone   - Dhrystone 2.1 benchmark
[guest0/uart0]             Usage: dhrystone [<iterations>]
[guest0/uart0]
[guest0/uart0] hexdump     - Dump memory contents in hex format
[guest0/uart0]             Usage: hexdump <addr> <count>
[guest0/uart0]             <addr>  = memory address in hex
[guest0/uart0]             <count> = byte count in hex
[guest0/uart0]
[guest0/uart0] copy        - Copy to target memory from source memory
[guest0/uart0]             Usage: copy <dest> <src> <count>
[guest0/uart0]             <dest>  = destination address in hex
[guest0/uart0]             <src>   = source address in hex
[guest0/uart0]             <count> = byte count in hex
[guest0/uart0]
[guest0/uart0] start_linux - Start linux kernel (device-tree mechanism)
[guest0/uart0]             Usage: start_linux <kernel_addr> <fdt_addr> [<initrd_addr>] [<initrd_size>]
[guest0/uart0]             <kernel_addr>  = kernel load address
[guest0/uart0]             <fdt_addr>     = fdt blob address
[guest0/uart0]             <initrd_addr>  = initrd load address (optional)
[guest0/uart0]             <initrd_size>  = initrd size (optional)
[guest0/uart0]
[guest0/uart0] fdt_override_u32 - Overrides an integer property in the device tree
[guest0/uart0]             Usage: fdt_override_u32 <fdt_addr> </path/to/property> <value>
[guest0/uart0]
[guest0/uart0] linux_cmdline - Show/Update linux command line
[guest0/uart0]             Usage: linux_cmdline [<new_linux_cmdline>]
[guest0/uart0]             <new_linux_cmdline>  = linux command line
[guest0/uart0]
[guest0/uart0] linux_memory_size - Show/Update linux memory size
[guest0/uart0]             Usage: linux_memory_size [<memory_size>]
[guest0/uart0]             <memory_size>  = memory size in hex
[guest0/uart0]
[guest0/uart0] autoexec    - autoexec command list from flash
[guest0/uart0]             Usage: autoexec
[guest0/uart0]
[guest0/uart0] go          - Jump to a given address
[guest0/uart0]             Usage: go <addr>
[guest0/uart0]             <addr>  = jump address in hex
[guest0/uart0]
[guest0/uart0] reset       - Reset the system
[guest0/uart0]
[guest0/uart0] basic# autoexec
[guest0/uart0] autoexec(copy 0x40080000 0x00100000 0xEF0000)
[guest0/uart0] copy took 2755221 usecs for 0xEF0000 bytes
[guest0/uart0] autoexec(copy 0x42000000 0x00FF0000 0x010000)
[guest0/uart0] copy took 11540 usecs for 0x010000 bytes
[guest0/uart0] autoexec(copy 0x42100000 0x01000000 0x800000)
[guest0/uart0] copy took 1475260 usecs for 0x800000 bytes
[guest0/uart0] autoexec(start_linux 0x40080000 0x42000000 0x42100000 0x800000)
[guest0/uart0] Trying to set new bootargs: root=/dev/ram rw earlycon=pl011,0x09000000 console=ttyAMA0 swiotlb=4096 mem=96M maxcpus=2
[guest0/uart0] Jumping into linux ...
[guest0/uart0] [    0.000000] Booting Linux on physical CPU 0x0
[guest0/uart0] [    0.000000] Linux version 4.9.0 (tono@sh76en) (gcc version 5.3.1 20160412 (Linaro GCC 5.3-2016.05-rc2) ) #1 SMP PREEMPT Sun Feb 5 16[guest0/uart0]
[guest0/uart0] [    0.000000] Boot CPU: AArch64 Processor [000f0000]
[guest0/uart0] [    0.000000] earlycon: pl11 at MMIO 0x0000000009000000 (options '')
[guest0/uart0] [    0.000000] bootconsole [pl11] enabled
[guest0/uart0] [    0.000000] Memory limited to 96MB
[guest0/uart0] [    0.000000] efi: Getting EFI parameters from FDT:
[guest0/uart0] [    0.000000] efi: UEFI not found.
[guest0/uart0] [    0.000000] cma: Reserved 16 MiB at 0x0000000045000000
[guest0/uart0] [    0.000000] psci: probing for conduit method from DT.
[guest0/uart0] [    0.000000] psci: PSCIv0.2 detected in firmware.
[guest0/uart0] [    0.000000] psci: Using standard PSCI v0.2 function IDs
[guest0/uart0] [    0.000000] psci: Trusted OS migration not required
[guest0/uart0] [    0.000000] percpu: Embedded 20 pages/cpu @ffff800004f9f000 s42752 r8192 d30976 u81920
[guest0/uart0] [    0.000000] Detected VIPT I-cache on CPU0
[guest0/uart0] [    0.000000] Built 1 zonelists in Zone order, mobility grouping on.  Total pages: 24192
[guest0/uart0] [    0.000000] Kernel command line: root=/dev/ram rw earlycon=pl011,0x09000000 console=ttyAMA0 swiotlb=4096 mem=96M maxcpus=2
[guest0/uart0] [    0.000000] log_buf_len individual max cpu contribution: 4096 bytes
[guest0/uart0] [    0.000000] log_buf_len total cpu_extra contributions: 12288 bytes
[guest0/uart0] [    0.000000] log_buf_len min size: 16384 bytes
[guest0/uart0] [    0.000000] log_buf_len: 32768 bytes
[guest0/uart0] [    0.000000] early log buf free: 14712(89%)
[guest0/uart0] [    0.000000] PID hash table entries: 512 (order: 0, 4096 bytes)
[guest0/uart0] [    0.000000] Dentry cache hash table entries: 16384 (order: 5, 131072 bytes)
[guest0/uart0] [    0.000000] Inode-cache hash table entries: 8192 (order: 4, 65536 bytes)
[guest0/uart0] [    0.000000] Memory: 48612K/98304K available (5308K kernel code, 348K rwdata, 1740K rodata, 384K init, 243K bss, 33308K reserved, 163[guest0/uart0] d)
[guest0/uart0] [    0.000000] Virtual kernel memory layout:
[guest0/uart0] [    0.000000]     modules : 0xffff000000000000 - 0xffff000008000000   (   128 MB)
[guest0/uart0] [    0.000000]     vmalloc : 0xffff000008000000 - 0xffff7dffbfff0000   (129022 GB)
[guest0/uart0] [    0.000000]       .text : 0xffff000008080000 - 0xffff0000085b0000   (  5312 KB)
[guest0/uart0] [    0.000000]     .rodata : 0xffff0000085b0000 - 0xffff000008770000   (  1792 KB)
[guest0/uart0] [    0.000000]       .init : 0xffff000008770000 - 0xffff0000087d0000   (   384 KB)
[guest0/uart0] [    0.000000]       .data : 0xffff0000087d0000 - 0xffff000008827200   (   349 KB)
[guest0/uart0] [    0.000000]        .bss : 0xffff000008827200 - 0xffff000008863fc0   (   244 KB)
[guest0/uart0] [    0.000000]     fixed   : 0xffff7dfffe7fd000 - 0xffff7dfffec00000   (  4108 KB)
[guest0/uart0] [    0.000000]     PCI I/O : 0xffff7dfffee00000 - 0xffff7dffffe00000   (    16 MB)
[guest0/uart0] [    0.000000]     vmemmap : 0xffff7e0000000000 - 0xffff800000000000   (  2048 GB maximum)
[guest0/uart0] [    0.000000]               0xffff7e0000000000 - 0xffff7e0000180000   (     1 MB actual)
[guest0/uart0] [    0.000000]     memory  : 0xffff800000000000 - 0xffff800006000000   (    96 MB)
[guest0/uart0] [    0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=4, Nodes=1
[guest0/uart0] [    0.000000] Preemptible hierarchical RCU implementation.
[guest0/uart0] [    0.000000]   Build-time adjustment of leaf fanout to 64.
[guest0/uart0] [    0.000000]   RCU restricting CPUs from NR_CPUS=64 to nr_cpu_ids=4.
[guest0/uart0] [    0.000000] RCU: Adjusting geometry for rcu_fanout_leaf=64, nr_cpu_ids=4
[guest0/uart0] [    0.000000] NR_IRQS:64 nr_irqs:64 0
[guest0/uart0] [    0.000000] arm_arch_timer: Architected cp15 timer(s) running at 19.20MHz (virt).
[guest0/uart0] [    0.000000] clocksource: arch_sys_counter: mask: 0xffffffffffffff max_cycles: 0x46d987e47, max_idle_ns: 440795202767 ns
[guest0/uart0] [    0.000006] sched_clock: 56 bits at 19MHz, resolution 52ns, wraps every 4398046511078ns
[guest0/uart0] [    0.010886] Console: colour dummy device 80x25
[guest0/uart0] [    0.017832] Calibrating delay loop (skipped), value calculated using timer frequency.. 38.40 BogoMIPS (lpj=76800)
[guest0/uart0] [    0.030769] pid_max: default: 32768 minimum: 301
[guest0/uart0] [    0.038135] Security Framework initialized
[guest0/uart0] [    0.044865] Mount-cache hash table entries: 512 (order: 0, 4096 bytes)
[guest0/uart0] [    0.054028] Mountpoint-cache hash table entries: 512 (order: 0, 4096 bytes)
[guest0/uart0] [    0.064801] ASID allocator initialised with 65536 entries
[guest0/uart0] [    0.104180] EFI services will not be available.
[guest0/uart0] [    0.150378] Detected VIPT I-cache on CPU1
[guest0/uart0] [    0.150520] CPU1: Booted secondary processor [000f0000]
[guest0/uart0] [    0.150666] Brought up 2 CPUs
[guest0/uart0] [    0.169460] SMP: Total of 2 processors activated.
[guest0/uart0] [    0.176844] CPU features: detected feature: 32-bit EL0 Support
[guest0/uart0] [    0.185617] CPU: All CPU(s) started at EL1
[guest0/uart0] [    0.192993] devtmpfs: initialized
[guest0/uart0] [    0.199858] DMI not present or invalid.
[guest0/uart0] [    0.205222] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns
[guest0/uart0] [    0.218902] NET: Registered protocol family 16
[guest0/uart0] [    0.240820] cpuidle: using governor ladder
[guest0/uart0] [    0.246312] vdso: 2 pages (1 code @ ffff0000085b7000, 1 data @ ffff0000087d4000)
[guest0/uart0] [    0.256274] hw-breakpoint: found 6 breakpoint and 4 watchpoint registers.
[guest0/uart0] [    0.266780] DMA: preallocated 256 KiB pool for atomic allocations
[guest0/uart0] [    0.274592] Serial: AMBA PL011 UART driver
[guest0/uart0] [    0.282329] 9000000.pl011: ttyAMA0 at MMIO 0x9000000 (irq = 5, base_baud = 0) is a PL011 rev1
[guest0/uart0] [    0.292488] console [ttyAMA0] enabled
[guest0/uart0] [    0.292488] console [ttyAMA0] enabled
[guest0/uart0] [    0.305139] bootconsole [pl11] disabled
[guest0/uart0] [    0.305139] bootconsole [pl11] disabled
[guest0/uart0] [    0.347570] HugeTLB registered 2 MB page size, pre-allocated 0 pages
[guest0/uart0] [    0.356999] SCSI subsystem initialized
[guest0/uart0] [    0.362575] pps_core: LinuxPPS API ver. 1 registered
[guest0/uart0] [    0.369663] pps_core: Software ver. 5.3.6 - Copyright 2005-2007 Rodolfo Giometti <giometti@linux.it>
[guest0/uart0] [    0.381508] dmi: Firmware registration failed.
[guest0/uart0] [    0.389696] clocksource: Switched to clocksource arch_sys_counter
[guest0/uart0] [    0.397596] VFS: Disk quotas dquot_6.6.0
[guest0/uart0] [    0.404047] VFS: Dquot-cache hash table entries: 512 (order 0, 4096 bytes)
[guest0/uart0] [    0.413986] simple-framebuffer 30000000.simplefb: framebuffer at 0x30000000, 0x1000000 bytes, mapped to 0xffff000009000000
[guest0/uart0] [    0.427248] simple-framebuffer 30000000.simplefb: format=r5g6b5, mode=1024x768x16, linelength=2048
[guest0/uart0] [    0.449322] Console: switching to colour frame buffer device 128x48
[guest0/uart0] [    0.465997] simple-framebuffer 30000000.simplefb: fb0: simplefb registered!
[guest0/uart0] [    0.487285] NET: Registered protocol family 2
[guest0/uart0] [    0.493805] TCP established hash table entries: 1024 (order: 1, 8192 bytes)
[guest0/uart0] [    0.502606] TCP bind hash table entries: 1024 (order: 2, 16384 bytes)
[guest0/uart0] [    0.511689] TCP: Hash tables configured (established 1024 bind 1024)
[guest0/uart0] [    0.520802] UDP hash table entries: 256 (order: 1, 8192 bytes)
[guest0/uart0] [    0.529221] UDP-Lite hash table entries: 256 (order: 1, 8192 bytes)
[guest0/uart0] [    0.538348] NET: Registered protocol family 1
[guest0/uart0] [    0.545811] RPC: Registered named UNIX socket transport module.
[guest0/uart0] [    0.553776] RPC: Registered udp transport module.
[guest0/uart0] [    0.561147] RPC: Registered tcp transport module.
[guest0/uart0] [    0.568545] RPC: Registered tcp NFSv4.1 backchannel transport module.
[guest0/uart0] [    0.577923] Trying to unpack rootfs image as initramfs...
[guest0/uart0] [    0.693049] Freeing initrd memory: 8192K (ffff800002100000 - ffff800002900000)
[guest0/uart0] [    0.701967] kvm [1]: HYP mode not available
[guest0/uart0] [    0.710345] futex hash table entries: 1024 (order: 5, 131072 bytes)
[guest0/uart0] [    0.718007] audit: initializing netlink subsys (disabled)
[guest0/uart0] [    0.726018] audit: type=2000 audit(0.659:1): initialized
[guest0/uart0] [    0.734639] workingset: timestamp_bits=46 max_order=15 bucket_order=0
[guest0/uart0] [    0.760508] squashfs: version 4.0 (2009/01/31) Phillip Lougher
[guest0/uart0] [    0.773005] NFS: Registering the id_resolver key type
[guest0/uart0] [    0.779360] Key type id_resolver registered
[guest0/uart0] [    0.786175] Key type id_legacy registered
[guest0/uart0] [    0.792891] nfs4filelayout_init: NFSv4 File Layout Driver Registering...
[guest0/uart0] [    0.802585] 9p: Installing v9fs 9p2000 file system support
[guest0/uart0] [    0.817159] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 250)
[guest0/uart0] [    0.825848] io scheduler noop registered
[guest0/uart0] [    0.832736] io scheduler cfq registered (default)
[guest0/uart0] [    0.853206] Unable to detect cache hierarchy from DT for CPU 0
[guest0/uart0] [    0.878030] brd: module loaded
[guest0/uart0] [    0.891802] loop: module loaded
[guest0/uart0] [    0.901834] nbd: registered device at major 43
[guest0/uart0] [    0.922832] hisi_sas: driver version v1.6
[guest0/uart0] [    0.928866] libphy: Fixed MDIO Bus: probed
[guest0/uart0] [    0.936633] tun: Universal TUN/TAP device driver, 1.6
[guest0/uart0] [    0.942951] tun: (C) 1999-2004 Max Krasnyansky <maxk@qualcomm.com>
[guest0/uart0] [    0.954473] mousedev: PS/2 mouse device common for all mice
[guest0/uart0] [    0.964160] NET: Registered protocol family 17
[guest0/uart0] [    0.970018] 9pnet: Installing 9P2000 support
[guest0/uart0] [    0.976964] Key type dns_resolver registered
[guest0/uart0] [    0.984830] registered taskstats version 1
[guest0/uart0] [    0.990859] hctosys: unable to open rtc device (rtc0)
[guest0/uart0] [    0.998562] uart-pl011 9000000.pl011: no DMA platform data
[guest0/uart0] [    1.006932] Freeing unused kernel memory: 384K (ffff800000770000 - ffff8000007d0000)
[guest0/uart0]            _  _
[guest0/uart0]           | ||_|
[guest0/uart0]           | | _ ____  _   _  _  _
[guest0/uart0]           | || |  _ \| | | |\ \/ /
[guest0/uart0]           | || | | | | |_| |/    \
[guest0/uart0]           |_||_|_| |_|\____|\_/\_/
[guest0/uart0]
[guest0/uart0]                Busybox Rootfs
[guest0/uart0]
[guest0/uart0] Please press Enter to activate this console.
[guest0/uart0] / # ls
[guest0/uart0] bin      etc      linuxrc  root     sys
[guest0/uart0] dev      init     proc     sbin     usr
[guest0/uart0] / #

^[+x+q return to Xvisor prompt.

Research: guest clock speed

tests/arm64/virt-v8/linux/linux-4.9_defconfig
1802:CONFIG_ARM_ARCH_TIMER=y
1803:CONFIG_ARM_ARCH_TIMER_EVTSTREAM=y

Research : guest cpu clock

tests/arm64/virt-v8/basic/gic_config.h
#define GIC_CPU_BASE		VIRT_V8_GIC_CPU
#define GIC_DIST_BASE		VIRT_V8_GIC_DIST

tests/arm64/virt-v8/basic/arm_plat.h
#define VIRT_V8_GIC			(0x08000000)
#define VIRT_V8_GIC_SIZE		(0x00020000)

tests/arm64/virt-v8/virt-v8x2.dts
		gic_dist {
			manifest_type = "virtual";
			address_type = "memory";
			guest_physical_addr = <0x08000000>; <- VIRT_V8_GIC
			physical_size = <0x1000>; <- 0x20000 overroded by one_guest_virt-v8.dts <- VIRT_V8_GIC_SIZE
			device_type = "pic";
			compatible = "arm,vgic,dist"; <- "virt,gic" overroded by one_guest_virt-v8.dts
			parent_irq = <6>;
			num_irq = <96>;
		};

arch/arm/board/generic/dts/bcm2837/one_guest_virt-v8.dts
		bootcmd = /* Mount initrd device */
              ...
			  /* Load guest0 device tree from file */
			  "vfs fdt_load /guests guest0 /images/arm64/virt-v8x2.dtb mem0,physical_size,physsize,0x06000000 net0,switch,string,br0",
			  /* Replace guest0 vGIC emulator with GIC emulator */
			  "devtree attr set /guests/guest0/aspace/gic_dist compatible string virt,gic",
			  "devtree attr set /guests/guest0/aspace/gic_dist physical_size physsize 0x20000",
			  "devtree node del /guests/guest0/aspace/gic_cpu",

tests/arm32/common/basic/arm_entry_v7.S:165:8:
	/* Enable the GIC CPU interface for this core */
	ldr	r0, _gic_cpu_addr
	mov	r1, #1
	str	r1, [r0]
	mov	r1, #0xFF
	str	r1, [r0, #4]
	ldr	r0, _sys_flags_addr
1:	
	/* Wait for interrupt before checking SPIN_ADDR */
	wfi	
...
_gic_cpu_addr:
	.word	GIC_CPU_BASE

tests/arm32/virt-v7/basic/README:48:283:
tests/arm32/virt-v7/linux/README:73:283:
# ${CROSS_COMPILE}gcc -nostdlib -march=armv7ve -mcpu=cortex-a15 -e start_boot -Wl,--build-id=none -Wl,-Ttext=0x80000000
 -DGENTIMER_FREQ=100000000 -DUART_PL011 -DUART_PL011_BASE=0x1c090000
 -DGICv2 -DGIC_DIST_BASE=0x2c001000 -DGIC_CPU_BASE=0x2c002000
 -DSPIN_LOOP_ADDR=0x14000000 -DSPIN_LOCATION=0x1c010030
 -DIMAGE=build/vmm.bin -DINITRD=build/disk.img
 -DDTB=build/arch/arm/board/generic/dts/vexpress/a15/one_guest_virt-v7.dtb
 ./docs/arm/fast_model_boot.S -o build/fast_model_boot.axf

tests/arm64/common/basic/arm_entry.S:79:9:
	/* Set GIC priority mask bit [7] = 1 */
	ldr	x0, __gic_cpu_base		/* CPU GIC base */
	mov	x1, #0x1
	str	w1, [x0]			/* GIC CPU CTRL */
	mov	x1, #0xFF
	str	w1, [x0, #0x4]			/* GIC CPU PMR */
__gic_cpu_base:
	.dword	GIC_CPU_BASE

tests/arm64/virt-v8/linux/README:74:202:
tests/arm64/virt-v8/basic/README:49:202:
# ${CROSS_COMPILE}gcc -nostdlib -nostdinc -e _start -Wl,--build-id=none -Wl,-Ttext=0x80000000
 -DGENTIMER_FREQ=100000000 -DUART_PL011 -DUART_PL011_BASE=0x1c090000
 -DGICv2 -DGIC_DIST_BASE=0x2c001000 -DGIC_CPU_BASE=0x2c002000
 -DSPIN_LOOP_ADDR=0x8000fff8
 -DIMAGE=./build/vmm.bin -DINITRD=./build/disk.img
 -DDTB=./build/arch/arm/board/generic/dts/foundation-v8/gicv2/one_guest_virt-v8.dtb
 ./docs/arm/foundation_v8_boot.S -o ./build/foundation_v8_boot.axf

probe gic_dist@arch/arm/board/generic/dts/bcm2837/one_guest_virt-v8.dts

  • einst ... emurator instance
  • reg ... region

struct vmm_emudev { struct gic_state *priv }

vmm_devemu_probe_region()@core/vmm_devemu.c
├─ reg->node = {name="gic_dist"}
├─match = {name="", type="pic", compatible="virt,gic", data=0x4}
├─gic_emulator_probe()@emulators/pic/gic.c
│  └─gic_state_alloc(name="gic_dist", type=GIC_TYPE_VIRT, num_cpu=2, is_child_pic=0, base_irq=0, num_irq=96, parent_irq=6)
└─gic_emulator_reset
   └─gic_state_reset((struct gic_state*)edev->priv)

[guest0/uart0] [    0.000000] arm_arch_timer: Architected cp15 timer(s) running at 19.20MHz (virt).
[guest0/uart0] [    0.000000] clocksource: arch_sys_counter: mask: 0xffffffffffffff max_cycles: 0x46d987e47, max_idle_ns: 440795202767 ns
[guest0/uart0] [    0.000006] sched_clock: 56 bits at 19MHz, resolution 52ns, wraps every 4398046511078ns

other emurator instances

hb core/vmm_devemu.c:1000

struct vmm_emudev { struct pl011_state *priv }

tests/arm64/virt-v8/virt-v8x2.dts
		uart0 {
			manifest_type = "virtual";
			address_type = "memory";
			guest_physical_addr = <0x09000000>;
			physical_size = <0x1000>;
			device_type = "serial";
			compatible = "primecell,arm,pl011";
			fifo_size = <1024>;
			interrupts = <33>;
		};

vmm_devemu_probe_region()@core/vmm_devemu.c
├─reg->node ={name="uart0"}
├─match {name="", type="serial", compatible="primecell,arm,pl011", data=<pl011_configs>}
├─pl011_emulator_probe()@emulators/serial/pl011.c
│  ├─vmm_vserial_create(name="guest0/uart0", can_send=<pl011_vserial_can_send>
│  │                  , send=<pl011_vserial_send>, receive_fifo_size=1024)@core/vio/vmm_vserial.c
│  ├─vser->receive_fifo = fifo_alloc(1, receive_fifo_size)
│  └─vmm_blocking_notifier_call(&vsctrl.notifier_chain, VMM_VSERIAL_EVENT_CREATE, &event)
└─pl011_emulator_reset()

[guest0/uart0] [    0.274592] Serial: AMBA PL011 UART driver
[guest0/uart0] [    0.282329] 9000000.pl011: ttyAMA0 at MMIO 0x9000000 (irq = 5, base_baud = 0) is a PL011 rev1

struct vmm_emudev { struct vminfo_state *priv }

tests/arm64/virt-v8/virt-v8x2.dts
        vminfo {
            manifest_type = "virtual";
            address_type = "memory";
            device_type = "sys";
            compatible = "vminfo-0.1";
            guest_physical_addr = <0x09001000>;
            physical_size = <0x1000>;
            ram0_base = <0x40000000>;
        };

vmm_devemu_probe_region()@core/vmm_devemu.c
├─reg->node ={name="vminfo"}
├─match {name="", type="sys", compatible="vminfo-0.1", data=0x1}
├─vminfo_emulator_probe(eid=<vminfo_emuid_table>)@/home/tono/Vagrant/clangX/xvisor-next/emulators/sys/vminfo.c
│  ├─s->nb.notifier_call = &vminfo_guest_aspace_notification;
│  └─vmm_guest_aspace_register_client(&s->nb)
└─vminfo_emulator_reset()

struct vmm_emudev { struct simplefb_state *priv { struct vmm_vdisplay *vdis } }

tests/arm64/virt-v8/virt-v8x2.dts
        simplefb {
            manifest_type = "virtual";
            address_type = "memory";
            device_type = "display";
            compatible = "simplefb-0.1";
            guest_physical_addr = <0x09002000>;
            physical_size = <0x1000>;
            width = <1024>; /* Override this if required */
            height = <768>; /* Override this if required */
            mode = "r5g6b5"; /* Override this if required */
            base = <0x30000000>; /* Override this if required */
        };

vmm_devemu_probe_region()@core/vmm_devemu.c
├─reg->node ={name="simplefb"}
├─match {name="", type="display", compatible="simplefb-0.1", data = 0x1}
├─simplefb_emulator_probe(eid=<simplefb_emuid_table>)@emulators/display/simplefb.c
│  ├─s->nb.notifier_call = &vminfo_guest_aspace_notification;
│  ├─vmm_guest_aspace_register_client(&s->nb)
│  ├─vmm_vdisplay_create(name="guest0/simplefb", &simplefb_ops, s)
│  └─vmm_blocking_notifier_call(&vsctrl.notifier_chain, VMM_VSERIAL_EVENT_CREATE, &event)
└─simplefb_emulator_reset()

[guest0/uart0] [    0.413986] simple-framebuffer 30000000.simplefb: framebuffer at 0x30000000, 0x1000000 bytes, mapped to 0xffff000009000000
[guest0/uart0] [    0.427248] simple-framebuffer 30000000.simplefb: format=r5g6b5, mode=1024x768x16, linelength=2048
[guest0/uart0] [    0.449322] Console: switching to colour frame buffer device 128x48
[guest0/uart0] [    0.465997] simple-framebuffer 30000000.simplefb: fb0: simplefb registered!

struct vmm_emudev { struct virtio_mmio_dev *priv { struct virtio_device dev } }

tests/arm64/virt-v8/virt-v8x2.dts
		NET0: virtio-net0 {
			manifest_type = "virtual";
			address_type = "memory";
			device_type = "virtio";
			compatible = "virtio,mmio";
			virtio_type = <1>;
			guest_physical_addr = <0x0A000000>;
			physical_size = <0x1000>;
			switch = ""; /* Override this before guest creation */
			interrupts = <48>;
		};

arch/arm/board/generic/dts/bcm2837/one_guest_virt-v8.dts
		bootcmd = /* Mount initrd device */
             "vfs fdt_load /guests guest0 /images/arm64/virt-v8x2.dtb mem0,physical_size,physsize,0x06000000 net0,switch,string,br0",

vmm_devemu_probe_region()@core/vmm_devemu.c
├─reg->node={name="virtio-net0"}
├─match {name="", type="virtio", compatible="virtio,mmio", data = 0x0}
├─virtio_mmio_probe(eid=<virtio_mmio_emuid_table>)@emulators/virtio/virtio_mmio.c
│  ├─m->dev.name="guest0/virtio-net0"
│  ├─m->irq=48
│  └─virtio_register_device(&m->dev)
└─virtio_mmio_reset()
  ├─vmm_devemu_emulate_irq(m->guest, m->irq, 0)
  └─virtio_reset(&m->dev)

struct vmm_emudev { struct virtio_mmio_dev *priv { struct virtio_device dev } }

tests/arm64/virt-v8/virt-v8x2.dts
		DISK0: virtio-blk0 {
			manifest_type = "virtual";
			address_type = "memory";
			device_type = "virtio";
			compatible = "virtio,mmio";
			virtio_type = <2>;
			guest_physical_addr = <0x0A001000>;
			physical_size = <0x1000>;
			blkdev = ""; /* Override this before guest creation */
			interrupts = <49>;
		};

vmm_devemu_probe_region()@core/vmm_devemu.c
├─reg->node={name="virtio-blk0"}
├─match {name="", type="virtio", compatible="virtio,mmio", data = 0x0}
├─virtio_mmio_probe(eid=<virtio_mmio_emuid_table>)@emulators/virtio/virtio_mmio.c
│  ├─m->dev.name="guest0/virtio-blk0"
│  ├─m->irq=49
│  └─virtio_register_device(&m->dev)
└─virtio_mmio_reset()
  ├─vmm_devemu_emulate_irq(m->guest, m->irq, 0)
  └─virtio_reset(&m->dev)

struct vmm_emudev { struct virtio_mmio_dev *priv { struct virtio_device dev } }

tests/arm64/virt-v8/virt-v8x2.dts
		virtio-con0 {
			manifest_type = "virtual";
			address_type = "memory";
			device_type = "virtio";
			compatible = "virtio,mmio";
			virtio_type = <3>;
			guest_physical_addr = <0x0A002000>;
			physical_size = <0x1000>;
			interrupts = <50>;
		};

vmm_devemu_probe_region()@core/vmm_devemu.c
├─reg->node={name="virtio-con0"}
├─match {name="", type="virtio", compatible="virtio,mmio", data = 0x0}
├─virtio_mmio_probe(eid=<virtio_mmio_emuid_table>)@emulators/virtio/virtio_mmio.c
│  ├─m->dev.name="guest0/virtio-con0"
│  ├─m->irq=50
│  └─virtio_register_device(&m->dev)
└─virtio_mmio_reset()
  ├─vmm_devemu_emulate_irq(m->guest, m->irq, 0)
  └─virtio_reset(&m->dev)

Research : Howto debug the guest

We should know about TLB and memory map of guest.

load guest image set (firmware included) to guest aspace

  • firmware.bin is relocatable
  1. guest image set is listed in tests/arm64/virt-v8/linux/nor_flash.list
- format: <paddr@guest> <path to file in `rbd0` mounted at `/`>

```
0x00000000 /images/arm64/virt-v8/firmware.bin
0x000FF000 /images/arm64/virt-v8/cmdlist
0x00100000 /images/arm64/virt-v8/Image
0x00FF0000 /images/arm64/virt-v8/virt-v8.dtb
0x01000000 /images/arm64/rootfs.img
```
  1. execute vfs guest_load_list command in chosen/bootcmd section of one_guest_virt-v8.dtb

    XVisor# vfs guest_load_list guest0 /images/arm64/virt-v8/nor_flash.list

    cmd_vfs_load_list()@commands/cmd_vfs.c
    └─cmd_vfs_load_list(guest!=NULL)
      └─vmm_guest_memory_write()
        ├─buf = vmm_malloc(VFS_LOAD_BUF_SZ)
        └─vmm_guest_memory_write(guest, wr_pa, buf, buf_count, FALSE)@core/vmm_guest_aspace.c
    
  2. files are loaded to guest padder

```
guest0: Loading 0x0000000000000000 with file /images/arm64/virt-v8/firmware.bin
guest0: Loaded 0x0000000000000000 with 74024 bytes
guest0: Loading 0x00000000000FF000 with file /images/arm64/virt-v8/cmdlist
guest0: Loaded 0x00000000000FF000 with 163 bytes
guest0: Loading 0x0000000000100000 with file /images/arm64/virt-v8/Image
guest0: Loaded 0x0000000000100000 with 8024576 bytes
guest0: Loading 0x0000000000FF0000 with file /images/arm64/virt-v8/virt-v8.dtb
guest0: Loaded 0x0000000000FF0000 with 2254 bytes
guest0: Loading 0x0000000001000000 with file /images/arm64/rootfs.img
guest0: Loaded 0x0000000001000000 with 1670656 bytes
```

firmware.bin is built from sources under tests/

  • tests/arm64/common/basic/Makefile.inc
  • tests/arm64/virt-v8/basic/Makefile

insider of firmware

  1. parse cmdlist
```
copy 0x40080000 0x00100000 0xEF0000
copy 0x42000000 0x00FF0000 0x010000
copy 0x42100000 0x01000000 0x800000
start_linux 0x40080000 0x42000000 0x42100000 0x800000
```

- autoexec(copy <dest> <src> <size>)

```
[guest0/uart0] basic# autoexec
[guest0/uart0] autoexec(copy 0x40080000 0x00100000 0xEF0000) <- Image@0x40080000
[guest0/uart0] copy took 2755221 usecs for 0xEF0000 bytes
[guest0/uart0] autoexec(copy 0x42000000 0x00FF0000 0x010000) <- virt-v8.dtb@0x42000000
[guest0/uart0] copy took 11540 usecs for 0x010000 bytes
[guest0/uart0] autoexec(copy 0x42100000 0x01000000 0x800000) <- rootfs.img@0x42100000
[guest0/uart0] copy took 1475260 usecs for 0x800000 bytes
[guest0/uart0] autoexec(start_linux 0x40080000 0x42000000 0x42100000 0x800000) <- start_linux 'Image 'virt-v8 'rootfs.img
```

Research : host device driver

INFO: mailbox@3f00b800: mailbox enabled
sdhci@3f300000: SDHCI controller v3 at 0x3f300000 irq 94 [PIO]
  • How to map physical device to guest

Sharing device memory mapping

  • vmm_guest_aspace_init(struct vmm_guest *guest)@core/vmm_guest_aspace.c

Research: how to select guest cpu mode (32bit/64bit)

decided by device tree (i.e. guest0/vcpus/cpu@0/compatible = "armv8,generic" in virt-v8.dtb)

arch_vcpu_init()@cpu_vcpu_helper.c

checked host capability

checked by cpu_supports_el1_a32()

About emurator devices

| struct vmm_emulator* | probe entry | |:--|:--|:--| | a9mpcore_emulator | | | | arm11mpcore_emulator | | | | arm11mpcore_emulator | | | | arm_sysregs_emulator | | | | gic_emulator | gic_emulator_probe | | l2x0_cc_emulator | | | lan9118_emulator | | | pl011_emulator | | | pl031_emulator | | | pl050_emulator | | | pl061_emulator | | | pl061_emulator | | | pl110_emulator | | | pl190_emulator | | | platform_pt_emulator | | | simplefb_emulator | | | smc91c111_emulator | | | sp804_emulator | | | sp805_emulator | | | sp810_emulator | | | virtio_mmio | | | vminfo_emulator | | | zero_emulator | |

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment