Bootstrapping custom virtual machine on chrome os without any dependencies (it reuses existing vm and it's kernel)

build-container-vm.sh

#!/usr/bin/env bash

export LC_ALL=C

set -eu

if test "$(id -u)" -ne 0; then
        echo "install: must be root"
        exit 1
fi

readonly LOCALDIR="/usr/local"
readonly DIR="$LOCALDIR/docker-vm"
readonly ROOTFS="$DIR/rootfs.img"

if test -d "$DIR"; then
	echo "Targed directory already exists, script will skip files that are already created"
fi

mkdir -p "$DIR"

###
### APK
###
readonly APK="$DIR/apk.static"
if ! test -e "$APK"; then
	( curl -L "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v2.14.0/$(uname -m)/apk.static" --output "$APK" ) &> /dev/null
	chmod +x "$APK"
	echo "Downloaded alpine package keeper to $APK"
fi

###
### KERNEL
###
readonly KERNEL="$DIR/kernel.img"
if ! test -e "$KERNEL"; then
	dlcservice_util --install --id=termina-dlc &> /dev/null
	cp /run/imageloader/termina-dlc/package/root/vm_kernel "$KERNEL"
	echo "Kernel has been copied from termina to $KERNEL"
fi

###
### CROS TOOLS
###
readonly CROSTOOLS="$DIR/cros-tools.img"
if ! test -e "$CROSTOOLS"; then
	dlcservice_util --install --id=termina-dlc &> /dev/null
	cp /run/imageloader/termina-dlc/package/root/vm_tools.img "$CROSTOOLS"
	echo "Cros-tools has been copied from termina to $CROSTOOLS"
fi

###
### SSH KEYS
###
readonly SSH_HOST_KEYS="dsa ecdsa ed25519 rsa"
KEYS_GEN=()
for key_type in $SSH_HOST_KEYS; do
	key_file="$DIR/ssh_host_${key_type}_key"
	if ! test -e "$key_file"; then
		ssh-keygen -q -f "$key_file" -N '' -t "$key_type"
		KEYS_GEN=("${KEYS_GEN[@]}" $key_type)
	fi
done

KEYS_GEN="${KEYS_GEN[@]}"
if test -n "$KEYS_GEN"; then
	echo "Generated new ssh host keys: $KEYS_GEN"
	if test -e "$ROOTFS"; then
		echo "Note: new keys won't be used until rootfs is regenerated"
	fi
fi
readonly USER="chronos"

###
### ROOTFS
###
gen_vshd_service() {
cat << 'EOF'
#!/sbin/openrc-run
supervisor=supervise-daemon

name="vshd"
description=""
description_reload=""

command="/opt/google/cros-containers/bin/vshd"
command_args=""

depend() {
        need sysfs 
}
EOF
}

gen_netstart_script() {
cat << 'EOF'
#!/bin/sh
set -eu
main() {
	local GUEST_DEV=eth0
	ip addr add $1/$3 dev "${GUEST_DEV}"
	ip link set "${GUEST_DEV}" up
	ip route add default via $2
	
	rc-service docker start

	if [ -e /etc/caddy/Caddyfile ]; then
		rc-service caddy start
	fi

	rc-service sshd start
	rc-service local start
}

if test $(id -u) -eq 0; then
	main "$@"
else
	sudo $0 "$@"
fi
EOF
}

gen_runstate_service() {
cat << 'EOF'
#!/sbin/openrc-run
name=""
description=""
description_reload=""

depend() {
	need sysfs
}

start() {
	if [ -x /opt/google/cros-containers/bin/vshd ]; then
		rc-service vshd start
	fi

	mkdir -p /var/lib/docker /var/home /var/.data
	chown 1000:1000 /var/home
	chown docker:docker /var/lib/docker
	
	mount LABEL=state "/var/.data" || true
	
	mkdir -p /var/.data/home /var/.data/docker /var/.data/local \
		/var/.data/etc/local.d /var/.data/etc/caddy

	chown 1000:1000 /var/.data/home

	if [ ! -f /var/.data/etc/resolv.conf ]; then
		echo 'nameserver 8.8.8.8' > /var/.data/etc/resolv.conf
	fi

	mount --bind /var/.data/home /var/home
	mount --bind /var/.data/etc/local.d /etc/local.d
	mount --bind /var/.data/docker /var/lib/docker
	mount --bind /var/.data/local /usr/local
	mount --bind /var/.data/etc/caddy /etc/caddy
}	
EOF
}

gen_inittab() {
	cat <<- 'EOF'
		::sysinit:/sbin/openrc sysinit
		::sysinit:/sbin/openrc boot
		::wait:/sbin/openrc default
		::respawn:/sbin/getty 38400 console
		::shutdown:/sbin/openrc shutdown
	EOF
}

gen_fstab() {
	cat <<- 'EOF'
		LABEL=rootfs / ext2 defaults,ro 0 0
		LABEL=cros-vm-tools /opt/google/cros-containers ext4 defaults,ro 0 0
		tmpfs /run tmpfs defaults,rw,exec 0 0
		tmpfs /var tmpfs defaults,rw,exec 0 0
		tmpfs /tmp tmpfs defaults,rw,exec 0 0
	EOF
}

gen_sshd_config() {
	cat <<- 'EOF'
		AllowAgentForwarding yes
		AllowTcpForwarding no
		GatewayPorts no
		X11Forwarding no
		PasswordAuthentication yes
		ChallengeResponseAuthentication no
		PermitEmptyPasswords yes
	EOF
}

build_rootfs() {
	local tmp_root="$1"
	test -n "${tmp_root:?}"
	
	mkdir -p "${tmp_root:?}/etc/apk"
	for repo in main community; do
		echo "http://dl-cdn.alpinelinux.org/alpine/latest-stable/$repo"
	done > "${tmp_root:?}/etc/apk/repositories"
	
	$APK add --root "${tmp_root:?}" --quiet --allow-untrusted --update-cache --initdb \
		alpine-baselayout-data openrc alpine-keys coreutils findutils grep \
		udev shadow sudo diffutils docker docker-compose caddy openssh \
		git micro curl
	
	passwd="$(cat "${tmp_root:?}/etc/passwd" | grep -v root)"
	echo -e "root:x:0:0:root,,,:/:/bin/nologin\n$passwd\n$USER:x:1000:1000:user,,,:/var/home:/bin/sh" > "${tmp_root:?}/etc/passwd"
	
	shadow="$(cat "${tmp_root:?}/etc/shadow" | grep -v root)"
	echo -e "root:!::0:::::\n$shadow\n$USER:::0:::::" > "${tmp_root:?}/etc/shadow"
	
	echo "$USER:!:1000:" >> "${tmp_root:?}/etc/group"
	sed -i -E "s/^(docker:x:.*:).*$/\1$USER/" "${tmp_root:?}/etc/group"
	
	echo "$USER:2000000:65536" > "${tmp_root:?}/etc/subuid"
	echo "root:1000000:65536" >> "${tmp_root:?}/etc/subuid"
	cp "${tmp_root:?}/etc/subuid" "${tmp_root:?}/etc/subgid"
	
	echo "$USER ALL=(ALL:ALL) NOPASSWD: ALL" > "${tmp_root:?}/etc/sudoers"
	
	rm -f "${tmp_root:?}/etc/resolv.conf"
	ln -s /var/.data/etc/resolv.conf "${tmp_root:?}/etc/resolv.conf"
	
	echo 'net.ipv4.ip_forward = 1' > "${tmp_root:?}/etc/sysctl.conf"
	
	echo -e "auto lo\niface lo inet loopback" > "${tmp_root:?}/etc/network/interfaces"

	gen_fstab > "${tmp_root:?}/etc/fstab"
	
	gen_inittab > "${tmp_root:?}/etc/inittab"
	
	for name in devfs dmesg udev udev-settle udev-trigger; do
		ln -s "/etc/init.d/$name" "${tmp_root:?}/etc/runlevels/sysinit/$name"
	done
	
	for name in bootmisc hostname loadkmap networking swap sysctl syslog urandom cgroups; do
		ln -s "/etc/init.d/$name" "${tmp_root:?}/etc/runlevels/boot/$name"
	done

	gen_vshd_service > "${tmp_root:?}/etc/init.d/vshd"
	gen_runstate_service > "${tmp_root:?}/etc/init.d/runstate"
	gen_netstart_script >  "${tmp_root:?}/etc/init.d/netstart"
	chmod a+x "${tmp_root:?}/etc/init.d/runstate" "${tmp_root:?}/etc/init.d/vshd" "${tmp_root:?}/etc/init.d/netstart"
	ln -s /etc/init.d/runstate "${tmp_root:?}/etc/runlevels/default/runstate"
	
	echo "" > ${tmp_root:?}/etc/motd
	
	for dir in home media mnt srv usr/local etc/apk usr/share/apk lib/apk root opt; do
		rm -fr "${tmp_root:?}/$dir"
	done
	
	for dir in tmp sys dev proc run var lib/modules etc/ssh; do
		rm -fr "${tmp_root:?}/$dir"
		mkdir -p "${tmp_root:?}/$dir"
	done
	
	mkdir -p "${tmp_root:?}/opt/google/cros-containers"

	gen_sshd_config > "${tmp_root:?}/etc/ssh/sshd_config"
	cp "$DIR/ssh_host_"* "${tmp_root:?}/etc/ssh"
	chown 0:0 -R "${tmp_root:?}/etc/ssh"
	chmod 600 -R "${tmp_root:?}/etc/ssh"
}

if ! test -f "$ROOTFS"; then
	echo "Creating VM rootfs"
	
	fallocate --length "416M" "$ROOTFS"
	mkfs.ext2 -L "rootfs" "$ROOTFS" &> /dev/null

    tmp_root="$LOCALDIR/tmp-$(date +%s)"
    cleanup_rootfs_build() {
    	set +eu
    	trap '' EXIT HUP INT TERM ERR
    	umount "${tmp_root:?}"
    	rm -fr "${tmp_root:?}" "$ROOTFS"
    	echo "Error encountered while building rootfs"
    	exit 1
    }
    
    trap cleanup_rootfs_build HUP INT TERM ERR
	mkdir "${tmp_root:?}"
	mount "$ROOTFS" "${tmp_root:?}"
	build_rootfs "${tmp_root:?}"
	umount "${tmp_root:?}"
    rm -fr "${tmp_root:?}"
    trap '' HUP INT TERM ERR

    echo "VM rootfs has been created and saved to $ROOTFS"
fi

###
### VMCTL
###
gen_startstop() {
cat << 'EOF'
#!/usr/bin/env bash

set -eu

if test "$(id -u)" -ne 0; then
	echo "vmctl: must be run as root"
	exit 1
fi

GATEWAY_MASK=24
NETWORK_NAME="vmtap0"
cd "$(dirname "$(readlink -f "$0")")"
source config.inc
readonly VM_CONTROL_SOCKET="control.sock"

create_network() {
	sysctl net.ipv4.ip_forward=1

	if ip link show dev "${NETWORK_NAME}" &> /dev/null; then
		return
	fi
	
	ip tuntap add mode tap user ${USER} vnet_hdr "${NETWORK_NAME}"
	ip addr add "${GATEWAY_IP}/${GATEWAY_MASK}" dev "${NETWORK_NAME}"
	ip link set "${NETWORK_NAME}" up

	for dev in wlan0 eth0; do
		iptables -t nat -A POSTROUTING -o "${dev}" -j MASQUERADE &&
			iptables -A FORWARD -i "${dev}" -o "${NETWORK_NAME}" -m state --state RELATED,ESTABLISHED -j ACCEPT &&
			iptables -A FORWARD -i "${NETWORK_NAME}" -o "${dev}" -j ACCEPT || true
	done
}

start_vm() {
	crosvm run ${EXTRA_ARGS:-} \
		--mem $MEM \
		--cpus $CPUS \
		--cid "$CID" \
		--socket "$VM_CONTROL_SOCKET" \
		--net tap-name=$NETWORK_NAME \
		--block "rootfs.img,ro,o_direct=true,root,sparse=false" \
		--block "cros-tools.img,ro,o_direct=true,sparse=false" \
		--block "$STATE_DISK,o_direct=true,sparse=false" \
		kernel.img			
}

case "${1:-}" in 
	start)
		if test -e "$VM_CONTROL_SOCKET"; then
			echo "vmctl: Seems already started"
			exit 1
		fi

		if [ -n "${DEBUG:-}" ]; then
			set -x
			create_network
			start_vm
		else
			create_network &> /dev/null
			( start_vm &> /dev/null & ) &

			attempts=60
			while test $attempts -gt 0; do
				./vmsh sudo /etc/init.d/netstart "${GUEST_IP}" "${GATEWAY_IP}" "${GATEWAY_MASK}" &> /dev/null && break
				echo -n .
				((attempts--))
				sleep 1s
				if ! test -e "$VM_CONTROL_SOCKET"; then
					echo -e"\rvmctl: Something went wrong              "
					exit 1
				fi
			done

			if test $attempts -eq 0; then
				echo -e "\rvmctl: Started without network              "
			else
				echo -e "\rvmctl: Started                              "
			fi
		fi				
	;;
	
	stop)
		if ! test -e "$VM_CONTROL_SOCKET"; then
			echo "vmctl: Seems already stopped"
			exit 1
		fi

		if crosvm stop "$VM_CONTROL_SOCKET" &> /dev/null; then
			echo "vmctl: Stopped"
		else
			echo "vmctl: Failed to stop"
		fi
	;;

	*)
		echo "Usage: vmctl {start|stop}"
		exit 1
	;;
esac
EOF
}

readonly VMCTL="$DIR/vmctl"
if ! test -e "$VMCTL"; then
	gen_startstop > "$VMCTL"
	chmod u+x "$VMCTL"
	echo "Created $VMCTL script"
fi

###
### VMSH
###
gen_vmsh() {
cat << 'EOF'
#!/usr/bin/env bash

set -eu
cd "$(dirname "$(readlink -f "$0")")"

readonly VM_CONTROL_SOCKET="control.sock"
if ! test -e "$VM_CONTROL_SOCKET"; then
	echo "vmsh: vm seems stopped"
	exit 1
fi

source config.inc

vsh --cid=$CID -- /usr/bin/env HOME=/var/home TERM=xterm PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin PWD=/var/home "${@:-sh -l}"
EOF
}

readonly VMSH="$DIR/vmsh"
if ! test -e "$VMSH"; then
	gen_vmsh > "$VMSH"
	chmod a+x "$VMSH"
	echo "Created $VMSH script"	
fi

###
### CONFIG
###
if ! test -e "$DIR/config.inc"; then
	cat <<- 'EOF' > $DIR/config.inc
		GATEWAY_IP=192.168.10.1
		GUEST_IP=192.168.10.2
		MEM=1024
		CPUS=4
		CID=10000
		STATE_DISK=state.img
		EXTRA_ARGS="--disable-sandbox --params nopci"
	EOF

	echo "Config example has been saved to $DIR/config.inc"
fi


###
### STATE DISK
###
source "$DIR/config.inc"
readonly STATE="$DIR/$STATE_DISK"
if ! test -e "$STATE"; then
	echo
	echo "Enter the size of a state disk or leave empty to skip"

	while true; do
		read -r -p '(e.g. 10M, 10G, <empty> to skip): ' size
		if test -z "$size"; then
			echo -en "\r! Disk creation has been skipped\n";
			break;
		fi

		if fallocate --length $size "$STATE"; then
			mkfs.ext4 -L state "$STATE" &> /dev/null
			echo -en "\rDisk for $size has been allocated and formatted\n"
			break;
		else
			echo "Retrying..."
		fi
	done
fi

###
### END
###
echo -e "\nFinished, here are some useful notes:\n"

echo "To control vm: $DIR/vmctl {start|stop}"
echo "To execute command remotely: $DIR/vmsh COMMAND"
echo "To login: ssh $USER@$GUEST_IP"
echo