cloud-init debug tools for azure ephemeral disk bug 1611074

README.md

debug tools for azure ephemeral disk bug 1611074

This just put together when debugging Bug 1611074

Testing

Things to test

1. upgrade-resize

2. clean image (simulated)

   • upgrade
   • clean /etc/fstab of any entry
   • simulate-first-boot

3. upgrade-reboot-resize

   This is present as it gets the /etc/fstab entry fixed before going into a resize. May not be necessary.

4. swap LP: #1642383

   • launch an instance with user-data from bug
   • upgrade
   • simulate-first-boot

launching and resizing

rgname="bug1611074" # resource group name
vmname="bugtest1"
azure config mode arm
azure vm quick-create $rgname $vmname centralus linux \
   Canonical:UbuntuServer:16.04.0-LTS:latest $USER \
   -M ~/.ssh/id_rsa.pub -z Standard_D1

azure vm set $rgname $vmname --vm-size=Standard_D1_v2

Sizes can be: Standard_D1 Standard_D1_v2 Standard_D2 and others.

clean-fstab

#!/bin/sh
exec sed -i '/\/mnt.*cloudconfig/d' /etc/fstab

disable-log-syslog

#!/bin/sh
sed -i.dist '/log_base.*log_syslog/s,^ ,# ,' /etc/cloud/cloud.cfg.d/05_logging.cfg

do-reboot

#!/bin/sh
me=${0}
PATH=${me%/*}:$PATH

error() { echo "$@" 1>&2; }
fail() { error "$@"; exit 1; }

[ "$(id -u)" = "0" ] || fail "not root"
clean=false
keep_logs=false
save=""
reboot=true
for i in "$@"; do
   case "$i" in
      clean) clean=true;;
      keep?logs) keep_logs=true;;
      save=*) save=${i#save=};;
      no-reboot) reboot=false;;
   esac
done

if [ -n "$save" ]; then
   save-old-data "$save" || fail "failed save data"
   error "saved to $save"
fi

if $clean; then
   rm -Rf /var/lib/cloud || fail "failed clean"
   error "cleared /var/lib/cloud"
fi

if ! $keep_logs; then
   rm -Rf /var/log/cloud* || fail "failed remove logs"
   error "cleared logs"
else
   error "kept logs"
fi

if $reboot; then
    error "rebooting"
    reboot
fi

ds-info.py

#!/usr/bin/python3
import json, sys
from cloudinit import stages
pkl_file = "/var/lib/cloud/instance/obj.pkl"
if len(sys.argv) > 1:
    pkl_file = sys.argv[1]
ds = stages._pkl_load(pkl_file)
print(json.dumps(ds.get_config_obj(), indent=1, sort_keys=True))

enable-proposed

#!/bin/sh
set -e
rel=$(lsb_release -sc)
line=$(awk '$1 == "deb" && $2 ~ /ubuntu.com/ {
  printf("%s %s %s-proposed main universe\n", $1, $2, rel); exit(0) };
  ' "rel=$rel" /etc/apt/sources.list)
echo "$line" | sudo tee /etc/apt/sources.list.d/proposed.list

go-format.sh

#!/bin/sh
disk=/dev/disk/cloud/azure_resource
part=/dev/disk/cloud/azure_resource-part1
mp=/mnt
size="${1:-100}" # size is in MB or "full" meaning full disk
if [ "$1" = "full" ]; then
   sectors=""
else
   sectors=$((2*1024*$size))
fi

mounted_at() {
    dev=$(awk '$2 == mp { print $1; exit(0); };' mp="$1" /proc/mounts)
    echo "$dev"
}
fail() { echo "$@" 1>&2; exit 1; }

dev=$(mounted_at $mp)
if [ -n "$dev" ]; then
   echo  "unmounting '$dev' at $mp"
   umount "$mp" || fail "failed unmount $dev"
   after_dev=$(mounted_at "$mp")
   [ -z "$after_dev" ] || fail "still mounted $after_dev at $mp"
else
   echo "nothing mounted at $mp"
fi

out=$(echo "2048,$sectors,7" | sfdisk --force "$disk" 2>&1) ||
   fail "failed sfdisk $disk: $out"
echo "repartitioned size=$size (sectors=$sectors)"
udevadm settle

dev=$(mounted_at "$mp")
if [ -n "$dev" ]; then
   echo "$mp got remounted ($dev). unmounting" 1>&2
   umount "$mp"
fi

time mkfs.ntfs --quick "$part"

save-old-data

#!/bin/sh
export LANG=C
[ -n "$1" ] || { echo need dir; exit 1; }
[ "$(id -u)" = "0" ] || { echo "not root"; exit 1; }
set -e
out_d=$1
rm -Rf "$out_d"
mkdir -p $out_d/logs $out_d/network/eni $out_d/ssh
rsync -a /var/lib/cloud/ $out_d/var_lib_cloud
rsync -a /run/cloud-init/ $out_d/run_cloud_init
cp /var/log/cloud-* $out_d/logs
cp /etc/network/interfaces $out_d/network/eni
cp -a /etc/network/interfaces.d $out_d/network/eni
dpkg-query --show > $out_d/package-manifest
cp /etc/cloud/ -r $out_d/etc-cloud
cp /etc/ssh/ssh_host_*.pub $out_d/ssh
apt-cache policy cloud-init > $out_d/apt-policy-cloud-init

cp /etc/fstab $out_d/etc-fstab
cp /proc/swaps $out_d/proc-swaps
cp /proc/mounts $out_d/proc-mounts
free > "$out_d/free.out"
blkid > "$out_d/blkid.out"
mount > "$out_d/mount.out"

systemctl list-dependencies > $out_d/systemctl-list-dependencies.out

find /run/systemd -type f > $out_d/systemd-run.list
find /run/systemd -type f -print0 |
   xargs -0 ls -altr --full-time > $out_d/systemd-run-ls.txt
journalctl -o short-monotonic > $out_d/journalctl-short-monotonic
journalctl -o short-precise > $out_d/journalctl-short-precise
systemctl --all --no-pager > $out_d/systemctl-all

items="
  cloud-init-local.service cloud-init.service cloud-config.service
  cloud-final.service cloud-init.target
  multi-user.target mnt.mount
"
for s in $items; do
   systemctl show --no-pager --full $s > $out_d/systemctl-show-$s 2>&1 || :
   systemctl status --no-pager --full $s > $out_d/systemctl-status-$s 2>&1 || :
   systemctl cat $s > $out_d/systemctl-$s 2>&1 || :
   systemctl list-dependencies $s > $out_d/systemctl-list-dependencies-$s 2>&1 || :
done

"${0%/*}/summary" "$out_d" > "$out_d/summary" 2>&1 || :
cat "$out_d/summary"

summary

#!/bin/sh

error() { echo "$@" 1>&2; }
fail() { echo "$@" 1>&2; exit 1; }
[ -d "$1" ] || fail "no dir $1"
last_match() {
    local f="$1" m="$2" out="" last=""
    out=$(egrep "$m" "$f") || return 1;
    _RET=$(echo "$out" | tail -n 1)
}

get_new() {
   local log="$1" mode="$2"
   last_match "$log" "\[$mode\].*new=(True|False)" || return
   case "$_RET" in
      *new=True) _RET="true";;
      *new=False) _RET="false";;
      *) error "bad $_RET"; return 1;;
   esac
}

get_reform() {
   last_match "$1" "reformattable=(True|False)" || return
   case "$_RET" in
      *=True*) _RET="true";;
      *=False*) _RET="false";;
      *) error "bad $_RET"; return 1;;
   esac
}

config_ran() {
   last_match "$1" "finish:.*config-$2 " || return
   case "$_RET" in
      *previously\ ran*) _RET="false";;
      *ran\ success*) _RET="true";;
      *) error "bad $_RET"; return 1;;
   esac
}

dir="$1"
log="$dir/logs/cloud-init.log"
# new instance: 
get_new "$log" local || _RET="not found"
echo "new instance local: $_RET"
get_new "$log" net || _RET="not found"
echo "new instance net  : $_RET"

# reformattable:
get_reform "$log" || _RET="not found"
echo "reformattable: $_RET"

# disk_setup ran:
config_ran "$log" disk_setup || _RET="not found"
echo "disk_setup ran: $_RET"

# mounts ran:
config_ran "$log" mounts || _RET="not found"
echo "mounts ran: $_RET"

# proc-mounts
ent=$(awk '$2 == "/mnt" { print $1, $2, $3 }' "$dir/proc-mounts")
echo "proc-mounts:" "$ent"

# /etc/fstab entry
ent=$(awk '$2 == "/mnt" { print $1, $2, $4 }' "$dir/etc-fstab")
echo "/etc/fstab: $ent"