Upgrade Ubuntu 24.04 to 26.04

upgrade-2604.sh

#!/bin/bash
# Upgrade one linux-x86_64 worker 24.04 -> 26.04. Runs on the MANAGEMENT host.
# Local: ocluster-admin pause/unpause. Remote (ssh): the upgrade steps.
# Usage: upgrade-2604.sh <fqdn> <pool>
set -uo pipefail
H="$1"; POOL="${2:-linux-x86_64}"
CAP="$HOME/admin.cap"
SSH="ssh -F /home/mtelvers/bot/ssh/config -i /home/mtelvers/bot/ssh/id_ed25519 -o ConnectTimeout=10 -o BatchMode=yes -o ServerAliveInterval=30"
say(){ echo "[$(date -u +%H:%M:%S)] $H: $*"; }
fail(){ say "FAILED: $*"; exit 1; }

wait_up(){ # wait for host ssh + optional kernel prefix
  local want="${1:-}"; local n=0
  while [ $n -lt 180 ]; do
    k=$($SSH "$H" 'uname -r' 2>/dev/null)
    if [ -n "$k" ]; then
      if [ -z "$want" ] || [[ "$k" == ${want}* ]]; then echo "$k"; return 0; fi
    fi
    sleep 10; n=$((n+1))
  done
  return 1
}

do_reboot(){ # issue reboot, wait until host goes DOWN (avoid pre-reboot race), then wait UP [kernel-prefix]
  local want="${1:-}"; local d=0
  $SSH "$H" 'nohup reboot >/dev/null 2>&1 &' >/dev/null 2>&1
  while [ $d -lt 24 ]; do
    sleep 10
    $SSH -o ConnectTimeout=5 "$H" 'true' 2>/dev/null || break  # ssh failed => host going down
    d=$((d+1))
  done
  wait_up "$want"
}

say "=== pause + drain ==="
ocluster-admin -c "$CAP" pause --wait "$POOL" "$H" >/dev/null 2>&1 || fail "pause"

say "=== fstab docker device->UUID (if separate device-name mount) ==="
$SSH "$H" 'set -e
  src=$(findmnt -n -o SOURCE /var/lib/docker 2>/dev/null || true)
  if [ -n "$src" ] && [ "${src#/dev/}" != "$src" ]; then
    uuid=$(blkid -s UUID -o value "$src"); fst=$(findmnt -n -o FSTYPE /var/lib/docker)
    cp /etc/fstab /etc/fstab.bak
    awk -v u="$uuid" -v f="$fst" '\''{ if ($2=="/var/lib/docker" && $1 ~ /^\/dev\//) print "UUID="u" /var/lib/docker "f" defaults 0 0"; else print }'\'' /etc/fstab.bak > /etc/fstab
    systemctl daemon-reload; mount -a && echo "fstab->UUID ($uuid)"
  else echo "no separate device-name docker mount; skip"; fi' || fail "fstab"

say "=== full-upgrade on 24.04 ==="
$SSH "$H" 'apt-get update -qq >/dev/null 2>&1; DEBIAN_FRONTEND=noninteractive apt-get -y full-upgrade >/dev/null 2>&1' || fail "full-upgrade"
say "reboot (24.04 latest)"; k=$(do_reboot) || fail "no return after pre-reboot"; say "back on $k"

say "=== do-release-upgrade -d ==="
$SSH "$H" 'tmux kill-server 2>/dev/null; rm -f /var/log/rel2604.log; tmux new-session -d -s relup "export DEBIAN_FRONTEND=noninteractive; do-release-upgrade -d -f DistUpgradeViewNonInteractive > /var/log/rel2604.log 2>&1; echo === EXIT \$? === >> /var/log/rel2604.log"' || fail "launch dru"
n=0; rc=""
while [ $n -lt 130 ]; do
  rc=$($SSH "$H" "grep -m1 '=== EXIT' /var/log/rel2604.log 2>/dev/null" 2>/dev/null)
  [ -n "$rc" ] && break; sleep 30; n=$((n+1))
done
[ -n "$rc" ] || fail "do-release-upgrade timed out"
echo "$rc" | grep -q "EXIT 0" || { tail=$($SSH "$H" 'tail -5 /var/log/dist-upgrade/main.log' 2>/dev/null); fail "do-release-upgrade $rc | $tail"; }
osnow=$($SSH "$H" 'lsb_release -ds' 2>/dev/null); say "upgrade EXIT 0, now: $osnow"
echo "$osnow" | grep -q "26.04" || fail "not 26.04 after upgrade ($osnow)"

say "=== post fixups: apparmor=0, remove CVE, grub ==="
$SSH "$H" 'set -e
  rm -f /etc/modprobe.d/cve-2026-31431.conf
  if ! grep -q "apparmor=0" /etc/default/grub; then
    cp /etc/default/grub /etc/default/grub.bak
    if grep -q "^GRUB_CMDLINE_LINUX=" /etc/default/grub; then
      sed -i -E "s/^GRUB_CMDLINE_LINUX=\"(.*)\"/GRUB_CMDLINE_LINUX=\"\1 apparmor=0\"/" /etc/default/grub
    else echo "GRUB_CMDLINE_LINUX=\"apparmor=0\"" >> /etc/default/grub; fi
    echo "apparmor=0 RE-ADDED"
  else echo "apparmor=0 already present in grub"; fi
  update-grub >/dev/null 2>&1' || fail "fixups"
say "reboot into 7.0"; k=$(do_reboot 7.0) || fail "no return on 7.0 kernel"
say "back on kernel $k"

say "=== verify ==="
V=$($SSH "$H" 'aa=$(grep -oE "apparmor=[0-9]" /proc/cmdline)
  for i in $(seq 1 8); do d=$(systemctl is-active docker); [ "$d" = active ] && break; sleep 3; done
  dr=$(docker run --rm busybox echo ok 2>&1 | tail -1)
  echo "cmdline=$aa docker=$d containerd=$(systemctl is-active containerd) worker=$(systemctl is-active ocluster-worker) run=$dr"' 2>&1)
say "$V"
echo "$V" | grep -q "cmdline=apparmor=0" || fail "apparmor=0 NOT live on cmdline: $V"
echo "$V" | grep -q "docker=active" || fail "docker not active: $V"
echo "$V" | grep -q "run=ok" || fail "docker run failed: $V"
echo "$V" | grep -q "worker=active" || fail "ocluster-worker not active: $V"

say "=== unpause ==="
ocluster-admin -c "$CAP" unpause "$POOL" "$H" >/dev/null 2>&1 || fail "unpause"
say "SUCCESS: on 26.04 / $k, apparmor=0 live, docker+worker active, unpaused"