Enphase IQ Gateway SD Card sync and tmpfs bind mount

setup_sd_longevity_v2.sh

#!/bin/bash
# setup_sd_longevity_v2.sh — Enphase Envoy eMMC Longevity Setup (tmpfs + async sync)
# Run as root on the Envoy after inserting the SD card (via USB adapter)
#
# Usage: bash setup_sd_longevity_v2.sh            # Install / re-install
#        bash setup_sd_longevity_v2.sh --uninstall # Remove everything
#
# Architecture:
#   - All high-write paths are tmpfs bind mounts (in RAM)
#   - Services write to tmpfs — fast, always available, never fails
#   - A sync daemon periodically copies tmpfs → USB/SD (if present)
#   - On boot, data is seeded: eMMC first, then USB overlay (if present)
#   - Critical DBs backed to eMMC daily/weekly via cron
#   - USB can vanish/reconnect without affecting services
#
# IMPORTANT: Review the detected device before confirming format!

set -e

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# ─── Configuration ──────────────────────────────────────────────
# tmpfs size limits (actual RAM usage = actual data stored, not the limit)
# Based on measured data: ~10MB total, these limits give ~10x headroom
TMPFS_DB_SIZE="64M"      # All database dirs combined
TMPFS_LOG_SIZE="32M"     # All log dirs combined
TMPFS_CORE_SIZE="16M"    # Core dumps
TMPFS_TMP_SIZE="64M"     # /tmp

# Sync interval in seconds
SYNC_INTERVAL=300

# eMMC backup location (for fallback seeding)
EMMC_BACKUP="/var/opt/emu/.emmc_backup"

# ─── Uninstall mode ─────────────────────────────────────────────
if [ "$1" = "--uninstall" ]; then
  if [ "$(id -u)" -ne 0 ]; then
    echo -e "${RED}ERROR: Must run as root${NC}"
    exit 1
  fi

  echo -e "${YELLOW}=== Uninstalling SD Longevity Setup (v2 — tmpfs) ===${NC}"
  echo ""

  # Stop sync daemon first
  if [ -f /tmp/sd_sync.pid ]; then
    echo "Stopping sync daemon..."
    kill "$(cat /tmp/sd_sync.pid)" 2>/dev/null || true
    rm -f /tmp/sd_sync.pid
  fi
  for pid in $(pgrep -f sd_sync_daemon.sh 2>/dev/null); do
    kill "$pid" 2>/dev/null || true
  done

  # Stop services for clean DB copy
  echo "Stopping services for clean data copy-back..."
  /etc/init.d/envoy_monit stop 2>/dev/null || true
  sleep 2

  # ─── Write back tmpfs data to eMMC ────────────────────────────
  echo -e "${YELLOW}Writing tmpfs data back to eMMC...${NC}"

  # Databases
  for dir in binned event pdm archive lifetime ens_device_commission; do
    SRC="/var/opt/emu/$dir"
    if [ -d "$SRC" ] && mountpoint -q "$SRC" 2>/dev/null; then
      echo "  Writing back $dir/..."
      # Create a temp copy, unmount tmpfs, then move data to eMMC
      TMP="/var/opt/emu/.uninstall_tmp_${dir}"
      mkdir -p "$TMP"
      cp -a "$SRC/." "$TMP/" 2>/dev/null || true
      umount -l "$SRC" 2>/dev/null || true
      mkdir -p "$SRC"
      cp -a "$TMP/." "$SRC/" 2>/dev/null && echo "    ✓ $dir/" || echo "    ✗ $dir/ (failed)"
      rm -rf "$TMP"
    fi
  done

  # Root-level DBs
  for db in emu.db emu.db-wal emu.db-shm agf_perf.db agf_perf.db-wal agf_perf.db-shm; do
    DB_PATH="/var/opt/emu/$db"
    if [ -f "$DB_PATH" ]; then
      TMP="/var/opt/emu/.uninstall_tmp_${db}"
      cp -a "$DB_PATH" "$TMP" 2>/dev/null || true
      umount -l "$DB_PATH" 2>/dev/null || true
      if [ -f "$TMP" ]; then
        cp -a "$TMP" "$DB_PATH" 2>/dev/null && echo "    ✓ $db" || echo "    ✗ $db (failed)"
        rm -f "$TMP"
      fi
    fi
  done

  # Logs
  for dir in emu nginx agf chrony tinyproxy; do
    SRC="/var/log/$dir"
    if [ -d "$SRC" ] && mountpoint -q "$SRC" 2>/dev/null; then
      TMP="/var/log/.uninstall_tmp_${dir}"
      mkdir -p "$TMP"
      cp -a "$SRC/." "$TMP/" 2>/dev/null || true
      umount -l "$SRC" 2>/dev/null || true
      mkdir -p "$SRC"
      cp -a "$TMP/." "$SRC/" 2>/dev/null && echo "    ✓ log/$dir/" || echo "    ✗ log/$dir/ (failed)"
      rm -rf "$TMP"
    fi
  done

  for logfile in messages dmesg; do
    TARGET="/var/log/$logfile"
    if [ -f "$TARGET" ] && [ ! -L "$TARGET" ]; then
      TMP="/var/log/.uninstall_tmp_${logfile}"
      cp -a "$TARGET" "$TMP" 2>/dev/null || true
      umount -l "$TARGET" 2>/dev/null || true
      if [ -f "$TMP" ]; then
        cp -a "$TMP" "$TARGET" 2>/dev/null && echo "    ✓ log/$logfile" || echo "    ✗ log/$logfile (failed)"
        rm -f "$TMP"
      fi
    fi
  done

  # Web logs
  if mountpoint -q /var/www/logs 2>/dev/null; then
    TMP="/var/www/.uninstall_tmp_logs"
    mkdir -p "$TMP"
    cp -a /var/www/logs/. "$TMP/" 2>/dev/null || true
    umount -l /var/www/logs 2>/dev/null || true
    mkdir -p /var/www/logs
    cp -a "$TMP/." /var/www/logs/ 2>/dev/null && echo "    ✓ www/logs/" || echo "    ✗ www/logs/ (failed)"
    rm -rf "$TMP"
  fi

  # Corefiles
  if mountpoint -q /corefiles 2>/dev/null; then
    umount -l /corefiles 2>/dev/null || true
  fi

  # Misc high-write paths (cfg, mal, agf meter logs, wifi)
  for pair in "cfg:/var/opt/cfg" "mal:/var/opt/mal" "agf_mlgr:/var/opt/agf/mlgr" "wifi:/etc/wifi"; do
    TDIR="${pair%%:*}"
    TARGET="${pair#*:}"
    if mountpoint -q "$TARGET" 2>/dev/null; then
      TMP="/tmp/.uninstall_tmp_${TDIR}"
      mkdir -p "$TMP"
      cp -a "$TARGET/." "$TMP/" 2>/dev/null || true
      umount -l "$TARGET" 2>/dev/null || true
      mkdir -p "$TARGET"
      cp -a "$TMP/." "$TARGET/" 2>/dev/null && echo "    ✓ $TDIR/" || echo "    ✗ $TDIR/ (failed)"
      rm -rf "$TMP"
    fi
  done

  # cache (regeneratable API XML) — raw.db is handled by Enphase's tmpfs_link
  if mountpoint -q /var/opt/emu/cache 2>/dev/null; then
    TMP="/tmp/.uninstall_tmp_cache"
    mkdir -p "$TMP"
    cp -a /var/opt/emu/cache/. "$TMP/" 2>/dev/null || true
    umount -l /var/opt/emu/cache 2>/dev/null || true
    mkdir -p /var/opt/emu/cache
    cp -a "$TMP/." /var/opt/emu/cache/ 2>/dev/null && echo "    ✓ cache/" || echo "    ✗ cache/ (failed)"
    rm -rf "$TMP"
  fi

  # Unmount /var/tmp tmpfs
  umount -l /var/tmp 2>/dev/null || true

  # Unmount utmp bind mount
  umount -l /var/run/utmp 2>/dev/null || true

  echo "Data write-back complete."
  echo ""

  # Also do a final sync to USB if it's available
  if mountpoint -q /mnt/sd 2>/dev/null; then
    echo "Final sync to USB drive..."
    /usr/local/bin/sd_sync_daemon.sh --once 2>/dev/null || true
    umount -l /mnt/sd 2>/dev/null || true
  fi

  # Restart services so they use eMMC copies
  /etc/init.d/envoy_monit start 2>/dev/null || true

  # Remove init script and boot/shutdown links
  echo "Removing init scripts and boot links..."
  rm -f /etc/rc3.d/S80sd_bind_mounts
  rm -f /etc/rc0.d/K97sd_bind_mounts
  rm -f /etc/rc6.d/K97sd_bind_mounts
  rm -f /etc/rcS.d/S03sd_bind_mounts /etc/rcS.d/S10sd_bind_mounts
  rm -f /etc/rcS.d/S11sd_bind_mounts /etc/rcS.d/S96sd_bind_mounts
  rm -f /etc/init.d/sd_bind_mounts

  # Remove sync daemon, old watchdog, cron jobs, tools
  echo "Removing tools and cron jobs..."
  rm -f /usr/local/bin/sd_sync_daemon.sh
  rm -f /usr/local/bin/sd_watchdog.sh
  rm -f /tmp/sd_sync.pid
  rm -f /tmp/sd_watchdog.pid
  rm -f /etc/cron.weekly/emmc_health
  rm -f /etc/cron.weekly/sd_backup_critical
  rm -f /etc/cron.daily/sd_emmc_backup
  rm -f /etc/cron.daily/write_stats
  rm -f /usr/local/bin/check_writes

  # Clean up eMMC backup dir
  rm -rf "$EMMC_BACKUP"

  # Revert sysctl.conf
  if grep -q "# eMMC longevity" /etc/sysctl.conf 2>/dev/null; then
    echo "Reverting sysctl.conf..."
    sed -i '/# eMMC longevity/d' /etc/sysctl.conf
    sed -i '/^vm.dirty_writeback_centisecs = 1500$/d' /etc/sysctl.conf
    sed -i '/^vm.dirty_expire_centisecs = 3000$/d' /etc/sysctl.conf
    sed -i 's/^#\(vm.dirty_writeback_centisecs.*\)  # disabled for eMMC longevity/\1/' /etc/sysctl.conf
    sed -i 's/^#\(vm.dirty_expire_centisecs.*\)  # disabled for eMMC longevity/\1/' /etc/sysctl.conf
  fi

  # Revert fstab
  echo "Reverting fstab..."
  sed -i '/^tmpfs \/tmp tmpfs.*eMMC\|^tmpfs \/tmp tmpfs defaults,noatime,nosuid,nodev/d' /etc/fstab 2>/dev/null || true
  sed -i 's/^#\(\/dev\/mmcblk0p6.*\)  # disabled for eMMC longevity/\1/' /etc/fstab 2>/dev/null || true
  sed -i 's/,commit=60//g' /etc/fstab 2>/dev/null || true

  # Restore original sysctl values
  sysctl -w vm.dirty_writeback_centisecs=100 > /dev/null 2>&1 || true
  sysctl -w vm.dirty_expire_centisecs=100 > /dev/null 2>&1 || true

  echo ""
  echo -e "${GREEN}=== Uninstall Complete ===${NC}"
  echo ""
  echo "What was removed:"
  echo "  ✓ tmpfs data written back to eMMC"
  echo "  ✓ Final sync to USB (if present)"
  echo "  ✓ All tmpfs bind mounts removed"
  echo "  ✓ Sync daemon stopped and removed"
  echo "  ✓ Init script and boot links removed"
  echo "  ✓ Cron jobs removed"
  echo "  ✓ sysctl.conf and fstab reverted"
  echo ""
  echo -e "${YELLOW}Reboot recommended to fully restore original behavior.${NC}"
  exit 0
fi

# ═══════════════════════════════════════════════════════════════════
# INSTALL MODE
# ═══════════════════════════════════════════════════════════════════

echo -e "${GREEN}=== Envoy eMMC Longevity Setup (v2 — tmpfs + async sync) ===${NC}"
echo ""

# ─── Step 0: Check prerequisites ────────────────────────────────
if [ "$(id -u)" -ne 0 ]; then
  echo -e "${RED}ERROR: Must run as root${NC}"
  exit 1
fi

echo -e "${YELLOW}Current eMMC health:${NC}"
ruby /opt/emu/scripts/mmc_utils.rb 2>/dev/null || echo "  (mmc_utils not available)"
echo ""

echo -e "${YELLOW}Current disk writes (ext4 lifetime):${NC}"
EMMC_TOTAL_MB=0
for part in /dev/mmcblk0p*; do
  if [ -b "$part" ]; then
    LW=$(tune2fs -l "$part" 2>/dev/null | grep "Lifetime writes:" | awk '{print $3, $4}')
    if [ -n "$LW" ]; then
      echo "  $part: $LW"
      VAL=$(echo "$LW" | awk '{print $1}')
      UNIT=$(echo "$LW" | awk '{print $2}')
      case "$UNIT" in
        GB) ADD=$(awk -v v="$VAL" 'BEGIN {printf "%.0f", v * 1024}') ;;
        MB) ADD=$(awk -v v="$VAL" 'BEGIN {printf "%.0f", v}') ;;
        kB) ADD=$(awk -v v="$VAL" 'BEGIN {printf "%.0f", v / 1024}') ;;
        *)  ADD=0 ;;
      esac
      EMMC_TOTAL_MB=$((EMMC_TOTAL_MB + ADD))
    fi
  fi
done
if [ "$EMMC_TOTAL_MB" -ge 1024 ]; then
  echo "  Total eMMC lifetime writes: $(awk -v v="$EMMC_TOTAL_MB" 'BEGIN {printf "%.1f", v / 1024}') GB"
else
  echo "  Total eMMC lifetime writes: ${EMMC_TOTAL_MB} MB"
fi
echo ""

echo -e "${YELLOW}Current RAM:${NC}"
free
echo ""

# ─── Step 1: Detect SD card ─────────────────────────────────────
echo -e "${YELLOW}Detecting SD card / USB storage...${NC}"

DEVS=()
for dev in /dev/sd[a-z] /dev/mmcblk[1-9]; do
  if [ -b "$dev" ]; then
    SIZE=0
    devname=$(basename "$dev")
    if [ -f "/sys/block/$devname/size" ]; then
      SECTORS=$(cat "/sys/block/$devname/size" 2>/dev/null || echo 0)
      SIZE=$((SECTORS * 512))
    fi
    if [ "$SIZE" -eq 0 ]; then
      SIZE=$(blockdev --getsize64 "$dev" 2>/dev/null || echo 0)
    fi
    if [ "$SIZE" -eq 0 ]; then
      if dd if="$dev" bs=512 count=1 of=/dev/null 2>/dev/null; then
        SIZE=1
      fi
    fi

    if [ "$SIZE" -gt 0 ] 2>/dev/null; then
      SIZE_GB=$(echo "$SIZE / 1073741824" | bc 2>/dev/null || echo "$((SIZE / 1073741824))")
      DEVS+=("$dev")
      echo "  [${#DEVS[@]}] $dev (${SIZE_GB} GB)"
    else
      echo "  SKIP: $dev (empty slot or unreadable)"
    fi
  fi
done

if [ ${#DEVS[@]} -eq 0 ]; then
  echo -e "${RED}No SD card / USB storage detected!${NC}"
  echo "  Make sure the USB adapter with SD card is plugged in."
  echo "  Check 'dmesg | tail -20' for USB detection messages."
  exit 1
fi

if [ ${#DEVS[@]} -eq 1 ]; then
  SD_DEV="${DEVS[0]}"
  echo ""
  echo -e "${YELLOW}Selected device: ${SD_DEV}${NC}"
else
  echo ""
  echo -e "${YELLOW}Multiple devices found. Which is your microSD card?${NC}"
  read -p "Enter number [1-${#DEVS[@]}]: " CHOICE
  if [ -z "$CHOICE" ] || [ "$CHOICE" -lt 1 ] || [ "$CHOICE" -gt ${#DEVS[@]} ] 2>/dev/null; then
    echo -e "${RED}Invalid selection.${NC}"
    exit 1
  fi
  SD_DEV="${DEVS[$((CHOICE-1))]}"
  echo -e "${YELLOW}Selected device: ${SD_DEV}${NC}"
fi

if echo "$SD_DEV" | grep -q mmcblk; then
  SD_PART="${SD_DEV}p1"
else
  SD_PART="${SD_DEV}1"
fi

# ─── Step 2: Confirm and format ─────────────────────────────────
echo ""

EXISTING_LABEL=$(blkid "$SD_PART" 2>/dev/null | grep -o 'LABEL="[^"]*"' | cut -d'"' -f2 || true)
SKIP_FORMAT=0

if [ "$EXISTING_LABEL" = "envoy-sd" ]; then
  echo -e "${GREEN}SD card already has 'envoy-sd' label — previously set up.${NC}"
  echo ""
  echo "  [1] Skip format, just reinstall scripts (preserves SD data)"
  echo "  [2] Full format (ERASES ALL DATA on SD)"
  echo ""
  read -p "Choose [1/2]: " FORMAT_CHOICE
  if [ "$FORMAT_CHOICE" = "1" ]; then
    SKIP_FORMAT=1
    echo "Skipping format."
  fi
fi

if [ "$SKIP_FORMAT" -eq 0 ]; then
  echo -e "${RED}WARNING: This will ERASE ALL DATA on ${SD_DEV}${NC}"
  echo "Contents:"
  fdisk -l "$SD_DEV" 2>/dev/null | head -10
  echo ""
  read -p "Type YES to format ${SD_DEV}: " CONFIRM
  if [ "$CONFIRM" != "YES" ]; then
    echo "Aborted."
    exit 1
  fi

  echo ""
  echo "Partitioning ${SD_DEV}..."
  (echo o; echo n; echo p; echo 1; echo; echo; echo w) | fdisk "$SD_DEV" 2>/dev/null || true
  sleep 1

  partprobe "$SD_DEV" 2>/dev/null || blockdev --rereadpt "$SD_DEV" 2>/dev/null || true
  sleep 1

  echo "Formatting ${SD_PART} as ext4 (no journal)..."
  mkfs.ext4 -L envoy-sd -m 1 "$SD_PART"
  tune2fs -c 0 -i 0 "$SD_PART"

  echo -e "${GREEN}Format complete.${NC}"
fi

# ─── Step 3: Mount SD card ──────────────────────────────────────
echo ""
echo "Mounting SD card..."
mkdir -p /mnt/sd
if mountpoint -q /mnt/sd 2>/dev/null; then
  echo "SD card already mounted at /mnt/sd, remounting..."
  mount -o remount,rw,noatime,nodiratime /mnt/sd
else
  mount -o rw,noatime,nodiratime "$SD_PART" /mnt/sd
fi

echo "SD card mounted at /mnt/sd"
df -h /mnt/sd

# ─── Step 4: Create directory structure on SD ────────────────────
echo ""
echo "Creating directory structure on SD card..."

mkdir -p /mnt/sd/var/log/emu
mkdir -p /mnt/sd/var/log/nginx
mkdir -p /mnt/sd/var/log/agf
mkdir -p /mnt/sd/var/log/chrony
mkdir -p /mnt/sd/var/log/tinyproxy
mkdir -p /mnt/sd/var/www/logs
mkdir -p /mnt/sd/var/log/misc
mkdir -p /mnt/sd/var/opt/emu/binned
mkdir -p /mnt/sd/var/opt/emu/event
mkdir -p /mnt/sd/var/opt/emu/pdm
mkdir -p /mnt/sd/var/opt/emu/archive
mkdir -p /mnt/sd/var/opt/emu/lifetime
mkdir -p /mnt/sd/var/opt/emu/ens_device_commission
mkdir -p /mnt/sd/var/opt/emu/root_dbs
mkdir -p /mnt/sd/corefiles
mkdir -p /mnt/sd/var/opt/cfg
mkdir -p /mnt/sd/var/opt/mal
mkdir -p /mnt/sd/var/opt/agf/mlgr
mkdir -p /mnt/sd/etc/wifi

echo "  Done."

# ─── Step 5: Copy existing data to SD ───────────────────────────
echo ""
echo "Copying existing data to SD card..."

for db_dir in binned event pdm archive lifetime ens_device_commission; do
  SRC="/var/opt/emu/$db_dir"
  DST="/mnt/sd/var/opt/emu/$db_dir"
  if [ -d "$SRC" ]; then
    echo "  Copying $SRC → $DST"
    cp -a "$SRC/"* "$DST/" 2>/dev/null || true
  fi
done

echo "Copying root-level databases..."
for db in emu.db emu.db-wal emu.db-shm agf_perf.db agf_perf.db-wal agf_perf.db-shm; do
  if [ -f "/var/opt/emu/$db" ]; then
    echo "  Copying /var/opt/emu/$db"
    cp -a "/var/opt/emu/$db" "/mnt/sd/var/opt/emu/root_dbs/$db" 2>/dev/null || true
  fi
done

echo "Copying logs..."
for log_dir in emu nginx agf chrony tinyproxy; do
  SRC="/var/log/$log_dir"
  DST="/mnt/sd/var/log/$log_dir"
  if [ -d "$SRC" ]; then
    echo "  Copying $SRC → $DST"
    cp -a "$SRC/"* "$DST/" 2>/dev/null || true
  fi
done
cp /var/log/messages /mnt/sd/var/log/misc/messages 2>/dev/null || true
cp /var/log/dmesg /mnt/sd/var/log/misc/dmesg 2>/dev/null || true

echo -e "${GREEN}Data copy complete.${NC}"

# ─── Step 5b: Create eMMC backup (for boot fallback seeding) ────
echo ""
echo "Creating eMMC backup for boot fallback..."

mkdir -p "$EMMC_BACKUP/root_dbs"
for dir in binned event pdm archive lifetime ens_device_commission; do
  mkdir -p "$EMMC_BACKUP/$dir"
  if [ -d "/var/opt/emu/$dir" ]; then
    cp -a "/var/opt/emu/$dir/"* "$EMMC_BACKUP/$dir/" 2>/dev/null || true
  fi
done
for db in emu.db emu.db-wal emu.db-shm agf_perf.db agf_perf.db-wal agf_perf.db-shm; do
  if [ -f "/var/opt/emu/$db" ]; then
    cp -a "/var/opt/emu/$db" "$EMMC_BACKUP/root_dbs/$db" 2>/dev/null || true
  fi
done
echo -e "  ${GREEN}eMMC backup created at $EMMC_BACKUP${NC}"

# eMMC backup for misc high-write paths
mkdir -p "$EMMC_BACKUP/cfg" "$EMMC_BACKUP/mal" "$EMMC_BACKUP/agf_mlgr" "$EMMC_BACKUP/wifi"
cp -a /var/opt/cfg/* "$EMMC_BACKUP/cfg/" 2>/dev/null || true
cp -a /var/opt/mal/* "$EMMC_BACKUP/mal/" 2>/dev/null || true
cp -a /var/opt/agf/mlgr/* "$EMMC_BACKUP/agf_mlgr/" 2>/dev/null || true
cp -a /etc/wifi/* "$EMMC_BACKUP/wifi/" 2>/dev/null || true

# ─── Step 6: Create init script (tmpfs + seed) ──────────────────
echo ""
echo "Installing tmpfs bind mount init script..."

cat > /etc/init.d/sd_bind_mounts << 'INITSCRIPT'
#!/bin/sh
### BEGIN INIT INFO
# Provides:          sd_bind_mounts
# Required-Start:    $local_fs
# Required-Stop:     $local_fs
# Default-Start:     3
# Default-Stop:      0 6
# Short-Description: tmpfs bind mounts + async USB sync for eMMC longevity
### END INIT INFO

SD_MOUNT="/mnt/sd"
EMMC_BACKUP="/var/opt/emu/.emmc_backup"
SYNC_PID="/tmp/sd_sync.pid"

# ─── Logging: echo + dmesg via /dev/kmsg ────────────────────────
log_msg() {
  echo "$1"
  echo "<6>sd_bind_mounts: $1" > /dev/kmsg 2>/dev/null || true
}

# ─── Helper: find SD/USB device by label ────────────────────────
find_sd_partition() {
  rm -f /etc/blkid.tab /run/blkid/blkid.tab 2>/dev/null
  for dev in /dev/sd[a-z]1 /dev/mmcblk[1-9]p1; do
    if [ -b "$dev" ]; then
      LABEL=$(blkid "$dev" 2>/dev/null | grep -o 'LABEL="[^"]*"' | cut -d'"' -f2)
      if [ "$LABEL" = "envoy-sd" ]; then
        echo "$dev"
        return 0
      fi
    fi
  done
  return 1
}

# ─── Helper: mount tmpfs and bind over target ───────────────────
# Usage: tmpfs_bind <backing_dir> <target_path> <size>
# Creates a tmpfs at a staging point, copies seed data in,
# then bind mounts it over the target.
tmpfs_bind_dir() {
  BACKING="$1"   # staging/backing dir under /var/run/sd_tmpfs/
  TARGET="$2"    # real path services write to
  SIZE="$3"      # tmpfs size limit

  mkdir -p "$BACKING"
  # Mount tmpfs at the backing location (if not already)
  if ! mountpoint -q "$BACKING" 2>/dev/null; then
    mount -t tmpfs -o size="$SIZE",noatime,nosuid,nodev tmpfs "$BACKING"
  fi

  # Peel any existing bind mount on target
  while mountpoint -q "$TARGET" 2>/dev/null; do
    umount -l "$TARGET" 2>/dev/null || break
  done

  mkdir -p "$TARGET"
  mount --bind "$BACKING" "$TARGET"
}

# Same but for individual files (emu.db, etc.)
tmpfs_bind_file() {
  BACKING="$1"   # backing file under /var/run/sd_tmpfs/
  TARGET="$2"    # real file path

  # Peel any existing bind mount
  while mountpoint -q "$TARGET" 2>/dev/null; do
    umount -l "$TARGET" 2>/dev/null || break
  done

  touch "$BACKING"
  mount --bind "$BACKING" "$TARGET"
}

case "$1" in
  start)
    log_msg "=== tmpfs bind mounts: starting ==="

    # ── Phase 1: Create tmpfs backing store ──────────────────────
    # One tmpfs for all DB data, one for all logs
    TMPFS_DB="/var/run/sd_tmpfs/db"
    TMPFS_LOG="/var/run/sd_tmpfs/log"
    TMPFS_CORE="/var/run/sd_tmpfs/core"

    mkdir -p "$TMPFS_DB" "$TMPFS_LOG" "$TMPFS_CORE"
    if ! mountpoint -q "$TMPFS_DB" 2>/dev/null; then
      mount -t tmpfs -o size=64M,noatime,nosuid tmpfs "$TMPFS_DB"
    fi
    if ! mountpoint -q "$TMPFS_LOG" 2>/dev/null; then
      mount -t tmpfs -o size=32M,noatime,nosuid tmpfs "$TMPFS_LOG"
    fi
    if ! mountpoint -q "$TMPFS_CORE" 2>/dev/null; then
      mount -t tmpfs -o size=16M,noatime,nosuid tmpfs "$TMPFS_CORE"
    fi

    # Create subdirectory structure in tmpfs
    mkdir -p "$TMPFS_DB/binned" "$TMPFS_DB/event" "$TMPFS_DB/pdm"    \
             "$TMPFS_DB/archive" "$TMPFS_DB/lifetime"                 \
             "$TMPFS_DB/ens_device_commission" "$TMPFS_DB/root_dbs"
    mkdir -p "$TMPFS_LOG/emu" "$TMPFS_LOG/nginx" "$TMPFS_LOG/agf"    \
             "$TMPFS_LOG/chrony" "$TMPFS_LOG/tinyproxy"               \
             "$TMPFS_LOG/www_logs" "$TMPFS_LOG/misc" "$TMPFS_LOG/stats"
    mkdir -p "$TMPFS_CORE/corefiles"

    # Additional tmpfs for high-write paths discovered via inotifywait
    TMPFS_MISC="/var/run/sd_tmpfs/misc"
    mkdir -p "$TMPFS_MISC"
    if ! mountpoint -q "$TMPFS_MISC" 2>/dev/null; then
      mount -t tmpfs -o size=32M,noatime,nosuid tmpfs "$TMPFS_MISC"
    fi
    mkdir -p "$TMPFS_MISC/cfg" "$TMPFS_MISC/mal" "$TMPFS_MISC/agf_mlgr" "$TMPFS_MISC/wifi"
    mkdir -p "$TMPFS_MISC/cache"  # API XML cache — volatile, no sync needed

    # Mount /var/tmp as tmpfs (SQLite etilqs temp files)
    if ! mountpoint -q /var/tmp 2>/dev/null; then
      mount -t tmpfs -o size=32M,noatime,nosuid,nodev tmpfs /var/tmp
      log_msg "  /var/tmp mounted as tmpfs"
    fi

    log_msg "tmpfs backing stores mounted"

    # ── Phase 2: Seed from eMMC (guaranteed available) ───────────
    log_msg "Seeding tmpfs from eMMC backup..."

    if [ -d "$EMMC_BACKUP" ]; then
      for dir in binned event pdm archive lifetime ens_device_commission; do
        if [ -d "$EMMC_BACKUP/$dir" ]; then
          cp -a "$EMMC_BACKUP/$dir/"* "$TMPFS_DB/$dir/" 2>/dev/null || true
        fi
      done
      for db in emu.db emu.db-wal emu.db-shm agf_perf.db agf_perf.db-wal agf_perf.db-shm; do
        if [ -f "$EMMC_BACKUP/root_dbs/$db" ]; then
          cp -a "$EMMC_BACKUP/root_dbs/$db" "$TMPFS_DB/root_dbs/$db" 2>/dev/null || true
        fi
      done
      log_msg "  eMMC seed complete"

      # Seed misc paths from eMMC backup
      TMPFS_MISC="/var/run/sd_tmpfs/misc"
      for pair in "cfg:cfg" "mal:mal" "agf_mlgr:agf_mlgr" "wifi:wifi"; do
        TDIR="${pair%%:*}"
        BDIR="${pair#*:}"
        if [ -d "$EMMC_BACKUP/$BDIR" ]; then
          cp -a "$EMMC_BACKUP/$BDIR/"* "$TMPFS_MISC/$TDIR/" 2>/dev/null || true
        fi
      done
    else
      # No backup dir — seed from live eMMC paths
      log_msg "  No eMMC backup dir, seeding from live paths..."
      for dir in binned event pdm archive lifetime ens_device_commission; do
        if [ -d "/var/opt/emu/$dir" ]; then
          cp -a "/var/opt/emu/$dir/"* "$TMPFS_DB/$dir/" 2>/dev/null || true
        fi
      done
      for db in emu.db emu.db-wal emu.db-shm agf_perf.db agf_perf.db-wal agf_perf.db-shm; do
        if [ -f "/var/opt/emu/$db" ]; then
          cp -a "/var/opt/emu/$db" "$TMPFS_DB/root_dbs/$db" 2>/dev/null || true
        fi
      done
    fi

    # Seed logs from eMMC (whatever is there currently)
    for dir in emu nginx agf chrony tinyproxy; do
      if [ -d "/var/log/$dir" ]; then
        cp -a "/var/log/$dir/"* "$TMPFS_LOG/$dir/" 2>/dev/null || true
      fi
    done
    for logfile in messages dmesg; do
      if [ -f "/var/log/$logfile" ] && [ ! -L "/var/log/$logfile" ]; then
        cp -a "/var/log/$logfile" "$TMPFS_LOG/misc/$logfile" 2>/dev/null || true
      fi
    done
    if [ -d /var/www/logs ]; then
      cp -a /var/www/logs/* "$TMPFS_LOG/www_logs/" 2>/dev/null || true
    fi

    # Seed misc high-write paths from eMMC
    TMPFS_MISC="/var/run/sd_tmpfs/misc"
    if [ -d /var/opt/cfg ]; then
      cp -a /var/opt/cfg/* "$TMPFS_MISC/cfg/" 2>/dev/null || true
    fi
    if [ -d /var/opt/mal ]; then
      cp -a /var/opt/mal/* "$TMPFS_MISC/mal/" 2>/dev/null || true
    fi
    if [ -d /var/opt/agf/mlgr ]; then
      cp -a /var/opt/agf/mlgr/* "$TMPFS_MISC/agf_mlgr/" 2>/dev/null || true
    fi
    if [ -d /etc/wifi ]; then
      cp -a /etc/wifi/* "$TMPFS_MISC/wifi/" 2>/dev/null || true
    fi
    # Note: /var/opt/emu/raw is handled by Enphase's tmpfs_link — don't override
    if [ -d /var/opt/emu/cache ]; then
      cp -a /var/opt/emu/cache/* "$TMPFS_MISC/cache/" 2>/dev/null || true
    fi

    # ── Phase 3: Try USB overlay (newer data) ────────────────────
    # Attempt to mount USB and overwrite tmpfs with fresher data.
    # This is best-effort — if USB isn't there, eMMC seed is fine.
    if ! mountpoint -q "$SD_MOUNT" 2>/dev/null; then
      SD_PART=$(find_sd_partition)
      if [ -n "$SD_PART" ]; then
        log_msg "Found USB drive at $SD_PART — overlaying with newer data..."
        mkdir -p "$SD_MOUNT"
        mount -o ro,noatime,nodiratime "$SD_PART" "$SD_MOUNT" 2>/dev/null
        if [ $? -eq 0 ]; then
          # Overlay DB data from USB (newer than eMMC backup)
          for dir in binned event pdm archive lifetime ens_device_commission; do
            if [ -d "$SD_MOUNT/var/opt/emu/$dir" ]; then
              cp -a "$SD_MOUNT/var/opt/emu/$dir/"* "$TMPFS_DB/$dir/" 2>/dev/null || true
            fi
          done
          for db in emu.db emu.db-wal emu.db-shm agf_perf.db agf_perf.db-wal agf_perf.db-shm; do
            if [ -f "$SD_MOUNT/var/opt/emu/root_dbs/$db" ]; then
              cp -a "$SD_MOUNT/var/opt/emu/root_dbs/$db" "$TMPFS_DB/root_dbs/$db" 2>/dev/null || true
            fi
          done
          # Overlay logs from USB
          for dir in emu nginx agf chrony tinyproxy; do
            if [ -d "$SD_MOUNT/var/log/$dir" ]; then
              cp -a "$SD_MOUNT/var/log/$dir/"* "$TMPFS_LOG/$dir/" 2>/dev/null || true
            fi
          done
          for logfile in messages dmesg; do
            if [ -f "$SD_MOUNT/var/log/misc/$logfile" ]; then
              cp -a "$SD_MOUNT/var/log/misc/$logfile" "$TMPFS_LOG/misc/$logfile" 2>/dev/null || true
            fi
          done
          if [ -d "$SD_MOUNT/var/www/logs" ]; then
            cp -a "$SD_MOUNT/var/www/logs/"* "$TMPFS_LOG/www_logs/" 2>/dev/null || true
          fi
          # Overlay misc paths from USB
          for mdir in cfg mal; do
            if [ -d "$SD_MOUNT/var/opt/$mdir" ]; then
              cp -a "$SD_MOUNT/var/opt/$mdir/"* "$TMPFS_MISC/$mdir/" 2>/dev/null || true
            fi
          done
          if [ -d "$SD_MOUNT/var/opt/agf/mlgr" ]; then
            cp -a "$SD_MOUNT/var/opt/agf/mlgr/"* "$TMPFS_MISC/agf_mlgr/" 2>/dev/null || true
          fi
          if [ -d "$SD_MOUNT/etc/wifi" ]; then
            cp -a "$SD_MOUNT/etc/wifi/"* "$TMPFS_MISC/wifi/" 2>/dev/null || true
          fi
          log_msg "  USB overlay complete"
          # Remount read-write for sync daemon later
          umount "$SD_MOUNT" 2>/dev/null
          mount -o rw,noatime,nodiratime "$SD_PART" "$SD_MOUNT" 2>/dev/null || true
        else
          log_msg "  USB mount failed — using eMMC seed only"
        fi
      else
        log_msg "No USB drive found — using eMMC seed only (this is fine)"
      fi
    else
      log_msg "USB already mounted at $SD_MOUNT — overlaying..."
      for dir in binned event pdm archive lifetime ens_device_commission; do
        if [ -d "$SD_MOUNT/var/opt/emu/$dir" ]; then
          cp -a "$SD_MOUNT/var/opt/emu/$dir/"* "$TMPFS_DB/$dir/" 2>/dev/null || true
        fi
      done
      for db in emu.db emu.db-wal emu.db-shm agf_perf.db agf_perf.db-wal agf_perf.db-shm; do
        if [ -f "$SD_MOUNT/var/opt/emu/root_dbs/$db" ]; then
          cp -a "$SD_MOUNT/var/opt/emu/root_dbs/$db" "$TMPFS_DB/root_dbs/$db" 2>/dev/null || true
        fi
      done
      log_msg "  USB overlay complete"
    fi

    # ── Phase 4: Bind mount tmpfs over service paths ─────────────
    log_msg "Activating bind mounts..."

    # Database directories
    for dir in binned event pdm archive lifetime ens_device_commission; do
      TARGET="/var/opt/emu/$dir"
      while mountpoint -q "$TARGET" 2>/dev/null; do
        umount -l "$TARGET" 2>/dev/null || break
      done
      mkdir -p "$TARGET"
      mount --bind "$TMPFS_DB/$dir" "$TARGET"
    done

    # Root-level DB files
    for db in emu.db emu.db-wal emu.db-shm agf_perf.db agf_perf.db-wal agf_perf.db-shm; do
      EMU_DB="/var/opt/emu/$db"
      TMPFS_FILE="$TMPFS_DB/root_dbs/$db"
      if [ -f "$EMU_DB" ] || [ -f "$TMPFS_FILE" ]; then
        # Ensure both source and target exist
        touch "$TMPFS_FILE" "$EMU_DB" 2>/dev/null
        while mountpoint -q "$EMU_DB" 2>/dev/null; do
          umount -l "$EMU_DB" 2>/dev/null || break
        done
        mount --bind "$TMPFS_FILE" "$EMU_DB"
      fi
    done

    # Log directories
    for dir in emu nginx agf chrony tinyproxy; do
      TARGET="/var/log/$dir"
      while mountpoint -q "$TARGET" 2>/dev/null; do
        umount -l "$TARGET" 2>/dev/null || break
      done
      mkdir -p "$TARGET"
      mount --bind "$TMPFS_LOG/$dir" "$TARGET" 2>/dev/null || true
      # nginx runs as nobody, needs write access to its log dir
      [ "$dir" = "nginx" ] && chmod 1777 "$TARGET"
    done

    # Web logs (mongrel2 runs as nobody, needs write access)
    while mountpoint -q /var/www/logs 2>/dev/null; do
      umount -l /var/www/logs 2>/dev/null || break
    done
    mkdir -p /var/www/logs
    mount --bind "$TMPFS_LOG/www_logs" /var/www/logs 2>/dev/null || true
    chmod 1777 /var/www/logs

    # System logs (only real files, not symlinks to /dev/null)
    for logfile in messages dmesg; do
      TARGET="/var/log/$logfile"
      if [ -f "$TARGET" ] && [ ! -L "$TARGET" ]; then
        while mountpoint -q "$TARGET" 2>/dev/null; do
          umount -l "$TARGET" 2>/dev/null || break
        done
        mount --bind "$TMPFS_LOG/misc/$logfile" "$TARGET"
      else
        log_msg "SKIP: /var/log/$logfile (symlink or missing)"
      fi
    done

    # Core dumps
    while mountpoint -q /corefiles 2>/dev/null; do
      umount -l /corefiles 2>/dev/null || break
    done
    mkdir -p /corefiles
    mount --bind "$TMPFS_CORE/corefiles" /corefiles 2>/dev/null || true

    # Misc high-write paths (cfg, mal, agf meter logs, wifi config)
    TMPFS_MISC="/var/run/sd_tmpfs/misc"
    for pair in "cfg:/var/opt/cfg" "mal:/var/opt/mal" "agf_mlgr:/var/opt/agf/mlgr" "wifi:/etc/wifi" "cache:/var/opt/emu/cache"; do
      TDIR="${pair%%:*}"
      TARGET="${pair#*:}"
      while mountpoint -q "$TARGET" 2>/dev/null; do
        umount -l "$TARGET" 2>/dev/null || break
      done
      mkdir -p "$TARGET"
      mount --bind "$TMPFS_MISC/$TDIR" "$TARGET"
    done
    # WiFi dir needs to be writable by services
    chmod 755 /etc/wifi 2>/dev/null || true

    # utmp — login tracking, written frequently, causes eMMC journal commits
    # /var/run is on eMMC on this Yocto system, so bind-mount utmp from /tmp (tmpfs)
    if [ -f /var/run/utmp ]; then
      cp -a /var/run/utmp /tmp/utmp 2>/dev/null || true
      mount --bind /tmp/utmp /var/run/utmp 2>/dev/null || true
      log_msg "utmp bind-mounted from /tmp (tmpfs)"
    fi

    log_msg "All tmpfs bind mounts active"

    # ── Phase 5: Start sync daemon ──────────────────────────────
    # Kill any existing sync daemon
    if [ -f "$SYNC_PID" ]; then
      kill "$(cat "$SYNC_PID")" 2>/dev/null || true
      rm -f "$SYNC_PID"
    fi
    if [ -x /usr/local/bin/sd_sync_daemon.sh ]; then
      /usr/local/bin/sd_sync_daemon.sh &
      log_msg "Sync daemon started (PID $!)"
    fi

    log_msg "=== tmpfs bind mounts: ready ==="
    ;;

  stop)
    echo "=== tmpfs bind mounts: stopping ==="

    # Stop sync daemon
    if [ -f "$SYNC_PID" ]; then
      kill "$(cat "$SYNC_PID")" 2>/dev/null || true
      rm -f "$SYNC_PID"
    fi
    for pid in $(pgrep -f sd_sync_daemon.sh 2>/dev/null); do
      [ "$pid" != "$$" ] && [ "$pid" != "$PPID" ] && kill "$pid" 2>/dev/null || true
    done

    # Final sync to USB before shutdown
    if [ -x /usr/local/bin/sd_sync_daemon.sh ]; then
      /usr/local/bin/sd_sync_daemon.sh --once 2>/dev/null || true
    fi

    # Final sync critical DBs to eMMC backup AND live eMMC paths
    # This ensures: (a) re-seeding works after SD reformat, (b) --uninstall
    # run after a reboot (bind mounts gone) still sees correct data.
    TMPFS_DB="/var/run/sd_tmpfs/db"
    TMPFS_MISC="/var/run/sd_tmpfs/misc"
    echo "Writing critical data back to eMMC..."

    # --- DB directories → EMMC_BACKUP AND live eMMC paths ---
    for dir in binned event pdm archive lifetime ens_device_commission; do
      TARGET="/var/opt/emu/$dir"
      if mountpoint -q "$TARGET" 2>/dev/null && [ -d "$TMPFS_DB/$dir" ]; then
        TMP="/tmp/stop_wb_${dir}"
        mkdir -p "$TMP"
        cp -a "$TMPFS_DB/$dir/." "$TMP/" 2>/dev/null || true
        umount -l "$TARGET" 2>/dev/null || true
        mkdir -p "$TARGET"
        cp -a "$TMP/." "$TARGET/" 2>/dev/null || true
        mkdir -p "$EMMC_BACKUP/$dir" 2>/dev/null || true
        cp -a "$TMP/." "$EMMC_BACKUP/$dir/" 2>/dev/null || true
        rm -rf "$TMP"
      elif [ -d "$TMPFS_DB/$dir" ]; then
        mkdir -p "$EMMC_BACKUP/$dir" 2>/dev/null || true
        cp -a "$TMPFS_DB/$dir/." "$EMMC_BACKUP/$dir/" 2>/dev/null || true
      fi
    done

    # --- emu.db root-level files → EMMC_BACKUP AND live eMMC paths ---
    if [ -d "$TMPFS_DB/root_dbs" ]; then
      mkdir -p "$EMMC_BACKUP/root_dbs" 2>/dev/null || true
      for db in emu.db emu.db-wal emu.db-shm agf_perf.db agf_perf.db-wal agf_perf.db-shm; do
        DB_LIVE="/var/opt/emu/$db"
        DB_TMPFS="$TMPFS_DB/root_dbs/$db"
        if [ -f "$DB_TMPFS" ]; then
          cp -a "$DB_TMPFS" "$EMMC_BACKUP/root_dbs/$db" 2>/dev/null || true
          if grep -q " $DB_LIVE " /proc/mounts 2>/dev/null; then
            # File-level bind mount active — unmount to expose eMMC, write back
            TMP="/tmp/stop_wb_${db}"
            cp -a "$DB_TMPFS" "$TMP" 2>/dev/null || true
            umount -l "$DB_LIVE" 2>/dev/null || true
            cp -a "$TMP" "$DB_LIVE" 2>/dev/null || true
            rm -f "$TMP"
          fi
        fi
      done
    fi

    # --- mal/ and cfg/ → EMMC_BACKUP AND live eMMC paths ---
    # These were never updated in EMMC_BACKUP after initial install, causing
    # revert to stale meter config when SD is reformatted or unavailable.
    for pair in "mal:/var/opt/mal" "cfg:/var/opt/cfg" "agf_mlgr:/var/opt/agf/mlgr"; do
      TDIR="${pair%%:*}"
      TARGET="${pair#*:}"
      if mountpoint -q "$TARGET" 2>/dev/null; then
        # Under the active bind mount, $TARGET reads from tmpfs.
        # Copy to a temp, unmount to expose bare eMMC, copy from temp.
        TMP="/tmp/stop_wb_${TDIR}"
        mkdir -p "$TMP"
        cp -a "$TARGET/." "$TMP/" 2>/dev/null || true
        umount -l "$TARGET" 2>/dev/null || true    # exposes underlying eMMC
        mkdir -p "$TARGET"
        cp -a "$TMP/." "$TARGET/" 2>/dev/null || true   # write to live eMMC
        mkdir -p "$EMMC_BACKUP/$TDIR" 2>/dev/null || true
        cp -a "$TMP/." "$EMMC_BACKUP/$TDIR/" 2>/dev/null || true  # update backup
        rm -rf "$TMP"
      elif [ -d "$TMPFS_MISC/$TDIR" ]; then
        # Bind mount already gone (e.g. called standalone) — copy from tmpfs directly
        mkdir -p "$TARGET"
        cp -a "$TMPFS_MISC/$TDIR/." "$TARGET/" 2>/dev/null || true
        mkdir -p "$EMMC_BACKUP/$TDIR" 2>/dev/null || true
        cp -a "$TMPFS_MISC/$TDIR/." "$EMMC_BACKUP/$TDIR/" 2>/dev/null || true
      fi
    done

    sync 2>/dev/null || true

    # Unmount bind mounts
    for db in emu.db emu.db-wal emu.db-shm agf_perf.db agf_perf.db-wal agf_perf.db-shm; do
      n=0; while [ $n -lt 5 ] && grep -q " /var/opt/emu/$db " /proc/mounts 2>/dev/null; do
        umount -l "/var/opt/emu/$db" 2>/dev/null || break
        n=$((n + 1))
      done
    done

    for d in /var/log/emu /var/log/nginx /var/log/agf /var/log/chrono \
             /var/log/tinyproxy /var/www/logs \
             /var/opt/emu/binned /var/opt/emu/event /var/opt/emu/pdm \
             /var/opt/emu/archive /var/opt/emu/lifetime \
             /var/opt/emu/ens_device_commission /corefiles \
             /var/opt/cfg /var/opt/mal /var/opt/agf/mlgr /etc/wifi \
             /var/opt/emu/cache; do
      n=0; while [ $n -lt 5 ] && grep -q " ${d} " /proc/mounts 2>/dev/null; do
        umount -l "$d" 2>/dev/null || break
        n=$((n + 1))
      done
    done

    for logfile in /var/log/messages /var/log/dmesg; do
      if [ -f "$logfile" ] && [ ! -L "$logfile" ]; then
        n=0; while [ $n -lt 5 ] && grep -q " ${logfile} " /proc/mounts 2>/dev/null; do
          umount -l "$logfile" 2>/dev/null || break
          n=$((n + 1))
        done
      fi
    done

    # Unmount utmp bind mount
    umount -l /var/run/utmp 2>/dev/null || true

    # Unmount tmpfs backing stores
    for d in /var/run/sd_tmpfs/db /var/run/sd_tmpfs/log /var/run/sd_tmpfs/core /var/run/sd_tmpfs/misc; do
      umount -l "$d" 2>/dev/null || true
    done

    # Unmount /var/tmp tmpfs
    umount -l /var/tmp 2>/dev/null || true

    # Unmount USB
    umount -l "$SD_MOUNT" 2>/dev/null || true

    echo "=== tmpfs bind mounts: stopped ==="
    ;;

  status)
    echo "=== tmpfs bind mount status ==="
    echo ""
    echo "tmpfs backing stores:"
    for d in /var/run/sd_tmpfs/db /var/run/sd_tmpfs/log /var/run/sd_tmpfs/core /var/run/sd_tmpfs/misc; do
      if mountpoint -q "$d" 2>/dev/null; then
        USED=$(du -sh "$d" 2>/dev/null | awk '{print $1}')
        echo "  ✓ $d ($USED used)"
      else
        echo "  ✗ $d (not mounted)"
      fi
    done
    echo ""
    echo "USB drive:"
    if mountpoint -q "$SD_MOUNT" 2>/dev/null; then
      echo "  ✓ Mounted at $SD_MOUNT"
      df -h "$SD_MOUNT"
    else
      echo "  ✗ Not mounted (sync daemon will retry)"
    fi
    echo ""
    echo "Sync daemon:"
    if [ -f /tmp/sd_sync.pid ] && kill -0 "$(cat /tmp/sd_sync.pid)" 2>/dev/null; then
      echo "  ✓ Running (PID $(cat /tmp/sd_sync.pid))"
      if [ -f /tmp/sd_sync_last ]; then
        echo "  Last sync: $(cat /tmp/sd_sync_last)"
      fi
    else
      echo "  ✗ Not running"
    fi
    echo ""
    echo "Active bind mounts:"
    grep "sd_tmpfs" /proc/mounts 2>/dev/null | awk '{print "  " $2}' || echo "  (none)"
    echo ""
    echo "RAM usage:"
    free
    ;;

  *)
    echo "Usage: $0 {start|stop|status}"
    exit 1
    ;;
esac
INITSCRIPT

chmod +x /etc/init.d/sd_bind_mounts

# Link into boot sequence (rc3.d S80 — after USB init, before app startup)
ln -sf ../init.d/sd_bind_mounts /etc/rc3.d/S80sd_bind_mounts
# Remove old rcS.d links if present
rm -f /etc/rcS.d/S03sd_bind_mounts /etc/rcS.d/S10sd_bind_mounts /etc/rcS.d/S11sd_bind_mounts /etc/rcS.d/S96sd_bind_mounts
# Shutdown links
ln -sf ../init.d/sd_bind_mounts /etc/rc0.d/K97sd_bind_mounts
ln -sf ../init.d/sd_bind_mounts /etc/rc6.d/K97sd_bind_mounts

echo -e "${GREEN}Init script installed (S80sd_bind_mounts).${NC}"

# ─── Step 6b: Install sync daemon ────────────────────────────────
echo ""
echo "Installing sync daemon..."

cat > /usr/local/bin/sd_sync_daemon.sh << 'SYNCDAEMON'
#!/bin/sh
# sd_sync_daemon.sh — Periodically sync tmpfs data to USB drive
# Lightweight, retry-friendly. If USB is gone, just skip and try next cycle.
#
# Usage:
#   sd_sync_daemon.sh          # Run as daemon (background loop)
#   sd_sync_daemon.sh --once   # Single sync pass then exit

SD_MOUNT="/mnt/sd"
TMPFS_DB="/var/run/sd_tmpfs/db"
TMPFS_LOG="/var/run/sd_tmpfs/log"
TMPFS_MISC="/var/run/sd_tmpfs/misc"
SYNC_INTERVAL=300
PIDFILE="/tmp/sd_sync.pid"
LAST_SYNC_FILE="/tmp/sd_sync_last"

ONE_SHOT=0
if [ "$1" = "--once" ]; then
  ONE_SHOT=1
fi

# ─── Find and mount USB if not already mounted ──────────────────
try_mount_usb() {
  if mountpoint -q "$SD_MOUNT" 2>/dev/null; then
    # Check if the underlying block device still exists BEFORE any I/O.
    # USB readers can disconnect and re-enumerate under a different node (e.g.
    # sda→sdb). Writing through a stale mount to a gone device causes an ext4
    # journal abort — so we detect and clear the stale mount first.
    CURRENT_DEV=$(awk "\$2 == \"$SD_MOUNT\" {print \$1}" /proc/mounts 2>/dev/null)
    if [ -n "$CURRENT_DEV" ] && [ ! -b "$CURRENT_DEV" ]; then
      # Block device is gone — lazy-unmount before any write attempt
      umount -l "$SD_MOUNT" 2>/dev/null || true
    elif touch "$SD_MOUNT/.mount_test" 2>/dev/null; then
      rm -f "$SD_MOUNT/.mount_test"
      return 0
    else
      # Write test failed (I/O error or read-only) — force remount
      umount -l "$SD_MOUNT" 2>/dev/null || true
    fi
  fi
  # Try to find our labeled device (handles sda→sdb re-enumeration)
  rm -f /etc/blkid.tab /run/blkid/blkid.tab 2>/dev/null
  for dev in /dev/sd[a-z]1 /dev/mmcblk[1-9]p1; do
    if [ -b "$dev" ]; then
      LABEL=$(blkid "$dev" 2>/dev/null | grep -o 'LABEL="[^"]*"' | cut -d'"' -f2)
      if [ "$LABEL" = "envoy-sd" ]; then
        mkdir -p "$SD_MOUNT"
        mount -o rw,noatime,nodiratime,errors=continue "$dev" "$SD_MOUNT" 2>/dev/null
        return $?
      fi
    fi
  done
  return 1
}

# ─── Sync tmpfs → USB ───────────────────────────────────────────
do_sync() {
  # Check USB is mounted and writable
  if ! mountpoint -q "$SD_MOUNT" 2>/dev/null; then
    return 1
  fi
  if ! touch "$SD_MOUNT/.sync_test" 2>/dev/null; then
    return 1
  fi
  rm -f "$SD_MOUNT/.sync_test"

  # Sync database directories
  for dir in binned event pdm archive lifetime ens_device_commission; do
    SRC="$TMPFS_DB/$dir"
    DST="$SD_MOUNT/var/opt/emu/$dir"
    if [ -d "$SRC" ]; then
      mkdir -p "$DST"
      cp -a "$SRC/"* "$DST/" 2>/dev/null || true
    fi
  done

  # Sync root-level DBs
  mkdir -p "$SD_MOUNT/var/opt/emu/root_dbs"
  for db in emu.db emu.db-wal emu.db-shm agf_perf.db agf_perf.db-wal agf_perf.db-shm; do
    if [ -f "$TMPFS_DB/root_dbs/$db" ]; then
      cp -a "$TMPFS_DB/root_dbs/$db" "$SD_MOUNT/var/opt/emu/root_dbs/$db" 2>/dev/null || true
    fi
  done

  # Sync logs
  for dir in emu nginx agf chrony tinyproxy; do
    SRC="$TMPFS_LOG/$dir"
    DST="$SD_MOUNT/var/log/$dir"
    if [ -d "$SRC" ]; then
      mkdir -p "$DST"
      cp -a "$SRC/"* "$DST/" 2>/dev/null || true
    fi
  done
  mkdir -p "$SD_MOUNT/var/log/misc"
  for logfile in messages dmesg; do
    if [ -f "$TMPFS_LOG/misc/$logfile" ]; then
      cp -a "$TMPFS_LOG/misc/$logfile" "$SD_MOUNT/var/log/misc/$logfile" 2>/dev/null || true
    fi
  done
  mkdir -p "$SD_MOUNT/var/www/logs"
  if [ -d "$TMPFS_LOG/www_logs" ]; then
    cp -a "$TMPFS_LOG/www_logs/"* "$SD_MOUNT/var/www/logs/" 2>/dev/null || true
  fi

  # Sync stats logs (write_stats.log, emmc_health.log)
  if [ -d "$TMPFS_LOG/stats" ]; then
    mkdir -p "$SD_MOUNT/stats"
    cp -a "$TMPFS_LOG/stats/"* "$SD_MOUNT/stats/" 2>/dev/null || true
  fi

  # Sync misc high-write paths (cfg, mal, agf meter logs, wifi)
  for pair in "cfg:var/opt/cfg" "mal:var/opt/mal" "agf_mlgr:var/opt/agf/mlgr" "wifi:etc/wifi"; do
    TDIR="${pair%%:*}"
    SDPATH="${pair#*:}"
    if [ -d "$TMPFS_MISC/$TDIR" ]; then
      mkdir -p "$SD_MOUNT/$SDPATH"
      cp -a "$TMPFS_MISC/$TDIR/"* "$SD_MOUNT/$SDPATH/" 2>/dev/null || true
    fi
  done
  # Note: /var/opt/emu/cache is NOT synced — it's regeneratable by the app

  # Record sync time
  date '+%Y-%m-%d %H:%M:%S' > "$LAST_SYNC_FILE"
  return 0
}

# ─── One-shot mode ──────────────────────────────────────────────
if [ "$ONE_SHOT" -eq 1 ]; then
  try_mount_usb
  if do_sync; then
    sync 2>/dev/null || true
  fi
  exit 0
fi

# ─── Daemon mode ────────────────────────────────────────────────
echo $$ > "$PIDFILE"
trap 'rm -f "$PIDFILE"; exit 0' INT TERM

logger -t sd_sync "Sync daemon started (PID $$, interval=${SYNC_INTERVAL}s)"

while true; do
  sleep "$SYNC_INTERVAL"

  # Try to mount USB if not already
  try_mount_usb

  if do_sync; then
    # Successful sync — no need to log every time (too noisy)
    :
  else
    # USB not available — log occasionally (not every cycle)
    FAIL_COUNT=$((${FAIL_COUNT:-0} + 1))
    if [ "$FAIL_COUNT" -eq 1 ] || [ $((FAIL_COUNT % 60)) -eq 0 ]; then
      logger -t sd_sync "USB not available (attempt #${FAIL_COUNT}) — data safe in tmpfs"
    fi
  fi
done
SYNCDAEMON

chmod +x /usr/local/bin/sd_sync_daemon.sh
echo -e "${GREEN}Sync daemon installed.${NC}"

# ─── Step 6c: Remove old watchdog if present ─────────────────────
if [ -f /usr/local/bin/sd_watchdog.sh ]; then
  echo "Removing old SD watchdog (replaced by sync daemon)..."
  if [ -f /tmp/sd_watchdog.pid ]; then
    kill "$(cat /tmp/sd_watchdog.pid)" 2>/dev/null || true
    rm -f /tmp/sd_watchdog.pid
  fi
  rm -f /usr/local/bin/sd_watchdog.sh
fi

# ─── Step 7: Fix sysctl writeback timers ─────────────────────────
echo ""
echo "Fixing kernel writeback timers..."

sysctl -w vm.dirty_writeback_centisecs=1500 > /dev/null
sysctl -w vm.dirty_expire_centisecs=3000 > /dev/null

if grep -q "# eMMC longevity" /etc/sysctl.conf 2>/dev/null; then
  echo "  sysctl.conf already patched."
else
  sed -i 's/^vm.dirty_writeback_centisecs.*/#&  # disabled for eMMC longevity/' /etc/sysctl.conf
  sed -i 's/^vm.dirty_expire_centisecs.*/#&  # disabled for eMMC longevity/' /etc/sysctl.conf
  cat >> /etc/sysctl.conf << 'EOF'

# eMMC longevity — reduce flush frequency (was 100/100 = 1 second!)
vm.dirty_writeback_centisecs = 1500
vm.dirty_expire_centisecs = 3000
EOF
  echo -e "  ${GREEN}sysctl.conf patched: 1s → 15s/30s writeback${NC}"
fi

# ─── Step 8: Mount /tmp as tmpfs ─────────────────────────────────
echo ""
echo "Setting up /tmp as tmpfs..."

if mount | grep -q "tmpfs on /tmp"; then
  echo "  /tmp is already tmpfs."
else
  umount /tmp 2>/dev/null || true
  mount -t tmpfs -o defaults,noatime,nosuid,nodev,size=64M tmpfs /tmp 2>/dev/null && \
    echo -e "  ${GREEN}/tmp mounted as tmpfs (64MB)${NC}" || \
    echo -e "  ${YELLOW}/tmp busy — will be tmpfs after reboot${NC}"
fi

if grep -q "^tmpfs.*/tmp" /etc/fstab; then
  echo "  fstab already has tmpfs /tmp."
else
  sed -i 's|^/dev/mmcblk0p6.*|#&  # disabled for eMMC longevity\ntmpfs /tmp tmpfs defaults,noatime,nosuid,nodev,size=64M 0 0|' /etc/fstab
  echo -e "  ${GREEN}fstab updated: /tmp → tmpfs${NC}"
fi

# ─── Step 9: Increase ext4 commit intervals ─────────────────────
echo ""
echo "Increasing ext4 journal commit intervals..."

mount -o remount,commit=60 / 2>/dev/null && echo "  / remounted with commit=60" || true
mount -o remount,commit=60 /var 2>/dev/null && echo "  /var remounted with commit=60" || true
mount -o remount,commit=60 /home 2>/dev/null && echo "  /home remounted with commit=60" || true

sed -i 's/\(,commit=60\)\{2,\}/,commit=60/g' /etc/fstab 2>/dev/null || true
for part in mmcblk0p3 mmcblk0p5 mmcblk0p7; do
  if grep -q "/dev/$part" /etc/fstab 2>/dev/null && ! grep "/dev/$part" /etc/fstab | grep -q "commit=60"; then
    sed -i "s|\(/dev/$part.*ext4.*\)rw,noatime|\1rw,noatime,commit=60|" /etc/fstab
    echo "  fstab: $part → commit=60"
  fi
done

# ─── Step 10: Set SQLite WAL mode ───────────────────────────────
echo ""
echo "Setting SQLite WAL mode on databases..."

for db in /mnt/sd/var/opt/emu/binned/binned.db \
          /mnt/sd/var/opt/emu/event/event.db \
          /mnt/sd/var/opt/emu/pdm/pdm.db \
          /mnt/sd/var/opt/emu/root_dbs/emu.db \
          /mnt/sd/var/opt/emu/lifetime/lifetime.db; do
  if [ -f "$db" ]; then
    sqlite3 "$db" "PRAGMA journal_mode=WAL;" 2>/dev/null && \
      echo "  WAL: $db" || echo "  SKIP: $db (in use?)"
  fi
done

# ─── Step 11: Activate tmpfs mounts + sync NOW ───────────────────
echo ""
echo "Stopping services before bind mount activation..."
# Services must be stopped so their open file descriptors are closed.
# Without this, processes holding open fds to eMMC files bypass the bind
# mount and continue writing directly to eMMC instead of tmpfs.
/etc/init.d/envoy_monit stop 2>/dev/null || true
sleep 2

echo "Activating tmpfs bind mounts and sync daemon..."
/etc/init.d/sd_bind_mounts start

echo "Restarting services against tmpfs paths..."
# Services now open files through the bind-mounted (tmpfs) paths.
/etc/init.d/envoy_monit start 2>/dev/null || true

# ─── Step 12: Install daily eMMC backup cron ─────────────────────
echo ""
echo "Installing daily eMMC backup of critical DBs..."

cat > /etc/cron.daily/sd_emmc_backup << 'CRONSCRIPT'
#!/bin/sh
# Daily backup: critical DBs from tmpfs → eMMC
# Ensures eMMC has a recent copy if USB and RAM are both lost (power failure)

TMPFS_DB="/var/run/sd_tmpfs/db"
EMMC_BACKUP="/var/opt/emu/.emmc_backup"

if ! mountpoint -q "$TMPFS_DB" 2>/dev/null; then
  logger -t sd_emmc_backup "tmpfs not mounted, skipping"
  exit 0
fi

mkdir -p "$EMMC_BACKUP/root_dbs" "$EMMC_BACKUP/lifetime"

# Critical: emu.db (device config, inverter pairing)
for db in emu.db emu.db-wal emu.db-shm; do
  if [ -f "$TMPFS_DB/root_dbs/$db" ]; then
    cp -a "$TMPFS_DB/root_dbs/$db" "$EMMC_BACKUP/root_dbs/$db" 2>/dev/null
  fi
done

# Critical: lifetime.db (production totals)
if [ -d "$TMPFS_DB/lifetime" ]; then
  cp -a "$TMPFS_DB/lifetime/"* "$EMMC_BACKUP/lifetime/" 2>/dev/null
fi

sync
logger -t sd_emmc_backup "Backed up emu.db + lifetime to eMMC"
CRONSCRIPT
chmod +x /etc/cron.daily/sd_emmc_backup

echo -e "  ${GREEN}Daily eMMC backup installed.${NC}"

# ─── Step 12b: Install weekly eMMC health monitor ────────────────
echo ""
echo "Installing weekly eMMC health monitor..."

cat > /etc/cron.weekly/emmc_health << 'CRONSCRIPT'
#!/bin/sh
LIFE=$(cat /sys/class/mmc_host/mmc0/mmc0:0001/life_time 2>/dev/null || echo "N/A")
EOL=$(cat /sys/class/mmc_host/mmc0/mmc0:0001/pre_eol_info 2>/dev/null || echo "N/A")
SECTORS=$(awk '/mmcblk0 / {print $10}' /proc/diskstats)
PE=$(ruby /opt/emu/scripts/mmc_utils.rb 2>/dev/null | tr -d '\n' || echo "N/A")
# Write to tmpfs — sync daemon carries it to USB
LOG_DIR="/var/run/sd_tmpfs/log/stats"
mkdir -p "$LOG_DIR"
echo "$(date '+%Y-%m-%d %H:%M'): life=$LIFE eol=$EOL sectors=$SECTORS pe=$PE" >> "$LOG_DIR/emmc_health.log"
CRONSCRIPT
chmod +x /etc/cron.weekly/emmc_health

# ─── Step 12c: Install daily write stats logger ──────────────────
echo ""
echo "Installing daily write stats logger..."

cat > /etc/cron.daily/write_stats << 'CRONSCRIPT'
#!/bin/sh
DATE=$(date '+%Y-%m-%d %H:%M')
LIFE=$(cat /sys/class/mmc_host/mmc0/mmc0:0001/life_time 2>/dev/null || echo "N/A")
EOL=$(cat /sys/class/mmc_host/mmc0/mmc0:0001/pre_eol_info 2>/dev/null || echo "N/A")
EMMC_TOTAL=$(awk '/mmcblk0 / {print $10}' /proc/diskstats)

EXT4_STATS=""
for dev in /sys/fs/ext4/*/lifetime_write_kbytes; do
  if [ -f "$dev" ]; then
    DEVNAME=$(echo "$dev" | sed 's|/sys/fs/ext4/||;s|/lifetime_write_kbytes||')
    KB=$(cat "$dev" 2>/dev/null || echo 0)
    MB=$((KB / 1024))
    EXT4_STATS="${EXT4_STATS} ${DEVNAME}=${MB}MB"
  fi
done

SD_SECTORS="N/A"
SD_DEV=$(mount | awk '/\/mnt\/sd / {print $1}')
if [ -n "$SD_DEV" ]; then
  SD_DISK=$(echo "$SD_DEV" | sed 's/[0-9]*$//' | sed 's|/dev/||')
  SD_SECTORS=$(awk -v d="$SD_DISK " '$0 ~ d {print $10}' /proc/diskstats)
  [ -z "$SD_SECTORS" ] && SD_SECTORS="N/A"
fi

# tmpfs usage
TMPFS_USED="N/A"
if mountpoint -q /var/run/sd_tmpfs/db 2>/dev/null; then
  TMPFS_USED=$(du -sh /var/run/sd_tmpfs/ 2>/dev/null | tail -1 | awk '{print $1}')
fi

# Write to tmpfs — sync daemon carries it to USB
LOG_DIR="/var/run/sd_tmpfs/log/stats"
mkdir -p "$LOG_DIR"
echo "${DATE} | emmc_life=${LIFE} eol=${EOL} | emmc_total=${EMMC_TOTAL}s | sd_total=${SD_SECTORS}s | tmpfs_used=${TMPFS_USED} | ext4:${EXT4_STATS}" >> "$LOG_DIR/write_stats.log"
CRONSCRIPT
chmod +x /etc/cron.daily/write_stats

echo -e "  ${GREEN}Daily write stats logger installed.${NC}"

# ─── Step 13: Install write rate monitor ─────────────────────────
cat > /usr/local/bin/check_writes << 'MONITOR'
#!/bin/sh
# Quick check of eMMC write rate + tmpfs usage
WAIT=${1:-60}

S1=$(awk '/mmcblk0 / {print $10}' /proc/diskstats)
echo "Measuring eMMC writes for ${WAIT} seconds..."
sleep "$WAIT"
S2=$(awk '/mmcblk0 / {print $10}' /proc/diskstats)

DELTA=$((S2 - S1))
KB=$((DELTA * 512 / 1024))
RATE=$((KB * 86400 / WAIT / 1024))

echo "Sectors written: $DELTA"
echo "KB written in ${WAIT}s: $KB"
echo "Estimated daily rate: ${RATE} MB/day"
echo ""
echo "tmpfs usage:"
for d in /var/run/sd_tmpfs/db /var/run/sd_tmpfs/log /var/run/sd_tmpfs/core; do
  if mountpoint -q "$d" 2>/dev/null; then
    echo "  $d: $(du -sh "$d" 2>/dev/null | awk '{print $1}')"
  fi
done
echo ""
echo "eMMC wear level:"
ruby /opt/emu/scripts/mmc_utils.rb 2>/dev/null || echo "  N/A"
MONITOR
chmod +x /usr/local/bin/check_writes

# Remove old weekly backup cron (replaced by daily)
rm -f /etc/cron.weekly/sd_backup_critical

# ─── Summary ─────────────────────────────────────────────────────
echo ""
echo -e "${GREEN}════════════════════════════════════════════${NC}"
echo -e "${GREEN}  Setup Complete! (v2 — tmpfs + async sync)${NC}"
echo -e "${GREEN}════════════════════════════════════════════${NC}"
echo ""
echo "Architecture:"
echo "  Services write to → tmpfs (RAM) → sync daemon → USB drive"
echo "  On boot: eMMC seed → USB overlay → tmpfs ready"
echo "  On shutdown: tmpfs → USB + critical DBs → eMMC"
echo ""
echo "What was done:"
echo "  ✓ tmpfs bind mounts for all high-write paths (in RAM)"
echo "  ✓ Data seeded from eMMC + USB overlay"
echo "  ✓ Sync daemon: tmpfs → USB every ${SYNC_INTERVAL}s"
echo "  ✓ Daily eMMC backup of critical DBs (emu.db, lifetime.db)"
echo "  ✓ Kernel writeback: 1s → 15s/30s"
echo "  ✓ /tmp → tmpfs (64MB RAM)"
echo "  ✓ ext4 commit interval → 60s"
echo "  ✓ SQLite WAL mode on databases"
echo "  ✓ Weekly eMMC health monitor"
echo "  ✓ Daily write stats logger"
echo "  ✓ Old watchdog removed (not needed anymore)"
echo ""
echo "If USB vanishes:"
echo "  - Services continue uninterrupted (tmpfs always available)"
echo "  - Sync daemon logs a warning and retries each cycle"
echo "  - When USB returns, sync resumes automatically"
echo "  - eMMC backup ensures data survives even total loss"
echo ""
echo "Commands:"
echo "  check_writes 60                      — measure eMMC write rate"
echo "  /etc/init.d/sd_bind_mounts status    — check tmpfs + sync status"
echo "  sd_sync_daemon.sh --once             — force sync to USB now"
echo "  ruby /opt/emu/scripts/mmc_utils.rb   — check eMMC wear"
echo ""
echo -e "${YELLOW}Reboot recommended to fully activate all changes.${NC}"