sync.sh

#!/bin/bash

trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT

LAST_ITEM=""
while :
do
  ssh -q -o ConnectTimeout=1 theflame exit
  if [[ $? != 0 ]]; then
    echo "SYNC: ssh is not available. waiting 5s..."
    sleep 5
    continue
  fi

  POOLS=$(ssh theflame 'zpool list | grep tanker')
  if [[ "$POOLS" == "" ]]; then
    echo "SYNC: mounting pool..."
    ssh theflame 'sudo zpool import -o readonly=on tanker'
  fi

  # Clear state from previous runs to ensure we can detect immediate hangs.
  truncate -s 0 rsync.txt

  echo "SYNC: starting rsync..."
  rsync -ahHv --append --progress \
    theflame:/tanker/media/* \
    /tanker/media | tee rsync.txt &

  sleep 60 # Let rsync get the file list done and transfer started.

  LAST_OUTPUT=""
  while sleep 10
  do
    NEW_OUTPUT=$(tail -2 rsync.txt)
    if [[ "$LAST_OUTPUT" == "$NEW_OUTPUT" ]]; then
      break # output has not changed, assume locked up.
    fi
    LAST_OUTPUT="$NEW_OUTPUT"

    # Send push notification of item currently being tranferred.
    NEW_ITEM=$(head -1 <<< "$LAST_OUTPUT" | cut -d'/' -f-3)
    if [[ "$LAST_ITEM" != "$NEW_ITEM" ]]; then
      curl -X POST -s -F "value1=$NEW_ITEM" https://maker.ifttt.com/trigger/rsync_update/with/key/... > /dev/null
    fi
    LAST_ITEM="$NEW_ITEM"
  done

  echo
  echo "SYNC: rsync output unchanged after 10s! power cycling..."

  echo "POWER: off"
  curl -X POST -s https://maker.ifttt.com/trigger/power_off/with/key/... > /dev/null

  # Loop until SSH disappears to confirm power off
  while :
  do
    sleep 5

    ssh -q -o ConnectTimeout=1 theflame exit
    if [[ $? != 0 ]]; then
      break # SSH is down!
    fi
    echo "POWER: ssh is still available. waiting 5s..."
  done

  echo "POWER: on"
  curl -X POST -s https://maker.ifttt.com/trigger/power_on/with/key/... > /dev/null

  echo "SYNC: waiting 50s for startup..."
  sleep 50
done