Restart the server unless ssh works.

restart_unless_ssh.sh

#!/bin/sh

USERNAME=user
SSH_CMD="ssh user@domain -i /home/user/.ssh/no_pw"
MAX_TRIES=4
RESTART_DELAY=`expr 15 \* 60` # seconds

tmpfile=/tmp/restart_unless_ssh-`date +%s`

lock=/tmp/restart_unless_ssh-lock
if [ -f $lock ]; then
  start=`cat $lock`
  duration=`expr $(date +%s) - $start`
  if [ $duration -le $RESTART_DELAY ]; then
    echo "Already running"
    exit 1
  fi
fi
date +%s > $lock

i=0
while [ $i -lt $MAX_TRIES ]; do
  echo "Trying to execute command via $SSH_CMD ..."
  $SSH_CMD touch $tmpfile
  echo "done"
  if [ -f $tmpfile ]; then
    rm $tmpfile
    echo "SSH worked."
    rm $lock
    exit 0
  else
    echo "Couldn't touch file via ssh."
  fi
  i=`expr $i + 1`
  echo $i
done

sendmail email@example.com <<EOF
TO: email@example.com
SUBJECT: SSH not working! Restarting soon!

Could not execute a command via ssh using \`$SSH_CMD\`.
Restarting in `expr $RESTART_DELAY / 60` minutes at `date -d +${RESTART_DELAY}seconds` (unless pid $$ is killed until then).
EOF

echo "waiting..."
sleep $RESTART_DELAY

$SSH_CMD touch $tmpfile
rm $lock
if [ -f $tmpfile ]; then
  rm $tmpfile
  echo "SSH worked after waiting; cancelling reboot"
else
  echo "rebooting"
  reboot
fi