Monitor a file for changes and backup to remote host

lnd-channel-backup-to-remote.sh

#!/bin/bash

# To configure settings for this script, run the script directly at the command
# line. A configuration file will be created as the same name as this script
# with a .json extension that will contain the values that will be used at
# runtime.

# As with any script, you should review it before running it.  The main entry
# point is at the end of the file after variable initializations and function
# declarations are made.

# =============================================================================

# Default values (these get overidden by configuration file)
# File to watch for backup, source and local backup paths
FILETOBACKUP="channel.backup"
SOURCEFOLDER="/home/bitcoin/.lnd/data/chain/bitcoin/mainnet/"
LOCALBACKUPFOLDER="/tmp/"
# This is where the file backup should be scp'd to and must be configured in
# the json file
REMOTESERVER="10.10.1.21"
REMOTEUSER="someuser"
REMOTEBACKUPFOLDER="/mnt/backup/lnd/"
# RSA Public Key Path
#   generate with:              ssh-keygen -t rsa
#   then copy to remote with:   ssh-copy-id user@server
RSAPUBKEYPATH="~/.ssh/id_rsa.pub"
# Identify this device (useful if you are running this on multiple nodes)
DEVICE=$(hostname)

# Color definitions for convenience
color_red='\033[0;31m'
color_green='\033[0;32m'
color_yellow='\033[0;33m'
color_blue='\033[0;34m'
color_purple='\033[0;35m'
color_cyan='\033[0;36m'
color_white='\033[0;37m'
color_normal=${color_white}
color_currentvalue=${color_purple}
color_newvalue=${color_yellow}
color_link=${color_green}
color_command=${color_cyan}
color_content=${color_red}

# This variable just tracks where we are in config process
configlevel=0 #92

# =============================================================================
# Declare some functions to make this logically easier to read through
# =============================================================================

setup_backup_filename() {
  bfe=true
  bfc=0
  while $bfe
  do
    # get a time stamp in seconds since epoch
    NEWFILEDATE=`date +%s`
    # determine backup file name
    BACKUPFILE="${LOCALBACKUPFILE}.${NEWFILEDATE}"
    # make sure the backup filename doesnt already exist
    if [ ! -f "${BACKUPFILE}" ]; then
      bfe=false
    else
      bfc=$((bfc++))
      if [ $bfc -ge 5 ]; then
        bfe=false
        break
      fi
      sleep 0.1
    fi
  done
}

backup_to_remote() {
  # file to backup is in ${1} arg passed in
  # filename itself is in ${2} arg passed in
  REMOTEBACKUPFILE="${REMOTEBACKUPFOLDER}${DEVICE}/${2}"
  # Depends on prior call to   eval `ssh-agent`    and     ssh-add
  scp -i ${RSAPUBKEYPATH} ${1} ${REMOTEUSER}@${REMOTESERVER}:${REMOTEBACKUPFILE}
}

make_remote_folder() {
  REMOTEFOLDERTOENSURE="${REMOTEBACKUPFOLDER}${DEVICE}"
  eval `ssh-agent`    # Eventually need to kill this when script exits
  ssh-add
  ssh ${REMOTEUSER}@${REMOTESERVER} "mkdir -p ${REMOTEFOLDERTOENSURE}"
}

service_loop() {
  config_load "validate"
  make_remote_folder
  three_hours=10800
  while true; do
    # wait for change to source file
    inotifywait --timeout $three_hours $SOURCEFILE
    ec=$(echo $?)
    if [ $ec -eq 0 ]; then
      # file changed
      setup_backup_filename
      MD5SUMFILE="${BACKUPFILE}.md5"
      # copy source to backup and get md5 sum file
      cp $SOURCEFILE $BACKUPFILE
      md5sum $BACKUPFILE > $MD5SUMFILE
      sed -i 's/\/.*\///g' $MD5SUMFILE
      # backup to remote
      backup_to_remote $BACKUPFILE ${FILETOBACKUP}.${NEWFILEDATE}
      backup_to_remote $MD5SUMFILE ${FILETOBACKUP}.${NEWFILEDATE}.md5
    fi
  done
}

guidance() {
  printf "\nThis script is intended to watch a single file (${color_currentvalue}${FILETOBACKUP}${color_normal}) and "
  printf "\nwhenever it changes, backup to a local folder, as well as to a remote "
  printf "\nserver.  Backups to the remote server are timestamped.\n"
  printf "\nYou must have an ssh key pair to use this script and you will need to "
  printf "\nhave copied it to the remote server."
  printf "\n- To create an ssh key pair use this command: ${color_command}ssh-keygen -t rsa${color_normal}"
  printf "\n- To copy to remote server use this command: ${color_command}ssh-copy-id user@server${color_normal}"
  printf "\n  replacing ${color_command}user${color_normal} and ${color_command}server${color_normal} with the remote server account and address\n"
  read -p "$(printf "\nWhen ready to complete configuration, press enter to continue.\nOr press CTRL+C to quit.")"
  configlevel=5
}

prompt_file_to_backup() {
  printf "\n\n${color_normal}File to Backup"
  while true
  do
    printf "\nWhat is the name of the file to be monitored for backup? Do not include the folder/path."
    printf "\nCurrent value: ${color_currentvalue}${FILETOBACKUP}${color_normal}"
    read -p "$(printf "\nProvide a new value, or simply press return to keep the current value ${color_newvalue}")" answer
    printf "${color_normal}"
    if [ -n "$answer" ]; then
      FILETOBACKUP=${answer}
    fi
    configlevel=6
    break
  done
}

prompt_source_folder() {
  printf "\n\n${color_normal}Source Folder"
  while true
  do
    printf "\nWhat is the full path to the folder containing the file to be monitored for backup?"
    printf "\nCurrent value: ${color_currentvalue}${SOURCEFOLDER}${color_normal}"
    read -p "$(printf "\nProvide a new value, or simply press return to keep the current value ${color_newvalue}")" answer
    printf "${color_normal}"
    if [ -n "$answer" ]; then
      SOURCEFOLDER=${answer}
    fi

    if [ ! -d ${SOURCEFOLDER} ]; then
      printf "\nDirectory ${SOURCEFOLDER} does not exist."
      read -p "Create it? [Yes or No] " answer
      printf "${color_normal}"
      if [ -n "$answer" ]; then
        answer=$(echo ${answer:0:1} | tr '[:upper:]' '[:lower:]')
        if [[ $answer == "y" ]]; then
          mkdir -p ${SOURCEFOLDER}
          configlevel=7
          break
        fi
      fi
    else
      configlevel=7
      break
    fi
  done
}

prompt_local_backup_folder() {
  printf "\n\n${color_normal}Local Backup Folder"
  while true
  do
    printf "\nWhere should the file be backed up to on the local server?"
    printf "\nWhen a backup is made, an MD5 hash file will also be created in this folder"
    printf "\nCurrent value: ${color_currentvalue}${LOCALBACKUPFOLDER}${color_normal}"
    read -p "$(printf "\nProvide a new value, or simply press return to keep the current value ${color_newvalue}")" answer
    printf "${color_normal}"
    if [ -n "$answer" ]; then
      LOCALBACKUPFOLDER=${answer}
    fi
    if [ ! -d ${LOCALBACKUPFOLDER} ]; then
      printf "\nDirectory ${LOCALBACKUPFOLDER} does not exist."
      read -p "Create it? [Yes or No] " answer
      printf "${color_normal}"
      if [ -n "$answer" ]; then
        answer=$(echo ${answer:0:1} | tr '[:upper:]' '[:lower:]')
        if [[ $answer == "y" ]]; then
          mkdir -p ${LOCALBACKUPFOLDER}
          configlevel=10
          break
        fi
      fi
    else
      configlevel=10
      break
    fi
  done
}

prompt_remote_server() {
  printf "\n\n${color_normal}Remote Server"
  while true
  do
    printf "\nWhat is the server address (either ip address for local network, or fully qualified domain name)"
    printf "\nCurrent value: ${color_currentvalue}${REMOTESERVER}${color_normal}"
    read -p "$(printf "\nProvide a new value, or simply press return to keep the current value ${color_newvalue}")" answer
    printf "${color_normal}"
    if [ -n "$answer" ]; then
      REMOTESERVER=${answer}
    fi
    configlevel=11
    break
  done
}

prompt_remote_folder() {
  printf "\n\n${color_normal}Remote Folder"
  while true
  do
    printf "\nWhich folder on the remote server should be the base directory for backups?"
    printf "\nCurrent value: ${color_currentvalue}${REMOTEBACKUPFOLDER}${color_normal}"
    read -p "$(printf "\nProvide a new value, or simply press return to keep the current value ${color_newvalue}")" answer
    printf "${color_normal}"
    if [ -n "$answer" ]; then
      REMOTEBACKUPFOLDER=${answer}
    fi
    configlevel=12
    break
  done
}

prompt_remote_user() {
  printf "\n\n${color_normal}Remote User"
  while true
  do
    printf "\nWhat is the username for logging into the server"
    printf "\nCurrent value: ${color_currentvalue}${REMOTEUSER}${color_normal}"
    read -p "$(printf "\nProvide a new value, or simply press return to keep the current value ${color_newvalue}")" answer
    printf "${color_normal}"
    if [ -n "$answer" ]; then
      REMOTEUSER=${answer}
    fi
    configlevel=20
    break
  done
}

prompt_rsa_pubkeypath() {
  printf "\n\n${color_normal}RSA Public Key"
  while true
  do
    printf "\nWhat is the path of the RSA public key file created with ssh-keygen"
    printf "\nCurrent value: ${color_currentvalue}${RSAPUBKEYPATH}${color_normal}"
    read -p "$(printf "\nProvide a new value, or simply press return to keep the current value ${color_newvalue}")" answer
    printf "${color_normal}"
    if [ -n "$answer" ]; then
      RSAPUBKEYPATH=${answer}
    fi
    configlevel=30
    break
  done
}

prompt_device_identity() {
  printf "\n\n${color_normal}Device Identity"
  while true
  do
    printf "\nYou can use the same script for backing up from multiple nodes but each should have a unique identifier."
    printf "\nA subfolder for the node identifier will be created under the remote folder."
    printf "\nIts recommended to use the alias for your node for this value so you can distinguish them."
    printf "\nCurrent value: ${color_currentvalue}${DEVICE}${color_normal}"
    read -p "$(printf "\nProvide a new value, or simply press return to keep the current value ${color_newvalue}")" answer
    printf "${color_normal}"
    if [ -n "$answer" ]; then
      DEVICE=${answer}
    fi
    configlevel=90
    break
  done
}

config_summary() {
  printf "${color_normal}"
  printf "\n\nConfiguration Summary"
  printf "\nRemote Server: ${color_current_value}${REMOTESERVER}${color_normal}"
  printf "\nRemote User: ${color_current_value}${REMOTEUSER}${color_normal}"
  printf "\nRemote Folder: ${color_current_value}${REMOTEBACKUPFOLDER}${color_normal}"
  printf "\nRSA Public Key Path: ${color_current_value}${RSAPUBKEYPATH}${color_normal}"
  printf "\nDevice Identity: ${color_current_value}${DEVICE}${color_normal}"
  printf "\nFile name to watch: ${color_current_value}${FILETOBACKUP}${color_normal}"
  printf "\nSource Folder: ${color_current_value}${SOURCEFOLDER}${color_normal}"
  printf "\nLocal Backup Folder: ${color_current_value}${LOCALBACKUPFOLDER}${color_normal}"
  configlevel=91
}

get_config_filename() {
  configfilename=`basename $0`
  configfilename="${configfilename}.json"
}

config_save() {
  printf "${color_normal}"
  get_config_filename
  printf "\n\nSaving updated configuration to ${configfilename}\n"
cat >${configfilename} <<EOF
{
  "remote": {
    "server": "${REMOTESERVER}",
    "user": "${REMOTEUSER}",
    "folder": "${REMOTEBACKUPFOLDER}"
  },
  "rsapubkeypath": "${RSAPUBKEYPATH}",
  "device": "${DEVICE}",
  "file": "${FILETOBACKUP}",
  "source_folder": "${SOURCEFOLDER}",
  "local_backup_folder": "${LOCALBACKUPFOLDER}"
}
EOF
}

prompt_save() {
  printf "${color_normal}"
  printf "\n\n"
  while true
  do
    read -p "Save Configuration? [Yes or No] " answer
    printf "${color_normal}"
    if [ -n "$answer" ]; then
      answer=$(echo ${answer:0:1} | tr '[:upper:]' '[:lower:]')
      if [[ $answer == "y" ]]; then
        config_save
        break
      fi
      if [[ $answer == "n" ]]; then
        break
      fi
    fi
  done
  configlevel=92
}

run_howto() {
  scriptname=`basename $0`
  scriptsanex=${scriptname}
  scriptsanex=`echo ${scriptname}|sed "s/.sh//"`
  pwd=`pwd`
  printf "\nTo run this script with the config, simply start it with"
  printf "\n  ${color_command}${scriptname} doloop${color_normal}\n"
  printf "\nTo run as a service, setup a systemd script "
  printf "\n  ${color_command}sudo nano /etc/systemd/system/${scriptsanex}.service${color_normal}"
  printf "\nAnd use contents as follows:"
  printf "\n${color_content}[Service]\nWorkingDirectory=${pwd}\nExecStart=/bin/sh -c '${pwd}/${scriptname} doloop'\nRestart=always\nRestartSec=1\nStandardOutput=syslog\nStandardError=syslog\nSyslogIdentifer=my-file-backup\nUser=bitcoin\nGroup=bitcoin\n\n[Install]WantedBy=multi-user.target${color_normal}\n"
  printf "\nEnable and start the service as follows"
  printf "\n  ${color_command}sudo systemctl enable ${scriptsanex}.service${color_normal}"
  printf "\n  ${color_command}sudo systemctl start ${scriptsanex}.service${color_normal}"
  printf "\nReviewing logs (press CTRL+C to stop)"
  printf "\n  ${color_command}sudo journalctl -fu ${scriptsanex}.service${color_normal}"
  printf "\nTest the backup"
  printf "\n  ${color_command}sudo touch ${SOURCEFILE}${color_normal}"
  configlevel=99
}

config_done() {
  printf "${color_normal}"
  printf "\n\nExiting\n"
  exit 0
}

config_require() {
  if [ -z "$2" ]; then
    printf "\nThe value for $1 is empty. Please run configuration.\n"
    exit 1
  fi
  if [ $2 == $value_not_set ]; then
    printf "\nThe value for $1 is ${value_not_set}. Please run configuration.\n"
    exit 1
  fi
  if [[ $2 == "null" ]]; then
    printf "\nThe value for $1 is null. Please run configuration.\n"
    exit 1
  fi
}

config_read_value() {
  # arg 1 = current value
  # arg 2 = jq path
  get_config_filename
  config_value=$(cat ${configfilename} | jq -r ${2})
  if [[ $config_value == "null" ]]; then
    config_value=$1
  fi
}

config_load() {
  printf "${color_normal}"
  get_config_filename
  printf "\nLoading configuration from ${configfilename}\n"
  # check that file exists
  if [ ! -f "${configfilename}" ]; then
    # if running service, then fail when no configuration
    if [ "$1" == "validate" ]; then
      printf "\nConfiguration file not found. Unable to run service loop.\n"
      exit 1
    fi
  else
    # read in values
    config_read_value "${REMOTESERVER}" ".remote.server"
    REMOTESERVER=${config_value}
    config_read_value "${REMOTEUSER}" ".remote.user"
    REMOTEUSER=${config_value}
    config_read_value ${REMOTEBACKUPFOLDER} ".remote.folder"
    REMOTEBACKUPFOLDER=${config_value}
    config_read_value ${RSAPUBKEYPATH} ".rsapubkeypath"
    RSAPUBKEYPATH=${config_value}
    config_read_value ${DEVICE} ".device"
    DEVICE=${config_value}
    config_read_value ${FILETOBACKUP} ".file"
    FILETOBACKUP=${config_value}
    config_read_value ${SOURCEFOLDER} ".source_folder"
    SOURCEFOLDER=${config_value}
    config_read_value ${LOCALBACKUPFOLDER} ".local_backup_folder"
    LOCALBACKUPFOLDER=${config_value}
    # validation - ensure required fields are set
    if [ "$1" == "validate" ]; then
      config_require "Remote Server" ${REMOTESERVER}
      config_require "Remote User" ${REMOTEUSER}
      config_require "Remote Backup Folder" ${REMOTEBACKUPFOLDER}
      config_require "RSA Public Key Path" ${RSAPUBKEYPATH}
      config_require "Device Identity" ${DEVICE}
      config_require "File to Backup" ${FILETOBACKUP}
      config_require "Source Folder" ${SOURCEFOLDER}
      config_require "Local Backup Folder" ${LOCALBACKUPFOLDER}
    fi
    # composite variables
    if [[ ! ${SOURCEFOLDER} == */ ]]; then
        SOURCEFOLDER="${SOURCEFOLDER}/"
    fi
    SOURCEFILE=${SOURCEFOLDER}${FILETOBACKUP}
    if [[ ! ${LOCALBACKUPFOLDER} == */ ]]; then
        LOCALBACKUPFOLDER="${LOCALBACKUPFOLDER}/"
    fi
    LOCALBACKUPFILE=${LOCALBACKUPFOLDER}${FILETOBACKUP}
  fi
}

configure() {
  printf "\n${color_normal}Configuring Remote Backup of File ${FILETOBACKUP}\n"
  config_load "ok"
  while true
  do
    case $configlevel in
      0)
        guidance ;;
      5)
        prompt_file_to_backup ;;
      6)
        prompt_source_folder ;;
      7)
        prompt_local_backup_folder ;;
      10)
        prompt_remote_server ;;
      11)
        prompt_remote_folder ;;
      12)
        prompt_remote_user ;;
      20)
        prompt_rsa_pubkeypath ;;
      30)
        prompt_device_identity ;;
      90)
        config_summary ;;
      91)
        prompt_save ;;
      92)
        run_howto ;;
      99)
        config_done ;;
      *)
        configlevel=0
    esac
  done
}

# =============================================================================
# Main starting point
# =============================================================================
# At the top of the script some initial variables were declared.  Now lets
# check if we are in configuration mode (no arguments), run mode (passed the
# value 'doloop' as first argument, or other (some other argument value was
# given). .

# Run loop or configure
if [ -z "$1" ]; then
  configure
else
  if [ "$1" == "doloop" ]; then
    service_loop
  else
    printf "${color_normal}"
    printf "\nTo configure settings, run this script without any command line arguments"
    printf "\nTo run the service, the first argument after the scriptname should be 'doloop'"
    printf "\n"
    exit 1
  fi
fi

Run with no arguments to enter configuration mode.

Introduction starts

Get prompts for each field. Accept existing value or default, or assign new value

A summary is presented for what will be saved as configuration file

And finally closing instructions for how to run and create service