Skip to content

Instantly share code, notes, and snippets.

@vicariousdrama
Last active December 18, 2022 09:30
Show Gist options
  • Save vicariousdrama/78c68ba7827eb6a16583c9df59134e0f to your computer and use it in GitHub Desktop.
Save vicariousdrama/78c68ba7827eb6a16583c9df59134e0f to your computer and use it in GitHub Desktop.
Monitor a file for changes and backup to WebDAV instance (Tested with Nextcloud)
#!/bin/bash
# To configure settings for this script, run the script directly at the command
# line. A configuration file will be created as the same name as this script
# with a .json extension that will contain the values that will be used at
# runtime.
# As with any script, you should review it before running it. The main entry
# point is at the end of the file after variable initializations and function
# declarations are made.
# =============================================================================
# Default values (these get overidden by configuration file)
# File to watch for backup, source and local backup paths
FILETOBACKUP="channel.backup"
SOURCEFOLDER="/home/bitcoin/.lnd/data/chain/bitcoin/mainnet/"
LOCALBACKUPFOLDER="/tmp/"
# This is where the file backup should be scp'd to and must be configured in
# the json file
WEBDAVBASEURL="https://10.10.1.21/remote.php/dav/files/someuser/path/"
WEBDAVUSER="someuser"
WEBDAVPASSWORD="password"
# Identify this device (useful if you are running this on multiple nodes)
DEVICE=$(hostname)
# Color definitions for convenience
color_red='\033[0;31m'
color_green='\033[0;32m'
color_yellow='\033[0;33m'
color_blue='\033[0;34m'
color_purple='\033[0;35m'
color_cyan='\033[0;36m'
color_white='\033[0;37m'
color_normal=${color_white}
color_currentvalue=${color_purple}
color_newvalue=${color_yellow}
color_link=${color_green}
color_command=${color_cyan}
color_content=${color_red}
# This variable just tracks where we are in config process
configlevel=0 #92
# These track which folders we worked with
OLDDEVICE=""
OLDYEAR=""
OLDMONTH=""
# =============================================================================
# Declare some functions to make this logically easier to read through
# =============================================================================
setup_backup_filename() {
bfe=true
bfc=0
while $bfe
do
# get a time stamp in seconds since epoch
NEWFILEDATE=`date +%s`
YEAR=`date +%Y`
MONTH=`date +%m`
# determine backup file name
BACKUPFILE="${LOCALBACKUPFILE}.${NEWFILEDATE}"
# make sure the backup filename doesnt already exist
if [ ! -f "${BACKUPFILE}" ]; then
bfe=false
else
bfc=$((bfc++))
if [ $bfc -ge 5 ]; then
bfe=false
break
fi
sleep 0.1
fi
done
}
backup_to_remote() {
# file to backup is in ${1} arg passed in
# filename itself is in ${2} arg passed in
REMOTEPATH="${WEBDAVBASEURL}${DEVICE}/${YEAR}/${MONTH}/${2}"
printf "\nUploading ${2} to ${REMOTEPATH}"
curl -T ${1} -u ${WEBDAVUSER}:${WEBDAVPASSWORD} ${REMOTEPATH}
}
make_remote_folder() {
cmdtodo="curl --silent --write-out '%{http_code}' --output /dev/null --head -X PROPFIND -u ${WEBDAVUSER}:${WEBDAVPASSWORD} ${1}"
response_code=`eval ${cmdtodo}`
if [[ ${response_code} == "'000'" ]]; then
printf "\nMalformed request while checking for folder resulted in 000 response code"
fi
if [[ ${response_code} == "404" ]]; then
printf "\nCreating folder for ${1}"
curl --silent -X MKCOL --output /dev/null -u ${WEBDAVUSER}:${WEBDAVPASSWORD} ${1}
fi
}
make_remote_folders() {
# Note: WEBDAVBASEURL must always exist
# Create device folder if not exists
if [[ "${OLDDEVICE}" != "${DEVICE}" ]]; then
make_remote_folder "${WEBDAVBASEURL}${DEVICE}"
OLDDEVICE=${DEVICE}
fi
# Create year folder if not exists
if [[ "${OLDYEAR}" != "${YEAR}" ]]; then
make_remote_folder "${WEBDAVBASEURL}${DEVICE}/${YEAR}"
OLDYEAR=${YEAR}
fi
# Create month folder if not exists
if [[ "${OLDMONTH}" != "${MONTH}" ]]; then
make_remote_folder "${WEBDAVBASEURL}${DEVICE}/${YEAR}/${MONTH}"
OLDMONTH=${MONTH}
fi
}
service_loop() {
config_load "validate"
three_hours=10800
while true; do
# wait for change to source file
inotifywait --timeout $three_hours $SOURCEFILE
ec=$(echo $?)
if [ $ec -eq 0 ]; then
printf "\nFile change detected!"
# file changed
setup_backup_filename
MD5SUMFILE="${BACKUPFILE}.md5"
# copy source to backup and get md5 sum file
printf "\nCopying from source to local backup file ${BACKUPFILE}"
cp $SOURCEFILE $BACKUPFILE
printf "\nPreparing MD5 file"
md5sum $BACKUPFILE > $MD5SUMFILE
sed -i 's/\/.*\///g' $MD5SUMFILE
# backup to remote
make_remote_folders
backup_to_remote $BACKUPFILE ${FILETOBACKUP}.${NEWFILEDATE}
backup_to_remote $MD5SUMFILE ${FILETOBACKUP}.${NEWFILEDATE}.md5
printf "\nok.\n"
fi
done
}
guidance() {
printf "\nThis script is intended to watch a single file (${color_currentvalue}${FILETOBACKUP}${color_normal}) and "
printf "\nwhenever it changes, backup to a local folder, as well as to a remote server via"
printf "\nWebDAV. Backups to the remote server are timestamped. You must have credentials"
printf "\nfor the WebDAV server, and have created the base path included in the URL to it."
printf "\nIntermediate folders for the device (node identifier), and year and month will"
printf "\nbe created automatically at run time if they do not exist.\n"
printf "\nThis script has been tested for use with Nextcloud instances and leverages the "
printf "\nMKCOL and PROPFIND methods.\n"
read -p "$(printf "\nWhen ready to complete configuration, press enter to continue.\nOr press CTRL+C to quit.")"
configlevel=5
}
prompt_file_to_backup() {
printf "\n\n${color_normal}File to Backup"
while true
do
printf "\nWhat is the name of the file to be monitored for backup? Do not include the folder/path."
printf "\nCurrent value: ${color_currentvalue}${FILETOBACKUP}${color_normal}"
read -p "$(printf "\nProvide a new value, or simply press return to keep the current value ${color_newvalue}")" answer
printf "${color_normal}"
if [ -n "$answer" ]; then
FILETOBACKUP=${answer}
fi
configlevel=6
break
done
}
prompt_source_folder() {
printf "\n\n${color_normal}Source Folder"
while true
do
printf "\nWhat is the full path to the folder containing the file to be monitored for backup?"
printf "\nCurrent value: ${color_currentvalue}${SOURCEFOLDER}${color_normal}"
read -p "$(printf "\nProvide a new value, or simply press return to keep the current value ${color_newvalue}")" answer
printf "${color_normal}"
if [ -n "$answer" ]; then
SOURCEFOLDER=${answer}
fi
if [ ! -d ${SOURCEFOLDER} ]; then
printf "\nDirectory ${SOURCEFOLDER} does not exist."
read -p "Create it? [Yes or No] " answer
printf "${color_normal}"
if [ -n "$answer" ]; then
answer=$(echo ${answer:0:1} | tr '[:upper:]' '[:lower:]')
if [[ $answer == "y" ]]; then
mkdir -p ${SOURCEFOLDER}
SOURCEFILE=${SOURCEFOLDER}${FILETOBACKUP}
configlevel=7
break
fi
fi
else
SOURCEFILE=${SOURCEFOLDER}${FILETOBACKUP}
configlevel=7
break
fi
done
}
prompt_local_backup_folder() {
printf "\n\n${color_normal}Local Backup Folder"
while true
do
printf "\nWhere should the file be backed up to on the local server?"
printf "\nWhen a backup is made, an MD5 hash file will also be created in this folder"
printf "\nCurrent value: ${color_currentvalue}${LOCALBACKUPFOLDER}${color_normal}"
read -p "$(printf "\nProvide a new value, or simply press return to keep the current value ${color_newvalue}")" answer
printf "${color_normal}"
if [ -n "$answer" ]; then
LOCALBACKUPFOLDER=${answer}
fi
if [ ! -d ${LOCALBACKUPFOLDER} ]; then
printf "\nDirectory ${LOCALBACKUPFOLDER} does not exist."
read -p "Create it? [Yes or No] " answer
printf "${color_normal}"
if [ -n "$answer" ]; then
answer=$(echo ${answer:0:1} | tr '[:upper:]' '[:lower:]')
if [[ $answer == "y" ]]; then
mkdir -p ${LOCALBACKUPFOLDER}
LOCALBACKUPFILE=${LOCALBACKUPFOLDER}${FILETOBACKUP}
configlevel=10
break
fi
fi
else
LOCALBACKUPFILE=${LOCALBACKUPFOLDER}${FILETOBACKUP}
configlevel=10
break
fi
done
}
prompt_webdav_baseurl() {
printf "\n\n${color_normal}WevDAV Base URL"
while true
do
printf "\nWhat is the base url for the WebDAV server for which you want to store backups?"
printf "\nYou should include the FQDN, webdav path, user info, and user path"
printf "\nCurrent value: ${color_currentvalue}${WEBDAVBASEURL}${color_normal}"
read -p "$(printf "\nProvide a new value, or simply press return to keep the current value ${color_newvalue}")" answer
printf "${color_normal}"
if [ -n "$answer" ]; then
WEBDAVBASEURL=${answer}
fi
configlevel=11
break
done
}
prompt_webdav_user() {
printf "\n\n${color_normal}WebDAV User"
while true
do
printf "\nWhat is the username for logging into the WebDAV instance"
printf "\nCurrent value: ${color_currentvalue}${WEBDAVUSER}${color_normal}"
read -p "$(printf "\nProvide a new value, or simply press return to keep the current value ${color_newvalue}")" answer
printf "${color_normal}"
if [ -n "$answer" ]; then
WEBDAVUSER=${answer}
fi
configlevel=12
break
done
}
prompt_webdav_password() {
printf "\n\n${color_normal}WebDAV Password"
while true
do
printf "\nWhat is the password for logging into the WebDAV instance"
printf "\nCurrent value: ${color_currentvalue}${WEBDAVPASSWORD}${color_normal}"
read -p "$(printf "\nProvide a new value, or simply press return to keep the current value ${color_newvalue}")" answer
printf "${color_normal}"
if [ -n "$answer" ]; then
WEBDAVPASSWORD=${answer}
fi
configlevel=30
break
done
}
prompt_device_identity() {
printf "\n\n${color_normal}Device Identity"
while true
do
printf "\nYou can use the same script for backing up from multiple nodes but each should have a unique identifier."
printf "\nA subfolder for the node identifier will be created under the remote folder."
printf "\nIts recommended to use the alias for your node for this value so you can distinguish them."
printf "\nCurrent value: ${color_currentvalue}${DEVICE}${color_normal}"
read -p "$(printf "\nProvide a new value, or simply press return to keep the current value ${color_newvalue}")" answer
printf "${color_normal}"
if [ -n "$answer" ]; then
DEVICE=${answer}
fi
configlevel=90
break
done
}
config_summary() {
printf "${color_normal}"
printf "\n\nConfiguration Summary"
printf "\nWebDAV Base URL: ${color_current_value}${WEBDAVBASEURL}${color_normal}"
printf "\nWebDAV User: ${color_current_value}${WEBDAVUSER}${color_normal}"
printf "\nWebDAV Password: ${color_current_value}${WEBDAVPASSWORD}${color_normal}"
printf "\nDevice Identity: ${color_current_value}${DEVICE}${color_normal}"
printf "\nFile name to watch: ${color_current_value}${FILETOBACKUP}${color_normal}"
printf "\nSource Folder: ${color_current_value}${SOURCEFOLDER}${color_normal}"
printf "\nLocal Backup Folder: ${color_current_value}${LOCALBACKUPFOLDER}${color_normal}"
configlevel=91
}
get_config_filename() {
configfilename=`basename $0`
configfilename="${configfilename}.json"
}
config_save() {
printf "${color_normal}"
get_config_filename
printf "\n\nSaving updated configuration to ${configfilename}\n"
cat >${configfilename} <<EOF
{
"webdav": {
"baseurl": "${WEBDAVBASEURL}",
"user": "${WEBDAVUSER}",
"password": "${WEBDAVPASSWORD}"
},
"device": "${DEVICE}",
"file": "${FILETOBACKUP}",
"source_folder": "${SOURCEFOLDER}",
"local_backup_folder": "${LOCALBACKUPFOLDER}"
}
EOF
}
prompt_save() {
printf "${color_normal}"
printf "\n\n"
while true
do
read -p "Save Configuration? [Yes or No] " answer
printf "${color_normal}"
if [ -n "$answer" ]; then
answer=$(echo ${answer:0:1} | tr '[:upper:]' '[:lower:]')
if [[ $answer == "y" ]]; then
config_save
break
fi
if [[ $answer == "n" ]]; then
break
fi
fi
done
configlevel=92
}
run_howto() {
scriptname=`basename $0`
scriptsanex=${scriptname}
scriptsanex=`echo ${scriptname}|sed "s/.sh//"`
pwd=`pwd`
printf "\nTo run this script with the config, simply start it with"
printf "\n ${color_command}${scriptname} doloop${color_normal}\n"
printf "\nTo run as a service, setup a systemd script "
printf "\n ${color_command}sudo nano /etc/systemd/system/${scriptsanex}.service${color_normal}"
printf "\nAnd use contents as follows:"
printf "\n${color_content}[Service]\nWorkingDirectory=${pwd}\nExecStart=/bin/sh -c '${pwd}/${scriptname} doloop'\nRestart=always\nRestartSec=1\nStandardOutput=syslog\nStandardError=syslog\nSyslogIdentifer=my-file-backup\nUser=bitcoin\nGroup=bitcoin\n\n[Install]\nWantedBy=multi-user.target${color_normal}\n"
printf "\nEnable and start the service as follows"
printf "\n ${color_command}sudo systemctl enable ${scriptsanex}.service${color_normal}"
printf "\n ${color_command}sudo systemctl start ${scriptsanex}.service${color_normal}"
printf "\nReviewing logs (press CTRL+C to stop)"
printf "\n ${color_command}sudo journalctl -fu ${scriptsanex}.service${color_normal}"
printf "\nTest the backup"
printf "\n ${color_command}sudo touch ${SOURCEFILE}${color_normal}"
configlevel=99
}
config_done() {
printf "${color_normal}"
printf "\n\nExiting\n"
exit 0
}
config_require() {
if [ -z "$2" ]; then
printf "\nThe value for $1 is empty. Please run configuration.\n"
exit 1
fi
if [[ $2 == "null" ]]; then
printf "\nThe value for $1 is null. Please run configuration.\n"
exit 1
fi
}
config_read_value() {
# arg 1 = current value
# arg 2 = jq path
get_config_filename
config_value=$(cat ${configfilename} | jq -r ${2})
if [[ $config_value == "null" ]]; then
config_value=$1
fi
}
config_load() {
printf "${color_normal}"
get_config_filename
printf "\nLoading configuration from ${configfilename}\n"
# check that file exists
if [ ! -f "${configfilename}" ]; then
# if running service, then fail when no configuration
if [ "$1" == "validate" ]; then
printf "\nConfiguration file not found. Unable to run service loop.\n"
exit 1
fi
else
# read in values
config_read_value "${WEBDAVBASEURL}" ".webdav.baseurl"
WEBDAVBASEURL=${config_value}
config_read_value "${WEBDAVUSER}" ".webdav.user"
WEBDAVUSER=${config_value}
config_read_value ${WEBDAVPASSWORD} ".webdav.password"
WEBDAVPASSWORD=${config_value}
config_read_value ${DEVICE} ".device"
DEVICE=${config_value}
config_read_value ${FILETOBACKUP} ".file"
FILETOBACKUP=${config_value}
config_read_value ${SOURCEFOLDER} ".source_folder"
SOURCEFOLDER=${config_value}
config_read_value ${LOCALBACKUPFOLDER} ".local_backup_folder"
LOCALBACKUPFOLDER=${config_value}
# validation - ensure required fields are set
if [ "$1" == "validate" ]; then
config_require "WebDAV Base URL" ${WEBDAVBASEURL}
config_require "WebDAV User" ${WEBDAVUSER}
config_require "WebDAV Password" ${WEBDAVPASSWORD}
config_require "Device Identity" ${DEVICE}
config_require "File to Backup" ${FILETOBACKUP}
config_require "Source Folder" ${SOURCEFOLDER}
config_require "Local Backup Folder" ${LOCALBACKUPFOLDER}
fi
# composite variables
if [[ ! ${SOURCEFOLDER} == */ ]]; then
SOURCEFOLDER="${SOURCEFOLDER}/"
fi
SOURCEFILE=${SOURCEFOLDER}${FILETOBACKUP}
if [[ ! ${LOCALBACKUPFOLDER} == */ ]]; then
LOCALBACKUPFOLDER="${LOCALBACKUPFOLDER}/"
fi
LOCALBACKUPFILE=${LOCALBACKUPFOLDER}${FILETOBACKUP}
fi
}
configure() {
printf "\n${color_normal}Configuring Remote Backup of File ${FILETOBACKUP}\n"
config_load "ok"
while true
do
case $configlevel in
0)
guidance ;;
5)
prompt_file_to_backup ;;
6)
prompt_source_folder ;;
7)
prompt_local_backup_folder ;;
10)
prompt_webdav_baseurl ;;
11)
prompt_webdav_user ;;
12)
prompt_webdav_password ;;
30)
prompt_device_identity ;;
90)
config_summary ;;
91)
prompt_save ;;
92)
run_howto ;;
99)
config_done ;;
*)
configlevel=0
esac
done
}
# =============================================================================
# Main starting point
# =============================================================================
# At the top of the script some initial variables were declared. Now lets
# check if we are in configuration mode (no arguments), run mode (passed the
# value 'doloop' as first argument, or other (some other argument value was
# given). .
# Run loop or configure
if [ -z "$1" ]; then
configure
else
if [ "$1" == "doloop" ]; then
service_loop
else
printf "${color_normal}"
printf "\nTo configure settings, run this script without any command line arguments"
printf "\nTo run the service, the first argument after the scriptname should be 'doloop'"
printf "\n"
exit 1
fi
fi
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment