Skip to content

Instantly share code, notes, and snippets.

@wrouesnel
Last active September 5, 2021 23:32
Show Gist options
  • Save wrouesnel/8f0c681e4bf598176203 to your computer and use it in GitHub Desktop.
Save wrouesnel/8f0c681e4bf598176203 to your computer and use it in GitHub Desktop.
A script for performing a remote backup to another server using rsync and bup for versioning.
# this is an rsync-excludes format list of files to exclude.
# This command is included by do-remote-backup.bsh to set parameters.
BACKUPNAME="name_to_save_as_in_bup"
# Backup set (array of root folders we backup
BACKUPSET=( "directories" "tobackup" "in" "the" "root" "directory" )
BACKUPROOT="/cygdrive/c"
# Backup destination settings
REMOTEUSER="backup"
REMOTEHOST="backup-server"
REMOTEROOT="backup"
REMOTETARSTORAGE="."
# File-transfer retry settings
MAX_RETRIES=10
# Email Settings
MAILTO=("your@email.com")
SMTPSERVER="smtp.gmail.com"
SMTPPORT="587"
SMTPFROM="host@provider.com"
SMTPUSER="host account"
SMTPPASSWORD="host account password"
# minimum time between sending notification emails of backups completed (in seconds)
# 43200 = 12 hours
MAILFREQUENCY=43200
# skip doing tar archive creation for failed snapshots (not recommended!)
SKIPRECOVERY=1
#!/bin/bash
# Script executed by a daily cron job in order to remotely backup med. This
# script must run as root so it has sufficient privileges to create ZFS
# snapshots.
#
# Method of operation:
# Windows Scheduler (or cron) initiates this job by running the vscsc.exe
# application with this scipt as the target for a Cygwin instance.
#
# The script then runs rsync and copies files to the remote server target using
# rsync over SSH.
#
# After rsync is complete, it connects to the remote host with SSH and invokes
# bup to make a snapshot version of the backed up files. This duel-action
# ensures we always have a copy of the data outside of the bup-archive.
#
# What we backup:
# c:\med
#
# System Settings
VERBOSE=1
LOGFILE=$(mktemp)
STATSFILE=$(mktemp)
PIDFILE=/var/run/remote-backup.pid
REMOTEFLAGFILE=.backup-in-progress
SNAPSHOTFLAGFILE=.last-bup-save-result
RETRIES=0
RSYNCEXCLUDES=.remote-backup-excludes.txt
. .remote-backup-settingsrc
## Function declarations ##
# Non-stdout echo command
echoerr ()
{
echo $@ 1>&2
}
# Log file output command
log ()
{
if [[ $VERBOSE == 1 ]]; then
echoerr "VERBOSE: $@"
fi
if [ ! -z $LOGFILE ]; then
echo $@ >> $LOGFILE
fi
}
# Function that packs up the logs and emails them.
# Depends on the excellent cygwin email package (should for Unix).
# Usage: sendmail <headerline>
sendmail()
{
cat << EOF | email -from-name "$(hostname)" \
-from-addr "$SMTPFROM" \
-smtp-server "$SMTPSERVER" \
-smtp-port $SMTPPORT \
-smtp-auth login \
-smtp-user "$SMTPUSER" \
-smtp-pass "$SMTPPASSWORD" \
-tls \
-subject "$BACKUPNAME Remote Backup: $1" \
$(echo ${MAILTO[@]} | tr ' ' ',')
$BACKUPNAME Remote Backup Report
========================
Backup Statistics
-----------------
$(cat "$STATSFILE")
Session Log File
----------------
$(cat "$LOGFILE")
EOF
}
# Fatal error that should terminate the script
# Usage: fatal <exit code> <error string>
fatal ()
{
echoerr "${@:2}"
# Always try to sendmail before failing.
sendmail "FATAL - ${@:2}"
exit $1
}
declare -a on_exit_handlers
on_exit()
{
for i in "${on_exit_handlers[@]}"
do
log "Running Exit Handler: $i"
eval "$i"
done
}
add_exit_handler()
{
local n=${#on_exit_handlers[*]}
on_exit_handlers[$n]="$*"
if [[ $n -eq 0 ]]; then
trap on_exit EXIT
fi
}
########################
## Script starts here ##
########################
# Trap for killing the log file
add_exit_handler "rm -f $LOGFILE"
add_exit_handler "rm -r $STATSFILE"
log "Backup started on $(date)"
# Check we are not already running (shouldn't be).
if [ -e $PIDFILE ]; then
PID=$(cat $PIDFILE)
if [ -e /proc/${PID} ]; then
log "Backup was already in progress when started."
log "Check backup job is not stalled."
fatal 1 "Another backup is already running"
fi
fi
# Create the pidfile (detects multiple attempted executions)
echo $$ > $PIDFILE
add_exit_handler "rm -f $PIDFILE"
# Check we have a volume to work with
if [ -z $1 ]; then
fatal 1 "No shadow copy volume supplied."
fi
# Shadow Copy devices in cygwin are easily accessible as regular drives under
# /proc/sys/Device/<shadowcopyvolume>. VSCSC gives us a windows label, from
# that we just need the name.
log "Shadow copy volume label: $1"
BACKUPROOT="/proc/sys/Device/$(echo $1 | cut -d'\' -f 5)"
log "Backup Root determined as: $BACKUPROOT"
# Verify the shadow copy volume exists
if [ ! -e "$BACKUPROOT" ]; then
log "Specified shadow copy volume not found!"
log "/proc/sys/Device had following volumes:"
log "$(ls /proc/sys/Device/Hard*)"
fatal 1 "Specified shadow copy volume not found."
fi
# Initial connection to server.
while [ 1 ]; do
# Read the time of the last backup - used at the bottom
LASTBACKUPTIME=$(ssh $REMOTEUSER@$REMOTEHOST cat "\"$REMOTEFLAGFILE\"" | sed -n 1p)
# And the count of the last backup - used at the bottom
BACKUPCOUNT=$(ssh $REMOTEUSER@$REMOTEHOST cat "\"$REMOTEFLAGFILE\"" | sed -n 2p)
# Last bup save result - we use this to avoid overwriting good data.
LASTSNAPSHOTRESULT=$(ssh $REMOTEUSER@$REMOTEHOST cat \"$SNAPSHOTFLAGFILE\" | sed -n 2p)
ssh $REMOTEUSER@$REMOTEHOST "echo -e \"$(date +%s)\n$BACKUPCOUNT\" > \"$REMOTEFLAGFILE\""
if [[ $? == 0 ]]; then
break;
elif [[ $RETRIES < $MAX_RETRIES ]]; then
let RETRIES+=1
else
fatal 2 "Maximum number of transfer retries exceeded on initial connection."
fi
done
if [[ $? != 0 ]]; then
fatal 1 "Failed initial connection to $REMOTEHOST"
fi
if [ -z $LASTBACKUPTIME ]; then
LASTBACKUPTIME=0
fi
if [ -z $BACKUPCOUNT ]; then
BACKUPCOUNT=0
fi
log "Last backup time: $LASTBACKUPTIME"
log "Backup count: $BACKUPCOUNT"
log "Last bup-save result: $LASTSNAPSHOTRESULT"
# if the last snapshot failed, make a tar archive on the remote server and
# tag it with the time. this is our failsafe behavior,
if [[ $SKIPRECOVERY != 1 ]]; then
if [[ $LASTSNAPSHOTRESULT != 0 ]]; then
while [ 1 ]; do
TARNAME="$BACKUPNAME-$(date +%Y-%m-%d-%H-%M-%S).tar.gz"
log "Last backup commit operation failed."
log "Creating a tar archive of last backup: $TARNAME"
ssh $REMOTEUSER@$REMOTEHOST "tar -zcf $REMOTETARSTORAGE/$TARNAME $REMOTEROOT/"
if [[ $? == 0 ]]; then
log "Created emergency tar archive for last backup. Proceeding."
break;
elif [[ $RETRIES < $MAX_RETRIES ]]; then
let RETRIES+=1
else
fatal 4 "Maximum number of transfer retries exceeded while creating emergency tar-archive. Aborting!"
fi
done
fi
fi
log "Backup in progress noted on remote server."
# Loop MAX_RETRIES times while trying to send backup set.
for dir in ${BACKUPSET[@]}; do
while [ 1 ]; do
log "Starting transfer of $dir"
echo "$BACKUPROOT/$dir Transfer Statistics:" >> $STATSFILE
rsync -rptlz -v \
--human-readable \
--delete \
--stats \
--exclude-from "$RSYNCEXCLUDES" \
-e "ssh" "$BACKUPROOT/./$dir" $REMOTEUSER@$REMOTEHOST:"$REMOTEROOT" | tee $STATSFILE
if [[ ${PIPESTATUS[0]} == 0 ]]; then
# Log behavior.
log "Finished transfer of $dir."
# Escape the loop.
break;
elif [[ $RETRIES < $MAX_RETRIES ]]; then
echo "$BACKUPROOT/$dir FAILED. Try $RETRIES" >> $LOGFILE
let RETRIES+=1
else
fatal 2 "Maximum number of transfer retries exceeded."
fi
done
done
log "Backup Completed."
# Log on to the remote server and do a backup with bup to maintain a version
# history.
log "Attempting to snapshot on remote server with bup..."
while [ 1 ]; do
if [[ $VERBOSE == 1 ]]; then
BUPSAVEVERBOSE="-vv"
fi
ssh -t -t $REMOTEUSER@$REMOTEHOST 2>&1 << EOF | tee -a $LOGFILE
bup index -um $REMOTEROOT
echo $? > $SNAPSHOTFLAGFILE
bup save -q $BUPSAVEVERBOSE --name $BACKUPNAME --strip-path $REMOTEROOT --tree --commit $REMOTEROOT
echo $? >> $SNAPSHOTFLAGFILE
exit
EOF
# ensure SSH succeeds
if [[ ${PIPESTATUS[0]} == 0 ]]; then
break;
elif [[ $RETRIES < $MAX_RETRIES ]]; then
let RETRIES+=1
else
fatal 2 "Maximum number of transfer retries exceeded while snapshotting."
fi
done
# ensure bup succeeded
BUPINDEXRESULT=$(ssh $REMOTEUSER@$REMOTEHOST "cat $SNAPSHOTFLAGFILE | sed -n 1p")
BUPSAVERESULT=$(ssh $REMOTEUSER@$REMOTEHOST "cat $SNAPSHOTFLAGFILE | sed -n 2p")
log "Bup index result: $BUPINDEXRESULT"
log "Bup save result: $BUPSAVERESULT"
if [[ $BUPINDEXRESULT != 0 ]]; then
fatal 3 "bup indexing operation failed."
fi
if [[ $BUPSAVERESULT != 0 ]]; then
fatal 3 "bup snapshotting operation failed."
fi
# tag the new backup count into the flag file
ssh $REMOTEUSER@$REMOTEHOST "echo $(($BACKUPCOUNT+1)) >> \"$REMOTEFLAGFILE\""
log "Backup Committed."
# Only mail logs if > MAILFREQUENCY has passed
if [[ $(($(date +%s) - $LASTBACKUPTIME)) > $MAILFREQUENCY ]]; then
log "Sending mail. $(($BACKUPCOUNT+1)) successes before this email. Elapsed time since last mail: $(($(date +%s) - $LASTBACKUPTIME))"
sendmail "Successful"
else
log "Not sending mail until cooldown period of $MAILFREQUENCY reached. At $(($(date +%s) - $LASTBACKUPTIME))"
fi
# Exit successfully (handlers will take care of temporary files)
exit 0
@echristopherson
Copy link

Are you planning on updating this for bup with the changes you indicated at http://blog.wrouesnel.com/articles/bup%20-%20towards%20the%20perfect%20backup/ integrated into it?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment