|
#!/bin/bash |
|
# Script executed by a daily cron job in order to remotely backup med. This |
|
# script must run as root so it has sufficient privileges to create ZFS |
|
# snapshots. |
|
# |
|
# Method of operation: |
|
# Windows Scheduler (or cron) initiates this job by running the vscsc.exe |
|
# application with this scipt as the target for a Cygwin instance. |
|
# |
|
# The script then runs rsync and copies files to the remote server target using |
|
# rsync over SSH. |
|
# |
|
# After rsync is complete, it connects to the remote host with SSH and invokes |
|
# bup to make a snapshot version of the backed up files. This duel-action |
|
# ensures we always have a copy of the data outside of the bup-archive. |
|
# |
|
# What we backup: |
|
# c:\med |
|
# |
|
|
|
# System Settings |
|
VERBOSE=1 |
|
LOGFILE=$(mktemp) |
|
STATSFILE=$(mktemp) |
|
PIDFILE=/var/run/remote-backup.pid |
|
|
|
REMOTEFLAGFILE=.backup-in-progress |
|
SNAPSHOTFLAGFILE=.last-bup-save-result |
|
|
|
RETRIES=0 |
|
|
|
RSYNCEXCLUDES=.remote-backup-excludes.txt |
|
|
|
. .remote-backup-settingsrc |
|
|
|
## Function declarations ## |
|
|
|
# Non-stdout echo command |
|
echoerr () |
|
{ |
|
echo $@ 1>&2 |
|
} |
|
|
|
# Log file output command |
|
log () |
|
{ |
|
if [[ $VERBOSE == 1 ]]; then |
|
echoerr "VERBOSE: $@" |
|
fi |
|
|
|
if [ ! -z $LOGFILE ]; then |
|
echo $@ >> $LOGFILE |
|
fi |
|
} |
|
|
|
# Function that packs up the logs and emails them. |
|
# Depends on the excellent cygwin email package (should for Unix). |
|
# Usage: sendmail <headerline> |
|
sendmail() |
|
{ |
|
cat << EOF | email -from-name "$(hostname)" \ |
|
-from-addr "$SMTPFROM" \ |
|
-smtp-server "$SMTPSERVER" \ |
|
-smtp-port $SMTPPORT \ |
|
-smtp-auth login \ |
|
-smtp-user "$SMTPUSER" \ |
|
-smtp-pass "$SMTPPASSWORD" \ |
|
-tls \ |
|
-subject "$BACKUPNAME Remote Backup: $1" \ |
|
$(echo ${MAILTO[@]} | tr ' ' ',') |
|
$BACKUPNAME Remote Backup Report |
|
======================== |
|
|
|
Backup Statistics |
|
----------------- |
|
$(cat "$STATSFILE") |
|
|
|
Session Log File |
|
---------------- |
|
$(cat "$LOGFILE") |
|
|
|
EOF |
|
} |
|
|
|
# Fatal error that should terminate the script |
|
# Usage: fatal <exit code> <error string> |
|
fatal () |
|
{ |
|
echoerr "${@:2}" |
|
|
|
# Always try to sendmail before failing. |
|
sendmail "FATAL - ${@:2}" |
|
exit $1 |
|
} |
|
|
|
declare -a on_exit_handlers |
|
|
|
on_exit() |
|
{ |
|
for i in "${on_exit_handlers[@]}" |
|
do |
|
log "Running Exit Handler: $i" |
|
eval "$i" |
|
done |
|
} |
|
|
|
add_exit_handler() |
|
{ |
|
local n=${#on_exit_handlers[*]} |
|
on_exit_handlers[$n]="$*" |
|
if [[ $n -eq 0 ]]; then |
|
trap on_exit EXIT |
|
fi |
|
} |
|
|
|
######################## |
|
## Script starts here ## |
|
######################## |
|
|
|
# Trap for killing the log file |
|
add_exit_handler "rm -f $LOGFILE" |
|
add_exit_handler "rm -r $STATSFILE" |
|
|
|
log "Backup started on $(date)" |
|
|
|
# Check we are not already running (shouldn't be). |
|
if [ -e $PIDFILE ]; then |
|
PID=$(cat $PIDFILE) |
|
if [ -e /proc/${PID} ]; then |
|
log "Backup was already in progress when started." |
|
log "Check backup job is not stalled." |
|
fatal 1 "Another backup is already running" |
|
fi |
|
fi |
|
|
|
# Create the pidfile (detects multiple attempted executions) |
|
echo $$ > $PIDFILE |
|
add_exit_handler "rm -f $PIDFILE" |
|
|
|
# Check we have a volume to work with |
|
if [ -z $1 ]; then |
|
fatal 1 "No shadow copy volume supplied." |
|
fi |
|
|
|
# Shadow Copy devices in cygwin are easily accessible as regular drives under |
|
# /proc/sys/Device/<shadowcopyvolume>. VSCSC gives us a windows label, from |
|
# that we just need the name. |
|
log "Shadow copy volume label: $1" |
|
BACKUPROOT="/proc/sys/Device/$(echo $1 | cut -d'\' -f 5)" |
|
log "Backup Root determined as: $BACKUPROOT" |
|
|
|
# Verify the shadow copy volume exists |
|
if [ ! -e "$BACKUPROOT" ]; then |
|
log "Specified shadow copy volume not found!" |
|
log "/proc/sys/Device had following volumes:" |
|
log "$(ls /proc/sys/Device/Hard*)" |
|
fatal 1 "Specified shadow copy volume not found." |
|
fi |
|
|
|
# Initial connection to server. |
|
while [ 1 ]; do |
|
# Read the time of the last backup - used at the bottom |
|
LASTBACKUPTIME=$(ssh $REMOTEUSER@$REMOTEHOST cat "\"$REMOTEFLAGFILE\"" | sed -n 1p) |
|
# And the count of the last backup - used at the bottom |
|
BACKUPCOUNT=$(ssh $REMOTEUSER@$REMOTEHOST cat "\"$REMOTEFLAGFILE\"" | sed -n 2p) |
|
# Last bup save result - we use this to avoid overwriting good data. |
|
LASTSNAPSHOTRESULT=$(ssh $REMOTEUSER@$REMOTEHOST cat \"$SNAPSHOTFLAGFILE\" | sed -n 2p) |
|
|
|
ssh $REMOTEUSER@$REMOTEHOST "echo -e \"$(date +%s)\n$BACKUPCOUNT\" > \"$REMOTEFLAGFILE\"" |
|
if [[ $? == 0 ]]; then |
|
break; |
|
elif [[ $RETRIES < $MAX_RETRIES ]]; then |
|
let RETRIES+=1 |
|
else |
|
fatal 2 "Maximum number of transfer retries exceeded on initial connection." |
|
fi |
|
done |
|
|
|
if [[ $? != 0 ]]; then |
|
fatal 1 "Failed initial connection to $REMOTEHOST" |
|
fi |
|
|
|
if [ -z $LASTBACKUPTIME ]; then |
|
LASTBACKUPTIME=0 |
|
fi |
|
|
|
if [ -z $BACKUPCOUNT ]; then |
|
BACKUPCOUNT=0 |
|
fi |
|
|
|
log "Last backup time: $LASTBACKUPTIME" |
|
log "Backup count: $BACKUPCOUNT" |
|
log "Last bup-save result: $LASTSNAPSHOTRESULT" |
|
|
|
# if the last snapshot failed, make a tar archive on the remote server and |
|
# tag it with the time. this is our failsafe behavior, |
|
if [[ $SKIPRECOVERY != 1 ]]; then |
|
if [[ $LASTSNAPSHOTRESULT != 0 ]]; then |
|
while [ 1 ]; do |
|
TARNAME="$BACKUPNAME-$(date +%Y-%m-%d-%H-%M-%S).tar.gz" |
|
log "Last backup commit operation failed." |
|
log "Creating a tar archive of last backup: $TARNAME" |
|
ssh $REMOTEUSER@$REMOTEHOST "tar -zcf $REMOTETARSTORAGE/$TARNAME $REMOTEROOT/" |
|
if [[ $? == 0 ]]; then |
|
log "Created emergency tar archive for last backup. Proceeding." |
|
break; |
|
elif [[ $RETRIES < $MAX_RETRIES ]]; then |
|
let RETRIES+=1 |
|
else |
|
fatal 4 "Maximum number of transfer retries exceeded while creating emergency tar-archive. Aborting!" |
|
fi |
|
done |
|
fi |
|
fi |
|
|
|
log "Backup in progress noted on remote server." |
|
|
|
# Loop MAX_RETRIES times while trying to send backup set. |
|
for dir in ${BACKUPSET[@]}; do |
|
while [ 1 ]; do |
|
log "Starting transfer of $dir" |
|
|
|
echo "$BACKUPROOT/$dir Transfer Statistics:" >> $STATSFILE |
|
|
|
rsync -rptlz -v \ |
|
--human-readable \ |
|
--delete \ |
|
--stats \ |
|
--exclude-from "$RSYNCEXCLUDES" \ |
|
-e "ssh" "$BACKUPROOT/./$dir" $REMOTEUSER@$REMOTEHOST:"$REMOTEROOT" | tee $STATSFILE |
|
|
|
if [[ ${PIPESTATUS[0]} == 0 ]]; then |
|
# Log behavior. |
|
log "Finished transfer of $dir." |
|
# Escape the loop. |
|
break; |
|
elif [[ $RETRIES < $MAX_RETRIES ]]; then |
|
echo "$BACKUPROOT/$dir FAILED. Try $RETRIES" >> $LOGFILE |
|
let RETRIES+=1 |
|
else |
|
fatal 2 "Maximum number of transfer retries exceeded." |
|
fi |
|
done |
|
done |
|
|
|
log "Backup Completed." |
|
|
|
# Log on to the remote server and do a backup with bup to maintain a version |
|
# history. |
|
log "Attempting to snapshot on remote server with bup..." |
|
while [ 1 ]; do |
|
if [[ $VERBOSE == 1 ]]; then |
|
BUPSAVEVERBOSE="-vv" |
|
fi |
|
|
|
ssh -t -t $REMOTEUSER@$REMOTEHOST 2>&1 << EOF | tee -a $LOGFILE |
|
bup index -um $REMOTEROOT |
|
echo $? > $SNAPSHOTFLAGFILE |
|
bup save -q $BUPSAVEVERBOSE --name $BACKUPNAME --strip-path $REMOTEROOT --tree --commit $REMOTEROOT |
|
echo $? >> $SNAPSHOTFLAGFILE |
|
exit |
|
EOF |
|
# ensure SSH succeeds |
|
if [[ ${PIPESTATUS[0]} == 0 ]]; then |
|
break; |
|
elif [[ $RETRIES < $MAX_RETRIES ]]; then |
|
let RETRIES+=1 |
|
else |
|
fatal 2 "Maximum number of transfer retries exceeded while snapshotting." |
|
fi |
|
done |
|
|
|
# ensure bup succeeded |
|
BUPINDEXRESULT=$(ssh $REMOTEUSER@$REMOTEHOST "cat $SNAPSHOTFLAGFILE | sed -n 1p") |
|
BUPSAVERESULT=$(ssh $REMOTEUSER@$REMOTEHOST "cat $SNAPSHOTFLAGFILE | sed -n 2p") |
|
|
|
log "Bup index result: $BUPINDEXRESULT" |
|
log "Bup save result: $BUPSAVERESULT" |
|
|
|
if [[ $BUPINDEXRESULT != 0 ]]; then |
|
fatal 3 "bup indexing operation failed." |
|
fi |
|
if [[ $BUPSAVERESULT != 0 ]]; then |
|
fatal 3 "bup snapshotting operation failed." |
|
fi |
|
|
|
# tag the new backup count into the flag file |
|
ssh $REMOTEUSER@$REMOTEHOST "echo $(($BACKUPCOUNT+1)) >> \"$REMOTEFLAGFILE\"" |
|
|
|
log "Backup Committed." |
|
|
|
# Only mail logs if > MAILFREQUENCY has passed |
|
if [[ $(($(date +%s) - $LASTBACKUPTIME)) > $MAILFREQUENCY ]]; then |
|
log "Sending mail. $(($BACKUPCOUNT+1)) successes before this email. Elapsed time since last mail: $(($(date +%s) - $LASTBACKUPTIME))" |
|
sendmail "Successful" |
|
else |
|
log "Not sending mail until cooldown period of $MAILFREQUENCY reached. At $(($(date +%s) - $LASTBACKUPTIME))" |
|
fi |
|
|
|
# Exit successfully (handlers will take care of temporary files) |
|
exit 0 |
This comment has been minimized.
Are you planning on updating this for bup with the changes you indicated at http://blog.wrouesnel.com/articles/bup%20-%20towards%20the%20perfect%20backup/ integrated into it?