wrouesnel/.remote-backup-excludes.txt

## .remote-backup-excludes.txt
# this is an rsync-excludes format list of files to exclude.

## .remote-backup-settingsrc
# This command is included by do-remote-backup.bsh to set parameters.

BACKUPNAME="name_to_save_as_in_bup"

# Backup set (array of root folders we backup
BACKUPSET=( "directories" "tobackup" "in" "the" "root" "directory" )
BACKUPROOT="/cygdrive/c"

# Backup destination settings
REMOTEUSER="backup"
REMOTEHOST="backup-server"
REMOTEROOT="backup"
REMOTETARSTORAGE="."

# File-transfer retry settings
MAX_RETRIES=10

# Email Settings
MAILTO=("your@email.com")
SMTPSERVER="smtp.gmail.com"
SMTPPORT="587"
SMTPFROM="host@provider.com"
SMTPUSER="host account"
SMTPPASSWORD="host account password"

# minimum time between sending notification emails of backups completed (in seconds)
# 43200 = 12 hours
MAILFREQUENCY=43200

# skip doing tar archive creation for failed snapshots (not recommended!)
SKIPRECOVERY=1

## do-remote-backup.bsh
#!/bin/bash
# Script executed by a daily cron job in order to remotely backup med. This
# script must run as root so it has sufficient privileges to create ZFS
# snapshots.
#
# Method of operation:
# Windows Scheduler (or cron) initiates this job by running the vscsc.exe
# application with this scipt as the target for a Cygwin instance.
#
# The script then runs rsync and copies files to the remote server target using
# rsync over SSH.
#
# After rsync is complete, it connects to the remote host with SSH and invokes
# bup to make a snapshot version of the backed up files. This duel-action
# ensures we always have a copy of the data outside of the bup-archive.
#
# What we backup:
# c:\med
#

# System Settings
VERBOSE=1
LOGFILE=$(mktemp)
STATSFILE=$(mktemp)
PIDFILE=/var/run/remote-backup.pid

REMOTEFLAGFILE=.backup-in-progress
SNAPSHOTFLAGFILE=.last-bup-save-result

RETRIES=0

RSYNCEXCLUDES=.remote-backup-excludes.txt

. .remote-backup-settingsrc

## Function declarations ##

# Non-stdout echo command
echoerr ()
{
    echo $@ 1>&2
}

# Log file output command
log ()
{
    if [[ $VERBOSE == 1 ]]; then
        echoerr "VERBOSE: $@"
    fi

    if [ ! -z $LOGFILE ]; then
        echo $@ >> $LOGFILE
    fi
}

# Function that packs up the logs and emails them.
# Depends on the excellent cygwin email package (should for Unix).
# Usage: sendmail <headerline>
sendmail()
{
    cat << EOF | email  -from-name "$(hostname)" \
                        -from-addr "$SMTPFROM" \
                        -smtp-server "$SMTPSERVER" \
                        -smtp-port $SMTPPORT \
                        -smtp-auth login \
                        -smtp-user "$SMTPUSER" \
                        -smtp-pass "$SMTPPASSWORD" \
                        -tls \
                        -subject "$BACKUPNAME Remote Backup: $1" \
                        $(echo ${MAILTO[@]} | tr ' ' ',')
$BACKUPNAME Remote Backup Report
========================

Backup Statistics
-----------------
$(cat "$STATSFILE")

Session Log File
----------------
$(cat "$LOGFILE")

EOF
}

# Fatal error that should terminate the script
# Usage: fatal <exit code> <error string>
fatal ()
{
    echoerr "${@:2}"

    # Always try to sendmail before failing.
    sendmail "FATAL - ${@:2}"
    exit $1
}

declare -a on_exit_handlers

on_exit()
{
    for i in "${on_exit_handlers[@]}"
    do
        log "Running Exit Handler: $i"
        eval "$i"
    done
}

add_exit_handler()
{
    local n=${#on_exit_handlers[*]}
    on_exit_handlers[$n]="$*"
    if [[ $n -eq 0 ]]; then
        trap on_exit EXIT
    fi
}

########################
## Script starts here ##
########################

# Trap for killing the log file
add_exit_handler "rm -f $LOGFILE"
add_exit_handler "rm -r $STATSFILE"

log "Backup started on $(date)"

# Check we are not already running (shouldn't be).
if [ -e $PIDFILE ]; then
    PID=$(cat $PIDFILE)
    if [ -e /proc/${PID} ]; then
        log "Backup was already in progress when started."
        log "Check backup job is not stalled."
        fatal 1 "Another backup is already running"
    fi
fi

# Create the pidfile (detects multiple attempted executions)
echo $$ > $PIDFILE
add_exit_handler "rm -f $PIDFILE"

# Check we have a volume to work with
if [ -z $1 ]; then
    fatal 1 "No shadow copy volume supplied."
fi

# Shadow Copy devices in cygwin are easily accessible as regular drives under
# /proc/sys/Device/<shadowcopyvolume>. VSCSC gives us a windows label, from
# that we just need the name.
log "Shadow copy volume label: $1"
BACKUPROOT="/proc/sys/Device/$(echo $1 | cut -d'\' -f 5)"
log "Backup Root determined as: $BACKUPROOT"

# Verify the shadow copy volume exists
if [ ! -e "$BACKUPROOT" ]; then
    log "Specified shadow copy volume not found!"
    log "/proc/sys/Device had following volumes:"
    log "$(ls /proc/sys/Device/Hard*)"
    fatal 1 "Specified shadow copy volume not found."
fi

# Initial connection to server.
while [ 1 ]; do
	# Read the time of the last backup - used at the bottom
	LASTBACKUPTIME=$(ssh $REMOTEUSER@$REMOTEHOST cat "\"$REMOTEFLAGFILE\"" | sed -n 1p)
	# And the count of the last backup - used at the bottom
	BACKUPCOUNT=$(ssh $REMOTEUSER@$REMOTEHOST cat "\"$REMOTEFLAGFILE\"" | sed -n 2p)
	# Last bup save result - we use this to avoid overwriting good data.
	LASTSNAPSHOTRESULT=$(ssh $REMOTEUSER@$REMOTEHOST cat \"$SNAPSHOTFLAGFILE\" | sed -n 2p)

    ssh $REMOTEUSER@$REMOTEHOST "echo -e \"$(date +%s)\n$BACKUPCOUNT\" > \"$REMOTEFLAGFILE\""
    if [[ $? == 0 ]]; then
        break;
    elif [[ $RETRIES < $MAX_RETRIES ]]; then
        let RETRIES+=1
    else
        fatal 2 "Maximum number of transfer retries exceeded on initial connection."
    fi
done

if [[ $? != 0 ]]; then
    fatal 1 "Failed initial connection to $REMOTEHOST"
fi

if [ -z $LASTBACKUPTIME ]; then
	LASTBACKUPTIME=0
fi

if [ -z $BACKUPCOUNT ]; then
	BACKUPCOUNT=0
fi

log "Last backup time: $LASTBACKUPTIME"
log "Backup count: $BACKUPCOUNT"
log "Last bup-save result: $LASTSNAPSHOTRESULT"

# if the last snapshot failed, make a tar archive on the remote server and
# tag it with the time. this is our failsafe behavior,
if [[ $SKIPRECOVERY != 1 ]]; then
	if [[ $LASTSNAPSHOTRESULT != 0 ]]; then
		while [ 1 ]; do
			TARNAME="$BACKUPNAME-$(date +%Y-%m-%d-%H-%M-%S).tar.gz"
			log "Last backup commit operation failed."
			log "Creating a tar archive of last backup: $TARNAME"
			ssh $REMOTEUSER@$REMOTEHOST "tar -zcf $REMOTETARSTORAGE/$TARNAME $REMOTEROOT/"
			if [[ $? == 0 ]]; then
				log "Created emergency tar archive for last backup. Proceeding."
				break;
			elif [[ $RETRIES < $MAX_RETRIES ]]; then
				let RETRIES+=1
			else
				fatal 4 "Maximum number of transfer retries exceeded while creating emergency tar-archive. Aborting!"
			fi
		done
	fi
fi

log "Backup in progress noted on remote server."

# Loop MAX_RETRIES times while trying to send backup set.
for dir in ${BACKUPSET[@]}; do
    while [ 1 ]; do
        log "Starting transfer of $dir"

		echo "$BACKUPROOT/$dir Transfer Statistics:" >> $STATSFILE

        rsync -rptlz -v \
		--human-readable \
		--delete \
		--stats \
		--exclude-from "$RSYNCEXCLUDES" \
		-e "ssh" "$BACKUPROOT/./$dir" $REMOTEUSER@$REMOTEHOST:"$REMOTEROOT" | tee $STATSFILE

        if [[ ${PIPESTATUS[0]} == 0 ]]; then
            # Log behavior.
            log "Finished transfer of $dir."
            # Escape the loop.
            break;
        elif [[ $RETRIES < $MAX_RETRIES ]]; then
			echo "$BACKUPROOT/$dir FAILED. Try $RETRIES" >> $LOGFILE
			let RETRIES+=1
        else
            fatal 2 "Maximum number of transfer retries exceeded."
        fi
    done
done

log "Backup Completed."

# Log on to the remote server and do a backup with bup to maintain a version
# history.
log "Attempting to snapshot on remote server with bup..."
while [ 1 ]; do
	if [[ $VERBOSE == 1 ]]; then
			BUPSAVEVERBOSE="-vv"
	fi

    ssh -t -t $REMOTEUSER@$REMOTEHOST 2>&1 << EOF | tee -a $LOGFILE
	bup index -um $REMOTEROOT
	echo $? > $SNAPSHOTFLAGFILE
	bup save -q $BUPSAVEVERBOSE --name $BACKUPNAME --strip-path $REMOTEROOT --tree --commit $REMOTEROOT
	echo $? >> $SNAPSHOTFLAGFILE
	exit
EOF
	# ensure SSH succeeds
    if [[ ${PIPESTATUS[0]} == 0 ]]; then
        break;
    elif [[ $RETRIES < $MAX_RETRIES ]]; then
        let RETRIES+=1
    else
        fatal 2 "Maximum number of transfer retries exceeded while snapshotting."
    fi
done

# ensure bup succeeded
BUPINDEXRESULT=$(ssh $REMOTEUSER@$REMOTEHOST "cat $SNAPSHOTFLAGFILE | sed -n 1p")
BUPSAVERESULT=$(ssh $REMOTEUSER@$REMOTEHOST "cat $SNAPSHOTFLAGFILE | sed -n 2p")

log "Bup index result: $BUPINDEXRESULT"
log "Bup save result: $BUPSAVERESULT"

if [[ $BUPINDEXRESULT != 0 ]]; then
	fatal 3 "bup indexing operation failed."
fi
if [[ $BUPSAVERESULT != 0 ]]; then
	fatal 3 "bup snapshotting operation failed."
fi

# tag the new backup count into the flag file
ssh $REMOTEUSER@$REMOTEHOST "echo $(($BACKUPCOUNT+1)) >> \"$REMOTEFLAGFILE\""

log "Backup Committed."

# Only mail logs if > MAILFREQUENCY has passed
if [[ $(($(date +%s) - $LASTBACKUPTIME)) > $MAILFREQUENCY ]]; then
	log "Sending mail. $(($BACKUPCOUNT+1)) successes before this email. Elapsed time since last mail: $(($(date +%s) - $LASTBACKUPTIME))"
	sendmail "Successful"
else
	log "Not sending mail until cooldown period of $MAILFREQUENCY reached. At $(($(date +%s) - $LASTBACKUPTIME))"
fi

# Exit successfully (handlers will take care of temporary files)
exit 0
	# This command is included by do-remote-backup.bsh to set parameters.

	BACKUPNAME="name_to_save_as_in_bup"

	# Backup set (array of root folders we backup
	BACKUPSET=( "directories" "tobackup" "in" "the" "root" "directory" )
	BACKUPROOT="/cygdrive/c"

	# Backup destination settings
	REMOTEUSER="backup"
	REMOTEHOST="backup-server"
	REMOTEROOT="backup"
	REMOTETARSTORAGE="."

	# File-transfer retry settings
	MAX_RETRIES=10

	# Email Settings
	MAILTO=("your@email.com")
	SMTPSERVER="smtp.gmail.com"
	SMTPPORT="587"
	SMTPFROM="host@provider.com"
	SMTPUSER="host account"
	SMTPPASSWORD="host account password"

	# minimum time between sending notification emails of backups completed (in seconds)
	# 43200 = 12 hours
	MAILFREQUENCY=43200

	# skip doing tar archive creation for failed snapshots (not recommended!)
	SKIPRECOVERY=1
	#!/bin/bash
	# Script executed by a daily cron job in order to remotely backup med. This
	# script must run as root so it has sufficient privileges to create ZFS
	# snapshots.
	#
	# Method of operation:
	# Windows Scheduler (or cron) initiates this job by running the vscsc.exe
	# application with this scipt as the target for a Cygwin instance.
	#
	# The script then runs rsync and copies files to the remote server target using
	# rsync over SSH.
	#
	# After rsync is complete, it connects to the remote host with SSH and invokes
	# bup to make a snapshot version of the backed up files. This duel-action
	# ensures we always have a copy of the data outside of the bup-archive.
	#
	# What we backup:
	# c:\med
	#

	# System Settings
	VERBOSE=1
	LOGFILE=$(mktemp)
	STATSFILE=$(mktemp)
	PIDFILE=/var/run/remote-backup.pid

	REMOTEFLAGFILE=.backup-in-progress
	SNAPSHOTFLAGFILE=.last-bup-save-result

	RETRIES=0

	RSYNCEXCLUDES=.remote-backup-excludes.txt

	. .remote-backup-settingsrc

	## Function declarations ##

	# Non-stdout echo command
	echoerr ()
	{
	echo $@ 1>&2
	}

	# Log file output command
	log ()
	{
	if [[ $VERBOSE == 1 ]]; then
	echoerr "VERBOSE: $@"
	fi

	if [ ! -z $LOGFILE ]; then
	echo $@ >> $LOGFILE
	fi
	}

	# Function that packs up the logs and emails them.
	# Depends on the excellent cygwin email package (should for Unix).
	# Usage: sendmail <headerline>
	sendmail()
	{
	cat << EOF \| email -from-name "$(hostname)" \
	-from-addr "$SMTPFROM" \
	-smtp-server "$SMTPSERVER" \
	-smtp-port $SMTPPORT \
	-smtp-auth login \
	-smtp-user "$SMTPUSER" \
	-smtp-pass "$SMTPPASSWORD" \
	-tls \
	-subject "$BACKUPNAME Remote Backup: $1" \
	$(echo ${MAILTO[@]} \| tr ' ' ',')
	$BACKUPNAME Remote Backup Report
	========================

	Backup Statistics
	-----------------
	$(cat "$STATSFILE")

	Session Log File
	----------------
	$(cat "$LOGFILE")

	EOF
	}

	# Fatal error that should terminate the script
	# Usage: fatal <exit code> <error string>
	fatal ()
	{
	echoerr "${@:2}"

	# Always try to sendmail before failing.
	sendmail "FATAL - ${@:2}"
	exit $1
	}

	declare -a on_exit_handlers

	on_exit()
	{
	for i in "${on_exit_handlers[@]}"
	do
	log "Running Exit Handler: $i"
	eval "$i"
	done
	}

	add_exit_handler()
	{
	local n=${#on_exit_handlers[*]}
	on_exit_handlers[$n]="$*"
	if [[ $n -eq 0 ]]; then
	trap on_exit EXIT
	fi
	}

	########################
	## Script starts here ##
	########################

	# Trap for killing the log file
	add_exit_handler "rm -f $LOGFILE"
	add_exit_handler "rm -r $STATSFILE"

	log "Backup started on $(date)"

	# Check we are not already running (shouldn't be).
	if [ -e $PIDFILE ]; then
	PID=$(cat $PIDFILE)
	if [ -e /proc/${PID} ]; then
	log "Backup was already in progress when started."
	log "Check backup job is not stalled."
	fatal 1 "Another backup is already running"
	fi
	fi

	# Create the pidfile (detects multiple attempted executions)
	echo $$ > $PIDFILE
	add_exit_handler "rm -f $PIDFILE"

	# Check we have a volume to work with
	if [ -z $1 ]; then
	fatal 1 "No shadow copy volume supplied."
	fi

	# Shadow Copy devices in cygwin are easily accessible as regular drives under
	# /proc/sys/Device/<shadowcopyvolume>. VSCSC gives us a windows label, from
	# that we just need the name.
	log "Shadow copy volume label: $1"
	BACKUPROOT="/proc/sys/Device/$(echo $1 \| cut -d'\' -f 5)"
	log "Backup Root determined as: $BACKUPROOT"

	# Verify the shadow copy volume exists
	if [ ! -e "$BACKUPROOT" ]; then
	log "Specified shadow copy volume not found!"
	log "/proc/sys/Device had following volumes:"
	log "$(ls /proc/sys/Device/Hard*)"
	fatal 1 "Specified shadow copy volume not found."
	fi

	# Initial connection to server.
	while [ 1 ]; do
	# Read the time of the last backup - used at the bottom
	LASTBACKUPTIME=$(ssh $REMOTEUSER@$REMOTEHOST cat "\"$REMOTEFLAGFILE\"" \| sed -n 1p)
	# And the count of the last backup - used at the bottom
	BACKUPCOUNT=$(ssh $REMOTEUSER@$REMOTEHOST cat "\"$REMOTEFLAGFILE\"" \| sed -n 2p)
	# Last bup save result - we use this to avoid overwriting good data.
	LASTSNAPSHOTRESULT=$(ssh $REMOTEUSER@$REMOTEHOST cat \"$SNAPSHOTFLAGFILE\" \| sed -n 2p)

	ssh $REMOTEUSER@$REMOTEHOST "echo -e \"$(date +%s)\n$BACKUPCOUNT\" > \"$REMOTEFLAGFILE\""
	if [[ $? == 0 ]]; then
	break;
	elif [[ $RETRIES < $MAX_RETRIES ]]; then
	let RETRIES+=1
	else
	fatal 2 "Maximum number of transfer retries exceeded on initial connection."
	fi
	done

	if [[ $? != 0 ]]; then
	fatal 1 "Failed initial connection to $REMOTEHOST"
	fi

	if [ -z $LASTBACKUPTIME ]; then
	LASTBACKUPTIME=0
	fi

	if [ -z $BACKUPCOUNT ]; then
	BACKUPCOUNT=0
	fi

	log "Last backup time: $LASTBACKUPTIME"
	log "Backup count: $BACKUPCOUNT"
	log "Last bup-save result: $LASTSNAPSHOTRESULT"

	# if the last snapshot failed, make a tar archive on the remote server and
	# tag it with the time. this is our failsafe behavior,
	if [[ $SKIPRECOVERY != 1 ]]; then
	if [[ $LASTSNAPSHOTRESULT != 0 ]]; then
	while [ 1 ]; do
	TARNAME="$BACKUPNAME-$(date +%Y-%m-%d-%H-%M-%S).tar.gz"
	log "Last backup commit operation failed."
	log "Creating a tar archive of last backup: $TARNAME"
	ssh $REMOTEUSER@$REMOTEHOST "tar -zcf $REMOTETARSTORAGE/$TARNAME $REMOTEROOT/"
	if [[ $? == 0 ]]; then
	log "Created emergency tar archive for last backup. Proceeding."
	break;
	elif [[ $RETRIES < $MAX_RETRIES ]]; then
	let RETRIES+=1
	else
	fatal 4 "Maximum number of transfer retries exceeded while creating emergency tar-archive. Aborting!"
	fi
	done
	fi
	fi

	log "Backup in progress noted on remote server."

	# Loop MAX_RETRIES times while trying to send backup set.
	for dir in ${BACKUPSET[@]}; do
	while [ 1 ]; do
	log "Starting transfer of $dir"

	echo "$BACKUPROOT/$dir Transfer Statistics:" >> $STATSFILE

	rsync -rptlz -v \
	--human-readable \
	--delete \
	--stats \
	--exclude-from "$RSYNCEXCLUDES" \
	-e "ssh" "$BACKUPROOT/./$dir" $REMOTEUSER@$REMOTEHOST:"$REMOTEROOT" \| tee $STATSFILE

	if [[ ${PIPESTATUS[0]} == 0 ]]; then
	# Log behavior.
	log "Finished transfer of $dir."
	# Escape the loop.
	break;
	elif [[ $RETRIES < $MAX_RETRIES ]]; then
	echo "$BACKUPROOT/$dir FAILED. Try $RETRIES" >> $LOGFILE
	let RETRIES+=1
	else
	fatal 2 "Maximum number of transfer retries exceeded."
	fi
	done
	done

	log "Backup Completed."

	# Log on to the remote server and do a backup with bup to maintain a version
	# history.
	log "Attempting to snapshot on remote server with bup..."
	while [ 1 ]; do
	if [[ $VERBOSE == 1 ]]; then
	BUPSAVEVERBOSE="-vv"
	fi

	ssh -t -t $REMOTEUSER@$REMOTEHOST 2>&1 << EOF \| tee -a $LOGFILE
	bup index -um $REMOTEROOT
	echo $? > $SNAPSHOTFLAGFILE
	bup save -q $BUPSAVEVERBOSE --name $BACKUPNAME --strip-path $REMOTEROOT --tree --commit $REMOTEROOT
	echo $? >> $SNAPSHOTFLAGFILE
	exit
	EOF
	# ensure SSH succeeds
	if [[ ${PIPESTATUS[0]} == 0 ]]; then
	break;
	elif [[ $RETRIES < $MAX_RETRIES ]]; then
	let RETRIES+=1
	else
	fatal 2 "Maximum number of transfer retries exceeded while snapshotting."
	fi
	done

	# ensure bup succeeded
	BUPINDEXRESULT=$(ssh $REMOTEUSER@$REMOTEHOST "cat $SNAPSHOTFLAGFILE \| sed -n 1p")
	BUPSAVERESULT=$(ssh $REMOTEUSER@$REMOTEHOST "cat $SNAPSHOTFLAGFILE \| sed -n 2p")

	log "Bup index result: $BUPINDEXRESULT"
	log "Bup save result: $BUPSAVERESULT"

	if [[ $BUPINDEXRESULT != 0 ]]; then
	fatal 3 "bup indexing operation failed."
	fi
	if [[ $BUPSAVERESULT != 0 ]]; then
	fatal 3 "bup snapshotting operation failed."
	fi

	# tag the new backup count into the flag file
	ssh $REMOTEUSER@$REMOTEHOST "echo $(($BACKUPCOUNT+1)) >> \"$REMOTEFLAGFILE\""

	log "Backup Committed."

	# Only mail logs if > MAILFREQUENCY has passed
	if [[ $(($(date +%s) - $LASTBACKUPTIME)) > $MAILFREQUENCY ]]; then
	log "Sending mail. $(($BACKUPCOUNT+1)) successes before this email. Elapsed time since last mail: $(($(date +%s) - $LASTBACKUPTIME))"
	sendmail "Successful"
	else
	log "Not sending mail until cooldown period of $MAILFREQUENCY reached. At $(($(date +%s) - $LASTBACKUPTIME))"
	fi

	# Exit successfully (handlers will take care of temporary files)
	exit 0