Backup Linux Box

backup_linuxbox.sh

#!/bin/bash

## backup script for my_env

TSTAMP=$(date +%d%b%y_%H%M%S%Z)

SOURCEDIR="/projects/my_env"
BKUPDIR="/backupdisk/my_env"

SOURCECHECK=$(test -d "${SOURCEDIR}" && echo Exists || echo Does not exist)
BKUPCHECK=$(test -d "${BKUPDIR}" && echo Exists || echo Does not exist)

## this script should run from a specific hostname or linux machine, and not from all of hostnames, e.g., 
## run only from a login node but not from compute nodes as compute node may not have access to all disk mounts
if [[ "$(hostname)" == "my-dev.jax.org" && -d "${SOURCEDIR}" && -d "${BKUPDIR}" ]]; then

	LOGFILE=$(printf "bkup_my_env_%s.log" "${TSTAMP}")
	LOGPATH=$(printf "%s/bkup_configs/logs/%s" "${SOURCEDIR}" "${LOGFILE}")

	## slack api
	## echo -e "\nSource location: ${SOURCEDIR}\nLog file: ${LOGPATH}\n" | slacktee -t "my_env backup started" -i "floppy_disk" >> /dev/null 2>&1 &

	df -h

	## preserve symlinks using -l flag
	##### DANGER: DO NOT EDIT PATHS #####
	## --delete-after is enabled, so changing paths in source or target location or exclude-from list
	## can potentially delete previously backuped data.
	## Read --delete-after specification for more.
	## keep trailing slash marks for both source and destination
	
	## DISABLE --delete-after rsync: rsync will NOT sync updated packages and will keep deprecated 
	## rsync -rtxl --partial --numeric-ids --info=progress2 --exclude-from="${SOURCEDIR}"/bkup_configs/exclude_dirs.txt --delete-after --log-file=/tmp/"${LOGFILE}" "${SOURCEDIR}"/ "${BKUPDIR}"/

	## rsync without --delete-after; keep LOGFILE under tmp/ or RAM disk else I/O error may occur because of too many
	## write requests over very short time
	rsync -rtxl --partial --numeric-ids --info=progress2 --log-file=/tmp/"${LOGFILE}" "${SOURCEDIR}"/ "${BKUPDIR}"/
	
	exitstat=$?

	rsync -avhP /tmp/"${LOGFILE}" "${LOGPATH}"
	
	rm -f /tmp/"${LOGFILE}"

	df -h

	# notify slack if error, require slacktee script
	if [[ "${exitstat}" != 0 && -s "${HOME}"/bin/slacktee ]]; then
		echo -e "\nmy_env backup failed on $(hostname)\nSource: ${SOURCEDIR}\nTarget: ${BKUPDIR}\nExit status: ${exitstat}\nLog file: ${LOGPATH}\n"

	    echo -e "\nmy_env backup failed on $(hostname)\nSource: ${SOURCEDIR}\nTarget: ${BKUPDIR}\nExit status: ${exitstat}\nLog file: ${LOGPATH}\n" | slacktee -t "my_env backup returned an error exit code: ${exitstat}" -i "warning" >> /dev/null 2>&1 &
	    # keep ssh into background but allow 5 seconds before exit of parent script so ssh job can ping slack
	    sleep 5
	elif [[ "${exitstat}" == 0 && -s "${HOME}"/bin/slacktee ]]; then
		echo -e "\nmy_env backup is done on $(hostname)\nSource: ${SOURCEDIR}\nTarget: ${BKUPDIR}\nExit status: ${exitstat}\nLog file: ${LOGPATH}\n"

	    echo -e "\nmy_env backup is done on $(hostname)\nSource: ${SOURCEDIR}\nTarget: ${BKUPDIR}\nExit status: ${exitstat}\nLog file: ${LOGPATH}\n" | slacktee -t "my_env backup is done" -i "white_check_mark" >> /dev/null 2>&1 &
	    # keep ssh into background but allow 5 seconds before exit of parent script so ssh job can ping slack
	    sleep 5
	fi
	
else
	echo -e "####\nSkipping backup my_env\nExecute script from rvdev host to start backup\nHostname: $(hostname)\nSource: ${SOURCEDIR} ${SOURCECHECK}\nTarget: ${BKUPDIR} ${BKUPCHECK}\n####"
	
	## slack api
	## echo -e "####\nSkipping backup my_env\nExecute script from rvdev host to start backup\nHostname: $(hostname)\nSource: ${SOURCEDIR} ${SOURCECHECK}\nTarget: ${BKUPDIR} ${BKUPCHECK}\n####" | slacktee -t "my_env backup skipped" -i "warning" >> /dev/null 2>&1 &
	## wait for slack api to send message before exit
	sleep 5	
fi


## full system backup
## Read https://wiki.archlinux.org/index.php/Rsync#Full_system_backup including snapshots
#### CAUTION: ####
## make sure target disk has enough space
## make sure exclude flag is excluding most of large and non-essential disk mounts not needed to be backuped
## we certainly do not need to backup NFS mounted disks from HPC!
## if using --delete-after flag, see above for more on that, including some dangers of changing paths.
#### Command to backup entire system ####
## rsync -aAXv --exclude={"/dev/*","/proc/*","/sys/*","/tmp/*","/run/*","/mnt/*","/media/*","/scratch/*","/lost+found"} / /mnt/bkup_cgp-small/

## END ##