fastscp.bash

#!/bin/bash


# deploys the distribution directory of a repo to a remote host
# remote host needs to be configured in .ssh/config with
# public key authentication, use LogLevel ERROR
# remote host must only have ls, rm and mkdir installed
# no other tools are needed on the remote host
# distribution directory must contain only files to be deployed
# only new files are transferred (using a saved checksum for
# each file saved in a directory tree in the repo)
# all other files and directories in the hostpath are deleted
# run in the root of the repo, e.g. with grunt-shell
# set distribution directory below

# Usage: fastscp host hostPath [debug]

# constants
# path to the directory which should be deployed
distdir=dist
# root of the checksum directory tree
ckroot=.cksums
# ignore list, list of root directories to ignore on server
ckignore=.ckignore
# hostname in ssh config
hostname=$1
#hostpath
hostpath=$2
#loglevel
loglevel=$3

if [ -z "$hostname" ]; then
    echo "No hostname supplied"
    exit 3
fi

if [ -z "$hostpath" ]; then
    echo "No host path supplied"
    exit 2
fi

if [[ ! "${ckroot}" == "${ckroot% *}" ]] ; then
	echo ckroot cannot contain spaces
	exit 4
fi

if [[ ! "${hostpath}" == "${hostpath% *}" ]] ; then
	echo hostpath cannot contain spaces
	exit 4
fi

# check if root of checksum directory tree exists
if [[ ! -d ${distdir} ]]; then
	echo deploy directory ${distdir} does not exist, exiting
	exit 1
fi

# check if root of checksum directory tree exists
if [[ ! -d ${ckroot} ]]; then
	echo creating root directory ${ckroot}
	mkdir ${ckroot}
fi

function debug {
	if [[ ${loglevel} = debug ]]; then
		echo $1
	fi
}

# read ignore list into variable
if [[ -e ${ckignore} ]]; then
	OFS=${IFS}
	IFS=$'\n'
	read -d '' -r -a ckignorelist < ${ckignore}
	IFS=${OFS}
fi

echo Ignored root directories on server: ${ckignorelist}

# update checksum tree with contents of dist
debug ==========================================================================================
echo Updating checksum tree with contents of ${distdir}
# loop via http://unix.stackexchange.com/questions/9496/looping-through-files-with-spaces-in-the-names
find ${distdir} -print0 | while IFS= read -r -d '' file; do
	#skip dist dir itself. find ${distdir}/* does not work because it skips hidden files like .htaccess
	if [[ ${file} = ${distdir} ]]; then
		continue
	fi
	#get only the file name
	file=$(echo ${file} | sed "s/${distdir}\///")
	debug ----------------------------------------------------------------------------------
	debug "working on ${file}"
	if [[ "${file}" == *".DS_Store"* ]]; then
		echo "ignoring file ${file}"
		continue
	fi
	found=false
	ckfile=${ckroot}/${file}
	# compare and create new checksums, deploy if file is new
	if [[ -d ${distdir}/${file} ]]; then
		debug "${file} is a directory, checking ${ckfile}"
		#check if directory exists in cksum tree
		if [[ -e ${ckfile} ]]; then
			#check if its a directory
			if [[ -d ${ckfile} ]]; then
				#all good
				found=true
				debug "Directory ${ckfile} found"
			else
				#something else is here, delete it
				debug "Found something else in ckroot instead of directory ${ckfile}, deleting it"
				rm -rf ${ckfile} 
			fi
		fi
	fi

	if [[ -f ${distdir}/${file} ]]; then
		debug "${file} is a file, checking ${ckfile}.cksum"
		#check if file exists in cksum tree
		if [[ -e ${ckfile}.cksum ]]; then
			#check if its a file
			if [[ -f ${ckfile}.cksum ]]; then
				#check if the checksum  matches
				oldchecksum=$(cat "${ckfile}.cksum")
				newchecksum=$(cksum "${distdir}/${file}")
				debug "old checksum is ${oldchecksum}"
				debug "new checksum is ${newchecksum}"
				if [[ ${oldchecksum} = ${newchecksum} ]]; then
					found=true
					debug "Checksum of file ${file} is not different, skipping it"
				fi
			else
				#something else is here, delete it
				debug "Found something else in ckroot instead of file ${ckfile}.cksum, deleting it"
				rm -rf ${ckfile}.cksum
			fi
		fi
	fi

	# create file or directory if it does not exist
	if [[ ${found} = false ]]; then
		echo ${file} was not found or checksum is different, deploying it
		hostfile=${hostpath}/${file}
		debug "hostfile is ${hostfile}"
		if [[ -d ${distdir}/${file} ]]; then
			#create directory on host
			debug "create directory ${hostfile} on host"
			ssh -n ${hostname} mkdir -p "\"${hostfile}\""
			if [[ ${?} -ne 0 ]]; then
				echo ERROR could not create directory ${hostfile}, there seems to be a file with that name
			else
				#create directory in ckroot tree
				debug "Creating directory ${ckfile}"
				mkdir -p "${ckfile}"
			fi
		fi
		if [[ -f ${distdir}/${file} ]]; then
			#transfer the file
			debug "transferring with scp"
			scp "${distdir}/${file}" ${hostname}:"\"${hostfile}\"" > /dev/null
			if [[ ${?} -ne 0 ]]; then
				echo ERROR scp transfer error while transferring ${hostfile}
			else
				#create a new checksum
				debug "creating new checksum for ${file} and storing it in ${ckfile}.cksum"
				cksum "${distdir}/${file}" > "${ckfile}.cksum" 
			fi
		fi
	fi
	debug "finished ${file}"
done

debug ==========================================================================================
# delete old files from checksum tree
echo Delete unmatched files in checksum tree ${ckroot}
# loop via http://unix.stackexchange.com/questions/9496/looping-through-files-with-spaces-in-the-names
find ${ckroot} -print0 | while IFS= read -r -d '' cksumfile; do
	#skip ckroot dir itself. find ${ckroot}/* does not work because it skips hidden files like .htaccess
	if [[ ${cksumfile} = ${ckroot} ]]; then
		continue
	fi
	debug ----------------------------------------------------------------------------------
	debug "working on ${cksumfile}"
	#remove leading directory
	file=$(echo ${cksumfile} | sed "s/^${ckroot}\///")
	if [[ -f ${cksumfile} ]]; then
		#remove .cksum ending
		file=$(echo ${file} | sed "s/\.cksum$//")
	fi
	debug "checking if checksum file ${cksumfile} has existing file ${file} in ${distdir}"
	#delete chksum file if file does not exist in distdir
	if [[ ! -e ${distdir}/${file} ]]; then
		echo removing ${cksumfile}
		rm -rf "${cksumfile}"
	fi
	debug "finished ${cksumfile}"
done

OFS=${IFS}
IFS=$'\n'

debug ==========================================================================================
echo Delete unmatched directories from host
# getting recursive full path ls output: http://stackoverflow.com/questions/1767384/ls-command-how-can-i-get-a-recursive-full-path-listing-one-line-per-file
# fixing case statements in command substitutions: http://wiki.bash-hackers.org/syntax/expansion/cmdsubst
dirs=$(ssh ${hostname} ls -apR1 ${hostpath} | while read l; do case $l in (*:) d=${l%:};; ("") d=;; (*) echo "$d/$l";; esac; done | grep -v "\/\.\.\/$" | grep -v "\/\.\/$" | grep \/$ | sed "s/${hostpath}\///")
for file in ${dirs}; do
	debug ----------------------------------------------------------------------------------
 	debug "working on host directory ${file}, see if ${ckroot}/${file} exists"
 	if [[ ! -d ${ckroot}/${file} ]]; then
 		# remove trailing slash from ${file} to compare
		echo "${ckignorelist[@]}"
		echo "${file//\/}"
      # get root directory by bash string substitution
		if [[ "${ckignorelist[@]}" =~ "${file%%/*}" ]]; then
	 		echo root directory ${file} is on ignore list, not deleting			
 		else
	 		echo directory ${file} does not exist in source, deleting
	 		ssh -n ${hostname} rm -rf "\"${hostpath}/${file}\""
 		fi
 	fi
 	debug "finished ${file}"
done

debug ==========================================================================================
echo Delete unmatched files from host
dirs=$(ssh ${hostname} ls -apR1 ${hostpath} | while read l; do case $l in (*:) d=${l%:};; ("") d=;; (*) echo "$d/$l";; esac; done | grep -v "\/\.\.\/$" | grep -v "\/\.\/$" | grep -v \/$)
for file in ${dirs}; do
	file=$(echo ${file} | sed "s/${hostpath}\///")
	#skip if its a directory
	if [[ ! -d ${ckroot}/${file} ]]; then
		debug ----------------------------------------------------------------------------------
		debug "working on host file ${file}, see if ${ckroot}/${file}.cksum exists"
		if [[ ! -e ${ckroot}/${file}.cksum ]]; then
	 		# just get root directory of ${file} by removing substring from end of variable by using bash parameter expansion. Neat.
	 		# https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html
			if [[ "${ckignorelist[@]}" =~ "${file%%/*}" ]]; then
		 		echo root directory of ${file} is on ignore list, not deleting			
	 		else
				echo file ${file} does not exist in source, deleting
				ssh -n ${hostname} rm -rf "\"${hostpath}/${file}\""
			fi
		fi
		debug "finished ${file}"
	fi
done

IFS=${OFS}