Uses strictly wget and grep to mirror a remote HTTP location recursively. Deletes old files. Supports http auth and self-signed certificates. Strictly POSIX compliant so that it works in "dash" as well as in "bash". Handy to use on arm devices that need syncing but do not have rsync or ssh. Currently used to sync multiple kobo ereaders.

wgetmirror.sh

#!/bin/bash
################ CONFIG ###################
username="user"
#HTTP Auth username
password="password"
#HTTP Auth password
basedirname="/directory/to/Sync"
#CA certificate in PEM format
certificate="/directory/to/cert.crt"
#url = Set to the https server running an nginx or apache file listing. Make sure its urlencoded and don't forget trailing slash.
url="https://url.com/sub/sub/sub%20directory/"
#allowed extensions
exts="epub,mobi,cbz,cbr"
############### END CONFIG #################
numdirs=$(($(echo "$url" | grep -o "/" | wc -l) - 3))
#get everything with wget, if this fails then everything fails..
wgetoutput=$(wget --user="$username" --password="$password" --mirror --ca-certificate="$certificate" --cut-dirs=$numdirs -P "$basedirname" --no-parent --no-host-directories --accept "$exts" "$url" 2>&1 )
if [ $? -ne 0 ]; then
	echo "Something went wrong. Stopping."
	exit 1
fi
echo "$wgetoutput"
#catch the rejections to subtract from total downloads.
urllength=${#url}
urllength=$((urllength+1))
log1loc=$(echo "$wgetoutput" | grep -Eio https://.+ | cut -c $urllength- | grep -v '.*/$')
fixurls ()
{
	echo "$log1loc" | while read line
		do name="$basedirname/$(echo "$line" | sed 's % \\\\x g' | xargs printf)"
		echo "$name"
	done
}
result=$(fixurls)
numindexes=$(echo "$wgetoutput" | grep -o "since it should be rejected." | wc -l)
wgetresult=$(echo "$wgetoutput" | tail -1)
numdownloads=$(echo "$wgetresult" | sed -e 's/Downloaded://' | sed -e 's/files.*//')
echo "----"
echo "Start removing files:"
currentfiles=$(find "$basedirname" -type f -print)
dostuff () 
{
	echo "$currentfiles" | while read i; do
		echo "$result" | grep -qF "$i"
		if [ $? -ne 0 ]
		then
			echo "@Removing $i.."
			rm -f "$i"
			if [ $? -ne 0 ]; then
				echo "Could not delete $i. Stopping"
				exit 1
			fi
		fi
	done
}
removefiles=$(dostuff)
echo $removefiles
echo $removefiles | grep -qF "@Removing"
deletestatus=$?
find "$basedirname" -type d -empty -delete
echo "Done removing files"
echo "----"
numrealdownloads=$(($numdownloads - $numindexes))
echo "Num indexes: $numindexes | Num downloads: $numrealdownloads"
echo "----"
echo "Sync complete"
echo "----"
if [ $numrealdownloads -ne 0 ] || [ $deletestatus -eq 0 ]; then
	echo "Disk changed, will notify library update"
fi