Skip to content

Instantly share code, notes, and snippets.

@huyz
Forked from Vartkat/buildresticexcludes.sh
Last active March 6, 2022 03:43
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save huyz/a3711154a5ac476f06240e13a148fcf6 to your computer and use it in GitHub Desktop.
Save huyz/a3711154a5ac476f06240e13a148fcf6 to your computer and use it in GitHub Desktop.
Bash script to build a restic exclude list that mimics Apple TimeMachine exclude list
#!/bin/bash
# This script intend to mimic TimeMachine exclude list.
# As the exclude list can evolve between backups it has to be rebuilt before every backup
# Apple uses 5 types of excludes:
# - 4 `/System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist`
# - and files from applications where metadata says to not backup, these can be found usinr
# `sudo mdfind "com_apple_backup_excludeItem = 'com.apple.backupd'"`
PLISTBUDDY=/usr/libexec/PlistBuddy
RESTIC=/usr/local/bin/restic
STDEXCLUSIONS=/System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist
if [[ ! -x $PLISTBUDDY ]]; then
syslog -s -k Facility com.apple.console Level Error Sender ResticBackupUserScript Message \
"Can't find $PLISTBUDDY"
exit 1
fi
if [[ $# -eq 1 ]]; then
exec > "$1"
fi
printf "# PathsExcluded\n"
$PLISTBUDDY $STDEXCLUSIONS -c "Print PathsExcluded" \
| sed -En 's,^[[:space:]]+,,p'
printf "\n# ContentsExcluded\n"
$PLISTBUDDY $STDEXCLUSIONS -c "Print ContentsExcluded" \
| sed -En 's,^[[:space:]]+(.*),\1/\*,p'
printf "\n# FileContentsExcluded\n"
$PLISTBUDDY $STDEXCLUSIONS -c "Print FileContentsExcluded" \
| sed -En 's,^[[:space:]]+,,p' \
| xargs -I _ find _ \! -type d
printf "\n# UserPathsExcluded\n"
$PLISTBUDDY $STDEXCLUSIONS -c "Print UserPathsExcluded" \
| sed -En 's,^[[:space:]]+,/Users/*/,p'
printf "\n# Applications files excluded\n"
mdfind "com_apple_backup_excludeItem = 'com.apple.backupd'"
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>org.resticuser.daemon</string>
<key>UserName</key>
<string>root</string>
<key>GroupName</key>
<string>admin</string>
<key>InitGroups</key>
<true/>
<key>Program</key>
<string>/usr/local/sbin/resticuser.sh</string>
<key>RunAtLoad</key>
<true/>
<key>StandardErrorPath</key>
<string>/tmp/org.resticuser.daemon.stderr</string>
<key>StandardOutPath</key>
<string>/tmp/org.resticuser.daemon.stdout</string>
<key>StartInterval</key>
<integer>3600</integer>
<key>WorkingDirectory</key>
<string>/Users/YOURUSERNAME</string>
<key>AbandonProcessGroup</key>
<true/>
</dict>
#!/bin/bash
# This script intend to optimize user dir backup by restic.
# It will run hourly (using launchd). Another daily script will do a complete backup.
# This script avoids running if another instance is already doing the job
# (as sometime a backup can last more than one hour, or the daily one can last).
# It builds a special exclude list, inspired by Apple TimeMachine exclude list before running restic.
# Edit lines 35 to 42
WHICH=/usr/bin/which;
BASENAME=$($WHICH basename);
CAT=$($WHICH cat);
DATE=$($WHICH date);
DSCL=$($WHICH dscl);
ECHO=$($WHICH echo);
GREP=$($WHICH grep);
HOSTNAME=$($WHICH hostname);
LOGGER=$($WHICH logger);
MAIL=$($WHICH mail);
MKDIR=$($WHICH mkdir);
PRINTF=$($WHICH printf);
RESTIC=$($WHICH restic);
RM=$($WHICH rm);
SLEEP=$($WHICH sleep);
TOUCH=$($WHICH touch);
WC=$($WHICH wc);
TEMPDIR=/tmp/restictemp;
LOCKFILE=$TEMPDIR/restic.pid;
EXCLUDEFILE=$TEMPDIR/tmpexcl.txt;
LOGFILE=$TEMPDIR/restic.log;
THISHOST=$($HOSTNAME -s);
ADMINMAIL=YourEmail;
REPO=YourRepoRef (i.e. 3:https://us-east-1@s3.wasabisys.com/repo-name);
#Restic repo on Wasabi
export RESTIC_PASSWORD="YourResticRepoPassword"
export RESTIC_REPOSITORY="YourResticRepository"
export AWS_ACCESS_KEY_ID="YourWasabiAccessKey"
export AWS_SECRET_ACCESS_KEY="YourWasabiSecretAccessKey"
#set -x
# trap handler: print location of last error and process it further
#
function my_trap_handler()
{
MYSELF="$0" # equals to my script name
LASTLINE="$1" # argument 1: last line of error occurence
LASTERR="$2" # argument 2: error code of last command
$LOGGER -i -t '#ResticUser' -f /var/log/restic.log "pid="$$ "Error in line ${LASTLINE}: exit status of last command: ${LASTERR}\n"
$PRINTF "ResticUser encountered an error at `$DATE` in ${MYSELF}: line ${LASTLINE}: exit status of last command: ${LASTERR}\n" >> $LOGFILE
}
function my_exit_handler()
{
$LOGGER -i -t '#ResticUser' -f /var/log/restic.log "pid="$$ "End of script. Exiting"
$PRINTF "Restic user script on $THISHOST ends at `$DATE`\n" >> $LOGFILE
$CAT $LOGFILE | $MAIL -s "Restic User script for $THISHOST" $ADMINMAIL
#if lockfile is mine remove temp directory
LOCKID=$($CAT $LOCKFILE)
if [ $LOCKID -eq $$ ]; then
$LOGGER -i -t '#ResticUser' -f /var/log/restic.log "pid="$$ "Removing temp directory."
[ -d ${TEMPDIR} ] && $RM -rf ${TEMPDIR}
fi
}
function adduserexcludes()
{
USERDIR=$1;
DESTFILE=$2;
#Add some user specific excludes
$FIND /Users/$USERDIR/.vagrant.d/ -iname '*.vmdk' 2>/dev/null>> $DESTFILE
$ECHO "/Users/$USERDIR/Documents/Compta_D_et_T/Backup/*.ova" >> $DESTFILE
$ECHO "/Users/$USERDIR/Documents/Compta_D_et_T/Backup/*.tgz" >> $DESTFILE
$ECHO "/Users/$USERDIR/Music/iTunes/iTunes Media/Mobile Applications/" >> $DESTFILE
$ECHO "/Users/$USERDIR/.Trash" >> $DESTFILE
$ECHO "/Users/$USERDIR/Library/Application Support/MobileSync/Backup/*" >> $DESTFILE
#Add some forgotten mail V3 excludes copied from V2
$ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/Envelope Index" >> $DESTFILE
$ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/Envelope Index-journal" >> $DESTFILE
$ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/AvailableFeeds" >> $DESTFILE
$ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/BackingStoreUpdateJournal" >> $DESTFILE
$ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/Envelope Index-shm" >> $DESTFILE
$ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/Envelope Index-wal" >> $DESTFILE
}
# trap commands with non-zero exit code
# trap script EXIT
#
trap 'my_trap_handler ${LINENO} $?' ERR
trap 'my_exit_handler' EXIT
#Verify if lock exists
if [ -f $LOCKFILE ]; then
$PRINTF "Restic : Other instance running on $THISHOST at `$DATE`, Restic script exits\n" >> $LOGFILE
$LOGGER -i -t '#ResticUser' -f /var/log/restic.log "pid="$$ "Other instance is running, exiting."
exit 0
fi
#Create a tempdir
if [ ! -d $TEMPDIR ]; then
$MKDIR $TEMPDIR
fi
#Create pidfile
$ECHO $$ > $LOCKFILE
#Create system log file
if [ ! -f /var/log/restic.log ]; then
$TOUCH /var/log/restic.log
fi
#Build Exclude list
$PRINTF "Restic user script on $THISHOST builds exclude list at `$DATE`\n\n" >> $LOGFILE
source /usr/local/sbin/buildresticexcludes.sh $EXCLUDEFILE
#Add restic temp dir to exclude list
$ECHO $TEMPDIR >> $EXCLUDEFILE
# As we're only dealing with user directory let's optimize our exclude list
mv $EXCLUDEFILE $TEMPDIR/temptemp.txt
$CAT $TEMPDIR/temptemp.txt | $GREP '/Users/*' >> $EXCLUDEFILE
# Find list of users
users=$($DSCL . list /Users | $GREP -v '^_')
for i in $users; do
# check if some file has been changed during last hour and forget about 'no such dir' errors
changed=$($FIND /Users/$i ! -name '.DS_Store' -type f -mmin -60 -print -quit 2>/dev/null | $WC -l)
if [ $changed -ne 0 ]; then
#As it seems better to not use wildcards in excludes we build the user specific excludes
adduserexcludes $i $EXCLUDEFILE
# do restic backup for this user
$LOGGER -i -t '#ResticUser' -f /var/log/restic.log "Restic backup begins"
$PRINTF "Backup for $THISHOST begins at `$DATE`\n" >> $LOGFILE
$RESTIC -r $REPO backup /Users/$i --exclude-file=$EXCLUDEFILE --tag UserDir --cleanup-cache 2>&1| tee -a $LOGFILE
$LOGGER -i -t '#ResticUser' -f /var/log/restic.log "Restic backup ends"
$PRINTF "Backup for $THISHOST ends at `$DATE`\n" >> $LOGFILE
fi
done
exit 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment