Bash script to build a restic exclude list that mimics Apple TimeMachine exclude list

buildresticexcludes.sh

#!/bin/bash

# This script intend to mimic TimeMachine exclude list.
# As the exclude list can evolve between backups it has to be rebuilt before every backup
# Apple uses 5 types of excludes, four from the /System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist file
# and files from applications where metadata says to not backup, these can be found usinf
# sudo mdfind "com_apple_backup_excludeItem = 'com.apple.backupd'"

SYSLOG=/usr/bin/syslog;
TEMPFILE=$1;
PLISTBUDDY=/usr/libexec/PlistBuddy;
MDFIND=/usr/bin/mdfind;
FIND=/usr/bin/find;
RESTIC=/usr/local/bin/restic;
ECHO=/bin/echo;

# Verify if PlistBuddy is installed if not exit
if [ ! -f $PLISTBUDDY ]; then
$SYSLOG -s -k Facility com.apple.console Level Error Sender ResticBackupUserScript Message  "Can't find PlistBuddy"
exit 1
fi

# Excludes to keep directory structure without files
$FIND /private/var/log -type f  > $TEMPFILE
$FIND /private/var/spool/cups -type f  >> $TEMPFILE
$FIND /private/var/spool/fax -type f  >> $TEMPFILE
$FIND /private/var/spool/uucp -type f  >> $TEMPFILE

$ECHO -e "# PathsExcluded\n#" >> $TEMPFILE
$PLISTBUDDY /System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist -c "Print PathsExcluded" | grep -v '{' |grep -v '}'| sed -e 's/^[[:space:]]*//' >> $TEMPFILE

$ECHO -e "#\n# ContentsExcluded\n#" >> $TEMPFILE
$PLISTBUDDY /System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist -c "Print ContentsExcluded" | grep -v '{' |grep -v '}'| sed -e 's/^[[:space:]]*//' | sed -e 's/$/\/\*/' >> $TEMPFILE


$ECHO -e "#\n# UserPathsExcluded\n#" >> $TEMPFILE
$PLISTBUDDY /System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist -c "Print UserPathsExcluded" | grep -v '{' |grep -v '}'| sed -e 's/^[[:space:]]*/\/Users\/\*\//'  >> $TEMPFILE

$ECHO -e "#\n# Applications files excluded\n#" >> $TEMPFILE
$MDFIND "com_apple_backup_excludeItem = 'com.apple.backupd'"  >> $TEMPFILE
# Exclude DocumentRevision files
$ECHO "/.DocumentRevisions-V100/" >> $TEMPFILE

org.resticuser.daemon.plist

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
	<key>Label</key>
	<string>org.resticuser.daemon</string>
	<key>UserName</key>
	<string>root</string>
	<key>GroupName</key>
	<string>admin</string>
	<key>InitGroups</key>
	<true/>
	<key>Program</key>
	<string>/usr/local/sbin/resticuser.sh</string>
	<key>RunAtLoad</key>
	<true/>
	<key>StandardErrorPath</key>
	<string>/tmp/org.resticuser.daemon.stderr</string>
	<key>StandardOutPath</key>
	<string>/tmp/org.resticuser.daemon.stdout</string>
	<key>StartInterval</key>
	<integer>3600</integer>
	<key>WorkingDirectory</key>
	<string>/Users/YOURUSERNAME</string>
	<key>AbandonProcessGroup</key>
	<true/>
</dict>

resticsuser.sh

#!/bin/bash

# This script intend to optimize user dir backup by restic.
# It will run hourly (using launchd). Another daily script will do a complete backup.
# This script avoids running if another instance is already doing the job
# (as sometime a backup can last more than one hour, or the daily one can last).
# It builds a special exclude list, inspired by Apple TimeMachine exclude list before running restic.

# Edit lines 35 to 42

WHICH=/usr/bin/which;

BASENAME=$($WHICH basename);
CAT=$($WHICH cat);
DATE=$($WHICH date);
DSCL=$($WHICH dscl);
ECHO=$($WHICH echo);
GREP=$($WHICH grep);
HOSTNAME=$($WHICH hostname);
LOGGER=$($WHICH logger);
MAIL=$($WHICH mail);
MKDIR=$($WHICH mkdir);
PRINTF=$($WHICH printf);
RESTIC=$($WHICH restic);
RM=$($WHICH rm);
SLEEP=$($WHICH sleep);
TOUCH=$($WHICH touch);
WC=$($WHICH wc);

TEMPDIR=/tmp/restictemp;
LOCKFILE=$TEMPDIR/restic.pid;
EXCLUDEFILE=$TEMPDIR/tmpexcl.txt;
LOGFILE=$TEMPDIR/restic.log;
THISHOST=$($HOSTNAME -s);
ADMINMAIL=YourEmail;
REPO=YourRepoRef (i.e. 3:https://us-east-1@s3.wasabisys.com/repo-name);

#Restic repo on Wasabi
export RESTIC_PASSWORD="YourResticRepoPassword"
export RESTIC_REPOSITORY="YourResticRepository"
export AWS_ACCESS_KEY_ID="YourWasabiAccessKey"
export AWS_SECRET_ACCESS_KEY="YourWasabiSecretAccessKey"
#set -x

# trap handler: print location of last error and process it further
#
function my_trap_handler()
{
        MYSELF="$0"               # equals to my script name
        LASTLINE="$1"            # argument 1: last line of error occurence
        LASTERR="$2"             # argument 2: error code of last command
	$LOGGER -i -t '#ResticUser' -f /var/log/restic.log "pid="$$ "Error in  line ${LASTLINE}: exit status of last command: ${LASTERR}\n"
        $PRINTF "ResticUser encountered an error at `$DATE` in ${MYSELF}: line ${LASTLINE}: exit status of last command: ${LASTERR}\n" >> $LOGFILE
}


function my_exit_handler()
{
	$LOGGER -i -t '#ResticUser' -f /var/log/restic.log "pid="$$ "End of script. Exiting"
	$PRINTF "Restic user script on $THISHOST ends at `$DATE`\n" >> $LOGFILE
    	$CAT $LOGFILE | $MAIL -s "Restic User script for $THISHOST" $ADMINMAIL
	#if lockfile is mine remove temp directory
	LOCKID=$($CAT $LOCKFILE)
	if [ $LOCKID -eq $$ ]; then
		$LOGGER -i -t '#ResticUser' -f /var/log/restic.log "pid="$$ "Removing temp directory."
		[ -d ${TEMPDIR} ] && $RM -rf ${TEMPDIR}
	fi
}


function adduserexcludes()
{
	USERDIR=$1;
	DESTFILE=$2;
	#Add some user specific excludes
	$FIND /Users/$USERDIR/.vagrant.d/ -iname '*.vmdk'  2>/dev/null>> $DESTFILE
	$ECHO "/Users/$USERDIR/Documents/Compta_D_et_T/Backup/*.ova" >> $DESTFILE
	$ECHO "/Users/$USERDIR/Documents/Compta_D_et_T/Backup/*.tgz" >> $DESTFILE
	$ECHO "/Users/$USERDIR/Music/iTunes/iTunes Media/Mobile Applications/" >> $DESTFILE
	$ECHO "/Users/$USERDIR/.Trash" >> $DESTFILE
	$ECHO "/Users/$USERDIR/Library/Application Support/MobileSync/Backup/*" >> $DESTFILE

	#Add some forgotten mail V3 excludes copied from V2
	$ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/Envelope Index"  >> $DESTFILE
	$ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/Envelope Index-journal" >> $DESTFILE
	$ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/AvailableFeeds" >> $DESTFILE
	$ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/BackingStoreUpdateJournal" >> $DESTFILE
	$ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/Envelope Index-shm" >> $DESTFILE
	$ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/Envelope Index-wal" >> $DESTFILE

}
# trap commands with non-zero exit code
# trap script EXIT
#

trap 'my_trap_handler ${LINENO} $?' ERR
trap 'my_exit_handler' EXIT


#Verify if lock exists
if [ -f $LOCKFILE ]; then
	$PRINTF "Restic : Other instance running on $THISHOST at `$DATE`, Restic script exits\n"  >> $LOGFILE
	$LOGGER -i -t '#ResticUser' -f /var/log/restic.log "pid="$$ "Other instance is running, exiting."
	exit 0
fi


#Create a tempdir
if [ ! -d $TEMPDIR ]; then
	$MKDIR $TEMPDIR
fi

#Create pidfile
$ECHO $$ > $LOCKFILE

#Create system log file
if [ ! -f /var/log/restic.log ]; then
	$TOUCH /var/log/restic.log
fi

#Build Exclude list
$PRINTF "Restic user script on $THISHOST builds exclude list at `$DATE`\n\n" >> $LOGFILE
source /usr/local/sbin/buildresticexcludes.sh $EXCLUDEFILE

#Add restic temp dir to exclude list
$ECHO $TEMPDIR >> $EXCLUDEFILE

# As we're only dealing with user directory let's optimize our exclude list
mv $EXCLUDEFILE $TEMPDIR/temptemp.txt
$CAT $TEMPDIR/temptemp.txt | $GREP '/Users/*' >> $EXCLUDEFILE



# Find list of users
users=$($DSCL . list /Users | $GREP -v '^_')
for i in $users; do
	# check if some file has been changed during last hour and forget about 'no such dir' errors
	changed=$($FIND /Users/$i  ! -name '.DS_Store' -type f -mmin -60 -print -quit 2>/dev/null | $WC -l)
	if [  $changed -ne 0 ]; then
		#As it seems better to not use wildcards in excludes we build the user specific excludes
		adduserexcludes $i $EXCLUDEFILE

		# do restic backup for this user
		$LOGGER -i -t '#ResticUser' -f /var/log/restic.log "Restic backup begins"
		$PRINTF "Backup for $THISHOST begins at `$DATE`\n" >> $LOGFILE
		$RESTIC -r $REPO backup /Users/$i --exclude-file=$EXCLUDEFILE --tag UserDir --cleanup-cache 2>&1| tee -a $LOGFILE
		$LOGGER -i -t '#ResticUser' -f /var/log/restic.log "Restic backup ends"
		$PRINTF "Backup for $THISHOST ends at `$DATE`\n" >> $LOGFILE
	fi
done
exit 0

Forgot to say: thanks for this super script! I'm using it myself now.

Cleaned the buildresticexcludes-sh script up: https://gist.github.com/huyz/a3711154a5ac476f06240e13a148fcf6#file-buildresticexcludes-sh

Anyone update to Big Sur yet? StdExclusions.plist is not a thing anymore. I took a look in backupd and seems like the exclusion rules are a bit more complex, but do get written out to the backup location.

This is painful to read. 👎

I did some research and it seems that the list is kept only at the top level of each time machine backup for Big Sur and beyond.

CCC maintains a list here. Will this work?

https://bombich.com/kb/ccc6/some-files-and-folders-are-automatically-excluded-from-backup-task

The CCC list isn't sufficient as it won't include the exclusions that you've manually added. This script above will include the exceptions that you've added for Time Machine.