-
-
Save Vartkat/1b097d8e9a6ad648bd3a356be86d97af to your computer and use it in GitHub Desktop.
#!/bin/bash | |
# This script intend to mimic TimeMachine exclude list. | |
# As the exclude list can evolve between backups it has to be rebuilt before every backup | |
# Apple uses 5 types of excludes, four from the /System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist file | |
# and files from applications where metadata says to not backup, these can be found usinf | |
# sudo mdfind "com_apple_backup_excludeItem = 'com.apple.backupd'" | |
SYSLOG=/usr/bin/syslog; | |
TEMPFILE=$1; | |
PLISTBUDDY=/usr/libexec/PlistBuddy; | |
MDFIND=/usr/bin/mdfind; | |
FIND=/usr/bin/find; | |
RESTIC=/usr/local/bin/restic; | |
ECHO=/bin/echo; | |
# Verify if PlistBuddy is installed if not exit | |
if [ ! -f $PLISTBUDDY ]; then | |
$SYSLOG -s -k Facility com.apple.console Level Error Sender ResticBackupUserScript Message "Can't find PlistBuddy" | |
exit 1 | |
fi | |
# Excludes to keep directory structure without files | |
$FIND /private/var/log -type f > $TEMPFILE | |
$FIND /private/var/spool/cups -type f >> $TEMPFILE | |
$FIND /private/var/spool/fax -type f >> $TEMPFILE | |
$FIND /private/var/spool/uucp -type f >> $TEMPFILE | |
$ECHO -e "# PathsExcluded\n#" >> $TEMPFILE | |
$PLISTBUDDY /System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist -c "Print PathsExcluded" | grep -v '{' |grep -v '}'| sed -e 's/^[[:space:]]*//' >> $TEMPFILE | |
$ECHO -e "#\n# ContentsExcluded\n#" >> $TEMPFILE | |
$PLISTBUDDY /System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist -c "Print ContentsExcluded" | grep -v '{' |grep -v '}'| sed -e 's/^[[:space:]]*//' | sed -e 's/$/\/\*/' >> $TEMPFILE | |
$ECHO -e "#\n# UserPathsExcluded\n#" >> $TEMPFILE | |
$PLISTBUDDY /System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist -c "Print UserPathsExcluded" | grep -v '{' |grep -v '}'| sed -e 's/^[[:space:]]*/\/Users\/\*\//' >> $TEMPFILE | |
$ECHO -e "#\n# Applications files excluded\n#" >> $TEMPFILE | |
$MDFIND "com_apple_backup_excludeItem = 'com.apple.backupd'" >> $TEMPFILE | |
# Exclude DocumentRevision files | |
$ECHO "/.DocumentRevisions-V100/" >> $TEMPFILE |
<?xml version="1.0" encoding="UTF-8"?> | |
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> | |
<plist version="1.0"> | |
<dict> | |
<key>Label</key> | |
<string>org.resticuser.daemon</string> | |
<key>UserName</key> | |
<string>root</string> | |
<key>GroupName</key> | |
<string>admin</string> | |
<key>InitGroups</key> | |
<true/> | |
<key>Program</key> | |
<string>/usr/local/sbin/resticuser.sh</string> | |
<key>RunAtLoad</key> | |
<true/> | |
<key>StandardErrorPath</key> | |
<string>/tmp/org.resticuser.daemon.stderr</string> | |
<key>StandardOutPath</key> | |
<string>/tmp/org.resticuser.daemon.stdout</string> | |
<key>StartInterval</key> | |
<integer>3600</integer> | |
<key>WorkingDirectory</key> | |
<string>/Users/YOURUSERNAME</string> | |
<key>AbandonProcessGroup</key> | |
<true/> | |
</dict> |
#!/bin/bash | |
# This script intend to optimize user dir backup by restic. | |
# It will run hourly (using launchd). Another daily script will do a complete backup. | |
# This script avoids running if another instance is already doing the job | |
# (as sometime a backup can last more than one hour, or the daily one can last). | |
# It builds a special exclude list, inspired by Apple TimeMachine exclude list before running restic. | |
# Edit lines 35 to 42 | |
WHICH=/usr/bin/which; | |
BASENAME=$($WHICH basename); | |
CAT=$($WHICH cat); | |
DATE=$($WHICH date); | |
DSCL=$($WHICH dscl); | |
ECHO=$($WHICH echo); | |
GREP=$($WHICH grep); | |
HOSTNAME=$($WHICH hostname); | |
LOGGER=$($WHICH logger); | |
MAIL=$($WHICH mail); | |
MKDIR=$($WHICH mkdir); | |
PRINTF=$($WHICH printf); | |
RESTIC=$($WHICH restic); | |
RM=$($WHICH rm); | |
SLEEP=$($WHICH sleep); | |
TOUCH=$($WHICH touch); | |
WC=$($WHICH wc); | |
TEMPDIR=/tmp/restictemp; | |
LOCKFILE=$TEMPDIR/restic.pid; | |
EXCLUDEFILE=$TEMPDIR/tmpexcl.txt; | |
LOGFILE=$TEMPDIR/restic.log; | |
THISHOST=$($HOSTNAME -s); | |
ADMINMAIL=YourEmail; | |
REPO=YourRepoRef (i.e. 3:https://us-east-1@s3.wasabisys.com/repo-name); | |
#Restic repo on Wasabi | |
export RESTIC_PASSWORD="YourResticRepoPassword" | |
export RESTIC_REPOSITORY="YourResticRepository" | |
export AWS_ACCESS_KEY_ID="YourWasabiAccessKey" | |
export AWS_SECRET_ACCESS_KEY="YourWasabiSecretAccessKey" | |
#set -x | |
# trap handler: print location of last error and process it further | |
# | |
function my_trap_handler() | |
{ | |
MYSELF="$0" # equals to my script name | |
LASTLINE="$1" # argument 1: last line of error occurence | |
LASTERR="$2" # argument 2: error code of last command | |
$LOGGER -i -t '#ResticUser' -f /var/log/restic.log "pid="$$ "Error in line ${LASTLINE}: exit status of last command: ${LASTERR}\n" | |
$PRINTF "ResticUser encountered an error at `$DATE` in ${MYSELF}: line ${LASTLINE}: exit status of last command: ${LASTERR}\n" >> $LOGFILE | |
} | |
function my_exit_handler() | |
{ | |
$LOGGER -i -t '#ResticUser' -f /var/log/restic.log "pid="$$ "End of script. Exiting" | |
$PRINTF "Restic user script on $THISHOST ends at `$DATE`\n" >> $LOGFILE | |
$CAT $LOGFILE | $MAIL -s "Restic User script for $THISHOST" $ADMINMAIL | |
#if lockfile is mine remove temp directory | |
LOCKID=$($CAT $LOCKFILE) | |
if [ $LOCKID -eq $$ ]; then | |
$LOGGER -i -t '#ResticUser' -f /var/log/restic.log "pid="$$ "Removing temp directory." | |
[ -d ${TEMPDIR} ] && $RM -rf ${TEMPDIR} | |
fi | |
} | |
function adduserexcludes() | |
{ | |
USERDIR=$1; | |
DESTFILE=$2; | |
#Add some user specific excludes | |
$FIND /Users/$USERDIR/.vagrant.d/ -iname '*.vmdk' 2>/dev/null>> $DESTFILE | |
$ECHO "/Users/$USERDIR/Documents/Compta_D_et_T/Backup/*.ova" >> $DESTFILE | |
$ECHO "/Users/$USERDIR/Documents/Compta_D_et_T/Backup/*.tgz" >> $DESTFILE | |
$ECHO "/Users/$USERDIR/Music/iTunes/iTunes Media/Mobile Applications/" >> $DESTFILE | |
$ECHO "/Users/$USERDIR/.Trash" >> $DESTFILE | |
$ECHO "/Users/$USERDIR/Library/Application Support/MobileSync/Backup/*" >> $DESTFILE | |
#Add some forgotten mail V3 excludes copied from V2 | |
$ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/Envelope Index" >> $DESTFILE | |
$ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/Envelope Index-journal" >> $DESTFILE | |
$ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/AvailableFeeds" >> $DESTFILE | |
$ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/BackingStoreUpdateJournal" >> $DESTFILE | |
$ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/Envelope Index-shm" >> $DESTFILE | |
$ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/Envelope Index-wal" >> $DESTFILE | |
} | |
# trap commands with non-zero exit code | |
# trap script EXIT | |
# | |
trap 'my_trap_handler ${LINENO} $?' ERR | |
trap 'my_exit_handler' EXIT | |
#Verify if lock exists | |
if [ -f $LOCKFILE ]; then | |
$PRINTF "Restic : Other instance running on $THISHOST at `$DATE`, Restic script exits\n" >> $LOGFILE | |
$LOGGER -i -t '#ResticUser' -f /var/log/restic.log "pid="$$ "Other instance is running, exiting." | |
exit 0 | |
fi | |
#Create a tempdir | |
if [ ! -d $TEMPDIR ]; then | |
$MKDIR $TEMPDIR | |
fi | |
#Create pidfile | |
$ECHO $$ > $LOCKFILE | |
#Create system log file | |
if [ ! -f /var/log/restic.log ]; then | |
$TOUCH /var/log/restic.log | |
fi | |
#Build Exclude list | |
$PRINTF "Restic user script on $THISHOST builds exclude list at `$DATE`\n\n" >> $LOGFILE | |
source /usr/local/sbin/buildresticexcludes.sh $EXCLUDEFILE | |
#Add restic temp dir to exclude list | |
$ECHO $TEMPDIR >> $EXCLUDEFILE | |
# As we're only dealing with user directory let's optimize our exclude list | |
mv $EXCLUDEFILE $TEMPDIR/temptemp.txt | |
$CAT $TEMPDIR/temptemp.txt | $GREP '/Users/*' >> $EXCLUDEFILE | |
# Find list of users | |
users=$($DSCL . list /Users | $GREP -v '^_') | |
for i in $users; do | |
# check if some file has been changed during last hour and forget about 'no such dir' errors | |
changed=$($FIND /Users/$i ! -name '.DS_Store' -type f -mmin -60 -print -quit 2>/dev/null | $WC -l) | |
if [ $changed -ne 0 ]; then | |
#As it seems better to not use wildcards in excludes we build the user specific excludes | |
adduserexcludes $i $EXCLUDEFILE | |
# do restic backup for this user | |
$LOGGER -i -t '#ResticUser' -f /var/log/restic.log "Restic backup begins" | |
$PRINTF "Backup for $THISHOST begins at `$DATE`\n" >> $LOGFILE | |
$RESTIC -r $REPO backup /Users/$i --exclude-file=$EXCLUDEFILE --tag UserDir --cleanup-cache 2>&1| tee -a $LOGFILE | |
$LOGGER -i -t '#ResticUser' -f /var/log/restic.log "Restic backup ends" | |
$PRINTF "Backup for $THISHOST ends at `$DATE`\n" >> $LOGFILE | |
fi | |
done | |
exit 0 |
Forgot to say: thanks for this super script! I'm using it myself now.
Cleaned the buildresticexcludes-sh
script up: https://gist.github.com/huyz/a3711154a5ac476f06240e13a148fcf6#file-buildresticexcludes-sh
Anyone update to Big Sur yet? StdExclusions.plist is not a thing anymore. I took a look in backupd and seems like the exclusion rules are a bit more complex, but do get written out to the backup location.
This is painful to read. 👎
I did some research and it seems that the list is kept only at the top level of each time machine backup for Big Sur and beyond.
CCC maintains a list here. Will this work?
https://bombich.com/kb/ccc6/some-files-and-folders-are-automatically-excluded-from-backup-task
The CCC list isn't sufficient as it won't include the exclusions that you've manually added. This script above will include the exceptions that you've added for Time Machine.
I don't knwow, I'm not a pro devel, I just found it on some examples and find it clean writing. Perhaps it's only necessary on Linux...