/RestoreExcludes.txt Secret
Last active
October 10, 2015 19:18
Star
You must be signed in to star a gist
DeployStudio add-on User Backup Restore script set
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/sh | |
### v.2 Allister Banks, 318 Inc. - No warranty granted or implied | |
echo "Checking for Backups to restore to currently booted machine with serial number $DS_SERIAL_NUMBER" | |
declare -x RESTORE_SOURCE="$DS_REPOSITORY_PATH/Backups/$DS_SERIAL_NUMBER/" | |
if [ ! "$(ls -A "$RESTORE_SOURCE")" ]; then | |
echo "HEY! Jumping the gun, please run Backup workflow first" | |
exit 1 | |
fi | |
exit 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
- .DS_Store | |
- /Shared | |
- /Guest | |
- /*Deleted* | |
- /*/Library/Application Support/SyncServices/data.version | |
- /*/Library/Application Support/Firefox/Crash Reports | |
- /*/Library/Application Support/Ubiquity | |
- /*/Library/Caches | |
- /*/Library/Logs | |
- /*/Library/Mail/Envelope Index | |
- /*/Library/Mail/Envelope Index-journal | |
- /*/Library/Mail/AvailableFeeds | |
- /*/Library/Mail/Metadata/BackingStoreUpdateJournal | |
- /*/Library/Mail/V2/MailData/Envelope Index | |
- /*/Library/Mail/V2/MailData/Envelope Index-journal | |
- /*/Library/Mail/V2/MailData/AvailableFeeds | |
- /*/Library/Mail/V2/MailData/BackingStoreUpdateJournal | |
- /*/Library/Mail/V2/MailData/Envelope Index-shm | |
- /*/Library/Mail/V2/MailData/Envelope Index-wal | |
- /*/Library/Mirrors | |
- /*/Library/PubSub/Database | |
- /*/Library/PubSub/Downloads | |
- /*/Library/PubSub/Feeds | |
- /*/Library/Safari/Icons.db | |
- /*/Library/Safari/WebpageIcons.db | |
- /*/Library/Safari/HistoryIndex.sk | |
- /*/.FileSync | |
- /*/.Trash | |
- /*/.dropbox | |
- /*/Dropbox | |
- /*/Downloads/*.dmg | |
- /*/Downloads/*.pkg |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
- /groups | |
- /hashes | |
- /*.plist |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
review hdiutil and rsync options, tee rsync section of log, die if no space on backup | |
step zero | |
from DeployStudio\ Admin.app/Contents/Frameworks/DSCore.framework/Versions/A/Resources/Tools/rsync, cp to /Backups/admin in repo | |
designate user data to backup | |
- only rsync excludes on backup step necessary | |
v1 WON'T IMPLEMENT: csv for moving backups from one machine to another | |
step one | |
backup script | |
functions: | |
√die = runtime abort | |
- check versions | |
√- check paths | |
logging (w/ tee to split between verbose/info) | |
√- both output of stats and size of sparseimage are important | |
detect | |
- OS, and sys_profile to plist w/backup? | |
- space on backup destination | |
√- admin group membership (starting with just that- if different OS may be unnecessary anyway?) | |
√excluded users - part of rsync exclude file, grabbing user records anyway | |
√- show du across all partitions (starting with backupSource for session) | |
v1 WON'T IMPLEMENT: filavault1/deleted_users detection/reporting? | |
v1 WON'T IMPLEMENT- prioritization based on size? | |
√exclusions (Caches, .Trashes, etc - what TM ignores as well) | |
√backup_destination(starting with default or hard-coded mountpoint) | |
√backup_userRecord(starting with all on mountpoint) | |
√backup_Old_userPassword(grabbing hash folder if not empty) | |
√backup_userdata | |
- tool | |
√ - rsync out (sparse, a la CCC, not worth time/compression) | |
√ - "$DS_REPOSITORY_PATH/Backups/admin/rsync" -aNHhAXx --protect-args --fileflags --force-change --stats --progress --exclude='*.ipsw' --filter="merge $DS_REPOSITORY_PATH/Backups/admin/excludes.txt" /Users/ $DS_REPOSITORY_PATH/Backups (a switch is same as rlptgoD) | |
WON'T IMPLEMENT: puppetstrings not working for hdiutil, probably not worth bothering since ~30 seconds for ~300GBs | |
v1 WON'T IMPLEMENT: filevault2 drive unlock | |
step two | |
√preflight verify backup first | |
step three | |
actual restore | |
die = runtime abort | |
- check versions | |
√- check paths | |
logging (w/ tee to split between verbose/info) | |
√path to restore users | |
√path to pull backups from | |
- default is most recent due to file naming | |
v1 WON'T IMPLEMENT: mtime, disregard filename | |
detect destination | |
v1 WON'T IMPLEMENT: - fix uid's to avoid collision | |
v1 WON'T IMPLEMENT: if root folders excluded, insert dummies (not paranoid) | |
v1 WON'T IMPLEMENT: - warn if newer home folders than destination | |
v1 WON'T IMPLEMENT: verify_df_vs_backups | |
√ restore_userRecord | |
√ restore_Old_userPassword | |
√ restore_userdata | |
√- open sparse and rsync out, a la CCC | |
√- progress | |
v1 WON'T IMPLEMENT: fix_uid's - plistBuddy, chown, ByHost | |
v1 WON'T IMPLEMENT: optional_fix_groupMembership |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/sh | |
### sonOfRestoreScript v .3 Allister Banks, 318 Inc. - No warranty granted or implied | |
### This process relies upon a patched version of rsync placed in the following path: | |
declare -r rsync="$DS_REPOSITORY_PATH/Backups/admin/rsync" | |
### Customiztion is allowed for sources to take the backed up user folders from, and destination to restore them | |
set -x | |
declare -rx mktemp="/usr/bin/mktemp" | |
declare -rx hdiutil="/usr/bin/hdiutil" | |
declare -rx rm="/bin/rm" | |
declare -rx cp="/bin/cp" | |
declare -rx cat="/bin/cat" | |
declare -rx mv="/bin/mv" | |
declare -rx date="/bin/date" | |
declare -rx mkdir="/bin/mkdir" | |
declare -rx basename="/usr/bin/basename" | |
declare -rx du="/usr/bin/du" | |
declare -rx tail="/usr/bin/tail" | |
declare -rx df="/bin/df" | |
declare -rx python="/usr/bin/python" | |
declare -x HARD_CODED_SPARSE_SOURCE='' | |
declare -x HARD_CODED_DESTINATION='/Volumes/318admin-Sept-MoLoClientRestored/Users/' | |
# declare -r MAIN_LOG="$DS_REPOSITORY_PATH/Logs/$DS_SERIAL_NUMBER.log" | |
# declare -r VERBOSE_LOG="$DS_REPOSITORY_PATH/Logs/Backup/$DS_SERIAL_NUMBER-verbose.log" | |
declare -rx DEFAULT_SPARSE_SOURCE="$DS_REPOSITORY_PATH/Backups/$DS_SERIAL_NUMBER" | |
declare -x DEFAULT_RESTORE_DESTINATION="/Volumes/$DS_LAST_RESTORED_VOLUME" | |
declare -x RSYNCEXCLUDESPATH="$DS_REPOSITORY_PATH/Backups/admin/RestoreExcludes.txt" | |
declare -rx TIMESTAMP=`$date "+%H:%M:%S"` | |
declare -rx SCRIPT="${0##*/}" | |
help(){ | |
$cat<<EOF | |
Usage: $SCRIPT [ -s "/Volumes/Macintosh HD" ] [ -d "/Volumes/External Drive/" ] | |
Variables can be set in DeployStudio variables window when running script. | |
BackupRestore Variables: | |
-s Source where backup is stored | |
Trailing slash on user folder is important! | |
Specify full path to sparseimage | |
Default is /tmp/DSNetworkRepository/Backups/$DS_SERIAL_NUMBER/ | |
-d Destination to store | |
Specify full path to mount point of volume | |
Default is the \$DS_LAST_RESTORED_VOLUME volume | |
e.g. "/Volumes/Macintosh HD" | |
EOF | |
} | |
header(){ | |
$python -c "print '-' * 120" | |
} | |
die(){ | |
echo "$1 Failed." | |
$hdiutil eject "$IMAGE_MOUNT_PT" | |
$rm -r $IMAGE_MOUNT_PT | |
exit 1 | |
} | |
checkSetup(){ | |
if [ ! -e $rsync ]; then | |
die "no custom rsync in expected location" | |
fi | |
} | |
checkRestoreSource(){ | |
IMAGE_MOUNT_PT=`$mktemp -d "/tmp/tmp-pt"` | |
if [ -z "$HARD_CODED_SPARSE_SOURCE" -a -e "$DEFAULT_SPARSE_SOURCE" ]; then | |
RESTORE_SOURCE=`ls $DEFAULT_SPARSE_SOURCE/$DS_SERIAL_NUMBER-*.sparseimage | $tail -n1` | |
$hdiutil mount "$RESTORE_SOURCE" -owners on -readwrite -noverify -nobrowse -mountpoint "$IMAGE_MOUNT_PT" | |
elif [ -d "$HARD_CODED_SPARSE_SOURCE" ]; then | |
RESTORE_SOURCE="$HARD_CODED_SPARSE_SOURCE" | |
$hdiutil mount "$RESTORE_SOURCE" -owners on -readwrite -noverify -nobrowse -mountpoint "$IMAGE_MOUNT_PT" | |
else | |
die "Neither default nor hard-coded custom path to restore sparse image available" | |
fi | |
header && echo "restore Started at $TIMESTAMP" | |
header && printf "\n%s\n" "***** Size of stored user folders:" | |
$du -h -d 1 -I "Shared" "$IMAGE_MOUNT_PT"/ | |
printf "\n" | |
} | |
checkRestoreDestination(){ | |
if [ ! -z "$HARD_CODED_DESTINATION" ]; then | |
if [ ! -d "$HARD_CODED_DESTINATION" ]; then | |
die "Custom backup destination not a directory or unreachable" | |
else | |
RESTORE_DESTINATION="$HARD_CODED_DESTINATION" | |
fi | |
else | |
RESTORE_DESTINATION="$DEFAULT_RESTORE_DESTINATION/Users/" | |
fi | |
header && printf "\n%s\n" "***** Free space and percentage full on destination before restore:" | |
$df -h $RESTORE_DESTINATION | awk '{print $4, $5}' | |
printf "\n" | |
} | |
restoreData(){ | |
header && echo "***** Starting to move the data, there may be some lag while the log is simultaneously being written to"; echo " " | |
$rsync -aANHhXx --stats --progress --fake-super --filter="merge $RSYNCEXCLUDESPATH" "$IMAGE_MOUNT_PT/" "$RESTORE_DESTINATION" >/dev/stdout | |
} | |
moveUsersPassAndFixup(){ | |
# declare -x GROUP_REC_PATH="$RESTORE_DESTINATION../private/var/db/dslocal/nodes/Default/groups" | |
USER_RECORD_PATH="$RESTORE_DESTINATION../private/var/db/dslocal/nodes/Default/users" | |
USER_HASH_PATH="$RESTORE_DESTINATION../private/var/db/shadow/hash" | |
header && printf "\n%s" "***** Data move done, copying user records and passwords (if in old format)" | |
echo " " | |
for u in $IMAGE_MOUNT_PT/*.plist; | |
do | |
$cp -v "$u" "$USER_RECORD_PATH" | |
done | |
if [ -d "$IMAGE_MOUNT_PT/hashes" -a "$(ls -A "$IMAGE_MOUNT_PT/hashes/")" ]; then | |
if [ ! -d "$USER_HASH_PATH" ]; then | |
$mkdir -p "$USER_HASH_PATH" || die | |
fi | |
$cp -R "$IMAGE_MOUNT_PT/hashes/" "$USER_HASH_PATH" | |
fi | |
$hdiutil eject "$IMAGE_MOUNT_PT" | |
header && printf "\n%s\n" "***** Disk image ejected, free space and percentage full on destination post-restore:" | |
$df -h $RESTORE_DESTINATION | awk '{print $4, $5}' | |
printf "\n" | |
} | |
cleanup(){ | |
$rm -r $IMAGE_MOUNT_PT | |
} | |
checkSetup | |
checkRestoreSource | |
checkRestoreDestination | |
while getopts :s:d:h opt; do | |
case "$opt" in | |
s) RESTORE_SOURCE="$OPTARG";; | |
d) RESTORE_DESTINATION="$OPTARG";; | |
h) | |
help | |
exit 0;; | |
\?) | |
echo "Sorry, "$OPTARG" is not understood. For more help, run: $SCRIPT -h" | |
echo "Usage: $SCRIPT [ -s Sparseimage Location for Users to Restore ] [ -d Destination to Restore Backup ]" | |
exit 0;; | |
esac | |
done | |
restoreData | |
moveUsersPassAndFixup | |
cleanup | |
exit 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/sh | |
### sonOfBackupScript v .8 Allister Banks, 318 Inc. - No warranty granted or implied | |
### This process relies upon a patched version of rsync placed in the following path: | |
declare -r rsync="$DS_REPOSITORY_PATH/Backups/admin/rsync" | |
### Customiztion is allowed for sources to take the user folders from, and destination to store them | |
# set -x | |
### Currently the minimum size of the sparseimage created is ~300MBs, there is a maximum of 100GB - | |
# if the home folders being backed up exceed this size, IT WILL FAIL!! | |
declare -rx mktemp="/usr/bin/mktemp" | |
declare -rx uuidgen="/usr/bin/uuidgen" | |
declare -rx hdiutil="/usr/bin/hdiutil" | |
declare -rx rm="/bin/rm" | |
declare -rx cp="/bin/cp" | |
declare -rx cat="/bin/cat" | |
declare -rx mv="/bin/mv" | |
declare -rx mkdir="/bin/mkdir" | |
declare -rx basename="/usr/bin/basename" | |
declare -rx du="/usr/bin/du" | |
declare -rx df="/bin/df" | |
declare -rx python="/usr/bin/python" | |
declare -rx dscl="/usr/bin/dscl" | |
declare -x HARD_CODED_USER_SOURCE='' # don't forget trailing slash! | |
### INCREASE THE FOLLOWING AS NECESSARY! | |
declare -r USER_DATA_DMG_SIZE="10g" # since large sparse images take up extra time to create | |
declare -r HARD_CODED_DESTINATION='' | |
# declare -r MAIN_LOG="$DS_REPOSITORY_PATH/Logs/$DS_SERIAL_NUMBER.log" | |
# declare -r VERBOSE_LOG="$DS_REPOSITORY_PATH/Logs/Backup/$DS_SERIAL_NUMBER-verbose.log" | |
declare -r DEFAULT_BACKUP_PATH="$DS_REPOSITORY_PATH/Backups/$DS_SERIAL_NUMBER" | |
declare -x DEFAULT_BACKUP_SOURCE=`system_profiler SPSerialATADataType | | |
awk -F': ' '/Mount Point/ { print $2}'|head -n1` # first mounted volume | |
declare -r RSYNCEXCLUDESPATH="$DS_REPOSITORY_PATH/Backups/admin/meSpecificExcludes.txt" | |
declare -r DATE_FOR_BACKUP_NAME=`date "+%m%d-%H%M%S"` #monthday-hourminutesecond, e.g. 0919-114348 | |
declare -x TIMESTAMP=`date "+%H:%M:%S"` | |
declare -x SCRIPT="${0##*/}" | |
help(){ | |
$cat<<EOF | |
Usage: $SCRIPT [ -s "/Volumes/Macintosh HD" ] [ -d "/Volumes/External Drive/" ] | |
Variables can be set in DeployStudio variables window when running script. | |
BackupRestore Variables: | |
-s Target volume | |
Specify full path to mount point of volume | |
Default is the \$DS_LAST_RESTORED_VOLUME volume | |
e.g. "/Volumes/Macintosh HD" | |
-d Backup destination | |
Specify full path to the backup volume | |
Default is /tmp/DSNetworkRepository | |
EOF | |
cleanup | |
} | |
header(){ | |
$python -c "print '-' * 80" | |
} | |
die(){ | |
echo "$1 Failed." | |
$rm -r $IMAGE_MOUNT_PT | |
$rm -r $IMAGE_ENCL_FOLDER | |
exit 1 | |
} | |
checkSetup(){ | |
header && echo "Backup Started at $TIMESTAMP" | |
if [ ! -e $rsync ]; then | |
die "no custom rsync in expected location" | |
fi | |
} | |
checkBackupSource(){ | |
if [ -z "$HARD_CODED_USER_SOURCE" -a -e "$DEFAULT_BACKUP_SOURCE" ]; then | |
BACKUP_SOURCE="$DEFAULT_BACKUP_SOURCE/Users/" | |
elif [ -d "$HARD_CODED_USER_SOURCE" ]; then | |
BACKUP_SOURCE="$HARD_CODED_USER_SOURCE" | |
else | |
die "Neither default nor hard-coded custom user data source available" | |
fi | |
header && printf "\n%s\n" "***** Size of user folders before exclusions:" | |
$du -h -d 1 -I "Shared" $BACKUP_SOURCE | |
printf "\n" | |
} | |
checkBackupDestination(){ | |
if [ ! -z "$HARD_CODED_DESTINATION" ]; then | |
if [ ! -d "$HARD_CODED_DESTINATION" ]; then | |
die "Custom backup destination not a directory or unreachable" | |
else | |
BACKUP_STORAGE="$HARD_CODED_DESTINATION" # string for path | |
fi | |
elif [ -z "$HARD_CODED_DESTINATION" -a ! -e "$DEFAULT_BACKUP_PATH" ]; then | |
$mkdir "$DEFAULT_BACKUP_PATH" || die "backup destination folder creation failed" | |
BACKUP_STORAGE="$DEFAULT_BACKUP_PATH" | |
else | |
BACKUP_STORAGE="$DEFAULT_BACKUP_PATH" | |
fi | |
header && printf "\n%s\n" "***** Free space and percentage full on destination before backup:" | |
$df -h $DS_REPOSITORY_PATH | awk '{print $4, $5}' | |
printf "\n" | |
} | |
createBackupSparseAndRsyncMinusExclusions(){ | |
IMAGE_MOUNT_PT=`$mktemp -d "/tmp/tmp-pt"` | |
IMAGE_ENCL_FOLDER=`$mktemp -d "$DS_REPOSITORY_PATH/.tmp-subfolder"` | |
IMAGE_FILENAME="$IMAGE_ENCL_FOLDER/`$uuidgen`" | |
IMAGE_FULLPATH="$IMAGE_FILENAME.sparseimage" | |
header && echo "Starting backup of $BACKUP_SOURCE" | |
header && echo "Creating $USER_DATA_DMG_SIZE disk image to retain ownership on files while backing up" | |
$hdiutil create -type SPARSE -size $USER_DATA_DMG_SIZE -fs JHFS+ -volname "$DS_SERIAL_NUMBER-Users" "$IMAGE_FULLPATH" | |
$hdiutil mount "$IMAGE_FULLPATH" -owners on -readwrite -noverify -nobrowse -mountpoint "$IMAGE_MOUNT_PT" | |
header && echo "***** Starting to move the data, there may be some lag while the log is simultaneously being written to"; echo " " | |
$rsync -aANHhXx --stats --progress --exclude='*.ipsw' --filter="merge $RSYNCEXCLUDESPATH" "$BACKUP_SOURCE" "$IMAGE_MOUNT_PT" >/dev/stdout | |
} | |
unmountAndmoveTheGoods(){ | |
header && printf "\n%s" "***** Data move done, copying user records and passwords (if in old format)" | |
GROUP_REC_PATH="$BACKUP_SOURCE../private/var/db/dslocal/nodes/Default/groups" | |
USER_RECORD_PATH="$BACKUP_SOURCE../private/var/db/dslocal/nodes/Default/users" | |
USER_HASH_PATH="$BACKUP_SOURCE../private/var/db/shadow/hash/" | |
echo " " | |
for u in "$BACKUP_SOURCE"*/; | |
do | |
$cp -v "$USER_RECORD_PATH/$($basename "$u").plist" "$IMAGE_MOUNT_PT" | |
done | |
$mkdir "$IMAGE_MOUNT_PT/groups" "$IMAGE_MOUNT_PT/hashes" | |
$cp -v "$GROUP_REC_PATH/admin.plist" "$IMAGE_MOUNT_PT/groups/" | |
if [ -d "$USER_HASH_PATH" -a "$(ls -A "$USER_HASH_PATH")" ]; then | |
$cp -Rv "$USER_HASH_PATH" "$IMAGE_MOUNT_PT/hashes" | |
fi | |
$hdiutil eject "$IMAGE_MOUNT_PT" | |
$mv "$IMAGE_FULLPATH" "$BACKUP_STORAGE/$DS_SERIAL_NUMBER-$DATE_FOR_BACKUP_NAME.sparseimage" | |
header && printf "\n%s\n" "***** Disk image ejected and moved, size and path to actual sparseimage:" | |
$du -h "$BACKUP_STORAGE/$DS_SERIAL_NUMBER-$DATE_FOR_BACKUP_NAME.sparseimage" | |
header && printf "\n%s\n" "***** Free space and percentage full on destination:" | |
$df -h $DS_REPOSITORY_PATH | awk '{print $4, $5}' | |
printf "\n" | |
} | |
cleanup(){ | |
$rm -r $IMAGE_MOUNT_PT | |
$rm -r $IMAGE_ENCL_FOLDER | |
header && echo "Backup finished $TIMESTAMP" | |
} | |
checkSetup | |
checkBackupSource | |
checkBackupDestination | |
while getopts :s:d:h opt; do | |
case "$opt" in | |
s) HARD_CODED_USER_SOURCE="$OPTARG";; | |
d) BACKUP_STORAGE="$OPTARG";; | |
h) | |
help | |
exit 0;; | |
\?) | |
echo "Sorry, "$OPTARG" is not understood. For more help, run: $SCRIPT -h" | |
echo "Usage: $SCRIPT [ -s Source of Users to Backup ] [ -d Destination to Store Backup ]" | |
exit 0;; | |
esac | |
done | |
createBackupSparseAndRsyncMinusExclusions | |
unmountAndmoveTheGoods | |
cleanup | |
exit 0 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment