secret
Last active

DeployStudio add-on User Backup Restore script set

  • Download Gist
RestoreExcludes.txt
1 2 3
- /groups
- /hashes
- /*.plist
failsafe.sh
Shell
1 2 3 4 5 6 7 8 9
#!/bin/sh
### v.2 Allister Banks, 318 Inc. - No warranty granted or implied
echo "Checking for Backups to restore to currently booted machine with serial number $DS_SERIAL_NUMBER"
declare -x RESTORE_SOURCE="$DS_REPOSITORY_PATH/Backups/$DS_SERIAL_NUMBER/"
if [ ! "$(ls -A "$RESTORE_SOURCE")" ]; then
echo "HEY! Jumping the gun, please run Backup workflow first"
exit 1
fi
exit 0
meSpecificExcludes.txt
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
- .DS_Store
- /Shared
- /Guest
- /*Deleted*
- /*/Library/Application Support/SyncServices/data.version
- /*/Library/Application Support/Firefox/Crash Reports
- /*/Library/Application Support/Ubiquity
- /*/Library/Caches
- /*/Library/Logs
- /*/Library/Mail/Envelope Index
- /*/Library/Mail/Envelope Index-journal
- /*/Library/Mail/AvailableFeeds
- /*/Library/Mail/Metadata/BackingStoreUpdateJournal
- /*/Library/Mail/V2/MailData/Envelope Index
- /*/Library/Mail/V2/MailData/Envelope Index-journal
- /*/Library/Mail/V2/MailData/AvailableFeeds
- /*/Library/Mail/V2/MailData/BackingStoreUpdateJournal
- /*/Library/Mail/V2/MailData/Envelope Index-shm
- /*/Library/Mail/V2/MailData/Envelope Index-wal
- /*/Library/Mirrors
- /*/Library/PubSub/Database
- /*/Library/PubSub/Downloads
- /*/Library/PubSub/Feeds
- /*/Library/Safari/Icons.db
- /*/Library/Safari/WebpageIcons.db
- /*/Library/Safari/HistoryIndex.sk
- /*/.FileSync
- /*/.Trash
- /*/.dropbox
- /*/Dropbox
- /*/Downloads/*.dmg
- /*/Downloads/*.pkg
sonOfBackupRestoreToDo.txt
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
review hdiutil and rsync options, tee rsync section of log, die if no space on backup
 
 
step zero
from DeployStudio\ Admin.app/Contents/Frameworks/DSCore.framework/Versions/A/Resources/Tools/rsync, cp to /Backups/admin in repo
designate user data to backup
- only rsync excludes on backup step necessary
v1 WON'T IMPLEMENT: csv for moving backups from one machine to another
 
step one
backup script
functions:
√die = runtime abort
- check versions
√- check paths
logging (w/ tee to split between verbose/info)
√- both output of stats and size of sparseimage are important
detect
- OS, and sys_profile to plist w/backup?
- space on backup destination
√- admin group membership (starting with just that- if different OS may be unnecessary anyway?)
√excluded users - part of rsync exclude file, grabbing user records anyway
√- show du across all partitions (starting with backupSource for session)
v1 WON'T IMPLEMENT: filavault1/deleted_users detection/reporting?
v1 WON'T IMPLEMENT- prioritization based on size?
√exclusions (Caches, .Trashes, etc - what TM ignores as well)
√backup_destination(starting with default or hard-coded mountpoint)
√backup_userRecord(starting with all on mountpoint)
√backup_Old_userPassword(grabbing hash folder if not empty)
√backup_userdata
- tool
√ - rsync out (sparse, a la CCC, not worth time/compression)
√ - "$DS_REPOSITORY_PATH/Backups/admin/rsync" -aNHhAXx --protect-args --fileflags --force-change --stats --progress --exclude='*.ipsw' --filter="merge $DS_REPOSITORY_PATH/Backups/admin/excludes.txt" /Users/ $DS_REPOSITORY_PATH/Backups (a switch is same as rlptgoD)
WON'T IMPLEMENT: puppetstrings not working for hdiutil, probably not worth bothering since ~30 seconds for ~300GBs
v1 WON'T IMPLEMENT: filevault2 drive unlock
 
 
step two
√preflight verify backup first
 
step three
actual restore
die = runtime abort
- check versions
√- check paths
logging (w/ tee to split between verbose/info)
√path to restore users
√path to pull backups from
- default is most recent due to file naming
v1 WON'T IMPLEMENT: mtime, disregard filename
detect destination
v1 WON'T IMPLEMENT: - fix uid's to avoid collision
v1 WON'T IMPLEMENT: if root folders excluded, insert dummies (not paranoid)
v1 WON'T IMPLEMENT: - warn if newer home folders than destination
v1 WON'T IMPLEMENT: verify_df_vs_backups
√ restore_userRecord
√ restore_Old_userPassword
√ restore_userdata
√- open sparse and rsync out, a la CCC
√- progress
v1 WON'T IMPLEMENT: fix_uid's - plistBuddy, chown, ByHost
v1 WON'T IMPLEMENT: optional_fix_groupMembership
sonOfRestore.sh
Shell
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146
#!/bin/sh
### sonOfRestoreScript v .3 Allister Banks, 318 Inc. - No warranty granted or implied
### This process relies upon a patched version of rsync placed in the following path:
declare -r rsync="$DS_REPOSITORY_PATH/Backups/admin/rsync"
### Customiztion is allowed for sources to take the backed up user folders from, and destination to restore them
set -x
declare -rx mktemp="/usr/bin/mktemp"
declare -rx hdiutil="/usr/bin/hdiutil"
declare -rx rm="/bin/rm"
declare -rx cp="/bin/cp"
declare -rx cat="/bin/cat"
declare -rx mv="/bin/mv"
declare -rx date="/bin/date"
declare -rx mkdir="/bin/mkdir"
declare -rx basename="/usr/bin/basename"
declare -rx du="/usr/bin/du"
declare -rx tail="/usr/bin/tail"
declare -rx df="/bin/df"
declare -rx python="/usr/bin/python"
declare -x HARD_CODED_SPARSE_SOURCE=''
declare -x HARD_CODED_DESTINATION='/Volumes/318admin-Sept-MoLoClientRestored/Users/'
# declare -r MAIN_LOG="$DS_REPOSITORY_PATH/Logs/$DS_SERIAL_NUMBER.log"
# declare -r VERBOSE_LOG="$DS_REPOSITORY_PATH/Logs/Backup/$DS_SERIAL_NUMBER-verbose.log"
declare -rx DEFAULT_SPARSE_SOURCE="$DS_REPOSITORY_PATH/Backups/$DS_SERIAL_NUMBER"
declare -x DEFAULT_RESTORE_DESTINATION="/Volumes/$DS_LAST_RESTORED_VOLUME"
declare -x RSYNCEXCLUDESPATH="$DS_REPOSITORY_PATH/Backups/admin/RestoreExcludes.txt"
declare -rx TIMESTAMP=`$date "+%H:%M:%S"`
declare -rx SCRIPT="${0##*/}"
help(){
$cat<<EOF
 
Usage: $SCRIPT [ -s "/Volumes/Macintosh HD" ] [ -d "/Volumes/External Drive/" ]
Variables can be set in DeployStudio variables window when running script.
BackupRestore Variables:
-s Source where backup is stored
Trailing slash on user folder is important!
Specify full path to sparseimage
Default is /tmp/DSNetworkRepository/Backups/$DS_SERIAL_NUMBER/
-d Destination to store
Specify full path to mount point of volume
Default is the \$DS_LAST_RESTORED_VOLUME volume
e.g. "/Volumes/Macintosh HD"
EOF
}
header(){
$python -c "print '-' * 120"
}
die(){
echo "$1 Failed."
$hdiutil eject "$IMAGE_MOUNT_PT"
$rm -r $IMAGE_MOUNT_PT
exit 1
}
 
checkSetup(){
if [ ! -e $rsync ]; then
die "no custom rsync in expected location"
fi
}
 
checkRestoreSource(){
IMAGE_MOUNT_PT=`$mktemp -d "/tmp/tmp-pt"`
if [ -z "$HARD_CODED_SPARSE_SOURCE" -a -e "$DEFAULT_SPARSE_SOURCE" ]; then
RESTORE_SOURCE=`ls $DEFAULT_SPARSE_SOURCE/$DS_SERIAL_NUMBER-*.sparseimage | $tail -n1`
$hdiutil mount "$RESTORE_SOURCE" -owners on -readwrite -noverify -nobrowse -mountpoint "$IMAGE_MOUNT_PT"
elif [ -d "$HARD_CODED_SPARSE_SOURCE" ]; then
RESTORE_SOURCE="$HARD_CODED_SPARSE_SOURCE"
$hdiutil mount "$RESTORE_SOURCE" -owners on -readwrite -noverify -nobrowse -mountpoint "$IMAGE_MOUNT_PT"
else
die "Neither default nor hard-coded custom path to restore sparse image available"
fi
header && echo "restore Started at $TIMESTAMP"
header && printf "\n%s\n" "***** Size of stored user folders:"
$du -h -d 1 -I "Shared" "$IMAGE_MOUNT_PT"/
printf "\n"
}
checkRestoreDestination(){
if [ ! -z "$HARD_CODED_DESTINATION" ]; then
if [ ! -d "$HARD_CODED_DESTINATION" ]; then
die "Custom backup destination not a directory or unreachable"
else
RESTORE_DESTINATION="$HARD_CODED_DESTINATION"
fi
else
RESTORE_DESTINATION="$DEFAULT_RESTORE_DESTINATION/Users/"
fi
header && printf "\n%s\n" "***** Free space and percentage full on destination before restore:"
$df -h $RESTORE_DESTINATION | awk '{print $4, $5}'
printf "\n"
}
 
restoreData(){
header && echo "***** Starting to move the data, there may be some lag while the log is simultaneously being written to"; echo " "
$rsync -aANHhXx --stats --progress --fake-super --filter="merge $RSYNCEXCLUDESPATH" "$IMAGE_MOUNT_PT/" "$RESTORE_DESTINATION" >/dev/stdout
}
 
moveUsersPassAndFixup(){
# declare -x GROUP_REC_PATH="$RESTORE_DESTINATION../private/var/db/dslocal/nodes/Default/groups"
USER_RECORD_PATH="$RESTORE_DESTINATION../private/var/db/dslocal/nodes/Default/users"
USER_HASH_PATH="$RESTORE_DESTINATION../private/var/db/shadow/hash"
header && printf "\n%s" "***** Data move done, copying user records and passwords (if in old format)"
echo " "
for u in $IMAGE_MOUNT_PT/*.plist;
do
$cp -v "$u" "$USER_RECORD_PATH"
done
if [ -d "$IMAGE_MOUNT_PT/hashes" -a "$(ls -A "$IMAGE_MOUNT_PT/hashes/")" ]; then
if [ ! -d "$USER_HASH_PATH" ]; then
$mkdir -p "$USER_HASH_PATH" || die
fi
$cp -R "$IMAGE_MOUNT_PT/hashes/" "$USER_HASH_PATH"
fi
$hdiutil eject "$IMAGE_MOUNT_PT"
header && printf "\n%s\n" "***** Disk image ejected, free space and percentage full on destination post-restore:"
$df -h $RESTORE_DESTINATION | awk '{print $4, $5}'
printf "\n"
}
 
cleanup(){
$rm -r $IMAGE_MOUNT_PT
}
 
checkSetup
checkRestoreSource
checkRestoreDestination
 
while getopts :s:d:h opt; do
case "$opt" in
s) RESTORE_SOURCE="$OPTARG";;
d) RESTORE_DESTINATION="$OPTARG";;
h)
help
exit 0;;
\?)
echo "Sorry, "$OPTARG" is not understood. For more help, run: $SCRIPT -h"
echo "Usage: $SCRIPT [ -s Sparseimage Location for Users to Restore ] [ -d Destination to Restore Backup ]"
exit 0;;
esac
done
 
restoreData
moveUsersPassAndFixup
cleanup
 
exit 0
sonOfUserBackup.sh
Shell
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169
#!/bin/sh
### sonOfBackupScript v .8 Allister Banks, 318 Inc. - No warranty granted or implied
### This process relies upon a patched version of rsync placed in the following path:
declare -r rsync="$DS_REPOSITORY_PATH/Backups/admin/rsync"
### Customiztion is allowed for sources to take the user folders from, and destination to store them
# set -x
### Currently the minimum size of the sparseimage created is ~300MBs, there is a maximum of 100GB -
# if the home folders being backed up exceed this size, IT WILL FAIL!!
 
declare -rx mktemp="/usr/bin/mktemp"
declare -rx uuidgen="/usr/bin/uuidgen"
declare -rx hdiutil="/usr/bin/hdiutil"
declare -rx rm="/bin/rm"
declare -rx cp="/bin/cp"
declare -rx cat="/bin/cat"
declare -rx mv="/bin/mv"
declare -rx mkdir="/bin/mkdir"
declare -rx basename="/usr/bin/basename"
declare -rx du="/usr/bin/du"
declare -rx df="/bin/df"
declare -rx python="/usr/bin/python"
declare -rx dscl="/usr/bin/dscl"
declare -x HARD_CODED_USER_SOURCE='' # don't forget trailing slash!
### INCREASE THE FOLLOWING AS NECESSARY!
declare -r USER_DATA_DMG_SIZE="10g" # since large sparse images take up extra time to create
declare -r HARD_CODED_DESTINATION=''
# declare -r MAIN_LOG="$DS_REPOSITORY_PATH/Logs/$DS_SERIAL_NUMBER.log"
# declare -r VERBOSE_LOG="$DS_REPOSITORY_PATH/Logs/Backup/$DS_SERIAL_NUMBER-verbose.log"
declare -r DEFAULT_BACKUP_PATH="$DS_REPOSITORY_PATH/Backups/$DS_SERIAL_NUMBER"
declare -x DEFAULT_BACKUP_SOURCE=`system_profiler SPSerialATADataType |
awk -F': ' '/Mount Point/ { print $2}'|head -n1` # first mounted volume
declare -r RSYNCEXCLUDESPATH="$DS_REPOSITORY_PATH/Backups/admin/meSpecificExcludes.txt"
declare -r DATE_FOR_BACKUP_NAME=`date "+%m%d-%H%M%S"` #monthday-hourminutesecond, e.g. 0919-114348
declare -x TIMESTAMP=`date "+%H:%M:%S"`
declare -x SCRIPT="${0##*/}"
 
help(){
$cat<<EOF
 
Usage: $SCRIPT [ -s "/Volumes/Macintosh HD" ] [ -d "/Volumes/External Drive/" ]
Variables can be set in DeployStudio variables window when running script.
BackupRestore Variables:
-s Target volume
Specify full path to mount point of volume
Default is the \$DS_LAST_RESTORED_VOLUME volume
e.g. "/Volumes/Macintosh HD"
-d Backup destination
Specify full path to the backup volume
Default is /tmp/DSNetworkRepository
 
EOF
cleanup
}
header(){
$python -c "print '-' * 80"
}
 
die(){
echo "$1 Failed."
$rm -r $IMAGE_MOUNT_PT
$rm -r $IMAGE_ENCL_FOLDER
exit 1
}
 
checkSetup(){
header && echo "Backup Started at $TIMESTAMP"
if [ ! -e $rsync ]; then
die "no custom rsync in expected location"
fi
}
 
checkBackupSource(){
if [ -z "$HARD_CODED_USER_SOURCE" -a -e "$DEFAULT_BACKUP_SOURCE" ]; then
BACKUP_SOURCE="$DEFAULT_BACKUP_SOURCE/Users/"
elif [ -d "$HARD_CODED_USER_SOURCE" ]; then
BACKUP_SOURCE="$HARD_CODED_USER_SOURCE"
else
die "Neither default nor hard-coded custom user data source available"
fi
header && printf "\n%s\n" "***** Size of user folders before exclusions:"
$du -h -d 1 -I "Shared" $BACKUP_SOURCE
printf "\n"
}
 
checkBackupDestination(){
if [ ! -z "$HARD_CODED_DESTINATION" ]; then
if [ ! -d "$HARD_CODED_DESTINATION" ]; then
die "Custom backup destination not a directory or unreachable"
else
BACKUP_STORAGE="$HARD_CODED_DESTINATION" # string for path
fi
elif [ -z "$HARD_CODED_DESTINATION" -a ! -e "$DEFAULT_BACKUP_PATH" ]; then
$mkdir "$DEFAULT_BACKUP_PATH" || die "backup destination folder creation failed"
BACKUP_STORAGE="$DEFAULT_BACKUP_PATH"
else
BACKUP_STORAGE="$DEFAULT_BACKUP_PATH"
fi
header && printf "\n%s\n" "***** Free space and percentage full on destination before backup:"
$df -h $DS_REPOSITORY_PATH | awk '{print $4, $5}'
printf "\n"
}
 
createBackupSparseAndRsyncMinusExclusions(){
IMAGE_MOUNT_PT=`$mktemp -d "/tmp/tmp-pt"`
IMAGE_ENCL_FOLDER=`$mktemp -d "$DS_REPOSITORY_PATH/.tmp-subfolder"`
IMAGE_FILENAME="$IMAGE_ENCL_FOLDER/`$uuidgen`"
IMAGE_FULLPATH="$IMAGE_FILENAME.sparseimage"
header && echo "Starting backup of $BACKUP_SOURCE"
header && echo "Creating $USER_DATA_DMG_SIZE disk image to retain ownership on files while backing up"
$hdiutil create -type SPARSE -size $USER_DATA_DMG_SIZE -fs JHFS+ -volname "$DS_SERIAL_NUMBER-Users" "$IMAGE_FULLPATH"
$hdiutil mount "$IMAGE_FULLPATH" -owners on -readwrite -noverify -nobrowse -mountpoint "$IMAGE_MOUNT_PT"
header && echo "***** Starting to move the data, there may be some lag while the log is simultaneously being written to"; echo " "
$rsync -aANHhXx --stats --progress --exclude='*.ipsw' --filter="merge $RSYNCEXCLUDESPATH" "$BACKUP_SOURCE" "$IMAGE_MOUNT_PT" >/dev/stdout
}
 
unmountAndmoveTheGoods(){
header && printf "\n%s" "***** Data move done, copying user records and passwords (if in old format)"
GROUP_REC_PATH="$BACKUP_SOURCE../private/var/db/dslocal/nodes/Default/groups"
USER_RECORD_PATH="$BACKUP_SOURCE../private/var/db/dslocal/nodes/Default/users"
USER_HASH_PATH="$BACKUP_SOURCE../private/var/db/shadow/hash/"
echo " "
for u in "$BACKUP_SOURCE"*/;
do
$cp -v "$USER_RECORD_PATH/$($basename "$u").plist" "$IMAGE_MOUNT_PT"
done
$mkdir "$IMAGE_MOUNT_PT/groups" "$IMAGE_MOUNT_PT/hashes"
$cp -v "$GROUP_REC_PATH/admin.plist" "$IMAGE_MOUNT_PT/groups/"
if [ -d "$USER_HASH_PATH" -a "$(ls -A "$USER_HASH_PATH")" ]; then
$cp -Rv "$USER_HASH_PATH" "$IMAGE_MOUNT_PT/hashes"
fi
$hdiutil eject "$IMAGE_MOUNT_PT"
$mv "$IMAGE_FULLPATH" "$BACKUP_STORAGE/$DS_SERIAL_NUMBER-$DATE_FOR_BACKUP_NAME.sparseimage"
header && printf "\n%s\n" "***** Disk image ejected and moved, size and path to actual sparseimage:"
$du -h "$BACKUP_STORAGE/$DS_SERIAL_NUMBER-$DATE_FOR_BACKUP_NAME.sparseimage"
header && printf "\n%s\n" "***** Free space and percentage full on destination:"
$df -h $DS_REPOSITORY_PATH | awk '{print $4, $5}'
printf "\n"
}
 
cleanup(){
$rm -r $IMAGE_MOUNT_PT
$rm -r $IMAGE_ENCL_FOLDER
header && echo "Backup finished $TIMESTAMP"
}
 
checkSetup
checkBackupSource
checkBackupDestination
 
while getopts :s:d:h opt; do
case "$opt" in
s) HARD_CODED_USER_SOURCE="$OPTARG";;
d) BACKUP_STORAGE="$OPTARG";;
h)
help
exit 0;;
\?)
echo "Sorry, "$OPTARG" is not understood. For more help, run: $SCRIPT -h"
echo "Usage: $SCRIPT [ -s Source of Users to Backup ] [ -d Destination to Store Backup ]"
exit 0;;
esac
done
 
createBackupSparseAndRsyncMinusExclusions
unmountAndmoveTheGoods
cleanup
 
exit 0

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.