Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
#!/bin/bash
#set -vx
SOURCE="...@s....gridserver.com:/home/.../"
# Where all the history of files are stored
DESTINATION_PARENT="/.../mediatemple/"
# Where will be the latest backup will be stored, relative to
# DESTINATION_PARENT. Note that ISO-8601 is used for the ability to remove old
# backups. If you want to add ":" to any of the folder names, you are on your
# own because rsync thinks there is a remote component even in local folders.
DESTINATION_DIR="$(date --iso-8601=date)"
# Excluded paths are looked up from this file
EXCLUDES=excludes
# 7x1: Keep last 7 days
# 4x7: Keep 1 for each week for last 4 weeks
# 12x30: Keep 1 for each month for last 12 months
# You need apt-get install chiark-scripts
EXPIRE_PATTERN="7x1 4x7 12x30"
(
# Working with relative paths are more preferable
cd ${DESTINATION_PARENT}
# --link-dest is the magical parameter of rsync. Whe it is backing up, it'll
# look at the previous folders and hard-link the file from there if it
# matches the remote file.
linkdests=$(find -maxdepth 1 -type d | while read line; do echo -n "--link-dest=../${line}/ "; done)
rsync \
--archive \
--verbose \
--partial \
--progress \
--human-readable \
--delete \
--delete-excluded \
--exclude-from=${EXCLUDES} \
${linkdests} \
"${SOURCE}/" \
"${DESTINATION_DIR}/"
# Backup done. Expire old directories
expire-iso8601 \
-n \
-r \
${EXPIRE_PATTERN}
# Expiration done, generate a report
du --max-depth=1 -h
)
#!/bin/bash
set -e
usage () {
cat <<END
usage:
expire-iso8601 [<options>] <number>x<interval> [<number>x<interval> ...]
options:
-u<unitlen> <interval> is measured in units of <unitlen> seconds
(default is 86400, so <interval> is in days)
-s<slop> allow kept items to be <slop> seconds shorter apart than
specified; default is 10% of <unitlen>
-n do not really delete
-r recursive removal (rm -r)
example:
/home/ian/junk/expire-iso8601 14x1 4x7
uses units of 86400s (1 day) with a slop of 8640
it keeps 14 daily items
(that is 14 items, dated no less than 86400-8640 apart)
and 7 weekly items
(that is 7 items, dated no less than 7*86400-8640 apart)
the 14 daily and 7 weekly items may be the same, or not
There is no need to sort the list of <number>x<interval> pairs.
exit status:
0 ok
4 rm failed
8 bad usage
16 catastrophic failure
END
}
# Copyright 2006 Ian Jackson <ian@chiark.greenend.org.uk>
#
# This script and its documentation (if any) are free software; you
# can redistribute it and/or modify them under the terms of the GNU
# General Public License as published by the Free Software Foundation;
# either version 3, or (at your option) any later version.
#
# chiark-named-conf and its manpage are distributed in the hope that
# it will be useful, but WITHOUT ANY WARRANTY; without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
# PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, consult the Free Software Foundation's
# website at www.fsf.org, or the GNU Project website at www.gnu.org.
trap 'exit 16' 0
badusage () { echo >&2 "bad usage: $*"; usage >&2; trap '' 0; exit 8; }
#-------------------- argument parsing --------------------
alldigits () {
[ "x${2##*[^0-9]}" = "x$2" ] || \
badusage "bad $1 \`$2'; must be all digits"
[ "$2" ] || badusage "bad $2; must be nonempty"
eval $1='$2'
}
rm=rm
recurse=''
unit=86400
slop=''
while [ $# -ge 1 ]; do
arg=$1; shift
case "$arg" in
--|-) break ;;
--help) usage; exit 0 ;;
--*) badusage "unknown option $arg" ;;
-*)
val=${arg#-?}
case "$arg" in
-n*) rm=: ;;
-r*) recurse=-r ;;
-u*) alldigits unit "$val"; arg='' ;;
-s*) alldigits slop "$val"; arg='' ;;
*) badusage "unknown option ${1:0:2}" ;;
esac
arg=-${arg#-?}
if test "x$arg" != x-; then set -- "$arg" "$@"; fi
;;
*) set "$arg" "$@"; break ;;
esac
done
[ $# -ge 1 ] || badusage 'too few arguments'
[ "$slop" ] || slop=$(( $unit / 10 ))
for ni in "$@"; do
case "$ni" in *x*);; *) badusage "bad <number>x<interval> $ni";; esac
alldigits number "${ni%%x*}"
alldigits interval "${ni#*x}"
done
#-------------------- scanning the directory ----------
# We build in $l a list of the relevant filenames and the time_t's
# they represent.
#
# Each entry in $l is $time_t/$filename, and the list is
# newline-separated for the benefit of sort(1).
ls=0
for cn in [0-9]*; do
case "$cn" in
????-??-??)
conv="$cn";;
????-??-??T[0-2][0-9]+[0-9][0-9][0-9][0-9]|\
????-??-??T[0-2][0-9]:[0-6][0-9]+[0-9][0-9][0-9][0-9]|\
????-??-??T[0-2][0-9]:[0-6][0-9]:[0-6][0-9]+[0-9][0-9][0-9][0-9])
conv="${cn%T*} ${cn#*T}";;
*)
echo >&2 "ignoring $cn"
continue;;
esac
cs=$(date -d "$conv" +%s)
l="$cs/$cn
$l"
done
#-------------------- main computation --------------------
# We process each minimum/extent pair, to have it select a bunch of
# versions to keep. We annotate entries in $l: if we are keeping
# an entry we prepend a colon; temporarily, if we are keeping an entry
# because of this particular minimum/extent, we prepend a comma.
# For each minimum/extent pair we look at the list from most recent
# to least recent,
# ie in order of increasing age
# ie in order of decreasing time_t
# and each time we're more than min older than the last item we kept,
# we mark the item to keep, until we have as many as we want.
#
# We build the new list (space-separated) in lnew.
l=$(sort -nr <<END
$l
END
)
for ni in "$@"; do
wantcount=${ni%x*}
div=1
while true; do
min=$(( (${ni#*x} * $unit) / $div - $slop ))
ls=''
lnew=''
skipped=0
for ce in $l; do
cn=${ce#*/}; cl=${ce%%/*}
cs=${cl#,}; cs=${cs#:}
case $cl in ,*) ls=$cs; continue;; esac
if [ $wantcount != 0 ]; then
if ! [ "$ls" ] || \
[ $(( $ls - $cs )) -ge $min ]; then
echo "keep (for $ni) $cn"
ce=,$ce
ls=$cs
wantcount=$(( $wantcount - 1 ))
else
skipped=$(( $skipped+1 ))
fi
fi
lnew="$lnew $ce"
done
l=$lnew
if [ $wantcount = 0 ]; then break; fi
printf "%s" "insufficient (for $ni) by $wantcount"
if [ $skipped = 0 ]; then echo; break; fi
div=$(( $div * 2 ))
echo " shortening interval ${div}x"
done
# s/([,:]+).*/:\1/g
lnew=''
for ce in $l; do
case $ce in ,*) ce=:${ce#,};; esac
case $ce in ::*) ce=${ce#:};; esac
lnew="$lnew $ce"
done
l=$lnew
done
#-------------------- execution --------------------
trap '' 0
exitstatus=0
nonbroken_echo () { (echo "$@"); }
# While we have subprocesses, we have to avoid bash calling write(1,...)
# because of a bug in bash (Debian #382798), so we arrange for a subshell
# for each echo.
jobs=''
for ce in $l; do
case $ce in
:*);;
*)
cn=${ce#*/}
nonbroken_echo "expire $cn"
$rm $recurse -- $cn &
jobs="$jobs $!"
;;
esac
done
if [ "$jobs" ]; then
nonbroken_echo "all running"
fi
for job in $jobs; do
wait $job || exitstatus=4
done
if [ $exitstatus = 0 ]; then
echo "complete"
else
echo "complete, but problems deleting"
fi
exit $exitstatus
#!/bin/bash
set -vx
BACKUP_BASE=/home/.../users/.home/backup/
DUMP_FILE="$(date --iso-8601=date)"
TMP=$(tempfile -d ${BACKUP_BASE})
DATABASE_HOST=internal-db.....gridserver.com
DATABASE_NAME=db...
DATABASE_USER=db...
DATABASE_PASSWORD='...'
cd ${BACKUP_BASE}
nice mysqldump \
-h ${DATABASE_HOST} \
${DATABASE_NAME} \
-u${DATABASE_USER} \
-p${DATABASE_PASSWORD} | nice bzip2 > ${TMP}
mv ${TMP} ${BACKUP_BASE}/${DUMP_FILE}
# 7x1: Keep last 7 days
# 4x7: Keep 1 for each week for last 4 weeks
# 48x30: Keep 1 for each month for last 48 months
# You need apt-get install chiark-scripts
./expire-iso8601 7x1 4x7 48x30
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment