A short script to backup the S3 buckets on an account, to a local directory.

s3backup.sh

#!/bin/bash

###
# A short script to sync S3 to a local directory.

# Read arguments
while getopts ":c:b:dts" opt ; do
	case $opt in
		c) 
			if [ -e ${OPTARG} ] ; then
				S3BU_CONFIG_OPT="--config=${OPTARG}"
			fi
			;;
		b)
			S3BU_BACKUP_DIR=$OPTARG
			;;
		d)
			S3BU_DRYRUN_OPT="--dry-run"
			;;
		t)
			S3BU_TAR=true
			;;
		s)
			S3BU_SILENT=true
			S3BU_SILENT_OPT="--quiet"
			;;
		:)
			if [ $OPTARG == "c" ] ; then 
				echo "You must specify a config file with -c."
			fi
			if [ $OPTARG == "b" ] ; then
				echo "You must specify a backup directory with -b."
			fi
			exit 1
			;;
		\?)
			echo && echo "Usage: `basename $0` [-c configfile] [-b backupdir] [-d] [-t] [-s]" && echo
			echo "  -c configfile    Specify a different s3cmd config file to use (default: ~/.s3cfg)."
			echo "  -b backupdir     Specify a backup directory (default: ./s3backups)"
			echo "  -d               Do a dry run (don't download any files.)"
			echo "  -t               Tar.gz and delete the downloaded files."
			echo "  -s               Run silently."
			echo "  -h               This help."
			echo && echo "  All arguments are optional."
			exit
			;;
	esac
done

# When `true`, only show error messages.
S3BU_SILENT=${S3BU_SILENT-false}

# Default backup directory.
S3BU_BACKUP_DIR=${S3BU_BACKUP_DIR-"./s3backups"}

# Default to not tarring the downloaded bucket.
S3BU_TAR=${S3BU_TAR-"false"}

# Default to the user's config file. 
S3BU_CONFIG_OPT=${S3BU_CONFIG_OPT-""}

# Default to actually copying the files.
S3BU_DRYRUN_OPT=${S3BU_DRYRUN_OPT-""}

# Default to verbose output.
S3BU_SILENT_OPT=${S3BU_SILENT_OPT-"--verbose"}

# Base command to sync backups.
S3BU_CMD="/usr/local/bin/s3cmd"

# Options for the command.
S3BU_OPTS="--no-check-md5 --recursive --no-delete-removed --skip-existing ${S3BU_CONFIG_OPT} ${S3BU_SILENT_OPT} ${S3BU_DRYRUN_OPT}"

TSTAMP=`date "+%Y%m%d-%H%M"`

# The function `check` will exit the script if the given command fails.
function check {
	"$@"
	status=$?
	if [ $status -ne 0 ]; then
		echo "ERROR: Encountered error (${status}) while running the following:" >&2
		echo "           $@"  >&2
		echo "       (at line ${BASH_LINENO[0]} of file $0.)"  >&2
		echo "       Aborting." >&2
		exit $status
	fi
}

# The function `tgz` will create a gzipped tar archive of the specified file ($1) and then remove the original
function tgz {
	check tar zcf $1-${TSTAMP}.tar.gz $1 && check rm -rf $1
}

$S3BU_SILENT || ( echo "" && echo "=== INITIALIZING backup at ${TSTAMP} ===" && echo "" ) 

$S3BU_SILENT || echo "Using backup directory $S3BU_BACKUP_DIR"
check mkdir -p $S3BU_BACKUP_DIR

$S3BU_SILENT || echo -n "Fetching list of buckets... "

BUCKET_LIST="$(${S3BU_CMD} ${S3BU_CONFIG_OPT} ls | grep s3 | awk '{ print $3 }')"

$S3BU_SILENT || echo "found `echo $BUCKET_LIST | wc -w | awk '$1=$1'` buckets."
echo "${S3BU_CMD} ${S3BU_CONFIG_OPT} ls" 

for bucket in $BUCKET_LIST; do

	bucket_name=${bucket//s3:\/\//}

	check mkdir -p ${S3BU_BACKUP_DIR}/${bucket_name}

	$S3BU_SILENT || echo "  Syncing $bucket."

	${S3BU_CMD} sync ${S3BU_OPTS} $bucket/ ${S3BU_BACKUP_DIR}/${bucket_name}/
	echo "${S3BU_CMD} sync ${S3BU_OPTS} $bucket/ ${S3BU_BACKUP_DIR}/${bucket_name}/"

	if [ ${S3BU_TAR} == true ] ; then
		$S3BU_SILENT || echo "Archiving ${bucket_name}."
		tgz ${S3BU_BACKUP_DIR}/${bucket_name}
	fi	

done

$S3BU_SILENT || (echo "=== S3 backup completed at" `date "+%Y%m%d-%H%M"` "===" && echo "")