Skip to content

Instantly share code, notes, and snippets.

@jaygooby
Created January 27, 2015 12:22
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save jaygooby/4e1a38311b2dc4247343 to your computer and use it in GitHub Desktop.
Save jaygooby/4e1a38311b2dc4247343 to your computer and use it in GitHub Desktop.
#!/bin/sh
#
# S3rep.sh version 1.0
# Copyright 2009 Matthew Staver
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>
pid="$$"
# Set type to first argument.
type="$1"
# Set path to the rest of the arguments.
path="${*:2:$#}"
# Default incron mask, if you never plan to use deletion=1 and your
# directories are fairly static you can set this to:
#
# mask="IN_CREATE,IN_CLOSE_WRITE,IN_MOVED_TO"
#
# This may slightly reduce system load, but your incrontab file will
# not be cleaned of watched directories that are moved or deleted.
# I highly recommend you leave this with the default setting.
mask="IN_CREATE,IN_DELETE,IN_CLOSE_WRITE,IN_MOVED_TO,IN_MOVED_FROM"
# The location of your spool/queue directory.
queue="/var/spool/s3rep"
lockfile=/var/run/s3rep.pid
# The location of s3sync
s3path="/root/s3sync"
# The location of your incrontab file
incrontab="/var/spool/incron/root"
# The s3bucket you want to use
bucket="yourbucket"
# Set this to 0 to turn off syslog logging
syslog=1
# Set this to 1 to enable deletion from S3 of removed files/directories
deletion=0
function log () {
[ $syslog -eq 1 ] && logger -t "s3rep[$pid]" "$@" || cat > /dev/null
}
case "$type" in
'IN_CREATE')
exit 0
;;
'IN_CREATE,IN_ISDIR' | 'IN_MOVED_TO,IN_ISDIR' | 'IN_CLOSE_WRITE' | 'IN_MOVED_TO' | '-a')
# If a file or directory is created, edited, or moved into a watched directory
# symlink the it in the queue directory. If no uploads are currently running,
# call this script with the UPLOAD option to start processing the queue directory.
symname="$pid`date +%N`"
ln -s "$path" "$queue/$symname"
log "Queued: '$path' in '$queue/$symname'"
[ ! -e $lockfile ] && $0 UPLOAD &
;;
'IN_DELETE' | 'IN_MOVED_FROM')
# If a file is deleted or moved out of a watched directory: Check to
# see if deletion is enabled in the script. If so, delete it from S3.
[ $deletion -eq 1 ] && $s3path/s3cmd.rb --ssl --verbose delete $bucket:"$path" 2>&1 | log
;;
'IN_DELETE,IN_ISDIR' | 'IN_MOVED_FROM,IN_ISDIR' | '-d')
# If a directory is deleted or removed from a watched directory: Remove
# the directory from the watched list in the incrontab file. Check
# to see if deletion is enabled in the script and if so, remove all files
# from within the watched directory.
sed -i "\|^${path// /\\\ }|d" $incrontab
log "Removing '$path' and all sub-folders from watch list '$incrontab'"
[ $deletion -eq 1 ] && $s3path/s3cmd.rb --ssl --verbose deleteall $bucket:"$path" 2>&1 | log
;;
'UPLOAD')
# Check for lockfile and being uploads if it doesn't exist.
if ( set -o noclobber; echo "$$" > "$lockfile") 2> /dev/null;
then
# Trap exits and clean up lockfile.
trap 'rm -f "$lockfile"; exit $?' INT TERM EXIT
# Pop oldest symlink, store it, and unlink it.
symlink=$(ls -rt1 $queue | sed -n -e '1,1p')
linkpath="$(readlink "$queue/$symlink")"
unlink "$queue/$symlink"
# If it is a directory s3sync the files inside.
if [ -d "$linkpath" ];
then
log "Syncing '$linkpath' and all sub-folders to S3 bucket: '$bucket'"
$s3path/s3sync.rb --ssl -r --verbose "$linkpath/" $bucket:"$linkpath" 2>&1 | log
# Recurse directories, stream edit them to escape any spaces with \,
# append each directory to the incrontab file. NB: For some reason
# incron does not pass trailing spaces in triggered events. I've
# added a trailing \ to the incrontab and this appears to resolve this.
# I assume it causes incron to pass the trailing space and the \, and
# bash ignores the \.
find "$linkpath" -type d |sed -e 's| |\\ |g' -e "s|\$| $mask $0 \$% \$@/\$#\\\|" >>$incrontab
log "Added '$linkpath' and all sub-folders to watch list '$incrontab'"
else
# Upload single file vis s3cmd put.
[ -e "$linkpath" ] && $s3path/s3cmd.rb --ssl --verbose put $bucket:"$linkpath" "$linkpath" 2>&1 | log
fi
# Clean up lockfile
rm -f "$lockfile"
trap - INT TERM EXIT
# Continue processing queue directory if symlinks remain.
if [ "$(ls $queue|wc -l)" -gt "0" ]; then
exec $0 UPLOAD
fi
else
log "Upload $(cat $lockfile) in progress. Failed to acquire lockfile: $lockfile."
fi
;;
*)
echo "Usage: s3rep.sh [OPTION] DIRECTORY"
echo
echo "Options:"
echo " -a add DIRECTORY and sub-folders to incrontab and sync to S3"
echo " NOTE: Do not include a trailing / after DIRECTORY"
echo " -d remove DIRECTORY and all sub-folders from incrontab and"
echo " if deletion is enabled, remove them from S3"
echo
exit 1
esac
exit 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment