Skip to content

Instantly share code, notes, and snippets.

@RichardBronosky RichardBronosky/aws
Last active May 23, 2019

Embed
What would you like to do?
A pattern for syncing a dir to AWS S3 using using only Docker
#!/bin/bash -eu
AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-PUT_DEFAULT_KEY_HERE}
AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-PUT_DEFAULT_SECRET_HERE}
args=" -e \"AWS_DEFAULT_REGION=us-east-1\" ${DOCKER_ARGS:-} "
for v in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_SECURITY_TOKEN; do
if [[ -n ${!v:-} ]]; then
args=" -e ${v}=${!v} ${args}"
fi
done
sudo docker run --rm -t $(tty &>/dev/null && echo "-i") \
$args \
-v /mnt/awssync:/mnt/awssync \
mesosphere/aws-cli "$@"
#!/bin/bash -eu
what=feeds
src=/mnt/awssync/$what/
dst=s3://bucket-prefix-$what/data-sync/$what/
script="$(basename "${BASH_SOURCE[0]}")"
log="$script.log"
last="$script.last"
temp="$script.temp"
export TZ=America/New_York
function main(){
printf '\n####\n# New execution of %s beginning %s\n####\n\n' "$script" "$(date)"
time aws s3 sync $src $dst
echo
}
function update_log(){
sed 's/\r/\n/g' $last >> $log
}
function clean_log(){
first="$(date "+%a %b %d" -d "-1 week")"
grep -q "$first" sync_feeds.sh.log || first=""
awk -v first="$first" -f <(cat - <<-'AWK'
function filter_until_first(){
if(first != "" && $0 ~ first){
print last;
first="";
}
if(first != ""){
last=$0;
next;
}
}
function filter_early_progress(){
if($1 == "Completed"){
progress=$0;
next;
}
}
function print_uploads_with_progress(){
if($1 == "upload:"){
print progress;
print;
next
}
}
{
filter_until_first()
filter_early_progress()
print_uploads_with_progress()
print
}
AWK
) <$log >$temp
mv $temp $log
}
main 2>&1 | tee $last
update_log
clean_log
DOCKER_ARGS="-v $(pwd)/$log:/tmp/$log" aws s3 cp /tmp/$log $dst
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.