Download all files from AWS S3 from a list generated with AWS CLI

aws-download.sh

#!/bin/bash

# USE CASE
# I have a bucket with a lot of files and someone else wants to download them.
# They don't have AWS CLI for rsyncing.

# PURPOSE
# It is possible to generate a list of files present in an AWS bucket with the
# AWS CLI:
# aws s3 ls s3://bucket-name/path/ --region eu-west-1 --recursive  | tee aws-files.txt
#
# This results in file with lines like this:
# 2014-05-27 11:51:26    3277380 production/path/to/file/IMG_0561.JPG
#
# Now we have a list of files. I sent this list of files to the other person
# along with this script.

# USAGE
# On the command line:
# ./aws-download <region> <bucket-name> <aws-file-list>
#
# Example:
# ./aws-download eu-west-1 my-bucket aws-files.txt
#
# Be patient.
#
# If for some reason the script stops, start it again. It won't download what
# already exists.

awk -v region=$1 -v bucket=$2 '{
  if(system("[ -e " $4 " ]") == 0) {
    system("curl https://s3-" region ".amazonaws.com/" bucket $4 " --create-dirs -o ./" $4)
  }
}' $3