A minimal authenticated S3 download script using only Bash, Curl, and OpenSSL.
#!/bin/bash | |
# s3dl | |
# by Alex Markley; released under CC0 Public Domain | |
# https://creativecommons.org/publicdomain/zero/1.0/ | |
# Hosted at: https://gist.github.com/alexmarkley/0b45af17fdc805756ac710aeaceb77be | |
# Example script demonstrating how to fetch a private AWS S3 Object using only basic shell scripting techniques. | |
################# | |
### CONSTANTS ### | |
################# | |
S3_DOMAIN=s3.amazonaws.com | |
################# | |
### VARIABLES ### | |
################# | |
S3_BUCKET= | |
S3_PATH= | |
LISTMODE=0 | |
SIGNURL_ONLY=0 | |
VALID_MINUTES=5 | |
EXPIRATION_DATE= | |
SIGNATURE= | |
SED= | |
CURL= | |
OPENSSL= | |
XMLSTARLET= | |
################# | |
### FUNCTIONS ### | |
################# | |
_log() { | |
echo s3dl: "${@}" 1>&2 | |
} | |
_die() { | |
_log "FATAL:" "${@}" | |
exit 1 | |
} | |
_usage() { | |
cat << EOF | |
s3dl | |
==== | |
Summary: | |
Example script demonstrating how to fetch a private AWS S3 Object using only basic shell scripting techniques. | |
Usage: | |
${0##*/} [-h] [-l] [-s] [-m MIN] [-p PATH/PREFIX] -b BUCKET | |
-h Display this help message and exit. | |
-l List the objects at PREFIX instead of downloading the object at PATH. | |
-s Generate a signed URL and dump it to STDOUT. Do not actually perform the action. | |
-m MIN Generate a URL which is valid until MIN minutes into the future, instead of 5 minutes. | |
-p PATH/PREFIX When downloading an object, this is the PATH to the S3 Object being downloaded. When listing objects, this is the PREFIX, which filters the list. | |
-b BUCKET What S3 Bucket are we acting on? | |
Output: | |
Output data, such as file contents or lists, go to STDOUT. Take care to direct STDOUT to a file and not to your terminal, especially if you are downloading a binary! | |
Credentials: | |
It is always a security risk to provide credentials on the command line. Instead, provide your AWS credentials in the form of Environment variables. | |
AWS_ACCESS_KEY_ID Your AWS Access Key ID | |
AWS_SECRET_ACCESS_KEY Your AWS Secret Access Key | |
X_AMZ_SECURITY_TOKEN An Optional AWS Session Token | |
Return Code: | |
Returns 0 if operation completed successfully, 1 otherwise. | |
EOF | |
} | |
_parse_input_arguments() { | |
#Parse options | |
OPTIND=1 | |
while getopts ":hlsm:p:b:" opt; do | |
case "${opt}" in | |
'h') | |
_usage | |
exit 1 | |
;; | |
's') | |
SIGNURL_ONLY=1 | |
;; | |
'l') | |
LISTMODE=1 | |
;; | |
'm') | |
VALID_MINUTES="${OPTARG}" | |
;; | |
'p') | |
S3_PATH="${OPTARG}" | |
;; | |
'b') | |
S3_BUCKET="${OPTARG}" | |
;; | |
'?') | |
_usage | |
_die "Invalid option: -${OPTARG}" | |
;; | |
':') | |
_usage | |
_die "Option -${OPTARG} requires an argument." | |
;; | |
*) | |
_die "Internal error" | |
;; | |
esac | |
done | |
#Validation | |
if [ "${VALID_MINUTES}" -eq "${VALID_MINUTES}" ] 2>/dev/null ; then | |
false | |
else | |
_usage | |
_die "Valid Minutes (-m) must be an integer." | |
fi | |
if [ "${VALID_MINUTES}" -lt 1 ]; then | |
_usage | |
_die "Valid Minutes (-m) of ${VALID_MINUTES} seems unreasonable." | |
fi | |
if [ -z "${AWS_ACCESS_KEY_ID}" -o -z "${AWS_SECRET_ACCESS_KEY}" ]; then | |
_usage | |
_die "One of AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY is not set." | |
fi | |
if [ -z "${S3_BUCKET}" ]; then | |
_usage | |
_die "S3 Bucket (-b) is required." | |
fi | |
if [ -z "${S3_PATH}" -a "${LISTMODE}" -ne 1 ]; then | |
_usage | |
_die "S3 Path/Prefix (-p) is required unless you are listing objects." | |
fi | |
} | |
_validate_environment() { | |
SED=$(which sed) || _die "No sed found" | |
CURL=$(which curl) || _die "No curl found" | |
OPENSSL=$(which openssl) || _die "No openssl found" | |
if [ "${LISTMODE}" -eq 1 ]; then | |
XMLSTARLET=$(which xmlstarlet) || _die "No xmlstarlet found. XMLStarlet is required when listing objects." | |
fi | |
} | |
_calculate_expiration_date() { | |
NOW=$(date +%s) | |
let "EXPIRATION_DATE = NOW + (VALID_MINUTES * 60)" | |
} | |
_generate_signature() { | |
#Handle supported canonicalized AMZ headers. | |
CANONICAL= | |
if [ ! -z "${X_AMZ_SECURITY_TOKEN}" ]; then | |
CANONICAL="${CANONICAL}x-amz-security-token:${X_AMZ_SECURITY_TOKEN}\n" | |
fi | |
#Build the string to sign. | |
if [ "${LISTMODE}" -ne 1 ]; then | |
#Downloading | |
SIGN_STRING="GET\n\n\n${EXPIRATION_DATE}\n${CANONICAL}/${S3_BUCKET}/${S3_PATH}" | |
else | |
#Listing | |
SIGN_STRING="GET\n\n\n${EXPIRATION_DATE}\n${CANONICAL}/${S3_BUCKET}" | |
fi | |
#Sign the string. | |
SIGNATURE=$(echo -en "${SIGN_STRING}" | "${OPENSSL}" sha1 -hmac "${AWS_SECRET_ACCESS_KEY}" -binary | base64) | |
} | |
_simple_urlencode() { | |
echo -n "${1}" | "${CURL}" -Gso /dev/null -w %{url_effective} --data-urlencode @- "" | cut -c 3- | |
if [ "${?}" -ne 3 ]; then | |
_die "Super curl hack failed. :(" | |
fi | |
true | |
} | |
############# | |
### ENTRY ### | |
############# | |
set -o pipefail | |
#Parse the command line arguments | |
_parse_input_arguments "${@}" | |
#Validate working environment. | |
_validate_environment | |
#Normalize PATH variable. (Strip any leading forward slash off the PATH.) | |
if [ ! -z "${S3_PATH}" ]; then | |
S3_PATH=$(echo "${S3_PATH}" | "${SED}" 's|^/||') | |
fi | |
#Calculate the expiration date. | |
_calculate_expiration_date | |
#Generate the actual signature based on the action we're taking. | |
_generate_signature | |
#URL Encode components that will be embedded in the URL | |
ENCODED_AWS_ACCESS_KEY_ID=$(_simple_urlencode "${AWS_ACCESS_KEY_ID}") || _die "URL encoding failed." | |
ENCODED_EXPIRATION_DATE=$(_simple_urlencode "${EXPIRATION_DATE}") || _die "URL encoding failed." | |
ENCODED_SIGNATURE=$(_simple_urlencode "${SIGNATURE}") || _die "URL encoding failed." | |
#Handle supported canonicalized AMZ headers. | |
CANONICAL= | |
if [ ! -z "${X_AMZ_SECURITY_TOKEN}" ]; then | |
CANONICAL="${CANONICAL}&x-amz-security-token="$(_simple_urlencode "${X_AMZ_SECURITY_TOKEN}") || _die "URL encoding failed." | |
fi | |
#Actual signed URL. | |
if [ "${LISTMODE}" -ne 1 ]; then | |
#Downloading | |
SIGNED_URL="https://${S3_DOMAIN}/${S3_BUCKET}/${S3_PATH}?AWSAccessKeyId=${ENCODED_AWS_ACCESS_KEY_ID}&Expires=${ENCODED_EXPIRATION_DATE}&Signature=${ENCODED_SIGNATURE}${CANONICAL}" | |
else | |
#Listing | |
PREFIXVAR= | |
if [ ! -z "${S3_PATH}" ]; then | |
PREFIXVAR="prefix="$(_simple_urlencode "${S3_PATH}")"&" || _die "URL encoding failed." | |
fi | |
SIGNED_URL="https://${S3_DOMAIN}/${S3_BUCKET}?list-type=2&${PREFIXVAR}AWSAccessKeyId=${ENCODED_AWS_ACCESS_KEY_ID}&Expires=${ENCODED_EXPIRATION_DATE}&Signature=${ENCODED_SIGNATURE}${CANONICAL}" | |
fi | |
#Perform requested action | |
if [ "${SIGNURL_ONLY}" -eq 0 ]; then | |
if [ "${LISTMODE}" -ne 1 ]; then | |
#Fetch the file and dump it to STDOUT | |
"${CURL}" --silent --fail --show-error "${SIGNED_URL}" | |
else | |
#Fetch the listing, filter it, and dump the filtered listing to STDOUT. | |
"${CURL}" --silent --fail --show-error "${SIGNED_URL}" | "${SED}" 's/xmlns="[a-z0-9.:\/-]*"//g' | "${XMLSTARLET}" sel -T -t -v "/ListBucketResult/Contents/Key" -n | |
fi | |
if [ "${?}" -ne 0 ]; then | |
_die "Something went wrong." | |
fi | |
else | |
#Dump the signed URL to STDOUT | |
echo "${SIGNED_URL}" | |
fi | |
exit 0 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment