A minimal authenticated S3 download script using only Bash, Curl, and OpenSSL.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# s3dl | |
# by Alex Markley; released under CC0 Public Domain | |
# https://creativecommons.org/publicdomain/zero/1.0/ | |
# Hosted at: https://gist.github.com/alexmarkley/0b45af17fdc805756ac710aeaceb77be | |
# Example script demonstrating how to fetch a private AWS S3 Object using only basic shell scripting techniques. | |
################# | |
### CONSTANTS ### | |
################# | |
S3_DOMAIN=s3.amazonaws.com | |
################# | |
### VARIABLES ### | |
################# | |
S3_BUCKET= | |
S3_PATH= | |
LISTMODE=0 | |
SIGNURL_ONLY=0 | |
VALID_MINUTES=5 | |
EXPIRATION_DATE= | |
SIGNATURE= | |
SED= | |
CURL= | |
OPENSSL= | |
XMLSTARLET= | |
################# | |
### FUNCTIONS ### | |
################# | |
_log() { | |
echo s3dl: "${@}" 1>&2 | |
} | |
_die() { | |
_log "FATAL:" "${@}" | |
exit 1 | |
} | |
_usage() { | |
cat << EOF | |
s3dl | |
==== | |
Summary: | |
Example script demonstrating how to fetch a private AWS S3 Object using only basic shell scripting techniques. | |
Usage: | |
${0##*/} [-h] [-l] [-s] [-m MIN] [-p PATH/PREFIX] -b BUCKET | |
-h Display this help message and exit. | |
-l List the objects at PREFIX instead of downloading the object at PATH. | |
-s Generate a signed URL and dump it to STDOUT. Do not actually perform the action. | |
-m MIN Generate a URL which is valid until MIN minutes into the future, instead of 5 minutes. | |
-p PATH/PREFIX When downloading an object, this is the PATH to the S3 Object being downloaded. When listing objects, this is the PREFIX, which filters the list. | |
-b BUCKET What S3 Bucket are we acting on? | |
Output: | |
Output data, such as file contents or lists, go to STDOUT. Take care to direct STDOUT to a file and not to your terminal, especially if you are downloading a binary! | |
Credentials: | |
It is always a security risk to provide credentials on the command line. Instead, provide your AWS credentials in the form of Environment variables. | |
AWS_ACCESS_KEY_ID Your AWS Access Key ID | |
AWS_SECRET_ACCESS_KEY Your AWS Secret Access Key | |
X_AMZ_SECURITY_TOKEN An Optional AWS Session Token | |
Return Code: | |
Returns 0 if operation completed successfully, 1 otherwise. | |
EOF | |
} | |
_parse_input_arguments() { | |
#Parse options | |
OPTIND=1 | |
while getopts ":hlsm:p:b:" opt; do | |
case "${opt}" in | |
'h') | |
_usage | |
exit 1 | |
;; | |
's') | |
SIGNURL_ONLY=1 | |
;; | |
'l') | |
LISTMODE=1 | |
;; | |
'm') | |
VALID_MINUTES="${OPTARG}" | |
;; | |
'p') | |
S3_PATH="${OPTARG}" | |
;; | |
'b') | |
S3_BUCKET="${OPTARG}" | |
;; | |
'?') | |
_usage | |
_die "Invalid option: -${OPTARG}" | |
;; | |
':') | |
_usage | |
_die "Option -${OPTARG} requires an argument." | |
;; | |
*) | |
_die "Internal error" | |
;; | |
esac | |
done | |
#Validation | |
if [ "${VALID_MINUTES}" -eq "${VALID_MINUTES}" ] 2>/dev/null ; then | |
false | |
else | |
_usage | |
_die "Valid Minutes (-m) must be an integer." | |
fi | |
if [ "${VALID_MINUTES}" -lt 1 ]; then | |
_usage | |
_die "Valid Minutes (-m) of ${VALID_MINUTES} seems unreasonable." | |
fi | |
if [ -z "${AWS_ACCESS_KEY_ID}" -o -z "${AWS_SECRET_ACCESS_KEY}" ]; then | |
_usage | |
_die "One of AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY is not set." | |
fi | |
if [ -z "${S3_BUCKET}" ]; then | |
_usage | |
_die "S3 Bucket (-b) is required." | |
fi | |
if [ -z "${S3_PATH}" -a "${LISTMODE}" -ne 1 ]; then | |
_usage | |
_die "S3 Path/Prefix (-p) is required unless you are listing objects." | |
fi | |
} | |
_validate_environment() { | |
SED=$(which sed) || _die "No sed found" | |
CURL=$(which curl) || _die "No curl found" | |
OPENSSL=$(which openssl) || _die "No openssl found" | |
if [ "${LISTMODE}" -eq 1 ]; then | |
XMLSTARLET=$(which xmlstarlet) || _die "No xmlstarlet found. XMLStarlet is required when listing objects." | |
fi | |
} | |
_calculate_expiration_date() { | |
NOW=$(date +%s) | |
let "EXPIRATION_DATE = NOW + (VALID_MINUTES * 60)" | |
} | |
_generate_signature() { | |
#Handle supported canonicalized AMZ headers. | |
CANONICAL= | |
if [ ! -z "${X_AMZ_SECURITY_TOKEN}" ]; then | |
CANONICAL="${CANONICAL}x-amz-security-token:${X_AMZ_SECURITY_TOKEN}\n" | |
fi | |
#Build the string to sign. | |
if [ "${LISTMODE}" -ne 1 ]; then | |
#Downloading | |
SIGN_STRING="GET\n\n\n${EXPIRATION_DATE}\n${CANONICAL}/${S3_BUCKET}/${S3_PATH}" | |
else | |
#Listing | |
SIGN_STRING="GET\n\n\n${EXPIRATION_DATE}\n${CANONICAL}/${S3_BUCKET}" | |
fi | |
#Sign the string. | |
SIGNATURE=$(echo -en "${SIGN_STRING}" | "${OPENSSL}" sha1 -hmac "${AWS_SECRET_ACCESS_KEY}" -binary | base64) | |
} | |
_simple_urlencode() { | |
echo -n "${1}" | "${CURL}" -Gso /dev/null -w %{url_effective} --data-urlencode @- "" | cut -c 3- | |
if [ "${?}" -ne 3 ]; then | |
_die "Super curl hack failed. :(" | |
fi | |
true | |
} | |
############# | |
### ENTRY ### | |
############# | |
set -o pipefail | |
#Parse the command line arguments | |
_parse_input_arguments "${@}" | |
#Validate working environment. | |
_validate_environment | |
#Normalize PATH variable. (Strip any leading forward slash off the PATH.) | |
if [ ! -z "${S3_PATH}" ]; then | |
S3_PATH=$(echo "${S3_PATH}" | "${SED}" 's|^/||') | |
fi | |
#Calculate the expiration date. | |
_calculate_expiration_date | |
#Generate the actual signature based on the action we're taking. | |
_generate_signature | |
#URL Encode components that will be embedded in the URL | |
ENCODED_AWS_ACCESS_KEY_ID=$(_simple_urlencode "${AWS_ACCESS_KEY_ID}") || _die "URL encoding failed." | |
ENCODED_EXPIRATION_DATE=$(_simple_urlencode "${EXPIRATION_DATE}") || _die "URL encoding failed." | |
ENCODED_SIGNATURE=$(_simple_urlencode "${SIGNATURE}") || _die "URL encoding failed." | |
#Handle supported canonicalized AMZ headers. | |
CANONICAL= | |
if [ ! -z "${X_AMZ_SECURITY_TOKEN}" ]; then | |
CANONICAL="${CANONICAL}&x-amz-security-token="$(_simple_urlencode "${X_AMZ_SECURITY_TOKEN}") || _die "URL encoding failed." | |
fi | |
#Actual signed URL. | |
if [ "${LISTMODE}" -ne 1 ]; then | |
#Downloading | |
SIGNED_URL="https://${S3_DOMAIN}/${S3_BUCKET}/${S3_PATH}?AWSAccessKeyId=${ENCODED_AWS_ACCESS_KEY_ID}&Expires=${ENCODED_EXPIRATION_DATE}&Signature=${ENCODED_SIGNATURE}${CANONICAL}" | |
else | |
#Listing | |
PREFIXVAR= | |
if [ ! -z "${S3_PATH}" ]; then | |
PREFIXVAR="prefix="$(_simple_urlencode "${S3_PATH}")"&" || _die "URL encoding failed." | |
fi | |
SIGNED_URL="https://${S3_DOMAIN}/${S3_BUCKET}?list-type=2&${PREFIXVAR}AWSAccessKeyId=${ENCODED_AWS_ACCESS_KEY_ID}&Expires=${ENCODED_EXPIRATION_DATE}&Signature=${ENCODED_SIGNATURE}${CANONICAL}" | |
fi | |
#Perform requested action | |
if [ "${SIGNURL_ONLY}" -eq 0 ]; then | |
if [ "${LISTMODE}" -ne 1 ]; then | |
#Fetch the file and dump it to STDOUT | |
"${CURL}" --silent --fail --show-error "${SIGNED_URL}" | |
else | |
#Fetch the listing, filter it, and dump the filtered listing to STDOUT. | |
"${CURL}" --silent --fail --show-error "${SIGNED_URL}" | "${SED}" 's/xmlns="[a-z0-9.:\/-]*"//g' | "${XMLSTARLET}" sel -T -t -v "/ListBucketResult/Contents/Key" -n | |
fi | |
if [ "${?}" -ne 0 ]; then | |
_die "Something went wrong." | |
fi | |
else | |
#Dump the signed URL to STDOUT | |
echo "${SIGNED_URL}" | |
fi | |
exit 0 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment