Skip to content

Instantly share code, notes, and snippets.

@alexmarkley
Last active January 27, 2022 17:36
Show Gist options
  • Save alexmarkley/0b45af17fdc805756ac710aeaceb77be to your computer and use it in GitHub Desktop.
Save alexmarkley/0b45af17fdc805756ac710aeaceb77be to your computer and use it in GitHub Desktop.
A minimal authenticated S3 download script using only Bash, Curl, and OpenSSL.
#!/bin/bash
# s3dl
# by Alex Markley; released under CC0 Public Domain
# https://creativecommons.org/publicdomain/zero/1.0/
# Hosted at: https://gist.github.com/alexmarkley/0b45af17fdc805756ac710aeaceb77be
# Example script demonstrating how to fetch a private AWS S3 Object using only basic shell scripting techniques.
#################
### CONSTANTS ###
#################
S3_DOMAIN=s3.amazonaws.com
#################
### VARIABLES ###
#################
S3_BUCKET=
S3_PATH=
LISTMODE=0
SIGNURL_ONLY=0
VALID_MINUTES=5
EXPIRATION_DATE=
SIGNATURE=
SED=
CURL=
OPENSSL=
XMLSTARLET=
#################
### FUNCTIONS ###
#################
_log() {
echo s3dl: "${@}" 1>&2
}
_die() {
_log "FATAL:" "${@}"
exit 1
}
_usage() {
cat << EOF
s3dl
====
Summary:
Example script demonstrating how to fetch a private AWS S3 Object using only basic shell scripting techniques.
Usage:
${0##*/} [-h] [-l] [-s] [-m MIN] [-p PATH/PREFIX] -b BUCKET
-h Display this help message and exit.
-l List the objects at PREFIX instead of downloading the object at PATH.
-s Generate a signed URL and dump it to STDOUT. Do not actually perform the action.
-m MIN Generate a URL which is valid until MIN minutes into the future, instead of 5 minutes.
-p PATH/PREFIX When downloading an object, this is the PATH to the S3 Object being downloaded. When listing objects, this is the PREFIX, which filters the list.
-b BUCKET What S3 Bucket are we acting on?
Output:
Output data, such as file contents or lists, go to STDOUT. Take care to direct STDOUT to a file and not to your terminal, especially if you are downloading a binary!
Credentials:
It is always a security risk to provide credentials on the command line. Instead, provide your AWS credentials in the form of Environment variables.
AWS_ACCESS_KEY_ID Your AWS Access Key ID
AWS_SECRET_ACCESS_KEY Your AWS Secret Access Key
X_AMZ_SECURITY_TOKEN An Optional AWS Session Token
Return Code:
Returns 0 if operation completed successfully, 1 otherwise.
EOF
}
_parse_input_arguments() {
#Parse options
OPTIND=1
while getopts ":hlsm:p:b:" opt; do
case "${opt}" in
'h')
_usage
exit 1
;;
's')
SIGNURL_ONLY=1
;;
'l')
LISTMODE=1
;;
'm')
VALID_MINUTES="${OPTARG}"
;;
'p')
S3_PATH="${OPTARG}"
;;
'b')
S3_BUCKET="${OPTARG}"
;;
'?')
_usage
_die "Invalid option: -${OPTARG}"
;;
':')
_usage
_die "Option -${OPTARG} requires an argument."
;;
*)
_die "Internal error"
;;
esac
done
#Validation
if [ "${VALID_MINUTES}" -eq "${VALID_MINUTES}" ] 2>/dev/null ; then
false
else
_usage
_die "Valid Minutes (-m) must be an integer."
fi
if [ "${VALID_MINUTES}" -lt 1 ]; then
_usage
_die "Valid Minutes (-m) of ${VALID_MINUTES} seems unreasonable."
fi
if [ -z "${AWS_ACCESS_KEY_ID}" -o -z "${AWS_SECRET_ACCESS_KEY}" ]; then
_usage
_die "One of AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY is not set."
fi
if [ -z "${S3_BUCKET}" ]; then
_usage
_die "S3 Bucket (-b) is required."
fi
if [ -z "${S3_PATH}" -a "${LISTMODE}" -ne 1 ]; then
_usage
_die "S3 Path/Prefix (-p) is required unless you are listing objects."
fi
}
_validate_environment() {
SED=$(which sed) || _die "No sed found"
CURL=$(which curl) || _die "No curl found"
OPENSSL=$(which openssl) || _die "No openssl found"
if [ "${LISTMODE}" -eq 1 ]; then
XMLSTARLET=$(which xmlstarlet) || _die "No xmlstarlet found. XMLStarlet is required when listing objects."
fi
}
_calculate_expiration_date() {
NOW=$(date +%s)
let "EXPIRATION_DATE = NOW + (VALID_MINUTES * 60)"
}
_generate_signature() {
#Handle supported canonicalized AMZ headers.
CANONICAL=
if [ ! -z "${X_AMZ_SECURITY_TOKEN}" ]; then
CANONICAL="${CANONICAL}x-amz-security-token:${X_AMZ_SECURITY_TOKEN}\n"
fi
#Build the string to sign.
if [ "${LISTMODE}" -ne 1 ]; then
#Downloading
SIGN_STRING="GET\n\n\n${EXPIRATION_DATE}\n${CANONICAL}/${S3_BUCKET}/${S3_PATH}"
else
#Listing
SIGN_STRING="GET\n\n\n${EXPIRATION_DATE}\n${CANONICAL}/${S3_BUCKET}"
fi
#Sign the string.
SIGNATURE=$(echo -en "${SIGN_STRING}" | "${OPENSSL}" sha1 -hmac "${AWS_SECRET_ACCESS_KEY}" -binary | base64)
}
_simple_urlencode() {
echo -n "${1}" | "${CURL}" -Gso /dev/null -w %{url_effective} --data-urlencode @- "" | cut -c 3-
if [ "${?}" -ne 3 ]; then
_die "Super curl hack failed. :("
fi
true
}
#############
### ENTRY ###
#############
set -o pipefail
#Parse the command line arguments
_parse_input_arguments "${@}"
#Validate working environment.
_validate_environment
#Normalize PATH variable. (Strip any leading forward slash off the PATH.)
if [ ! -z "${S3_PATH}" ]; then
S3_PATH=$(echo "${S3_PATH}" | "${SED}" 's|^/||')
fi
#Calculate the expiration date.
_calculate_expiration_date
#Generate the actual signature based on the action we're taking.
_generate_signature
#URL Encode components that will be embedded in the URL
ENCODED_AWS_ACCESS_KEY_ID=$(_simple_urlencode "${AWS_ACCESS_KEY_ID}") || _die "URL encoding failed."
ENCODED_EXPIRATION_DATE=$(_simple_urlencode "${EXPIRATION_DATE}") || _die "URL encoding failed."
ENCODED_SIGNATURE=$(_simple_urlencode "${SIGNATURE}") || _die "URL encoding failed."
#Handle supported canonicalized AMZ headers.
CANONICAL=
if [ ! -z "${X_AMZ_SECURITY_TOKEN}" ]; then
CANONICAL="${CANONICAL}&x-amz-security-token="$(_simple_urlencode "${X_AMZ_SECURITY_TOKEN}") || _die "URL encoding failed."
fi
#Actual signed URL.
if [ "${LISTMODE}" -ne 1 ]; then
#Downloading
SIGNED_URL="https://${S3_DOMAIN}/${S3_BUCKET}/${S3_PATH}?AWSAccessKeyId=${ENCODED_AWS_ACCESS_KEY_ID}&Expires=${ENCODED_EXPIRATION_DATE}&Signature=${ENCODED_SIGNATURE}${CANONICAL}"
else
#Listing
PREFIXVAR=
if [ ! -z "${S3_PATH}" ]; then
PREFIXVAR="prefix="$(_simple_urlencode "${S3_PATH}")"&" || _die "URL encoding failed."
fi
SIGNED_URL="https://${S3_DOMAIN}/${S3_BUCKET}?list-type=2&${PREFIXVAR}AWSAccessKeyId=${ENCODED_AWS_ACCESS_KEY_ID}&Expires=${ENCODED_EXPIRATION_DATE}&Signature=${ENCODED_SIGNATURE}${CANONICAL}"
fi
#Perform requested action
if [ "${SIGNURL_ONLY}" -eq 0 ]; then
if [ "${LISTMODE}" -ne 1 ]; then
#Fetch the file and dump it to STDOUT
"${CURL}" --silent --fail --show-error "${SIGNED_URL}"
else
#Fetch the listing, filter it, and dump the filtered listing to STDOUT.
"${CURL}" --silent --fail --show-error "${SIGNED_URL}" | "${SED}" 's/xmlns="[a-z0-9.:\/-]*"//g' | "${XMLSTARLET}" sel -T -t -v "/ListBucketResult/Contents/Key" -n
fi
if [ "${?}" -ne 0 ]; then
_die "Something went wrong."
fi
else
#Dump the signed URL to STDOUT
echo "${SIGNED_URL}"
fi
exit 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment