A minimal authenticated S3 download script using only Bash, Curl, and OpenSSL.

s3dl

#!/bin/bash

# s3dl
# by Alex Markley; released under CC0 Public Domain
# https://creativecommons.org/publicdomain/zero/1.0/
# Hosted at: https://gist.github.com/alexmarkley/0b45af17fdc805756ac710aeaceb77be
# Example script demonstrating how to fetch a private AWS S3 Object using only basic shell scripting techniques.


#################
### CONSTANTS ###
#################

S3_DOMAIN=s3.amazonaws.com


#################
### VARIABLES ###
#################

S3_BUCKET=
S3_PATH=
LISTMODE=0
SIGNURL_ONLY=0
VALID_MINUTES=5
EXPIRATION_DATE=
SIGNATURE=
SED=
CURL=
OPENSSL=
XMLSTARLET=


#################
### FUNCTIONS ###
#################

_log() {
	echo s3dl: "${@}" 1>&2
}

_die() {
	_log "FATAL:" "${@}"
	exit 1
}

_usage() {
cat << EOF

s3dl
====

Summary:
    Example script demonstrating how to fetch a private AWS S3 Object using only basic shell scripting techniques.

Usage:
    ${0##*/} [-h] [-l] [-s] [-m MIN] [-p PATH/PREFIX] -b BUCKET

    -h              Display this help message and exit.
    -l              List the objects at PREFIX instead of downloading the object at PATH.
    -s              Generate a signed URL and dump it to STDOUT. Do not actually perform the action.
    -m MIN          Generate a URL which is valid until MIN minutes into the future, instead of 5 minutes.
    -p PATH/PREFIX  When downloading an object, this is the PATH to the S3 Object being downloaded. When listing objects, this is the PREFIX, which filters the list.
    -b BUCKET       What S3 Bucket are we acting on?

Output:
    Output data, such as file contents or lists, go to STDOUT. Take care to direct STDOUT to a file and not to your terminal, especially if you are downloading a binary!

Credentials:
    It is always a security risk to provide credentials on the command line. Instead, provide your AWS credentials in the form of Environment variables.

    AWS_ACCESS_KEY_ID       Your AWS Access Key ID
    AWS_SECRET_ACCESS_KEY   Your AWS Secret Access Key
    X_AMZ_SECURITY_TOKEN    An Optional AWS Session Token

Return Code:
    Returns 0 if operation completed successfully, 1 otherwise.

EOF
}

_parse_input_arguments() {
	#Parse options
	OPTIND=1
	while getopts ":hlsm:p:b:" opt; do
		case "${opt}" in
			'h')
				_usage
				exit 1
				;;
			's')
				SIGNURL_ONLY=1
				;;
			'l')
				LISTMODE=1
				;;
			'm')
				VALID_MINUTES="${OPTARG}"
				;;
			'p')
				S3_PATH="${OPTARG}"
				;;
			'b')
				S3_BUCKET="${OPTARG}"
				;;
			'?')
				_usage
				_die "Invalid option: -${OPTARG}"
				;;
			':')
				_usage
				_die "Option -${OPTARG} requires an argument."
				;;
			*)
				_die "Internal error"
				;;
		esac
	done

	#Validation
	if [ "${VALID_MINUTES}" -eq "${VALID_MINUTES}" ] 2>/dev/null ; then
		false
	else
		_usage
		_die "Valid Minutes (-m) must be an integer."
	fi
	if [ "${VALID_MINUTES}" -lt 1 ]; then
		_usage
		_die "Valid Minutes (-m) of ${VALID_MINUTES} seems unreasonable."
	fi
	if [ -z "${AWS_ACCESS_KEY_ID}" -o -z "${AWS_SECRET_ACCESS_KEY}" ]; then
		_usage
		_die "One of AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY is not set."
	fi
	if [ -z "${S3_BUCKET}" ]; then
		_usage
		_die "S3 Bucket (-b) is required."
	fi
	if [ -z "${S3_PATH}" -a "${LISTMODE}" -ne 1 ]; then
		_usage
		_die "S3 Path/Prefix (-p) is required unless you are listing objects."
	fi
}

_validate_environment() {
	SED=$(which sed) || _die "No sed found"
	CURL=$(which curl) || _die "No curl found"
	OPENSSL=$(which openssl) || _die "No openssl found"
	if [ "${LISTMODE}" -eq 1 ]; then
		XMLSTARLET=$(which xmlstarlet) || _die "No xmlstarlet found. XMLStarlet is required when listing objects."
	fi
}

_calculate_expiration_date() {
	NOW=$(date +%s)
	let "EXPIRATION_DATE = NOW + (VALID_MINUTES * 60)"
}

_generate_signature() {
	#Handle supported canonicalized AMZ headers.
	CANONICAL=
	if [ ! -z "${X_AMZ_SECURITY_TOKEN}" ]; then
		CANONICAL="${CANONICAL}x-amz-security-token:${X_AMZ_SECURITY_TOKEN}\n"
	fi
	#Build the string to sign.
	if [ "${LISTMODE}" -ne 1 ]; then
		#Downloading
		SIGN_STRING="GET\n\n\n${EXPIRATION_DATE}\n${CANONICAL}/${S3_BUCKET}/${S3_PATH}"
	else
		#Listing
		SIGN_STRING="GET\n\n\n${EXPIRATION_DATE}\n${CANONICAL}/${S3_BUCKET}"
	fi
	#Sign the string.
	SIGNATURE=$(echo -en "${SIGN_STRING}" | "${OPENSSL}" sha1 -hmac "${AWS_SECRET_ACCESS_KEY}" -binary | base64)
}

_simple_urlencode() {
	echo -n "${1}" | "${CURL}" -Gso /dev/null -w %{url_effective} --data-urlencode @- "" | cut -c 3-
	if [ "${?}" -ne 3 ]; then
		_die "Super curl hack failed. :("
	fi
	true
}


#############
### ENTRY ###
#############

set -o pipefail

#Parse the command line arguments
_parse_input_arguments "${@}"

#Validate working environment.
_validate_environment

#Normalize PATH variable. (Strip any leading forward slash off the PATH.)
if [ ! -z "${S3_PATH}" ]; then
	S3_PATH=$(echo "${S3_PATH}" | "${SED}" 's|^/||')
fi

#Calculate the expiration date.
_calculate_expiration_date

#Generate the actual signature based on the action we're taking.
_generate_signature

#URL Encode components that will be embedded in the URL
ENCODED_AWS_ACCESS_KEY_ID=$(_simple_urlencode "${AWS_ACCESS_KEY_ID}") || _die "URL encoding failed."
ENCODED_EXPIRATION_DATE=$(_simple_urlencode "${EXPIRATION_DATE}") || _die "URL encoding failed."
ENCODED_SIGNATURE=$(_simple_urlencode "${SIGNATURE}") || _die "URL encoding failed."

#Handle supported canonicalized AMZ headers.
CANONICAL=
if [ ! -z "${X_AMZ_SECURITY_TOKEN}" ]; then
	CANONICAL="${CANONICAL}&x-amz-security-token="$(_simple_urlencode "${X_AMZ_SECURITY_TOKEN}") || _die "URL encoding failed."
fi

#Actual signed URL.
if [ "${LISTMODE}" -ne 1 ]; then
	#Downloading
	SIGNED_URL="https://${S3_DOMAIN}/${S3_BUCKET}/${S3_PATH}?AWSAccessKeyId=${ENCODED_AWS_ACCESS_KEY_ID}&Expires=${ENCODED_EXPIRATION_DATE}&Signature=${ENCODED_SIGNATURE}${CANONICAL}"
else
	#Listing
	PREFIXVAR=
	if [ ! -z "${S3_PATH}" ]; then
		PREFIXVAR="prefix="$(_simple_urlencode "${S3_PATH}")"&" || _die "URL encoding failed."
	fi
	SIGNED_URL="https://${S3_DOMAIN}/${S3_BUCKET}?list-type=2&${PREFIXVAR}AWSAccessKeyId=${ENCODED_AWS_ACCESS_KEY_ID}&Expires=${ENCODED_EXPIRATION_DATE}&Signature=${ENCODED_SIGNATURE}${CANONICAL}"
fi

#Perform requested action
if [ "${SIGNURL_ONLY}" -eq 0 ]; then
	if [ "${LISTMODE}" -ne 1 ]; then
		#Fetch the file and dump it to STDOUT
		"${CURL}" --silent --fail --show-error "${SIGNED_URL}"
	else
		#Fetch the listing, filter it, and dump the filtered listing to STDOUT.
		"${CURL}" --silent --fail --show-error "${SIGNED_URL}" | "${SED}" 's/xmlns="[a-z0-9.:\/-]*"//g' | "${XMLSTARLET}" sel -T -t -v "/ListBucketResult/Contents/Key" -n
	fi
	if [ "${?}" -ne 0 ]; then
		_die "Something went wrong."
	fi
else
	#Dump the signed URL to STDOUT
	echo "${SIGNED_URL}"
fi

exit 0