|
#!/usr/bin/env bash |
|
|
|
# REF: https://repost.aws/knowledge-center/s3-undelete-configuration |
|
|
|
# Requirements: |
|
# - jq: sudo apt-get install jq |
|
# - awk |
|
|
|
# Exit on error. Append "|| true" if you expect an error. |
|
set -o errexit |
|
# Exit On Error Inside Any Functions Or Subshells. |
|
set -o errtrace |
|
# Do Not Allow Use Of Undefined Vars. Use ${VAR:-} To Use An Undefined VAR |
|
set -o nounset |
|
# Catch The Error In Case Mysqldump Fails (But Gzip Succeeds) In `mysqldump |gzip` |
|
# set -o pipefail |
|
# Turn On Traces, Useful While Debugging But Commented Out By Default |
|
# set -o xtrace |
|
|
|
# Set magic variables for current file, directory, os, etc. |
|
readonly __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
|
readonly __log_dir="${__dir}/logs" |
|
readonly __file="${__dir}/$(basename "${BASH_SOURCE[0]}")" |
|
readonly __base="$(basename "${__file}" .sh)" |
|
readonly __invocation="$(printf %q "${__file}")$( (($#)) && printf ' %q' "$@" || true)" |
|
readonly __paramlist="${@}" |
|
__indent="" |
|
|
|
# Constants |
|
readonly NA="##NA##" |
|
readonly TRUE="TRUE" |
|
readonly FALSE="FALSE" |
|
|
|
# Set the initial runtime |
|
readonly RUNTIME=$(date '+%Y%m%d%H%M%S') |
|
|
|
# Capture the start time |
|
readonly START_TIME=$(date +%s) |
|
|
|
unset -v LOG_FILE_PATH |
|
# >>> Pretty Print Functions >>> |
|
############################################################################## |
|
log() { |
|
local timestamp="$(date +"%Y-%m-%d %H:%M:%S")" |
|
local log_message="${__indent}|${timestamp} => ${@}" |
|
|
|
# Write to console |
|
echo -e "${log_message}" |
|
|
|
local log_file_path="${LOG_FILE_PATH:-${NA}}" |
|
|
|
# Write to log file if LOG_FILE_PATH is defined |
|
if [ "${log_file_path}" != "${NA}" ]; then |
|
echo -e "${log_message}" >> "${LOG_FILE_PATH}" |
|
fi |
|
} |
|
|
|
add_indent() { |
|
__indent=" ${__indent}" |
|
} |
|
|
|
subtract_indent() { |
|
__indent="${__indent# }" |
|
} |
|
|
|
newline() { |
|
local num_of_newlines=${1:-1} |
|
for _ in $(seq "${num_of_newlines}"); do |
|
echo -e "\n" |
|
done |
|
} |
|
# <<< Pretty Print Functions. <<< |
|
|
|
|
|
# >>> Define Usage And Helptext >>> |
|
############################################################################## |
|
__sample_usage="SAMPLE USAGE: ${0} [ -s s3_path ] [-d] # e.g. bash ${0} -s 's3://my-bucket/path/to/my/dir' " |
|
|
|
[[ "${__usage+_}" ]] || read -r -d '' __usage <<-'EOF'|| true # exits non-zero when EOF encountered |
|
-s prefix <<MANDATORY PARAMETER>>: S3 path. |
|
|
|
-d prefix <<OPTIONAL FLAG>>: Flag indicating whether to delete delete-markers. |
|
|
|
-h --help This Page |
|
EOF |
|
|
|
# shellcheck disable=SC2015 |
|
[[ "${__helptext+_}" ]] || read -r -d '' __helptext <<-'EOF' || true # exits non-zero when EOF encountered |
|
A script to list or delete delete-markers in a versioning-enabled S3 bucket. Deleting delete-markers restores the deleted object. |
|
EOF |
|
|
|
help () { |
|
echo "" 1>&2 |
|
echo " ${*}" 1>&2 |
|
echo "" 1>&2 |
|
echo " ${__sample_usage}" |
|
echo "" 1>&2 |
|
echo " ${__usage:-No usage available}" 1>&2 |
|
echo "" 1>&2 |
|
|
|
if [[ "${__helptext:-}" ]]; then |
|
echo " ${__helptext}" 1>&2 |
|
echo "" 1>&2 |
|
fi |
|
exit 1 |
|
} |
|
# <<< Define Usage And Helptext. <<< |
|
|
|
|
|
# >>> Print Script Invocation Info >>> |
|
############################################################################## |
|
|
|
log "----------------------------------" |
|
log "SCRIPT INVOCATION DETAILS" |
|
log "----------------------------------" |
|
log "##############################################################################" |
|
add_indent |
|
log "Script Dir: '${__dir}'" |
|
log "Log Dir: '${__log_dir}'" |
|
log "Script Complete Path: '${__file}'" |
|
log "Script File Base Name: '${__base}'" |
|
log "Script Invocation: '${__invocation}'" |
|
log "Script Param List: '${__paramlist}'" |
|
subtract_indent |
|
log "##############################################################################" |
|
newline |
|
# <<< Print Script Invocation Info. <<< |
|
|
|
# Create log dir |
|
mkdir -p "${__log_dir}" |
|
|
|
# >>> Parse Opt Args >>> |
|
############################################################################## |
|
# Initialize variables |
|
bucket_name="${NA}" |
|
prefix="${NA}" |
|
deletion_flag="${FALSE}" # Set deletion_flag to FALSE by default |
|
|
|
|
|
# Read Options |
|
while getopts ":s:dh" o; do |
|
case "${o}" in |
|
s) |
|
# Trim whitespaces and parse S3 path |
|
s3_path="${OPTARG}" |
|
log "Input S3 Path: '${s3_path}'" |
|
newline |
|
|
|
# Extract bucket_name and prefix |
|
s3_path="$(echo "${s3_path}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | sed 's/^s3:\/\///')" |
|
|
|
bucket_name="${s3_path%%/*}" |
|
prefix="${s3_path#*/}" |
|
|
|
;; |
|
d) |
|
deletion_flag="${TRUE}" # Set deletion_flag to TRUE if -d option is provided |
|
;; |
|
h) |
|
help "Help Using: '${0}'" |
|
;; |
|
:) |
|
echo "ERROR: Option '-${OPTARG}' Requires An Argument" |
|
exit 1 |
|
;; |
|
\?) |
|
help "ERROR: Invalid Option '-${OPTARG}'" |
|
exit 1 |
|
;; |
|
esac |
|
done |
|
|
|
shift $((OPTIND-1)) |
|
|
|
if [[ "${bucket_name}" == "${NA}" ]]; then |
|
help "ERROR: Missing Required Argument '[ -s s3_path ]'" |
|
exit 1 |
|
fi |
|
|
|
if [[ "${prefix}" == "${NA}" ]]; then |
|
help "ERROR: Unable to derive 'prefix' from the provided S3 path." |
|
exit 1 |
|
fi |
|
|
|
if [[ -z "${bucket_name}" ]]; then |
|
help "ERROR: Derived 'bucket_name' is empty." |
|
exit 1 |
|
fi |
|
|
|
if [[ -z "${prefix}" ]]; then |
|
help "ERROR: Derived 'prefix' is empty." |
|
exit 1 |
|
fi |
|
# <<< Parse Opt Args. <<< |
|
|
|
|
|
# >>> Parse Nonopt Args, Set Defaults For Variables >>> |
|
############################################################################## |
|
__num_of_args=${#} |
|
|
|
if [[ ${__num_of_args} -gt 0 ]]; |
|
then |
|
help "ERROR: Script only requires the S3 path. Remove extra parameters and try again." |
|
exit 1 |
|
fi |
|
# <<< Parse Nonopt Args, Set Defaults For Variables. <<< |
|
|
|
|
|
# >>> Validate Parameter Values & Set Execution variables>>> |
|
############################################################################## |
|
readonly BUCKET_NAME="$(echo "${bucket_name}" | sed 's|^/||;s|/$||')" |
|
readonly PREFIX="$(echo "${prefix}" | sed 's|^/||;s|/$||')" |
|
readonly DELETION_FLAG="${deletion_flag}" |
|
readonly LOG_FILE="$(echo "${PREFIX}" | tr '/' '_' | sed 's/[[:space:]]//g')_${RUNTIME}.log" |
|
readonly LOG_FILE_PATH="${__log_dir}/${LOG_FILE}" |
|
# <<< Validate Parameter Values & Set Execution variables. <<< |
|
|
|
# >>> Print Script Execution Varaible Details >>> |
|
############################################################################## |
|
log "----------------------------------" |
|
log "SCRIPT EXECUTION VARIABLE DETAILS" |
|
log "----------------------------------" |
|
log "##############################################################################" |
|
add_indent |
|
log "BUCKET_NAME: '${BUCKET_NAME}'" |
|
log "PREFIX: '${PREFIX}'" |
|
log "DELETION_FLAG: '${DELETION_FLAG}'" |
|
log "LOG_FILE: '${LOG_FILE}'" |
|
log "LOG_FILE_PATH: '${LOG_FILE_PATH}'" |
|
log "START_TIME: '${START_TIME}'" |
|
subtract_indent |
|
log "##############################################################################" |
|
newline |
|
# <<< Print Script Execution Variable Details. <<< |
|
|
|
log "----------------------------------" |
|
if [[ ${DELETION_FLAG} == "${TRUE}" ]]; |
|
then |
|
log "BEGIN PROCESSING IN [DELETE] MODE. RUN WITHOUT -d OPTION TO LIST FILES WITH DELETE-MARKERS." |
|
log "Example Usage: bash ${0} -s '${s3_path}'" |
|
else |
|
log "BEGIN PROCESSING IN [LIST] MODE. USE OPTION -d TO DELETE DELETE-MARKERS." |
|
log "Example Usage: bash ${0} -s '${s3_path}' -d" |
|
fi |
|
log "----------------------------------" |
|
log "##############################################################################" |
|
|
|
|
|
log "Using below command to get the list of the delete-markers:" |
|
log "aws s3api list-object-versions --bucket \"${BUCKET_NAME}\" --prefix \"${PREFIX}\" --output json --query 'DeleteMarkers[?IsLatest==\`true\`].[Key, VersionId, LastModified]'" |
|
|
|
add_indent |
|
log ">>> S3 DELETED OBJECT LIST >>>" |
|
CTR=0 |
|
aws s3api list-object-versions --bucket "${BUCKET_NAME}" --prefix "${PREFIX}" --output json --query 'DeleteMarkers[?IsLatest==`true`].[Key, VersionId, LastModified]' | |
|
grep -v '^null$' | jq -r '.[] | "'\''" + .[0] + "'\''~" + .[1] + "~" + .[2]' | while read result |
|
do |
|
CTR=$((CTR+1)) |
|
S3_OBJECT_KEY="$( echo "${result}"| awk -F\' '{print $2}' )" |
|
VERSION_ID="$( echo "${result}" | awk -F '~' '{print $2}' )" |
|
LAST_MODIFIED="$( echo "${result}" | awk -F '~' '{print $3}' )" |
|
|
|
if [[ ${DELETION_FLAG} == "${TRUE}" ]]; |
|
then |
|
log "[DELETE] ${CTR}): S3_OBJECT_KEY='${S3_OBJECT_KEY}' | VERSION='${VERSION_ID}' | LAST_MODIFIED='${LAST_MODIFIED}'" |
|
aws s3api delete-object --bucket "${BUCKET_NAME}" --key "${S3_OBJECT_KEY}" --version-id "${VERSION_ID}" > /dev/null 2>&1 |
|
else |
|
log "[LIST] ${CTR}): S3_OBJECT_KEY='${S3_OBJECT_KEY}' | VERSION='${VERSION_ID}' | LAST_MODIFIED='${LAST_MODIFIED}'" |
|
fi |
|
done |
|
log "<<< S3 DELETED OBJECT LIST <<<" |
|
subtract_indent |
|
|
|
log "##############################################################################" |
|
newline |
|
|
|
# Capture the end time |
|
END_TIME=$(date +%s) |
|
log "END_TIME: '${END_TIME}'" |
|
|
|
# Calculate the elapsed time |
|
ELAPSED_TIME=$((END_TIME - START_TIME)) |
|
|
|
# Print elapsed time |
|
log "Elapsed Time: ${ELAPSED_TIME} seconds" |
