Cleanup s3 bucket from build packages and keep only the last 30 items of each project Raw

cleanup-s3-artifacts.sh

#!/bin/bash
AWS_BUCKET_NAME="<insert_bucket>"
AWS_BUCKET_REGION="<insert_region>"
AWS_KEY="<insert_key>"
AWS_SECRET="<insert_secret>"
MAX_COUNT_TO_KEEP=30
PROFILE="<insert_profile>"

log()       { echo -e "["$(date "+%Y%m%dT%H%M%S")"] $1"; }
echo_blu(){ log "${BLUE}$1${NC}"; }
echo_red(){ log "${RED}$1${NC}"; }
echo_grn(){ log "${GREEN}$1${NC}"; }
echo_cya(){ log "${CYAN}$1${NC}"; }
echo_yel(){ log "${YELLOW}$1${NC}"; }
echo_mag(){ log "${MAGENTA}$1${NC}"; }

info()    { echo_blu "[INFO]    $1"; }
warning() { echo_yel "[WARN]    $1"; }
error()   { echo_red "[ERROR]   $1"; }
fatal()   { echo_red "[FATAL]   $1"; exit 1 ; }
debug()   { if [ "${DEBUG}" == "true" ]; then echo_cya "[DEBUG]   :: ${FUNCNAME[1]} :: $1"; fi }

configure_aws_cli() {
  debug

  info "Setting keys for bucket: $AWS_BUCKET_NAME region: $AWS_BUCKET_REGION"

  aws configure set aws_access_key_id ${AWS_KEY} --profile ${PROFILE}
  aws configure set aws_secret_access_key ${AWS_SECRET} --profile ${PROFILE}
  aws configure set output 'json' --profile ${PROFILE}
  aws configure set region ${AWS_BUCKET_REGION} --profile ${PROFILE}
}

separator() {
  SEP=$(printf '%*s' 101 | tr ' ' '#')
  echo_blu "[INFO]    $SEP"
 }

STEP=0

bump_step(){
  ((STEP++))
  echo_blu "[INFO]    ($STEP) $1"
}

set_colors(){
    debug
    RED='\033[0;31m'
    BLUE='\033[0;34m'
    GREEN='\033[0;32m'
    YELLOW='\033[33;m'
    CYAN='\033[0;36m'
    MAGENTA='\033[35m'
    BOLD_WHITE="\033[1m"
    NC='\033[0m'
}

unset_colors(){
    debug
    RED=''
    BLUE=''
    GREEN=''
    YELLOW=''
    CYAN=''
    MAGENTA=''
    BOLD_WHITE=''
    NC=''
}

configure_colors(){
  debug
  if test -t 1; then
    number_colors=$(tput colors)
    if test -n "${number_colors}" && test ${number_colors} -ge 6; then
        set_colors      
    else
        unset_colors
    fi
  fi
}

iterate_all_items(){
    # pattern -->>> client-project-component-TIMESTAMP-TIMESTAMP-ID-xxx-xxxx.tar.gz
    PATTERN='([a-z]+-[a-z]+-*[a-z]*)-[0-9]{8}-[0-9]{6}-.*'
    COUNTER=0
    ARTIFACT_COUNTER=0
    REMOVED_COUNTER=0

    PREVIOUS_ITEM=''
    IFS=$'\n'
    for item in ${RAW_LIST} 
    do
        (( COUNTER++ ))
        (( ARTIFACT_COUNTER++ ))

        [[ $item =~ $PATTERN ]] 

        ARTIFACT_NAME=${BASH_REMATCH[1]}

        if [ "${PREVIOUS_ITEM}" != "${ARTIFACT_NAME}" ]; then
            ARTIFACT_COUNTER=0
        else 
            if (( $ARTIFACT_COUNTER <= $MAX_COUNT_TO_KEEP )); then
                info "${GREEN}(${ARTIFACT_COUNTER}) ${BLUE}keeping ${item}"
            else
                info "${GREEN}(${ARTIFACT_COUNTER}) ${RED}removing ${item}"
                item=$(echo $item | cut -d ' ' -f 1)
                remove_item "${item}"
                (( REMOVED_COUNTER++ ))
            fi
        fi

        PREVIOUS_ITEM=$ARTIFACT_NAME
    done

    separator
    bump_step "FINISHED WITH SUCCESS !!!"
    separator
    info 
    separator
    info "# TOTAL ARTIFACTS IN REPOSITORY \t${YELLOW}${COUNTER}${NC}\t\t\t\t\t\t\t\t${BLUE}#"
    info "# TOTAL ITEMS REMOVED \t\t${RED}${REMOVED_COUNTER}${NC}\t\t\t\t\t\t\t\t${BLUE}#"
    info "# TOTAL ITEMS REMAINING \t\t${GREEN}$(( $COUNTER - $REMOVED_COUNTER ))${NC}\t\t\t\t\t\t\t\t${BLUE}#"
    separator
}

remove_item(){
    local ITEM_TO_REMOVE=$1
    aws s3 rm s3://${AWS_BUCKET_NAME}/${ITEM_TO_REMOVE} --region ${AWS_BUCKET_REGION} --profile ${PROFILE}
}

is_mac_osx(){
  OSPLATFORM=$(uname)
  [ "$OSPLATFORM" == "Darwin" ]
}

main_function(){

    set_colors

    info
    separator
    bump_step "CLEANING UP ARTIFACTS BUCKET ${AWS_BUCKET_NAME}"
    separator
    info

    bump_step "CONFIGURE AWS CLI"
    separator
    configure_aws_cli

    bump_step "GET LIST OF PACKAGES FROM AWS S3"
    separator
    if is_mac_osx; then
        RAW_LIST=$(aws s3 ls s3://${AWS_BUCKET_NAME} --region ${AWS_BUCKET_REGION} --human-readable --profile ${PROFILE} | tail -r | awk '{ print $5 FS $3 $4 }' | grep tar.gz)
    else
        RAW_LIST=$(aws s3 ls s3://${AWS_BUCKET_NAME} --region ${AWS_BUCKET_REGION} --human-readable --profile ${PROFILE}| tac | awk '{ print $5 FS $3 $4 }' | grep tar.gz)
    fi

    bump_step "ITERATE AND DELETE PACKAGES WITH MORE THAN $MAX_COUNT_TO_KEEP ITEMS"
    separator

    info
    info

    iterate_all_items
}

main_function