Download AWS RDS / Aurora logs between specified dates - working around truncated files from broken API

fetch_rds_logs.sh

#!/bin/bash

set -e

REGION=eu-west-1
DB_INSTANCE=
USE_CSV=0
START_DATE=`gdate -u +'%Y-%m-%d'`
END_DATE=`gdate -u +'%Y-%m-%d' --date 'tomorrow'`

usage () {
    echo "USAGE: $0 [OPTIONS]"
    echo "Options:"
    echo "    -r  : AWS Region. Default '$REGION'"
    echo "    -d  : RDS PostgreSQL Instance Name. Default '$DB_INSTANCE'"
    echo "    -c  : Use CSV log files. Allowed values 0 or 1. Default '$USE_CSV'"
    echo "    -s  : Start date of log file [YYYY-MM-DD or YYYY-MM-DD-HH]. Defaults to current date (in UTC) '$START_DATE'"
    echo "    -e  : End date of log file [YYYY-MM-DD or YYYY-MM-DD-HH]. Defaults to next day (in UTC) '$END_DATE'"
    echo
    exit 1
}

log () {
    echo "`gdate +%Y%m%d-%H%M%S`: ${@}"
}

while getopts :r:d:c:s:e:h option
do
    case "${option}" in
        r)
            REGION=${OPTARG};;
        d)
            DB_INSTANCE=${OPTARG};;
        c)
            USE_CSV=${OPTARG}
            if [ "$USE_CSV" -ne "0" -a "$USE_CSV" -ne "1" ]; then
                echo "ERROR: Invalid value for argument '-c'"
                usage
            fi
            ;;
        s)
            START_DATE=${OPTARG};;
        e)
            END_DATE=${OPTARG};;
        h)
            usage;;
        \?)
            echo 1>&2 "ERROR: Invalid argument passed."
            usage;;
    esac
done

echo
log "Fetching logs generated between dates [ $START_DATE ] and [ $END_DATE ] (UTC)."

download=0

if [ $USE_CSV -eq 1 ]; then
    logfilelist=$(aws --region $REGION rds describe-db-log-files --db-instance-identifier $DB_INSTANCE --no-paginate --output text | grep "error/postgresql.log.*.csv" | cut -f3 | cut -d '/' -f2 | sort)
else
    logfilelist=$(aws --region $REGION rds describe-db-log-files --db-instance-identifier $DB_INSTANCE --no-paginate --output text | grep -v ".csv" | grep "error/postgresql.log.*" | cut -f3 | cut -d '/' -f2 | sort)
fi

for LOGFILE in $logfilelist; do
    if [[ $LOGFILE == *"$START_DATE"* ]]; then
        download=1
    fi

    if [ $download -eq 1 ]
    then
        COUNTER=1
        LASTFOUNDTOKEN=0
        PREVIOUSTOKEN=0

        rm -f ${LOGFILE}

        while [  $COUNTER -lt 100 ]; do
          log "Downloading $LOGFILE (part ${COUNTER} from marker ${LASTFOUNDTOKEN})"
          PREVIOUSTOKEN=${LASTFOUNDTOKEN}

          aws --region $REGION rds download-db-log-file-portion --db-instance-identifier $DB_INSTANCE --log-file-name error/${LOGFILE} --starting-token ${LASTFOUNDTOKEN} --debug --output text 2>>${LOGFILE}.${COUNTER}.debug >> ${LOGFILE}.${COUNTER}
          LASTFOUNDTOKEN=`tac ${LOGFILE}.${COUNTER}.debug | grep -m1 -oP "<Marker>([0-9:]+)</Marker>" | tr -d "<Marker>/"`

          if [ ${PREVIOUSTOKEN} == ${LASTFOUNDTOKEN} ]; then
            rm -f ${LOGFILE}.${COUNTER}.debug
            rm -f ${LOGFILE}.${COUNTER}
            break;
          else
            rm -f ${LOGFILE}.${COUNTER}.debug
            PREVIOUSTOKEN=${LASTFOUNDTOKEN}
          fi

          cat ${LOGFILE}.${COUNTER} >> ${LOGFILE}
          rm -f ${LOGFILE}.${COUNTER}

          let COUNTER=COUNTER+1
        done
    fi

    if [[ $LOGFILE == *"$END_DATE"* ]]; then
        break
    fi
done

log "PostgreSQL Logs download completed."
echo