Fetch AWS Cloudwatch exported S3 logs, extract, combine and sort.

get-log.sh

#!/bin/bash

set -eE

mkdir -p .log
rm -rf .log/*

aws s3 --profile PROFILE --region REGION sync s3://my-log-bucket/exportedlogs .log/

cd .log

mkdir -p ./extracted
mkdir -p ./combined

LIST=$(find . -type f -name '*.gz')

C=1
for FILE in $LIST; do
  echo $FILE
  gzip -dc < $FILE > ./extracted/$C.log
  C=$(($C+1))
done;

find . -type f -name '*.log' -exec cat {} + >> ./combined/combined.log

# This step can take an incredibly long time.  Comment out for large log sets.
sort ./combined/combined.log