Clean your S3 logs

clean-logs.sh

#!/bin/bash

echo "Prerequisites:"
echo "sudo apt-get install awscli"
echo "aws configure"

echo "Usage:"
echo "aws s3 ls s3://BUCKET/PREFIX/ | cut -c32- | ./clean-logs.sh"
bucket=SET_BUCKET_NAME
prefix=SET_PREFIX

# set bash verbose mode
set -x
# iterate on file list from stdin
while read l
do
  # download from s3
  aws s3 cp s3://$bucket/$prefix/$l .
  # uncompress
  gunzip $l
  # get the uncompressed name
  f=`basename $l .gz`
  # do the actual cleaning, your mileage may vary
  cat $f | grep -v ' heroku .*sample#' | grep -v ' app prediction' > tmp
  mv tmp $f
  # compress back the new file
  gzip $f
  # upload again to s3
  aws s3 cp $l s3://$bucket/$prefix/$l
  # cleanup
  rm $l
done