goaccess_s3_sync.sh

#!/bin/sh

###
#
# GoAccess report generator
# -------------------------
#
# This script syncs logfiles from an S3 bucket, and aggregates them.
# Then it creates a GoAccess report out of this log and pushes it back to an S3 bucket.
#
#
# The MIT License (MIT)
#
# Copyright (c) 2013 WooRank sprl
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
# the Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
###

s3_log_path=""
s3_report_path=""
base_log_dir="~/s3logs/"
log_pattern="*"
combined_log="/tmp/access.front.log-combined"
report="~/$(echo report_`date +%F_%H-%M-%S`.html)"
since_day=$(date +%F)

usage() {
  echo "Usage: $0 [-hvd] -i s3_log_path -o s3_report_path -p log_file_pattern"
  echo ""
  echo "\t -h\tPrint this help output"
  echo "\t -v\tVerbose output"
  echo "\t -d\tThe date to run the report on"
  echo "\t -o\tOutput s3 path to store report (s3://[BUCKET]/[PATH]/)"
  echo "\t -i\tInput s3 path where the logs are to be found (s3://[BUCKET]/[PATH]/)"
  echo "\t -p\tLog filename pattern"
  echo ""
  exit 1
}

log() {
  if [ "$verbose" = 1 ]; then
    echo "\n---"
    echo "$1"
    echo "---\n"
  fi
}

while getopts "hvd:i:o:p:" opt; do
    case "$opt" in
    h)
        usage
        exit 0
        ;;
    v)  verbose=1
        ;;
    d)  since_day=$(date +%F -d "$OPTARG")
        ;;
    i)  s3_log_path=$OPTARG
        ;;
    o)  s3_report_path=$OPTARG
        ;;
    p)  log_pattern=$OPTARG
        ;;
    esac
done

shift $(($OPTIND - 1))

[ -z "$s3_log_path" ] && echo "No input s3 path provided" && usage
[ -z "$s3_report_path" ] && echo "No output s3 path provided" && usage
[ -z "$since_day" ] && usage

log "Syncing $s3_log_path to $base_log_dir..."
mkdir -p $base_log_dir
s3cmd sync $s3_log_path $base_log_dir

# Loop log files and change modified time based on filename timestamp suffix
log "Fixing modified times on $log_pattern..."
find $base_log_dir -name "$log_pattern" | while read filename;
do
  origdate=$(date -d @`echo $filename | sed 's/.*-//'` +%Y%m%d%H%M.%S);
  touch -t  $origdate "$filename";
done;

# Merge all files modified since start of the day
until_day=$(date +%F -d "$since_day +1 day")
log "Merging into $combined_log between $since_day and $until_day..."
find $base_log_dir -newermt "$since_day" ! -newermt "$until_day" -name "$log_pattern"
cat `find $base_log_dir -newermt "$since_day" ! -newermt "$until_day" -name "$log_pattern"` > $combined_log

# Write goaccess config
cat > ~/.goaccessrc << EOL
color_scheme 0
date_format %d/%b/%Y
log_format %^ %^[%d:%^] "%r" %s %b "%R" "%u" "%h"
EOL

# Create html report
log "Creating report file $report"
goaccess -f $combined_log -a > $report

# Upload to s3
log "Uploading to $s3_report_path"
s3cmd put --mime-type=text/html $report $s3_report_path

# Cleanup
log "Cleaning up..."
rm -rf $combined_log $report

log "Done!"

I got the following error every time even without specifying a date with -d.

---
Fixing modified times on *...

---

usage: date [-jnu] [-d dst] [-r seconds] [-t west] [-v[+|-]val[ymwdHMS]] ... 
            [-f fmt date | [[[mm]dd]HH]MM[[cc]yy][.ss]] [+format]
touch: out of range or illegal time specification: [[CC]YY]MMDDhhmm[.SS]

I'm on OSX, do you have any ideas?