Syncing and splunking Amazon S3 logs

access_s3_transformation

^[[nspaces:s3_bucket_owner]]\s++[[nspaces:vhost]]\s++[[sbstring:req_time]]\s++[[nspaces:clientip]]\s++[[nspaces:user]]\s++[[nspaces:s3_request_id]]\s++[[nspaces:s3_operation]]\s++[[nspaces:s3_key]]\s++[[access-request]]\s++[[nspaces:status]]\s++[[nspaces:s3_error_code]]\s++[[nspaces:bytes]]\s++[[nspaces:s3_object_size]]\s++[[nspaces:s3_total_time]]\s++[[nspaces:s3_turnaround_time]]\s++"(?[[bc_domain:referer_]]?+[^"]*+)"\s++[[qstring:useragent]]\s++[[nspaces:s3_version_id]][[all:other]]

inputs.conf

# snippet from /opt/splunk/etc/apps/search/local/inputs.conf
[monitor:///var/log/s3]
disabled = false
followTail = 0
host =
host_segment = 4
sourcetype = access_s3

s3_log_sync.sh

#!/bin/bash
LOG_BUCKET=logs # replace this with the name of your log bucket
LOG_DIR=/var/log/s3
S3CMD=/usr/bin/s3cmd

tmpdir=$(mktemp -d)

$S3CMD sync --skip-existing --delete-removed s3://$LOG_BUCKET/ $LOG_DIR/ &> $tmpdir/s3cmd.log
if [[ $? -ne 0 ]]
then
	cat $tmpdir/s3cmd.log
fi
rm -rf $tmpdir