Exports the Cloudflare ELS log file and transforms the format to be consumed by goaccess

cf-els-goaccess.sh

#!/bin/bash

# Purpose: Exports the Cloudflare ELS log file and transforms the format to be consumed by goaccess.

# Developed by Tom Kaminski <tom@mobilenations.com>

# Requires the following tools:
# wget
# jq - https://stedolan.github.io/jq/
# goaccess - https://goaccess.io/

EMAIL=monkey@zoo.com   # Cloudflare API Email
KEY=banana12345        # Cloudflare API Key
ZONEID=123             # Cloudflare Zone ID
START=`date -u +"%Y-%m-%dT%H:%M:%SZ" -d "-11 minutes"`
END=`date -u +"%Y-%m-%dT%H:%M:%SZ" -d "-10 minutes"`
SAMPLE=0.01

wget -q --header "X-Auth-Email: $EMAIL" --header "X-Auth-Key: $KEY" "https://api.cloudflare.com/client/v4/zones/$ZONEID/logs/received?start=$START&end=$END&sample=$SAMPLE&fields=ClientIP,EdgeStartTimestamp,EdgeEndTimestamp,ClientRequestMethod,ClientRequestProtocol,ClientRequestHost,ClientRequestURI,EdgeResponseStatus,EdgeResponseBytes,ClientRequestReferer,ClientRequestUserAgent" -O - \
| \
jq -cr '[.ClientIP,(.EdgeStartTimestamp/1000|floor),((.EdgeEndTimestamp-.EdgeStartTimestamp)/1000|floor),.ClientRequestMethod,.ClientRequestProtocol,.ClientRequestHost,.ClientRequestURI,.EdgeResponseStatus,.EdgeResponseBytes,.ClientRequestReferer,.ClientRequestUserAgent]|@tsv' \
| \
goaccess --time-format="%f" --date-format="%f" --log-format="%h\t%x\t%D\t%m\t%H\t%v\t%U\t%s\t%b\t%R\t%u"