extract data from gorouter access logs

extract_access_data.sh

#!/bin/bash
set -euo pipefail
shopt -s nullglob

# extract data about every request from all access logs across all gorouters

deployment_logs_dir="$1"

LOGREGEX='^(.*?) - \[(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}).(\d+)(.*?)] "(.*?) (.*?) (.*?)" (\d+) (\d+) (\d+) "(.*?)" "(.*?)" "(.*?)" "(.*?)" x_forwarded_for:"(.*?)" x_forwarded_proto:"(.*?)" vcap_request_id:"(.*?)" response_time:(\d+\.\d+) app_id:"(.*?)" app_index:"(.*?)" x_b3_traceid:"(.*?)" x_b3_spanid:"(.*?)" x_b3_parentspanid:"(.*?)"$'

extract_basics () {
  for vm in $deployment_logs_dir/router.*; do
    router_guid=$(basename $vm | cut -d '.' -f2)
    for al in $vm/gorouter/access.log.*.gz; do
      gunzip "$al"
    done;
    for al in $vm/gorouter/access.log.?; do
      # router_guid METHOD backend_ip:port] status_code response_time
      perl -n -e '/'"$LOGREGEX"'/ && print "'"$router_guid"' ".$10." ".$19."] ".$13." ".$23."\n"' < "$al"
    done
  done;
}

# replace - with -:- to simplify parsing
expand_empty_backend () {
  sed -e 's/\ -\]/ -:-]/'
}

# we don't care about the backend port
drop_port () {
  sed -e 's/:.*\]//g'
}

to_csv () {
  sed -e 's/ /,/g'
}

extract_basics | expand_empty_backend | drop_port | to_csv