KPCC Logstash Config for Podcasts

nginx_filter

filter {
  if [type] == "nginx" {
    grok {
      match => ["message", "%{NGINXACCESS}" ]
    }
    grok {
      match => ["request","%{URIPATH:request_path}(?:%{URIPARAM:request_params})?"]
    }
    date {
      match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z", "dd/MMM/yyyy:HH:mm:ss" ]
    }
    kv {
      source => "request_params"
      field_split => "&?"
      prefix => "q"
    }

    geoip {
      source => "clientip"
    }

    mutate {
      gsub => [
        "agent", "(?:^\"|\"$)","",
        "range_request", "(?:^\"|\"$)",""
      ]
    }
  }
}

nginx_patterns

NGUSERNAME [a-zA-Z\.\@\-\+_%]+
NGUSER %{NGUSERNAME}
NGINXACCESS %{IPORHOST:clientip} %{NGUSER:ident} %{NGUSER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response} (?:%{NUMBER:bytes_sent:int}|-) (?:"(?:%{URI:referrer}|-)"|%{QS:referrer}) %{QS:agent}(?: %{NUMBER:request_time:float} (?:%{NUMBER:upstream_time:float}|\-) [\w\.](?: %{QS:range_request})?)?