haproxy_status_code.awk

# File dropped of by Chef, local modifications will be overwritten!

# 2013-05-08T00:00:02+00:00 10.87.125.117 haproxy[30746]: 10.44.133.138:39039 [07/May/2013:23:59:56.974] public dogdispatcher/i-5e4d843b:9000 4/0/5001/1002/6009 202 166 - - ---- 132/132/35/11/1 0/0 "POST /api/v1/series/?api_key=<redacted> HTTP/1.1"

# Field numbers, description, example
# Ref: https://code.google.com/p/haproxy-docs/wiki/HTTPLogFormat
# 1 (rsyslog) log time
# 2 (rsyslog) haproxy server
# 3   process_name '[' pid ']:'   haproxy[14389]:
# 4   client_ip ':' client_port   10.0.1.2:33317
# 5   '[' accept_date ']' [06/Feb/2009:12:14:14.655]
# 6   frontend_name   http-in
# 7   backend_name '/' server_name    static/srv1
# 8   Tq '/' Tw '/' Tc '/' Tr '/' Tt* 10/0/30/69/109
# 9   status_code 200
# 10   bytes_read* 2750
# 11  captured_request_cookie -
# 12  captured_response_cookie    -
# 13  termination_state   ----
# 14  actconn '/' feconn '/' beconn '/' srv_conn '/' retries* 1/1/1/1/0
# 15  srv_queue '/' backend_queue 0/0
# 16  '{' captured_request_headers* '}'   {haproxy.1wt.eu}
# 17  '{' captured_response_headers* '}'  {}
# 18  '"' http_request '"'    "GET /index.html HTTP/1.1"

BEGIN {
    now = systime();
    one_hour_ago = now - 3600;
}
{
    # only report on log entries that match the time range
    # need to do some fun string conversions, and by fun I mean not.  
    log_line_time_raw = $1

    gsub(/[-:]|T|+/," ",log_line_time_raw);
    split(log_line_time_raw, d, " ");

    # Now we have an array 'd' that maps to:
    # 1 YYYY
    # 2 MM
    # 3 DD
    # 4 HH
    # 5 MM
    # 6 SS
    # 7 TZ-HH
    # 8 TZ-MM

    log_line_time_epoch = mktime(d[1]" "d[2]" "d[3]" "d[4]" "d[5]" "d[6])

    status_code = $9

    # compare converted timestamp to determine if in time range
    if ( log_line_time_epoch <= now && log_line_time_epoch >= one_hour_ago )

        # then test to see if the status_code is indeed a status_code
        valid = match(status_code, /^[0-9]+/);
        if (valid != 0)
            freq[status_code]++ ;

    # No 'else' statements, simply pass on lines that don't match
}
END {
    for (code in freq)
        printf "%s,%s\n", code, freq[code]

    # Dump some debug info to stderr
    print now, NR, "Records Processed" > "/dev/stderr";
}

submit_haproxy_metrics.py

# File dropped of by Chef, local modifications will be overwritten!

import csv
import sys

# use dogstatsd instead of the API directly
from statsd import statsd

reader = csv.reader(sys.stdin)
for line in reader:
    tag = "code:%s" % (line[0])
    statsd.gauge('dd.haproxy.response_codes_per_hour', line[1], tags=[tag])