Example syslog-ng configuration for parsing nginx-timing logs

nginx.conf

@module mod-python
@define logpath "/home/cchq/www/{{ deploy_env }}/logs"
@define environment "{{ env_monitoring_id }}"
@define statsd_host "{{ datadog_statsd_host|default('localhost') }}"
@define statsd_port "{{ datadog_statsd_port|default(8125) }}"

source src {
    file("`logpath`/{{ deploy_env }}-timing.log" follow-freq(1) flags(no-parse));
};

template t_apdex "$(if ("${.nginx.duration}" <= "3") "1" $(if ("${.nginx.duration}" <= "12") "0.5" "0"))";

# Templates to create dogstatsd messages
template common_tags "#environment:`environment`,http_method:${.nginx.verb},status_code:${.nginx.status_code},url_group:${.nginx.request_group}";
template apdex_metric "nginx.apdex:$(template t_apdex)f|g|$(template common_tags)";
template timing_metric "nginx.timing:${.nginx.duration}f|g|$(template common_tags),referer_group:${.nginx.referer_group}";
template requests_metric "nginx.requests:1.0|c|$(template common_tags),cache_status:${.nginx.cache_status},referer_group:${.nginx.referer_group}";
template combined_metrics "$(template apdex_metric)\n$(template timing_metric)\n$(template requests_metric)\n";

# send to statsd
destination d_statsd-timing {
    network(
        "`statsd_host`"
        port(`statsd_port`)
        transport(udp)
        template(combined_metrics)
    );
};

destination d_test {
    file("/var/log/test.txt"
        template(combined_metrics)
    );
};

# Define URL Group parser
python {
{% include './files/url_group_parser.py' %}
};

# main log line parser
block parser nginx-timing-parser(prefix(".nginx.") template("${MESSAGE}")) {
    channel {
        parser {
            csv-parser(
                prefix(`prefix`)
                dialect(escape-double-char)
                flags(strip-whitespace)
                delimiters(" ")
                template(`template`)
                quote-pairs('""[]')
                columns("timestamp", "cache_status", "rawrequest", "status_code", "duration", "referer"));

            csv-parser(
                prefix(`prefix`)
                template("${`prefix`rawrequest}")
                delimiters(" ")
                dialect(escape-none)
                flags(strip-whitespace)
                columns("verb", "request", "httpversion"));

	          date-parser(format("%d/%b/%Y:%H:%M:%S %z") template("${`prefix`timestamp}"));

            python(
                class("UrlGroupParser")
                options("source", "`prefix`request", "destination", "`prefix`request_group")
            );


            python(
                class("UrlGroupParser")
                options("source", "`prefix`referer", "destination", "`prefix`referer_group")
            );

    ## TODO: timing bucket

	    };
        rewrite {
            subst("^HTTP/(.*)$", "$1", value("`prefix`httpversion"));
        };
    };
};


log {
    source(src);
    parser { nginx-timing-parser();};
    destination(d_statsd_timing);
    destination(d_test);
};

url_group_parser.py

import re

STATIC_GROUPS = {
    '/home/': '/home/',
    '/pricing/': '/pricing/',
    '/accounts/login/': 'login'
}

MM_MAPPING = {
    'CommCareAudio': 'mm/audio',
    'CommCareVideo': 'mm/video',
    'CommCareImage': 'mm/image',
}

DOMAIN_PATTERN = re.compile('^/a/[0-9a-z-]+')


class UrlGroupParser(object):
    def init(self, options):
        self.source = options['source']
        self.destination = options['destination']
        return True

    def deinit(self):
        pass

    def parse(self, log_message):
        url = log_message[self.source]
        if not url:
            return True

        log_message[self.destination] = self._get_group(url)
        return True

    def _get_group(self, url):
        default = 'other'
        if DOMAIN_PATTERN.match(url):
            parts = url.split('/')
            group = parts[3] if len(parts) >= 4 else default
            if group == 'phone':
                return 'phone/{}'.format(parts[4])
            return group
        elif url.startswith('/hq/multimedia/file/'):
            parts = url.split('/')
            return MM_MAPPING.get(parts[4], 'mm/other')
        elif url.startswith('/static/') or url.startswith('favicon'):
            return 'static'
        else:
            return STATIC_GROUPS.get(url, default)