Logstash config example for parsing nginx-timing logs

logstash.conf

##
# {{ ansible_managed }}
# See https://github.com/dimagi/commcare-cloud/compare/sk/logstash?expand=1

filter {
    if [type] == "nginx-timing" {
        grok {
            match => [
                "message", "\[%{HTTPDATE:http_timestamp}\] (?<cache_status>[-\w]+) %{WORD:http_method} %{DATA:request} HTTP/%{NUMBER:httpversion} %{NUMBER:status_code} %{NUMBER:request_time}"
            ]
        }
        # Add domain
        grok {
            match => { "message" => "/a/(?<domain>[0-9a-z-]+)" }
            tag_on_failure => ["_domainparsefailure"]
        }
        if "_domainparsefailure" in [tags] {
            mutate {
                add_field => { "domain" => "" }
                remove_tag => [ "_domainparsefailure" ]
            }
        }

        # Parse date
        date {
            match => [ "http_timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
        }
        # Parse request duration
        mutate {
            convert => { "request_time" => "float" }
        }

        # Set apdex
        if [request_time] > 12 {
            mutate {
                add_field => { "apdex" => 0 }
            }
        } else if [request_time] > 3 {
            mutate {
                add_field => { "apdex" => 0.5 }
            }
        } else {
            mutate {
                add_field => { "apdex" => 1 }
            }
        }
        mutate {
            convert => { "apdex" => "float" }
        }

        # Parse url_group
        if [request] =~ "/hq/multimedia/file/" {
            mutate {
                split => ["request", "/"]
                add_field => { "url_group" => "%{[request][4]}" }
            }
        } else if [request] =~ "/static/" or [request] =~ "favicon.ico" {
            mutate {
                add_field => { "url_group" => "static" }
            }
        } else if [request] =~ "^/a/[0-9a-z-]+" {
            mutate {
                split => ["request", "/"]
            }
            if "phone" in [request] {
                mutate {
                    add_field => { "url_group" => "%{[request][3]}/%{[request][4]}" }
                }
            } else {
                mutate {
                    add_field => { "url_group" => "%{[request][3]}" }
                }
                alter {
                    coalesce => [
                        "url_group", "%{url_group}", "other"
                    ]
                }
            }
        } else {
            mutate {
                add_field => { "url_group" => "%{request}" }
            }
        }
        
        # TODO: referer group
        # TODO: timing bucket

        # convert url groups to desired values
        alter {
            condrewrite => [
                "url_group", "CommCareAudio", "mm/audio",
                "url_group", "CommCareVideo", "mm/video",
                "url_group", "CommCareImage", "mm/image",
                "url_group", "/accounts/login/", "login"
            ]
        }

        # Cleanup
        mutate {
            remove_field => [ "request", "bytes", "httpversion", "message" ]
        }

    }

    if "_grokparsefailure" in [tags] {
        drop { }
    }
}

output {
  upd {
    host => "{{ datadog_statsd_host|default('localhost') }}"
    port => {{ datadog_statsd_port|default(8125) }}
    codec => codec => plain {
        format => "nginx.timing:%<request_time>f|g|#environment:{{ env_monitoring_id }},domain:%{domain},http_method:%{http_method},status_code:%{status_code},url_group:%{url_group}"
    }
    id => "logstash-nginx-timing"
  }

  upd {
    host => "{{ datadog_statsd_host|default('localhost') }}"
    port => {{ datadog_statsd_port|default(8125) }}
    codec => codec => plain {
        format => "nginx.requests:1.0|c|#environment:{{ env_monitoring_id }},domain:%{domain},http_method:%{http_method},status_code:%{status_code},url_group:%{url_group},cache_status:%{cache_status}"
    }
    dd_tags => [ "environment:{{ env_monitoring_id }}", "domain:%{domain}", "http_method:%{http_method}", "status_code:%{status_code}", "url_group:%{url_group}", "cache_status:%{cache_status}"]
    id => "logstash-nginx-requests"
  }

  upd {
    host => "{{ datadog_statsd_host|default('localhost') }}"
    port => {{ datadog_statsd_port|default(8125) }}
    codec => codec => plain {
        format => "nginx.apdex:%<apdex>f|g|#environment:{{ env_monitoring_id }},domain:%{domain},http_method:%{http_method},status_code:%{status_code},url_group:%{url_group}"
    }
    id => "logstash-nginx-apdex"
  }
}