Logstash setup

ls1.conf

input {

        file {
                path => "/var/nginx/chroot/var/log/nginx/pec.tls.access.log"
                type => "nginx-access"
                start_position => "beginning"
        }
}

filter {
        if [type] == "nginx-access" {
                                        grok {
                                                match => [ "message", '%{NGINXACCESS1}',
                                                           "message", '%{NGINXACCESS2}',
                                                           "message", '%{NGINXACCESS3}',
                                                           "message", '%{NGINXACCESS4}',
                                                           "message", '%{NGINXACCESS5}' ]
                                        }
                                        date {
                                                match=> [ "logdate", "dd/MMM/YYYY:HH:mm:ss +0000" ]
                                        }
                                        geoip {
                                                source => "clientip"
                                                target => "geoip"
                                                database => "/etc/logstash/GeoLiteCity.dat"
                                                add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
                                                add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
                                        }
                                        mutate {
                                                convert => [ "[geoip][coordinates]", "float"]
                                        }

        }
}

output {
        stdout { codec => rubydebug }

        if [type] == "nginx-access" and "_grokparsefailure" in [tags] {
                file { path => "/var/log/logstash/failed-nginx-events-%{+YYYY-MM-dd}" }
        } else {

        elasticsearch {
                        hosts => [ "192.168.37.50:9200","192.168.37.24:9200"]
                        index => "logstash-%{+YYYY.MM.dd}"
                        manage_template => true
                        sniffing => true
        }
        }
}

nginxpatterns

NGUSERNAME [a-zA-Z\.\@\-\+_%]+
NGUSER %{NGUSERNAME}
NGINXACCESS1 %{HTTPDATE:logdate} %{IP:clientip}%{NOTSPACE}%{NOTSPACE}(\s+)"Status:%{NUMBER:status}"(\s+)"%{WORD:httpmethod}(\s+)%{URIPATHPARAM:uripath}(\s+)(?<httpversion>(?:(?!").)*)"(\s+)"{(?<request_body_part_a>(.+))mac\\x22: \\x22(?<mac>(\w+))\\x22(?<request_body_part_b>(.+))}"(\s+)%{NUMBER:connnumber}(\s+)%{NUMBER:connreq}(\s+)(?<cipher>(\w+-\w+){3,4})(\s+)%{NUMBER:bodybytessent}(\s+)%{QS:refer}(\s+)%{QS:useragent}
NGINXACCESS2 %{HTTPDATE:logdate} %{IP:clientip}%{NOTSPACE}%{NOTSPACE}(\s+)"Status:%{NUMBER:status}"(\s+)"%{WORD:httpmethod}(\s+)%{URIPATHPARAM:uripath}(\s+)(?<httpversion>(?:(?!").)*)"(\s+)"(?<request_body_part_a>(.+))\\x22mac\\x22\\x0D\\x0A\\x0D\\x0A(?<mac>(.{12}))\\x0D(?<request_body_part_b>(.+))(\s+)%{NUMBER:connnumber}(\s+)%{NUMBER:connreq}(\s+)(?<cipher>(\w+-\w+){3,4})(\s+)%{NUMBER:bodybytessent}(\s+)%{QS:refer}(\s+)%{QS:useragent}
NGINXACCESS3 %{HTTPDATE:logdate} %{IP:clientip}%{NOTSPACE}%{NOTSPACE}(\s+)"Status:%{NUMBER:status}"(\s+)"%{WORD:httpmethod}(\s+)%{URIPATHPARAM:uripath}(\s+)(?<httpversion>(?:(?!").)*)"(\s+)%{QS:request_body_part_a}(\s+)%{NUMBER:connnumber}(\s+)%{NUMBER:connreq}(\s+)(?<cipher>(\w+-\w+){3,4})(\s+)%{NUMBER:bodybytessent}(\s+)%{QS:refer}(\s+)%{QS:useragent}
NGINXACCESS4 %{HTTPDATE:logdate} %{IP:clientip}%{NOTSPACE}%{NOTSPACE}(\s+)"Status:%{NUMBER:status}"(\s+)"%{WORD:httpmethod}(\s+)%{URIPATHPARAM:uripath}(\s+)(?<httpversion>(?:(?!").)*)"(\s+)"(?<request_body>(?:(?!").)*)"(\s+)%{NUMBER:connnumber}(\s+)%{NUMBER:connreq}(\s+)(?<cipher>([^\s]+)(\s+)%{NUMBER:bodybytessent})(\s+)"(?<refer>(?:(?!").)*)"(\s+)"(?<useragent>(?:(?!").)*)"
NGINXACCESS5 %{HTTPDATE:logdate} %{IP:clientip}%{NOTSPACE}%{NOTSPACE}(\s+)"Status:%{NUMBER:status}"(\s+)"(?<httpmethod>(?:(?!").)*)"(\s+)"(?<request_body>(?:(?!").)*)"(\s+)%{NUMBER:connnumber}(\s+)%{NUMBER:connreq}(\s+)(?<cipher>([^\s]+)(\s+)%{NUMBER:bodybytessent})(\s+)"(?<refer>(?:(?!").)*)"(\s+)"(?<useragent>(?:(?!").)*)"