nginx grok

input.conf

#/etc/logstash/02-nginx.conf
input {
  file {
    type => "nginx"
    path => "/var/log/nginx/*"
    exclude => "*.gz"
  }
}
filter {
  if [type] == "nginx" {
    grok {
	patterns_dir => "/etc/logstash/patterns"
	match => { "message" => "%{NGINX_ACCESS}" }
	remove_tag => ["_grokparsefailure"]
	add_tag => ["nginx_access"]
    }
    geoip {
      source => "clientip"
    }
  }
}

nginx

#/etc/logstash/patterns/nginx
NGINX_ACCESS %{IPORHOST:clientip} (?:-|(%{WORD}.%{WORD})) %{USER:ident} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} %{QS:forwarder}

NGINX-GROK-ELASTICSEARCH.md

http://grokdebug.herokuapp.com

SAMPLE

127.0.0.1 - - [26/Mar/2016:19:09:19 -0400] "GET / HTTP/1.1" 401 194 "" "Mozilla/5.0 Gecko" "-"

MATCH

%{IPORHOST:clientip} (?:-|(%{WORD}.%{WORD})) %{USER:ident} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} %{QS:forwarder}

Output

{
  "clientip": [
    [
      "127.0.0.1"
    ]
  ],
  "HOSTNAME": [
    [
      "127.0.0.1"
    ]
  ],
  "IP": [
    [
      null
    ]
  ],
  "IPV6": [
    [
      null
    ]
  ],
  "IPV4": [
    [
      null
    ]
  ],
  "WORD": [
    [
      null,
      null
    ]
  ],
  "ident": [
    [
      "-"
    ]
  ],
  "USERNAME": [
    [
      "-"
    ]
  ],
  "timestamp": [
    [
      "26/Mar/2016:19:09:19 -0400"
    ]
  ],
  "MONTHDAY": [
    [
      "26"
    ]
  ],
  "MONTH": [
    [
      "Mar"
    ]
  ],
  "YEAR": [
    [
      "2016"
    ]
  ],
  "TIME": [
    [
      "19:09:19"
    ]
  ],
  "HOUR": [
    [
      "19"
    ]
  ],
  "MINUTE": [
    [
      "09"
    ]
  ],
  "SECOND": [
    [
      "19"
    ]
  ],
  "INT": [
    [
      "-0400"
    ]
  ],
  "verb": [
    [
      "GET"
    ]
  ],
  "request": [
    [
      "/"
    ]
  ],
  "httpversion": [
    [
      "1.1"
    ]
  ],
  "BASE10NUM": [
    [
      "1.1",
      "401",
      "194"
    ]
  ],
  "rawrequest": [
    [
      null
    ]
  ],
  "response": [
    [
      "401"
    ]
  ],
  "bytes": [
    [
      "194"
    ]
  ],
  "referrer": [
    [
      """"
    ]
  ],
  "QUOTEDSTRING": [
    [
      """",
      ""Mozilla/5.0 Gecko"",
      ""-""
    ]
  ],
  "agent": [
    [
      ""Mozilla/5.0 Gecko""
    ]
  ],
  "forwarder": [
    [
      ""-""
    ]
  ]
}

Permissions.md

setfacl -m u:logstash:wrx /var/log/nginx/* setfacl -m u:logstash:wrx /var/log/nginx/

to use the quotes in the grok expression its possible to scape with \ before the ". so the filter will look like this:

%{IPORHOST:remoteAddr} (?:-|(%{WORD}.%{WORD})) %{USER:ident} [%{HTTPDATE:timestamp}] "(?:%{WORD:method} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:status} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} %{QS:forwarder}

and then the config for the pipeline its just

filter {
        grok {
                  match => { "log" => "%{IPORHOST:remoteAddr} (?:-|(%{WORD}.%{WORD})) %{USER:ident} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:method} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:status} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} %{QS:forwarder}" }
        }
}

note that in my case the fild its log and not message as usual.

Hi, I'm getting [[main]-pipeline-manager] javapipeline - Pipeline aborted due to error {:pipeline_id=>"main", :exception=>#<Grok::PatternError: pattern %{NGINX_ACCESS} not defined> when running the above.

My conf file looks like this:

input {
beats {
port => 5044
}
}

filter {
if [type] == "nginx" {
grok {
patterns_dir => "/etc/logstash/patterns"
match => { "message" => "%{NGINX_ACCESS}" }
remove_tag => ["_grokparsefailure"]
add_tag => ["nginx_access"]
}
geoip {
source => "clientip"
}
}
}

output { stdout { codec => rubydebug } }

#output {
# elasticsearch {
# hosts => ["192.168.1.35:9200"]
# manage_template => false
# index => "www-access-%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
# document_type => "%{[@metadata][type]}"
# }
#}

I commented out the output to ES to test it out on the console first ..

Any ideas??

I also added grok patterns for Tomcat and Spring https://gist.github.com/petrov9/4740c61459a5dcedcef2f27c7c2900fd