Skip to content

Instantly share code, notes, and snippets.

@mhzawadi
Last active June 12, 2020 19:17
Show Gist options
  • Save mhzawadi/dc47c2707c1149a09d34 to your computer and use it in GitHub Desktop.
Save mhzawadi/dc47c2707c1149a09d34 to your computer and use it in GitHub Desktop.
Logstash Grok filters
filter {
if [type] == "apache_access" {
grok {
match => { "message" => "%{COMMONAPACHELOG}" }
}
date {
match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
}
}
elseif [type] == "nginx_access" {
grok {
match => { "message" => "%{COMMONAPACHELOG} %{QUOTEDSTRING:referrer} %{QUOTEDSTRING:http_x_forwarded_for} %{QUOTEDSTRING:upstream_addr}" }
}
date {
match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
}
geoip {
add_tag => [ "geoip" ]
source => "clientip"
}
}
else if [type] == "apache_error" {
grok {
match => { "message" => "\[%{DAY} (?<timestamp>%{MONTH} %{MONTHDAY} %{TIME} %{YEAR})\] \[(?<apache-level>[^\]]+)\] %{GREEDYDATA:content}" }
}
date {
match => [ "timestamp" , "MMM dd HH:mm:ss yyyy" ]
}
}
elseif [type] == "nginx_error" {
grok {
match => { "message" => "(?<timestamp>%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME}) \[(?<log-level>[^\]]+)\] %{GREEDYDATA:content}" }
}
date {
match => [ "timestamp" , "YYYY/MM/dd HH:mm:ss" ]
}
}
else if [type] == "php-fpm" {
grok {
match => { "message" => "\[(?<timestamp>%{MONTHDAY}-%{MONTH}-%{YEAR} %{TIME})\] (?<php-level>[^:]+):[ ]*%{GREEDYDATA:content}" }
}
date {
match => [ "timestamp" , "dd-MMM-yyyy HH:mm:ss" ]
}
}
else if [type] =~ "iaptus-log" {
grok {
match => { "message" => "\[(?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME})\] (?<log-level>[^:]+): (?<content>[^{]+)*%{GREEDYDATA:context}" }
}
grok {
match => { "message" => "(?<location>.fileName.:.[^,]+,.line.:[0-9]+)" }
}
date {
match => [ "timestamp" , "yyy-MM-dd HH:mm:ss" ]
}
}
else if [type] == "mysql" {
grok {
match => { "message" => "(?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}) %{NUMBER} \[(?<mysql-level>[^\]]+)\] %{GREEDYDATA:content}" }
}
date {
match => [ "timestamp" , "yyy-MM-dd HH:mm:ss" ]
}
}
else if [type] == "mysql-slow" {
}
else if [type] == "squid_access" {
grok {
match => { "message" => "%{NUMBER:timestamp}\s+%{NUMBER:request_msec:float} %{IP:src_ip} %{WORD:cache_result}/%{NUMBER:response_status:int} %{NUMBER:response_size:int} %{WORD:http_method} (%{URIPROTO:http_proto}://)?%{IPORHOST:dst_host}(?::%{POSINT:port})?(?:%{URIPATHPARAM:uri_param})? %{USERNAME:cache_user} %{WORD:request_route}/(%{IPORHOST:forwarded_to}|-) %{GREEDYDATA:content_type}" }
add_tag => ["squid"]
}
}
else if [type] == "openvpn" {
grok {
match => { "message" => "(?<timestamp>%{DAY} %{MONTH}\s+%{MONTHDAY} %{TIME} %{YEAR}) us=%{NUMBER} TCP connection established with \[AF_INET\]%{IP:vpnhost}:%{NUMBER}" }
}
grok {
match => { "message" => "(?<timestamp>%{DAY} %{MONTH}\s+%{MONTHDAY} %{TIME} %{YEAR}) us=%{NUMBER} %{IP:vpnhost}:%{NUMBER} TLS: Username/Password authentication succeeded for username '%{USERNAME:vpnuser}'" }
}
grok {
match => { "message" => "(?<timestamp>%{DAY} %{MONTH}\s+%{MONTHDAY} %{TIME} %{YEAR}) us=%{NUMBER} %{USERNAME:vpnuser}/%{IP:vpnhost}:%{NUMBER} MULTI: Learn: %{IP:vpnip} %{GREEDYDATA}" }
}
grok {
match => { "message" => "(?<timestamp>%{DAY} %{MONTH}\s+%{MONTHDAY} %{TIME} %{YEAR}) us=%{NUMBER} %{IP:vpnhost}:%{NUMBER} TLS Error: TLS handshake failed" }
}
grok {
match => { "message" => "(?<timestamp>%{DAY} %{MONTH}\s+%{MONTHDAY} %{TIME} %{YEAR}) us=%{NUMBER} %{IP:vpnhost}:%{NUMBER} TLS Auth Error: Auth Username/Password verification failed for peer" }
}
grok {
match => { "message" => "(?<timestamp>%{DAY} %{MONTH}\s+%{MONTHDAY} %{TIME} %{YEAR}) us=%{NUMBER} %{IP:vpnhost}:%{NUMBER} SENT CONTROL \[%{USERNAME:vpnuser}\]: 'AUTH_FAILED' \(status=1\)" }
}
geoip {
add_tag => [ "geoip" ]
source => "vpnhost"
}
}
else if [type] == "sSMTP" {
grok {
match => { "message" => "%{MONTH}\s+%{MONTHDAY} %{TIME} %{HOST} sSMTP\[%{NUMBER}\]: %{WORD} %{WORD} %{WORD} (?<email>%{USERNAME}@%{HOST:host}) \(%{NUMBER} %{HOST:mta}\)"}
}
grok {
match => { "message" => "%{MONTH}\s+%{MONTHDAY} %{TIME} %{HOST} sSMTP\[%{NUMBER}\]: %{WORD} %{WORD}:\<(?<email>%{USERNAME}@%{HOST:host})\> %{GREEDYDATA:error}"}
}
}
else if [type] == "sendmail" {
grok {
match => { "message" => "%{MONTH}\s+%{MONTHDAY} %{TIME} %{HOST} %{PROG}\[%{NUMBER}\]: %{WORD:qid}: to=%{DATA:email_to}, delay%{DATA}, %{DATA}, %{DATA}, %{DATA}, relay=%{DATA:relay_mx} \[%{HOST:relay_ip}\], %{DATA}, stat=%{WORD:status} \(%{GREEDYDATA:status_msg}\)" }
}
grok {
match => { "message" => "(?<timestamp>%{MONTH}\s+%{MONTHDAY} %{TIME}) %{HOST} %{PROG}\[%{NUMBER}\]: %{GREEDYDATA}" }
}
}
else {
mutate { replace => { "type" => "unknown" } }
date {
match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
}
}
}
@lazyCoder10
Copy link

hello,
What's the meaning of [ if [type] == "apache_error" ] ... i am newbie to logstash and failing to figure out what's the meaning of if[type] =="some_parameter"... from where i will get this parameter?

@mhzawadi
Copy link
Author

its how the logs are tagged, this is a very out of date filter example. Will see if I can get an updated filter from work next week

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment