Skip to content

Instantly share code, notes, and snippets.

@jamiew
Forked from radu-gheorghe/logstash.conf
Created October 12, 2017 20:05
Show Gist options
  • Save jamiew/ff5c7d5819561a26918d7c4157ac2133 to your computer and use it in GitHub Desktop.
Save jamiew/ff5c7d5819561a26918d7c4157ac2133 to your computer and use it in GitHub Desktop.
logstash grok filter for Elasticsearch logs
filter {
if [type] == "elasticsearch" {
grok {
match => [ "message", "\[%{TIMESTAMP_ISO8601:timestamp}\]\[%{DATA:severity}%{SPACE}\]\[%{DATA:source}%{SPACE}\]%{SPACE}(?<message>(.|\r|\n)*)" ]
overwrite => [ "message" ]
}
if "_grokparsefailure" not in [tags] {
grok { # regular logs
match => [
"message", "^\[%{DATA:node}\] %{SPACE}\[%{DATA:index}\]%{SPACE}(?<short_message>(.|\r|\n)*)",
"message", "^\[%{DATA:node}\]%{SPACE}(?<short_message>(.|\r|\n)*)" ]
tag_on_failure => []
}
grok { # slow logs
match => [ "message", "took\[%{DATA:took}\], took_millis\[%{NUMBER:took_millis}\], types\[%{DATA:types}\], stats\[%{DATA:stats}\], search_type\[%{DATA:search_type}\], total_shards\[%{NUMBER:total_shards}\], source\[%{DATA:source_query}\], extra_source\[%{DATA:extra_source}\]," ]
tag_on_failure => []
add_tag => [ "elasticsearch-slowlog" ]
}
date { # use timestamp from the log
"match" => [ "timestamp", "YYYY-MM-dd HH:mm:ss,SSS" ]
target => "@timestamp"
}
mutate {
remove_field => [ "timestamp" ] # remove unused stuff
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment