Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Overcast's logstash config for parsing log files created by our custom log4j2 config
input {
file {
path => "/minecraft/logs/*/server.log"
type => "server"
# stacktrace java as one message
codec => multiline {
pattern => "^([0-9\-: ]+ \[[A-Z]+\] )?(((\w+\.)+\w+Exception.*)|((\s|\t)+at .+)|(\s+... \d+ more)|(\s*Caused by:.+))"
what => "previous"
multiline_tag => "stacktrace"
}
}
}
# attempt to simplify host by removing any suffixes
filter {
grok {
match => [ "host", "(?<simplehost>(phx|ams)\d\d)\.?" ]
}
if "_grokparsefailure" in [tags] {
mutate {
remove_tag => [ "_grokparsefailure" ]
}
} else {
mutate {
replace => [ "host", "%{simplehost}" ]
remove_field => "simplehost"
}
}
}
# extract server name from path
filter {
grok {
match => [ "path", "/logs/(?<server>[A-Za-z0-9-]+)/*" ]
}
if "_grokparsefailure" in [tags] {
mutate {
replace => [ "server", "unknown" ]
remove_tag => [ "_grokparsefailure" ]
}
} else {
if [host] =~ /^phx\d\d/ {
mutate {
replace => [ "server", "us-%{server}" ]
}
} else if [host] =~ /^ams\d\d/ {
mutate {
replace => [ "server", "eu-%{server}" ]
}
}
mutate {
remove_field => "path"
}
}
}
# get rid of color codes
filter {
mutate {
gsub => [ "message", "\e\[(0;)?([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]", "" ]
}
}
filter {
# match lines
grok {
match => [ "message", "(?m)%{DATESTAMP:minecraft_timestamp} \[%{WORD:level}\] %{GREEDYDATA:minecraft_message}" ]
add_field => [ "received_at", "%{@timestamp}" ]
named_captures_only => true
}
# read timestamp from lines
date {
match => [ "minecraft_timestamp", "YY-MM-dd HH:mm:ss" ]
}
# set message
if "_grokparsefailure" not in [tags] {
mutate {
replace => [ "message", "%{minecraft_message}" ]
}
}
# remove parsed fieds
mutate {
remove_field => [ "minecraft_message", "minecraft_timestamp" ]
}
}
output {
elasticsearch_http {
host => "es"
index => "logstash-%{+xxxx.ww}"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment