Created
October 26, 2017 09:36
-
-
Save poldeuce-sys/17fc067d1047a95f8ff6c7e17167f7b4 to your computer and use it in GitHub Desktop.
A couple notes on the setting a spring boot log format to include trace and span id for zipkin, and a logstash pipeline for reading these logs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
######### application.properties | |
# trace and span taken from MDC context. Hostname could be redundant is using FileBeat on same host, | |
# but maybe you are feeding logs to ES in another way. Including it in the log covers this possibility | |
logging.pattern.file=%d{ABSOLUTE} [%X{traceId}-%X{spanId}] %-5p ${HOSTNAME} ${PID} [%t] [%C{2}] [%F:%L] - %m%n | |
######### pipeline/logstash.conf | |
# There is an old saying. You had a problem. You solved it with regex. Now you have 2 problems. | |
# Really, you can address this is a simpler way by using || or [] separators between each field, | |
# but anyhow, where's the fun in that. | |
# Note that if you do not add the date mutation, the timestamp in ES will be the time it was received from the FileBeat | |
# as opposed to the timestamp of the log message. And that is bad. | |
input { | |
beats { | |
port => 5044 | |
} | |
} | |
filter { | |
grok { | |
match => [ "message", | |
"(?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}) \[(?<traceid>[a-f0-9]*)-(?<spanid>[a-f0-9]*)\] %{LOGLEVEL:level}\s+(?<hostname>[a-zA-Z0-9.-]*)\s+%{NUMBER:pid}\s*\[(?<thread>[A-Za-z0-9-]+)\]\s*\[[A-Za-z0-9.]*\.(?<class>[A-Za-z0-9#_]+)\] \[(?<file>[A-Za-z0-9.]+)\:(?<line>[0-9]+)\]\s*\-\s*(?<logmessage>.*)" ] | |
} | |
date { | |
locale => "en" | |
timezone => "UTC" | |
match => ["timestamp", "yyyy-MM-dd HH:mm:ss,SSS"] | |
target => "@timestamp" | |
remove_field => ["timestamp", "monthday", "year", "month", "day", "time"] | |
} | |
} | |
output { | |
elasticsearch { | |
hosts => ["elasticsearch:9200"] | |
user => elastic | |
password => changeme | |
index => "logstash-%{+YYYY.MM.dd}" | |
} | |
stdout { | |
codec => rubydebug | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment