Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
trying for a logstash conf file that works with java's logback logger
input {
# file {
# path => ["/var/log/knewton/*/*.log"]
# type => "knewton_logback"
# }
stdin {
type => "knewton_logback"
}
}
filter {
# Logback logs junk like this:
# 2012-05-23 15:07:28,876 WARN [PZDiscoveryMonitor STARTING] c.k.p.d.PZDiscoveryMonitor : Failed to load fileConfig
# com.knewton.knewtonconfig.exception.NotFoundException: java.io.FileNotFoundException: /etc/knewton/discovery/grover/platform.yml (No such file or directory)
# at com.knewton.knewtonconfig.configuration.FileInputStreamProvider.get(FileInputStreamProvider.java:30) ~[KPIP.jar:na]
# at com.knewton.knewtonconfig.discovery.StreamDiscoveryProvider.get(StreamDiscoveryProvider.java:116) ~[KPIP.jar:na]
# at com.knewton.knewtonconfig.discovery.StreamDiscoveryProvider.get(StreamDiscoveryProvider.java:49) ~[KPIP.jar:na]
# at com.knewton.pettingzoo.discovery.PZDiscoveryMonitor.startUp(PZDiscoveryMonitor.java:245) ~[KPIP.jar:na]
# at com.google.common.util.concurrent.AbstractIdleService$1$1.run(AbstractIdleService.java:43) [KPIP.jar:na]
# at java.lang.Thread.run(Thread.java:679) [na:1.6.0_22]
# Caused by: java.io.FileNotFoundException: /etc/knewton/discovery/grover/platform.yml (No such file or directory)
# at java.io.FileInputStream.open(Native Method) ~[na:1.6.0_22]
# at java.io.FileInputStream.<init>(FileInputStream.java:137) ~[na:1.6.0_22]
# at java.io.FileInputStream.<init>(FileInputStream.java:96) ~[na:1.6.0_22]
# at com.knewton.knewtonconfig.configuration.FileInputStreamProvider.get(FileInputStreamProvider.java:28) ~[KPIP.jar:na]
# ... 5 common frames omitted
date {
type => "knewton_logback"
timestamp => "yyyy-MM-dd hh:mm:ss,SSS"
}
multiline {
type => "knewton_logback"
pattern => "^(\\s+|com|Caused)"
what => "previous"
}
# Returns the wrong level
# gelfify {
# type => "knewton_logback"
# }
grok {
type => "knewton_logback"
patterns_path => "/usr/local/share/grok/patterns"
pattern => ["%{JAVALOGBACK}"]
}
mutate {
type => "knewton_logback"
add_field => ["level", "%{severity}"]
}
}
output {
gelf {
host => "graylog2.utility.knewton.net"
sender => "KPIP"
level => "info"
facility => "KPIP"
}
# stdout {
# debug => true
# debug_format => json
# type => "knewton_logback"
# }
}
JAVACLASS (?:[a-z-]+\.)[A-Za-z0-9]+
JAVAFILE (?:[A-Za-z0-9_.-]})
JAVASTACKTRACEPART "at %{JAVACLASS:class}\.%{WORD:method}\(%{JAVAFILE:file}:%{NUMBER:line}\)
MILLISECOND (\d{3})
JAVALOGBACKTIMESTAMP %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND}),%{MILLISECOND}
JAVALOGBACK %{JAVALOGBACKTIMESTAMP:timestamp} (?:%{WORD:severity})
@pcn

This comment has been minimized.

Copy link
Owner Author

commented May 24, 2012

Test data:

2012-05-23 15:07:33,015 WARN  [PZDiscoveryMonitor STARTING] c.k.p.d.PZDiscoveryMonitor    : Failed to load fileConfig
com.knewton.knewtonconfig.exception.NotFoundException: java.io.FileNotFoundException: /etc/knewton/discovery/haag/platform.yml (No such file or directory)
    at com.knewton.knewtonconfig.configuration.FileInputStreamProvider.get(FileInputStreamProvider.java:30) ~[KPIP.jar:na]
    at com.knewton.knewtonconfig.discovery.StreamDiscoveryProvider.get(StreamDiscoveryProvider.java:116) ~[KPIP.jar:na]
    at com.knewton.knewtonconfig.discovery.StreamDiscoveryProvider.get(StreamDiscoveryProvider.java:49) ~[KPIP.jar:na]
    at com.knewton.pettingzoo.discovery.PZDiscoveryMonitor.startUp(PZDiscoveryMonitor.java:245) ~[KPIP.jar:na]
    at com.google.common.util.concurrent.AbstractIdleService$1$1.run(AbstractIdleService.java:43) [KPIP.jar:na]
    at java.lang.Thread.run(Thread.java:679) [na:1.6.0_22]
Caused by: java.io.FileNotFoundException: /etc/knewton/discovery/haag/platform.yml (No such file or directory)
    at java.io.FileInputStream.open(Native Method) ~[na:1.6.0_22]
    at java.io.FileInputStream.<init>(FileInputStream.java:137) ~[na:1.6.0_22]
    at java.io.FileInputStream.<init>(FileInputStream.java:96) ~[na:1.6.0_22]
    at com.knewton.knewtonconfig.configuration.FileInputStreamProvider.get(FileInputStreamProvider.java:28) ~[KPIP.jar:na]
    ... 5 common frames omitted
2012-05-23 15:07:33,414 INFO  [main] com.knewton.kpip.KpipMain     : Setting up outgoing connections...
2012-05-23 15:07:33,415 DEBUG [OutgoingConnections STARTING] c.k.k.s.OutgoingConnections   : Starting connection with KPIPClientContext[NEW]{lifeguard=Lifeguard [NEW]service=Haag}...
@iamloivx

This comment has been minimized.

Copy link

commented Dec 19, 2014

Thanks Peter N for your Java patterns. They help me a lots.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.