Skip to content

Instantly share code, notes, and snippets.

@bdelbosc
Created March 12, 2014 15:06
Show Gist options
  • Save bdelbosc/9508821 to your computer and use it in GitHub Desktop.
Save bdelbosc/9508821 to your computer and use it in GitHub Desktop.
logstash conf to get info from Apache/Nuxeo/PostgreSQL stack
input {
file {
path => "/var/log/nuxeo/server.log"
type => "nuxeo"
}
file {
path => "/var/log/postgresql/postgresql-9.1-main.log"
type => "postgresql"
}
file {
type => "apache"
path => "/opt/graphite/storage/log/webapp/access.log"
}
}
filter {
# Add a tag to all events to identify the instance
mutate { add_tag => ["nxmon"] }
if [type] == "nuxeo" {
# Group all lines until they start with a new timestamp (useful for stacktraces)
multiline {
pattern => "^20"
negate => true
what => "previous"
}
grok {
match => ["message", "(?m)%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level}(?: \[%{JAVAFILE:class}\])?(?: %{GREEDYDATA:logmessage})?"]
}
}
else if [type] == "postgresql" {
multiline {
pattern => "^\s"
what => "previous"
}
if "COPY" in [message] { drop {} }
else if "duration" in [message] {
if "execute" in [message] {
mutate { add_tag => ["sql-query"] }
grok {
match => ["message", "%{DATESTAMP:timestamp} CET \[%{POSINT:pid}\]: \[%{POSINT}-1] user=%{NOTSPACE:connection_id} %{WORD:level}: duration: %{NUMBER:duration_ms} ms execute %{GREEDYDATA}: %{GREEDYDATA:request}"]
}
} else if "parse" in [message] {
mutate { add_tag => ["sql-parse"] }
grok {
match => ["message", "%{DATESTAMP:timestamp} CET \[%{POSINT:pid}\]: \[%{POSINT}-1] user=%{GREEDYDATA:connection_id} %{WORD:level}: duration: %{NUMBER:duration_ms} ms parse %{GREEDYDATA}: %{GREEDYDATA:request}"]
}
} else if "bind" in [message] {
mutate { add_tag => ["sql-bind"] }
grok {
match => ["message", "%{DATESTAMP:timestamp} CET \[%{POSINT:pid}\]: \[%{POSINT}-1] user=%{GREEDYDATA:connection_id} %{WORD:level}: duration: %{NUMBER:duration_ms} ms bind %{GREEDYDATA}: %{GREEDYDATA:request}"]
}
} else if "statement" in [message] {
mutate { add_tag => ["sql-query"] }
grok {
match => ["message", "%{DATESTAMP:timestamp} CET \[%{POSINT:pid}\]: \[%{POSINT}-1] user=%{NOTSPACE:connection_id} %{WORD:level}: duration: %{NUMBER:duration_ms} ms statement: %{GREEDYDATA:request}"]
}
}
mutate { convert => [ "execution_ms", "float" ] }
ruby { code => "event['elapsed'] = event['duration_ms'].to_f / 1000" }
} else {
mutate { add_tag => "db-message" }
grok {
match => ["message", "%{DATESTAMP:timestamp} CET \[%{POSINT:pid}\]: \[%{POSINT}-1] user=%{NOTSPACE:connection_id} %{WORD:level}: %{GREEDYDATA:message}"]
}
}
}
else if [type] == "apache" {
grok {
match => ["message", "%{COMBINEDAPACHELOG} \-\- %{INT:elapsed_us:int}"]
}
if "/nuxeo/" not in [request] { drop { } }
mutate { convert => [ "elapsed_us", "integer" ] }
mutate { convert => [ "bytes", "integer" ] }
ruby { code => "event['elapsed'] = event['elapsed_us'].to_f / 1000000" }
}
}
output {
# stdout { }
elasticsearch { host => localhost }
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment