Skip to content

Instantly share code, notes, and snippets.

@ssp
Created May 16, 2013 07:14
Show Gist options
  • Star 7 You must be signed in to star a gist
  • Fork 3 You must be signed in to fork a gist
  • Save ssp/5589952 to your computer and use it in GitHub Desktop.
Save ssp/5589952 to your computer and use it in GitHub Desktop.
input {
stdin {
type => "stdin-type"
}
file {
type => "apache"
path => ["/var/log/apache2/access_log", "/var/log/apache2/error_log", "/var/log/apache2/ssl_request_log"]
}
file {
type => "pazpar2"
path => "/var/log/pazpar2/pazpar2.log"
}
file {
type => "metaproxy"
path => "/var/log/metaproxy/metaproxy.log"
}
file {
type => "couchdb"
path => "/var/log/couchdb/couch.log"
}
file {
type => "mysql"
path => "/var/log/mysqld.log"
}
file {
type => "nagios"
path => "/var/log/nagios/nagios.log"
}
file {
type => "tivoli"
path => ["/var/log/tivoli/dsmerror.log", "/var/log/tivoli/dsmsched.log"]
}
}
filter {
grok {
type => "apache"
# See the following URL for a complete list of named patterns
# logstash/grok ships with by default:
# https://github.com/logstash/logstash/tree/master/patterns
#
# The grok filter will use the below pattern and on successful match use
# any captured values as new fields in the event.
pattern => "%{COMBINEDAPACHELOG}"
}
date {
type => "apache"
# Try to pull the timestamp from the 'timestamp' field (parsed above with
# grok). The apache time format looks like: "18/Aug/2011:05:44:34 -0700"
locale => en
match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
}
grok {
type => "pazpar2"
# Try to match pazpar2's log format, e.g.:
# 2013-05-15 23:19:27 pazpar2 [log] Request: GET /search.pz2?command=server-status
# 2013-05-16 00:05:50 pazpar2 [log] Request: GET /search.pz2?command=ping&session=335780868&windowid=
break_on_match => false
pattern => "%{DATESTAMP:timestamp} pazpar2 \[%{WORD:loglevel}\] %{GREEDYDATA:logmessage}"
pattern => "%{GREEDYDATA} Session %{NOTSPACE:session}: %{GREEDYDATA:sessionmessage}"
pattern => "%{GREEDYDATA} PQF for Client %{NOTSPACE:target}: %{GREEDYDATA:pqf}"
pattern => "%{GREEDYDATA} Client %{NOTSPACE:target}: Search CQL: %{GREEDYDATA:cql}"
pattern => "%{GREEDYDATA} Request: %{NOTSPACE:method} %{GREEDYDATA:request}"
pattern => "%{GREEDYDATA} Request: %{GREEDYDATA}session=%{WORD:session}"
pattern => "%{GREEDYDATA} Request: %{GREEDYDATA}command=%{WORD:command}"
pattern => "%{GREEDYDATA} Request: %{GREEDYDATA}block=%{WORD:block}"
}
date {
type => "pazpar2"
# Try to pull the timestamp from the 'timestamp' field (parsed above with
# grok). The time format looks like: "2013-05-15 23:19:27"
locale => en
match => ["timestamp", "yyyy-MM-dd HH:mm:ss"]
}
grok {
type => "metaproxy"
# Try to match pazpar2's log format, e.g.:
# 2013-05-16--00:24:32 10868 [log] metaproxy start 1.3.55 3e8ea72503dab6e988c622e647b7bbd3abb338e6
# 2013-05-16--00:26:07 10868 [log] FN 134.76.22.139:2 2 0.000000 HTTP_Request POST /dspace/geo-leo
break_on_match => false
pattern => "%{WORD:date} %{NUMBER:pid} \[%{WORD:loglevel}\] %{GREEDYDATA:logmessage}"
pattern => "%{WORD:date} %{NUMBER:pid} \[%{WORD:loglevel}\] %{GREEDYDATA:filtermessage} %{IP:ip}:%{GREEDYDATA:commandmessage}"
}
date {
type => "metaproxy"
# Try to pull the timestamp from the 'timestamp' field (parsed above with
# grok). The time format looks like: "2013-05-15--23:19:27"
locale => en
match => ["timestamp", "yyyy-MM-dd--HH:mm:ss"]
}
grok {
type => couchdb
pattern => "\[%{GREEDYDATA:timestamp}\] \[%{GREEDYDATA:loglevel}\] \[%{GREEDYDATA:pid}\] %{IP:clientip} %{USER:ident} %{USER:auth} '%{WORD:verb}' %{NOTSPACE:path} %{NUMBER:response}"
}
date {
type => "couchdb"
# Try to pull the timestamp from the 'timestamp' field (parsed above with
# grok). The time format looks like: "Wed, 15 May 2013 22:16:16 GMT"
locale => en
match => ["timestamp", "EEE, dd MMM yyyy HH:mm:ss z"]
}
grok {
type => nagios
pattern => "%{NAGIOSLOGLINE}"
}
date {
type => "nagios"
# Try to pull the timestamp from the 'timestamp' field (parsed above with
# grok). The time format looks like: "Wed, 15 May 2013 22:16:16 GMT"
locale => en
match => ["nagios_epoch", "UNIX"]
}
grok {
type => "tivoli"
# 05/15/13 02:11:26 ANS1802E Incremental backup of '/' finished with 4 failure
pattern => "(?<tivoli_time>.{19}) %{GREEDYDATA:message}"
}
date {
type => "tivoli"
# Try to pull the timestamp from the 'timestamp' field (parsed above with
# grok). The time format looks like: "05/15/13 02:11:26"
locale => en
match => ["tivoli_time", "MM/dd/yy HH:mm:ss"]
}
}
output {
stdout {
debug => true
debug_format => "json"
}
elasticsearch {
embedded => true
}
}
@jparagon
Copy link

https://gist.github.com/jparagon/6122333 Updated and tested for CouchDB.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment