logstash.conf

input {
  stdin {
		type => "stdin-type"
	}
	
	file {
		type => "apache"
		path => ["/var/log/apache2/access_log", "/var/log/apache2/error_log", "/var/log/apache2/ssl_request_log"]
	}
	
	file {
		type => "pazpar2"
		path => "/var/log/pazpar2/pazpar2.log"
	}

	file {
		type => "metaproxy"
		path => "/var/log/metaproxy/metaproxy.log"
	}
	
	file {
		type => "couchdb"
		path => "/var/log/couchdb/couch.log"
	}
	
	file {
		type => "mysql"
		path => "/var/log/mysqld.log"
	}

	file {
		type => "nagios"
		path => "/var/log/nagios/nagios.log"
	}
	
	file {
		type => "tivoli"
		path => ["/var/log/tivoli/dsmerror.log", "/var/log/tivoli/dsmsched.log"]
	}

}


filter {
	grok {
		type => "apache"
		# See the following URL for a complete list of named patterns
		# logstash/grok ships with by default:
		# https://github.com/logstash/logstash/tree/master/patterns
		#
		# The grok filter will use the below pattern and on successful match use
		# any captured values as new fields in the event.
		pattern => "%{COMBINEDAPACHELOG}"
	}

	date {
		type => "apache"
		# Try to pull the timestamp from the 'timestamp' field (parsed above with
		# grok). The apache time format looks like: "18/Aug/2011:05:44:34 -0700"
		locale => en
		match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
	}

	grok {
		type => "pazpar2"
		# Try to match pazpar2's log format, e.g.:
		# 2013-05-15 23:19:27 pazpar2 [log] Request: GET /search.pz2?command=server-status
		# 2013-05-16 00:05:50 pazpar2 [log] Request: GET /search.pz2?command=ping&session=335780868&windowid= 
		break_on_match => false
		pattern => "%{DATESTAMP:timestamp} pazpar2 \[%{WORD:loglevel}\] %{GREEDYDATA:logmessage}"
		pattern => "%{GREEDYDATA} Session %{NOTSPACE:session}: %{GREEDYDATA:sessionmessage}"
		pattern => "%{GREEDYDATA} PQF for Client %{NOTSPACE:target}: %{GREEDYDATA:pqf}"
		pattern => "%{GREEDYDATA} Client %{NOTSPACE:target}: Search CQL: %{GREEDYDATA:cql}"
		pattern => "%{GREEDYDATA} Request: %{NOTSPACE:method} %{GREEDYDATA:request}"
		pattern => "%{GREEDYDATA} Request: %{GREEDYDATA}session=%{WORD:session}"
		pattern => "%{GREEDYDATA} Request: %{GREEDYDATA}command=%{WORD:command}"
		pattern => "%{GREEDYDATA} Request: %{GREEDYDATA}block=%{WORD:block}"
	}

	date {
		type => "pazpar2"
		# Try to pull the timestamp from the 'timestamp' field (parsed above with
		# grok). The time format looks like: "2013-05-15 23:19:27"
		locale => en
		match => ["timestamp", "yyyy-MM-dd HH:mm:ss"]
	}

	grok {
		type => "metaproxy"
		# Try to match pazpar2's log format, e.g.:
		# 2013-05-16--00:24:32 10868 [log] metaproxy start 1.3.55 3e8ea72503dab6e988c622e647b7bbd3abb338e6 
		# 2013-05-16--00:26:07 10868 [log] FN 134.76.22.139:2 2 0.000000 HTTP_Request POST /dspace/geo-leo
		break_on_match => false
		pattern => "%{WORD:date} %{NUMBER:pid} \[%{WORD:loglevel}\] %{GREEDYDATA:logmessage}"
		pattern => "%{WORD:date} %{NUMBER:pid} \[%{WORD:loglevel}\] %{GREEDYDATA:filtermessage} %{IP:ip}:%{GREEDYDATA:commandmessage}"
	}

	date {
		type => "metaproxy"
		# Try to pull the timestamp from the 'timestamp' field (parsed above with
		# grok). The time format looks like: "2013-05-15--23:19:27"
		locale => en
		match => ["timestamp", "yyyy-MM-dd--HH:mm:ss"]
	}
	
	grok {
		type => couchdb
		pattern => "\[%{DATA:timestamp}\] \[%{DATA:loglevel}\] \[%{DATA:pid}\] %{IP:clientip} %{USER:ident} %{USER:auth} %{WORD:verb} %{NOTSPACE:path} %{NUMBER:response}"
	}
	
	date {
		type => "couchdb"
		# Try to pull the timestamp from the 'timestamp' field (parsed above with
		# grok). The time format looks like: "Wed, 15 May 2013 22:16:16 GMT"
		locale => en
		match => ["timestamp", "EEE, dd MMM yyyy HH:mm:ss z"]
	}
	
	grok {
		type => nagios
		pattern => "%{NAGIOSLOGLINE}"
	}
	
	date {
		type => "nagios"
		# Try to pull the timestamp from the 'timestamp' field (parsed above with
		# grok). The time format looks like: "Wed, 15 May 2013 22:16:16 GMT"
		locale => en
		match => ["nagios_epoch", "UNIX"]
	}
	
	grok {
		type => "tivoli"
		# 05/15/13   02:11:26 ANS1802E Incremental backup of '/' finished with 4 failure
		pattern => "(?<tivoli_time>.{19}) %{GREEDYDATA:message}"
	}
	
	date {
		type => "tivoli"
		# Try to pull the timestamp from the 'timestamp' field (parsed above with
		# grok). The time format looks like: "05/15/13   02:11:26"
		locale => en
		match => ["tivoli_time", "MM/dd/yy   HH:mm:ss"]
	}

}


output {
	stdout {
		debug => true
		debug_format => "json"
	}
	elasticsearch {
		embedded => true
	}
}