Skip to content

Instantly share code, notes, and snippets.

@vmadman
Created April 27, 2013 06:43
Show Gist options
  • Save vmadman/5472127 to your computer and use it in GitHub Desktop.
Save vmadman/5472127 to your computer and use it in GitHub Desktop.
A configuration for a logstash agent. This logstash agent collects logs from apache access, apache error, and syslog via UDP.
input {
tcp {
port => 5000
type => "syslog"
host => "127.0.0.1"
}
udp {
port => 5000
type => "syslog"
host => "127.0.0.1"
}
udp {
port => 5001
host => "127.0.0.1"
type => "apache"
buffer_size => 8192
format => "json_event"
}
udp {
port => 5002
host => "127.0.0.1"
type => "apache-error"
tags => [ "Apache", "Error", "_SERVER_NAME_" ]
format => "plain"
}
udp {
port => 5003
host => "127.0.0.1"
type => "apache-error"
tags => [ "Apache", "Error", "_SERVER_NAME_._TLD_" ]
format => "plain"
}
udp {
port => 5004
host => "127.0.0.1"
type => "apache-error"
tags => [ "Apache", "Error", "logging._TLD_" ]
format => "plain"
}
udp {
port => 5010
host => "127.0.0.1"
type => "apache-error"
tags => [ "Apache", "Error", "ajenti._SERVER_NAME_._TLD_" ]
format => "plain"
}
udp {
port => 5011
host => "127.0.0.1"
type => "apache-error"
tags => [ "Apache", "Error", "graphite._SERVER_NAME_._TLD_" ]
format => "plain"
}
udp {
port => 5012
host => "127.0.0.1"
type => "apache-error"
tags => [ "Apache", "Error", "graylog._SERVER_NAME_._TLD_" ]
format => "plain"
}
udp {
port => 5013
host => "127.0.0.1"
type => "apache-error"
tags => [ "Apache", "Error", "logstash._SERVER_NAME_._TLD_" ]
format => "plain"
}
udp {
port => 5014
host => "127.0.0.1"
type => "apache-error"
tags => [ "Apache", "Error", "rabbit._SERVER_NAME_._TLD_" ]
format => "plain"
}
udp {
port => 5015
host => "127.0.0.1"
type => "apache-error"
tags => [ "Apache", "Error", "sensu._SERVER_NAME_._TLD_" ]
format => "plain"
}
udp {
port => 5016
host => "127.0.0.1"
type => "apache-error"
tags => [ "Apache", "Error", "supervisor._SERVER_NAME_._TLD_" ]
format => "plain"
}
}
filter {
# SYSLOG PROCESSING
grok {
type => "syslog"
pattern => [ "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" ]
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{@source_host}" ]
}
syslog_pri {
type => "syslog"
}
date {
type => "syslog"
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
mutate {
type => "syslog"
exclude_tags => "_grokparsefailure"
replace => [ "@source_host", "%{syslog_hostname}" ]
replace => [ "@message", "%{syslog_message}" ]
}
mutate {
type => "syslog"
remove => [ "syslog_hostname", "syslog_message", "syslog_timestamp" ]
}
# APACHE ACCESS LOG PROCESSING
date {
type => "apache"
timestamp => "ISO8601"
}
mutate {
type => "apache"
remove => [ "timestamp" ]
}
# APACHE ERROR LOG PROCESSING
grok {
type => "apache-error"
pattern => [ "%{GENERICAPACHEERROR}" ]
patterns_dir => "/usr/local/logstash/conf/grok_patterns"
}
date {
type => "apache-error"
timestamp => "EEE MMM dd HH:mm:ss yyyy"
}
mutate {
type => "apache-error"
replace => [ "severity", "%{apacheseverity}" ]
remove => [ "apacheseverity", "timestamp" ]
}
mutate {
type => "apache-error"
tags => [ "Apache", "Error", "ajenti._SERVER_NAME_._TLD_" ]
replace => [ "@source_host", "ajenti._SERVER_NAME_._TLD_" ]
}
mutate {
type => "apache-error"
tags => [ "Apache", "Error", "graphite._SERVER_NAME_._TLD_" ]
replace => [ "@source_host", "graphite._SERVER_NAME_._TLD_" ]
}
mutate {
type => "apache-error"
tags => [ "Apache", "Error", "graylog._SERVER_NAME_._TLD_" ]
replace => [ "@source_host", "graylog._SERVER_NAME_._TLD_" ]
}
mutate {
type => "apache-error"
tags => [ "Apache", "Error", "logstash._SERVER_NAME_._TLD_" ]
replace => [ "@source_host", "logstash._SERVER_NAME_._TLD_" ]
}
mutate {
type => "apache-error"
tags => [ "Apache", "Error", "rabbit._SERVER_NAME_._TLD_" ]
replace => [ "@source_host", "rabbit._SERVER_NAME_._TLD_" ]
}
mutate {
type => "apache-error"
tags => [ "Apache", "Error", "sensu._SERVER_NAME_._TLD_" ]
replace => [ "@source_host", "sensu._SERVER_NAME_._TLD_" ]
}
mutate {
type => "apache-error"
tags => [ "Apache", "Error", "supervisor._SERVER_NAME_._TLD_" ]
replace => [ "@source_host", "supervisor._SERVER_NAME_._TLD_" ]
}
mutate {
type => "apache-error"
tags => [ "Apache", "Error", "logging._TLD_" ]
replace => [ "@source_host", "logging._TLD_" ]
}
mutate {
type => "apache-error"
tags => [ "Apache", "Error", "_SERVER_NAME_._TLD_" ]
replace => [ "@source_host", "_SERVER_NAME_._TLD_" ]
}
mutate {
type => "apache-error"
tags => [ "Apache", "Error", "_SERVER_NAME_" ]
replace => [ "@source_host", "_SERVER_NAME_" ]
}
}
output {
redis {
host => "###REDIS_IP###"
data_type => "list"
key => "logstash"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment