Skip to content

Instantly share code, notes, and snippets.

@reyjrar
Created July 5, 2012 12:48
Show Gist options
  • Save reyjrar/3053503 to your computer and use it in GitHub Desktop.
Save reyjrar/3053503 to your computer and use it in GitHub Desktop.
LogStash Configuration
input {
tcp {
type => "syslog"
port => 8514
}
}
filter {
## DISCARD IMPROPERLY FORMATTED MESSAGES
grep {
match => [ "@message", "^\<[0-9]+\>[A-Za-z]{3}\s+[0-9]{1,2} ([0-9]{2}:){2}[0-9]{2} \S+ \S+:" ]
add_tag => "syslog"
}
## DECODE SYSLOG MESSAGE
grok {
tags => "syslog"
pattern => "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:timestamp} %{SYSLOGHOST:logsource} %{GREEDYDATA:message_remainder}"
}
grok {
tags => "syslog"
pattern => "%{SYSLOGTIMESTAMP} %{HOSTSHORT:hostname}"
patterns_dir => "/etc/logstash/pattern.d"
}
mutate {
tags => "syslog"
replace => [ "@message", "%{syslog_pri}"]
}
## Syslog Priority Parser Plugin
#syslog_pri {
# tags => "syslog"
#}
## Restore @message
mutate {
tags => "syslog"
replace => [ "@message", "%{message_remainder}" ]
rename => [ "syslog_facility", "facility", "syslog_severity", "severity" ]
}
mutate {
tags => "syslog"
remove => [ "message_remainder", "syslog_facility_code", "syslog_severity_code", "syslog_pri" ]
}
## DETERMINE PATH ##
grok {
tags => "syslog"
pattern => "^%{SYSLOGPROG}: %{GREEDYDATA:message_remainder}"
add_tag => "%{program}"
}
mutate {
tags => "syslog"
replace => [ "@message", "%{message_remainder}" ]
}
## OSSEC
grok {
tags => "ossec"
patterns_dir => "/etc/logstash/pattern.d"
pattern => "\"component\": \"\(%{HOSTSHORT:src}"
}
mutate {
tags => "ossec"
add_field => [ "json_data", "%{message_remainder}" ]
add_tag => [ "json", "pull_message" ]
}
## APACHE
grok {
tags => "apache"
patterns_dir => "/etc/logstash/pattern.d"
pattern => "%{HOST:vhost} %{BOOKINGAPACHELOG}"
}
## nginx
grok {
tags => "nginx"
pattern => "%{HOST:vhost} %{GREEDYDATA:json_data}"
}
mutate {
tags => [ "nginx_parsed" ]
add_tag => "json"
}
## GLOBAL JSON Parser
json {
tags => "json"
json_data => "cee"
add_tag => "parsed"
}
## GLOBAL CLEANUP
# Remove the message_remainder
mutate {
remove => [ "message_remainder" ]
}
# Remove the json_data field only if the parse was successful
mutate {
tags => [ "json", "parsed" ]
remove => [ "json_data" ]
}
# Pull message from the message field
mutate {
tags => "pull_message"
rename => [ "message", "@message" ]
}
}
output {
#stdout { debug => true }
elasticsearch {
embedded => false
cluster => "log-ng"
host => "logsearch-cluster.example.com"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment