Skip to content

Instantly share code, notes, and snippets.

@piavlo
Created January 15, 2013 23:54
Show Gist options
  • Save piavlo/4543318 to your computer and use it in GitHub Desktop.
Save piavlo/4543318 to your computer and use it in GitHub Desktop.
indexer-apache-errorlog.conf is causing the slowness problem, even though there are ZERO event in it's flow
input {
redis {
host => "log1a.us-east-1"
type => "apache-accesslog"
data_type => "list"
key => "logstash:apache:accesslog"
message_format => "json_event"
charset => "UTF-8"
db => 0
debug => false
threads => 1
}
}
output {
elasticsearch_http {
tags => [ "apache-accesslog" ]
host => "log1a.us-east-1"
index => "apache-accesslog-%{+YYYY.MM.dd}"
flush_size => 100
exclude_tags => [ "metric" ]
}
}
input {
redis {
host => "log1a.us-east-1"
type => "apache-errorlog"
tags => [ "apache-errorlog" ]
data_type => "list"
key => "logstash:apache:errorlog"
format => "plain"
charset => "UTF-8"
db => 0
debug => false
threads => 1
}
}
filter {
grok {
tags => [ "apache-errorlog" ]
patterns_dir => [ "/opt/logstash/patterns", "/etc/logstash/patterns.d" ]
match => [ "@message", "%{PHPAPACHEERROR}" ]
match => [ "@message", "%{GENERICAPACHEERROR}" ]
break_on_match => true
}
date {
tags => [ "apache-errorlog" ]
match => [ "timestamp", "EEE MMM dd HH:mm:ss yyyy" ]
exclude_tags => [ "_grokparsefailure" ]
}
mutate {
tags => [ "apache-errorlog" ]
replace => [ "@message", "%{message}" ]
remove => [ "message" ]
}
}
output {
elasticsearch_http {
tags => [ "apache-accesslog" ]
host => "log1a.us-east-1"
index => "apache-errorlog-%{+YYYY.MM.dd}"
flush_size => 1
exclude_tags => [ "metric" ]
}
}
filter {
metrics {
meter => "events"
add_tag => [ "metric" ]
}
grep {
tags => [ "metric" ]
match => [ "events.rate_1m", ".*" ]
drop => false
negate => true
add_field => [ "events.rate_1m", "0.0" ]
}
}
output {
# stdout {
# debug => true
# debug_format => "json"
# }
stdout {
tags => [ "metric" ]
message => "events rate: %{events.rate_1m}"
}
}
APACHEERRORTIME %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}
APACHEERRORPREFIX \[%{APACHEERRORTIME:timestamp}\] \[%{NOTSPACE:apacheseverity}\] \[client %{IPORHOST:sourcehost}\]
GENERICAPACHEERROR %{APACHEERRORPREFIX} %{GREEDYDATA:message}
PHPSEVERITY PHP %{WORD:phpseverity}( error)?\:
PHPMESSAGE %{PHPSEVERITY} %{GREEDYDATA}
PHPAPACHEERROR %{APACHEERRORPREFIX} %{PHPMESSAGE:message}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment