Last active
February 19, 2022 12:00
-
-
Save unreal4u/23ad9d6df99ebfe1fd80744ef650d4f2 to your computer and use it in GitHub Desktop.
Filter for logstash for access and error logs generated by apache2.(2|4) and php7
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
filter { | |
# Analyze access log | |
if [fields][type] == "apache-access" { | |
# First part: get every common message | |
grok { | |
match => { "message" => "^%{IP:[apache2][access][remote_ip]} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] " } | |
overwrite => "message" | |
} | |
# Get the specifics | |
grok { | |
match => { "message" => "\"(?:%{WORD:[apache2][access][method]} %{NOTSPACE:[apache2][access][url]}(?: HTTP/%{BASE16FLOAT:[apache2][access][http_version]})?|%{DATA:rawrequest})\" %{INT:[apache2][access][response_code]} (?:%{INT:[apache2][access][body_sent][bytes]}|-) %{QS:[apache2][access][referrer]} %{QS:[apache2][access][agent]}" } | |
remove_field => [ "message" ] | |
} | |
# Add the user-agent data | |
useragent { | |
source => "[apache2][access][agent]" | |
target => "[apache2][access][user_agent]" | |
} | |
# Get the quotes out of the data we are going to save | |
mutate { | |
gsub => [ | |
"[apache2][access][referrer]", '"', "", | |
"[apache2][access][agent]", '"', "" | |
] | |
} | |
} | |
# Analyze error log | |
if [fields][type] == "apache-error" { | |
# First part and always equal: get the header of the log | |
grok { | |
match => { "message" => "^\[%{DAY:weekday} %{MONTH:month} %{MONTHDAY:day} %{TIME:time}(?:.%{INT:microseconds})? %{YEAR:year}\] \[(?:php7\:)?%{WORD:[apache2][error][level]}\] (?:\[pid %{INT:[apache2][error][pid]}\] )?\[client %{IP:[apache2][access][remote_ip]}(?:\:%{INT:internalPort})?\] PHP %{DATA:[php][errorLevel]}\: %{GREEDYDATA:message}" } | |
overwrite => "message" | |
} | |
# This can have a lot of variations, so perform this in a second run | |
grok { | |
match => { "message" => "%{DATA:[php][errorMessage]} in %{DATA:[php][file]}(?:\:%{INT}\\nStack trace:\\n%{DATA:[php][stackTrace]} in %{DATA})? on line %{INT:[php][line]}(?:, referer\: %{DATA:[php][referer]})?" } | |
remove_field => [ "weekday", "month", "day", "time", "microseconds", "year", "internalport", "message" ] | |
} | |
# Transform certain parts: get whitespaces out and convert stacktrace to an actual array | |
mutate { | |
strip => [ "[php][errorLevel]", "[php][errorMessage]", "[php][file]" ] | |
split => { "[php][stackTrace]" => "\n" } | |
} | |
} | |
# Lastly, set the geoip data | |
geoip { | |
source => "[apache2][access][remote_ip]" | |
target => "[apache2][access][geoip]" | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This was a big help. Thanks for sharing it!