Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Logstash configuration file for throwing CloudFront logs to Amazon ES
input {
s3 {
bucket => "<CLOUDFRONT_LOG_BUCKET>"
prefix => "<CLOUDFRONT_LOG_KEY_PREFIX>"
region => "<BUCKET_REGION_NAME>"
}
}
filter {
grok {
match => { "message" => "%{DATE_EU:date}\t%{TIME:time}\t%{WORD:x_edge_location}\t(?:%{NUMBER:sc_bytes:int}|-)\t%{IPORHOST:c_ip}\t%{WORD:cs_method}\t%{HOSTNAME:cs_host}\t%{NOTSPACE:cs_uri_stem}\t%{NUMBER:sc_status:int}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:User_Agent}\t%{GREEDYDATA:cs_uri_stem}\t%{GREEDYDATA:cookies}\t%{WORD:x_edge_result_type}\t%{NOTSPACE:x_edge_request_id}\t%{HOSTNAME:x_host_header}\t%{URIPROTO:cs_protocol}\t%{INT:cs_bytes:int}\t%{GREEDYDATA:time_taken}\t%{GREEDYDATA:x_forwarded_for}\t%{GREEDYDATA:ssl_protocol}\t%{GREEDYDATA:ssl_cipher}\t%{GREEDYDATA:x_edge_response_result_type}" }
}
mutate {
add_field => [ "listener_timestamp", "%{date} %{time}" ]
}
date {
match => [ "listener_timestamp", "yy-MM-dd HH:mm:ss" ]
target => "@timestamp"
}
geoip {
source => "c_ip"
}
useragent {
source => "User_Agent"
target => "useragent"
}
mutate {
remove_field => ["date", "time", "listener_timestamp", "cloudfront_version", "message", "cloudfront_fields", "User_Agent"]
}
}
output {
amazon_es {
hosts => ["<AMAZON_ES_DOMAIN_ENDPOINT>"]
region => "<AMAZON_ES_DOMAIN_REGION_NAME>"
index => "cloudfront-logs-%{+YYYY.MM.dd}"
template => "/etc/logstash/cloudfront.template.json"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment