Skip to content

Instantly share code, notes, and snippets.

@grem11n
Last active October 29, 2018 09:44
Show Gist options
  • Save grem11n/b00b2b4b90d76689222a48f798318075 to your computer and use it in GitHub Desktop.
Save grem11n/b00b2b4b90d76689222a48f798318075 to your computer and use it in GitHub Desktop.
Logstash configuration for Cloudfront logs
input {
s3 {
bucket => "<S3_BUCKET_NAME_WITH_CLOUDFRONT_LOGS>"
prefix => "PREFIX_FOR_LOG_FILES (CF DISTRIBUTION NAME)"
region => "BUCKET_REGION"
}
}
filter {
grok {
match => { "message" => "(?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY}\t%{TIME})\t%{WORD:x_edge_location}\t(?:%{NUMBER:sc_bytes:int}|-)\t%{IPORHOST:clientip}\t%{WORD:cs_method}\t%{HOSTNAME:cs_host}\t%{NOTSPACE:cs_uri_stem}\t%{NUMBER:sc_status:int}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:agent}\t%{GREEDYDATA:cs_uri_query}\t%{GREEDYDATA:cookies}\t%{WORD:x_edge_result_type}\t%{NOTSPACE:x_edge_request_id}\t%{HOSTNAME:x_host_header}\t%{URIPROTO:cs_protocol}\t%{INT:cs_bytes:int}\t%{GREEDYDATA:time_taken:float}\t%{GREEDYDATA:x_forwarded_for}\t%{GREEDYDATA:ssl_protocol}\t%{GREEDYDATA:ssl_cipher}\t%{GREEDYDATA:x_edge_response_result_type}" }
}
mutate {
add_field => [ "listener_timestamp", "%{date} %{time}" ]
}
date {
match => [ "listener_timestamp", "yy-MM-dd HH:mm:ss" ]
target => "@timestamp"
}
geoip {
source => "c_ip"
}
useragent {
source => "User_Agent"
target => "useragent"
}
mutate {
remove_field => ["date", "time", "listener_timestamp", "cloudfront_version", "message", "cloudfront_fields", "User_Agent"]
}
}
output {
amazon_es {
hosts => ["<ELASTIC_SEARCH_HOST>"]
region => "REGION (IF AWS ES)"
index => "cloudfront-logs-%{+YYYY.MM.dd}"
template => "<PATH_TO_TEMPLATE>"
}
}
{
"template": "cloudfront-logs-*",
"mappings": {
"logs": {
"_source": {
"enabled": false
},
"_all": {
"enabled": false
},
"dynamic_templates": [
{
"string_fields": {
"mapping": {
"index": "not_analyzed",
"type": "string"
},
"match_mapping_type": "string",
"match": "*"
}
}
],
"properties": {
"geoip" : {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "float" },
"longitude" : { "type" : "float" }
}
}
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment