-
-
Save chrisan/1c5ce5beacfc0e124d39fa842f051857 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| GET /_cat/indices?v | |
| health status index uuid pri rep docs.count docs.deleted store.size pri.store.size | |
| yellow open logstash-2018.02.03 CagddD3xTrKLsGGZ586mZQ 5 1 56627 0 15.4mb 15.4mb | |
| yellow open logstash-2018.04.13 Sg9NvyL5SJacHhAvYAtg2w 5 1 147773 0 49.6mb 49.6mb | |
| yellow open logstash-2018.03.03 FZBQ5O0dRiCLVVQS8E6Ugw 5 1 161512 0 32.1mb 32.1mb | |
| yellow open logstash-2018.01.18 5n7AldZlQ4usrbgwDMxTag 5 1 96455 0 31.8mb 31.8mb | |
| yellow open logstash-2018.01.11 D8eDEOGFTK-F6C5PN_s52Q 5 1 51229 0 14.9mb 14.9mb |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| input { | |
| s3 { | |
| bucket => "cloudfront-logs" | |
| delete => false | |
| interval => 60 # seconds | |
| prefix => "api/" | |
| region => "us-east-1" | |
| type => "api_logs" | |
| codec => "plain" | |
| sincedb_path => "/var/lib/logstash/api.sincedb" | |
| } | |
| } | |
| filter { | |
| if [type] == "api_logs" { | |
| if ( ("#Version: 1.0" in [message]) or ("#Fields: date" in [message])) { | |
| drop {} | |
| } | |
| grok { | |
| match => { | |
| "message" => "(?<timestamp>%{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day}\t%{TIME:time})\t%{WORD:x_edge_location}\t(?:%{NUMBER:sc_bytes:int}|-)\t%{IPORHOST:clientip}\t%{WORD:cs_method}\t%{HOSTNAME:cs_host}\t(?<cs_uri_stem>(/v1/rest/%{USERNAME:company})?%{GREEDYDATA})\t%{NUMBER:sc_status:int}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:agent}\t%{GREEDYDATA:cs_uri_query}\t%{GREEDYDATA:cookies}\t%{WORD:x_edge_result_type}\t%{NOTSPACE:x_edge_request_id}\t%{HOSTNAME:x_host_header}\t%{URIPROTO:cs_protocol}\t%{INT:cs_bytes:int}\t%{GREEDYDATA:time_taken:float}\t%{GREEDYDATA:x_forwarded_for}\t%{GREEDYDATA:ssl_protocol}\t%{GREEDYDATA:ssl_cipher}\t%{GREEDYDATA:x_edge_response_result_type}" | |
| } | |
| } | |
| mutate { | |
| add_field => [ "listener_timestamp", "%{year}-%{month}-%{day} %{time}" ] | |
| } | |
| date { | |
| match => [ "listener_timestamp", "yy-MM-dd HH:mm:ss" ] | |
| target => "@timestamp" | |
| } | |
| geoip { | |
| source => "c_ip" | |
| } | |
| useragent { | |
| source => "User_Agent" | |
| target => "useragent" | |
| } | |
| mutate { | |
| remove_field => ["year","month", "day", "time", "listener_timestamp", "timestamp","cloudfront_version", "message", "cloudfront_fields", "User_Agent", "cookies", "x_edge_request_id", "x_edge_location", "@version", "x_forwarded_for", "ssl_protocol", "ssl_cipher", "x_edge_response_result_type"] | |
| } | |
| } | |
| } | |
| output { | |
| if [type] == "api_logs" { | |
| elasticsearch { | |
| hosts => [ "localhost:9200" ] | |
| document_type => "api_logs" | |
| } | |
| } | |
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| GET /*/_mapping/ | |
| { | |
| ..., | |
| "logstash-2018.03.31": { | |
| "mappings": { | |
| "cdn_api_logs": { | |
| "dynamic_templates": [ | |
| { | |
| "message_field": { | |
| "path_match": "message", | |
| "match_mapping_type": "string", | |
| "mapping": { | |
| "norms": false, | |
| "type": "text" | |
| } | |
| } | |
| }, | |
| { | |
| "string_fields": { | |
| "match": "*", | |
| "match_mapping_type": "string", | |
| "mapping": { | |
| "fields": { | |
| "keyword": { | |
| "ignore_above": 256, | |
| "type": "keyword" | |
| } | |
| }, | |
| "norms": false, | |
| "type": "text" | |
| } | |
| } | |
| } | |
| ], | |
| "properties": { | |
| "@timestamp": { | |
| "type": "date" | |
| }, | |
| "@version": { | |
| "type": "keyword" | |
| }, | |
| "agent": { | |
| "type": "text", | |
| "norms": false, | |
| "fields": { | |
| "keyword": { | |
| "type": "keyword", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "clientip": { | |
| "type": "text", | |
| "norms": false, | |
| "fields": { | |
| "keyword": { | |
| "type": "keyword", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "cs_bytes": { | |
| "type": "long" | |
| }, | |
| "cs_host": { | |
| "type": "text", | |
| "norms": false, | |
| "fields": { | |
| "keyword": { | |
| "type": "keyword", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "cs_method": { | |
| "type": "text", | |
| "norms": false, | |
| "fields": { | |
| "keyword": { | |
| "type": "keyword", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "cs_protocol": { | |
| "type": "text", | |
| "norms": false, | |
| "fields": { | |
| "keyword": { | |
| "type": "keyword", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "cs_uri_query": { | |
| "type": "text", | |
| "norms": false, | |
| "fields": { | |
| "keyword": { | |
| "type": "keyword", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "cs_uri_stem": { | |
| "type": "text", | |
| "norms": false, | |
| "fields": { | |
| "keyword": { | |
| "type": "keyword", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "geoip": { | |
| "dynamic": "true", | |
| "properties": { | |
| "ip": { | |
| "type": "ip" | |
| }, | |
| "latitude": { | |
| "type": "half_float" | |
| }, | |
| "location": { | |
| "type": "geo_point" | |
| }, | |
| "longitude": { | |
| "type": "half_float" | |
| } | |
| } | |
| }, | |
| "company": { | |
| "type": "text", | |
| "norms": false, | |
| "fields": { | |
| "keyword": { | |
| "type": "keyword", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "referrer": { | |
| "type": "text", | |
| "norms": false, | |
| "fields": { | |
| "keyword": { | |
| "type": "keyword", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "sc_bytes": { | |
| "type": "long" | |
| }, | |
| "sc_status": { | |
| "type": "long" | |
| }, | |
| "tags": { | |
| "type": "text", | |
| "norms": false, | |
| "fields": { | |
| "keyword": { | |
| "type": "keyword", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "time_taken": { | |
| "type": "float" | |
| }, | |
| "type": { | |
| "type": "text", | |
| "norms": false, | |
| "fields": { | |
| "keyword": { | |
| "type": "keyword", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "x_edge_result_type": { | |
| "type": "text", | |
| "norms": false, | |
| "fields": { | |
| "keyword": { | |
| "type": "keyword", | |
| "ignore_above": 256 | |
| } | |
| } | |
| }, | |
| "x_host_header": { | |
| "type": "text", | |
| "norms": false, | |
| "fields": { | |
| "keyword": { | |
| "type": "keyword", | |
| "ignore_above": 256 | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "_default_": { | |
| "dynamic_templates": [ | |
| { | |
| "message_field": { | |
| "path_match": "message", | |
| "match_mapping_type": "string", | |
| "mapping": { | |
| "norms": false, | |
| "type": "text" | |
| } | |
| } | |
| }, | |
| { | |
| "string_fields": { | |
| "match": "*", | |
| "match_mapping_type": "string", | |
| "mapping": { | |
| "fields": { | |
| "keyword": { | |
| "ignore_above": 256, | |
| "type": "keyword" | |
| } | |
| }, | |
| "norms": false, | |
| "type": "text" | |
| } | |
| } | |
| } | |
| ], | |
| "properties": { | |
| "@timestamp": { | |
| "type": "date" | |
| }, | |
| "@version": { | |
| "type": "keyword" | |
| }, | |
| "geoip": { | |
| "dynamic": "true", | |
| "properties": { | |
| "ip": { | |
| "type": "ip" | |
| }, | |
| "latitude": { | |
| "type": "half_float" | |
| }, | |
| "location": { | |
| "type": "geo_point" | |
| }, | |
| "longitude": { | |
| "type": "half_float" | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| ... | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment