Filebeat config for AWS CloudFront log processing

filebeat-cloudfront.yml

---
logging.metrics.enabled: false
filebeat.inputs:
- type: aws-s3
  queue_url: <SQS_NAME>
  processors:
    - drop_event:
        when:
          regexp:
            message: "^#.*"
    - dissect:
        tokenizer: "%{date}\t%{time}\t%{edge_location}\t%{response_bytes|long}\t%{c_ip|ip}\t%{cs_method}\t%{cs_host}\t%{uri_path}\t%{response|integer}\t%{referer}\t%{user_agent}\t%{query}\t%{cookies}\t%{result_type}\t%{request_id}\t%{host_header}\t%{protocol}\t%{cs_bytes|long}\t%{duration|float}\t%{x_forwarded_for}\t%{ssl_protocol}\t%{ssl_cipher}\t%{edge_response_result_type}\t%{cs_protocol_version}\t%{fle_status}\t%{fle_encrypted_fields}\t%{c_port|integer}\t%{time_to_first_byte|float}\t%{x_edge_detail_result_type}\t%{sc_content_type|long}\t%{sc_content_len|long}\t%{sc_range_start}\t%{sc_range_end}"
        field: "message"
        target_prefix: ""
        overwrite_keys: true
        ignore_failure: true
    - script:
        lang: javascript
        id: cf_merge_ts
        source: >
          function process(event) {
            event.Put('datetime', event.Get('date') + 'T' + event.Get('time') + 'Z');
          }
    - timestamp:
        field: "datetime"
        layouts:
          - '2006-01-02T15:04:05Z'
    - urldecode:
        fields:
          - from: "user_agent"
          - from: "sc_content_type"
    - drop_fields:
        fields:
          - "aws"
          - "time"
          - "date"
          - "input"
          - "datetime"
          - "fle_status"
          - "fle_encrypted_fields"