Graylog Pipeline: Parse AWS CloudFront Raw Logs

graylog-pipeline-cloudfront-logs

rule "Parse AWS CloudFront Raw Logs"

when
  regex("^(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)", to_string($message.message)).matches == true
then
  let m = regex("^(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)", to_string($message.message));
  
  let t = parse_unix_milliseconds(to_long(replace(to_string(m["0"]), ".", "")));
  set_field("cf_unix_timestamp", m["0"]);
  set_field("cf_timestamp", t);
  
  set_field("cf_client_ip", m["1"]);
  
  set_field("cf_time_to_first_byte", m["2"]);
  set_field("cf_status", m["3"]);
  set_field("cf_bytes", m["4"]);
  set_field("cf_method", m["5"]);
  set_field("cf_protocol", m["6"]);
  set_field("cf_host", m["7"]);
  set_field("cf_uri_stem", m["8"]);
  set_field("cf_bytes", m["9"]);
  set_field("cf_x_edge_location", m["10"]);
  set_field("cf_x_edge_request_id", m["11"]);
  set_field("cf_x_host_header", m["12"]);
  set_field("cf_time_taken", m["13"]);
  set_field("cf_protocol_version", m["14"]);
  set_field("cf_ip_version", m["15"]);
  set_field("cf_user_agent", m["16"]);
  set_field("cf_referer", m["17"]);
  set_field("cf_cookie", m["18"]);
  set_field("cf_uri_query", m["19"]);
  set_field("cf_x_edge_respose_result_type", m["20"]);
  set_field("cf_x_forwarded_for", m["21"]);
  set_field("cf_ssl_protocol", m["22"]);
  set_field("cf_ssl_cipher", m["23"]);
  set_field("cf_x_edge_result_type", m["24"]);
  set_field("cf_file_encrypted_fields", m["25"]);
  set_field("cf_file_status", m["26"]);
  set_field("cf_content_type", m["27"]);
  set_field("cf_content_len", m["28"]);
  set_field("cf_range_start", m["29"]);
  set_field("cf_range_end", m["30"]);
  set_field("cf_port", m["31"]);
  set_field("cf_x_edge_detailed_result_type", m["32"]);
  set_field("cf_country", m["33"]);
  set_field("cf_accept_encoding", m["34"]);
  set_field("cf_accept", m["35"]);
  set_field("cf_cache_behavior_path_pattern", m["36"]);
  set_field("cf_headers", m["37"]);
  set_field("cf_header_names", m["38"]);
  set_field("cf_header_count", m["39"]);
end