companylogs.conf

# /etc/logstash/conf.d/userlogs.conf
input {

  s3 {
     bucket => "example-logs"
     delete => false
     interval => 60 # seconds
     prefix => "user-logs/"
     region => "us-east-1"
     type => "cloudfront"
     codec => "plain"
     access_key_id => "XXX"
     secret_access_key => "YYY"
     use_ssl => true
  }

}


filter {
  if ( ("#Version: 1.0" in [message]) or ("#Fields: date" in [message])) {
    drop {}
  }

  grok {
    match => {
       "message" => "(?<timestamp>%{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day}\t%{TIME:time})\t%{WORD:x_edge_location}\t(?:%{NUMBER:sc_bytes:int}|-)\t%{IPORHOST:clientip}\t%{WORD:cs_method}\t%{HOSTNAME:cs_host}\t(?<cs_uri_stem>(/v1/users/%{USERNAME:user})?%{GREEDYDATA})\t%{NUMBER:sc_status:int}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:agent}\t%{GREEDYDATA:cs_uri_query}\t%{GREEDYDATA:cookies}\t%{WORD:x_edge_result_type}\t%{NOTSPACE:x_edge_request_id}\t%{HOSTNAME:x_host_header}\t%{URIPROTO:cs_protocol}\t%{INT:cs_bytes:int}\t%{GREEDYDATA:time_taken:float}\t%{GREEDYDATA:x_forwarded_for}\t%{GREEDYDATA:ssl_protocol}\t%{GREEDYDATA:ssl_cipher}\t%{GREEDYDATA:x_edge_response_result_type}"
    }
  }

  mutate {
    add_field => [ "listener_timestamp", "%{year}-%{month}-%{day} %{time}" ]
    add_field => [ "raw_data", "%{message}" ]
  }

  date {
    match => [ "listener_timestamp", "yy-MM-dd HH:mm:ss" ]
    target => "@timestamp"
  }

  geoip {
    source => "c_ip"
  }

  useragent {
    source => "User_Agent"
    target => "useragent"
  }

  mutate {
    remove_field => ["year","month", "day", "time", "listener_timestamp", "timestamp","cloudfront_version", "message", "cloudfront_fields", "User_Agent", "cookies", "x_edge_request_id", "x_edge_location", "@version", "x_forwarded_for", "ssl_protocol", "ssl_cipher", "x_edge_response_result_type"]
  }
}

output {
  stdout { codec => rubydebug }
  elasticsearch {
    hosts => [ "localhost:9200" ]
    document_type => "user_logs"
  }
}

userlogs.conf

#/etc/logstash/conf.d/companylogs.conf
input {

  s3 {
     bucket => "example-logs"
     delete => false
     interval => 60 # seconds
     prefix => "company-logs/"
     region => "us-east-1"
     type => "cloudfront"
     codec => "plain"
     access_key_id => "XXX"
     secret_access_key => "YYY"
     use_ssl => true
  }

}


filter {
  if ( ("#Version: 1.0" in [message]) or ("#Fields: date" in [message])) {
    drop {}
  }

  grok {
    match => {
      "message" => "(?<timestamp>%{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day}\t%{TIME:time})\t%{WORD:x_edge_location}\t(?:%{NUMBER:sc_bytes:int}|-)\t%{IPORHOST:clientip}\t%{WORD:cs_method}\t%{HOSTNAME:cs_host}\t(?<cs_uri_stem>/%{WORD:company}%{NOTSPACE}|/%{GREEDYDATA}?)\t%{NUMBER:sc_status:int}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:agent}\t%{GREEDYDATA:cs_uri_query}\t%{GREEDYDATA:cookies}\t%{WORD:x_edge_result_type}\t%{NOTSPACE:x_edge_request_id}\t%{HOSTNAME:x_host_header}\t%{URIPROTO:cs_protocol}\t%{INT:cs_bytes:int}\t%{GREEDYDATA:time_taken:float}\t%{GREEDYDATA:x_forwarded_for}\t%{GREEDYDATA:ssl_protocol}\t%{GREEDYDATA:ssl_cipher}\t%{GREEDYDATA:x_edge_response_result_type}"
    }
  }

  mutate {
    add_field => [ "listener_timestamp", "%{year}-%{month}-%{day} %{time}" ]
    add_field => [ "raw_data", "%{message}" ]
  }

  date {
    match => [ "listener_timestamp", "yy-MM-dd HH:mm:ss" ]
    target => "@timestamp"
  }

  geoip {
    source => "c_ip"
  }

  useragent {
    source => "User_Agent"
    target => "useragent"
  }

  mutate {
    remove_field => ["year","month", "day", "time", "listener_timestamp", "timestamp","cloudfront_version", "message", "cloudfront_fields", "User_Agent", "cookies", "x_edge_request_id", "x_edge_location", "@version", "x_forwarded_for", "ssl_protocol", "ssl_cipher", "x_edge_response_result_type"]
  }
}

output {
  stdout { codec => rubydebug }
  elasticsearch {
    hosts => [ "localhost:9200" ]
    document_type => "company_logs"
  }
}