An ElasticSearch Index template for CloudTrail events

cloudtrail-template-elasticsearch.json

PUT _template/cloudtrail
{
  "index_patterns": ["cloudtrail-*"],
  "settings": {
    "number_of_shards": 1,
    "mapping": {
      "total_fields": {
        "limit": 10000
      }
    }
  },
  "mappings": {
    "doc": {
      "properties": {
        "@timestamp" : {
          "type" : "date"
        },
        "@version" : {
          "type" : "integer"
        },
        "additionalEventData" : {
          "properties" : {
            "LoginTo" : {
              "type" : "keyword"
            },
            "MFAUsed" : {
              "type" : "keyword"
            },
            "MobileVersion" : {
              "type" : "keyword"
            },
            "Note" : {
              "type" : "text"
            },
            "SamlProviderArn" : {
              "type" : "keyword"
            },
            "vpcEndpointId" : {
              "type" : "keyword"
            }
          }
        },
        "apiVersion" : {
          "type" : "keyword"
        },
        "awsRegion" : {
          "type" : "keyword"
        },
        "errorCode" : {
          "type" : "keyword"
        },
        "errorMessage" : {
          "type" : "text"
        },
        "eventID" : {
          "type" : "keyword"
        },
        "eventName" : {
          "type" : "keyword"
        },
        "eventSource" : {
          "type" : "keyword"
        },
        "eventType" : {
          "type" : "keyword"
        },
        "eventVersion" : {
          "type" : "double"
        },
        "readOnly" : {
          "type" : "boolean"
        },
        "recipientAccountId" : {
          "type" : "keyword"
        },
        "requestID" : {
          "type" : "keyword"
        },
        "requestParameters" : {
          "type": "object",
          "properties": {
            "iamInstanceProfile": {
              "type": "object",
              "properties": {
                "name": {
                  "type": "keyword"
                },
                "arn": {
                  "type": "keyword"
                }
              }
            }
          }
        },
        "resources" : {
          "properties" : {
            "ARN" : {
              "type" : "keyword"
            },
            "accountId" : {
              "type" : "keyword"
            },
            "type" : {
              "type" : "keyword"
            }
          }
        },
        "responseElements" : {
          "type": "object"
        },
        "serviceEventDetails" : {
          "type": "object"
        },
        "sharedEventID" : {
          "type" : "keyword"
        },
        "sourceIPAddress" : {
          "type" : "keyword"
        },
        "tags" : {
          "type" : "keyword"
        },
        "type" : {
          "type" : "keyword"
        },
        "userAgent" : {
          "type" : "keyword"
        },
        "userIdentity" : {
          "properties" : {
            "accessKeyId" : {
              "type" : "keyword"
            },
            "accountId" : {
              "type" : "keyword"
            },
            "arn" : {
              "type" : "keyword"
            },
            "identityProvider" : {
              "type" : "keyword"
            },
            "invokedBy" : {
              "type" : "keyword"
            },
            "principalId" : {
              "type" : "keyword"
            },
            "sessionContext" : {
              "properties" : {
                "attributes" : {
                  "properties" : {
                    "creationDate" : {
                      "type" : "date"
                    },
                    "mfaAuthenticated" : {
                      "type" : "boolean"
                    }
                  }
                },
                "sessionIssuer" : {
                  "properties" : {
                    "accountId" : {
                      "type" : "keyword"
                    },
                    "arn" : {
                      "type" : "keyword"
                    },
                    "principalId" : {
                      "type" : "keyword"
                    },
                    "type" : {
                      "type" : "keyword"
                    },
                    "userName" : {
                      "type" : "keyword"
                    }
                  }
                }
              }
            },
            "type" : {
              "type" : "keyword"
            },
            "userName" : {
              "type" : "keyword"
            }
          }
        },
        "vpcEndpointId" : {
          "type" : "keyword"
        }
      }
    }
  }
}

logstash.conf

input {
  s3 {
    bucket => "our-happy-cloudtrail-bucket"
    type => "cloudtrail"
    codec => "cloudtrail"
    # the s3-input plugins lists ALL objects in this prefix each time
    # it checks if something was added ... very expensive for big buckets
    prefix => "CT/AWSLogs/123456789012/CloudTrail/us-east-1/2019/05/"
    exclude_pattern => "/(2018|2019/01|2019/02|2019/03|2019/04)/"
    watch_for_new_files => true
  }
}

filter {
  metrics {
    meter => [ "events" ]
    add_tag => "metric"
  }
  if [type] == "cloudtrail" {
    if [requestParameters][iamInstanceProfile] {
      # coerce requestParameters.iamInstanceProfile
      # from "x" to {"name"=>"x"} because in "ec2:CreateLaunchConfiguration"
      # it is a string, but in "ec2:RunInstances" it is an object
      ruby {
        code => "
          profile = event.get('[requestParameters][iamInstanceProfile]')
          if profile.is_a? String
            event.set('[requestParameters][iamInstanceProfile]', {
              'name' => profile
            })
          end
        "
      }
    }
  }
}

output {
  if "metric" in [tags] {
    stdout {
      codec => line {
        format => "rate: %{[events][rate_1m]}"
      }
    }z
  }
  if [type] == "cloudtrail" {
     elasticsearch {
       action => update
       doc_as_upsert => true
       hosts => ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"]
       index => "cloudtrail-%{+YYYY.MM.dd}"
       document_id => "%{eventID}"
       manage_template => false
     }
  }
}