Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
An ElasticSearch Index template for CloudTrail events
PUT _template/cloudtrail
{
"index_patterns": ["cloudtrail-*"],
"settings": {
"number_of_shards": 1,
"mapping": {
"total_fields": {
"limit": 10000
}
}
},
"mappings": {
"doc": {
"properties": {
"@timestamp" : {
"type" : "date"
},
"@version" : {
"type" : "integer"
},
"additionalEventData" : {
"properties" : {
"LoginTo" : {
"type" : "keyword"
},
"MFAUsed" : {
"type" : "keyword"
},
"MobileVersion" : {
"type" : "keyword"
},
"Note" : {
"type" : "text"
},
"SamlProviderArn" : {
"type" : "keyword"
},
"vpcEndpointId" : {
"type" : "keyword"
}
}
},
"apiVersion" : {
"type" : "keyword"
},
"awsRegion" : {
"type" : "keyword"
},
"errorCode" : {
"type" : "keyword"
},
"errorMessage" : {
"type" : "text"
},
"eventID" : {
"type" : "keyword"
},
"eventName" : {
"type" : "keyword"
},
"eventSource" : {
"type" : "keyword"
},
"eventType" : {
"type" : "keyword"
},
"eventVersion" : {
"type" : "double"
},
"readOnly" : {
"type" : "boolean"
},
"recipientAccountId" : {
"type" : "keyword"
},
"requestID" : {
"type" : "keyword"
},
"requestParameters" : {
"type": "object",
"properties": {
"iamInstanceProfile": {
"type": "object",
"properties": {
"name": {
"type": "keyword"
},
"arn": {
"type": "keyword"
}
}
}
}
},
"resources" : {
"properties" : {
"ARN" : {
"type" : "keyword"
},
"accountId" : {
"type" : "keyword"
},
"type" : {
"type" : "keyword"
}
}
},
"responseElements" : {
"type": "object"
},
"serviceEventDetails" : {
"type": "object"
},
"sharedEventID" : {
"type" : "keyword"
},
"sourceIPAddress" : {
"type" : "keyword"
},
"tags" : {
"type" : "keyword"
},
"type" : {
"type" : "keyword"
},
"userAgent" : {
"type" : "keyword"
},
"userIdentity" : {
"properties" : {
"accessKeyId" : {
"type" : "keyword"
},
"accountId" : {
"type" : "keyword"
},
"arn" : {
"type" : "keyword"
},
"identityProvider" : {
"type" : "keyword"
},
"invokedBy" : {
"type" : "keyword"
},
"principalId" : {
"type" : "keyword"
},
"sessionContext" : {
"properties" : {
"attributes" : {
"properties" : {
"creationDate" : {
"type" : "date"
},
"mfaAuthenticated" : {
"type" : "boolean"
}
}
},
"sessionIssuer" : {
"properties" : {
"accountId" : {
"type" : "keyword"
},
"arn" : {
"type" : "keyword"
},
"principalId" : {
"type" : "keyword"
},
"type" : {
"type" : "keyword"
},
"userName" : {
"type" : "keyword"
}
}
}
}
},
"type" : {
"type" : "keyword"
},
"userName" : {
"type" : "keyword"
}
}
},
"vpcEndpointId" : {
"type" : "keyword"
}
}
}
}
}
input {
s3 {
bucket => "our-happy-cloudtrail-bucket"
type => "cloudtrail"
codec => "cloudtrail"
# the s3-input plugins lists ALL objects in this prefix each time
# it checks if something was added ... very expensive for big buckets
prefix => "CT/AWSLogs/123456789012/CloudTrail/us-east-1/2019/05/"
exclude_pattern => "/(2018|2019/01|2019/02|2019/03|2019/04)/"
watch_for_new_files => true
}
}
filter {
metrics {
meter => [ "events" ]
add_tag => "metric"
}
if [type] == "cloudtrail" {
if [requestParameters][iamInstanceProfile] {
# coerce requestParameters.iamInstanceProfile
# from "x" to {"name"=>"x"} because in "ec2:CreateLaunchConfiguration"
# it is a string, but in "ec2:RunInstances" it is an object
ruby {
code => "
profile = event.get('[requestParameters][iamInstanceProfile]')
if profile.is_a? String
event.set('[requestParameters][iamInstanceProfile]', {
'name' => profile
})
end
"
}
}
}
}
output {
if "metric" in [tags] {
stdout {
codec => line {
format => "rate: %{[events][rate_1m]}"
}
}z
}
if [type] == "cloudtrail" {
elasticsearch {
action => update
doc_as_upsert => true
hosts => ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"]
index => "cloudtrail-%{+YYYY.MM.dd}"
document_id => "%{eventID}"
manage_template => false
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.