Skip to content

Instantly share code, notes, and snippets.

@eredding-rmn
Forked from matetsu/README
Last active August 29, 2015 14:06
Show Gist options
  • Save eredding-rmn/fbfe479652393de94072 to your computer and use it in GitHub Desktop.
Save eredding-rmn/fbfe479652393de94072 to your computer and use it in GitHub Desktop.
* cloudtrail_template_1.json
dateのformatを YYYY-MM-dd'T'HH:mm:ssZ としたもの。
* cloudtrail_template_2.json
dateのformatを date_time_no_millis としたもの。
* mapping_after_insert.json
cloudtrail_template_1.json でデータ投入後のmapping (dynamic_templateが効いていない?のは要調査)
* data.json
実際のログデータ
* データの流れ
AWS API操作 --[CloudTrail]--> S3 --[SNS通知]--> 自前スクリプト(S3のログを取得) --[fluent-logger-ruby]--> fluentd --[fluent-plugin-elasticsearch]--> Elasticsearch --> Kibana3
{
"template" : "aws-*",
"mappings" : {
"cloudtrail": {
"_source": { "compress": true },
"_ttl" : {
"enabled" : true,
"default" : "60d"
},
"dynamic_templates": [
{
"string_template": {
"match": "*",
"mapping": { "type": "string", "index": "not_analyzed" },
"match_mapping_type": "string"
}
}
],
"properties": {
"eventTime": {"type": "date", "format":"YYYY-MM-dd'T'HH:mm:ssZ"},
"userIdentity" : {
"properties"
{
"template" : "aws-*",
"mappings" : {
"cloudtrail": {
"_source": { "compress": true },
"_ttl" : {
"enabled" : true,
"default" : "60d"
},
"dynamic_template": [
{
"string_template": {
"match": "*",
"mapping": { "type": "string", "index": "not_analyzed" },
"match_mapping_type": "string"
}
}
],
"properties": {
"eventTime": {"type": "date", "format":"date_time_no_millis"},
"userIdentity" : {
"properties" : {
"sessionContext" : {
"properties" : {
"attributes" : {
"properties" : {
"creationDate" : {
"type" : "date",
"format" : "date_time_no_millis"
}
}
}
}
}
}
},
"@timestamp": {"type": "date", "format": "dateOptionalTime"}
}
}
}
}
}
{
"_index": "aws-20140709",
"_type": "cloudtrail",
"_id": "99ci9IzjT-aA8JNcMBPbPw",
"_score": null,
"_source": {
"eventVersion": "1.01",
"userIdentity": {
"type": "IAMUser",
"principalId": "PRINCIPAL_ID",
"arn": "arn:aws:iam::ACCOUNT_ID:user/USER",
"accountId": "ACCOUNT_ID",
"accessKeyId": "ACCESS_KEY_ID",
"userName": "USER",
"sessionContext": {
"attributes": {
"mfaAuthenticated": "false",
"creationDate": "2014-07-01T01:32:43Z"
}
},
"invokedBy": "signin.amazonaws.com"
},
"eventTime": "2014-07-01T13:07:43Z",
"eventSource": "monitoring.amazonaws.com",
"eventName": "DescribeAlarms",
"awsRegion": "ap-northeast-1",
"sourceIPAddress": "signin.amazonaws.com",
"userAgent": "signin.amazonaws.com",
"requestParameters": {
"maxRecords": 100
},
"responseElements": null,
"requestID": "REQUEST_ID",
"eventID": "EVENT_ID",
"tag": "cloudtrail",
"@timestamp": "2014-07-09T10:14:47+09:00"
},
"sort": [
1404220063000
]
}
{
"aws-20140709" : {
"mappings" : {
"cloudtrail" : {
"_ttl" : {
"enabled" : true,
"default" : 5184000000
},
"_source" : {
"compress" : true
},
"properties" : {
"@timestamp" : {
"type" : "date",
"format" : "dateOptionalTime"
},
"awsRegion" : {
"type" : "string"
},
"eventID" : {
"type" : "string"
},
"eventName" : {
"type" : "string"
},
"eventSource" : {
"type" : "string"
},
"eventTime" : {
"type" : "date",
"format" : "YYYY-MM-dd'T'HH:mm:ssZ"
},
"eventVersion" : {
"type" : "string"
},
"requestID" : {
"type" : "string"
},
"requestParameters" : {
"properties" : {
"maxRecords" : {
"type" : "long"
}
}
},
"sourceIPAddress" : {
"type" : "string"
},
"tag" : {
"type" : "string"
},
"time" : {
"type" : "date",
"format" : "dateOptionalTime"
},
"userAgent" : {
"type" : "string"
},
"userIdentity" : {
"properties" : {
"accessKeyId" : {
"type" : "string"
},
"accountId" : {
"type" : "string"
},
"arn" : {
"type" : "string"
},
"invokedBy" : {
"type" : "string"
},
"principalId" : {
"type" : "string"
},
"sessionContext" : {
"properties" : {
"attributes" : {
"properties" : {
"creationDate" : {
"type" : "date",
"format" : "YYYY-MM-dd'T'HH:mm:ssZ"
},
"mfaAuthenticated" : {
"type" : "string"
}
}
}
}
},
"type" : {
"type" : "string"
},
"userName" : {
"type" : "string"
}
}
}
}
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment