Created
August 4, 2021 06:08
-
-
Save ankitdevnalkar/d0ead1e65778b38ff060230a0f063135 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
"signals-aws-cloudtrail" : { | |
"description" : "Pipeline for AWS CloudTrail Logs", | |
"processors" : [ | |
{ | |
"rename" : { | |
"field" : "message", | |
"target_field" : "event.original" | |
} | |
}, | |
{ | |
"json" : { | |
"target_field" : "json", | |
"field" : "event.original" | |
} | |
}, | |
{ | |
"rename" : { | |
"target_field" : "aws.cloudtrail.event_version", | |
"ignore_failure" : true, | |
"field" : "json.eventVersion" | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.userIdentity.type", | |
"target_field" : "aws.cloudtrail.user_identity.type", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"target_field" : "user.name", | |
"ignore_failure" : true, | |
"field" : "json.userIdentity.userName" | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.userIdentity.principalId", | |
"target_field" : "user.id", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.userIdentity.arn", | |
"target_field" : "aws.cloudtrail.user_identity.arn", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.userIdentity.accountId", | |
"target_field" : "cloud.account.id", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"ignore_failure" : true, | |
"field" : "json.userIdentity.accessKeyId", | |
"target_field" : "aws.cloudtrail.user_identity.access_key_id" | |
} | |
}, | |
{ | |
"rename" : { | |
"target_field" : "aws.cloudtrail.user_identity.session_context.mfa_authenticated", | |
"ignore_failure" : true, | |
"field" : "json.userIdentity.sessionContext.attributes.mfaAuthenticated" | |
} | |
}, | |
{ | |
"date" : { | |
"field" : "json.userIdentity.sessionContext.attributes.creationDate", | |
"target_field" : "aws.cloudtrail.user_identity.session_context.creation_date", | |
"ignore_failure" : true, | |
"formats" : [ | |
"ISO8601" | |
] | |
} | |
}, | |
{ | |
"rename" : { | |
"target_field" : "aws.cloudtrail.user_identity.session_context.session_issuer.type", | |
"ignore_failure" : true, | |
"field" : "json.userIdentity.sessionContext.sessionIssuer.type" | |
} | |
}, | |
{ | |
"rename" : { | |
"ignore_failure" : true, | |
"field" : "json.userIdentity.sessionContext.sessionIssuer.userName", | |
"target_field" : "user.name" | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.userIdentity.sessionContext.sessionIssuer.principalId", | |
"target_field" : "aws.cloudtrail.user_identity.session_context.session_issuer.principal_id", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"target_field" : "aws.cloudtrail.user_identity.session_context.session_issuer.arn", | |
"ignore_failure" : true, | |
"field" : "json.userIdentity.sessionContext.sessionIssuer.arn" | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.userIdentity.sessionContext.sessionIssuer.accountId", | |
"target_field" : "aws.cloudtrail.user_identity.session_context.session_issuer.account_id", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"ignore_failure" : true, | |
"field" : "json.userIdentity.invokedBy", | |
"target_field" : "aws.cloudtrail.user_identity.invoked_by" | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.eventSource", | |
"target_field" : "event.provider", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"set" : { | |
"ignore_empty_value" : true, | |
"field" : "event.action", | |
"value" : "{{json.eventName}}", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.awsRegion", | |
"target_field" : "cloud.region", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.sourceIPAddress", | |
"target_field" : "source.address", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"grok" : { | |
"field" : "source.address", | |
"ignore_failure" : true, | |
"patterns" : [ | |
"^%{IP:source.ip}$" | |
] | |
} | |
}, | |
{ | |
"geoip" : { | |
"target_field" : "source.geo", | |
"ignore_failure" : true, | |
"ignore_missing" : true, | |
"field" : "source.ip" | |
} | |
}, | |
{ | |
"geoip" : { | |
"field" : "source.ip", | |
"target_field" : "source.as", | |
"properties" : [ | |
"asn", | |
"organization_name" | |
], | |
"ignore_missing" : true, | |
"database_file" : "GeoLite2-ASN.mmdb" | |
} | |
}, | |
{ | |
"rename" : { | |
"target_field" : "source.as.number", | |
"ignore_missing" : true, | |
"field" : "source.as.asn" | |
} | |
}, | |
{ | |
"rename" : { | |
"target_field" : "source.as.organization.name", | |
"ignore_missing" : true, | |
"field" : "source.as.organization_name" | |
} | |
}, | |
{ | |
"user_agent" : { | |
"target_field" : "user_agent", | |
"on_failure" : [ | |
{ | |
"rename" : { | |
"target_field" : "user_agent.original", | |
"ignore_failure" : true, | |
"field" : "json.userAgent" | |
} | |
} | |
], | |
"field" : "json.userAgent" | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.errorCode", | |
"target_field" : "aws.cloudtrail.error_code", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"target_field" : "aws.cloudtrail.error_message", | |
"ignore_failure" : true, | |
"field" : "json.errorMessage" | |
} | |
}, | |
{ | |
"rename" : { | |
"if" : "ctx.json.requestParameters != null", | |
"field" : "json.requestParameters", | |
"target_field" : "aws.cloudtrail.flattened.request_parameters" | |
} | |
}, | |
{ | |
"script" : { | |
"lang" : "painless", | |
"source" : """if (ctx.aws.cloudtrail.flattened.request_parameters != null) { | |
ctx.aws.cloudtrail.request_parameters = ctx.aws.cloudtrail.flattened.request_parameters.toString(); | |
} | |
""", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.responseElements", | |
"target_field" : "aws.cloudtrail.flattened.response_elements", | |
"if" : "ctx.json.responseElements != null" | |
} | |
}, | |
{ | |
"script" : { | |
"ignore_failure" : true, | |
"lang" : "painless", | |
"source" : """if (ctx.aws.cloudtrail.flattened.response_elements != null) { | |
ctx.aws.cloudtrail.response_elements = ctx.aws.cloudtrail.flattened.response_elements.toString(); | |
} | |
""" | |
} | |
}, | |
{ | |
"rename" : { | |
"if" : "ctx?.json?.additionalEventData != null", | |
"field" : "json.additionalEventData", | |
"target_field" : "aws.cloudtrail.flattened.additional_eventdata" | |
} | |
}, | |
{ | |
"script" : { | |
"lang" : "painless", | |
"source" : """if (ctx.aws.cloudtrail.flattened.additional_eventdata != null) { | |
ctx.aws.cloudtrail.additional_eventdata = ctx.aws.cloudtrail.flattened.additional_eventdata.toString(); | |
} | |
""", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.requestId", | |
"target_field" : "aws.cloudtrail.request_id", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.eventID", | |
"target_field" : "event.id", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"target_field" : "aws.cloudtrail.event_type", | |
"ignore_failure" : true, | |
"field" : "json.eventType" | |
} | |
}, | |
{ | |
"rename" : { | |
"ignore_failure" : true, | |
"field" : "json.apiVersion", | |
"target_field" : "aws.cloudtrail.api_version" | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.managementEvent", | |
"target_field" : "aws.cloudtrail.management_event", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.readOnly", | |
"target_field" : "aws.cloudtrail.read_only", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"target_field" : "aws.cloudtrail.resources.arn", | |
"ignore_failure" : true, | |
"field" : "json.resources.ARN" | |
} | |
}, | |
{ | |
"rename" : { | |
"ignore_failure" : true, | |
"field" : "json.resources.accountId", | |
"target_field" : "aws.cloudtrail.resources.account_id" | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.resources.type", | |
"target_field" : "aws.cloudtrail.resources.type", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.recipientAccountId", | |
"target_field" : "aws.cloudtrail.recipient_account_id", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.serviceEventDetails", | |
"target_field" : "aws.cloudtrail.flattened.service_event_details", | |
"if" : "ctx.json.serviceEventDetails != null" | |
} | |
}, | |
{ | |
"script" : { | |
"lang" : "painless", | |
"source" : """if (ctx.aws.cloudtrail.flattened.service_event_details != null) { | |
ctx.aws.cloudtrail.service_event_details = ctx.aws.cloudtrail.flattened.service_event_details.toString(); | |
} | |
""", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.sharedEventId", | |
"target_field" : "aws.cloudtrail.shared_event_id", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "json.vpcEndpointId", | |
"target_field" : "aws.cloudtrail.vpc_endpoint_id", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"script" : { | |
"lang" : "painless", | |
"ignore_failure" : true, | |
"source" : """void addRelatedUser(def ctx, String userName) { | |
if (ctx.related == null) { | |
Map map = new HashMap(); | |
ctx.put("related", map); | |
} | |
if (ctx.related.user == null) { | |
ArrayList al = new ArrayList(); | |
ctx.related.put("user", al); | |
} | |
ctx.related.user.add(userName); | |
} if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.userName != null) { | |
addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.userName); | |
} if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.newUserName != null) { | |
addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.newUserName); | |
}""" | |
} | |
}, | |
{ | |
"script" : { | |
"lang" : "painless", | |
"ignore_failure" : true, | |
"source" : """if (ctx.json?.eventName != 'ConsoleLogin') { | |
return; | |
} Map aed_map = new HashMap(); if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.MobileVersion != null) { | |
if (ctx.aws.cloudtrail.flattened.additional_eventdata.MobileVersion == 'No') { | |
aed_map.put("mobile_version", false); | |
} else { | |
aed_map.put("mobile_version", true); | |
} | |
} if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.LoginTo != null) { | |
aed_map.put("login_to", ctx.aws.cloudtrail.flattened.additional_eventdata.LoginTo); | |
} if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.MFAUsed != null) { | |
if (ctx.aws.cloudtrail.flattened.additional_eventdata.MFAUsed == 'No') { | |
aed_map.put("mfa_used", false); | |
} else { | |
aed_map.put("mfa_used", true); | |
} | |
} if (aed_map.size() > 0) { | |
Map cl_map = new HashMap(); | |
cl_map.put("additional_eventdata", aed_map); | |
ctx.aws.cloudtrail.put("console_login", cl_map); | |
}""" | |
} | |
}, | |
{ | |
"script" : { | |
"lang" : "painless", | |
"ignore_failure" : true, | |
"params" : { | |
"DetachGroupPolicy" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"group", | |
"change" | |
] | |
}, | |
"ConsoleLogin" : { | |
"category" : [ | |
"authentication" | |
], | |
"type" : [ | |
"info" | |
] | |
}, | |
"EnableMFADevice" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"change" | |
] | |
}, | |
"ListUserPolicies" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"info" | |
] | |
}, | |
"PutUserPolicy" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"change" | |
] | |
}, | |
"UpdateGroup" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"group", | |
"change" | |
] | |
}, | |
"CreateGroup" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"group", | |
"creation" | |
] | |
}, | |
"CreateKeyPair" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"admin", | |
"creation" | |
] | |
}, | |
"DeleteUser" : { | |
"type" : [ | |
"user", | |
"deletion" | |
], | |
"category" : [ | |
"iam" | |
] | |
}, | |
"ListGroupPolicies" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"group", | |
"info" | |
] | |
}, | |
"ListAttachedUserPolicies" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"info" | |
] | |
}, | |
"CreateBucket" : { | |
"category" : [ | |
"file" | |
], | |
"type" : [ | |
"creation" | |
] | |
}, | |
"DeleteBucket" : { | |
"category" : [ | |
"file" | |
], | |
"type" : [ | |
"deletion" | |
] | |
}, | |
"UntagUser" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"change" | |
] | |
}, | |
"PutUserPermissionsBoundary" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"change" | |
] | |
}, | |
"DeleteUserPermissionsBoundary" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"change" | |
] | |
}, | |
"AssumeRole" : { | |
"category" : [ | |
"authentication" | |
], | |
"type" : [ | |
"info" | |
] | |
}, | |
"CreateAccessKey" : { | |
"type" : [ | |
"user", | |
"change" | |
], | |
"category" : [ | |
"iam" | |
] | |
}, | |
"DeleteVirtualMFADevice" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"change" | |
] | |
}, | |
"GetGroupPolicy" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"group", | |
"info" | |
] | |
}, | |
"AttachGroupPolicy" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"group", | |
"change" | |
] | |
}, | |
"CreateVirtualMFADevice" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"change" | |
] | |
}, | |
"SetSecurityTokenServicePreferences" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"admin", | |
"change" | |
] | |
}, | |
"GetUserPolicy" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"info" | |
] | |
}, | |
"ListUsers" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"info" | |
] | |
}, | |
"CreateUser" : { | |
"type" : [ | |
"user", | |
"creation" | |
], | |
"category" : [ | |
"iam" | |
] | |
}, | |
"DeleteGroupPolicy" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"group", | |
"change" | |
] | |
}, | |
"DeleteGroup" : { | |
"type" : [ | |
"group", | |
"deletion" | |
], | |
"category" : [ | |
"iam" | |
] | |
}, | |
"ChangePassword" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"change" | |
] | |
}, | |
"TagUser" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"change" | |
] | |
}, | |
"UpdateRole" : { | |
"type" : [ | |
"admin", | |
"change" | |
], | |
"category" : [ | |
"iam" | |
] | |
}, | |
"UpdateSSHPublicKey" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"change" | |
] | |
}, | |
"DeleteSSHPublicKey" : { | |
"type" : [ | |
"user", | |
"change" | |
], | |
"category" : [ | |
"iam" | |
] | |
}, | |
"RemoveUserFromGroup" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"group", | |
"change" | |
] | |
}, | |
"GetUser" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"info" | |
] | |
}, | |
"UpdateAccessKey" : { | |
"type" : [ | |
"user", | |
"change" | |
], | |
"category" : [ | |
"iam" | |
] | |
}, | |
"GetGroup" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"group", | |
"info" | |
] | |
}, | |
"PutGroupPolicy" : { | |
"type" : [ | |
"group", | |
"change" | |
], | |
"category" : [ | |
"iam" | |
] | |
}, | |
"AttachUserPolicy" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"change" | |
] | |
}, | |
"UpdateUser" : { | |
"type" : [ | |
"user", | |
"change" | |
], | |
"category" : [ | |
"iam" | |
] | |
}, | |
"SetDefaultPolicyVersion" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"admin", | |
"change" | |
] | |
}, | |
"UpdateLoginProfile" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"change" | |
] | |
}, | |
"DeleteUserPolicy" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"change" | |
] | |
}, | |
"ListGroups" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"group", | |
"info" | |
] | |
}, | |
"ListUserTags" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"info" | |
] | |
}, | |
"DeactivateMFADevice" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"change" | |
] | |
}, | |
"DetachUserPolicy" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"user", | |
"change" | |
] | |
}, | |
"DeleteAccessKey" : { | |
"type" : [ | |
"user", | |
"change" | |
], | |
"category" : [ | |
"iam" | |
] | |
}, | |
"ListAttachedGroupPolicies" : { | |
"type" : [ | |
"group", | |
"info" | |
], | |
"category" : [ | |
"iam" | |
] | |
}, | |
"AddUserToGroup" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"group", | |
"change" | |
] | |
}, | |
"UpdateAccountPasswordPolicy" : { | |
"category" : [ | |
"iam" | |
], | |
"type" : [ | |
"admin", | |
"change" | |
] | |
}, | |
"ListGroupsForUser" : { | |
"type" : [ | |
"user", | |
"info" | |
], | |
"category" : [ | |
"iam" | |
] | |
} | |
}, | |
"source" : """ctx.event.kind = 'event'; ctx.event.type = 'info'; | |
if (ctx.aws.cloudtrail.error_code != null || ctx.aws.cloudtrail.error_message != null) { | |
ctx.event.outcome = 'failure' | |
} else { | |
ctx.event.outcome = 'success' | |
} | |
if (ctx?.event?.action == null) { | |
return; | |
} | |
if (ctx.event.action == 'ConsoleLogin' && ctx?.aws?.cloudtrail?.flattened?.response_elements.ConsoleLogin != null) { | |
ctx.event.outcome = Processors.lowercase(ctx.aws.cloudtrail.flattened.response_elements.ConsoleLogin); | |
} | |
def hm = new HashMap(params.get(ctx.event.action)); hm.forEach((k, v) -> ctx.event[k] = v);""" | |
} | |
}, | |
{ | |
"remove" : { | |
"ignore_missing" : true, | |
"field" : [ | |
"json" | |
] | |
} | |
} | |
], | |
"on_failure" : [ | |
{ | |
"set" : { | |
"field" : "error.message", | |
"value" : "{{ _ingest.on_failure_message }}" | |
} | |
} | |
] | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment