Skip to content

Instantly share code, notes, and snippets.

@ankitdevnalkar
Created August 4, 2021 06:08
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save ankitdevnalkar/d0ead1e65778b38ff060230a0f063135 to your computer and use it in GitHub Desktop.
Save ankitdevnalkar/d0ead1e65778b38ff060230a0f063135 to your computer and use it in GitHub Desktop.
"signals-aws-cloudtrail" : {
"description" : "Pipeline for AWS CloudTrail Logs",
"processors" : [
{
"rename" : {
"field" : "message",
"target_field" : "event.original"
}
},
{
"json" : {
"target_field" : "json",
"field" : "event.original"
}
},
{
"rename" : {
"target_field" : "aws.cloudtrail.event_version",
"ignore_failure" : true,
"field" : "json.eventVersion"
}
},
{
"rename" : {
"field" : "json.userIdentity.type",
"target_field" : "aws.cloudtrail.user_identity.type",
"ignore_failure" : true
}
},
{
"rename" : {
"target_field" : "user.name",
"ignore_failure" : true,
"field" : "json.userIdentity.userName"
}
},
{
"rename" : {
"field" : "json.userIdentity.principalId",
"target_field" : "user.id",
"ignore_failure" : true
}
},
{
"rename" : {
"field" : "json.userIdentity.arn",
"target_field" : "aws.cloudtrail.user_identity.arn",
"ignore_failure" : true
}
},
{
"rename" : {
"field" : "json.userIdentity.accountId",
"target_field" : "cloud.account.id",
"ignore_failure" : true
}
},
{
"rename" : {
"ignore_failure" : true,
"field" : "json.userIdentity.accessKeyId",
"target_field" : "aws.cloudtrail.user_identity.access_key_id"
}
},
{
"rename" : {
"target_field" : "aws.cloudtrail.user_identity.session_context.mfa_authenticated",
"ignore_failure" : true,
"field" : "json.userIdentity.sessionContext.attributes.mfaAuthenticated"
}
},
{
"date" : {
"field" : "json.userIdentity.sessionContext.attributes.creationDate",
"target_field" : "aws.cloudtrail.user_identity.session_context.creation_date",
"ignore_failure" : true,
"formats" : [
"ISO8601"
]
}
},
{
"rename" : {
"target_field" : "aws.cloudtrail.user_identity.session_context.session_issuer.type",
"ignore_failure" : true,
"field" : "json.userIdentity.sessionContext.sessionIssuer.type"
}
},
{
"rename" : {
"ignore_failure" : true,
"field" : "json.userIdentity.sessionContext.sessionIssuer.userName",
"target_field" : "user.name"
}
},
{
"rename" : {
"field" : "json.userIdentity.sessionContext.sessionIssuer.principalId",
"target_field" : "aws.cloudtrail.user_identity.session_context.session_issuer.principal_id",
"ignore_failure" : true
}
},
{
"rename" : {
"target_field" : "aws.cloudtrail.user_identity.session_context.session_issuer.arn",
"ignore_failure" : true,
"field" : "json.userIdentity.sessionContext.sessionIssuer.arn"
}
},
{
"rename" : {
"field" : "json.userIdentity.sessionContext.sessionIssuer.accountId",
"target_field" : "aws.cloudtrail.user_identity.session_context.session_issuer.account_id",
"ignore_failure" : true
}
},
{
"rename" : {
"ignore_failure" : true,
"field" : "json.userIdentity.invokedBy",
"target_field" : "aws.cloudtrail.user_identity.invoked_by"
}
},
{
"rename" : {
"field" : "json.eventSource",
"target_field" : "event.provider",
"ignore_failure" : true
}
},
{
"set" : {
"ignore_empty_value" : true,
"field" : "event.action",
"value" : "{{json.eventName}}",
"ignore_failure" : true
}
},
{
"rename" : {
"field" : "json.awsRegion",
"target_field" : "cloud.region",
"ignore_failure" : true
}
},
{
"rename" : {
"field" : "json.sourceIPAddress",
"target_field" : "source.address",
"ignore_failure" : true
}
},
{
"grok" : {
"field" : "source.address",
"ignore_failure" : true,
"patterns" : [
"^%{IP:source.ip}$"
]
}
},
{
"geoip" : {
"target_field" : "source.geo",
"ignore_failure" : true,
"ignore_missing" : true,
"field" : "source.ip"
}
},
{
"geoip" : {
"field" : "source.ip",
"target_field" : "source.as",
"properties" : [
"asn",
"organization_name"
],
"ignore_missing" : true,
"database_file" : "GeoLite2-ASN.mmdb"
}
},
{
"rename" : {
"target_field" : "source.as.number",
"ignore_missing" : true,
"field" : "source.as.asn"
}
},
{
"rename" : {
"target_field" : "source.as.organization.name",
"ignore_missing" : true,
"field" : "source.as.organization_name"
}
},
{
"user_agent" : {
"target_field" : "user_agent",
"on_failure" : [
{
"rename" : {
"target_field" : "user_agent.original",
"ignore_failure" : true,
"field" : "json.userAgent"
}
}
],
"field" : "json.userAgent"
}
},
{
"rename" : {
"field" : "json.errorCode",
"target_field" : "aws.cloudtrail.error_code",
"ignore_failure" : true
}
},
{
"rename" : {
"target_field" : "aws.cloudtrail.error_message",
"ignore_failure" : true,
"field" : "json.errorMessage"
}
},
{
"rename" : {
"if" : "ctx.json.requestParameters != null",
"field" : "json.requestParameters",
"target_field" : "aws.cloudtrail.flattened.request_parameters"
}
},
{
"script" : {
"lang" : "painless",
"source" : """if (ctx.aws.cloudtrail.flattened.request_parameters != null) {
ctx.aws.cloudtrail.request_parameters = ctx.aws.cloudtrail.flattened.request_parameters.toString();
}
""",
"ignore_failure" : true
}
},
{
"rename" : {
"field" : "json.responseElements",
"target_field" : "aws.cloudtrail.flattened.response_elements",
"if" : "ctx.json.responseElements != null"
}
},
{
"script" : {
"ignore_failure" : true,
"lang" : "painless",
"source" : """if (ctx.aws.cloudtrail.flattened.response_elements != null) {
ctx.aws.cloudtrail.response_elements = ctx.aws.cloudtrail.flattened.response_elements.toString();
}
"""
}
},
{
"rename" : {
"if" : "ctx?.json?.additionalEventData != null",
"field" : "json.additionalEventData",
"target_field" : "aws.cloudtrail.flattened.additional_eventdata"
}
},
{
"script" : {
"lang" : "painless",
"source" : """if (ctx.aws.cloudtrail.flattened.additional_eventdata != null) {
ctx.aws.cloudtrail.additional_eventdata = ctx.aws.cloudtrail.flattened.additional_eventdata.toString();
}
""",
"ignore_failure" : true
}
},
{
"rename" : {
"field" : "json.requestId",
"target_field" : "aws.cloudtrail.request_id",
"ignore_failure" : true
}
},
{
"rename" : {
"field" : "json.eventID",
"target_field" : "event.id",
"ignore_failure" : true
}
},
{
"rename" : {
"target_field" : "aws.cloudtrail.event_type",
"ignore_failure" : true,
"field" : "json.eventType"
}
},
{
"rename" : {
"ignore_failure" : true,
"field" : "json.apiVersion",
"target_field" : "aws.cloudtrail.api_version"
}
},
{
"rename" : {
"field" : "json.managementEvent",
"target_field" : "aws.cloudtrail.management_event",
"ignore_failure" : true
}
},
{
"rename" : {
"field" : "json.readOnly",
"target_field" : "aws.cloudtrail.read_only",
"ignore_failure" : true
}
},
{
"rename" : {
"target_field" : "aws.cloudtrail.resources.arn",
"ignore_failure" : true,
"field" : "json.resources.ARN"
}
},
{
"rename" : {
"ignore_failure" : true,
"field" : "json.resources.accountId",
"target_field" : "aws.cloudtrail.resources.account_id"
}
},
{
"rename" : {
"field" : "json.resources.type",
"target_field" : "aws.cloudtrail.resources.type",
"ignore_failure" : true
}
},
{
"rename" : {
"field" : "json.recipientAccountId",
"target_field" : "aws.cloudtrail.recipient_account_id",
"ignore_failure" : true
}
},
{
"rename" : {
"field" : "json.serviceEventDetails",
"target_field" : "aws.cloudtrail.flattened.service_event_details",
"if" : "ctx.json.serviceEventDetails != null"
}
},
{
"script" : {
"lang" : "painless",
"source" : """if (ctx.aws.cloudtrail.flattened.service_event_details != null) {
ctx.aws.cloudtrail.service_event_details = ctx.aws.cloudtrail.flattened.service_event_details.toString();
}
""",
"ignore_failure" : true
}
},
{
"rename" : {
"field" : "json.sharedEventId",
"target_field" : "aws.cloudtrail.shared_event_id",
"ignore_failure" : true
}
},
{
"rename" : {
"field" : "json.vpcEndpointId",
"target_field" : "aws.cloudtrail.vpc_endpoint_id",
"ignore_failure" : true
}
},
{
"script" : {
"lang" : "painless",
"ignore_failure" : true,
"source" : """void addRelatedUser(def ctx, String userName) {
if (ctx.related == null) {
Map map = new HashMap();
ctx.put("related", map);
}
if (ctx.related.user == null) {
ArrayList al = new ArrayList();
ctx.related.put("user", al);
}
ctx.related.user.add(userName);
} if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.userName != null) {
addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.userName);
} if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.newUserName != null) {
addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.newUserName);
}"""
}
},
{
"script" : {
"lang" : "painless",
"ignore_failure" : true,
"source" : """if (ctx.json?.eventName != 'ConsoleLogin') {
return;
} Map aed_map = new HashMap(); if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.MobileVersion != null) {
if (ctx.aws.cloudtrail.flattened.additional_eventdata.MobileVersion == 'No') {
aed_map.put("mobile_version", false);
} else {
aed_map.put("mobile_version", true);
}
} if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.LoginTo != null) {
aed_map.put("login_to", ctx.aws.cloudtrail.flattened.additional_eventdata.LoginTo);
} if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.MFAUsed != null) {
if (ctx.aws.cloudtrail.flattened.additional_eventdata.MFAUsed == 'No') {
aed_map.put("mfa_used", false);
} else {
aed_map.put("mfa_used", true);
}
} if (aed_map.size() > 0) {
Map cl_map = new HashMap();
cl_map.put("additional_eventdata", aed_map);
ctx.aws.cloudtrail.put("console_login", cl_map);
}"""
}
},
{
"script" : {
"lang" : "painless",
"ignore_failure" : true,
"params" : {
"DetachGroupPolicy" : {
"category" : [
"iam"
],
"type" : [
"group",
"change"
]
},
"ConsoleLogin" : {
"category" : [
"authentication"
],
"type" : [
"info"
]
},
"EnableMFADevice" : {
"category" : [
"iam"
],
"type" : [
"user",
"change"
]
},
"ListUserPolicies" : {
"category" : [
"iam"
],
"type" : [
"user",
"info"
]
},
"PutUserPolicy" : {
"category" : [
"iam"
],
"type" : [
"user",
"change"
]
},
"UpdateGroup" : {
"category" : [
"iam"
],
"type" : [
"group",
"change"
]
},
"CreateGroup" : {
"category" : [
"iam"
],
"type" : [
"group",
"creation"
]
},
"CreateKeyPair" : {
"category" : [
"iam"
],
"type" : [
"admin",
"creation"
]
},
"DeleteUser" : {
"type" : [
"user",
"deletion"
],
"category" : [
"iam"
]
},
"ListGroupPolicies" : {
"category" : [
"iam"
],
"type" : [
"group",
"info"
]
},
"ListAttachedUserPolicies" : {
"category" : [
"iam"
],
"type" : [
"user",
"info"
]
},
"CreateBucket" : {
"category" : [
"file"
],
"type" : [
"creation"
]
},
"DeleteBucket" : {
"category" : [
"file"
],
"type" : [
"deletion"
]
},
"UntagUser" : {
"category" : [
"iam"
],
"type" : [
"user",
"change"
]
},
"PutUserPermissionsBoundary" : {
"category" : [
"iam"
],
"type" : [
"user",
"change"
]
},
"DeleteUserPermissionsBoundary" : {
"category" : [
"iam"
],
"type" : [
"user",
"change"
]
},
"AssumeRole" : {
"category" : [
"authentication"
],
"type" : [
"info"
]
},
"CreateAccessKey" : {
"type" : [
"user",
"change"
],
"category" : [
"iam"
]
},
"DeleteVirtualMFADevice" : {
"category" : [
"iam"
],
"type" : [
"user",
"change"
]
},
"GetGroupPolicy" : {
"category" : [
"iam"
],
"type" : [
"group",
"info"
]
},
"AttachGroupPolicy" : {
"category" : [
"iam"
],
"type" : [
"group",
"change"
]
},
"CreateVirtualMFADevice" : {
"category" : [
"iam"
],
"type" : [
"user",
"change"
]
},
"SetSecurityTokenServicePreferences" : {
"category" : [
"iam"
],
"type" : [
"admin",
"change"
]
},
"GetUserPolicy" : {
"category" : [
"iam"
],
"type" : [
"user",
"info"
]
},
"ListUsers" : {
"category" : [
"iam"
],
"type" : [
"user",
"info"
]
},
"CreateUser" : {
"type" : [
"user",
"creation"
],
"category" : [
"iam"
]
},
"DeleteGroupPolicy" : {
"category" : [
"iam"
],
"type" : [
"group",
"change"
]
},
"DeleteGroup" : {
"type" : [
"group",
"deletion"
],
"category" : [
"iam"
]
},
"ChangePassword" : {
"category" : [
"iam"
],
"type" : [
"user",
"change"
]
},
"TagUser" : {
"category" : [
"iam"
],
"type" : [
"user",
"change"
]
},
"UpdateRole" : {
"type" : [
"admin",
"change"
],
"category" : [
"iam"
]
},
"UpdateSSHPublicKey" : {
"category" : [
"iam"
],
"type" : [
"user",
"change"
]
},
"DeleteSSHPublicKey" : {
"type" : [
"user",
"change"
],
"category" : [
"iam"
]
},
"RemoveUserFromGroup" : {
"category" : [
"iam"
],
"type" : [
"group",
"change"
]
},
"GetUser" : {
"category" : [
"iam"
],
"type" : [
"user",
"info"
]
},
"UpdateAccessKey" : {
"type" : [
"user",
"change"
],
"category" : [
"iam"
]
},
"GetGroup" : {
"category" : [
"iam"
],
"type" : [
"group",
"info"
]
},
"PutGroupPolicy" : {
"type" : [
"group",
"change"
],
"category" : [
"iam"
]
},
"AttachUserPolicy" : {
"category" : [
"iam"
],
"type" : [
"user",
"change"
]
},
"UpdateUser" : {
"type" : [
"user",
"change"
],
"category" : [
"iam"
]
},
"SetDefaultPolicyVersion" : {
"category" : [
"iam"
],
"type" : [
"admin",
"change"
]
},
"UpdateLoginProfile" : {
"category" : [
"iam"
],
"type" : [
"user",
"change"
]
},
"DeleteUserPolicy" : {
"category" : [
"iam"
],
"type" : [
"user",
"change"
]
},
"ListGroups" : {
"category" : [
"iam"
],
"type" : [
"group",
"info"
]
},
"ListUserTags" : {
"category" : [
"iam"
],
"type" : [
"user",
"info"
]
},
"DeactivateMFADevice" : {
"category" : [
"iam"
],
"type" : [
"user",
"change"
]
},
"DetachUserPolicy" : {
"category" : [
"iam"
],
"type" : [
"user",
"change"
]
},
"DeleteAccessKey" : {
"type" : [
"user",
"change"
],
"category" : [
"iam"
]
},
"ListAttachedGroupPolicies" : {
"type" : [
"group",
"info"
],
"category" : [
"iam"
]
},
"AddUserToGroup" : {
"category" : [
"iam"
],
"type" : [
"group",
"change"
]
},
"UpdateAccountPasswordPolicy" : {
"category" : [
"iam"
],
"type" : [
"admin",
"change"
]
},
"ListGroupsForUser" : {
"type" : [
"user",
"info"
],
"category" : [
"iam"
]
}
},
"source" : """ctx.event.kind = 'event'; ctx.event.type = 'info';
if (ctx.aws.cloudtrail.error_code != null || ctx.aws.cloudtrail.error_message != null) {
ctx.event.outcome = 'failure'
} else {
ctx.event.outcome = 'success'
}
if (ctx?.event?.action == null) {
return;
}
if (ctx.event.action == 'ConsoleLogin' && ctx?.aws?.cloudtrail?.flattened?.response_elements.ConsoleLogin != null) {
ctx.event.outcome = Processors.lowercase(ctx.aws.cloudtrail.flattened.response_elements.ConsoleLogin);
}
def hm = new HashMap(params.get(ctx.event.action)); hm.forEach((k, v) -> ctx.event[k] = v);"""
}
},
{
"remove" : {
"ignore_missing" : true,
"field" : [
"json"
]
}
}
],
"on_failure" : [
{
"set" : {
"field" : "error.message",
"value" : "{{ _ingest.on_failure_message }}"
}
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment