Created
April 28, 2021 20:55
-
-
Save ankitdevnalkar/6fbc7fcfe5c60a4d45f0be118f6ae04a to your computer and use it in GitHub Desktop.
ingest pipeline for Azure activity log
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
PUT _ingest/pipeline/my-azure-activity-log | |
{ | |
"my-azure-activitylogs-azure-shared-pipeline" : { | |
"description" : "Pipeline for parsing azure activity logs.", | |
"processors" : [ | |
{ | |
"set" : { | |
"field" : "cloud.provider", | |
"value" : "azure" | |
} | |
}, | |
{ | |
"grok" : { | |
"ignore_failure" : true, | |
"field" : "azure.resource_id", | |
"patterns" : [ | |
"/SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule}" | |
], | |
"pattern_definitions" : { | |
"GROUPID" : ".+", | |
"PROVIDERNAME" : ".+", | |
"NAMESPACE" : ".+", | |
"RULE" : ".+", | |
"SUBID" : """(\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}""" | |
} | |
} | |
}, | |
{ | |
"grok" : { | |
"field" : "azure.resource_id", | |
"patterns" : [ | |
"/SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}" | |
], | |
"pattern_definitions" : { | |
"SUBID" : """(\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}""", | |
"GROUPID" : ".+", | |
"PROVIDERNAME" : """([A-Z])\w+.([A-Z])\w+/([A-Z])\w+.""", | |
"NAME" : "((?!AUTHORIZATIONRULES).)*$" | |
}, | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"grok" : { | |
"field" : "azure.resource_id", | |
"patterns" : [ | |
"/providers/%{PROVIDER:azure.resource.provider}" | |
], | |
"pattern_definitions" : { | |
"PROVIDER" : ".+" | |
}, | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "azure.resource_id", | |
"target_field" : "azure.resource.id", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"lowercase" : { | |
"ignore_missing" : true, | |
"field" : "event.outcome" | |
} | |
} | |
], | |
"on_failure" : [ | |
{ | |
"set" : { | |
"field" : "error.message", | |
"value" : "{{ _ingest.on_failure_message }}" | |
} | |
} | |
] | |
}, | |
"my-azure-activitylogs-pipeline" : { | |
"description" : "Pipeline for parsing azure activity logs.", | |
"processors" : [ | |
{ | |
"rename" : { | |
"field" : "azure", | |
"target_field" : "azure-eventhub", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"script" : { | |
"source" : "ctx.message = ctx.message.replace(params.empty_field_name, '')", | |
"params" : { | |
"empty_field_name" : "\"\":\"\"," | |
}, | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"json" : { | |
"field" : "message", | |
"target_field" : "azure.activitylogs" | |
} | |
}, | |
{ | |
"date" : { | |
"ignore_failure" : true, | |
"formats" : [ | |
"ISO8601" | |
], | |
"field" : "azure.activitylogs.time", | |
"target_field" : "@timestamp" | |
} | |
}, | |
{ | |
"remove" : { | |
"field" : [ | |
"message", | |
"azure.activitylogs.time" | |
], | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"target_field" : "azure.resource_id", | |
"ignore_missing" : true, | |
"field" : "azure.activitylogs.resourceId" | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "azure.activitylogs.callerIpAddress", | |
"target_field" : "source.ip", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"ignore_missing" : true, | |
"field" : "azure.activitylogs.level", | |
"target_field" : "log.level" | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "azure.activitylogs.durationMs", | |
"target_field" : "event.duration", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"script" : { | |
"ignore_failure" : true, | |
"lang" : "painless", | |
"source" : "if (ctx.event.duration!= null) {ctx.event.duration = ctx.event.duration * params.param_nano;}", | |
"params" : { | |
"param_nano" : 1000000 | |
} | |
} | |
}, | |
{ | |
"rename" : { | |
"ignore_missing" : true, | |
"field" : "azure.activitylogs.location", | |
"target_field" : "geo.name" | |
} | |
}, | |
{ | |
"script" : { | |
"lang" : "painless", | |
"source" : """if (ctx?.azure?.activitylogs?.properties?.eventCategory != null) { | |
ctx.azure.activitylogs.event_category = ctx.azure.activitylogs.properties.eventCategory; | |
} else if (ctx?.azure?.activitylogs?.properties?.policies != null) { | |
ctx.azure.activitylogs.event_category = 'Policy'; | |
} else { | |
ctx.azure.activitylogs.event_category = 'Administrative'; | |
}""", | |
"ignore_failure" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "azure.activitylogs.resultType", | |
"target_field" : "azure.activitylogs.result_type", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"convert" : { | |
"field" : "azure.activitylogs.result_type", | |
"target_field" : "event.outcome", | |
"type" : "string", | |
"if" : "ctx?.azure?.activitylogs?.resultType != null && ctx.azure.activitylogs.resultType instanceof String && (ctx.azure.activitylogs.resultType.toLowerCase() == 'success' || ctx.azure.activitylogs.resultType.toLowerCase() == 'failure')" | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "azure.activitylogs.operationName", | |
"target_field" : "azure.activitylogs.operation_name", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"convert" : { | |
"type" : "string", | |
"ignore_missing" : true, | |
"field" : "azure.activitylogs.operation_name", | |
"target_field" : "event.action" | |
} | |
}, | |
{ | |
"rename" : { | |
"target_field" : "azure.activitylogs.result_signature", | |
"ignore_missing" : true, | |
"field" : "azure.activitylogs.resultSignature" | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "azure.activitylogs.identity.authorization.evidence.roleAssignmentScope", | |
"target_field" : "azure.activitylogs.identity.authorization.evidence.role_assignment_scope", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "azure.activitylogs.identity.authorization.evidence.roleDefinitionId", | |
"target_field" : "azure.activitylogs.identity.authorization.evidence.role_definition_id", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "azure.activitylogs.identity.authorization.evidence.roleAssignmentId", | |
"target_field" : "azure.activitylogs.identity.authorization.evidence.role_assignment_id", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "azure.activitylogs.identity.authorization.evidence.principalId", | |
"target_field" : "azure.activitylogs.identity.authorization.evidence.principal_id", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "azure.activitylogs.identity.authorization.evidence.principalType", | |
"target_field" : "azure.activitylogs.identity.authorization.evidence.principal_type", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "azure.activitylogs.correlationId", | |
"target_field" : "azure.correlation_id", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "azure.activitylogs.properties.serviceRequestId", | |
"target_field" : "azure.activitylogs.properties.service_request_id", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "azure.activitylogs.properties.statusMessage", | |
"target_field" : "message", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "azure.activitylogs.properties.statusCode", | |
"target_field" : "azure.activitylogs.properties.status_code", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"geoip" : { | |
"field" : "source.ip", | |
"target_field" : "geo", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"geoip": { | |
"properties": { | |
"city_name": { | |
"type": "keyword" | |
}, | |
"continent_code": { | |
"type": "keyword" | |
}, | |
"country_code2": { | |
"type": "keyword" | |
}, | |
"country_code3": { | |
"type": "keyword" | |
}, | |
"country_name": { | |
"type": "keyword" | |
}, | |
"ip": { | |
"type": "keyword" | |
}, | |
"location": { | |
"type": "geo_point" | |
}, | |
"postal_code": { | |
"type": "keyword" | |
}, | |
"region_code": { | |
"type": "keyword" | |
}, | |
"region_name": { | |
"type": "keyword" | |
}, | |
"timezone": { | |
"type": "keyword" | |
} | |
} | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "azure.activitylogs.identity.claims.name", | |
"target_field" : "azure.activitylogs.identity.claims_initiated_by_user.fullname", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"script" : { | |
"ignore_failure" : true, | |
"lang" : "painless", | |
"source" : """if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname'] != null) { | |
ctx.azure.activitylogs.identity.claims_initiated_by_user.surname = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname']; | |
}""" | |
} | |
}, | |
{ | |
"script" : { | |
"source" : """if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name'] != null) { | |
ctx.azure.activitylogs.identity.claims_initiated_by_user.name = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name']; | |
}""", | |
"ignore_failure" : true, | |
"lang" : "painless" | |
} | |
}, | |
{ | |
"script" : { | |
"ignore_failure" : true, | |
"lang" : "painless", | |
"source" : """if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname'] != null) { | |
ctx.azure.activitylogs.identity.claims_initiated_by_user.givenname = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname']; | |
}""" | |
} | |
}, | |
{ | |
"set" : { | |
"ignore_failure" : true, | |
"if" : "ctx.azure.activitylogs.identity!= null && ctx.azure.activitylogs.identity.claims_initiated_by_user != null && ctx.azure.activitylogs.identity.claims_initiated_by_user.name != null", | |
"field" : "azure.activitylogs.identity.claims_initiated_by_user.schema", | |
"value" : "http://schemas.xmlsoap.org/ws/2005/05/identity/claims" | |
} | |
}, | |
{ | |
"script" : { | |
"ignore_failure" : true, | |
"lang" : "painless", | |
"source" : """if (ctx.azure.activitylogs.identity.claims != null) { | |
ctx.temp_claims = new HashMap(); | |
for (String key : ctx.azure.activitylogs.identity.claims.keySet()) { | |
ctx.temp_claims[key.replace('.', '_')] = ctx.azure.activitylogs.identity.claims.get(key); | |
} | |
ctx.azure.activitylogs.identity.claims = ctx.temp_claims; ctx.remove('temp_claims'); | |
}""" | |
} | |
}, | |
{ | |
"script" : { | |
"source" : """if (ctx?.azure?.activitylogs?.category == null) { | |
return; | |
} def hm = new HashMap(params.get(ctx.azure.activitylogs.category.toLowerCase())); hm.forEach((k, v) -> ctx.event[k] = v);""", | |
"lang" : "painless", | |
"ignore_failure" : true, | |
"params" : { | |
"read" : { | |
"type" : [ | |
"access" | |
] | |
}, | |
"delete" : { | |
"type" : [ | |
"deletion" | |
] | |
}, | |
"action" : { | |
"type" : [ | |
"change" | |
] | |
}, | |
"write" : { | |
"type" : [ | |
"change" | |
] | |
} | |
} | |
} | |
}, | |
{ | |
"geoip" : { | |
"field" : "source.ip", | |
"target_field" : "source.geo" | |
} | |
}, | |
{ | |
"geoip" : { | |
"properties" : [ | |
"asn", | |
"organization_name" | |
], | |
"ignore_missing" : true, | |
"database_file" : "GeoLite2-ASN.mmdb", | |
"field" : "source.ip", | |
"target_field" : "source.as" | |
} | |
}, | |
{ | |
"rename" : { | |
"ignore_missing" : true, | |
"field" : "source.as.asn", | |
"target_field" : "source.as.number" | |
} | |
}, | |
{ | |
"rename" : { | |
"field" : "source.as.organization_name", | |
"target_field" : "source.as.organization.name", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"grok" : { | |
"ignore_missing" : true, | |
"field" : "azure.activitylogs.identity.claims_initiated_by_user.name", | |
"patterns" : [ | |
"%{USERNAME:user.name}@%{HOSTNAME:user.domain}" | |
] | |
} | |
}, | |
{ | |
"convert" : { | |
"field" : "azure.activitylogs.identity.claims_initiated_by_user.fullname", | |
"target_field" : "user.full_name", | |
"type" : "string", | |
"ignore_missing" : true | |
} | |
}, | |
{ | |
"set" : { | |
"field" : "event.kind", | |
"value" : "event" | |
} | |
}, | |
{ | |
"pipeline" : { | |
"name" : "filebeat-7.9.1-azure-activitylogs-azure-shared-pipeline" | |
} | |
} | |
], | |
"on_failure" : [ | |
{ | |
"set" : { | |
"field" : "error.message", | |
"value" : "{{ _ingest.on_failure_message }}" | |
} | |
} | |
] | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment