ingest_pipeline_in_use

gistfile1.txt

PUT _ingest/pipeline/signals-azure-activitylogs-pipeline
{
    "description" : "Pipeline for parsing azure activity logs.",
    "processors" : [
      {
        "rename" : {
          "field" : "azure",
          "target_field" : "azure-eventhub",
          "ignore_missing" : true
        }
      },
      {
        "script" : {
          "source" : "ctx.message = ctx.message.replace(params.empty_field_name, '')",
          "params" : {
            "empty_field_name" : "\"\":\"\","
          },
          "ignore_failure" : true
        }
      },
      {
        "json" : {
          "field" : "message",
          "target_field" : "azure.activitylogs"
        }
      },
      {
        "date" : {
          "ignore_failure" : true,
          "formats" : [
            "ISO8601"
          ],
          "field" : "azure.activitylogs.time",
          "target_field" : "@timestamp"
        }
      },
      {
        "remove" : {
          "field" : [
            "message",
            "azure.activitylogs.time"
          ],
          "ignore_missing" : true
        }
      },
      {
        "rename" : {
          "target_field" : "azure.resource_id",
          "ignore_missing" : true,
          "field" : "azure.activitylogs.resourceId"
        }
      },
      {
        "rename" : {
          "field" : "azure.activitylogs.callerIpAddress",
          "target_field" : "source.ip",
          "ignore_missing" : true
        }
      },
      {
        "rename" : {
          "ignore_missing" : true,
          "field" : "azure.activitylogs.level",
          "target_field" : "log.level"
        }
      },
      {
        "rename" : {
          "field" : "azure.activitylogs.durationMs",
          "target_field" : "event.duration",
          "ignore_missing" : true
        }
      },
      {
        "script" : {
          "ignore_failure" : true,
          "lang" : "painless",
          "source" : "if (ctx.event.duration!= null) {ctx.event.duration = ctx.event.duration * params.param_nano;}",
          "params" : {
            "param_nano" : 1000000
          }
        }
      },
      {
        "rename" : {
          "ignore_missing" : true,
          "field" : "azure.activitylogs.location",
          "target_field" : "geo.name"
        }
      },
      {
        "script" : {
          "lang" : "painless",
          "source" : """if (ctx?.azure?.activitylogs?.properties?.eventCategory != null) {
  ctx.azure.activitylogs.event_category = ctx.azure.activitylogs.properties.eventCategory;
} else if (ctx?.azure?.activitylogs?.properties?.policies != null)  {
  ctx.azure.activitylogs.event_category = 'Policy';
} else {
  ctx.azure.activitylogs.event_category = 'Administrative';
}""",
          "ignore_failure" : true
        }
      },
      {
        "rename" : {
          "field" : "azure.activitylogs.resultType",
          "target_field" : "azure.activitylogs.result_type",
          "ignore_missing" : true
        }
      },
      {
        "convert" : {
          "field" : "azure.activitylogs.result_type",
          "target_field" : "event.outcome",
          "type" : "string",
          "if" : "ctx?.azure?.activitylogs?.resultType != null && ctx.azure.activitylogs.resultType instanceof String && (ctx.azure.activitylogs.resultType.toLowerCase() == 'success' || ctx.azure.activitylogs.resultType.toLowerCase() == 'failure')"
        }
      },
      {
        "rename" : {
          "field" : "azure.activitylogs.operationName",
          "target_field" : "azure.activitylogs.operation_name",
          "ignore_missing" : true
        }
      },
      {
        "convert" : {
          "type" : "string",
          "ignore_missing" : true,
          "field" : "azure.activitylogs.operation_name",
          "target_field" : "event.action"
        }
      },
      {
        "rename" : {
          "target_field" : "azure.activitylogs.result_signature",
          "ignore_missing" : true,
          "field" : "azure.activitylogs.resultSignature"
        }
      },
      {
        "rename" : {
          "field" : "azure.activitylogs.identity.authorization.evidence.roleAssignmentScope",
          "target_field" : "azure.activitylogs.identity.authorization.evidence.role_assignment_scope",
          "ignore_missing" : true
        }
      },
      {
        "rename" : {
          "field" : "azure.activitylogs.identity.authorization.evidence.roleDefinitionId",
          "target_field" : "azure.activitylogs.identity.authorization.evidence.role_definition_id",
          "ignore_missing" : true
        }
      },
      {
        "rename" : {
          "field" : "azure.activitylogs.identity.authorization.evidence.roleAssignmentId",
          "target_field" : "azure.activitylogs.identity.authorization.evidence.role_assignment_id",
          "ignore_missing" : true
        }
      },
      {
        "rename" : {
          "field" : "azure.activitylogs.identity.authorization.evidence.principalId",
          "target_field" : "azure.activitylogs.identity.authorization.evidence.principal_id",
          "ignore_missing" : true
        }
      },
      {
        "rename" : {
          "field" : "azure.activitylogs.identity.authorization.evidence.principalType",
          "target_field" : "azure.activitylogs.identity.authorization.evidence.principal_type",
          "ignore_missing" : true
        }
      },
      {
        "rename" : {
          "field" : "azure.activitylogs.correlationId",
          "target_field" : "azure.correlation_id",
          "ignore_missing" : true
        }
      },
      {
        "rename" : {
          "field" : "azure.activitylogs.properties.serviceRequestId",
          "target_field" : "azure.activitylogs.properties.service_request_id",
          "ignore_missing" : true
        }
      },
      {
        "rename" : {
          "field" : "azure.activitylogs.properties.statusMessage",
          "target_field" : "message",
          "ignore_missing" : true
        }
      },
      {
        "rename" : {
          "field" : "azure.activitylogs.properties.statusCode",
          "target_field" : "azure.activitylogs.properties.status_code",
          "ignore_missing" : true
        }
      },
      {
        "geoip" : {
          "field" : "source.ip",
          "target_field" : "source.geo",
          "properties" : [
            "continent_name",
            "country_name",
            "country_iso_code",
            "region_iso_code",
            "region_name",
            "city_name",
            "location"
          ],
          "ignore_missing" : true
        }
      },
      {
        "rename" : {
          "field" : "azure.activitylogs.identity.claims.name",
          "target_field" : "azure.activitylogs.identity.claims_initiated_by_user.fullname",
          "ignore_missing" : true
        }
      },
      {
        "script" : {
          "ignore_failure" : true,
          "lang" : "painless",
          "source" : """if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname'] != null) {
  ctx.azure.activitylogs.identity.claims_initiated_by_user.surname = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname'];
}"""
        }
      },
      {
        "script" : {
          "source" : """if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name'] != null) {
  ctx.azure.activitylogs.identity.claims_initiated_by_user.name = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name'];
}""",
          "ignore_failure" : true,
          "lang" : "painless"
        }
      },
      {
        "script" : {
          "ignore_failure" : true,
          "lang" : "painless",
          "source" : """if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname'] != null) {
  ctx.azure.activitylogs.identity.claims_initiated_by_user.givenname = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname'];
}"""
        }
      },
      {
        "set" : {
          "ignore_failure" : true,
          "if" : "ctx.azure.activitylogs.identity!= null && ctx.azure.activitylogs.identity.claims_initiated_by_user != null && ctx.azure.activitylogs.identity.claims_initiated_by_user.name != null",
          "field" : "azure.activitylogs.identity.claims_initiated_by_user.schema",
          "value" : "http://schemas.xmlsoap.org/ws/2005/05/identity/claims"
        }
      },
      {
        "script" : {
          "ignore_failure" : true,
          "lang" : "painless",
          "source" : """if (ctx.azure.activitylogs.identity.claims != null) {
  ctx.temp_claims = new HashMap();
  for (String key : ctx.azure.activitylogs.identity.claims.keySet()) {
    ctx.temp_claims[key.replace('.', '_')] = ctx.azure.activitylogs.identity.claims.get(key);
  }
  ctx.azure.activitylogs.identity.claims = ctx.temp_claims; ctx.remove('temp_claims');
}"""
        }
      },
      {
        "script" : {
          "source" : """if (ctx?.azure?.activitylogs?.category == null) {
  return;
} def hm = new HashMap(params.get(ctx.azure.activitylogs.category.toLowerCase())); hm.forEach((k, v) -> ctx.event[k] = v);""",
          "lang" : "painless",
          "ignore_failure" : true,
          "params" : {
            "read" : {
              "type" : [
                "access"
              ]
            },
            "delete" : {
              "type" : [
                "deletion"
              ]
            },
            "action" : {
              "type" : [
                "change"
              ]
            },
            "write" : {
              "type" : [
                "change"
              ]
            }
          }
        }
      },
      {
        "geoip" : {
          "field" : "source.ip",
          "target_field" : "source.geo"
        }
      },
      {
        "geoip" : {
          "properties" : [
            "asn",
            "organization_name"
          ],
          "ignore_missing" : true,
          "database_file" : "GeoLite2-ASN.mmdb",
          "field" : "source.ip",
          "target_field" : "source.as"
        }
      },
      {
        "rename" : {
          "ignore_missing" : true,
          "field" : "source.as.asn",
          "target_field" : "source.as.number"
        }
      },
      {
        "rename" : {
          "field" : "source.as.organization_name",
          "target_field" : "source.as.organization.name",
          "ignore_missing" : true
        }
      },
      {
        "grok" : {
          "ignore_missing" : true,
          "field" : "azure.activitylogs.identity.claims_initiated_by_user.name",
          "patterns" : [
            "%{USERNAME:user.name}@%{HOSTNAME:user.domain}"
          ]
        }
      },
      {
        "convert" : {
          "field" : "azure.activitylogs.identity.claims_initiated_by_user.fullname",
          "target_field" : "user.full_name",
          "type" : "string",
          "ignore_missing" : true
        }
      },
      {
        "set" : {
          "field" : "event.kind",
          "value" : "event"
        }
      },
      {
        "pipeline" : {
          "name" : "signals-azure-activitylogs-shared-pipeline"
        }
      }
    ],
    "on_failure" : [
      {
        "set" : {
          "field" : "error.message",
          "value" : "{{ _ingest.on_failure_message }}"
        }
      }
    ]
  }