Created
August 6, 2021 22:45
-
-
Save bvader/fc316eabf2ac6ca2b62e3ed705b39da2 to your computer and use it in GitHub Desktop.
filebeat-panos-pipeline-7.10.0.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
DELETE _ingest/pipeline/filebeat-7.10.0-panw-panos-pipeline | |
PUT _ingest/pipeline/filebeat-7.10.0-panw-panos-pipeline | |
{ | |
"description": "Pipeline for Palo Alto Networks PAN-OS Logs", | |
"processors": [ | |
{ | |
"set": { | |
"field": "event.ingested", | |
"value": "{{_ingest.timestamp}}" | |
} | |
}, | |
{ | |
"rename": { | |
"field": "message", | |
"target_field": "log.original" | |
} | |
}, | |
{ | |
"date": { | |
"if": "ctx.event.timezone == null", | |
"field": "_temp_.generated_time", | |
"formats": [ | |
"yyyy/MM/dd HH:mm:ss" | |
], | |
"on_failure": [ | |
{ | |
"append": { | |
"field": "error.message", | |
"value": "{{ _ingest.on_failure_message }}" | |
} | |
} | |
] | |
} | |
}, | |
{ | |
"date": { | |
"field": "_temp_.generated_time", | |
"formats": [ | |
"yyyy/MM/dd HH:mm:ss" | |
], | |
"timezone": "{{ event.timezone }}", | |
"on_failure": [ | |
{ | |
"append": { | |
"field": "error.message", | |
"value": "{{ _ingest.on_failure_message }}" | |
} | |
} | |
], | |
"if": "ctx.event.timezone != null" | |
} | |
}, | |
{ | |
"date": { | |
"formats": [ | |
"yyyy/MM/dd HH:mm:ss" | |
], | |
"on_failure": [ | |
{ | |
"append": { | |
"field": "error.message", | |
"value": "{{ _ingest.on_failure_message }}" | |
} | |
} | |
], | |
"if": "ctx.event.timezone == null && ctx.event.created != null ", | |
"field": "event.created", | |
"target_field": "event.created" | |
} | |
}, | |
{ | |
"date": { | |
"timezone": "{{ event.timezone }}", | |
"on_failure": [ | |
{ | |
"append": { | |
"value": "{{ _ingest.on_failure_message }}", | |
"field": "error.message" | |
} | |
} | |
], | |
"if": "ctx.event.timezone != null && ctx.event.created != null ", | |
"field": "event.created", | |
"target_field": "event.created", | |
"formats": [ | |
"yyyy/MM/dd HH:mm:ss" | |
] | |
} | |
}, | |
{ | |
"date": { | |
"field": "event.start", | |
"target_field": "event.start", | |
"formats": [ | |
"yyyy/MM/dd HH:mm:ss" | |
], | |
"on_failure": [ | |
{ | |
"append": { | |
"field": "error.message", | |
"value": "{{ _ingest.on_failure_message }}" | |
} | |
} | |
], | |
"if": "ctx.event.timezone == null && ctx.event.start != null" | |
} | |
}, | |
{ | |
"date": { | |
"if": "ctx.event.timezone != null && ctx.event.start != null", | |
"field": "event.start", | |
"target_field": "event.start", | |
"timezone": "{{ event.timezone }}", | |
"formats": [ | |
"yyyy/MM/dd HH:mm:ss" | |
], | |
"on_failure": [ | |
{ | |
"append": { | |
"field": "error.message", | |
"value": "{{ _ingest.on_failure_message }}" | |
} | |
} | |
] | |
} | |
}, | |
{ | |
"convert": { | |
"field": "client.bytes", | |
"type": "long", | |
"ignore_missing": true | |
} | |
}, | |
{ | |
"convert": { | |
"ignore_missing": true, | |
"field": "client.packets", | |
"type": "long" | |
} | |
}, | |
{ | |
"convert": { | |
"type": "long", | |
"ignore_missing": true, | |
"field": "client.port" | |
} | |
}, | |
{ | |
"convert": { | |
"ignore_missing": true, | |
"field": "server.bytes", | |
"type": "long" | |
} | |
}, | |
{ | |
"convert": { | |
"type": "long", | |
"ignore_missing": true, | |
"field": "server.packets" | |
} | |
}, | |
{ | |
"convert": { | |
"ignore_missing": true, | |
"field": "server.port", | |
"type": "long" | |
} | |
}, | |
{ | |
"convert": { | |
"type": "long", | |
"ignore_missing": true, | |
"field": "source.bytes" | |
} | |
}, | |
{ | |
"convert": { | |
"type": "long", | |
"ignore_missing": true, | |
"field": "source.packets" | |
} | |
}, | |
{ | |
"convert": { | |
"type": "long", | |
"ignore_missing": true, | |
"field": "source.port" | |
} | |
}, | |
{ | |
"convert": { | |
"type": "long", | |
"ignore_missing": true, | |
"field": "destination.bytes" | |
} | |
}, | |
{ | |
"convert": { | |
"field": "destination.packets", | |
"type": "long", | |
"ignore_missing": true | |
} | |
}, | |
{ | |
"convert": { | |
"field": "destination.port", | |
"type": "long", | |
"ignore_missing": true | |
} | |
}, | |
{ | |
"convert": { | |
"type": "long", | |
"ignore_missing": true, | |
"field": "network.bytes" | |
} | |
}, | |
{ | |
"convert": { | |
"ignore_missing": true, | |
"field": "network.packets", | |
"type": "long" | |
} | |
}, | |
{ | |
"convert": { | |
"type": "long", | |
"ignore_missing": true, | |
"field": "event.duration" | |
} | |
}, | |
{ | |
"convert": { | |
"type": "long", | |
"ignore_missing": true, | |
"field": "_temp_.labels" | |
} | |
}, | |
{ | |
"convert": { | |
"type": "long", | |
"ignore_missing": true, | |
"field": "panw.panos.sequence_number" | |
} | |
}, | |
{ | |
"convert": { | |
"ignore_missing": true, | |
"field": "source.nat.port", | |
"type": "long" | |
} | |
}, | |
{ | |
"convert": { | |
"ignore_missing": true, | |
"field": "destination.nat.port", | |
"type": "long" | |
} | |
}, | |
{ | |
"convert": { | |
"ignore_missing": true, | |
"field": "client.nat.port", | |
"type": "long" | |
} | |
}, | |
{ | |
"convert": { | |
"type": "long", | |
"ignore_missing": true, | |
"field": "server.nat.port" | |
} | |
}, | |
{ | |
"remove": { | |
"if": "ctx?.panw?.panos?.network?.pcap_id == \"0\"", | |
"field": [ | |
"panw.panos.network.pcap_id" | |
] | |
} | |
}, | |
{ | |
"script": { | |
"if": "ctx?._temp_?.labels != null && ctx._temp_.labels != 0", | |
"params": { | |
"nat_translated": 4194304, | |
"http_proxy": 262144, | |
"symmetric_return": 2048, | |
"ssl_decrypted": 16777216, | |
"container_page": 32768, | |
"pcap_included": 2147483648, | |
"url_filter_denied": 8388608, | |
"x_forwarded_for": 524288, | |
"temporary_match": 8192, | |
"ipv6_session": 33554432, | |
"captive_portal": 2097152 | |
}, | |
"source": """def labels = ctx?.labels; if (labels == null) { | |
labels = new HashMap(); | |
ctx['labels'] = labels; | |
} long value = ctx._temp_.labels; for (entry in params.entrySet()) { | |
if ((value & entry.getValue()) != 0) { | |
labels[entry.getKey()] = true; | |
} | |
} | |
""", | |
"lang": "painless" | |
} | |
}, | |
{ | |
"script": { | |
"lang": "painless", | |
"if": "ctx?.event?.duration != null", | |
"params": { | |
"NANOS_IN_A_SECOND": 1000000000 | |
}, | |
"source": """long nanos = ctx['event']['duration'] * params.NANOS_IN_A_SECOND; ctx['event']['duration'] = nanos; def start = ctx.event?.start; if (start != null) { | |
ctx.event['end'] = ZonedDateTime.parse(start).plusNanos(nanos); | |
} | |
""" | |
} | |
}, | |
{ | |
"set": { | |
"field": "network.direction", | |
"value": "inbound", | |
"if": "ctx?.panw?.panos?.type == \"TRAFFIC\" && ctx?.panw?.panos?.source?.zone == \"untrust\" && ctx?.panw?.panos?.destination?.zone == \"trust\"" | |
} | |
}, | |
{ | |
"set": { | |
"field": "network.direction", | |
"value": "outbound", | |
"if": "ctx?.panw?.panos?.type == \"TRAFFIC\" && ctx?.panw?.panos?.source?.zone == \"trust\" && ctx?.panw?.panos?.destination?.zone == \"untrust\"" | |
} | |
}, | |
{ | |
"set": { | |
"field": "network.direction", | |
"value": "internal", | |
"if": "ctx?.panw?.panos?.type == \"TRAFFIC\" && ctx?.panw?.panos?.source?.zone == \"trust\" && ctx?.panw?.panos?.destination?.zone == \"trust\"" | |
} | |
}, | |
{ | |
"set": { | |
"value": "external", | |
"if": "ctx?.panw?.panos?.type == \"TRAFFIC\" && ctx?.panw?.panos?.source?.zone == \"untrust\" && ctx?.panw?.panos?.destination?.zone == \"untrust\"", | |
"field": "network.direction" | |
} | |
}, | |
{ | |
"set": { | |
"value": "unknown", | |
"if": """ctx?.panw?.panos?.type == "TRAFFIC" && ((ctx?.panw?.panos?.source?.zone != "trust" && ctx?.panw?.panos?.source?.zone != "untrust") || (ctx?.panw?.panos?.destination?.zone != "trust" && ctx?.panw?.panos?.destination?.zone != "untrust"))""", | |
"field": "network.direction" | |
} | |
}, | |
{ | |
"set": { | |
"field": "network.direction", | |
"value": "inbound", | |
"if": """ctx?.panw?.panos?.type == "THREAT" && (ctx?._temp_?.direction == "0" || ctx?._temp_?.direction == "client-to-server")""" | |
} | |
}, | |
{ | |
"set": { | |
"field": "network.direction", | |
"value": "outbound", | |
"if": """ctx?.panw?.panos?.type == "THREAT" && (ctx?._temp_?.direction == "1" || ctx?._temp_?.direction == "server-to-client")""" | |
} | |
}, | |
{ | |
"set": { | |
"field": "network.direction", | |
"value": "unknown", | |
"if": """ctx?.panw?.panos?.type == "THREAT" && ctx?.network?.direction == null""" | |
} | |
}, | |
{ | |
"set": { | |
"value": "ipv4", | |
"if": """ctx?.panw?.panos?.type == "TRAFFIC" && ctx?.labels?.ipv6_session == null""", | |
"field": "network.type" | |
} | |
}, | |
{ | |
"set": { | |
"field": "network.type", | |
"value": "ipv6", | |
"if": """ctx?.panw?.panos?.type == "TRAFFIC" && ctx?.labels?.ipv6_session != null""" | |
} | |
}, | |
{ | |
"set": { | |
"value": "event", | |
"if": "ctx?.panw?.panos?.type == \"TRAFFIC\"", | |
"field": "event.kind" | |
} | |
}, | |
{ | |
"append": { | |
"field": "event.category", | |
"value": [ | |
"network_traffic", | |
"network" | |
], | |
"if": "ctx?.panw?.panos?.type == \"TRAFFIC\"" | |
} | |
}, | |
{ | |
"set": { | |
"field": "event.kind", | |
"value": "alert", | |
"if": "ctx?.panw?.panos?.type == \"THREAT\"" | |
} | |
}, | |
{ | |
"append": { | |
"field": "event.category", | |
"value": [ | |
"security_threat", | |
"intrusion_detection", | |
"network" | |
], | |
"if": "ctx?.panw?.panos?.type == \"THREAT\"" | |
} | |
}, | |
{ | |
"append": { | |
"field": "event.type", | |
"value": "allowed", | |
"if": "ctx?.panw?.panos?.action != null && ['alert', 'allow', 'continue'].contains(ctx.panw.panos.action)" | |
} | |
}, | |
{ | |
"append": { | |
"field": "event.type", | |
"value": "denied", | |
"if": "ctx?.panw?.panos?.action != null && ['deny', 'drop', 'reset-client', 'reset-server', 'reset-both', 'block-url', 'block-ip', 'random-drop', 'sinkhole', 'block'].contains(ctx.panw.panos.action)" | |
} | |
}, | |
{ | |
"set": { | |
"field": "event.outcome", | |
"value": "success" | |
} | |
}, | |
{ | |
"set": { | |
"value": "flow_started", | |
"if": "ctx?.panw?.panos?.sub_type == \"start\"", | |
"field": "event.action" | |
} | |
}, | |
{ | |
"append": { | |
"value": [ | |
"start", | |
"connection" | |
], | |
"if": "ctx?.panw?.panos?.sub_type == \"start\"", | |
"field": "event.type" | |
} | |
}, | |
{ | |
"set": { | |
"if": "ctx?.panw?.panos?.sub_type == \"end\"", | |
"field": "event.action", | |
"value": "flow_terminated" | |
} | |
}, | |
{ | |
"append": { | |
"field": "event.type", | |
"value": [ | |
"end", | |
"connection" | |
], | |
"if": "ctx?.panw?.panos?.sub_type == \"end\"" | |
} | |
}, | |
{ | |
"set": { | |
"if": "ctx?.panw?.panos?.sub_type == \"drop\"", | |
"field": "event.action", | |
"value": "flow_dropped" | |
} | |
}, | |
{ | |
"append": { | |
"value": [ | |
"denied", | |
"connection" | |
], | |
"if": "ctx?.panw?.panos?.sub_type == \"drop\"", | |
"field": "event.type" | |
} | |
}, | |
{ | |
"set": { | |
"field": "event.action", | |
"value": "flow_denied", | |
"if": "ctx?.panw?.panos?.sub_type == \"deny\"" | |
} | |
}, | |
{ | |
"append": { | |
"value": [ | |
"denied", | |
"connection" | |
], | |
"if": "ctx?.panw?.panos?.sub_type == \"deny\"", | |
"field": "event.type" | |
} | |
}, | |
{ | |
"set": { | |
"if": "ctx?.panw?.panos?.sub_type == \"data\"", | |
"field": "event.action", | |
"value": "data_match" | |
} | |
}, | |
{ | |
"set": { | |
"field": "event.action", | |
"value": "file_match", | |
"if": "ctx?.panw?.panos?.sub_type == \"file\"" | |
} | |
}, | |
{ | |
"set": { | |
"field": "event.action", | |
"value": "flood_detected", | |
"if": "ctx?.panw?.panos?.sub_type == \"flood\"" | |
} | |
}, | |
{ | |
"set": { | |
"if": "ctx?.panw?.panos?.sub_type == \"packet\"", | |
"field": "event.action", | |
"value": "packet_attack" | |
} | |
}, | |
{ | |
"set": { | |
"field": "event.action", | |
"value": "scan_detected", | |
"if": "ctx?.panw?.panos?.sub_type == \"scan\"" | |
} | |
}, | |
{ | |
"set": { | |
"field": "event.action", | |
"value": "spyware_detected", | |
"if": "ctx?.panw?.panos?.sub_type == \"spyware\"" | |
} | |
}, | |
{ | |
"set": { | |
"if": "ctx?.panw?.panos?.sub_type == \"url\"", | |
"field": "event.action", | |
"value": "url_filtering" | |
} | |
}, | |
{ | |
"set": { | |
"if": "ctx?.panw?.panos?.sub_type == \"virus\"", | |
"field": "event.action", | |
"value": "virus_detected" | |
} | |
}, | |
{ | |
"set": { | |
"value": "exploit_detected", | |
"if": "ctx?.panw?.panos?.sub_type == \"vulnerability\"", | |
"field": "event.action" | |
} | |
}, | |
{ | |
"set": { | |
"field": "event.action", | |
"value": "wildfire_verdict", | |
"if": "ctx?.panw?.panos?.sub_type == \"wildfire\"" | |
} | |
}, | |
{ | |
"set": { | |
"field": "event.action", | |
"value": "wildfire_virus_detected", | |
"if": "ctx?.panw?.panos?.sub_type == \"wildfire-virus\"" | |
} | |
}, | |
{ | |
"set": { | |
"value": 1, | |
"field": "event.severity", | |
"if": "ctx.log.level == \"critical\"" | |
} | |
}, | |
{ | |
"set": { | |
"field": "event.severity", | |
"if": "ctx.log.level == \"high\"", | |
"value": 2 | |
} | |
}, | |
{ | |
"set": { | |
"value": 3, | |
"field": "event.severity", | |
"if": "ctx.log.level == \"medium\"" | |
} | |
}, | |
{ | |
"set": { | |
"if": "ctx.log.level == \"low\"", | |
"value": 4, | |
"field": "event.severity" | |
} | |
}, | |
{ | |
"set": { | |
"field": "event.severity", | |
"if": "ctx.log.level == \"informational\"", | |
"value": 5 | |
} | |
}, | |
{ | |
"set": { | |
"field": "panw.panos.action", | |
"value": "drop-icmp", | |
"if": "ctx?.panw?.panos?.action == \"drop icmp\" || ctx?.panw?.panos?.action == \"drop ICMP\"" | |
} | |
}, | |
{ | |
"set": { | |
"field": "panw.panos.action", | |
"value": "reset-both", | |
"if": "ctx?.panw?.panos?.action == \"reset both\"" | |
} | |
}, | |
{ | |
"set": { | |
"field": "panw.panos.action", | |
"value": "reset-client", | |
"if": "ctx?.panw?.panos?.action == \"reset client\"" | |
} | |
}, | |
{ | |
"set": { | |
"field": "panw.panos.action", | |
"value": "reset-server", | |
"if": "ctx?.panw?.panos?.action == \"reset server\"" | |
} | |
}, | |
{ | |
"append": { | |
"value": [ | |
"{{source.ip}}" | |
], | |
"if": "ctx?.source?.ip != null", | |
"field": "related.ip" | |
} | |
}, | |
{ | |
"append": { | |
"field": "related.ip", | |
"value": [ | |
"{{destination.ip}}" | |
], | |
"if": "ctx?.destination?.ip != null" | |
} | |
}, | |
{ | |
"append": { | |
"value": [ | |
"{{source.nat.ip}}" | |
], | |
"if": "ctx?.source?.nat?.ip != null", | |
"field": "related.ip" | |
} | |
}, | |
{ | |
"append": { | |
"if": "ctx?.destination?.nat?.ip != null", | |
"field": "related.ip", | |
"value": [ | |
"{{destination.nat.ip}}" | |
] | |
} | |
}, | |
{ | |
"geoip": { | |
"if": "ctx?.source?.ip != null", | |
"field": "source.ip", | |
"target_field": "source.geo" | |
} | |
}, | |
{ | |
"geoip": { | |
"if": "ctx?.destination?.ip != null", | |
"field": "destination.ip", | |
"target_field": "destination.geo" | |
} | |
}, | |
{ | |
"geoip": { | |
"ignore_missing": true, | |
"database_file": "GeoLite2-ASN.mmdb", | |
"field": "source.ip", | |
"target_field": "source.as", | |
"properties": [ | |
"asn", | |
"organization_name" | |
] | |
} | |
}, | |
{ | |
"geoip": { | |
"ignore_missing": true, | |
"database_file": "GeoLite2-ASN.mmdb", | |
"field": "destination.ip", | |
"target_field": "destination.as", | |
"properties": [ | |
"asn", | |
"organization_name" | |
] | |
} | |
}, | |
{ | |
"rename": { | |
"field": "source.as.asn", | |
"target_field": "source.as.number", | |
"ignore_missing": true | |
} | |
}, | |
{ | |
"rename": { | |
"target_field": "source.as.organization.name", | |
"ignore_missing": true, | |
"field": "source.as.organization_name" | |
} | |
}, | |
{ | |
"rename": { | |
"target_field": "destination.as.number", | |
"ignore_missing": true, | |
"field": "destination.as.asn" | |
} | |
}, | |
{ | |
"rename": { | |
"field": "destination.as.organization_name", | |
"target_field": "destination.as.organization.name", | |
"ignore_missing": true | |
} | |
}, | |
{ | |
"rename": { | |
"if": "ctx.source?.geo?.name == null", | |
"field": "_temp_.srcloc", | |
"target_field": "source.geo.name", | |
"ignore_missing": true | |
} | |
}, | |
{ | |
"rename": { | |
"target_field": "destination.geo.name", | |
"ignore_missing": true, | |
"if": "ctx.destination?.geo?.name == null", | |
"field": "_temp_.dstloc" | |
} | |
}, | |
{ | |
"append": { | |
"if": "ctx?.panw?.panos?.network?.nat?.community_id != null && ctx.panw.panos.network.nat.community_id != ctx?.network?.community_id", | |
"field": "network.community_id", | |
"value": [ | |
"{{panw.panos.network.nat.community_id}}" | |
] | |
} | |
}, | |
{ | |
"grok": { | |
"field": "panw.panos.threat.name", | |
"ignore_failure": true, | |
"patterns": [ | |
"""%{GREEDYDATA:panw.panos.threat.name}\(\s*%{GREEDYDATA:panw.panos.threat.id}\s*\)""" | |
], | |
"if": "ctx?.panw?.panos?.threat?.name != null" | |
} | |
}, | |
{ | |
"set": { | |
"field": "panw.panos.threat.name", | |
"value": "URL-filtering", | |
"if": "ctx?.panw?.panos?.threat?.id == \"9999\"" | |
} | |
}, | |
{ | |
"set": { | |
"field": "rule.name", | |
"value": "{{panw.panos.ruleset}}", | |
"ignore_empty_value": true | |
} | |
}, | |
{ | |
"append": { | |
"value": "{{client.user.name}}", | |
"if": "ctx?.client?.user?.name != null", | |
"field": "related.user" | |
} | |
}, | |
{ | |
"append": { | |
"value": "{{source.user.name}}", | |
"if": "ctx?.source?.user?.name != null", | |
"field": "related.user" | |
} | |
}, | |
{ | |
"append": { | |
"field": "related.user", | |
"value": "{{server.user.name}}", | |
"if": "ctx?.server?.user?.name != null" | |
} | |
}, | |
{ | |
"append": { | |
"if": "ctx?.destination?.user?.name != null", | |
"field": "related.user", | |
"value": "{{destination.user.name}}" | |
} | |
}, | |
{ | |
"append": { | |
"value": "{{panw.panos.file.hash}}", | |
"if": "ctx?.panw?.panos?.file?.hash != null", | |
"field": "related.hash" | |
} | |
}, | |
{ | |
"append": { | |
"field": "related.hosts", | |
"value": "{{observer.hostname}}", | |
"if": "ctx?.observer?.hostname != null && ctx.observer?.hostname != ''", | |
"allow_duplicates": false | |
} | |
}, | |
{ | |
"remove": { | |
"field": [ | |
"_temp_" | |
], | |
"ignore_missing": true | |
} | |
}, | |
{ | |
"remove": { | |
"field": [ | |
"source.nat.ip", | |
"source.nat.port", | |
"client.nat.ip", | |
"client.nat.port" | |
], | |
"if": """ctx?.source?.nat?.ip == "0.0.0.0" && ctx?.source?.nat?.port == 0""" | |
} | |
}, | |
{ | |
"remove": { | |
"if": """ctx?.destination?.nat?.ip == "0.0.0.0" && ctx?.destination?.nat?.port == 0""", | |
"field": [ | |
"destination.nat.ip", | |
"destination.nat.port", | |
"server.nat.ip", | |
"server.nat.port" | |
] | |
} | |
} | |
], | |
"on_failure": [ | |
{ | |
"set": { | |
"field": "error.message", | |
"value": "{{ _ingest.on_failure_message }}" | |
} | |
}, | |
{ | |
"remove": { | |
"field": [ | |
"_temp_" | |
], | |
"ignore_missing": true | |
} | |
} | |
] | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment