Skip to content

Instantly share code, notes, and snippets.

@bvader
Created August 6, 2021 22:45
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save bvader/fc316eabf2ac6ca2b62e3ed705b39da2 to your computer and use it in GitHub Desktop.
Save bvader/fc316eabf2ac6ca2b62e3ed705b39da2 to your computer and use it in GitHub Desktop.
filebeat-panos-pipeline-7.10.0.json
DELETE _ingest/pipeline/filebeat-7.10.0-panw-panos-pipeline
PUT _ingest/pipeline/filebeat-7.10.0-panw-panos-pipeline
{
"description": "Pipeline for Palo Alto Networks PAN-OS Logs",
"processors": [
{
"set": {
"field": "event.ingested",
"value": "{{_ingest.timestamp}}"
}
},
{
"rename": {
"field": "message",
"target_field": "log.original"
}
},
{
"date": {
"if": "ctx.event.timezone == null",
"field": "_temp_.generated_time",
"formats": [
"yyyy/MM/dd HH:mm:ss"
],
"on_failure": [
{
"append": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}
]
}
},
{
"date": {
"field": "_temp_.generated_time",
"formats": [
"yyyy/MM/dd HH:mm:ss"
],
"timezone": "{{ event.timezone }}",
"on_failure": [
{
"append": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}
],
"if": "ctx.event.timezone != null"
}
},
{
"date": {
"formats": [
"yyyy/MM/dd HH:mm:ss"
],
"on_failure": [
{
"append": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}
],
"if": "ctx.event.timezone == null && ctx.event.created != null ",
"field": "event.created",
"target_field": "event.created"
}
},
{
"date": {
"timezone": "{{ event.timezone }}",
"on_failure": [
{
"append": {
"value": "{{ _ingest.on_failure_message }}",
"field": "error.message"
}
}
],
"if": "ctx.event.timezone != null && ctx.event.created != null ",
"field": "event.created",
"target_field": "event.created",
"formats": [
"yyyy/MM/dd HH:mm:ss"
]
}
},
{
"date": {
"field": "event.start",
"target_field": "event.start",
"formats": [
"yyyy/MM/dd HH:mm:ss"
],
"on_failure": [
{
"append": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}
],
"if": "ctx.event.timezone == null && ctx.event.start != null"
}
},
{
"date": {
"if": "ctx.event.timezone != null && ctx.event.start != null",
"field": "event.start",
"target_field": "event.start",
"timezone": "{{ event.timezone }}",
"formats": [
"yyyy/MM/dd HH:mm:ss"
],
"on_failure": [
{
"append": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}
]
}
},
{
"convert": {
"field": "client.bytes",
"type": "long",
"ignore_missing": true
}
},
{
"convert": {
"ignore_missing": true,
"field": "client.packets",
"type": "long"
}
},
{
"convert": {
"type": "long",
"ignore_missing": true,
"field": "client.port"
}
},
{
"convert": {
"ignore_missing": true,
"field": "server.bytes",
"type": "long"
}
},
{
"convert": {
"type": "long",
"ignore_missing": true,
"field": "server.packets"
}
},
{
"convert": {
"ignore_missing": true,
"field": "server.port",
"type": "long"
}
},
{
"convert": {
"type": "long",
"ignore_missing": true,
"field": "source.bytes"
}
},
{
"convert": {
"type": "long",
"ignore_missing": true,
"field": "source.packets"
}
},
{
"convert": {
"type": "long",
"ignore_missing": true,
"field": "source.port"
}
},
{
"convert": {
"type": "long",
"ignore_missing": true,
"field": "destination.bytes"
}
},
{
"convert": {
"field": "destination.packets",
"type": "long",
"ignore_missing": true
}
},
{
"convert": {
"field": "destination.port",
"type": "long",
"ignore_missing": true
}
},
{
"convert": {
"type": "long",
"ignore_missing": true,
"field": "network.bytes"
}
},
{
"convert": {
"ignore_missing": true,
"field": "network.packets",
"type": "long"
}
},
{
"convert": {
"type": "long",
"ignore_missing": true,
"field": "event.duration"
}
},
{
"convert": {
"type": "long",
"ignore_missing": true,
"field": "_temp_.labels"
}
},
{
"convert": {
"type": "long",
"ignore_missing": true,
"field": "panw.panos.sequence_number"
}
},
{
"convert": {
"ignore_missing": true,
"field": "source.nat.port",
"type": "long"
}
},
{
"convert": {
"ignore_missing": true,
"field": "destination.nat.port",
"type": "long"
}
},
{
"convert": {
"ignore_missing": true,
"field": "client.nat.port",
"type": "long"
}
},
{
"convert": {
"type": "long",
"ignore_missing": true,
"field": "server.nat.port"
}
},
{
"remove": {
"if": "ctx?.panw?.panos?.network?.pcap_id == \"0\"",
"field": [
"panw.panos.network.pcap_id"
]
}
},
{
"script": {
"if": "ctx?._temp_?.labels != null && ctx._temp_.labels != 0",
"params": {
"nat_translated": 4194304,
"http_proxy": 262144,
"symmetric_return": 2048,
"ssl_decrypted": 16777216,
"container_page": 32768,
"pcap_included": 2147483648,
"url_filter_denied": 8388608,
"x_forwarded_for": 524288,
"temporary_match": 8192,
"ipv6_session": 33554432,
"captive_portal": 2097152
},
"source": """def labels = ctx?.labels; if (labels == null) {
labels = new HashMap();
ctx['labels'] = labels;
} long value = ctx._temp_.labels; for (entry in params.entrySet()) {
if ((value & entry.getValue()) != 0) {
labels[entry.getKey()] = true;
}
}
""",
"lang": "painless"
}
},
{
"script": {
"lang": "painless",
"if": "ctx?.event?.duration != null",
"params": {
"NANOS_IN_A_SECOND": 1000000000
},
"source": """long nanos = ctx['event']['duration'] * params.NANOS_IN_A_SECOND; ctx['event']['duration'] = nanos; def start = ctx.event?.start; if (start != null) {
ctx.event['end'] = ZonedDateTime.parse(start).plusNanos(nanos);
}
"""
}
},
{
"set": {
"field": "network.direction",
"value": "inbound",
"if": "ctx?.panw?.panos?.type == \"TRAFFIC\" && ctx?.panw?.panos?.source?.zone == \"untrust\" && ctx?.panw?.panos?.destination?.zone == \"trust\""
}
},
{
"set": {
"field": "network.direction",
"value": "outbound",
"if": "ctx?.panw?.panos?.type == \"TRAFFIC\" && ctx?.panw?.panos?.source?.zone == \"trust\" && ctx?.panw?.panos?.destination?.zone == \"untrust\""
}
},
{
"set": {
"field": "network.direction",
"value": "internal",
"if": "ctx?.panw?.panos?.type == \"TRAFFIC\" && ctx?.panw?.panos?.source?.zone == \"trust\" && ctx?.panw?.panos?.destination?.zone == \"trust\""
}
},
{
"set": {
"value": "external",
"if": "ctx?.panw?.panos?.type == \"TRAFFIC\" && ctx?.panw?.panos?.source?.zone == \"untrust\" && ctx?.panw?.panos?.destination?.zone == \"untrust\"",
"field": "network.direction"
}
},
{
"set": {
"value": "unknown",
"if": """ctx?.panw?.panos?.type == "TRAFFIC" && ((ctx?.panw?.panos?.source?.zone != "trust" && ctx?.panw?.panos?.source?.zone != "untrust") || (ctx?.panw?.panos?.destination?.zone != "trust" && ctx?.panw?.panos?.destination?.zone != "untrust"))""",
"field": "network.direction"
}
},
{
"set": {
"field": "network.direction",
"value": "inbound",
"if": """ctx?.panw?.panos?.type == "THREAT" && (ctx?._temp_?.direction == "0" || ctx?._temp_?.direction == "client-to-server")"""
}
},
{
"set": {
"field": "network.direction",
"value": "outbound",
"if": """ctx?.panw?.panos?.type == "THREAT" && (ctx?._temp_?.direction == "1" || ctx?._temp_?.direction == "server-to-client")"""
}
},
{
"set": {
"field": "network.direction",
"value": "unknown",
"if": """ctx?.panw?.panos?.type == "THREAT" && ctx?.network?.direction == null"""
}
},
{
"set": {
"value": "ipv4",
"if": """ctx?.panw?.panos?.type == "TRAFFIC" && ctx?.labels?.ipv6_session == null""",
"field": "network.type"
}
},
{
"set": {
"field": "network.type",
"value": "ipv6",
"if": """ctx?.panw?.panos?.type == "TRAFFIC" && ctx?.labels?.ipv6_session != null"""
}
},
{
"set": {
"value": "event",
"if": "ctx?.panw?.panos?.type == \"TRAFFIC\"",
"field": "event.kind"
}
},
{
"append": {
"field": "event.category",
"value": [
"network_traffic",
"network"
],
"if": "ctx?.panw?.panos?.type == \"TRAFFIC\""
}
},
{
"set": {
"field": "event.kind",
"value": "alert",
"if": "ctx?.panw?.panos?.type == \"THREAT\""
}
},
{
"append": {
"field": "event.category",
"value": [
"security_threat",
"intrusion_detection",
"network"
],
"if": "ctx?.panw?.panos?.type == \"THREAT\""
}
},
{
"append": {
"field": "event.type",
"value": "allowed",
"if": "ctx?.panw?.panos?.action != null && ['alert', 'allow', 'continue'].contains(ctx.panw.panos.action)"
}
},
{
"append": {
"field": "event.type",
"value": "denied",
"if": "ctx?.panw?.panos?.action != null && ['deny', 'drop', 'reset-client', 'reset-server', 'reset-both', 'block-url', 'block-ip', 'random-drop', 'sinkhole', 'block'].contains(ctx.panw.panos.action)"
}
},
{
"set": {
"field": "event.outcome",
"value": "success"
}
},
{
"set": {
"value": "flow_started",
"if": "ctx?.panw?.panos?.sub_type == \"start\"",
"field": "event.action"
}
},
{
"append": {
"value": [
"start",
"connection"
],
"if": "ctx?.panw?.panos?.sub_type == \"start\"",
"field": "event.type"
}
},
{
"set": {
"if": "ctx?.panw?.panos?.sub_type == \"end\"",
"field": "event.action",
"value": "flow_terminated"
}
},
{
"append": {
"field": "event.type",
"value": [
"end",
"connection"
],
"if": "ctx?.panw?.panos?.sub_type == \"end\""
}
},
{
"set": {
"if": "ctx?.panw?.panos?.sub_type == \"drop\"",
"field": "event.action",
"value": "flow_dropped"
}
},
{
"append": {
"value": [
"denied",
"connection"
],
"if": "ctx?.panw?.panos?.sub_type == \"drop\"",
"field": "event.type"
}
},
{
"set": {
"field": "event.action",
"value": "flow_denied",
"if": "ctx?.panw?.panos?.sub_type == \"deny\""
}
},
{
"append": {
"value": [
"denied",
"connection"
],
"if": "ctx?.panw?.panos?.sub_type == \"deny\"",
"field": "event.type"
}
},
{
"set": {
"if": "ctx?.panw?.panos?.sub_type == \"data\"",
"field": "event.action",
"value": "data_match"
}
},
{
"set": {
"field": "event.action",
"value": "file_match",
"if": "ctx?.panw?.panos?.sub_type == \"file\""
}
},
{
"set": {
"field": "event.action",
"value": "flood_detected",
"if": "ctx?.panw?.panos?.sub_type == \"flood\""
}
},
{
"set": {
"if": "ctx?.panw?.panos?.sub_type == \"packet\"",
"field": "event.action",
"value": "packet_attack"
}
},
{
"set": {
"field": "event.action",
"value": "scan_detected",
"if": "ctx?.panw?.panos?.sub_type == \"scan\""
}
},
{
"set": {
"field": "event.action",
"value": "spyware_detected",
"if": "ctx?.panw?.panos?.sub_type == \"spyware\""
}
},
{
"set": {
"if": "ctx?.panw?.panos?.sub_type == \"url\"",
"field": "event.action",
"value": "url_filtering"
}
},
{
"set": {
"if": "ctx?.panw?.panos?.sub_type == \"virus\"",
"field": "event.action",
"value": "virus_detected"
}
},
{
"set": {
"value": "exploit_detected",
"if": "ctx?.panw?.panos?.sub_type == \"vulnerability\"",
"field": "event.action"
}
},
{
"set": {
"field": "event.action",
"value": "wildfire_verdict",
"if": "ctx?.panw?.panos?.sub_type == \"wildfire\""
}
},
{
"set": {
"field": "event.action",
"value": "wildfire_virus_detected",
"if": "ctx?.panw?.panos?.sub_type == \"wildfire-virus\""
}
},
{
"set": {
"value": 1,
"field": "event.severity",
"if": "ctx.log.level == \"critical\""
}
},
{
"set": {
"field": "event.severity",
"if": "ctx.log.level == \"high\"",
"value": 2
}
},
{
"set": {
"value": 3,
"field": "event.severity",
"if": "ctx.log.level == \"medium\""
}
},
{
"set": {
"if": "ctx.log.level == \"low\"",
"value": 4,
"field": "event.severity"
}
},
{
"set": {
"field": "event.severity",
"if": "ctx.log.level == \"informational\"",
"value": 5
}
},
{
"set": {
"field": "panw.panos.action",
"value": "drop-icmp",
"if": "ctx?.panw?.panos?.action == \"drop icmp\" || ctx?.panw?.panos?.action == \"drop ICMP\""
}
},
{
"set": {
"field": "panw.panos.action",
"value": "reset-both",
"if": "ctx?.panw?.panos?.action == \"reset both\""
}
},
{
"set": {
"field": "panw.panos.action",
"value": "reset-client",
"if": "ctx?.panw?.panos?.action == \"reset client\""
}
},
{
"set": {
"field": "panw.panos.action",
"value": "reset-server",
"if": "ctx?.panw?.panos?.action == \"reset server\""
}
},
{
"append": {
"value": [
"{{source.ip}}"
],
"if": "ctx?.source?.ip != null",
"field": "related.ip"
}
},
{
"append": {
"field": "related.ip",
"value": [
"{{destination.ip}}"
],
"if": "ctx?.destination?.ip != null"
}
},
{
"append": {
"value": [
"{{source.nat.ip}}"
],
"if": "ctx?.source?.nat?.ip != null",
"field": "related.ip"
}
},
{
"append": {
"if": "ctx?.destination?.nat?.ip != null",
"field": "related.ip",
"value": [
"{{destination.nat.ip}}"
]
}
},
{
"geoip": {
"if": "ctx?.source?.ip != null",
"field": "source.ip",
"target_field": "source.geo"
}
},
{
"geoip": {
"if": "ctx?.destination?.ip != null",
"field": "destination.ip",
"target_field": "destination.geo"
}
},
{
"geoip": {
"ignore_missing": true,
"database_file": "GeoLite2-ASN.mmdb",
"field": "source.ip",
"target_field": "source.as",
"properties": [
"asn",
"organization_name"
]
}
},
{
"geoip": {
"ignore_missing": true,
"database_file": "GeoLite2-ASN.mmdb",
"field": "destination.ip",
"target_field": "destination.as",
"properties": [
"asn",
"organization_name"
]
}
},
{
"rename": {
"field": "source.as.asn",
"target_field": "source.as.number",
"ignore_missing": true
}
},
{
"rename": {
"target_field": "source.as.organization.name",
"ignore_missing": true,
"field": "source.as.organization_name"
}
},
{
"rename": {
"target_field": "destination.as.number",
"ignore_missing": true,
"field": "destination.as.asn"
}
},
{
"rename": {
"field": "destination.as.organization_name",
"target_field": "destination.as.organization.name",
"ignore_missing": true
}
},
{
"rename": {
"if": "ctx.source?.geo?.name == null",
"field": "_temp_.srcloc",
"target_field": "source.geo.name",
"ignore_missing": true
}
},
{
"rename": {
"target_field": "destination.geo.name",
"ignore_missing": true,
"if": "ctx.destination?.geo?.name == null",
"field": "_temp_.dstloc"
}
},
{
"append": {
"if": "ctx?.panw?.panos?.network?.nat?.community_id != null && ctx.panw.panos.network.nat.community_id != ctx?.network?.community_id",
"field": "network.community_id",
"value": [
"{{panw.panos.network.nat.community_id}}"
]
}
},
{
"grok": {
"field": "panw.panos.threat.name",
"ignore_failure": true,
"patterns": [
"""%{GREEDYDATA:panw.panos.threat.name}\(\s*%{GREEDYDATA:panw.panos.threat.id}\s*\)"""
],
"if": "ctx?.panw?.panos?.threat?.name != null"
}
},
{
"set": {
"field": "panw.panos.threat.name",
"value": "URL-filtering",
"if": "ctx?.panw?.panos?.threat?.id == \"9999\""
}
},
{
"set": {
"field": "rule.name",
"value": "{{panw.panos.ruleset}}",
"ignore_empty_value": true
}
},
{
"append": {
"value": "{{client.user.name}}",
"if": "ctx?.client?.user?.name != null",
"field": "related.user"
}
},
{
"append": {
"value": "{{source.user.name}}",
"if": "ctx?.source?.user?.name != null",
"field": "related.user"
}
},
{
"append": {
"field": "related.user",
"value": "{{server.user.name}}",
"if": "ctx?.server?.user?.name != null"
}
},
{
"append": {
"if": "ctx?.destination?.user?.name != null",
"field": "related.user",
"value": "{{destination.user.name}}"
}
},
{
"append": {
"value": "{{panw.panos.file.hash}}",
"if": "ctx?.panw?.panos?.file?.hash != null",
"field": "related.hash"
}
},
{
"append": {
"field": "related.hosts",
"value": "{{observer.hostname}}",
"if": "ctx?.observer?.hostname != null && ctx.observer?.hostname != ''",
"allow_duplicates": false
}
},
{
"remove": {
"field": [
"_temp_"
],
"ignore_missing": true
}
},
{
"remove": {
"field": [
"source.nat.ip",
"source.nat.port",
"client.nat.ip",
"client.nat.port"
],
"if": """ctx?.source?.nat?.ip == "0.0.0.0" && ctx?.source?.nat?.port == 0"""
}
},
{
"remove": {
"if": """ctx?.destination?.nat?.ip == "0.0.0.0" && ctx?.destination?.nat?.port == 0""",
"field": [
"destination.nat.ip",
"destination.nat.port",
"server.nat.ip",
"server.nat.port"
]
}
}
],
"on_failure": [
{
"set": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
},
{
"remove": {
"field": [
"_temp_"
],
"ignore_missing": true
}
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment