Skip to content

Instantly share code, notes, and snippets.

@andrewkroh
Last active December 15, 2020 14:12
Show Gist options
  • Save andrewkroh/4642be6718ac76bda8e0c0df6aaad1f5 to your computer and use it in GitHub Desktop.
Save andrewkroh/4642be6718ac76bda8e0c0df6aaad1f5 to your computer and use it in GitHub Desktop.
Citrix Netscaler Elasticsearch Ingest Node Pipeline
{
"description": "Pipeline for parsing Citrix Netscaler logs",
"processors": [
{
"script": {
"description": "set event.original",
"lang": "painless",
"source": "def event = ctx.event;\nif (event == null) {\n event = [:];\n ctx['event'] = event;\n}\nevent['original'] = ctx.message;\n"
}
},
{
"dissect": {
"description": "trim exabeam headers",
"if": "ctx?.message.contains(\"KAFKA_CONNECT_SYSLOG\")",
"field": "message",
"pattern": "%{} - - - - %{message}"
}
},
{
"set": {
"field": "event.ingested",
"value": "{{_ingest.timestamp}}"
}
},
{
"set": {
"field": "observer.vendor",
"value": "Citrix"
}
},
{
"set": {
"field": "observer.product",
"value": "Netscaler"
}
},
{
"set": {
"field": "observer.type",
"value": "firewall"
}
},
{
"set": {
"field": "event.module",
"value": "citrix"
}
},
{
"set": {
"field": "event.dataset",
"value": "citrix.netscaler"
}
},
{
"grok": {
"description": "grok syslog header",
"field": "message",
"patterns": [
"<%{NONNEGINT:log.syslog.facility.code:long}> %{NOTSPACE:timestamp} ?%{WORD:event.timezone}? %{HOSTNAME:host.name} %{HOSTNAME} : %{GREEDYDATA:message}"
]
}
},
{
"grok": {
"description": "grok citrix header",
"field": "message",
"patterns": [
"%{WORD} %{WORD:citrix.netscaler.feature} %{WORD:citrix.netscaler.message_type} %{NUMBER} %{NUMBER} :%{SPACE}+%{GREEDYDATA:message}"
]
}
},
{
"lowercase": {
"field": "citrix.netscaler.feature"
}
},
{
"lowercase": {
"field": "citrix.netscaler.message_type"
}
},
{
"date": {
"field": "timestamp",
"target_field": "@timestamp",
"formats": [
"MM/dd/yyyy:HH:mm:ss"
],
"timezone": "{{event.timezone}}",
"if": "ctx?.event?.timezone != null"
}
},
{
"date": {
"field": "timestamp",
"target_field": "@timestamp",
"formats": [
"MM/dd/yyyy:HH:mm:ss"
],
"if": "ctx?.event?.timezone == null"
}
},
{
"remove": {
"field": [
"timestamp"
]
}
},
{
"set": {
"description": "set event.code",
"field": "event.code",
"value": "{{citrix.netscaler.feature}}-{{citrix.netscaler.message_type}}"
}
},
{
"set": {
"description": "set generic event.action",
"field": "event.action",
"value": "{{citrix.netscaler.feature}} {{citrix.netscaler.message_type}}"
}
},
{
"script": {
"description": "add event category and type",
"lang": "painless",
"tag": "ecs_categorize",
"params": {
"api-cmd_executed": {
"category": [
"process"
],
"type": [
"start"
],
"action": "command executed"
},
"aaa-login_failed": {
"category": [
"authentication"
],
"type": [
"denied"
],
"outcome": "failure",
"action": "login failed"
},
"sslvpn-icastart": {
"category": [
"process",
"network"
],
"type": [
"start"
],
"action": "ICA application started"
},
"sslvpn-httprequest": {
"category": [
"network",
"web"
],
"type": [
"protocol",
"start",
"info"
],
"action": "http request"
},
"sslvpn-login": {
"category": [
"authentication"
],
"type": [
"start"
]
},
"sslvpn-logout": {
"category": [
"authentication"
],
"type": [
"end"
]
}
},
"source": "ctx.event.kind = 'event';\nctx.event.type = ['info'];\nctx.event.outcome = 'success';\n\ndef metadata = params.get(ctx.event.code);\nif (metadata == null) {\n return;\n}\nmetadata.forEach((k, v) -> ctx.event[k] = v);\n"
}
},
{
"dissect": {
"description": "api-cmd_executed",
"if": "ctx?.event?.code == \"api-cmd_executed\"",
"field": "message",
"pattern": "User %{user.name} - Remote_ip %{source.ip} - Command \"%{process.command_line}\" - Status \"%{citrix.netscaler.status}\""
}
},
{
"dissect": {
"description": "sslvpn-httprequest",
"if": "ctx?.event?.code == \"sslvpn-httprequest\"",
"field": "message",
"pattern": "Context %{user.name}@%{source.ip} - SessionId: %{citrix.netscaler.session_id}- %{destination.domain} User %{} : Group(s) %{citrix.netscaler.group} : Vserver %{citrix.netscaler.vserver}:%{destination.port} - %{} : SSO is %{} : %{http.request.method} %{url.original} - -"
}
},
{
"set": {
"description": "sslvpn-httprequest network.protocol http",
"if": "ctx?.event?.code == \"sslvpn-httprequest\"",
"field": "network.protocol",
"value": "http"
}
},
{
"grok": {
"field": "url.original",
"ignore_failure": true,
"patterns": [
"%{URIPATH:url.path}(?:%{URIPARAM:url.query})?"
]
}
},
{
"dissect": {
"description": "sslvpn-login",
"if": "ctx?.event?.code == \"sslvpn-login\"",
"field": "message",
"pattern": "Context %{user.name}@%{source.ip} - SessionId: %{citrix.netscaler.session_id}- User %{} - Client_ip %{} - Nat_ip \"%{}\" - Vserver %{citrix.netscaler.vserver}:%{destination.port} - Browser_type \"%{user_agent.original}\" - SSLVPN_client_type %{citrix.netscaler.vpn_client_type} - Group(s) \"%{citrix.netscaler.group}\""
}
},
{
"dissect": {
"description": "aaa-login_failed",
"if": "ctx?.event?.code == \"aaa-login_failed\"",
"field": "message",
"pattern": "User %{user.name} - Client_ip %{source.ip} - Failure_reason \"%{citrix.netscaler.error_message}\" - Browser %{user_agent.original}"
}
},
{
"dissect": {
"description": "cli-cmd_execute",
"if": "ctx?.event?.code == \"cli-cmd_executed\"",
"field": "message",
"pattern": "User %{user.name} - Remote_ip %{source.ip} - Command \"%{process.command_line}\" - Status \"%{citrix.netscaler.status}\""
}
},
{
"dissect": {
"description": "sslvpn-icastart",
"if": "ctx?.event?.code == \"sslvpn-icastart\"",
"field": "message",
"pattern": "Source %{source.ip}:%{source.port} - Destination %{destination.ip}:%{destination.port} - username:domainname %{user.name}: - applicationName %{process.name} - startTime \"%{event.start}\" - connectionId %{citrix.netscaler.connection_id}"
}
},
{
"dissect": {
"description": "sslvpn-logout-with-context-header",
"if": "ctx?.event?.code == \"sslvpn-logout\" && ctx?.message.startsWith(\"Context\")",
"field": "message",
"pattern": "Context %{user.name}@%{source.ip} - SessionId: %{citrix.netscaler.session_id}- User %{} - %{}"
}
},
{
"grok": {
"description": "sslvpn-logout-without-context-header",
"if": "ctx?.event?.code == \"sslvpn-logout\" && !ctx?.message.startsWith(\"Context\")",
"field": "message",
"patterns": [
"User %{NOTSPACE:user.name}?%{SPACE}?- "
]
}
},
{
"dissect": {
"description": "sslvpn-logout-trailer",
"if": "ctx?.event?.code == \"sslvpn-logout\"",
"field": "message",
"pattern": "%{} - Client_ip %{source.ip} - Nat_ip \"%{}\" - Vserver %{citrix.netscaler.vserver}:%{destination.port} - Start_time \"%{event.start}\" - End_time \"%{event.end}\" - Duration %{} - Http_resources_accessed %{citrix.netscaler.http_resources_accessed} - NonHttp_services_accessed %{citrix.netscaler.non_http_resources_accessed} - Total_TCP_connections %{citrix.netscaler.total_tcp_connections} - Total_UDP_flows %{citrix.netscaler.total_udp_flows} - Total_policies_allowed %{citrix.netscaler.total_policies_allowed} - Total_policies_denied %{citrix.netscaler.total_policies_denied} - Total_bytes_send %{source.bytes} - Total_bytes_recv %{destination.bytes} - Total_compressedbytes_send %{citrix.netscaler.total_compressed_bytes_send} - Total_compressedbytes_recv %{citrix.netscaler.total_compressed_bytes_recv} - Compression_ratio_send %{} - Compression_ratio_recv %{} - LogoutMethod \"%{event.reason}\" - Group(s) \"%{citrix.netscaler.group}\""
}
},
{
"set": {
"description": "copy user.name to user.email",
"if": "ctx?.user?.name != null && ctx.user.name.contains(\"@\")",
"field": "user.email",
"value": "{{user.name}}"
}
},
{
"dissect": {
"description": "parser user.name from email",
"if": "ctx?.user?.name != null && ctx.user.name.contains(\"@\")",
"field": "user.name",
"pattern": "%{user.name}@%{}"
}
},
{
"lowercase": {
"description": "event.reason",
"field": "event.reason",
"ignore_missing": true
}
},
{
"lowercase": {
"description": "citrix.netscaler.status",
"field": "citrix.netscaler.status",
"ignore_missing": true
}
},
{
"set": {
"if": "ctx?.citrix?.netscaler?.status != null && ctx.citrix.netscaler.status != \"success\"",
"field": "event.outcome",
"value": "failed"
}
},
{
"remove": {
"description": "drop N/A group",
"if": "ctx?.citrix?.netscaler?.group == \"N/A\"",
"field": "citrix.netscaler.group"
}
},
{
"remove": {
"description": "drop empty user.name",
"if": "ctx?.user?.name != null && ctx.user.name.trim().isEmpty()",
"field": "user.name"
}
},
{
"date": {
"description": "event.start",
"if": "ctx?.event?.start != null",
"field": "event.start",
"target_field": "event.start",
"formats": [
"MM/dd/yyyy:HH:mm:ss z",
"MM/dd/yyyy:HH:mm:ss"
],
"on_failure": [
{
"remove": {
"field": "event.start"
}
}
]
}
},
{
"date": {
"description": "event.end",
"if": "ctx?.event?.end != null",
"field": "event.end",
"target_field": "event.end",
"formats": [
"MM/dd/yyyy:HH:mm:ss z",
"MM/dd/yyyy:HH:mm:ss"
],
"on_failure": [
{
"remove": {
"field": "event.end"
}
}
]
}
},
{
"script": {
"description": "compute event.duration",
"if": "ctx?.event?.start != null && ctx?.event?.end != null",
"lang": "painless",
"ignore_failure": true,
"source": "ctx.event.duration = Duration.between(Instant.parse(ctx.event.start), Instant.parse(ctx.event.end)).toNanos();\n"
}
},
{
"convert": {
"description": "make destination.bytes a number",
"field": "destination.bytes",
"type": "long",
"ignore_missing": true,
"on_failure": [
{
"remove": {
"field": "destination.bytes"
}
}
]
}
},
{
"convert": {
"description": "make destination.port a number",
"field": "destination.port",
"type": "long",
"ignore_missing": true,
"on_failure": [
{
"remove": {
"field": "destination.port"
}
}
]
}
},
{
"convert": {
"description": "make source.bytes a number",
"field": "source.bytes",
"type": "long",
"ignore_missing": true,
"on_failure": [
{
"remove": {
"field": "source.bytes"
}
}
]
}
},
{
"convert": {
"description": "make source.port a number",
"field": "source.port",
"type": "long",
"ignore_missing": true,
"on_failure": [
{
"remove": {
"field": "source.port"
}
}
]
}
},
{
"convert": {
"description": "make citrix.netscaler.http_resources_accessed a number",
"field": "citrix.netscaler.http_resources_accessed",
"type": "long",
"ignore_missing": true,
"on_failure": [
{
"remove": {
"field": "citrix.netscaler.http_resources_accessed"
}
}
]
}
},
{
"convert": {
"description": "make citrix.netscaler.non_http_resources_accessed a number",
"field": "citrix.netscaler.non_http_resources_accessed",
"type": "long",
"ignore_missing": true,
"on_failure": [
{
"remove": {
"field": "citrix.netscaler.non_http_resources_accessed"
}
}
]
}
},
{
"convert": {
"description": "make citrix.netscaler.total_compressed_bytes_recv a number",
"field": "citrix.netscaler.total_compressed_bytes_recv",
"type": "long",
"ignore_missing": true,
"on_failure": [
{
"remove": {
"field": "citrix.netscaler.total_compressed_bytes_recv"
}
}
]
}
},
{
"convert": {
"description": "make citrix.netscaler.total_compressed_bytes_send a number",
"field": "citrix.netscaler.total_compressed_bytes_send",
"type": "long",
"ignore_missing": true,
"on_failure": [
{
"remove": {
"field": "citrix.netscaler.total_compressed_bytes_send"
}
}
]
}
},
{
"convert": {
"description": "make citrix.netscaler.total_policies_allowed a number",
"field": "citrix.netscaler.total_policies_allowed",
"type": "long",
"ignore_missing": true,
"on_failure": [
{
"remove": {
"field": "citrix.netscaler.total_policies_allowed"
}
}
]
}
},
{
"convert": {
"description": "make citrix.netscaler.total_policies_denied a number",
"field": "citrix.netscaler.total_policies_denied",
"type": "long",
"ignore_missing": true,
"on_failure": [
{
"remove": {
"field": "citrix.netscaler.total_policies_denied"
}
}
]
}
},
{
"convert": {
"description": "make citrix.netscaler.total_tcp_connections a number",
"field": "citrix.netscaler.total_tcp_connections",
"type": "long",
"ignore_missing": true,
"on_failure": [
{
"remove": {
"field": "citrix.netscaler.total_tcp_connections"
}
}
]
}
},
{
"convert": {
"description": "make citrix.netscaler.total_udp_flows a number",
"field": "citrix.netscaler.total_udp_flows",
"type": "long",
"ignore_missing": true,
"on_failure": [
{
"remove": {
"field": "citrix.netscaler.total_udp_flows"
}
}
]
}
},
{
"script": {
"description": "compute network.bytes",
"if": "ctx?.source?.bytes != null && ctx?.destination?.bytes != null",
"ignore_failure": true,
"lang": "painless",
"source": "def network = ctx.network;\nif (network == null) {\n network = [:];\n ctx.network = network;\n}\nnetwork.bytes = ctx.source.bytes + ctx.destination.bytes;\n"
}
},
{
"user_agent": {
"field": "user_agent.original",
"ignore_missing": true
}
},
{
"geoip": {
"if": "ctx.source?.geo == null",
"field": "source.ip",
"target_field": "source.geo",
"ignore_missing": true
}
},
{
"geoip": {
"if": "ctx.destination?.geo == null",
"field": "destination.ip",
"target_field": "destination.geo",
"ignore_missing": true
}
},
{
"geoip": {
"database_file": "GeoLite2-ASN.mmdb",
"field": "source.ip",
"target_field": "source.as",
"properties": [
"asn",
"organization_name"
],
"ignore_missing": true
}
},
{
"geoip": {
"database_file": "GeoLite2-ASN.mmdb",
"field": "destination.ip",
"target_field": "destination.as",
"properties": [
"asn",
"organization_name"
],
"ignore_missing": true
}
},
{
"rename": {
"field": "source.as.asn",
"target_field": "source.as.number",
"ignore_missing": true
}
},
{
"rename": {
"field": "source.as.organization_name",
"target_field": "source.as.organization.name",
"ignore_missing": true
}
},
{
"rename": {
"field": "destination.as.asn",
"target_field": "destination.as.number",
"ignore_missing": true
}
},
{
"rename": {
"field": "destination.as.organization_name",
"target_field": "destination.as.organization.name",
"ignore_missing": true
}
}
],
"on_failure": [
{
"set": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}
]
}

Citrix Netscaler Pipeline POC Usage

Elasticsearch Setup

  1. Install the pipeline definition to Elasticsearch using Kibana Dev Tools Console or use curl.
    PUT _ingest/pipeline/citrix-netscaler
    { /* JSON pipeline content */ }
    
    OR
    curl -XPUT "https://es:9200/_ingest/pipeline/citrix-netscaler" -H 'Content-Type: application/json' -d@citrix-netscaler-pipeline.json
    

Filebeat Setup

  1. Add a log file input to the filebeat.yml.

    filebeat.inputs:
    - type: log
      paths:
        - /var/log/citrix-netscaler*.log
      tags: [citrix-netscaler, forwarded]
      pipeline: citrix-netscaler
    
  2. Restart Filebeat.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment