Citrix Netscaler Elasticsearch Ingest Node Pipeline

citrix-netscaler-pipeline.json

{
  "description": "Pipeline for parsing Citrix Netscaler logs",
  "processors": [
    {
      "script": {
        "description": "set event.original",
        "lang": "painless",
        "source": "def event = ctx.event;\nif (event == null) {\n    event = [:];\n    ctx['event'] = event;\n}\nevent['original'] = ctx.message;\n"
      }
    },
    {
      "dissect": {
        "description": "trim exabeam headers",
        "if": "ctx?.message.contains(\"KAFKA_CONNECT_SYSLOG\")",
        "field": "message",
        "pattern": "%{} - - - - %{message}"
      }
    },
    {
      "set": {
        "field": "event.ingested",
        "value": "{{_ingest.timestamp}}"
      }
    },
    {
      "set": {
        "field": "observer.vendor",
        "value": "Citrix"
      }
    },
    {
      "set": {
        "field": "observer.product",
        "value": "Netscaler"
      }
    },
    {
      "set": {
        "field": "observer.type",
        "value": "firewall"
      }
    },
    {
      "set": {
        "field": "event.module",
        "value": "citrix"
      }
    },
    {
      "set": {
        "field": "event.dataset",
        "value": "citrix.netscaler"
      }
    },
    {
      "grok": {
        "description": "grok syslog header",
        "field": "message",
        "patterns": [
          "<%{NONNEGINT:log.syslog.facility.code:long}> %{NOTSPACE:timestamp} ?%{WORD:event.timezone}? %{HOSTNAME:host.name} %{HOSTNAME} : %{GREEDYDATA:message}"
        ]
      }
    },
    {
      "grok": {
        "description": "grok citrix header",
        "field": "message",
        "patterns": [
          "%{WORD} %{WORD:citrix.netscaler.feature} %{WORD:citrix.netscaler.message_type} %{NUMBER} %{NUMBER} :%{SPACE}+%{GREEDYDATA:message}"
        ]
      }
    },
    {
      "lowercase": {
        "field": "citrix.netscaler.feature"
      }
    },
    {
      "lowercase": {
        "field": "citrix.netscaler.message_type"
      }
    },
    {
      "date": {
        "field": "timestamp",
        "target_field": "@timestamp",
        "formats": [
          "MM/dd/yyyy:HH:mm:ss"
        ],
        "timezone": "{{event.timezone}}",
        "if": "ctx?.event?.timezone != null"
      }
    },
    {
      "date": {
        "field": "timestamp",
        "target_field": "@timestamp",
        "formats": [
          "MM/dd/yyyy:HH:mm:ss"
        ],
        "if": "ctx?.event?.timezone == null"
      }
    },
    {
      "remove": {
        "field": [
          "timestamp"
        ]
      }
    },
    {
      "set": {
        "description": "set event.code",
        "field": "event.code",
        "value": "{{citrix.netscaler.feature}}-{{citrix.netscaler.message_type}}"
      }
    },
    {
      "set": {
        "description": "set generic event.action",
        "field": "event.action",
        "value": "{{citrix.netscaler.feature}} {{citrix.netscaler.message_type}}"
      }
    },
    {
      "script": {
        "description": "add event category and type",
        "lang": "painless",
        "tag": "ecs_categorize",
        "params": {
          "api-cmd_executed": {
            "category": [
              "process"
            ],
            "type": [
              "start"
            ],
            "action": "command executed"
          },
          "aaa-login_failed": {
            "category": [
              "authentication"
            ],
            "type": [
              "denied"
            ],
            "outcome": "failure",
            "action": "login failed"
          },
          "sslvpn-icastart": {
            "category": [
              "process",
              "network"
            ],
            "type": [
              "start"
            ],
            "action": "ICA application started"
          },
          "sslvpn-httprequest": {
            "category": [
              "network",
              "web"
            ],
            "type": [
              "protocol",
              "start",
              "info"
            ],
            "action": "http request"
          },
          "sslvpn-login": {
            "category": [
              "authentication"
            ],
            "type": [
              "start"
            ]
          },
          "sslvpn-logout": {
            "category": [
              "authentication"
            ],
            "type": [
              "end"
            ]
          }
        },
        "source": "ctx.event.kind = 'event';\nctx.event.type = ['info'];\nctx.event.outcome = 'success';\n\ndef metadata = params.get(ctx.event.code);\nif (metadata == null) {\n    return;\n}\nmetadata.forEach((k, v) -> ctx.event[k] = v);\n"
      }
    },
    {
      "dissect": {
        "description": "api-cmd_executed",
        "if": "ctx?.event?.code == \"api-cmd_executed\"",
        "field": "message",
        "pattern": "User %{user.name} - Remote_ip %{source.ip} - Command \"%{process.command_line}\" - Status \"%{citrix.netscaler.status}\""
      }
    },
    {
      "dissect": {
        "description": "sslvpn-httprequest",
        "if": "ctx?.event?.code == \"sslvpn-httprequest\"",
        "field": "message",
        "pattern": "Context %{user.name}@%{source.ip} - SessionId: %{citrix.netscaler.session_id}- %{destination.domain} User %{} : Group(s) %{citrix.netscaler.group} : Vserver %{citrix.netscaler.vserver}:%{destination.port} - %{} : SSO is %{} : %{http.request.method} %{url.original} - -"
      }
    },
    {
      "set": {
        "description": "sslvpn-httprequest network.protocol http",
        "if": "ctx?.event?.code == \"sslvpn-httprequest\"",
        "field": "network.protocol",
        "value": "http"
      }
    },
    {
      "grok": {
        "field": "url.original",
        "ignore_failure": true,
        "patterns": [
          "%{URIPATH:url.path}(?:%{URIPARAM:url.query})?"
        ]
      }
    },
    {
      "dissect": {
        "description": "sslvpn-login",
        "if": "ctx?.event?.code == \"sslvpn-login\"",
        "field": "message",
        "pattern": "Context %{user.name}@%{source.ip} - SessionId: %{citrix.netscaler.session_id}- User %{} - Client_ip %{} - Nat_ip \"%{}\" - Vserver %{citrix.netscaler.vserver}:%{destination.port} - Browser_type \"%{user_agent.original}\" - SSLVPN_client_type %{citrix.netscaler.vpn_client_type} - Group(s) \"%{citrix.netscaler.group}\""
      }
    },
    {
      "dissect": {
        "description": "aaa-login_failed",
        "if": "ctx?.event?.code == \"aaa-login_failed\"",
        "field": "message",
        "pattern": "User %{user.name} - Client_ip %{source.ip} - Failure_reason \"%{citrix.netscaler.error_message}\" - Browser %{user_agent.original}"
      }
    },
    {
      "dissect": {
        "description": "cli-cmd_execute",
        "if": "ctx?.event?.code == \"cli-cmd_executed\"",
        "field": "message",
        "pattern": "User %{user.name} - Remote_ip %{source.ip} - Command \"%{process.command_line}\" - Status \"%{citrix.netscaler.status}\""
      }
    },
    {
      "dissect": {
        "description": "sslvpn-icastart",
        "if": "ctx?.event?.code == \"sslvpn-icastart\"",
        "field": "message",
        "pattern": "Source %{source.ip}:%{source.port} - Destination %{destination.ip}:%{destination.port} - username:domainname %{user.name}: - applicationName %{process.name} - startTime \"%{event.start}\" - connectionId %{citrix.netscaler.connection_id}"
      }
    },
    {
      "dissect": {
        "description": "sslvpn-logout-with-context-header",
        "if": "ctx?.event?.code == \"sslvpn-logout\" && ctx?.message.startsWith(\"Context\")",
        "field": "message",
        "pattern": "Context %{user.name}@%{source.ip} - SessionId: %{citrix.netscaler.session_id}- User %{} - %{}"
      }
    },
    {
      "grok": {
        "description": "sslvpn-logout-without-context-header",
        "if": "ctx?.event?.code == \"sslvpn-logout\" && !ctx?.message.startsWith(\"Context\")",
        "field": "message",
        "patterns": [
          "User %{NOTSPACE:user.name}?%{SPACE}?- "
        ]
      }
    },
    {
      "dissect": {
        "description": "sslvpn-logout-trailer",
        "if": "ctx?.event?.code == \"sslvpn-logout\"",
        "field": "message",
        "pattern": "%{} - Client_ip %{source.ip} - Nat_ip \"%{}\" - Vserver %{citrix.netscaler.vserver}:%{destination.port} - Start_time \"%{event.start}\" - End_time \"%{event.end}\" - Duration %{} - Http_resources_accessed %{citrix.netscaler.http_resources_accessed} - NonHttp_services_accessed %{citrix.netscaler.non_http_resources_accessed} - Total_TCP_connections %{citrix.netscaler.total_tcp_connections} - Total_UDP_flows %{citrix.netscaler.total_udp_flows} - Total_policies_allowed %{citrix.netscaler.total_policies_allowed} - Total_policies_denied %{citrix.netscaler.total_policies_denied} - Total_bytes_send %{source.bytes} - Total_bytes_recv %{destination.bytes} - Total_compressedbytes_send %{citrix.netscaler.total_compressed_bytes_send} - Total_compressedbytes_recv %{citrix.netscaler.total_compressed_bytes_recv} - Compression_ratio_send %{} - Compression_ratio_recv %{} - LogoutMethod \"%{event.reason}\" - Group(s) \"%{citrix.netscaler.group}\""
      }
    },
    {
      "set": {
        "description": "copy user.name to user.email",
        "if": "ctx?.user?.name != null && ctx.user.name.contains(\"@\")",
        "field": "user.email",
        "value": "{{user.name}}"
      }
    },
    {
      "dissect": {
        "description": "parser user.name from email",
        "if": "ctx?.user?.name != null && ctx.user.name.contains(\"@\")",
        "field": "user.name",
        "pattern": "%{user.name}@%{}"
      }
    },
    {
      "lowercase": {
        "description": "event.reason",
        "field": "event.reason",
        "ignore_missing": true
      }
    },
    {
      "lowercase": {
        "description": "citrix.netscaler.status",
        "field": "citrix.netscaler.status",
        "ignore_missing": true
      }
    },
    {
      "set": {
        "if": "ctx?.citrix?.netscaler?.status != null && ctx.citrix.netscaler.status != \"success\"",
        "field": "event.outcome",
        "value": "failed"
      }
    },
    {
      "remove": {
        "description": "drop N/A group",
        "if": "ctx?.citrix?.netscaler?.group == \"N/A\"",
        "field": "citrix.netscaler.group"
      }
    },
    {
      "remove": {
        "description": "drop empty user.name",
        "if": "ctx?.user?.name != null && ctx.user.name.trim().isEmpty()",
        "field": "user.name"
      }
    },
    {
      "date": {
        "description": "event.start",
        "if": "ctx?.event?.start != null",
        "field": "event.start",
        "target_field": "event.start",
        "formats": [
          "MM/dd/yyyy:HH:mm:ss z",
          "MM/dd/yyyy:HH:mm:ss"
        ],
        "on_failure": [
          {
            "remove": {
              "field": "event.start"
            }
          }
        ]
      }
    },
    {
      "date": {
        "description": "event.end",
        "if": "ctx?.event?.end != null",
        "field": "event.end",
        "target_field": "event.end",
        "formats": [
          "MM/dd/yyyy:HH:mm:ss z",
          "MM/dd/yyyy:HH:mm:ss"
        ],
        "on_failure": [
          {
            "remove": {
              "field": "event.end"
            }
          }
        ]
      }
    },
    {
      "script": {
        "description": "compute event.duration",
        "if": "ctx?.event?.start != null && ctx?.event?.end != null",
        "lang": "painless",
        "ignore_failure": true,
        "source": "ctx.event.duration = Duration.between(Instant.parse(ctx.event.start), Instant.parse(ctx.event.end)).toNanos();\n"
      }
    },
    {
      "convert": {
        "description": "make destination.bytes a number",
        "field": "destination.bytes",
        "type": "long",
        "ignore_missing": true,
        "on_failure": [
          {
            "remove": {
              "field": "destination.bytes"
            }
          }
        ]
      }
    },
    {
      "convert": {
        "description": "make destination.port a number",
        "field": "destination.port",
        "type": "long",
        "ignore_missing": true,
        "on_failure": [
          {
            "remove": {
              "field": "destination.port"
            }
          }
        ]
      }
    },
    {
      "convert": {
        "description": "make source.bytes a number",
        "field": "source.bytes",
        "type": "long",
        "ignore_missing": true,
        "on_failure": [
          {
            "remove": {
              "field": "source.bytes"
            }
          }
        ]
      }
    },
    {
      "convert": {
        "description": "make source.port a number",
        "field": "source.port",
        "type": "long",
        "ignore_missing": true,
        "on_failure": [
          {
            "remove": {
              "field": "source.port"
            }
          }
        ]
      }
    },
    {
      "convert": {
        "description": "make citrix.netscaler.http_resources_accessed a number",
        "field": "citrix.netscaler.http_resources_accessed",
        "type": "long",
        "ignore_missing": true,
        "on_failure": [
          {
            "remove": {
              "field": "citrix.netscaler.http_resources_accessed"
            }
          }
        ]
      }
    },
    {
      "convert": {
        "description": "make citrix.netscaler.non_http_resources_accessed a number",
        "field": "citrix.netscaler.non_http_resources_accessed",
        "type": "long",
        "ignore_missing": true,
        "on_failure": [
          {
            "remove": {
              "field": "citrix.netscaler.non_http_resources_accessed"
            }
          }
        ]
      }
    },
    {
      "convert": {
        "description": "make citrix.netscaler.total_compressed_bytes_recv a number",
        "field": "citrix.netscaler.total_compressed_bytes_recv",
        "type": "long",
        "ignore_missing": true,
        "on_failure": [
          {
            "remove": {
              "field": "citrix.netscaler.total_compressed_bytes_recv"
            }
          }
        ]
      }
    },
    {
      "convert": {
        "description": "make citrix.netscaler.total_compressed_bytes_send a number",
        "field": "citrix.netscaler.total_compressed_bytes_send",
        "type": "long",
        "ignore_missing": true,
        "on_failure": [
          {
            "remove": {
              "field": "citrix.netscaler.total_compressed_bytes_send"
            }
          }
        ]
      }
    },
    {
      "convert": {
        "description": "make citrix.netscaler.total_policies_allowed a number",
        "field": "citrix.netscaler.total_policies_allowed",
        "type": "long",
        "ignore_missing": true,
        "on_failure": [
          {
            "remove": {
              "field": "citrix.netscaler.total_policies_allowed"
            }
          }
        ]
      }
    },
    {
      "convert": {
        "description": "make citrix.netscaler.total_policies_denied a number",
        "field": "citrix.netscaler.total_policies_denied",
        "type": "long",
        "ignore_missing": true,
        "on_failure": [
          {
            "remove": {
              "field": "citrix.netscaler.total_policies_denied"
            }
          }
        ]
      }
    },
    {
      "convert": {
        "description": "make citrix.netscaler.total_tcp_connections a number",
        "field": "citrix.netscaler.total_tcp_connections",
        "type": "long",
        "ignore_missing": true,
        "on_failure": [
          {
            "remove": {
              "field": "citrix.netscaler.total_tcp_connections"
            }
          }
        ]
      }
    },
    {
      "convert": {
        "description": "make citrix.netscaler.total_udp_flows a number",
        "field": "citrix.netscaler.total_udp_flows",
        "type": "long",
        "ignore_missing": true,
        "on_failure": [
          {
            "remove": {
              "field": "citrix.netscaler.total_udp_flows"
            }
          }
        ]
      }
    },
    {
      "script": {
        "description": "compute network.bytes",
        "if": "ctx?.source?.bytes != null && ctx?.destination?.bytes != null",
        "ignore_failure": true,
        "lang": "painless",
        "source": "def network = ctx.network;\nif (network == null) {\n    network = [:];\n    ctx.network = network;\n}\nnetwork.bytes = ctx.source.bytes + ctx.destination.bytes;\n"
      }
    },
    {
      "user_agent": {
        "field": "user_agent.original",
        "ignore_missing": true
      }
    },
    {
      "geoip": {
        "if": "ctx.source?.geo == null",
        "field": "source.ip",
        "target_field": "source.geo",
        "ignore_missing": true
      }
    },
    {
      "geoip": {
        "if": "ctx.destination?.geo == null",
        "field": "destination.ip",
        "target_field": "destination.geo",
        "ignore_missing": true
      }
    },
    {
      "geoip": {
        "database_file": "GeoLite2-ASN.mmdb",
        "field": "source.ip",
        "target_field": "source.as",
        "properties": [
          "asn",
          "organization_name"
        ],
        "ignore_missing": true
      }
    },
    {
      "geoip": {
        "database_file": "GeoLite2-ASN.mmdb",
        "field": "destination.ip",
        "target_field": "destination.as",
        "properties": [
          "asn",
          "organization_name"
        ],
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "source.as.asn",
        "target_field": "source.as.number",
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "source.as.organization_name",
        "target_field": "source.as.organization.name",
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "destination.as.asn",
        "target_field": "destination.as.number",
        "ignore_missing": true
      }
    },
    {
      "rename": {
        "field": "destination.as.organization_name",
        "target_field": "destination.as.organization.name",
        "ignore_missing": true
      }
    }
  ],
  "on_failure": [
    {
      "set": {
        "field": "error.message",
        "value": "{{ _ingest.on_failure_message }}"
      }
    }
  ]
}

usage.md

Citrix Netscaler Pipeline POC Usage

Elasticsearch Setup

1. Install the pipeline definition to Elasticsearch using Kibana Dev Tools Console or use curl.

   PUT _ingest/pipeline/citrix-netscaler
   { /* JSON pipeline content */ }
   

   OR

   curl -XPUT "https://es:9200/_ingest/pipeline/citrix-netscaler" -H 'Content-Type: application/json' -d@citrix-netscaler-pipeline.json
   

Filebeat Setup

1. Add a log file input to the filebeat.yml.

   filebeat.inputs:
   - type: log
     paths:
       - /var/log/citrix-netscaler*.log
     tags: [citrix-netscaler, forwarded]
     pipeline: citrix-netscaler
   

2. Restart Filebeat.