pfsense Elastic ECS Compliant Ingest Pipeline for filterlog & openvpn (INW)

pfsense ingest pipeline

PUT _ingest/pipeline/plex
{
  "processors": [
    {
      "set": {
        "field": "event.module",
        "value": "plex",
        "ignore_failure": true
      }
    },
    {
      "set": {
        "field": "event.original",
        "value": "{{message}}",
        "ignore_failure": true
      }
    },
    {
      "grok": {
        "field": "message",
        "patterns": [
          "(?<timestamp>\\w{3} \\d{2}\\, \\d{4} \\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}) \\[%{INT:process.pid}\\] %{WORD:log.level} \\- %{GREEDYDATA:message}",
          "(?<message>.*)"
        ],
        "ignore_failure": true,
        "description": "Initial parsing"
      }
    },
    {
      "grok": {
        "field": "message",
        "patterns": [
          "(?<event.dataset>Auth|Request|Completed|Statistics)\\: %{GREEDYDATA:message}",
          "%{GREEDYDATA:message}"
        ],
        "ignore_failure": true,
        "description": "event.dataset parsing"
      }
    },
    {
      "grok": {
        "field": "message",
        "patterns": [
          "\\[%{IP:source.ip}\\:%{INT:source.port} \\(%{WORD:network.zone}\\)\\] %{WORD:http.request.method} (?<url.path>\\/?[^\\ ]+) \\(%{INT:process.args_count} live\\) %{GREEDYDATA:message}"
        ],
        "if": "ctx.event?.dataset == 'Request'",
        "ignore_failure": true,
        "description": "Request event.dataset parsing"
      }
    },
    {
      "grok": {
        "field": "message",
        "patterns": [
          "(?<event.action>Signed-in)",
          "(?<event.action>TLS GZIP Signed-in Token|TLS Signed-in Token) \\((?<client.user.name>[^\\)]+)\\)"
        ],
        "if": "ctx.event?.dataset == 'Request'",
        "ignore_failure": true,
        "description": "Request dataset event.action parsing"
      }
    },
    {
      "grok": {
        "field": "message",
        "patterns": [
          "\\[%{IP:source.ip}\\:%{INT:source.port}\\] %{INT:http.response.status_code} %{WORD:http.request.method} (?<url.path>\\/?[^\\ ]+) \\(%{INT:process.args_count} live\\) %{GREEDYDATA:message}",
          "%{GREEDYDATA:message}"
        ],
        "if": "ctx.event?.dataset == 'Completed'",
        "ignore_failure": true,
        "description": "Completed event.dataset parsing"
      }
    },
    {
      "grok": {
        "field": "message",
        "patterns": [
          "%{INT:process.uptime}ms %{INT:server.bytes} bytes",
          "(?<tls.version_protocol>TLS|TLS GZIP) %{INT:process.uptime}ms %{INT:server.bytes} bytes \\(pipelined\\: %{INT:process.args_count}\\)"
        ],
        "if": "ctx.event?.dataset == 'Completed'",
        "ignore_failure": true,
        "description": "Completed dataset event.action parsing"
      }
    },
    {
      "grok": {
        "field": "message",
        "patterns": [
          "(?<event.action>authenticated user) %{INT:client.user.id} as %{NOTSPACE:client.user.name}",
          "%{GREEDYDATA:message}"
        ],
        "if": "ctx.event?.dataset == 'Auth'",
        "ignore_failure": true,
        "description": "Auth event.dataset parsing"
      }
    },
    {
      "dissect": {
        "field": "message",
        "pattern": "Library item %{file.uid} '%{file.name}' got played by account %{client.user.id}!",
        "ignore_failure": true,
        "description": "Library item"
      }
    },
    {
      "dissect": {
        "field": "message",
        "pattern": "Play progress on %{file.uid} '%{file.name}' - got played %{process.uptime} ms by account %{client.user.id}!",
        "ignore_failure": true,
        "description": "Play progress"
      }
    },
    {
      "grok": {
        "field": "message",
        "patterns": [
          "\\[(?<event.dataset>Now|Transcode)\\] %{GREEDYDATA:message}",
          "\\[(?<event.dataset>Transcode[^\\/])\\/(?<file.uid>[^\\]]+)\\] %{GREEDYDATA:message}"
        ],
        "ignore_failure": true,
        "description": "Now|Transcode event.dataset parsing"
      }
    },
    {
      "grok": {
        "field": "message",
        "patterns": [
          "(?<process.thread.name>User|Device|Profile|Updated|Adding) %{GREEDYDATA:message}"
        ],
        "if": "ctx.event?.dataset == 'Now'",
        "ignore_failure": true,
        "description": "Now process.thread.name parsing"
      }
    },
    {
      "grok": {
        "field": "message",
        "patterns": [
          "is %{NOTSPACE:client.user.name} \\(ID\\: %{INT:client.user.id}\\)"
        ],
        "if": "ctx.process.thread?.name == 'User'",
        "ignore_failure": true,
        "description": "Now process.thread.name:User parsing"
      }
    },
    {
      "grok": {
        "field": "message",
        "patterns": [
          "is %{NOTSPACE:user_agent.device.name} \\((?<client.host.name>[^\\)]+)\\)\\."
        ],
        "if": "ctx.process.thread?.name == 'Device'",
        "ignore_failure": true,
        "description": "Now process.thread.name:Device parsing"
      }
    },
    {
      "grok": {
        "field": "message",
        "patterns": [
          "is %{NOTSPACE:user_agent.name}"
        ],
        "if": "ctx.process.thread?.name == 'Device'",
        "ignore_failure": true,
        "description": "Now process.thread.name:Profile parsing"
      }
    }
  ]
}