Filebeat - Haproxy module - ingest pipeline - TCP log

gistfile1.txt

PUT _ingest/pipeline/filebeat-7.15.1-haproxy-log-pipeline-custom
{
    "description" : "Pipeline for parsing HAProxy http, tcp and default logs. Requires the geoip plugin.",
    "processors" : [
      {
        "set" : {
          "field" : "event.ingested",
          "value" : "{{_ingest.timestamp}}"
        }
      },
      {
        "grok" : {
          "field" : "message",
          "patterns" : [
            """%{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?: (%{IP:source.address}|-):%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy_server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.total_session_duration_time_ms:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}""",
            """%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\[%{POSINT:process.pid:long}\])?: %{GREEDYDATA} (%{IPORHOST:source.address}|-):%{POSINT:source.port:long} %{WORD} %{IPORHOST:destination.ip}:%{POSINT:destination.port:long} \(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\)""",
            "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?(%{IP:source.address}|-):%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} (%{IPORHOST:destination.address} )?%{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:temp.duration:long} %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} |\\{%{DATA}\\} )?\"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"",
            """(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?(%{IP:source.address}|-):%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}""",
            """%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?(%{IP:source.address}|-):%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:temp.duration:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}"""
          ],
          "ignore_missing" : false,
          "pattern_definitions" : {
            "HAPROXY_DATE" : "(%{MONTHDAY}[/-]%{MONTH}[/-]%{YEAR}:%{HOUR}:%{MINUTE}:%{SECOND})|%{SYSLOGTIMESTAMP}"
          }
        }
      },
      {
        "date" : {
          "field" : "haproxy.request_date",
          "target_field" : "@timestamp",
          "formats" : [
            "dd/MMM/yyyy:HH:mm:ss.SSS",
            "MMM dd HH:mm:ss"
          ],
          "on_failure" : [
            {
              "append" : {
                "field" : "error.message",
                "value" : "{{ _ingest.on_failure_message }}"
              }
            }
          ],
          "if" : "ctx.event.timezone == null"
        }
      },
      {
        "date" : {
          "target_field" : "@timestamp",
          "formats" : [
            "dd/MMM/yyyy:HH:mm:ss.SSS",
            "MMM dd HH:mm:ss"
          ],
          "timezone" : "{{ event.timezone }}",
          "on_failure" : [
            {
              "append" : {
                "field" : "error.message",
                "value" : "{{ _ingest.on_failure_message }}"
              }
            }
          ],
          "if" : "ctx.event.timezone != null",
          "field" : "haproxy.request_date"
        }
      },
      {
        "remove" : {
          "field" : "haproxy.request_date"
        }
      },
      {
        "remove" : {
          "field" : "message"
        }
      },
      {
        "grok" : {
          "field" : "haproxy.http.request.raw_request_line",
          "patterns" : [
            "%{WORD:http.request.method}%{SPACE}%{URIPATHPARAM:url.original}%{SPACE}HTTP/%{NUMBER:http.version}"
          ],
          "ignore_missing" : true
        }
      },
      {
        "uri_parts" : {
          "field" : "url.original",
          "ignore_failure" : true,
          "if" : "ctx?.url?.original != null"
        }
      },
      {
        "grok" : {
          "ignore_failure" : true,
          "patterns" : [
            "^%{IP:source.ip}$"
          ],
          "field" : "source.address"
        }
      },
      {
        "grok" : {
          "field" : "destination.address",
          "patterns" : [
            "^%{IP:destination.ip}$"
          ],
          "on_failure" : [
            {
              "set" : {
                "field" : "destination.domain",
                "value" : "{{destination.address}}",
                "ignore_empty_value" : true
              }
            }
          ]
        }
      },
      {
        "geoip" : {
          "field" : "source.ip",
          "target_field" : "source.geo",
          "ignore_missing" : true
        }
      },
      {
        "geoip" : {
          "database_file" : "GeoLite2-ASN.mmdb",
          "field" : "source.ip",
          "target_field" : "source.as",
          "properties" : [
            "asn",
            "organization_name"
          ],
          "ignore_missing" : true
        }
      },
      {
        "rename" : {
          "ignore_missing" : true,
          "field" : "source.as.asn",
          "target_field" : "source.as.number"
        }
      },
      {
        "rename" : {
          "field" : "source.as.organization_name",
          "target_field" : "source.as.organization.name",
          "ignore_missing" : true
        }
      },
      {
        "split" : {
          "ignore_failure" : true,
          "field" : "haproxy.http.request.captured_headers",
          "separator" : """\|"""
        }
      },
      {
        "split" : {
          "field" : "haproxy.http.response.captured_headers",
          "separator" : """\|""",
          "ignore_failure" : true
        }
      },
      {
        "script" : {
          "lang" : "painless",
          "source" : "ctx.event.duration = Math.round(ctx.temp.duration * params.scale)",
          "params" : {
            "scale" : 1000000
          },
          "if" : "ctx.temp?.duration != null"
        }
      },
      {
        "remove" : {
          "field" : "temp.duration",
          "ignore_missing" : true
        }
      },
      {
        "convert" : {
          "field" : "haproxy.bytes_read",
          "target_field" : "http.response.bytes",
          "type" : "long",
          "if" : "ctx.containsKey('http')"
        }
      },
      {
        "append" : {
          "if" : "ctx?.source?.ip != null",
          "field" : "related.ip",
          "value" : "{{source.ip}}"
        }
      },
      {
        "append" : {
          "field" : "related.ip",
          "value" : "{{destination.ip}}",
          "if" : "ctx?.destination?.ip != null"
        }
      },
      {
        "append" : {
          "field" : "related.hosts",
          "value" : "{{destination.domain}}",
          "if" : "ctx?.destination?.domain != null"
        }
      },
      {
        "set" : {
          "field" : "event.kind",
          "value" : "event"
        }
      },
      {
        "append" : {
          "value" : "web",
          "if" : "ctx?.haproxy?.mode == 'HTTP' || ctx?.haproxy?.http != null",
          "field" : "event.category"
        }
      },
      {
        "append" : {
          "field" : "event.category",
          "value" : "network",
          "if" : "ctx?.source.ip != null && ctx?.destination?.ip != null"
        }
      },
      {
        "append" : {
          "field" : "event.type",
          "value" : "connection",
          "if" : "ctx?.source.ip != null && ctx?.destination?.ip != null"
        }
      },
      {
        "set" : {
          "field" : "event.outcome",
          "value" : "success",
          "if" : "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"
        }
      },
      {
        "set" : {
          "field" : "event.outcome",
          "value" : "failure",
          "if" : "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400"
        }
      },
      {
        "script" : {
          "lang" : "painless",
          "description" : "This script processor iterates over the whole document to remove fields with null values.",
          "source" : """void handleMap(Map map) {
  for (def x : map.values()) {
    if (x instanceof Map) {
        handleMap(x);
    } else if (x instanceof List) {
        handleList(x);
    }
  }
  map.values().removeIf(v -> v == null);
}
void handleList(List list) {
  for (def x : list) {
      if (x instanceof Map) {
          handleMap(x);
      } else if (x instanceof List) {
          handleList(x);
      }
  }
}
handleMap(ctx);
"""
        }
      }
    ],
    "on_failure" : [
      {
        "set" : {
          "field" : "error.message",
          "value" : "{{ _ingest.on_failure_message }}"
        }
      }
    ]
  }
}