Configurations of Filebeat, Fluent-bit, and Vector used during comparison tests

filebeat.yml

filebeat.modules:
  - module: system
    syslog:
      enabled: true
    auth:
      enabled: true
filebeat.autodiscover:
  providers:
    - type: kubernetes
      node: ${NODE_NAME}
      hints.enabled: true
    - type: docker
      templates:
        - condition.or:
            - contains:
                docker.container.image: hyperkube
            - contains:
                docker.container.image: coreos-etcd
          config:
            - type: container
              paths:
                - '/var/lib/docker/containers/${data.docker.container.id}/*-json.log'
output.kafka:
  hosts: ["logging-kafka-bootstrap.logging-kafka.svc.cluster.local:9092"]
  topic: 'filebeat-benchmark'

fluent-bit.conf

[SERVICE]
    Flush                     5
    Log_Level                 info
    Parsers_File              parsers.conf
    HTTP_Server               On
    HTTP_Listen               0.0.0.0
    HTTP_Port                 2020

[INPUT]
    Name             tail
    Alias            kube_tail
    Tag              kube.*
    Path             /var/log/containers/*.log
    Path_Key         log_file
    DB               /var/lib/fluent-bit/kube.db
    Mem_Buf_Limit    5MB
    Skip_Long_lines  On
    Refresh_Interval 10

[INPUT]
    Name             tail
    Alias            kube_audit_tail
    Tag              kube.audit
    Path             /var/log/kube-audit/audit-log.json
    Path_Key         log_file
    DB               /var/lib/fluent-bit/kube-audit.db
    Parser           json
    Mem_Buf_Limit    5MB
    Skip_Long_lines  On
    Refresh_Interval 10

[INPUT]
    Name             tail
    Alias            rke_tail
    Tag              rke.*
    Path             /var/lib/rancher/rke/log/*.log
    Path_Key         log_file
    DB               /var/lib/fluent-bit/rke.db
    Mem_Buf_Limit    5MB
    Skip_Long_lines  On
    Refresh_Interval 10
    Parser           docker

[INPUT]
    Name             tail
    Alias            syslog_tail
    Tag              syslog
    Path             /var/log/syslog
    Path_Key         log_file
    DB               /var/lib/fluent-bit/syslog.db
    Mem_Buf_Limit    5MB
    Skip_Long_lines  On
    Refresh_Interval 10

[INPUT]
    Name             tail
    Alias            auth_tail
    Tag              auth
    Path             /var/log/auth.log
    Path_Key         log_file
    DB               /var/lib/fluent-bit/auth.db
    Mem_Buf_Limit    5MB
    Skip_Long_lines  On
    Refresh_Interval 10

[FILTER]
    Name                kubernetes
    Alias               kube_metadata
    Match               kube.var.*
    Kube_URL            https://kubernetes.default.svc:443
    Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
    Kube_Tag_Prefix     kube.var.log.containers.
    Merge_Log           On
    K8S-Logging.Exclude On

[OUTPUT]
    Name             kafka
    Match            *
    Brokers          logging-kafka-bootstrap.logging-kafka.svc:9092
    Message_Key      message
    Topics           flb-benchmark
    Timestamp_Key    @timestamp
    Timestamp_Format iso8601
    Retry_Limit   false
    # hides errors "Receive failed: Disconnected" when kafka kills idle connections
    rdkafka.log.connection.close false
    # producer buffer is not included in http://fluentbit.io/documentation/0.12/configuration/memory_usage.html#estimating
    rdkafka.queue.buffering.max.kbytes 10240
    # for logs you'll probably want this ot be 0 or 1, not more
    rdkafka.request.required.acks 1

vector.toml

# Configuration for vector.
# Docs: https://vector.dev/docs/

# Data dir is location controlled at the `DaemonSet`.
data_dir = "/vector-data-dir"
[log_schema]
  host_key = "host"
  message_key = "message"
  source_type_key = "source_type"
  timestamp_key = "timestamp"
# Ingest logs from Kubernetes.
[sources.kubernetes_logs]
  type = "kubernetes_logs"
[sources.json_logs]
  type = "file"
  include = ["/var/log/kube-audit/*.json", "/var/lib/rancher/rke/log/*.log"]
  
[sources.system_logs]
  type = "file"
  include = ["/var/log/syslog*", "/var/log/auth.log*"]
  
[transforms.json_parser]
  type = "json_parser"
  inputs = ["json_logs"]
  drop_invalid = true
  
[sinks.kafka]
  type = "kafka"
  inputs = ["kubernetes_logs","json_parser","system_logs"]
  bootstrap_servers = "logging-kafka-bootstrap.logging-kafka:9092"
  topic = "vector-benchmark"
  librdkafka_options."request.required.acks" = "1"
  [sinks.kafka.encoding]
    codec = "json"