fluentd-helm.md

fluentd Installation with Helm

• Fluentd on Kubernetes

Repo

• https://github.com/fluent/helm-charts/tree/main/charts/fluent-bit

Prerequisites

• Kubernetes 1.19+
• Helm 3.2.0+
• ECK Operator (https://gist.github.com/taking/43a6aafdff1e86e888b72bb6d06fc22f)
• A persistent storage resource and RW access to it
• Kubernetes StorageClass for dynamic provisioning

helm update

helm repo add fluent https://fluent.github.io/helm-charts
helm repo update

values Override

PW=$(kubectl get secret quickstart-es-elastic-user -n elastic-system -o go-template='{{.data.elastic | base64decode}}')
cat <<EOF > fluentd-value-override.yaml
env:
- name: K8S_NODE_NAME
  valueFrom:
    fieldRef:
      fieldPath: spec.nodeName
- name: "FLUENTD_CONF"
  value: "../../../etc/fluent/fluent.conf"
# The host depends on your elasticsearch deployment name if you use dns because the operator deploys a service with its name.
- name: FLUENT_ELASTICSEARCH_HOST
  value: quickstart-es-http
# The default elasticsearch user
- name: FLUENT_ELASTICSEARCH_USER
  value: elastic
# Is already present from the elasticsearch deployment secret
- name: FLUENT_ELASTICSEARCH_PASSWORD
  valueFrom:
    secretKeyRef:
      name: quickstart-es-elastic-user
      key: elastic
# Elasticsearch standard port
- name: FLUENT_ELASTICSEARCH_PORT
  value: "9200"
# The operator comes with encryption so force the collector to use https
- name: FLUENT_ELASTICSEARCH_SCHEME
  value: "https"
# To keep it simple for demo purposes
- name: FLUENT_ELASTICSEARCH_SSL_VERIFY
  value: "false"
# In case you want to disable systemd logs
- name: FLUENTD_SYSTEMD_CONF
  value: disable
# to avoid issue https://github.com/uken/fluent-plugin-elasticsearch/issues/525
- name: FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS
  value: "false"
# for better separation in kibana dashboards
- name: FLUENT_ELASTICSEARCH_INDEX_NAME
  value: fluentd
fileConfigs:
  01_sources.conf: |-
    ## logs from podman
    <source>
      @type tail
      @id in_tail_container_logs
      @label @KUBERNETES
      path /var/log/containers/*.log
      pos_file /var/log/fluentd-containers.log.pos
      tag kubernetes.*
      read_from_head true
      <parse>
        @type multi_format
        <pattern>
          format json
          time_key time
          time_type string
          time_format "%Y-%m-%dT%H:%M:%S.%NZ"
          keep_time_key false
        </pattern>
        <pattern>
          format regexp
          expression /^(?<time>.+) (?<stream>stdout|stderr)( (.))? (?<log>.*)$/
          time_format '%Y-%m-%dT%H:%M:%S.%NZ'
          keep_time_key false
        </pattern>
      </parse>
      emit_unmatched_lines true
    </source>
  02_filters.conf: |-
    <label @KUBERNETES>
      <match kubernetes.var.log.containers.fluentd**>
        @type relabel
        @label @FLUENT_LOG
      </match>
      # <match kubernetes.var.log.containers.**_kube-system_**>
      #   @type null
      #   @id ignore_kube_system_logs
      # </match>
      <filter kubernetes.**>
        @type kubernetes_metadata
        @id filter_kube_metadata
        skip_labels false
        skip_container_metadata false
        skip_namespace_metadata true
        skip_master_url true
      </filter>
      <match **>
        @type relabel
        @label @DISPATCH
      </match>
    </label>
  03_dispatch.conf: |-
    <label @DISPATCH>
      <filter **>
        @type prometheus
        <metric>
          name fluentd_input_status_num_records_total
          type counter
          desc The total number of incoming records
          <labels>
            tag ${tag}
            hostname ${hostname}
          </labels>
        </metric>
      </filter>
      <match **>
        @type relabel
        @label @OUTPUT
      </match>
    </label>
  04_outputs.conf: |-
    <label @OUTPUT>
      <match **>
        @type elasticsearch
        host "quickstart-es-all-nodes"
        port 9200
        logstash_format true
        path ""
        scheme https
        ssl_verify false
        logstash_prefix "fluentd"
        index_name "fluentd"
        type_name "fluentd"
        user elastic
        password ${PW}
      </match>
    </label>
EOF

installation

helm install fluentd fluent/fluentd \
  --create-namespace \
  --namespace elastic-system \
  -f fluentd-value-override.yaml

(Option) fluent-bit

fluent-bit-value-override.yaml

PW=$(kubectl get secret quickstart-es-elastic-user -n elastic-system -o go-template='{{.data.elastic | base64decode}}')
cat <<EOF > fluent-bit-value-override.yaml
config:
  service: |
    [SERVICE]
        Daemon Off
        Flush {{ .Values.flush }}
        Log_Level {{ .Values.logLevel }}
        Parsers_File parsers.conf
        Parsers_File custom_parsers.conf
        HTTP_Server On
        HTTP_Listen 0.0.0.0
        HTTP_Port {{ .Values.metricsPort }}
        Health_Check On
  ## https://docs.fluentbit.io/manual/pipeline/inputs
  inputs: |
    [INPUT]
        Name tail
        Tag kube.*
        Path /var/log/containers/*.log
        multiline.parser docker, cri
        Mem_Buf_Limit 5MB
        Skip_Long_Lines On
    [INPUT]
        Name systemd
        Tag host.*
        Systemd_Filter _SYSTEMD_UNIT=kubelet.service
        Read_From_Tail On
  ## https://docs.fluentbit.io/manual/pipeline/filters
  filters: |
    [FILTER]
        Name kubernetes
        Match kube.*
        Merge_Log On
        Merge_Log_Trim      On
        Keep_Log Off
        K8S-Logging.Parser On
        K8S-Logging.Exclude Off
        Annotations         Off
        Labels              On

    [FILTER]
        Name          nest
        Match         kube.*
        Operation     lift
        Nested_under  kubernetes
        Add_prefix    kubernetes_

    [FILTER]
        Name          nest
        Match         kube.*
        Operation     lift
        Nested_under  kubernetes_labels
        Add_prefix    kubernetes_labels_
  ## https://docs.fluentbit.io/manual/pipeline/outputs
  outputs: |
    [OUTPUT]
        Name es
        Match *
        Host quickstart-es-all-nodes
        Port 9200
        HTTP_User elastic
        HTTP_Passwd ${PW}
        Logstash_Format On
        Logstash_Prefix fluentd
        tls On
        tls.verify Off
        Replace_Dots On
        Retry_Limit False
        Suppress_Type_Name On
  ## https://docs.fluentbit.io/manual/pipeline/parsers
  customParsers: |
    [PARSER]
        Name        json
        Format      json
        Time_Key    time
        Time_Format %d/%b/%Y:%H:%M:%S %z
        Time_Keep   On

    [PARSER]
        Name docker
        Format json
        Time_Key time
        Time_Format %Y-%m-%dT%H:%M:%S.%L
        Time_Keep On

    [PARSER]
        Name        cri
        Format      regex
        Regex       ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L%z
        Decode_Field json message

    [PARSER]
        Name        syslog
        Format      regex
        Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
        Time_Key    time
        Time_Format %b %d %H:%M:%S
EOF

helm install fluent-bit fluent/fluent-bit \
  --create-namespace \
  --namespace elastic-system \
  -f fluent-bit-value-override.yaml