Skip to content

Instantly share code, notes, and snippets.

@taking
Last active Jul 22, 2022
Embed
What would you like to do?

fluentd Installation with Helm

  • Fluentd on Kubernetes

Repo

Prerequisites

helm update

helm repo add fluent https://fluent.github.io/helm-charts
helm repo update

values Override

PW=$(kubectl get secret quickstart-es-elastic-user -n elastic-system -o go-template='{{.data.elastic | base64decode}}')
cat <<EOF > fluentd-value-override.yaml
env:
- name: K8S_NODE_NAME
  valueFrom:
    fieldRef:
      fieldPath: spec.nodeName
- name: "FLUENTD_CONF"
  value: "../../../etc/fluent/fluent.conf"
# The host depends on your elasticsearch deployment name if you use dns because the operator deploys a service with its name.
- name: FLUENT_ELASTICSEARCH_HOST
  value: quickstart-es-http
# The default elasticsearch user
- name: FLUENT_ELASTICSEARCH_USER
  value: elastic
# Is already present from the elasticsearch deployment secret
- name: FLUENT_ELASTICSEARCH_PASSWORD
  valueFrom:
    secretKeyRef:
      name: quickstart-es-elastic-user
      key: elastic
# Elasticsearch standard port
- name: FLUENT_ELASTICSEARCH_PORT
  value: "9200"
# The operator comes with encryption so force the collector to use https
- name: FLUENT_ELASTICSEARCH_SCHEME
  value: "https"
# To keep it simple for demo purposes
- name: FLUENT_ELASTICSEARCH_SSL_VERIFY
  value: "false"
# In case you want to disable systemd logs
- name: FLUENTD_SYSTEMD_CONF
  value: disable
# to avoid issue https://github.com/uken/fluent-plugin-elasticsearch/issues/525
- name: FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS
  value: "false"
# for better separation in kibana dashboards
- name: FLUENT_ELASTICSEARCH_INDEX_NAME
  value: fluentd
fileConfigs:
  01_sources.conf: |-
    ## logs from podman
    <source>
      @type tail
      @id in_tail_container_logs
      @label @KUBERNETES
      path /var/log/containers/*.log
      pos_file /var/log/fluentd-containers.log.pos
      tag kubernetes.*
      read_from_head true
      <parse>
        @type multi_format
        <pattern>
          format json
          time_key time
          time_type string
          time_format "%Y-%m-%dT%H:%M:%S.%NZ"
          keep_time_key false
        </pattern>
        <pattern>
          format regexp
          expression /^(?<time>.+) (?<stream>stdout|stderr)( (.))? (?<log>.*)$/
          time_format '%Y-%m-%dT%H:%M:%S.%NZ'
          keep_time_key false
        </pattern>
      </parse>
      emit_unmatched_lines true
    </source>
  02_filters.conf: |-
    <label @KUBERNETES>
      <match kubernetes.var.log.containers.fluentd**>
        @type relabel
        @label @FLUENT_LOG
      </match>
      # <match kubernetes.var.log.containers.**_kube-system_**>
      #   @type null
      #   @id ignore_kube_system_logs
      # </match>
      <filter kubernetes.**>
        @type kubernetes_metadata
        @id filter_kube_metadata
        skip_labels false
        skip_container_metadata false
        skip_namespace_metadata true
        skip_master_url true
      </filter>
      <match **>
        @type relabel
        @label @DISPATCH
      </match>
    </label>
  03_dispatch.conf: |-
    <label @DISPATCH>
      <filter **>
        @type prometheus
        <metric>
          name fluentd_input_status_num_records_total
          type counter
          desc The total number of incoming records
          <labels>
            tag ${tag}
            hostname ${hostname}
          </labels>
        </metric>
      </filter>
      <match **>
        @type relabel
        @label @OUTPUT
      </match>
    </label>
  04_outputs.conf: |-
    <label @OUTPUT>
      <match **>
        @type elasticsearch
        host "quickstart-es-all-nodes"
        port 9200
        logstash_format true
        path ""
        scheme https
        ssl_verify false
        logstash_prefix "fluentd"
        index_name "fluentd"
        type_name "fluentd"
        user elastic
        password ${PW}
      </match>
    </label>
EOF

installation

helm install fluentd fluent/fluentd \
  --create-namespace \
  --namespace elastic-system \
  -f fluentd-value-override.yaml
@taking
Copy link
Author

taking commented Jul 21, 2022

(Option) fluent-bit

fluent-bit-value-override.yaml

PW=$(kubectl get secret quickstart-es-elastic-user -n elastic-system -o go-template='{{.data.elastic | base64decode}}')
cat <<EOF > fluent-bit-value-override.yaml
config:
  service: |
    [SERVICE]
        Daemon Off
        Flush {{ .Values.flush }}
        Log_Level {{ .Values.logLevel }}
        Parsers_File parsers.conf
        Parsers_File custom_parsers.conf
        HTTP_Server On
        HTTP_Listen 0.0.0.0
        HTTP_Port {{ .Values.metricsPort }}
        Health_Check On
  ## https://docs.fluentbit.io/manual/pipeline/inputs
  inputs: |
    [INPUT]
        Name tail
        Path /var/log/containers/*.log
        multiline.parser docker, cri
        Tag kube.*
        Mem_Buf_Limit 5MB
        Skip_Long_Lines On
    [INPUT]
        Name systemd
        Tag host.*
        Systemd_Filter _SYSTEMD_UNIT=kubelet.service
        Read_From_Tail On
  ## https://docs.fluentbit.io/manual/pipeline/filters
  filters: |
    [FILTER]
        Name kubernetes
        Match kube.*
        Merge_Log On
        Keep_Log Off
        K8S-Logging.Parser On
        K8S-Logging.Exclude On
  ## https://docs.fluentbit.io/manual/pipeline/outputs
  outputs: |
    [OUTPUT]
        Name es
        Match *
        Host quickstart-es-all-nodes
        Port 9200
        HTTP_User elastic
        HTTP_Passwd ${PW}
        Logstash_Format On
        Logstash_Prefix fluentd
        tls On
        tls.verify Off
        Replace_Dots On
        Retry_Limit False
        Suppress_Type_Name On
  ## https://docs.fluentbit.io/manual/pipeline/parsers
  customParsers: |
    [PARSER]
        Name docker
        Format json
        Time_Key time
        Time_Format %Y-%m-%dT%H:%M:%S.%L
        Time_Keep On

    [PARSER]
        Name        cri
        Format      regex
        Regex       ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L%z
        Decode_Field json message

    [PARSER]
        Name        syslog
        Format      regex
        Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
        Time_Key    time
        Time_Format %b %d %H:%M:%S

    [PARSER]
        Name   json
        Format json
        Time_Key time
        Time_Format %d/%b/%Y:%H:%M:%S %z
EOF
helm install fluent-bit fluent/fluent-bit \
  --create-namespace \
  --namespace elastic-system \
  -f fluent-bit-value-override.yaml

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment