set of fluentd configs for kubernetes aggregated logging into elasticsearch using fluentd, on k8s v1.11+ using docker 1.13+ on centos 7 - kubectl apply the 06-fluentd-daemonset.yaml after placing the other files below into a directory (such as "fluentd-configs") and create configmap using: kubectl -n monitoring create configmap es-fluentd --from…

06-fluentd-daemonset.yaml

---
# Source: fluentd-elasticsearch/templates/service-account.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: es-fluentd
  namespace: monitoring
  labels:
    app: fluentd
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile
---
# Source: fluentd-elasticsearch/templates/clusterrole.yaml
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: es-fluentd
  labels:
    app: fluentd
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile
rules:
- apiGroups:
  - ""
  resources:
  - "namespaces"
  - "pods"
  verbs:
  - "get"
  - "watch"
  - "list"
---
# Source: fluentd-elasticsearch/templates/clusterrolebinding.yaml
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: es-fluentd
  labels:
    app: fluentd
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile
subjects:
- kind: ServiceAccount
  name: es-fluentd
  apiGroup: ""
  namespace: monitoring
roleRef:
  kind: ClusterRole
  name: es-fluentd
  apiGroup: ""
---
# Source: fluentd-elasticsearch/templates/daemonset.yaml
apiVersion: apps/v1beta2
kind: DaemonSet
metadata:
  name: es-fluentd
  namespace: monitoring
  labels:
    app: es-fluentd
    version: v2.3.0
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile
spec:
  selector:
    matchLabels:
      app: es-fluentd
  template:
    metadata:
      labels:
        app: es-fluentd
        version: v2.3.0
        kubernetes.io/cluster-service: "true"
      # This annotation ensures that fluentd does not get evicted if the node
      # supports critical pod annotation based priority scheme.
      # Note that this does not guarantee admission on the nodes (#40573).
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
        #checksum/config: fcf8b484a327b35a768cd1da3d990ae833df14f95edee32071a088a7c983612e
    spec:
      serviceAccountName: es-fluentd
      containers:
      - name: es-fluentd
        image:  "aleksmariusz/fluentd-elasticsearch:v2.3.0"
        imagePullPolicy:
        env:
        - name: FLUENTD_ARGS
          value: --no-supervisor -q
        - name: OUTPUT_HOST
          value: "elasticsearch"
        - name: OUTPUT_PORT
          value: "9200"
        - name: OUTPUT_BUFFER_CHUNK_LIMIT
          value: "2M"
        - name: OUTPUT_BUFFER_QUEUE_LIMIT
          value: "8"
        - name: K8S_NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        resources:
          {}

        volumeMounts:
        - name: runlog
          mountPath: /run/log
        - name: varlog
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: libsystemddir
          mountPath: /host/lib
          readOnly: true
        - name: config-volume-es-fluentd
          mountPath: /etc/fluent/config.d
        ports:
        # Liveness probe is aimed to help in situarions where fluentd
        # silently hangs for no apparent reasons until manual restart.
        # The idea of this probe is that if fluentd is not queueing or
        # flushing chunks for 5 minutes, something is not right. If
        # you want to change the fluentd configuration, reducing amount of
        # logs fluentd collects, consider changing the threshold or turning
        # liveness probe off completely.
        livenessProbe:
          initialDelaySeconds: 600
          periodSeconds: 60
          exec:
            command:
            - '/bin/sh'
            - '-c'
            - >
              LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-300};
              STUCK_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-900};
              if [ ! -e /var/log/fluentd-buffers ];
              then
                exit 1;
              fi;
              touch -d "${STUCK_THRESHOLD_SECONDS} seconds ago" /tmp/marker-stuck;
              if [[ -z "$(find /var/log/fluentd-buffers -type f -newer /tmp/marker-stuck -print -quit)" ]];
              then
                rm -rf /var/log/fluentd-buffers;
                exit 1;
              fi;
              touch -d "${LIVENESS_THRESHOLD_SECONDS} seconds ago" /tmp/marker-liveness;
              if [[ -z "$(find /var/log/fluentd-buffers -type f -newer /tmp/marker-liveness -print -quit)" ]];
              then
                exit 1;
              fi;
      tolerations:
        - effect: NoSchedule
          operator: Exists
      terminationGracePeriodSeconds: 30
      volumes:
      - name: runlog
        hostPath:
          path: /run/log
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      # It is needed to copy systemd library to decompress journals
      - name: libsystemddir
        hostPath:
          path: /usr/lib64
      - name: config-volume-es-fluentd
        configMap:
          name: es-fluentd

input.systemd.conf

# Logs from systemd-journal for interesting services.
#
<source>
  @id journald
  @type systemd
  <storage>
    persistent true
    path /var/log/journald.pos
  </storage>
  tag journal
</source>

#Tag Kubernetes containers
<match journal>
  @type rewrite_tag_filter
  <rule>
    key SYSLOG_IDENTIFIER
    pattern /^(.*)/
    tag systemd.$1
  </rule>
  <rule>
    key CONTAINER_NAME
    pattern /^k8s_/
    tag kubernetes.journal.container
  </rule>
</match> 

kubernetes.conf

#Tell kubernetes_metadata that the logs are coming from journal
<filter kubernetes.journal.container>
  @type kubernetes_metadata
  use_journal true
</filter>

# rewrite_tag_filter does not support nested fields like
# kubernetes.container_name, so this exists to flatten the fields
# so we can use them in our rewrite_tag_filter
<filter kubernetes.journal.container>
  @type record_transformer
  enable_ruby true
  <record>
    kubernetes_namespace_container_name ${record["kubernetes"]["namespace_name"]}.${record["kubernetes"]["container_name"]}
  </record>
</filter>

# retag based on the namespace and container name of the log message
<match kubernetes.journal.container>
  @type rewrite_tag_filter
  # Update the tag have a structure of kube.<namespace>.<containername>
  <rule>
    key kubernetes_namespace_container_name
    pattern /^(.+)$/
    tag kube.$1
  </rule>
</match>

monitoring.conf

# Prometheus Exporter Plugin

# input plugin that exports metrics
<source>
  @type prometheus
</source>
<source>
  @type monitor_agent
</source>

# input plugin that collects metrics from MonitorAgent
<source>
  @type prometheus_monitor
  <labels>
    host ${hostname}
  </labels>
</source>

# input plugin that collects metrics for output plugin
<source>
  @type prometheus_output_monitor
  <labels>
    host ${hostname}
  </labels>
</source>

# input plugin that collects metrics for in_tail plugin
<source>
  @type prometheus_tail_monitor
  <labels>
    host ${hostname}
  </labels>
</source>

output.conf

<match **>
  @id elasticsearch
  @type elasticsearch
  @log_level info
  include_tag_key true
  type_name fluentd
  host "#{ENV['OUTPUT_HOST']}"
  port "#{ENV['OUTPUT_PORT']}"
  logstash_format true
  <buffer>
    @type file
    path /var/log/fluentd-buffers/kubernetes.system.buffer
    flush_mode interval
    retry_type exponential_backoff
    flush_thread_count 2
    flush_interval 5s
    retry_forever
    retry_max_interval 30
    chunk_limit_size "#{ENV['OUTPUT_BUFFER_CHUNK_LIMIT']}"
    queue_limit_length "#{ENV['OUTPUT_BUFFER_QUEUE_LIMIT']}"
    overflow_action block
  </buffer>
</match>

system.conf

<system>
  root_dir /tmp/fluentd-buffers/
</system>