fluentd-logging-kubernetes.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: fluentd-config
  namespace: fluentd
  labels:
    app: fluentd
data:
  fluentd.conf: |
    @include prometheus.conf
    @include systemd.conf
    @include kubernetes.conf
    @include apiserver-audit.conf

    # Prevent fluentd from handling records containing its own logs. Otherwise
    # it can lead to an infinite loop, when error in sending one message generates
    # another message which also fails to be sent and so on.
    <match fluent.**>
      type null
    </match>

    # Used for health checking
    <source>
      @type http
      port 9880
      bind 0.0.0.0
    </source>

    # Emits internal metrics to every minute, and also exposes them on port
    # 24220. Useful for determining if an output plugin is retryring/erroring,
    # or determining the buffer queue length.
    <source>
      @type monitor_agent
      bind 0.0.0.0
      port 24220
      tag fluentd.monitor.metrics
    </source>

    @include elasticsearch-output.conf

  prometheus.conf: |
    # input plugin that is required to expose metrics by other prometheus
    # plugins, such as the prometheus_monitor input below.
    <source>
      @type prometheus
      bind 0.0.0.0
      port 24231
      metrics_path /metrics
    </source>

    # input plugin that collects metrics from MonitorAgent and exposes them
    # as prometheus metrics
    <source>
      @type prometheus_monitor
      # update the metrics every 5 seconds
      interval 5
    </source>

    <source>
      @type prometheus_output_monitor
      interval 5
    </source>

    <source>
      @type prometheus_tail_monitor
      interval 5
    </source>

  systemd.conf: |
    <source>
      @type systemd
      pos_file /var/log/fluentd-journald-systemd.pos
      read_from_head true
      strip_underscores true
      tag systemd
    </source>

    <match systemd>
      @type rewrite_tag_filter
      rewriterule1 SYSTEMD_UNIT   ^(.+).service$  systemd.$1
      rewriterule2 SYSTEMD_UNIT   !^(.+).service$ systemd.unmatched
    </match>

    <filter kubelet>
      type parser
      format kubernetes
      reserve_data true
      key_name MESSAGE
    </filter>

    <filter docker>
      type parser
      format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
      reserve_data true
      key_name MESSAGE
      suppress_parse_error_log true
      ignore_key_not_exist true
    </filter>

    # Filter filter ssh logs since it's mostly bots trying to login
    <filter systemd.**>
      @type grep
      exclude1 SYSTEMD_UNIT (sshd@.*\.service)
    </filter>

  kubernetes.conf: |
    # Capture Kubernetes pod logs
    # The kubelet creates symlinks that capture the pod name, namespace,
    # container name & Docker container ID to the docker logs for pods in the
    # /var/log/containers directory on the host.
    <source>
      type tail
      path /var/log/containers/*.log
      pos_file /var/log/fluentd-containers.log.pos
      time_format %Y-%m-%dT%H:%M:%S.%NZ
      tag kubernetes.*
      format json
      read_from_head true
    </source>

    # Query the API for extra metadata.
    <filter kubernetes.**>
      type kubernetes_metadata
      # If the logs begin with '{' and end with '}' then it's JSON so merge
      # the JSON log field into the log event
      merge_json_log true
      preserve_json_log true
    </filter>

    # rewrite_tag_filter does not support nested fields like
    # kubernetes.container_name, so this exists to flatten the fields
    # so we can use them in our rewrite_tag_filter
    <filter kubernetes.**>
      @type record_transformer
      enable_ruby true
      <record>
        kubernetes_namespace_container_name ${record["kubernetes"]["namespace_name"]}.${record["kubernetes"]["container_name"]}
      </record>
    </filter>

    # retag based on the container name of the log message
    <match kubernetes.**>
      @type rewrite_tag_filter
      rewriterule1 kubernetes_namespace_container_name  ^(.+)$ kube.$1
    </match>

    # Remove the unnecessary field as the information is already available on
    # other fields.
    <filter kube.**>
      @type record_transformer
      remove_keys kubernetes_namespace_container_name
    </filter>

    <filter kube.kube-system.**>
      type parser
      format kubernetes
      reserve_data true
      key_name log
      suppress_parse_error_log true
      ignore_key_not_exist true
    </filter>

  apiserver-audit.conf: |
    # Example:
    # 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods"
    # 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200"
    <source>
      type tail
      format multiline
      multiline_flush_interval 5s
      format_firstline /^\S+\s+AUDIT:/
      # Fields must be explicitly captured by name to be parsed into the record.
      # Fields may not always be present, and order may change, so this just looks
      # for a list of key="\"quoted\" value" pairs separated by spaces.
      # Unknown fields are ignored.
      # Note: We can't separate query/response lines as format1/format2 because
      #       they don't always come one after the other for a given query.
      # TODO: Maybe add a JSON output mode to audit log so we can get rid of this?
      format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/
      time_format %FT%T.%L%Z
      path /var/log/kube-apiserver-audit.log
      pos_file /var/log/kube-apiserver-audit.log.pos
      tag kube-apiserver-audit
    </source>

  elasticsearch-output.conf: |
    <match **>
       type elasticsearch
       log_level info
       include_tag_key true
       # Replace with the host/port to your elasticsearch cluster.
       # This assumes a service `elasticsearch` exists in the default namespace
       host elasticsearch.default.svc.cluster.local
       port 9200
       logstash_format true
       buffer_chunk_limit 2M
       buffer_queue_limit 32
       flush_interval 5s
       max_retry_wait 30
       disable_retry_limit
       num_threads 8

       template_file /fluentd/etc/elasticsearch-template-es5x.json
       template_name elasticsearch-template-es5x.json
    </match>

  elasticsearch-template-es5x.json: |
    {
      "template" : "logstash-*",
      "version" : 50001,
      "settings" : {
        "index.refresh_interval" : "5s",
        "number_of_shards": 1
      },
      "mappings" : {
        "_default_" : {
          "_all" : {"enabled" : true, "norms" : false},
          "dynamic_templates" : [ {
            "message_field" : {
              "path_match" : "message",
              "match_mapping_type" : "string",
              "mapping" : {
                "type" : "text",
                "norms" : false
              }
            }
          }, {
            "string_fields" : {
              "match" : "*",
              "match_mapping_type" : "string",
              "mapping" : {
                "type" : "text", "norms" : false,
                "fields" : {
                  "keyword" : { "type": "keyword" }
                }
              }
            }
          } ],
          "properties" : {
            "@timestamp": { "type": "date", "include_in_all": false },
            "@version": { "type": "keyword", "include_in_all": false },
            "geoip"  : {
              "dynamic": true,
              "properties" : {
                "ip": { "type": "ip" },
                "location" : { "type" : "geo_point" },
                "latitude" : { "type" : "half_float" },
                "longitude" : { "type" : "half_float" }
              }
            }
          }
        }
      }
    }

---

apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: fluentd
  namespace: fluentd
  labels:
    app: fluentd
spec:
  minReadySeconds: 10
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  template:
    metadata:
      labels:
        app: fluentd
    spec:
      containers:
      - name: fluentd
        image: quay.io/coreos/fluentd-kubernetes:v0.12-debian-elasticsearch
        command: ["sh", "-c", "gem install fluent-plugin-rewrite-tag-filter fluent-plugin-prometheus && fluentd -c /fluentd/etc/fluentd.conf -p /fluentd/plugins"]
        # command: ["fluentd", "-c", "/fluentd/etc/fluentd.conf", "-p", "/fluentd/plugins"]
        resources:
          limits:
            memory: 200Mi
          requests:
            cpu: 500m
            memory: 200Mi
        ports:
        - name: prom-metrics
          containerPort: 24231
          protocol: TCP
        - name: monitor-agent
          containerPort: 24220
          protocol: TCP
        - name: http-input
          containerPort: 9880
          protocol: TCP
        livenessProbe:
          httpGet:
            # Use percent encoding for query param.
            # The value is {"log": "health check"}.
            # the endpoint itself results in a new fluentd
            # tag 'fluentd.pod-healthcheck'
            path: /fluentd.pod.healthcheck?json=%7B%22log%22%3A+%22health+check%22%7D
            port: 9880
          initialDelaySeconds: 5
          timeoutSeconds: 1
        volumeMounts:
        - name: varlog
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: fluentd-config
          mountPath: /fluentd/etc
      volumes:
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: fluentd-config
        configMap:
          name: "fluentd-config"
      terminationGracePeriodSeconds: 30
      serviceAccountName: fluentd
      tolerations:
      - key: node-role.kubernetes.io/master
        operator: Exists
        effect: NoSchedule

---

apiVersion: v1
kind: Namespace
metadata:
  name: fluentd

---

apiVersion: monitoring.coreos.com/v1alpha1
kind: ServiceMonitor
metadata:
  name: fluentd
  namespace: fluentd
  labels:
    app: fluentd
spec:
  jobLabel: app
  selector:
    matchLabels:
      app: fluentd
  namespaceSelector:
    any: false
    matchNames:
    - fluentd
  endpoints:
  - port: prometheus-metrics
    interval: 15s

---

apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: fluentd-read
  namespace: fluentd
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: fluentd-read
subjects:
- kind: ServiceAccount
  name: fluentd
  namespace: fluentd

---

apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: fluentd-read
  namespace: fluentd
rules:
- apiGroups: [""]
  resources:
  - namespaces
  - pods
  verbs: ["get", "list", "watch"]

---

apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluentd
  namespace: fluentd

---

apiVersion: v1
kind: Service
metadata:
  name: fluentd
  namespace: fluentd
  labels:
    app: fluentd
spec:
  type: ClusterIP
  clusterIP: None
  selector:
    app: fluentd
  ports:
  # Exposes Prometheus metrics
  - name: prometheus-metrics
    port: 24231
    targetPort: prom-metrics
    protocol: TCP
  # Can be accessed using "kubectl proxy" at:
  # http://127.0.0.1:8001/api/v1/proxy/namespaces/kube-system/services/fluentd:monitor-agent/api/plugins.json
  - name: monitor-agent
    port: 24220
    targetPort: monitor-agent
    protocol: TCP

---

apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: prometheus-fluentd
  namespace: fluentd
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus-k8s
subjects:
- kind: ServiceAccount
  name: prometheus-fluentd
  namespace: fluentd

---

apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus-fluentd
  namespace: fluentd

---

apiVersion: monitoring.coreos.com/v1alpha1
kind: Prometheus
metadata:
  name: fluentd
  namespace: fluentd
  labels:
    prometheus: fluentd
spec:
  replicas: 1
  serviceAccountName: prometheus-fluentd
  serviceMonitorSelector:
    matchLabels:
      app: fluentd
  resources:
    requests:
      memory: 250Mi

---