EFK Stack: Logs Beyond Debugging

cluster-role-binding.yaml

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: fluentd
roleRef:
  kind: ClusterRole
  name: fluentd
  apiGroup: rbac.authorization.k8s.io
subjects:
  - kind: ServiceAccount
    name: fluentd
    namespace: kube-system

cluster-role.yaml

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: fluentd
  namespace: kube-system
rules:
  - apiGroups:
      - ""
    resources:
      - pods
      - namespaces
    verbs:
      - get
      - list
      - watch

config-map.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: fluentd
  namespace: kube-system
data:
  fluent.conf: |-

    <source>
      @type tail
      @id in_tail_container_logs
      path "/var/log/containers/efk*.log"
      pos_file "/var/log/fluentd-containers.log.pos"
      tag efk
      read_from_head true
      <parse>
        @type regexp
        expression /LogRecord\((type=(?<type>[^\"][\W\s\w\[\]`!@#$%\^&*()={}:;<>+'-]*)?, uuid=(?<uuid>[^\"][\W\s\w\[\]`!@#$%\^&*()={}:;<>+'-]*)?, created=(?<created>[^\"][\W\s\w\[\]`!@#$%\^&*()={}:;<>+'-]*)?)\)/
        time_format %Y-%m-%dT%H:%M:%S.%NZ
      </parse>
    </source>

    <match efk>
      @type elasticsearch
      @id out_es
      @log_level info
      include_tag_key true
      host "#{ENV["FLUENT_ELASTICSEARCH_HOST"]}"
      port "#{ENV["FLUENT_ELASTICSEARCH_PORT"]}"
      scheme "#{ENV["FLUENT_ELASTICSEARCH_SCHEME"]}"
      ssl_verify true
      reload_connections true
      index_name ${tag}-%Y-%m
      type_name fluentd
      include_timestamp true

      <buffer tag, time>
        @type memory
        timekey 3600
      </buffer>
    </match>

daemon-set.yaml

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluentd
  namespace: kube-system
  labels:
    k8s-app: fluentd-logging
    version: v1
    kubernetes.io/cluster-service: "true"
spec:
  selector:
    matchLabels:
      k8s-app: fluentd-logging
      version: v1
  template:
    metadata:
      labels:
        k8s-app: fluentd-logging
        version: v1
        kubernetes.io/cluster-service: "true"
    spec:
      serviceAccount: fluentd
      serviceAccountName: fluentd
      containers:
        - name: fluentd
          image: fluent/fluentd-kubernetes-daemonset:v1-debian-elasticsearch
          env:
            - name:  FLUENT_ELASTICSEARCH_HOST
              value: "elasticsearch.efk"
            - name:  FLUENT_ELASTICSEARCH_PORT
              value: "9200"
            - name: FLUENT_ELASTICSEARCH_SCHEME
              value: "http"
            - name: FLUENT_UID
              value: "0"
            - name: FLUENT_ELASTICSEARCH_USER
              valueFrom:
                secretKeyRef:
                  name: fluentd
                  key: ELASTICSEARCH_USER
            - name: FLUENT_ELASTICSEARCH_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: fluentd
                  key: ELASTICSEARCH_PASSWORD
          resources:
            limits:
              memory: 200Mi
            requests:
              cpu: 100m
              memory: 200Mi
          securityContext:
            privileged: true
          volumeMounts:
            - name: varlog
              mountPath: /var/log
            - name: varlibdockercontainers
              mountPath: /var/lib/docker/containers
            - name: config
              mountPath: /fluentd/etc/fluent.conf
              subPath: fluent.conf
      volumes:
        - name: varlog
          hostPath:
            path: /var/log
        - name: varlibdockercontainers
          hostPath:
            path: /var/lib/docker/containers
        - name: config
          configMap:
            name: fluentd

EFK Stack: Logs Beyond Debugging

Article

elastic.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: elasticsearch
  namespace: efk
spec:
  selector:
    matchLabels:
      component: elasticsearch
  template:
    metadata:
      labels:
        component: elasticsearch
    spec:
      containers:
        - name: elasticsearch
          image: docker.elastic.co/elasticsearch/elasticsearch:6.5.4
          env:
            - name: discovery.type
              value: single-node
          ports:
            - containerPort: 9200
              name: http
              protocol: TCP
          resources:
            limits:
              cpu: 500m
              memory: 4Gi
            requests:
              cpu: 500m
              memory: 4Gi

---

apiVersion: v1
kind: Service
metadata:
  name: elasticsearch
  namespace: efk
  labels:
    service: elasticsearch
spec:
  type: NodePort
  selector:
    component: elasticsearch
  ports:
    - port: 9200
      targetPort: 9200

kibana.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: kibana
  namespace: efk
spec:
  selector:
    matchLabels:
      run: kibana
  template:
    metadata:
      labels:
        run: kibana
    spec:
      containers:
        - name: kibana
          image: docker.elastic.co/kibana/kibana:6.5.4
          env:
            - name: ELASTICSEARCH_URL
              value: http://elasticsearch:9200
            - name: XPACK_SECURITY_ENABLED
              value: "true"
          ports:
            - containerPort: 5601
              name: http
              protocol: TCP

---

apiVersion: v1
kind: Service
metadata:
  name: kibana
  namespace: efk
  labels:
    service: kibana
spec:
  type: NodePort
  selector:
    run: kibana
  ports:
    - port: 5601
      targetPort: 5601

LogRecord.kt

package com.github.biancacristina.data

import java.time.LocalDateTime

data class LogRecord (
    val type: String,
    val uuid: String,
    val created: LocalDateTime = LocalDateTime.now()
)

namespace.yaml

apiVersion: v1
kind: Namespace
metadata:
  name: efk

secret.yaml

apiVersion: v1
kind: Secret
metadata:
  name: fluentd
  namespace: kube-system
type: Opaque
data:
  ELASTICSEARCH_USER: bm9uZQ==
  ELASTICSEARCH_PASSWORD: bm9uZQ==

service-account.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluentd
  namespace: kube-system

UserController.kt

package com.github.biancacristina.controller

import com.github.biancacristina.data.request.UserRequest
import com.github.biancacristina.data.response.UserResponse
import com.github.biancacristina.service.UserService
import io.micronaut.http.annotation.Body
import io.micronaut.http.annotation.Controller
import io.micronaut.http.annotation.Delete
import io.micronaut.http.annotation.Post

@Controller("/users")
class UserController(private val userService: UserService) {

    @Post
    fun save(@Body request: UserRequest): UserResponse {
        return userService.save(request)
    }

    @Delete("/{uuid}")
    fun delete(uuid: String) {
        userService.delete(uuid)
    }
}

UserRequest.kt

package com.github.biancacristina.data.request

import io.micronaut.core.annotation.Introspected

@Introspected
data class UserRequest (
    val username: String,
    val name: String
)

UserService.kt

package com.github.biancacristina.service

import com.github.biancacristina.data.LogRecord
import com.github.biancacristina.data.model.User
import com.github.biancacristina.data.request.UserRequest
import com.github.biancacristina.data.response.UserResponse
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.util.*
import javax.inject.Singleton

@Singleton
class UserService {
    private val logger: Logger = LoggerFactory.getLogger(UserService::class.java)

    fun save(request: UserRequest): UserResponse {
        val user: User = User(uuid = UUID.randomUUID().toString(),
                              username = request.username,
                              name = request.name)
        logger.info(LogRecord(type = "SAVE_USER", uuid = user.uuid).toString())
        return UserResponse.from(user)
    }

    fun delete(uuid: String) {
        logger.info(LogRecord(type = "DELETE_USER", uuid = uuid).toString())
    }
}