Fluent Bit log forwarding to CloudWatch

README.md

Set Up Fluent Bit as a DaemonSet to Send Logs to CloudWatch

Create a new 1.13 or 1.14 EKS cluster called container-insights.

Enable IRSA:

eksctl utils associate-iam-oidc-provider \
               --name container-insights \
               --approve

Create a namespace:

kubectl create ns cw

Create SA with arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy policy:

eksctl create iamserviceaccount \
                --name fluentbitds \
                --namespace cw \
                --cluster container-insights \
                --attach-policy-arn arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy \
                --approve

Set up Fluent Bit as DS:

kubectl apply -f eks-fluent-bit-configmap.yaml
kubectl apply -f eks-fluent-bit-daemonset-rbac.yaml
kubectl apply -f eks-fluent-bit-daemonset.yaml

Set up NGINX for generating logs:

kubectl apply -f eks-nginx-app.yaml

Verify if all is running:

kubectl get po,ds,cm
NAME                       READY   STATUS    RESTARTS   AGE
pod/fluentbit-bkntf        1/1     Running   0          6m11s
pod/nginx-8c5ddb5c-576hm   1/1     Running   0          6m4s

NAME                             DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE
daemonset.extensions/fluentbit   1         1         1       1            1           <none>          6m12s

NAME                          DATA   AGE
configmap/fluent-bit-config   2      6m32s

Generate load on NGINX to produce logs, for example using kubectl port-forward service/nginx 9090:80 and then curl localhost:9090.

eks-fluent-bit-configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: fluent-bit-config
  namespace: cw
  labels:
    app.kubernetes.io/name: fluentbit
data:
  fluent-bit.conf: |
    [SERVICE]
        Parsers_File  parsers.conf
    [INPUT]
        Name              tail
        Tag               kube.*
        Path              /var/log/containers/*.log
        Parser            docker
        DB                /var/log/flb_kube.db
        Mem_Buf_Limit     5MB
        Skip_Long_Lines   On
        Refresh_Interval  10
    [FILTER]
        Name parser
        Match **
        Parser nginx
        Key_Name log
    [OUTPUT]
        Name cloudwatch
        Match *
        region us-west-2 
        log_group_name fluent-bit-cloudwatch
        log_stream_prefix from-fluent-bit-
        auto_create_group true
  parsers.conf: |
    [PARSER]
        Name   nginx
        Format regex
        Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")? \"-\"$
        Time_Key time
        Time_Format %d/%b/%Y:%H:%M:%S %z
    [PARSER]
        Name        docker
        Format      json
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L
        Time_Keep   On
        # Command      |  Decoder | Field | Optional Action
        # =============|==================|=================
        Decode_Field_As   escaped    log

eks-fluent-bit-daemonset-rbac.yaml

apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: pod-log-reader
rules:
- apiGroups: [""]
  resources:
  - namespaces
  - pods
  verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: pod-log-crb
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: pod-log-reader
subjects:
- kind: ServiceAccount
  name: fluentbitds
  namespace: cw

eks-fluent-bit-daemonset.yaml

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluentbit
  namespace: cw
  labels:
    app.kubernetes.io/name: fluentbit
spec:
  selector:
    matchLabels:
      name: fluentbit
  template:
    metadata:
      labels:
        name: fluentbit
    spec:
      serviceAccountName: fluentbitds
      containers:
      - name: aws-for-fluent-bit
        image: 906394416424.dkr.ecr.us-west-2.amazonaws.com/aws-for-fluent-bit:latest
        volumeMounts:
        - name: varlog
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: fluent-bit-config
          mountPath: /fluent-bit/etc/
        - name: mnt
          mountPath: /mnt
          readOnly: true
        resources:
          limits:
            memory: 500Mi
          requests:
            cpu: 500m
            memory: 100Mi
      volumes:
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: fluent-bit-config
        configMap:
          name: fluent-bit-config
      - name: mnt
        hostPath:
          path: /mnt

eks-nginx-app.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
  namespace: cw
  labels:
    app.kubernetes.io/name: nginx
spec:
  replicas: 1 
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx:1.17
        ports:
        - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: nginx
  labels:
    app: nginx
spec:
  ports:
  - port: 80
    targetPort: 80
  selector:
    app: nginx

@mhausenblas Do workloads have to be in the same namespace for this to work?