Forwarding Logs to Cloudwatch Using Firelens

eksctl.yml

apiVersion: eksctl.io/v1alpha5
kind: ClusterConfig

metadata:
  name: logging-demo-cluster
  region: us-east-2

vpc:
  cidr: "10.1.0.0/16"
  clusterEndpoints:
    publicAccess:  true
    privateAccess: true

cloudWatch:
  clusterLogging:
    enableTypes: ["audit", "authenticator"]

nodeGroups:
  - name: NodeGroup1
    instanceType: t3.large
    desiredCapacity: 3
    privateNetworking: true
    tags:
      k8s.io/cluster-autoscaler/enabled: "true"
      k8s.io/cluster-autoscaler/appmesh-demo-cluster: "owned"
    iam:
      withAddonPolicies:
        ebs: true
        efs: true

iam:
  withOIDC: true
  serviceAccounts:
    - metadata:
        name: fluentbit
        namespace: fluentbit-system
      attachPolicy:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Resource: "*"
            Action:
              - "logs:CreateLogGroup"
              - "logs:PutRetentionPolicy"
              - "logs:CreateLogStream"
              - "logs:PutLogEvents"
              - "logs:DescribeLogStreams"
              - "logs:GetLogEvents"
              - "firehose:DeleteDeliveryStream"
              - "firehose:PutRecord"
              - "firehose:PutRecordBatch"
              - "firehose:UpdateDestination"
    - metadata:
        name: cluster-autoscaler
        namespace: kube-system
      attachPolicy:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Resource: "*"
            Action:
              - "autoscaling:DescribeAutoScalingGroups"
              - "autoscaling:DescribeAutoScalingInstances"
              - "autoscaling:DescribeLaunchConfigurations"
              - "autoscaling:DescribeTags"
              - "autoscaling:SetDesiredCapacity"
              - "autoscaling:TerminateInstanceInAutoScalingGroup"
    - metadata:
        name: alb-ingress-controller
        namespace: kube-system
      attachPolicy:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Resource: "*"
            Action:
              - "acm:DescribeCertificate"
              - "acm:ListCertificates"
              - "acm:GetCertificate"
              - "ec2:AuthorizeSecurityGroupIngress"
              - "ec2:CreateSecurityGroup"
              - "ec2:CreateTags"
              - "ec2:DeleteTags"
              - "ec2:DeleteSecurityGroup"
              - "ec2:DescribeAccountAttributes"
              - "ec2:DescribeAddresses"
              - "ec2:DescribeInstances"
              - "ec2:DescribeInstanceStatus"
              - "ec2:DescribeInternetGateways"
              - "ec2:DescribeNetworkInterfaces"
              - "ec2:DescribeSecurityGroups"
              - "ec2:DescribeSubnets"
              - "ec2:DescribeTags"
              - "ec2:DescribeVpcs"
              - "ec2:ModifyInstanceAttribute"
              - "ec2:ModifyNetworkInterfaceAttribute"
              - "ec2:RevokeSecurityGroupIngress"
              - "elasticloadbalancing:AddListenerCertificates"
              - "elasticloadbalancing:AddTags"
              - "elasticloadbalancing:CreateListener"
              - "elasticloadbalancing:CreateLoadBalancer"
              - "elasticloadbalancing:CreateRule"
              - "elasticloadbalancing:CreateTargetGroup"
              - "elasticloadbalancing:DeleteListener"
              - "elasticloadbalancing:DeleteLoadBalancer"
              - "elasticloadbalancing:DeleteRule"
              - "elasticloadbalancing:DeleteTargetGroup"
              - "elasticloadbalancing:DeregisterTargets"
              - "elasticloadbalancing:DescribeListenerCertificates"
              - "elasticloadbalancing:DescribeListeners"
              - "elasticloadbalancing:DescribeLoadBalancers"
              - "elasticloadbalancing:DescribeLoadBalancerAttributes"
              - "elasticloadbalancing:DescribeRules"
              - "elasticloadbalancing:DescribeSSLPolicies"
              - "elasticloadbalancing:DescribeTags"
              - "elasticloadbalancing:DescribeTargetGroups"
              - "elasticloadbalancing:DescribeTargetGroupAttributes"
              - "elasticloadbalancing:DescribeTargetHealth"
              - "elasticloadbalancing:ModifyListener"
              - "elasticloadbalancing:ModifyLoadBalancerAttributes"
              - "elasticloadbalancing:ModifyRule"
              - "elasticloadbalancing:ModifyTargetGroup"
              - "elasticloadbalancing:ModifyTargetGroupAttributes"
              - "elasticloadbalancing:RegisterTargets"
              - "elasticloadbalancing:RemoveListenerCertificates"
              - "elasticloadbalancing:RemoveTags"
              - "elasticloadbalancing:SetIpAddressType"
              - "elasticloadbalancing:SetSecurityGroups"
              - "elasticloadbalancing:SetSubnets"
              - "elasticloadbalancing:SetWebACL"
              - "iam:CreateServiceLinkedRole"
              - "iam:GetServerCertificate"
              - "iam:ListServerCertificates"
              - "waf-regional:GetWebACLForResource"
              - "waf-regional:GetWebACL"
              - "waf-regional:AssociateWebACL"
              - "waf-regional:DisassociateWebACL"
              - "tag:GetResources"
              - "tag:TagResources"
              - "waf:GetWebACL"

fluentbit.yml

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: fluentbit
rules:
  - apiGroups: [""]
    resources:
      - namespaces
      - pods
    verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: fluentbit
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: fluentbit
subjects:
  - kind: ServiceAccount
    name: fluentbit
    namespace: fluentbit-system
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: fluentbit-config
  namespace: fluentbit-system
  labels:
    app.kubernetes.io/name: fluentbit
data:
  fluent-bit.conf: |
    [SERVICE]
        Parsers_File /fluent-bit/parsers/parsers.conf

    [INPUT]
        Name              tail
        Tag               kube.*
        Path              /var/log/containers/*.log
        DB                /var/log/flb_kube.db
        Parser            docker
        Docker_Mode       On
        Mem_Buf_Limit     5MB
        Skip_Long_Lines   On
        Refresh_Interval  10

    [FILTER]
        Name                kubernetes
        Match               kube.*
        Kube_URL            https://kubernetes.default.svc.cluster.local:443
        Merge_Log           On
        Merge_Log_Key       data
        K8S-Logging.Parser  On
        K8S-Logging.Exclude On

    [OUTPUT]
        Name cloudwatch
        Match   **
        region us-east-2
        log_group_name fluentbit-cloudwatch
        log_stream_prefix fluentbit-
        auto_create_group true
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluentbit
  namespace: fluentbit-system
  labels:
    app.kubernetes.io/name: fluentbit
spec:
  selector:
    matchLabels:
      name: fluentbit
  template:
    metadata:
      labels:
        name: fluentbit
    spec:
      serviceAccountName: fluentbit
      containers:
        - name: aws-for-fluent-bit
          imagePullPolicy: Always
          image: amazon/aws-for-fluent-bit:latest
          volumeMounts:
            - name: varlog
              mountPath: /var/log
            - name: varlibdockercontainers
              mountPath: /var/lib/docker/containers
              readOnly: true
            - name: fluentbit-config
              mountPath: /fluent-bit/etc/
            - name: mnt
              mountPath: /mnt
              readOnly: true
          resources:
            limits:
              memory: 500Mi
            requests:
              cpu: 500m
              memory: 100Mi
      volumes:
        - name: varlog
          hostPath:
            path: /var/log
        - name: varlibdockercontainers
          hostPath:
            path: /var/lib/docker/containers
        - name: fluentbit-config
          configMap:
            name: fluentbit-config
        - name: mnt
          hostPath:
            path: /mnt