EFK stack

README.md

EFK stack

• Elasticsearch

• Fluent Bit

• Kibana

Prerequisites

Network

• web

docker network create \
  --subnet 172.23.0.0/16 \
  --gateway 172.23.0.1 \
  --ipv6 \
  --subnet fc00:1:a1a1::/64 \
  --gateway fc00:1:a1a1::1 \
  web

Volumes

• elasticsearch

docker volume create elasticsearch

• kibana

docker volume create kibana

Usage

• Start the services

docker compose up -d

Security

• Create(reset) a password for the elastic user

docker compose exec -it elasticsearch /usr/share/elasticsearch/bin/elasticsearch-reset-password -u elastic

• Create(reset) the registration kibana token

docker compose exec -it elasticsearch /usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s kibana

compose.yaml

services:
  fluentbit:
    container_name: fluentbit
    # command: ["fluent-bit", "-c", "/fluent-bit/etc/fluent-bit.yaml"]
    restart: unless-stopped
    image: cr.fluentbit.io/fluent/fluent-bit:latest
    pull_policy: always
    configs:
      - source: fb_config
        target: /fluent-bit/etc/fluent-bit.conf
      - source: fb_parsers
        target: /fluent-bit/etc/parsers.conf
    ports:
      - 24224:24224
      - 24224:24224/udp
    networks:
      - web
  elasticsearch:
    container_name: elasticsearch
    depends_on:
      - fluentbit
    deploy:
      resources:
        limits:
          memory: 4gb
    restart: unless-stopped
    image: elasticsearch:8.13.0
    pull_policy: missing
    volumes:
      - elasticsearch:/usr/share/elasticsearch/data
    env_file: elasticsearch.env
    networks:
      - web
  kibana:
    container_name: kibana
    depends_on:
      - elasticsearch
    deploy:
      resources:
        limits:
          memory: 4gb
    restart: unless-stopped
    image: kibana:8.13.0
    pull_policy: missing
    volumes:
      - kibana:/usr/share/kibana/data
    env_file: kibana.env
    networks:
      - web
configs:
  fb_config:
    file: ./fluent-bit.conf
  fb_parsers:
    file: ./parsers.conf
networks:
  web:
    external: true
volumes:
  elasticsearch:
    external: true
  kibana:
    external: true

elasticsearch.env

# https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery-settings.html
discovery.type=single-node
# https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html
xpack.security.enabled=false

fluent-bit.conf

# https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/configuration-file#config_section
[SERVICE]
  Parsers_File parsers.conf
  HTTP_Server On
  HTTP_Listen 0.0.0.0
  HTTP_PORT 2020
  Health_Check On
  Log_level debug
# https://docs.fluentbit.io/manual/pipeline/inputs/forward
[INPUT]
  Name forward
  Listen 0.0.0.0
  Port 24224
  Buffer_Chunk_Size 1M
  Buffer_Max_Size 6M
# https://docs.fluentbit.io/manual/pipeline/filters/parser
[FILTER]
  Name parser
  Match *
  Parser docker
  Key_Name log
  Reserve_Data True
# https://docs.fluentbit.io/manual/pipeline/outputs/elasticsearch
[OUTPUT]
  Name es
  Host elasticsearch
  Suppress_Type_Name On
  Port 9200
  Match *
  Index fluentbit

kibana.env

# https://www.elastic.co/guide/en/kibana/current/docker.html
ELASTICSEARCH_HOSTS=http://elasticsearch:9200
ELASTIC_PASSWORD=<ELASTIC_PASSWORD>

parsers.conf

# https://docs.fluentbit.io/manual/pipeline/parsers/json
[PARSER]
  Name docker
  Format json
  Time_Key time
  Time_Format %Y-%m-%dT%H:%M:%S %z