Filebeat + Dgraph Docker Exploration - First Attempt

README.md

Filebeat/Dgraph on Docker Exploration

This is a small experiment to see if I can get logs from Dgraph running on Docker. Steps to get this to work (Linux system with docker deamon running and docker-cli + docker-compose):

sudo su -
chown root:root filebeat.yml

# launch E-K
docker-compose up -d elasticsearch
docker-compose up -d kibana

# launch Filebeat when Kibana is read
docker-compose up -d filebeat

# launch Dgraph services
docker-compose up -d zero
docker-compose up -d alpha

Searching Logs

Through some tinkering with Kibana, I was able to run this URL to get at the Date + Container + logs.

http://localhost:5601/app/discover#/?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(columns:!(container.name,message),filters:!(),index:'filebeat-*',interval:auto,query:(language:kuery,query:''),sort:!())

Links

• FileBeat
  • Autodiscover
    • Autodiscover
    • What is Autodiscover for Filebeat? And why do we need it?
    • Filebeat Modules with Docker & Kubernetes
    • https://github.com/jimxliu/docker_elk_filebeat
    • Docker loggin using filebeat
  • Other
    • Make Filebeat read all stack trace lines as one entry
    • https://logz.io/blog/filebeat-vs-logstash/
• ElasticSearch
  • Encrypting communications in an Elasticsearch Docker Container
  • Built-in users
  • Run with Docker Compose
  • https://www.docker.elastic.co/
  • https://github.com/elastic/helm-charts
• Kibana
  • Configure Kibana
  • Install Kibana with Docker
  • https://github.com/elastic/dockerfiles/tree/7.12/kibana

docker-compose.yml

version: "3.5"
services:
  zero:
    image: dgraph/dgraph:${DGRAPH_VERSION}
    container_name: zero
    command: dgraph zero --my=zero:5080 --replicas 1 --idx 1
    labels:
      co.elastic.logs/enabled: true

  alpha:
    image: dgraph/dgraph:${DGRAPH_VERSION}
    container_name: alpha
    command: dgraph alpha --my=alpha:7080 --zero=zero:5080
    labels:
      co.elastic.logs/enabled: true
    ports:
      - 8080:8080
      - 9080:9080

  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
    container_name: elasticsearch
    labels:
      co.elastic.logs/module: elasticsearch
    environment:
      discovery.type: single-node
      bootstrap.memory_lock: "true"
      ES_JAVA_OPTS: "-Xms750m -Xmx750m"
    ports:
      - 9200:9200

  kibana:
    image: docker.elastic.co/kibana/kibana:${ELASTIC_VERSION}
    container_name: kibana
    labels:
      co.elastic.logs/module: kibana
    depends_on:
      - elasticsearch
    ports:
      - 5601:5601

  filebeat:
    image: docker.elastic.co/beats/filebeat:${ELASTIC_VERSION}
    container_name: filebeat
    command: filebeat -e -strict.perms=false
    privileged: true 
    user: root
    volumes:
      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml
      - /var/lib/docker/containers:/var/lib/docker/containers
      - /var/run/docker.sock:/var/run/docker.sock

example_event.json

{
  "_index": "filebeat-7.12.0-2021.03.28-000001",
  "_type": "_doc",
  "_id": "5aw_e3gBGKxZUvo67lzo",
  "_version": 1,
  "_score": null,
  "fields": {
    "docker.container.labels.com_docker_compose_container-number": [
      "1"
    ],
    "container.labels.com_docker_compose_project_working_dir": [
      "/home/joaquin/area51/scratch/filebeat"
    ],
    "container.labels.com_docker_compose_config-hash": [
      "b12183b264ec829b57cb0f02bd692df1e1d079f2b98825b2c883f6ff2ca7c8e6"
    ],
    "container.labels.com_docker_compose_service": [
      "zero"
    ],
    "docker.container.labels.com_docker_compose_oneoff": [
      "False"
    ],
    "docker.container.labels.com_docker_compose_version": [
      "1.25.0"
    ],
    "docker.container.labels.com_docker_compose_project_config_files": [
      "docker-compose.yml"
    ],
    "container.id": [
      "c3a8edf179d8848235059eb2ade1bd1faab469f787bf547fdf770e0ea4b90083"
    ],
    "agent.type": [
      "filebeat"
    ],
    "docker.container.labels.co_elastic_logs/enabled": [
      "True"
    ],
    "container.name": [
      "zero"
    ],
    "container.image.name": [
      "dgraph/dgraph:v20.11.2"
    ],
    "stream": [
      "stderr"
    ],
    "agent.name": [
      "0de235cf1d47"
    ],
    "container.labels.co_elastic_logs/enabled": [
      "True"
    ],
    "host.name": [
      "0de235cf1d47"
    ],
    "docker.container.labels.com_docker_compose_service": [
      "zero"
    ],
    "docker.container.labels.maintainer": [
      "Dgraph Labs <contact@dgraph.io>"
    ],
    "docker.container.labels.com_docker_compose_project_working_dir": [
      "/home/joaquin/area51/scratch/filebeat"
    ],
    "input.type": [
      "container"
    ],
    "log.offset": [
      8997
    ],
    "agent.hostname": [
      "0de235cf1d47"
    ],
    "message": [
      "W0328 23:51:25.235656      19 pool.go:267] Connection lost with alpha:7080. Error: rpc error: code = Unavailable desc = transport is closing"
    ],
    "container.labels.com_docker_compose_version": [
      "1.25.0"
    ],
    "docker.container.labels.com_docker_compose_project": [
      "filebeat"
    ],
    "container.labels.com_docker_compose_oneoff": [
      "False"
    ],
    "container.labels.com_docker_compose_project": [
      "filebeat"
    ],
    "container.labels.maintainer": [
      "Dgraph Labs <contact@dgraph.io>"
    ],
    "@timestamp": [
      "2021-03-28T23:51:25.235Z"
    ],
    "agent.id": [
      "fbff0850-502a-4752-86ad-5d7cb9ba397a"
    ],
    "ecs.version": [
      "1.8.0"
    ],
    "log.file.path": [
      "/var/lib/docker/containers/c3a8edf179d8848235059eb2ade1bd1faab469f787bf547fdf770e0ea4b90083/c3a8edf179d8848235059eb2ade1bd1faab469f787bf547fdf770e0ea4b90083-json.log"
    ],
    "docker.container.labels.com_docker_compose_config-hash": [
      "b12183b264ec829b57cb0f02bd692df1e1d079f2b98825b2c883f6ff2ca7c8e6"
    ],
    "agent.ephemeral_id": [
      "28d869a2-6ae6-4c47-863d-13cf0d98e0a7"
    ],
    "container.labels.com_docker_compose_container-number": [
      "1"
    ],
    "agent.version": [
      "7.12.0"
    ],
    "container.labels.com_docker_compose_project_config_files": [
      "docker-compose.yml"
    ]
  },
  "highlight": {
    "container.name": [
      "@kibana-highlighted-field@zero@/kibana-highlighted-field@"
    ]
  },
  "sort": [
    1888
  ]
}

filebeat.yml

filebeat.config:
  modules:
    path: ${path.config}/modules.d/*.yml
    reload.enabled: false

filebeat.modules:
  - module: elasticsearch
  - module: kibana

filebeat.autodiscover:
  providers:
    - type: docker
      hints:
         enabled: true
         default_config:
           enabled: false

processors:
- add_cloud_metadata: ~
- add_docker_metadata:
    host: unix:///var/run/docker.sock

output.elasticsearch:
  hosts: ["http://elasticsearch:9200"]

setup.kibana:
  host: http://kibana:5601

setup.dashboards.enabled: true