Monitoring Elastic Stack

docker-compose.yml

version: "3"
services:

  kibana:
    image: docker.elastic.co/kibana/kibana:6.2.3
    ports:
      - 5601:5601
    depends_on:
      - elasticsearch
      
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:6.2.3
    expose:
      - 9200
    environment:
      - cluster.name=test-cluster
      - node.name=master
    volumes:
      - esdata1:/usr/share/elasticsearch/data

  elasticsearch2:
    image: docker.elastic.co/elasticsearch/elasticsearch:6.2.3
    expose:
      - 9200
    environment:
      - cluster.name=test-cluster
      - node.name=slave
      - "discovery.zen.ping.unicast.hosts=elasticsearch"
    depends_on:
      - elasticsearch
    volumes:
      - esdata2:/usr/share/elasticsearch/data

  metrics:
    image: docker.elastic.co/beats/metricbeat:6.2.3
    command: "-system.hostfs=/hostfs"
    user: root
    network_mode: "host"
    depends_on:
      - elasticsearch
    environment:
      ELASTICSEARCH_URL: http://elasticsearch:9200
    volumes:
      - ./metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro
      - /proc:/hostfs/proc:ro
      - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro
      - /:/hostfs:ro

  logs:
    image: docker.elastic.co/beats/filebeat:6.2.3
    user: root
    command: "--strict.perms=false -v -e -d autodiscover,docker"
    depends_on:
      - elasticsearch
    environment:
      ELASTICSEARCH_URL: http://elasticsearch:9200
    volumes:
      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml
      - /var/log/messages:/var/mounted/logs/syslog:ro

volumes:
  esdata1:
    driver: local
  esdata2:
    driver: local

filebeat.yml

# this file sets up filebeat to watch logs at a certain path
filebeat.prospectors:
  - input_type: log
    paths:
      - /var/mounted/logs/*

filebeat.modules:
- module: system

output.elasticsearch:
  hosts: ["elasticsearch:9200"]
setup.kibana:
  host: "kibana:5601"

metricbeat.yml

# sets up metricbeat for monitoring system resources
metricbeat.modules:
- module: system
  period: 10s
  metricsets:
    - cpu
    - load
    - memory
    - network
    - process
    - process_summary
    - core
    - diskio
    - socket
  processes: ['.*']
  process.include_top_n:
    by_cpu: 5      # include top 5 processes by CPU
    by_memory: 5   # include top 5 processes by memory

- module: system
  period: 1m
  metricsets:
    - filesystem
    - fsstat
  processors:
  - drop_event.when.regexp:
      system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)'

- module: system
  period: 15m
  metricsets:
    - uptime

output.elasticsearch:
  hosts: ["elasticsearch:9200"]
setup.kibana:
  host: "kibana:5601"