Observability Compose

readme.md

observability-stack

alternative (mostly opensource) observability stack

• Kibana as a events front end
• Grafana as a metrics front end
• Events and logs served by Elasticsearch
• Metrics by Prometheus
• Distributed tracing by Jaeger
• Deployed as a Docker stack to Amazon
• Traefik for a Docker native reverse proxy
• Oroborous for dynamic container updates
• Test rig provided by Uber's hotROD

note: this assumes resolution for observability.local is applied to the docker host and it's various backends

quickstart

1. docker-compose up
2. wait a few moments
3. open http://kibana.observability.local

docker-compose.yml

---
version: "3"
services:
  # utility
  ouroboros:
    image: pyouroboros/ouroboros
    container_name: ouroboros
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    environment:
      - CLEANUP=true
      - SELF_UPDATE=true

  # router
  reverse-proxy:
    image: traefik:latest
    command: --api --docker --docker.watch --docker.domain=observability.local
    container_name: traefik
    hostname: traefik
    ports:
      - "80:80"
      - "443:443"
      - "8000:8000"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - /traefik/traefik.toml:/etc/traefik/traefik.toml
    labels:
      - "traefik.enable=true"
      - "traefik.backend=traefik"
      - "traefik.frontend.rule=Host:traefik.observability.local"
      - "traefik.port=8080"

  # elastic-stack
  elasticsearch:
    image: elasticsearch:6.6.1
    container_name: elasticsearch
    restart: unless-stopped
    ports:
      - "9200:9200"
    volumes:
      - "elastic_data:/usr/share/elasticsearch/data"
    labels:
      - "traefik.enable=true"
      - "traefik.frontend.rule=Host:elasticsearch.observability.local"

  kibana:
    image: elastic/kibana:6.6.1
    container_name: kibana
    restart: unless-stopped
    ports:
      - "5601:5601"
    depends_on:
      - "elasticsearch"
    labels:
      - "traefik.enable=true"
      - "traefik.frontend.rule=Host:kibana.observability.local"

  # jaeger
  jaeger:
    image: jaegertracing/all-in-one:latest
    container_name: jaeger
    restart: unless-stopped
    ports:
      - "16686:16686" # ui
      - "5775:5775/udp"
      - "6831:6831/udp"
      - "6832:6832/udp"
      - "5778:5778"
      - "14268:14268"
      - "9411:9411"
    environment:
      - COLLECTOR_ZIPKIN_HTTP_PORT=9411
    labels:
      - "traefik.enable=true"
      - "traefik.frontend.rule=Host:jaeger.observability.local"
      - "traefik.port=16686"

  # prometheus
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    restart: unless-stopped
    ports:
      - "9090:9090"
    volumes:
      - "prometheus_data:/prometheus_data"
    labels:
      - "traefik.enable=true"
      - "traefik.frontend.rule=Host:prometheus.observability.local"


  # grafana
  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    restart: unless-stopped
    ports:
      - "3000:3000"
    environment:
      - GF_SERVER_DOMAIN=grafana.observability.local
    volumes:
      - grafana_data:/var/lib/grafana
    labels:
      - "traefik.enable=true"
      - "traefik.frontend.rule=Host:grafana.observability.local"


  ## test app
  webapp:
    image: jaegertracing/example-hotrod:latest
    container_name: hotrod
    command: all
    ports:
      - "8080-8083:8080-8083"
    environment:
      - JAEGER_AGENT_HOST=jaeger
      - JAEGER_AGENT_PORT=6831
    labels:
      - "traefik.frontend.rule=Host:webapp.observability.local"

volumes:
  elastic_data:
  prometheus_data:
  grafana_data:

traefik.toml

################################################################
# Global configuration
################################################################

# Enable debug mode
#
# Optional
# Default: false
#
# debug = true

# Log level
#
# Optional
# Default: "ERROR"
#
# logLevel = "DEBUG"

# Entrypoints to be used by frontends that do not specify any entrypoint.
# Each frontend can specify its own entrypoints.
#
# Optional
# Default: ["http"]
#
# defaultEntryPoints = ["http", "https"]

################################################################
# Entrypoints configuration
################################################################

# Entrypoints definition
#
# Optional
# Default:
[entryPoints]
    [entryPoints.http]
    address = ":80"

################################################################
# Traefik logs configuration
################################################################

# Traefik logs
# Enabled by default and log to stdout
#
# Optional
#
# [traefikLog]

# Sets the filepath for the traefik log. If not specified, stdout will be used.
# Intermediate directories are created if necessary.
#
# Optional
# Default: os.Stdout
#
# filePath = "log/traefik.log"

# Format is either "json" or "common".
#
# Optional
# Default: "common"
#
# format = "common"

################################################################
# Access logs configuration
################################################################

# Enable access logs
# By default it will write to stdout and produce logs in the textual
# Common Log Format (CLF), extended with additional fields.
#
# Optional
#
[accessLog]

# Sets the file path for the access log. If not specified, stdout will be used.
# Intermediate directories are created if necessary.
#
# Optional
# Default: os.Stdout
#
# filePath = "/path/to/log/log.txt"

# Format is either "json" or "common".
#
# Optional
# Default: "common"
#
# format = "common"

################################################################
# API and dashboard configuration
################################################################

# Enable API and dashboard
[api]

  # Name of the related entry point
  #
  # Optional
  # Default: "traefik"
  #
  # entryPoint = "traefik"

  # Enabled Dashboard
  #
  # Optional
  # Default: true
  #
  # dashboard = false

################################################################
# Ping configuration
################################################################

# Enable ping
[ping]

  # Name of the related entry point
  #
  # Optional
  # Default: "traefik"
  #
  # entryPoint = "traefik"

################################################################
# Docker configuration backend
################################################################

# Enable Docker configuration backend
[docker]

# Docker server endpoint. Can be a tcp or a unix socket endpoint.
#
# Required
# Default: "unix:///var/run/docker.sock"
#
# endpoint = "tcp://10.10.10.10:2375"

# Default domain used.
# Can be overridden by setting the "traefik.domain" label on a container.
#
# Optional
# Default: ""
#
# domain = "docker.localhost"

# Expose containers by default in traefik
#
# Optional
# Default: true
#
# exposedByDefault = true