alert.rules

ALERT deleted_container_rate_on_hosts
  IF sum without(instance) (rate(engine_daemon_container_actions_seconds_count{action="delete"}[1m])) > 0.1
  FOR 1m
  ANNOTATIONS {
      summary = "DELETED CONTAINER RATE WARNING ON '{{ $labels.instance }}'",
      description = "{{ $labels.instance }} has a HIGH rate of deleted containers. Deleted container rate is {{ humanize $value}}%.",
  }

meetup.yml

version: '3.3'

networks:
  monitoring:
    attachable: true

configs:
  prometheus_config_step_7:
    file: ./prometheus.yml
  prometheus_alerts_step_7:
    file: ./alert.rules

services:

  cadvisor:
    image: 'google/cadvisor:v0.26.1'
    networks:
      - monitoring
    volumes:
      - '/var/run/docker.sock:/var/run/docker.sock,readonly'
      - '/:/rootfs'
      - '/var/run:/var/run'
      - '/sys:/sys'
      - '/var/lib/docker/:/var/lib/docker'
    deploy:
      mode: global
  
  node-exporter:
    image: 'basi/node-exporter'
    networks:
      - monitoring
    volumes:
      - '/proc:/host/proc'
      - '/sys:/host/sys'
      - '/:/rootfs'
    hostname: '{{.Node.ID}}'
    command: >-
      --path.procfs "/host/proc"
      --path.sysfs /host/sys
      --collector.textfile.directory /etc/node-exporter/
      --collector.filesystem.ignored-mount-points "^/(sys|proc|dev|host|etc)($$|/)"
    deploy:
      mode: global

  docker-exporter:
    image: 'basi/socat:v0.1.0'
    networks:
      - monitoring
    deploy:
      mode: global

  prometheus:
    image: 'prom/prometheus:v1.8.2'
    ports:
      - '9090:9090'
    networks:
      - monitoring
    command: >-
      -config.file=/etc/prometheus/prometheus.yml
      -storage.local.path=/prometheus
      -web.console.libraries=/etc/prometheus/console_libraries
      -web.console.templates=/etc/prometheus/consoles
      -alertmanager.url=http://alertmanager:9093
    configs:
      - source: prometheus_config_step_7
        target: /etc/prometheus/prometheus.yml
      - source: prometheus_alerts_step_7
        target: /etc/prometheus/alert.rules
    deploy:
      mode: replicated
      replicas: 1

  postgres:
    image: postgres:9
    networks:
      - monitoring
    environment:
      POSTGRES_USER: "postgres"
      POSTGRES_PASSWORD: "password"
    deploy:
      mode: replicated
      replicas: 1

  postgres-exporter:
    image: wrouesnel/postgres_exporter:v0.2.3
    environment:
      DATA_SOURCE_NAME: "postgresql://postgres:password@postgres/?sslmode=disable"
    networks:
        - monitoring
    deploy:
      mode: replicated
      replicas: 1

  traefik:
    image: traefik:v1.4-alpine
    networks:
      - monitoring
    command: >-
      --debug
      --web.metrics
      --web.metrics.prometheus
    ports:
      - '80:80'
    deploy:
      mode: replicated
      replicas: 3

  grafana:
    image: 'basi/grafana:4.4.3'
    ports:
      - '3000:3000'
    networks:
      - monitoring
    environment:
      GF_SECURITY_ADMIN_PASSWORD: 'admin'
      PROMETHEUS_ENDPOINT: 'http://prometheus:9090'
    deploy:
      mode: replicated
      replicas: 1

  alertmanager:
    image: 'basi/alertmanager:v0.8.0'
    networks:
      - monitoring
    ports:
      - '9093:9093'
    environment:
      SLACK_API: 'https://hooks.slack.com/services/YOUR/API/key'
    command: '-config.file=/etc/alertmanager/config.yml'
    deploy:
      mode: replicated
      replicas: 1

  test:
    image: alpine
    command: exit 1
    networks:
      - monitoring
    deploy:
      mode: replicated
      replicas: 5

prometheus.yml

global:
  scrape_interval:     30s
  evaluation_interval: 30s

  # Attach these labels to any time series or alerts when communicating with
  # external systems (federation, remote storage, Alertmanager).
  external_labels:
    monitor: 'prometheus-swarm'

rule_files:
  - "alert.rules"

scrape_configs:
  - job_name: 'prometheus'
    dns_sd_configs:
    - names:
      - 'tasks.prometheus'
      type: 'A'
      port: 9090

  - job_name: 'cadvisor'
    dns_sd_configs:
    - names:
      - 'tasks.cadvisor'
      type: 'A'
      port: 8080

  - job_name: 'node-exporter'
    dns_sd_configs:
    - names:
      - 'tasks.node-exporter'
      type: 'A'
      port: 9100

  - job_name: 'docker-exporter'
    dns_sd_configs:
    - names:
      - 'tasks.docker-exporter'
      type: 'A'
      port: 4999

  - job_name: 'postgres'
    dns_sd_configs:
      - names:
          - 'tasks.postgres-exporter'
        type: 'A'
        port: 9187

  - job_name: 'traefik'
    dns_sd_configs:
      - names:
          - 'tasks.traefik'
        type: 'A'
        port: 8080