Observability with Grafana and docker.

observability_app.md

Simulate app

need install loki plugin

docker plugin install grafana/loki-docker-driver:2.9.2 --alias loki --grant-all-permissions

docker command for up container and save logs with loki driver plugin

docker run -d \
  --name $CONTAINER_NAME \
  -p $PORT:$PORT \
  -e PORT=$PORT \
  -e AUTH_TRUST_HOST=$AUTH_TRUST_HOST \
  -e AUTH_SECRET=$AUTH_SECRET \
  -e DATABASE_URL="$DATABASE_URL" \
  --log-driver=loki \
  --log-opt loki-url="http://127.0.0.1:3100/loki/api/v1/push" \
  --log-opt loki-retries=5 \
  --log-opt loki-batch-size=400 \
  --restart unless-stopped \
  $IMAGE_NAME:latest

observability_docker.md

Observability Docker config

docker-compose.yml

version: "3"

networks:
  loki:

services:
  loki:
    image: grafana/loki:2.9.2
    ports:
      - "3100:3100"
    command: -config.file=/etc/loki/local-config.yaml
    networks:
      - loki

  promtail:
    image: grafana/promtail:2.9.2
    volumes:
      - /var/log/nginx:/var/log/nginx
      - ./promtail.config.yml:/etc/promtail/config.yml
    command: -config.file=/etc/promtail/config.yml
    networks:
      - loki

  prometheus:
    image: prom/prometheus:latest
    volumes:
      - ./prometheus.config.yml:/etc/prometheus/prometheus.yml
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
    ports:
      - "9090:9090"
    networks:
      - loki

  node-exporter:
    image: prom/node-exporter:latest
    ports:
      - "9100:9100"
    networks:
      - loki

  grafana:
    environment:
      - GF_PATHS_PROVISIONING=/etc/grafana/provisioning
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
    entrypoint:
      - sh
      - -euc
      - |
        mkdir -p /etc/grafana/provisioning/datasources
        cat <<EOF > /etc/grafana/provisioning/datasources/ds.yaml
        apiVersion: 1
        datasources:
        - name: Loki
          type: loki
          access: proxy
          id: 1
          orgId: 1
          url: http://loki:3100
          basicAuth: false
          isDefault: true
          version: 1
          editable: false
        - name: Prometheus
          type: prometheus
          access: proxy
          id: 2
          orgId: 1
          url: http://prometheus:9090
          basicAuth: false
          isDefault: false
          version: 1
          editable: false
        EOF
        /run.sh
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    networks:
      - loki

promtail.config.yml

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://loki:3100/loki/api/v1/push

scrape_configs:
  - job_name: nginx
    static_configs:
      - targets:
          - localhost
        labels:
          job: nginx
          __path__: /var/log/nginx/*.log
    relabel_configs:
      - source_labels: [__line__]
        regex: .*monitor\.xxx\.com.*
        action: drop

prometheus.config.yml

global:
  scrape_interval: 15s

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'node_exporter'
    static_configs:
      - targets: ['node-exporter:9100']