Tinkering with Loki, Promtail, Grafana, Prometheus, Nginx and Dnsmasq

dnsmasq.conf

log-queries
log-facility=/var/log/dnsmasq.log
no-resolv
server=8.8.4.4
server=8.8.8.8
address=/router/10.1.1.1
address=/server/10.1.1.2

docker-compose.yml

version: '2.4'

networks:
  loki:

services:

  loki:
    image: grafana/loki:master-2739551
    ports:
      - "3100:3100"
    networks:
      - loki
    restart: unless-stopped

  grafana:
    image: grafana/grafana:6.4.4
    ports:
      - "3000:3000"
    networks:
      - loki
    restart: unless-stopped

  promtail:
    image: grafana/promtail:master-2739551
    networks:
      - loki
    volumes:
      - ./promtail.yml:/etc/promtail/promtail.yml
    command: 
      - '-config.file=/etc/promtail/promtail.yml'
    restart: unless-stopped

  promtail-nginx:
    image: grafana/promtail:master-2739551
    networks:
      - loki
    volumes:
      - ./promtail-nginx.yml:/etc/promtail/promtail.yml
      - ./nginx:/var/log/nginx
    command:
      - '-config.file=/etc/promtail/promtail.yml'
    restart: unless-stopped

  nginx:
    image: nginx
    networks:
      - loki
    volumes:
      - ./nginx:/var/log/nginx
    ports:
      - 8084:80

  promtail-dnsmasq:
    image: grafana/promtail:master-2739551
    networks:
      - loki
    volumes:
      - ./promtail-dnsmasq.yml:/etc/promtail/promtail.yml
      - ./dnsmasq:/var/log
    command:
      - '-config.file=/etc/promtail/promtail.yml'
    restart: unless-stopped

  dnsmasq:
    image: strm/dnsmasq
    networks:
      - loki
    cap_add:
      - NET_ADMIN
    volumes:
      - ./dnsmasq:/var/log
      - ./dnsmasq.conf:/etc/dnsmasq.conf 

promtail-dnsmasq.yml

server:

  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

client:

  url: http://loki:3100/loki/api/v1/push

scrape_configs:

  - job_name: dnsmasq
    static_configs:
    - targets:
        - localhost
      labels:
        job: dnsmasq
        env: production
        host: ruan-prod-dnsmasq
        __path__: /var/log/dnsmasq.log
    pipeline_stages:
    - match:
        selector: '{job="dnsmasq"}'
        stages:
        - regex:
            expression: ".*query\\[A\\] (?P<query>.*\\s)"
        - labels:
            query:

promtail-journal.yml

server:
  http_listen_port: 9080
  grpc_listen_port: 0
positions:
  filename: /var/lib/promtail/positions.yaml
clients:
  - url: http://localhost:3100/loki/api/v1/push
scrape_configs:
  - job_name: journal
    journal:
      max_age: 1h
      path: /var/log/journal
      labels:
        job: systemd
        env: production
    relabel_configs:
      - source_labels: ['__journal__systemd_unit']
        target_label: 'unit'

promtail-nginx.yml

server:

  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

client:

  url: http://loki:3100/loki/api/v1/push

scrape_configs:

  - job_name: nginx
    static_configs:
    - targets:
        - localhost
      labels:
        job: nginx
        env: production
        host: ruan-prod-nginx
        __path__: /var/log/nginx/*.log

promtail-nginx_with_pipeline.yml

server:
  http_listen_port: 9080
  grpc_listen_port: 0
positions:
  filename: /var/lib/promtail/positions.yaml
clients:
  - url: http://prometheus.internal.eng.mamamoney.co.za:3100/loki/api/v1/push
scrape_configs:
- job_name: nginx
  static_configs:
  - targets:
      - localhost
    labels:
      app: nginx
      env: production
      host: ruan-prod-nginx
      __path__: /var/log/nginx/*.log
  pipeline_stages:
  - match:
      selector: '{app="nginx"}'
      stages:
      - regex:
          # logline example: 127.0.0.1 - - [21/Apr/2020:13:59:45 +0000] "GET /?foo=bar HTTP/1.1" 200 612 "http://example.com/lekkebot.html" "curl/7.58.0"
          expression: '^(?P<host>[\w\.]+) - (?P<user>[^ ]*) \[(?P<timestamp>.*)\] "(?P<method>[^ ]*) (?P<request_url>[^ ]*) (?P<request_http_protocol>[^ ]*)" (?P<status>[\d]+) (?P<bytes_out>[\d]+) "(?P<http_referer>[^"]*)" "(?P<user_agent>[^"]*)"?'
      - labels:
          host:
          method:
          request_url:
          status:
          user_agent:

promtail.yml

server:

  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

client:

  url: http://loki:3100/loki/api/v1/push

scrape_configs:

  - job_name: messages
    static_configs:
    - targets:
        - localhost
      labels:
        job: all-logs
        env: dev
        host: monitoring-dev
        __path__: /var/log/*.log

queries.md

Prometheus (loki) as Datasource

Requests for last 60 Seconds:

count_over_time( {job="nginx"} [60s])

Rate over 60s:

rate( ( {env="production", job="nginx"} ) [60s])

Show metrics with filter patterns:

rate( ( {env="production", job="nginx"} |~ "GET (/er|/ax)" ) [10s])

Dnsmasq Queries:

sum (rate( ( {env="production",query=~".*\\..*"} )[5m] )) by (query)

Loki as a Datasource

All logs:

{job="all-logs"}

Logs for nginx job:

{job="nginx"}

Generating Queries on DnsMasq Container

apk add bind-tools
for domain in yahoo.com google.com facebook.com; do dig A $domain @127.0.0.1; done

More

• https://itnext.io/grafana-loki-the-logqls-prometheus-like-counters-aggregation-functions-and-dnsmasq-s-1e622c25c7e4
• https://grafana.com/blog/2019/07/15/lokis-path-to-ga-docker-logging-driver-plugin-support-for-systemd/
• https://medium.com/@mightywomble/using-grafana-to-visualise-syslog-files-with-loki-9a8869081fb5 (includes queries)