Integrate Grafana with Promtail + Loki

README.md

Integrate Grafana with Promtail + Loki

Install Loki Binary and Start as a Service

cd /usr/local/bin
curl -s https://api.github.com/repos/grafana/loki/releases/latest | grep browser_download_url |  cut -d '"' -f 4 | grep loki-linux-amd64.zip | wget -i -

sudo apt install unzip
sudo unzip loki-linux-amd64.zip
mv loki-linux-amd64 loki
sudo chmod a+x loki

Create Loki Config

sudo nano config-loki.yml

auth_enabled: false

server:
  http_listen_port: 3100

ingester:
  lifecycler:
    address: 127.0.0.1
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
    final_sleep: 0s
  chunk_idle_period: 5m
  chunk_retain_period: 30s
  max_transfer_retries: 0

schema_config:
  configs:
    - from: 2022-07-24
      store: boltdb
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

storage_config:
  boltdb:
    directory: /tmp/loki/index

  filesystem:
    directory: /tmp/loki/chunks

limits_config:
  enforce_metric_name: false
  reject_old_samples: true
  reject_old_samples_max_age: 168h

chunk_store_config:
  max_look_back_period: 0s

table_manager:
  retention_deletes_enabled: false
  retention_period: 0s

Configure Loki as a Service

sudo nano /etc/systemd/system/loki.service

[Unit]
Description=Loki service
After=network.target

[Service]
Type=simple
User=root
ExecStart=/usr/local/bin/loki -config.file /usr/local/bin/config-loki.yml

[Install]
WantedBy=multi-user.target

systemctl enable loki
sudo service loki start
sudo service loki status

Access Loki

http://[Your-Server-Domain-or-IP]:3100/metrics

sudo service loki stop
sudo service loki status

Install Promtail Binary and Start a Service

cd /usr/local/bin
curl -s https://api.github.com/repos/grafana/loki/releases/latest | grep browser_download_url |  cut -d '"' -f 4 | grep promtail-linux-amd64.zip | wget -i -
sudo unzip promtail-linux-amd64.zip
mv promtail-linux-amd64 promtail

sudo chmod a+x promtail

Create Promtail Config

sudo nano config-promtail.yml

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://127.0.0.1:3100/loki/api/v1/push

scrape_configs:
  - job_name: journal
    journal:
      max_age: 12h
      labels:
        job: systemd-journal
    relabel_configs:
      - source_labels: ['__journal__systemd_unit']
        target_label: 'unit'

Create Promatail Service

sudo nano /etc/systemd/system/promtail.service

[Unit]
Description=Promtail service
After=network.target

[Service]
Type=simple
User=root
ExecStart=/usr/local/bin/promtail -config.file /usr/local/bin/config-promtail.yml

[Install]
WantedBy=multi-user.target

systemctl enable promtail
sudo service promtail start
sudo service promtail status

Integrate Grafana with Loki as Data Source

Create new data source in Grafana with Loki and Endpoint http://127.0.0.1:3100

• In the Log Labels text area, try these LogQL example :

  # Show All log lines 
  {job="systemd-journal"}
  
  # Show lines containing "info"
  {job="systemd-journal"} |= "info"
  
  # Show lines containing "error"
  {job="systemd-journal"} |= "error"
  
  # Show lines containing "info" or "error"
  {job="systemd-journal"} |~ "info|error"
  
  # Show lines NOT containing "info"
  {job="systemd-journal"} != "info"
  
  # Show lines including the text "status 403" or "status 503"
  {job="systemd-journal"} |~ "status [45]03"
  

Trigger

To trigger log on Grafana, exec this :

echo 'abc123 this is a fake error def678' | systemd-cat

Then, monitor on your panel on Grafana.