Promtail configuration for systemd and logs from all nomad task directories using consul service discovery

promtail.yml

positions:
  filename: /tmp/data/positions.yaml

server:
  log_level: warn
  #disable: true
  http_listen_address: {{ env "NOMAD_IP_http" }}
  http_listen_port: {{ env "NOMAD_PORT_http" }}
  grpc_listen_address: {{ env "NOMAD_IP_grpc" }}
  grpc_listen_port: {{ env "NOMAD_PORT_grpc" }}

clients:
  - url: http://${loki_addr}/loki/api/v1/push

scrape_configs:
  - job_name: journal
    journal:
      json: false
      max_age: 1h
      path: /var/log/journal
      labels:
        job: systemd-journal
    relabel_configs:
      - source_labels:
        - __journal__systemd_unit
        target_label: systemd_unit
      - source_labels:
        - __journal__hostname
        target_label: instance
      - source_labels:
        - __journal_syslog_identifier
        target_label: syslog_identifier
  - job_name: 'nomad-logs'
    consul_sd_configs:
      - server: '127.0.0.1:8501'
        scheme: https
        tls_config:
          insecure_skip_verify: true
          ca_file: /etc/pki/CA/consul_ca_cert.pem
          cert_file: /etc/pki/tls/certs/consul_server_cert.pem
          key_file: /etc/pki/tls/private/consul_server_key.pem
    relabel_configs:
      - source_labels: [__meta_consul_node]
        target_label: __host__
      - source_labels: [__meta_consul_service]
        target_label: job
      - source_labels: ['__meta_consul_node']
        target_label:  'instance'
      - source_labels: [__meta_consul_service_id]
        regex: '_nomad-task-([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})-.*'
        target_label:  'alloc_id'
      - source_labels: [__meta_consul_service_id]
        regex: '_nomad-task-([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})-.*'
        target_label:  '__path__'
        replacement: '/opt/nomad/alloc/$${1}/alloc/logs/{*.log,*std*.{?,??}}'