Pino, docker, promtail, loki and grafana

local-config.yaml

auth_enabled: false

target: all
http_prefix: ""
ballast_bytes: 0
use_buffered_logger: true
use_sync_logger: true

common:
  path_prefix: /loki
  storage:
    filesystem:
      chunks_directory: /loki/chunks
      rules_directory: /loki/rules
  replication_factor: 1
  ring:
    kvstore:
      store: inmemory

server:
  http_listen_port: 3100
  grpc_listen_port: 9095

schema_config:
  configs:
    - from: "2020-10-24"
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

limits_config:
  ingestion_rate_strategy: global
  ingestion_rate_mb: 4
  ingestion_burst_size_mb: 6
  max_label_name_length: 1024
  max_label_value_length: 2048
  max_label_names_per_series: 30
  reject_old_samples: true
  reject_old_samples_max_age: 1w
  creation_grace_period: 10m
  enforce_metric_name: true
  max_line_size: 0
  max_line_size_truncate: false
  increment_duplicate_timestamp: false
  max_streams_per_user: 0
  max_global_streams_per_user: 5000
  unordered_writes: true
  per_stream_rate_limit: 3145728
  per_stream_rate_limit_burst: 15728640
  max_chunks_per_query: 2000000
  max_query_series: 500
  max_query_lookback: 0s
  max_query_length: 10d1h
  max_query_parallelism: 32
  cardinality_limit: 100000
  max_streams_matchers_per_query: 1000
  max_concurrent_tail_requests: 10
  max_entries_limit_per_query: 5000
  max_cache_freshness_per_query: 1m
  max_queriers_per_tenant: 0
  query_ready_index_num_days: 0
  query_timeout: 1m
  split_queries_by_interval: 30m
  min_sharding_lookback: 0s

  deletion_mode: filter-only
  retention_period: 30d
  per_tenant_override_config: ""
  per_tenant_override_period: 10s
  allow_deletes: true
  shard_streams:
    enabled: false
    logging_enabled: false
    desired_rate: 3145728
table_manager:
  throughput_updates_disabled: false
  retention_deletes_enabled: true
  retention_period: 30d
  poll_interval: 2m0s
  creation_grace_period: 10m0s

compactor:
  working_directory: /loki/compactor
  shared_store: filesystem
  shared_store_key_prefix: index/
  compaction_interval: 10m0s
  apply_retention_interval: 0s
  retention_enabled: true
  retention_delete_delay: 15m0s
  retention_delete_worker_count: 150
  retention_table_timeout: 0s
  delete_batch_size: 70
  delete_request_cancel_period: 15m0s
  delete_max_interval: 0s
  max_compaction_parallelism: 1
  upload_parallelism: 10

logger.ts

import pino from 'pino';
import util from 'util';

const formatOptions = {
  // 8 levels should be enough to log content of protocol messages stored in an object property.
  depth: 8,
  breakLength: Infinity,
  compact: false
};

export type PinoLogger = pino.Logger<{ customLevels: { noise: number } }>;

class Logger {
  private logger?: PinoLogger;
  private mods: { [key: string]: PinoLogger } = {};

  init(): void {
    const destination = process.env.LOG_DESTINATION ?? 1;
    const opts = {
      customLevels: { noise: 5 },
      level: process.env.LOG_LEVEL || 'debug',
      hooks: {
        logMethod(args: string[], method: (...args: string[]) => void): void {
          if (args.length > 1) {
            args = [util.formatWithOptions(formatOptions, ...args)];
          }
          method.apply(this, args);
        }
      }
    };
    if (process.env.LOG_PRETTY != null) {
      const transport = {
        target: 'pino-pretty',
        options: {
          levelFirst: true,
          translateTime: true,
          destination
        }
      };
      // From pino docs: If the transport option is supplied to pino, a destination parameter may not also be passed as a separate argument to pino.
      this.logger = pino({ ...opts, transport });
    } else {
      this.logger = pino(opts, pino.destination(destination));
    }

    (global as unknown as { [k: string]: unknown }).logger = this.logger;
  }

  getLogger(mod?: string): PinoLogger {
    if (this.logger == null) {
      throw new Error('logger was not initialized properly: call init() first');
    }
    if (mod == null) {
      return this.logger;
    }
    if (!(mod in this.mods)) {
      this.mods[mod] = this.logger.child<object>({ mod });
    }
    return this.mods[mod];
  }
}

const logger = new Logger();
logger.init();

export default logger;

monitoring__docker-compose.yml

version: '3.9'

services:

  prometheus:
    image: prom/prometheus:v2.47.2
    user: ":"
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.console.libraries=/etc/prometheus/console_libraries'
      - '--web.console.templates=/etc/prometheus/consoles'
      - '--storage.tsdb.retention.time=60d'
      - '--web.enable-lifecycle'
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - ./prometheus_data:/prometheus
    networks:
      - metrics-network
    restart: unless-stopped
  loki:
    image: grafana/loki:2.9.2
    ports:
      - "3100:3100"
    command: -config.file=/etc/loki/local-config.yaml
    volumes:
      - type: bind
        source: ./local-config.yaml
        target: /etc/loki/local-config.yaml
    networks:
      - metrics-network

  grafana:
    user: ":"
    image: grafana/grafana:10.2.0
    environment:
      GF_SERVER_ROOT_URL: 'http://{{SERVER_IP}}'
    ports:
      - "80:3000"
    volumes:
      - ./grafana:/var/lib/grafana
    networks:
      - metrics-network
    restart: unless-stopped

networks:
  metrics-network:

promtail.yml

server:
  http_listen_address: 0.0.0.0
  http_listen_port: 9080

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://{{MONITORING_IP}}:3100/loki/api/v1/push
    external_labels:
      env: "dev"

scrape_configs:
- job_name: containers
  static_configs:
    - targets:
        - localhost
      labels:
        __path__: /var/lib/docker/containers/*/*-json.log

  pipeline_stages:
    - json:
        expressions:
          log:
          timestamp: time
          service_name: attrs.service_name
    - json:
        expressions:
          level: level
          mod: mod
          hostname: hostname
          msg: msg
        source: log
    - labels:
        app: service_name
        level:
        mod:
        hostname: 
    - match:
        selector: '{app=""}'
        action: drop
    - labeldrop:
        - filename
        - attrs
        - attrs_production_status
        - attrs_service_name
        - stream
        - msg
        - log
    - labelallow:
        - app
        - env
        - hostname
        - job
        - level
        - mod
        - time
    - output:
       source: msg

services__docker-compose.yml

version: '3.9'
services:
  nodejsapp:
    restart: unless-stopped
    logging:
      driver: 'json-file'
      options:
        max-size: 10m
        max-file: '3'
        labels: 'production_status,service_name'
    labels:
      production_status: 'prod'
      service_name: 'tgbot'

  promtail:
    image: grafana/promtail:2.9.0
    volumes:
      - ./promtail.yml:/etc/promtail/config.yml
      - ./promtail-data:/var/lib/promtail/positions
      - /var/lib/docker/containers:/var/lib/docker/containers:ro