Open Source Grafana Stack on k8s - k8staz.com

Chart.yaml

apiVersion: v2
name: kube-prometheus-stack
version: 1.0.0-SNAPSHOT
description: Chart to deploy kube-prometheus-stack

dependencies:
  # https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/Chart.yaml
  - name: kube-prometheus-stack
    version: 34.6.0
    repository: https://prometheus-community.github.io/helm-charts

loki.yaml

# If you need edit values -- https://github.com/grafana/helm-charts/blob/main/charts/loki-distributed/values.yaml

minio.yaml

apiVersion: v1
kind: Service
metadata:
  labels:
    name: minio
  name: minio
  namespace: default
spec:
  ports:
  - port: 9000
    targetPort: 9000
  selector:
    app: minio
    name: minio
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: minio
  namespace: default
spec:
  minReadySeconds: 10
  replicas: 1
  selector:
    matchLabels:
      app: minio
      name: minio
  template:
    metadata:
      labels:
        app: minio
        name: minio
    spec:
      containers:
      - env:
        - name: MINIO_ACCESS_KEY
          value: tempo
        - name: MINIO_SECRET_KEY
          value: supersecret
        image: minio/minio:latest
        imagePullPolicy: IfNotPresent
        name: minio
        command: ["sh"]
        args: ["-euc", "mkdir -p /data/tempo && /opt/bin/minio server /data --console-address ':9001'"]

promtail.yaml

# https://github.com/grafana/helm-charts/blob/main/charts/promtail/values.yaml
config:
  snippets:
    pipelineStages:
      # not cri, because we're running in k3s with docker runtime
      - docker: {}
  # Loki service url
  lokiAddress: http://loki-loki-distributed-gateway/loki/api/v1/push

run.sh

# Create namespace
kubectl create ns observability

# Add Grafana Helm charts to local repo
helm repo add grafana https://grafana.github.io/helm-charts
helm repo update

# Install promtail
helm upgrade --install promtail grafana/promtail -n observability -f promtail.yaml

# Install Loki
helm upgrade --install loki grafana/loki-distributed -n observability

# Install Prometheus + Grafana
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

mkdir kube-prometheus-stack && cd kube-prometheus-stack
# Copy Chart.yaml and values.yaml files 
helm dependency update
helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack -n observability .

# Run Spring boot app

kubectl apply -f springboot-app.yaml

springboot-app.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: springboot-app
  labels:
    app: springboot-app
spec:
  selector:
    matchLabels:
      app: springboot-app
  template:
    metadata:
      labels:
        app: springboot-app
    spec:
      containers:
        - name: springboot-app
          image: "stazdx/otel-springboot-prometheus"
          imagePullPolicy: Always
          ports:
            - name: http
              containerPort: 8080
              protocol: TCP
            - name: actuator
              containerPort: 8081
              protocol: TCP
          readinessProbe:
            httpGet:
              path: /actuator/health/readiness
              port: 8081
          livenessProbe:
            httpGet:
              path: /actuator/health/liveness
              port: 8081
          startupProbe:
            httpGet:
              path: /actuator/health/liveness
              port: 8081
            periodSeconds: 5
            failureThreshold: 30
          env:
            - name: SERVER_PORT
              value: '8080'
            - name: MANAGEMENT_SERVER_PORT
              value: '8081'
            # Setting Tempo Distributor Service using GRPC Port -> 4317
            - name: OTEL_EXPORTER_OTLP_ENDPOINT
              value: http://tempo-tempo-distributed-distributor:4317
            - name: OTEL_SERVICE_NAME
              value: springboot-app
            - name: KUBE_POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: OTEL_RESOURCE_ATTRIBUTES
              value: app=springboot-app
---
apiVersion: v1
kind: Service
metadata:
  name: springboot-app
  labels:
    app: springboot-app
  annotations:
    # Annotations for Prometheus - scrape config 
    prometheus.io/path: /actuator/prometheus
    prometheus.io/port: '8080'
    prometheus.io/scrape: 'true'
spec:
  type: LoadBalancer
  ports:
    - name: http
      port: 8080
      targetPort: 8080
      protocol: TCP
    - name: actuator
      port: 8081
      targetPort: 8081
      protocol: TCP
  selector:
    app: springboot-app
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: springboot-app-grafana-dashboard
  labels:
    app: springboot-app
    grafana_dashboard: "1"
data:
  # Grafana Dashboar for Exemplar Demo
  spring-boot-app.json: |-
    {
      "annotations": {
        "list": [
          {
            "builtIn": 1,
            "datasource": "-- Grafana --",
            "enable": true,
            "hide": true,
            "iconColor": "rgba(0, 211, 255, 1)",
            "name": "Annotations & Alerts",
            "target": {
              "limit": 100,
              "matchAny": false,
              "tags": [],
              "type": "dashboard"
            },
            "type": "dashboard"
          }
        ]
      },
      "editable": true,
      "fiscalYearStartMonth": 0,
      "graphTooltip": 0,
      "links": [],
      "liveNow": false,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "latency / s",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "auto",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              }
            },
            "overrides": []
          },
          "gridPos": {
            "h": 9,
            "w": 12,
            "x": 0,
            "y": 0
          },
          "id": 2,
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "single",
              "sort": "none"
            }
          },
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "prometheus"
              },
              "exemplar": false,
              "expr": "rate(http_server_requests_seconds_sum[1m]) / rate(http_server_requests_seconds_count[1m])",
              "interval": "",
              "legendFormat": "avg",
              "refId": "A"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "prometheus"
              },
              "exemplar": true,
              "expr": "histogram_quantile(0.7, rate(http_server_requests_seconds_bucket[1m]))",
              "hide": false,
              "interval": "",
              "legendFormat": "0.7",
              "refId": "C"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "prometheus"
              },
              "exemplar": true,
              "expr": "histogram_quantile(0.9, rate(http_server_requests_seconds_bucket[1m]))",
              "hide": false,
              "interval": "",
              "legendFormat": "0.9",
              "refId": "B"
            }
          ],
          "title": "Requests latency",
          "type": "timeseries"
        }
      ],
      "refresh": false,
      "schemaVersion": 35,
      "style": "dark",
      "tags": [],
      "templating": {
        "list": []
      },
      "time": {
        "from": "now-1h",
        "to": "now"
      },
      "timepicker": {},
      "timezone": "",
      "title": "Spring Boot Demo",
      "uid": "qiDRdxsnk",
      "version": 1,
      "weekStart": ""
    }

tempo.yaml

traces:
  otlp:
    grpc:
      # -- Enable Tempo to ingest Jaeger GRPC traces
      enabled: true
      # -- Jaeger GRPC receiver config
      receiverConfig: {}

queryFrontend:
  service:
    type: ClusterIP
    annotations: {}
      # service.beta.kubernetes.io/aws-load-balancer-internal: "true"
      # service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
      # service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "tcp"
config: |
  query_frontend:
    search:
      max_duration: 0
  multitenancy_enabled: false
  search_enabled: true
  compactor:
    compaction:
      block_retention: 1440h
  distributor:
    receivers:
      jaeger:
        protocols:
          thrift_compact:
            endpoint: 0.0.0.0:6831
          thrift_binary:
            endpoint: 0.0.0.0:6832
          thrift_http:
            endpoint: 0.0.0.0:14268
          grpc:
            endpoint: 0.0.0.0:14250
      otlp:
        protocols:
          http:
            endpoint: 0.0.0.0:55681
          grpc:
            endpoint: 0.0.0.0:4317
  querier:
    frontend_worker:
      frontend_address: tempo-tempo-distributed-query-frontend-discovery:9095
  ingester:
    lifecycler:
      ring:
        replication_factor: 1
        kvstore:
          store: memberlist
      tokens_file_path: /var/tempo/tokens.json
  memberlist:
    abort_if_cluster_join_fails: false
    join_members:
      - tempo-tempo-distributed-gossip-ring
  overrides:
    max_search_bytes_per_trace: 0
    per_tenant_override_config: /conf/overrides.yaml
  server:
    http_listen_port: 3100
    log_level: info
    grpc_server_max_recv_msg_size: 4.194304e+06
    grpc_server_max_send_msg_size: 4.194304e+06
  storage:
    trace:
      backend: s3
      s3:
        bucket: tempo
        endpoint: minio.default.svc.cluster.local:9000
        access_key: tempo
        secret_key: supersecret
        insecure: true
      blocklist_poll: 5m
      wal:
        path: /var/tempo/wal

values.yaml

kube-prometheus-stack:
  prometheus:
    prometheusSpec:
      # Enable Exemplar Storage - Prometheus 
      enableFeatures:
        - exemplar-storage
      additionalScrapeConfigs:
        - job_name: 'kubernetes-service-endpoints'
          scrape_interval: 5s
          scrape_timeout: 2s
          kubernetes_sd_configs:
            - role: endpoints
          relabel_configs:
            - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
              action: keep
              regex: true
            - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
              action: replace
              target_label: __scheme__
              regex: (https?)
            - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
              action: replace
              target_label: __metrics_path__
              regex: (.+)
            - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
              action: replace
              target_label: __address__
              regex: (.+)(?::\d+);(\d+)
              replacement: $1:$2
            - action: labelmap
              regex: __meta_kubernetes_service_label_(.+)
            - source_labels: [__meta_kubernetes_namespace]
              action: replace
              target_label: kubernetes_namespace
            - source_labels: [__meta_kubernetes_service_name]
              action: replace
              target_label: kubernetes_name
  grafana:
    service:
      type: LoadBalancer
    # Anonymous login without credentials -- ONLY NON PRODUCTION ENVIRONMENTS!
    grafana.ini:
      auth:
        disable_login_form: true
      auth.anonymous:
        enabled: true
        org_name: "Main Org."
        org_role: Admin
    sidecar:
      dashboards:
        searchNamespace: ALL
    # Import Prometheus, Loki and Tempo Datasources
    additionalDataSources:
      - name: Prometheus
        type: prometheus
        uid: prometheus
        access: proxy
        url: http://kube-prometheus-stack-prometheus:9090
        jsonData:
          exemplarTraceIdDestinations:
            # Prometheus Exemplar Metrics relationated with Tempo trace_id variable.
            - name: trace_id
              datasourceUid: tempo
              urlDisplayLabel: View in Tempo
      - name: Loki
        type: loki
        uid: loki
        access: proxy
        url: http://loki-loki-distributed-gateway
        jsonData:
          derivedFields:
            # Opentelemetry set trace_id in log and Loki search this metric for relating with Tempo traces.
            - name: trace_id
              datasourceUid: tempo
              matcherRegex: "trace_id=(\\w+)"
              url: '$${__value.raw}'
      - name: Tempo
        type: tempo
        uid: tempo
        access: proxy
        url: http://tempo-tempo-distributed-query-frontend:3100
        jsonData:
          httpMethod: GET
          # Tempo redirects to Loki logs by specific trace_id
          tracesToLogs:
            datasourceUid: 'loki'
            tags: [ 'app' ]
            spanStartTimeShift: 1s
            spandEndTimeShift: 1s
            lokiSearch: true
          serviceMap:
            datasourceUid: 'prometheus'