Skip to content

Instantly share code, notes, and snippets.

@stazdx
Last active June 1, 2022 08:20
Embed
What would you like to do?
Open Source Grafana Stack on k8s - k8staz.com
apiVersion: v2
name: kube-prometheus-stack
version: 1.0.0-SNAPSHOT
description: Chart to deploy kube-prometheus-stack
dependencies:
# https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/Chart.yaml
- name: kube-prometheus-stack
version: 34.6.0
repository: https://prometheus-community.github.io/helm-charts
# If you need edit values -- https://github.com/grafana/helm-charts/blob/main/charts/loki-distributed/values.yaml
apiVersion: v1
kind: Service
metadata:
labels:
name: minio
name: minio
namespace: default
spec:
ports:
- port: 9000
targetPort: 9000
selector:
app: minio
name: minio
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: minio
namespace: default
spec:
minReadySeconds: 10
replicas: 1
selector:
matchLabels:
app: minio
name: minio
template:
metadata:
labels:
app: minio
name: minio
spec:
containers:
- env:
- name: MINIO_ACCESS_KEY
value: tempo
- name: MINIO_SECRET_KEY
value: supersecret
image: minio/minio:latest
imagePullPolicy: IfNotPresent
name: minio
command: ["sh"]
args: ["-euc", "mkdir -p /data/tempo && /opt/bin/minio server /data --console-address ':9001'"]
# https://github.com/grafana/helm-charts/blob/main/charts/promtail/values.yaml
config:
snippets:
pipelineStages:
# not cri, because we're running in k3s with docker runtime
- docker: {}
# Loki service url
lokiAddress: http://loki-loki-distributed-gateway/loki/api/v1/push
# Create namespace
kubectl create ns observability
# Add Grafana Helm charts to local repo
helm repo add grafana https://grafana.github.io/helm-charts
helm repo update
# Install promtail
helm upgrade --install promtail grafana/promtail -n observability -f promtail.yaml
# Install Loki
helm upgrade --install loki grafana/loki-distributed -n observability
# Install Prometheus + Grafana
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update
mkdir kube-prometheus-stack && cd kube-prometheus-stack
# Copy Chart.yaml and values.yaml files
helm dependency update
helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack -n observability .
# Run Spring boot app
kubectl apply -f springboot-app.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: springboot-app
labels:
app: springboot-app
spec:
selector:
matchLabels:
app: springboot-app
template:
metadata:
labels:
app: springboot-app
spec:
containers:
- name: springboot-app
image: "stazdx/otel-springboot-prometheus"
imagePullPolicy: Always
ports:
- name: http
containerPort: 8080
protocol: TCP
- name: actuator
containerPort: 8081
protocol: TCP
readinessProbe:
httpGet:
path: /actuator/health/readiness
port: 8081
livenessProbe:
httpGet:
path: /actuator/health/liveness
port: 8081
startupProbe:
httpGet:
path: /actuator/health/liveness
port: 8081
periodSeconds: 5
failureThreshold: 30
env:
- name: SERVER_PORT
value: '8080'
- name: MANAGEMENT_SERVER_PORT
value: '8081'
# Setting Tempo Distributor Service using GRPC Port -> 4317
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: http://tempo-tempo-distributed-distributor:4317
- name: OTEL_SERVICE_NAME
value: springboot-app
- name: KUBE_POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: OTEL_RESOURCE_ATTRIBUTES
value: app=springboot-app
---
apiVersion: v1
kind: Service
metadata:
name: springboot-app
labels:
app: springboot-app
annotations:
# Annotations for Prometheus - scrape config
prometheus.io/path: /actuator/prometheus
prometheus.io/port: '8080'
prometheus.io/scrape: 'true'
spec:
type: LoadBalancer
ports:
- name: http
port: 8080
targetPort: 8080
protocol: TCP
- name: actuator
port: 8081
targetPort: 8081
protocol: TCP
selector:
app: springboot-app
---
apiVersion: v1
kind: ConfigMap
metadata:
name: springboot-app-grafana-dashboard
labels:
app: springboot-app
grafana_dashboard: "1"
data:
# Grafana Dashboar for Exemplar Demo
spring-boot-app.json: |-
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "latency / s",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(http_server_requests_seconds_sum[1m]) / rate(http_server_requests_seconds_count[1m])",
"interval": "",
"legendFormat": "avg",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": true,
"expr": "histogram_quantile(0.7, rate(http_server_requests_seconds_bucket[1m]))",
"hide": false,
"interval": "",
"legendFormat": "0.7",
"refId": "C"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": true,
"expr": "histogram_quantile(0.9, rate(http_server_requests_seconds_bucket[1m]))",
"hide": false,
"interval": "",
"legendFormat": "0.9",
"refId": "B"
}
],
"title": "Requests latency",
"type": "timeseries"
}
],
"refresh": false,
"schemaVersion": 35,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Spring Boot Demo",
"uid": "qiDRdxsnk",
"version": 1,
"weekStart": ""
}
traces:
otlp:
grpc:
# -- Enable Tempo to ingest Jaeger GRPC traces
enabled: true
# -- Jaeger GRPC receiver config
receiverConfig: {}
queryFrontend:
service:
type: ClusterIP
annotations: {}
# service.beta.kubernetes.io/aws-load-balancer-internal: "true"
# service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
# service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "tcp"
config: |
query_frontend:
search:
max_duration: 0
multitenancy_enabled: false
search_enabled: true
compactor:
compaction:
block_retention: 1440h
distributor:
receivers:
jaeger:
protocols:
thrift_compact:
endpoint: 0.0.0.0:6831
thrift_binary:
endpoint: 0.0.0.0:6832
thrift_http:
endpoint: 0.0.0.0:14268
grpc:
endpoint: 0.0.0.0:14250
otlp:
protocols:
http:
endpoint: 0.0.0.0:55681
grpc:
endpoint: 0.0.0.0:4317
querier:
frontend_worker:
frontend_address: tempo-tempo-distributed-query-frontend-discovery:9095
ingester:
lifecycler:
ring:
replication_factor: 1
kvstore:
store: memberlist
tokens_file_path: /var/tempo/tokens.json
memberlist:
abort_if_cluster_join_fails: false
join_members:
- tempo-tempo-distributed-gossip-ring
overrides:
max_search_bytes_per_trace: 0
per_tenant_override_config: /conf/overrides.yaml
server:
http_listen_port: 3100
log_level: info
grpc_server_max_recv_msg_size: 4.194304e+06
grpc_server_max_send_msg_size: 4.194304e+06
storage:
trace:
backend: s3
s3:
bucket: tempo
endpoint: minio.default.svc.cluster.local:9000
access_key: tempo
secret_key: supersecret
insecure: true
blocklist_poll: 5m
wal:
path: /var/tempo/wal
kube-prometheus-stack:
prometheus:
prometheusSpec:
# Enable Exemplar Storage - Prometheus
enableFeatures:
- exemplar-storage
additionalScrapeConfigs:
- job_name: 'kubernetes-service-endpoints'
scrape_interval: 5s
scrape_timeout: 2s
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
action: replace
target_label: __scheme__
regex: (https?)
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
action: replace
target_label: __address__
regex: (.+)(?::\d+);(\d+)
replacement: $1:$2
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_service_name]
action: replace
target_label: kubernetes_name
grafana:
service:
type: LoadBalancer
# Anonymous login without credentials -- ONLY NON PRODUCTION ENVIRONMENTS!
grafana.ini:
auth:
disable_login_form: true
auth.anonymous:
enabled: true
org_name: "Main Org."
org_role: Admin
sidecar:
dashboards:
searchNamespace: ALL
# Import Prometheus, Loki and Tempo Datasources
additionalDataSources:
- name: Prometheus
type: prometheus
uid: prometheus
access: proxy
url: http://kube-prometheus-stack-prometheus:9090
jsonData:
exemplarTraceIdDestinations:
# Prometheus Exemplar Metrics relationated with Tempo trace_id variable.
- name: trace_id
datasourceUid: tempo
urlDisplayLabel: View in Tempo
- name: Loki
type: loki
uid: loki
access: proxy
url: http://loki-loki-distributed-gateway
jsonData:
derivedFields:
# Opentelemetry set trace_id in log and Loki search this metric for relating with Tempo traces.
- name: trace_id
datasourceUid: tempo
matcherRegex: "trace_id=(\\w+)"
url: '$${__value.raw}'
- name: Tempo
type: tempo
uid: tempo
access: proxy
url: http://tempo-tempo-distributed-query-frontend:3100
jsonData:
httpMethod: GET
# Tempo redirects to Loki logs by specific trace_id
tracesToLogs:
datasourceUid: 'loki'
tags: [ 'app' ]
spanStartTimeShift: 1s
spandEndTimeShift: 1s
lokiSearch: true
serviceMap:
datasourceUid: 'prometheus'
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment