Skip to content

Instantly share code, notes, and snippets.

@bravosierrasierra

bravosierrasierra/tempo-operational.json

Created April 11, 2024 18:26
Show Gist options
  • Save bravosierrasierra/26395507b65467fef7bfecc7bba94675 to your computer and use it in GitHub Desktop.
Save bravosierrasierra/26395507b65467fef7bfecc7bba94675 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
tempo-operational.json
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
},
{
"datasource": {
"uid": "$logsds"
},
"enable": true,
"expr": "{diff_namespace=\"$namespace\", container=\"kube-diff-logger\"}",
"hide": true,
"iconColor": "rgba(255, 96, 96, 1)",
"name": "diffs",
"showIn": 0,
"target": {}
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": 1651,
"links": [
{
"asDropdown": true,
"icon": "external link",
"includeVars": true,
"keepTime": true,
"tags": [
"tempo"
],
"targetBlank": false,
"title": "Tempo Dashboards",
"type": "dashboards"
}
],
"liveNow": false,
"panels": [
{
"collapsed": false,
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 19,
"panels": [],
"title": "Write",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 0,
"y": 1
},
"id": 10,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d452322apre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P666011C0B63BDCA4"
},
"editorMode": "code",
"expr": "sum by(cluster) (rate(tempo_distributor_spans_received_total{namespace=\"$namespace\"}[$__rate_interval]))",
"interval": "",
"legendFormat": "{{pod}}",
"range": true,
"refId": "A"
}
],
"title": "Spans Received",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 3,
"y": 1
},
"id": 9,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d452322apre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"editorMode": "code",
"expr": "sum by(cluster) (rate(tempo_ingester_traces_created_total{namespace=\"$namespace\"}[$__rate_interval]))",
"interval": "",
"legendFormat": "",
"range": true,
"refId": "A"
}
],
"title": "Traces Created",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "none",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 6,
"y": 1
},
"id": 72,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d452322apre",
"targets": [
{
"expr": "sum(rate(tempo_receiver_accepted_spans{namespace=\"$namespace\"}[$__rate_interval]))",
"interval": "",
"legendFormat": "accepted",
"refId": "A"
},
{
"expr": "sum(rate(tempo_receiver_refused_spans{namespace=\"$namespace\"}[$__rate_interval]))",
"interval": "",
"legendFormat": "refused",
"refId": "B"
}
],
"title": "OTEL Distributor Spans/second",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 9,
"y": 1
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d452322apre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".99",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".9",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".5",
"refId": "C"
}
],
"title": "Push Latency (Ingester)",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 12,
"y": 1
},
"id": 26,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d452322apre",
"targets": [
{
"expr": "sum(increase(tempo_ingester_blocks_cleared_total{namespace=\"$namespace\"}[1h]))",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Blocks Cleared (1h)",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "points",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 6,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "always",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 15,
"y": 1
},
"id": 36,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d452322apre",
"targets": [
{
"expr": "histogram_quantile(.99, sum(rate(tempo_ingester_flush_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".99",
"refId": "A"
},
{
"expr": "histogram_quantile(.9, sum(rate(tempo_ingester_flush_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".9",
"refId": "B"
},
{
"expr": "histogram_quantile(.5, sum(rate(tempo_ingester_flush_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".5",
"refId": "C"
}
],
"title": "Flush Duration",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 18,
"y": 1
},
"id": 8,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d452322apre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "sum(increase(tempo_ingester_blocks_flushed_total{namespace=\"$namespace\", job=\"tempo-grafana-tempo-ingester\"}[1h]))",
"interval": "",
"legendFormat": "{{pod}}",
"range": true,
"refId": "A"
}
],
"title": "Blocks Flushed (1h)",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 21,
"y": 1
},
"id": 12,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d452322apre",
"targets": [
{
"expr": "increase(tempo_ingester_failed_flushes_total{namespace=\"$namespace\"}[5m]) > 0",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"title": "Failed Flushes",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 18,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "infra"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 5,
"w": 5,
"x": 0,
"y": 6
},
"id": 128,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max",
"last"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d452322apre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P666011C0B63BDCA4"
},
"editorMode": "code",
"expr": "sum by(cluster) (rate(tempo_discarded_spans_total{namespace=\"$namespace\"}[$__rate_interval]))",
"interval": "",
"legendFormat": "{{pod}}",
"range": true,
"refId": "A"
}
],
"title": "Spans discarded",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "infra"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 5,
"w": 5,
"x": 5,
"y": 6
},
"id": 129,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max",
"last"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d452322apre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P666011C0B63BDCA4"
},
"editorMode": "code",
"expr": "tempo_distributor_queue_length{namespace=\"$namespace\"}",
"interval": "",
"legendFormat": "Queue: {{name}}",
"range": true,
"refId": "A"
}
],
"title": "distributor queue length",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 7,
"x": 10,
"y": 6
},
"id": 133,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editorMode": "code",
"expr": "sum(rate(tempo_distributor_traces_per_batch_count[$__rate_interval]))",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Distributor traces per batch",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 7,
"x": 17,
"y": 6
},
"id": 134,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editorMode": "code",
"expr": "sum(rate(tempo_distributor_traces_per_batch_sum[$__rate_interval])) / sum(rate(tempo_distributor_traces_per_batch_count[$__rate_interval]))",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Distributor traces per batch sum/count",
"type": "timeseries"
},
{
"collapsed": false,
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 11
},
"id": 21,
"panels": [],
"title": "Read",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 4,
"x": 0,
"y": 12
},
"id": 90,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "sum(rate(tempo_request_duration_seconds_count{namespace=\"$namespace\", route=~\".*api_traces_traceid\", job=\"tempo-grafana-tempo-query-frontend\"}[$__rate_interval])) by (status_code)",
"hide": false,
"interval": "",
"legendFormat": "{{status_code}}",
"range": true,
"refId": "A"
}
],
"title": "Queries/Sec (Query Frontend)",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 8,
"x": 4,
"y": 12
},
"id": 89,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-querier\", route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".99",
"refId": "A"
},
{
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-querier\", route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".9",
"refId": "B"
},
{
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-querier\", route=~\"querier_.*api_traces_traceid\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".5",
"refId": "C"
}
],
"title": "Query Latency (Querier)",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 4,
"x": 12,
"y": 12
},
"id": 98,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"exemplar": true,
"expr": "sum(rate(tempo_request_duration_seconds_count{namespace=\"$namespace\", route=~\".*api_search.*\", job=\"tempo-grafana-tempo-query-frontend\"}[$__rate_interval])) by (status_code)",
"hide": false,
"interval": "",
"legendFormat": "{{status_code}}",
"refId": "A",
"stepMode": "min"
}
],
"title": "Search Queries/Sec (Query Frontend)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 8,
"x": 16,
"y": 12
},
"id": 94,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max",
"last"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"exemplar": true,
"expr": "histogram_quantile(1, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
"hide": false,
"interval": "",
"legendFormat": "1",
"range": true,
"refId": "E",
"stepMode": "min"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"exemplar": true,
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".99",
"range": true,
"refId": "A",
"stepMode": "min"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"exemplar": true,
"expr": "histogram_quantile(.95, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
"hide": false,
"interval": "",
"legendFormat": ".95",
"range": true,
"refId": "D",
"stepMode": "min"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"exemplar": true,
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".9",
"refId": "B",
"stepMode": "min"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"exemplar": true,
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".5",
"refId": "C",
"stepMode": "min"
}
],
"title": "Search Query Latency (Query Frontend)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "500"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 5,
"w": 4,
"x": 0,
"y": 17
},
"id": 91,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"querier_.*api_traces_traceid\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"tempo-grafana-tempo-querier\"}[$__rate_interval])) by (status_code)",
"hide": false,
"interval": "",
"legendFormat": "{{status_code}}",
"range": true,
"refId": "A"
}
],
"title": "Queries/Sec (Querier)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 8,
"x": 4,
"y": 17
},
"id": 17,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".99",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".9",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".5",
"refId": "C"
}
],
"title": "Query Latency (Query Frontend)",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 4,
"x": 12,
"y": 17
},
"id": 99,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"exemplar": true,
"expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"querier_.*api_search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"tempo-grafana-tempo-querier\"}[$__rate_interval])) by (status_code)",
"hide": false,
"interval": "",
"legendFormat": "{{status_code}}",
"refId": "A",
"stepMode": "min"
}
],
"title": "Search Queries/Sec (Querier)",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 8,
"x": 16,
"y": 17
},
"id": 95,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"exemplar": true,
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".99",
"refId": "A",
"stepMode": "min"
},
{
"exemplar": true,
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".9",
"refId": "B",
"stepMode": "min"
},
{
"exemplar": true,
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".5",
"refId": "C",
"stepMode": "min"
}
],
"title": "Search Query Latency (Querier)",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "success"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "cancel"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 5,
"w": 4,
"x": 0,
"y": 22
},
"id": 92,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "sum(rate(tempo_request_duration_seconds_count{route=\"/tempopb.Querier/FindTraceByID\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"tempo-grafana-tempo-ingester\"}[$__rate_interval])) by (status_code)",
"hide": false,
"interval": "",
"legendFormat": "{{status_code}}",
"refId": "A"
}
],
"title": "Queries/Sec (Ingester)",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s",
"unitScale": true
},
"overrides": [
{
"__systemRef": "hideSeriesFrom",
"matcher": {
"id": "byNames",
"options": {
"mode": "exclude",
"names": [
".99"
],
"prefix": "All except:",
"readOnly": true
}
},
"properties": [
{
"id": "custom.hideFrom",
"value": {
"legend": false,
"tooltip": false,
"viz": true
}
}
]
}
]
},
"gridPos": {
"h": 5,
"w": 8,
"x": 4,
"y": 22
},
"id": 3,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".99",
"refId": "A"
},
{
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".9",
"refId": "B"
},
{
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".5",
"refId": "C"
}
],
"title": "Query Latency (Ingester)",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 4,
"x": 12,
"y": 22
},
"id": 100,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"exemplar": true,
"expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"/tempopb.Querier/Search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"tempo-grafana-tempo-ingester\"}[$__rate_interval])) by (status_code)",
"hide": false,
"interval": "",
"legendFormat": "{{status_code}}",
"refId": "A",
"stepMode": "min"
}
],
"title": "Search Queries/Sec (Ingester)",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s",
"unitScale": true
},
"overrides": [
{
"__systemRef": "hideSeriesFrom",
"matcher": {
"id": "byNames",
"options": {
"mode": "exclude",
"names": [
".99"
],
"prefix": "All except:",
"readOnly": true
}
},
"properties": [
{
"id": "custom.hideFrom",
"value": {
"legend": false,
"tooltip": false,
"viz": true
}
}
]
}
]
},
"gridPos": {
"h": 5,
"w": 8,
"x": 16,
"y": 22
},
"id": 96,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"exemplar": true,
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-ingester\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".99",
"refId": "A",
"stepMode": "min"
},
{
"exemplar": true,
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-ingester\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".9",
"refId": "B",
"stepMode": "min"
},
{
"exemplar": true,
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-ingester\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".5",
"refId": "C",
"stepMode": "min"
}
],
"title": "Search Query Latency (Ingester)",
"type": "timeseries"
},
{
"collapsed": false,
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 27
},
"id": 74,
"panels": [],
"title": "Memcached",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [
{
"options": {
"match": "null",
"result": {
"text": "N/A"
}
},
"type": "special"
}
],
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "rgba(50, 172, 45, 0.97)",
"value": null
},
{
"color": "rgba(237, 129, 40, 0.89)",
"value": 0.7
},
{
"color": "rgba(245, 54, 54, 0.9)",
"value": 0.9
}
]
},
"unit": "percentunit",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 0,
"y": 28
},
"id": 122,
"links": [],
"maxDataPoints": 100,
"options": {
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showThresholdLabels": false,
"showThresholdMarkers": true,
"sizing": "auto"
},
"pluginVersion": "10.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editorMode": "code",
"expr": "sum(memcached_current_bytes{namespace=~\"$namespace\"}) / sum(memcached_limit_bytes{namespace=~\"$namespace\"})",
"format": "time_series",
"intervalFactor": 1,
"range": true,
"refId": "A",
"step": 20
}
],
"title": "Memory usage",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 2,
"mappings": [
{
"options": {
"match": "null",
"result": {
"text": "N/A"
}
},
"type": "special"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "bytes",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 3,
"w": 4,
"x": 4,
"y": 28
},
"id": 124,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "none",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "10.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editorMode": "code",
"expr": "sum (memcached_current_bytes{namespace=~\"$namespace\"})",
"format": "time_series",
"intervalFactor": 1,
"range": true,
"refId": "A",
"step": 20
}
],
"title": "Used",
"type": "stat"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"unitScale": true
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 6,
"w": 4,
"x": 8,
"y": 28
},
"height": "240px",
"hiddenSeries": false,
"id": 123,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "10.3.1",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editorMode": "code",
"expr": "sum (irate (memcached_commands_total{status=\"hit\",command=\"get\",namespace=~\"$namespace\"}[5m])) / sum (irate (memcached_commands_total{command=\"get\",namespace=~\"$namespace\"}[5m]\n))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "Hit",
"range": true,
"refId": "A",
"step": 5
}
],
"thresholds": [],
"timeRegions": [],
"title": "Hit rate",
"tooltip": {
"msResolution": true,
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": [
"total"
]
},
"yaxes": [
{
"format": "percentunit",
"logBase": 1,
"min": 0,
"show": true
},
{
"format": "short",
"logBase": 1,
"show": false
}
],
"yaxis": {
"align": false
}
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "decbytes",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 12,
"y": 28
},
"id": 102,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"editorMode": "code",
"expr": "sum(rate(memcached_read_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))",
"hide": false,
"instant": false,
"legendFormat": "read",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"editorMode": "code",
"expr": "sum(rate(memcached_written_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))",
"hide": false,
"instant": false,
"legendFormat": "write",
"range": true,
"refId": "B"
}
],
"title": "read/write bytes",
"type": "timeseries"
},
{
"aliasColors": {
"evicts": "#890F02",
"memcached_items_evicted_total{instance=\"172.17.0.1:9150\",job=\"prometheus\"}": "#890F02",
"reclaims": "#3F6833"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"unitScale": true
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 6,
"w": 6,
"x": 18,
"y": 28
},
"height": "240px",
"hiddenSeries": false,
"id": 126,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "10.3.1",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "reclaims",
"yaxis": 2
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editorMode": "code",
"expr": "sum (irate (memcached_items_evicted_total{namespace=~\"$namespace\"}[5m]))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "evicts",
"range": true,
"refId": "A",
"step": 5,
"target": ""
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editorMode": "code",
"expr": "sum (irate (memcached_items_reclaimed_total{namespace=~\"$namespace\"}[5m]))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "reclaims",
"range": true,
"refId": "B",
"step": 5
}
],
"thresholds": [],
"timeRegions": [],
"title": "Evicts & Reclaims rate",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"show": true
},
{
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 2,
"mappings": [
{
"options": {
"match": "null",
"result": {
"text": "N/A"
}
},
"type": "special"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "bytes",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 3,
"w": 4,
"x": 4,
"y": 31
},
"id": 125,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "none",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "10.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editorMode": "code",
"expr": "sum (memcached_limit_bytes{namespace=~\"$namespace\"})",
"format": "time_series",
"intervalFactor": 1,
"range": true,
"refId": "A",
"step": 20
}
],
"title": "Total",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 0,
"y": 34
},
"id": 75,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "sum(rate(memcached_connections_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))",
"interval": "",
"legendFormat": "{{status_code}}-{{method}}",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "sum(rate(cortex_memcache_request_duration_seconds_count{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}[$__rate_interval])) by (status_code, method)",
"hide": true,
"interval": "",
"legendFormat": "{{status_code}}-{{method}}",
"range": true,
"refId": "B"
}
],
"title": "Requests/Second",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 6,
"y": 34
},
"id": 101,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"editorMode": "code",
"expr": "memcached_max_connections{cluster=\"$cluster\",namespace=\"$namespace\"} - memcached_current_connections{cluster=\"$cluster\",namespace=\"$namespace\"}",
"hide": false,
"instant": false,
"legendFormat": "{{pod}}",
"range": true,
"refId": "A"
}
],
"title": "connections free",
"type": "timeseries"
},
{
"datasource": {
"type": "loki",
"uid": "${logsds}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unitScale": true
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "from querier"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "dark-red",
"mode": "fixed"
}
}
]
},
{
"__systemRef": "hideSeriesFrom",
"matcher": {
"id": "byNames",
"options": {
"mode": "exclude",
"names": [
"{component=\"querier\"}"
],
"prefix": "All except:",
"readOnly": true
}
},
"properties": [
{
"id": "custom.hideFrom",
"value": {
"legend": false,
"tooltip": false,
"viz": true
}
}
]
}
]
},
"gridPos": {
"h": 6,
"w": 6,
"x": 12,
"y": 34
},
"id": 104,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max",
"last"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${logsds}"
},
"editorMode": "code",
"expr": "sum by (component)(count_over_time({cluster=\"infra\",namespace=\"tempo-tracing\"} | logfmt | level = \"error\" | msg = `Failed to get keys from memcached`[1m]))",
"queryType": "range",
"refId": "A"
}
],
"title": "[logs] access problems from component",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 18,
"y": 34
},
"id": 127,
"links": [],
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "right",
"showLegend": true,
"width": 120
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "10.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editorMode": "code",
"expr": "sum (irate (memcached_commands_total{namespace=~\"$namespace\"}[5m])) by (command)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{command}}",
"range": true,
"refId": "A",
"step": 4,
"target": ""
}
],
"title": "Commands",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 40
},
"id": 117,
"panels": [],
"title": "service graph / metrics-generator paths",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 5,
"x": 0,
"y": 41
},
"id": 113,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editorMode": "code",
"expr": "sum (rate(tempo_distributor_metrics_generator_pushes_total{}[$__rate_interval]))",
"legendFormat": "pushes",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editorMode": "code",
"expr": "sum (rate(tempo_distributor_metrics_generator_pushes_failures_total{}[$__rate_interval]))",
"hide": false,
"legendFormat": "failures",
"range": true,
"refId": "B"
}
],
"title": "distributor metrics generator pushes",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 5,
"y": 41
},
"id": 114,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editorMode": "code",
"expr": "sum (rate(tempo_metrics_generator_spans_received_total{}[$__rate_interval]))",
"legendFormat": "spans",
"range": true,
"refId": "A"
}
],
"title": "Metrics-generator ingest spans",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "decbytes",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 9,
"y": 41
},
"id": 121,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editorMode": "code",
"expr": "sum (rate(tempo_metrics_generator_bytes_received_total{}[$__rate_interval]))",
"hide": false,
"legendFormat": "bytes",
"range": true,
"refId": "B"
}
],
"title": "Metrics-generator ingest bytes",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 7,
"x": 13,
"y": 41
},
"id": 115,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editorMode": "code",
"expr": "sum(tempo_metrics_generator_registry_active_series{}) by (pod)",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Amount of active series in the registry",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"description": "https://github.com/grafana/tempo/issues/2732#issuecomment-1658626251",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 20,
"y": 41
},
"id": 116,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editorMode": "code",
"expr": "sum (rate(prometheus_agent_samples_appended_total{pod=~'.*tempo-metrics-generator.*'}[$__rate_interval]))",
"legendFormat": "samples appended",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editorMode": "code",
"expr": "sum (rate(prometheus_remote_storage_samples_failed_total{pod=~'.*tempo-metrics-generator.*'}[$__rate_interval]))",
"hide": false,
"legendFormat": "samples failed",
"range": true,
"refId": "B"
}
],
"title": "Amount samples appended to the Prometheus WAL",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"description": "https://grafana.com/docs/tempo/latest/troubleshooting/metrics-generator/#remote-write-failures",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 8,
"x": 0,
"y": 47
},
"id": 118,
"options": {
"legend": {
"calcs": [
"min",
"max"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "sum(rate(prometheus_remote_storage_samples_failed_total{}[$__rate_interval]))",
"instant": false,
"legendFormat": "samples failed",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "sum(rate(prometheus_remote_storage_samples_dropped_total{}[$__rate_interval]))\n",
"hide": false,
"instant": false,
"legendFormat": "samples dropped",
"range": true,
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "sum(rate(prometheus_remote_storage_exemplars_failed_total{}[$__rate_interval]))",
"hide": false,
"instant": false,
"legendFormat": "exemplars failed",
"range": true,
"refId": "C"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "sum(rate(prometheus_remote_storage_exemplars_dropped_total{}[$__rate_interval]))",
"hide": false,
"instant": false,
"legendFormat": "exemplars dropped",
"range": true,
"refId": "D"
}
],
"title": "Prometheus remote write failure",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"description": "https://grafana.com/docs/tempo/latest/troubleshooting/metrics-generator/#expired-edges",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 8,
"x": 8,
"y": 47
},
"id": 119,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "sum(rate(tempo_metrics_generator_processor_service_graphs_edges{}[$__rate_interval]))",
"instant": false,
"legendFormat": "Rate of edges that have expired without a match",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "sum(rate(tempo_metrics_generator_processor_service_graphs_expired_edges{}[$__rate_interval]))",
"hide": false,
"instant": false,
"legendFormat": "Rate of all edges",
"range": true,
"refId": "B"
}
],
"title": "Expired edges",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"description": "https://grafana.com/docs/tempo/latest/troubleshooting/metrics-generator/#expired-edges",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 8,
"x": 16,
"y": 47
},
"id": 120,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "sum(rate(tempo_metrics_generator_processor_service_graphs_dropped_spans{}[$__rate_interval]))",
"instant": false,
"legendFormat": "edges are being dropped due to service graph max items",
"range": true,
"refId": "A"
}
],
"title": "edges are being dropped due to service graph max items",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 53
},
"id": 103,
"panels": [],
"title": "Problems check",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${logsds}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percentunit",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 6,
"x": 0,
"y": 54
},
"id": 105,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max",
"last"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true,
"sortBy": "Mean",
"sortDesc": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${logsds}"
},
"editorMode": "code",
"expr": "sum by (component) (count_over_time({cluster=\"$cluster\",namespace=\"$namespace\"} |~ `dial tcp 213.180.193.243:80: i/o timeout`[1m])) / sum by (component) (count_over_time({cluster=\"$cluster\",namespace=\"$namespace\"} [1m]))",
"legendFormat": "{{ component }}",
"queryType": "range",
"refId": "A"
},
{
"datasource": {
"type": "loki",
"uid": "${logsds}"
},
"editorMode": "code",
"expr": "sum by (component) (count_over_time({cluster=\"$cluster\",namespace=\"$namespace\"} |~ `dial tcp 213.180.193.243:80: i/o timeout|storage.yandexcloud.net`[1m])) / sum by (component) (count_over_time({cluster=\"$cluster\",namespace=\"$namespace\"} [1m]))",
"hide": true,
"legendFormat": "{{ component }}",
"queryType": "range",
"refId": "B"
}
],
"title": "S3 access problems by component by log messages",
"type": "timeseries"
},
{
"datasource": {
"type": "loki",
"uid": "${logsds}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percentunit",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 6,
"x": 6,
"y": 54
},
"id": 106,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max",
"last"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true,
"sortBy": "Mean",
"sortDesc": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${logsds}"
},
"editorMode": "code",
"expr": "count_over_time({site=\"platform-dev\", app=\"internal-infra-proxy\"} |~ \"55681|4317|9411\"[1m])",
"legendFormat": "{{ component }}",
"queryType": "range",
"refId": "A"
}
],
"title": "internal-infra-proxy to tempo access problem",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 6,
"x": 12,
"y": 54
},
"id": 107,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max",
"last"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "max(tempodb_backend_objects_total{cluster=\"$cluster\",namespace=\"$namespace\"}) by (tenant,job)",
"instant": false,
"legendFormat": "{{tenant}} - {{ job }}",
"range": true,
"refId": "A"
}
],
"title": "максимальное количество трейсов глазами сервисов",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "decbytes",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 6,
"x": 18,
"y": 54
},
"id": 108,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max",
"last"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true,
"sortBy": "Last",
"sortDesc": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "avg (container_memory_working_set_bytes{cluster=\"$cluster\",namespace=\"$namespace\"}[$__interval]) by (pod)",
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Kubernetes pod memory size",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 6,
"x": 0,
"y": 62
},
"id": 109,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max",
"last"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "increase(kube_pod_container_status_restarts_total{namespace=\"tempo-tracing\"}[5m]) > 0",
"hide": false,
"interval": "",
"legendFormat": "{{pod}}",
"range": true,
"refId": "A"
}
],
"title": "Container Restarts",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unitScale": true
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Reason"
},
"properties": [
{
"id": "custom.width",
"value": 104
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 8,
"x": 6,
"y": 62
},
"id": 110,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"enablePagination": true,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "Count"
}
]
},
"pluginVersion": "10.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"exemplar": false,
"expr": "sum(changes(kube_pod_container_status_last_terminated_reason{cluster=\"infra\",namespace=\"tempo-tracing\"}[1m]) > 0) by (pod,reason)",
"format": "table",
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"exemplar": false,
"expr": "count(kube_pod_container_status_last_terminated_reason{reason=\"OOMKilled\",cluster=\"infra\",namespace=\"tempo-tracing\"}) by (reason, container, pod)",
"format": "table",
"hide": true,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"exemplar": false,
"expr": "kube_pod_container_status_last_terminated_reason{cluster=\"infra\",namespace=\"tempo-tracing\"}",
"format": "table",
"hide": true,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "C"
}
],
"title": "Restart count by pod",
"transformations": [
{
"id": "groupBy",
"options": {
"fields": {
"Value": {
"aggregations": [
"count"
],
"operation": "aggregate"
},
"container": {
"aggregations": [],
"operation": "groupby"
},
"count": {
"aggregations": [
"count"
],
"operation": "aggregate"
},
"pod": {
"aggregations": [],
"operation": "groupby"
},
"reason": {
"aggregations": [
"count"
],
"operation": "groupby"
}
}
}
},
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true,
"Value": false
},
"indexByName": {
"Time": 0,
"Value": 4,
"container": 3,
"pod": 2,
"reason": 1
},
"renameByName": {
"Value": "count",
"Value (count)": "Count",
"container": "Container",
"pod": "Pod",
"reason": "Reason"
}
}
}
],
"type": "table"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"description": "Если график не в 0, то под тротлит по CPU",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"axisSoftMax": 1,
"axisSoftMin": 0,
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percentunit",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 10,
"x": 14,
"y": 62
},
"id": 130,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max",
"last"
],
"displayMode": "table",
"placement": "right",
"showLegend": true,
"sortBy": "Last",
"sortDesc": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"editorMode": "code",
"exemplar": false,
"expr": "sum(increase(container_cpu_cfs_throttled_periods_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1m])) by (container) / sum(increase(container_cpu_cfs_periods_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1m])) by (container)\n",
"instant": false,
"legendFormat": "{{container}}",
"range": true,
"refId": "A"
}
],
"title": "CPU Throttling",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percentunit",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 70
},
"id": 131,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max",
"last"
],
"displayMode": "table",
"placement": "right",
"showLegend": true,
"sortBy": "Last",
"sortDesc": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "avg (container_memory_working_set_bytes{namespace=\"$namespace\"}[1m]) by (container, pod) / avg (container_memory_limit_bytes{namespace=\"$namespace\"}>0 [1m]) by (container, pod)",
"instant": false,
"legendFormat": "{{ container }} - {{pod}}",
"range": true,
"refId": "A"
}
],
"title": "Memory utilization from limit, %",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percentunit",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 80
},
"id": 132,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max",
"last"
],
"displayMode": "table",
"placement": "right",
"showLegend": true,
"sortBy": "Last",
"sortDesc": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "avg (container_memory_working_set_bytes{namespace=\"$namespace\"}[1m]) by (container, pod) / avg (container_memory_request_bytes{namespace=\"$namespace\"}>0 [1m]) by (container, pod)",
"instant": false,
"legendFormat": "{{ container }} - {{pod}}",
"range": true,
"refId": "A"
}
],
"title": "Memory utilization from request, %",
"type": "timeseries"
},
{
"datasource": {
"type": "loki",
"uid": "${logsds}"
},
"gridPos": {
"h": 27,
"w": 24,
"x": 0,
"y": 90
},
"id": 112,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${logsds}"
},
"editorMode": "code",
"expr": "{cluster=\"$cluster\",namespace=\"$namespace\"} | logfmt | level=\"error\" !~ \"rpc error: code = Canceled|Failed to get keys from memcached\"",
"queryType": "range",
"refId": "A"
}
],
"title": "Errors in tempo (wo \"context canceled\" and \"Failed to get keys from memcached\")",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "${logsds}"
},
"gridPos": {
"h": 13,
"w": 24,
"x": 0,
"y": 117
},
"id": 111,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${logsds}"
},
"editorMode": "code",
"expr": "{cluster=\"$cluster\",namespace=\"$namespace\"} |= `TRACE_TOO_LARGE`",
"queryType": "range",
"refId": "A"
}
],
"title": "Trace too large: overweighted traces",
"type": "logs"
},
{
"collapsed": false,
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 130
},
"id": 49,
"panels": [],
"title": "Maintenance",
"type": "row"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percentunit",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 0,
"y": 131
},
"id": 33,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "tempodb_work_queue_length{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"} / tempodb_work_queue_max{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"title": "%age total work queue",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 3,
"y": 131
},
"id": 32,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "sum(increase(tempodb_compaction_errors_total{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}[$__rate_interval])) by (job)",
"legendFormat": "compaction_err",
"refId": "B"
},
{
"expr": "sum(increase(tempodb_retention_errors_total{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}[$__rate_interval])) by (job)",
"legendFormat": "retention_err",
"refId": "C"
},
{
"expr": "sum(increase(tempodb_blocklist_poll_errors_total{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}[$__rate_interval])) by (job)",
"legendFormat": "blocklist_err",
"refId": "D"
}
],
"title": "maintenance errors",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "points",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 6,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "always",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 6,
"y": 131
},
"id": 35,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"editorMode": "code",
"expr": "histogram_quantile(.99, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".99",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"expr": "histogram_quantile(.9, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}[$__rate_interval])) by (le))",
"legendFormat": ".9",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"expr": "histogram_quantile(.5, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".5",
"refId": "C"
}
],
"title": "Blocklist Poll Duration",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 9,
"y": 131
},
"id": 47,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"exemplar": true,
"expr": "avg(tempodb_blocklist_length{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}) by (tenant)",
"instant": false,
"interval": "",
"legendFormat": "{{tenant}}",
"refId": "A"
}
],
"title": "Blocklist Length",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "points",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 6,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "always",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 12,
"y": 131
},
"id": 51,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "histogram_quantile(.99, sum(rate(tempodb_retention_duration_seconds_bucket{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-compactor\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".99",
"refId": "A"
},
{
"expr": "histogram_quantile(.9, sum(rate(tempodb_retention_duration_seconds_bucket{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-compactor\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".9",
"refId": "B"
},
{
"expr": "histogram_quantile(.5, sum(rate(tempodb_retention_duration_seconds_bucket{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-compactor\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".5",
"refId": "C"
}
],
"title": "retention duration",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 15,
"y": 131
},
"id": 53,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "sum(increase(tempodb_retention_deleted_total{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}[$__rate_interval]))",
"interval": "",
"legendFormat": "deleted",
"refId": "A"
},
{
"expr": "sum(increase(tempodb_retention_marked_for_deletion_total{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}[$__rate_interval]))",
"interval": "",
"legendFormat": "marked_for_deletion",
"refId": "B"
}
],
"title": "retention",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 18,
"y": 131
},
"id": 70,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "increase(kube_pod_container_status_restarts_total{namespace=\"$namespace\"}[5m]) > 0",
"hide": false,
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"title": "Container Restarts",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "decbytes",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 21,
"y": 131
},
"id": 45,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "asc"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "kubelet_volume_stats_available_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\"$component.*\"}",
"legendFormat": "{{persistentvolumeclaim}}",
"refId": "A"
}
],
"title": "data volume free",
"type": "timeseries"
},
{
"collapsed": false,
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 136
},
"id": 28,
"panels": [],
"title": "Backend",
"type": "row"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 137
},
"id": 31,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "sum(rate(tempodb_backend_request_duration_seconds_count{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}[$__rate_interval])) by (status_code, operation)",
"interval": "",
"legendFormat": "{{status_code}}-{{operation}}",
"refId": "A"
}
],
"title": "Requests/Second",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 137
},
"id": 30,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "histogram_quantile(.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}[$__rate_interval])) by (operation, le))",
"legendFormat": ".99-{{operation}}",
"refId": "A"
},
{
"expr": "histogram_quantile(.9, sum(rate(tempodb_backend_request_duration_seconds_bucket{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}[$__rate_interval])) by (operation, le))",
"legendFormat": ".9-{{operation}}",
"refId": "B"
},
{
"expr": "histogram_quantile(.5, sum(rate(tempodb_backend_request_duration_seconds_bucket{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}[$__rate_interval])) by (operation, le))",
"legendFormat": ".5-{{operation}}",
"refId": "C"
}
],
"title": "Latency By Operation",
"type": "timeseries"
},
{
"collapsed": false,
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 145
},
"id": 56,
"panels": [],
"title": "Ring/Memberlist",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 6,
"x": 0,
"y": 146
},
"id": 58,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P666011C0B63BDCA4"
},
"editorMode": "code",
"expr": "gauge_memberlist_health_score{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}",
"interval": "",
"legendFormat": "{{instance}}",
"range": true,
"refId": "A"
}
],
"title": "Gossip Health",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 6,
"x": 6,
"y": 146
},
"id": 62,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "tempo_memberlist_client_cluster_node_health_score{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}",
"interval": "",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"title": "Node Health",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 6,
"x": 12,
"y": 146
},
"id": 63,
"options": {
"legend": {
"calcs": [
"min",
"mean",
"max",
"last"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "min(tempo_memberlist_client_cluster_members_count{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"})",
"interval": "",
"legendFormat": "min",
"refId": "A"
},
{
"expr": "max(tempo_memberlist_client_cluster_members_count{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"})",
"interval": "",
"legendFormat": "max",
"refId": "B"
}
],
"title": "Member Count",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 6,
"x": 18,
"y": 146
},
"id": 60,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P666011C0B63BDCA4"
},
"expr": "min(tempo_memberlist_client_kv_store_count{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"})",
"interval": "",
"legendFormat": "min",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "P666011C0B63BDCA4"
},
"expr": "max(tempo_memberlist_client_kv_store_count{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"})",
"interval": "",
"legendFormat": "max",
"refId": "B"
}
],
"title": "KV Store Count",
"type": "timeseries"
},
{
"collapsed": false,
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 154
},
"id": 69,
"panels": [],
"title": "Vulture",
"type": "row"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": []
},
"unit": "percentunit",
"unitScale": true
},
"overrides": [
{
"__systemRef": "hideSeriesFrom",
"matcher": {
"id": "byNames",
"options": {
"mode": "exclude",
"names": [
"requestfailed"
],
"prefix": "All except:",
"readOnly": true
}
},
"properties": [
{
"id": "custom.hideFrom",
"value": {
"legend": false,
"tooltip": false,
"viz": true
}
}
]
}
]
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 155
},
"id": 77,
"options": {
"graph": {},
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "7.5.0-14141pre",
"targets": [
{
"expr": "sum(rate(tempo_vulture_trace_error_total{namespace=\"$namespace\"}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{namespace=\"$namespace\"}[$__rate_interval]))",
"interval": "",
"legendFormat": "{{error}}",
"refId": "A"
}
],
"title": "Vulture Query Errors",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
}
]
},
"unit": "percentunit",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 155
},
"id": 67,
"options": {
"displayMode": "gradient",
"maxVizHeight": 300,
"minVizHeight": 10,
"minVizWidth": 0,
"namePlacement": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"showUnfilled": true,
"sizing": "auto",
"text": {},
"valueMode": "color"
},
"pluginVersion": "10.3.1",
"targets": [
{
"expr": "sum(rate(tempo_vulture_trace_error_total{namespace=\"$namespace\"}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{namespace=\"$namespace\"}[$__rate_interval]))",
"interval": "",
"legendFormat": "{{secondsago}}",
"refId": "A"
}
],
"title": "Average Vulture Query Errors",
"type": "bargauge"
},
{
"collapsed": false,
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 164
},
"id": 81,
"panels": [],
"title": "Compactor",
"type": "row"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 0,
"y": 165
},
"id": 83,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "sum(rate(tempodb_compaction_objects_combined_total{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-compactor\"}[$__rate_interval])) by (level)",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Objects Combined / s",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 4,
"y": 165
},
"id": 85,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "sum(rate(tempodb_compaction_objects_written_total{namespace=\"$namespace\", job=\"tempo-grafana-tempo-compactor\"}[$__rate_interval])) by (level)",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Objects Written / s",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 8,
"y": 165
},
"id": 88,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P666011C0B63BDCA4"
},
"editorMode": "builder",
"expr": "sum(rate(tempodb_compaction_bytes_written_total{namespace=\"$namespace\", job=\"tempo-grafana-tempo-compactor\"}[$__rate_interval])) by (level)",
"interval": "",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Bytes Written / s",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 12,
"y": 165
},
"id": 86,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "P666011C0B63BDCA4"
},
"editorMode": "code",
"expr": "sum(increase(tempodb_compaction_blocks_total{namespace=\"$namespace\", job=\"tempo-grafana-tempo-compactor\"}[5m])) by (level)",
"interval": "",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Blocks Compacted",
"type": "timeseries"
},
{
"collapsed": true,
"datasource": {
"type": "prometheus",
"uid": "P1809F7CD0C75ACF3"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 171
},
"id": 38,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 0,
"y": 169
},
"id": 24,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"expr": "rate(go_gc_duration_seconds_count{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}[$__rate_interval])",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"title": "gcs",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "decbytes"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 3,
"y": 169
},
"id": 25,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"expr": "go_memstats_heap_inuse_bytes{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"title": "go heap",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 6,
"y": 169
},
"id": 23,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "victoriametrics-platform"
},
"expr": "go_goroutines{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"title": "goroutines",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "continuous-BlYlRd"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 9,
"y": 169
},
"id": 42,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"$component.*\", container!=\"POD\"}[$__rate_interval])",
"interval": "",
"intervalFactor": 5,
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"title": "cpu",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "decbytes"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 12,
"y": 169
},
"id": 43,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"$component.*\", container!=\"POD\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"title": "working set",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "decbytes"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 15,
"y": 169
},
"id": 44,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "rate(container_network_receive_bytes_total{namespace=\"$namespace\", pod=~\"$component.*\"}[$__rate_interval])",
"hide": false,
"interval": "",
"legendFormat": "rx-{{pod}}",
"refId": "A"
},
{
"expr": "rate(container_network_transmit_bytes_total{namespace=\"$namespace\", pod=~\"$component.*\"}[$__rate_interval])",
"hide": false,
"interval": "",
"legendFormat": "tx-{{pod}}",
"refId": "B"
}
],
"title": "tx/rx",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 18,
"y": 169
},
"id": 46,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "rate(promtail_custom_bad_words_total{exported_namespace=\"$namespace\", app=~\"$component.*\"}[$__rate_interval])",
"interval": "",
"legendFormat": "{{exported_pod}}",
"refId": "A"
}
],
"title": "bad words",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 4,
"x": 0,
"y": 174
},
"id": 34,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "sum(rate(tempo_request_duration_seconds_count{route=~\".*api_traces_traceid\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"tempo-grafana-tempo-cortex-gw\"}[$__rate_interval])) by (status_code)",
"hide": false,
"interval": "",
"legendFormat": "{{status_code}}",
"refId": "A"
}
],
"title": "Queries/Sec (cortex-gw)",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 8,
"x": 4,
"y": 174
},
"id": 78,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-cortex-gw\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".99",
"refId": "A"
},
{
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-cortex-gw\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".9",
"refId": "B"
},
{
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-cortex-gw\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".5",
"refId": "C"
}
],
"title": "Query Latency (Gateway)",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 4,
"x": 12,
"y": 174
},
"id": 97,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"exemplar": true,
"expr": "sum(rate(tempo_request_duration_seconds_count{route=~\".*api_search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"tempo-grafana-tempo-cortex-gw\"}[$__rate_interval])) by (status_code)",
"hide": false,
"interval": "",
"legendFormat": "{{status_code}}",
"refId": "A",
"stepMode": "min"
}
],
"title": "Search Queries/Sec (cortex-gw)",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 8,
"x": 16,
"y": 174
},
"id": 93,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"exemplar": true,
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-cortex-gw\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".99",
"refId": "A",
"stepMode": "min"
},
{
"exemplar": true,
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-cortex-gw\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
"hide": false,
"interval": "",
"legendFormat": ".9",
"refId": "B",
"stepMode": "min"
},
{
"exemplar": true,
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-cortex-gw\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
"hide": false,
"interval": "",
"legendFormat": ".5",
"refId": "C",
"stepMode": "min"
}
],
"title": "Search Query Latency (Gateway)",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 7,
"x": 0,
"y": 179
},
"id": 71,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d452322apre",
"targets": [
{
"expr": "sum(rate(tempo_request_duration_seconds_count{namespace=\"$namespace\", job=\"tempo-grafana-tempo-cortex-gw\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (status_code)",
"interval": "",
"legendFormat": "{{status_code}}",
"refId": "A"
}
],
"title": "Pushes/sec (gateway)",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 6,
"x": 7,
"y": 179
},
"id": 79,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d452322apre",
"targets": [
{
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-cortex-gw\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".99",
"refId": "A"
},
{
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-cortex-gw\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".9",
"refId": "B"
},
{
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{namespace=\"$namespace\", job=\"tempo-grafana-tempo-cortex-gw\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))",
"interval": "",
"legendFormat": ".5",
"refId": "C"
}
],
"title": "Push Latency (Gateway)",
"type": "timeseries"
},
{
"datasource": {
"uid": "$ds"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 7,
"y": 184
},
"id": 76,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "9.0.0-d373beebpre",
"targets": [
{
"expr": "histogram_quantile(.99, sum(rate(cortex_memcache_request_duration_seconds_bucket{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}[$__rate_interval])) by (method, le))",
"interval": "",
"legendFormat": ".99-{{method}}",
"refId": "A"
},
{
"expr": "histogram_quantile(.9, sum(rate(cortex_memcache_request_duration_seconds_bucket{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}[$__rate_interval])) by (method, le))",
"interval": "",
"legendFormat": ".9-{{method}}",
"refId": "B"
},
{
"expr": "histogram_quantile(.5, sum(rate(cortex_memcache_request_duration_seconds_bucket{namespace=\"$namespace\", job=~\"tempo-grafana-tempo-$component\"}[$__rate_interval])) by (method, le))",
"interval": "",
"legendFormat": ".5-{{method}}",
"refId": "C"
}
],
"title": "memcached Latency By Operation",
"type": "timeseries"
}
],
"title": "unusable info",
"type": "row"
}
],
"refresh": "5m",
"schemaVersion": 39,
"tags": [
"tempo"
],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "VictoriaMetrics-platform",
"value": "victoriametrics-platform"
},
"hide": 0,
"includeAll": false,
"multi": false,
"name": "ds",
"options": [],
"query": "prometheus",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"current": {
"selected": false,
"text": "Loki-platform",
"value": "loki-platform"
},
"hide": 0,
"includeAll": false,
"multi": false,
"name": "logsds",
"options": [],
"query": "loki",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"current": {
"selected": false,
"text": "infra",
"value": "infra"
},
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"definition": "label_values(tempo_build_info, cluster)",
"hide": 0,
"includeAll": false,
"multi": false,
"name": "cluster",
"options": [],
"query": {
"query": "label_values(tempo_build_info, cluster)",
"refId": "ops-cortex-cluster-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"current": {
"selected": false,
"text": "tempo-tracing",
"value": "tempo-tracing"
},
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"definition": "label_values(tempo_build_info, namespace)",
"hide": 0,
"includeAll": false,
"multi": false,
"name": "namespace",
"options": [],
"query": {
"query": "label_values(tempo_build_info, namespace)",
"refId": "ops-cortex-namespace-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": false,
"text": "All",
"value": "$__all"
},
"hide": 0,
"includeAll": true,
"multi": false,
"name": "component",
"options": [
{
"selected": true,
"text": "All",
"value": "$__all"
},
{
"selected": false,
"text": "compactor",
"value": "compactor"
},
{
"selected": false,
"text": "distributor",
"value": "distributor"
},
{
"selected": false,
"text": "ingester",
"value": "ingester"
},
{
"selected": false,
"text": "metrics-generator",
"value": "metrics-generator"
},
{
"selected": false,
"text": "query-frontend",
"value": "query-frontend"
},
{
"selected": false,
"text": "querier",
"value": "querier"
},
{
"selected": false,
"text": "cortex-gw",
"value": "cortex-gw"
}
],
"query": "compactor,distributor,ingester,metrics-generator,query-frontend,querier,cortex-gw",
"queryValue": "",
"skipUrlSync": false,
"type": "custom"
}
]
},
"time": {
"from": "now-2d",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
]
},
"timezone": "",
"title": "Tempo Operational",
"uid": "a6175b9cc7ec20591890117c39580030",
"version": 55,
"weekStart": ""
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment