Created
May 21, 2020 20:10
-
-
Save chancez/9dddbde493db1f1be0fad0e3773fbc60 to your computer and use it in GitHub Desktop.
Loki dashboards that work with monolithic mode or microservices mode
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "annotations": { | |
| "list": [ ] | |
| }, | |
| "editable": true, | |
| "gnetId": null, | |
| "graphTooltip": 0, | |
| "hideControls": false, | |
| "links": [ ], | |
| "refresh": "10s", | |
| "rows": [ | |
| { | |
| "collapse": false, | |
| "height": "250px", | |
| "panels": [ | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "id": 1, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=\"$namespace/ingester\"})", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "series", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Series", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "id": 2, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=\"$namespace/ingester\"}) / sum(loki_ingester_memory_streams{job=\"$namespace/ingester\"})", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "chunks", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Chunks per series", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| } | |
| ], | |
| "repeat": null, | |
| "repeatIteration": null, | |
| "repeatRowId": null, | |
| "showTitle": true, | |
| "title": "Active Series / Chunks", | |
| "titleSize": "h6" | |
| }, | |
| { | |
| "collapse": false, | |
| "height": "250px", | |
| "panels": [ | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "id": 3, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=\"$namespace/ingester\"}[$__interval])) by (le)) * 1", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "99th Percentile", | |
| "refId": "A", | |
| "step": 10 | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=\"$namespace/ingester\"}[$__interval])) by (le)) * 1", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "50th Percentile", | |
| "refId": "B", | |
| "step": 10 | |
| }, | |
| { | |
| "expr": "sum(rate(loki_ingester_chunk_utilization_sum{cluster=\"$cluster\", job=\"$namespace/ingester\"}[$__interval])) * 1 / sum(rate(loki_ingester_chunk_utilization_count{cluster=\"$cluster\", job=\"$namespace/ingester\"}[$__interval]))", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "Average", | |
| "refId": "C", | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Utilization", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "percentunit", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "id": 4, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/ingester\"}[$__interval])) by (le)) * 1e3", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "99th Percentile", | |
| "refId": "A", | |
| "step": 10 | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/ingester\"}[$__interval])) by (le)) * 1e3", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "50th Percentile", | |
| "refId": "B", | |
| "step": 10 | |
| }, | |
| { | |
| "expr": "sum(rate(loki_ingester_chunk_age_seconds_sum{cluster=\"$cluster\", job=\"$namespace/ingester\"}[$__interval])) * 1e3 / sum(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=\"$namespace/ingester\"}[$__interval]))", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "Average", | |
| "refId": "C", | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Age", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "ms", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| } | |
| ], | |
| "repeat": null, | |
| "repeatIteration": null, | |
| "repeatRowId": null, | |
| "showTitle": true, | |
| "title": "Flush Stats", | |
| "titleSize": "h6" | |
| }, | |
| { | |
| "collapse": false, | |
| "height": "250px", | |
| "panels": [ | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "id": 5, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=\"$namespace/ingester\"}[$__interval])) by (le)) * 1", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "99th Percentile", | |
| "refId": "A", | |
| "step": 10 | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=\"$namespace/ingester\"}[$__interval])) by (le)) * 1", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "50th Percentile", | |
| "refId": "B", | |
| "step": 10 | |
| }, | |
| { | |
| "expr": "sum(rate(loki_ingester_chunk_entries_sum{cluster=\"$cluster\", job=\"$namespace/ingester\"}[$__interval])) * 1 / sum(rate(loki_ingester_chunk_entries_count{cluster=\"$cluster\", job=\"$namespace/ingester\"}[$__interval]))", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "Average", | |
| "refId": "C", | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Size", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "id": 6, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum(rate(cortex_chunk_store_index_entries_per_chunk_sum{cluster=\"$cluster\", job=\"$namespace/ingester\"}[5m])) / sum(rate(cortex_chunk_store_index_entries_per_chunk_count{cluster=\"$cluster\", job=\"$namespace/ingester\"}[5m]))", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "entries", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Entries", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| } | |
| ], | |
| "repeat": null, | |
| "repeatIteration": null, | |
| "repeatRowId": null, | |
| "showTitle": true, | |
| "title": "Flush Stats", | |
| "titleSize": "h6" | |
| }, | |
| { | |
| "collapse": false, | |
| "height": "250px", | |
| "panels": [ | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 1, | |
| "id": 7, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "cortex_ingester_flush_queue_length{cluster=\"$cluster\", job=\"$namespace/ingester\"}", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "{{instance}}", | |
| "legendLink": null, | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Queue Length", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| }, | |
| { | |
| "aliasColors": { | |
| "1xx": "#EAB839", | |
| "2xx": "#7EB26D", | |
| "3xx": "#6ED0E0", | |
| "4xx": "#EF843C", | |
| "5xx": "#E24D42", | |
| "error": "#E24D42", | |
| "success": "#7EB26D" | |
| }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fill": 10, | |
| "id": 8, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 0, | |
| "links": [ ], | |
| "nullPointMode": "null as zero", | |
| "percentage": false, | |
| "pointradius": 5, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "span": 6, | |
| "stack": true, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum by (status) (label_replace(label_replace(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=\"$namespace/ingester\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", | |
| "format": "time_series", | |
| "intervalFactor": 2, | |
| "legendFormat": "{{status}}", | |
| "refId": "A", | |
| "step": 10 | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Flush Rate", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": 0, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ] | |
| } | |
| ], | |
| "repeat": null, | |
| "repeatIteration": null, | |
| "repeatRowId": null, | |
| "showTitle": true, | |
| "title": "Flush Stats", | |
| "titleSize": "h6" | |
| } | |
| ], | |
| "schemaVersion": 14, | |
| "style": "dark", | |
| "tags": [ ], | |
| "templating": { | |
| "list": [ | |
| { | |
| "current": { | |
| "text": "default", | |
| "value": "default" | |
| }, | |
| "hide": 0, | |
| "label": null, | |
| "name": "datasource", | |
| "options": [ ], | |
| "query": "prometheus", | |
| "refresh": 1, | |
| "regex": "", | |
| "type": "datasource" | |
| }, | |
| { | |
| "allValue": null, | |
| "current": { | |
| "text": "prod", | |
| "value": "prod" | |
| }, | |
| "datasource": "$datasource", | |
| "hide": 0, | |
| "includeAll": false, | |
| "label": "cluster", | |
| "multi": false, | |
| "name": "cluster", | |
| "options": [ ], | |
| "query": "label_values(kube_pod_container_info{image=~\".*loki.*\"}, cluster)", | |
| "refresh": 1, | |
| "regex": "", | |
| "sort": 2, | |
| "tagValuesQuery": "", | |
| "tags": [ ], | |
| "tagsQuery": "", | |
| "type": "query", | |
| "useTags": false | |
| }, | |
| { | |
| "allValue": null, | |
| "current": { | |
| "text": "prod", | |
| "value": "prod" | |
| }, | |
| "datasource": "$datasource", | |
| "hide": 0, | |
| "includeAll": false, | |
| "label": "namespace", | |
| "multi": false, | |
| "name": "namespace", | |
| "options": [ ], | |
| "query": "label_values(kube_pod_container_info{image=~\".*loki.*\"}, namespace)", | |
| "refresh": 1, | |
| "regex": "", | |
| "sort": 2, | |
| "tagValuesQuery": "", | |
| "tags": [ ], | |
| "tagsQuery": "", | |
| "type": "query", | |
| "useTags": false | |
| } | |
| ] | |
| }, | |
| "time": { | |
| "from": "now-1h", | |
| "to": "now" | |
| }, | |
| "timepicker": { | |
| "refresh_intervals": [ | |
| "5s", | |
| "10s", | |
| "30s", | |
| "1m", | |
| "5m", | |
| "15m", | |
| "30m", | |
| "1h", | |
| "2h", | |
| "1d" | |
| ], | |
| "time_options": [ | |
| "5m", | |
| "15m", | |
| "1h", | |
| "6h", | |
| "12h", | |
| "24h", | |
| "2d", | |
| "7d", | |
| "30d" | |
| ] | |
| }, | |
| "timezone": "utc", | |
| "title": "Loki / Chunks", | |
| "uid": "", | |
| "version": 0 | |
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "annotations": { | |
| "list": [ | |
| { | |
| "builtIn": 1, | |
| "datasource": "-- Grafana --", | |
| "enable": true, | |
| "hide": true, | |
| "iconColor": "rgba(0, 211, 255, 1)", | |
| "name": "Annotations & Alerts", | |
| "type": "dashboard" | |
| } | |
| ] | |
| }, | |
| "editable": true, | |
| "gnetId": null, | |
| "graphTooltip": 1, | |
| "id": 8, | |
| "iteration": 1583185057230, | |
| "links": [ ], | |
| "panels": [ | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$metrics", | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 4, | |
| "w": 3, | |
| "x": 0, | |
| "y": 0 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 35, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": false, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum(go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$deployment.*\", instance=~\"$pod.*\"})", | |
| "refId": "A" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "goroutines", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$metrics", | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 4, | |
| "w": 3, | |
| "x": 3, | |
| "y": 0 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 41, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": false, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum(go_gc_duration_seconds{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$deployment.*\", instance=~\"$pod.*\"}) by (quantile)", | |
| "legendFormat": "{{quantile}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "gc duration", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$metrics", | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 4, | |
| "w": 3, | |
| "x": 6, | |
| "y": 0 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 36, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": false, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=~\"$deployment.*\", pod_name=~\"$pod\", container_name=~\"$container\"}[5m]))", | |
| "refId": "A" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "cpu", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$metrics", | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 4, | |
| "w": 3, | |
| "x": 9, | |
| "y": 0 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 40, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": false, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=~\"$deployment.*\", pod_name=~\"$pod\", container_name=~\"$container\"})", | |
| "refId": "A" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "working set", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "bytes", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$metrics", | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 4, | |
| "w": 3, | |
| "x": 12, | |
| "y": 0 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 38, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": false, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=~\"$deployment.*\", pod_name=~\"$pod\"}[5m]))", | |
| "refId": "A" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "tx", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "bytes", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$metrics", | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 4, | |
| "w": 3, | |
| "x": 15, | |
| "y": 0 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 39, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": false, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=~\"$deployment.*\", pod_name=~\"$pod\"}[5m]))", | |
| "refId": "A" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "rx", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "decbytes", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$metrics", | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 4, | |
| "w": 3, | |
| "x": 18, | |
| "y": 0 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 37, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": false, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "increase(kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[30m]) > 0", | |
| "legendFormat": "{{reason}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "restarts", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "bytes", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$metrics", | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 4, | |
| "w": 3, | |
| "x": 21, | |
| "y": 0 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 42, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": false, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum(rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", exported_instance=~\"$deployment.*\", exported_instance=~\"$pod.*\", container_name=~\"$container\"}[5m])) by (level)", | |
| "legendFormat": "{{level}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "bad words", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "bytes", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": true, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$logmetrics", | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 2, | |
| "w": 24, | |
| "x": 0, | |
| "y": 4 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 31, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": false, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": false, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ | |
| { | |
| "alias": "warn", | |
| "color": "#FF780A" | |
| }, | |
| { | |
| "alias": "error", | |
| "color": "#E02F44" | |
| }, | |
| { | |
| "alias": "info", | |
| "color": "#56A64B" | |
| }, | |
| { | |
| "alias": "debug", | |
| "color": "#3274D9" | |
| } | |
| ], | |
| "spaceLength": 10, | |
| "stack": true, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$deployment.*\", instance=~\"$pod.*\", container_name=~\"$container\", level=~\"$level\"}$filter[5m])) by (level)", | |
| "intervalFactor": 3, | |
| "legendFormat": "{{level}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 2, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": false, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": false | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "datasource": "$logs", | |
| "gridPos": { | |
| "h": 19, | |
| "w": 24, | |
| "x": 0, | |
| "y": 6 | |
| }, | |
| "id": 29, | |
| "maxDataPoints": "", | |
| "options": { | |
| "showLabels": false, | |
| "showTime": true, | |
| "sortOrder": "Descending", | |
| "wrapLogMessage": true | |
| }, | |
| "targets": [ | |
| { | |
| "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", instance=~\"$deployment.*\", instance=~\"$pod.*\", container_name=~\"$container\", level=~\"$level\"} $filter", | |
| "refId": "A" | |
| } | |
| ], | |
| "timeFrom": null, | |
| "timeShift": null, | |
| "title": "Logs", | |
| "type": "logs" | |
| } | |
| ], | |
| "refresh": false, | |
| "schemaVersion": 22, | |
| "style": "dark", | |
| "tags": [ ], | |
| "templating": { | |
| "list": [ | |
| { | |
| "hide": 0, | |
| "includeAll": false, | |
| "label": null, | |
| "multi": false, | |
| "name": "logs", | |
| "options": [ ], | |
| "query": "loki", | |
| "refresh": 1, | |
| "regex": "", | |
| "skipUrlSync": false, | |
| "type": "datasource" | |
| }, | |
| { | |
| "hide": 0, | |
| "includeAll": false, | |
| "label": null, | |
| "multi": false, | |
| "name": "logmetrics", | |
| "options": [ ], | |
| "query": "prometheus", | |
| "refresh": 1, | |
| "regex": "", | |
| "skipUrlSync": false, | |
| "type": "datasource" | |
| }, | |
| { | |
| "hide": 0, | |
| "includeAll": false, | |
| "label": null, | |
| "multi": false, | |
| "name": "metrics", | |
| "options": [ ], | |
| "query": "prometheus", | |
| "refresh": 1, | |
| "regex": "", | |
| "skipUrlSync": false, | |
| "type": "datasource" | |
| }, | |
| { | |
| "allValue": null, | |
| "datasource": "$metrics", | |
| "definition": "label_values(kube_pod_container_info, cluster)", | |
| "hide": 0, | |
| "includeAll": false, | |
| "label": null, | |
| "multi": false, | |
| "name": "cluster", | |
| "options": [ ], | |
| "query": "label_values(kube_pod_container_info, cluster)", | |
| "refresh": 1, | |
| "regex": "", | |
| "skipUrlSync": false, | |
| "sort": 0, | |
| "tagValuesQuery": "", | |
| "tags": [ ], | |
| "tagsQuery": "", | |
| "type": "query", | |
| "useTags": false | |
| }, | |
| { | |
| "allValue": null, | |
| "datasource": "$metrics", | |
| "definition": "label_values(kube_pod_container_info{cluster=\"$cluster\"}, namespace)", | |
| "hide": 0, | |
| "includeAll": false, | |
| "label": null, | |
| "multi": false, | |
| "name": "namespace", | |
| "options": [ ], | |
| "query": "label_values(kube_pod_container_info{cluster=\"$cluster\"}, namespace)", | |
| "refresh": 1, | |
| "regex": "", | |
| "skipUrlSync": false, | |
| "sort": 0, | |
| "tagValuesQuery": "", | |
| "tags": [ ], | |
| "tagsQuery": "", | |
| "type": "query", | |
| "useTags": false | |
| }, | |
| { | |
| "allValue": ".*", | |
| "datasource": "$metrics", | |
| "definition": "label_values(kube_deployment_created{cluster=\"$cluster\", namespace=\"$namespace\"}, deployment)", | |
| "hide": 0, | |
| "includeAll": false, | |
| "label": null, | |
| "multi": false, | |
| "name": "deployment", | |
| "options": [ ], | |
| "query": "label_values(kube_deployment_created{cluster=\"$cluster\", namespace=\"$namespace\"}, deployment)", | |
| "refresh": 1, | |
| "regex": "", | |
| "skipUrlSync": false, | |
| "sort": 0, | |
| "tagValuesQuery": "", | |
| "tags": [ ], | |
| "tagsQuery": "", | |
| "type": "query", | |
| "useTags": false | |
| }, | |
| { | |
| "allValue": ".*", | |
| "datasource": "$metrics", | |
| "definition": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\"}, pod)", | |
| "hide": 0, | |
| "includeAll": true, | |
| "label": null, | |
| "multi": false, | |
| "name": "pod", | |
| "options": [ ], | |
| "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\"}, pod)", | |
| "refresh": 1, | |
| "regex": "", | |
| "skipUrlSync": false, | |
| "sort": 0, | |
| "tagValuesQuery": "", | |
| "tags": [ ], | |
| "tagsQuery": "", | |
| "type": "query", | |
| "useTags": false | |
| }, | |
| { | |
| "allValue": ".*", | |
| "datasource": "$metrics", | |
| "definition": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\", pod=~\"$deployment.*\"}, container)", | |
| "hide": 0, | |
| "includeAll": true, | |
| "label": null, | |
| "multi": false, | |
| "name": "container", | |
| "options": [ ], | |
| "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\", pod=~\"$deployment.*\"}, container)", | |
| "refresh": 1, | |
| "regex": "", | |
| "skipUrlSync": false, | |
| "sort": 0, | |
| "tagValuesQuery": "", | |
| "tags": [ ], | |
| "tagsQuery": "", | |
| "type": "query", | |
| "useTags": false | |
| }, | |
| { | |
| "allValue": ".*", | |
| "hide": 0, | |
| "includeAll": true, | |
| "label": null, | |
| "multi": true, | |
| "name": "level", | |
| "options": [ | |
| { | |
| "selected": true, | |
| "text": "All", | |
| "value": "$__all" | |
| }, | |
| { | |
| "selected": false, | |
| "text": "debug", | |
| "value": "debug" | |
| }, | |
| { | |
| "selected": false, | |
| "text": "info", | |
| "value": "info" | |
| }, | |
| { | |
| "selected": false, | |
| "text": "warn", | |
| "value": "warn" | |
| }, | |
| { | |
| "selected": false, | |
| "text": "error", | |
| "value": "error" | |
| } | |
| ], | |
| "query": "debug,info,warn,error", | |
| "skipUrlSync": false, | |
| "type": "custom" | |
| }, | |
| { | |
| "hide": 0, | |
| "label": "LogQL Filter", | |
| "name": "filter", | |
| "options": [ | |
| { | |
| "selected": true, | |
| "text": "", | |
| "value": "" | |
| } | |
| ], | |
| "query": "", | |
| "skipUrlSync": false, | |
| "type": "textbox" | |
| } | |
| ] | |
| }, | |
| "time": { | |
| "from": "now-12h", | |
| "to": "now" | |
| }, | |
| "timepicker": { | |
| "refresh_intervals": [ | |
| "5s", | |
| "10s", | |
| "30s", | |
| "1m", | |
| "5m", | |
| "15m", | |
| "30m", | |
| "1h", | |
| "2h", | |
| "1d" | |
| ] | |
| }, | |
| "timezone": "", | |
| "title": "Logs", | |
| "uid": "a7e130cb82be229d6f3edbfd0a438001", | |
| "version": 1 | |
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "annotations": { | |
| "list": [ | |
| { | |
| "builtIn": 1, | |
| "datasource": "-- Grafana --", | |
| "enable": true, | |
| "hide": true, | |
| "iconColor": "rgba(0, 211, 255, 1)", | |
| "name": "Annotations & Alerts", | |
| "type": "dashboard" | |
| }, | |
| { | |
| "datasource": "$logs", | |
| "enable": true, | |
| "expr": "{cluster=\"$cluster\", diff_namespace=\"$namespace\", container_name=\"kube-diff-logger\"}", | |
| "hide": true, | |
| "iconColor": "rgba(255, 96, 96, 1)", | |
| "name": "deployments", | |
| "showIn": 0, | |
| "target": { } | |
| } | |
| ] | |
| }, | |
| "editable": true, | |
| "gnetId": null, | |
| "graphTooltip": 1, | |
| "id": 68, | |
| "iteration": 1588704280892, | |
| "links": [ | |
| { | |
| "icon": "dashboard", | |
| "includeVars": true, | |
| "keepTime": true, | |
| "tags": [ ], | |
| "targetBlank": true, | |
| "title": "Chunks", | |
| "tooltip": "", | |
| "type": "link", | |
| "url": "/grafana/d/586c2916e59d3f81282a776dbe8333e9/loki-chunks" | |
| }, | |
| { | |
| "icon": "dashboard", | |
| "includeVars": true, | |
| "keepTime": true, | |
| "tags": [ ], | |
| "targetBlank": true, | |
| "title": "Memcached", | |
| "tooltip": "", | |
| "type": "link", | |
| "url": "/grafana/d/8127d7f60e3a3230d7c633c8b5ddf68c/memcached" | |
| }, | |
| { | |
| "icon": "dashboard", | |
| "includeVars": false, | |
| "keepTime": true, | |
| "tags": [ ], | |
| "targetBlank": true, | |
| "title": "Promtail", | |
| "type": "link", | |
| "url": "/grafana/d/739c15a23c04c6ace0d03f6e14ba26e8/loki-promtail?var-datasource=$datasource&var-cluster=$cluster&var-namespace=default" | |
| }, | |
| { | |
| "icon": "dashboard", | |
| "includeVars": true, | |
| "keepTime": true, | |
| "tags": [ ], | |
| "targetBlank": true, | |
| "title": "BigTable Backup", | |
| "type": "link", | |
| "url": "/grafana/d/da5ce918801d6d9f775a8c7e9f9f339a/loki-bigtable-backup" | |
| }, | |
| { | |
| "icon": "dashboard", | |
| "keepTime": true, | |
| "tags": [ ], | |
| "targetBlank": true, | |
| "title": "Consul", | |
| "type": "link", | |
| "url": "/grafana/d/836793349c8ce200baa98f045123d19b/consul?var-datasource=$datasource&var-job=$namespace%2Fconsul" | |
| } | |
| ], | |
| "panels": [ | |
| { | |
| "collapsed": false, | |
| "datasource": null, | |
| "gridPos": { | |
| "h": 1, | |
| "w": 24, | |
| "x": 0, | |
| "y": 0 | |
| }, | |
| "id": 17, | |
| "panels": [ ], | |
| "title": "Main", | |
| "type": "row" | |
| }, | |
| { | |
| "aliasColors": { | |
| "5xx": "red" | |
| }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fieldConfig": { | |
| "defaults": { | |
| "custom": { } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 5, | |
| "w": 4, | |
| "x": 0, | |
| "y": 1 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 6, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/cortex-gw\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", | |
| "legendFormat": "{{status}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "Queries/Second", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 10, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { | |
| "5xx": "red" | |
| }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fieldConfig": { | |
| "defaults": { | |
| "custom": { } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 5, | |
| "w": 4, | |
| "x": 4, | |
| "y": 1 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 7, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/cortex-gw\", route=~\"api_prom_push|loki_api_v1_push\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", | |
| "legendFormat": "{{status}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "Pushes/Second", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 10, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fieldConfig": { | |
| "defaults": { | |
| "custom": { } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 5, | |
| "w": 4, | |
| "x": 8, | |
| "y": 1 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 11, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "hideEmpty": false, | |
| "hideZero": false, | |
| "max": false, | |
| "min": false, | |
| "show": false, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "topk(5, sum by (name,level) (rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\"}[$__interval])) - \nsum by (name,level) (rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\"}[$__interval] offset 1h)))", | |
| "legendFormat": "{{name}}-{{level}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "Bad Words", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 2, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fieldConfig": { | |
| "defaults": { | |
| "custom": { } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 5, | |
| "w": 4, | |
| "x": 12, | |
| "y": 1 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 2, | |
| "interval": "", | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": false, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "topk(10, sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant))", | |
| "legendFormat": "{{tenant}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "Lines Per Tenant (top 10)", | |
| "tooltip": { | |
| "shared": false, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fieldConfig": { | |
| "defaults": { | |
| "custom": { } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 5, | |
| "w": 4, | |
| "x": 16, | |
| "y": 1 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 4, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "hideEmpty": true, | |
| "hideZero": true, | |
| "max": false, | |
| "min": false, | |
| "show": false, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant)) / 1024 / 1024", | |
| "legendFormat": "{{tenant}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "MBs Per Tenant (Top 10)", | |
| "tooltip": { | |
| "shared": false, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fieldConfig": { | |
| "defaults": { | |
| "custom": { } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 5, | |
| "w": 4, | |
| "x": 20, | |
| "y": 1 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 24, | |
| "legend": { | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "show": false, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}[10m]) > 0", | |
| "hide": false, | |
| "interval": "", | |
| "legendFormat": "{{container}}-{{pod}}", | |
| "refId": "B" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "Container Restarts", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 2, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fieldConfig": { | |
| "defaults": { | |
| "custom": { } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 10, | |
| "w": 12, | |
| "x": 0, | |
| "y": 6 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 9, | |
| "legend": { | |
| "alignAsTable": true, | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "rightSide": true, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "histogram_quantile(0.99, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/cortex-gw\", route=~\"api_prom_push|loki_api_v1_push\", cluster=~\"$cluster\"})) * 1e3", | |
| "legendFormat": ".99", | |
| "refId": "A" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.75, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/cortex-gw\", route=~\"api_prom_push|loki_api_v1_push\", cluster=~\"$cluster\"})) * 1e3", | |
| "legendFormat": ".9", | |
| "refId": "B" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.5, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/cortex-gw\", route=~\"api_prom_push|loki_api_v1_push\", cluster=~\"$cluster\"})) * 1e3", | |
| "legendFormat": ".5", | |
| "refId": "C" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "Push Latency", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fieldConfig": { | |
| "defaults": { | |
| "custom": { } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 5, | |
| "w": 6, | |
| "x": 12, | |
| "y": 6 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 12, | |
| "legend": { | |
| "alignAsTable": true, | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "rightSide": true, | |
| "show": false, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "histogram_quantile(0.99, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/distributor\", cluster=~\"$cluster\"})) * 1e3", | |
| "legendFormat": ".99", | |
| "refId": "A" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.9, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/distributor\", cluster=~\"$cluster\"})) * 1e3", | |
| "legendFormat": ".9", | |
| "refId": "B" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.5, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/distributor\", cluster=~\"$cluster\"})) * 1e3", | |
| "legendFormat": ".5", | |
| "refId": "C" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "Distributor Latency", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fieldConfig": { | |
| "defaults": { | |
| "custom": { } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "fill": 0, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 5, | |
| "w": 6, | |
| "x": 18, | |
| "y": 6 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 71, | |
| "legend": { | |
| "alignAsTable": true, | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "rightSide": true, | |
| "show": false, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/distributor\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/distributor\"}[5m])) by (route)", | |
| "interval": "", | |
| "legendFormat": "{{route}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "Distributor Success Rate", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "decimals": null, | |
| "format": "percentunit", | |
| "label": "", | |
| "logBase": 1, | |
| "max": "1", | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fieldConfig": { | |
| "defaults": { | |
| "custom": { } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 5, | |
| "w": 6, | |
| "x": 12, | |
| "y": 11 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 13, | |
| "legend": { | |
| "alignAsTable": true, | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "rightSide": true, | |
| "show": false, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "histogram_quantile(0.99, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/ingester\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", | |
| "legendFormat": ".99", | |
| "refId": "A" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.9, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/ingester\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", | |
| "hide": false, | |
| "legendFormat": ".9", | |
| "refId": "B" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.5, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/ingester\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", | |
| "hide": false, | |
| "legendFormat": ".5", | |
| "refId": "C" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "Ingester Latency Write", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fieldConfig": { | |
| "defaults": { | |
| "custom": { } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "fill": 0, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 5, | |
| "w": 6, | |
| "x": 18, | |
| "y": 11 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 72, | |
| "legend": { | |
| "alignAsTable": true, | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "rightSide": true, | |
| "show": false, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", status_code!~\"5[0-9]{2}\", route=\"/logproto.Pusher/Push\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/logproto.Pusher/Push\"}[5m])) by (route)", | |
| "interval": "", | |
| "legendFormat": "{{route}}", | |
| "refId": "A" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "Ingester Success Rate Write", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "decimals": null, | |
| "format": "percentunit", | |
| "label": "", | |
| "logBase": 1, | |
| "max": "1", | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fieldConfig": { | |
| "defaults": { | |
| "custom": { } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 10, | |
| "w": 12, | |
| "x": 0, | |
| "y": 16 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 10, | |
| "legend": { | |
| "alignAsTable": true, | |
| "avg": false, | |
| "current": false, | |
| "hideEmpty": true, | |
| "hideZero": false, | |
| "max": false, | |
| "min": false, | |
| "rightSide": true, | |
| "show": true, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "histogram_quantile(0.99, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=\"$namespace/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", | |
| "legendFormat": "{{route}}-.99", | |
| "refId": "A" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.9, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=\"$namespace/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", | |
| "legendFormat": "{{route}}-.9", | |
| "refId": "B" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.5, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=\"$namespace/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", | |
| "legendFormat": "{{route}}-.5", | |
| "refId": "C" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "Query Latency", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fieldConfig": { | |
| "defaults": { | |
| "custom": { } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "fill": 1, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 5, | |
| "w": 6, | |
| "x": 12, | |
| "y": 16 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 14, | |
| "legend": { | |
| "alignAsTable": true, | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |
| "min": false, | |
| "rightSide": true, | |
| "show": false, | |
| "total": false, | |
| "values": false | |
| }, | |
| "lines": true, | |
| "linewidth": 1, | |
| "nullPointMode": "null", | |
| "options": { | |
| "dataLinks": [ ] | |
| }, | |
| "percentage": false, | |
| "pointradius": 2, | |
| "points": false, | |
| "renderer": "flot", | |
| "seriesOverrides": [ ], | |
| "spaceLength": 10, | |
| "stack": false, | |
| "steppedLine": false, | |
| "targets": [ | |
| { | |
| "expr": "histogram_quantile(0.99, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=\"$namespace/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", | |
| "legendFormat": ".99-{{route}}", | |
| "refId": "A" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.9, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=\"$namespace/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", | |
| "legendFormat": ".9-{{route}}", | |
| "refId": "B" | |
| }, | |
| { | |
| "expr": "histogram_quantile(0.5, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=\"$namespace/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", | |
| "legendFormat": ".5-{{route}}", | |
| "refId": "C" | |
| } | |
| ], | |
| "thresholds": [ ], | |
| "timeFrom": null, | |
| "timeRegions": [ ], | |
| "timeShift": null, | |
| "title": "Querier Latency", | |
| "tooltip": { | |
| "shared": true, | |
| "sort": 0, | |
| "value_type": "individual" | |
| }, | |
| "type": "graph", | |
| "xaxis": { | |
| "buckets": null, | |
| "mode": "time", | |
| "name": null, | |
| "show": true, | |
| "values": [ ] | |
| }, | |
| "yaxes": [ | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| }, | |
| { | |
| "format": "short", | |
| "label": null, | |
| "logBase": 1, | |
| "max": null, | |
| "min": null, | |
| "show": true | |
| } | |
| ], | |
| "yaxis": { | |
| "align": false, | |
| "alignLevel": null | |
| } | |
| }, | |
| { | |
| "aliasColors": { }, | |
| "bars": false, | |
| "dashLength": 10, | |
| "dashes": false, | |
| "datasource": "$datasource", | |
| "fieldConfig": { | |
| "defaults": { | |
| "custom": { } | |
| }, | |
| "overrides": [ ] | |
| }, | |
| "fill": 0, | |
| "fillGradient": 0, | |
| "gridPos": { | |
| "h": 5, | |
| "w": 6, | |
| "x": 18, | |
| "y": 16 | |
| }, | |
| "hiddenSeries": false, | |
| "id": 73, | |
| "legend": { | |
| "alignAsTable": true, | |
| "avg": false, | |
| "current": false, | |
| "max": false, | |