Created
October 8, 2025 18:05
-
-
Save realvz/13807b77e04c4b0a4a1c3f4288adf0f8 to your computer and use it in GitHub Desktop.
This dashboard visualizes metrics for the Envoy AI Gateway (EAIG), providing insights into model usage, latency, and traffic patterns. It relies on Prometheus as its data source and is designed for environments where EAIG proxies requests to foundation models such as Anthropic Claude and OpenAI GPT. The dashboard displays token usage, latency, a…
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "annotations": { | |
| "list": [ | |
| { | |
| "builtIn": 1, | |
| "datasource": { | |
| "type": "grafana", | |
| "uid": "-- Grafana --" | |
| }, | |
| "enable": true, | |
| "hide": true, | |
| "iconColor": "rgba(0, 211, 255, 1)", | |
| "name": "Annotations & Alerts", | |
| "type": "dashboard" | |
| } | |
| ] | |
| }, | |
| "editable": true, | |
| "fiscalYearStartMonth": 0, | |
| "graphTooltip": 0, | |
| "id": 1, | |
| "links": [], | |
| "panels": [ | |
| { | |
| "datasource": { | |
| "type": "prometheus", | |
| "uid": "df09ihq40n9j4c" | |
| }, | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 14, | |
| "w": 19, | |
| "x": 2, | |
| "y": 0 | |
| }, | |
| "id": 1, | |
| "options": { | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| } | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "datasource": { | |
| "type": "prometheus", | |
| "uid": "df09ihq40n9j4c" | |
| }, | |
| "editorMode": "code", | |
| "expr": "sum(gen_ai_client_token_usage_token_sum{gateway_envoyproxy_io_owning_gateway_name = \"eaig-bedrock\"}) by (gen_ai_request_model, gen_ai_token_type)", | |
| "instant": true, | |
| "key": "Q-d2308b02-182d-476c-b7ef-ca985e90efb6-1", | |
| "legendFormat": "__auto", | |
| "range": true, | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "I/O tokens for the gateway", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "prometheus", | |
| "uid": "df09ihq40n9j4c" | |
| }, | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 12, | |
| "x": 0, | |
| "y": 14 | |
| }, | |
| "id": 6, | |
| "options": { | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| } | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "adhocFilters": [], | |
| "datasource": { | |
| "type": "prometheus", | |
| "uid": "df09ihq40n9j4c" | |
| }, | |
| "expr": "sum(rate(gen_ai_client_token_usage_token_sum{ }[$__rate_interval]))", | |
| "fromExploreMetrics": true, | |
| "interval": "", | |
| "legendFormat": "sum(rate)", | |
| "refId": "gen_ai_client_token_usage_token_sum-sum(rate)" | |
| } | |
| ], | |
| "title": "gen_ai_response_model ", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "prometheus", | |
| "uid": "df09ihq40n9j4c" | |
| }, | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [ | |
| { | |
| "__systemRef": "hideSeriesFrom", | |
| "matcher": { | |
| "id": "byNames", | |
| "options": { | |
| "mode": "exclude", | |
| "names": [ | |
| "sum(increase(gen_ai_client_token_usage_token_sum{gen_ai_token_type=\"input\"}[1h]))\n" | |
| ], | |
| "prefix": "All except:", | |
| "readOnly": true | |
| } | |
| }, | |
| "properties": [ | |
| { | |
| "id": "custom.hideFrom", | |
| "value": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": true | |
| } | |
| } | |
| ] | |
| } | |
| ] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 12, | |
| "x": 12, | |
| "y": 14 | |
| }, | |
| "id": 3, | |
| "options": { | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| } | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "editorMode": "code", | |
| "expr": "sum(increase(gen_ai_client_token_usage_token_sum{gen_ai_token_type=\"input\"}[1h]))\n", | |
| "legendFormat": "__auto", | |
| "range": true, | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Total input tokens used (last hour)", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "prometheus", | |
| "uid": "df09ihq40n9j4c" | |
| }, | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 12, | |
| "x": 0, | |
| "y": 22 | |
| }, | |
| "id": 4, | |
| "options": { | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| } | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "editorMode": "code", | |
| "expr": "rate(gen_ai_server_request_duration_seconds_sum[5m]) \n / rate(gen_ai_server_request_duration_seconds_count[5m])\n", | |
| "legendFormat": "__auto", | |
| "range": true, | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Average Latency", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "prometheus", | |
| "uid": "df09ihq40n9j4c" | |
| }, | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "line", | |
| "fillOpacity": 0, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 12, | |
| "x": 12, | |
| "y": 22 | |
| }, | |
| "id": 5, | |
| "options": { | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| } | |
| }, | |
| "pluginVersion": "12.1.1", | |
| "targets": [ | |
| { | |
| "adhocFilters": [], | |
| "datasource": { | |
| "type": "prometheus", | |
| "uid": "df09ihq40n9j4c" | |
| }, | |
| "editorMode": "code", | |
| "expr": "sum(rate(envoy_cluster_external_upstream_rq_xx{ }[$__rate_interval]))", | |
| "fromExploreMetrics": true, | |
| "instant": true, | |
| "interval": "", | |
| "legendFormat": "sum(rate)", | |
| "range": true, | |
| "refId": "envoy_cluster_external_upstream_rq_xx-sum(rate)" | |
| } | |
| ], | |
| "title": "External Upstream Request Rate", | |
| "type": "timeseries" | |
| } | |
| ], | |
| "preload": false, | |
| "refresh": "auto", | |
| "schemaVersion": 41, | |
| "tags": [], | |
| "templating": { | |
| "list": [] | |
| }, | |
| "time": { | |
| "from": "2025-10-08T16:40:29.026Z", | |
| "to": "2025-10-08T17:40:29.026Z" | |
| }, | |
| "timepicker": {}, | |
| "timezone": "browser", | |
| "title": "EAIG", | |
| "uid": "dd897f8c-17d2-4d94-a3cf-cd6c6082f3a9", | |
| "version": 19 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment