Created
July 29, 2026 11:16
-
-
Save 0GiS0/962280f135918ffd18d89bd8e56b962e to your computer and use it in GitHub Desktop.
GitHub Copilot Custom Agents - Grafana Dashboard para Application Insights
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "__inputs": [ | |
| { | |
| "name": "DS_AZURE_MONITOR", | |
| "label": "Azure Monitor", | |
| "description": "", | |
| "type": "datasource", | |
| "pluginId": "grafana-azure-monitor-datasource", | |
| "pluginName": "Azure Monitor" | |
| } | |
| ], | |
| "__elements": {}, | |
| "__requires": [ | |
| { | |
| "type": "panel", | |
| "id": "barchart", | |
| "name": "Bar chart", | |
| "version": "" | |
| }, | |
| { | |
| "type": "grafana", | |
| "id": "grafana", | |
| "name": "Grafana", | |
| "version": "12.4.3" | |
| }, | |
| { | |
| "type": "datasource", | |
| "id": "grafana-azure-monitor-datasource", | |
| "name": "Azure Monitor", | |
| "version": "12.4.3" | |
| }, | |
| { | |
| "type": "panel", | |
| "id": "piechart", | |
| "name": "Pie chart", | |
| "version": "" | |
| }, | |
| { | |
| "type": "panel", | |
| "id": "stat", | |
| "name": "Stat", | |
| "version": "" | |
| }, | |
| { | |
| "type": "panel", | |
| "id": "table", | |
| "name": "Table", | |
| "version": "" | |
| }, | |
| { | |
| "type": "panel", | |
| "id": "timeseries", | |
| "name": "Time series", | |
| "version": "" | |
| }, | |
| { | |
| "type": "panel", | |
| "id": "traces", | |
| "name": "Traces", | |
| "version": "" | |
| } | |
| ], | |
| "annotations": { | |
| "list": [ | |
| { | |
| "builtIn": 1, | |
| "datasource": { | |
| "type": "grafana", | |
| "uid": "-- Grafana --" | |
| }, | |
| "enable": true, | |
| "hide": true, | |
| "iconColor": "rgba(0, 211, 255, 1)", | |
| "name": "Annotations & Alerts", | |
| "type": "dashboard" | |
| } | |
| ] | |
| }, | |
| "description": "Monitors GitHub Copilot custom agent (chat mode) adoption and usage via Application Insights OpenTelemetry data, in addition to the standard model/token/session metrics from the official GitHub Copilot dashboard. Highlights which custom agents are used, how often, and by how many sessions, alongside the default GitHub Copilot Chat agent.", | |
| "editable": true, | |
| "fiscalYearStartMonth": 0, | |
| "graphTooltip": 0, | |
| "links": [ | |
| { | |
| "asDropdown": false, | |
| "icon": "external link", | |
| "includeVars": false, | |
| "keepTime": false, | |
| "tags": [], | |
| "targetBlank": true, | |
| "title": "Doc", | |
| "tooltip": "Documentation", | |
| "type": "link", | |
| "url": "https://aka.ms/amg/dash-doc/gh-copilot" | |
| } | |
| ], | |
| "panels": [ | |
| { | |
| "collapsed": false, | |
| "gridPos": { | |
| "h": 1, | |
| "w": 24, | |
| "x": 0, | |
| "y": 0 | |
| }, | |
| "id": 100, | |
| "panels": [], | |
| "title": "Summary", | |
| "type": "row" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Key performance indicators for agent operations including total operations executed, token consumption (input and output), and average response time across all agents. Est. Total Cost is an estimate derived from per-model token pricing — not actual billed cost.", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "thresholds" | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| } | |
| ] | |
| }, | |
| "unit": "short" | |
| }, | |
| "overrides": [ | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "Total Operations" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "color", | |
| "value": { | |
| "fixedColor": "blue", | |
| "mode": "fixed" | |
| } | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "Total Input Tokens" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "color", | |
| "value": { | |
| "fixedColor": "purple", | |
| "mode": "fixed" | |
| } | |
| }, | |
| { | |
| "id": "unit", | |
| "value": "short" | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "Total Output Tokens" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "color", | |
| "value": { | |
| "fixedColor": "green", | |
| "mode": "fixed" | |
| } | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "Avg Response Time" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "color", | |
| "value": { | |
| "fixedColor": "orange", | |
| "mode": "fixed" | |
| } | |
| }, | |
| { | |
| "id": "unit", | |
| "value": "s" | |
| }, | |
| { | |
| "id": "decimals", | |
| "value": 2 | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "Est. Total Cost" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "color", | |
| "value": { | |
| "fixedColor": "dark-yellow", | |
| "mode": "fixed" | |
| } | |
| }, | |
| { | |
| "id": "unit", | |
| "value": "currencyUSD" | |
| }, | |
| { | |
| "id": "decimals", | |
| "value": 2 | |
| } | |
| ] | |
| } | |
| ] | |
| }, | |
| "gridPos": { | |
| "h": 6, | |
| "w": 12, | |
| "x": 0, | |
| "y": 1 | |
| }, | |
| "id": 12, | |
| "options": { | |
| "colorMode": "background", | |
| "graphMode": "none", | |
| "justifyMode": "center", | |
| "orientation": "auto", | |
| "percentChangeColorMode": "standard", | |
| "reduceOptions": { | |
| "calcs": [ | |
| "lastNotNull" | |
| ], | |
| "fields": "", | |
| "values": false | |
| }, | |
| "showPercentChange": false, | |
| "text": {}, | |
| "textMode": "auto", | |
| "wideLayout": true | |
| }, | |
| "pluginVersion": "12.4.3", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "let cost = toscalar(\n dependencies\n | where cloud_RoleName in (${source:singlequote})\n | where isnotempty(customDimensions[\"gen_ai.response.model\"])\n | extend c = todouble(customDimensions[\"github.copilot.cost\"])\n | summarize sum(c));\ndependencies\n| where cloud_RoleName in (${source:singlequote})\n| where name contains \"invoke_agent\"\n| extend\n input_tokens = toint(customDimensions[\"gen_ai.usage.input_tokens\"]),\n output_tokens = toint(customDimensions[\"gen_ai.usage.output_tokens\"])\n| summarize\n Operations = count(),\n InputTokens = sum(input_tokens),\n OutputTokens = sum(output_tokens),\n AvgRespSec = round(avg(duration) / 1000, 2)\n| project\n ['Est. Total Cost'] = cost,\n ['Total Operations'] = Operations,\n ['Total Input Tokens'] = InputTokens,\n ['Total Output Tokens'] = OutputTokens,\n ['Avg Response Time'] = AvgRespSec", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "time_series", | |
| "timeColumn": "timestamp" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Agent Summary Statistics", | |
| "type": "stat" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Key metrics across all LLM chat completions: total calls, total tokens consumed, and average time-to-first-token (TTFT).", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "thresholds" | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| } | |
| ] | |
| }, | |
| "unit": "short" | |
| }, | |
| "overrides": [ | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "LLM Calls" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "color", | |
| "value": { | |
| "fixedColor": "semi-dark-blue", | |
| "mode": "fixed" | |
| } | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "Chat Sessions" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "color", | |
| "value": { | |
| "fixedColor": "semi-dark-green", | |
| "mode": "fixed" | |
| } | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "Avg TTFT" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "color", | |
| "value": { | |
| "fixedColor": "semi-dark-orange", | |
| "mode": "fixed" | |
| } | |
| }, | |
| { | |
| "id": "unit", | |
| "value": "ms" | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "Tool Calls" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "color", | |
| "value": { | |
| "fixedColor": "semi-dark-purple", | |
| "mode": "fixed" | |
| } | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "Reasoning Calls" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "color", | |
| "value": { | |
| "fixedColor": "semi-dark-yellow", | |
| "mode": "fixed" | |
| } | |
| } | |
| ] | |
| } | |
| ] | |
| }, | |
| "gridPos": { | |
| "h": 6, | |
| "w": 12, | |
| "x": 12, | |
| "y": 1 | |
| }, | |
| "id": 13, | |
| "options": { | |
| "colorMode": "background", | |
| "graphMode": "none", | |
| "justifyMode": "center", | |
| "orientation": "auto", | |
| "percentChangeColorMode": "standard", | |
| "reduceOptions": { | |
| "calcs": [ | |
| "lastNotNull" | |
| ], | |
| "fields": "", | |
| "values": false | |
| }, | |
| "showPercentChange": false, | |
| "text": {}, | |
| "textMode": "auto", | |
| "wideLayout": true | |
| }, | |
| "pluginVersion": "12.4.3", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "dependencies\n| where cloud_RoleName in (${source:singlequote})\n| extend\n isLlm = isnotempty(customDimensions[\"gen_ai.response.model\"]),\n isTool = tostring(customDimensions[\"gen_ai.operation.name\"]) == \"execute_tool\",\n isReasoning = isnotempty(tostring(customDimensions[\"copilot_chat.reasoning_content\"])),\n sessId = coalesce(\n tostring(customDimensions[\"copilot_chat.chat_session_id\"]),\n tostring(customDimensions[\"gen_ai.conversation.id\"])),\n ttft = coalesce(\n toint(customDimensions[\"copilot_chat.time_to_first_token\"]),\n toint(customDimensions[\"github.copilot.server_duration\"]))\n| summarize\n ['LLM Calls'] = countif(isLlm),\n ['Chat Sessions'] = dcountif(sessId, isnotempty(sessId)),\n ['Tool Calls'] = countif(isTool),\n ['Reasoning Calls'] = countif(isReasoning and isLlm),\n ['Avg TTFT'] = round(avgif(toreal(ttft), isLlm and isnotnull(ttft)), 0)", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "time_series", | |
| "timeColumn": "timestamp" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Chat & Tool Summary", | |
| "type": "stat" | |
| }, | |
| { | |
| "collapsed": false, | |
| "gridPos": { | |
| "h": 1, | |
| "w": 24, | |
| "x": 0, | |
| "y": 7 | |
| }, | |
| "id": 200, | |
| "panels": [], | |
| "title": "Custom Agents", | |
| "type": "row" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Adoption of custom agents (chat modes) vs. the default GitHub Copilot Chat agent.", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "thresholds" | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| } | |
| ] | |
| }, | |
| "unit": "short" | |
| }, | |
| "overrides": [ | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "Total Invocations" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "color", | |
| "value": { | |
| "fixedColor": "blue", | |
| "mode": "fixed" | |
| } | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "Custom Agent Invocations" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "color", | |
| "value": { | |
| "fixedColor": "purple", | |
| "mode": "fixed" | |
| } | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "Distinct Custom Agents" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "color", | |
| "value": { | |
| "fixedColor": "orange", | |
| "mode": "fixed" | |
| } | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "Custom Agent Sessions" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "color", | |
| "value": { | |
| "fixedColor": "green", | |
| "mode": "fixed" | |
| } | |
| } | |
| ] | |
| } | |
| ] | |
| }, | |
| "gridPos": { | |
| "h": 6, | |
| "w": 24, | |
| "x": 0, | |
| "y": 8 | |
| }, | |
| "id": 201, | |
| "options": { | |
| "colorMode": "background", | |
| "graphMode": "none", | |
| "justifyMode": "center", | |
| "orientation": "auto", | |
| "percentChangeColorMode": "standard", | |
| "reduceOptions": { | |
| "calcs": [ | |
| "lastNotNull" | |
| ], | |
| "fields": "", | |
| "values": false | |
| }, | |
| "showPercentChange": false, | |
| "text": {}, | |
| "textMode": "auto", | |
| "wideLayout": true | |
| }, | |
| "pluginVersion": "12.4.3", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "dependencies\n| where cloud_RoleName in (${source:singlequote})\n| where name contains \"invoke_agent\"\n| extend agentType = tostring(customDimensions[\"github.copilot.agent.type\"]),\n agentMode = tostring(customDimensions[\"copilot_chat.mode_name\"]),\n sessionId = tostring(customDimensions[\"copilot_chat.session_id\"])\n| summarize\n ['Total Invocations'] = count(),\n ['Custom Agent Invocations'] = countif(agentType == \"custom\"),\n ['Distinct Custom Agents'] = dcountif(agentMode, agentType == \"custom\"),\n ['Custom Agent Sessions'] = dcountif(sessionId, agentType == \"custom\")", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "time_series", | |
| "timeColumn": "timestamp" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Custom Agent Adoption", | |
| "type": "stat" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Breakdown of agent invocations by custom agent (chat mode). Shows which custom agents are used and how frequently, compared to the default GitHub Copilot Chat agent.", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| } | |
| }, | |
| "mappings": [] | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 8, | |
| "x": 0, | |
| "y": 14 | |
| }, | |
| "id": 202, | |
| "options": { | |
| "displayLabels": [ | |
| "name", | |
| "percent" | |
| ], | |
| "legend": { | |
| "displayMode": "table", | |
| "placement": "right", | |
| "showLegend": true, | |
| "values": [ | |
| "value", | |
| "percent" | |
| ] | |
| }, | |
| "pieType": "donut", | |
| "reduceOptions": { | |
| "calcs": [ | |
| "lastNotNull" | |
| ], | |
| "fields": "", | |
| "values": true | |
| }, | |
| "sort": "desc", | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| } | |
| }, | |
| "pluginVersion": "12.4.3", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "dependencies\n| where cloud_RoleName in (${source:singlequote})\n| where name contains \"invoke_agent\"\n| extend agentType = tostring(customDimensions[\"github.copilot.agent.type\"]),\n agentMode = tostring(customDimensions[\"copilot_chat.mode_name\"])\n| extend agent = iff(agentType == \"custom\", agentMode, \"GitHub Copilot (default)\")\n| summarize Invocations = count() by agent\n| order by Invocations desc", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "table" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Invocations by Agent", | |
| "type": "piechart" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Distinct chat sessions per agent. Helps distinguish one-off invocations from sustained usage of a custom agent.", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "fillOpacity": 80, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "lineWidth": 1, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| } | |
| ] | |
| }, | |
| "unit": "short" | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 8, | |
| "x": 8, | |
| "y": 14 | |
| }, | |
| "id": 203, | |
| "options": { | |
| "barRadius": 0.05, | |
| "barWidth": 0.7, | |
| "fullHighlight": false, | |
| "groupWidth": 0.7, | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "orientation": "horizontal", | |
| "showValue": "auto", | |
| "stacking": "none", | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| }, | |
| "xTickLabelRotation": 0, | |
| "xTickLabelSpacing": 0 | |
| }, | |
| "pluginVersion": "12.4.3", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "dependencies\n| where cloud_RoleName in (${source:singlequote})\n| where name contains \"invoke_agent\"\n| extend agentType = tostring(customDimensions[\"github.copilot.agent.type\"]),\n agentMode = tostring(customDimensions[\"copilot_chat.mode_name\"])\n| extend agent = iff(agentType == \"custom\", agentMode, \"GitHub Copilot (default)\")\n| summarize Sessions = dcount(tostring(customDimensions[\"copilot_chat.session_id\"])) by agent\n| order by Sessions desc", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "table" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Sessions by Agent", | |
| "type": "barchart" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Custom agent invocations over time, one series per agent.", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "bars", | |
| "fillOpacity": 80, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "never", | |
| "showValues": false, | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "normal" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 8, | |
| "x": 16, | |
| "y": 14 | |
| }, | |
| "id": 204, | |
| "options": { | |
| "legend": { | |
| "calcs": [ | |
| "sum" | |
| ], | |
| "displayMode": "table", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": true, | |
| "mode": "multi", | |
| "sort": "desc" | |
| } | |
| }, | |
| "pluginVersion": "12.4.3", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "dependencies\n| where cloud_RoleName in (${source:singlequote})\n| where name contains \"invoke_agent\"\n| extend agentType = tostring(customDimensions[\"github.copilot.agent.type\"]),\n agentMode = tostring(customDimensions[\"copilot_chat.mode_name\"])\n| extend agent = iff(agentType == \"custom\", agentMode, \"GitHub Copilot (default)\")\n| summarize Invocations = count() by bin(timestamp, $__interval), agent\n| order by timestamp asc", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "time_series", | |
| "timeColumn": "timestamp" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Custom Agent Usage Over Time", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "collapsed": false, | |
| "gridPos": { | |
| "h": 1, | |
| "w": 24, | |
| "x": 0, | |
| "y": 22 | |
| }, | |
| "id": 101, | |
| "panels": [], | |
| "title": "Usage Over Time", | |
| "type": "row" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Number of LLM API calls over time, grouped by operation type (chat completions, agent invocations, tool executions).", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "bars", | |
| "fillOpacity": 80, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "never", | |
| "showValues": false, | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "normal" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 12, | |
| "x": 0, | |
| "y": 23 | |
| }, | |
| "id": 1, | |
| "maxDataPoints": 60, | |
| "options": { | |
| "legend": { | |
| "calcs": [ | |
| "sum" | |
| ], | |
| "displayMode": "table", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": true, | |
| "mode": "multi", | |
| "sort": "desc" | |
| } | |
| }, | |
| "pluginVersion": "12.4.3", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "dependencies\n| where cloud_RoleName in (${source:singlequote})\n| extend opName = tostring(customDimensions[\"gen_ai.operation.name\"])\n| where isnotempty(opName)\n| summarize Count = count() by bin(timestamp, $__interval), opName\n| order by timestamp asc", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "time_series", | |
| "timeColumn": "timestamp" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Operations Over Time", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Total token consumption (input + output) over time, broken down by model.", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "Tokens", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "bars", | |
| "fillOpacity": 80, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "never", | |
| "showValues": false, | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "normal" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 12, | |
| "x": 12, | |
| "y": 23 | |
| }, | |
| "id": 2, | |
| "maxDataPoints": 60, | |
| "options": { | |
| "legend": { | |
| "calcs": [ | |
| "sum" | |
| ], | |
| "displayMode": "table", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": true, | |
| "mode": "multi", | |
| "sort": "desc" | |
| } | |
| }, | |
| "pluginVersion": "12.4.3", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "let topModels = materialize(\n dependencies\n | where cloud_RoleName in (${source:singlequote})\n | where isnotempty(customDimensions[\"gen_ai.response.model\"])\n | extend m = tostring(customDimensions[\"gen_ai.response.model\"]),\n inputTokens = toint(customDimensions[\"gen_ai.usage.input_tokens\"]),\n outputTokens = toint(customDimensions[\"gen_ai.usage.output_tokens\"])\n | summarize Total = sum(inputTokens) + sum(outputTokens) by m\n | top 7 by Total\n | project m);\ndependencies\n| where cloud_RoleName in (${source:singlequote})\n| where isnotempty(customDimensions[\"gen_ai.response.model\"])\n| extend m = tostring(customDimensions[\"gen_ai.response.model\"]),\n inputTokens = toint(customDimensions[\"gen_ai.usage.input_tokens\"]),\n outputTokens = toint(customDimensions[\"gen_ai.usage.output_tokens\"])\n| extend model = iff(m in (topModels), m, \"other\")\n| summarize TotalTokens = sum(inputTokens) + sum(outputTokens) by bin(timestamp, $__interval), model\n| order by timestamp asc", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "time_series", | |
| "timeColumn": "timestamp" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Token Consumption Over Time by Model", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "collapsed": false, | |
| "gridPos": { | |
| "h": 1, | |
| "w": 24, | |
| "x": 0, | |
| "y": 31 | |
| }, | |
| "id": 102, | |
| "panels": [], | |
| "title": "Model Performance", | |
| "type": "row" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Breakdown of LLM calls by model. Shows which AI models are being used and how frequently.", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| } | |
| }, | |
| "mappings": [] | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 8, | |
| "x": 0, | |
| "y": 32 | |
| }, | |
| "id": 3, | |
| "options": { | |
| "displayLabels": [ | |
| "name", | |
| "percent" | |
| ], | |
| "legend": { | |
| "displayMode": "table", | |
| "placement": "right", | |
| "showLegend": true, | |
| "values": [ | |
| "value", | |
| "percent" | |
| ] | |
| }, | |
| "pieType": "donut", | |
| "reduceOptions": { | |
| "calcs": [ | |
| "lastNotNull" | |
| ], | |
| "fields": "", | |
| "values": true | |
| }, | |
| "sort": "desc", | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| } | |
| }, | |
| "pluginVersion": "12.4.3", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "let topModels = materialize(\n dependencies\n | where cloud_RoleName in (${source:singlequote})\n | where isnotempty(customDimensions[\"gen_ai.response.model\"])\n | extend m = tostring(customDimensions[\"gen_ai.response.model\"])\n | summarize Total = count() by m\n | top 7 by Total\n | project m);\ndependencies\n| where cloud_RoleName in (${source:singlequote})\n| where isnotempty(customDimensions[\"gen_ai.response.model\"])\n| extend m = tostring(customDimensions[\"gen_ai.response.model\"])\n| extend model = iff(m in (topModels), m, \"other\")\n| summarize CallCount = count() by model\n| order by CallCount desc", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "table" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Model Usage Distribution", | |
| "type": "piechart" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Average and P90 response duration per model. Helps identify which models are slower.", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "fillOpacity": 80, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "lineWidth": 1, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| } | |
| ] | |
| }, | |
| "unit": "ms" | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 8, | |
| "x": 8, | |
| "y": 32 | |
| }, | |
| "id": 4, | |
| "options": { | |
| "barRadius": 0.05, | |
| "barWidth": 0.7, | |
| "fullHighlight": false, | |
| "groupWidth": 0.7, | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "orientation": "horizontal", | |
| "showValue": "auto", | |
| "stacking": "none", | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| }, | |
| "xTickLabelRotation": 0, | |
| "xTickLabelSpacing": 0 | |
| }, | |
| "pluginVersion": "12.4.3", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "let topModels = materialize(\n dependencies\n | where cloud_RoleName in (${source:singlequote})\n | where isnotempty(customDimensions[\"gen_ai.response.model\"])\n | extend m = tostring(customDimensions[\"gen_ai.response.model\"])\n | summarize Total = count() by m\n | top 7 by Total\n | project m);\ndependencies\n| where cloud_RoleName in (${source:singlequote})\n| where isnotempty(customDimensions[\"gen_ai.response.model\"])\n| extend m = tostring(customDimensions[\"gen_ai.response.model\"])\n| extend model = iff(m in (topModels), m, \"other\")\n| summarize ['Avg Duration'] = round(avg(duration), 0), ['P90 Duration'] = round(percentile(duration, 90), 0) by model\n| order by ['Avg Duration'] desc", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "table" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Response Duration by Model (Avg / P90)", | |
| "type": "barchart" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Time to First Token (TTFT) percentiles per model. TTFT measures how quickly the user starts seeing a response stream. Lower is better for user experience.", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "fillOpacity": 80, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "lineWidth": 1, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green", | |
| "value": 0 | |
| } | |
| ] | |
| }, | |
| "unit": "ms" | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 8, | |
| "x": 16, | |
| "y": 32 | |
| }, | |
| "id": 5, | |
| "options": { | |
| "barRadius": 0.05, | |
| "barWidth": 0.7, | |
| "fullHighlight": false, | |
| "groupWidth": 0.7, | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "orientation": "horizontal", | |
| "showValue": "auto", | |
| "stacking": "none", | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| }, | |
| "xTickLabelRotation": 0, | |
| "xTickLabelSpacing": 0 | |
| }, | |
| "pluginVersion": "12.4.3", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "let topModels = materialize(\n dependencies\n | where cloud_RoleName in (${source:singlequote})\n | extend ttft = coalesce(\n toint(customDimensions[\"copilot_chat.time_to_first_token\"]),\n toint(customDimensions[\"github.copilot.server_duration\"]))\n | where isnotnull(ttft)\n | extend m = tostring(customDimensions[\"gen_ai.response.model\"])\n | where isnotempty(m)\n | summarize Total = count() by m\n | top 7 by Total\n | project m);\ndependencies\n| where cloud_RoleName in (${source:singlequote})\n| extend ttft = coalesce(\n toint(customDimensions[\"copilot_chat.time_to_first_token\"]),\n toint(customDimensions[\"github.copilot.server_duration\"]))\n| where isnotnull(ttft)\n| extend m = tostring(customDimensions[\"gen_ai.response.model\"])\n| where isnotempty(m)\n| extend model = iff(m in (topModels), m, \"other\")\n| summarize ['P50 TTFT'] = round(percentile(ttft, 50), 0), ['P90 TTFT'] = round(percentile(ttft, 90), 0) by model\n| order by ['P50 TTFT'] desc", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "table" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Time to First Token by Model (P50 / P90)", | |
| "type": "barchart" | |
| }, | |
| { | |
| "collapsed": false, | |
| "gridPos": { | |
| "h": 1, | |
| "w": 24, | |
| "x": 0, | |
| "y": 40 | |
| }, | |
| "id": 104, | |
| "panels": [], | |
| "title": "Agent & Feature Activity", | |
| "type": "row" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Breakdown of Copilot feature/agent usage. Shows which Copilot features (Chat, Edit Agent, Commit Generator, Explore, etc.) are being used most.", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| } | |
| }, | |
| "mappings": [] | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 8, | |
| "x": 0, | |
| "y": 41 | |
| }, | |
| "id": 8, | |
| "options": { | |
| "displayLabels": [ | |
| "name", | |
| "percent" | |
| ], | |
| "legend": { | |
| "displayMode": "table", | |
| "placement": "right", | |
| "showLegend": true, | |
| "values": [ | |
| "value", | |
| "percent" | |
| ] | |
| }, | |
| "pieType": "donut", | |
| "reduceOptions": { | |
| "calcs": [ | |
| "lastNotNull" | |
| ], | |
| "fields": "", | |
| "values": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| } | |
| }, | |
| "pluginVersion": "11.6.9", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "let topAgents = materialize(\n dependencies\n | where cloud_RoleName in (${source:singlequote})\n | extend a = tostring(customDimensions[\"gen_ai.agent.name\"])\n | where isnotempty(a)\n | summarize Total = count() by a\n | top 7 by Total\n | project a);\ndependencies\n| where cloud_RoleName in (${source:singlequote})\n| extend a = tostring(customDimensions[\"gen_ai.agent.name\"])\n| where isnotempty(a)\n| extend agentName = iff(a in (topAgents), a, \"other\")\n| summarize CallCount = count() by agentName\n| order by CallCount desc", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "table" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Copilot Feature Usage", | |
| "type": "piechart" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Top tools invoked by Copilot during agent interactions. Includes file operations, search, MCP tool calls, and terminal commands.", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "fillOpacity": 80, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "lineWidth": 1, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green" | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 8, | |
| "x": 8, | |
| "y": 41 | |
| }, | |
| "id": 9, | |
| "options": { | |
| "barRadius": 0.05, | |
| "barWidth": 0.7, | |
| "fullHighlight": false, | |
| "groupWidth": 0.7, | |
| "legend": { | |
| "calcs": [], | |
| "displayMode": "list", | |
| "placement": "bottom", | |
| "showLegend": false | |
| }, | |
| "orientation": "horizontal", | |
| "showValue": "auto", | |
| "stacking": "none", | |
| "tooltip": { | |
| "hideZeros": false, | |
| "mode": "single", | |
| "sort": "none" | |
| }, | |
| "xTickLabelRotation": 0, | |
| "xTickLabelSpacing": 0 | |
| }, | |
| "pluginVersion": "11.6.9", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "let topTools = materialize(\n dependencies\n | where cloud_RoleName in (${source:singlequote})\n | where tostring(customDimensions[\"gen_ai.operation.name\"]) == \"execute_tool\"\n | extend t = tostring(customDimensions[\"gen_ai.tool.name\"])\n | where isnotempty(t)\n | summarize Total = count() by t\n | top 7 by Total\n | project t);\ndependencies\n| where cloud_RoleName in (${source:singlequote})\n| where tostring(customDimensions[\"gen_ai.operation.name\"]) == \"execute_tool\"\n| extend t = tostring(customDimensions[\"gen_ai.tool.name\"])\n| where isnotempty(t)\n| extend toolName = iff(t in (topTools), t, \"other\")\n| summarize CallCount = count() by toolName\n| order by CallCount desc", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "table" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Top Tool Calls", | |
| "type": "barchart" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Copilot activity over time: chat sessions started, agent conversation turns, and tool calls executed.", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "bars", | |
| "fillOpacity": 80, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "never", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "normal" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green" | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 8, | |
| "x": 16, | |
| "y": 41 | |
| }, | |
| "id": 10, | |
| "maxDataPoints": 60, | |
| "options": { | |
| "legend": { | |
| "calcs": [ | |
| "sum" | |
| ], | |
| "displayMode": "table", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": true, | |
| "mode": "multi", | |
| "sort": "desc" | |
| } | |
| }, | |
| "pluginVersion": "11.6.9", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "let sessions = dependencies\n| where cloud_RoleName in (${source:singlequote})\n| extend sessId = coalesce(\n tostring(customDimensions[\"copilot_chat.chat_session_id\"]),\n tostring(customDimensions[\"gen_ai.conversation.id\"]))\n| where isnotempty(sessId)\n| summarize Count = dcount(sessId) by bin(timestamp, $__interval)\n| extend eventName = \"Chat Sessions\";\nlet llmCalls = dependencies\n| where cloud_RoleName in (${source:singlequote})\n| where tostring(customDimensions[\"gen_ai.operation.name\"]) == \"chat\"\n| summarize Count = count() by bin(timestamp, $__interval)\n| extend eventName = \"LLM Calls\";\nlet tools = dependencies\n| where cloud_RoleName in (${source:singlequote})\n| where tostring(customDimensions[\"gen_ai.operation.name\"]) == \"execute_tool\"\n| summarize Count = count() by bin(timestamp, $__interval)\n| extend eventName = \"Tool Calls\";\nunion sessions, llmCalls, tools\n| order by timestamp asc", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "time_series", | |
| "timeColumn": "timestamp" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Sessions, Turns & Tool Calls Over Time", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "collapsed": false, | |
| "gridPos": { | |
| "h": 1, | |
| "w": 24, | |
| "x": 0, | |
| "y": 49 | |
| }, | |
| "id": 105, | |
| "panels": [], | |
| "title": "Token Efficiency & Cost", | |
| "type": "row" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Detailed breakdown of token usage per model: total input tokens, output tokens, average tokens per call, and call count.", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "thresholds" | |
| }, | |
| "custom": { | |
| "align": "auto", | |
| "cellOptions": { | |
| "type": "auto" | |
| }, | |
| "filterable": true, | |
| "inspect": false | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green" | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [ | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "TotalInputTokens" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "custom.cellOptions", | |
| "value": { | |
| "mode": "gradient", | |
| "type": "gauge" | |
| } | |
| }, | |
| { | |
| "id": "color", | |
| "value": { | |
| "mode": "continuous-blues" | |
| } | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "TotalOutputTokens" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "custom.cellOptions", | |
| "value": { | |
| "mode": "gradient", | |
| "type": "gauge" | |
| } | |
| }, | |
| { | |
| "id": "color", | |
| "value": { | |
| "mode": "continuous-greens" | |
| } | |
| } | |
| ] | |
| } | |
| ] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 12, | |
| "x": 0, | |
| "y": 50 | |
| }, | |
| "id": 14, | |
| "options": { | |
| "cellHeight": "sm", | |
| "footer": { | |
| "countRows": false, | |
| "enablePagination": false, | |
| "fields": [], | |
| "reducer": [ | |
| "sum" | |
| ], | |
| "show": true | |
| }, | |
| "showHeader": true, | |
| "sortBy": [ | |
| { | |
| "desc": true, | |
| "displayName": "TotalInputTokens" | |
| } | |
| ] | |
| }, | |
| "pluginVersion": "11.6.9", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "dependencies\n| where cloud_RoleName in (${source:singlequote})\n| where isnotempty(customDimensions[\"gen_ai.response.model\"])\n| extend model = tostring(customDimensions[\"gen_ai.response.model\"]),\n inputTokens = toint(customDimensions[\"gen_ai.usage.input_tokens\"]),\n outputTokens = toint(customDimensions[\"gen_ai.usage.output_tokens\"])\n| summarize Calls = count(),\n TotalInputTokens = sum(inputTokens),\n TotalOutputTokens = sum(outputTokens),\n AvgInputPerCall = round(avg(inputTokens), 0),\n AvgOutputPerCall = round(avg(outputTokens), 0)\n by model\n| order by TotalInputTokens desc", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "table" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Token Usage by Model", | |
| "type": "table" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Input vs Output token ratio over time. A high ratio indicates large prompts with relatively small responses, which is typical for code context.", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "Tokens", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "line", | |
| "fillOpacity": 30, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "smooth", | |
| "lineWidth": 2, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "auto", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "none" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green" | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 12, | |
| "x": 12, | |
| "y": 50 | |
| }, | |
| "id": 15, | |
| "options": { | |
| "legend": { | |
| "calcs": [ | |
| "mean", | |
| "sum" | |
| ], | |
| "displayMode": "table", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": true, | |
| "mode": "multi", | |
| "sort": "desc" | |
| } | |
| }, | |
| "pluginVersion": "11.6.9", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "dependencies\n| where cloud_RoleName in (${source:singlequote})\n| where isnotempty(customDimensions[\"gen_ai.response.model\"])\n| extend inputTokens = toint(customDimensions[\"gen_ai.usage.input_tokens\"]),\n outputTokens = toint(customDimensions[\"gen_ai.usage.output_tokens\"])\n| summarize ['Input Tokens'] = sum(inputTokens), ['Output Tokens'] = sum(outputTokens) by bin(timestamp, $__interval)\n| order by timestamp asc", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "time_series", | |
| "timeColumn": "timestamp" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Input vs Output Tokens Over Time", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "collapsed": false, | |
| "gridPos": { | |
| "h": 1, | |
| "w": 24, | |
| "x": 0, | |
| "y": 58 | |
| }, | |
| "id": 106, | |
| "panels": [], | |
| "title": "Errors & Reliability", | |
| "type": "row" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Exceptions thrown by Copilot Chat over time. Includes canceled operations, MCP server errors, and other failures.", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "palette-classic" | |
| }, | |
| "custom": { | |
| "axisBorderShow": false, | |
| "axisCenteredZero": false, | |
| "axisColorMode": "text", | |
| "axisLabel": "", | |
| "axisPlacement": "auto", | |
| "barAlignment": 0, | |
| "barWidthFactor": 0.6, | |
| "drawStyle": "bars", | |
| "fillOpacity": 80, | |
| "gradientMode": "none", | |
| "hideFrom": { | |
| "legend": false, | |
| "tooltip": false, | |
| "viz": false | |
| }, | |
| "insertNulls": false, | |
| "lineInterpolation": "linear", | |
| "lineWidth": 1, | |
| "pointSize": 5, | |
| "scaleDistribution": { | |
| "type": "linear" | |
| }, | |
| "showPoints": "never", | |
| "spanNulls": false, | |
| "stacking": { | |
| "group": "A", | |
| "mode": "normal" | |
| }, | |
| "thresholdsStyle": { | |
| "mode": "off" | |
| } | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green" | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 12, | |
| "x": 0, | |
| "y": 59 | |
| }, | |
| "id": 11, | |
| "maxDataPoints": 60, | |
| "options": { | |
| "legend": { | |
| "calcs": [ | |
| "sum" | |
| ], | |
| "displayMode": "table", | |
| "placement": "bottom", | |
| "showLegend": true | |
| }, | |
| "tooltip": { | |
| "hideZeros": true, | |
| "mode": "multi", | |
| "sort": "desc" | |
| } | |
| }, | |
| "pluginVersion": "11.6.9", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "exceptions\n| where cloud_RoleName in (${source:singlequote})\n| extend exType = coalesce(type, outerType, tostring(customDimensions[\"exception.type\"]))\n| summarize Count = count() by bin(timestamp, $__interval), exType\n| order by timestamp asc", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "time_series", | |
| "timeColumn": "timestamp" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Exceptions Over Time", | |
| "type": "timeseries" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Detailed exception log showing each error with timestamp, type, and message. Useful for debugging Copilot issues like MCP server failures or canceled requests.", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "thresholds" | |
| }, | |
| "custom": { | |
| "align": "auto", | |
| "cellOptions": { | |
| "type": "auto" | |
| }, | |
| "filterable": true, | |
| "inspect": true | |
| }, | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green" | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [ | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "ExceptionType" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "custom.width", | |
| "value": 120 | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "timestamp" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "custom.width", | |
| "value": 180 | |
| } | |
| ] | |
| } | |
| ] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 12, | |
| "x": 12, | |
| "y": 59 | |
| }, | |
| "id": 16, | |
| "options": { | |
| "cellHeight": "sm", | |
| "footer": { | |
| "countRows": true, | |
| "enablePagination": true, | |
| "fields": [], | |
| "reducer": [ | |
| "count" | |
| ], | |
| "show": true | |
| }, | |
| "showHeader": true, | |
| "sortBy": [ | |
| { | |
| "desc": true, | |
| "displayName": "timestamp" | |
| } | |
| ] | |
| }, | |
| "pluginVersion": "11.6.9", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "exceptions\n| where cloud_RoleName in (${source:singlequote})\n| extend ExceptionType = coalesce(type, outerType, tostring(customDimensions[\"exception.type\"])),\n Message = coalesce(outerMessage, innermostMessage, problemId)\n| project timestamp, ExceptionType, Message\n| order by timestamp desc", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "table" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Exception Details", | |
| "type": "table" | |
| }, | |
| { | |
| "collapsed": false, | |
| "gridPos": { | |
| "h": 1, | |
| "w": 24, | |
| "x": 0, | |
| "y": 67 | |
| }, | |
| "id": 107, | |
| "panels": [], | |
| "title": "Select a trace", | |
| "type": "row" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Interactive table listing recent Copilot Chat operations. Click any row to drill down into detailed trace information for that specific operation.", | |
| "fieldConfig": { | |
| "defaults": { | |
| "color": { | |
| "mode": "thresholds" | |
| }, | |
| "custom": { | |
| "align": "auto", | |
| "cellOptions": { | |
| "type": "auto" | |
| }, | |
| "inspect": false | |
| }, | |
| "links": [ | |
| { | |
| "targetBlank": false, | |
| "title": "Open trace with trace Id \"${__data.fields[TraceId]}\"", | |
| "url": "/d/${__dashboard.uid}/${__dashboard.slug}?var-copilotTraceId=${__data.fields[TraceId]}" | |
| } | |
| ], | |
| "mappings": [], | |
| "thresholds": { | |
| "mode": "absolute", | |
| "steps": [ | |
| { | |
| "color": "green" | |
| }, | |
| { | |
| "color": "red", | |
| "value": 80 | |
| } | |
| ] | |
| } | |
| }, | |
| "overrides": [ | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "timestamp" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "custom.width", | |
| "value": 209 | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "duration" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "unit", | |
| "value": "ms" | |
| }, | |
| { | |
| "id": "custom.width", | |
| "value": 122 | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "success" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "custom.cellOptions", | |
| "value": { | |
| "applyToRow": false, | |
| "type": "color-background" | |
| } | |
| }, | |
| { | |
| "id": "mappings", | |
| "value": [ | |
| { | |
| "options": { | |
| "False": { | |
| "color": "red", | |
| "index": 0 | |
| } | |
| }, | |
| "type": "value" | |
| } | |
| ] | |
| }, | |
| { | |
| "id": "custom.width", | |
| "value": 77 | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "operation" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "custom.width", | |
| "value": 287 | |
| } | |
| ] | |
| }, | |
| { | |
| "matcher": { | |
| "id": "byName", | |
| "options": "model" | |
| }, | |
| "properties": [ | |
| { | |
| "id": "custom.width", | |
| "value": 189 | |
| } | |
| ] | |
| } | |
| ] | |
| }, | |
| "gridPos": { | |
| "h": 8, | |
| "w": 24, | |
| "x": 0, | |
| "y": 68 | |
| }, | |
| "id": 108, | |
| "options": { | |
| "cellHeight": "sm", | |
| "footer": { | |
| "countRows": false, | |
| "enablePagination": true, | |
| "fields": "", | |
| "reducer": [ | |
| "sum" | |
| ], | |
| "show": false | |
| }, | |
| "showHeader": true, | |
| "sortBy": [ | |
| { | |
| "desc": true, | |
| "displayName": "timestamp" | |
| } | |
| ] | |
| }, | |
| "pluginVersion": "11.6.9", | |
| "targets": [ | |
| { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "dependencies\n| where cloud_RoleName in (${source:singlequote})\n| where name contains \"invoke_agent\"\n| extend operation = tostring(customDimensions[\"gen_ai.operation.name\"]),\n model = tostring(customDimensions[\"gen_ai.response.model\"]),\n input_tokens = toint(customDimensions[\"gen_ai.usage.input_tokens\"]),\n output_tokens = toint(customDimensions[\"gen_ai.usage.output_tokens\"])\n| project timestamp, Source = cloud_RoleName, operation, model, success, duration, input_tokens, output_tokens, TraceId=operation_Id\n| order by timestamp desc", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "logs", | |
| "timeColumn": "timestamp" | |
| }, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| } | |
| ], | |
| "title": "Recent Operations", | |
| "type": "table" | |
| }, | |
| { | |
| "collapsed": false, | |
| "gridPos": { | |
| "h": 1, | |
| "w": 24, | |
| "x": 0, | |
| "y": 76 | |
| }, | |
| "id": 109, | |
| "panels": [], | |
| "title": "Trace detail", | |
| "type": "row" | |
| }, | |
| { | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "description": "Detailed trace visualization showing the complete execution flow of a selected Copilot Chat operation, including all dependencies, timing, and hierarchical span relationships.", | |
| "gridPos": { | |
| "h": 27, | |
| "w": 24, | |
| "x": 0, | |
| "y": 77 | |
| }, | |
| "id": 110, | |
| "options": { | |
| "spanFilters": { | |
| "criticalPathOnly": false, | |
| "matchesOnly": false, | |
| "serviceNameOperator": "=", | |
| "spanNameOperator": "=", | |
| "tags": [ | |
| { | |
| "id": "copilot-trace-001", | |
| "operator": "=" | |
| } | |
| ] | |
| } | |
| }, | |
| "pluginVersion": "11.6.9", | |
| "targets": [ | |
| { | |
| "azureTraces": { | |
| "operationId": "$copilotTraceId", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "resultFormat": "trace", | |
| "traceTypes": [ | |
| "availabilityResults", | |
| "dependencies", | |
| "customEvents", | |
| "exceptions", | |
| "pageViews", | |
| "requests", | |
| "traces" | |
| ] | |
| }, | |
| "queryType": "Azure Traces", | |
| "refId": "A", | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| } | |
| } | |
| ], | |
| "title": "Trace Visualization", | |
| "type": "traces" | |
| } | |
| ], | |
| "preload": false, | |
| "refresh": "30m", | |
| "schemaVersion": 42, | |
| "tags": [ | |
| "github-copilot", | |
| "copilot", | |
| "ai", | |
| "llm", | |
| "opentelemetry", | |
| "application-insights", | |
| "Azure-managed" | |
| ], | |
| "templating": { | |
| "list": [ | |
| { | |
| "current": {}, | |
| "label": "Data Source", | |
| "name": "am_ds", | |
| "options": [], | |
| "query": "grafana-azure-monitor-datasource", | |
| "refresh": 1, | |
| "regex": "", | |
| "type": "datasource" | |
| }, | |
| { | |
| "current": {}, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "definition": "", | |
| "description": "Subscription of the Application Insights", | |
| "includeAll": false, | |
| "label": "Subscription", | |
| "name": "sub", | |
| "options": [], | |
| "query": { | |
| "grafanaTemplateVariableFn": { | |
| "kind": "SubscriptionsQuery", | |
| "rawQuery": "subscriptions()" | |
| }, | |
| "queryType": "Azure Subscriptions", | |
| "refId": "A" | |
| }, | |
| "refresh": 1, | |
| "regex": "", | |
| "regexApplyTo": "value", | |
| "sort": 5, | |
| "type": "query" | |
| }, | |
| { | |
| "current": {}, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "definition": "", | |
| "label": "Resource Group", | |
| "name": "rg", | |
| "options": [], | |
| "query": { | |
| "azureLogAnalytics": { | |
| "query": "", | |
| "resources": [] | |
| }, | |
| "azureResourceGraph": { | |
| "query": "resources\r\n| where type =~ \"microsoft.insights/components\"\r\n| project resourceGroup" | |
| }, | |
| "queryType": "Azure Resource Graph", | |
| "refId": "A", | |
| "subscriptions": [ | |
| "$sub" | |
| ] | |
| }, | |
| "refresh": 1, | |
| "regex": "", | |
| "regexApplyTo": "value", | |
| "type": "query" | |
| }, | |
| { | |
| "current": {}, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${DS_AZURE_MONITOR}" | |
| }, | |
| "definition": "", | |
| "label": "Application Insights", | |
| "name": "res", | |
| "options": [], | |
| "query": { | |
| "azureLogAnalytics": { | |
| "query": "", | |
| "resources": [] | |
| }, | |
| "namespace": "microsoft.insights/components", | |
| "queryType": "Azure Resource Names", | |
| "refId": "A", | |
| "resourceGroup": "$rg", | |
| "subscription": "$sub" | |
| }, | |
| "refresh": 1, | |
| "regex": "", | |
| "regexApplyTo": "value", | |
| "sort": 5, | |
| "type": "query" | |
| }, | |
| { | |
| "current": {}, | |
| "datasource": { | |
| "type": "grafana-azure-monitor-datasource", | |
| "uid": "${am_ds}" | |
| }, | |
| "definition": "", | |
| "description": "Trace Id of the Copilot Chat operations", | |
| "hide": 2, | |
| "label": "Trace Id", | |
| "name": "copilotTraceId", | |
| "options": [], | |
| "query": { | |
| "azureLogAnalytics": { | |
| "dashboardTime": true, | |
| "query": "dependencies\r\n| where cloud_RoleName == \"copilot-chat\"\r\n| distinct operation_Id\r\n| limit 50", | |
| "resources": [ | |
| "/subscriptions/$sub/resourceGroups/$rg/providers/microsoft.insights/components/$res" | |
| ], | |
| "timeColumn": "timestamp" | |
| }, | |
| "queryType": "Azure Log Analytics", | |
| "refId": "A" | |
| }, | |
| "refresh": 2, | |
| "regex": "", | |
| "regexApplyTo": "value", | |
| "type": "query" | |
| }, | |
| { | |
| "current": { | |
| "text": [ | |
| "copilot-chat", | |
| "github-copilot" | |
| ], | |
| "value": [ | |
| "copilot-chat", | |
| "github-copilot" | |
| ] | |
| }, | |
| "description": "Filter by sender: VS Code Copilot Chat (copilot-chat), Copilot CLI (github-copilot), or both", | |
| "includeAll": false, | |
| "label": "Source", | |
| "multi": true, | |
| "name": "source", | |
| "options": [ | |
| { | |
| "selected": true, | |
| "text": "VS Code Copilot", | |
| "value": "copilot-chat" | |
| }, | |
| { | |
| "selected": true, | |
| "text": "Copilot CLI", | |
| "value": "github-copilot" | |
| } | |
| ], | |
| "query": "VS Code Copilot : copilot-chat, Copilot CLI : github-copilot", | |
| "type": "custom", | |
| "valuesFormat": "csv" | |
| } | |
| ] | |
| }, | |
| "time": { | |
| "from": "now-7d", | |
| "to": "now" | |
| }, | |
| "timepicker": {}, | |
| "timezone": "browser", | |
| "title": "GitHub Copilot - Custom Agents", | |
| "uid": "github-copilot-custom-agents", | |
| "version": 1, | |
| "weekStart": "" | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment