Dashboard to optimize baseline alert condition thresholds and examine model behavior

baseline_optimization.json

{
  "name": "Baseline Alert Optimization",
  "description": null,
  "permissions": "PUBLIC_READ_WRITE",
  "pages": [
    {
      "name": "Baseline Alert Optimization",
      "description": null,
      "widgets": [
        {
          "title": "",
          "layout": {
            "column": 1,
            "row": 1,
            "width": 5,
            "height": 3
          },
          "linkedEntityGuids": null,
          "visualization": {
            "id": "viz.markdown"
          },
          "rawConfiguration": {
            "text": "# NRQL Baseline Optimization\nThis view aims to optimize the standard deviation threshold within baseline alert conditions. A good rule of thumb is to set critical threshold in a baseline alert condition to the p75 (75th percentile) std deviation value shown.\n## How to use this dashboard\n1. Select a condition id from dropdown above, and specify your condition's current std deviation threshold\n2. Review widgets to determine optimal deviation threshold\n\n## Additional detail\n- Plot your condition's query in a timeseries format alongside this data to correlate signal behavior with corresponding deviation trends (i.e - **100-200 ms latency may equate to 10-20 deviations from the normal trend**)\n- Current model behavior can be examined by looking at the `Model Behavior` widget - This displays the actual signal value, predicted value by the model (which is updated as it learns the patterns), and the upper threshold calculated based on the std dev threshold set. If the signal value is consistently above the upper threshold, that is a good indicator the threshold needs to be tuned. If the predictedValue trend is not in close alignment with the signalValue trend, that is an indicator the model is not trained well enough yet."
          }
        },
        {
          "title": "Std Deviation Candidate Thresholds",
          "layout": {
            "column": 6,
            "row": 1,
            "width": 7,
            "height": 3
          },
          "linkedEntityGuids": null,
          "visualization": {
            "id": "viz.table"
          },
          "rawConfiguration": {
            "facet": {
              "showOtherSeries": false
            },
            "nrqlQueries": [
              {
                "accountIds": [
                  123
                ],
                "query": "SELECT average(deviations) as 'avg', percentile(deviations, 75) as 'p75', percentile(deviations, 95) as 'p95' FROM (FROM NrAiSignal SELECT max(abs(numberOfDeviations)) as 'deviations' WHERE conditionId = {{condition_id}} TIMESERIES 1 hour LIMIT MAX) since 1 week ago compare with 2 weeks ago FACET string({{threshold}}) as 'Current Threshold'"
              }
            ],
            "platformOptions": {
              "ignoreTimeRange": false
            }
          }
        },
        {
          "title": "Actual deviatons vs Current Threshold Trend",
          "layout": {
            "column": 1,
            "row": 4,
            "width": 12,
            "height": 3
          },
          "linkedEntityGuids": null,
          "visualization": {
            "id": "viz.line"
          },
          "rawConfiguration": {
            "facet": {
              "showOtherSeries": false
            },
            "legend": {
              "enabled": true
            },
            "nrqlQueries": [
              {
                "accountIds": [
                  123
                ],
                "query": "FROM NrAiSignal SELECT max(abs(numberOfDeviations)), ({{threshold}} or 0) as `Current Threshold` WHERE conditionId = {{condition_id}} SINCE 1 week ago TIMESERIES 30 minutes COMPARE WITH 2 weeks ago"
              }
            ],
            "platformOptions": {
              "ignoreTimeRange": false
            },
            "yAxisLeft": {
              "zero": true
            },
            "yAxisRight": {
              "zero": true
            }
          }
        },
        {
          "title": "Model Behavior",
          "layout": {
            "column": 1,
            "row": 7,
            "width": 12,
            "height": 3
          },
          "linkedEntityGuids": null,
          "visualization": {
            "id": "viz.line"
          },
          "rawConfiguration": {
            "facet": {
              "showOtherSeries": false
            },
            "legend": {
              "enabled": true
            },
            "nrqlQueries": [
              {
                "accountIds": [
                  123
                ],
                "query": "FROM NrAiSignal SELECT latest(signalValue), latest(predictedValue), latest(predictedValue + (standardDeviation * {{threshold}})) as 'Upper Threshold' WHERE conditionId = {{condition_id}}  SINCE 1 week ago TIMESERIES 30 minutes"
              }
            ],
            "platformOptions": {
              "ignoreTimeRange": false
            },
            "yAxisLeft": {
              "zero": true
            },
            "yAxisRight": {
              "zero": true
            }
          }
        }
      ]
    }
  ],
  "variables": [
    {
      "name": "condition_id",
      "items": null,
      "defaultValues": [],
      "nrqlQuery": {
        "accountIds": [
          123
        ],
        "query": "SELECT uniques(conditionId, 5000) FROM NrAiSignal where numberOfDeviations is not null since 1 month ago"
      },
      "options": {
        "ignoreTimeRange": true
      },
      "title": "Condition ID",
      "type": "NRQL",
      "isMultiSelection": false,
      "replacementStrategy": "NUMBER"
    },
    {
      "name": "threshold",
      "items": null,
      "defaultValues": [
        {
          "value": {
            "string": "7"
          }
        }
      ],
      "nrqlQuery": null,
      "title": "Current Threshold",
      "type": "STRING",
      "isMultiSelection": null,
      "replacementStrategy": "NUMBER"
    }
  ]
}