Script for creating an alert for a service

create_alert.py

#
# create_alert.py - Create an alert for an email address
#

import sys

import requests
import json


#
# Parameters: "Token" "Cluster Name" "Service to Monitor" "Email to send alerts to"
#
def main():
    token = sys.argv[1]
    cluster = sys.argv[2]
    service = sys.argv[3]
    email = sys.argv[4]

    headers = {
        'X-SF-TOKEN': token,
        'Content-Type': 'application/json'
    }

    common_filter = f"filter('metric_source', 'prometheus_discovery') and " \
                    f"filter('kubernetes_cluster', '{cluster}') and " \
                    f"filter('app', '{service}')"
    alert_program = f"data('counter_status_500*', filter={common_filter}).delta().sum(over='5m').publish(label='A')"

    alert_data = {
        'name': f"Service {service} 500 error alert",
        'programText': alert_program,
        'rules': [{
            'detectLabel': 'A',
            'severity': 'Major',
            'parameterizedSubject': '{{ruleSeverity}} Alert: {{{ruleName}}} ({{{detectorName}}})',
            'parameterizedBody': """{{#if anomalous}}
	Rule "{{{ruleName}}}" in detector "{{{detectorName}}}" triggered at {{timestamp}}.
{{else}}
	Rule "{{{ruleName}}}" in detector "{{{detectorName}}}" cleared at {{timestamp}}.
{{/if}}

{{#if anomalous}}
Triggering condition: {{{readableRule}}}
{{/if}}

{{#if anomalous}}Signal value for counter_status_500* - Delta - Sum(30m): {{inputs.A.value}}
{{else}}Current signal value for counter_status_500* - Delta - Sum(30m): {{inputs.A.value}}
{{/if}}

{{#notEmpty dimensions}}
Signal details:
{{{dimensions}}}
{{/notEmpty}}

{{#if anomalous}}
{{#if runbookUrl}}Runbook: {{{runbookUrl}}}{{/if}}
{{#if tip}}Tip: {{{tip}}}{{/if}}
{{/if}}""",
            'notifications': [{
                'email': email,
                'type': 'Email'
            }]
        }]
    }

    alert_api = 'https://api.signalfx.com/v2/detector'
    response = requests.post(alert_api, headers=headers, json=alert_data)
    if response.status_code != 200:
        print(f"Failed to create alert {response.status_code}")
        return -1
    r_json = json.loads(response.text)
    return r_json.id