Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Monzo's Alertmanager Slack templates
###################################################
##
## Alertmanager YAML configuration for routing.
##
## Will route alerts with a code_owner label to the slack-code-owners receiver
## configured above, but will continue processing them to send to both a
## central Slack channel (slack-monitoring) and PagerDuty receivers
## (pd-warning and pd-critical)
##
routes:
###################################################
## Duplicate code_owner routes to teams
## These will send alerts to team channels but continue
## processing through the rest of the tree to handled by on-call
- match_re:
code_owner: '.+'
routes:
- match: {severity: info|warning|critical}
continue: true
receiver: slack-code-owners
###################################################
## Standard on-call routes
- match_re:
severity: info|warning|critical
receiver: slack-monitoring
continue: true
###################################################
## Pagerduty routes
- match: {severity: warning}
routes:
- receiver: pd-warning
- match: {severity: critical}
routes:
- receiver: pd-critical
receivers:
###################################################
## Slack Receivers
- name: slack-code-owners
slack_configs:
- channel: '#{{- template "slack.monzo.code_owner_channel" . -}}'
send_resolved: true
title: '{{ template "slack.monzo.title" . }}'
icon_emoji: '{{ template "slack.monzo.icon_emoji" . }}'
color: '{{ template "slack.monzo.color" . }}'
text: '{{ template "slack.monzo.text" . }}'
actions:
- type: button
text: 'Runbook :green_book:'
url: '{{ (index .Alerts 0).Annotations.runbook }}'
- type: button
text: 'Query :mag:'
url: '{{ (index .Alerts 0).GeneratorURL }}'
- type: button
text: 'Dashboard :grafana:'
url: '{{ (index .Alerts 0).Annotations.dashboard }}'
- type: button
text: 'Silence :no_bell:'
url: '{{ template "__alert_silence_link" . }}'
- type: button
text: '{{ template "slack.monzo.link_button_text" . }}'
url: '{{ .CommonAnnotations.link_url }}'
# This builds the silence URL. We exclude the alertname in the range
# to avoid the issue of having trailing comma separator (%2C) at the end
# of the generated URL
{{ define "__alert_silence_link" -}}
{{ .ExternalURL }}/#/silences/new?filter=%7B
{{- range .CommonLabels.SortedPairs -}}
{{- if ne .Name "alertname" -}}
{{- .Name }}%3D"{{- .Value -}}"%2C%20
{{- end -}}
{{- end -}}
alertname%3D"{{ .CommonLabels.alertname }}"%7D
{{- end }}
{{ define "__alert_severity_prefix" -}}
{{ if ne .Status "firing" -}}
:lgtm:
{{- else if eq .Labels.severity "critical" -}}
:fire:
{{- else if eq .Labels.severity "warning" -}}
:warning:
{{- else -}}
:question:
{{- end }}
{{- end }}
{{ define "__alert_severity_prefix_title" -}}
{{ if ne .Status "firing" -}}
:lgtm:
{{- else if eq .CommonLabels.severity "critical" -}}
:fire:
{{- else if eq .CommonLabels.severity "warning" -}}
:warning:
{{- else if eq .CommonLabels.severity "info" -}}
:information_source:
{{- else -}}
:question:
{{- end }}
{{- end }}
{{/* First line of Slack alerts */}}
{{ define "slack.monzo.title" -}}
[{{ .Status | toUpper -}}
{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{- end -}}
] {{ template "__alert_severity_prefix_title" . }} {{ .CommonLabels.alertname }}
{{- end }}
{{/* Color of Slack attachment (appears as line next to alert )*/}}
{{ define "slack.monzo.color" -}}
{{ if eq .Status "firing" -}}
{{ if eq .CommonLabels.severity "warning" -}}
warning
{{- else if eq .CommonLabels.severity "critical" -}}
danger
{{- else -}}
#439FE0
{{- end -}}
{{ else -}}
good
{{- end }}
{{- end }}
{{/* Emoji to display as user icon (custom emoji supported!) */}}
{{ define "slack.monzo.icon_emoji" }}:prometheus:{{ end }}
{{/* The test to display in the alert */}}
{{ define "slack.monzo.text" -}}
{{ range .Alerts }}
{{- if .Annotations.message }}
{{ .Annotations.message }}
{{- end }}
{{- if .Annotations.description }}
{{ .Annotations.description }}
{{- end }}
{{- end }}
{{- end }}
{{- /* If none of the below matches, send to #monitoring-no-owner, and we
can then assign the expected code_owner to the alert or map the code_owner
to the correct channel */ -}}
{{ define "__get_channel_for_code_owner" -}}
{{- if eq . "platform-team" -}}
platform-alerts
{{- else if eq . "security-team" -}}
security-alerts
{{- else -}}
monitoring-no-owner
{{- end -}}
{{- end }}
{{- /* Select the channel based on the code_owner. We only expect to get
into this template function if the code_owners label is present on an alert.
This is to defend against us accidentally breaking the routing logic. */ -}}
{{ define "slack.monzo.code_owner_channel" -}}
{{- if .CommonLabels.code_owner }}
{{ template "__get_channel_for_code_owner" .CommonLabels.code_owner }}
{{- else -}}
monitoring
{{- end }}
{{- end }}
{{ define "slack.monzo.link_button_text" -}}
{{- if .CommonAnnotations.link_text -}}
{{- .CommonAnnotations.link_text -}}
{{- else -}}
Link
{{- end }} :link:
{{- end }}
@brovoca
Copy link

brovoca commented Aug 25, 2022

@ioncoa Remove the indentation from the go template function. E.g.

{{/* The text to display in the alert */}}
{{ define "slack.monzo.text" -}}
{{ range .Alerts }}
{{- if .Annotations.description }}
{{ .Annotations.description }}
{{- end }}
{{- end }}
{{- end }}

@aorfanos
Copy link

aorfanos commented Oct 1, 2022

Addition for Grafana Alerting integration with AlertManager:

ℹ️ You have to set alertmanger.externalUrl to your Grafana instance external URL

    {{ define "__grafana_alert_silence_link" -}}
        {{ .ExternalURL }}/alerting/silence/new?
        {{- range .CommonLabels.SortedPairs -}}
            matcher={{- .Name }}%3D{{- .Value | urlquery -}}&
        {{- end -}}
    {{- end }}

@audunsolemdal
Copy link

audunsolemdal commented Nov 24, 2022

Addition for Grafana Alerting integration with AlertManager:

ℹ️ You have to set alertmanger.externalUrl to your Grafana instance external URL

    {{ define "__grafana_alert_silence_link" -}}
        {{ .ExternalURL }}/alerting/silence/new?
        {{- range .CommonLabels.SortedPairs -}}
            matcher={{- .Name }}%3D{{- .Value | urlquery -}}&
        {{- end -}}
    {{- end }}

Thanks @aorfanos

Have you also found a way to get the query button working towards Grafana? Alternatively, an external Prometheus?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment