Monzo's Alertmanager Slack templates

monzo-alertmanager-config.yaml

###################################################
##
## Alertmanager YAML configuration for routing.
##
## Will route alerts with a code_owner label to the slack-code-owners receiver 
## configured above, but will continue processing them to send to both a
## central Slack channel (slack-monitoring) and PagerDuty receivers
## (pd-warning and pd-critical)
##

routes:
  ###################################################
  ## Duplicate code_owner routes to teams
  ## These will send alerts to team channels but continue
  ## processing through the rest of the tree to handled by on-call
  - match_re:
      code_owner: '.+'
    routes:
      - match: {severity: info|warning|critical}
        continue: true
        receiver: slack-code-owners

  ###################################################
  ## Standard on-call routes
  - match_re:
      severity: info|warning|critical
    receiver: slack-monitoring
    continue: true


  ###################################################
  ## Pagerduty routes
  - match: {severity: warning}
    routes:
    - receiver: pd-warning
  - match: {severity: critical}
    routes:
    - receiver: pd-critical

monzo-alertmanager-receiver.yaml

receivers:
###################################################
## Slack Receivers
- name: slack-code-owners
  slack_configs:
  - channel: '#{{- template "slack.monzo.code_owner_channel" . -}}'
    send_resolved: true
    title: '{{ template "slack.monzo.title" . }}'
    icon_emoji: '{{ template "slack.monzo.icon_emoji" . }}'
    color: '{{ template "slack.monzo.color" . }}'
    text: '{{ template "slack.monzo.text" . }}'
    actions:
    - type: button
      text: 'Runbook :green_book:'
      url: '{{ (index .Alerts 0).Annotations.runbook }}'
    - type: button
      text: 'Query :mag:'
      url: '{{ (index .Alerts 0).GeneratorURL }}'
    - type: button
      text: 'Dashboard :grafana:'
      url: '{{ (index .Alerts 0).Annotations.dashboard }}'
    - type: button
      text: 'Silence :no_bell:'
      url: '{{ template "__alert_silence_link" . }}'
    - type: button
      text: '{{ template "slack.monzo.link_button_text" . }}'
      url: '{{ .CommonAnnotations.link_url }}'

monzo-slack-templates.tmpl

# This builds the silence URL.  We exclude the alertname in the range
# to avoid the issue of having trailing comma separator (%2C) at the end
# of the generated URL
{{ define "__alert_silence_link" -}}
    {{ .ExternalURL }}/#/silences/new?filter=%7B
    {{- range .CommonLabels.SortedPairs -}}
        {{- if ne .Name "alertname" -}}
            {{- .Name }}%3D"{{- .Value -}}"%2C%20
        {{- end -}}
    {{- end -}}
    alertname%3D"{{ .CommonLabels.alertname }}"%7D
{{- end }}



{{ define "__alert_severity_prefix" -}}
    {{ if ne .Status "firing" -}}
    :lgtm:
    {{- else if eq .Labels.severity "critical" -}}
    :fire:
    {{- else if eq .Labels.severity "warning" -}}
    :warning:
    {{- else -}}
    :question:
    {{- end }}
{{- end }}

{{ define "__alert_severity_prefix_title" -}}
    {{ if ne .Status "firing" -}}
    :lgtm:
    {{- else if eq .CommonLabels.severity "critical" -}}
    :fire:
    {{- else if eq .CommonLabels.severity "warning" -}}
    :warning:
    {{- else if eq .CommonLabels.severity "info" -}}
    :information_source:
    {{- else -}}
    :question:
    {{- end }}
{{- end }}


{{/* First line of Slack alerts */}}
{{ define "slack.monzo.title" -}}
    [{{ .Status | toUpper -}}
    {{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{- end -}}
    ] {{ template "__alert_severity_prefix_title" . }} {{ .CommonLabels.alertname }}
{{- end }}


{{/* Color of Slack attachment (appears as line next to alert )*/}}
{{ define "slack.monzo.color" -}}
    {{ if eq .Status "firing" -}}
        {{ if eq .CommonLabels.severity "warning" -}}
            warning
        {{- else if eq .CommonLabels.severity "critical" -}}
            danger
        {{- else -}}
            #439FE0
        {{- end -}}
    {{ else -}}
    good
    {{- end }}
{{- end }}


{{/* Emoji to display as user icon (custom emoji supported!) */}}
{{ define "slack.monzo.icon_emoji" }}:prometheus:{{ end }}

{{/* The test to display in the alert */}}
{{ define "slack.monzo.text" -}}
    {{ range .Alerts }}
        {{- if .Annotations.message }}
            {{ .Annotations.message }}
        {{- end }}
        {{- if .Annotations.description }}
            {{ .Annotations.description }}
        {{- end }}
    {{- end }}
{{- end }}



{{- /* If none of the below matches, send to #monitoring-no-owner, and we 
can then assign the expected code_owner to the alert or map the code_owner
to the correct channel */ -}}
{{ define "__get_channel_for_code_owner" -}}
    {{- if eq . "platform-team" -}}
        platform-alerts
    {{- else if eq . "security-team" -}}
        security-alerts
    {{- else -}}
        monitoring-no-owner
    {{- end -}}
{{- end }}

{{- /* Select the channel based on the code_owner. We only expect to get
into this template function if the code_owners label is present on an alert.
This is to defend against us accidentally breaking the routing logic. */ -}}
{{ define "slack.monzo.code_owner_channel" -}}
    {{- if .CommonLabels.code_owner }}
        {{ template "__get_channel_for_code_owner" .CommonLabels.code_owner }}
    {{- else -}}
        monitoring
    {{- end }}
{{- end }}

{{ define "slack.monzo.link_button_text" -}}
    {{- if .CommonAnnotations.link_text -}}
        {{- .CommonAnnotations.link_text -}}
    {{- else -}}
        Link
    {{- end }} :link:
{{- end }}

Addition for Grafana Alerting integration with AlertManager:

ℹ️ You have to set alertmanger.externalUrl to your Grafana instance external URL

    {{ define "__grafana_alert_silence_link" -}}
        {{ .ExternalURL }}/alerting/silence/new?
        {{- range .CommonLabels.SortedPairs -}}
            matcher={{- .Name }}%3D{{- .Value | urlquery -}}&
        {{- end -}}
    {{- end }}

Addition for Grafana Alerting integration with AlertManager:

ℹ️ You have to set alertmanger.externalUrl to your Grafana instance external URL

    {{ define "__grafana_alert_silence_link" -}}
        {{ .ExternalURL }}/alerting/silence/new?
        {{- range .CommonLabels.SortedPairs -}}
            matcher={{- .Name }}%3D{{- .Value | urlquery -}}&
        {{- end -}}
    {{- end }}

Thanks @aorfanos

Have you also found a way to get the query button working towards Grafana? Alternatively, an external Prometheus?

Hello,

Thanks for sharing :)

In case you have built in Grafana Alerting/silence but also using Alertmanager as an additional Data source you have to append alertmanager=Alertmanager at the end of the url .

      {{ define "__grafana_alert_silence_link" -}}
        {{ .ExternalURL }}/alerting/silence/new?
        {{- range .CommonLabels.SortedPairs -}}
            matcher={{- .Name }}%3D{{- .Value | urlquery -}}&
        {{- end -}}
        alertmanager=Alertmanager
        {{- end }}

P.S : tested on alertmanager 0.25 / grafana 9.1.6

Cheers

Hey everyone,
I want to share some findings as i tried to implement what is written here and got some insights that might help you:

1. name field for slack button:
   if you dont specify a name for a slack button, you will only have 1 slack button as they need it to diffrenciate between the buttons in the list. here is where i found it in their documents: https://api.slack.com/legacy/interactive-message-field-guide

2. if your button links to a url it must contain http:// or https:// i had a button with: grafana.com and it didn't work, only after i made it https://grafana.com it worked.

3. the advise by @80kk was crucial for the buttons, the value: 'test' field was mandatory for me to make it work.

so this is my configuration for working buttons:

          - type: button
            name: mutebutton
            text: 'Mute :prometheus:'
            value: 'text'
            url: "https://{{ (index .Alerts 0).Labels.alertManagerUrl }}" # just a link for our internal alert manager, with added https://
          - type: button
            name: readmebutton
            text: 'README :green_book:'
            value: 'text'
            url: "{{ (index .Alerts 0).Labels.readmeUrl }}" # just a link for our internal readme, it comes with https:// so no need to add
          - type: button
            name: grafana
            text: 'Grafana :grafana:'
            value: 'text'
            url: 'https://{{ (index .Alerts 0).Labels.grafanaAlertUrl }}' # same as first button

this is the final result:

and thanks everyone for all the comments here!

      actions:
      - type: button
        text: 'README :green_book:'
        url: '{{ (index .Alerts 0).Annotations.runbook_url }}'
      - type: button
        text: 'Query :mag:'
        url: '{{ (index .Alerts 0).GeneratorURL }}'
      - type: button
        text: 'Dashboard :chart_with_upwards_trend:'
        url: '{{ (index .Alerts 0).Annotations.dashboard_url }}'
      - type: button
        text: 'Silence :no_bell:'
        url: '{{ template "__alert_silence_link" . }}'
EOF

I have my buttons set up like this and I can only get in Slack the "Runbook" one to be shown. Any idea why it's not showing the Query, Dashboard or Silence button as well?

I have tried the value: 'text' but it seems not to make a difference.

@stefaneacsu147 see my previous answer, you might need to add name: field to each button.