Monzo's Alertmanager Slack templates

monzo-alertmanager-config.yaml

###################################################
##
## Alertmanager YAML configuration for routing.
##
## Will route alerts with a code_owner label to the slack-code-owners receiver 
## configured above, but will continue processing them to send to both a
## central Slack channel (slack-monitoring) and PagerDuty receivers
## (pd-warning and pd-critical)
##

routes:
  ###################################################
  ## Duplicate code_owner routes to teams
  ## These will send alerts to team channels but continue
  ## processing through the rest of the tree to handled by on-call
  - match_re:
      code_owner: '.+'
    routes:
      - match: {severity: info|warning|critical}
        continue: true
        receiver: slack-code-owners

  ###################################################
  ## Standard on-call routes
  - match_re:
      severity: info|warning|critical
    receiver: slack-monitoring
    continue: true


  ###################################################
  ## Pagerduty routes
  - match: {severity: warning}
    routes:
    - receiver: pd-warning
  - match: {severity: critical}
    routes:
    - receiver: pd-critical

monzo-alertmanager-receiver.yaml

receivers:
###################################################
## Slack Receivers
- name: slack-code-owners
  slack_configs:
  - channel: '#{{- template "slack.monzo.code_owner_channel" . -}}'
    send_resolved: true
    title: '{{ template "slack.monzo.title" . }}'
    icon_emoji: '{{ template "slack.monzo.icon_emoji" . }}'
    color: '{{ template "slack.monzo.color" . }}'
    text: '{{ template "slack.monzo.text" . }}'
    actions:
    - type: button
      text: 'Runbook :green_book:'
      url: '{{ (index .Alerts 0).Annotations.runbook }}'
    - type: button
      text: 'Query :mag:'
      url: '{{ (index .Alerts 0).GeneratorURL }}'
    - type: button
      text: 'Dashboard :grafana:'
      url: '{{ (index .Alerts 0).Annotations.dashboard }}'
    - type: button
      text: 'Silence :no_bell:'
      url: '{{ template "__alert_silence_link" . }}'
    - type: button
      text: '{{ template "slack.monzo.link_button_text" . }}'
      url: '{{ .CommonAnnotations.link_url }}'

monzo-slack-templates.tmpl

# This builds the silence URL.  We exclude the alertname in the range
# to avoid the issue of having trailing comma separator (%2C) at the end
# of the generated URL
{{ define "__alert_silence_link" -}}
    {{ .ExternalURL }}/#/silences/new?filter=%7B
    {{- range .CommonLabels.SortedPairs -}}
        {{- if ne .Name "alertname" -}}
            {{- .Name }}%3D"{{- .Value -}}"%2C%20
        {{- end -}}
    {{- end -}}
    alertname%3D"{{ .CommonLabels.alertname }}"%7D
{{- end }}



{{ define "__alert_severity_prefix" -}}
    {{ if ne .Status "firing" -}}
    :lgtm:
    {{- else if eq .Labels.severity "critical" -}}
    :fire:
    {{- else if eq .Labels.severity "warning" -}}
    :warning:
    {{- else -}}
    :question:
    {{- end }}
{{- end }}

{{ define "__alert_severity_prefix_title" -}}
    {{ if ne .Status "firing" -}}
    :lgtm:
    {{- else if eq .CommonLabels.severity "critical" -}}
    :fire:
    {{- else if eq .CommonLabels.severity "warning" -}}
    :warning:
    {{- else if eq .CommonLabels.severity "info" -}}
    :information_source:
    {{- else -}}
    :question:
    {{- end }}
{{- end }}


{{/* First line of Slack alerts */}}
{{ define "slack.monzo.title" -}}
    [{{ .Status | toUpper -}}
    {{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{- end -}}
    ] {{ template "__alert_severity_prefix_title" . }} {{ .CommonLabels.alertname }}
{{- end }}


{{/* Color of Slack attachment (appears as line next to alert )*/}}
{{ define "slack.monzo.color" -}}
    {{ if eq .Status "firing" -}}
        {{ if eq .CommonLabels.severity "warning" -}}
            warning
        {{- else if eq .CommonLabels.severity "critical" -}}
            danger
        {{- else -}}
            #439FE0
        {{- end -}}
    {{ else -}}
    good
    {{- end }}
{{- end }}


{{/* Emoji to display as user icon (custom emoji supported!) */}}
{{ define "slack.monzo.icon_emoji" }}:prometheus:{{ end }}

{{/* The test to display in the alert */}}
{{ define "slack.monzo.text" -}}
    {{ range .Alerts }}
        {{- if .Annotations.message }}
            {{ .Annotations.message }}
        {{- end }}
        {{- if .Annotations.description }}
            {{ .Annotations.description }}
        {{- end }}
    {{- end }}
{{- end }}



{{- /* If none of the below matches, send to #monitoring-no-owner, and we 
can then assign the expected code_owner to the alert or map the code_owner
to the correct channel */ -}}
{{ define "__get_channel_for_code_owner" -}}
    {{- if eq . "platform-team" -}}
        platform-alerts
    {{- else if eq . "security-team" -}}
        security-alerts
    {{- else -}}
        monitoring-no-owner
    {{- end -}}
{{- end }}

{{- /* Select the channel based on the code_owner. We only expect to get
into this template function if the code_owners label is present on an alert.
This is to defend against us accidentally breaking the routing logic. */ -}}
{{ define "slack.monzo.code_owner_channel" -}}
    {{- if .CommonLabels.code_owner }}
        {{ template "__get_channel_for_code_owner" .CommonLabels.code_owner }}
    {{- else -}}
        monitoring
    {{- end }}
{{- end }}

{{ define "slack.monzo.link_button_text" -}}
    {{- if .CommonAnnotations.link_text -}}
        {{- .CommonAnnotations.link_text -}}
    {{- else -}}
        Link
    {{- end }} :link:
{{- end }}

hi guys how can use this? should you create docs dude

@amirnoshian https://prometheus.io/docs/alerting/latest/configuration/

If someone has missing Silence button, then double check your alert name. If alert name has spaces, then your link is malformed.
You can use this template

{{ define "__alert_silence_link" -}}
          {{ .ExternalURL }}/#/silences/new?filter=%7B
          {{- range .CommonLabels.SortedPairs -}}
              {{- if ne .Name "alertname" -}}
                  {{- .Name }}%3D"{{- .Value | urlquery -}}"%2C%20
              {{- end -}}
          {{- end -}}
          alertname%3D"{{- .CommonLabels.alertname | urlquery  -}}"%7D
      {{- end }}

Notice .CommonLabels.alertname | urlquery change in the end.
Buttons are going to disappear if you link is malformed (i.e. non-urlencoded spaces or other chars). The best way to check it is to add the same link to message text and check the result:

  receivers:
    - name: slack-receiver
      slack_configs:
        - channel: "#alerts" # slack channel for notrifications
          send_resolved: true
          color: '{{ template "slack.monzo.color" . }}'
          title: '{{ template "slack.monzo.title" . }}'
          text: '{{ template "slack.monzo.text" . }} LINK: `{{ template "__alert_silence_link" . }}` '

Hope this will help!

My case is there are too many labels value with a space " ",
this is how i solved it.

{{ define "__alert_silence_link" -}}
    {{ .ExternalURL }}/#/silences/new?filter=%7B
    {{- range .CommonLabels.SortedPairs -}}
        {{- if ne .Name "alertname" -}}
            {{- .Name }}%3D"{{- urlquery .Value | reReplaceAll "\\+" "%20" -}}"%2C%20
        {{- end -}}
    {{- end -}}
    alertname%3D"{{- urlquery .CommonLabels.alertname | reReplaceAll "\\+" "%20" -}}"%7D
{{- end }}

Hope this will help solving the same case

alertmanger.externalUrl

@aorfanos can you please tell me where should I put it in Grafana?

@fd239 https://github.com/prometheus-community/helm-charts/blob/8fd0872073e36db06e3dcab2dbf836de182adc77/charts/alertmanager/values.yaml#L19 here in alertmanager config.

Looks like the new alertmanager 0.32 breaks this. Anyone with a clue as to why?

Looks like the new alertmanager 0.32 breaks this. Anyone with a clue as to why?

VictoriaMetrics/helm-charts#2869 :D