-
-
Save milesbxf/e2744fc90e9c41b47aa47925f8ff6512 to your computer and use it in GitHub Desktop.
################################################### | |
## | |
## Alertmanager YAML configuration for routing. | |
## | |
## Will route alerts with a code_owner label to the slack-code-owners receiver | |
## configured above, but will continue processing them to send to both a | |
## central Slack channel (slack-monitoring) and PagerDuty receivers | |
## (pd-warning and pd-critical) | |
## | |
routes: | |
################################################### | |
## Duplicate code_owner routes to teams | |
## These will send alerts to team channels but continue | |
## processing through the rest of the tree to handled by on-call | |
- match_re: | |
code_owner: '.+' | |
routes: | |
- match: {severity: info|warning|critical} | |
continue: true | |
receiver: slack-code-owners | |
################################################### | |
## Standard on-call routes | |
- match_re: | |
severity: info|warning|critical | |
receiver: slack-monitoring | |
continue: true | |
################################################### | |
## Pagerduty routes | |
- match: {severity: warning} | |
routes: | |
- receiver: pd-warning | |
- match: {severity: critical} | |
routes: | |
- receiver: pd-critical |
receivers: | |
################################################### | |
## Slack Receivers | |
- name: slack-code-owners | |
slack_configs: | |
- channel: '#{{- template "slack.monzo.code_owner_channel" . -}}' | |
send_resolved: true | |
title: '{{ template "slack.monzo.title" . }}' | |
icon_emoji: '{{ template "slack.monzo.icon_emoji" . }}' | |
color: '{{ template "slack.monzo.color" . }}' | |
text: '{{ template "slack.monzo.text" . }}' | |
actions: | |
- type: button | |
text: 'Runbook :green_book:' | |
url: '{{ (index .Alerts 0).Annotations.runbook }}' | |
- type: button | |
text: 'Query :mag:' | |
url: '{{ (index .Alerts 0).GeneratorURL }}' | |
- type: button | |
text: 'Dashboard :grafana:' | |
url: '{{ (index .Alerts 0).Annotations.dashboard }}' | |
- type: button | |
text: 'Silence :no_bell:' | |
url: '{{ template "__alert_silence_link" . }}' | |
- type: button | |
text: '{{ template "slack.monzo.link_button_text" . }}' | |
url: '{{ .CommonAnnotations.link_url }}' |
# This builds the silence URL. We exclude the alertname in the range | |
# to avoid the issue of having trailing comma separator (%2C) at the end | |
# of the generated URL | |
{{ define "__alert_silence_link" -}} | |
{{ .ExternalURL }}/#/silences/new?filter=%7B | |
{{- range .CommonLabels.SortedPairs -}} | |
{{- if ne .Name "alertname" -}} | |
{{- .Name }}%3D"{{- .Value -}}"%2C%20 | |
{{- end -}} | |
{{- end -}} | |
alertname%3D"{{ .CommonLabels.alertname }}"%7D | |
{{- end }} | |
{{ define "__alert_severity_prefix" -}} | |
{{ if ne .Status "firing" -}} | |
:lgtm: | |
{{- else if eq .Labels.severity "critical" -}} | |
:fire: | |
{{- else if eq .Labels.severity "warning" -}} | |
:warning: | |
{{- else -}} | |
:question: | |
{{- end }} | |
{{- end }} | |
{{ define "__alert_severity_prefix_title" -}} | |
{{ if ne .Status "firing" -}} | |
:lgtm: | |
{{- else if eq .CommonLabels.severity "critical" -}} | |
:fire: | |
{{- else if eq .CommonLabels.severity "warning" -}} | |
:warning: | |
{{- else if eq .CommonLabels.severity "info" -}} | |
:information_source: | |
{{- else -}} | |
:question: | |
{{- end }} | |
{{- end }} | |
{{/* First line of Slack alerts */}} | |
{{ define "slack.monzo.title" -}} | |
[{{ .Status | toUpper -}} | |
{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{- end -}} | |
] {{ template "__alert_severity_prefix_title" . }} {{ .CommonLabels.alertname }} | |
{{- end }} | |
{{/* Color of Slack attachment (appears as line next to alert )*/}} | |
{{ define "slack.monzo.color" -}} | |
{{ if eq .Status "firing" -}} | |
{{ if eq .CommonLabels.severity "warning" -}} | |
warning | |
{{- else if eq .CommonLabels.severity "critical" -}} | |
danger | |
{{- else -}} | |
#439FE0 | |
{{- end -}} | |
{{ else -}} | |
good | |
{{- end }} | |
{{- end }} | |
{{/* Emoji to display as user icon (custom emoji supported!) */}} | |
{{ define "slack.monzo.icon_emoji" }}:prometheus:{{ end }} | |
{{/* The test to display in the alert */}} | |
{{ define "slack.monzo.text" -}} | |
{{ range .Alerts }} | |
{{- if .Annotations.message }} | |
{{ .Annotations.message }} | |
{{- end }} | |
{{- if .Annotations.description }} | |
{{ .Annotations.description }} | |
{{- end }} | |
{{- end }} | |
{{- end }} | |
{{- /* If none of the below matches, send to #monitoring-no-owner, and we | |
can then assign the expected code_owner to the alert or map the code_owner | |
to the correct channel */ -}} | |
{{ define "__get_channel_for_code_owner" -}} | |
{{- if eq . "platform-team" -}} | |
platform-alerts | |
{{- else if eq . "security-team" -}} | |
security-alerts | |
{{- else -}} | |
monitoring-no-owner | |
{{- end -}} | |
{{- end }} | |
{{- /* Select the channel based on the code_owner. We only expect to get | |
into this template function if the code_owners label is present on an alert. | |
This is to defend against us accidentally breaking the routing logic. */ -}} | |
{{ define "slack.monzo.code_owner_channel" -}} | |
{{- if .CommonLabels.code_owner }} | |
{{ template "__get_channel_for_code_owner" .CommonLabels.code_owner }} | |
{{- else -}} | |
monitoring | |
{{- end }} | |
{{- end }} | |
{{ define "slack.monzo.link_button_text" -}} | |
{{- if .CommonAnnotations.link_text -}} | |
{{- .CommonAnnotations.link_text -}} | |
{{- else -}} | |
Link | |
{{- end }} :link: | |
{{- end }} |
Addition for Grafana Alerting integration with AlertManager:
ℹ️ You have to set
alertmanger.externalUrl
to your Grafana instance external URL{{ define "__grafana_alert_silence_link" -}} {{ .ExternalURL }}/alerting/silence/new? {{- range .CommonLabels.SortedPairs -}} matcher={{- .Name }}%3D{{- .Value | urlquery -}}& {{- end -}} {{- end }}
Thanks @aorfanos
Have you also found a way to get the query button working towards Grafana? Alternatively, an external Prometheus?
Hello,
Thanks for sharing :)
In case you have built in Grafana Alerting/silence but also using Alertmanager as an additional Data source you have to append alertmanager=Alertmanager at the end of the url .
{{ define "__grafana_alert_silence_link" -}}
{{ .ExternalURL }}/alerting/silence/new?
{{- range .CommonLabels.SortedPairs -}}
matcher={{- .Name }}%3D{{- .Value | urlquery -}}&
{{- end -}}
alertmanager=Alertmanager
{{- end }}
P.S : tested on alertmanager 0.25 / grafana 9.1.6
Cheers
Hey everyone,
I want to share some findings as i tried to implement what is written here and got some insights that might help you:
-
name field for slack button:
if you dont specify a name for a slack button, you will only have 1 slack button as they need it to diffrenciate between the buttons in the list. here is where i found it in their documents: https://api.slack.com/legacy/interactive-message-field-guide -
if your button links to a url it must contain
http://
orhttps://
i had a button with:grafana.com
and it didn't work, only after i made ithttps://grafana.com
it worked. -
the advise by @80kk was crucial for the buttons, the
value: 'test'
field was mandatory for me to make it work.
so this is my configuration for working buttons:
- type: button
name: mutebutton
text: 'Mute :prometheus:'
value: 'text'
url: "https://{{ (index .Alerts 0).Labels.alertManagerUrl }}" # just a link for our internal alert manager, with added https://
- type: button
name: readmebutton
text: 'README :green_book:'
value: 'text'
url: "{{ (index .Alerts 0).Labels.readmeUrl }}" # just a link for our internal readme, it comes with https:// so no need to add
- type: button
name: grafana
text: 'Grafana :grafana:'
value: 'text'
url: 'https://{{ (index .Alerts 0).Labels.grafanaAlertUrl }}' # same as first button
and thanks everyone for all the comments here!
actions:
- type: button
text: 'README :green_book:'
url: '{{ (index .Alerts 0).Annotations.runbook_url }}'
- type: button
text: 'Query :mag:'
url: '{{ (index .Alerts 0).GeneratorURL }}'
- type: button
text: 'Dashboard :chart_with_upwards_trend:'
url: '{{ (index .Alerts 0).Annotations.dashboard_url }}'
- type: button
text: 'Silence :no_bell:'
url: '{{ template "__alert_silence_link" . }}'
EOF
I have my buttons set up like this and I can only get in Slack the "Runbook" one to be shown. Any idea why it's not showing the Query, Dashboard or Silence button as well?
I have tried the value: 'text' but it seems not to make a difference.
@stefaneacsu147 see my previous answer, you might need to add name:
field to each button.
Addition for Grafana Alerting integration with AlertManager:
ℹ️ You have to set
alertmanger.externalUrl
to your Grafana instance external URL