alert manager config example

alertmanager.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-alertmanager
  namespace: monitoring
data:
  alertmanager.yml: |-
    global:

    ##
    # tool to help visualize where your alerts are going
    # https://prometheus.io/webtools/alerting/routing-tree-editor/
    ##

    # The directory from which notification templates are read.
    templates:
    - '/etc/alertmanager/templates/*.tmpl'

    # The root route on which each incoming alert enters.
    route:
      # The labels by which incoming alerts are grouped together. For example,
      # multiple alerts coming in for cluster=A and alertname=LatencyHigh would
      # be batched into a single group.
      group_by: ['alertname', 'cluster', 'service']

      # When a new group of alerts is created by an incoming alert, wait at
      # least 'group_wait' to send the initial notification.
      # This way ensures that you get multiple alerts for the same group that start
      # firing shortly after another are batched together on the first
      # notification.
      group_wait: 30s

      # When the first notification was sent, wait 'group_interval' to send a batch
      # of new alerts that started firing for that group.
      group_interval: 5m

      # If an alert has successfully been sent, wait 'repeat_interval' to
      # resend them.
      repeat_interval: 3h

      # A default receiver
      receiver: slack_prometheus

      # All the above attributes are inherited by all child routes and can
      # overwritten on each.

      # The child route trees.
      routes:
      # This routes performs a regular expression match on alert labels to
      # catch alerts that are related to a list of services.
      # - match_re:
      #     service: ^(foo1|foo2|baz)$
      #   receiver: team-X-mails
      #   # The service has a sub-route for critical alerts, any alerts
      #   # that do not match, i.e. severity != critical, fall-back to the
      #   # parent node and are sent to 'team-X-mails'
      #   routes:
      #   - match:
      #       severity: critical
      #     receiver: team-X-pager
      - match:
          severity: critical
          receiver: critical_alert

      - match:
          severity: warning
          receiver: slack_prometheus

    # Inhibition rules allow to mute a set of alerts given that another alert is
    # firing.
    # We use this to mute any warning-level notifications if the same alert is
    # already critical.
    inhibit_rules:
    - source_match:
        severity: 'critical'
      target_match:
        severity: 'warning'
      # Apply inhibition if the alertname is the same.
      equal: ['cluster', 'service']


    receivers:

    # receivers for critical alerts. For example Sends to pager duty _and_ slack
    - name: critical_alert
      pagerduty_configs:
       - send_resolved:  true
         service_key: '<your service key>'

      slack_configs:
        - send_resolved: true
          api_url: '<your slack api url>'
          channel: '#<your channel name>'
          text: '{{ template "slack.default.text" . }}'
          title: "{{ range .Alerts }}{{ .Annotations.summary }}\n{{ end }}"


    # send to a slack channel. This is being used by the warning severity
    - name: 'slack_prometheus'
      slack_configs:
        - send_resolved: true
          api_url: '<your slack api url>'
          channel: '#<your channel name>'
          title: "{{ range .Alerts }}{{ .Annotations.summary }}\n{{ end }}"
          text: "{{ range .Alerts }}{{ .Annotations.description }}\n{{ end }}"