OCP user-workload monitoring and custom alerting

OCP user-workload monitoring and custom alerting.md

OCP user-workload monitoring and custom alerting

1. (Admin) Enable user workload monitoring and alerting

# Enable user workload monitoring (for user's namespace PrometheusRule)
oc -n openshift-monitoring get configmap cluster-monitoring-config -o=jsonpath='{.data.config\.yaml}' | yq w - 'enableUserWorkload' true > /tmp/config.yaml
oc set data  -n openshift-monitoring configmap cluster-monitoring-config --from-file=config.yaml=/tmp/config.yaml

# Enable user workload alerting (for user's namespace AlertmanagerConfig)
cat << EOF| oc apply -f -
apiVersion: v1
kind: ConfigMap
metadata:
  name: user-workload-monitoring-config
  namespace: openshift-user-workload-monitoring
data:
  config.yaml: |
    alertmanager:
      enabled: true
      enableAlertmanagerConfig: true
EOF

# Create user (test / example) namespace
cat << EOF| oc apply -f -
apiVersion: v1
kind: Namespace
metadata:
  name: ds-1
EOF

Refs:

• https://docs.openshift.com/container-platform/4.15/observability/monitoring/enabling-monitoring-for-user-defined-projects.html
• https://docs.openshift.com/container-platform/4.15/observability/monitoring/enabling-alert-routing-for-user-defined-projects.html

2. (Developer) Apply PrometheusRule (alert) and AlermanagerConfig (alert receiver - logger)

cat << EOF| oc apply -f -
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: my-alerts
  namespace: ds-1
  labels:
    role: alert-rules
spec:
  groups:
    - name: ExampleAlertGroup
      rules:
        - alert: ExampleAlert
          expr: "vector(1)"
          labels:
            myalert: "true"
---
apiVersion: monitoring.coreos.com/v1beta1
kind: AlertmanagerConfig
metadata:
  name: my-alerts
  namespace: ds-1
spec:
  route:
    groupBy: [ 'alertname']
    groupWait: 30s
    groupInterval: 1m
    repeatInterval: 2m
    receiver: 'webhook'
  receivers:
    - name: 'webhook'
      webhookConfigs:
        - url: 'http://alertmanager-webhook-logger.ds-1.svc'
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: alertmanager-webhook-logger
  name: alertmanager-webhook-logger
  namespace: ds-1
spec:
  replicas: 1
  selector:
    matchLabels:
      app: alertmanager-webhook-logger
  template:
    metadata:
      labels:
        app: alertmanager-webhook-logger
    spec:
      containers:
        - image: quay.io/pierdipi/alertmanager-webhook-logger-3b5723ee51a404f252f8a30d03820a50@sha256:4fb10b7cedbdb1cbb2c24e46d4236c384f808446c634b1fe8c11c1ea49f3dc4b
          name: alertmanager-webhook-logger
          resources:
            requests:
              cpu: 200m
              memory: 100M
            limits:
              cpu: 200m
              memory: 100M
          ports:
            - containerPort: 6725
              name: http
              protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
  name: alertmanager-webhook-logger
  namespace: ds-1
  labels:
    app: alertmanager-webhook-logger
spec:
  ports:
    - name: http
      port: 80
      protocol: TCP
      targetPort: 6725
  selector:
    app: alertmanager-webhook-logger
  type: ClusterIP
---
EOF

3. (Developer) Verification

oc logs -n ds-1 app=alertmanager-webhook-logger -f