prom-reload.sh

prom-reload.sh

#!/bin/bash
VOLUME_DIRECTORY="/etc/prometheus/rules/changeme"
for arg in "$@"
do
    case $arg in
        -VD=*|--volume-dir=*)
        VOLUME_DIRECTORY="${arg#*=}"
        shift
        ;;
    esac
done
inotifywait -e delete -m -r $VOLUME_DIRECTORY |
    while read path action file; do
        # second delete check is to ignore events that look like DELETE,ISDIR .. we want file deletes only
        if [[ $action = "DELETE" ]]; then
          pkill -HUP prometheus
          echo "$(date) - Reloaded prometheus because the $action event occurred on file: [$path $file]  "
        fi
    done

prom.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: prometheus
  name: prometheus
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus
  template:
    metadata:
      labels:
        app: prometheus
    spec:
      shareProcessNamespace: true
      containers:
      - name: reloader
        args:
        - --volume-dir=/etc/alerts/
        volumeMounts:
        - name: alert-volume
          mountPath: /etc/alerts
        image: docker.io/scott2449/prom-reload:v14
      - name: prometheus
        args:
        - --storage.tsdb.retention=6h
        - --config.file=/etc/prometheus/prometheus.yml
        - --storage.tsdb.path=/data/
        # commenting this out to stop the api (also allows for reload from thanos)
        #- --web.enable-lifecycle
        - --storage.tsdb.no-lockfile
        - --storage.tsdb.min-block-duration=2h
        - --storage.tsdb.max-block-duration=2h
        - --query.max-concurrency=10
        image: docker.io/prom/prometheus:v2.15.1
        livenessProbe:
          httpGet:
            path: /-/healthy
            port: 9090
        ports:
        - containerPort: 9090
          name: http
        readinessProbe:
          httpGet:
            path: /-/ready
            port: 9090
        volumeMounts:
        - mountPath: /data/
          name: config-shared
        - name: alert-volume
          mountPath: /etc/alerts
        - mountPath: /etc/prometheus
          name: config-volume
      volumes:
      - name: config-shared
        emptyDir: {}
      - name: alert-volume
        configMap:
          name: alerts
      - configMap:
          name: prometheus
        name: config-volume

Because of how symlinks and config map updates work, delete is the best event to use here that doesn't cause loops or missed events with the absolute smallest amount of code. I tried like 30 different versions of events and inotifywait flags analyzing what precisely k8s is doing when reloading the config map.