block15.rule

job:stashdef_lag:seconds =
  (time() - max(stashdef_last_received_timestamp_seconds)) + max(stashdef_lag_seconds)

ALERT StashDeferredLagHigh
  IF job:stashdef_lag:seconds > 5 * 60
  FOR 2m
  LABELS {
    slack_channel="stash-deferred"
  }
  ANNOTATIONS {
    description="Stash deferred messages are arriving {{ $value }} seconds after they were scheduled (threshold 5m)"
  }