Riemann heartbeat with continuous alerting (with exponential backoff time)

heartbeat.clj

(defn- heartbeat-bounce
  [alpha min-ttl max-ttl & children]
  (fn [event]
    (let [ttl (max min-ttl (:ttl event))
          ttl (min max-ttl (* alpha ttl))]
      (call-rescue (assoc event :ttl ttl :time (unix-time)) children))))

(defn- heartbeat-process
  [svc desc index bounce & children]
  (fn [event]
    (let [state (if (or (expired? event) (not= (:state event) "ok"))
                  "crit"
                  "ok")
          desc  (if (= state "ok")
                  (format "%s: hearbeat resumed" desc)
                  (format "%s: hearbeat missing for %.02f seconds" desc (float (:ttl event))))]
      (let [event (-> event
                    (assoc :service svc :state state)
                    (dissoc :metric :description))]
        (if-not (= state "ok")
          (do
            (call-rescue (assoc event :description desc) children)
            (bounce event))
          (do
            (index event)))))))

(defn heartbeat
  [pattern svc desc index & [params & children]]
  (let [[params children] (if (map? params)
                            [params children]
                            [{} (apply vector params children)])
         alpha             (get params :alpha 2)
         min-ttl           (get params :min-ttl 60)
         max-ttl           (get params :max-ttl 3600)
         bounce            (heartbeat-bounce alpha min-ttl max-ttl index)
         process           (apply heartbeat-process svc desc index bounce children)]
    (sdo
      (where (and (expired? event) (service svc))
        process)
      (where (and (not (expired? event)) (service pattern))
        (default {:ttl min-ttl} process)))))