Created
March 15, 2017 17:44
-
-
Save danielkza/6390154a16340f77bc8861a2762efdcb to your computer and use it in GitHub Desktop.
Riemann heartbeat with continuous alerting (with exponential backoff time)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
(defn- heartbeat-bounce | |
[alpha min-ttl max-ttl & children] | |
(fn [event] | |
(let [ttl (max min-ttl (:ttl event)) | |
ttl (min max-ttl (* alpha ttl))] | |
(call-rescue (assoc event :ttl ttl :time (unix-time)) children)))) | |
(defn- heartbeat-process | |
[svc desc index bounce & children] | |
(fn [event] | |
(let [state (if (or (expired? event) (not= (:state event) "ok")) | |
"crit" | |
"ok") | |
desc (if (= state "ok") | |
(format "%s: hearbeat resumed" desc) | |
(format "%s: hearbeat missing for %.02f seconds" desc (float (:ttl event))))] | |
(let [event (-> event | |
(assoc :service svc :state state) | |
(dissoc :metric :description))] | |
(if-not (= state "ok") | |
(do | |
(call-rescue (assoc event :description desc) children) | |
(bounce event)) | |
(do | |
(index event))))))) | |
(defn heartbeat | |
[pattern svc desc index & [params & children]] | |
(let [[params children] (if (map? params) | |
[params children] | |
[{} (apply vector params children)]) | |
alpha (get params :alpha 2) | |
min-ttl (get params :min-ttl 60) | |
max-ttl (get params :max-ttl 3600) | |
bounce (heartbeat-bounce alpha min-ttl max-ttl index) | |
process (apply heartbeat-process svc desc index bounce children)] | |
(sdo | |
(where (and (expired? event) (service svc)) | |
process) | |
(where (and (not (expired? event)) (service pattern)) | |
(default {:ttl min-ttl} process))))) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment