common.clj

(defn moving-trigger-fraction-trigger
  [time-window trigger-predicate cancel-predicate & children]
  (let [state-ok (atom true)
        s (fn [event]
              (reset! state-ok (= "ok" (:state event)))
              (call-rescue event children))]
    (moving-time-window time-window
      (fn [events]
        (prn events)
        ((split
          (trigger-predicate events)
            (smap peek (with :state "critical" s))
          (and (not @state-ok) #(cancel-predicate events))
            (smap peek (with :state "critical" s))
          (smap peek (with :state "ok" s))) events)))))

output

[#riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 1, :ttl nil}]
"child: " #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 1, :ttl nil}
[#riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 1, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 2, :ttl nil}]
"child: " #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 2, :ttl nil}
[#riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 1, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 2, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 3, :ttl nil}]
"child: " #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 3, :ttl nil}
[#riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 1, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 2, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 3, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 4, :ttl nil}]
"child: " #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 4, :ttl nil}
[#riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 1, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 2, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 3, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 4, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 200, :ttl nil}]
"child: " #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 200, :ttl nil}
[#riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 1, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 2, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 3, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 4, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 200, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 201, :ttl nil}]
"child: " #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 201, :ttl nil}
[#riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 200, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 201, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 310, :ttl nil}]
"child: " #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 310, :ttl nil}
[#riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 200, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 201, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 310, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 310, :ttl nil}]
"child: " #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 310, :ttl nil}
[#riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 200, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 201, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 310, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 310, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 432, :ttl nil}]
"child: " #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 432, :ttl nil}
[#riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 310, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 310, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 432, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 532, :ttl nil}]
"child: " #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 532, :ttl nil}
[#riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 532, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 765, :ttl nil}]
"child: " #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 765, :ttl nil}
[#riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 532, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 765, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 800, :ttl nil}]
"child: " #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 800, :ttl nil}
[#riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 765, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 800, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 910, :ttl nil}]
"child: " #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 910, :ttl nil}
[#riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 765, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 800, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 910, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 910, :ttl nil}]
"child: " #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 910, :ttl nil}
[#riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 765, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 800, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 910, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 910, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 970, :ttl nil}]
"child: " #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 970, :ttl nil}
[#riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 765, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 800, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 910, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 910, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 970, :ttl nil} #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "ok", :description nil, :metric nil, :tags ["class::ciamon"], :time 999, :ttl nil}]
"child: " #riemann.codec.Event{:host "somehost", :service "git-clone-status/repo", :state "critical", :description nil, :metric nil, :tags ["class::ciamon"], :time 999, :ttl nil}

rule.clj

(where (service #"^git-clone-status/.*")
        ;; Raise an alarm if during the last 300s,
        ;; greater than 2/3 has state "critical"
        ;; Will not split by host.
        (by [:service]
          (moving-trigger-fraction-trigger 300
            (fn [events]
              (let [fraction (/ (count (filter #(= (:state %) "critical") events)) (count events))]
                (>= fraction 75/100)))
            (fn [events]
              (let [fraction (/ (count (filter #(= (:state %) "ok") events)) (count events))]
                (>= fraction 75/100)))
            alert-stream)))

test.clj

(deftest test-gitclone
  (testing ""
    (send-events
      {:end-time 1000}
      [(make-event 1 "ok")
       (make-event 2 "critical")
       (make-event 3 "critical")
       (make-event 4 "ok")
       (make-event 200 "critical")
       (make-event 201 "critical")
       (make-event 310 "critical")
       (make-event 310 "critical")
       (make-event 432 "ok")
       (make-event 532 "ok")
       (make-event 765 "ok")
       (make-event 800 "ok")
       (make-event 910 "critical")
       (make-event 910 "ok")
       (make-event 970 "critical")
       (make-event 999 "ok")
       ]
      )
    (check-externals
      ;:pagerduty { :event { :state "critical"}}
      ;:pagerduty { :event { :state "ok"}}
      )))