percent-alerts.clj

(streams
  (where (and metric (service "app1")(tagged "sign_in")
              ; We want to get alerted about failed sign ins. However we expect that there will be failures
              ; due to incorect passwords etc. So we only want to get alerted if more than 50% of the signins
              ; in a 60 second period are failures. The app tags failed signins with a warning state.
              (fixed-time-window 60
                                 (smap (fn [events]
                                         ; count all warnings and count all received events and work out the percentage
                                         (let [percent (/ (count (filter #(= (:state %) "warning") events))
                                                          (count events))]
                                           ; then send alerts based on the percentage, the else it just a debugging statement
                                           ; that we would probably remove
                                           (cond
                                             (> percent 0.7) (page-ops (format "sign_in is CRITICAL percent %f" (float percent)))
                                             (> percent 0.5) (mail-devs (format "sign_in is BAD percent %f" (float percent)))
                                             :else (prn (format "sign in is %f" (float percent)))
                                             )
                                           )))
                                 )))
  )