Example how to do dynamic threshold in riemann

riemann-dynamic-threshold.rb

vagrant@precise64:~$ irb
1.9.3-p484 :002 > require "riemann"
 => true
# Set the threshold for the service "load" for "precise64" to 1
1.9.3-p484 :016 > Riemann::Client.new << {service: "dynamic-threshold-load", host: "precise64", metric: 1.0, ttl: Float::INFINITY}
# Set the threshold for the service "load" for "precise64" to 0
1.9.3-p484 :015 > Riemann::Client.new << {service: "dynamic-threshold-load", host: "precise64", metric: 0, ttl: Float::INFINITY}
# Set the threshold to 10
1.9.3-p484 :017 > Riemann::Client.new << {service: "dynamic-threshold-load", host: "precise64", metric: 10, ttl: Float::INFINITY}
# Clear the threshold 
1.9.3-p484 :014 > Riemann::Client.new << {service: "dynamic-threshold-load", host: "precise64", metric: nil, ttl: Float::INFINITY}

riemann.config

; vim: filetype=clojure
(logging/init :file "/var/log/riemann.log")
; everything is available on the public interface

; probably not the best idea for a production system
(tcp-server :host "0.0.0.0")
(udp-server :host "0.0.0.0")
(ws-server :host "0.0.0.0")
(repl-server :host "0.0.0.0")
(periodically-expire 10)

(defn dynamic-threshold [host service default]
  "What is the current threshold for this host/service?"
  (let [service (apply str ["dynamic-threshold-" service])]
    (or (->> (list 'and (list '= 'host host)
                        (list '= 'service service))
         ; Search the current Riemann core's index for any matching events
         (riemann.index/search (:index @core))
         ; Take the first match
         first
         ; Find its metric
         :metric)
     ; if not found, use its default value
     default)))

(let [client (tcp-client)
  ; Keep events for 5 minutes by default
  index (default :ttl 300 (update-index (index)))]
  (streams
    (where (service "load")
      (where (> metric (dynamic-threshold "precise64" "load" 0.5))
        #(info %)))
    index
))