riemann collectd watchdog

mcorbin_config.clj

(streams
  (with {:service "heartbeat" :ttl 120 } (index))
  (where (service "heartbeat")
         (by :host (...))))

riemann-config.clj

  (streams (default {:state "ok"}
    index
    (by :host
      (where (service "heartbeat")
        (changed expired?
          (where (expired? event)
            (with {:state "critical"}
              alert!)
            (else alert!))))
      (where (and (tagged "collectd") (not (service "heartbeat")) (not (expired? event)))
        (with {:service "heartbeat" :ttl heartbeat-timeout}
          (throttle 1 (/ heartbeat-timeout 2)
            reinject)))))))