riemann flapping — II

riemman-alert.rb

require 'riemann/client'

DEFAULT = {:host => 'ch01', :metric => nil, :service => "serf", :ttl => 10}
CLIENT = Riemann::Client.new

def event(e)
  CLIENT << DEFAULT.merge(e).tap { |p| puts "#{Time.now} — #{e}" }
end

delay = (ARGV[0] || 2).to_i

event(:state => 'ok', :description => "ch01 is reachable")
sleep(delay)
event(:state => 'critical', :description => "ch01 is not reachable")
sleep(delay)
event(:state => 'ok', :description => "ch01 is reachable")

30.times.each do |n|
  sleep(delay)
  event(:state => 'ok', :description => "ch01 is reachable")
end

riemann.clj

(logging/init {:file "/var/log/riemann/riemann.log"})

(udp-server {:host "::"})

(periodically-expire 1)

(let [index (index)]

  (streams
    index
    (match :service #"^serf"
           #(info %)
           (changed-state {:init "ok"}
                          (stable 60 :state
                                  #(error %)))))
)

riemman-alert.log

bin/riemman-alert 5                                                                              
2015-12-24 02:58:35 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 02:58:40 +0000 — {:state=>"critical", :description=>"ch01 is not reachable"}
2015-12-24 02:58:45 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 02:58:50 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 02:58:55 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 02:59:00 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 02:59:05 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 02:59:10 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 02:59:15 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 02:59:20 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 02:59:25 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 02:59:30 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 02:59:35 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 02:59:40 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 02:59:45 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 02:59:50 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 02:59:55 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 03:00:00 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 03:00:05 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 03:00:10 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 03:00:15 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 03:00:20 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 03:00:25 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 03:00:30 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 03:00:35 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 03:00:40 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 03:00:45 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 03:00:50 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 03:00:55 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 03:01:00 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 03:01:05 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 03:01:10 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}
2015-12-24 03:01:15 +0000 — {:state=>"ok", :description=>"ch01 is reachable"}

riemman.log

INFO [2015-12-24 02:58:35,546] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450925915, :ttl 10.0}
INFO [2015-12-24 02:58:40,524] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state critical, :description ch01 is not reachable, :metric nil, :tags nil, :time 1450925920, :ttl 10.0}
INFO [2015-12-24 02:58:45,524] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450925925, :ttl 10.0}
INFO [2015-12-24 02:58:50,525] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450925930, :ttl 10.0}
INFO [2015-12-24 02:58:55,525] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450925935, :ttl 10.0}
INFO [2015-12-24 02:59:00,525] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450925940, :ttl 10.0}
INFO [2015-12-24 02:59:05,526] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450925945, :ttl 10.0}
INFO [2015-12-24 02:59:10,526] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450925950, :ttl 10.0}
INFO [2015-12-24 02:59:15,526] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450925955, :ttl 10.0}
INFO [2015-12-24 02:59:20,527] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450925960, :ttl 10.0}
INFO [2015-12-24 02:59:25,527] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450925965, :ttl 10.0}
INFO [2015-12-24 02:59:30,528] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450925970, :ttl 10.0}
INFO [2015-12-24 02:59:35,528] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450925975, :ttl 10.0}
INFO [2015-12-24 02:59:40,529] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450925980, :ttl 10.0}
ERROR [2015-12-24 02:59:45,002] riemann task 3 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450925925, :ttl 10.0}
INFO [2015-12-24 02:59:45,529] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450925985, :ttl 10.0}
INFO [2015-12-24 02:59:50,529] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450925990, :ttl 10.0}
INFO [2015-12-24 02:59:55,530] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450925995, :ttl 10.0}
INFO [2015-12-24 03:00:00,530] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450926000, :ttl 10.0}
INFO [2015-12-24 03:00:05,531] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450926005, :ttl 10.0}
INFO [2015-12-24 03:00:10,531] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450926010, :ttl 10.0}
INFO [2015-12-24 03:00:15,531] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450926015, :ttl 10.0}
INFO [2015-12-24 03:00:20,532] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450926020, :ttl 10.0}
INFO [2015-12-24 03:00:25,532] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450926025, :ttl 10.0}
INFO [2015-12-24 03:00:30,532] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450926030, :ttl 10.0}
INFO [2015-12-24 03:00:35,533] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450926035, :ttl 10.0}
INFO [2015-12-24 03:00:40,533] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450926040, :ttl 10.0}
INFO [2015-12-24 03:00:45,533] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450926045, :ttl 10.0}
INFO [2015-12-24 03:00:50,534] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450926050, :ttl 10.0}
INFO [2015-12-24 03:00:55,534] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450926055, :ttl 10.0}
INFO [2015-12-24 03:01:00,534] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450926060, :ttl 10.0}
INFO [2015-12-24 03:01:05,535] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450926065, :ttl 10.0}
INFO [2015-12-24 03:01:10,536] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450926070, :ttl 10.0}
INFO [2015-12-24 03:01:15,535] defaultEventExecutorGroup-2-1 - riemann.config - #riemann.codec.Event{:host ch01, :service serf, :state ok, :description ch01 is reachable, :metric nil, :tags nil, :time 1450926075, :ttl 10.0}
INFO [2015-12-24 03:01:25,041] Thread-7 - riemann.config - {:time 362731521259/250, :state expired, :service serf, :host ch01}
ERROR [2015-12-24 03:02:25,036] riemann task 2 - riemann.config - {:time 362731521259/250, :state expired, :service serf, :host ch01}