Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
Stashboard <-> Pingdom Integration
#!/usr/bin/env ruby
require 'time'
require 'logger'
require 'rubygems'
require 'pingdom-client'
require 'active_support/core_ext/numeric/time' # time extensions, e.g., 5.days
require 'stashboard'
TIME_FORMAT = "%Y-%m-%d %H:%M:%S %Z" # YYYY-MM-DD HH:MM:SS ZZZ
logger = Logger.new(STDOUT)
logger.level = Logger::WARN
pingdom_auth = {
:username => 'email@example.com',
:password => 'xxxxxxx',
:key => '1234567890123456789012345678901234567890'
}
stashboard_auth = {
:url => 'https://yourstashboard.appspot.com',
:oauth_token => 'AaBbCcDdEeFfgG0102030405060708',
:oauth_secret => 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
}
# Stashboard service id => Regex matching pingdom check name(s)
services = {
'foo' => /foo/i,
'bar' => /bar/i,
'quux' => /moopie/i,
}
stashboard = Stashboard::Stashboard.new(
stashboard_auth[:url],
stashboard_auth[:oauth_token],
stashboard_auth[:oauth_secret]
)
pingdom = Pingdom::Client.new pingdom_auth.merge(:logger => logger)
stashboard.service_ids.each do |service|
puts "WARNING: Missing mapping for stashboard service '#{service}'" unless services[service]
end
up_services = services
warning_services = {}
# Synchronize recent pingdom outages over to stashboard and determine which services
# are currently up.
pingdom.checks.each do |check|
service = services.keys.find do |service|
regex = services[service]
check.name =~ regex
end
unless service
next
end
puts "Check: #{check.name} => #{check.status} [#{service}]"
yesterday = Time.now - 1.day
recent_outages = check.summary.outages.select do |outage|
outage.timefrom > yesterday || outage.timeto > yesterday
end
recent_events = stashboard.events(service, "start" => yesterday.strftime("%Y-%m-%d"))
recent_outages.each do |outage|
msg = "Service #{check.name} unavailable: " +
"#{outage.timefrom.strftime(TIME_FORMAT)} - #{outage.timeto.strftime(TIME_FORMAT)}"
unless recent_events.any? { |event| event["message"] == msg }
puts "New outage: " + msg
stashboard.create_event(service, "down", msg)
end
end
unless recent_outages.empty?
up_services.delete(service)
warning_services[service] = true
end
# if any pingdom check fails for a given service, consider the service down.
if check.status == "down"
up_services.delete(service)
end
end
# Mark any up/warning services as such in stashboard
#
# Note: Custom stashboard status messages *will not* be changed unless
# they match the regex /Service .* unavailable/i or there is a new outage
# reported by Pingdom. This is intentional to allow overriding "up" automated
# updates.
puts "Up services: #{up_services.inspect}"
up_services.each_key do |service|
current = stashboard.current_event(service)
if current["message"] =~ /(Service .* unavailable)|(Service operational but has experienced outage)/i
puts "Service now operating normally: #{service}"
stashboard.create_event(service, "up", "Service operating normally.")
end
end
puts "Warning services: #{warning_services.inspect}"
warning_services.each_key do |service|
current = stashboard.current_event(service)
if current["message"] =~ /Service .* unavailable/i
puts "Service experienced outages in past 24 hours: #{service}"
stashboard.create_event(service, "warning", "Service operational but has experienced outage(s) in past 24 hours.")
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment