Skip to content

Instantly share code, notes, and snippets.

@steveberryman
Created October 1, 2014 18:11
Show Gist options
  • Save steveberryman/c57dd6595429e86d6a12 to your computer and use it in GitHub Desktop.
Save steveberryman/c57dd6595429e86d6a12 to your computer and use it in GitHub Desktop.
sensu hipchat extension
#/usr/bin/env ruby
# Sends events to hipchat for wonderful chatty notifications
#
# This extension requires the hipchat gem
#
# The reason I wrote this instead of using the normal hipchat handler, is that with Flapjack
# all events are handled unless you do crazy filtering stuff. Also with a large number of events
# and checks the sensu server can get overloaded with forking stuff. So anyway, hipchat extension :)
#
# Here is an example of what the Sensu configuration for hipchat should
# look like. It's fairly configurable:
#
# {
# "hipchat": {
# "apiversion": "{hipchat_apiversion}",
# "apikey": "{hipchat_apikey}",
# "room": "{hipchat_room}",
# "from": "{hipchat_from}",
# "keepalive": {
# "room": "{hipchat_keepalive_room}",
# "occurrences": {hipchat_keepalive_occurrences}
# }
# }
# }
#
# The first four variables should be fairly self-explanatory.
# The 'keepalive' block is for keepalive check settings, in case you want to have keepalive alerts
# going to a different room, repeating at different intervals. This could probably be done better.
#
# Checks can also define a hipchat room, and other options.
# Options that can be passed in event data are as follows:
# "hipchat_room" => Room to send hipchat events for this check to
# "hipchat_from" => Name to send message from (defaults to Sensu)
# "hipchat_notify" => Turn on/off the hipchat 'notify' option (defines if the room goes red on new message)
# - Defaults to 'true' for OK, Critical, Warning and 'false' for Unknown
# "playbook" => URL or HTML for playbook for that check
#
# Copyright 2014 Steve Berryman and contributors.
#
# Released under the same terms as Sensu (the MIT license); see LICENSE for details.
require 'rubygems' if RUBY_VERSION < '1.9.0'
require 'hipchat'
require 'timeout'
require 'net/http'
require 'slack-notifier'
module Sensu::Extension
class Hipchat < Handler
# The post_init hook is called after the main event loop has started
# At this time EventMachine is available for interaction.
def post_init
end
# Must at a minimum define type and name. These are used by
# Sensu during extension initialization.
def definition
{
type: 'extension', # Always.
name: 'hipchat', # Usually just class name lower case.
mutator: 'ruby_hash'
}
end
# Simple accessor for the extension's name. You could put a different
# value here that is slightly more descriptive, or you could simply
# refer to the definition name.
def name
definition[:name]
end
# A simple, brief description of the extension's function.
def description
'Hipchat extension. Because otherwise the sensu server will forking die.'
end
# Sends an event to the specified hipchat room etc
def send_hipchat(room, from, message, color, notify)
# I've found that the v2 api basically doesn't work YMMV
apiversion = @settings["hipchat"]["apiversion"] || 'v1'
apikey = @settings["hipchat"]["apikey"]
hipchatmsg = HipChat::Client.new(apikey, :api_version => apiversion)
begin
timeout(3) do
hipchatmsg[room].send(from, "#{message}.", :color => color, :notify => notify)
return "Sent hipchat message"
end
rescue Timeout::Error
return "Timed out while attempting to message #{room} [#{message}]"
rescue HipChat::UnknownResponseCode
return "Hipchat returned an unknown response code (rate limited?)"
end
end
# Sends an event to the specified slack room etc
def send_slack(message)
slack_token = @settings["hipchat"]["slack_token"]
slack_org = @settings["hipchat"]["slack_team"]
slack_client = Slack::Notifier.new(slack_org, slack_token)
begin
timeout(3) do
slack_client.ping("Sensu Alert", message)
return "Sent slack message"
end
rescue Timeout::Error
return "Timed out while attempting to message"
rescue
return "Exception occurred sending slack message"
end
end
# Log something and return false.
def bail(msg, event)
@logger.info("Hipchat handler: " + msg + ': ' + event[:client][:name] + '/' + event[:check][:name])
false
end
# Lifted from the sensu-plugin gem, makes an api request to sensu
def api_request(method, path, &blk)
http = Net::HTTP.new(@settings['api']['host'], @settings['api']['port'])
req = net_http_req_class(method).new(path)
if @settings['api']['user'] && @settings['api']['password']
req.basic_auth(@settings['api']['user'], @settings['api']['password'])
end
yield(req) if block_given?
http.request(req)
end
# also lifted from the sensu-plugin gem. In fact, most of the rest was.
def net_http_req_class(method)
case method.to_s.upcase
when 'GET'
Net::HTTP::Get
when 'POST'
Net::HTTP::Post
when 'DELETE'
Net::HTTP::Delete
when 'PUT'
Net::HTTP::Put
end
end
def stash_exists?(path)
api_request(:GET, '/stash' + path).code == '200'
end
def event_exists?(client, check)
api_request(:GET, '/event/' + client + '/' + check).code == '200'
end
# Has this check been disabled from handlers?
def filter_disabled(event)
if event[:check].has_key?(:handler)
if event[:check][:handler] == false
bail 'alert disabled', event
end
end
return true
end
# Don't spam hipchat too much!
def filter_repeated(event)
defaults = {
'occurrences' => 1,
'interval' => 60,
'refresh' => 1800
}
occurrences = event[:check][:occurrences] || defaults['occurrences']
interval = event[:check][:interval] || defaults['interval']
refresh = event[:check][:refresh] || defaults['refresh']
if event[:occurrences] < occurrences
return bail 'not enough occurrences', event
end
if event[:occurrences] > occurrences && event[:action] == :create
number = refresh.fdiv(interval).to_i
unless number == 0 || event[:occurrences] % number == 0
return bail 'only handling every ' + number.to_s + ' occurrences', event
end
end
return true
end
# Has the event been silenced through the API?
def filter_silenced(event)
stashes = [
['client', '/silence/' + event[:client][:name]],
['check', '/silence/' + event[:client][:name] + '/' + event[:check][:name]],
['check', '/silence/all/' + event[:check][:name]]
]
stashes.each do |(scope, path)|
begin
timeout(2) do
if stash_exists?(path)
return bail scope + ' alerts silenced', event
end
end
rescue Timeout::Error
@logger.warn('timed out while attempting to query the sensu api for a stash')
end
end
true
end
# Does this event have dependencies?
def filter_dependencies(event)
if event[:check].has_key?(:dependencies)
if event[:check][:dependencies].is_a?(Array)
event[:check][:dependencies].each do |dependency|
begin
timeout(2) do
check, client = dependency.split('/').reverse
if event_exists?(client || event[:client][:name], check)
return bail 'check dependency event exists', event
end
end
rescue Timeout::Error
@logger.warn( 'timed out while attempting to query the sensu api for an event')
end
end
end
end
return true
end
# Run all the filters in some order. Only run the handler if they all return true
def filters(event_data)
return false unless filter_repeated(event_data)
return false unless filter_silenced(event_data)
return false unless filter_dependencies(event_data)
return false unless filter_disabled(event_data)
@logger.info("#{event_data[:client][:name]}/#{event_data[:check][:name]} not being filtered!")
return true
end
def color_to_hex(color)
if color == "green"
return "#09B524"
elsif color == "red"
return "#E31717"
elsif color == "yellow"
return "#FFFF00"
else
return "#FFFFFF"
end
end
def build_link(type, url, text)
if type == :slack
return "<#{url}|#{text}>"
else
return "<a href='#{url}'>#{text}</a>"
end
end
def build_playbook(check, type)
# If the playbook attribute exists and is a URL, "[<a href='url'>playbook</a>]" will be output.
# To control the link name, set the playbook value to the HTML output you would like.
if check[:playbook]
begin
uri = URI.parse(check[:playbook])
if %w( http https ).include?(uri.scheme)
return "[#{build_link(type, check[:playbook], "Playbook")}]"
else
return "Playbook: #{check[:playbook]}"
end
rescue
return "Playbook: #{check[:playbook]}"
end
else
return nil
end
end
def build_hipchat_message(event, state_msg, status_msg)
check = event[:check]
client_name = check[:source] || event[:client][:name]
check_name = check[:name]
output = check[:notification] || check[:output]
playbook = build_playbook(check, :hipchat)
return "#{status_msg} #{client_name}/#{check_name} - #{state_msg}: #{output} #{playbook}"
end
def build_slack_message(event, room, from, color, state_msg, status_msg)
check = event[:check]
client_name = check[:source] || event[:client][:name]
check_name = check[:name]
output = check[:notification] || check[:output]
duration = check[:duration] || Time.at(check[:occurrences] * check[:interval]).utc.strftime("%Hh %Mm %Ss") || "Unknown"
playbook = build_playbook(check, :slack)
snoopy_link = build_link(:slack, "https://internal-tool-thing", "Snoopy")
dash_link = build_link(:slack, "http://sensu-dashboard.wherever.com:3000/#/client/Sensu/#{client_name}", "Sensu")
if room == "compose"
room = "#general"
else
room = "#" + room.downcase.gsub(/\s+/, "-")
end
fields = [
{
"title" => "Output",
"value" => output,
"short" => false
}
]
playbook = build_playbook(event[:check], :slack)
if !playbook.nil?
fields << {
"title" => "Playbook",
"value" => playbook,
"short" => false
}
end
return {
"channel" => room,
"username" => from,
"icon_url" => "https://files.slack.com/files-pri/T024FNVQB-F02G53U5L/sensu_small.png",
"attachments" => [
{
"mrkdwn_in" => ["text", "title", "fallback"],
"fallback" => "Host: #{client_name}, Check: #{check_name}\nOutput: #{output}\nAlerting for: #{duration} | [ #{snoopy_link} ] [ #{dash_link} ]",
"color" => color_to_hex(color),
"text" => "*Host*: #{client_name}, *Check*: #{check_name}\n*Alerting for:* #{duration} | [ #{snoopy_link} ] [ #{dash_link} ]",
"fields" => fields
}
]
}
end
def clarify_state(state, check)
if state == 0
state_msg = 'OK'
color = 'green'
notify = check[:hipchat_notify] || true
elsif state == 1
state_msg = 'WARNING'
color = 'yellow'
notify = check[:hipchat_notify] || true
elsif state == 2
state_msg = 'CRITICAL'
color = 'red'
notify = check[:hipchat_notify] || true
else
state_msg = 'UNKNOWN'
color = 'gray'
notify = check[:hipchat_notify] || false
end
return state_msg, color, notify
end
# run() is passed a copy of the event_data hash
def run(event_data)
event = event_data
# Is this event a resolution?
resolved = true if event[:action].eql?("resolve")
# Is this event a keepalive?
# Adding extra config on every client is annoying. Just make some extra settings for it.
keepalive = @settings["hipchat"]["keepalive"] || {}
if event[:check][:name] == "keepalive"
event[:check][:occurrences] = keepalive["occurrences"] || 6
event[:check][:hipchat_room] = keepalive["room"] || @settings["hipchat"]["room"]
event[:check][:hipchat_from] = keepalive["from"] || @settings["hipchat"]["from"] || 'Sensu'
end
# If this event is resolved, or in one of the 'bad' states, and it passes all the filters,
# send the message to hipchat
if (resolved or [1,2,3].include?(event[:check][:status])) && filters(event)
check = event[:check]
room = check[:hipchat_room] || @settings["hipchat"]["room"]
from = check[:hipchat_from] || @settings["hipchat"]["from"] || 'Sensu'
state = check[:status]
state_msg, color, notify = clarify_state(state, check)
status_msg = resolved ? "[RESOLVED]" : "[PROBLEM]"
hipchat_msg = build_hipchat_message(event, state_msg, status_msg)
slack_msg = build_slack_message(event, room, from, color, state_msg, status_msg)
operation = proc {
send_hipchat(room, from, hipchat_msg, color, notify)
send_slack(slack_msg)
}
callback = proc {|result|
yield "Hipchat message: #{result}", 0
}
EM::defer(operation, callback)
else
yield "", 0
end
end
# Called when Sensu begins to shutdown.
def stop
true
end
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment