Skip to content

Instantly share code, notes, and snippets.

@hanynowsky
Created March 3, 2016 10:19
Show Gist options
  • Save hanynowsky/9c700af0913d96ee4c99 to your computer and use it in GitHub Desktop.
Save hanynowsky/9c700af0913d96ee4c99 to your computer and use it in GitHub Desktop.
# Sends events to Flapjack for notification routing. See http://flapjack.io/
#
# This extension requires Flapjack >= 0.8.7 and Sensu >= 0.13.1
#
# In order for Flapjack to keep its entities up to date, it is necssary to set
# metric to "true" for each check that is using the flapjack handler extension.
#
# Here is an example of what the Sensu configuration for flapjack should
# look like, assuming your Flapjack's redis service is running on the
# same machine as the Sensu server:
#
# {
# "flapjack": {
# "host": "localhost",
# "port": 6380,
# "db": "0"
# }
# }
#
# Copyright 2014 Jive Software and contributors.
#
# Released under the same terms as Sensu (the MIT license); see LICENSE for details.
require 'net/http'
require 'uri'
require 'json'
require 'sensu-plugin/utils'
require 'mixlib/cli'
require 'sensu/redis'
module Sensu
module Extension
class Flapjack < Handler
def name
'flapjack'
end
def description
'sends sensu events to the flapjack redis queue'
end
def options
return @options if @options
@options = {
host: '127.0.0.1',
port: 6379,
channel: 'events',
db: 0
}
if @settings[:flapjack].is_a?(Hash)
@options.merge!(@settings[:flapjack])
end
@options
end
def definition
{
type: 'extension',
name: name,
mutator: 'ruby_hash'
}
end
def post_init
@redis = Sensu::Redis.connect(options)
@redis.on_error do |_error|
@logger.warn('Flapjack Redis instance not available on ' + options[:host])
end
end
### CUSTOM
# Log something and return false.
def bail(msg, event)
@logger.info("Flapjack handler: #{msg} : #{event[:client][:name]}/#{event[:check][:name]}")
false
end
# Lifted from the sensu-plugin gem, makes an api request to sensu
def api_request(method, path, &blk)
http = Net::HTTP.new(@settings['api']['host'], @settings['api']['port'])
req = net_http_req_class(method).new(path)
if @settings['api']['user'] && @settings['api']['password']
req.basic_auth(@settings['api']['user'], @settings['api']['password'])
end
yield(req) if block_given?
http.request(req)
end
# Lifted from sensu-plugin for REST
def net_http_req_class(method)
case method.to_s.upcase
when 'GET'
Net::HTTP::Get
when 'POST'
Net::HTTP::Post
when 'DELETE'
Net::HTTP::Delete
when 'PUT'
Net::HTTP::Put
end
end
# Check stashes
def stash_exists?(path)
api_request(:GET, '/stash' + path).code == '200'
end
# Check whether a given client's check event exists
def event_exists?(client, check)
api_request(:GET, '/event/' + client + '/' + check).code == '200'
end
# Has this check been disabled from handlers?
def filter_disabled(event)
if event[:check].key?(:alert)
if event[:check][:alert] == false
bail 'alert disabled', event
end
end
true
end
# Don't spam flapjack too much!
@not_occured = "_no?_"
@occured = "_o?_"
def filter_repeated(event)
defaults = {
'occurrences' => 1,
'interval' => 60,
'refresh' => 1800
}
occurrences = event[:check][:occurrences] || defaults['occurrences']
interval = event[:check][:interval] || defaults['interval']
refresh = event[:check][:refresh] || defaults['refresh']
if event[:occurrences].to_i < occurrences.to_i
@not_occured = "Not enough occurrences"
return bail 'not enough occurrences', event
end
if event[:occurrences].to_i > occurrences.to_i && event[:action] == :create
number = refresh.fdiv(interval).to_i
occudiff = "#{event[:occurrences]} VS #{occurrences}"
@occured = "Event occurrences > Check occurrences #{occudiff}"
modulo = (event[:occurrences] - occurrences) % number
unless number == 0 || modulo == 0
@occured = 'only handling every ' + number.to_s + ' occurrences' + " : #{occudiff}" + " MODULO: #{modulo}"
return bail 'only handling every ' + number.to_s + ' occurrences', event
else
@occured = "Event occurrences > Check occurrences MODULO HANDLE ON: #{occudiff} MODULO: #{modulo}"
end
else
@occured = "Event occurrences < Check occurrences OR action not CREATE #{event[:action]} - #{occudiff} - MODULO: #{modulo}"
end
true
end
# Has the event been silenced through the API?
def filter_silenced(event)
stashes = [
['client', '/silence/' + event[:client][:name]],
['check', '/silence/' + event[:client][:name] + '/' + event[:check][:name]],
['check', '/silence/all/' + event[:check][:name]]
]
stashes.each do |(scope, path)|
begin
timeout(2) do
if stash_exists?(path)
return bail scope + ' alerts silenced', event
end
end
rescue Timeout::Error
@logger.warn('timed out while attempting to query the sensu api for a stash')
end
end
true
end
# Does this event have dependencies?
def filter_dependencies(event)
if event[:check].has_key?(:dependencies)
if event[:check][:dependencies].is_a?(Array)
event[:check][:dependencies].each do |dependency|
begin
timeout(2) do
check, client = dependency.split('/').reverse
if event_exists?(client || event[:client][:name], check)
return bail 'check dependency event exists', event
end
end
rescue Timeout::Error
@logger.warn('timed out while attempting to query the sensu api for an event')
end
end
end
end
true
end
# Run all the filters in some order. Only run the handler if they all return true
def filters(event_data)
return false unless filter_repeated(event_data)
return false unless filter_silenced(event_data)
return false unless filter_dependencies(event_data)
return false unless filter_disabled(event_data)
@logger.info("#{event_data[:client][:name]}/#{event_data[:check][:name]} not being filtered!")
true
end
def run(event)
client = event[:client]
check = event[:check]
tags = []
tags.concat(client[:tags]) if client[:tags].is_a?(Array)
tags.concat(check[:tags]) if check[:tags].is_a?(Array)
tags << client[:environment] unless client[:environment].nil?
# #YELLOW
unless check[:subscribers].nil? || check[:subscribers].empty? # rubocop:disable UnlessElse
tags.concat(client[:subscriptions] - (client[:subscriptions] - check[:subscribers]))
else
tags.concat(client[:subscriptions])
end
details = ['Address:' + client[:address]]
details << 'Tags:' + tags.join(',')
details << "Raw Output: #{check[:output]}" if check[:notification]
flapjack_event = {
entity: client[:name],
check: check[:name],
type: 'service',
state: Sensu::SEVERITIES[check[:status]] || 'unknown',
summary: check[:notification] || check[:output],
details: details.join(' '),
time: check[:executed],
tags: tags
}
# CUSTOM TEST
dependencies = check[:dependencies]
has_dep = false
if dependencies.is_a?(Array)
if dependencies.size > 0
has_dep = true
end
end
apihost = @settings['api']['host'] || 'NO_HOST_DEFINED'
# CUSTOM TEST END
resolved = event[:action].eql?(:resolve)
fired = [1, 2, 3].include?(event[:check][:status])
if (filter_repeated(event) == false or filter_silenced(event) == false or filter_dependencies(event) == false or filter_disabled(event) == false ) && !resolved
yield "FLAPJACK Skipping Flapjack event delivery. FILTERS: #{filter_repeated(event)} #{filter_silenced(event)} #{filter_dependencies(event)} #{filter_disabled(event)} - STATUS: #{event[:check][:status]} Occurrence: #{event[:occurrences]} vs #{event[:check][:occurrences]} - #{client[:name]} #{check[:name]} - ACTION: #{event[:action]} - RESOLUTION: #{resolved} - OCC #{@occured} vs NOT_OCC #{@not_occured}", 0
else
@redis.lpush(options[:channel], MultiJson.dump(flapjack_event))
yield "FLAPJACK filters: #{filter_repeated(event)} #{filter_silenced(event)} #{filter_dependencies(event)} #{filter_disabled(event)} #{client[:name]} : #{check[:name]} . Occurrence: #{event[:occurrences]} vs #{event[:check][:occurrences]} : STATUS: #{event[:check][:status]} - ACTION: #{event[:action]} - RESOLUTIOn: #{resolved} - OCC #{@occured} vs NOT_OCC #{@not_occured} : sent an event to the flapjack redis queue", 0
end
end
# Called when Sensu begins to shutdown.
def stop
true
end
end
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment