mycheck.json

{
  "checks": {
    "mycheck": {
      "pagerduty": {
        "service": "oncall"
      },
      "sendgrid": {
        "mail_to": "ops@myemail.com"
      },
      "runbook": "https://wiki.url.to.runbook",
      "subscribers": [
        "core"
      ],
      "interval": 60,
      "standalone": true,
      "command": "/usr/lib/nagios/plugins/check_http -e 200 -H localhost -p 8080",
      "handlers": [
        "pagerduty",
        "sendgrid"
      ]
    }
  }
}

pagerduty.json

{
  "pagerduty": {
    "keys" : {
      "default": "apikey1",
      "data": "apikey2",
      "hbase": "apikey3",
      "oncall": "apikey4"
    }
  },
  "handlers": {
    "pagerduty": {
      "severities": [
        "ok",
        "critical"
      ],
      "command": "/etc/sensu/handlers/pagerduty.rb",
      "type": "pipe"
    }
  }
}

pagerduty.rb

#!/usr/bin/env ruby
#
# This handler creates and resolves PagerDuty incidents, refreshing
# stale incident details every 30 minutes
#
# Copyright 2011 Sonian, Inc <chefs@sonian.net>
#
# Released under the same terms as Sensu (the MIT license); see LICENSE
# for details.

require 'rubygems' if RUBY_VERSION < '1.9.0'
require 'sensu-handler'
require 'redphone/pagerduty'

class Pagerduty < Sensu::Handler

  def api_key
    api_key_list = settings['pagerduty']['keys']
    # Set a default key if no service specified
    default_key = settings['pagerduty']['keys']['default']
    # If PagerDuty key given, then see if it's in the list.
    # If it's in the list, then use it to act on the event.
    begin
      service = @event['check']['pagerduty']['service']
      if settings['pagerduty']['keys'].has_key?(service)
        api_key = settings['pagerduty']['keys'][service]
      else
        api_key = default_key
      end
    rescue
      api_key = default_key
    end
    return api_key
  end

  def incident_key
    @event['client']['name'] + '/' + @event['check']['name']
  end

  def handle
    description = @event['notification'] || [@event['client']['name'], @event['check']['name'], @event['check']['output']].join(' : ')
    service_key = api_key
    begin
      timeout(3) do
        response = case @event['action']
                   when 'create'
                     Redphone::Pagerduty.trigger_incident(
                                                          :service_key => service_key,
                                                          :incident_key => incident_key,
                                                          :description => description,
                                                          :details => @event
                                                          )
                   when 'resolve'
                     Redphone::Pagerduty.resolve_incident(
                                                          :service_key => service_key,
                                                          :incident_key => incident_key
                                                          )
                   end
        if response['status'] == 'success'
          puts 'pagerduty -- ' + @event['action'].capitalize + 'd incident -- ' + incident_key
        else
          puts 'pagerduty -- failed to ' + @event['action'] + ' incident -- ' + incident_key
        end
      end
    rescue Timeout::Error
      puts 'pagerduty -- timed out while attempting to ' + @event['action'] + ' a incident -- ' + incident_key
    end
  end

end

sendgrid.json

{
  "sendgrid": {
    "smtp_password": "<password>",
    "smtp_domain": "<domain>",
    "smtp_user": "sensu",
    "mail_to": "ops+sensu@<domain>.com",
    "mail_from": "sensu@<domain>.com"
  },
  "handlers": {
    "sendgrid": {
      "severities": [
        "ok",
        "unknown",
        "warning",
        "critical"
      ],
      "command": "/etc/sensu/handlers/sendgrid.rb",
      "type": "pipe"
    }
  }
}

sendgrid.rb

#!/usr/bin/env ruby
#
# Sensu Handler: sendgrid
#
# This handler formats alerts as mails and sends them off to a pre-defined recipient.
#
# Released under the same terms as Sensu (the MIT license); see LICENSE
# for details.

require 'rubygems' if RUBY_VERSION < '1.9.0'
require 'sensu-handler'
require 'mail'
require 'timeout'

class Sendgrid < Sensu::Handler
  def short_name
    @event['client']['name'] + '/' + @event['check']['name']
  end

  def action_to_string
   @event['action'].eql?('resolve') ? "RESOLVED" : "ALERT"
  end

  def default_config
    #_ Global Settings
    smtp_address = settings['sendgrid']['smtp_address'] || 'smtp.sendgrid.net'
    smtp_port = settings['sendgrid']['smtp_port'] || '587'
    smtp_domain = settings['sendgrid']['smtp_domain'] || 'localhost.localdomain'
    smtp_user = settings['sendgrid']['smtp_user'] || 'yourusername@domain.com'
    smtp_password = settings['sendgrid']['smtp_password'] || 'yourPassword'
    smtp_auth = settings['sendgrid']['smtp_auth'] || 'plain'

    defaults = {
      "mail_from" => settings['sendgrid']['mail_from'] || 'localhost',
      "mail_to" => settings['sendgrid']['mail_to'] || 'root@localhost'
    }

    # Merge per-check configs
    defaults.merge!(@event['check']['sendgrid'] || {})

    params = {
      :mail_to   => defaults['mail_to'],
      :mail_from => defaults['mail_from'],
      :smtp_addr => smtp_address,
      :smtp_port => smtp_port,
      :smtp_domain => smtp_domain,
      :smtp_user => smtp_user,
      :smtp_password => smtp_password,
      :smtp_auth => smtp_auth
    }
  end

  def handle
    params = self.default_config

    body = <<-BODY.gsub(/^ {14}/, '')
            #{@event['check']['output']}
            Host: #{@event['client']['name']}
            Timestamp: #{Time.at(@event['check']['issued'])}
            Address:  #{@event['client']['address']}
            Check Name:  #{@event['check']['name']}
            Command:  #{@event['check']['command']}
            Status:  #{@event['check']['status']}
            Occurrences:  #{@event['occurrences']}
          BODY
    body << "Runbook:  #{@event['check']['runbook']}" if @event['check']['runbook']
    subject = "#{action_to_string} - #{short_name}: #{@event['check']['notification']}"

    Mail.defaults do
      delivery_method :smtp, {
        :address              => params[:smtp_addr],
        :port                 => params[:smtp_port],
        :domain               => params[:smtp_domain],
        :user_name            => params[:smtp_user],
        :password             => params[:smtp_password],
        :enable_starttls_auto => true
      }
    end

    begin
      timeout 10 do
        Mail.deliver do
          to      params[:mail_to]
          from    params[:mail_from]
          subject subject
          body    body
        end

        puts 'sendgrid -- sent alert for ' + short_name + ' to ' + params[:mail_to]
      end
    rescue Timeout::Error
      puts 'sendgrid -- timed out while attempting to ' + @event['action'] + ' an incident -- ' + short_name
    end
  end
end