Skip to content

Instantly share code, notes, and snippets.

@ezmobius
Created July 31, 2009 00:36
Show Gist options
  • Save ezmobius/159010 to your computer and use it in GitHub Desktop.
Save ezmobius/159010 to your computer and use it in GitHub Desktop.
diff --git a/app/controllers/reporting.rb b/app/controllers/reporting.rb
index 3a6768f..662bdd9 100644
--- a/app/controllers/reporting.rb
+++ b/app/controllers/reporting.rb
@@ -24,34 +24,6 @@ class Reporting < Application
if instance.status == :running
Merb.logger.error! "instance(#{instance.amazon_id}) ALERT!\n SEVERITY: #{params[:data]['Severity']}\n PLUGIN:#{params[:data]['Plugin']} TYPE:#{params[:data]['Type']}\n MESSAGE:#{params[:data]['alert_message']}"
Alert.create_from_json(instance, params['data'])
- environment = Environment.get instance.environment.id
- if environment.alerting_enabled
- if throttled = environment.throttled_at.blank? || 1.hour.ago.to_datetime > environment.throttled_at
- alerts_in_last_hour = Alert.count :environment_id => environment.id,:created_at.gt => 1.hour.ago
- if alerts_in_last_hour >= Environment::THROTTLE_TRIGGER
- environment.update_attributes(:throttled_at => DateTime.now) if throttled
- info = {
- :text => "You received #{alerts_in_last_hour} alerts in the last hour. We have throttled your alert emails and will not send any alerts for the next hour."
- }
- send_alert(:throttled, info, {
- :subject => "[IMPORTANT] Your '#{environment.name}' environment alerting has been throttled",
- :to => environment.alert_email,
- :link => resource(environment, :alerts)
- })
- else
- subject = format_alert(params[:data], environment.name)
- params['data']['Environment'] = environment.name
- params['data']['Instance Id'] = instance.amazon_id
- params['data']['Instance Role'] = instance.pretty_instance_role
- params['data']['Public Hostname'] = instance.public_hostname
- send_alert(:alert, params['data'], {
- :subject => subject,
- :to => environment.alert_email,
- :link => resource(environment, :alerts)
- })
- end
- end
- end
end
elsif message == 'bootstrap'
instance.status = :building
@@ -71,19 +43,5 @@ class Reporting < Application
"failure"
end
end
-
-
- def send_alert(alert_type, alert, opts)
- send_mail(
- AlertMailer,
- alert_type,
- {
- :from => 'no-reply@engineyard.com',
- :to => opts[:to],
- :subject => opts[:subject]
- },
- {:data => alert.merge(:link => opts[:link])}
- )
-
- end
+
end
diff --git a/app/helpers/reporting_helper.rb b/app/helpers/reporting_helper.rb
index ea0b674..0296a0a 100644
--- a/app/helpers/reporting_helper.rb
+++ b/app/helpers/reporting_helper.rb
@@ -1,28 +1,5 @@
module Merb
module ReportingHelper
- def format_alert(data, env=nil)
- current = format_float(data['CurrentValue'])
- warn = format_float(data['WarningMax'])
- fail = format_float(data['FailureMax'])
- env_details = "for Environment: #{env} " if env
- position = case data['Type']
- when 'memory-free', 'df'
- 'below'
- else
- 'above'
- end
- case data['Severity']
- when 'OKAY'
- "Alert(#{data['Severity']}) #{env_details}#{data['Type']}: #{current} is within acceptable range"
- when 'WARNING'
- "Alert(#{data['Severity']}) #{env_details}#{data['Type']}: #{current} is #{position}: #{warn}"
- when 'FAILURE'
- "Alert(#{data['Severity']}) #{env_details}#{data['Type']}: #{current} is #{position}: #{fail}"
- end
- end
- def format_float(f)
- sprintf("%.02f", Float(f))
- end
end
end # Merb
\ No newline at end of file
diff --git a/app/models/alert.rb b/app/models/alert.rb
index a15ec63..962864a 100644
--- a/app/models/alert.rb
+++ b/app/models/alert.rb
@@ -1,6 +1,6 @@
class Alert
include DataMapper::Resource
-
+ extend Merb::MailerMixin
property :id, Serial
property :created_at, DateTime
property :amazon_id, String
@@ -18,6 +18,35 @@ class Alert
:alert_type => pretty_type(json_data),
:severity => json_data['Severity'],
:message => format_alert(json_data)
+ environment = Environment.get instance.environment.id
+ if environment.alerting_enabled
+ if throttled = environment.throttled_at.blank? || 1.hour.ago.to_datetime > environment.throttled_at
+ alerts_in_last_hour = Alert.count :environment_id => environment.id,:created_at.gt => 1.hour.ago
+ if alerts_in_last_hour >= Environment::THROTTLE_TRIGGER
+ environment.update_attributes(:throttled_at => DateTime.now) if throttled
+ info = {
+ :text => "You received #{alerts_in_last_hour} alerts in the last hour. We have throttled your alert emails and will not send any alerts for the next hour."
+ }
+ send_alert(:throttled, info, {
+ :subject => "[IMPORTANT] Your '#{environment.name}' environment alerting has been throttled",
+ :to => environment.alert_email,
+ :link => "/environments/#{environment.id}/alerts"
+ })
+ else
+ subject = format_subject(json_data, environment.name)
+ json_data['Environment'] = environment.name
+ json_data['Instance Id'] = instance.amazon_id
+ json_data['Instance Role'] = instance.pretty_instance_role
+ json_data['Public Hostname'] = instance.public_hostname
+ send_alert(:alert, json_data, {
+ :subject => subject,
+ :to => environment.alert_email,
+ :link => "/environments/#{environment.id}/alerts"
+ })
+ end
+ end
+ end
+
end
def self.pretty_type(json_data)
@@ -33,6 +62,27 @@ class Alert
end
end
+ def self.format_subject(data, env=nil)
+ current = format_float(data['CurrentValue'])
+ warn = format_float(data['WarningMax'])
+ fail = format_float(data['FailureMax'])
+ env_details = "for Environment: #{env} " if env
+ position = case data['Type']
+ when 'memory-free', 'df'
+ 'below'
+ else
+ 'above'
+ end
+ case data['Severity']
+ when 'OKAY'
+ "Alert(#{data['Severity']}) #{env_details}#{data['Type']}: #{current} is within acceptable range"
+ when 'WARNING'
+ "Alert(#{data['Severity']}) #{env_details}#{data['Type']}: #{current} is #{position}: #{warn}"
+ when 'FAILURE'
+ "Alert(#{data['Severity']}) #{env_details}#{data['Type']}: #{current} is #{position}: #{fail}"
+ end
+ end
+
def self.format_alert(data, env=nil)
current = format_float(data['CurrentValue'])
warn = format_float(data['WarningMax'])
@@ -74,4 +124,18 @@ class Alert
sprintf("%.02f", Float(f))
end
+
+ def self.send_alert(alert_type, alert, opts)
+ send_mail(
+ AlertMailer,
+ alert_type,
+ {
+ :from => 'no-reply@engineyard.com',
+ :to => opts[:to],
+ :subject => opts[:subject]
+ },
+ {:data => alert.merge(:link => opts[:link])}
+ )
+
+ end
end
\ No newline at end of file
diff --git a/spec/requests/reporting_spec.rb b/spec/requests/reporting_spec.rb
index 3e51ec2..28609cd 100644
--- a/spec/requests/reporting_spec.rb
+++ b/spec/requests/reporting_spec.rb
@@ -116,7 +116,6 @@ describe Reporting do
it "should send an email with the severity in the subject and a link to view the instance's stats" do
request_report.should be_successful
-
email.to_s.grep(/Alert\(OKAY\)/).should_not be_empty
email.to_s.grep(/#{HOSTNAME}\/environments\/#{@environment.id}\/alerts/).should_not be_empty
email.to_s.grep(/Environment\: #{@environment.name}/).should_not be_empty
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment