public
Created

Watches heroku router logs and summerizes and notifies

  • Download Gist
heroku_watcher.rake
Ruby
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
# Excute by the following:
# bundle exec rake GMAIL_USERNAME=name@example.com GMAIL_PASSWORD=password watch_heroku
task :watch_heroku do
 
puts "need to set GMAIL_USERNAME and GMAIL_PASSWORD environment variables" unless (ENV['GMAIL_USERNAME'] && ENV['GMAIL_PASSWORD'])
 
@critical_notified = false
@warning_notified = false
 
@total_lines = 0
@total_service = 0
@total_wait = 0
@total_queue = 0
@total_errors = 0
@max_service = 0
 
@mutex = Mutex.new
 
# print headers
puts
puts
printf "%7s %7s %7s %7s %7s %7s %40s\n",
'ops', 'service', 'wait', 'queue', 'max_serv', 'errors', 'time'
 
threads = []
threads << Thread.new('heroku_logs') do
IO.popen("heroku logs --ps router --tail --app #{ENV['APP_NAME']}") do |f|
while line = f.gets
parse_heroku_line(line) if line =~ /^/
end
end
end
 
# every 60 seconds print out averages, and other stats and reset memos
threads << Thread.new('summarize') do
while true do
sleep(60)
average_service = @total_lines > 0 ? @total_service / @total_lines : 'N/A'
average_wait = @total_lines > 0 ? @total_wait / @total_lines : 'N/A'
average_queue = @total_lines > 0 ? @total_queue / @total_lines : 'N/A'
 
color_print @total_lines
color_print average_service, warning: 1000, critical: 10_000
color_print average_wait, warning: 10, critical: 100
color_print average_queue, warning: 10, critical: 100
color_print @max_service, warning: 10_000, critical: 20_000
color_print @total_errors, warning: 1, critical: 10
color_print Time.now, length: 40
printf "\n"
 
# if any errors or average service about a threshold notify
check_and_notify(average_service, @total_errors)
 
#reset
@mutex.synchronize do
@total_service = @total_lines = @total_wait = @total_queue = @total_errors = @max_service = 0
end
end
end
 
threads.each { |t| t.join }
 
end
 
def color_print(field, options ={})
options[:length] = 7 unless options[:length]
if options[:critical] && is_number?(field) && Integer(field) > options[:critical]
print "\a" #beep
# notify_critical
print Term::ANSIColor.red
print Term::ANSIColor.bold
elsif options[:warning] && is_number?(field) && Integer(field) > options[:warning]
print Term::ANSIColor.yellow
end
printf "%#{options[:length]}s", field
print Term::ANSIColor.clear
end
 
def parse_heroku_line(line)
# 2012-07-05T20:24:10+00:00 heroku[router]: GET my-app.com/pxl/4fdbc97dc6b36c0030001160?value=1 dyno=web.14 queue=0 wait=0ms service=8ms status=200 bytes=35
 
# or if error
 
#2012-07-05T20:17:12+00:00 heroku[router]: Error H12 (Request timeout) -> GET my-app.com/crossdomain.xml dyno=web.4 queue= wait= service=30000ms status=503 bytes=0
 
items = line.split
 
if line =~ /Error/
@total_errors += 1
else
 
 
time = items[0]
process = items[1]
http_type = items[2]
url = items[3]
dyno = items[4].split('=').last if items[4]
queue = items[5].split('=').last.sub('ms', '') if items[5]
wait = items[6].split('=').last.sub('ms', '') if items[6]
service = items[7].split('=').last.sub('ms', '') if items[7]
status = items[8].split('=').last if items[8]
bytes = items[9].split('=').last if items[9]
 
if is_number?(service) && is_number?(wait) && is_number?(queue)
@mutex.synchronize do
@total_lines +=1
@total_service += Integer(service) if service
@total_wait += Integer(wait) if wait
@total_queue += Integer(queue) if queue
@max_service = Integer(service) if Integer(service) > @max_service
end
end
 
 
end
 
end
 
def is_number?(string)
_is_number = true
begin
num = Integer(string)
rescue
_is_number = false
end
_is_number
end
 
 
def check_and_notify(average_service, errors)
if average_service > 10_000 || errors > 10
notify_critical
elsif average_service > 600 || errors > 5
notify_warning
end
end
 
 
def notify_kaboom
Thread.new('notify_admins') do
# send emails
end
end
 
def notify_critical
unless @critical_notified
Thread.new('notify_admins') do
# send emails
end
@critical_notified = true
end
end
 
def notify_warning
 
unless @warning_notified
Thread.new('notify_admins') do
# send emails
end
@warning_notified = true
end
 
end
 
 
def send_email(to, msg)
 
content = <<EOF
From: #{ENV['GMAIL_USERNAME']}
To: #{to}
Subject: #{msg}
 
 
#{msg}
EOF
 
content = [
"From: Heroku Dyno Watcher <#{ENV['GMAIL_USERNAME']}>",
"To: #{to}",
"Subject: #{msg}",
"",
"#{msg}"
].join("\r\n")
 
Net::SMTP.enable_tls(OpenSSL::SSL::VERIFY_NONE)
Net::SMTP.start('smtp.gmail.com', 587, 'gmail.com', ENV['GMAIL_USERNAME'], ENV['GMAIL_PASSWORD'], :login) do |smtp|
smtp.send_message(content, ENV['GMAIL_USERNAME'], to)
end
 
 
end

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.