Skip to content

Instantly share code, notes, and snippets.

@leejones
Created September 25, 2010 02:49
Show Gist options
  • Save leejones/596401 to your computer and use it in GitHub Desktop.
Save leejones/596401 to your computer and use it in GitHub Desktop.
Remove orphaned passenger processes.
require 'chronic'
class PassengerProcess
attr_accessor :pid, :uptime
def initialize(passenger_status_line)
values = passenger_status_line.match(/PID:\s(\d*).*Uptime:\s*(.*)$/)
@pid = values[1].to_i
@uptime = values[2]
end
def uptime_in_seconds
uptime_values = uptime.split(/\s/).map { |u| u.to_i }
seconds = uptime_values[-1] || 0
minutes = uptime_values[-2] || 0
hours = uptime_values[-3] || 0
seconds + (minutes*60) + (hours*3600)
end
def self.passenger_status
@passenger_status ||= `passenger-status | grep PID`
end
def self.active
passengers = []
passenger_status.each_line do |line|
passengers << PassengerProcess.new(line)
end
passengers
end
def self.stale
stale_passengers = []
potentially_stale_processes = active.select { |p| p.uptime_in_seconds > 600 }
potentially_stale_processes.each do |process|
process_last_log_entry = last_log_entry(process.pid)
etime = (Time.now - parse_time_from_log_entry(process_last_log_entry)).to_i
if etime > 600
puts "Stale process last log entry: #{process_last_log_entry}" if debug?
stale_passengers << process
end
end
stale_passengers
end
def self.last_log_entry(pid)
pwd = `pwd`.chomp
`grep 'rails\\[#{pid}\\]' #{pwd}/log/production.log | tail -n 1`.chomp
end
def self.last_log_entry_time(pid)
log_entry = last_log_entry(pid)
log_entry_time = parse_time_from_log_entry(log_entry)
(Time.now - log_entry_time).to_i
end
def self.parse_time_from_log_entry(entry)
Chronic.parse(entry.match(/.*\s.*\s\d{1,2}:\d{1,2}:\d{1,2}/)[0])
end
def self.active_passenger_pids
active.map{ |p| p.pid }
end
def self.passenger_memory_stats
# 17630 287.0 MB 64.5 MB Rails: /var/www/apps/gldl/current
# 17761 285.9 MB 64.9 MB Rails: /var/www/apps/gldl/current
# 18242 293.1 MB 71.4 MB Rails: /var/www/apps/gldl/current
# 18255 285.9 MB 60.6 MB Rails: /var/www/apps/gldl/current
@passenger_memory_stats ||= `passenger-memory-stats | grep 'Rails:.*\/current'`
end
def self.all_passenger_pids
passengers = []
passenger_memory_stats.each_line do |line|
matcher = line.match(/\s?(\d*)\s/)
passengers << matcher[1] if matcher
end
passengers
end
def self.inactive_passenger_pids
inactive_passenger_pids = []
(all_passenger_pids - active_passenger_pids).each do |pid|
raw_etime = `ps -p #{pid} --no-headers -o etime`.chomp
etime = PsEtime.new(raw_etime)
inactive_passenger_pids << pid unless (etime.age_in_seconds < (MINIMUM_ETIME || 600))
end
inactive_passenger_pids
end
def self.kill_inactive_passengers
inactive_passenger_pids.each do |pid|
kill(pid)
end
end
def self.kill_stale_passengers
stale.each do |p|
kill(p.pid)
end
end
def self.kill(pid)
signal = ARGV.include?('--hard') ? 9 : 15
command = "kill -#{signal} #{pid}"
puts command
`#{command}` unless noop?
end
def self.inactive_passengers_last_log_entry
inactive_passenger_pids.each do |pid|
puts last_log_entry(pid)[0, 150]
end
end
private
def self.noop?
@noop ||= ARGV.include?('--noop')
end
def self.debug?
@noop ||= ARGV.include?('--debug')
end
end
#!/usr/bin/env ruby
require 'rubygems'
require File.join(File.dirname(__FILE__), 'ps_etime')
require File.join(File.dirname(__FILE__), 'passenger_process')
MINIMUM_ETIME = 600 # 10 minutes
if %W(active inactive debug status).include?(ARGV.first)
case ARGV.first
when 'active'
PassengerProcess.kill_stale_passengers
when 'inactive'
PassengerProcess.kill_inactive_passengers
when 'debug'
PassengerProcess.inactive_passengers_last_log_entry
when 'status'
puts "Total of passengers: #{PassengerProcess.all_passenger_pids.count}"
puts "Stale passengers: #{PassengerProcess.stale.count}"
end
else
help =<<EOF
Error: please use the following syntax:
passenger_reaper <command> <options>
Commands:
status displays the number of total passenger workers and the number of active workers
active kills stale workers that passengers has in the pool
inactive kills workers that passenger no longer controls
debug shows the last log entry from each inactive worker
Options:
--noop don't actually kill processes but show which ones would have been killed
--hard send the KILL signal
EOF
puts help
exit 1
end
require 'ps_etime'
require 'passenger_process'
require 'chronic'
describe PassengerProcess do
before do
@passenger_status =<<EOF
PID: 14548 Sessions: 0 Processed: 42 Uptime: 40s
PID: 14521 Sessions: 0 Processed: 29 Uptime: 1h 20m 43s
PID: 14525 Sessions: 0 Processed: 32 Uptime: 20m 5s
PID: 14007 Sessions: 0 Processed: 36 Uptime: 1m 47s
PID: 14830 Sessions: 1 Processed: 1 Uptime: 5s
EOF
PassengerProcess.stub!(:passenger_status).and_return(@passenger_status)
end
it "should return active passenger processes" do
all_passenger_processes = PassengerProcess.active
all_passenger_processes.count.should eql(5)
passenger = all_passenger_processes.first
passenger.pid.should eql(14548)
passenger.uptime.should eql('40s')
end
it "should parse the uptime into seconds" do
all_passenger_processes = PassengerProcess.active
all_passenger_processes[0].uptime_in_seconds.should eql(40)
all_passenger_processes[4].uptime_in_seconds.should eql(5)
all_passenger_processes[3].uptime_in_seconds.should eql(107)
all_passenger_processes[1].uptime_in_seconds.should eql(4843)
end
it "should return the stale passengers" do
stub_time = Chronic.parse('Sep 26 17:40:34')
Time.stub!(:now).and_return(stub_time)
PassengerProcess.stub!(:last_log_entry).with(14521).and_return('Sep 26 16:40:34 web1 rails[14521]: Rendering promotions/index')
PassengerProcess.stub!(:last_log_entry).with(14525).and_return('Sep 26 17:39:34 web1 rails[14525]: Rendering promotions/index')
stale_passengers = PassengerProcess.stale
stale_passengers.count.should eql(1)
stale_passengers[0].pid.should eql(14521)
end
end
class PsEtime
attr_accessor :seconds, :minutes, :hours, :days
attr_reader :etime
def initialize(raw_etime)
@etime = raw_etime
split_etime = @etime.split(/\-|\:/).map {|a| a.to_i}
@seconds = split_etime[-1] || 0
@minutes = split_etime[-2] || 0
@hours = split_etime[-3] || 0
@days = split_etime[-4] || 0
end
def age_in_seconds
@seconds + (@minutes*60) + (@hours*3600) + (@days*86400)
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment