gistfile1.rb

#!/usr/bin/env ruby
# based on http://gist.github.com/29838
# logging to syslog added
# added killing orphaned procs
# added culling applications to maintain some per application limits
# Find bloating passengers and kill them gracefully. Run from cron every so often.
#

require "rubygems"
require "logging"

logger = Logging.logger['PassengerGrooming']
logger.level = :debug
logger.add_appenders(
    Logging::Appenders::Syslog.new(''),
    Logging.appenders.stdout
)

# required for passenger since cron has no environment
ENV['HTTPD'] = 'httpd'
 
module Process
  def self.running?(pid)
    begin
      return Process.getpgid(pid) != -1
    rescue Errno::ESRCH
      return false
    end
  end
end
      
class PassengerGrooming
  
  attr_reader :logger
  
  def initialize(options={})
    @logger         = Logging.logger[self]
    @mem_limit      = options[:mem_limit] || 200
    @ruby_user      = options[:ruby_user] || "www-data"
    @max_instances  = options[:max_instances] || {}
    @kill_time_wait = options[:kill_time_wait] || 8
    @safe_pids      = []
    @all_pids       = get_all_pids
    vacate_bloaters
    vacate_orphans
    vacate_unneeded
  end
  
  def get_all_pids
    all_pids = `ps -U #{@ruby_user} -o pid,args | grep  -e "Rails\\|Rack" | cut -f1 -d"R"`.split.map {|x| x.to_i}
    logger.debug "all_pids: #{all_pids.inspect}"
    all_pids
  end
  
  def vacate(pid,signal="USR1") 
    logger.debug "#{pid}: Killing with #{signal} ..."
    begin 
      Process.kill(signal, pid)
    rescue Errno::ESRCH
      logger.info "#{pid}: Pid not found to kill, moving on."
    else
      logger.info "#{pid}: Finished kill attempt with #{signal}. Sleeping for #{@kill_time_wait} seconds..."
      sleep @kill_time_wait
      if Process.running?(pid)
        logger.warn "#{pid}: Process is still running, so killing with some predjudice!"
        Process.kill("ABRT", pid)
        logger.debug "#{pid}: Finished kill attempt with ABRT. Sleeping for #{@kill_time_wait} seconds..."
        sleep @kill_time_wait
        if Process.running?(pid)
          logger.error "#{pid}: Process is still running, so killing with extreme predjudice!"
          Process.kill("TERM", pid)
        end
      end
    end
  end
  
  def vacate_bloaters
    #Trim the ruby procs passenger knows about
    `passenger-memory-stats`.each_line do |line|
      if line =~ /Rails: |Rack: /
        parts = line.split
        pid, private_dirty_rss = parts[0].to_i, parts[4].to_f
        if private_dirty_rss > @mem_limit
          logger.warn "#{pid}: Vacating bloater with size #{private_dirty_rss.to_s}"
          vacate(pid)
          @all_pids.delete(pid)
        else
          @safe_pids << pid
        end
      end
    end
    logger.debug "safe_pids: #{@safe_pids.inspect}"
  end

  def vacate_orphans
    #Kill the ruby procs that passenger has orphaned use abrt to try and find out why
    (@all_pids - @safe_pids).each do |pid|
      logger.warn "#{pid} Vacating orphan"
      vacate(pid,"ABRT") #SIGABRT tells passenger to leave a backtrace in the logs
    end
  end
  
  def vacate_unneeded
    processed = {}
    current_app = ''
    `passenger-status`.each_line do |line|
      case line 
      when /^\//
        current_app = line.chop
      when /PID:/
        parts = line.split
        processed[current_app] = Array.new unless processed[current_app].is_a?(Array)
        processed[current_app] << { :pid => parts[1].to_i, :processed => parts[5].to_i }
      end
    end
    application = []
    @max_instances.each do |k,v| #find first index to match
      processed.each{|index,obj|
        if index =~ /#{k.to_s}/ 
          application = obj
          break 
        end
      }
      application.sort_by{|x| x[:processed]} #least processed first
      logger.debug "#{k} procs: #{application.size} limit: #{v}"
      v.times { application.shift } #pop the ones were going to keep, off the stack
      application.each do |app| #vacate the rest
        logger.warn "#{app[:pid]}: Vacating unneeded #{k}"
        vacate(app[:pid])
      end
    end
  end
end

PassengerGrooming.new(
  :kill_time_wait => 12,
  :max_instances => {:spanky => 1} #This is an application that we didn't need more than one process for, but Passenger didn't allow us to control the limit of instances of a specific application, and this particular one was taking a lot of memory while idle.
)