public
Last active

Rake taks to launch multiple Resque workers in development/production with simple management included

  • Download Gist
workers.rake
Ruby
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163
# Rake task to launch multiple Resque workers in development/production with simple management included
 
require 'resque/tasks' # Require Resque tasks
 
namespace :workers do
 
# = $ rake workers:start
#
# Launch multiple Resque workers with the Rails environment loaded,
# so they have access to your models, etc.
#
# Each worker is being run in their own separate process (_not_ thread).
#
# On clean shutdown (SIGINT / Ctrl-C, SIGQUIT, SIGTERM), the task will clean
# after itself: kill its workers and delete PID files, when appropriate. It
# will deal fine with already dead workers.
#
#
# Default options like COUNT can and should be over-ridden when invoking, of course:
#
# $ rake workers:start COUNT=10 QUEUE=my_queue
#
#
# To daemonize, simply run with nohup:
#
# $ nohup rake workers:start > log/workers.log 2>&1 &
#
#
# You can and should set up your monitoring tool to watch for process with PID
# from `cat tmp/pids/resque/master.pid`.
#
# For proper monitoring of _individual_ workers, use provided examples for God or Monit:
# http://github.com/defunkt/resque/blob/master/examples/.
#
#
# A task for killing all workers on the machine (`rake workers:killall`) is also provided,
# for pruning orphaned workers etc.
#
desc "Run and manage group of Resque workers with some default options"
task :start => :environment do
 
# - CONFIGURATION ----
ENV['QUEUE'] ||= '*'
ENV['COUNT'] ||= '3'
# --------------------
 
def queue
ENV['QUEUE']
end
 
def count
ENV['COUNT']
end
 
def Process.exists?(pid)
kill(0, pid.to_i) rescue false
end
 
def pid_directory
@pid_directory ||= Rails.root.join('tmp', 'pids', "resque")
end
 
def pid_directory_for_group
@pid_directory_for_group ||= Rails.root.join('tmp', 'pids', "resque", queue)
end
 
def group_master_pid
File.read pid_directory.join("#{queue}.pid").to_s rescue nil
end
 
def group?
!group_master_pid || group_master_pid.to_s == Process.pid.to_s
end
 
def group_running?
Process.exists?(group_master_pid)
end
 
def kill_worker(pid)
Process.kill("QUIT", pid)
puts "Killed worker with PID #{pid}"
rescue Errno::ESRCH => e
puts " STALE worker with PID #{pid}"
end
 
def kill_workers
@pids.each { |pid| kill_worker(pid) }
end
 
def kill_workers_and_remove_pids_for_group
Dir.glob(pid_directory_for_group.join('worker_*.pid').to_s) do |pidfile|
begin
pid = pidfile[/(\d+)\.pid/, 1].to_i
kill_worker(pid)
ensure
FileUtils.rm pidfile, :force => true
end
end
if group_master_pid
FileUtils.rm pid_directory.join("#{queue}.pid").to_s
FileUtils.rm_rf pid_directory_for_group.to_s
end
end
 
def shutdown
puts "\n*** Exiting"
if group?
kill_workers_and_remove_pids_for_group
else
kill_workers
end
exit(0)
end
 
# Clean up after dead group from before -- and become one
unless group_running?
puts "--- Cleaning up after previous group (PID: #{group_master_pid})"
kill_workers_and_remove_pids_for_group
end
 
# Handle exit
trap('INT') { shutdown }
trap('QUIT') { shutdown }
trap('TERM') { shutdown }
trap('KILL') { shutdown }
trap('SIGKILL') { shutdown }
 
puts "=== Launching #{ENV['COUNT']} worker(s) on '#{ENV['QUEUE']}' queue(s) with PID #{Process.pid}"
 
# Launch workers in separate processes, saving their PIDs
@pids = []
ENV['COUNT'].to_i.times do
@pids << Process.fork { Rake::Task['resque:work'].invoke }
end
 
if group?
# Make sure we have directory for pids
FileUtils.mkdir_p pid_directory.to_s
FileUtils.mkdir_p pid_directory_for_group.to_s
# Create PID files for workers
File.open( pid_directory.join("#{queue}.pid").to_s, 'w' ) do |f| f.write Process.pid end
@pids.each do |pid|
File.open( pid_directory_for_group.join("worker_#{pid}.pid").to_s, 'w' ) { |f| f.write pid }
end
# Stay in foreground, if any of our workers dies, we'll get killed so Monit/God etc can come to the resq^Hcue
Process.wait
else
# Stay in foreground, if any of our workers dies, continue running
Process.waitall
end
end
 
desc "Kill ALL workers on this machine"
task :killall do
require 'resque'
Resque::Worker.all.each do |worker|
puts "Shutting down worker #{worker}"
host, pid, queues = worker.id.split(':')
Process.kill("QUIT", pid.to_i)
end
end
 
end

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.