Skip to content

Instantly share code, notes, and snippets.

@stympy
Created February 8, 2017 20:53
Show Gist options
  • Save stympy/d51c42d7026b6bc43a3cc430190d5a7e to your computer and use it in GitHub Desktop.
Save stympy/d51c42d7026b6bc43a3cc430190d5a7e to your computer and use it in GitHub Desktop.
Sidekiq cluster control script and systemd service
#!/usr/bin/env ruby
require 'sidekiq'
require 'sidekiq/cli'
# Default to running one process per core
def process_count
return ENV['SK_PROCESS_COUNT'].to_i unless ENV['SK_PROCESS_COUNT'].to_i == 0
case RbConfig::CONFIG['host_os']
when /linux/
`grep processor /proc/cpuinfo | wc -l`.to_i
when /darwin9/
`hwprefs cpu_count`.to_i
when /darwin/
((`which hwprefs` != '') ? `hwprefs thread_count` : `sysctl -n hw.ncpu`).to_i
when /freebsd/
`sysctl -n hw.ncpu`.to_i
end
end
def handle_sig(sig)
# If we're shutting down, we don't need to respawn child processes that die
if sig == "INT" || sig == "TERM"
@watch_children = false
end
@pids.each do |pid|
Process.kill(sig, pid)
end
end
def fork_child
Process.fork do
begin
cli = Sidekiq::CLI.instance
cli.parse
cli.run
rescue => e
raise e if $DEBUG
STDERR.puts e.message
STDERR.puts e.backtrace.join("\n")
exit 1
end
end
end
def start_children
process_count.times.map do
fork_child
end
end
# Limit each child process to a chunk of RAM based on how
# many processes we will run, with a bit of RAM left over for headroom.
# E.g., for four cores we'll run four child processes,
# so let each child use 20% of RAM instead of 25%.
@memory_percentage_limit = ENV['SK_MEMORY_PCT_LIMIT'].nil? ? (1 / (process_count + 1).to_f) * 100 : ENV['SK_MEMORY_PCT_LIMIT'].to_f
puts "Starting up"
@pids = start_children
@watch_children = true
Thread.new do
puts "Watching for misbehaving children"
while @watch_children do
@pids.each do |pid|
memory_percent_used = `ps -o %mem= -p #{pid}`.to_f
if memory_percent_used == 0.0 # child died
@pids.delete(pid)
@pids << new_pid = fork_child
puts "Replaced lost pid #{pid} with #{new_pid}"
elsif memory_percent_used > @memory_percentage_limit
puts "Pid #{pid} crossed memory threshold -- replacing"
@pids.delete(pid)
Process.kill("USR1", pid)
sleep 4
Process.kill("TERM", pid)
@pids << new_pid = fork_child
elsif $DEBUG
puts "#{pid}: #{memory_percent_used.round(2)}"
end
end
puts "" if $DEBUG
sleep 10
end
puts "Letting the children run wild"
end
%w(INT USR1 TERM).each do |sig|
Signal.trap(sig) do
handle_sig(sig)
end
end
Process.waitall
puts "Shutting down"
[Unit]
Description=Sidekiq Cluster
Requires=network.target pgbouncer.service
After=network.target pgbouncer.service
[Service]
Type=simple
User=honeybadger
Group=honeybadger
WorkingDirectory=/home/honeybadger/current
EnvironmentFile=-/home/honeybadger/shared/sidekiq.env
ExecStart=/usr/local/bin/bundle exec ./script/skcluster.rb --require /home/honeybadger/current -C /home/honeybadger/current/config/sidekiq.yml --environment production -L /home/honeybadger/shared/log/sidekiq.log
KillSignal=SIGTERM
Restart=on-failure
SyslogIdentifier=sidekiq
[Install]
WantedBy=multi-user.target
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment