Sidekiq cluster control script and systemd service

skcluster.rb

#!/usr/bin/env ruby

require 'sidekiq'
require 'sidekiq/cli'

# Default to running one process per core
def process_count
  return ENV['SK_PROCESS_COUNT'].to_i unless ENV['SK_PROCESS_COUNT'].to_i == 0

  case RbConfig::CONFIG['host_os']
  when /linux/
    `grep processor /proc/cpuinfo | wc -l`.to_i
  when /darwin9/
    `hwprefs cpu_count`.to_i
  when /darwin/
    ((`which hwprefs` != '') ? `hwprefs thread_count` : `sysctl -n hw.ncpu`).to_i
  when /freebsd/
    `sysctl -n hw.ncpu`.to_i
  end
end

def handle_sig(sig)
  # If we're shutting down, we don't need to respawn child processes that die
  if sig == "INT" || sig == "TERM"
    @watch_children = false
  end

  @pids.each do |pid|
    Process.kill(sig, pid)
  end
end

def fork_child
  Process.fork do
    begin
      cli = Sidekiq::CLI.instance
      cli.parse
      cli.run
    rescue => e
      raise e if $DEBUG
      STDERR.puts e.message
      STDERR.puts e.backtrace.join("\n")
      exit 1
    end
  end
end

def start_children
  process_count.times.map do
    fork_child
  end
end

# Limit each child process to a chunk of RAM based on how
# many processes we will run, with a bit of RAM left over for headroom.
# E.g., for four cores we'll run four child processes,
# so let each child use 20% of RAM instead of 25%.
@memory_percentage_limit = ENV['SK_MEMORY_PCT_LIMIT'].nil? ? (1 / (process_count + 1).to_f) * 100 : ENV['SK_MEMORY_PCT_LIMIT'].to_f

puts "Starting up"

@pids = start_children
@watch_children = true

Thread.new do
  puts "Watching for misbehaving children"
  while @watch_children do
    @pids.each do |pid|
      memory_percent_used = `ps -o %mem= -p #{pid}`.to_f
      if memory_percent_used == 0.0 # child died
        @pids.delete(pid)
        @pids << new_pid = fork_child
        puts "Replaced lost pid #{pid} with #{new_pid}"
      elsif memory_percent_used > @memory_percentage_limit
        puts "Pid #{pid} crossed memory threshold -- replacing"
        @pids.delete(pid)
        Process.kill("USR1", pid)
        sleep 4
        Process.kill("TERM", pid)
        @pids << new_pid = fork_child
      elsif $DEBUG
        puts "#{pid}: #{memory_percent_used.round(2)}"
      end
    end
    puts "" if $DEBUG
    sleep 10
  end
  puts "Letting the children run wild"
end

%w(INT USR1 TERM).each do |sig|
  Signal.trap(sig) do
    handle_sig(sig)
  end
end

Process.waitall

puts "Shutting down"

skcluster.service

[Unit]
Description=Sidekiq Cluster
Requires=network.target pgbouncer.service
After=network.target pgbouncer.service

[Service]
Type=simple
User=honeybadger
Group=honeybadger
WorkingDirectory=/home/honeybadger/current
EnvironmentFile=-/home/honeybadger/shared/sidekiq.env
ExecStart=/usr/local/bin/bundle exec ./script/skcluster.rb --require /home/honeybadger/current -C /home/honeybadger/current/config/sidekiq.yml --environment production -L /home/honeybadger/shared/log/sidekiq.log
KillSignal=SIGTERM
Restart=on-failure
SyslogIdentifier=sidekiq

[Install]
WantedBy=multi-user.target