Maintenance (with god) scripts of Rails (unicorn, resque/resque-scheduler)

app.god

# encoding: utf-8
# god -c config/app.god

module YourApp
  module God
    def self.generic_monitoring(w, options = {})
      # start if process is not running
      w.start_if do |start|
        start.condition(:process_running) do |c|
          c.interval = 10.seconds
          c.running = false
        end
      end

      # restart if memory or cpu gets too high
      w.restart_if do |restart|
        restart.condition(:memory_usage) do |c|
          c.above = options[:memory_limit]
          c.times = [3, 5] # 3 out of 5 intervals
        end

        restart.condition(:cpu_usage) do |c|
          c.above = options[:cpu_limit]
          c.times = 5
        end
      end

      w.lifecycle do |on|
        on.condition(:flapping) do |c|
          c.to_state = [:start, :restart]
          c.times = 5
          c.within = 5.minute
          c.transition = :unmonitored
          c.retry_in = 10.minutes
          c.retry_times = 5
          c.retry_within = 2.hours
        end
      end

      # determine the state on startup
      w.transition(:init, { true => :up, false => :start }) do |on|
        on.condition(:process_running) do |c|
          c.running = true
        end
      end

      # determine when process has finished starting
      w.transition([:start, :restart], :up) do |on|
        on.condition(:process_running) do |c|
          c.running = true
          c.interval = 5.seconds
        end

        # failsafe
        on.condition(:tries) do |c|
          c.times = 5
          c.transition = :start
          c.interval = 5.seconds
        end
      end
    end
  end
end

rails_env   = ENV['RAILS_ENV'] || 'production'
rails_root  = ENV['RAILS_ROOT'] || File.expand_path('../../', __FILE__)
worker_uid  = ENV['WORKER_UID']
worker_gid  = ENV['WORKER_GID']

God.log_file = "#{rails_root}/log/god.log"
God.pid_file_directory = "#{rails_root}/tmp/pids"

# Unicorn
# http://unicorn.bogomips.org/SIGNALS.html
God.watch do |w|
  w.dir = rails_root
  w.name = 'unicorn'
  w.interval = 30.seconds
  w.start = "cd #{rails_root} && RAILS_ENV=#{rails_env} bin/unicornctl start"
  w.stop = "cd #{rails_root} && bin/unicornctl stop"
  w.restart = "cd #{rails_root} && RAILS_ENV=#{rails_env} bin/unicornctl restart"
  w.start_grace = 10.seconds
  w.restart_grace = 10.seconds
  w.uid = worker_uid if worker_uid
  w.gid = worker_gid if worker_gid
  w.pid_file = "#{rails_root}/tmp/pids/unicorn.pid"
  w.behavior(:clean_pid_file)

  YourApp::God.generic_monitoring(w, cpu_limit: 50.percent, memory_limit: 300.megabytes)
end

# Resque
num_workers = (rails_env == 'production' ? 4 : 1) # base on CPUs
num_workers.times do |num|
  God.watch do |w|
    w.dir = rails_root
    w.name = "resque-work-#{num}"
    w.group = 'resque'
    w.interval = 30.seconds
    w.start = "cd #{rails_root} && WORKER=#{num} RAILS_ENV=#{rails_env} bin/resquectl start work"
    w.stop = "cd #{rails_root} && WORKER=#{num} bin/resquectl stop work"
    w.restart = "cd #{rails_root} && WORKER=#{num} RAILS_ENV=#{rails_env} bin/resquectl restart work"
    w.start_grace = 5.seconds
    w.restart_grace = 5.seconds
    w.uid = worker_uid if worker_uid
    w.gid = worker_gid if worker_gid
    w.pid_file = "#{rails_root}/tmp/pids/resque-work-#{num}.pid"
    w.behavior(:clean_pid_file)

    YourApp::God.generic_monitoring(w, cpu_limit: 30.percent, memory_limit: 200.megabytes)
  end
end

# Resque Scheduler
God.watch do |w|
  w.dir = rails_root
  w.name = 'resque-scheduler-0'
  w.interval = 30.seconds
  w.start = "cd #{rails_root} && RAILS_ENV=#{rails_env} bin/resquectl start scheduler"
  w.stop = "cd #{rails_root} && bin/resquectl stop scheduler"
  w.restart = "cd #{rails_root} && RAILS_ENV=#{rails_env} bin/resquectl restart scheduler"
  w.start_grace = 5.seconds
  w.restart_grace = 5.seconds
  w.uid = worker_uid if worker_uid
  w.gid = worker_gid if worker_gid
  w.pid_file = "#{rails_root}/tmp/pids/resque-scheduler-0.pid"
  w.behavior(:clean_pid_file)

  YourApp::God.generic_monitoring(w, cpu_limit: 30.percent, memory_limit: 200.megabytes)
end

rainbows.rb

# encoding: utf-8
# rainbows -c /path_to_RAILS_ROOT/config/rainbows.rb -E production -D
# See http://rainbows.rubyforge.org/Rainbows/Configurator.html for complete
# documentation

rails_env  = ENV['RAILS_ENV'] || ENV["RACK_ENV"] || 'production'
rails_root = ENV['RAILS_ROOT'] || File.expand_path('../../', __FILE__)
worker_uid = ENV['WORKER_UID']
worker_gid = ENV['WORKER_GID']

# If running the master process as root and the workers as an unprivileged
# user, do this to switch euid/egid in the workers (also chowns logs):
# user "unprivileged_user", "unprivileged_group"
worker_processes (rails_env == 'production' ? 4 : 1) # assuming four CPU cores

Rainbows! do
  use :ThreadPool
  # use :ThreadSpawn
  worker_connections 64
end

# Help ensure your application will always spawn in the symlinked
# "current" directory that Capistrano sets up.
# working_directory "/path/to/app/current"
working_directory rails_root

# listen on both a Unix domain socket and a TCP port,
# we use a shorter backlog for quicker failover when busy
#listen 8080, tcp_nopush: true
listen "#{rails_root}/tmp/sockets/rainbows.sock", backlog: 64

# nuke workers after 30 seconds instead of 60 seconds (the default)
timeout 30

# feel free to point this anywhere accessible on the filesystem
pid "#{rails_root}/tmp/pids/rainbows.pid"

# By default, the Unicorn logger will write to stderr.
# Additionally, ome applications/frameworks log to stderr or stdout,
# so prevent them from going to /dev/null when daemonized here:
stderr_path "#{rails_root}/log/rainbows.stderr.log"
stdout_path "#{rails_root}/log/rainbows.stdout.log"

preload_app true
# http://www.rubyenterpriseedition.com/faq.html#adapt_apps_for_cow
if GC.respond_to?(:copy_on_write_friendly=)
  GC.copy_on_write_friendly = true
end

before_fork do |server, worker|
  # the following is highly recomended for Rails + "preload_app true"
  # as there's no need for the master process to hold a connection
  defined?(ActiveRecord::Base) and
      ActiveRecord::Base.connection.disconnect!

  # The following is only recommended for memory/DB-constrained
  # installations.  It is not needed if your system can house
  # twice as many worker_processes as you have configured.
  #
  # # This allows a new master process to incrementally
  # # phase out the old master process with SIGTTOU to avoid a
  # # thundering herd (especially in the "preload_app false" case)
  # # when doing a transparent upgrade.  The last worker spawned
  # # will then kill off the old master process with a SIGQUIT.
  old_pid = "#{server.config[:pid]}.oldbin"
  if old_pid != server.pid
    begin
      sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
      Process.kill(sig, File.read(old_pid).to_i)
    rescue Errno::ENOENT, Errno::ESRCH
      # do nothing
    end
  end
  #
  # Throttle the master from forking too quickly by sleeping.  Due
  # to the implementation of standard Unix signal handlers, this
  # helps (but does not completely) prevent identical, repeated signals
  # from being lost when the receiving process is busy.
  # sleep 1
end

after_fork do |server, worker|
  ##
  # Rainbows master loads the app then forks off workers - because of the way
  # Unix forking works, we need to make sure we aren't using any of the parent's
  # sockets, e.g. db connection

  defined?(ActiveRecord::Base) and
      ActiveRecord::Base.establish_connection
  # Redis and Memcached would go here but their connections are established
  # on demand, so the master never opens a socket

  if worker_uid && worker_gid
    begin
      uid, gid = Process.euid, Process.egid
      target_uid = Etc.getpwnam(worker_uid).uid
      target_gid = Etc.getgrnam(worker_gid).gid
      worker.tmp.chown(target_uid, target_gid)
      if uid != target_uid || gid != target_gid
        Process.initgroups(worker_uid, target_gid)
        Process::GID.change_privilege(target_gid)
        Process::UID.change_privilege(target_uid)
      end
    rescue => e
      if rails_env == 'development'
        STDERR.puts "couldn't change user, oh well"
      else
        raise e
      end
    end
  end
end

rainbowsctl

#!/bin/sh

# Put the file into <YOUR APP>/bin directory and it should be executable.
#
# Examples:
#
#   RAILS_ENV=production bin/rainbowsctl start
#   RAILS_ENV=production bin/rainbowsctl stop

# set ruby GC parameters
RUBY_HEAP_MIN_SLOTS=600000
RUBY_FREE_MIN=200000
RUBY_GC_MALLOC_LIMIT=60000000
export RUBY_HEAP_MIN_SLOTS RUBY_FREE_MIN RUBY_GC_MALLOC_LIMIT

relative_path=`dirname "$0"`
rails_root="`(cd \"${relative_path}/../\" && pwd)`"

if [ -z $RAILS_ENV ]; then RAILS_ENV="production"; fi
pid="${rails_root}/tmp/pids/rainbows.pid"

case "$1" in
  start)
    if [ -f "$pid" ]; then
      echo "** PID file already exists. Please make sure the PID `cat $pid` is gone, and remove the \"$pid\" file."
    else
      cd $rails_root && bundle exec rainbows -c ${rails_root}/config/rainbows.rb -E $RAILS_ENV -D
    fi
    ;;
  stop)
    kill -QUIT `cat $pid` && rm -f "$pid"
    ;;
  restart)
    $0 stop
    while [ -f "$pid" ]
    do
     sleep 1
    done
    $0 start
    ;;
  reload)
    kill -USR2 `cat $pid`
    ;;
  force-stop)
    kill -INT `cat $pid` && rm -f "$pid"
    ;;
  shutdown-workers)
    kill -WINCH `cat $pid`
    ;;
  increment-worker)
    kill -TTIN `cat $pid`
    ;;
  decrement-worker)
    kill -TTOU `cat $pid`
    ;;
  logrotate)
    kill -USR1 `cat $pid`
    ;;
  *)
    echo "Usage: $0 <start|stop|restart|reload|force-stop|shutdown-workers|increment-worker|decrement-worker|logrotate>"
    ;;
esac

exit 0

resquectl

#!/bin/sh

# Put the file into <YOUR APP>/bin directory and it should be executable.
#
# Examples:
#
#   RAILS_ENV=production bin/resquectl start             # Start the task 0 of the resque work
#   RAILS_ENV=production WORKER=2 bin/resquectl start    # Start the task 2 of the resque work
#   RAILS_ENV=production bin/resquectl start scheduler   # Start the task 0 of the resque scheduler

relative_path=`dirname "$0"`
rails_root="`(cd \"${relative_path}/../\" && pwd)`"

if [ -z "$RAILS_ENV" ]; then RAILS_ENV="production"; fi
if [ -z "$WORKER" ]; then WORKER=0; fi

# Ensure scheduler uses worker 0
case "$2" in
  scheduler)
    WORKER=0
    cmd="$2"
    ;;
  *)
    cmd="work"
    ;;
esac

pid="${rails_root}/tmp/pids/resque-${cmd}-${WORKER}.pid"

case "$1" in
  start)
    if [ -f "$pid" ] && [ -z "$FORCE" ]; then
      echo "** PID file already exists. Please make sure the PID `cat $pid` is gone, and remove the \"$pid\" file."
    else
      cd $rails_root && BACKGROUND=yes PIDFILE=$pid QUEUE=* RAILS_ENV=$RAILS_ENV \
      bundle exec rake -f ${rails_root}/Rakefile environment resque:$cmd
    fi
    ;;
  stop)
    kill -QUIT `cat $pid` && rm -f "$pid"
    ;;
  restart)
    $0 stop $cmd
    while [ -f "$pid" ]
    do
      sleep 1
    done
    $0 start $cmd
    ;;
  force-stop)
    kill -TERM `cat $pid` && rm -f "$pid"
    ;;
  *)
    echo "Usage: [WORKER=<worker number>] [RAILS_ENV=<rails env>] $0 <start|stop|restart|force-stop> [scheduler]"
    ;;
esac

exit 0

unicorn.rb

# encoding: utf-8
# unicorn_rails -c /path_to_RAILS_ROOT/config/unicorn.rb -E production -D
# See http://unicorn.bogomips.org/Unicorn/Configurator.html for complete
# documentation

rails_env = ENV['RAILS_ENV'] || ENV["RACK_ENV"] || 'production'
rails_root = ENV['RAILS_ROOT'] || File.expand_path('../../', __FILE__)
worker_uid = ENV['WORKER_UID']
worker_gid = ENV['WORKER_GID']

worker_processes (rails_env == 'production' ? 4 : 1) # base on CPUs

# Since Unicorn is never exposed to outside clients, it does not need to
# run on the standard HTTP port (80), there is no reason to start Unicorn
# as root unless it's from system init scripts.
# If running the master process as root and the workers as an unprivileged
# user, do this to switch euid/egid in the workers (also chowns logs):
# user "unprivileged_user", "unprivileged_group"

# Help ensure your application will always spawn in the symlinked
                                         # "current" directory that Capistrano sets up.
# working_directory "/path/to/app/current" # available in 0.94.0+
working_directory rails_root

# listen on both a Unix domain socket and a TCP port,
# we use a shorter backlog for quicker failover when busy
#listen 8080, tcp_nopush: true
listen "#{rails_root}/tmp/sockets/unicorn.sock", backlog: 64

# Restart any workers that haven't responded in 30 seconds
timeout 30

# feel free to point this anywhere accessible on the filesystem
pid "#{rails_root}/tmp/pids/unicorn.pid"

# By default, the Unicorn logger will write to stderr.
# Additionally, ome applications/frameworks log to stderr or stdout,
# so prevent them from going to /dev/null when daemonized here:
stderr_path "#{rails_root}/log/unicorn.stderr.log"
stdout_path "#{rails_root}/log/unicorn.stdout.log"

preload_app true
# http://www.rubyenterpriseedition.com/faq.html#adapt_apps_for_cow
if GC.respond_to?(:copy_on_write_friendly=)
  GC.copy_on_write_friendly = true
end

# Enable this flag to have unicorn test client connections by writing the
# beginning of the HTTP headers before calling the application.  This
# prevents calling the application for connections that have disconnected
# while queued.  This is only guaranteed to detect clients on the same
# host unicorn runs on, and unlikely to detect disconnects even on a
# fast LAN.
check_client_connection false

before_fork do |server, worker|
  # the following is highly recomended for Rails + "preload_app true"
  # as there's no need for the master process to hold a connection
  defined?(ActiveRecord::Base) and
      ActiveRecord::Base.connection.disconnect!

  # The following is only recommended for memory/DB-constrained
  # installations.  It is not needed if your system can house
  # twice as many worker_processes as you have configured.
  #
  # # This allows a new master process to incrementally
  # # phase out the old master process with SIGTTOU to avoid a
  # # thundering herd (especially in the "preload_app false" case)
  # # when doing a transparent upgrade.  The last worker spawned
  # # will then kill off the old master process with a SIGQUIT.
  old_pid = "#{server.config[:pid]}.oldbin"
  if old_pid != server.pid
    begin
      sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
      Process.kill(sig, File.read(old_pid).to_i)
    rescue Errno::ENOENT, Errno::ESRCH
      # do nothing
    end
  end
  #
  # Throttle the master from forking too quickly by sleeping.  Due
  # to the implementation of standard Unix signal handlers, this
  # helps (but does not completely) prevent identical, repeated signals
  # from being lost when the receiving process is busy.
  # sleep 1
end

after_fork do |server, worker|
  ##
  # Unicorn master loads the app then forks off workers - because of the way
  # Unix forking works, we need to make sure we aren't using any of the parent's
  # sockets, e.g. db connection

  defined?(ActiveRecord::Base) and
      ActiveRecord::Base.establish_connection
  # Redis and Memcached would go here but their connections are established
  # on demand, so the master never opens a socket

  if worker_uid && worker_gid
    begin
      uid, gid = Process.euid, Process.egid
      target_uid = Etc.getpwnam(worker_uid).uid
      target_gid = Etc.getgrnam(worker_gid).gid
      worker.tmp.chown(target_uid, target_gid)
      if uid != target_uid || gid != target_gid
        Process.initgroups(worker_uid, target_gid)
        Process::GID.change_privilege(target_gid)
        Process::UID.change_privilege(target_uid)
      end
    rescue => e
      if rails_env == 'development'
        STDERR.puts "couldn't change user, oh well"
      else
        raise e
      end
    end
  end
end

unicornctl

#!/bin/sh

# Put the file into <YOUR APP>/bin directory and it should be executable.
#
# Examples:
#
#   RAILS_ENV=production bin/unicornctl start
#   RAILS_ENV=production bin/unicornctl stop

# set ruby GC parameters
RUBY_HEAP_MIN_SLOTS=600000
RUBY_FREE_MIN=200000
RUBY_GC_MALLOC_LIMIT=60000000
export RUBY_HEAP_MIN_SLOTS RUBY_FREE_MIN RUBY_GC_MALLOC_LIMIT

relative_path=`dirname "$0"`
rails_root="`(cd \"${relative_path}/../\" && pwd)`"

if [ -z "$RAILS_ENV" ]; then RAILS_ENV="production"; fi
pid="${rails_root}/tmp/pids/unicorn.pid"

case "$1" in
  start)
    if [ -f "$pid" ]; then
      echo "** PID file already exists. Please make sure the PID `cat $pid` is gone, and remove the \"$pid\" file."
    else
      cd $rails_root && bundle exec unicorn_rails -c ${rails_root}/config/unicorn.rb -E $RAILS_ENV -D
    fi
    ;;
  stop)
    kill -QUIT `cat $pid` && rm -f "$pid"
    ;;
  restart)
    $0 stop
    while [ -f "$pid" ]
    do
      sleep 1
    done
    $0 start
    ;;
  reload)
    kill -USR2 `cat $pid`
    ;;
  force-stop)
    kill -INT `cat $pid` && rm -f "$pid"
    ;;
  shutdown-workers)
    kill -WINCH `cat $pid`
    ;;
  increment-worker)
    kill -TTIN `cat $pid`
    ;;
  decrement-worker)
    kill -TTOU `cat $pid`
    ;;
  logrotate)
    kill -USR1 `cat $pid`
    ;;
  *)
    echo "Usage: $0 <start|stop|restart|reload|force-stop|shutdown-workers|increment-worker|decrement-worker|logrotate>"
    ;;
esac

exit 0