bluepill.log

Nov 24 02:21:47 Dev001 bluepilld[16400]: [mcp:mcp:sark2] Going from starting => down
Nov 24 02:21:47 Dev001 bluepilld[16400]: [mcp:mcp:sark3] Going from starting => down
Nov 24 02:21:47 Dev001 bluepilld[16400]: [mcp:mcp:sark4] Going from starting => down
Nov 24 02:21:47 Dev001 bluepilld[16400]: [mcp:mcp:sark5] Going from starting => down
Nov 24 02:21:47 Dev001 bluepilld[16400]: [mcp:mcp:sark1] Going from starting => down
Nov 24 02:21:48 Dev001 bluepilld[16400]: [mcp:mcp:sark2] Executing start command: /prod/mcp/releases/20091124013157/app/services/runners/sark.rb -c
 /prod/mcp/releases/20091124013157/config/homeq.cfg -q sark2 -P /prod/mcp/releases/20091124013157/tmp/pids/sark-29016.pid -f 
Nov 24 02:21:48 Dev001 bluepilld[16400]: [mcp:mcp:sark2] Going from down => starting
Nov 24 02:21:48 Dev001 bluepilld[16400]: [mcp:mcp:sark3] Executing start command: /prod/mcp/releases/20091124013157/app/services/runners/sark.rb -c
 /prod/mcp/releases/20091124013157/config/homeq.cfg -q sark3 -P /prod/mcp/releases/20091124013157/tmp/pids/sark-29017.pid -f 
Nov 24 02:21:48 Dev001 bluepilld[16400]: [mcp:mcp:sark3] Going from down => starting
Nov 24 02:21:48 Dev001 bluepilld[16400]: [mcp:mcp:sark4] Executing start command: /prod/mcp/releases/20091124013157/app/services/runners/sark.rb -c
 /prod/mcp/releases/20091124013157/config/homeq.cfg -q sark4 -P /prod/mcp/releases/20091124013157/tmp/pids/sark-29018.pid -f 
Nov 24 02:21:48 Dev001 bluepilld[16400]: [mcp:mcp:sark4] Going from down => starting
Nov 24 02:21:48 Dev001 bluepilld[16400]: [mcp:mcp:sark5] Executing start command: /prod/mcp/releases/20091124013157/app/services/runners/sark.rb -c
 /prod/mcp/releases/20091124013157/config/homeq.cfg -q sark5 -P /prod/mcp/releases/20091124013157/tmp/pids/sark-29019.pid -f 
Nov 24 02:21:48 Dev001 bluepilld[16400]: [mcp:mcp:sark5] Going from down => starting
Nov 24 02:21:48 Dev001 bluepilld[16400]: [mcp:mcp:sark1] Executing start command: /prod/mcp/releases/20091124013157/app/services/runners/sark.rb -c
 /prod/mcp/releases/20091124013157/config/homeq.cfg -q sark1 -P /prod/mcp/releases/20091124013157/tmp/pids/sark-29015.pid -f 
Nov 24 02:21:48 Dev001 bluepilld[16400]: [mcp:mcp:sark1] Going from down => starting
Nov 24 02:21:51 Dev001 bluepilld[16400]: [mcp:mcp:sark2] Going from starting => up
Nov 24 02:21:51 Dev001 bluepilld[16400]: [mcp:mcp:sark2] mem_usage: [31420.0]
Nov 24 02:21:52 Dev001 bluepilld[16400]: [mcp:mcp:sark2] cpu_usage: [39.5*]
Nov 24 02:21:52 Dev001 bluepilld[16400]: [mcp:mcp:sark3] Going from starting => up
Nov 24 02:21:52 Dev001 bluepilld[16400]: [mcp:mcp:sark3] mem_usage: [23128.0]
Nov 24 02:21:52 Dev001 bluepilld[16400]: [mcp:mcp:sark3] cpu_usage: [29.7*]
Nov 24 02:21:52 Dev001 bluepilld[16400]: [mcp:mcp:sark4] Going from starting => up
Nov 24 02:21:52 Dev001 bluepilld[16400]: [mcp:mcp:sark4] mem_usage: [30328.0]
Nov 24 02:21:52 Dev001 bluepilld[16400]: [mcp:mcp:sark4] cpu_usage: [35.7*]
Nov 24 02:21:52 Dev001 bluepilld[16400]: [mcp:mcp:sark5] Going from starting => up
Nov 24 02:21:52 Dev001 bluepilld[16400]: [mcp:mcp:sark5] mem_usage: [23160.0]
Nov 24 02:21:52 Dev001 bluepilld[16400]: [mcp:mcp:sark5] cpu_usage: [40.6*]
Nov 24 02:21:52 Dev001 bluepilld[16400]: [mcp:mcp:sark1] Going from starting => up
Nov 24 02:21:52 Dev001 bluepilld[16400]: [mcp:mcp:sark1] mem_usage: [23260.0]
Nov 24 02:21:52 Dev001 bluepilld[16400]: [mcp:mcp:sark1] cpu_usage: [43.3*]
Nov 24 02:22:01 Dev001 bluepilld[16400]: [mcp:mcp:sark2] cpu_usage: [39.5*, 12.4*]
Nov 24 02:22:01 Dev001 bluepilld[16400]: [mcp:mcp:sark2] mem_usage: [31420.0, 33684.0]
Nov 24 02:22:02 Dev001 bluepilld[16400]: [mcp:mcp:sark3] cpu_usage: [29.7*, 11.2*]
Nov 24 02:22:02 Dev001 bluepilld[16400]: [mcp:mcp:sark3] mem_usage: [23128.0, 33348.0]
Nov 24 02:22:02 Dev001 bluepilld[16400]: [mcp:mcp:sark4] mem_usage: [30328.0, 33464.0]
Nov 24 02:22:02 Dev001 bluepilld[16400]: [mcp:mcp:sark4] cpu_usage: [35.7*, 11.4*]
Nov 24 02:22:02 Dev001 bluepilld[16400]: [mcp:mcp:sark5] mem_usage: [23160.0, 33648.0]
Nov 24 02:22:02 Dev001 bluepilld[16400]: [mcp:mcp:sark5] cpu_usage: [40.6*, 12.1*]
Nov 24 02:22:02 Dev001 bluepilld[16400]: [mcp:mcp:sark1] mem_usage: [23260.0, 33588.0]
Nov 24 02:22:02 Dev001 bluepilld[16400]: [mcp:mcp:sark1] cpu_usage: [43.3*, 12.3*]
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp:mcp:sark2] mem_usage: [31420.0, 33684.0, 33976.0]
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp:mcp:sark2] cpu_usage: [39.5*, 12.4*, 7.3*]
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp:mcp:sark2] cpu_usage dispatched: restart
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp:mcp:sark2] Flapping detected: retrying in 7 seconds
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp:mcp:sark2] Going from up => unmonitored
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] Got exception: StateMachine::InvalidTransition `Cannot transition state via :restart from :unmonitor
ed`
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/state_machine-0.8.0/lib/state_machine/event.rb:248:in `add_actions'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/state_machine-0.8.0/lib/state_machine/machine.rb:512:in `call'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/state_machine-0.8.0/lib/state_machine/machine.rb:512:in `restart!'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/../lib/bluepill/process.rb:138:in `send'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/../lib/bluepill/process.rb:138:in `dispatch!'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/1.8/monitor.rb:242:in `synchronize'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/../lib/bluepill/process.rb:137:in `dispatch!'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/../lib/bluepill/process.rb:189:in `run_watches'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/../lib/bluepill/process.rb:180:in `each'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/../lib/bluepill/process.rb:180:in `run_watches'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/../lib/bluepill/process.rb:120:in `tick'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/../lib/bluepill/group.rb:19:in `tick'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/../lib/bluepill/group.rb:18:in `each'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/../lib/bluepill/group.rb:18:in `tick'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/../lib/bluepill/application.rb:222:in `run'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/../lib/bluepill/application.rb:221:in `each'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/../lib/bluepill/application.rb:221:in `run'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/../lib/bluepill/application.rb:218:in `loop'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/../lib/bluepill/application.rb:218:in `run'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/../lib/bluepill/application.rb:214:in `start_server'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/../lib/bluepill/application.rb:22:in `load'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/../lib/bluepill/dsl.rb:71:in `application'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] (eval):66
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/bin/bluepill:19:in `eval'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/lib/ruby/gems/1.8/gems/bluepill-0.0.21/bin/bluepill:53
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/bin/bluepill:19:in `load'
Nov 24 02:22:11 Dev001 bluepilld[16400]: [mcp] /usr/bin/bluepill:19

bluepill_config.rb

#!/usr/bin/env ruby

# 
# Typical invocation:
# 
#   $ cd /prod/mcp/current
#   $ sudo bluepill load /prod/mcp/current/config/bluepill_config.rb

require 'rubygems'
require 'homeq'
topology        = HomeQ.calculated_homeq_topology
queues          = HomeQ.queue_list_for_host_from_topology

POD             = (ENV['MCP_POD'] || ENV['POD'] || 1).to_i
MCP_ROOT        = ENV['MCP_ROOT'] ||
                  File.expand_path(File.dirname(__FILE__))
HOMEQ_ENV       ||= ENV['HOMEQ_ENV']

#
# Figure out the directory where we're running.
#
parent_dir = File.expand_path(File.join(File.dirname(__FILE__), '..'))

def watch_it(app, process_name, port, queue_name)
  app.process(queue_name) do |process|
    pid_file = File.join(MCP_ROOT, "tmp/pids/#{process_name}-#{port}.pid")
    log_file = "#{MCP_ROOT}/log/#{queue_name}.log"
    cmd = "#{MCP_ROOT}/app/services/runners/#{process_name}.rb "
    cmd << "-c #{MCP_ROOT}/config/homeq.cfg "
    cmd << "-q #{queue_name} "
    cmd << "-P #{pid_file} "
    cmd << "-f "

    p cmd

    process.group = 'mcp'
    process.stop_command = "kill -INT {{PID}}"
    process.start_command = cmd
    process.pid_file = pid_file
    process.checks(:cpu_usage,
                   :every => 10.seconds,
                   :below => 5,
                   :times => 3)
    process.checks(:mem_usage,
                   :every => 10.seconds,
                   :below => 300.megabytes,
                   :times => [3,5])
    process.checks(:flapping,
                   :times => 2,
                   :within => 30.seconds,
                   :retry_in => 7.seconds)

    process.stdout = process.stderr = log_file

    process.working_dir = MCP_ROOT
    process.start_grace_time = 3.seconds
    process.stop_grace_time = 5.seconds
    process.restart_grace_time = 8.seconds
    process.daemonize = true

  end
end

require 'bluepill'

Bluepill.application("mcp") do |app|
  queues.each { |q|
    watch_it(app, q.gsub(/\d/, ''), topology[q][2], q)
  }
end