Skip to content

Instantly share code, notes, and snippets.

@csfrancis
Created July 28, 2016 18:49
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save csfrancis/02da80579605d4c70c112bc7f2ccf281 to your computer and use it in GitHub Desktop.
Save csfrancis/02da80579605d4c70c112bc7f2ccf281 to your computer and use it in GitHub Desktop.
#!/usr/bin/env ruby
NGINX_PID_FILE = "/var/run/nginx.pid"
NGINX_OLD_PID_FILE = "/var/run/nginx.pid.oldbin"
NGINX_CONF_FILE = "/etc/nginx/nginx.conf"
NGINX_BIN = "/usr/sbin/nginx"
LD_OVERRIDES = "/etc/nginx/ld-overrides"
DEFAULT_NGINX_WORKERS = 2
def with_term_color(color, eol: true)
@has_colors ||= `tput colors`.to_i > 0 ? true : false
$stdout.print "#{color}" if @has_colors
yield
$stdout.print "\e[0m" if @has_colors
$stdout.print "\n" if eol
end
def error(msg, should_exit: true)
with_term_color("\e[0;31m") { $stdout.print(msg) }
exit 1 if should_exit
end
def warning(msg)
with_term_color("\e[0;32m") { $stdout.print(msg) }
end
def with_timeout(seconds, timeout: 90, error_message: "error", warning_message: nil, exit_on_error: true)
count = 0
timeout.times do |i|
sleep 1
$stdout.print "."
count += 1
break if yield i
end
if count == timeout
if warning_message
warning(warning_message)
else
error(error_message, should_exit: exit_on_error)
end
else
$stdout.puts " done."
end
end
def get_child_processes(pid)
`pgrep -P #{pid}`.split.map{ |p| Integer(p) }
end
def get_process_cmdline(pids)
pids = [pids] unless pids.class == Array
pids.map do |pid|
begin
IO.read("/proc/#{pid}/cmdline").strip
rescue Errno::ENOENT
""
end
end
end
def get_nginx_master_pid
Integer(IO.read(NGINX_PID_FILE).chomp)
end
def get_nginx_config_value(key)
match = File.read(NGINX_CONF_FILE).match(/#{key}\s+(\S+);/)
match.captures.first if match
end
def get_num_workers(pid)
get_process_cmdline(get_child_processes(pid)).count { |p| p.start_with? "nginx: worker process" }
end
def test_nginx_config
ld_library_path = ""
if Dir.exist? LD_OVERRIDES
ld_library_path = "LD_LIBRARY_PATH=#{LD_OVERRIDES} "
end
$stdout.print("Testing nginx configuration .. ")
unless system("#{ld_library_path}#{NGINX_BIN} -c #{NGINX_CONF_FILE} -t > /dev/null 2>&1")
error "nginx configuration is invalid!"
end
$stdout.puts("done.")
end
def spawn_new_nginx_master(master_pid)
$stdout.print "Spawning new nginx master .."
Process.kill("USR2", master_pid)
old_master_pid = master_pid
with_timeout 20, error_message: "timed out waiting for new master to spawn!" do
if File.exist?(NGINX_OLD_PID_FILE) && File.exist?(NGINX_PID_FILE)
new_pid = get_nginx_master_pid
error("error spawning new master - old master still running.") if new_pid == old_master_pid
end
# Check if the new master process has spawned with correct number of workers
expected_workers = Integer(get_nginx_config_value('worker_processes')) || DEFAULT_NGINX_WORKERS
get_num_workers(get_nginx_master_pid) == expected_workers
end
end
def shutdown_nginx_workers(old_master_pid)
$stdout.print "Shutting down workers on master .."
Process.kill("WINCH", old_master_pid)
with_timeout 90, warning_message: "not all workers shut down on time - killing stragglers." do
get_process_cmdline(get_child_processes(old_master_pid)).all? { |p| p.start_with? "nginx: master process" }
end
%w(TERM KILL).each do |sig|
sleep 0.1
workers = get_child_processes(old_master_pid).select { |p| get_process_cmdline(p)[0].start_with? "nginx: worker process" }
workers.each do |pid|
warning "Sending #{pid} #{sig}."
Process.kill(sig, pid) rescue Errno::ESRCH
end
end
end
def shutdown_old_nginx_master(master_pid)
$stdout.print "Waiting for old master to shut down .."
Process.kill("QUIT", master_pid)
with_timeout 10, error_message: "timed out waiting for old master to shut down!" do
! File.exist? NGINX_OLD_PID_FILE
end
end
master_pid = get_nginx_master_pid
test_nginx_config
spawn_new_nginx_master(master_pid)
shutdown_nginx_workers(master_pid)
shutdown_old_nginx_master(master_pid)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment