Starman Killer

starman-killer.pl

#!/usr/bin/env perl
# Kill deadlocked starman processes.
# See https://github.com/miyagawa/Starman/pull/44, and
#     https://github.com/miyagawa/Plack/issues/208 for possible reason.
# Solution: Use Starlet maybe?
use strict;
use warnings;

my $email_to = 'foo@example.com';
my $master_pid_file = '/tmp/starman-master.pid';

my $max_time = '00:16:00';
my $max_cpu_percentage = 80;
my $debug = 0;
# Uncomment these for debugging:
# $debug = 1;
# $max_time = '00:00:01';
# $max_cpu_percentage = 10;

my $ps_cmd = 'ps -o pid,time,pcpu,comm --pid `pgrep -d , starman`';
chomp (my $master_pid = `cat $master_pid_file`);

chomp (my $date = `date`);
my @mail_content = ("$date");

open(my $pid_time, '-|', $ps_cmd)
	or die "Could not start process [$ps_cmd]: $!";

chomp (my $header = <$pid_time>);
push @mail_content, $header;

my @pid_to_kill;
my $send_email = 0;
while(my $line = <$pid_time>) {
	chomp $line;
	push @mail_content, $line;
	$line =~ s/^\s*//;
	$line =~ s/\s*$//;
	my ($pid, $time, $pcpu, $starman, $role) = split(/\s\s*/, $line);
	if ($role eq 'master') {
		next;
	}
	# Check for these conditions:
	# 1. Time is more than a day
	# 2. Time is greater than $max_time
	# 3. Cpu usage percentage is more than $max_cpu_percentage
	if (   ($time !~ /^\d\d:\d\d:\d\d$/ || $time gt $max_time)
		&& $pcpu > $max_cpu_percentage) {
		$mail_content[$#mail_content] .= ' <-- DEADLOCKED';
		push @pid_to_kill, $pid;
	}
}
close $pid_time;

if (scalar @pid_to_kill > 0) {
	my $kill_command = "kill -9 " . join(' ', @pid_to_kill);
	my $mail_message = join("\n", @mail_content);
	if ($debug) {
		system(qq(
			(
				echo "$mail_message"
				echo
				echo "$kill_command"
			)
		));
	} else {
		system(qq(
			(
				echo "$mail_message"
				echo
				echo "Deadlock threshold:"
				echo "  TIME: $max_time"
				echo "   CPU: $max_cpu_percentage"
				echo
				echo "$kill_command"
				$kill_command
			) | mailx -s '(CRON) Starman Assassin - $date' $email_to
		));
	}
}

# Report if there are no starman processes running
my $report_dead_starman = qq(
if ! pgrep starman >/dev/null 2>&1; then
	echo 'No starman is running! Please check live server' |
		mailx $email_to -s 'No starman is running?'
fi
);
system($report_dead_starman);