cli-logchecker.pl

#!/usr/bin/perl

use strict;
use warnings;

use File::Basename;
use File::Temp qw/ tempfile /;
use IO::Handle;
use IO::File;
use Getopt::Long;
use Cwd qw/ realpath /;;
use Term::ReadKey;

use WWW::Mechanize;
use HTTP::Cookies::Netscape; 
use HTML::TreeBuilder;

#use Data::Dumper;

my $prog_name = basename($0);
my $prog_ver  = "0.9";
my $prog_agent = "$prog_name/$prog_ver";

#######################################################################
#######################################################################
###                                                                 ###
###   options and related functions                                 ###
###                                                                 ###
#######################################################################
#######################################################################

my %options = (
	# program verbosity
		"verbose"			=> 0,
		"quiet"				=> 1,

	# auth cookie creation and usage
		"auth_cookie"		=> "$ENV{HOME}/.what.cookie",
		"create_cookie"		=> "",
		"dont_save_cookie"	=> 0,
		"test_cookie"		=> 0,
		"logout_cookie"		=> 0,

	# logchecker.php output
		"delimeter"			=> "\t",

	# temporary files
		# we wan't all the files we create to be readable to the
		# current user only, this actually means cookies
		"old_umask"			=> umask(0077),

	# reading logs filenames
		"use_stdin"			=> 0,

	# interaction with logchecker.php
		"upload"			=> 1,
		"paste"				=> 0,
		"sleep_time"			=> 8,
		"timeout"			=> 180,
		"retry"				=> 5,
		"recover"			=> 0,

	# internal vars
		"debug"				=> 0,
		"time_started"			=> scalar localtime,

	# batch status
		"batch_report"			=> undef,
		"do_batch_report"		=> 0,
		"logs_succesfull"		=> [],
		"logs_unsuccesfull"		=> [],
	);

sub process_options {
	usage(0) if $options{usage};
	$options{quiet} = 0 if $options{verbose};
	$options{upload} = 0 if $options{paste};
	$options{logout_cookie} = 1 if $options{dont_save_cookie};
	$options{do_batch_report} = 1 if defined($options{batch_report}) and $options{batch_report} ne "";
}

sub usage($) {
	my $exit_code = shift;

	my $usage_str = <<"USAGE_TEXT";
Command line interface to logchecker.php, version $prog_ver

usage: $prog_name [options] <.log -files>

OPTIONS

Program verbosity:

	--verbose	Be verbose.


Auth cookie creation and usage:

  All of the options here that deal with cookie creation ask for your
 username and password, and will login to the site to create the
 cookie. If you don't give any of these options, you might need to
 create the default file ~/.what.cookie by using
 --create-auth-cookie=\$HOME/.what.cookie

  The cookie file is in Netscape format, so if you have one already,
 use it with this script.

	--create-auth-cookie=cookies.txt	Create auth cookie to
					cookies.txt, if not present
					~/.what.cookie is assumed.

	--auth-cookie=cookies.txt	Use auth cookie from cookies.txt, if
			not present ~/.what.cookie is used. If the file doesn't
			exist, assume --create-auth-cookie.

	--dont-save-auth-cookie		Create an auth cookie and discard it
			after this session.

	--test-cookie			Test if login is succesfull, do not do
					any interaction with logchecker.php.

	--logout-cookie			Log out the cookie used, automatically
					done if --dont-save-auth-cookie is used.


Logchecker.php output:

	The results are presented in the following format:

	log's filename <delimeter> score <delimeter> reasons for score
	adjusts (separated by <delimeter>)


Logchecker.php interaction:

	--paste		Instead of uploading log, paste it to the text box.

	--sleep=s	Sleep s seconds between submissions, the default
			value is $options{sleep_time} seconds.


Connection setup:

	--timeout=s	Timeout after s seconds, the default value is $options{timeout}.

	--retry=N	Retry N times if failing to get logchecker.php, defaults to 5.
			Sleep --sleep seconds between attempts.

	--dont-panic	Don't allow error in logchecker interaction to cause fatal error.

Batch setup:

	--stdin		Read log filenames from stdin.

	--report=file	Save batch report to a file. This is a separate file from log
			scores! To save those you must redirect output to a file.

USAGE_TEXT

	print STDERR $usage_str if 		$exit_code;
	print STDOUT $usage_str unless	$exit_code;

	exit($exit_code);
}

#######################################################################
#######################################################################
###                                                                 ###
###   cookie creation and validation                                ###
###                                                                 ###
#######################################################################
#######################################################################

sub ask_for_login {

	print STDERR "$prog_name: Please enter your username and password:\n" if $options{verbose};

	# get username
	STDOUT->printflush("user: ");
	my $username = <STDIN>;
	chomp($username);

	# get password, don't echo characters
	print "password: ";
	ReadMode 'noecho';
	my $password = ReadLine 0;
	chomp $password;
	ReadMode 'normal';
	print "\n";

	($username, $password);
}

sub check_auth_cookie($) {
	my $cookie_file = shift;

	# basic check
	unless ( -f $cookie_file or -e _ ) {
		print STDERR "ERROR: File $cookie_file can't be read.\n";
		exit(1);
	}

	my $mech = new WWW::Mechanize(
		cookie_jar => HTTP::Cookies::Netscape->new( file => $options{auth_cookie} ),
		timeout => $options{timeout},
	);
	$mech->agent($prog_agent);

	print STDERR "Retrieving logchecker.php.\n" if $options{verbose};
	$mech->get("http://what.cd/logchecker.php");

	# turn warnings off now, since we hope -not- to find the
	# form, so a warning just annoys
	$mech->quiet(1);
	my $form = $mech->form_with_fields( ('username','password') );

	my $status = 1;
	$status = 0 if ( defined($form) );

	$status;
}

sub create_cookie_file($) {

	if ( $options{use_stdin} ) {
		print STDERR "ERROR: Can't create cookie interactively while using --stdin. Exiting...\n";
		exit(1);
	}

	my $cfh = shift;

	print STDERR "Creating cookie to $cfh.\n" if $options{verbose};

	my $mech = new WWW::Mechanize(
		cookie_jar => HTTP::Cookies::Netscape->new( file => $options{auth_cookie}, autosave => 1 ),
		timeout => $options{timeout},
	);
	$mech->agent($prog_agent);

	print STDERR "Trying to fetch login page.\n" if $options{verbose};

	$mech->get("http://what.cd/login.php");

	my $form = $mech->form_with_fields( ('username','password') );
	if ( defined($form) ) {
		(my $user, my $pw) = &ask_for_login;
		print STDERR "Trying to log in.\n" if $options{verbose};
		$mech->set_fields(
			username => $user,
			password => $pw,
			keeplogged => 1,
		);
		$mech->submit();
	}

	1;
}

sub logout_cookie($) {
	print STDERR "Logging out.\n" if $options{verbose};

	my $mech = new WWW::Mechanize(
		cookie_jar => HTTP::Cookies::Netscape->new( file => $options{auth_cookie}, autosave => 1 ),
		timeout	=> $options{timeout},
	);
	$mech->agent($prog_agent);

	$mech->get("http://what.cd/");
	$mech->follow_link( text => 'Logout' );


}


#######################################################################
#######################################################################
###                                                                 ###
###   submitting log to logchecker.php and parsing its output       ###
###                                                                 ###
#######################################################################
#######################################################################

sub retrieve_logchecker_score($) {
	my $log = shift;

	print STDERR "Checking $log, real path is " . realpath($log) . "\n" if $options{verbose};

	$log = realpath($log);

	my $retries = $options{retry};

	my $mech = new WWW::Mechanize(
		cookie_jar => HTTP::Cookies::Netscape->new( file => $options{auth_cookie} ),
		timeout => $options{timeout},
	);
	$mech->agent($prog_agent);

	my $html_tree = HTML::TreeBuilder->new;

	print STDERR "Retrieving logchecker.php." if $options{verbose};
	my $success = 0;
	while ( not $success and $retries ) {
		eval {
			my $attempt = $mech->get("http://what.cd/logchecker.php");
			$attempt->is_success or die $attempt->status_line;
		};
		if ( $@ ) {
			print STDERR "WARNING: Attempt ", $options{retry} - $retries + 1, " of $options{retry} to get logchecker.php for log \"$log\" failed for reason: $@";
			if ( $retries > 1 and $options{sleep_time}) {
				print STDERR "Sleeping $options{sleep_time} seconds before attempting again.\n" if $options{verbose};
				sleep($options{sleep_time});
			}
		}
		else {
			$success = 1;
		}
		--$retries;
	}
	if ( $success ) {
		my $form = $mech->form_with_fields( ('log','log_contents') );
		#my $form = $mech->form_number(7);
		if ( defined($form) ) {
			print STDERR "Submitting logchecker.php form.\n" if $options{verbose};
			$form->dump if $options{debug};
			if ( $options{upload} ) {
				# upload the file
				$mech->field('log' => $log);
			} else {
				# paste the file, I usually would trust the upload way
				# more, but as there was this one incident where uploads
				# didn't work at all, I decided to add this as a workaround
				my $log_contents = "";
				open(my $lcfh, "<", $log) or die "Couldn't open file: $!";
				$log_contents .= $_ while (<$lcfh>);
				$mech->field('log_contents', $log_contents )
			}
			
			my $submit_success = 0;
			my $submit_retries = $options{retry};
			while ( not $submit_success and $submit_retries ) {
				eval {
					my $submit_attempt = $mech->submit();
					$submit_attempt->is_success() or die $submit_attempt->status_line();
					$html_tree->parse_content($submit_attempt->decoded_content);
				};
				if ( $@ ) {
					print STDERR "WARNING: Attempt ", $options{retry} - $submit_retries + 1, " of $options{retry} to submit log \"$log\" failed for reason: $@";
					if ( $submit_retries > 1 and $options{sleep_time}) {
						print STDERR "Sleeping $options{sleep_time} seconds before attempting again.\n" if $options{verbose};
						sleep($options{sleep_time});
					}
				}
				else {
					$submit_success = 1;
				}
				--$submit_retries;
			}
			#$html_tree->parse_content($mech->submit()->decoded_content);
		} else {
			print STDERR "ERROR: Couldn't find the logchecker.php submit form!\n";
			print $mech->content(format => 'text') if $options{debug};
			$html_tree->delete();
			exit(1);
		}
		return ($log, $html_tree);
	}
	
	# error return
	print STDERR "WARNING: Returning undef!\n" if $options{verbose};
	return ($log, undef);
}

sub format_score {
	my $log = shift;
	my $html = shift;

	unless ( defined($html) ) {
		print STDERR "ERROR: Received undefined ParseTree.\n" if $options{verbose};
		exit(1) unless $options{recover};
		push(@{$options{logs_unsuccesfull}}, $log);
		return "";
	}
	else {
	
		push(@{$options{logs_succesfull}}, $log);

		$html->dump if $options{debug};

		# the report is under a tag that has id='content'
		# so we'll search for that
		my $content = $html->look_down('_tag', 'div', sub {
			$_[0]->attr('id') eq "content";
		});

		# try to find out if there was an error of some kind
		my $error = $content->look_down('_tag', 'h2');
		my $ret = "";
		if ( defined($error) and $error->as_text =~ /error/i ) {
			my $error_str = $content->look_down('class', 'box pad')->as_text;
			print STDERR "There was an error uploading the log: $error_str\n";
			$ret = "$log$options{delimeter}0$options{delimeter}$error_str\n";
		} else {
		
			my $score = $content->look_down('_tag', 'span')->as_text;
			my @res = $content->look_down('_tag', 'li');
			my @reasons = ();
			if ( @res ) {
				$reasons[$_] = $res[$_]->as_text for (0..$#res);
			}
			$ret = "$log$options{delimeter}$score$options{delimeter}" . join($options{delimeter}, @reasons) . "\n";
		}

		$html->delete();
		return $ret;
	}
}

#######################################################################
#######################################################################
###                                                                 ###
###   main                                                          ###
###                                                                 ###
#######################################################################
#######################################################################

my $result = GetOptions (
				"verbose"		=> \$options{verbose},
				"quiet"			=> \$options{quiet},
				"create-auth-cookie=s"	=> \$options{create_cookie},
				"auth-cookie=s"		=> \$options{auth_cookie},
				"dont-save-auth-cookie"	=> \$options{dont_save_cookie},
				"test-cookie"		=> \$options{test_cookie},
				"usage|help"    	=> \$options{usage},
				"logout-cookie"		=> \$options{logout_cookie},
				"sleep=i"		=> \$options{sleep_time},
				"timeout=i"		=> \$options{timeout},
				"stdin"			=> \$options{use_stdin},
				"paste"			=> \$options{paste},
				"retry=i"		=> \$options{retry},
				"report=s"		=> \$options{batch_report},
				"dont-panic"		=> \$options{recover},
);

&process_options;

# if we're not saving the cookie, use temporary file
if ( $options{dont_save_cookie} and not $options{test_cookie} ) {
	$options{create_cookie} = "";
	(my $fh_cookie, my $cookie_filename) = tempfile();
	$options{auth_cookie} = $cookie_filename;

	create_cookie_file $cookie_filename;
}

# we're creating lasting cookie file
if ( $options{create_cookie} ne "" ) {
	$options{auth_cookie} = $options{create_cookie};

	create_cookie_file $options{auth_cookie};
}

# check that we really can access the site with this cookie
unless( check_auth_cookie $options{auth_cookie} ) {
	print STDERR "ERROR: Couldn't get in with cookies in $options{auth_cookie}!\n";
	exit(1);
}

# only checking if the cookie is valid, no need to process further
if ( $options{test_cookie} ) {
	logout_cookie($options{auth_cookie}) if $options{logout_cookie};
	unlink($options{auth_cookie}) if $options{dont_save_cookie};
	exit(0);
}


# read logs from command line, or stdin if so desired
my @all_logs;
if ( $options{use_stdin} ) {
	@all_logs = <STDIN>;
	chomp(@all_logs);
} else {
	@all_logs = @ARGV;
}

# process only those logs that we can read
my @logs = grep { -r $_ } @all_logs;

print STDERR "WARNINGS: Unable to process all given logs." if @logs != @all_logs and $options{verbose};

my $reportfh = undef;
if ( $options{do_batch_report} ) {
	print STDERR "Opening \"$options{batch_report}\" for writing batch information!\n" if $options{verbose};
	open($reportfh, ">", $options{batch_report}) or die "Couldn't open file \"$options{batch_report}\" for writing batch report: $!";
}

# process all logs
for ( 0 .. $#logs ) {
	sleep($options{sleep_time}) if $_;;
	print &format_score(retrieve_logchecker_score $logs[$_]);
}

if ( defined($reportfh) ) {
	print {$reportfh} $_ for
		("BATCH CONNECTION REPORT\n\n",
		"STARTED: ", $options{time_started}, "\n",
		"ENDED: ", scalar localtime, "\n\n",
		"LOGS\n",
		"\tGIVEN: ", scalar @all_logs, "\n",
		"\tACCEPTED: ", scalar @logs, "\n\n",
		"PROCESSED\n",
		"\tSUCCESFULLY: ", scalar @{$options{logs_succesfull}}, "\n",
		"\tUNSUCCESFULLY: ", scalar @{$options{logs_unsuccesfull}}, "\n\n",
		"FAILED TO SCORE LOGS:\n",
		join("\n", @{$options{logs_unsuccesfull}}), "\n");
}

# finally, if we're not saving the cookie, delete it
logout_cookie($options{auth_cookie}) if $options{logout_cookie};
unlink($options{auth_cookie}) if $options{dont_save_cookie};