evandhoffman/nginx.conf

## nginx.conf
# Logformat fed into the script is this:

    log_format  cachemiss  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for" "$upstream_cache_status"';

## spidercounter.pl
#!/usr/bin/perl

use Text::CSV_XS;
use Time::HiRes qw/gettimeofday/;

my @rows;
my $csv = Text::CSV_XS->new ( { binary => 1, sep_char => ' ' })
or die "Cannot use CSV: ".Text::CSV->error_diag ();

my $user_agent = 9;
my $request = 5;
my $referrer = 8;
my $cache_status = 11;

my %bots = ();
my %cache_status = ();

my $line = 0;
my $last_time = gettimeofday();
my $interval = 10000;

my %cache_statuses = qw/HIT 1 MISS 1 EXPIRED 1 - 1 BYPASS 1/;
my $engine_regex = lc(join '|', qw( Searcharoo spinn3r majestic12.co.uk ezooms Gigabot baidu Googlebot msnbot slurp|exabot pipl 80legs bingbot ahrefs yandexbot discobot pingdom websitepulse wget curl mediapartners-google sosospider sogou bingpreview), 'google web preview');

$max_engine_name_length = 0;

print "Engine regex: $engine_regex\n";
while(<>) {
        chomp;
        $csv->parse ($_);
        @rows = $csv->fields();
        my $engine = lc(($rows[$user_agent] =~ /($engine_regex)/i)[0]);
        $engine = "(no engine)" unless $engine;
        if (length($engine) > $max_engine_name_length) { $max_engine_name_length = length($engine); };
#       my $engine = ($_ =~ /((\w+)(\s?)(bot|spider|robot))/)[0];

        my $cache = $rows[$cache_status];

        if (exists($bots{$engine})) {
                if (exists($bots{$engine}{$cache})) {
                        $bots{$engine}{$cache} = $bots{$engine}{$cache} + 1;
                        $bots{$engine}{'count'} = $bots{$engine}{'count'} + 1;
                } else {
                        $bots{$engine}{$cache} = 1;
                }
        } else {
                $cache_statuses{$cache} = 1;    # Add this to the hash if it isn't already there
                $bots{$engine} = ();
                foreach $s (keys %cache_statuses) {
                        $bots{$engine}{$s} = 0;
                }
                $bots{$engine}{'count'} = 1;
        }

        if (exists($cache_status{$cache})) {
                $cache_status{$cache} = $cache_status{$cache} + 1;
        } else {
                $cache_status{$cache} = 1;
        }

        $line++;
        if (($line % $interval) == 0) {
                my $now = gettimeofday();
                my $elapsed = $now - $last_time;
                $last_time = $now;
                my $rate = $interval / $elapsed;
                print "line $line, rate: $rate\n";

                print "------- RESULTS -----------\n";
                print "Total: $line\n";
                my $width = $max_engine_name_length + 18;
                printf("%".$width."s","header");
                foreach $st (sort keys %cache_statuses) {
                        my $width = 18;
                        printf("| %".$width."s",$st);
                }
                print "\n";

                foreach $bot (sort keys %bots) {
                        my $cnt = $bots{$bot}{'count'};
                        my $pct = $cnt * 100 / $line;
                        printf('%'.($max_engine_name_length+1)."s%8d%7.2f%% ",$bot,$cnt,$pct);
                        #foreach $c (sort keys %{$bots{$bot}}) {
                        foreach $c (sort keys %cache_statuses) {
#                               next if $c eq 'count';
                                my $hitcnt = $bots{$bot}{$c};
                                my $pct = $hitcnt * 100/ $cnt;
                                printf('|%8d (%6.2f%%) ',$hitcnt,$pct);
                        }
                        print "\n";
                }
                print "--------- Cache Stats --------------\n";
                foreach $status (sort keys %cache_status) {
                        my $pct = $cache_status{$status} * 100 / $line;
                        printf("%10s\t%7d\5%8.2f%%\n",$status,$cache_status{$status},$pct);
                }
        }
}
	# Logformat fed into the script is this:

	log_format cachemiss '$remote_addr - $remote_user [$time_local] "$request" '
	'$status $body_bytes_sent "$http_referer" '
	'"$http_user_agent" "$http_x_forwarded_for" "$upstream_cache_status"';
	#!/usr/bin/perl

	use Text::CSV_XS;
	use Time::HiRes qw/gettimeofday/;

	my @rows;
	my $csv = Text::CSV_XS->new ( { binary => 1, sep_char => ' ' })
	or die "Cannot use CSV: ".Text::CSV->error_diag ();

	my $user_agent = 9;
	my $request = 5;
	my $referrer = 8;
	my $cache_status = 11;

	my %bots = ();
	my %cache_status = ();

	my $line = 0;
	my $last_time = gettimeofday();
	my $interval = 10000;

	my %cache_statuses = qw/HIT 1 MISS 1 EXPIRED 1 - 1 BYPASS 1/;
	my $engine_regex = lc(join '\|', qw( Searcharoo spinn3r majestic12.co.uk ezooms Gigabot baidu Googlebot msnbot slurp\|exabot pipl 80legs bingbot ahrefs yandexbot discobot pingdom websitepulse wget curl mediapartners-google sosospider sogou bingpreview), 'google web preview');

	$max_engine_name_length = 0;

	print "Engine regex: $engine_regex\n";
	while(<>) {
	chomp;
	$csv->parse ($_);
	@rows = $csv->fields();
	my $engine = lc(($rows[$user_agent] =~ /($engine_regex)/i)[0]);
	$engine = "(no engine)" unless $engine;
	if (length($engine) > $max_engine_name_length) { $max_engine_name_length = length($engine); };
	# my $engine = ($_ =~ /((\w+)(\s?)(bot\|spider\|robot))/)[0];

	my $cache = $rows[$cache_status];

	if (exists($bots{$engine})) {
	if (exists($bots{$engine}{$cache})) {
	$bots{$engine}{$cache} = $bots{$engine}{$cache} + 1;
	$bots{$engine}{'count'} = $bots{$engine}{'count'} + 1;
	} else {
	$bots{$engine}{$cache} = 1;
	}
	} else {
	$cache_statuses{$cache} = 1; # Add this to the hash if it isn't already there
	$bots{$engine} = ();
	foreach $s (keys %cache_statuses) {
	$bots{$engine}{$s} = 0;
	}
	$bots{$engine}{'count'} = 1;
	}

	if (exists($cache_status{$cache})) {
	$cache_status{$cache} = $cache_status{$cache} + 1;
	} else {
	$cache_status{$cache} = 1;
	}

	$line++;
	if (($line % $interval) == 0) {
	my $now = gettimeofday();
	my $elapsed = $now - $last_time;
	$last_time = $now;
	my $rate = $interval / $elapsed;
	print "line $line, rate: $rate\n";

	print "------- RESULTS -----------\n";
	print "Total: $line\n";
	my $width = $max_engine_name_length + 18;
	printf("%".$width."s","header");
	foreach $st (sort keys %cache_statuses) {
	my $width = 18;
	printf("\| %".$width."s",$st);
	}
	print "\n";

	foreach $bot (sort keys %bots) {
	my $cnt = $bots{$bot}{'count'};
	my $pct = $cnt * 100 / $line;
	printf('%'.($max_engine_name_length+1)."s%8d%7.2f%% ",$bot,$cnt,$pct);
	#foreach $c (sort keys %{$bots{$bot}}) {
	foreach $c (sort keys %cache_statuses) {
	# next if $c eq 'count';
	my $hitcnt = $bots{$bot}{$c};
	my $pct = $hitcnt * 100/ $cnt;
	printf('\|%8d (%6.2f%%) ',$hitcnt,$pct);
	}
	print "\n";
	}
	print "--------- Cache Stats --------------\n";
	foreach $status (sort keys %cache_status) {
	my $pct = $cache_status{$status} * 100 / $line;
	printf("%10s\t%7d\5%8.2f%%\n",$status,$cache_status{$status},$pct);
	}
	}
	}