Analyze puppet's apache logs.

puppet apache log analyzer

#!/usr/bin/perl

use strict;
use warnings;
use Data::Dumper;

my $file = $ARGV[0] or die('Must pass an apache access logfile');

open IN, "<$file" or die("Can't open $file");

my %indirectors;

while(my $line=<IN>) {
    chomp($line);
    my @pieces = split(/\s+/, $line);
#    print Dumper(\@pieces);
    my $method = $pieces[5];
    $method=~s/"//g;
    my ($blah,$env, $indirector, $resource) = split(/\//, $pieces[6], 4);
    #print "$env|$indirector|$resource\n";
    my $ms = $pieces[10]/1000; # convert to milliseconds

    my $key = sprintf('%-4s %s', $method, $indirector);
    $indirectors{$key}{'count'}++;
    $indirectors{$key}{'sum'}+= $ms;
    push(@{$indirectors{$key}{'raw'}}, $ms);
    $indirectors{$key}{'min'} = $ms if(!defined($indirectors{$key}{'min'}) || $ms < $indirectors{$key}{'min'});
    $indirectors{$key}{'max'} = $ms if(!defined($indirectors{$key}{'max'}) || $ms > $indirectors{$key}{'max'});
}

print "Counts Per Indirector:\n";
foreach my $indirector(sort {$indirectors{$b}{'count'} <=> $indirectors{$a}{'count'}} keys %indirectors) {
    printf("  %7d %s\n", $indirectors{$indirector}{'count'}, $indirector);
}

print "\n\n";
print "Statistics by Indirector (milliseconds):\n";

printf("  %-25s %7s %7s %7s %7s\n", '', 'AVERAGE', 'STD_DEV','MIN', 'MAX');
foreach my $indirector(sort keys %indirectors) {
    $indirectors{$indirector}{'avg'} = $indirectors{$indirector}{'sum'}/$indirectors{$indirector}{'count'};

    my $sum_squares=0;
    foreach my $value(@{$indirectors{$indirector}{'raw'}}) {
        $sum_squares += ($value - $indirectors{$indirector}{'avg'})**2;
    }

    $indirectors{$indirector}{'stddev'} = ($indirectors{$indirector}{'count'} - 1) == 0
        ? 0
        : sqrt($sum_squares / ($indirectors{$indirector}{'count'} - 1));

    printf("  %-25s %7d %7d %7d %7d\n",
        $indirector,
        $indirectors{$indirector}{'avg'},
        $indirectors{$indirector}{'stddev'},
        $indirectors{$indirector}{'min'},
        $indirectors{$indirector}{'max'}
    );
}

This was inspired by this post: http://paperairoplane.net/?p=380

To run it: /path/to/script.pl /path/to/access_log

Example output:

Counts Per Indirector:
   589952 GET  file_metadata
    10380 GET  file_metadatas
    10289 PUT  report
    10280 POST catalog
      155 HEAD file_bucket_file
       27 PUT  file_bucket_file
        2 GET  facts
        1 GET  file_content


Statistics by Indirector (milliseconds):
                            AVERAGE STD_DEV     MIN     MAX
  GET  facts                      5       4       2       8
  GET  file_content              29       0      29      29
  GET  file_metadata              5      65       1   11961
  GET  file_metadatas            42     126       6    4825
  HEAD file_bucket_file           4       9       1     109
  POST catalog                 6873    9664      55  169981
  PUT  file_bucket_file          59      43      39     232
  PUT  report                  1851    3964      59   62103