Create a gist now

Instantly share code, notes, and snippets.

Analyze puppet's apache logs.
#!/usr/bin/perl
use strict;
use warnings;
use Data::Dumper;
my $file = $ARGV[0] or die('Must pass an apache access logfile');
open IN, "<$file" or die("Can't open $file");
my %indirectors;
while(my $line=<IN>) {
chomp($line);
my @pieces = split(/\s+/, $line);
# print Dumper(\@pieces);
my $method = $pieces[5];
$method=~s/"//g;
my ($blah,$env, $indirector, $resource) = split(/\//, $pieces[6], 4);
#print "$env|$indirector|$resource\n";
my $ms = $pieces[10]/1000; # convert to milliseconds
my $key = sprintf('%-4s %s', $method, $indirector);
$indirectors{$key}{'count'}++;
$indirectors{$key}{'sum'}+= $ms;
push(@{$indirectors{$key}{'raw'}}, $ms);
$indirectors{$key}{'min'} = $ms if(!defined($indirectors{$key}{'min'}) || $ms < $indirectors{$key}{'min'});
$indirectors{$key}{'max'} = $ms if(!defined($indirectors{$key}{'max'}) || $ms > $indirectors{$key}{'max'});
}
print "Counts Per Indirector:\n";
foreach my $indirector(sort {$indirectors{$b}{'count'} <=> $indirectors{$a}{'count'}} keys %indirectors) {
printf(" %7d %s\n", $indirectors{$indirector}{'count'}, $indirector);
}
print "\n\n";
print "Statistics by Indirector (milliseconds):\n";
printf(" %-25s %7s %7s %7s %7s\n", '', 'AVERAGE', 'STD_DEV','MIN', 'MAX');
foreach my $indirector(sort keys %indirectors) {
$indirectors{$indirector}{'avg'} = $indirectors{$indirector}{'sum'}/$indirectors{$indirector}{'count'};
my $sum_squares=0;
foreach my $value(@{$indirectors{$indirector}{'raw'}}) {
$sum_squares += ($value - $indirectors{$indirector}{'avg'})**2;
}
$indirectors{$indirector}{'stddev'} = ($indirectors{$indirector}{'count'} - 1) == 0
? 0
: sqrt($sum_squares / ($indirectors{$indirector}{'count'} - 1));
printf(" %-25s %7d %7d %7d %7d\n",
$indirector,
$indirectors{$indirector}{'avg'},
$indirectors{$indirector}{'stddev'},
$indirectors{$indirector}{'min'},
$indirectors{$indirector}{'max'}
);
}
@jasonhancock
Owner

This was inspired by this post: http://paperairoplane.net/?p=380

To run it: /path/to/script.pl /path/to/access_log

Example output:

Counts Per Indirector:
   589952 GET  file_metadata
    10380 GET  file_metadatas
    10289 PUT  report
    10280 POST catalog
      155 HEAD file_bucket_file
       27 PUT  file_bucket_file
        2 GET  facts
        1 GET  file_content


Statistics by Indirector (milliseconds):
                            AVERAGE STD_DEV     MIN     MAX
  GET  facts                      5       4       2       8
  GET  file_content              29       0      29      29
  GET  file_metadata              5      65       1   11961
  GET  file_metadatas            42     126       6    4825
  HEAD file_bucket_file           4       9       1     109
  POST catalog                 6873    9664      55  169981
  PUT  file_bucket_file          59      43      39     232
  PUT  report                  1851    3964      59   62103

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment