script which splits bmo syslog files

split.pl

#!/usr/bin/perl

# expects logs rsync'ed into a raw/ sub-directory (ie. raw/pp-app-bugs01, raw/pp-app-bugs02/, ..)
# generates logs in a logs/ sub-directory

use strict;
use warnings;
use autodie;

use FindBin qw($Bin);
use File::Basename;
use File::Slurp;

my $raw_path = "$Bin/raw";
my @hosts = map { s/^\Q$raw_path\///; $_ } glob("$raw_path/*");

my @processed = -e "$Bin/processed" ? read_file("$Bin/processed") : ();
chomp(@processed);

foreach my $host (@hosts) {
    print "$host ..\n";
    foreach my $file (glob("$raw_path/$host/*.log")) {
        my $filename = basename($file);
        next if grep { $_ eq "$host/$filename" } @processed;
        print "  $filename ..\n";
        my ($date) = $filename =~ /^(.+)\./;

        my @access;
        my @error;
        foreach my $line (read_file($file)) {
            # remove syslog prefix
            $line =~ s/^.+?apache: //;
            # remove passwords
            $line =~ s/(Bugzilla_password)=[^\s&]+/$1=*/;
            # skip zeus pings
            next if $line =~ /HTTP-Monitor\/1\.1/;

            if ($line =~ /\[(error|notice|warn)\]/) {
                push @error, $line;
            } else {
                push @access, $line;
            }
        }
        write_file("$Bin/logs/error_${date}_${host}.log", @error);
        write_file("$Bin/logs/access_${date}_${host}.log", @access);

        push @processed, "$host/$filename";
    }

    # always treat the last file processed as incomplete
    pop(@processed);
}

write_file("$Bin/processed", map { "$_\n" } sort @processed);