method

log_to_yaml.pl

my $requests = 0;
my %status;
my %type;
my %method;
my %kind;


while (<>) {
 next if (/request: /);
 next if (/\[(info|warn)\]/);
 if (/\]\s+\"([A-Z]+)\sh?t?t?p?:?\/?(\/[^\s?\&]*).*HTTP.1..\"\s(...)\s.*\"([^\"]*)\".*$/) {
  $requests++;
  $method{$1}++;
  $status{$3}++;

  $path = $2;
  $ua = $4;
  if ($path =~ /\....?$/) {
   $type{'FILE'}++;
  } else {
   $type{'PATH'}++;
  }
  if ($ua =~ /(bot|spider|crawler)/i) {
   $kind{'BOT'}++;
  } else {
   $kind{'USER'}++;
  }
 }
}

print "stats:\n";
print " requests: $requests\n";
for $k (sort(keys(%method))){print " $k: ".$method{$k}."\n"};
print " status: \n";
for $k (sort(keys(%status))){print "  $k: ".$status{$k}."\n"};
for $k (sort(keys(%kind))){print " $k: ".$kind{$k}."\n"};
for $k (sort(keys(%type))){print " $k: ".$type{$k}."\n"};

yaml_tocsv.rb

require 'yaml'
require 'csv'

CSV.open("res.csv","w") do |csv|
  csv << %w(date peticiones bots users files paths gets posts ok not_found client_err client_closed errors)
  Dir["*.yml"].sort.each do |file|
    puts file
    stats = YAML.load_file(file)['stats']
    status = stats['status']
    date = file[28..29].to_str + "/" + file[26..27].to_str + "/" +   file[22..25].to_str
    st_ok = status[200].to_i + status[206].to_i + status[301].to_i + status[302].to_i + status[304].to_i
    st_client_err = status[400].to_i + status[403].to_i + status[401].to_i + status[408].to_i + status[406].to_i
    csv << [date, stats['requests'], stats['BOT'].to_i, stats['USER'].to_i, stats['FILE'].to_i, stats['PATH'].to_i, stats['GET'].to_i, stats['POST'].to_i, st_ok, status[404].to_i, st_client_err, status[499].to_i, status[500].to_i]
  end 
end