Mopping up the floor after ABingo shits itself

abingo_logAnal.thor

#encoding: UTF-8
require 'rubygems'
require 'builder'
#Started POST "/gyorskereso/repjegy/purchase?url=http%3A%2F%2Fflight1.onlinetravel.ch%2Fcgi-bin%2Fflightmore" for 188.36.35.62 at 2011-12-03 04:11:57 +0000
REQUEST_LINE = /^Started ([A-Z]+) "(.*?)" for ([0-9\.]+) at (.*?)$/

LAYOUT_RENDER_LINE = /^Rendered (.*?) within (.*?) \(/


ABINGO_BINGO = /ABingo#bingo!/
ABINGO_UPDATE_PARTICIPATION = /^ABINGO: Updating paticipation (.*?)$/

URL_CLEANER = //

FILTERED_URLS = [
  /autocomplete_list.js/,
  /autocomplete.js/,
  /\.(php(3?)|asp|inc)(\?.*?)?$/,
  /\/en\/(.*?)\/show/
  # /\?/
]

class HttpAccess

  attr_reader :url
  attr_reader :date
  attr_reader :ip
  attr_reader :method

  attr_reader :is_bingo

  attr_accessor :page
  attr_accessor :layout

  attr_reader :line_parts

  def initialize line_parts
    @line_parts = line_parts
    @is_bingo = false
    process_req_line
  end


  def process_req_line
    @url = @line_parts[2].gsub /\?(.*?)$/, ''
    @ip = @line_parts[3]
    @date = Time.parse @line_parts[4]
    @method = @line_parts[1]
  end


  def bingo!
    @is_bingo = true
  end


  def page_key
    "#{layout} | #{page}"
  end
end


class PageVisit
  attr_accessor :key
  attr_accessor :count
end


class LogProcess < Thor

  desc "preprocess FILE", "Preprocess the given log file to a basic TSV format"
  def preprocess file
    counter = 0
    accesses = []
    ips = {}
    bingo_count = 0
    contents = File.read( file)

    current_access = nil

    @results = Results.new

    contents.lines.each do |l|
      md = REQUEST_LINE.match l
      if md
        # skip filtered urls
        skip = false
        FILTERED_URLS.each do |fu| 
          skip = true if md[2] =~ fu
        end
        next if skip
        # put into the result list
        current_access = HttpAccess.new(md)
        @results.accesses << current_access
        @results.ips[ current_access.ip ] = [] unless ips[ current_access.ip ]
        @results.ips[ current_access.ip ] << current_access
      end

      if l =~ ABINGO_BINGO
        current_access.bingo! if current_access
        bingo_count += 1
      end


      if l =~ ABINGO_UPDATE_PARTICIPATION
        current_access
      end

      md = LAYOUT_RENDER_LINE.match l
      if md
        current_access.page = md[1]
        current_access.layout = md[2]
      end

      counter += 1

    end

    puts "analyzed #{counter} lines"


    @printer = ResultPrinter.new @results
    File.open( file + ".html", 'w' ) do |f|
      @printer.print_to f
      puts "Written #{f.path}..."
    end


  end

end



class Results
  attr_reader :accesses
  attr_reader :ips

  def initialize
    @accesses = []
    @ips = {}
  end
end

class ResultPrinter

  attr_reader :results
  alias :res :results

  def accesses; @results.accesses; end
  def ips; @results.ips; end

  def initialize results
    @results = results
    @buffer = []
    calculate!
  end


  def print_to output
    @o = Builder::XmlMarkup.new
    @o.html do |html|
      html.head do |head|
        head.title "Konverziók"
        head.meta :charset => 'utf-8'
        head.comment! 'Le HTML5 shim, for IE6-8 support of HTML elements'
        head.comment! '[if lt IE 9]>
      <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
    <![endif]'
        head.link href: 'http://twitter.github.com/bootstrap/1.4.0/bootstrap.css', rel: 'stylesheet'
        head.link href: 'http://twitter.github.com/bootstrap/assets/css/docs.css', rel: 'stylesheet'
        head.link href: 'http://twitter.github.com/bootstrap/assets/js/google-code-prettify/prettify.css', rel: 'stylesheet'

      end
      html.body style: 'padding: 50px 0; width:1140px; margin:0 auto;' do |b|
        summary
        print_ab_results
        print_page_results b
        print_url_results b
      end
    end
    output << @o
  end

  private

def tag name, contents, opts={}
  "<#{name}>#{contents}</#{name}>"
end

  def summary
    @o.h1 "A/B postmortem"
    @o.ul do |u|
      u.li "-- kezdődik: #{@start_date}"
      u.li "-- végződik: #{@end_date}"
      u.li "-- #{ accesses.size } látogatás"
      u.li "-- #{ ips.size } egyedi IP"
      u.li "-- #{ @bingo_count } bingó (nem csak az AB tesztben résztvevőktől)"
    end
  end


  def calculate!
    @bingo_count = 0
    @page_results = {}
    @url_results = {}
    accesses.each do |access|
      url_results[access.url] = [] unless url_results[ access.url ]
      url_results[access.url] << access.ip

      key = "#{access.layout} | #{access.page}"
      next unless key.size > 3
      page_results[key] = [] unless page_results[key]
      page_results[key] << access.ip

      @bingo_count += 1 if access.is_bingo
    end

    @ip_abingo_mainpage = {}
    accesses.each do |v|
      @ip_abingo_mainpage[v.ip] = v.page_key if v.url == '/'
    end

    @ips_vith_bingo = {}
    ips.each_pair do |ip, visits|
      puts "ips_with_bingo build: #{ip}"
      visits.each do |v|
        puts "   Visit: #{ v.line_parts[0] }"
        @ips_vith_bingo[ip] = true if v.is_bingo
      end
    end

    p @ip_abingo_mainpage
    p @ips_vith_bingo



    @ab_test_source_results = {}
    @ip_abingo_mainpage.each_pair do |ip, page_key|
      @ab_test_source_results[page_key] = { total: 0, conversions: 0 } unless @ab_test_source_results[page_key]
      @ab_test_source_results[page_key][:total] += 1
      @ab_test_source_results[page_key][:conversions] += 1 if @ips_vith_bingo[ip] != nil
    end
    p @ab_test_source_results


    @start_date = accesses.first.date
    @end_date = accesses.last.date
  end

  attr_reader :page_results
  attr_reader :url_results

  def print_ab_results
    @o.h1  "Konverziók:"
    @o.table do |t|
      t.tr do |tr|
        tr.th 'opció'
        tr.th 'összes látogató'
        tr.th 'ebből konverzió'
      end
      @ab_test_source_results.each_pair do |key, results|
        @o.tr do |tr|
          tr.td key, class: :key
          tr.td results[:total], class: :total
          tr.td results[:conversions], class: :conversions
        end
      end
    end
  end


  def print_page_results o
    sort_array = []

    page_results.each_pair do |k,v|
      v.uniq!
      sort_array.push( [v.size, k ] )
    end

    sort_array.sort_by! { |a| a[0] }

    o.h1 "oldalak alaján csoportosítva:"
    o.table do |t|
      t.tr do |tr|
        tr.th 'összes látogató'
        tr.th 'url'
      end
      sort_array.reverse.each do |k|
        t.tr do |tr|
          tr.td k[0], class: 'number count'
          tr.td k[1], class: :key
        end
      end
    end
  end


  def print_url_results o
    sort_array = []

    url_results.each_pair do |k,v|
      v.uniq!
      sort_array << [v.size, k]
    end

    sort_array.sort_by! {|a| a[0] }

    o.h1 "URL-ek alapján csoportosítva:"
    o.table do |t|
      t.tr do |tr|
        tr.th 'összes látogató'
        tr.th 'url'
      end

      sort_array.reverse.each do |k|
        t.tr do |tr|
          tr.td k[0], class: 'number count'
          tr.td k[1], class: :key
        end
      end

    end
  end
end


  # filter accesses

def filter_accesses_of accesses, urls

  meaningful_accesses = []
  accesses.each do |acc|
    urls.each do |url|
      if acc.url =~ url || acc.is_bingo
        meaningful_accesses << acc
      end
    end
  end

  meaningful_accesses
end