puboo-performance-index.rb

# -*- coding: utf-8 -*-
require 'time'
require 'pathname'
require 'optparse'
require 'optparse/pathname'
require 'net/scp'
require 'google_drive'
require 'highline'
require 'net/http'

module Puboo
  module Performance
    class Index
      include OptionParser::Arguable

      def initialize
        @hosts = %w[app001.booklog.jp app002.booklog.jp]
        @dir = '/var/log/httpd'
        @file_prefix = 'access_log'
        @log_num = 1
        @config = {}

        @groups = %w[bookInfo read buy find write set communicate top other]
        @users = %w[guest user]
        @media = %w[pc sp]
        @tags = @groups.product(@media, @users).inject({}) {|tags, elem|
          tags[elem.join('.')] = {response_time: 0, count: 0}
          tags
        }
      end

      def start(argv=ARGV)
        options do |opt|
          opt.on '-c', '--config=FILE', 'config file', Pathname do |file|
            raise OptionParser::InvalidArgument, 'no such file' unless file.file?
            raise OptionParser::InvalidArgument, 'not readable' unless file.readable?
            @config = YAML.load_file(file.to_s)
          end
          opt.on '-n', '--lognum=NUMBER', 'log file number', OptionParser::DecimalInteger do |num|
            @log_num = [1, num].max
          end
          opt.on '-p', '--password', 'Use password for SSH connection. If given, key not used' do
            @ssh_password = HighLine.new.ask('password for SSH:') {|pw| pw.echo = false}
          end
          opt.parse!(ARGV)
        end

        @config['account'] = 'account@example.co.jp'
        @config['password'] ||= HighLine.new.ask('password> ') {|pw| pw.echo = false}

        if [@config['account'], @config['password']].any? {|env| env.nil?}
          abort "account and password is required"
        end
        @session = GoogleDrive.login(@config['account'], @config['password'])

        run
      end

      def run
        start_at = Time.now
        $stderr.puts "start at #{start_at}"

        format /^[^ ]+ [^ ]+ [^ ]+ [^ ]+ \[(?<time>[^\]]+)\] "(?<method>[^ ]+) (?<path>[^ ]+) [^"]+" (?<status>[^ ]+) [^ ]+ [^ ]+ "[^"]+" "[^"]+" "(?<uriGroup>[^ ]+)" "(?<loggedIn>[^ ]+)" "(?<media>[^ ]+)" "(?<isOwner>[^ ]+)" (?<response_time>[^ ]+) (?<profile>.+)$/

        Dir.mktmpdir do |dir|
          Dir.chdir dir do
            get_logfiles.each do |file|
              open file do |f|
                f.each_line do |line|
                  parse(line)
                end
              end
            end
          end
          calc_stats
          record
        end

        notify

        end_at = Time.now
        $stderr.puts "end at #{end_at}"
        $stderr.puts "took #{end_at - start_at} seconds"
      end

      def format(pattern)
        @pattern = pattern
      end

      def parse(log)
        matched = @pattern.match(log)
        return unless matched

        unless @date
          @date = Time.strptime(matched['time'], '%d/%b/%Y:%T %z')
        end

        group = matched['uriGroup']
        group = 'other' if group == '-'
        media = matched['media'] == 'sp' ? 'sp' : 'pc'
        user = matched['loggedIn'] == 'true' ? 'user' : 'guest'
        tag = [group, media, user].join('.')
        @tags[tag][:response_time] += (matched['response_time'].to_f / 1000) # micro second -> milisecond
        @tags[tag][:count] += 1

        @tags[tag]
      end

      def calc_stats
        @tags.each_pair do |tag, values|
          next if values[:count] == 0
          values[:response_time_per_request] = values[:response_time] / values[:count]
        end
        @tags
      end

      def get_logfiles
        files = []
        threads = []
        @hosts.each do |host|
          threads << Thread.new {
            file = "#{@file_prefix}.#{@log_num}.#{host}"
            files << file
            remote_file = "#{@dir}/#{@file_prefix}.#{@log_num}"
            cmd = "scp #{host}:#{remote_file} #{file}"
            $stderr.puts cmd
            options = @ssh_password ? {password: @ssh_password} : {keys: @config['keys']}
            Net::SCP.start host, `whoami`.chomp, options do |scp|
              scp.download! remote_file, file, verbose: true
            end
          }
        end
        threads.each &:join
        files
      end

      def record
        ss_title = @date.strftime('%Y-%m')
        ws_title = @date.strftime('%m-%d')
        @spreadsheet = @session.spreadsheet_by_title(ss_title)
        unless @spreadsheet
          @spreadsheet = @session.create_spreadsheet(ss_title)

          index_ss = @session.spreadsheet_by_key(@config['index_spreadsheet_key'])
          index_ws = index_ss.worksheet_by_title(@config['index_worksheet_title'])
          row_num = index_ws.num_rows + 1
          index_ws[row_num, 1] = ss_title
          index_ws[row_num, 2] = @spreadsheet.human_url
          index_ws.save
        end
        ws = @spreadsheet.worksheet_by_title(ws_title)
        if ws
          $stderr.puts "overwrite worksheet: #{ws_title}"
        else
          $stderr.puts "add worksheet: #{ws_title}"
          ws = @spreadsheet.add_worksheet(ws_title)
        end
        headers = ['tag', 'avg(msec)', 'total time(msec)', 'total request', 'ratio', 'total amount(sec)']
        headers.each.with_index do |header, col|
          ws[1, col+1] = header
        end
        body_row_count = @tags.length
        total_time = @tags.each_value.inject(0.0) {|sum, values| sum += values[:response_time]}
        total_request = @tags.each_value.inject(0) {|sum, values| sum += values[:count]}
        @tags.each_pair do |tag, values|
          values[:ratio] = values[:response_time] / total_time
        end
        tags = @tags.sort_by {|tag, values| values[:ratio]}
        tags.reverse_each.with_index 2 do |(tag, values), row|
          ws[row, 1] = tag
          ws[row, 2] = values[:response_time_per_request].to_i
          ws[row, 3] = values[:response_time].to_i
          ws[row, 4] = values[:count]
          ws[row, 5] = sprintf('%0.1f%%', values[:ratio] * 100)
        end
        ws[body_row_count+2, 3] = total_time.to_i
        ws[2, headers.length] = (total_time/1000).to_i
        ws[3, headers.length] = 'total request'
        ws[4, headers.length] = total_request
        ws[5, headers.length] = 'avg(msec/req)'
        ws[6, headers.length] = (total_time.to_f / total_request).round

        ws.save
      end

      def notify
        ikachan = 'http://example.co.jp'
        channel = '#dev_booklog'
        msg = "パフォーマンス指標更新しました：#{@spreadsheet.human_url}"
        $stderr.puts 'Notifying to IRC'
        Net::HTTP.post_form URI(ikachan + '/join'), channel: channel
        Net::HTTP.post_form URI(ikachan + '/privmsg'), channel: channel, message: msg
        Net::HTTP.post_form URI(ikachan + '/leave'), channel: channel
      end
    end
  end
end