Instantly share code, notes, and snippets.

Embed
What would you like to do?
# -*- coding: utf-8 -*-
require 'time'
require 'pathname'
require 'optparse'
require 'optparse/pathname'
require 'net/scp'
require 'google_drive'
require 'highline'
require 'net/http'
module Puboo
module Performance
class Index
include OptionParser::Arguable
def initialize
@hosts = %w[app001.booklog.jp app002.booklog.jp]
@dir = '/var/log/httpd'
@file_prefix = 'access_log'
@log_num = 1
@config = {}
@groups = %w[bookInfo read buy find write set communicate top other]
@users = %w[guest user]
@media = %w[pc sp]
@tags = @groups.product(@media, @users).inject({}) {|tags, elem|
tags[elem.join('.')] = {response_time: 0, count: 0}
tags
}
end
def start(argv=ARGV)
options do |opt|
opt.on '-c', '--config=FILE', 'config file', Pathname do |file|
raise OptionParser::InvalidArgument, 'no such file' unless file.file?
raise OptionParser::InvalidArgument, 'not readable' unless file.readable?
@config = YAML.load_file(file.to_s)
end
opt.on '-n', '--lognum=NUMBER', 'log file number', OptionParser::DecimalInteger do |num|
@log_num = [1, num].max
end
opt.on '-p', '--password', 'Use password for SSH connection. If given, key not used' do
@ssh_password = HighLine.new.ask('password for SSH:') {|pw| pw.echo = false}
end
opt.parse!(ARGV)
end
@config['account'] = 'account@example.co.jp'
@config['password'] ||= HighLine.new.ask('password> ') {|pw| pw.echo = false}
if [@config['account'], @config['password']].any? {|env| env.nil?}
abort "account and password is required"
end
@session = GoogleDrive.login(@config['account'], @config['password'])
run
end
def run
start_at = Time.now
$stderr.puts "start at #{start_at}"
format /^[^ ]+ [^ ]+ [^ ]+ [^ ]+ \[(?<time>[^\]]+)\] "(?<method>[^ ]+) (?<path>[^ ]+) [^"]+" (?<status>[^ ]+) [^ ]+ [^ ]+ "[^"]+" "[^"]+" "(?<uriGroup>[^ ]+)" "(?<loggedIn>[^ ]+)" "(?<media>[^ ]+)" "(?<isOwner>[^ ]+)" (?<response_time>[^ ]+) (?<profile>.+)$/
Dir.mktmpdir do |dir|
Dir.chdir dir do
get_logfiles.each do |file|
open file do |f|
f.each_line do |line|
parse(line)
end
end
end
end
calc_stats
record
end
notify
end_at = Time.now
$stderr.puts "end at #{end_at}"
$stderr.puts "took #{end_at - start_at} seconds"
end
def format(pattern)
@pattern = pattern
end
def parse(log)
matched = @pattern.match(log)
return unless matched
unless @date
@date = Time.strptime(matched['time'], '%d/%b/%Y:%T %z')
end
group = matched['uriGroup']
group = 'other' if group == '-'
media = matched['media'] == 'sp' ? 'sp' : 'pc'
user = matched['loggedIn'] == 'true' ? 'user' : 'guest'
tag = [group, media, user].join('.')
@tags[tag][:response_time] += (matched['response_time'].to_f / 1000) # micro second -> milisecond
@tags[tag][:count] += 1
@tags[tag]
end
def calc_stats
@tags.each_pair do |tag, values|
next if values[:count] == 0
values[:response_time_per_request] = values[:response_time] / values[:count]
end
@tags
end
def get_logfiles
files = []
threads = []
@hosts.each do |host|
threads << Thread.new {
file = "#{@file_prefix}.#{@log_num}.#{host}"
files << file
remote_file = "#{@dir}/#{@file_prefix}.#{@log_num}"
cmd = "scp #{host}:#{remote_file} #{file}"
$stderr.puts cmd
options = @ssh_password ? {password: @ssh_password} : {keys: @config['keys']}
Net::SCP.start host, `whoami`.chomp, options do |scp|
scp.download! remote_file, file, verbose: true
end
}
end
threads.each &:join
files
end
def record
ss_title = @date.strftime('%Y-%m')
ws_title = @date.strftime('%m-%d')
@spreadsheet = @session.spreadsheet_by_title(ss_title)
unless @spreadsheet
@spreadsheet = @session.create_spreadsheet(ss_title)
index_ss = @session.spreadsheet_by_key(@config['index_spreadsheet_key'])
index_ws = index_ss.worksheet_by_title(@config['index_worksheet_title'])
row_num = index_ws.num_rows + 1
index_ws[row_num, 1] = ss_title
index_ws[row_num, 2] = @spreadsheet.human_url
index_ws.save
end
ws = @spreadsheet.worksheet_by_title(ws_title)
if ws
$stderr.puts "overwrite worksheet: #{ws_title}"
else
$stderr.puts "add worksheet: #{ws_title}"
ws = @spreadsheet.add_worksheet(ws_title)
end
headers = ['tag', 'avg(msec)', 'total time(msec)', 'total request', 'ratio', 'total amount(sec)']
headers.each.with_index do |header, col|
ws[1, col+1] = header
end
body_row_count = @tags.length
total_time = @tags.each_value.inject(0.0) {|sum, values| sum += values[:response_time]}
total_request = @tags.each_value.inject(0) {|sum, values| sum += values[:count]}
@tags.each_pair do |tag, values|
values[:ratio] = values[:response_time] / total_time
end
tags = @tags.sort_by {|tag, values| values[:ratio]}
tags.reverse_each.with_index 2 do |(tag, values), row|
ws[row, 1] = tag
ws[row, 2] = values[:response_time_per_request].to_i
ws[row, 3] = values[:response_time].to_i
ws[row, 4] = values[:count]
ws[row, 5] = sprintf('%0.1f%%', values[:ratio] * 100)
end
ws[body_row_count+2, 3] = total_time.to_i
ws[2, headers.length] = (total_time/1000).to_i
ws[3, headers.length] = 'total request'
ws[4, headers.length] = total_request
ws[5, headers.length] = 'avg(msec/req)'
ws[6, headers.length] = (total_time.to_f / total_request).round
ws.save
end
def notify
ikachan = 'http://example.co.jp'
channel = '#dev_booklog'
msg = "パフォーマンス指標更新しました:#{@spreadsheet.human_url}"
$stderr.puts 'Notifying to IRC'
Net::HTTP.post_form URI(ikachan + '/join'), channel: channel
Net::HTTP.post_form URI(ikachan + '/privmsg'), channel: channel, message: msg
Net::HTTP.post_form URI(ikachan + '/leave'), channel: channel
end
end
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment