dcsaszar/analyze_obj_load.rb

## analyze_obj_load.rb
DATE_REGEX = /at\s(?<date>20\d\d-\d\d-\d\d\s)\d\d:\d\d:\d\d/
MS_REGEX = /Obj Load \((?<ms>.*?)ms\)\s+(?<q>.*)/

ms_by_query = Hash.new { |hash, key| hash[key] = [] }
ms_by_date = Hash.new { |hash, key| hash[key] = [] }
histogram_ms = Hash.new { |hash, key| hash[key] = 0 }

date = nil
IO.foreach("log/development.log") do |line|
  line = line.force_encoding('ISO-8859-1')
  has_date = line.match DATE_REGEX
  date = has_date[:date] + "12:00:00" if has_date
  hit = line.match MS_REGEX
  next unless hit
  ms = hit[:ms].to_f
  q = hit[:q]
  ms_by_query[q] << ms
  ms_by_date[date] << ms
  histogram_ms[ms] += 1
end

require 'time'
file = "/tmp/stats.html"
data_by_q = []
ms_by_query.map do |k,values|
  col = case k.split.first
  when "id" then k[","] ? 1 : 0
  when "ppath" then 2
  when "path" then 3
  when "permalink" then 4
  else 5
  end
  data_by_q += values.map { |v| [nil] * col + [v] + [nil] * (5 - col) }
end
data_by_time = []
ms_by_date.map do |date,values|
  next unless date
  jsdate = :"new Date('#{date.gsub("-","/")}')"
  avg = values.reduce(&:+) / values.size
  median = values[values.size / 2]
  data_by_time << [jsdate, avg, values.min, median]
end
IO.write file, DATA.read.
    sub("$MS", data_by_q.inspect).
    sub("$DATES", data_by_time.inspect).
    gsub("nil", "null").
    gsub(/:"([^"]*)"/, '\1')

system "firefox #{file} || open #{file}"

__END__
<html>
  <head>
    <script type="text/javascript" src="https://www.google.com/jsapi"></script>
    <script type="text/javascript">
    google.load('visualization', '1.0', {'packages':['corechart']});
    google.setOnLoadCallback(drawChart);
    function drawChart() {
      var data = new google.visualization.DataTable();
      data.addColumn('number', 'ms ids');
      data.addColumn('number', 'ms id');
      data.addColumn('number', 'ms ppath');
      data.addColumn('number', 'ms path');
      data.addColumn('number', 'ms permalink');
      data.addColumn('number', 'ms unknown');
      data.addRows($MS);
      (new google.visualization.Histogram(document.getElementById('chart1'))).draw(
        data,
        {
          height: 350,
          title:'Scrivito Obj Load (94%)',
          isStacked: true,
          histogram: { lastBucketPercentile: 3 }
        }
      );
      (new google.visualization.Histogram(document.getElementById('chart2'))).draw(
        data,
        {
          height: 350,
          title:'Scrivito Obj Load (70%)',
          isStacked: true,
          histogram: { lastBucketPercentile: 15 }
        }
      );
      data = new google.visualization.DataTable();
      data.addColumn('date', 'date');
      data.addColumn('number', 'ms avg');
      data.addColumn('number', 'ms min');
      data.addColumn('number', 'ms median');
      data.addRows($DATES);
      (new google.visualization.ScatterChart(document.getElementById('chart3'))).draw(
        data,
        {
          height: 350,
          lineWidth: 1,
          title:'Average Scrivito Obj Load by Age in Days',
          histogram: { lastBucketPercentile: 3 }
        }
      );
    }
    </script>
  </head>
  <body>
    <div id="chart1"></div>
    <div id="chart2"></div>
    <div id="chart3"></div>
  </body>
</html>
	DATE_REGEX = /at\s(?<date>20\d\d-\d\d-\d\d\s)\d\d:\d\d:\d\d/
	MS_REGEX = /Obj Load \((?<ms>.?)ms\)\s+(?<q>.)/

	ms_by_query = Hash.new { \|hash, key\| hash[key] = [] }
	ms_by_date = Hash.new { \|hash, key\| hash[key] = [] }
	histogram_ms = Hash.new { \|hash, key\| hash[key] = 0 }

	date = nil
	IO.foreach("log/development.log") do \|line\|
	line = line.force_encoding('ISO-8859-1')
	has_date = line.match DATE_REGEX
	date = has_date[:date] + "12:00:00" if has_date
	hit = line.match MS_REGEX
	next unless hit
	ms = hit[:ms].to_f
	q = hit[:q]
	ms_by_query[q] << ms
	ms_by_date[date] << ms
	histogram_ms[ms] += 1
	end

	require 'time'
	file = "/tmp/stats.html"
	data_by_q = []
	ms_by_query.map do \|k,values\|
	col = case k.split.first
	when "id" then k[","] ? 1 : 0
	when "ppath" then 2
	when "path" then 3
	when "permalink" then 4
	else 5
	end
	data_by_q += values.map { \|v\| [nil] * col + [v] + [nil] * (5 - col) }
	end
	data_by_time = []
	ms_by_date.map do \|date,values\|
	next unless date
	jsdate = :"new Date('#{date.gsub("-","/")}')"
	avg = values.reduce(&:+) / values.size
	median = values[values.size / 2]
	data_by_time << [jsdate, avg, values.min, median]
	end
	IO.write file, DATA.read.
	sub("$MS", data_by_q.inspect).
	sub("$DATES", data_by_time.inspect).
	gsub("nil", "null").
	gsub(/:"([^"]*)"/, '\1')

	system "firefox #{file} \|\| open #{file}"

	__END__
	<html>
	<head>
	<script type="text/javascript" src="https://www.google.com/jsapi"></script>
	<script type="text/javascript">
	google.load('visualization', '1.0', {'packages':['corechart']});
	google.setOnLoadCallback(drawChart);
	function drawChart() {
	var data = new google.visualization.DataTable();
	data.addColumn('number', 'ms ids');
	data.addColumn('number', 'ms id');
	data.addColumn('number', 'ms ppath');
	data.addColumn('number', 'ms path');
	data.addColumn('number', 'ms permalink');
	data.addColumn('number', 'ms unknown');
	data.addRows($MS);
	(new google.visualization.Histogram(document.getElementById('chart1'))).draw(
	data,
	{
	height: 350,
	title:'Scrivito Obj Load (94%)',
	isStacked: true,
	histogram: { lastBucketPercentile: 3 }
	}
	);
	(new google.visualization.Histogram(document.getElementById('chart2'))).draw(
	data,
	{
	height: 350,
	title:'Scrivito Obj Load (70%)',
	isStacked: true,
	histogram: { lastBucketPercentile: 15 }
	}
	);
	data = new google.visualization.DataTable();
	data.addColumn('date', 'date');
	data.addColumn('number', 'ms avg');
	data.addColumn('number', 'ms min');
	data.addColumn('number', 'ms median');
	data.addRows($DATES);
	(new google.visualization.ScatterChart(document.getElementById('chart3'))).draw(
	data,
	{
	height: 350,
	lineWidth: 1,
	title:'Average Scrivito Obj Load by Age in Days',
	histogram: { lastBucketPercentile: 3 }
	}
	);
	}
	</script>
	</head>
	<body>
	<div id="chart1"></div>
	<div id="chart2"></div>
	<div id="chart3"></div>
	</body>
	</html>