Generated munin graph from HAR file

README.md

Parses a HAR archive (e.g. generated with https://gist.github.com/mfn/5111680 ) and generate three type of graphs: hosts, mimetypes and timings.

Requires environment variables to be used:

• HAR pointing to the HAR file
• SECTION to choose between HOSTS, MIMETYPES and TIMINGS
• GRAPH_CATEGORY,
• GRAPH_TITLE

To use the plugin usually requires creating a symlink for each graph type you want and configuring them via plugin-conf.d/.

Example:

Note: this plugin is not capable of generated the HAR archive, only consuming. You need a separate way, e.g. using phantomjs with https://gist.github.com/mfn/5111680 and generate it somewhere this munin plugin has access to.

har_munin.rb

#!/usr/bin/env ruby

require 'date'
require 'json'
require 'uri'

RE_DOMAIN = /([^\.]+\.[^\.]+)$/
HAR_SPEC= "http://www.softwareishard.com/blog/har-12-spec/"
SECTIONS = 'HOSTS', 'MIMETYPES', 'TIMINGS'
COLORS = {
  html: {
    wait: "FFFF66",
    receive: "FFFF00",
  },
  css: {
    started: "66CCFF",
    wait: "3399FF",
    receive: "0066FF",
  },
  javascript: {
    started: "99FF66",
    wait: "66FF33",
    receive: "669900",
  },
  image: {
    started: "FF6666",
    wait: "FF3300",
    receive: "FF6600",
  },
}

def mime_to_content(mimeType)
  # remove any possible charset
  mt = mimeType.gsub(/;.*/, '').downcase
  return :javascript if mt =~ /javascript/
  return :css if mt =~ /css/
  return :html if mt =~ /html/
  return :png if mt =~ /png/
  return :jpg if mt =~ /(jpeg|jpg|)/
  return :gif if mt =~ /gif/
  STDERR.puts "Unknown: #{mt}"
  :unknown
end

def error(msg)
  STDERR.puts "ERROR: #{msg}"
  exit 1
end

def get_domain(uri)
  if uri.respond_to?(:host)
    uri.host.match(RE_DOMAIN)[1]
  else
    URI.parse(uri).host.match(RE_DOMAIN)[1]
  end
end

def usage(errmsg=nil)
  if errmsg
    STDERR.puts "ERROR: #{errmsg}"
    STDERR.puts
  end
  puts <<HELP
#{$0} [config]
Munin script to provide graphs based on HAR [1] files.

Supported environment variables:
HAR             - Path to HAR file to parse; required
GRAPH_CATEGORY  - Graph category; required
GRAPH_TITLE     - Graph title; required
SECTION         - Select section for which to generated graph.
                  Supported section:
                    HOSTS
                    MIMETYPES
                    TIMINGS

[1] #{HAR_SPEC}

HELP
  exit 1
end

STDOUT.sync

usage if ARGV[0] == "-h" || ARGV[0] == "--help"

graph_category = ENV['GRAPH_CATEGORY']
usage("No GRAPH_CATEGORY provided") if not graph_category

graph_title = ENV['GRAPH_TITLE']
usage("No GRAPH_TITLE provided") if not graph_title

har_file = ENV['HAR'] ? ENV['HAR'] : ARGV.shift
usage("No HAR provided") if not har_file

begin
  har = JSON.parse(File.read(har_file).force_encoding('utf-8'), :symbolize_names => true)
rescue
  error "Invalid HAR file, see #{HAR_SPEC} for specification"
end
if not har[:log]
  error "Doesn't look like a HAR archive, see #{HAR_SPEC} for specification"
end
if har[:log][:pages].length == 0
  error "No pages found (empty har?)"
end
page= har[:log][:pages][0]
page_url = page[:id]
data = {
  hosts: Hash.new { |h,k| h[k] = 0 },
  content: Hash.new { |h,k| h[k] = 0 },
  startTime: nil,
  onContentLoaded: nil,
  onLoad: nil,
  response: {
    html: nil,
    css: {
      started: nil,
      wait: nil,
      receive: nil,
    },
    javascript: {
      started: nil,
      wait: nil,
      receive: nil,
    },
    image: {
      started: nil,
      wait: nil,
      receive: nil,
    },
  }
}
domain = get_domain(page_url)
entries = har[:log][:entries].select do |e|
  next if e[:pageref] != page_url
  # Convert datetime
  e[:startedDateTime] = DateTime.iso8601( e[:startedDateTime] )
  e[:started] = e[:startedDateTime].strftime("%Q").to_i
  # Find first request
  if data[:startTime]
    data[:startTime] = e[:started] if e[:started] < data[:startTime]
  else
    data[:startTime] = e[:started]
  end
  # accumulate hosts
  url = URI.parse e[:request][:url]
  # Ignore data URLs
  next if url.scheme == 'data'
  data[:hosts][url.host] += 1
  mimeType = mime_to_content( e[:response][:content][:mimeType] )
  data[:content][ mimeType ] += 1
  if mimeType == :png or mimeType == :jpg or mimeType == :gif
    data[:content][ :image ] += 1
  end
  if mimeType == :html
    if not data[:response][:html]
      # record the very first html response
      data[:response][:html] = {
        started: data[:startTime],
        wait: data[:startTime] + e[:timings][:wait],
        receive: data[:startTime] + e[:timings][:wait] + e[:timings][:receive],
      }
    end
  else
    if mimeType == :png or mimeType == :jpg or mimeType == :gif
      mimeType = :image
    end
    res = data[:response][mimeType]
    wait = e[:started] + e[:timings][:wait]
    res[:wait] = wait if not res[:wait]
    res[:wait] = wait if wait < res[:wait]
    receive = wait + e[:timings][:receive]
    res[:receive] = receive if not res[:receive]
    res[:receive] = receive if receive > res[:receive]
    started = e[:started]
    res[:started] = started if not res[:started]
    res[:started] = started if started < res[:started]
  end
  # indicate we want to keep this entry
  true
end
if entries.length == 0
  error "No entries matching #{page} found"
end
data[:response].each do |_,v|
  v[:started] -= data[:startTime]
  v[:wait] -= data[:startTime]
  v[:receive] -= data[:startTime]
end

case ENV['SECTION']
when 'HOSTS'
  if ARGV[0] == "config"
    puts <<EOT
graph_title #{graph_title}
graph_vlabel Requests
graph_args --base 1000
graph_scale no
graph_category #{graph_category}
EOT
    data[:hosts].each do |host,_|
      puts "#{host.gsub('.', '_')}.label #{host}"
    end
    exit 0
  else
    data[:hosts].each do |host,count|
      puts "#{host.gsub('.', '_')}.value #{count}"
    end
  end
when 'MIMETYPES'
  if ARGV[0] == "config"
    puts <<EOT
graph_title #{graph_title}
graph_vlabel Requests
graph_args --base 1000
graph_scale no
graph_category #{graph_category}
EOT
    data[:content].each do |type,_|
      puts "#{type}.label #{type}"
    end
    exit 0
  else
    data[:content].each do |type,count|
      puts "#{type}.value #{count}"
    end
  end
when 'TIMINGS'
  if ARGV[0] == "config"
    puts <<EOT
graph_title #{graph_title}
graph_vlabel Time [ms]
graph_args --base 1000
graph_scale no
graph_category #{graph_category}
EOT
    puts "oncontentload.label oncontentload"
    puts "oncontentload.draw LINE1"
    puts "onload.label onload"
    puts "onload.draw LINE1"
    data[:response].each do |type,timings|
      timings.each do |timing,_|
        next if type == :html and timing == :started
        puts "#{type}_#{timing}.label #{type} #{timing}"
        puts "#{type}_#{timing}.colour #{COLORS[type][timing]}"
        puts "#{type}_#{timing}.draw LINE1"
      end
    end
    exit 0
  else
    puts "oncontentload.value #{page[:pageTimings][:onContentLoad]}" if page[:pageTimings][:onContentLoad]
    puts "onload.value #{page[:pageTimings][:onLoad]}" if page[:pageTimings][:onLoad]
    data[:response].each do |type,timings|
      timings.each do |timing,value|
        next if type == :html and timing == :started
        puts "#{type}_#{timing}.value #{value}"
      end
    end
  end
else
  error "Unknown section '#{ENV['SECTION']}'"
end

har_munin_rvm_wrapper.sh

#!/bin/sh
. /usr/local/rvm/environments/default
ruby $(dirname $(readlink -f $0))/har_munin.rb "$@"