JSON log stats

json_log_stats.rb

#!/usr/bin/env ruby
require 'json'
require 'descriptive_statistics'

def print_data_summary(key_name, data)
  puts "-----------------------------------------------------------"
  puts "-- #{key_name} --"
  puts "Data Points:\t\t#{data.length}"
  puts "Mean:\t\t#{data.mean}"
  puts "Median:\t\t#{data.median}"
  puts "Min:\t\t#{data.min}"
  puts "Max:\t\t#{data.max}"
  puts "90th:\t\t#{data.percentile(90)}"
  puts "95th:\t\t#{data.percentile(95)}"
  puts "99th:\t\t#{data.percentile(99)}"
end

keys = ARGV.to_a
datas = {}

at_exit {
  datas.each do |k, d|
    print_data_summary(k, d)
  end
}

STDIN.each do |line|
  begin
    j = JSON.parse(line.gsub(/^.*\{/, '{'))
    keys.each do |k|
      puts "#{k}: #{j[k].to_f}"
      datas[k] ||= []
      datas[k] << j[k].to_f
    end
  rescue => e
  end
end

 $ ./agency -c /etc/agency/agency.cfg | ./json_log_stats.rb "compress.encode" "compress.bytes_saved"
....
compress.encode: 16.162778
compress.bytes_saved: 1104.0
compress.encode: 9.670084
compress.bytes_saved: 496.0
compress.encode: 21.121682
compress.bytes_saved: -141.0
compress.encode: 6.795091
compress.bytes_saved: 2445.0
^C-----------------------------------------------------------
-- compress.encode --
Data Points:        302
Mean:       21.12750755960265
Median:     18.1611245
Min:        2.124988
Max:        99.886733
90th:       32.11705680000001
95th:       41.86232154999997
99th:       69.32423362000006
-----------------------------------------------------------
-- compress.bytes_saved --
Data Points:        302
Mean:       1224.453642384106
Median:     606.0
Min:        -275.0
Max:        14353.0
90th:       2877.0
95th:       3223.0499999999993
99th:       11254.08000000006