calculate ensemble average and statistical error of dat files

analyze_dat.rb

# calculate ensemble average and its statistical error of dat files
#
# the format of the input file is defined as follows
#   each line contains key and values separated by space as follows
#     <key> <val1> <val2> <val3> ...
#   all the file do not necessary have the same keys
#     When key is not found, the value is assumed to be zero
#
#   output file format is
#     <key> <ave_val1> <err_val1> <ave_val2> <err_val2> .....
#
# Usage
#   ruby analyze_dat.rb 1.dat 2.dat 3.dat
#

require 'pp'

class ErrorAnalysis

  def self.analyze_datfiles(filenames, io = nil)
    calculated = calc_average_and_error(filenames)

    if io
      calculated.sort.each do |key,values|
        io.puts "#{key} #{values.join(' ')}"
      end
    else
      calculated
    end
  end

  private
  def self.load_file(filename)
    parsed = {}
    File.open(filename).each do |line|
      next if line =~ /^\#/
      mapped = line.split.map(&:to_f)
      parsed[ mapped[0] ] = mapped[1..-1]
    end
    parsed
  end

  def self.calc_average_and_error(filenames)
    datas = filenames.map {|path| load_file(path) }
    keys = datas.map(&:keys).flatten.uniq.sort
    first_data = datas.first
    num_col = first_data[first_data.keys.first].size

    calculated = {}
    keys.each do |key|
      calculated[key] = []
      num_col.times do |col|
        values = datas.map {|data| data[key] ? data[key][col] : 0.0 }
        calculated[key] += average_error(values)
      end
    end
    calculated
  end

  def self.average_error(values)
    average = values.inject(:+).to_f / values.size
    variance = values.map {|v| (v - average)**2 }.inject(:+) / values.size
    unless values.size == 1
      error = Math.sqrt( variance / (values.size-1) )
    else
      error = 0.0
    end
    return average, error
  end
end

if __FILE__ == $0
  ErrorAnalysis.analyze_datfiles( ARGV, $stdout )
end