Calculate 60-month smoothed average

calc-avg.rb

FINAL_DIR = "data/ushcn.v2.5.5.20220824"
ELEMENT = "tavg"
DATASET = "FLs.52j"

FINAL_LINE_LENGTH = 124
NUM_MONTHS_IN_YEAR = 12

def get_final_filenames
  Dir.glob("#{FINAL_DIR}/*.#{DATASET}.#{ELEMENT}")
end

def chunkify_string(s, size)
  (0 .. (s.length - 1) / size).map { |i| s[i * size,size] }
end

class FinalRecord
  attr_reader :year, :values, :dmflags, :qcflags, :dsflags
  def initialize(attrs)
    @year    = attrs.fetch(:year)
    @values  = attrs.fetch(:values)
    @dmflags = attrs.fetch(:dmflags)
    @qcflags = attrs.fetch(:qcflags)
    @dsflags = attrs.fetch(:dsflags)
  end
  def to_s
    dmf = @dmflags.map{|e| e.nil? ? "." : e}.join()
    qcf = @qcflags.map{|e| e.nil? ? "." : e}.join()
    dsf = @dsflags.map{|e| e.nil? ? "." : e}.join()
    "f #{@year} #{@values.join(",")} #{dmf} #{qcf} #{dsf}"
  end
end

class Final
  def self.from_file(filename)
    data = File.read(filename)
    Final.from_data(data)
  end
  def self.from_data(data)
    id = nil
    records = []
    lines = data.split("\n")
    lines.each do |line|
      if line.length != FINAL_LINE_LENGTH
        raise "line length #{line.length} != #{FINAL_LINE_LENGTH}"
      end
      x = line[0,11]
      if id.nil?
        id = x
      else
        raise "id #{x} != id #{id}" if x != id
      end
      year = line[12,4].to_i
      values  = Array.new(NUM_MONTHS_IN_YEAR)
      dmflags = Array.new(NUM_MONTHS_IN_YEAR)
      qcflags = Array.new(NUM_MONTHS_IN_YEAR)
      dsflags = Array.new(NUM_MONTHS_IN_YEAR)
      chunks = chunkify_string(line[16..], 9)
      chunks.each_with_index do |chunk, idx|
        value  = chunk[0,6].to_i
        dmflag = chunk[6]
        qcflag = chunk[7]
        dsflag = chunk[8]
        value  = nil if value == -9999
        dmflag = nil if dmflag == " "
        qcflag = nil if qcflag == " "
        dsflag = nil if dsflag == " "
        values[idx]  = value
        dmflags[idx] = dmflag
        qcflags[idx] = qcflag
        dsflags[idx] = dsflag
      end
      records << FinalRecord.new(
        year:year, values:values, dmflags:dmflags, qcflags:qcflags, dsflags:dsflags
      )
    end
    Final.new(id:id, records:records)
  end

  attr_reader :id, :records
  def initialize(attrs)
    @id      = attrs.fetch(:id)
    @records = attrs.fetch(:records)
  end
  def to_s
    "Final #{@id}: #{@records.length} records"
  end
end

def moving_average(a, n, precision)
  a.each_cons(n).map { |e| e.reduce(&:+).fdiv(n).round(precision) }
end

if $0 == __FILE__
  db = {}
  final_filenames = get_final_filenames
  final_filenames.each do |final_filename|
    final = Final.from_file(final_filename)
    final.records.each do |f|
      next if f.year < 1900
      f.values.each_with_index do |v, idx|
        if v
          month = idx + 1
          db[f.year]        ||= {}
          db[f.year][month] ||= []
          db[f.year][month] << v
        end
      end
    end
  end
  avgs = []
  db.each_pair do |year, months|
    months.each_pair do |month, values|
      avg = values.sum(0.0) / values.length
      avgs << ["#{year}-#{month}", avg / 100]
    end
  end
  avgs = avgs.sort {|a,b| a.first <=> b.first }
  temps = avgs.map(&:last)
  mavgs = moving_average(temps, 60, 2)
  r = avgs.map(&:first)[60..].zip(mavgs)
  r.each do |y|
    puts y.join(",")
  end
end