AU-NSW COVID-19 data aggregation

aggregate.rb

# frozen_string_literal: true

require 'csv'
require 'fileutils'
require 'open-uri'


TEMP_DIRECTORY = 'tmp'

# Create temp file
FileUtils.mkdir_p(File.join(Dir.getwd, TEMP_DIRECTORY))

FILES = {
  raw: 'https://data.nsw.gov.au/data/dataset/60616720-3c60-4c52-b499-751f31e3b132/resource/945c6204-272a-4cad-8e33-dde791f5059a/download/pcr_testing_table1_location.csv',
  agg: 'https://data.nsw.gov.au/data/dataset/60616720-3c60-4c52-b499-751f31e3b132/resource/fb95de01-ad82-4716-ab9a-e15cf2c78556/download/pcr_testing_table1_location_agg.csv'
}.freeze

# Download files to work with
# FILES.each_pair do |_, url|
#   filename = url.split('/').last
#   File.open(File.join(Dir.getwd, TEMP_DIRECTORY, filename), 'wb') do |file|
#     file.write(URI.open(url).read)
#   end
# end

# Summarise by day
data_by_day = { raw: {}, agg: {} }

FILES.each_pair do |key, url|
  filename = url.split('/').last

  CSV.foreach(File.join(Dir.getwd, TEMP_DIRECTORY, filename), headers: true) do |row|
    data_by_day[key][row['test_date']] ||= 0
    data_by_day[key][row['test_date']] += row['test_count'].nil? ? 1 : row['test_count'].to_i
  end
end

dates = (data_by_day[:raw].keys | data_by_day[:agg].keys).sort

# Output a CSV with a column for agg and a column for raw
CSV.open(File.join(Dir.getwd, TEMP_DIRECTORY, 'summary.csv'), 'wb') do |csv|
  csv << ['date', 'raw', 'agg']

  dates.each do |date|
    csv << [date, data_by_day[:raw][date].to_i, data_by_day[:agg][date].to_i]
  end
end

summary.csv