De-duplicate queries w/counts

dedupe.rb

require 'csv'
require 'json'

# Needs an in.csv with query and count columns.

def normalise_query(query)
  query.downcase.gsub('+', ' ').strip # .split(' ').sort.join(' ')
end

data = File.open('in.csv').read
default_h = Hash.new(0)
h = CSV.parse(data).each_with_object(default_h) do |(query,count), hsh|
  hsh[normalise_query(query)] += Integer(count, 10)
end

CSV.open("out.csv", "wb") {|csv| h.to_a.each {|elem| csv << elem} }