cleaner.rb

require 'iconv'

def shift
  line = ''
  in_quote = 0
  loop do
    if read_line = $stdin.gets("\n")
      line += read_line.chomp
    else
      return nil
    end
    csv = []
    current = ''
    line.split(',', -1).each do |cell|
      cell = Iconv.iconv('ascii//translit//ignore', 'utf-8', cell).first
      if cell.count('"').zero? && current.empty?
        csv << cell.squeeze(' ').strip
        next
      end
      in_quote += cell.count '"'
      current += cell
      if in_quote % 2 == 0
        csv << current.gsub(/(\n|"|\r|,)/, '').dup.squeeze(' ').strip
        current = ''
      end
    end
    if in_quote % 2 == 0
      break csv
    end
  end
end

csv = []
while !csv.nil?
  csv = shift
  $stdout.puts(csv.join ",") unless csv.nil?
end