Utf8Cleaner for encoding problems.

gistfile1.txt

module Utf8Cleaner
  def to_utf8(new_value)
    if new_value.is_a? String
      begin
        # Try it as UTF-8 directly
        new_value.force_encoding('UTF-8')
        unless new_value.valid_encoding?
          # Some of it might be old Windows code page
          new_value.encode!( 'UTF-8', 'Windows-1252' )
        end
      rescue EncodingError
        # Force it to UTF-8, throwing out invalid bits
        new_value.encode!( 'UTF-8', invalid: :replace, undef: :replace )
      end
    end
    new_value
  end
end