Normalize to UTF8 (remove invalid utf8 characters)

utf8_normalization.rb

  #
  # Takes possible invalid UTF-8 output and ignores invalid bytes
  #
  # http://po-ru.com/diary/fixing-invalid-utf-8-in-ruby-revisited/
  def self.normalize_to_utf8_output(response)

    if response[:data]
      ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
      output = response[:data][:output]||''
      valid_output = ic.iconv(output + ' ')[0..-2]
      response[:data][:output] = valid_output
    else
      Log.warn "Skipping UTF-8 normalization since provided output does not have :data element. Response #{response}"
    end

    response
  end