read_csv_with_bad_unicode.py

## For Python 2
# import codecs
# from future.utils.surrogateescape import register_surrogateescape
# register_surrogateescape()
## Python 2 doesn't have an errors parameter in its global 'open' method, so, import codecs and use codecs.open

# with codecs.open('./youtube_quils_results.csv','r', encoding="utf-8", errors="surrogateescape") as the_file:
with open('./youtube_quils_results.csv','r', encoding="utf-8", errors="surrogateescape") as the_file:
    original = pandas.read_csv(the_file, iterator=True, encoding="utf-8")
    unsafe_set = set(pandas.read_csv('unsafe_list.csv', header=None, squeeze=True))