utf8-magic.py

import sys

UTF8Writer = codecs.getwriter('utf8')

class SafeStdoutWriter(UTF8Writer):
    """ A safe stdout writer class which expects to output only unicode string objects
    but if it encounters a decoding error (because the given string is not unicode but
    a normal 'str') it will attempt to convert it to unicode and try again.  If that
    conversion fails we fall back to decoding iso-8859-1 which is the default codec
    the CSVOutput writer uses.
    """
    def encode(self, obj, errors):
        try:
            return UTF8Writer.encode(obj, errors)
        except UnicodeDecodeError:
            try:
                return UTF8Writer.encode(obj.decode('utf-8'), errors)
            except UnicodeDecodeError:
                return UTF8Writer.encode(obj.decode('iso-8859-1'), errors)