Created
August 27, 2011 01:05
-
-
Save eightysteele/1174811 to your computer and use it in GitHub Desktop.
Unicode DictReader and DictWriter
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
id | name | age | |
---|---|---|---|
0 | aaron | 34 | |
1 | tina | 36 | |
2 | noah | © |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
import csv, codecs, cStringIO, sys | |
class UTF8Recoder: | |
""" | |
Iterator that reads an encoded stream and reencodes the input to UTF-8 | |
""" | |
def __init__(self, f, encoding): | |
self.reader = codecs.getreader(encoding)(f) | |
def __iter__(self): | |
return self | |
def next(self): | |
return self.reader.next().encode("utf-8") | |
class UnicodeDictReader: | |
""" | |
A CSV reader which will iterate over lines in the CSV file "f", | |
which is encoded in the given encoding. | |
""" | |
def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): | |
f = UTF8Recoder(f, encoding) | |
self.reader = csv.reader(f, dialect=dialect, **kwds) | |
self.header = self.reader.next() | |
def next(self): | |
row = self.reader.next() | |
vals = [unicode(s, "utf-8") for s in row] | |
return dict((self.header[x], vals[x]) for x in range(len(self.header))) | |
def __iter__(self): | |
return self | |
class UnicodeDictWriter: | |
""" | |
A CSV writer which will write rows to CSV file "f", | |
which is encoded in the given encoding. | |
""" | |
def __init__(self, f, fieldnames, dialect=csv.excel, encoding="utf-8", **kwds): | |
# Redirect output to a queue | |
self.fieldnames = fieldnames | |
self.queue = cStringIO.StringIO() | |
self.writer = csv.writer(self.queue, dialect=dialect, **kwds) | |
self.stream = f | |
self.encoder = codecs.getincrementalencoder(encoding)() | |
def writeheader(self): | |
self.writer.writerow(self.fieldnames) | |
def writerow(self, row): | |
self.writer.writerow([row[x].encode("utf-8") for x in self.fieldnames]) | |
# Fetch UTF-8 output from the queue ... | |
data = self.queue.getvalue() | |
data = data.decode("utf-8") | |
# ... and reencode it into the target encoding | |
data = self.encoder.encode(data) | |
# write to the target stream | |
self.stream.write(data) | |
# empty queue | |
self.queue.truncate(0) | |
def writerows(self, rows): | |
for row in rows: | |
self.writerow(row) | |
if __name__ == '__main__': | |
args = sys.argv | |
if len(args) > 1: | |
# Make sure your file header is: id,name,age | |
filename = sys.argv[1] | |
else: | |
filename = 'data.csv' | |
data = 'id,name,age\n0,aaron,34\n1,john\u00A9,50'.encode('utf-8') | |
f = codecs.open(filename, encoding='utf-8', mode='w+') | |
f.write(data) | |
f.close() | |
r = UnicodeDictReader(open(filename, 'r'), skipinitialspace=True) | |
w = UnicodeDictWriter(open(filename.replace('.csv', '.out.csv'), 'w'), ['id', 'name', 'age']) | |
w.writeheader() | |
for row in r: | |
print row | |
w.writerow(row) | |
Excellent.
Very helpful thanks a lot!
it's exacly what i need, thx !
on line 55 you can use
row[x].encode("utf-8") if isinstance(row[x],basestring) else row[x]
since encode will fail if row[x] is not a string
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Thanks for this.