Skip to content

Instantly share code, notes, and snippets.

@bnewbold
Last active October 16, 2017 11:52
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save bnewbold/8919a20b1f01532b0da4f1c5594c6a05 to your computer and use it in GitHub Desktop.
Save bnewbold/8919a20b1f01532b0da4f1c5594c6a05 to your computer and use it in GitHub Desktop.
Broken python3 gluish TSV unicode

Do the whole thing:

PYTHONPATH='.' luigi --module small Small --local-scheduler
diff --git a/gluish/format.py b/gluish/format.py
index 40ddf64..06872fb 100644
--- a/gluish/format.py
+++ b/gluish/format.py
@@ -59,7 +59,7 @@ def write_tsv(output_stream, *tup):
Write argument list in `tup` out as a tab-separeated row to the stream.
"""
value = '\t'.join([s for s in tup]) + '\n'
- output_stream.write(str(value))
+ output_stream.write(value.encode('utf8'))
def iter_tsv(input_stream, cols=None):
@@ -83,10 +83,10 @@ def iter_tsv(input_stream, cols=None):
for c in cols]
Record = collections.namedtuple('Record', cols)
for line in input_stream:
- yield Record._make(str(line).rstrip('\n').split('\t'))
+ yield Record._make(line.decode('utf8').rstrip('\n').split('\t'))
else:
for line in input_stream:
- yield tuple(str(line).rstrip('\n').split('\t'))
+ yield tuple(line.decode('utf8').rstrip('\n').split('\t'))
class TSVFormat(luigi.format.Format):
1 3 4
asdfasdf sdjf ģģģ 98
. ,
import luigi
from gluish.format import TSV
class SampleFile(luigi.ExternalTask):
file_name = luigi.Parameter(default="input_sample.tsv")
def output(self):
return luigi.LocalTarget(self.file_name, format=TSV)
class Small(luigi.Task):
def requires(self):
return SampleFile()
def run(self):
with self.input().open() as in_file, self.output().open('w') as out_file:
for row in in_file.iter_tsv(
cols=('col1', 'col2', 'col3')):
out_file.write_tsv(row.col2, row.col3)
def output(self):
return luigi.LocalTarget('output.tsv', format=TSV)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment