Skip to content

Instantly share code, notes, and snippets.

@tuan3w
Created June 21, 2016 15:14
Show Gist options
  • Save tuan3w/af2305dc02e4f2ee6af213c61289abd5 to your computer and use it in GitHub Desktop.
Save tuan3w/af2305dc02e4f2ee6af213c61289abd5 to your computer and use it in GitHub Desktop.
fast way to read big file line by line
from functools import partial
import codecs
def fast_read(name, bytes):
with codecs.open(name, 'r', 'utf-8') as f:
prev = ''
f_read = partial(f.read, bytes)
for text in iter(f_read, ''):
if text == '':
return
if not text.endswith('\n'):
text, rest = text.rsplit('\n', 1)
text = prev + text
prev = rest
else:
text = prev + text[:-1]
prev = ''
for l in text.split('\n'):
yield l
def normal_read(name):
with codecs.open(name, 'r', 'utf-8') as f:
for l in f:
yield l
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment