Skip to content

Instantly share code, notes, and snippets.

@shuson
Last active November 6, 2022 09:43
Show Gist options
  • Save shuson/67b8bfcda372b59ac1710d4f403ef750 to your computer and use it in GitHub Desktop.
Save shuson/67b8bfcda372b59ac1710d4f403ef750 to your computer and use it in GitHub Desktop.
handle huge file reading in python
from functools import partial, wraps
from typing import TextIO, Callable
def chunked_file(fp, block_size):
for chunk in iter(partial(fp.read, block_size), ''):
yield chunk
def transformer(data):
//do transformation for input data
output = data
return output
def read_file(file_to_read, file_to_write):
with open(file_to_read) as f_read:
for chunk in chunked_file(f_read):
//do transformation
output = transformer(chunk)
write_file(file_to_write, output)
def write_file(file_path, data):
with open(file_path, "w+") as f_write:
f_write.write(data)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment