Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
#!/usr/bin/env python3
import boto3
import gzip
import io
import sys
import tempfile
BUCKET = 'SOME_BUCKET'
PATH = 'SOME_PATH'
# also see http://pandas.pydata.org/pandas-docs/stable/generated/pandas.read_csv.html - 'compression' setting
def iterate_gz_file(filename='file_name'):
s3 = boto3.resource('s3')
with tempfile.NamedTemporaryFile() as tmp_file:
s3.meta.client.download_file(LOG_BUCKET, '%s/%s' % (LOG_PATH, filename) , tmp_file.name)
with io.TextIOWrapper(io.BufferedReader(gzip.open(tmp_file.name))) as file:
for line in file:
print(line)
if __name__ == '__main__':
iterate_gz_file()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment