Skip to content

Instantly share code, notes, and snippets.

@a-hisame
Created July 3, 2018 10:41
Show Gist options
  • Star 17 You must be signed in to star a gist
  • Fork 6 You must be signed in to fork a gist
  • Save a-hisame/be1c1746520b2b5589901a1045bb2174 to your computer and use it in GitHub Desktop.
Save a-hisame/be1c1746520b2b5589901a1045bb2174 to your computer and use it in GitHub Desktop.
To use gzip file between python application and S3 directly for Python3
#!/usr/bin/python
# -*- coding: utf-8 -*-
'''To use gzip file between python application and S3 directly for Python3.
Python 2 version - https://gist.github.com/a-hisame/f90815f4fae695ad3f16cb48a81ec06e
'''
import io
import gzip
import json
import boto3
def upload_json_gz(s3client, bucket, key, obj, default=None, encoding='utf-8'):
''' upload python dict into s3 bucket with gzip archive '''
inmem = io.BytesIO()
with gzip.GzipFile(fileobj=inmem, mode='wb') as fh:
with io.TextIOWrapper(fh, encoding=encoding) as wrapper:
wrapper.write(json.dumps(obj, ensure_ascii=False, default=default))
inmem.seek(0)
s3client.put_object(Bucket=bucket, Body=inmem, Key=key)
def download_json_gz(s3client, bucket, key):
''' download gzipped json file from s3 and convert to dict '''
response = s3client.get_object(Bucket=bucket, Key=key)
content = response['Body'].read()
with gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb') as fh:
return json.load(fh)
if __name__ == '__main__':
bucketname = '' # input for your bucketname
key = 'tmp.json.gz' # input for your key on S3 (means S3 object fullpath)
s3 = boto3.client('s3')
upload_json_gz(s3, bucketname, key, {'あ': 'いうえお'})
actual = download_json_gz(s3, bucketname, key)
assert actual == {'あ': 'いうえお'}
@stmeigs
Copy link

stmeigs commented Aug 3, 2020

This was really helpful, thank you!

@prideloki
Copy link

👍

@bruno-uy
Copy link

Really useful. Thanks!

@adwirawan
Copy link

Works really fine and super helpful! Thanks!

@MichaelLan
Copy link

👍🏾

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment