Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Create tar from S3 bucket files using boto3
import boto3
import tarfile
import os.path
s3Client = boto3.client('s3')
s3object = boto3.resource('s3')
def lambda_handler(event, context):
agtBucket = "angularbuildbucket"
key=""
tar = tarfile.open('/tmp/example.tar', 'w')
source_dir="/tmp/"
for fname in get_matching_s3_keys(bucket=agtBucket, prefix=key, suffix='.js'):
print(fname)
file_obj = s3object.Object(agtBucket, fname)
#file_content = file_obj.get()['Body'].read()
#tar.add(file_content)
s3object.Bucket(agtBucket).download_file(fname, '/tmp/'+fname)
tar.add(source_dir, arcname=os.path.basename(source_dir))
tar.close()
s3object.meta.client.upload_file(source_dir+"example.tar", agtBucket, 'example.tar')
def get_matching_s3_keys(bucket, prefix='', suffix=''):
"""
Generate the keys in an S3 bucket.
:param bucket: Name of the S3 bucket.
:param prefix: Only fetch keys that start with this prefix (optional).
:param suffix: Only fetch keys that end with this suffix (optional).
"""
kwargs = {'Bucket': bucket, 'Prefix': prefix}
while True:
resp = s3Client.list_objects_v2(**kwargs)
for obj in resp['Contents']:
key = obj['Key']
if key.endswith(suffix):
yield key
try:
kwargs['ContinuationToken'] = resp['NextContinuationToken']
except KeyError:
break
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment