Skip to content

Instantly share code, notes, and snippets.

@AvinashDalvi89
Created October 13, 2020 19:24
Show Gist options
  • Save AvinashDalvi89/31fc1fc450edb2c4af19d47f36ab1259 to your computer and use it in GitHub Desktop.
Save AvinashDalvi89/31fc1fc450edb2c4af19d47f36ab1259 to your computer and use it in GitHub Desktop.
Create tar from S3 bucket files using boto3
import boto3
import tarfile
import os.path
s3Client = boto3.client('s3')
s3object = boto3.resource('s3')
def lambda_handler(event, context):
agtBucket = "angularbuildbucket"
key=""
tar = tarfile.open('/tmp/example.tar', 'w')
source_dir="/tmp/"
for fname in get_matching_s3_keys(bucket=agtBucket, prefix=key, suffix='.js'):
print(fname)
file_obj = s3object.Object(agtBucket, fname)
#file_content = file_obj.get()['Body'].read()
#tar.add(file_content)
s3object.Bucket(agtBucket).download_file(fname, '/tmp/'+fname)
tar.add(source_dir, arcname=os.path.basename(source_dir))
tar.close()
s3object.meta.client.upload_file(source_dir+"example.tar", agtBucket, 'example.tar')
def get_matching_s3_keys(bucket, prefix='', suffix=''):
"""
Generate the keys in an S3 bucket.
:param bucket: Name of the S3 bucket.
:param prefix: Only fetch keys that start with this prefix (optional).
:param suffix: Only fetch keys that end with this suffix (optional).
"""
kwargs = {'Bucket': bucket, 'Prefix': prefix}
while True:
resp = s3Client.list_objects_v2(**kwargs)
for obj in resp['Contents']:
key = obj['Key']
if key.endswith(suffix):
yield key
try:
kwargs['ContinuationToken'] = resp['NextContinuationToken']
except KeyError:
break
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment