Skip to content

Instantly share code, notes, and snippets.

@kouk
Last active June 19, 2020 19:39
Show Gist options
  • Save kouk/0f8b19979c0bc636c604cc4f2a664b54 to your computer and use it in GitHub Desktop.
Save kouk/0f8b19979c0bc636c604cc4f2a664b54 to your computer and use it in GitHub Desktop.
import io
import boto3
import datetime
from zipfile import ZipFile
from dateutil.tz import tzutc
from botocore.config import Config
outfile = sys.argv[1]
fromdate = datetime.datetime(2020, 5, 7, 0, 0, 0, tzinfo=tzutc())
bucket = 'endlesspit'
prefix = 'somefiles/'
class Buf(io.BytesIO):
def getAndReset(self):
try:
return self.getvalue()
finally:
self.seek(0)
self.truncate()
b = boto3.resource('s3', config=Config(
retries = {
'max_attempts': 10,
'mode': 'adaptive'
}
)
).Bucket(bucket)
buf = Buf()
with ZipFile(outfile, 'w') as zf:
for i in b.objects.filter(Prefix=prefix):
if i.last_modified >= fromdate:
i.Object().download_fileobj(buf)
member = ZipInfo(i.key, i.last_modified.timetuple())
zf.writestr(member, buf.getAndReset())
@kouk
Copy link
Author

kouk commented Jun 19, 2020

Example:
aws-vault exec -d 24h myprofile -- python3 zip-some-s3-files.py output.zip

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment