Skip to content

Instantly share code, notes, and snippets.

@jworl
Created February 24, 2021 00:21
Show Gist options
  • Save jworl/d73970343f9abe09820bc02f5f5bcf8f to your computer and use it in GitHub Desktop.
Save jworl/d73970343f9abe09820bc02f5f5bcf8f to your computer and use it in GitHub Desktop.
boto3 s3 pagination example
/usr/bin/env python3
import boto3
from boto3.session import Session
from io import BytesIO
from sys import argv
pn = argv[1]
bucket = argv[2]
sub_folder_path = argv[3]
session = Session(profile_name=pn)
s3c = session.client("s3")
gimme = s3c.list_objects_v2(Bucket=bucket,Prefix=sub_folder_path)
my_keys = [x['Key'] for x in gimme['Contents']]
if gimme['IsTruncated']:
pages = paginator.paginate(Bucket=bucket,Prefix=sub_folder_path, PaginationConfig={'MaxItems':1000000,'PageSize':1000,'StartingToken':gimme['NextContinuationToken']})
for page in pages:
for obj in page['Contents']:
my_keys.append(obj['Key'])
accts = {}
my_keys = sorted(set(my_keys))
for this in my_keys:
f = BytesIO()
s3c.download_fileobj(bucket, this, f)
j = json.loads(f.getvalue().decode('utf-8'))
accts[j['awsAccountId']] = j
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment