download_snapshots.py

import json
import os

import boto3

# We don't need to manually specify the credentials if they have these names, but
# including for clarity
client = boto3.client(
    "s3",
    aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
    aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"),
)

provider = "testprovider"

snapshots = []

# Prefix can also be created with datetime.strftime
for snapshot in client.list_objects_v2(
    Bucket=os.getenv("S3_BUCKET"), Prefix=f"{provider}/2020/08/25", MaxKeys=1000
)["Contents"]:
    # Ignore irrelevant keys
    if "samplestring" not in snapshot["Key"]:
        continue
    print(snapshot["Key"])
    snapshot_obj = client.get_object(Bucket=os.getenv("S3_BUCKET"), Key=snapshot["Key"])
    snapshot_data = json.load(snapshot_obj["Body"])
    snapshots.append(snapshot_data)

print(f"Loaded {len(snapshots)}")