Skip to content

Instantly share code, notes, and snippets.

@pjsier
Created August 26, 2020 14:00
Show Gist options
  • Save pjsier/3a813244545163e3cdf276328e539186 to your computer and use it in GitHub Desktop.
Save pjsier/3a813244545163e3cdf276328e539186 to your computer and use it in GitHub Desktop.
import json
import os
import boto3
# We don't need to manually specify the credentials if they have these names, but
# including for clarity
client = boto3.client(
"s3",
aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"),
)
provider = "testprovider"
snapshots = []
# Prefix can also be created with datetime.strftime
for snapshot in client.list_objects_v2(
Bucket=os.getenv("S3_BUCKET"), Prefix=f"{provider}/2020/08/25", MaxKeys=1000
)["Contents"]:
# Ignore irrelevant keys
if "samplestring" not in snapshot["Key"]:
continue
print(snapshot["Key"])
snapshot_obj = client.get_object(Bucket=os.getenv("S3_BUCKET"), Key=snapshot["Key"])
snapshot_data = json.load(snapshot_obj["Body"])
snapshots.append(snapshot_data)
print(f"Loaded {len(snapshots)}")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment