threaded aws s3 bulk object modify and move

rewrite.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""bulk object modify and move."""
import concurrent.futures as cf
import json
import boto3

from mypy_boto3_s3.service_resource import Bucket, ObjectSummary

BUCKET_NAME = "some-bucket"


def modify(bucket: Bucket, i: ObjectSummary):
    cont = i.get()["Body"].read()
    obj = json.loads(cont)
    obj["created_at"] = int(i.last_modified.timestamp())
    out = json.dumps(obj)
    nbase = i.key.split("/")[-1]
    nkey = f"queue/webhooks/{nbase}"
    bucket.put_object(Key=nkey, Body=out)
    i.delete()
    return nkey

def get_bucket() -> Bucket:
    session = boto3.Session()
    return session.resource("s3").Bucket(BUCKET_NAME) # type: ignore


def main():
    """Run main function."""
    bucket = get_bucket()
    with cf.ThreadPoolExecutor(max_workers=20) as executor:
        futures = []
        for i in bucket.objects.filter(Prefix="payload/").all():
            futures.append(executor.submit(modify, bucket, i))
        for future in cf.as_completed(futures):
            print(f"{future.result()} Done")


if __name__ == "__main__":
    main()