send bucket cleanup script for https://send.ephemeral.land

lifecycle-s3-compatible.py

# send bucket cleanup script for https://send.ephemeral.land
# Editable variables
# __
aws_access_key_id = "AKIAIOSFODNN7EXAMPLE"
aws_secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
bucket = "mysendbucket"
lower_limit_days = 1
upper_limit_days = 28
prefixes = range(lower_limit_days, upper_limit_days+1)
endpoint = "https://s3.eu-central-1.wasabisys.com"  # Endpoint of bucket
# __

from boto3 import client, Session
from botocore.exceptions import ClientError
from datetime import datetime, timezone
from os import sys
import logging

logger = logging.getLogger()
logger.setLevel(logging.DEBUG) # process everything, even if everything isn't printed

ch = logging.StreamHandler()
ch.setLevel(logging.INFO) # or any other level
logger.addHandler(ch)
fh = logging.FileHandler('/path/to/send-cron.log')
fh.setLevel(logging.WARN) # or any level you want
logger.addHandler(fh)

if __name__ == '__main__':

    # get current date
    today = datetime.now(timezone.utc)


    try:
        # create a connection to Wasabi
        s3_client = client(
            's3',
            endpoint_url=endpoint,
            aws_access_key_id=aws_access_key_id,
            aws_secret_access_key=aws_secret_access_key)
    except Exception as e:
        raise e

    try:
        # list all the buckets under the account
        list_buckets = s3_client.list_buckets()
    except ClientError:
        # invalid access keys
        raise Exception("Invalid Access or Secret key")

    # instantiate temp variables.
    delete_list = []
    count_current = 0
    count_non_current = 0

    logger.info("$ Paginating bucket " + bucket)
    for prefix in prefixes:
        # create a paginator for all objects.
        object_response_paginator = s3_client.get_paginator('list_object_versions')
        if prefix:
            operation_parameters = {'Bucket': bucket,
                                    'Prefix': str(prefix) + "-"}
        else:
            operation_parameters = {'Bucket': bucket}

        for object_response_itr in object_response_paginator.paginate(**operation_parameters):
            if 'DeleteMarkers' in object_response_itr:
                for delete_marker in object_response_itr['DeleteMarkers']:
                    if (today - delete_marker['LastModified']).days > int(prefix):
                        delete_list.append({'Key': delete_marker['Key'], 'VersionId': delete_marker['VersionId']})
            if 'Versions' in object_response_itr:
                for version in object_response_itr['Versions']:
                    if version["IsLatest"] is True:
                        count_current += 1
                    elif version["IsLatest"] is False:
                        count_non_current += 1
                    if (today - version['LastModified']).days > int(prefix):
                        delete_list.append({'Key': version['Key'], 'VersionId': version['VersionId']})

    # print objects count
    logger.info("-" * 20)
    logger.warning(today)
    logger.info("-" * 20)
    logger.info("$ Before deleting objects")
    logger.info("$ current objects: " + str(count_current))
    logger.info("$ non-current objects: " + str(count_non_current))
    logger.info("-" * 20)

    # delete objects 1000 at a time
    logger.warning("$ Deleting objects from bucket " + bucket)
    for i in range(0, len(delete_list), 1000):
        response = s3_client.delete_objects(
            Bucket=bucket,
            Delete={
                'Objects': delete_list[i:i + 1000],
                'Quiet': True
            }
        )
        logger.info(response)

    # reset counts
    count_current = 0
    count_non_current = 0

    # paginate and recount
    logger.info("$ Paginating bucket " + bucket)
    for object_response_itr in object_response_paginator.paginate(Bucket=bucket):
        if 'Versions' in object_response_itr:
            for version in object_response_itr['Versions']:
                if version["IsLatest"] is True:
                    count_current += 1
                elif version["IsLatest"] is False:
                    count_non_current += 1

    # print objects count
    logger.info("-" * 20)
    logger.info("$ After deleting objects")
    logger.info("$ current objects: " + str(count_current))
    logger.info("$ non-current objects: " + str(count_non_current))
    logger.info("-" * 20)
    logger.info("$ task complete")

requirements.txt

boto3==1.17.45
botocore==1.20.45
jmespath==0.10.0
python-dateutil==2.8.1
s3transfer==0.3.6
six==1.15.0
urllib3==1.26.4