Purge all items from an AWS dynamodb table with an exponential timing back-off

dynamodb_purge_table.py

#!/usr/bin/env python3

""" Purge all items from an AWS dynamodb table with an exponential timing back-off
"""

import logging
from time import sleep

import boto3
from botocore.exceptions import ClientError

AWS_RESOURCE = 'dynamodb'
AWS_REGION = 'us-east-1'


def _purge_table(purge_table, key_attribute, pause_time):
    """Purge all items in purge_table"""

    batch = purge_table.batch_writer()

    response = purge_table.scan()

    sleep(pause_time)

    for item in response['Items']:
        batch.delete_item(
            Key={
                key_attribute: item[key_attribute]
            }
        )


    while 'LastEvaluatedKey' in response:

        response = purge_table.scan(
            ExclusiveStartKey=response['LastEvaluatedKey']
        )

        sleep(pause_time)

        for item in response['Items']:
            batch.delete_item(
                Key={
                    key_attribute: item[key_attribute]
                }
            )


def table_purge(purge_table, key_attribute):
    """ Purge all items from purge_table
        Catch quota exceptions, backoff, and retry as required
    """

    retry_exceptions = ('ProvisionedThroughputExceededException',
                        'ThrottlingException')
    retries = 0
    pause_time = 0

    while True:
        try:
            _purge_table(purge_table, key_attribute, pause_time)
            break
        except ClientError as err:
            if err.response['Error']['Code'] not in retry_exceptions:
                raise
            pause_time = (2 ** retries)
            logging.info('Back-off set to %d seconds', pause_time)
            retries += 1


if __name__ == "__main__":

    logging.getLogger("").setLevel(logging.INFO)
    logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%d/%m/%Y %I:%M:%S %p')

    DB = boto3.resource(AWS_RESOURCE, region_name=AWS_REGION)

    table_purge(DB.Table('MyTable'), 'KeyAttribute')

    logging.info("Processing Complete")
    logging.shutdown()