Download AWS RDS / Aurora logs between specified dates - working around truncated files from broken API

fetch_rds_logs.py

import os
import argparse

import botocore.session
import botocore.awsrequest

# Arguments
parser = argparse.ArgumentParser(
    description=
        '''
        Download AWS RDS/Aurora log files in full using a hidden deprecated API given the pagination of
        `aws-cli rds download-db-log-file-portion` is completely broken, truncating files with invalid markers as well
        as leaving junk in the output.
        '''
)
parser.add_argument('--region', dest="region", default='eu-west-1', help='AWS region in which the AWS RDS/Aurora database is')
parser.add_argument('--database-identifier', dest="db_name", required=True, help='AWS RDS/Aurora database identifier')
parser.add_argument('--log-name-filter', dest='log_filter', default='', help='Filters log files containing the string')
args = parser.parse_args()

# Helper function
def convert_bytes(num):
    step_unit = 1000.0
    for x in ['bytes', 'KB', 'MB', 'GB', 'TB']:
        if num < step_unit:
            return "%3.1f %s" % (num, x)
        num /= step_unit

# AWS client
session = botocore.session.get_session()
client = session.create_client('rds', region_name=args.region)

# For all log files matching.
response = client.describe_db_log_files(DBInstanceIdentifier=args.db_name, FilenameContains=args.log_filter)
for log_file in response['DescribeDBLogFiles']:
    while True:
        print('Downloading ' + log_file['LogFileName'] + ' (' + convert_bytes(log_file['Size']) + ')')

        request = botocore.awsrequest.create_request_object({
            'method': 'GET',
            'url': client.meta.endpoint_url + '/v13/downloadCompleteLogFile/' + args.db_name + '/' + log_file['LogFileName'],
            'headers': {'User-Agent': client.meta.config.user_agent},
            'context': {'client_config': client.meta.config},
            'body': None,
        })

        try:
            client.meta.events.emit('request-created.rds', request=request)
            response = client._endpoint.http_session.send(request.prepare())

            if response.status_code != 200:
                print('Got status code ' + response.status + '.. Retrying')
                continue

            log_file_path = os.path.join(args.db_name, log_file['LogFileName'])
            os.makedirs(os.path.dirname(log_file_path), exist_ok=True)
            with open(log_file_path, mode='wb') as f:
                f.write(response.content)

            break
        except KeyboardInterrupt:
            raise
        except:
            print('Got exception.. Retrying')
            pass