s3-client-benchmark.py

# Sorry for the scrappiness of this. It's supposed to be scrappy.

import time
from botocore.exceptions import ClientError
import boto3
import boto3.session
# boto3.set_stream_logger('')

session = boto3.session.Session()
s3 = session.client('s3')

B = 'symbols-public-dev'
K_base = 'test_boto3_issue_1128/test_boto3_issue_1128_{}.txt'


def _key_existing_size__list(client, bucket, key):
    """return the key's size if it exist, else None"""
    response = client.list_objects_v2(
        Bucket=bucket,
        Prefix=key,
    )
    for obj in response.get('Contents', []):
        if obj['Key'] == key:
            return obj['Size']


def _key_existing_size__head(client, bucket, key):
    try:
        obj = client.head_object(Bucket=bucket, Key=key)
        return obj['ContentLength']
    except ClientError as exc:
        if exc.response['Error']['Code'] != '404':
            raise

import random
functions = [_key_existing_size__list, _key_existing_size__head]
times = {k.__name__: [] for k in functions}

for i in range(1000):

    K = K_base.format(i)
    t0 = time.time()

    func = random.choice(functions)
    size = func(s3, B, K)
    if size is not None:
        print("EXISTED ALREADY", repr(size))
        t1 = time.time()
    else:
        print("DID NOT EXIST")
        t1 = time.time()
        s3.put_object(Bucket=B, Key=K, Body=b'asdfasdf')

    times[func.__name__].append(t1 - t0)

import statistics
for name, numbers in times.items():
    print('FUNCTION:', name, 'Used', len(numbers), 'times')
    print('\tSUM   ', sum(numbers))
    print('\tMEAN  ', statistics.mean(numbers))
    print('\tMEDIAN', statistics.median(numbers))
    print('\tSTDEV ', statistics.stdev(numbers))
    print()