Script to measure the memory used by instances for various configurations

measure_instance_resources.py

from itertools import product, islice
import argparse
import math
import string
import psutil
import keyword


CLI = argparse.ArgumentParser('Test memory consumption of instances')
CLI.add_argument(
    'VARIANT',
    choices=['dict', 'slots'],
    help='class implementation to test',
)
CLI.add_argument(
    '--fields',
    help='number of fields',
    default=4,
    type=int,
)
CLI.add_argument(
    '--instances-exp',
    help='exponent for maximum number of instances',
    default=6,
    type=int,
)
CLI.add_argument(
    '--instances-base',
    help='base for maximum number of instances',
    default=10,
    type=int,
)


def make_names(count: int, characters=string.ascii_lowercase):
    if count == 1:
        return characters[:1]
    return list(islice(filter(lambda x: not keyword.iskeyword(x),
        (
            ''.join(_chars) for _chars in
            product(characters, repeat=math.ceil(
                math.log(count, len(characters))
            ))
        )),
        count
    ))


def make_init(names):
    init_source = "def __init__(self, {0}):\n".format(', '.join(names))
    init_source += '\n'.join(
        f"""    self.{name} = {name}"""
        for name in names
    )
    namespace = {}
    exec(init_source, namespace)
    return namespace['__init__']


def make_type(size: int):
    names = make_names(size)
    init = make_init(names)

    class DictType:
        __init__ = init

    class SlotsType:
        __slots__ = names
        __init__ = init

    return DictType, SlotsType


def measure_consumption(variant, size, base, max_exp):
    this_process = psutil.Process()
    instance_store = [None] * (base ** max_exp)
    data = [0] * size
    yield 0, this_process.memory_full_info().uss
    instance_store[0] = variant(*data)
    yield 1, this_process.memory_full_info().uss
    for exponent in range(1, max_exp + 1):
        for i in range(base ** (exponent - 1), base ** exponent):
            instance_store[i] = variant(*data)
        yield base ** exponent, this_process.memory_full_info().uss


def measure(variant, size, base, max_exp):
    print('Count'.rjust(16), 'Field'.rjust(16), 'Instance'.rjust(16), 'Absolute'.rjust(16))
    base_memory = 0
    for count, abs_memory in measure_consumption(variant, size, base, max_exp):
        if count == 0:
            base_memory = abs_memory
        else:
            memory = abs_memory - base_memory
            relative = memory / count
            instance = relative / size
            print(
                '%16d %16.1f %16.1f %16d' % (count, instance, relative, memory)
            )


def main():
    args = CLI.parse_args()
    variants = make_type(args.fields)
    variant = variants[{'dict': 0, 'slots': 1}[args.VARIANT]]
    measure(variant, args.fields, args.instances_base, args.instances_exp)


if __name__ == "__main__":
    main()