Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Script to measure the memory used by instances for various configurations
from itertools import product, islice
import argparse
import math
import string
import psutil
import keyword
CLI = argparse.ArgumentParser('Test memory consumption of instances')
CLI.add_argument(
'VARIANT',
choices=['dict', 'slots'],
help='class implementation to test',
)
CLI.add_argument(
'--fields',
help='number of fields',
default=4,
type=int,
)
CLI.add_argument(
'--instances-exp',
help='exponent for maximum number of instances',
default=6,
type=int,
)
CLI.add_argument(
'--instances-base',
help='base for maximum number of instances',
default=10,
type=int,
)
def make_names(count: int, characters=string.ascii_lowercase):
if count == 1:
return characters[:1]
return list(islice(filter(lambda x: not keyword.iskeyword(x),
(
''.join(_chars) for _chars in
product(characters, repeat=math.ceil(
math.log(count, len(characters))
))
)),
count
))
def make_init(names):
init_source = "def __init__(self, {0}):\n".format(', '.join(names))
init_source += '\n'.join(
f""" self.{name} = {name}"""
for name in names
)
namespace = {}
exec(init_source, namespace)
return namespace['__init__']
def make_type(size: int):
names = make_names(size)
init = make_init(names)
class DictType:
__init__ = init
class SlotsType:
__slots__ = names
__init__ = init
return DictType, SlotsType
def measure_consumption(variant, size, base, max_exp):
this_process = psutil.Process()
instance_store = [None] * (base ** max_exp)
data = [0] * size
yield 0, this_process.memory_full_info().uss
instance_store[0] = variant(*data)
yield 1, this_process.memory_full_info().uss
for exponent in range(1, max_exp + 1):
for i in range(base ** (exponent - 1), base ** exponent):
instance_store[i] = variant(*data)
yield base ** exponent, this_process.memory_full_info().uss
def measure(variant, size, base, max_exp):
print('Count'.rjust(16), 'Field'.rjust(16), 'Instance'.rjust(16), 'Absolute'.rjust(16))
base_memory = 0
for count, abs_memory in measure_consumption(variant, size, base, max_exp):
if count == 0:
base_memory = abs_memory
else:
memory = abs_memory - base_memory
relative = memory / count
instance = relative / size
print(
'%16d %16.1f %16.1f %16d' % (count, instance, relative, memory)
)
def main():
args = CLI.parse_args()
variants = make_type(args.fields)
variant = variants[{'dict': 0, 'slots': 1}[args.VARIANT]]
measure(variant, args.fields, args.instances_base, args.instances_exp)
if __name__ == "__main__":
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.