Skip to content

Instantly share code, notes, and snippets.

@maxfischer2781
Created June 19, 2019 16:39
Show Gist options
  • Save maxfischer2781/d6b617771e76ddd518f1de8b1221e57e to your computer and use it in GitHub Desktop.
Save maxfischer2781/d6b617771e76ddd518f1de8b1221e57e to your computer and use it in GitHub Desktop.
Script to measure the memory used by instances for various configurations
from itertools import product, islice
import argparse
import math
import string
import psutil
import keyword
CLI = argparse.ArgumentParser('Test memory consumption of instances')
CLI.add_argument(
'VARIANT',
choices=['dict', 'slots'],
help='class implementation to test',
)
CLI.add_argument(
'--fields',
help='number of fields',
default=4,
type=int,
)
CLI.add_argument(
'--instances-exp',
help='exponent for maximum number of instances',
default=6,
type=int,
)
CLI.add_argument(
'--instances-base',
help='base for maximum number of instances',
default=10,
type=int,
)
def make_names(count: int, characters=string.ascii_lowercase):
if count == 1:
return characters[:1]
return list(islice(filter(lambda x: not keyword.iskeyword(x),
(
''.join(_chars) for _chars in
product(characters, repeat=math.ceil(
math.log(count, len(characters))
))
)),
count
))
def make_init(names):
init_source = "def __init__(self, {0}):\n".format(', '.join(names))
init_source += '\n'.join(
f""" self.{name} = {name}"""
for name in names
)
namespace = {}
exec(init_source, namespace)
return namespace['__init__']
def make_type(size: int):
names = make_names(size)
init = make_init(names)
class DictType:
__init__ = init
class SlotsType:
__slots__ = names
__init__ = init
return DictType, SlotsType
def measure_consumption(variant, size, base, max_exp):
this_process = psutil.Process()
instance_store = [None] * (base ** max_exp)
data = [0] * size
yield 0, this_process.memory_full_info().uss
instance_store[0] = variant(*data)
yield 1, this_process.memory_full_info().uss
for exponent in range(1, max_exp + 1):
for i in range(base ** (exponent - 1), base ** exponent):
instance_store[i] = variant(*data)
yield base ** exponent, this_process.memory_full_info().uss
def measure(variant, size, base, max_exp):
print('Count'.rjust(16), 'Field'.rjust(16), 'Instance'.rjust(16), 'Absolute'.rjust(16))
base_memory = 0
for count, abs_memory in measure_consumption(variant, size, base, max_exp):
if count == 0:
base_memory = abs_memory
else:
memory = abs_memory - base_memory
relative = memory / count
instance = relative / size
print(
'%16d %16.1f %16.1f %16d' % (count, instance, relative, memory)
)
def main():
args = CLI.parse_args()
variants = make_type(args.fields)
variant = variants[{'dict': 0, 'slots': 1}[args.VARIANT]]
measure(variant, args.fields, args.instances_base, args.instances_exp)
if __name__ == "__main__":
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment