python get memory size of object

nested_sizeof.py

# Recursively get size of objects
# https://stackoverflow.com/a/59228005/4126114
# https://stackoverflow.com/a/38515297/4126114
# Published at https://gist.github.com/shadiakiki1986/6d2b641264b054bc63b224a726a1ead8

def nested_apply(x, func, max_depth, depth_current=0):
  s1 = func(x)
  if depth_current >= max_depth: return s1

  wrap_napply = lambda v: nested_apply(v, func, max_depth, depth_current+1)

  if hasattr(x, "__dict__"):
    s2 = {}
    for k,v in x.__dict__.items():
      s2[k] = wrap_napply(getattr(x,k))
    return {"self": s1, "children": s2}

  if isinstance(x, dict):
    s2 = {k: wrap_napply(v) for k,v in x.items()}
    return s2

  if isinstance(x, list):
    s2 = [wrap_napply(v) for v in x]
    return s2

  return s1


import pickle
def nested_sizeof(x, factor, depth):
  """
  factor: use 1e9 to get values in GB
  """
  func = lambda x: round(len(pickle.dumps(x))/factor,1)
  return nested_apply(x, func, depth)

tests.py

import pprint
import numpy as np
f_mb, f_gb = int(1e6), int(1e9)
 
x = np.zeros((100, 10000))
pprint.pprint(nested_sizeof(x, f_mb, 1))
# 0.8

b = {"x1": x, "a": {"x2": x, "c": [1,2,3]}, "x3": np.copy(x)}
pprint.pprint(nested_sizeof(b, f_mb, 0))
pprint.pprint(nested_sizeof(b, f_mb, 1))
pprint.pprint(nested_sizeof(b, f_mb, 2))
pprint.pprint(nested_sizeof(b, f_mb, 3))
# 16
# {'a': 8,                         'x1': 8, 'x3': 8}
# {'a': {'c': 0,         'x2': 8}, 'x1': 8, 'x3': 8}
# {'a': {'c': [0, 0, 0], 'x2': 8}, 'x1': 8, 'x3': 8}


nested_apply(b, lambda x: type(x), 10)
# {'a': {'c': [int, int, int], 'x2': numpy.ndarray}, 'x1': numpy.ndarray}