import os, time, gc, psutil
from pympler import tracker
import numpy as np
from sklearn.ensemble import ExtraTreesClassifier
tracker.memory_tracker = tracker.SummaryTracker()
def get_mem():
return "{:.0f}MB".format(p.memory_info().rss / 1e6)
p = psutil.Process()
X = np.random.normal(size=(11000, 12))
y = np.random.randint(0, 3, size=(11000, ))
def sleep_1s_and_print_mem(title):
time.sleep(1)
tracker.memory_tracker.print_diff()
print("\n\n" + title + " : " + get_mem() + " " + "=" * 50 + "\n\n")
sleep_1s_and_print_mem("Initial memory")
for i in range(7):
et = ExtraTreesClassifier(n_estimators=1000, n_jobs=8).fit(X, y)
del et
gc.collect()
sleep_1s_and_print_mem("After iteration %d" % i)types | # objects | total size
=================================== | =========== | ============
<class 'numpy.ndarray | 2 | 1.09 MB
<class 'list | 6441 | 606.73 KB
<class 'str | 6402 | 453.00 KB
<class 'int | 1214 | 33.26 KB
<class 'dict | 17 | 4.62 KB
<class 'wrapper_descriptor | 32 | 2.50 KB
<class 'weakref | 16 | 1.25 KB
<class 'type | 0 | 672 B
function (store_info) | 1 | 136 B
function (sleep_1s_and_print_mem) | 1 | 136 B
function (get_mem) | 1 | 136 B
<class 'cell | 2 | 96 B
<class 'psutil._pslinux.Process | 1 | 72 B
<class 'psutil.Process | 1 | 56 B
<class 'method | -1 | -64 B
Initial memory : 74MB ==================================================
types | # objects | total size
=========================== | =========== | ============
<class 'type | 5 | 9.09 KB
<class 'dict | 11 | 7.44 KB
<class 'code | 28 | 3.94 KB
<class 'str | 38 | 2.84 KB
<class 'abc.ABCMeta | 0 | 2.06 KB
<class 'set | 3 | 1.16 KB
<class 'weakref | 14 | 1.09 KB
<class 'list | 11 | 1.03 KB
<class 'tuple | 13 | 848 B
function (__init__) | 5 | 680 B
<class 'getset_descriptor | 8 | 576 B
function (close) | 2 | 272 B
function (__exit__) | 2 | 272 B
function (__repr__) | 2 | 272 B
function (__enter__) | 2 | 272 B
After iteration 0 : 194MB ==================================================
types | # objects | total size
============== | =========== | ============
<class 'set | 0 | 512 B
<class 'list | 3 | 264 B
<class 'str | 3 | 233 B
<class 'int | 7 | 196 B
After iteration 1 : 604MB ==================================================
types | # objects | total size
======= | =========== | ============
After iteration 2 : 564MB ==================================================
types | # objects | total size
======= | =========== | ============
After iteration 3 : 353MB ==================================================
types | # objects | total size
======= | =========== | ============
After iteration 4 : 441MB ==================================================
types | # objects | total size
======= | =========== | ============
After iteration 5 : 397MB ==================================================
types | # objects | total size
======= | =========== | ============
After iteration 6 : 429MB ==================================================