Create a gist now

Instantly share code, notes, and snippets.

Plot memory usage of qr_multiply using numpy and scipy
"""
Plot memory usage of a numeric computation using numpy and scipy
"""
"""Get process information"""
import time, sys, os
import linecache
if sys.platform.startswith('linux'):
# FIXME: kernel page size might vary between archs
def _get_memory(pid):
try:
f = open('/proc/%s/statm' % pid)
res = (int(f.read().split(' ')[1]) * 4) / 1024
f.close()
return res
except IOError:
return 0
else:
# ..
# .. better to be safe than sorry ..
raise NotImplementedError
def memory(proc= -1, num= -1, interval=.1, locals={}):
"""
Return the memory usage of a process or piece of code
Parameters
----------
proc : {int, string}
The process to monitor. Can be given by a PID or by a string
containing a filename or the code to be executed. Set to -1
(default)for current process.
interval : int, optional
num : int, optional
Number of samples to generate. In the case of
defaults to -1, meaning
to wait until the process has finished if proc is a string or
to get just one if proc is an integer.
locals : dict
Local variables.
Returns
-------
mm : list of integers
memory usage, in MB
"""
ret = []
if isinstance(proc, str):
if proc.endswith('.py'):
f = open('proc', 'r')
proc = f.read()
f.close()
# TODO: make sure script's directory is on sys.path
from multiprocessing import Process
def f(x, locals):
# function interface for exec
exec(x, locals)
p = Process(target=f, args=(proc, locals))
p.start()
while p.is_alive(): # FIXME: or num
ret.append(_get_memory(p.pid))
time.sleep(interval)
else:
if proc == -1:
proc = os.getpid()
if num == -1:
num = 1
for _ in range(num):
ret.append(_get_memory(proc))
time.sleep(interval)
return ret
import numpy as np
from scipy import linalg
X = np.random.randn(1000, 1000)
y = np.random.randn(1000)
mm = memory("linalg.qr_multiply(X, y)", interval=.01, locals=locals())
mm2 = memory("R, Q = linalg.qr(X);np.dot(Q.T, y)", interval=.01, locals=locals())
mm1 = (np.array(mm) * 4.) / 1024
mm2 = (np.array(mm2) * 4.) / 1024
mm1.resize(mm2.shape)
import pylab as pl
x = np.linspace(0, np.max(mm1), len(mm1))
p = pl.fill_between(x, mm1)
p.set_facecolors('none')
ax = pl.axes()
from matplotlib.patches import PathPatch
for path in p.get_paths():
p1 = PathPatch(path, color='green', fc="none", hatch="//")
ax.add_patch(p1)
p1.set_zorder(p.get_zorder() - 0.1)
p = pl.fill_between(x, mm2)
p.set_facecolors('none')
ax = pl.axes()
from matplotlib.patches import PathPatch
for path in p.get_paths():
p2 = PathPatch(path, color='red', fc="none", hatch="/")
ax.add_patch(p2)
p2.set_zorder(p.get_zorder() - .1)
pl.legend([p2, p1], ["naive computation", "qr_multiply"])
pl.xlabel('time')
pl.ylabel('Memory consumption (in MB)')
pl.show()
@GaelVaroquaux

Hey Fabian,

What 'minimon'? It looks useful.

@fabianp
Owner
fabianp commented Oct 15, 2011

Yo, gets memory usage:

https://github.com/fabianp/minimon/blob/master/minimon.py

I'll make a python package with some additions soon.

@fabianp
Owner
fabianp commented Oct 20, 2011

updated to avoid the dependence

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment