Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
Plot memory usage of qr_multiply using numpy and scipy
"""
Plot memory usage of a numeric computation using numpy and scipy
"""
"""Get process information"""
import time, sys, os
import linecache
if sys.platform.startswith('linux'):
# FIXME: kernel page size might vary between archs
def _get_memory(pid):
try:
f = open('/proc/%s/statm' % pid)
res = (int(f.read().split(' ')[1]) * 4) / 1024
f.close()
return res
except IOError:
return 0
else:
# ..
# .. better to be safe than sorry ..
raise NotImplementedError
def memory(proc= -1, num= -1, interval=.1, locals={}):
"""
Return the memory usage of a process or piece of code
Parameters
----------
proc : {int, string}
The process to monitor. Can be given by a PID or by a string
containing a filename or the code to be executed. Set to -1
(default)for current process.
interval : int, optional
num : int, optional
Number of samples to generate. In the case of
defaults to -1, meaning
to wait until the process has finished if proc is a string or
to get just one if proc is an integer.
locals : dict
Local variables.
Returns
-------
mm : list of integers
memory usage, in MB
"""
ret = []
if isinstance(proc, str):
if proc.endswith('.py'):
f = open('proc', 'r')
proc = f.read()
f.close()
# TODO: make sure script's directory is on sys.path
from multiprocessing import Process
def f(x, locals):
# function interface for exec
exec(x, locals)
p = Process(target=f, args=(proc, locals))
p.start()
while p.is_alive(): # FIXME: or num
ret.append(_get_memory(p.pid))
time.sleep(interval)
else:
if proc == -1:
proc = os.getpid()
if num == -1:
num = 1
for _ in range(num):
ret.append(_get_memory(proc))
time.sleep(interval)
return ret
import numpy as np
from scipy import linalg
X = np.random.randn(1000, 1000)
y = np.random.randn(1000)
mm = memory("linalg.qr_multiply(X, y)", interval=.01, locals=locals())
mm2 = memory("R, Q = linalg.qr(X);np.dot(Q.T, y)", interval=.01, locals=locals())
mm1 = (np.array(mm) * 4.) / 1024
mm2 = (np.array(mm2) * 4.) / 1024
mm1.resize(mm2.shape)
import pylab as pl
x = np.linspace(0, np.max(mm1), len(mm1))
p = pl.fill_between(x, mm1)
p.set_facecolors('none')
ax = pl.axes()
from matplotlib.patches import PathPatch
for path in p.get_paths():
p1 = PathPatch(path, color='green', fc="none", hatch="//")
ax.add_patch(p1)
p1.set_zorder(p.get_zorder() - 0.1)
p = pl.fill_between(x, mm2)
p.set_facecolors('none')
ax = pl.axes()
from matplotlib.patches import PathPatch
for path in p.get_paths():
p2 = PathPatch(path, color='red', fc="none", hatch="/")
ax.add_patch(p2)
p2.set_zorder(p.get_zorder() - .1)
pl.legend([p2, p1], ["naive computation", "qr_multiply"])
pl.xlabel('time')
pl.ylabel('Memory consumption (in MB)')
pl.show()

Hey Fabian,

What 'minimon'? It looks useful.

Owner

fabianp commented Oct 15, 2011

Yo, gets memory usage:

https://github.com/fabianp/minimon/blob/master/minimon.py

I'll make a python package with some additions soon.

Owner

fabianp commented Oct 20, 2011

updated to avoid the dependence

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment