Plot memory usage of qr_multiply using numpy and scipy

plot_memory.py

"""
Plot memory usage of a numeric computation using numpy and scipy
"""

"""Get process information"""

import time, sys, os
import linecache

if sys.platform.startswith('linux'):
    # FIXME: kernel page size might vary between archs
    def _get_memory(pid):
        try:
            f = open('/proc/%s/statm' % pid)
            res = (int(f.read().split(' ')[1]) * 4) / 1024
            f.close()
            return res
        except IOError:
            return 0
else:
    # ..
    # .. better to be safe than sorry ..
    raise NotImplementedError

def memory(proc= -1, num= -1, interval=.1, locals={}):
    """
    Return the memory usage of a process or piece of code

    Parameters
    ----------
    proc : {int, string}
        The process to monitor. Can be given by a PID or by a string
        containing a filename or the code to be executed. Set to -1
        (default)for current process.

    interval : int, optional

    num : int, optional
        Number of samples to generate. In the case of
        defaults to -1, meaning
        to wait until the process has finished if proc is a string or
        to get just one if proc is an integer.

    locals : dict
        Local variables.

    Returns
    -------
    mm : list of integers
        memory usage, in MB
    """
    ret = []

    if isinstance(proc, str):
        if proc.endswith('.py'):
            f = open('proc', 'r')
            proc = f.read()
            f.close()
            # TODO: make sure script's directory is on sys.path
        from multiprocessing import Process
        def f(x, locals):
            # function interface for exec
            exec(x, locals)

        p = Process(target=f, args=(proc, locals))
        p.start()
        while p.is_alive(): # FIXME: or num
            ret.append(_get_memory(p.pid))
            time.sleep(interval)
    else:
        if proc == -1:
            proc = os.getpid()
        if num == -1:
            num = 1
        for _ in range(num):
            ret.append(_get_memory(proc))
            time.sleep(interval)
    return ret


import numpy as np
from scipy import linalg

X = np.random.randn(1000, 1000)
y = np.random.randn(1000)
mm = memory("linalg.qr_multiply(X, y)", interval=.01, locals=locals())
mm2 = memory("R, Q = linalg.qr(X);np.dot(Q.T, y)", interval=.01, locals=locals())
mm1 = (np.array(mm) * 4.) / 1024
mm2 = (np.array(mm2) * 4.) / 1024
mm1.resize(mm2.shape)

import pylab as pl
x = np.linspace(0, np.max(mm1), len(mm1))
p = pl.fill_between(x, mm1)
p.set_facecolors('none')
ax = pl.axes()
from matplotlib.patches import PathPatch
for path in p.get_paths():
    p1 = PathPatch(path, color='green', fc="none", hatch="//")
    ax.add_patch(p1)
    p1.set_zorder(p.get_zorder() - 0.1)

p = pl.fill_between(x, mm2)
p.set_facecolors('none')
ax = pl.axes()
from matplotlib.patches import PathPatch
for path in p.get_paths():
    p2 = PathPatch(path, color='red', fc="none", hatch="/")
    ax.add_patch(p2)
    p2.set_zorder(p.get_zorder() - .1)
pl.legend([p2, p1], ["naive computation", "qr_multiply"])
pl.xlabel('time')
pl.ylabel('Memory consumption (in MB)')
pl.show()

Hey Fabian,

What 'minimon'? It looks useful.

Yo, gets memory usage:

https://github.com/fabianp/minimon/blob/master/minimon.py

I'll make a python package with some additions soon.

updated to avoid the dependence