Profiling Jedi Memory using memory_profiler

jedi_mem_profiler.py

# -*- coding: utf-8 -*-
"""
@author Steven Silvester <steven.silvester@ieee.org>
@license MIT
"""
import pkgutil
import os
from memory_profiler import memory_usage

import jedi

mods = ['numpy', 'scipy', 'sympy', 'pandas', 'networkx', 'statsmodels',
        'matplotlib.pyplot', 'sklearn', 'skimage', 'mpmath', 'os', 'PIL',
        'OpenGL', 'array', 'audioop', 'binascii', 'cPickle', 'cStringIO',
        'cmath', 'collections', 'datetime', 'errno', 'exceptions', 'gc',
        'imageop', 'imp', 'itertools', 'marshal', 'math', 'mmap', 'msvcrt',
        'nt', 'operator', 'parser', 'rgbimg', 'signal', 'strop', 'sys',
        'thread', 'time', 'wx', 'xxsubtype', 'zipimport', 'zlib', 'nose',
        'PyQt4', 'PySide', 'os.path']


def get_submodules(mod):
    """Get all submodules of a given module"""
    def catch_exceptions(module):
        pass
    try:
        m = __import__(mod)
        submodules = [mod]
        submods = pkgutil.walk_packages(m.__path__, m.__name__ + '.',
                                        catch_exceptions)
        for sm in submods:
            sm_name = sm[1]
            submodules.append(sm_name)
    except ImportError:
        return []
    except:
        return [mod]

    return submodules

submodules = []
for m in mods:
    submods = get_submodules(m)
    submodules += submods

index = 0
submodules = get_submodules(mods[index])


def parse_submodule(mod):
    code = 'import %s' % mod
    script = jedi.Script(code, 1, len(code))
    script.goto_definitions()

print 'MB  | Import'
print '----------------------'
baseline = memory_usage((parse_submodule, ('re',)))[0]

for mod in submodules:
    new = memory_usage((parse_submodule, (mod,)))[0]
    if new - baseline > 10:
        print '%3d | %s' % (new - baseline, mod)
    baseline = new
print '----------------------'
print 'Total MB: %s' % baseline
print 'Total files: %s' % len(submodules)