Skip to content

Instantly share code, notes, and snippets.

@benoitbryon
Created January 21, 2015 16:40
Show Gist options
  • Save benoitbryon/b674717c6dc8d3677543 to your computer and use it in GitHub Desktop.
Save benoitbryon/b674717c6dc8d3677543 to your computer and use it in GitHub Desktop.
Draft utility to figure out what are the primary dependencies of a Python package. Let's list modules that are imported in the code (ignore stdlib and internals).
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
"""Helps you maintain ``install_requires``."""
#: List of modules in standard library.
#: See https://docs.python.org/3/py-modindex.html
PY3_MODULES = [
'__future__',
'__main__',
'_dummy_thread',
'_thread',
'abc',
'aifc',
'argparse',
'array',
'ast',
'asynchat',
'asyncio',
'asyncore',
'atexit',
'audioop',
'base64',
'bdb',
'binascii',
'binhex',
'bisect',
'builtins',
'bz2',
'calendar',
'cgi',
'cgitb',
'chunk',
'cmath',
'cmd',
'code',
'codecs',
'codeop',
'collections',
'colorsys',
'compileall',
'concurrent',
'configparser',
'contextlib',
'copy',
'copyreg',
'cProfile',
'crypt',
'csv',
'ctypes',
'curses',
'datetime',
'dbm',
'decimal',
'difflib',
'dis',
'distutils',
'doctest',
'dummy_threading',
'email',
'encodings',
'ensurepip',
'enum',
'errno',
'faulthandler',
'fcntl',
'filecmp',
'fileinput',
'fnmatch',
'formatter',
'fpectl',
'fractions',
'ftplib',
'functools',
'gc',
'getopt',
'getpass',
'gettext',
'glob',
'grp',
'gzip',
'hashlib',
'heapq',
'hmac',
'html',
'http',
'imaplib',
'imghdr',
'imp',
'importlib',
'inspect',
'io',
'ipaddress',
'itertools',
'json',
'keyword',
'lib2to3',
'linecache',
'locale',
'logging',
'lzma',
'macpath',
'mailbox',
'mailcap',
'marshal',
'math',
'mimetypes',
'mmap',
'modulefinder',
'msilib',
'msvcrt',
'multiprocessing',
'netrc',
'nis',
'nntplib',
'numbers',
'operator',
'optparse',
'os',
'ossaudiodev',
'parser',
'pathlib',
'pdb',
'pickle',
'pickletools',
'pipes',
'pkgutil',
'platform',
'plistlib',
'poplib',
'posix',
'pprint',
'profile',
'pstats',
'pty',
'pwd',
'py_compile',
'pyclbr',
'pydoc',
'queue',
'quopri',
'random',
're',
'readline',
'reprlib',
'resource',
'rlcompleter',
'runpy',
'sched',
'select',
'selectors',
'shelve',
'shlex',
'shutil',
'signal',
'site',
'smtpd',
'smtplib',
'sndhdr',
'socket',
'socketserver',
'spwd',
'sqlite3',
'ssl',
'stat',
'statistics',
'string',
'stringprep',
'struct',
'subprocess',
'sunau',
'symbol',
'symtable',
'sys',
'sysconfig',
'syslog',
'tabnanny',
'tarfile',
'telnetlib',
'tempfile',
'termios',
'test',
'textwrap',
'threading',
'time',
'timeit',
'tkinter',
'token',
'tokenize',
'trace',
'traceback',
'tracemalloc',
'tty',
'turtle',
'turtledemo',
'types',
'unicodedata',
'unittest',
'urllib',
'uu',
'uuid',
'venv',
'warnings',
'wave',
'weakref',
'webbrowser',
'winreg',
'winsound',
'wsgiref',
'xdrlib',
'xml',
'xmlrpc',
'zipfile',
'zipimport',
'zlib',
]
#: List of modules in standard library.
#: See https://docs.python.org/2/py-modindex.html
PY2_MODULES = [
'__builtin__',
'__future__',
'__main__',
'_winreg',
'abc',
'aepack',
'aetools',
'aetypes',
'aifc',
'al',
'AL',
'anydbm',
'applesingle',
'argparse',
'array',
'ast',
'asynchat',
'asyncore',
'atexit',
'audioop',
'autoGIL',
'base64',
'BaseHTTPServer',
'Bastion',
'bdb',
'binascii',
'binhex',
'bisect',
'bsddb',
'buildtools',
'bz2',
'calendar',
'Carbon',
'cd',
'cfmfile',
'cgi',
'CGIHTTPServer',
'cgitb',
'chunk',
'cmath',
'cmd',
'code',
'codecs',
'codeop',
'collections',
'ColorPicker',
'colorsys',
'commands',
'compileall',
'compiler',
'ConfigParser',
'contextlib',
'Cookie',
'cookielib',
'copy',
'copy_reg',
'cPickle',
'cProfile',
'crypt',
'cStringIO',
'csv',
'ctypes',
'curses',
'datetime',
'dbhash',
'dbm',
'decimal',
'DEVICE',
'difflib',
'dircache',
'dis',
'distutils',
'dl',
'doctest',
'DocXMLRPCServer',
'dumbdbm',
'dummy_thread',
'dummy_threading',
'EasyDialogs',
'email',
'encodings',
'ensurepip',
'errno',
'exceptions',
'fcntl',
'filecmp',
'fileinput',
'findertools',
'FL',
'fl',
'flp',
'fm',
'fnmatch',
'formatter',
'fpectl',
'fpformat',
'fractions',
'FrameWork',
'ftplib',
'functools',
'future_builtins ',
'gc',
'gdbm',
'gensuitemodule',
'getopt',
'getpass',
'gettext',
'gl',
'GL',
'glob',
'grp',
'gzip',
'hashlib',
'heapq',
'hmac',
'hotshot',
'htmlentitydefs',
'htmllib',
'HTMLParser',
'httplib',
'ic',
'icopen',
'imageop',
'imaplib',
'imgfile',
'imghdr',
'imp',
'importlib',
'imputil',
'inspect',
'io',
'itertools',
'jpeg',
'json',
'keyword',
'lib2to3',
'linecache',
'locale',
'logging',
'macerrors',
'MacOS',
'macostools',
'macpath',
'macresource',
'mailbox',
'mailcap',
'marshal',
'math',
'md5',
'mhlib',
'mimetools',
'mimetypes',
'MimeWriter',
'mimify',
'MiniAEFrame',
'mmap',
'modulefinder',
'msilib',
'msvcrt',
'multifile',
'multiprocessing',
'mutex',
'Nav',
'netrc',
'new',
'nis',
'nntplib',
'numbers',
'operator',
'optparse',
'os',
'ossaudiodev',
'parser',
'pdb',
'pickle',
'pickletools',
'pipes',
'PixMapWrapper',
'pkgutil',
'platform',
'plistlib',
'popen2',
'poplib',
'posix',
'posixfile',
'pprint',
'profile',
'pstats',
'pty',
'pwd',
'py_compile',
'pyclbr',
'pydoc',
'Queue',
'quopri',
'random',
're',
'readline',
'resource',
'rexec',
'rfc822',
'rlcompleter',
'robotparser',
'runpy',
'sched',
'ScrolledText',
'select',
'sets',
'sgmllib',
'sha',
'shelve',
'shlex',
'shutil',
'signal',
'SimpleHTTPServer',
'SimpleXMLRPCServer',
'site',
'smtpd',
'smtplib',
'sndhdr',
'socket',
'SocketServer',
'spwd',
'sqlite3',
'ssl',
'stat',
'statvfs',
'string',
'StringIO',
'stringprep',
'struct',
'subprocess',
'sunau',
'sunaudiodev',
'SUNAUDIODEV',
'symbol',
'symtable',
'sys',
'sysconfig',
'syslog',
'tabnanny',
'tarfile',
'telnetlib',
'tempfile',
'termios',
'test',
'textwrap',
'thread',
'threading',
'time',
'timeit',
'Tix',
'Tkinter',
'token',
'tokenize',
'trace',
'traceback',
'ttk',
'tty',
'turtle',
'types',
'unicodedata',
'unittest',
'urllib',
'urllib2',
'urlparse',
'user',
'UserDict',
'UserList',
'UserString',
'uu',
'uuid',
'videoreader',
'W',
'warnings',
'wave',
'weakref',
'webbrowser',
'whichdb',
'winsound',
'wsgiref',
'xdrlib',
'xml',
'xmlrpclib',
'zipfile',
'zipimport',
'zlib',
]
import argparse
import os
import re
IMPORT_REGEXP = re.compile(r'^[\s]*(from [\w._]+ |)import ([\w._, ]+)')
def is_dependency(module, root_module=None):
"""Return ``True`` if ``module`` implies external dependency."""
if not module:
return False
if module.startswith('.'):
return False
if re.match(r'^{0}(\.\w+)?$'.format(root_module), module):
return False
if module in PY2_MODULES:
return False
return True
def root_module(module):
"""Return only root package in module name."""
module = module.split(' ')[0]
parts = module.split('.')
return parts[0]
def modules_in_import_line(line):
"""Yield modules in import statement."""
line = line.lstrip()
if line.startswith('import '):
line = line[len('import '):]
matches = re.findall(r'([\w._]+( as [\w_]+)?)', line)
for match in matches:
yield root_module(match[0])
elif line.startswith('from '):
line = line[len('from '):]
match = re.match(r'([\w._]+)', line)
if match:
yield root_module(match.group(0))
def imports_in_file(lines):
"""Yield import statements in ``lines``."""
for line in lines:
match = re.match(IMPORT_REGEXP, line)
if match:
for module in modules_in_import_line(match.group(0)):
yield module
def python_files(path):
"""Yield path of Python scripts in path."""
for (dirpath, dirnames, filenames) in os.walk(path):
for filename in filenames:
if filename.endswith('.py'):
yield dirpath + "/" + filename
def main(args=None):
parser = argparse.ArgumentParser(
description='Show source code dependencies.')
parser.add_argument(
'source',
type=str,
help='an integer for the accumulator')
args = parser.parse_args()
root_module = os.path.basename(args.source)
root_path = args.source
modules = set()
for filename in python_files(root_path):
with open(filename) as lines:
for module in imports_in_file(lines):
modules.add(module)
for module in sorted(modules):
if is_dependency(module, root_module):
print module
if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment