List imported modules of Python projects

lsimports.py

#! /usr/bin/env python3
#  lsimports.py - List imported modules of Python projects.
#
#  Copyright (C) 2020 Richard Neumann <mail at richard dash neumann period de>
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program.  If not, see <https://www.gnu.org/licenses/>.
"""Lists imports of python modules and packages."""

from argparse import ArgumentParser, Namespace
from ast import Import, ImportFrom, Module, parse, walk
from itertools import chain
from logging import INFO, WARNING, basicConfig, getLogger
from pathlib import Path
from typing import Iterable, Iterator


DESCRIPTION = __doc__
LOG_FORMAT = '[%(levelname)s] %(name)s: %(message)s'
LOGGER = getLogger(Path(__file__).stem)
# Standard library module list for python 3.9 taken from:
#   https://github.com/jackmaney/python-stdlib-list
# Credits to Jack Maney.
STDLIB39 = {
    '__future__', '__main__', '__phello__', '_aix_support', '_bootlocale',
    '_bootsubprocess', '_collections_abc', '_compat_pickle', '_compression',
    '_markupbase', '_osx_support', '_py_abc', '_pydecimal', '_pyio',
    '_sitebuiltins', '_strptime', '_sysconfigdata_x86_64_conda_cos6_linux_gnu',
    '_sysconfigdata_x86_64_conda_linux_gnu', '_thread', '_threading_local',
    '_weakrefset', 'abc', 'aifc', 'antigravity', 'argparse', 'array', 'ast',
    'asynchat', 'asyncio', 'asyncore', 'atexit', 'audioop', 'base64', 'bdb',
    'binascii', 'binhex', 'bisect', 'builtins', 'bz2', 'cProfile', 'calendar',
    'cgi', 'cgitb', 'chunk', 'cmath', 'cmd', 'code', 'codecs', 'codeop',
    'collections', 'colorsys', 'compileall', 'concurrent', 'configparser',
    'contextlib', 'contextvars', 'copy', 'copyreg', 'crypt', 'csv', 'ctypes',
    'curses', 'dataclasses', 'datetime', 'dbm', 'decimal', 'difflib', 'dis',
    'distutils', 'doctest', 'email', 'encodings', 'ensurepip', 'enum', 'errno',
    'faulthandler', 'fcntl', 'filecmp', 'fileinput', 'fnmatch', 'formatter',
    'fractions', 'ftplib', 'functools', 'gc', 'genericpath', 'getopt',
    'getpass', 'gettext', 'glob', 'graphlib', 'grp', 'gzip', 'hashlib',
    'heapq', 'hmac', 'html', 'http', 'idlelib', 'imaplib', 'imghdr', 'imp',
    'importlib', 'inspect', 'io', 'ipaddress', 'itertools', 'json', 'keyword',
    'lib', 'lib2to3', 'linecache', 'locale', 'logging', 'lzma', 'mailbox',
    'mailcap', 'marshal', 'math', 'mimetypes', 'mmap', 'modulefinder',
    'msilib', 'msvcrt', 'multiprocessing', 'netrc', 'nis', 'nntplib',
    'ntpath', 'nturl2path', 'numbers', 'opcode', 'operator', 'optparse', 'os',
    'ossaudiodev', 'parser', 'pathlib', 'pdb', 'pickle', 'pickletools',
    'pipes', 'pkgutil', 'platform', 'plistlib', 'poplib', 'posix', 'posixpath',
    'pprint', 'profile', 'pstats', 'pty', 'pwd', 'py_compile', 'pyclbr',
    'pydoc', 'pydoc_data', 'queue', 'quopri', 'random', 're', 'readline',
    'reprlib', 'resource', 'rlcompleter', 'runpy', 'sched', 'secrets',
    'select', 'selectors', 'shelve', 'shlex', 'shutil', 'signal', 'site',
    'smtpd', 'smtplib', 'sndhdr', 'socket', 'socketserver', 'spwd', 'sqlite3',
    'sre_compile', 'sre_constants', 'sre_parse', 'ssl', 'stat', 'statistics',
    'string', 'stringprep', 'struct', 'subprocess', 'sunau', 'symbol',
    'symtable', 'sys', 'sysconfig', 'syslog', 'tabnanny', 'tarfile',
    'telnetlib', 'tempfile', 'termios', 'test', 'textwrap', 'this',
    'threading', 'time', 'timeit', 'tkinter', 'token', 'tokenize', 'trace',
    'traceback', 'tracemalloc', 'tty', 'turtle', 'turtledemo', 'types',
    'typing', 'unicodedata', 'unittest', 'urllib', 'uu', 'uuid', 'venv',
    'warnings', 'wave', 'weakref', 'webbrowser', 'winreg', 'winsound',
    'wsgiref', 'xdrlib', 'xml', 'xmlrpc', 'zipapp', 'zipfile', 'zipimport',
    'zlib', 'zoneinfo'
}


def get_imports_from_module(module: Module) -> Iterator[str]:
    """Lists imports from a module AST."""

    for node in walk(module):
        if isinstance(node, Import):
            for name in node.names:
                yield name.name
        elif isinstance(node, ImportFrom):
            if node.level == 0:
                yield node.module


def get_imports_from_file(path: Path) -> Iterator[str]:
    """Lists imports of the given file."""

    with path.open('r') as file:
        source_code = file.read()

    try:
        module = parse(source_code)
    except SyntaxError:
        LOGGER.warning('Cannot parse "%s" due to syntax error.', path)
        return

    yield from get_imports_from_module(module)


def get_imports_from_files(paths: Iterable[Path]) -> Iterator[str]:
    """Returns the imports from multiple files."""

    for path in paths:
        LOGGER.info('Checking: %s', path)
        yield from get_imports_from_file(path)


def get_module_root(module: str) -> str:
    """Returns the root of the given module path."""

    return module.split('.')[0]


def is_module(path: Path) -> bool:
    """Determines whether the given file is a python module."""

    return path.is_file() and path.suffix == '.py'


def iterfiles(path: Path) -> Iterator[Path]:
    """Recursively yields files in a directory."""

    if not path.exists():
        LOGGER.error('No such file: %s', path)
        return

    if path.is_dir():
        for subdir in path.iterdir():
            yield from iterfiles(subdir)
    elif path.is_file():
        yield path


def get_args() -> Namespace:
    """Parses and returns the command line arguments."""

    parser = ArgumentParser(description=DESCRIPTION)
    parser.add_argument('path', nargs='*', type=Path, default=[Path.cwd()],
                        help='the files and folders to scan for imports')
    parser.add_argument('-E', '--exclude-stdlib', action='store_true',
                        help='exclude imports from the standard library')
    parser.add_argument('-e', '--exclude-modules', nargs='+', default=(),
                        metavar='module', help='exclude the specified modules')
    parser.add_argument('-v', '--verbose', action='store_true',
                        help='print verbose messages')
    return parser.parse_args()


def main():
    """Runs the script."""

    args = get_args()
    basicConfig(format=LOG_FORMAT, level=INFO if args.verbose else WARNING)
    files = filter(is_module, chain(*map(iterfiles, args.path)))
    imports = set(map(get_module_root, get_imports_from_files(files)))
    imports -= set(args.exclude_modules)

    if args.exclude_stdlib:
        imports -= STDLIB39

    for module in imports:
        print(module)


if __name__ == '__main__':
    main()