convert python "*" imports to named imports

rename_imports.py

#!/usr/bin/env python

import ast
import sys
import os
from collections import defaultdict
import importlib
import inspect


def check_import(filename, module, pw_packages, pw_modules, verbose=False):
    # first check for 'module' in pw_packages
    # if they are plotwatt. or plotwattdark.

    # if module.startswith('plotwatt') or module.startswith('tools.') or module=='tools':
    #    if verbose: print module, filename, "plotwatt module can be imported"
    #    return True

    mdl = None
    try:
        package = os.path.dirname(filename)
        sys.path.append(package)
        module_found = False
        module_base = module
        if '.' in module:
            module_base = module.split('.')[-1]

#        while package in pw_packages:
#            module_found = "%s.py"%module_base in pw_packages[package]
#            if module_found: break
#            # check if this is a relative packge import
#            subpackage = "%s/%s" %(package, module_base)
#            module_found = "__init__.py" in pw_packages[subpackage]
#            if module_found: break
#            package  = os.path.dirname(package)
#
#        # module_found contains if module was found
#
#        if module_found:
#            if verbose: print module, filename, "found in", package
#            return True, mdl
#
#        # search for module in *any* pw_packages
#        if "%s.py"%module_base in pw_modules:
#            if verbose: print module, filename, "found", pw_modules["%s.py"%module]
#            return True, mdl

        # now try import ...
        # this is surely not a plotwatt module
        module_found = False
        try:
            if verbose:
                print module, filename, "via import",
            #mdl=__import__(module, globals(), locals(), [], -1)
            mdl = importlib.import_module(module)
            if verbose:
                print "found", mdl
            module_found = True
        except ImportError, e:
            print module, filename, "could not be loaded, relative?", str(e)

        # could not be found
        return module_found, mdl
    finally:
        sys.path.pop()

def find_imports(file_list, verbose=False, init_file=None):
    not_found = defaultdict(list)
    modules = set()
    pw_packages = defaultdict(list)
    pw_modules = defaultdict(list)
    init_modules = defaultdict(dict)
    used_names = defaultdict(list)
    for _fl in file_list:
        fl = os.path.abspath(_fl)
        pw_packages[os.path.dirname(fl)].append(os.path.basename(fl))
        pw_modules[os.path.basename(fl)].append(os.path.dirname(fl))

    for _fl in file_list:
        init_module = os.path.basename(_fl).startswith('__init__.py')
        fl = os.path.abspath(_fl)
        if 'site-packages' in fl or 'deprecated' in fl:
            continue
        if not os.path.exists(fl):
            continue
        src = open(fl).read()
        try:
            tree = ast.parse(src, fl)
            for stmt in ast.walk(tree):
                stmttype = type(stmt)
                if stmttype == ast.Import:
                    for name in stmt.names:
                        if verbose:
                            print fl, "ast.Import", stmt.lineno, name.name
                        found, mdl = check_import(fl, name.name, pw_packages, pw_modules, verbose)
                        print mdl
                        if found is True:
                            modules.add(name.name)
                        else:
                            not_found[name.name].append("%s:%s" % (fl,
                                                                   stmt.lineno))
                elif stmttype == ast.ImportFrom:
                    names = [(a.name, a.asname) for a in stmt.names]
                    if verbose:
                        print fl, "ast.ImportFrom", stmt.lineno, stmt.module, names
                    if stmt.module == 'plotwatt.util':
                        print fl, "ast.ImportFrom", stmt.lineno, stmt.module, names
                    found, mdl = check_import(fl, stmt.module, pw_packages, pw_modules, verbose)
                    print mdl
                    if found is True:
                        modules.add(stmt.module)
                        if init_module is True:
                            if mdl is not None:
                                mod_name = mdl.__file__[len(sorted([p for p in sys.path if mdl.__file__.startswith(
                                    p)])[-1]) + 1:].replace(".pyc", "").replace(".py", "").replace("/", ".")
                            else:
                                mod_name = stmt.module
                            if mod_name.endswith("__init__"):
                                dname = mod_name.split(".")
                                dname[-1] = dname[-2]
                                mod_name = ".".join(dname)
                            pkg_name = fl[len(sorted([p for p in sys.path if fl.startswith(p)])[-1]) +
                                          1:].replace(".pyc", "").replace(".py", "").replace("/", ".").replace(".__init__", "")
                            init_modules[pkg_name].update({nn: mod_name for nn in names})
                        else:
                            used_names[fl].append((stmt.module, stmt.lineno, names, mdl))
                    else:
                        not_found[stmt.module].append("%s:%s" % (fl,
                                                                 stmt.lineno))
                elif stmttype == ast.Call:
                    if hasattr(stmt.func, 'id'):
                        pass
                        # print stmt.func.id, stmt.func.lineno
        except Exception, e:
            print "***", e
    return modules, not_found, init_modules, used_names


import errno


def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:  # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise


def main():
    usage = """
        %prog [options] list_of_files
        examples
        cd ~/plotwatt; ~/plotwatt/runpy ~/plotwatt/find_imports.py $(find . -name '*.py')
        cd ~/plotwattdark; ~/plotwatt/runpy ~/plotwatt/find_imports.py $(find . -name '*.py')
    """
    from optparse import OptionParser
    parser = OptionParser(usage=usage)
    parser.add_option("--verbose", default=False,
                      action="store_true")

    (options, args) = parser.parse_args()

    if len(args) == 0:
        parser.print_help()
        exit(1)

    modules, not_found, init_modules, used_names = find_imports(args, options.verbose)

    print sorted(modules)

    print " "
    print "**** Modules that were not found ****"
    print " "
    for module, lst in not_found.items():
        print module, lst

    print 80 * "*"
    import linecache
    for used_name, pkglist in used_names.items():
        ln = []
        _lineno = -1
        edfl_name = "/tmp/ed/{}".format(used_name).replace(".py", ".ed")
        mkdir_p(os.path.dirname(edfl_name))
        edfl = None
        for (pkg, lineno, syms, mdl) in sorted(pkglist, key=lambda x: x[1], reverse=True):
            if not inspect.ismodule(mdl):
                continue
             # only process plotwatt modules
            if not inspect.getfile(mdl).startswith("/home/ubuntu/plotwatt"):
                continue
            for (sym, assym) in syms:
                modname = mdl.__name__
                if (sym in mdl.__dict__ and hasattr(mdl.__dict__[sym], "__module__")
                    and inspect.getfile(inspect.getmodule(mdl.__dict__[sym])).
                    startswith("/home/ubuntu/plotwatt")
                    ):
                    modname = mdl.__dict__[sym].__module__
                if _lineno != lineno:
                    # next line, print this one
                    if len(ln) > 0:
                        edfl = edfl or open(edfl_name, "wt")
                        print >>edfl, "{}c".format(_lineno)
                        for lnn in ln:
                            print >>edfl, lnn
                        print >>edfl, "."
                    _lineno = lineno
                    lnx = linecache.getline(used_name, _lineno)
                    indent = len(lnx) - len(lnx.lstrip())
                    ln = []
                sedln = "{}from {} import {}".format(lnx[:indent], modname, sym)
                if assym:
                    sedln += " as {}".format(assym)

                ln.append(sedln)
                print used_name, lineno, sym, modname

        if len(ln) > 0:
            edfl = edfl or open(edfl_name, "wt")
            print >>edfl, "{}c".format(_lineno)
            for lnn in ln:
                print >>edfl, lnn
            print >>edfl, "."

        if edfl is not None:
            print >>edfl, "w"
            edfl.close()
            # run ed command on the file
            os.system("/bin/ed {} < {}".format(used_name, edfl_name))

    print "ok"


#
# cd ~/plotwatt; ~/plotwatt/runpy ~/plotwatt/find_imports.py $(find . -name '*.py')py')
#
if __name__ == "__main__":
    main()