Last active
June 21, 2018 17:44
-
-
Save piec/ae22efa87d17a7ae0e8a4ca1873827d0 to your computer and use it in GitHub Desktop.
Dirty patch for pycharm stub generator
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# encoding: utf-8 | |
import atexit | |
import zipfile | |
from pycharm_generator_utils.clr_tools import * | |
from pycharm_generator_utils.module_redeclarator import * | |
from pycharm_generator_utils.util_methods import * | |
# TODO: Move all CLR-specific functions to clr_tools | |
debug_mode = False | |
def redo_module(module_name, outfile, module_file_name, doing_builtins): | |
# gobject does 'del _gobject' in its __init__.py, so the chained attribute lookup code | |
# fails to find 'gobject._gobject'. thus we need to pull the module directly out of | |
# sys.modules | |
mod = sys.modules.get(module_name) | |
mod_path = module_name.split('.') | |
if not mod and sys.platform == 'cli': | |
# "import System.Collections" in IronPython 2.7 doesn't actually put System.Collections in sys.modules | |
# instead, sys.modules['System'] get set to a Microsoft.Scripting.Actions.NamespaceTracker and Collections can be | |
# accessed as its attribute | |
mod = sys.modules[mod_path[0]] | |
for component in mod_path[1:]: | |
try: | |
mod = getattr(mod, component) | |
except AttributeError: | |
mod = None | |
report("Failed to find CLR module " + module_name) | |
break | |
if mod: | |
action("restoring") | |
r = ModuleRedeclarator(mod, outfile, module_file_name, doing_builtins=doing_builtins) | |
r.redo(module_name, ".".join(mod_path[:-1]) in MODULES_INSPECT_DIR) | |
action("flushing") | |
r.flush() | |
else: | |
report("Failed to find imported module in sys.modules " + module_name) | |
# find_binaries functionality | |
def cut_binary_lib_suffix(path, f): | |
""" | |
@param path where f lives | |
@param f file name of a possible binary lib file (no path) | |
@return f without a binary suffix (that is, an importable name) if path+f is indeed a binary lib, or None. | |
Note: if for .pyc or .pyo file a .py is found, None is returned. | |
""" | |
if not f.endswith(".pyc") and not f.endswith(".typelib") and not f.endswith(".pyo") and not f.endswith(".so") and not f.endswith( | |
".pyd"): | |
return None | |
ret = None | |
match = BIN_MODULE_FNAME_PAT.match(f) | |
if match: | |
ret = match.group(1) | |
modlen = len('module') | |
retlen = len(ret) | |
if ret.endswith('module') and retlen > modlen and f.endswith('.so'): # what for? | |
ret = ret[:(retlen - modlen)] | |
if f.endswith('.pyc') or f.endswith('.pyo'): | |
fullname = os.path.join(path, f[:-1]) # check for __pycache__ is made outside | |
if os.path.exists(fullname): | |
ret = None | |
pat_match = TYPELIB_MODULE_FNAME_PAT.match(f) | |
if pat_match: | |
ret = "gi.repository." + pat_match.group(1) | |
return ret | |
def is_posix_skipped_module(path, f): | |
if os.name == 'posix': | |
name = os.path.join(path, f) | |
for mod in POSIX_SKIP_MODULES: | |
if name.endswith(mod): | |
return True | |
return False | |
def is_mac_skipped_module(path, f): | |
fullname = os.path.join(path, f) | |
m = MAC_STDLIB_PATTERN.match(fullname) | |
if not m: return 0 | |
relpath = m.group(2) | |
for module in MAC_SKIP_MODULES: | |
if relpath.startswith(module): return 1 | |
return 0 | |
def is_skipped_module(path, f): | |
return is_mac_skipped_module(path, f) or is_posix_skipped_module(path, f[:f.rindex('.')]) or 'pynestkernel' in f | |
def is_module(d, root): | |
return (os.path.exists(os.path.join(root, d, "__init__.py")) or | |
os.path.exists(os.path.join(root, d, "__init__.pyc")) or | |
os.path.exists(os.path.join(root, d, "__init__.pyo")) or | |
is_valid_implicit_namespace_package_name(d)) | |
def walk_python_path(path): | |
for root, dirs, files in os.walk(path): | |
if root.endswith('__pycache__'): | |
continue | |
dirs_copy = list(dirs) | |
for d in dirs_copy: | |
if d.endswith('__pycache__') or not is_module(d, root): | |
dirs.remove(d) | |
# some files show up but are actually non-existent symlinks | |
yield root, [f for f in files if os.path.exists(os.path.join(root, f))] | |
def list_binaries(paths): | |
""" | |
Finds binaries in the given list of paths. | |
Understands nested paths, as sys.paths have it (both "a/b" and "a/b/c"). | |
Tries to be case-insensitive, but case-preserving. | |
@param paths: list of paths. | |
@return: dict[module_name, full_path] | |
""" | |
SEP = os.path.sep | |
res = {} # {name.upper(): (name, full_path)} # b/c windows is case-oblivious | |
if not paths: | |
return {} | |
if IS_JAVA: # jython can't have binary modules | |
return {} | |
paths = sorted_no_case(paths) | |
for path in paths: | |
if path == os.path.dirname(sys.argv[0]): continue | |
for root, files in walk_python_path(path): | |
cutpoint = path.rfind(SEP) | |
if cutpoint > 0: | |
preprefix = path[(cutpoint + len(SEP)):] + '.' | |
else: | |
preprefix = '' | |
prefix = root[(len(path) + len(SEP)):].replace(SEP, '.') | |
if prefix: | |
prefix += '.' | |
note("root: %s path: %s prefix: %s preprefix: %s", root, path, prefix, preprefix) | |
for f in files: | |
name = cut_binary_lib_suffix(root, f) | |
if name and not is_skipped_module(root, f): | |
note("cutout: %s", name) | |
if preprefix: | |
note("prefixes: %s %s", prefix, preprefix) | |
pre_name = (preprefix + prefix + name).upper() | |
if pre_name in res: | |
res.pop(pre_name) # there might be a dupe, if paths got both a/b and a/b/c | |
note("done with %s", name) | |
the_name = prefix + name | |
file_path = os.path.join(root, f) | |
res[the_name.upper()] = (the_name, file_path, os.path.getsize(file_path), int(os.stat(file_path).st_mtime)) | |
return list(res.values()) | |
def list_sources(paths): | |
# noinspection PyBroadException | |
try: | |
for path in paths: | |
if path == os.path.dirname(sys.argv[0]): continue | |
path = os.path.normpath(path) | |
if path.endswith('.egg') and os.path.isfile(path): | |
say("%s\t%s\t%d", path, path, os.path.getsize(path)) | |
for root, files in walk_python_path(path): | |
for name in files: | |
if name.endswith('.py') or name.endswith('-nspkg.pth'): | |
file_path = os.path.join(root, name) | |
say("%s\t%s\t%d", os.path.normpath(file_path), path, os.path.getsize(file_path)) | |
say('END') | |
sys.stdout.flush() | |
except: | |
import traceback | |
traceback.print_exc() | |
sys.exit(1) | |
# noinspection PyBroadException | |
def zip_sources(zip_path): | |
if not os.path.exists(zip_path): | |
os.makedirs(zip_path) | |
zip_filename = os.path.normpath(os.path.sep.join([zip_path, "skeletons.zip"])) | |
try: | |
zip = zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) | |
except: | |
zip = zipfile.ZipFile(zip_filename, 'w') | |
try: | |
try: | |
while True: | |
line = sys.stdin.readline() | |
line = line.strip() | |
if line == '-': | |
break | |
if line: | |
# This line will break the split: | |
# /.../dist-packages/setuptools/script template (dev).py setuptools/script template (dev).py | |
split_items = line.split() | |
if len(split_items) > 2: | |
match_two_files = re.match(r'^(.+\.py)\s+(.+\.py)$', line) | |
if not match_two_files: | |
report("Error(zip_sources): invalid line '%s'" % line) | |
continue | |
split_items = match_two_files.group(1, 2) | |
(path, arcpath) = split_items | |
zip.write(path, arcpath) | |
else: | |
# busy waiting for input from PyCharm... | |
time.sleep(0.10) | |
say('OK: ' + zip_filename) | |
sys.stdout.flush() | |
except: | |
import traceback | |
traceback.print_exc() | |
say('Error creating archive.') | |
sys.exit(1) | |
finally: | |
zip.close() | |
def add_to_zip(zip, paths): | |
# noinspection PyBroadException | |
try: | |
for path in paths: | |
print("Walking root %s" % path) | |
if path == os.path.dirname(sys.argv[0]): continue | |
path = os.path.normpath(path) | |
if path.endswith('.egg') and os.path.isfile(path): | |
pass # TODO: handle eggs | |
for root, files in walk_python_path(path): | |
for name in files: | |
if name.endswith('.py') or name.endswith('-nspkg.pth'): | |
file_path = os.path.join(root, name) | |
arcpath = os.path.relpath(file_path, path) | |
zip.write(file_path, os.path.join(str(hash(path)), arcpath)) | |
except: | |
import traceback | |
traceback.print_exc() | |
sys.exit(1) | |
def zip_stdlib(zip_path): | |
if not os.path.exists(zip_path): | |
os.makedirs(zip_path) | |
import platform | |
zip_filename = os.path.normpath(os.path.sep.join([zip_path, "%s-%s-stdlib-%s.zip" % ( | |
'Anaconda' if sys.version.find('Anaconda') != -1 else 'Python', | |
'.'.join(map(str, sys.version_info)), | |
platform.platform())])) | |
print("Adding file to %s" % zip_filename) | |
try: | |
zip = zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) | |
except: | |
zip = zipfile.ZipFile(zip_filename, 'w') | |
try: | |
add_to_zip(zip, sys.path) | |
finally: | |
zip.close() | |
# command-line interface | |
# noinspection PyBroadException | |
def process_one(name, mod_file_name, doing_builtins, subdir): | |
""" | |
Processes a single module named name defined in file_name (autodetect if not given). | |
Returns True on success. | |
""" | |
if has_regular_python_ext(name): | |
report("Ignored a regular Python file %r", name) | |
return True | |
if not quiet: | |
say(name) | |
sys.stdout.flush() | |
action("doing nothing") | |
try: | |
fname = build_output_name(subdir, name) | |
action("opening %r", fname) | |
old_modules = list(sys.modules.keys()) | |
imported_module_names = [] | |
class MyFinder: | |
# noinspection PyMethodMayBeStatic | |
def find_module(self, fullname, path=None): | |
if fullname != name: | |
imported_module_names.append(fullname) | |
return None | |
my_finder = None | |
if hasattr(sys, 'meta_path'): | |
my_finder = MyFinder() | |
sys.meta_path.append(my_finder) | |
else: | |
imported_module_names = None | |
action("importing") | |
__import__(name) # sys.modules will fill up with what we want | |
if my_finder: | |
sys.meta_path.remove(my_finder) | |
if imported_module_names is None: | |
imported_module_names = [m for m in sys.modules.keys() if m not in old_modules] | |
redo_module(name, fname, mod_file_name, doing_builtins) | |
# The C library may have called Py_InitModule() multiple times to define several modules (gtk._gtk and gtk.gdk); | |
# restore all of them | |
path = name.split(".") | |
redo_imports = not ".".join(path[:-1]) in MODULES_INSPECT_DIR | |
if imported_module_names and redo_imports: | |
for m in sys.modules.keys(): | |
if m.startswith("pycharm_generator_utils"): continue | |
action("looking at possible submodule %r", m) | |
# if module has __file__ defined, it has Python source code and doesn't need a skeleton | |
if m not in old_modules and m not in imported_module_names and m != name and not hasattr( | |
sys.modules[m], '__file__'): | |
if not quiet: | |
say(m) | |
sys.stdout.flush() | |
fname = build_output_name(subdir, m) | |
action("opening %r", fname) | |
try: | |
redo_module(m, fname, mod_file_name, doing_builtins) | |
finally: | |
action("closing %r", fname) | |
except: | |
exctype, value = sys.exc_info()[:2] | |
msg = "Failed to process %r while %s: %s" | |
args = name, CURRENT_ACTION, str(value) | |
report(msg, *args) | |
if debug_mode: | |
if sys.platform == 'cli': | |
import traceback | |
traceback.print_exc(file=sys.stderr) | |
raise | |
return False | |
return True | |
def get_help_text(): | |
return ( | |
# 01234567890123456789012345678901234567890123456789012345678901234567890123456789 | |
'Generates interface skeletons for python modules.' '\n' | |
'Usage: ' '\n' | |
' generator [options] [module_name [file_name]]' '\n' | |
' generator [options] -L ' '\n' | |
'module_name is fully qualified, and file_name is where the module is defined.' '\n' | |
'E.g. foo.bar /usr/lib/python/foo_bar.so' '\n' | |
'For built-in modules file_name is not provided.' '\n' | |
'Output files will be named as modules plus ".py" suffix.' '\n' | |
'Normally every name processed will be printed and stdout flushed.' '\n' | |
'directory_list is one string separated by OS-specific path separtors.' '\n' | |
'\n' | |
'Options are:' '\n' | |
' -h -- prints this help message.' '\n' | |
' -d dir -- output dir, must be writable. If not given, current dir is used.' '\n' | |
' -b -- use names from sys.builtin_module_names' '\n' | |
' -q -- quiet, do not print anything on stdout. Errors still go to stderr.' '\n' | |
' -x -- die on exceptions with a stacktrace; only for debugging.' '\n' | |
' -v -- be verbose, print lots of debug output to stderr' '\n' | |
' -c modules -- import CLR assemblies with specified names' '\n' | |
' -p -- run CLR profiler ' '\n' | |
' -s path_list -- add paths to sys.path before run; path_list lists directories' '\n' | |
' separated by path separator char, e.g. "c:\\foo;d:\\bar;c:\\with space"' '\n' | |
' -L -- print version and then a list of binary module files found ' '\n' | |
' on sys.path and in directories in directory_list;' '\n' | |
' lines are "qualified.module.name /full/path/to/module_file.{pyd,dll,so}"' '\n' | |
' -i -- read module_name, file_name and list of imported CLR assemblies from stdin line-by-line' '\n' | |
' -S -- lists all python sources found in sys.path and in directories in directory_list\n' | |
' -z archive_name -- zip files to archive_name. Accepts files to be archived from stdin in format <filepath> <name in archive>' | |
) | |
if __name__ == "__main__": | |
from getopt import getopt | |
f = open('/tmp/pierre.txt', 'a') | |
print(sys.argv, file=f) | |
helptext = get_help_text() | |
opts, args = getopt(sys.argv[1:], "d:hbqxvc:ps:LiSzu") | |
opts = dict(opts) | |
quiet = '-q' in opts | |
_is_verbose = '-v' in opts | |
subdir = opts.get('-d', '') | |
if not opts or '-h' in opts: | |
say(helptext) | |
sys.exit(0) | |
if '-L' not in opts and '-b' not in opts and '-S' not in opts and '-i' not in opts and '-u' not in opts and not args: | |
report("Neither -L nor -b nor -S nor any module name given") | |
sys.exit(1) | |
if "-x" in opts: | |
debug_mode = True | |
# patch sys.path? | |
extra_path = opts.get('-s', None) | |
if extra_path: | |
source_dirs = extra_path.split(os.path.pathsep) | |
for p in source_dirs: | |
if p and p not in sys.path: | |
sys.path.append(p) # we need this to make things in additional dirs importable | |
note("Altered sys.path: %r", sys.path) | |
sys.path.append('/usr/lib/girepository-1.0') | |
print("sys.path", sys.path, file=f) | |
# find binaries? | |
if "-L" in opts: | |
if len(args) > 0: | |
report("Expected no args with -L, got %d args", len(args)) | |
sys.exit(1) | |
say(VERSION) | |
results = list(list_binaries(sys.path)) | |
results.sort() | |
for name, path, size, last_modified in results: | |
say("%s\t%s\t%d\t%d", name, path, size, last_modified) | |
sys.exit(0) | |
if "-S" in opts: | |
if len(args) > 0: | |
report("Expected no args with -S, got %d args", len(args)) | |
sys.exit(1) | |
say(VERSION) | |
list_sources(sys.path) | |
sys.exit(0) | |
if "-z" in opts: | |
if len(args) != 1: | |
report("Expected 1 arg with -z, got %d args", len(args)) | |
sys.exit(1) | |
zip_sources(args[0]) | |
sys.exit(0) | |
if "-u" in opts: | |
if len(args) != 1: | |
report("Expected 1 arg with -u, got %d args", len(args)) | |
sys.exit(1) | |
zip_stdlib(args[0]) | |
sys.exit(0) | |
# build skeleton(s) | |
timer = Timer() | |
# determine names | |
if '-b' in opts: | |
if args: | |
report("No names should be specified with -b") | |
sys.exit(1) | |
names = list(sys.builtin_module_names) | |
if not BUILTIN_MOD_NAME in names: | |
names.append(BUILTIN_MOD_NAME) | |
if '__main__' in names: | |
names.remove('__main__') # we don't want ourselves processed | |
ok = True | |
for name in names: | |
ok = process_one(name, None, True, subdir) and ok | |
if not ok: | |
sys.exit(1) | |
else: | |
if '-i' in opts: | |
if args: | |
report("No names should be specified with -i") | |
sys.exit(1) | |
name = sys.stdin.readline().strip() | |
mod_file_name = sys.stdin.readline().strip() | |
if not mod_file_name: | |
mod_file_name = None | |
refs = sys.stdin.readline().strip() | |
else: | |
if len(args) > 2: | |
report("Only module_name or module_name and file_name should be specified; got %d args", len(args)) | |
sys.exit(1) | |
name = args[0] | |
if len(args) == 2: | |
mod_file_name = args[1] | |
else: | |
mod_file_name = None | |
refs = opts.get('-c', '') | |
if sys.platform == 'cli': | |
# noinspection PyUnresolvedReferences | |
import clr | |
if refs: | |
for ref in refs.split(';'): clr.AddReferenceByPartialName(ref) | |
if '-p' in opts: | |
atexit.register(print_profile) | |
# We take module name from import statement | |
name = get_namespace_by_name(name) | |
if not process_one(name, mod_file_name, False, subdir): | |
sys.exit(1) | |
say("Generation completed in %d ms", timer.elapsed()) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment