Skip to content

Instantly share code, notes, and snippets.

@jkbjh
Last active August 18, 2020 08:15
Show Gist options
  • Save jkbjh/73d7ea63bf78a3d77d9cf18779f31620 to your computer and use it in GitHub Desktop.
Save jkbjh/73d7ea63bf78a3d77d9cf18779f31620 to your computer and use it in GitHub Desktop.
Get PIP Requirements for current folder. Hacky.
import importlib
import subprocess
import re
import os
import joblib
from joblib import parallel_backend, Parallel, delayed
found_imports = sorted(list(set([x.lstrip().rstrip() for x in subprocess.check_output("findimports -p -l 1 |grep -v ':' |sort -u", shell=True).decode("utf-8").rstrip().lstrip().split("\n")])))
installed_packages = subprocess.check_output('pip list | tail -n +3 | cut -d" " -f1', shell=True).decode("utf-8").rstrip().lstrip()
def get_package_files(package):
output = subprocess.check_output('pip show -f {package}'.format(package=package), shell=True).decode("utf-8").strip()
pip_search = re.match(r'Name:\s*(?P<name>[^ \n]+).+Location:\s+(?P<location>[^ \n]+).+Files:\s+(?P<files>.+)', output, re.MULTILINE|re.DOTALL)
pip_dict = pip_search.groupdict()
name = pip_dict["name"]
location = pip_dict["location"].lstrip().rstrip()
files = pip_dict["files"]
if files.startswith("Cannot locate"):
files = subprocess.check_output("find {location} -name '*.py' ".format(location=pip_search.groupdict()["location"]), shell=True).decode("utf-8").lstrip().rstrip()
files = [os.path.join(location, f.rstrip().lstrip()) for f in files.strip().split("\n")]
file2package = {f: name for f in files}
package2files = {name: files}
return file2package, package2files
file2package = {}
package2files = {}
installed_package_files = [delayed(get_package_files)(package) for package in installed_packages.strip().split("\n")]
for f2p, p2f in Parallel(n_jobs=-1, backend="multiprocessing")(installed_package_files):
file2package.update(f2p)
package2files.update(p2f)
#####################################################################################
import_files = [];
for imp in found_imports:
try:
import_files.append(importlib.import_module(imp).__file__)
except Exception as e:
print("error in %s, %s" % (imp, e))
############################################################
not_found_packages = set([f for f in import_files if f not in file2package])
used_packages = set([file2package.get(f, None) for f in import_files])
package_filter = re.compile("(=|^)(%s)(==|$)" % ("|".join([re.escape(x) for x in used_packages if x]),))
frozen = subprocess.check_output("pip freeze", shell=True).decode("utf-8").rstrip().lstrip().split("\n")
filtered_frozen = [f for f in frozen if package_filter.search(f)]
filtered_frozen.sort()
print("\n".join(filtered_frozen))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment