Skip to content

Instantly share code, notes, and snippets.

@Flamefire
Forked from boegel/deptree2exts_list.py
Last active September 12, 2023 11:07
Show Gist options
  • Save Flamefire/49426e502cd8983757bd01a08a10ae0d to your computer and use it in GitHub Desktop.
Save Flamefire/49426e502cd8983757bd01a08a10ae0d to your computer and use it in GitHub Desktop.
Find order of dependencies of a python package, put both into a PATH accessible folder (e.g. ~/.local/bin) and make executable
#!/usr/bin/env python
import json
import sys
import pkg_resources
from pprint import pprint
if len(sys.argv) != 3:
sys.stderr.write("ERROR: Usage: %s <path to JSON file with output of `pipdeptree -j <pkg>`> <pkg name>\n" % sys.argv[0])
sys.exit(1)
json_file, target_pkg = sys.argv[1:]
with open(json_file) as fp:
deptree = json.loads(fp.read())
def find_deps(pkgs):
res = []
for pkg in pkgs:
matching_entries = [entry for entry in deptree if pkg in (entry['package']['package_name'], entry['package']['key'])]
if not matching_entries:
raise RuntimeError("Found no installed package for '%s'" % pkg)
if len(matching_entries) > 1:
raise RuntimeError("Found multiple installed packages for '%s'" % pkg)
entry = matching_entries[0]
res.append((entry['package']['package_name'], entry['package']['installed_version']))
deps = (dep['package_name'] for dep in entry['dependencies'])
res.extend(find_deps(deps))
return res
deps = find_deps([target_pkg])
installed_modules = {mod.project_name for mod in pkg_resources.working_set}
print("Installed modules: %s" % installed_modules)
# iterate over deps in reverse order, get rid of duplicates along the way
# also filter out Python packages that are already installed in current environment
res = []
handled = set()
for dep in reversed(deps):
if dep not in handled:
handled.add(dep)
if dep[0] in installed_modules:
print("Skipping installed module '%s'" % dep[0])
else:
res.append(dep)
print("List of dependencies in (likely) install order:")
pprint(res, indent=4)
print("Sorted list of dependencies:")
pprint(sorted(res), indent=4)
#!/usr/bin/env bash
set -euo pipefail
# Find dependencies of Python packages by installing it in a temporary virtualenv.
# Usage: findPythonDeps.sh <python-pkg-spec>
# Example usage with EasyBuild:
# eb TensorFlow-2.3.4.eb --dump-env
# source TensorFlow-2.3.4.env
# findPythonDeps.sh tensorflow==2.3.4
package_to_check="${1}"
package_name="${package_to_check%%=*}"
# prevent pip from (ab)using $HOME/.cache/pip
export XDG_CACHE_HOME="$(mktemp -t -d pip_cache.XXXXXXXXXX)"
venv_dir="$(mktemp -t -d "venv_${package_name}.XXXXXXXXXX")"
# create virtualenv, install package in it
virtualenv --system-site-packages "${venv_dir}"
source "${venv_dir}"/bin/activate
pip install "${package_to_check}"
# install pipdeptree, figure out dependency tree for installed package
pip install pipdeptree
dep_tree_file="${venv_dir}/deptree.json"
pipdeptree -j -p "${package_name}" > "${dep_tree_file}"
deactivate
deptree2exts_list.py "${dep_tree_file}" "${package_name}"
rm -rf "${venv_dir}" "${XDG_CACHE_HOME}"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment