vendoring Python modules with Habitat

module_vendoring.md

The following code can be put in a file we source via a plan.sh inside a directory representing the module to be vendored, such as Cython:

pkg_origin="bixu"
pkg_name="${PWD##*/}" # use the name of the directory containing our `plan.sh` to get the `pip` module name
pkg_bin_dirs=(bin)
pkg_lib_dirs=(lib)

# libs commonly needed during Python module installs:
pkg_build_deps=(
  core/gcc
  core/libffi
  core/pcre
  core/python
)

requirements_file="./requirements.txt"

do_before() {
  update_pkg_version
}

pkg_version() {
  grep "^${pkg_name}==" ${requirements_file} | cut -d= -f3 # extract the version number for our vendored module
}

do_setup_environment() {
  push_runtime_env   PYTHONPATH      "${pkg_prefix}/lib/python3.6/site-packages"
  push_buildtime_env LD_LIBRARY_PATH "$(pkg_path_for core/gcc)/lib"
  push_buildtime_env LD_LIBRARY_PATH "$(pkg_path_for core/libffi)/lib"
  push_buildtime_env LD_LIBRARY_PATH "$(pkg_path_for core/pcre)/lib"
}

do_prepare() {
  python -m venv "${pkg_prefix}"
  source "${pkg_prefix}/bin/activate"
}

do_build() {
  return 0
}

do_install() {
  pip install --no-cache-dir "${pkg_name}==${pkg_version}"
  build_line "${pkg_name} version:"
  python -c "import ${pkg_name}; print(${pkg_name}.__version__)" # ensure we can import our vendored module
}

do_strip() {
  return 0
}

An example of how we might mutate pkg_deps to include vendored modules, using our project's requirements.txt as our source of truth for module versions (this approach would be useful for scripting):

pkg_deps=(core/python/3.6.3)

pkg_deps+=($(echo origin/$(grep "^botocore"   requirements.txt) | sed 's@==@/@'))
pkg_deps+=($(echo origin/$(grep "^Cython"     requirements.txt) | sed 's@==@/@'))
pkg_deps+=($(echo origin/$(grep "^pandas"     requirements.txt) | sed 's@==@/@'))
pkg_deps+=($(echo origin/$(grep "^scipy"      requirements.txt) | sed 's@==@/@'))
pkg_deps+=($(echo origin/$(grep "^tensorflow" requirements.txt) | sed 's@==@/@'))

Alternatively, we can statically do it:

pkg_deps=(
  core/python/3.6.3
  $(echo origin/$(grep "^botocore"   requirements.txt) | sed 's@==@/@')
  $(echo origin/$(grep "^Cython"     requirements.txt) | sed 's@==@/@')
  $(echo origin/$(grep "^pandas"     requirements.txt) | sed 's@==@/@')
  $(echo origin/$(grep "^scipy"      requirements.txt) | sed 's@==@/@')
  $(echo origin/$(grep "^tensorflow" requirements.txt) | sed 's@==@/@')
)