pyvenv

conanfile.py

from conans import ConanFile, tools
from conans.model import Generator

import os
import re
from pyvenv import venv

class PyvenvConanFile(ConanFile):
    name = "pyvenv"
    version = "0.3.1"
    description = "Generates Python venv based on Python hosting Conan"
    exports = ['pyvenv.py']

    no_copy_source = True

    def package_info(self):
        self.env_info.PYTHONPATH.append(os.path.dirname(__file__))

class pyvenv(Generator):
    @property
    def filename(self):
        pass

    @property
    def content(self):
        pip_requirements = []
        for dep_name, user_info in self.conanfile.deps_user_info.items():
            if 'pyvenv_pip_requirements' in user_info.vars:
                pip_requirements.append(os.path.join(self.conanfile.deps_cpp_info[dep_name].rootpath, user_info.pyvenv_pip_requirements))

        output_pyvenv = venv(self.conanfile)
        # clear=True because Conan generators typically overwrite the contents completely
        # in order to remove any trace of previously installed packages, files, etc.
        output_pyvenv.create(os.path.join(self.output_path, "pyvenv"), clear=True)

        if pip_requirements:
            self.conanfile.run(tools.args_to_string([output_pyvenv.pip, 'install', *(arg for requirements in pip_requirements for arg in ('-r',requirements))]))

        def import_executable(name, **properties):
            if not 'IMPORTED_LOCATION' in properties:
                try:
                    path = output_pyvenv.which(name, required=True)
                except FileNotFoundError as e:
                    self.conanfile.output.warn("pyvenv(Generator): FileNotFoundError: {e} (omitted pyvenv::{name} from pyvenv-config.cmake)".format(name=name,e=e))
                    return ""
                else:
                    properties['IMPORTED_LOCATION'] = cmake_path(path)


            cmake_properties = ["        %s %s" % (cmake_escape(prop_name), cmake_quoted(prop_val))
                                for prop_name, prop_val in properties.items() if not prop_val is None]
            return """
IF(NOT TARGET pyvenv::{name})
    add_executable(pyvenv::{name} IMPORTED)
    set_target_properties(pyvenv::{name} PROPERTIES
{properties}
    )
ENDIF()
""".format(name=name, properties='\n'.join(cmake_properties))

        pyvenv_config = """ # Creates python3 venv imported target
# uses same version of Python as Conan client
"""
        pyvenv_config += import_executable('python', IMPORTED_LOCATION=cmake_path(output_pyvenv.python))

        entry_points = output_pyvenv.entry_points()
        for name in entry_points.get('console_scripts',[]):
            pyvenv_config += import_executable(name)
        for name in entry_points.get('gui_scripts',[]):
            pyvenv_config += import_executable(name, WIN32_EXECUTABLE='ON')

        return { "pyvenv/cmake/pyvenv-config.cmake": pyvenv_config }

# and https://cmake.org/cmake/help/latest/manual/cmake-language.7.html#escape-sequences
escape_encoded = { '\t': r'\t', '\r': r'\r', '\n':r'\n' }
escape_identity = re.compile('[^A-Za-z0-9_;]')

# See https://cmake.org/cmake/help/latest/manual/cmake-language.7.html#quoted-argument
def cmake_quoted(string):
    def quoted_element(c):
        if c in ('\\', '"'):
            return '\\' + c
        if c in escape_encoded: #cosmetic, but I prefer escaping these, and it's allowed
            return escape_encoded[c]
        else:
            return c

    return '"' + ''.join(quoted_element(c) for c in string) + '"'

def cmake_escape(string):
    def escape_character(c):
        if c in escape_encoded:
            return escape_encoded[c]
        elif escape_identity.match(c):
            return '\\' + c
        else:
            return c

    return ''.join(escape_character(c) for c in string)

# as in FILE(TO_CMAKE_PATH ...)
def cmake_path(path):
    path = ';'.join(path.split(os.pathsep))
    return path.replace(os.path.sep,'/')

pyvenv.py

from conans import tools
import os
import sys
from contextlib import contextmanager
import pathlib
import operator
import itertools

# mostly like shutil.which, but allows searching for alternate filenames,
# and never falls back to %PATH% or curdir
def _which(files, paths, access=os.F_OK | os.X_OK):
    if isinstance(files,str): files = [files]
    if sys.platform == "win32":
        pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
        def expand_pathext(cmd):
            if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
                yield cmd # already has an extension, so check only that one
            else:
                yield from (cmd + ext for ext in pathext) # check all possibilities
        files = [x for cmd in files for x in expand_pathext(cmd)]

        # Windows filesystems are (usually) case-insensitive, so match might be spelled differently than the searched name
        # And in particular, the extensions from PATHEXT are usually uppercase, and yet the real file seldom is.
        # Using pathlib.resolve() for now because os.path.realpath() was a no-op on win32 
        # until nt symlink support landed in python 3.9 (based on GetFinalPathNameByHandleW)
        # https://github.com/python/cpython/commit/75e064962ee0e31ec19a8081e9d9cc957baf6415
        #
        # realname() canonicalizes *only* the searched-for filename, but keeps the caller-provided path verbatim:
        # they might have been short paths, or via some symlink, and that's fine

        def realname(file):
            path = pathlib.Path(file)
            realname = path.resolve(strict=True).name
            return str(path.with_name(realname))
    else:
        def realname(path): return path # no-op

    for path in paths:
        for file in files:
            filepath = os.path.join(path, file)
            if os.path.exists(filepath) and os.access(filepath,access) and not os.path.isdir(filepath): # is executable
                return realname(filepath)
    return None

def _default_python():
    base_exec_prefix = sys.base_exec_prefix

    if hasattr(sys, 'real_prefix'): # in a virtualenv, which sets this instead of base_exec_prefix like venv
        base_exec_prefix = getattr(sys,'real_prefix')

    if sys.exec_prefix != base_exec_prefix: # alread running in a venv
        # we want to create the new virtualenv off the base python installation,
        # rather than create a grandchild (child of of the current venv)
        names = [ os.path.basename(sys.executable), "python3", "python" ]

        prefixes = [base_exec_prefix]

        suffixes = ["bin", "Scripts"]
        exec_prefix_suffix = os.path.relpath(os.path.dirname(sys.executable), sys.exec_prefix) # e.g. bin or Scripts
        if exec_prefix_suffix and exec_prefix_suffix != '.':
            suffixes.insert(0,exec_prefix_suffix)

        def add_suffix(prefix, suffixes):
            yield prefix
            yield from (os.path.join(prefix,suffix) for suffix in suffixes)

        dirs = [ x for prefix in prefixes for x in add_suffix(prefix, suffixes) ]
        return _which(names, dirs)
    else:
        return sys.executable

# build helper for making venv
class venv():
    def __init__(self, conanfile, python = _default_python()):
        self._conanfile = conanfile
        self.base_python = python
        self.env_folder = None

    # symlink logic borrowed from python -m venv
    # See venv.main() in /Lib/venv/__init__
    def create(self, folder, *, clear=True, symlinks=(os.name != 'nt'), with_pip=True):
        self.env_folder = folder

        self._conanfile.output.info("creating venv at %s based on %s" % (self.env_folder,self.base_python or '<conanfile>'))

        if self.base_python:
            # another alternative (if we ever wanted to support more customization) would be to launch
            # a `python -` subprocess and feed it the script text `import venv venv.EnvBuilder() ...` on stdin
            venv_options = ['--symlinks' if symlinks else '--copies']
            if clear: venv_options.append('--clear')
            if not with_pip: venv_options.append('--without-pip')
            with tools.environment_append({'__PYVENV_LAUNCHER__': None}):
                self._conanfile.run(tools.args_to_string([self.base_python, '-mvenv', *venv_options, self.env_folder]))
        else:
            # fallback to using the python this script is running in
            # (risks the new venv having an inadvertant dependency if conan itself is virtualized somehow, but it will *work*)
            import venv
            builder = venv.EnvBuilder(clear=clear, symlinks=symlinks, with_pip=with_pip)
            builder.create(self.env_folder)

    def entry_points(self, package=None):
        import importlib.metadata # Python 3.8 or greater
        entry_points = itertools.chain.from_iterable(
            dist.entry_points for dist in importlib.metadata.distributions(name=package, path=self.lib_paths))

        by_group = operator.attrgetter('group')
        ordered = sorted(entry_points, key=by_group)
        grouped = itertools.groupby(ordered, by_group)

        return {
            group: [x.name for x in entry_points]
            for group, entry_points in grouped
        }

    def setup_entry_points(self, package, folder):
        # create target folder
        try:
            os.makedirs(folder)
        except Exception:
            pass

        def copy_executable(name, target_folder, type):
            import shutil

            # locate script in venv
            try:
                path = self.which(name, required=True)
            except FileNotFoundError as e:
                # avoid FileNotFound if the no launcher script for this name was found, or
                self._conanfile.output.warn("pyvenv.setup_entry_points: FileNotFoundError: %s" % e)
                return

            root, ext = os.path.splitext(path)

            try:
                # copy venv script to target folder
                shutil.copy2(path, target_folder)

                # copy entry point script
                # if it exists
                if type == 'gui':
                    ext = '-script.pyw'
                else:
                    ext = '-script.py'

                entry_point_script = root + ext

                if(os.path.isfile(entry_point_script)):
                    shutil.copy2(entry_point_script, target_folder)
            except shutil.SameFileError:
                # SameFileError if the launcher script is *already* in the target_folder
                # e.g. on posix systems the venv scripts are already in bin/
                self._conanfile.output.error("pyvenv.setup_entry_points: SameFileError: command '%s' already found in '%s'. Other entry_points may also be unintentionally visible." % (name, folder))

        entry_points = self.entry_points(package)
        for name in entry_points.get('console_scripts',[]):
            copy_executable(name, folder, type='console')
        for name in entry_points.get('gui_scripts',[]):
            copy_executable(name, folder, type='gui')

    @property
    def bin_paths(self):
        # this should be the same logic as as
        # context.bin_name = ... in venv.ensure_directories
        if sys.platform == 'win32':
            binname = 'Scripts'
        else:
            binname = 'bin'
        bindirs = [binname]
        return [os.path.join(self.env_folder, x) for x in bindirs]

    @property
    def lib_paths(self):
        # this should be the same logic as as
        # libpath = ... in venv.ensure_directories
        if sys.platform == 'win32':
            libpath = os.path.join(self.env_folder, 'Lib','site-packages')
        else:
            libpath = os.path.join(self.env_folder, 'lib', 'python%d.%d' % sys.version_info[:2],'site-packages')
        return [libpath]

    #return the path to a command within the venv, None if only found outside
    def which(self, command, required = False, **kwargs):
        found = _which(command, self.bin_paths, **kwargs)
        if found:
            return found
        elif required:
            raise FileNotFoundError("command %s not in venv bin_paths %s" % (command, os.pathsep.join(self.bin_paths)))
        else:
            return None

    # convenience wrappers for python/pip since they are so commonly needed
    @property
    def python(self): return self.which("python", required=True)

    @property
    def pip(self): return self.which("pip", required = True)

    # environment variables like the usual venv `activate` script, i.e.
    # with tools.environment_append(venv.env):
    #     ...
    @property
    def env(self):
        return {
            '__PYVENV_LAUNCHER__': None, # this might already be set if conan was launched through a venv
            'PYTHONHOME': None,
            'VIRTUAL_ENV': self.env_folder,
            'PATH': self.bin_paths
        }

    # Setup environment and add site_packages of this this venv to sys.path
    # (importing from the venv only works if it contains python modules compatible
    #  with conan's python interrpreter as well as the venv one
    # But they're generally the same per _default_python(), so this will let you try
    # with venv.activate():
    #     ...
    @contextmanager
    def activate(self):
        old_path = sys.path[:]
        sys.path.extend(self.lib_paths)
        with tools.environment_append(self.env):
            yield
        sys.path = old_path

sphinx conanfile.py

class SphinxConan(ConanFile):
    name = "python-sphinx"
    version = "4.4.0"
    homepage = "http://www.sphinx-doc.org"
    description = "Sphinx is used to generate the help documentation"
    settings = "os_build", "arch_build"
    build_requires = [
        'pyvenv/0.3.1@jdps-focus-desktop-tools/release'
    ]

    # python venvs are not relocatable, so we will not have binaries for this on artifactory. Just build it on first use
    build_policy = "missing"
    exports_sources = [ "SphinxMacros.cmake", "python-sphinx-config.cmake" ]

    def package(self):
        from pyvenv import venv
        venv = venv(self)
        venv.create(folder=os.path.join(self.package_folder))

        self.run('{pip} install sphinx=={version}'.format(pip=venv.pip, version=self.version))
        self.run('{pip} install sphinx-rtd-theme==0.5.2'.format(pip=venv.pip))
#        self.run(tools.args_to_string([venv.pip, 'install', 'recommonmark==0.5.0']))
        self.copy("*.cmake", dst="Modules")

        venv.setup_entry_points("sphinx", os.path.join(self.package_folder,"bin"))

    def package_info(self):
        self.cpp_info.builddirs = [os.path.join(self.package_folder, 'Modules')]

test_package CMakeLists.txt

project(pyvenv_test LANGUAGES)

set(CONAN_DISABLE_CHECK_COMPILER 1) # not compiling anything here
include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake)
conan_basic_setup()

find_package(pyvenv REQUIRED HINTS ${CMAKE_BINARY_DIR})
enable_testing()
add_test(NAME python_version COMMAND pyvenv::python --version)
add_test(NAME pip_list COMMAND pyvenv::pip list)

test_package conanfile.py

from conans import ConanFile, tools, CMake
import os

class TestPyvenv(ConanFile):
    generators = 'pyvenv','cmake'

    def build(self):
        from pyvenv import venv
        build_env = venv(self)
        build_env.create("build_env")
        self.build_env = build_env

        build_env.setup_entry_points("pip", os.path.join(self.build_folder, "bin"))

        cmake = CMake(self)
        cmake.configure()

    def test(self):
        # just run something quick to test that we can call pip
        self.run(tools.args_to_string([self.build_env.python, '--version']))
        self.run(tools.args_to_string([self.build_env.pip, 'list']))

        # try running pip copied to bin folder
        self.run(tools.args_to_string([os.path.join(self.build_folder, "bin", "pip"), 'list']))
        cmake = CMake(self)
        cmake.test()