build_embed_dist.py

"""Creates a built program with Python embedded that can be compressed and distributed."""

import contextlib
import http.client
import logging
import os
import pathlib
import shutil
import subprocess
import sysconfig
from dataclasses import dataclass
from string import Template
from zipfile import ZipFile

"""
This script compiles a Python application that can be distributed in a zip file.
Unzip the contents in a folder, and run [exe_name].exe.

Prerequisites
=============

Put this script (`build_embed_dist.py`), `custom_python_pth`, and `main.c` in a folder in your source code
named 'support'.

If you want to change this, change the places in this script that reference `support/`.

Decide on a python version
--------------------------

These instructions work for one, and only one, Python version.
Choose this first.


A 'full' install of the same Python version
-------------------------------------------

The full Python install includes the headers and libraries, which are missing from the embedded Python.
Note that the patch version must also match.

This is also needed to be able to run pip to install 
Use `py --list-paths` to get the path to the matching version of the 'full install' of Python.

The path to 'python.exe' is the 'full_python_file' path.


Visual Studio tools for building the exe
----------------------------------------

Using Visual Studio Installer,
select "Individual Components for Visual Studio Community 2022",
Then:
- "MSVC v143 - VS 2022 C++ x64/x86 build tools (Latest)".
- "C++ core features"
- "C++ 2022 Redistributable Update"
- "Windows 10 SDK ([version])" - the most recent version (for 10, not 11)



Running
=======

Change directory to the top level of your application source tree.

Run the `./support/build_embed_dist.py` build script using the path to the full install `python.exe` that you found as part of the prerequisites.

Wait for the build process.

When prompted, start the 'x64 Native Tools Command Prompt for VS 2022' from the Start menu.

Run the commands provided.

Refs
====

- https://python-packaging-user-guide--1778.org.readthedocs.build/en/1778/guides/windows-applications-embedding/
- https://stevedower.id.au/blog/build-a-python-app
- https://discuss.python.org/t/how-to-create-a-window-command-line-application-and-its-installer-from-wheels/76136/17
- https://learn.microsoft.com/en-us/cpp/build/how-to-enable-a-64-bit-visual-cpp-toolset-on-the-command-line

"""


logging.basicConfig(
    format="%(asctime)s [%(levelname)-8s]: %(message)s",
    level=logging.DEBUG,
)
logger = logging.getLogger(__name__)


@dataclass(frozen=True)
class EmbedDetails:
    output_dir: pathlib.Path
    python_version: str
    arch: str
    full_python_file: pathlib.Path
    exe_name: str
    module: str
    function: str


class EmbedBuilder:
    def __init__(self, details: EmbedDetails):
        self._details = details

    @property
    def python_version(self):
        return self._details.python_version

    @property
    def python_collapsed_version(self):
        return self._details.python_version.replace(".", "")

    @property
    def python_collapsed_major_minor_version(self):
        ver = "".join(self._details.python_version.split(".")[:-1])
        return ver

    @property
    def output_dir(self):
        return self._details.output_dir

    @property
    def working_dir(self):
        return self._details.output_dir / "working"

    @property
    def dist_dir(self):
        return self._details.output_dir / "dist"

    @property
    def interp_dir(self):
        return self.dist_dir / "interp"

    @property
    def lib_dir(self):
        return self.dist_dir / "lib"

    @property
    def embed_zip_path(self):
        return self.working_dir / "python-embed.zip"

    @property
    def driver_code_file(self):
        return self.working_dir / "main.c"

    @property
    def driver_obj_file(self):
        return self.working_dir / "main.obj"

    @property
    def exe_file(self):
        return self.dist_dir / f"{self._details.exe_name}.exe"

    def install_lib(self):
        logger.info("Installing to lib dir %s", self.lib_dir)
        python = self._details.full_python_file

        self._run_cmd(
            [
                str(python),
                "-m",
                "pip",
                "install",
                "--target",
                str(self.lib_dir),
                os.getcwd(),
            ],
            env={**os.environ},
        )

    def test_lib(self):
        logger.info("Testing lib in %s", self.lib_dir)
        d = self._details
        env = {
            "PYTHONPATH": str(self.lib_dir),
        }

        to_run = f"from {d.module} import {d.function}; {d.function}()"

        cmds = [str(d.full_python_file), "-c", to_run]
        result = self._run_cmd(cmds, env)
        logger.debug(result)

    def download_python_embed(self):
        target = self.embed_zip_path
        self._create_dir(target.parent)

        ver = self._details.python_version
        arch = self._details.arch

        url = "/".join(["/ftp/python", ver, f"python-{ver}-embed-{arch}.zip"])

        logger.info(
            "Downloading embedded Python version %s arch %s from %s to %s",
            ver,
            arch,
            url,
            target,
        )

        conn = None
        try:
            conn = http.client.HTTPSConnection("www.python.org")
            conn.request("GET", url)
            r = conn.getresponse()

            if r.status != http.HTTPStatus.OK:
                raise ValueError(
                    f"Unexpected response from '{r.geturl()}': {r.status} - {r.reason}"
                )

            content = r.read()
            target.write_bytes(content)
        finally:
            if conn is not None:
                conn.close()

    def unpack_python_embed(self):
        target = self.interp_dir
        self._create_dir(target)

        logger.info("Unpack embedded Python to %s", target)

        with contextlib.chdir(target), ZipFile(self.embed_zip_path, "r") as f:
            f.extractall(target)

    def template_python_pth(self):
        ver = self.python_collapsed_major_minor_version
        target = self.interp_dir / f"python{ver}._pth"

        logger.info("Template embedded Python to %s", target)

        source = pathlib.Path("support/custom_python_pth").resolve(strict=True)
        content = self._template(source, python_ver=ver)
        target.write_text(content, encoding="utf-8")

    def template_driver_code(self):
        target = self.driver_code_file

        logger.info("Template driver C code file to %s", target)

        source = pathlib.Path("support/main.c").resolve(strict=True)
        d = self._details
        content = self._template(source, module=d.module, function=d.function)
        target.write_text(content, encoding="utf-8")

    def find_python_headers_library(self):
        logger.info("Find Python include and library paths")
        return {
            "include": sysconfig.get_path("include"),
            "library": sysconfig.get_config_vars().get("LIBDIR"),
        }

    def compile_driver_application(self):
        d = self._details
        paths = self.find_python_headers_library()

        logger.info("Compile C driver application")
        cmds_cl = [
            "cl",
            "/c",
            f'/Fo:"{str(self.driver_obj_file)}"',
            f'/I"{paths["include"]!s}"',
            f'"{str(self.driver_code_file)}"',
        ]

        logger.info("Link C driver application")
        ver = self.python_collapsed_major_minor_version
        cmds_link = [
            "link",
            f'"{str(self.driver_obj_file)}"',
            f'/OUT:"{str(self.exe_file)}"',
            f"/DELAYLOAD:python{ver}.dll",
            f'/LIBPATH:"{paths["library"]!s}"',
        ]

        logger.warning(
            "Run these commands in a 'x64 Native Tools Command Prompt for VS 2022' terminal:"
        )
        logger.warning("cd %s", os.getcwd())
        logger.warning(" ".join(cmds_cl))
        logger.warning(" ".join(cmds_link))

    def test_app(self):
        logger.info("Testing lib in %s", self.lib_dir)
        cmds = [str(self.exe_file)]

        logger.warning("Run these commands in a terminal:")
        logger.warning(" ".join(cmds))

    def run(self):
        d = self._details

        self._clean_dir(d.output_dir)

        self.download_python_embed()

        self.unpack_python_embed()

        self.template_python_pth()

        self.install_lib()

        self.test_lib()

        self.template_driver_code()

        self.compile_driver_application()

        self.test_app()

    def _run_cmd(self, cmds: list[str], env: dict | None = None):
        env = env or {}
        try:
            result = subprocess.run(
                args=cmds,
                env=env,
                check=True,
                capture_output=True,
                shell=False,
            )
        except subprocess.CalledProcessError as e:
            logger.exception(f"Command failed", e)
            logger.error("Stdout: %s", e.stdout)
            logger.error("Stderr %s", e.stderr)
            raise

        return result

    def _create_dir(self, path: pathlib.Path) -> None:
        logger.info("Creating dir %s", path)
        path.mkdir(exist_ok=True, parents=True)

    def _clean_dir(self, path: pathlib.Path) -> None:
        logger.info("Cleaning dir %s", path)
        shutil.rmtree(path, ignore_errors=True)

    def _clean_file(self, path: pathlib.Path) -> None:
        logger.info("Deleting file %s", path)
        path.unlink(missing_ok=True)

    def _template(self, source: pathlib.Path, **kwargs) -> str:
        content = source.read_text(encoding="utf-8", errors="strict")
        s = Template(content)
        return s.substitute(**kwargs)


def main():
    details = EmbedDetails(
        output_dir=pathlib.Path("./embed").resolve(),
        python_version="3.13.1",
        arch="amd64",
        full_python_file=pathlib.Path(r"C:\Program Files\Python313\python.exe"),
        exe_name="my-app",
        module="my_app.cli",
        function="my_app_run",
    )
    EmbedBuilder(details).run()


if __name__ == "__main__":
    main()

custom_python_pth

python${python_ver}.zip
.
../lib

# Uncomment to run site.main() automatically
#import site

main.c

/* Include the Python headers */
#include <Python.h>

#define PYTHON_LOCATION L"interp"
#define APP_MODULE "${module}"
#define APP_FUNCTION "${function}"

/* Finding the Python interpreter */
#include <windows.h>
#include <pathcch.h>

/* Tell the Visual Studio linker what libraries we need */
#pragma comment(lib, "delayimp")
#pragma comment(lib, "pathcch")

int dll_dir(wchar_t *path) {
    wchar_t interp_dir[PATHCCH_MAX_CCH];
    if (GetModuleFileNameW(NULL, interp_dir, PATHCCH_MAX_CCH) &&
        SUCCEEDED(PathCchRemoveFileSpec(interp_dir, PATHCCH_MAX_CCH)) &&
        SUCCEEDED(PathCchCombineEx(interp_dir, PATHCCH_MAX_CCH, interp_dir, path, PATHCCH_ALLOW_LONG_PATHS)) &&
        SetDefaultDllDirectories(LOAD_LIBRARY_SEARCH_DEFAULT_DIRS) &&
        AddDllDirectory(interp_dir) != 0) {
                return 1;
    }
    return 0;
}

/* Your application main program */
int wmain(int argc, wchar_t **argv)
{
    PyStatus status;
    PyConfig config;

    /* Tell the loader where to find the Python interpreter.
     * This is the name, relative to the directory containing
     * the application executable, of the directory where you
     * placed the embeddable Python distribution.
     *
     * This MUST be called before any functions from the Python
     * runtime are called.
     */
    if (!dll_dir(PYTHON_LOCATION))
        return -1;

    /* Initialise the Python configuration */
    PyConfig_InitIsolatedConfig(&config);
    /* Pass the C argv array to ``sys.argv`` */
    PyConfig_SetArgv(&config, argc, argv);
    /* Install the standard Python KeyboardInterrupt handler */
    config.install_signal_handlers = 1;
    /* Initialise the runtime */
    status = Py_InitializeFromConfig(&config);
    /* Deal with any errors */
    if (PyStatus_Exception(status)) {
        PyConfig_Clear(&config);
        if (PyStatus_IsExit(status)) {
            return status.exitcode;
        }
        Py_ExitStatusException(status);
        return -1;
    }

    /* CPython is now initialised.
     * Now load and run your application code.
     */

    int exitCode = -1;
    PyObject *module = PyImport_ImportModule(APP_MODULE);
    if (module) {
        // Pass any more arguments here
        PyObject *result = PyObject_CallMethod(module, APP_FUNCTION, NULL);
        if (result) {
            exitCode = 0;
            Py_DECREF(result);
        }
        Py_DECREF(module);
    }
    if (exitCode != 0) {
        PyErr_Print();
    }
    Py_Finalize();
    return exitCode;
}