A demo of importing Python modules directly from URLs

url_loader_demo.py

#!/usr/bin/env python

import contextlib
import importlib.abc
import importlib.machinery
import importlib.util
import logging
import sys
import types
from urllib.request import urlopen
from urllib.error import URLError


__log__ = logging.getLogger(__name__)


class URLLoader(importlib.abc.Loader):
    """An import loader that loads packages and modules directly from URLs"""

    def __init__(self, packages = None):
        self._packages = packages if packages is not None else {}
        super().__init__()

    def register(self, package_name, url_template):
        self._packages[package_name] = url_template

    def _path_to_package_name(self, path):
        return path.split(".", 1)[0].lower()

    def provides(self, path: str) -> bool:
        return self._path_to_package_name(path) in self._packages

    def _get_url_options(self, path):
        url_template = self._packages.get(self._path_to_package_name(path))
        if not url_template:
            return []

        # Allow a fallback to __init__.py (for top-level modules)
        return [
            url_template.format(path=p.replace(".", "/"))
            for p in (path, path + ".__init__")
        ]

    def download_module(self, path):
        for url in self._get_url_options(path):
            with contextlib.suppress(URLError):
                with urlopen(url) as resp:
                    __log__.debug("Downloaded module '%s' from '%s'", path, url)
                    return resp.read()
            __log__.debug("Failed to download module '%s' from '%s'", path, url)
        return None

    def create_module(self, spec: importlib.machinery.ModuleSpec) -> types.ModuleType:
        if spec.name not in self._packages:
            # Is a module - use the default module creation semantics
            return None

        # Create the package
        module = types.ModuleType(spec.name)
        module.__path__ = []
        module.__package__ = spec.name
        return module

    def exec_module(self, module: types.ModuleType) -> None:
        code = self.download_module(module.__name__)
        if not code:
            raise ImportError(
                "{} failed to download configured '{}' module".format(
                    self.__class__.__name__,
                    module.__name__
                )
            )
        exec(code, module.__dict__)


class URLFinder(importlib.abc.MetaPathFinder):
    """An import path finder that uses the configured URLLoader to find and download modules"""
    def __init__(self, loader: URLLoader):
        self._loader = loader

    def find_spec(self, fullname: str, path, target=None):
        if self._loader.provides(fullname):
            return importlib.util.spec_from_loader(fullname, self._loader)
        return None


class LoggingPathFinder(importlib.abc.MetaPathFinder):
    """An import path finder that logs every module that is requested to be loaded"""

    def find_spec(self, fullname, path, target):
        __log__.debug("Importing '%s' (path=%s)", fullname, path)
        return None


if __name__ == "__main__":

    logging.basicConfig(level=logging.DEBUG)
    #sys.meta_path.append(LoggingPathFinder())

    print("Creating the URLLoader and adding it to sys.meta_path")
    url_loader = URLLoader()
    sys.meta_path.append(URLFinder(url_loader))

    print("Loading colorama to use colored output")
    url_loader.register("colorama", "https://raw.githubusercontent.com/tartley/colorama/master/{path}.py")
    from colorama import init, Fore, Style
    init()
    green = lambda m: print(Fore.GREEN + m + Style.RESET_ALL)
    green("Colors loaded")

    green("Loading zipstream and test it works")
    url_loader.register("zipstream", "https://raw.githubusercontent.com/pR0Ps/zipstream-ng/master/{path}.py")
    from zipstream import ZipStream
    zs = ZipStream.from_path(".")
    computed_size = len(zs)
    actual_size = len(bytes(b"".join(zs)))
    data = zs.get_info()
    green(
        "Created streamed zip with {} files ({}), {} bytes ({} bytes estimated)".format(
            len(data),
            ", ".join(x["name"] for x in data),
            actual_size,
            computed_size,
        )
    )

    url_loader.register("pipdeptree", "https://raw.githubusercontent.com/naiquevin/pipdeptree/master/{path}.py")
    green("Using pipdeptree to show no packages are installed")
    from pipdeptree import main as show_packages

    sys.argv, args = sys.argv[0:1], sys.argv[1:]  # prevent args from being passed to it
    print(Fore.YELLOW, end="")
    show_packages()
    print(Style.RESET_ALL)

    green("Using click to say hello")
    url_loader.register("click", "https://raw.githubusercontent.com/pallets/click/main/src/{path}.py")
    import click

    @click.command
    @click.option("--name", prompt="Your name", help="The person to greet.", default="unknown")
    def hello(name):
        green(f"Hello {name}!")

    sys.argv += args  # put args back
    hello()