Testing Python scripts using pytest

.gitignore

__pycache__/
.pytest_cache/

README.md

An approach to https://twitter.com/ezyang/status/1735987289795215724

Output:

$ pytest -v
============================= test session starts ==============================
platform linux -- Python 3.11.6, pytest-7.2.2, pluggy-1.0.0 -- /usr/bin/python3
cachedir: .pytest_cache
rootdir: ..., configfile: pytest.ini
plugins: anyio-3.5.0
collecting ... collected 2 items

test_scripts.py::test_script[scripta.py] PASSED                          [ 50%]
test_scripts.py::test_script[scriptb.py] FAILED                          [100%]

=================================== FAILURES ===================================
___________________________ test_script[scriptb.py] ____________________________

script_path = PosixPath('.../scriptb.py')

    def test_script(script_path: Path):
>       _run_script(script_path)
E       AssertionError: Script exited with non-zero exit code
E       assert 1 == 0
E        +  where 1 = CompletedProcess(args=['/usr/bin/python3', PosixPath('.../scriptb.py')], returncode=1).returncode

test_scripts.py:35: AssertionError
----------------------------- Captured stdout call -----------------------------
This one fails
----------------------------- Captured stderr call -----------------------------
Traceback (most recent call last):
  File ".../scriptb.py", line 3, in <module>
    assert success
AssertionError
=========================== short test summary info ============================
FAILED test_scripts.py::test_script[scriptb.py] - AssertionError: Script exit...
========================= 1 failed, 1 passed in 0.21s ==========================

pytest.ini

[pytest]
python_files = test_scripts.py

scripta.py

print("This one succeeds")
success = True
assert success

scriptb.py

print("This one fails")
success = False
assert success

test_scripts.py

import sys
import subprocess
from pathlib import Path


def pytest_generate_tests(metafunc):
    if "script_path" in metafunc.fixturenames:
        this_module = Path(__file__)

        scripts = [
            path for path in this_module.parent.glob("**/*.py") if path != this_module
        ]

        metafunc.parametrize(
            "script_path",
            sorted(scripts),
            ids=lambda path: str(path.relative_to(this_module.parent)),
        )


def _run_script(path: Path):
    __tracebackhide__ = True

    env = {
        "PYTHONUNBUFFERED": "1",
        "PYTHONDONTWRITEBYTECODE": "1",
    }
    args = [sys.executable, path]
    proc = subprocess.run(args, env=env)

    assert proc.returncode == 0, "Script exited with non-zero exit code"


def test_script(script_path: Path):
    _run_script(script_path)