A Python test script for testing simple input based scripts using Pytest

test-py-script.sh

#!/bin/sh
# Generates a python script that checks another input-based python script
# Uses another python file as the reference for "true" answers ($header file)
#
# $1: path to script to test
# $2: path to the file holding answers variable
# $3: optional pytest flags
# $4: if "keep", the script doesn't remove the constructed file
#
# A `header.py` example:
# ```
# # Pairs to check a program that takes 4 inputs and prints the minimum
# VALUES_RESULTS_PAIRS = [
#     ([43, 123, -4, 23], iof(-4)),  # negative
#     ([1, 2, 4, 1],      iof(1)),   # multiple
#     ([1, 2, 4, 5],      iof(1)),   # First number
#     ([2, 1, 4, 5],      iof(1)),   # Second number
#     ([2, 2, 1, 5],      iof(1)),   # Third number
#     ([3, 1, 42, 0],     iof(0)),   # Forth number
# ]
# ```
file="/tmp/test-script.py"
header="${2-:$1}"
cat >"$file" <<EOF
#!/usr/bin/env python3
# requires python-console-scripts (script_runner)
# it gets included in other scripts with these two variables modified
# SCRIPT_NAME: str
# VALUES_RESULTS_PAIRS: list[tuple[list[str], str|list|tuple]]

import pytest

from collections import deque

# Helper functions
def iof(x) -> tuple[int, float]:
    """Return int and float of the same number x."""
    return (int(x), float(x))


SCRIPT_NAME: str = "$1"

$(cat "$header")


def make_multiple_inputs(inputs):
    """Make values to patch on the __builtins__.input.

    this could be used in a monkeypatch like
    m.setitem(__builtins__, "input", make_multiple_inputs(values))
    where values is a sequence of inputs that is expected for a "user" to
    put in the input prompt

    """
    if not isinstance(inputs, (list, tuple)):
        inputs = [inputs]
    inputs = [str(i) for i in inputs]
    inputs = deque(inputs)
    return lambda _: inputs.popleft()


@pytest.mark.parametrize("values, results", VALUES_RESULTS_PAIRS)
def test_script(values, results, monkeypatch, script_runner):
    """Test the script with the included VALUES_RESULTS_PAIRS."""
    with monkeypatch.context() as m:
        m.setitem(__builtins__, "input", make_multiple_inputs(values))
        ret = script_runner.run(SCRIPT_NAME)
        assert ret.success

        if not isinstance(results, (list, tuple)):
            value = ret.stdout == str(results) + "\n"
        else:  # if a collection of results
            value = ret.stdout == str(results[0]) + "\n"
            for i in results[1:]:
                if value:  # if it's already True, break
                    break
                value = value or (ret.stdout == str(i) + "\n")
        if not value:
            pytest.fail(
                f"{ret.stdout[:-1]} didn't match any of results: {results}",
            )
EOF
pytest $3 "$file"
x=$?
[ "$4" = "keep" ] && rm "$file"
exit $x

test_script.py

#!/usr/bin/env python3
"""Make a dynamic pytest script for input based scripts and run it.

A rewrite of my test-py-script, originally written in shell.

Requires pytest, python 3.9 and python-console-scripts plugin for pytest
(`script_runner` fixture)

Takes 2 inputs (from shell or built-in input function):
- $1: Path to the script that you want to test (`script_path`)
- $2: Path to the answer file (`header_path`)

Optional there can be a third and forth argument for the program, only passable
through shell.
- $3: Pytest space-separated flags (e.g "-s")
- $4: If equal to "keep", prevents the deletion of the constructed test script
  from the header files.

The header file is another python script having a mandatory
`VALUES_RESULTS_PAIRS: list[tuple[list[str], str|list|tuple]]`.
BUG I couldn't be bothered to implement `SCRIPT_NAME` variable support
like the shell version. Contributions are welcome.

NOTE Ironically, I never tested, nor ran the script for I basically have no use
for it since I just use the shell version. Use at your own risk.

There are "helper functions" like "iof" in this file that will implecitely get
injected in the final cloned/constructed test file. It means one can safely
ignore their lint's errors about "iof" not found and use it safely.

Examples (assuming the script name is `script.py`):
- script.py # providing sufficient permissions are granted
- python script.py
- python script.py /path/to/script-to-be-tested.py
- python script.py /path/to/script-to-be-tested.py /path/to/answers.py
- python script.py /path/to/script-to-be-tested.py /path/to/answers.py "-s"
- python script.py /path/to/script-to-be-tested.py /path/to/answers.py -s keep

A `header.py` example:
```
# Pairs to check a program that takes 4 inputs and prints the minimum
VALUES_RESULTS_PAIRS = [
    ([43, 123, -4, 23], iof(-4)),  # negative
    ([1, 2, 4, 1],      iof(1)),   # multiple
    ([1, 2, 4, 5],      iof(1)),   # First number
    ([2, 1, 4, 5],      iof(1)),   # Second number
    ([2, 2, 1, 5],      iof(1)),   # Third number
    ([3, 1, 42, 0],     iof(0)),   # Forth number
]
```

"""

import inspect
import os
import sys
import tempfile
from collections import deque
from shutil import rmtree

import pytest


# Helper functions
def iof(x) -> tuple[int, float]:
    """Return int and float of the same number x."""
    return (int(x), float(x))


def make_multiple_inputs(inputs):
    """Make values to patch on the `__builtins__.input`.

    this could be used in a monkeypatch like
    `m.setitem(__builtins__, "input", make_multiple_inputs(values))`
    where values is a sequence of inputs that is expected for a "user" to
    put in the input prompt

    """
    if not isinstance(inputs, (list, tuple)):
        inputs = [inputs]
    inputs = [str(i) for i in inputs]
    inputs = deque(inputs)
    return lambda _: inputs.popleft()


def the_actual_test_script(
    script_path: str,
    values,
    results,
    monkeypatch,
    script_runner,
) -> None:
    """Test the script with the included `VALUES_RESULTS_PAIRS`."""
    with monkeypatch.context() as m:
        m.setitem(__builtins__, "input", make_multiple_inputs(values))
        ret = script_runner.run(script_path)
        assert ret.success

        if not isinstance(results, (list, tuple)):
            value = ret.stdout == str(results) + "\n"
        else:  # if a collection of results
            value = ret.stdout == str(results[0]) + "\n"
            for i in results[1:]:
                if value:  # if it's already True, break
                    break
                value = value or (ret.stdout == str(i) + "\n")
        if not value:
            pytest.fail(
                f"{ret.stdout[:-1]} didn't match any of results: {results}",
            )


# List helper functions this far
HELPER_FUNCTIONS: list[str] = [
    name
    for name, func in inspect.getmembers(sys.modules[__name__])
    if inspect.isfunction(func) and func.__module__ == __name__
    # and not name.startswith("invisible_")
]

# Don't run the block if the file is getting included (prevents recursion)
if __name__ == "__main__":

    # NOTE invisible_* functions won't be included in generated tests
    def argv_or_input(index, prompt) -> str:
        """Take an input from argv[index], if undefined, ask interactively."""
        try:
            value = sys.argv[index]
        except IndexError:
            print("Insufficient inputs")
            value = input(prompt)
        return value

    # Variables
    script_path: str = os.path.expanduser(
        argv_or_input(1, "Input the path of the script you want to test: ")
    )
    header_path: str = os.path.expanduser(
        argv_or_input(2, "Input the path of the header (answers) file: ")
    )
    file_split_path: tuple[str, str] = os.path.split(__file__)
    header_split_path: tuple[str, str] = os.path.split(header_path)
    parent_module_name: str = os.path.splitext(
        file_split_path[1],
    )[0]
    tmp_dir = tempfile.mkdtemp()
    clone_path: str = os.path.join(tmp_dir, header_split_path[1])
    clone_data: tuple[str, str] = os.path.split(clone_path)

    print("Turn the header in a full blown test")
    # Make a clone of the header file, inject it with the functions, add some
    # more pytest related ones and run the test
    # Clone is the newly generated test file from the header
    with open(header_path, "r") as file:
        with open(clone_path, "w") as clone:
            clone.write(
                "from importlib import import_module\n"
                "import pytest, os, sys\n"
                f'sys.path.append("{file_split_path[0]}")\n\n'
            )
            for i in HELPER_FUNCTIONS:
                clone.write(f"from {parent_module_name} import {i}\n")
            clone.write("\n\n")
            clone.write(file.read())  # The original header contents
            clone.write(
                f"""@pytest.mark.parametrize("values, results", VALUES_RESULTS_PAIRS)
def test_script(
    values,
    results,
    monkeypatch,
    script_runner,
):
    the_actual_test_script(
        "{script_path}",
        values,
        results,
        monkeypatch,
        script_runner,
    )
"""
            )
    print(f"Created the test {clone_path}\n\n")

    # Manage the final args and run the test
    args = [clone_path]
    keep_the_tmp_dir = False
    if len(sys.argv) >= 4:
        args = [*sys.argv[3].split(), *args]
        if len(sys.argv) == 5 and sys.argv[5] == "keep":
            keep_the_tmp_dir = True
    pytest_status = pytest.main(args)
    if not keep_the_tmp_dir:
        rmtree(tmp_dir)  # Delete the temp files after successful load
    exit(pytest_status)