pytest exceptiongroups helpers

exceptiongroups.py

from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union

from ._compat import BaseExceptionGroup

E = TypeVar("E", bound=BaseException)
ExceptionMatch = Union[
    Type[E],
    Tuple[Type[E], ...],
    Callable[[E], bool],
]


def flattened_exceptions(group: BaseExceptionGroup) -> List[Exception]:
    exceptions = []

    for e in group.exceptions:
        if isinstance(e, BaseExceptionGroup):
            exceptions += flattened_exceptions(e)
        else:
            exceptions.append(e)

    return exceptions


def _flattened_exceptions_if_not_none(
    group: Optional[BaseExceptionGroup],
) -> Optional[List[Exception]]:
    if group is not None:
        return flattened_exceptions(group)
    return None


def filtered_flattened_exceptions(
    exception_match: ExceptionMatch,
    group: BaseExceptionGroup,
) -> Tuple[Optional[List[Exception]], Optional[List[Exception]]]:
    matched, others = group.split(exception_match)

    return (_flattened_exceptions_if_not_none(matched), _flattened_exceptions_if_not_none(others))

pytest_utils.py

"""WARNING: This module is experimental.
Only import this from within a test suite! This is not meant to be used at runtime.
Motivation:
Filtering exception groups for certain exceptions while maintaining expected behavior
from pytest.raises which fails a test if either:
 - the expected exception is not thrown
 - a different kind is thrown
    - `raises_in_group` takes this one step further and fails the test
      if the group contains an exception that was not expected
cattrs is using the exceptiongroup backport package from PyPI
This is also further inconvenient because:
  - PEP-654 (exception groups) includes language features in CPython >= 3.11
  - We aim to support Python 3.6
  - The backport package's `except*` implementation tries its best but isn't all that great
  - Python 3.11 is in its pre-release stages and exceptiongroup adoption among libraries is limited
  - Pytest currently has no ongoing work or discussions for this
  - Even the `unittest` module in the current 3.11 pre-release does not offer any additional support
It relies on internal pytest APIs which are succeptible to change at any time.
Where needed, we pass `_ispytest=True` to suppress warnings.
"""
import warnings
from inspect import getfullargspec
from types import TracebackType
from typing import Optional, Type, cast

from _pytest._code import ExceptionInfo
from pytest import fail

from ._compat import BaseExceptionGroup, ExceptionGroup

from .exceptiongroups import ExceptionMatch, flattened_exceptions

BYPASS_PYTEST_INTERNAL_WARNING = "_ispytest" in getfullargspec(ExceptionInfo.__init__).kwonlyargs


class PytestInternalAPIChangedError(Exception):
    ...


class RaisesExceptionGroupContext(object):
    """Captures the only (or first of, if allow_multiple=True) exception in a group.
    This is based on _pytest.python_api.RaisesExceptionContext.
    """

    def __init__(
        self, exception_match: ExceptionMatch, match_expr: str = None, allow_multiple=False
    ):
        self.exception_match = exception_match
        self.match_expr = match_expr
        self.allow_multiple = allow_multiple
        self.excinfo: Optional[ExceptionInfo] = None

    def __enter__(self):
        self.excinfo = ExceptionInfo.for_later()
        return self.excinfo

    def __exit__(
        self,
        exc_type: Type[BaseException] = None,
        exc_value: BaseException = None,
        traceback: TracebackType = None,
    ):
        assert self.excinfo

        if exc_type is None:
            fail(f"NO EXCEPTIONS RAISED, EXPECTED GROUP CONTAINING {self.exception_match}")

        if not issubclass(exc_type, BaseExceptionGroup):
            fail(f"NAKED EXCEPTION WAS RAISED: {repr(exc_value)}")

        exc_group = cast(BaseExceptionGroup, exc_value)

        (matched_exc, others) = exc_group.split(self.exception_match)

        if matched_exc is None:
            fail(f"DID NOT RAISE {self.exception_match} IN EXCEPTION GROUP")

        if others is not None:
            warnings.warn(f"Unexpected exceptions: {others!r}", stacklevel=2)
            raise ExceptionGroup(
                "Thrown group contained unexpected exceptions", others.exceptions
            ) from exc_value

        exceptions = flattened_exceptions(matched_exc)

        if len(exceptions) > 1 and not self.allow_multiple:
            fail(f"GROUP CONTAINED MORE THAN ONE {self.exception_match}")

        exc = exceptions[0]

        try:
            exc_info = (type(exc), exc, exc.__traceback__)
            if BYPASS_PYTEST_INTERNAL_WARNING:
                self.excinfo.__init__(exc_info, _ispytest=True)  # type: ignore
            else:
                self.excinfo.__init__(exc_info)  # type: ignore
        except BaseException as e:
            msg = (
                "Failed to populate exception info, please open an issue over at "
                "https://github.com/xoeye/typecats/issues."
                "Make sure to mention your Python and Pytest versions!"
            )
            raise PytestInternalAPIChangedError(msg) from e
        if self.match_expr is not None:
            self.excinfo.match(self.match_expr)

        return True  # suppress exception


def raises_in_group(exception_match: ExceptionMatch, match_expr: str = None, allow_multiple=False):
    """Captures the only (or first of, if allow_multiple=True) exception in a group.
    Fails the test in the following order:
      - No exceptions are thrown
      - Thrown exception is not an exception group
      - Thrown group does not contain expected exception
      - Thrown group contained unexpected exceptions
      - (if allow_multiple=False): Thrown group contains more than one of the expected exception
      - (if matched_expr is not None): First matched exception does not match the passed regex
    Parameters:
    exception_match: Same as ExceptionGroup.subgroup
    match_expr: Same as pytest.raises. If set, will only check against the first found exception.
    If you need to capture multiple exceptions from a group, you should instead
    use `pytest.raises(BaseExceptionGroup)` and consider
    using `typecats.exceptiongroups.filter_flattened_exceptions`, though you will
    need to account for any other unmatched exceptions on your own.
    """
    return RaisesExceptionGroupContext(exception_match, match_expr, allow_multiple)

test_exceptiongroups.py

from functools import partial

import pytest

from housecats._compat import ExceptionGroup

from housecats.exceptiongroups import filtered_flattened_exceptions, flattened_exceptions


@pytest.fixture
def sample_group():
    try:
        raise ExceptionGroup(
            "Sample Error",
            [
                ValueError(1),
                ExceptionGroup(
                    "Sample Error 2",
                    [
                        TypeError(2),
                        NameError("a"),
                        ExceptionGroup(
                            "Sample Error 3",
                            [
                                ValueError(3),
                            ],
                        ),
                    ],
                ),
                ExceptionGroup(
                    "Sample Error 2.1",
                    [
                        TypeError(2.1),
                        ValueError(2.1),
                    ],
                ),
            ],
        )
    except ExceptionGroup as e:
        return e


@pytest.fixture
def sample_flat():
    """Manually flattened sample_group, in lexical (or line-by-line) order from top to bottom"""
    return [
        ValueError(1),
        TypeError(2),
        NameError("a"),
        ValueError(3),
        TypeError(2.1),
        ValueError(2.1),
    ]


def test_flattened_exceptions(sample_group, sample_flat):
    for i, exc in enumerate(flattened_exceptions(sample_group)):
        assert isinstance(exc, type(sample_flat[i]))
        assert exc.args == sample_flat[i].args


get_value_errors = partial(filtered_flattened_exceptions, ValueError)
get_type_errors = partial(filtered_flattened_exceptions, TypeError)


def test_filtered_flattened_exceptions(sample_group):
    value_errors, non_value_errors = get_value_errors(sample_group)
    assert value_errors
    assert len(value_errors) == 3

    type_errors, _others = get_type_errors(sample_group)
    assert type_errors
    assert len(type_errors) == 2


def test_filtered_flattened_exceptions_tuple(sample_group):
    expected = [
        ValueError(1),
        TypeError(2),
        ValueError(3),
        TypeError(2.1),
        ValueError(2.1),
    ]

    matched, others = filtered_flattened_exceptions((ValueError, TypeError), sample_group)
    assert matched
    assert others

    for i, exc in enumerate(matched):
        assert isinstance(exc, type(expected[i]))
        assert exc.args == expected[i].args

    assert len(others) == 1
    assert isinstance(others[0], NameError)
    assert others[0].args == ("a",)


def test_filtered_flattened_exceptions_predicate(sample_group):
    def _is_value_under_3(exc: Exception):
        try:
            return int(exc.args[0]) < 3
        except ValueError:
            return False

    get_errors_under_3 = partial(filtered_flattened_exceptions, _is_value_under_3)

    expected_matched = [
        ValueError(1),
        TypeError(2),
        TypeError(2.1),
        ValueError(2.1),
    ]

    expected_others = [
        NameError("a"),
        ValueError(3),
    ]

    matched, others = get_errors_under_3(sample_group)
    assert matched
    assert others

    for i, exc in enumerate(matched):
        assert isinstance(exc, type(expected_matched[i]))
        assert exc.args == expected_matched[i].args

    for i, exc in enumerate(others):
        assert isinstance(exc, type(expected_others[i]))
        assert exc.args == expected_others[i].args


def test_filtered_flattened_exceptions_match_all_others_return_none(sample_group, sample_flat):
    matched, others = filtered_flattened_exceptions(lambda _: True, sample_group)
    assert others is None
    assert matched

    for i, exc in enumerate(matched):
        assert isinstance(exc, type(sample_flat[i]))
        assert exc.args == sample_flat[i].args


def test_filtered_flattened_exceptions_match_none(sample_group, sample_flat):
    matched, others = filtered_flattened_exceptions(lambda _: False, sample_group)
    assert matched is None
    assert others

    for i, exc in enumerate(others):
        assert isinstance(exc, type(sample_flat[i]))
        assert exc.args == sample_flat[i].args

test_pytest_utils.py

import warnings
from unittest.mock import MagicMock

import pytest
from pytest_mock import MockerFixture

from housecats._compat import ExceptionGroup

import housecats.pytest_utils as pytest_utils
from housecats.pytest_utils import PytestInternalAPIChangedError, raises_in_group


class UnitTestFailure(Exception):
    def __init__(self, *args):
        super().__init__(self, *args)


def _throw_instead_of_fail(*args):
    raise UnitTestFailure(*args)


@pytest.fixture
def mocked_fail(mocker: MockerFixture):
    mocked_fail = mocker.patch.object(pytest_utils, "fail")
    mocked_fail.side_effect = _throw_instead_of_fail
    return mocked_fail


def test_fails_if_nothing_raised(mocked_fail):
    with pytest.raises(UnitTestFailure, match="NO EXCEPTIONS RAISED"):
        with raises_in_group(TypeError):
            pass


def test_fails_if_naked_exception_raised(mocked_fail):
    with pytest.raises(UnitTestFailure, match="NAKED EXCEPTION WAS RAISED"):
        with raises_in_group(TypeError):
            raise TypeError("This is not encapsulated in a group")


def test_fails_if_group_does_not_contain_type(mocked_fail):
    with pytest.raises(UnitTestFailure, match="DID NOT RAISE <class 'TypeError'>"):
        with raises_in_group(TypeError):
            raise ExceptionGroup("", [ValueError()])

    with pytest.raises(
        UnitTestFailure, match=r"DID NOT RAISE \(<class 'TypeError'>, <class 'NameError'>\)"
    ):
        with raises_in_group((TypeError, NameError)):
            raise ExceptionGroup("", [ValueError()])

    with pytest.raises(UnitTestFailure, match=r"DID NOT RAISE <function"):

        def match_nothing(x: BaseException):
            return x.args == ("a", "b")

        with raises_in_group(match_nothing):
            raise ExceptionGroup("", [ValueError()])


def test_fails_if_group_has_more_than_one(mocked_fail: MagicMock):
    with pytest.raises(UnitTestFailure, match="GROUP CONTAINED MORE THAN ONE <class 'TypeError'>"):
        with raises_in_group(TypeError):
            raise ExceptionGroup("", [TypeError(), TypeError()])

    mocked_fail.reset_mock()

    with raises_in_group(TypeError, allow_multiple=True):
        raise ExceptionGroup("", [TypeError(), TypeError()])

    mocked_fail.assert_not_called()


def test_fails_if_group_has_unexpected_error():
    with pytest.raises(ExceptionGroup, match="unexpected exceptions"):
        with warnings.catch_warnings(record=True) as w:
            with raises_in_group(ValueError):
                raise ExceptionGroup("", [TypeError(), ValueError()])

            assert len(w) == 1
            warning = w[0]
            assert "Unexpected exceptions" in warning
            assert "ExceptionGroup('', [TypeError()])" in warning


def test_throws_pytest_api_changed_error(mocker: MockerFixture):
    with pytest.raises(PytestInternalAPIChangedError, match="github.com/xoeye/typecats/issues"):
        with raises_in_group(TypeError) as e:
            mocked_init = mocker.patch.object(e, "__init__")
            mocked_init.side_effect = RuntimeError("Pytest api changed")
            raise ExceptionGroup("", [TypeError()])

        mocked_init.assert_called_once()


def test_correctly_populates_forward_exception_reference(mocked_fail):
    with raises_in_group(TypeError) as e:
        raise ExceptionGroup("", [TypeError(1)])

    assert isinstance(e.value, TypeError)
    assert e.value.args[0] == 1


def test_correctly_populates_reference_with_first_instance_in_group(mocked_fail):
    with raises_in_group(TypeError, allow_multiple=True) as e:
        raise ExceptionGroup("", [TypeError(1), TypeError(2), ExceptionGroup("", [TypeError(3)])])

    assert isinstance(e.value, TypeError)
    assert e.value.args[0] == 1


def test_match_expr(mocked_fail):
    with raises_in_group(TypeError, "something happened!"):
        raise ExceptionGroup("", [TypeError("something happened!")])

    with raises_in_group(TypeError, "something happened!", allow_multiple=True):
        raise ExceptionGroup("", [TypeError("something happened!"), TypeError(2)])

    with pytest.raises(AssertionError):
        with raises_in_group(TypeError, "something happened!", allow_multiple=True):
            raise ExceptionGroup("", [TypeError("something else happened!"), TypeError(2)])