Pytest --fail-unstable using unstable_fail fixture

conftest.py

import warnings
from typing import List, Callable

import pytest


def pytest_addoption(parser: pytest.Parser) -> None:
    parser.addoption("--fail-unstable", action="store_true")


def pytest_configure(config: pytest.Config) -> None:
    config.addinivalue_line("markers", "unstable: mark test as unstable")


def pytest_collection_modifyitems(
    config: pytest.Config, items: List[pytest.Item]
) -> None:
    for item in items:
        if "unstable_fail" in item.fixturenames:
            item.add_marker("unstable")


@pytest.fixture(scope="session")
def fail_unstable_tests(pytestconfig: pytest.Config) -> bool:
    return pytestconfig.getoption("--fail-unstable")


@pytest.fixture
def unstable_fail(
    request: pytest.FixtureRequest, fail_unstable_tests: bool
) -> Callable[[str], None]:
    def fail(msg: str) -> None:
        if fail_unstable_tests:
            pytest.fail(msg)
        else:
            warnings.warn(msg)

    return fail

test.py

# Invoke with:
#  - All tests: 'pytest' (outputs warning)
#  - Unstable tests: 'pytest -m unstable' (outputs warning)
#  - Make warnings failures: 'pytest --fail-unstable' (fails)
#  - Make warnings failures but only run stable: 'pytest --fail-unstable -m "not unstable"' (passes)


def test_foo(unstable_fail):
    unstable_fail("error!")

def test_bar():
    pass