test_1.py

# test_1.py
def test_w_number():
    """
    Correctly Passes

    Examples
    --------
    >>> 1.  # doctest: +NUMBER
    0.
    """

def test_w_off_by_2_number():
    """
    Correctly Fails

    Examples
    --------
    >>> 0.  # doctest: +NUMBER
    2.
    """

def test_wo_number():
    """
    Incorrectly Passes

    Examples
    --------
    >>> 1.
    0.
    """

test_1_result.txt

============================= test session starts ==============================
platform linux -- Python 3.9.2, pytest-7.1.2, pluggy-1.0.0 -- /home/matt/pytest_issue/venv/bin/python3.9
cachedir: .pytest_cache
rootdir: /home/matt/pytest_issue
collecting ... collected 6 items

test_1.py::test_1.test_w_number PASSED                                   [ 16%]
test_1.py::test_1.test_w_off_by_2_number FAILED                          [ 33%]
test_1.py::test_1.test_wo_number PASSED                                  [ 50%]
test_1.py::test_w_number PASSED                                          [ 66%]
test_1.py::test_w_off_by_2_number PASSED                                 [ 83%]
test_1.py::test_wo_number PASSED                                         [100%]

=================================== FAILURES ===================================
___________________ [doctest] test_1.test_w_off_by_2_number ____________________
013 
014     Correctly Fails
015 
016     Examples
017     --------
018     >>> 0.  # doctest: +NUMBER
Expected:
    2.
Got:
    0.0

/home/matt/pytest_issue/test_1.py:18: DocTestFailure
=========================== short test summary info ============================
FAILED test_1.py::test_1.test_w_off_by_2_number
========================= 1 failed, 5 passed in 0.02s ==========================

test_2.py

# test_2.py
def test_w_off_by_2_number():
    """
    Correctly Fails

    Examples
    --------
    >>> 0.  # doctest: +NUMBER
    2.
    """


def test_wo_number():
    """
    Incorrectly Passes

    Examples
    --------
    >>> 1.
    0.
    """

test_2_result.txt

============================= test session starts ==============================
platform linux -- Python 3.9.2, pytest-7.1.2, pluggy-1.0.0 -- /home/matt/pytest_issue/venv/bin/python3.9
cachedir: .pytest_cache
rootdir: /home/matt/pytest_issue
collecting ... collected 4 items

test_2.py::test_2.test_w_off_by_2_number FAILED                          [ 25%]
test_2.py::test_2.test_wo_number PASSED                                  [ 50%]
test_2.py::test_w_off_by_2_number PASSED                                 [ 75%]
test_2.py::test_wo_number PASSED                                         [100%]

=================================== FAILURES ===================================
___________________ [doctest] test_2.test_w_off_by_2_number ____________________
003 
004     Correctly Fails
005 
006     Examples
007     --------
008     >>> 0.  # doctest: +NUMBER
Expected:
    2.
Got:
    0.0

/home/matt/pytest_issue/test_2.py:8: DocTestFailure
=========================== short test summary info ============================
FAILED test_2.py::test_2.test_w_off_by_2_number
========================= 1 failed, 3 passed in 0.01s ==========================

test_3.py

# test_3.py
def test_w_number():
    """
    Correctly Passes

    Examples
    --------
    >>> 1.  # doctest: +NUMBER
    0.
    """

def test_wo_number():
    """
    Correctly Fails

    Examples
    --------
    >>> 1.
    0.
    """

test_3_result.txt

============================= test session starts ==============================
platform linux -- Python 3.9.2, pytest-7.1.2, pluggy-1.0.0 -- /home/matt/pytest_issue/venv/bin/python3.9
cachedir: .pytest_cache
rootdir: /home/matt/pytest_issue
collecting ... collected 4 items

test_3.py::test_3.test_w_number PASSED                                   [ 25%]
test_3.py::test_3.test_wo_number FAILED                                  [ 50%]
test_3.py::test_w_number PASSED                                          [ 75%]
test_3.py::test_wo_number PASSED                                         [100%]

=================================== FAILURES ===================================
_______________________ [doctest] test_3.test_wo_number ________________________
013 
014     Correctly Fails
015 
016     Examples
017     --------
018     >>> 1.
Expected:
    0.
Got:
    1.0

/home/matt/pytest_issue/test_3.py:18: DocTestFailure
=========================== short test summary info ============================
FAILED test_3.py::test_3.test_wo_number
========================= 1 failed, 3 passed in 0.01s ==========================