diff --git a/changelog/8375.bugfix.rst b/changelog/8375.bugfix.rst new file mode 100644 index 00000000000..1415562ba0a --- /dev/null +++ b/changelog/8375.bugfix.rst @@ -0,0 +1 @@ +Session fixture teardown errors now surface as errors instead of being converted to expected failures when the last test is marked ``xfail``. Other ``xfail`` behavior is unchanged. diff --git a/src/_pytest/runner.py b/src/_pytest/runner.py index 27c5739845a..5478c747fdd 100644 --- a/src/_pytest/runner.py +++ b/src/_pytest/runner.py @@ -35,6 +35,7 @@ from _pytest.outcomes import OutcomeException from _pytest.outcomes import Skipped from _pytest.outcomes import TEST_OUTCOME +from _pytest.stash import StashKey if sys.version_info < (3, 11): @@ -436,6 +437,24 @@ def collect() -> list[Item | Collector]: return rep +# Set when session teardown raises; consumed once by the teardown report (#8375). +session_teardown_error_key = StashKey[bool]() + + +def consume_session_teardown_error(item: Item, call: CallInfo[None]) -> bool: + """Consume a recorded session teardown error for a teardown report. + + Returns True once if session teardown raised, else False (#8375). + """ + if call.when != "teardown": + return False + stash = item.session.stash + if session_teardown_error_key not in stash: + return False + del stash[session_teardown_error_key] + return True + + class SetupState: """Shared state for setting up/tearing down test items or collectors in a session. @@ -575,6 +594,9 @@ def teardown_exact(self, nextitem: Item | None) -> None: msg = f"errors while tearing down {node!r}" exceptions.append(BaseExceptionGroup(msg, these_exceptions[::-1])) + if node.parent is None and these_exceptions: + node.session.stash[session_teardown_error_key] = True + if len(exceptions) == 1: raise exceptions[0] elif exceptions: diff --git a/src/_pytest/skipping.py b/src/_pytest/skipping.py index aa97e3b7dc6..6965186cea6 100644 --- a/src/_pytest/skipping.py +++ b/src/_pytest/skipping.py @@ -23,6 +23,7 @@ from _pytest.reports import BaseReport from _pytest.reports import TestReport from _pytest.runner import CallInfo +from _pytest.runner import consume_session_teardown_error from _pytest.stash import StashKey @@ -279,6 +280,7 @@ def pytest_runtest_makereport( ) -> Generator[None, TestReport, TestReport]: rep = yield xfailed = item.stash.get(xfailed_key, None) + session_teardown_error = consume_session_teardown_error(item, call) if item.config.option.runxfail: pass # don't interfere elif call.excinfo and isinstance(call.excinfo.value, xfail.Exception): @@ -286,7 +288,7 @@ def pytest_runtest_makereport( rep.wasxfail = call.excinfo.value.msg rep.outcome = "skipped" elif not rep.skipped and xfailed: - if call.excinfo: + if call.excinfo and not session_teardown_error: raises = xfailed.raises if raises is None or ( ( diff --git a/testing/test_runner.py b/testing/test_runner.py index b5c3839c79e..980d8838f90 100644 --- a/testing/test_runner.py +++ b/testing/test_runner.py @@ -22,6 +22,7 @@ if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup from exceptiongroup import ExceptionGroup @@ -140,6 +141,46 @@ def raiser(exc): assert isinstance(func.exceptions[0], TypeError) assert isinstance(func.exceptions[1], ValueError) + def test_teardown_exact_records_session_errors(self, pytester) -> None: + """Session teardown errors are recorded out-of-band, others are not (#8375).""" + + def raiser(exc): + raise exc + + item = pytester.getitem("def test_func(): pass") + ss = item.session._setupstate + ss.setup(item) + ss.addfinalizer(partial(raiser, RuntimeError("session")), item.session) + ss.addfinalizer(partial(raiser, ValueError("item")), item) + with pytest.raises(BaseExceptionGroup, match="errors during test teardown"): + ss.teardown_exact(None) + key = runner.session_teardown_error_key + assert item.session.stash.get(key, False) is True + + def test_teardown_exact_no_flag_for_item_errors(self, pytester) -> None: + """Item-only teardown errors leave no session attribution (#8375).""" + + def raiser(exc): + raise exc + + item = pytester.getitem("def test_func(): pass") + ss = item.session._setupstate + ss.setup(item) + ss.addfinalizer(partial(raiser, ValueError("item")), item) + with pytest.raises(ValueError, match="item"): + ss.teardown_exact(None) + assert runner.session_teardown_error_key not in item.session.stash + + def test_consume_session_teardown_error_one_shot(self, pytester) -> None: + """Consuming the session flag deletes it; second consume is False (#8375).""" + item = pytester.getitem("def test_func(): pass") + call = runner.CallInfo.from_call(lambda: None, when="teardown") + key = runner.session_teardown_error_key + assert runner.consume_session_teardown_error(item, call) is False + item.session.stash[key] = True + assert runner.consume_session_teardown_error(item, call) is True + assert runner.consume_session_teardown_error(item, call) is False + def test_cached_exception_doesnt_get_longer(self, pytester: Pytester) -> None: """Regression test for #12204 (the "BTW" case).""" pytester.makepyfile(test="") diff --git a/testing/test_skipping.py b/testing/test_skipping.py index fbe196915e8..b36902152a7 100644 --- a/testing/test_skipping.py +++ b/testing/test_skipping.py @@ -843,6 +843,106 @@ def test_func(my_fix): ] ) + def test_xfail_session_teardown_error_is_error(self, pytester: Pytester) -> None: + """Session teardown errors are errors even for xfail tests (#8375).""" + pytester.makepyfile( + test_case=""" + import pytest + + @pytest.fixture(autouse=True, scope="session") + def failme(): + yield + raise RuntimeError("cleanup fails for some reason") + + def test_ok(): + assert 1 == 1 + + @pytest.mark.xfail() + def test_xfail(): + assert 0 == 1 + """ + ) + result = pytester.runpytest() + result.assert_outcomes(passed=1, xfailed=1, errors=1) + result.stdout.fnmatch_lines( + [ + "*ERROR at teardown of test_xfail*", + "*RuntimeError: cleanup fails for some reason*", + ] + ) + + def test_xfail_non_session_teardown_stays_xfail(self, pytester: Pytester) -> None: + """Non-session teardown failures on xfail tests stay XFAIL (#8375).""" + pytester.makepyfile( + test_case=""" + import pytest + + @pytest.fixture() + def fail_func(): + yield + raise RuntimeError("func teardown fails") + + @pytest.fixture(scope="module") + def fail_mod(): + yield + raise RuntimeError("module teardown fails") + + class TestClass: + @pytest.fixture(scope="class", autouse=True) + @classmethod + def fail_cls(cls): + yield + raise RuntimeError("class teardown fails") + + @pytest.mark.xfail() + def test_cls(self): + assert 0 == 1 + + @pytest.mark.xfail() + def test_func(fail_func): + assert 0 == 1 + + @pytest.mark.xfail() + def test_mod(fail_mod): + assert 0 == 1 + """ + ) + result = pytester.runpytest() + result.assert_outcomes(xfailed=6) + + def test_xfail_mixed_session_and_item_teardown_stays_error( + self, pytester: Pytester + ) -> None: + """A session error grouped with an item error is still an error (#8375).""" + pytester.makepyfile( + test_case=""" + import pytest + + @pytest.fixture(autouse=True, scope="session") + def fail_session(): + yield + raise RuntimeError("session teardown fails") + + @pytest.fixture() + def fail_item(): + yield + raise RuntimeError("item teardown fails") + + @pytest.mark.xfail() + def test_mixed(fail_item): + assert 0 == 1 + """ + ) + result = pytester.runpytest() + result.assert_outcomes(xfailed=1, errors=1) + result.stdout.fnmatch_lines( + [ + "*ERROR at teardown of test_mixed*", + "*RuntimeError: session teardown fails*", + "*RuntimeError: item teardown fails*", + ] + ) + class TestSkip: def test_skip_class(self, pytester: Pytester) -> None: