python: warn when an abstract test class swallows its tests

changelog/13546.improvement.rst

@@ -0,0 +1,7 @@
+pytest now warns with :class:`~pytest.PytestCollectionWarning` when a test class is skipped
+because it is abstract, but looks like it was meant to be collected -- it inherits abstract
+methods it does not implement and does not declare any of its own, yet carries tests.
+
+Such a class is most often a test class whose base class grew a new abstract method, which
+silently stops its tests from running. Classes that declare abstract methods of their own are
+still skipped silently, as are classes that set ``__test__ = False`` in their body.

src/_pytest/python.py

@@ -227,6 +227,52 @@ def pytest_collect_file(file_path: Path, parent: nodes.Collector) -> Module | No
     return None
 
 
+def _warn_if_abstract_tests_go_dark(
+    collector: Module | Class, name: str, obj: object
+) -> None:
+    """Warn if an abstract class silently swallows tests that look collectible.
+
+    Abstract classes are not collected (#12275), which is right for a base class
+    written to be subclassed, but indistinguishable from a leaf class that failed
+    to implement a blank -- for instance because a base class grew a new abstract
+    method. In that case the tests stop running and nothing says so (#13546).
+
+    A class that declares abstract methods of its own is taken to mean it; one
+    that only inherits unimplemented ones, yet carries tests, is warned about.
+    ``__test__`` in the class body opts out, deliberately read off the class
+    itself, since an inherited ``__test__`` would silence every subclass too.
+    """
+    assert isinstance(obj, type)
+    missing: frozenset[str] = getattr(obj, "__abstractmethods__", frozenset())
+    if not missing or "__test__" in obj.__dict__:
+        return
+    if any(attr in obj.__dict__ for attr in missing):
+        # Declares blanks of its own, so it is meant as a base class.
+        return
+    if not any(
+        collector.istestfunction(value, attr)
+        for klass in obj.__mro__
+        for attr, value in klass.__dict__.items()
+    ):
+        return
+
+    def declared_in(attr: str) -> str:
+        return next(k.__name__ for k in obj.__mro__ if attr in k.__dict__)
+
+    unimplemented = ", ".join(
+        f"{attr!r} (declared abstract in {declared_in(attr)!r})"
+        for attr in sorted(missing)
+    )
+    collector.warn(
+        PytestCollectionWarning(
+            f"cannot collect test class {name!r} because it is abstract: "
+            f"it does not implement {unimplemented}. Set '__test__ = False' in "
+            f"its body if it is meant to be a base class "
+            f"(from: {collector.nodeid})"
+        )
+    )
+
+
 def path_matches_patterns(path: Path, patterns: Iterable[str]) -> bool:
     """Return whether path matches any of the patterns in the list of globs given."""
     return any(fnmatch_ex(pattern, path) for pattern in patterns)
@@ -245,6 +291,8 @@ def pytest_pycollect_makeitem(
     if safe_isclass(obj):
         if collector.istestclass(obj, name):
             return Class.from_parent(collector, name=name, obj=obj)
+        if collector.classnamefilter(name) or collector.isnosetest(obj):
+            _warn_if_abstract_tests_go_dark(collector, name, obj)
     elif collector.istestfunction(obj, name):
         # mock seems to store unbound methods (issue473), normalize it.
         obj = getattr(obj, "__func__", obj)

src/_pytest/unittest.py

@@ -31,6 +31,7 @@
 from _pytest.outcomes import fail
 from _pytest.outcomes import skip
 from _pytest.outcomes import xfail
+from _pytest.python import _warn_if_abstract_tests_go_dark
 from _pytest.python import Class
 from _pytest.python import Function
 from _pytest.python import Module
@@ -58,7 +59,7 @@
 
 def pytest_pycollect_makeitem(
     collector: Module | Class, name: str, obj: object
-) -> UnitTestCase | None:
+) -> UnitTestCase | list[Item | Collector] | None:
     try:
         # Has unittest been imported?
         ut = sys.modules["unittest"]
@@ -71,7 +72,10 @@ def pytest_pycollect_makeitem(
     # Is obj a concrete class?
     # Abstract classes can't be instantiated so no point collecting them.
     if inspect.isabstract(obj):
-        return None
+        # Return a result rather than None so that the python plugin does not
+        # warn a second time about a TestCase subclass named like a test class.
+        _warn_if_abstract_tests_go_dark(collector, name, obj)
+        return []
     # Yes, so let's collect it.
     return UnitTestCase.from_parent(collector, name=name, obj=obj)
 

testing/python/collect.py

@@ -352,10 +352,79 @@ class TestConcrete(TestPartial):
                 def abstract2(self): pass
             """
         )
-        result = pytester.runpytest()
+        result = pytester.runpytest("-Wignore::pytest.PytestCollectionWarning")
         assert result.ret == ExitCode.OK
         result.assert_outcomes(passed=1)
 
+    def test_abstract_class_missing_implementation_warns(
+        self, pytester: Pytester
+    ) -> None:
+        """A class that inherits tests but no longer instantiates says so (#13546).
+
+        ``TestForgot`` looks like a leaf test class and carries an inherited
+        test, but is still abstract, so its tests never run.
+        """
+        pytester.makepyfile(
+            """
+            import abc
+
+            class TestBase(abc.ABC):
+                @abc.abstractmethod
+                def impl(self): pass
+
+                def test_it(self):
+                    assert self.impl() == 1
+
+            class TestForgot(TestBase):
+                pass
+
+            class TestOk(TestBase):
+                def impl(self):
+                    return 1
+            """
+        )
+        result = pytester.runpytest("-Wdefault::pytest.PytestCollectionWarning")
+        result.assert_outcomes(passed=1, warnings=1)
+        result.stdout.fnmatch_lines(
+            [
+                "*PytestCollectionWarning: cannot collect test class 'TestForgot'"
+                " because it is abstract: it does not implement 'impl'"
+                " (declared abstract in 'TestBase')*"
+            ]
+        )
+
+    def test_abstract_base_class_is_not_warned_about(self, pytester: Pytester) -> None:
+        """A class declaring blanks of its own means to be a base class (#13546).
+
+        ``__test__`` in the class body opts a class out of the warning even when
+        it declares no blanks of its own.
+        """
+        pytester.makepyfile(
+            """
+            import abc
+
+            class TestBase(abc.ABC):
+                @abc.abstractmethod
+                def impl(self): pass
+
+                def test_it(self):
+                    assert self.impl() == 1
+
+            class TestStillAbstract(TestBase):
+                __test__ = False
+
+            class TestNoTests(abc.ABC):
+                @abc.abstractmethod
+                def impl(self): pass
+
+            class TestOk(TestBase):
+                def impl(self):
+                    return 1
+            """
+        )
+        result = pytester.runpytest("-Wdefault::pytest.PytestCollectionWarning")
+        result.assert_outcomes(passed=1, warnings=0)
+
 
 class TestFunction:
     def test_getmodulecollector(self, pytester: Pytester) -> None:

testing/test_unittest.py

@@ -1738,6 +1738,42 @@ class TestConcrete(TestPartial):
             def abstract2(self): pass
         """
     )
-    result = pytester.runpytest()
+    result = pytester.runpytest("-Wignore::pytest.PytestCollectionWarning")
     assert result.ret == ExitCode.OK
     result.assert_outcomes(passed=1)
+
+
+def test_abstract_testcase_missing_implementation_warns(pytester: Pytester) -> None:
+    """A TestCase that inherits tests but no longer instantiates says so (#13546).
+
+    The class name deliberately does not match ``python_classes``: unittest
+    classes are collected regardless of it.
+    """
+    pytester.makepyfile(
+        """
+        import abc
+        import unittest
+
+        class BaseTestCase(unittest.TestCase, abc.ABC):
+            @abc.abstractmethod
+            def get_application(self): pass
+
+            def test_it(self):
+                assert self.get_application() == 1
+
+        class InvalidTestCase(BaseTestCase):
+            pass
+
+        class ValidTestCase(BaseTestCase):
+            def get_application(self):
+                return 1
+        """
+    )
+    result = pytester.runpytest("-Wdefault::pytest.PytestCollectionWarning")
+    result.assert_outcomes(passed=1, warnings=1)
+    result.stdout.fnmatch_lines(
+        [
+            "*PytestCollectionWarning: cannot collect test class 'InvalidTestCase'"
+            " because it is abstract: it does not implement 'get_application'*"
+        ]
+    )