perf: reduce fixture dependency collection overhead

AUTHORS

@@ -210,6 +210,7 @@ Harshna
 Henk-Jaap Wagenaar
 Henry Schreiner
 Holger Kohr
+host452b
 Hugo van Kemenade
 Hui Wang (coldnight)
 Ian Bicking

bench/unittest_methods.py

@@ -0,0 +1,25 @@
+"""Collect many generated methods sharing the same unittest class fixtures.
+
+Run with ``pytest --collect-only bench/unittest_methods.py``.
+"""
+
+from __future__ import annotations
+
+from unittest import TestCase
+
+
+class TestManyMethods(TestCase):
+    @classmethod
+    def setUpClass(cls):
+        pass
+
+
+def test_method(self):
+    pass
+
+
+for i in range(35000):
+    setattr(TestManyMethods, f"test_{i:05d}", test_method)
+
+# Only collect the bound methods, not the helper function itself.
+del test_method

changelog/15060.improvement.rst

@@ -0,0 +1 @@
+Improved collection performance for large test suites, particularly those with many unittest methods, and avoided recursion errors while collecting deeply nested fixture dependencies.

src/_pytest/fixtures.py

@@ -416,35 +416,44 @@ def traverse_fixture_closure(
     # last, etc.
     current_indices: dict[str, int] = {}
 
-    def process_argname(argname: str) -> Iterator[str]:
-        index = current_indices.get(argname)
-
-        # Optimization: already processed this argname.
-        if index == -1:
-            return
+    # Save the iterator and override index when descending into dependencies.
+    # An explicit stack avoids a recursive generator for each dependency and
+    # the reference cycle created by a recursive nested function.
+    stack: list[tuple[str, int, Iterator[str]]] = []
+    pending = iter(initialnames)
+    while True:
+        for argname in pending:
+            index = current_indices.get(argname)
+
+            # Already processed this argname outside the active override chain.
+            if index == -1:
+                continue
 
-        # Only yield each argname once.
-        if index is None:
-            yield argname
-            current_indices[argname] = -1
+            if index is None:
+                yield argname
+                current_indices[argname] = -1
 
-        fixturedefs = getfixturedefs(argname)
-        if not fixturedefs:
-            return
-
-        index = current_indices.get(argname, -1)
-        if -index > len(fixturedefs):
-            # Exhausted the override chain (will error during runtest).
-            return
-        fixturedef = fixturedefs[index]
+            fixturedefs = getfixturedefs(argname)
+            if not fixturedefs:
+                continue
 
-        current_indices[argname] = index - 1
-        for dep in fixturedef.argnames:
-            yield from process_argname(dep)
-        current_indices[argname] = index
+            index = current_indices.get(argname, -1)
+            if -index > len(fixturedefs):
+                # Exhausted the override chain (will error during runtest).
+                continue
+            fixturedef = fixturedefs[index]
+            if not fixturedef.argnames:
+                continue
 
-    for argname in initialnames:
-        yield from process_argname(argname)
+            current_indices[argname] = index - 1
+            stack.append((argname, index, pending))
+            pending = iter(fixturedef.argnames)
+            break
+        else:
+            if not stack:
+                return
+            argname, index, pending = stack.pop()
+            current_indices[argname] = index
 
 
 @dataclasses.dataclass(frozen=True)
@@ -1943,9 +1952,10 @@ def _getautousenames(self, node: nodes.Node) -> Iterator[str]:
             if basenames:
                 yield from basenames
             # Legacy fallback: check string-based nodeid autouse names.
-            nodeid_basenames = self._nodeid_autousenames.get(parentnode.nodeid)
-            if nodeid_basenames:
-                yield from nodeid_basenames
+            if self._nodeid_autousenames:
+                nodeid_basenames = self._nodeid_autousenames.get(parentnode.nodeid)
+                if nodeid_basenames:
+                    yield from nodeid_basenames
 
     def _getusefixturesnames(self, node: nodes.Item) -> Iterator[str]:
         """Return the names of usefixtures fixtures visible to node."""
@@ -2387,14 +2397,23 @@ def getfixturedefs(
         nodeid2fixturedefs = self._arg2nodeid2fixturedefs.get(argname, {})
         if not node2fixturedefs and not nodeid2fixturedefs:
             return None
-        fixturedefs = [
-            fixturedef
-            for parent in node.iter_parents()
-            for fixturedef in [
-                *node2fixturedefs.get(parent, ()),
-                *nodeid2fixturedefs.get(parent.nodeid, ()),
+        if not nodeid2fixturedefs:
+            # Avoid string nodeid lookups and intermediate lists when no plugin
+            # has registered fixtures through the legacy nodeid-based API.
+            fixturedefs = [
+                fixturedef
+                for parent in node.iter_parents()
+                for fixturedef in node2fixturedefs.get(parent, ())
+            ]
+        else:
+            fixturedefs = [
+                fixturedef
+                for parent in node.iter_parents()
+                for fixturedef in [
+                    *node2fixturedefs.get(parent, ()),
+                    *nodeid2fixturedefs.get(parent.nodeid, ()),
+                ]
             ]
-        ]
         fixturedefs.reverse()
         return fixturedefs
 

testing/python/fixtures.py

@@ -5894,6 +5894,26 @@ def test_something(self, request, app):
     result.assert_outcomes(passed=1)
 
 
+@pytest.mark.parametrize("parametrize", [False, True])
+def test_fixture_closure_handles_deep_dependencies(
+    pytester: Pytester, parametrize: bool
+) -> None:
+    """Collection must not use a Python frame for every fixture dependency."""
+    depth = sys.getrecursionlimit() + 50
+    source = ["import pytest", "@pytest.fixture", "def fix0(): pass"]
+    for index in range(1, depth):
+        source.extend(["@pytest.fixture", f"def fix{index}(fix{index - 1}): pass"])
+    if parametrize:
+        source.append('@pytest.mark.parametrize("fix0", [0])')
+    source.append(f"def test_deep(fix{depth - 1}): pass")
+    pytester.makepyfile("\n".join(source))
+
+    items, _hookrec = pytester.inline_genitems()
+    assert len(items) == 1
+    assert isinstance(items[0], Function)
+    assert items[0].fixturenames == [f"fix{i}" for i in reversed(range(depth))]
+
+
 def test_fixture_closure_handles_circular_dependencies(pytester: Pytester) -> None:
     """Test that getfixtureclosure properly handles circular dependencies.