From f5249b7239d65a7c602c4c1059bd8ceed25868ad Mon Sep 17 00:00:00 2001 From: host452b <32806348+host452b@users.noreply.github.com> Date: Fri, 18 Sep 2026 07:08:03 +0000 Subject: [PATCH 1/2] perf: reduce fixture dependency collection overhead Use an explicit DFS stack for fixture closures and avoid legacy nodeid lookups when fixtures use node-based registration. Preserve override ordering and add deep-closure collection tests and a unittest benchmark. Co-authored-by: OpenAI Codex --- AUTHORS | 1 + bench/unittest_methods.py | 25 +++++++++++ src/_pytest/fixtures.py | 89 +++++++++++++++++++++++--------------- testing/python/fixtures.py | 20 +++++++++ 4 files changed, 100 insertions(+), 35 deletions(-) create mode 100644 bench/unittest_methods.py diff --git a/AUTHORS b/AUTHORS index e2fad5e8364..5b31a5653b3 100644 --- a/AUTHORS +++ b/AUTHORS @@ -210,6 +210,7 @@ Harshna Henk-Jaap Wagenaar Henry Schreiner Holger Kohr +host452b Hugo van Kemenade Hui Wang (coldnight) Ian Bicking diff --git a/bench/unittest_methods.py b/bench/unittest_methods.py new file mode 100644 index 00000000000..a0bab6f498a --- /dev/null +++ b/bench/unittest_methods.py @@ -0,0 +1,25 @@ +"""Collect many generated methods sharing the same unittest class fixtures. + +Run with ``pytest --collect-only bench/unittest_methods.py``. +""" + +from __future__ import annotations + +from unittest import TestCase + + +class TestManyMethods(TestCase): + @classmethod + def setUpClass(cls): + pass + + +def test_method(self): + pass + + +for i in range(35000): + setattr(TestManyMethods, f"test_{i:05d}", test_method) + +# Only collect the bound methods, not the helper function itself. +del test_method diff --git a/src/_pytest/fixtures.py b/src/_pytest/fixtures.py index 6f5d7082311..d39ba9701f9 100644 --- a/src/_pytest/fixtures.py +++ b/src/_pytest/fixtures.py @@ -416,35 +416,44 @@ def traverse_fixture_closure( # last, etc. current_indices: dict[str, int] = {} - def process_argname(argname: str) -> Iterator[str]: - index = current_indices.get(argname) - - # Optimization: already processed this argname. - if index == -1: - return + # Save the iterator and override index when descending into dependencies. + # An explicit stack avoids a recursive generator for each dependency and + # the reference cycle created by a recursive nested function. + stack: list[tuple[str, int, Iterator[str]]] = [] + pending = iter(initialnames) + while True: + for argname in pending: + index = current_indices.get(argname) + + # Already processed this argname outside the active override chain. + if index == -1: + continue - # Only yield each argname once. - if index is None: - yield argname - current_indices[argname] = -1 + if index is None: + yield argname + current_indices[argname] = -1 - fixturedefs = getfixturedefs(argname) - if not fixturedefs: - return - - index = current_indices.get(argname, -1) - if -index > len(fixturedefs): - # Exhausted the override chain (will error during runtest). - return - fixturedef = fixturedefs[index] + fixturedefs = getfixturedefs(argname) + if not fixturedefs: + continue - current_indices[argname] = index - 1 - for dep in fixturedef.argnames: - yield from process_argname(dep) - current_indices[argname] = index + index = current_indices.get(argname, -1) + if -index > len(fixturedefs): + # Exhausted the override chain (will error during runtest). + continue + fixturedef = fixturedefs[index] + if not fixturedef.argnames: + continue - for argname in initialnames: - yield from process_argname(argname) + current_indices[argname] = index - 1 + stack.append((argname, index, pending)) + pending = iter(fixturedef.argnames) + break + else: + if not stack: + return + argname, index, pending = stack.pop() + current_indices[argname] = index @dataclasses.dataclass(frozen=True) @@ -1943,9 +1952,10 @@ def _getautousenames(self, node: nodes.Node) -> Iterator[str]: if basenames: yield from basenames # Legacy fallback: check string-based nodeid autouse names. - nodeid_basenames = self._nodeid_autousenames.get(parentnode.nodeid) - if nodeid_basenames: - yield from nodeid_basenames + if self._nodeid_autousenames: + nodeid_basenames = self._nodeid_autousenames.get(parentnode.nodeid) + if nodeid_basenames: + yield from nodeid_basenames def _getusefixturesnames(self, node: nodes.Item) -> Iterator[str]: """Return the names of usefixtures fixtures visible to node.""" @@ -2387,14 +2397,23 @@ def getfixturedefs( nodeid2fixturedefs = self._arg2nodeid2fixturedefs.get(argname, {}) if not node2fixturedefs and not nodeid2fixturedefs: return None - fixturedefs = [ - fixturedef - for parent in node.iter_parents() - for fixturedef in [ - *node2fixturedefs.get(parent, ()), - *nodeid2fixturedefs.get(parent.nodeid, ()), + if not nodeid2fixturedefs: + # Avoid string nodeid lookups and intermediate lists when no plugin + # has registered fixtures through the legacy nodeid-based API. + fixturedefs = [ + fixturedef + for parent in node.iter_parents() + for fixturedef in node2fixturedefs.get(parent, ()) + ] + else: + fixturedefs = [ + fixturedef + for parent in node.iter_parents() + for fixturedef in [ + *node2fixturedefs.get(parent, ()), + *nodeid2fixturedefs.get(parent.nodeid, ()), + ] ] - ] fixturedefs.reverse() return fixturedefs diff --git a/testing/python/fixtures.py b/testing/python/fixtures.py index bc7b5a40cc2..6fff76aab11 100644 --- a/testing/python/fixtures.py +++ b/testing/python/fixtures.py @@ -5894,6 +5894,26 @@ def test_something(self, request, app): result.assert_outcomes(passed=1) +@pytest.mark.parametrize("parametrize", [False, True]) +def test_fixture_closure_handles_deep_dependencies( + pytester: Pytester, parametrize: bool +) -> None: + """Collection must not use a Python frame for every fixture dependency.""" + depth = sys.getrecursionlimit() + 50 + source = ["import pytest", "@pytest.fixture", "def fix0(): pass"] + for index in range(1, depth): + source.extend(["@pytest.fixture", f"def fix{index}(fix{index - 1}): pass"]) + if parametrize: + source.append('@pytest.mark.parametrize("fix0", [0])') + source.append(f"def test_deep(fix{depth - 1}): pass") + pytester.makepyfile("\n".join(source)) + + items, _hookrec = pytester.inline_genitems() + assert len(items) == 1 + assert isinstance(items[0], Function) + assert items[0].fixturenames == [f"fix{i}" for i in reversed(range(depth))] + + def test_fixture_closure_handles_circular_dependencies(pytester: Pytester) -> None: """Test that getfixtureclosure properly handles circular dependencies. From 65dae4350aae07a5269929cdb2e909571abb93b2 Mon Sep 17 00:00:00 2001 From: host452b <32806348+host452b@users.noreply.github.com> Date: Fri, 18 Sep 2026 07:08:37 +0000 Subject: [PATCH 2/2] docs: add collection performance changelog for #15060 Co-authored-by: OpenAI Codex --- changelog/15060.improvement.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/15060.improvement.rst diff --git a/changelog/15060.improvement.rst b/changelog/15060.improvement.rst new file mode 100644 index 00000000000..d09eb10613e --- /dev/null +++ b/changelog/15060.improvement.rst @@ -0,0 +1 @@ +Improved collection performance for large test suites, particularly those with many unittest methods, and avoided recursion errors while collecting deeply nested fixture dependencies.