From 60c4d25e65bad6c3712b037cdc71e2bd44369a58 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Tue, 1 Sep 2026 16:30:09 +0530 Subject: [PATCH] feat(selenium-devtools-py): configure capture from pytest, not just the shell --- .../selenium/python-test/test_login_pytest.py | 2 +- package.json | 2 +- packages/selenium-devtools-py/README.md | 28 +- .../src/selenium_devtools/pytest_plugin.py | 131 +++++++- .../tests/test_pytest_config.py | 290 ++++++++++++++++++ 5 files changed, 444 insertions(+), 9 deletions(-) create mode 100644 packages/selenium-devtools-py/tests/test_pytest_config.py diff --git a/examples/selenium/python-test/test_login_pytest.py b/examples/selenium/python-test/test_login_pytest.py index 79435591..158fb15b 100644 --- a/examples/selenium/python-test/test_login_pytest.py +++ b/examples/selenium/python-test/test_login_pytest.py @@ -16,7 +16,7 @@ Run it (the plugin is inert unless a devtools env var opts the run in): pip install -e packages/selenium-devtools-py - DEVTOOLS_ENABLE=1 python -m pytest examples/selenium/python-test/test_login_pytest.py + python -m pytest --devtools examples/selenium/python-test/test_login_pytest.py The driver fixture is function-scoped, so each test gets its own browser session — which also exercises the adapter's per-driver capture state. diff --git a/package.json b/package.json index d020b2d7..996f9480 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "demo:selenium": "pnpm --filter @wdio/selenium-devtools example", "demo:python": "python3 examples/selenium/python-test/web_form.py", "demo:python:login": "python3 examples/selenium/python-test/login.py", - "demo:python:pytest": "DEVTOOLS_ENABLE=1 python3 -m pytest examples/selenium/python-test/test_login_pytest.py", + "demo:python:pytest": "python3 -m pytest --devtools examples/selenium/python-test/test_login_pytest.py", "dev": "pnpm --parallel dev", "preview": "pnpm --parallel preview", "test": "vitest run", diff --git a/packages/selenium-devtools-py/README.md b/packages/selenium-devtools-py/README.md index 5c540838..23b8e768 100644 --- a/packages/selenium-devtools-py/README.md +++ b/packages/selenium-devtools-py/README.md @@ -45,9 +45,35 @@ adapter says so on the first command instead of degrading quietly. **With pytest (recommended) — no code changes to your tests:** ```bash -DEVTOOLS_ENABLE=1 pytest tests/ # DEVTOOLS_PORT= also opts in (and attaches) +pytest --devtools tests/ # dashboard +pytest --devtools-trace tests/ # trace archive instead (implies --devtools) ``` +Or commit it, so everyone on the project gets it without remembering a flag: + +```toml +[tool.pytest.ini_options] +devtools = true +# devtools_trace = true # trace archive instead of a dashboard +``` + +Capture is always opt-in — the plugin auto-loads when the package is installed, +so installing it must never change how an existing suite behaves. What you +choose is only *how* you say yes: + +| | | +|---|---| +| `--devtools` / `--devtools-trace` | this run | +| `devtools` / `devtools_trace` in `[tool.pytest.ini_options]` | this project | +| `DEVTOOLS_ENABLE=1` (or `DEVTOOLS_PORT=`, which also attaches) | this shell — for CI | + +Highest wins: CLI, then ini, then environment. `pytest -o devtools=false` turns +a project default off for one run, which is why there is no `--no-devtools`. + +`DEVTOOLS_TRACE=1` selects trace mode but does **not** switch capture on by +itself — it is a mode fallback you may have exported for your own scripts, and +reading it as an opt-in would capture pytest runs you never asked for. + The bundled plugin auto-captures the run, opens the dashboard in a dedicated window, and — after the run — **keeps it open so you can inspect it**; close the window (or Ctrl-C) to finish. Nothing devtools-specific goes in your test files. diff --git a/packages/selenium-devtools-py/src/selenium_devtools/pytest_plugin.py b/packages/selenium-devtools-py/src/selenium_devtools/pytest_plugin.py index 2a446f67..aa3d0043 100644 --- a/packages/selenium-devtools-py/src/selenium_devtools/pytest_plugin.py +++ b/packages/selenium-devtools-py/src/selenium_devtools/pytest_plugin.py @@ -1,9 +1,10 @@ """pytest plugin — feeds the suite/test tree to the dashboard. The analogue of the JS adapter's mocha/jest hooks. Inert unless the run opts in -via ``DEVTOOLS_ENABLE=1`` or ``DEVTOOLS_PORT=...`` so installing the package never -hijacks an unrelated pytest run. On opt-in it enables capture at session start, -stamps per-test timing, and re-sends the ``suites`` frame as each test reports. +— ``--devtools``, a ``devtools`` ini option, or ``DEVTOOLS_ENABLE``/``DEVTOOLS_PORT``, +in that precedence — so installing the package never hijacks an unrelated pytest +run. On opt-in it enables capture at session start, stamps per-test timing, and +re-sends the ``suites`` frame as each test reports. """ from __future__ import annotations @@ -25,10 +26,119 @@ _comparisons = assertions.ComparisonBuffer() -def _opted_in() -> bool: +#: Resolved once in `pytest_configure`. Most hooks — `pytest_runtest_logstart`, +#: `pytest_runtest_logreport` — are handed no `config`, so the answer cannot be +#: recomputed where it is needed. +_enabled = False + + +def pytest_addoption(parser) -> None: # noqa: ANN001 + """Register the CLI flags and ini options that switch capture on. + + `default=None` on all four, so "not given" is distinguishable from "given + false" — that is what makes CLI -> ini -> env a real precedence chain rather + than three sources ORed together. + + No `--no-devtools`: pytest's own `-o devtools=false` already overrides an ini + option for one run, and a second spelling of the same thing is a second + thing to keep consistent. + """ + group = parser.getgroup("devtools", "WebdriverIO DevTools") + group.addoption( + "--devtools", + action="store_true", + default=None, + help="Capture this run for the DevTools dashboard.", + ) + group.addoption( + "--devtools-trace", + action="store_true", + default=None, + help="Write a trace archive instead of opening a dashboard. Implies --devtools.", + ) + parser.addini( + "devtools", + "Capture pytest runs for the DevTools dashboard.", + type="bool", + default=None, + ) + parser.addini( + "devtools_trace", + "Write a trace archive instead of opening a dashboard. Implies devtools.", + type="bool", + default=None, + ) + + +def _ini(config, name: str) -> Optional[bool]: # noqa: ANN001 + """An ini option's value, or None when the project did not set it.""" + try: + value = config.getini(name) + except (ValueError, KeyError): # option not registered (another plugin's parser) + return None + return None if value is None or value == "" else bool(value) + + +def _resolve_trace(config) -> Optional[bool]: # noqa: ANN001 + """Whether this run writes a trace archive: CLI, else ini, else undecided. + + None rather than False when nothing said, so `enable()` still reads + DEVTOOLS_TRACE — the env layer lives there and is not duplicated here. + """ + if config.getoption("--devtools-trace", None): + return True + return _ini(config, "devtools_trace") + + +def _resolve_enabled(config) -> bool: # noqa: ANN001 + """Whether to capture at all: CLI, else ini, else the environment. + + Asking for a trace is asking for capture, so `--devtools-trace` and + `devtools_trace` imply it. DEVTOOLS_TRACE deliberately does NOT: it is a + mode fallback that a user may have exported for their own scripts, and + reading it as an opt-in would capture pytest runs they never asked for. + """ + # Nothing runs under --collect-only, so there is nothing to capture: opting + # in would still launch a backend and open a dashboard window, and leave it + # sitting empty for a run that never happened. + if config.getoption("--collect-only", False): + return False + if config.getoption("--devtools", None) or config.getoption( + "--devtools-trace", None + ): + return True + for name in ("devtools", "devtools_trace"): + value = _ini(config, name) + if value is not None: + if value: + return True + # An explicit `devtools = false` is the project's answer; the + # environment does not get to overturn it. + if name == "devtools": + return False return bool(os.environ.get(ENV_OPT_IN) or os.environ.get(ENV_PORT)) +def _opted_in() -> bool: + return _enabled + + +def _teardown_unused_run() -> None: + """Undo an opt-in for a run that turned out to have nothing to capture. + + Closes the dashboard window and drops every later hook out, so the run ends + the way it would have without the flag. `disable()` exports nothing here — + the backend refuses a trace with no commands, console or network. + """ + global _enabled + _enabled = False + _log.info("no tests collected; capture is off for this run") + try: + devtools.disable() + except Exception as exc: # noqa: BLE001 — teardown must never fail a run + _log.debug("could not tear down an unused run: %s", exc) + + def _rolled_up_state(tests: list, suites: list) -> str: """A group is failed if anything under it failed, else running while any child still has work left, else its children's outcome. Mirrors how the JS @@ -247,6 +357,8 @@ def _configure_rerun(config, rootdir: Optional[str]) -> None: # noqa: ANN001 def pytest_configure(config) -> None: # noqa: ANN001 + global _enabled + _enabled = _resolve_enabled(config) if _opted_in(): _enable_assertion_pass_hook(config) global _rootdir @@ -254,7 +366,7 @@ def pytest_configure(config) -> None: # noqa: ANN001 root = getattr(config, "rootpath", None) or getattr(config, "rootdir", None) _rootdir = str(root) if root else None _configure_rerun(config, _rootdir) - capturer = devtools.enable() + capturer = devtools.enable(trace=_resolve_trace(config)) # pytest owns the suite tree — suppress the adapter's default script suite. from . import instrumentation @@ -279,10 +391,17 @@ def pytest_collection_finish(session) -> None: # noqa: ANN001 """ if not _opted_in(): return + items = getattr(session, "items", []) or [] + if not items: + # A run that collected nothing has nothing to show, and `sessionfinish` + # would still park on the dashboard window — so a mistyped path leaves + # the terminal blocked on an empty UI. Collection is the first point + # this is knowable; `configure` opened the window before it. + _teardown_unused_run() + return capturer = devtools.get_capturer() if capturer is None: return - items = getattr(session, "items", []) or [] for item in items: file, line, name = item.location _registry.record( diff --git a/packages/selenium-devtools-py/tests/test_pytest_config.py b/packages/selenium-devtools-py/tests/test_pytest_config.py new file mode 100644 index 00000000..528ec096 --- /dev/null +++ b/packages/selenium-devtools-py/tests/test_pytest_config.py @@ -0,0 +1,290 @@ +"""The pytest plugin's config surface: CLI flags, ini options, env fallback. + +The precedence matrix runs against a stand-in config, because it is pure logic. +What a stand-in cannot prove — that the options register at all, that +`addini(default=None)` really yields None for an unset option, and that pytest's +own `-o` override reaches us — is checked by running real pytest in a subprocess. +""" + +from __future__ import annotations + +import os +import pathlib +import subprocess +import sys +import tempfile +import textwrap +import unittest +from unittest import mock + +from selenium_devtools import pytest_plugin as plugin + +SRC = str(pathlib.Path(__file__).resolve().parents[1] / "src") +_DEVTOOLS_ENV = ("DEVTOOLS_ENABLE", "DEVTOOLS_PORT", "DEVTOOLS_TRACE") + + +class _Config: + """pytest's `config`, reduced to what the resolvers ask of it. + + Both accessors return None for "not given", which is what registering every + option with `default=None` buys — and the whole reason a precedence chain is + expressible rather than three sources ORed together. + """ + + def __init__(self, options=None, ini=None): + self._options = options or {} + self._ini = ini or {} + + def getoption(self, name, default=None): + return self._options.get(name, default) + + def getini(self, name): + return self._ini.get(name) + + +class ResolveEnabledTest(unittest.TestCase): + def setUp(self): + patcher = mock.patch.dict(os.environ, {}, clear=False) + patcher.start() + self.addCleanup(patcher.stop) + for name in _DEVTOOLS_ENV: + os.environ.pop(name, None) + + def test_nothing_anywhere_stays_inert(self): + # Installing the package must never change how an existing suite behaves. + self.assertFalse(plugin._resolve_enabled(_Config())) + + def test_the_cli_flag_enables(self): + self.assertTrue(plugin._resolve_enabled(_Config({"--devtools": True}))) + + def test_the_ini_option_enables(self): + self.assertTrue(plugin._resolve_enabled(_Config(ini={"devtools": True}))) + + def test_the_environment_still_enables(self): + # CI and the pnpm demo scripts use these; removing them breaks setups + # that already exist. + for name in ("DEVTOOLS_ENABLE", "DEVTOOLS_PORT"): + with self.subTest(env=name): + os.environ.pop("DEVTOOLS_ENABLE", None) + os.environ.pop("DEVTOOLS_PORT", None) + os.environ[name] = "1" + self.assertTrue(plugin._resolve_enabled(_Config())) + + def test_asking_for_a_trace_is_asking_for_capture(self): + self.assertTrue(plugin._resolve_enabled(_Config({"--devtools-trace": True}))) + self.assertTrue(plugin._resolve_enabled(_Config(ini={"devtools_trace": True}))) + + def test_the_trace_env_var_alone_does_not_enable(self): + # It is a mode fallback a user may have exported for their own scripts; + # reading it as an opt-in captures pytest runs nobody asked for. + os.environ["DEVTOOLS_TRACE"] = "1" + self.assertFalse(plugin._resolve_enabled(_Config())) + + def test_collect_only_captures_nothing(self): + # Nothing runs, so opting in would launch a backend and leave a dashboard + # window sitting empty for a run that never happened. + for source in ({"--devtools": True}, {"--devtools-trace": True}): + with self.subTest(source=source): + config = _Config({**source, "--collect-only": True}) + self.assertFalse(plugin._resolve_enabled(config)) + os.environ["DEVTOOLS_ENABLE"] = "1" + self.assertFalse( + plugin._resolve_enabled(_Config({"--collect-only": True})) + ) + self.assertFalse( + plugin._resolve_enabled( + _Config({"--collect-only": True}, {"devtools": True}) + ) + ) + + def test_the_cli_beats_an_ini_that_says_no(self): + self.assertTrue( + plugin._resolve_enabled( + _Config({"--devtools": True}, {"devtools": False}) + ) + ) + + def test_an_ini_that_says_no_beats_the_environment(self): + os.environ["DEVTOOLS_ENABLE"] = "1" + self.assertFalse(plugin._resolve_enabled(_Config(ini={"devtools": False}))) + + def test_a_trace_ini_that_says_no_leaves_the_environment_to_answer(self): + # `devtools_trace = false` picks a MODE; it is not a refusal to capture. + os.environ["DEVTOOLS_ENABLE"] = "1" + self.assertTrue( + plugin._resolve_enabled(_Config(ini={"devtools_trace": False})) + ) + + +class ResolveTraceTest(unittest.TestCase): + def test_undecided_is_none_so_enable_reads_the_environment(self): + # The env layer lives in `enable()`; duplicating it here would give + # DEVTOOLS_TRACE two readers that could disagree. + self.assertIsNone(plugin._resolve_trace(_Config())) + + def test_the_cli_flag_selects_trace_mode(self): + self.assertTrue(plugin._resolve_trace(_Config({"--devtools-trace": True}))) + + def test_the_ini_option_selects_it(self): + self.assertTrue(plugin._resolve_trace(_Config(ini={"devtools_trace": True}))) + + def test_an_ini_that_says_no_is_an_answer_not_a_shrug(self): + self.assertIs(plugin._resolve_trace(_Config(ini={"devtools_trace": False})), False) + + +class ConfigureResolvesOnceTest(unittest.TestCase): + """Most hooks are handed no `config`, so the answer has to be cached.""" + + def setUp(self): + self._was = plugin._enabled + self.addCleanup(lambda: setattr(plugin, "_enabled", self._was)) + + def test_opted_in_reports_what_configure_resolved(self): + plugin._enabled = False + with mock.patch.object(plugin, "_resolve_enabled", return_value=True), \ + mock.patch.object(plugin, "_enable_assertion_pass_hook"), \ + mock.patch.object(plugin, "_configure_rerun"), \ + mock.patch.object(plugin.devtools, "enable", return_value=None), \ + mock.patch.object(plugin.devtools, "dashboard_url", return_value=None): + plugin.pytest_configure(_Config()) + + self.assertTrue(plugin._opted_in()) + + def test_a_run_that_did_not_opt_in_leaves_every_hook_inert(self): + plugin._enabled = True + with mock.patch.object(plugin, "_resolve_enabled", return_value=False), \ + mock.patch.object(plugin.devtools, "enable") as enable: + plugin.pytest_configure(_Config()) + + self.assertFalse(plugin._opted_in()) + enable.assert_not_called() + + def test_the_resolved_mode_is_what_enable_is_asked_for(self): + with mock.patch.object(plugin, "_resolve_enabled", return_value=True), \ + mock.patch.object(plugin, "_resolve_trace", return_value=True), \ + mock.patch.object(plugin, "_enable_assertion_pass_hook"), \ + mock.patch.object(plugin, "_configure_rerun"), \ + mock.patch.object(plugin.devtools, "enable", return_value=None) as enable, \ + mock.patch.object(plugin.devtools, "dashboard_url", return_value=None): + plugin.pytest_configure(_Config()) + + enable.assert_called_once_with(trace=True) + + +class EmptyRunTest(unittest.TestCase): + """`configure` opens the window before collection, so a run that collects + nothing has to undo it — otherwise `sessionfinish` parks on an empty + dashboard and a mistyped path blocks the terminal.""" + + def setUp(self): + self._was = plugin._enabled + self.addCleanup(lambda: setattr(plugin, "_enabled", self._was)) + plugin._enabled = True + + def test_collecting_nothing_tears_the_run_down(self): + with mock.patch.object(plugin.devtools, "disable") as disable: + plugin.pytest_collection_finish(mock.Mock(items=[])) + + disable.assert_called_once() + self.assertFalse(plugin._opted_in()) + + def test_a_teardown_that_throws_never_fails_the_run(self): + with mock.patch.object(plugin.devtools, "disable", side_effect=OSError("x")): + plugin.pytest_collection_finish(mock.Mock(items=[])) + + self.assertFalse(plugin._opted_in()) + + def test_a_run_with_tests_is_left_alone(self): + item = mock.Mock(nodeid="t.py::a", location=("t.py", 1, "a")) + with mock.patch.object(plugin.devtools, "disable") as disable, \ + mock.patch.object(plugin.devtools, "get_capturer", return_value=None): + plugin.pytest_collection_finish(mock.Mock(items=[item])) + + disable.assert_not_called() + self.assertTrue(plugin._opted_in()) + + +def _pytest_available() -> bool: + try: + import pytest # noqa: F401 + except ImportError: + return False + return True + + +@unittest.skipUnless(_pytest_available(), "needs pytest to drive a real config") +class RealPytestTest(unittest.TestCase): + """What a stand-in config cannot prove: that the options exist.""" + + @classmethod + def setUpClass(cls): + cls._dir = tempfile.TemporaryDirectory() + d = pathlib.Path(cls._dir.name) + (d / "test_probe.py").write_text("def test_one():\n assert True\n") + (d / "conftest.py").write_text( + textwrap.dedent( + """ + from selenium_devtools import pytest_plugin as plugin + def pytest_configure(config): + print(f"RESOLVED={plugin._resolve_enabled(config)}" + f",{plugin._resolve_trace(config)}") + """ + ) + ) + cls._path = d + + @classmethod + def tearDownClass(cls): + cls._dir.cleanup() + + def _run(self, *args, ini="[pytest]\n", env=None): + (self._path / "pytest.ini").write_text(ini) + environ = {k: v for k, v in os.environ.items() if k not in _DEVTOOLS_ENV} + environ["PYTHONPATH"] = SRC + environ.update(env or {}) + out = subprocess.run( + [sys.executable, "-m", "pytest", "-p", "no:cacheprovider", "-q", "-s", + *args, "test_probe.py"], + cwd=self._path, env=environ, capture_output=True, text=True, + ) + for line in (out.stdout + out.stderr).splitlines(): + if line.startswith("RESOLVED="): + return line[len("RESOLVED="):].split(",") + self.fail(f"no resolution reported:\n{out.stdout}\n{out.stderr}") + + def test_the_flags_are_registered_and_documented(self): + out = subprocess.run( + [sys.executable, "-m", "pytest", "-p", "no:cacheprovider", "--help"], + cwd=self._path, env={**os.environ, "PYTHONPATH": SRC}, + capture_output=True, text=True, + ).stdout + + self.assertIn("--devtools", out) + self.assertIn("--devtools-trace", out) + # Undiscoverable is the whole complaint in #339; `pytest --help` showing + # nothing is what an env var could never fix. + self.assertIn("devtools_trace (bool)", out) + + def test_an_unset_ini_option_really_reads_as_none(self): + # addini(default=None). Were it pytest's usual `False`, an unset option + # would look like an explicit no and shut the env layer out. + self.assertEqual(self._run(env={"DEVTOOLS_ENABLE": "1"}), ["True", "None"]) + + def test_the_ini_options_parse_as_booleans(self): + self.assertEqual( + self._run(ini="[pytest]\ndevtools_trace = true\n"), ["True", "True"] + ) + + def test_collect_only_does_not_start_a_backend(self): + self.assertEqual(self._run("--collect-only", "--devtools")[0], "False") + + def test_pytests_own_override_switches_it_off_for_one_run(self): + # Why there is no --no-devtools: pytest already ships the spelling. + self.assertEqual( + self._run("-o", "devtools=false", ini="[pytest]\ndevtools = true\n"), + ["False", "None"], + ) + + +if __name__ == "__main__": + unittest.main()