feat(selenium-devtools-py): per-test trace archives and tracePolicy

.github/pull_request_template.md

@@ -23,9 +23,11 @@
 - [ ] `service` (WebdriverIO adapter)
 - [ ] `nightwatch-devtools` (Nightwatch adapter)
 - [ ] `selenium-devtools` (Selenium adapter)
+- [ ] `selenium-devtools-py` (Selenium Python adapter)
 - [ ] `backend` (server)
 - [ ] `app` (UI)
 - [ ] `script` (page-injected runtime)
+- [ ] `trace` (Trace mode)
 
 ## Notes for reviewers
 

.gitignore

@@ -48,6 +48,10 @@ packages/nightwatch-devtools/nightwatch-video-*.webm
 # trace output (mode: 'trace')
 trace-*.zip
 examples/**/trace-*/
+# ...but this one is a committed example, not output. The rule above exists for
+# unpacked archives (`trace-<hash>/`), and a folder named for what it
+# demonstrates collides with it.
+!examples/selenium/python-test/trace-py-test/
 
 # test results
 examples/**/test-results*/

examples/selenium/python-test/trace-py-test/pytest.ini

@@ -0,0 +1,123 @@
+; DevTools capture, committed with the project.
+;
+; Why here and not on the command line:
+;   - a contributor who clones this gets the same capture, without being told
+;   - a flag has to be retyped every run; an exported variable is per-shell
+;
+; Run it (no flags — pytest reads this file because it sits beside the test):
+;   pytest examples/selenium/python-test/trace-py-test/
+;   pnpm show-trace examples/selenium/python-test/trace-py-test/test-results/*.zip
+;
+; Override one setting for one run, without editing this file:
+;   pytest -o devtools_trace_policy=on  examples/.../trace-py-test/
+;   pytest -o devtools_trace_granularity=session  examples/.../trace-py-test/
+;   pytest --devtools  examples/.../trace-py-test/          ; live dashboard
+;   pytest -o devtools=false  examples/.../trace-py-test/   ; capture nothing
+
+[pytest]
+
+; ─────────────────────────────────────────────────────────────────────────────
+; devtools — opt in
+; ─────────────────────────────────────────────────────────────────────────────
+;   - capture is ALWAYS opt-in: the plugin auto-loads when the package is
+;     installed, so it must never change an existing suite's behaviour
+;   - precedence, highest first:  CLI flag  >  this file  >  environment
+;   - environment fallback: DEVTOOLS_ENABLE=1, or DEVTOOLS_PORT=<n>
+;   - `-o devtools=false` turns it off for one run
+;     (which is why there is no --no-devtools flag)
+;   - not set below: devtools_trace already implies it
+
+; ─────────────────────────────────────────────────────────────────────────────
+; devtools_trace — a dashboard, or a file
+; ─────────────────────────────────────────────────────────────────────────────
+;   false   live mode: streams to a dashboard window (the default)
+;   true    trace mode: writes an archive to test-results/, opens no window
+;
+;   - trace mode is what you want in CI, and when the run already happened
+;   - open an archive with `pnpm show-trace <zip>`
+devtools_trace = true
+
+; ─────────────────────────────────────────────────────────────────────────────
+; devtools_trace_granularity — how many archives
+; ─────────────────────────────────────────────────────────────────────────────
+;   session   one archive for the whole run (the default)
+;   test      one per test
+;
+;   - a per-test archive holds only that test's own commands, console, network,
+;     DOM mutations, a11y trees and screencast frames
+;   - `spec` is not offered: this adapter's spec IS its test file, so it could
+;     only silently mean one of the two above
+devtools_trace_granularity = test
+
+; ─────────────────────────────────────────────────────────────────────────────
+; devtools_trace_policy — which archives to keep
+; ─────────────────────────────────────────────────────────────────────────────
+; Values:
+;   on                              keep everything (the default)
+;   retain-on-failure               keep only what failed
+;   retain-on-first-failure         )
+;   on-first-retry                  )  accepted, but see below
+;   on-all-retries                  )
+;   retain-on-failure-and-retries   )
+;
+; Those last four currently behave EXACTLY like retain-on-failure:
+;   - nothing this adapter puts on the wire carries an attempt number
+;   - so a retried test overwrites its own earlier outcome
+;   - so the retry-aware question cannot be asked at all
+;   - the backend logs the degradation rather than pretending otherwise
+;   - pick one only if you want retain-on-failure under a name that will mean
+;     more later
+;
+; What it combines with:
+;   granularity=test     + retain-on-failure   ->  only the tests that failed
+;   granularity=session  + retain-on-failure   ->  the whole run, if any failed
+;   either granularity   + on                  ->  everything
+;
+; As set here, a green run writes nothing at all — which is the point: the
+; archives you have are the ones worth opening.
+devtools_trace_policy = retain-on-failure
+
+; ─────────────────────────────────────────────────────────────────────────────
+; Environment-only settings
+; ─────────────────────────────────────────────────────────────────────────────
+; No ini option yet. Export them, or pass the matching keyword to
+; devtools.enable() in a plain script (see ../login.py).
+;
+; Connecting:
+;   DEVTOOLS_ENABLE=1             opt in (what `devtools = true` does)
+;   DEVTOOLS_HOST=<host>          backend host (default: localhost)
+;   DEVTOOLS_PORT=<n>             attach to a backend already listening
+;   DEVTOOLS_BACKEND_CMD=<cmd>    launch the backend your own way
+;
+; Capture:
+;   DEVTOOLS_BIDI=0               skip BiDi; console and network go quiet
+;   DEVTOOLS_OPEN=0               never open a dashboard window (CI)
+;
+; Trace mode:
+;   DEVTOOLS_TRACE=1              selects trace mode, but does NOT opt a pytest
+;                                 run in on its own — it is a mode fallback you
+;                                 may have exported for your own scripts
+;   DEVTOOLS_TRACE_GRANULARITY=test   )  ambient, so neither turns trace mode
+;   DEVTOOLS_TRACE_POLICY=...         )  on by itself — pair with DEVTOOLS_TRACE=1.
+;                                     )  The CLI flag, the ini option and the
+;                                     )  enable() argument DO imply it. A run
+;                                     )  that ignores one of these says so.
+;   DEVTOOLS_FILMSTRIP=0          drop the dense screencast from the archive
+;   DEVTOOLS_A11Y=0               drop the per-action a11y tree and element
+;                                 rects (loses the A11y tab and the overlay)
+;
+; Set for you, not by you — listed so an unexpected value is recognisable:
+;   DEVTOOLS_RUN_ID               one id shared by every process of a run
+;   DEVTOOLS_RUNNER_CWD           where a Rerun spawns (pytest's rootdir)
+;   DEVTOOLS_APP_REUSE            )
+;   DEVTOOLS_APP_HOST             )  point a spawned rerun back at the
+;   DEVTOOLS_APP_PORT             )  backend that asked for it
+;
+; The same choices in a plain script:
+;   devtools.enable(
+;       trace=True,
+;       trace_granularity="test",
+;       trace_policy="retain-on-failure",
+;       filmstrip=True,
+;       a11y=True,
+;   )

examples/selenium/python-test/test_login_pytest.py → examples/selenium/python-test/trace-py-test/test_login_pytest.py

@@ -13,10 +13,31 @@
 pytest has no ``describe``/``it`` blocks; a class IS the grouping construct, so
 this is the closest equivalent to a nested ``describe`` in the JS examples.
 
-Run it (the plugin is inert unless a devtools env var opts the run in):
+This folder is the "committed config" shape: `pytest.ini` beside it already
+says how to capture, so running it needs no flags and no environment. Read that
+file — it documents every DevTools setting the adapter has, including the ones
+that are environment-only.
 
     pip install -e packages/selenium-devtools-py
-    python -m pytest --devtools examples/selenium/python-test/test_login_pytest.py
+    pytest examples/selenium/python-test/trace-py-test/
+    pnpm show-trace examples/selenium/python-test/trace-py-test/test-results/*.zip
+
+As committed it writes ONE archive PER FAILING TEST — `devtools_trace = true`,
+`devtools_trace_granularity = test`, `devtools_trace_policy = retain-on-failure`.
+So a green run writes nothing at all, which is the point: the archives you have
+are the ones worth opening.
+
+To see the difference, break an assertion below — the flash-message one in
+`test_rejects_invalid_credentials` is the easiest — and compare:
+
+    pytest -o devtools_trace_policy=on   examples/.../trace-py-test/   # 3 archives
+    pytest                               examples/.../trace-py-test/   # 1, the broken test
+    pytest -o devtools_trace_granularity=session examples/.../trace-py-test/
+                                                                       # 1, the whole run
+
+`-o` overrides one setting for one run without editing the file. A sibling
+example, `../login.py`, shows the same choices passed to `devtools.enable()`
+instead, for a script with no test runner.
 
 The driver fixture is function-scoped, so each test gets its own browser session
 — which also exercises the adapter's per-driver capture state.

package.json

@@ -15,7 +15,7 @@
     "demo:selenium": "pnpm --filter @wdio/selenium-devtools example",
     "demo:python": "python3 examples/selenium/python-test/web_form.py",
     "demo:python:login": "python3 examples/selenium/python-test/login.py",
-    "demo:python:pytest": "python3 -m pytest --devtools examples/selenium/python-test/test_login_pytest.py",
+    "demo:python:pytest": "python3 -m pytest examples/selenium/python-test/trace-py-test/",
     "dev": "pnpm --parallel dev",
     "preview": "pnpm --parallel preview",
     "test": "vitest run",

packages/backend/src/trace-export-message.ts

@@ -11,7 +11,11 @@ import {
   type TraceExportResult
 } from '@wdio/devtools-shared'
 import type { ActiveRun } from './baseline/types.js'
-import { exportActiveRunTrace } from './trace-export.js'
+import {
+  exportActiveRunTrace,
+  exportPerTestTraces,
+  retentionDecision
+} from './trace-export.js'
 
 const log = logger('@wdio/devtools-backend')
 
@@ -56,11 +60,45 @@ export async function runTraceExport(
       JSON.stringify({ scope: TRACE_EXPORT_SCOPE.result, data: result })
     )
   try {
-    const path = await exportActiveRunTrace(deps.activeRun(), {
+    const run = deps.activeRun()
+    if (request.traceGranularity === 'test') {
+      // Retention is per slice here, so the run-wide decision below would be
+      // the wrong question: one failing test must not keep the passing ones.
+      const paths = await exportPerTestTraces(run, request)
+      log.info(
+        paths.length
+          ? `Trace exported for session ${request.sessionId}: ${paths.length} test slice(s)`
+          : `No test slice for session ${request.sessionId} was retained by ` +
+              `policy '${request.tracePolicy}'`
+      )
+      reply({
+        requestId: request.requestId,
+        paths,
+        ...(paths.length ? {} : { declinedByPolicy: true })
+      })
+      return
+    }
+    // Asked before exporting rather than reported after: a decline is the
+    // policy working, and routing it through the catch below would log a
+    // passing run's own success as an export failure.
+    const decision = retentionDecision(run, request.tracePolicy)
+    if (!decision.retain) {
+      log.info(
+        `Trace for session ${request.sessionId} not retained by policy ` +
+          `'${request.tracePolicy}'` +
+          (decision.degradedToFailure
+            ? ' (no attempt info; degraded to retain-on-failure)'
+            : '')
+      )
+      reply({ requestId: request.requestId, declinedByPolicy: true })
+      return
+    }
+    const path = await exportActiveRunTrace(run, {
       outputDir: request.outputDir,
       sessionId: request.sessionId,
       format: request.format,
-      fileStem: request.fileStem
+      fileStem: request.fileStem,
+      tracePolicy: request.tracePolicy
     })
     log.info(`Trace exported for session ${request.sessionId}: ${path}`)
     reply({ requestId: request.requestId, path })

packages/backend/src/trace-export.ts

@@ -10,14 +10,20 @@
 import type {
   ActionSnapshot,
   TestMetadataMap,
-  TraceExportRequest
+  TraceExportRequest,
+  TraceRetentionPolicy
 } from '@wdio/devtools-shared'
 import { serializeWebSnapshot } from '@wdio/devtools-trace/a11y-snapshot'
 import {
   writeTraceZip,
   type TraceCapturer
 } from '@wdio/devtools-trace/trace-exporter'
+import {
+  shouldRetainTrace,
+  type RetentionDecision
+} from '@wdio/devtools-trace/trace-retention'
 import type { ActiveRun, TimeWindowNode } from './baseline/types.js'
+import { sliceRunByTest } from './trace-slice.js'
 
 /**
  * Test titles for `Tracing.tracingGroup` events, derived from the suite tree
@@ -94,16 +100,42 @@ export function hasExportableData(run: Readonly<ActiveRun>): boolean {
   )
 }
 
+/**
+ * Whether this run is worth an archive, by the same rule and the same function
+ * `core/trace-finalizer.ts` `writeSessionTrace` applies for an in-process
+ * adapter — one implementation, not one per language.
+ *
+ * `attemptInfoAvailable` is false because nothing on the wire carries an
+ * attempt number: the accumulator's node tree keeps one state per uid, so a
+ * retried test overwrites its own earlier outcome. The retry-aware policies
+ * therefore degrade to `retain-on-failure`, which `shouldRetainTrace` reports
+ * through `degradedToFailure` rather than silently.
+ */
+export function retentionDecision(
+  run: Readonly<ActiveRun>,
+  policy: TraceRetentionPolicy | undefined
+): RetentionDecision {
+  const outcomes = Array.from(run.nodes.values())
+    .filter((node) => node.kind === 'test')
+    .map((node) => ({ state: node.state }))
+  return shouldRetainTrace(policy, { outcomes, attemptInfoAvailable: false })
+}
+
 export async function exportActiveRunTrace(
   run: Readonly<ActiveRun>,
   request: Pick<
     TraceExportRequest,
-    'outputDir' | 'sessionId' | 'format' | 'fileStem'
+    'outputDir' | 'sessionId' | 'format' | 'fileStem' | 'tracePolicy'
   >
 ): Promise<string> {
   if (!hasExportableData(run)) {
     throw new Error('nothing captured for this run')
   }
+  if (!retentionDecision(run, request.tracePolicy).retain) {
+    // Distinct from the error above: that one means the run captured nothing,
+    // this one means it captured a run nobody asked to keep.
+    throw new Error(`trace not retained by policy '${request.tracePolicy}'`)
+  }
   // `capabilities` is not passed separately: writeTraceZip spreads the
   // capturer's own metadata, which already carries it.
   return writeTraceZip(toCapturer(run), {
@@ -126,3 +158,53 @@ export async function exportActiveRunTrace(
     testMetadata: testMetadataFromNodes(run.nodes)
   })
 }
+
+/** Filesystem-safe fragment of a test's identity, for its artifact name. */
+function slugForTest(uid: string, title: string): string {
+  const base = (title || uid)
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, 60)
+  // A hash of the UID, not of the title: two tests can share a title (a
+  // parametrised case), and colliding names would have one overwrite the other.
+  let hash = 0
+  for (let i = 0; i < uid.length; i++) {
+    hash = (hash * 31 + uid.charCodeAt(i)) | 0
+  }
+  return `${base || 'test'}-${(hash >>> 0).toString(36).slice(0, 6)}`
+}
+
+/**
+ * One archive per test that the policy retains.
+ *
+ * A test whose slice captured nothing is skipped rather than written empty —
+ * the same rule `hasExportableData` applies to a whole run, for the same
+ * reason: an empty archive reads in the viewer as a run that captured nothing.
+ */
+export async function exportPerTestTraces(
+  run: Readonly<ActiveRun>,
+  request: Pick<
+    TraceExportRequest,
+    'outputDir' | 'sessionId' | 'format' | 'tracePolicy'
+  >
+): Promise<string[]> {
+  const paths: string[] = []
+  for (const slice of sliceRunByTest(run)) {
+    if (!hasExportableData(slice.run)) {
+      continue
+    }
+    if (!retentionDecision(slice.run, request.tracePolicy).retain) {
+      continue
+    }
+    paths.push(
+      await exportActiveRunTrace(slice.run, {
+        outputDir: request.outputDir,
+        sessionId: request.sessionId,
+        ...(request.format ? { format: request.format } : {}),
+        fileStem: `trace-${slugForTest(slice.uid, slice.title)}`
+      })
+    )
+  }
+  return paths
+}