From fdc940ced84ade51fa5144d0b479d3934d75f65a Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:52:18 -0500 Subject: [PATCH 1/2] test(code-metrics): accept an unavailable measure in the complexity coverage grader A host run of the change-scope complexity case reached its report with every python measure unavailable (no collector visible in the eval's throwaway home), and the coverage grader split FAIL PASS PASS because it asked for a collector and version per measure. The grader now accepts a measure reported unavailable with its reason, as the type-debt case's coverage grader already does. Refs: #12940 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015te8CrNfM6v83JDbLRFDA9 --- .../code-metrics/test-12940-complexity-coverage-grader.md | 5 +++++ .../case.yaml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changes/code-metrics/test-12940-complexity-coverage-grader.md diff --git a/.changes/code-metrics/test-12940-complexity-coverage-grader.md b/.changes/code-metrics/test-12940-complexity-coverage-grader.md new file mode 100644 index 00000000000..f26a02158ea --- /dev/null +++ b/.changes/code-metrics/test-12940-complexity-coverage-grader.md @@ -0,0 +1,5 @@ +--- +bump: none +--- + +Eval suite only: the change-scope complexity case's coverage grader accepts a measure reported unavailable with its reason, and the shipped skills are unchanged. diff --git a/plugins/code-metrics/evals/audit-complexity-reports-the-change-with-provenance-and-no-verdict/case.yaml b/plugins/code-metrics/evals/audit-complexity-reports-the-change-with-provenance-and-no-verdict/case.yaml index fe08cfa51e3..9b20c4d961d 100644 --- a/plugins/code-metrics/evals/audit-complexity-reports-the-change-with-provenance-and-no-verdict/case.yaml +++ b/plugins/code-metrics/evals/audit-complexity-reports-the-change-with-provenance-and-no-verdict/case.yaml @@ -21,7 +21,7 @@ graders: focus: trace - name: expectation-2 type: llm - criteria: 'Presents the Coverage of this run table naming the collector and version used per lane and measure' + criteria: 'Presents the Coverage of this run table with, for each lane and measure, either the collector and version used or the measure as unavailable with the reason the run gave, without installing a collector and without stating a number for a measure it did not collect' focus: trace - name: expectation-3 type: llm From 0c9fb81ce0fa091de7666e04be521a005e76ff90 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 3 Nov 2026 23:58:03 -0500 Subject: [PATCH 2/2] test(code-metrics): say the complexity case goes unmeasured in the sandbox The grader accepts a measure reported unavailable, so the case's description and expected_outcome now say the same, matching the type-debt case. Rename the fragment to the form check-plugins requires. Co-Authored-By: Claude Opus 5.5 --- ...r.md => test-12940-code-metrics-scaffold-runs-7c41e2a9.md} | 0 .../case.yaml | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename .changes/code-metrics/{test-12940-complexity-coverage-grader.md => test-12940-code-metrics-scaffold-runs-7c41e2a9.md} (100%) diff --git a/.changes/code-metrics/test-12940-complexity-coverage-grader.md b/.changes/code-metrics/test-12940-code-metrics-scaffold-runs-7c41e2a9.md similarity index 100% rename from .changes/code-metrics/test-12940-complexity-coverage-grader.md rename to .changes/code-metrics/test-12940-code-metrics-scaffold-runs-7c41e2a9.md diff --git a/plugins/code-metrics/evals/audit-complexity-reports-the-change-with-provenance-and-no-verdict/case.yaml b/plugins/code-metrics/evals/audit-complexity-reports-the-change-with-provenance-and-no-verdict/case.yaml index 9b20c4d961d..adfd8de5f3e 100644 --- a/plugins/code-metrics/evals/audit-complexity-reports-the-change-with-provenance-and-no-verdict/case.yaml +++ b/plugins/code-metrics/evals/audit-complexity-reports-the-change-with-provenance-and-no-verdict/case.yaml @@ -1,8 +1,8 @@ schema_version: "1.1" name: audit-complexity-reports-the-change-with-provenance-and-no-verdict -description: on a scaffolded repository with a default branch and an uncommitted change (a Python function with several branches, in the lane lizard and radon serve), the complexity report measures the change +description: on a scaffolded repository with a default branch and an uncommitted change (a Python function with several branches, in the lane lizard and radon serve), the complexity report covers the change. The scaffold adds no collector, so a measure is taken only on a host where its collector is on PATH; on any other host, the eval sandbox included, it reads unavailable with its reason, and the measured numbers go unchecked there tags: [audit, complexity, change-scope] -expected_outcome: 'Runs the bundled audit-complexity.sh against the change (merge-base diff plus uncommitted files), presents the markdown report with its Coverage of this run table, the cyclomatic reference of 20 cited to ISO/IEC 5055:2021 section 8.2.117, and no reference for cognitive complexity or Halstead difficulty, and renders no pass/fail verdict.' +expected_outcome: 'Runs the bundled audit-complexity.sh against the change (merge-base diff plus uncommitted files), presents the markdown report with its Coverage of this run table naming the collector or the reason a measure is unavailable, the cyclomatic reference of 20 cited to ISO/IEC 5055:2021 section 8.2.117, and no reference for cognitive complexity or Halstead difficulty, and renders no pass/fail verdict.' context: scaffold_script: scaffold.sh execution: