diff --git a/.changes/code-metrics/test-12940-code-metrics-scaffold-runs-7c41e2a9.md b/.changes/code-metrics/test-12940-code-metrics-scaffold-runs-7c41e2a9.md new file mode 100644 index 00000000000..f26a02158ea --- /dev/null +++ b/.changes/code-metrics/test-12940-code-metrics-scaffold-runs-7c41e2a9.md @@ -0,0 +1,5 @@ +--- +bump: none +--- + +Eval suite only: the change-scope complexity case's coverage grader accepts a measure reported unavailable with its reason, and the shipped skills are unchanged. diff --git a/plugins/code-metrics/evals/audit-complexity-reports-the-change-with-provenance-and-no-verdict/case.yaml b/plugins/code-metrics/evals/audit-complexity-reports-the-change-with-provenance-and-no-verdict/case.yaml index fe08cfa51e3..adfd8de5f3e 100644 --- a/plugins/code-metrics/evals/audit-complexity-reports-the-change-with-provenance-and-no-verdict/case.yaml +++ b/plugins/code-metrics/evals/audit-complexity-reports-the-change-with-provenance-and-no-verdict/case.yaml @@ -1,8 +1,8 @@ schema_version: "1.1" name: audit-complexity-reports-the-change-with-provenance-and-no-verdict -description: on a scaffolded repository with a default branch and an uncommitted change (a Python function with several branches, in the lane lizard and radon serve), the complexity report measures the change +description: on a scaffolded repository with a default branch and an uncommitted change (a Python function with several branches, in the lane lizard and radon serve), the complexity report covers the change. The scaffold adds no collector, so a measure is taken only on a host where its collector is on PATH; on any other host, the eval sandbox included, it reads unavailable with its reason, and the measured numbers go unchecked there tags: [audit, complexity, change-scope] -expected_outcome: 'Runs the bundled audit-complexity.sh against the change (merge-base diff plus uncommitted files), presents the markdown report with its Coverage of this run table, the cyclomatic reference of 20 cited to ISO/IEC 5055:2021 section 8.2.117, and no reference for cognitive complexity or Halstead difficulty, and renders no pass/fail verdict.' +expected_outcome: 'Runs the bundled audit-complexity.sh against the change (merge-base diff plus uncommitted files), presents the markdown report with its Coverage of this run table naming the collector or the reason a measure is unavailable, the cyclomatic reference of 20 cited to ISO/IEC 5055:2021 section 8.2.117, and no reference for cognitive complexity or Halstead difficulty, and renders no pass/fail verdict.' context: scaffold_script: scaffold.sh execution: @@ -21,7 +21,7 @@ graders: focus: trace - name: expectation-2 type: llm - criteria: 'Presents the Coverage of this run table naming the collector and version used per lane and measure' + criteria: 'Presents the Coverage of this run table with, for each lane and measure, either the collector and version used or the measure as unavailable with the reason the run gave, without installing a collector and without stating a number for a measure it did not collect' focus: trace - name: expectation-3 type: llm