diff --git a/.changes/code-metrics/refactor-12491-code-metrics-audit-descriptions-dbcc5296.md b/.changes/code-metrics/refactor-12491-code-metrics-audit-descriptions-dbcc5296.md new file mode 100644 index 00000000000..dfca88c1203 --- /dev/null +++ b/.changes/code-metrics/refactor-12491-code-metrics-audit-descriptions-dbcc5296.md @@ -0,0 +1,5 @@ +--- +bump: none +--- + +Adds frozen trigger probes for three audit skills; no shipped behavior changes. diff --git a/plugins/code-metrics/probes/audit-coverage.json b/plugins/code-metrics/probes/audit-coverage.json new file mode 100644 index 00000000000..1421a62e62d --- /dev/null +++ b/plugins/code-metrics/probes/audit-coverage.json @@ -0,0 +1,123 @@ +{ + "skill": "code-metrics:audit-coverage", + "plugin": "code-metrics", + "skill_dir": "plugins/code-metrics/skills/audit-coverage", + "ranking_note": "The target reads test-execution artifacts a build already wrote and reports per-function line figures and CRAP, while its competitors measure complexity alone, compare against a baseline, plan tests, or judge whether tests can catch faults. The set was authored blind from before descriptions.", + "competitors": [ + "code-metrics:audit", + "verification:measure", + "testing:plan", + "testing:audit", + "mutation-testing:audit" + ], + "queries": [ + { + "id": "pos-train-01", + "split": "train", + "expect_trigger": true, + "request": "The nightly build left build/lcov.info in the billing-service repo. Without rerunning anything, give me the line percentage for each function in src/invoice/tax.ts." + }, + { + "id": "pos-train-02", + "split": "train", + "expect_trigger": true, + "request": "Which functions in the payments module have the worst CRAP numbers? The cobertura.xml from CI is already sitting in target/site." + }, + { + "id": "pos-train-03", + "split": "train", + "expect_trigger": true, + "request": "pytest-cov already wrote its JSON report for orders-api. How many lines of the functions I changed on this branch actually ran?" + }, + { + "id": "pos-train-04", + "split": "train", + "expect_trigger": true, + "request": "go test saved profile.out yesterday for the ledger package. Break it down by function so I can see which ones never got hit." + }, + { + "id": "pos-train-05", + "split": "train", + "expect_trigger": true, + "request": "I don't need a before-and-after or a baseline here. Just pull today's line percentage for shipping/rates.py out of the lcov.info our pipeline produced." + }, + { + "id": "pos-val-01", + "split": "validation", + "expect_trigger": true, + "request": "Find the gnarly functions in checkout-web that nobody tests: lots of branching, few lines hit. The lcov.info is already in the reports folder." + }, + { + "id": "pos-val-02", + "split": "validation", + "expect_trigger": true, + "request": "My suite for parser.go seems thin. Tell me what percent of each function's lines ran, using the profile CI saved, and don't start any tests." + }, + { + "id": "pos-val-03", + "split": "validation", + "expect_trigger": true, + "request": "Open target/cobertura.xml and tell me how well exercised src/Auth/TokenStore.cs is, method by method." + }, + { + "id": "pos-val-04", + "split": "validation", + "expect_trigger": true, + "request": "A mutation score isn't what I'm after. Give me a CRAP number for every function under services/notifier, from the report the last CI job uploaded." + }, + { + "id": "neg-train-01", + "split": "train", + "expect_trigger": false, + "request": "What's the cyclomatic complexity per function in src/router.ts? I don't need any coverage numbers, only how tangled the code is." + }, + { + "id": "neg-train-02", + "split": "train", + "expect_trigger": false, + "request": "Capture a coverage baseline before I refactor the cart module, so that afterwards we can compare and prove the number went up." + }, + { + "id": "neg-train-03", + "split": "train", + "expect_trigger": false, + "request": "I just changed the scheduler. What kinds of tests does this need, where are the coverage gaps, and what should I tackle first given regression risk?" + }, + { + "id": "neg-train-04", + "split": "train", + "expect_trigger": false, + "request": "Our line coverage is 90% but I suspect plenty of those tests have no assertions at all. Find the ones in tests/api that can't fail." + }, + { + "id": "neg-train-05", + "split": "train", + "expect_trigger": false, + "request": "Coverage on the pricing engine is high but I don't trust it. Run mutation testing on this diff and show me the survivors." + }, + { + "id": "neg-val-01", + "split": "validation", + "expect_trigger": false, + "request": "How much of packages/core is typed? Give me the type coverage and how many anys are left." + }, + { + "id": "neg-val-02", + "split": "validation", + "expect_trigger": false, + "request": "Did coverage actually rise after my test PR, compared with the baseline we took before starting it?" + }, + { + "id": "neg-val-03", + "split": "validation", + "expect_trigger": false, + "request": "I'm prepping the PR for the webhook handler. Draw up a test plan: which changed files need unit or integration tests to close the coverage holes?" + }, + { + "id": "neg-val-04", + "split": "validation", + "expect_trigger": false, + "request": "The tests I just wrote for discount.py hit every line. Would they notice a broken implementation? Get me a mutation score." + } + ] +} diff --git a/plugins/code-metrics/probes/audit-duplication.json b/plugins/code-metrics/probes/audit-duplication.json new file mode 100644 index 00000000000..5486fe2d2be --- /dev/null +++ b/plugins/code-metrics/probes/audit-duplication.json @@ -0,0 +1,121 @@ +{ + "skill": "code-metrics:audit-duplication", + "plugin": "code-metrics", + "skill_dir": "plugins/code-metrics/skills/audit-duplication", + "ranking_note": "The target reports repeated source code as clone groups with their locations, while its competitors consolidate repeated markdown prose, count lines or complexity, or compare a metric against a captured baseline. The set was authored blind from before descriptions.", + "competitors": [ + "code-metrics:audit", + "docs-hygiene:extract-ssot", + "verification:measure" + ], + "queries": [ + { + "id": "pos-train-01", + "split": "train", + "expect_trigger": true, + "request": "I think the handlers in orders-api/src/routes were pasted from each other. Show me which blocks repeat and exactly where each copy sits." + }, + { + "id": "pos-train-02", + "split": "train", + "expect_trigger": true, + "request": "Put a number on how much repeated code my branch introduces. No baseline needed, just today's figure." + }, + { + "id": "pos-train-03", + "split": "train", + "expect_trigger": true, + "request": "src/billing and src/invoicing feel like the same TypeScript written twice. Is there actually repeated logic between them?" + }, + { + "id": "pos-train-04", + "split": "train", + "expect_trigger": true, + "request": "Do a DRY pass over the code in packages/checkout: list groups of identical or near-identical blocks and where each one lives. Don't change anything yet." + }, + { + "id": "pos-train-05", + "split": "train", + "expect_trigger": true, + "request": "Our generated protocol stubs are copied on purpose and the repo says so. Leaving those out, how many clone groups are there across the whole codebase?" + }, + { + "id": "pos-val-01", + "split": "validation", + "expect_trigger": true, + "request": "Is utils/date_helpers.py so big because someone pasted the same functions several times? Show me the repeated blocks, not a line count." + }, + { + "id": "pos-val-02", + "split": "validation", + "expect_trigger": true, + "request": "The same retry helper keeps turning up in our Go services. Find every place that block of code appears under services/, with line numbers." + }, + { + "id": "pos-val-03", + "split": "validation", + "expect_trigger": true, + "request": "How much copy-and-paste is in the C# under src/Reports? I want it as clone groups and total lines." + }, + { + "id": "pos-val-04", + "split": "validation", + "expect_trigger": true, + "request": "Are the new Python lines in my diff just repeats of code that already exists in ingest/?" + }, + { + "id": "neg-train-01", + "split": "train", + "expect_trigger": false, + "request": "The same branch-protection paragraph is duplicated in five rule files. Pull it into one doc and make the others cite it." + }, + { + "id": "neg-train-02", + "split": "train", + "expect_trigger": false, + "request": "We suspect duplication has bloated this repo, but for now just list lines per file and flag anything over 1000." + }, + { + "id": "neg-train-03", + "split": "train", + "expect_trigger": false, + "request": "Capture a baseline of the code's shape before I refactor the importer to remove duplicated paths, then compare afterwards to prove it improved." + }, + { + "id": "neg-train-04", + "split": "train", + "expect_trigger": false, + "request": "DRY this prose: our ADRs and skill bodies duplicate the same timeout value, so every bump touches several markdown files." + }, + { + "id": "neg-train-05", + "split": "train", + "expect_trigger": false, + "request": "This parser in legacy/ got copied around and it's hard to follow. What are its cyclomatic and cognitive complexity per function?" + }, + { + "id": "neg-val-01", + "split": "validation", + "expect_trigger": false, + "request": "README and CONTRIBUTING both carry a duplicate of the release checklist. Make one source of truth and migrate the references to it." + }, + { + "id": "neg-val-02", + "split": "validation", + "expect_trigger": false, + "request": "I removed the duplicated handlers yesterday. Did complexity go down compared with the numbers we captured before the change?" + }, + { + "id": "neg-val-03", + "split": "validation", + "expect_trigger": false, + "request": "The duplicated DTOs in packages/api made me wonder about typing. What percentage is typed and how many anys are in there?" + }, + { + "id": "neg-val-04", + "split": "validation", + "expect_trigger": false, + "request": "After deleting the duplicated cache code, is the search endpoint any faster? Benchmark it before and after." + } + ] +} diff --git a/plugins/code-metrics/probes/audit-suppressions.json b/plugins/code-metrics/probes/audit-suppressions.json new file mode 100644 index 00000000000..79d230db8a4 --- /dev/null +++ b/plugins/code-metrics/probes/audit-suppressions.json @@ -0,0 +1,121 @@ +{ + "skill": "code-metrics:audit-suppressions", + "plugin": "code-metrics", + "skill_dir": "plugins/code-metrics/skills/audit-suppressions", + "ranking_note": "The target inventories the lint, type-checker and evidence-disabling directives in code with their rules and justification, while its competitors freeze a count as a CI ceiling, run linters and formatters, or measure size, complexity and typed percentage. The set was authored blind from before descriptions.", + "competitors": [ + "code-metrics:audit", + "toolchain:ratchet", + "toolchain:lint" + ], + "queries": [ + { + "id": "pos-train-01", + "split": "train", + "expect_trigger": true, + "request": "How many eslint-disable comments does the web-client repo have right now, and which rules do they switch off? No CI changes, I only want the list." + }, + { + "id": "pos-train-02", + "split": "train", + "expect_trigger": true, + "request": "Before anyone talks about a CI ceiling, tally every noqa and type: ignore in the analytics package so I know the real number." + }, + { + "id": "pos-train-03", + "split": "train", + "expect_trigger": true, + "request": "Someone sprinkled # noqa all over ingest/. Which of those comments give no justification at all?" + }, + { + "id": "pos-train-04", + "split": "train", + "expect_trigger": true, + "request": "Did my branch add any @ts-expect-error or eslint-disable lines? Show each one with the rule it silences and why the author gave for it." + }, + { + "id": "pos-train-05", + "split": "train", + "expect_trigger": true, + "request": "Are people dodging our checks in services/payments? Find every istanbul ignore, Stryker disable, nosec and nosemgrep in there." + }, + { + "id": "pos-val-01", + "split": "validation", + "expect_trigger": true, + "request": "The linter passes on legacy-api, but I suspect that's only thanks to nolint comments. List them all with the linters they silence." + }, + { + "id": "pos-val-02", + "split": "validation", + "expect_trigger": true, + "request": "Is type debt hiding behind ignores? Inventory each type: ignore and pyright: ignore in src/models along with its error codes." + }, + { + "id": "pos-val-03", + "split": "validation", + "expect_trigger": true, + "request": "Give me a table of every rubocop:disable and shellcheck disable in the repo, saying whether each one has a decent explanation." + }, + { + "id": "pos-val-04", + "split": "validation", + "expect_trigger": true, + "request": "Which #pragma warning disable lines in our C# projects turn off correctness rules rather than style ones?" + }, + { + "id": "neg-train-01", + "split": "train", + "expect_trigger": false, + "request": "Freeze our eslint-disable count at today's value in CI so that any PR adding one fails." + }, + { + "id": "neg-train-02", + "split": "train", + "expect_trigger": false, + "request": "Run eslint and prettier on my changed files and fix the formatting. I added a couple of eslint-disable comments along the way." + }, + { + "id": "neg-train-03", + "split": "train", + "expect_trigger": false, + "request": "Lots of type: ignore lines landed in this package lately. How much of it is typed according to mypy, as a percentage?" + }, + { + "id": "neg-train-04", + "split": "train", + "expect_trigger": false, + "request": "Ratchet the lint suppressions in the monorepo: check in a ceiling and lower it whenever the count falls." + }, + { + "id": "neg-train-05", + "split": "train", + "expect_trigger": false, + "request": "I suppressed the complexity rule on process_order. What are its actual cyclomatic and cognitive scores?" + }, + { + "id": "neg-val-01", + "split": "validation", + "expect_trigger": false, + "request": "Does ruff pass on this branch now that I removed the noqa comments? Just a quick lint, no full build." + }, + { + "id": "neg-val-02", + "split": "validation", + "expect_trigger": false, + "request": "Stop the number of pylint disables from growing. Propose the CI files for a human to merge." + }, + { + "id": "neg-val-03", + "split": "validation", + "expect_trigger": false, + "request": "We suppressed max-lines a while back. Which files in src/ are too long now? Give me lines per file." + }, + { + "id": "neg-val-04", + "split": "validation", + "expect_trigger": false, + "request": "Apply the eslint autofixes on this diff so I can drop some of these eslint-disable lines." + } + ] +}