From 1707929caac9f2d38520bfef9fb3ba50e3eeca15 Mon Sep 17 00:00:00 2001 From: nivSwisa1 Date: Wed, 4 Nov 2026 14:51:19 +0200 Subject: [PATCH 1/2] LINBEE-30104 | feat: verdict says Reduced/Kept/Raised effort against the model's current level (v1.5.0) Co-Authored-By: Claude Opus 5.5 --- .claude-plugin/marketplace.json | 2 +- .../.claude-plugin/plugin.json | 4 +- plugins/agentic-advisor/CHANGELOG.md | 7 + plugins/agentic-advisor/README.md | 6 +- .../hooks/agentic-advisor-report.sh | 31 +++- .../hooks/agentic-advisor-trigger.sh | 12 +- .../skills/agentic-advisor/SKILL.md | 55 +++--- tests/agentic-advisor/hooks_test.sh | 173 +++++++++++++++++- 8 files changed, 248 insertions(+), 42 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index ddc6aa5..1d0c525 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "plugins": [ { "name": "agentic-advisor", - "description": "Before writing code, grades how fragile the target is from LinearB health signals + local git history and holds the agent to a matching effort level (LOW/MEDIUM/HIGH) — lean on calm repos, defensive on fragile ones.", + "description": "Before writing code, grades how fragile the target is from LinearB health signals + local git history and reports whether it Reduced, Kept or Raised effort against the model's current level — lean on calm repos, defensive on fragile ones.", "author": { "name": "LinearB", "email": "support@linearb.io" diff --git a/plugins/agentic-advisor/.claude-plugin/plugin.json b/plugins/agentic-advisor/.claude-plugin/plugin.json index b101870..4d74e8f 100644 --- a/plugins/agentic-advisor/.claude-plugin/plugin.json +++ b/plugins/agentic-advisor/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agentic-advisor", - "version": "1.4.7", - "description": "Before writing code, grades how fragile the target is from LinearB health signals + local git history (rework, incidents, fix/revert density, code ownership) and holds the agent to a matching effort level (LOW/MEDIUM/HIGH) — lean on calm repos, defensive on fragile ones. Auto-triggers via a bundled hook and can report each effort decision to your LinearB org as usage telemetry.", + "version": "1.5.0", + "description": "Before writing code, grades how fragile the target is from LinearB health signals + local git history (rework, incidents, fix/revert density, code ownership) and reports whether it Reduced, Kept or Raised effort against the model's current level — lean on calm repos, defensive on fragile ones. Auto-triggers via a bundled hook and can report each effort decision to your LinearB org as usage telemetry.", "author": { "name": "LinearB", "email": "support@linearb.io" diff --git a/plugins/agentic-advisor/CHANGELOG.md b/plugins/agentic-advisor/CHANGELOG.md index 3c29e83..e2a61d3 100644 --- a/plugins/agentic-advisor/CHANGELOG.md +++ b/plugins/agentic-advisor/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.5.0] - 2026-11-04 + +### Changed + +- The verdict now compares the grade with the model's current effort (`/effort`): `Raised effort (medium → high; …)`, `Kept effort (high; …)` or `Reduced effort (max → medium; …)`. It never raises past high, and a HIGH grade on xhigh or max is Kept. +- The decision event adds `effort_change` and `target_effort`; `effort_level` stays LOW/MEDIUM/HIGH, and the older `MEDIUM effort` lines are still reported. + ## [1.4.7] - 2026-10-26 ### Fixed diff --git a/plugins/agentic-advisor/README.md b/plugins/agentic-advisor/README.md index e14708d..b6aeca1 100644 --- a/plugins/agentic-advisor/README.md +++ b/plugins/agentic-advisor/README.md @@ -2,7 +2,7 @@ Sizes the agent's coding **effort** to how fragile the target is — before it writes code. -At the start of a code task, a bundled hook auto-invokes the `agentic-advisor` skill. It reads the repo's current health (LinearB rework %, unreviewed merges, recent incidents) plus local git history on the files being touched, then grades **LOW / MEDIUM / HIGH effort** and holds the agent to it: lean and token-efficient on calm repos, defensive (smallest change, guards, tests, clarifying questions) on fragile ones. It never blocks; if LinearB data is unavailable it defaults to MEDIUM. +At the start of a code task, a bundled hook auto-invokes the `agentic-advisor` skill. It reads the repo's current health (LinearB rework %, unreviewed merges, recent incidents) plus local git history on the files being touched, then compares the grade with the model's current effort and says whether it **Reduced**, **Kept** or **Raised** effort, and holds the agent to it: lean and token-efficient on calm repos, defensive (smallest change, guards, tests, clarifying questions) on fragile ones. It never blocks; if LinearB data is unavailable it defaults to MEDIUM. **Changelog:** [CHANGELOG.md](CHANGELOG.md) @@ -19,9 +19,9 @@ Then add `LINEARB_API_TOKEN` to `~/.claude/settings.json` under `env` (see [Setu The agent prints a one-line verdict before writing code, e.g.: -> LinearB: api-service — LOW effort (healthy: rework 0.4%, 0 unreviewed merges, no incidents). +> LinearB: api-service — Reduced effort (high → low; healthy: rework 0.4%, 0 unreviewed merges, no incidents). -> LinearB: payments-service — HIGH effort (rework 9.5% NEEDS FOCUS; target PaymentForm.tsx: ~140 lines rewritten/90d) — smallest viable change, defensive validation, focused test. +> LinearB: payments-service — Raised effort (medium → high; rework 9.5% NEEDS FOCUS; target PaymentForm.tsx: ~140 lines rewritten/90d) — smallest viable change, defensive validation, focused test. ## How it decides diff --git a/plugins/agentic-advisor/hooks/agentic-advisor-report.sh b/plugins/agentic-advisor/hooks/agentic-advisor-report.sh index 24f932e..2f6d733 100755 --- a/plugins/agentic-advisor/hooks/agentic-advisor-report.sh +++ b/plugins/agentic-advisor/hooks/agentic-advisor-report.sh @@ -117,6 +117,9 @@ if ! { [ -d "$marker_dir" ] && [ -O "$marker_dir" ] && [ -w "$marker_dir" ]; }; state="$marker_dir/${session_id:-default}.reported" touch "$state" 2>/dev/null || true +# A graded verdict line: the Reduced/Kept/Raised shape, or the older LOW/MEDIUM/HIGH one. +GRADED_RE='LinearB: .+ ((Reduced|Kept|Raised) effort \((low|medium|high|xhigh|max)|(LOW|MEDIUM|HIGH) effort)' + # Cheap fast-exits for the graded path, BEFORE the git + jq repo-resolution work — so a # per-turn Stop that has nothing new to report costs only a grep or two, not a full # transcript parse. (The whole graded path requires the skill to have actually run.) @@ -126,7 +129,7 @@ if [ "$label" = "graded" ]; then # Has the set of verdict lines changed since we last processed a Stop? SKILL.md # example lines are constant, so this count only rises on a real new verdict (incl. # a phase-2 re-emit). Unchanged -> nothing to do -> skip the O(n) scrape + resolution. - vcount="$(grep -cE 'LinearB: .+ (LOW|MEDIUM|HIGH) effort' "$transcript" 2>/dev/null)"; vcount="${vcount:-0}" + vcount="$(grep -cE "$GRADED_RE" "$transcript" 2>/dev/null)"; vcount="${vcount:-0}" vseen="$marker_dir/${session_id:-default}.vseen" [ -f "$vseen" ] && [ "$(cat "$vseen" 2>/dev/null)" = "$vcount" ] && { dbg "Stop: skip: no new verdict since last report"; exit 0; } fi @@ -253,7 +256,7 @@ for line in open(sys.argv[1]): # verdict must be ASSISTANT-authored text AFTER the Skill call — not the # SKILL.md examples that load on invocation (they also match the pattern). elif role == "assistant" and skill is not None and verdict is None \ - and re.search(r"LinearB: .+ (LOW|MEDIUM|HIGH) effort", (x.get("text") or "") if typ == "text" else + and re.search(r"LinearB: .+ ((Reduced|Kept|Raised) effort \((low|medium|high|xhigh|max)|(LOW|MEDIUM|HIGH) effort)", (x.get("text") or "") if typ == "text" else str((x.get("input") or {}).get("command", "")) if typ == "tool_use" and x.get("name") == "Bash" and re.search(r"(^|\n|;|&&)\s*printf\s[^\n]*LinearB: ", str((x.get("input") or {}).get("command", ""))) else ""): verdict = te @@ -329,19 +332,29 @@ fi # GATE 2 — a real assistant-authored verdict line exists, as text or as the hold's # `printf` Bash call (grep, not python, so it still works without python3). verdicts="$(jq -r 'select(.message.role? == "assistant") | .message.content? // [] | .[]? | if .type? == "text" then .text? // empty elif .type? == "tool_use" and .name? == "Bash" and ((.input.command? // "") | test("(^|\\n|;|&&)\\s*printf\\s[^\\n]*LinearB: ")) then .input.command else empty end' "$transcript" 2>/dev/null \ - | grep -E 'LinearB: .+ (LOW|MEDIUM|HIGH) effort' || true)" + | grep -E "$GRADED_RE" || true)" [ -n "$verdicts" ] || { dbg "$event: skip: skill ran but no verdict line yet"; exit 0; } re='LinearB: (.+) (LOW|MEDIUM|HIGH) effort' +# Current shape: the change against the model's effort, e.g. "Raised effort (medium → high; …)" or "Kept effort (high; …)". +re_change='LinearB: (.+) (Reduced|Kept|Raised) effort \((low|medium|high|xhigh|max)( (→|->) (low|medium|high|xhigh|max))?' er='\(([^\)]*)\)' -# Parse a verdict line into repo/effort/evidence globals. +# Parse a verdict line into repo/effort/change/target/evidence globals; effort stays LOW/MEDIUM/HIGH. parse_verdict() { # $1 = line - vrepo=""; veffort=""; vev="" - [[ "$1" =~ $re ]] || return 1 - vrepo="${BASH_REMATCH[1]}"; veffort="${BASH_REMATCH[2]}" + vrepo=""; veffort=""; vchange=""; vtarget=""; vev="" + if [[ "$1" =~ $re_change ]]; then + vrepo="${BASH_REMATCH[1]}"; vtarget="${BASH_REMATCH[6]:-${BASH_REMATCH[3]}}" + case "${BASH_REMATCH[2]}" in Reduced) vchange=reduced ;; Kept) vchange=kept ;; *) vchange=raised ;; esac + case "$vtarget" in low) veffort=LOW ;; medium) veffort=MEDIUM ;; *) veffort=HIGH ;; esac + elif [[ "$1" =~ $re ]]; then + vrepo="${BASH_REMATCH[1]}"; veffort="${BASH_REMATCH[2]}" + else + return 1 + fi vrepo="${vrepo%% —}"; vrepo="${vrepo%%—}"; vrepo="${vrepo%"${vrepo##*[![:space:]]}"}" [[ "$1" =~ $er ]] && vev="${BASH_REMATCH[1]}" + if [ -n "$vchange" ]; then case "$vev" in *\;*) vev="${vev#*; }" ;; *) vev="" ;; esac; fi return 0 } effort_val() { case "$1" in LOW) echo 1 ;; MEDIUM) echo 2 ;; HIGH) echo 3 ;; *) echo 0 ;; esac; } @@ -363,6 +376,7 @@ report_decisions() { --arg effort "$veffort" --arg repo "$vrepo" --arg ev "$vev" \ --arg email "$email" --arg repourl "$repo_url" --arg branch "$branch" \ --arg actual "$effort_actual" --arg ticket "$ticket" --arg sname "$session_name" \ + --arg change "$vchange" --arg target "$vtarget" \ --arg gdur "$grading_duration" --arg model "$model" --arg gtok "$grading_tokens" --arg pver "$plugin_version" --arg idsrc "$identity_source" \ --arg backfill "$backfill" ' { @@ -376,6 +390,7 @@ report_decisions() { (if $ev != "" then {evidence: $ev} else {} end) + (if $branch != "" then {branch: $branch} else {} end) + (if $actual != "" then {actual_effort: $actual} else {} end) + + (if $change != "" then {effort_change: $change, target_effort: $target} else {} end) + (if $ticket != "" then {ticket: $ticket} else {} end) + (if $sname != "" then {session_name: $sname} else {} end) + (if $gdur != "" then {grading_duration_s: $gdur} else {} end) + @@ -397,7 +412,7 @@ report_decisions # event == SessionEnd, graded: TOKENS event — the whole-session coding_tokens, attributed # to the final (last) verdict. value 0 so it doesn't double-count on grade charts; the # effort_level tag is kept for stratified coding_tokens comparison. -last_verdict="$(printf '%s\n' "$verdicts" | grep -E 'LinearB: .+ (LOW|MEDIUM|HIGH) effort' | tail -1)" +last_verdict="$(printf '%s\n' "$verdicts" | grep -E "$GRADED_RE" | tail -1)" [ -n "$last_verdict" ] || exit 0 parse_verdict "$last_verdict" || exit 0 key="$(printf 'tokens|%s|%s' "$vrepo" "$veffort" | sha1)" diff --git a/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh b/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh index 943b9bb..4df71e5 100755 --- a/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh +++ b/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh @@ -58,8 +58,8 @@ skill_ran() { # Is the LATEST advisor Skill call still missing its verdict (assistant text, or the # hold's `printf` Bash call)? Prints "pending" or "ok"; prints nothing on parse failure. -VERDICT_RE='LinearB: .+ ((LOW|MEDIUM|HIGH) effort|skipped \(not a code task\))' -GRADED_RE='LinearB: .+ (LOW|MEDIUM|HIGH) effort' +VERDICT_RE='LinearB: .+ (((Reduced|Kept|Raised) effort \((low|medium|high|xhigh|max)|(LOW|MEDIUM|HIGH) effort)|skipped \(not a code task\))' +GRADED_RE='LinearB: .+ ((Reduced|Kept|Raised) effort \((low|medium|high|xhigh|max)|(LOW|MEDIUM|HIGH) effort)' verdict_state() { # $1 = the verdict regex required (default: a grade or a skip) jq -rs --arg n "agentic-advisor" --arg re "${1:-$VERDICT_RE}" ' [.[] | select(.message.role? == "assistant") | .message.content? // [] | .[]?] as $c @@ -107,7 +107,7 @@ set_marker() { # $1 = a directory inside the repo # Imperative, silent guidance — no permission question, no announcement. skill_file="$(cd "$(dirname "$0")/.." 2>/dev/null && pwd)/skills/agentic-advisor/SKILL.md" skill_hint=""; [ -f "$skill_file" ] && skill_hint=" If the skill is not in your skill list, read $skill_file and follow it." -guidance="MANDATORY FIRST STEP for this code task: before you read files, search, explore, spawn any subagent, or edit anything, you MUST invoke the \`agentic-advisor\` skill (via the Skill tool) to grade this repo's effort level. Do not start the task, do not touch code, until you have run it and printed its verdict line verbatim in this shape: > LinearB: — effort () — (a hook parses that line; a prose summary is not recorded). This is required, not optional — do not skip it even if the task seems obvious. Invoke it directly (no need to ask the user or announce it). It degrades gracefully: if LinearB is unavailable, default to MEDIUM effort and continue. If the task will not change any code, print > LinearB: — skipped (not a code task) instead of a grade.$skill_hint" +guidance="MANDATORY FIRST STEP for this code task: before you read files, search, explore, spawn any subagent, or edit anything, you MUST invoke the \`agentic-advisor\` skill (via the Skill tool) to grade this repo's effort level. Do not start the task, do not touch code, until you have run it and printed its verdict line verbatim in this shape: > LinearB: — effort ( → ; ) — , or > LinearB: — Kept effort (; ) — (a hook parses that line; a prose summary is not recorded). This is required, not optional — do not skip it even if the task seems obvious. Invoke it directly (no need to ask the user or announce it). It degrades gracefully: if LinearB is unavailable, work at medium effort and continue. If the task will not change any code, print > LinearB: — skipped (not a code task) instead of a grade.$skill_hint" # Mark handled (this session+repo) and emit additionalContext for the event. emit() { # $1 = hookEventName, $2 = context text @@ -173,7 +173,7 @@ case "$event" in state="$(verdict_state)" if [ -n "$state" ] && : > "$vdone" 2>/dev/null && [ "$state" = "pending" ]; then dbg "PreToolUse: reminded the agent to print the verdict line" - note "LinearB: your effort grade isn't recorded yet. It's only recorded from the verdict line, not from thinking, so print it now (a prose summary doesn't count): > LinearB: — effort () — . If you aren't writing text, run: printf '%s\\n' '> LinearB: — effort () — '. If no code will change, print > LinearB: — skipped (not a code task) instead." + note "LinearB: your effort grade isn't recorded yet. It's only recorded from the verdict line, not from thinking, so print it now (a prose summary doesn't count): > LinearB: — effort ( → ; ) — , or > LinearB: — Kept effort (; ) — . If you aren't writing text, run: printf '%s\\n' '> LinearB: — '. If no code will change, print > LinearB: — skipped (not a code task) instead." fi fi fi @@ -234,9 +234,9 @@ case "$event" in printf '%s' "$((n + 1))" > "$held" 2>/dev/null || exit 0 dbg "PreToolUse: held $tool until a verdict line exists (hold $((n + 1)))" if skill_ran; then - reason="Your effort grade isn't recorded yet: print the verdict line itself, not a prose summary — > LinearB: — effort () — (or run: printf '%s\\n' '> LinearB: …'). A skipped (not a code task) line doesn't count once you change code. Then make the same call again." + reason="Your effort grade isn't recorded yet: print the verdict line itself, not a prose summary — > LinearB: — effort ( → ; ) — , or > LinearB: — Kept effort (; ) — (or run: printf '%s\\n' '> LinearB: …'). A skipped (not a code task) line doesn't count once you change code. Then make the same call again." else - reason="One quick first step before ${what}: run the \`agentic-advisor\` skill to grade this repo's effort.$skill_hint Print its verdict line (> LinearB: — effort (…) — …), then make the same call again; this hold repeats until it's printed. If LinearB isn't available it just defaults to MEDIUM." + reason="One quick first step before ${what}: run the \`agentic-advisor\` skill to grade this repo's effort.$skill_hint Print its verdict line (> LinearB: — effort ( → ; ) — , or > LinearB: — Kept effort (; ) — ), then make the same call again; this hold repeats until it's printed. If LinearB isn't available it just works at medium effort." fi [ "$what" = "this edit" ] || reason="$reason Tell the subagent the effort level in its prompt, so the delegated work follows it." deny "$reason" diff --git a/plugins/agentic-advisor/skills/agentic-advisor/SKILL.md b/plugins/agentic-advisor/skills/agentic-advisor/SKILL.md index 69c4807..4f58e84 100644 --- a/plugins/agentic-advisor/skills/agentic-advisor/SKILL.md +++ b/plugins/agentic-advisor/skills/agentic-advisor/SKILL.md @@ -11,21 +11,22 @@ Run this at the start of a code task to load the repo's current health signals, ## Off switch (check first) -Unless the user asked for this skill by name, your first Bash call starts with `printf 'enabled=%s\n' "${LINEARB_AGENTIC_ADVISOR_ENABLED:-}"`. If it prints `0`, `false`, `off` or `no` (any case), the user turned this skill off: stop here, print no verdict, and carry on with the task normally. +Your first Bash call starts with `printf 'enabled=%s effort=%s\n' "${LINEARB_AGENTIC_ADVISOR_ENABLED:-}" "${CLAUDE_EFFORT:-}"`. `effort` is the model's current effort level, used by the verdict (see Effort Change). Unless the user asked for this skill by name, if `enabled` prints `0`, `false`, `off` or `no` (any case), the user turned this skill off: stop here, print no verdict, and carry on with the task normally. ## Output contract (read this first) Right after the repo sweep, your first assistant text **must be this verdict line, verbatim in shape**, as a markdown blockquote: -> LinearB: — effort () — +> LinearB: — effort ( → ; ) — +> LinearB: — Kept effort (; ) — -- **A hook parses this exact line** to record the decision. Prose such as "repo health looks solid… I'll treat it as HIGH effort" is not recognized: the grade is lost and nothing is reported. -- The level is one word in capitals (`LOW`, `MEDIUM` or `HIGH`) followed immediately by the word `effort`. +- **A hook parses this exact line** to record the decision. Prose such as "repo health looks solid… I'll work harder" is not recognized: the decision is lost and nothing is reported. +- `Raised effort` or `Reduced effort`, then in the parentheses the model's current level, `→`, and the level to work at; `Kept effort` shows only the current level. Levels are lowercase (`low`, `medium`, `high`, `xhigh`, `max`); see Effort Change. - Explanation may follow on later lines, but the verdict line comes first and is never reworded. - **No code will change?** Print `> LinearB: — skipped (not a code task)` instead of a grade and stop (see the cheap exits). Nothing is reported. -- If you aren't writing visible text at this point, record it with one Bash call instead: `printf '%s\n' '> LinearB: — effort () — '`. If neither exists, a hook reminds you on your next file read or edit. +- If you aren't writing visible text at this point, record it with one Bash call instead: `printf '%s\n' '> LinearB: — effort ( → ; ) — '` (or `Kept effort (; )`). If neither exists, a hook reminds you on your next file read or edit. -Example: `> LinearB: api-service — HIGH effort (rework 3.4%, calm repo — but auth refactor with wide blast radius) — small behavior-preserving steps, tests green, confirm scope first.` +Example: `> LinearB: api-service — Raised effort (medium → high; rework 3.4%, calm repo — but auth refactor with wide blast radius) — small behavior-preserving steps, tests green, confirm scope first.` ## Access (read first) @@ -58,7 +59,7 @@ Two checks that prevent wasted work — do them before any LinearB call: f="${XDG_CACHE_HOME:-$HOME/.cache}/agentic-advisor/verdicts/$h.json" # private per-user dir, not shared /tmp cat "$f" 2>/dev/null # read BEFORE any LinearB call ``` - - **On a hit** (file exists, **is owned by you** — check `[ -O "$f" ]` and ignore it otherwise, since a cache file someone else wrote could fake a LOW grade — AND `computed_at` is within the last 24h — or the file's mtime is <24h old): **skip the repo-health LinearB calls** (repo list, measurements, incidents/search) and reuse `repo_effort`/`repo_evidence` **as the repo-health axis only**. You MUST still grade **task-complexity and change-area (files) fresh for the current task**, then combine — the cache replaces the phase-1 *data*, NOT the final verdict. Note it — **keep the standard `… effort (…)` shape** (with the literal word `effort`) so the reporter still captures the decision: `LinearB: — effort (repo-health cached h ago; task + files fresh)`. (The ticket lookup under Task Complexity is per task, not repo health, so it still runs on a hit.) + - **On a hit** (file exists, **is owned by you** — check `[ -O "$f" ]` and ignore it otherwise, since a cache file someone else wrote could fake a LOW grade — AND `computed_at` is within the last 24h — or the file's mtime is <24h old): **skip the repo-health LinearB calls** (repo list, measurements, incidents/search) and reuse `repo_effort`/`repo_evidence` **as the repo-health axis only**. You MUST still grade **task-complexity and change-area (files) fresh for the current task**, then combine — the cache replaces the phase-1 *data*, NOT the final verdict. Note it — **keep the standard verdict shape** so the reporter still captures the decision: `LinearB: — effort (; repo-health cached h ago; task + files fresh)`. (The ticket lookup under Task Complexity is per task, not repo health, so it still runs on a hit.) - **On a miss/stale:** compute the repo-health verdict fresh, then **write the cache atomically** (a per-process temp file then `mv` — atomic on the same filesystem, and the `$$` suffix means two concurrent sessions never share a temp file): ```text (umask 077; mkdir -p "${XDG_CACHE_HOME:-$HOME/.cache}/agentic-advisor/verdicts") @@ -144,6 +145,16 @@ The rework bands match **LinearB's own benchmark** (ELITE <3% and STRONG 3-6% LOW effort is the reward for *confirmed* health — it requires positive evidence the repo is calm. When signals are missing or a tool errors, do **not** assume healthy: fall back to **MEDIUM** and continue (never block the task). +### Effort Change (what the verdict says) + +LOW / MEDIUM / HIGH is your internal grade; the verdict states it as a change against the model's current effort (`effort=` from your first Bash call). The grade's level is `low`, `medium` or `high`. Rank the model's levels low < medium < high < xhigh < max; an empty `effort` counts as `medium`. + +- Grade's level above the current → `Raised effort ( → ; …)`. +- Grade's level below the current → `Reduced effort ( → ; …)`. +- Same → `Kept effort (; …)`. A HIGH grade on `xhigh` or `max` is also `Kept effort (; …)`: the advisor never raises past `high`, and never reduces fragile work. + +Then work at the grade's level as described below. + ## Task Complexity (a third effort axis) Repo health + file history tell you how **fragile the ground** is — not how **hard the task** is. A calm repo (LOW health signals) can still host a demanding change, and that change deserves care *regardless* of how healthy the repo is. Grading purely from repo signals **under-invests on big tasks in calm repos** — the agent goes light and the human ends up fixing the output. @@ -168,14 +179,14 @@ key="$(printf '%s' "$key" | tr '[:lower:]' '[:upper:]')" # lookups are case-se ``` - **Raise, never lower.** Priority `Highest`/`Critical`/`Blocker`, or type `bug`/`incident`/`hotfix`, means the task is at least **MEDIUM**; a `Highest`/`Critical` bug or incident means **HIGH**. A large estimate (`points` >= 8) raises the task axis one notch. A `Low` priority or small estimate never lowers the grade below what the prompt and repo already show. -- **Name it in the verdict** when it drove the grade, e.g. `… HIGH effort (repo calm — rework 1.2% — but ticket ABC-123 per LinearB: Highest-priority bug)`. +- **Name it in the verdict** when it drove the grade, e.g. `… Raised effort (medium → high; repo calm — rework 1.2% — but ticket ABC-123 per LinearB: Highest-priority bug)`. - **Freshness:** ticket data comes from LinearB's Jira sync and can lag Jira slightly. Treat it as a hint; that's also why it only ever raises the grade. - **Labels:** a `security`, `incident`, `hotfix`, `migration` or `breaking` label means the task is at least **MEDIUM**; any other label is context only. - **HTTP 401/403 → the token rule in Access applies** (grade MEDIUM, tell the user to refresh `LINEARB_API_TOKEN`), even on a cache hit, where this may be the only call. **Empty output, any other failure, or no key → ignore it** and grade the task from the prompt as usual. Never block, and don't retry. Same LOW/MEDIUM/HIGH scale: **LOW** = small, localized, well-specified, little logic; **MEDIUM** = a normal feature/fix (several files, real logic, a few edge cases); **HIGH** = large, cross-cutting, multi-repo, tricky, ambiguous, or high blast-radius. -**Combine all axes: `effort = max(repo-health, change-area, task-complexity)`** — any one being high raises the bar, and **name which axis drove it** (e.g. `MEDIUM effort (repo calm — rework 1%, no incidents — but the task spans 3 components with shared state)`). This keeps the token win — a *routine* change to a calm repo still grades LOW — while stopping the under-investment on demanding tasks in calm repos. +**Combine all axes: `effort = max(repo-health, change-area, task-complexity)`** — any one being high raises the bar, and **name which axis drove it** (e.g. `Kept effort (medium; repo calm — rework 1%, no incidents — but the task spans 3 components with shared state)`). This keeps the token win — a *routine* change to a calm repo still grades LOW — while stopping the under-investment on demanding tasks in calm repos. **Re-assess as the task unfolds:** if it turns out to touch more repos/files or more logic than first expected, **bump the effort mid-task and say so** — effort tracks the real task, not the first impression. @@ -245,13 +256,13 @@ Phase 2 was only ever deferred because it needs the target file paths. So: - **If the target file(s) are already known at task start** — named in the prompt, or unambiguous from cwd — run the phase-2 probe **together with the repo sweep** and fold both into **one combined verdict line** (no separate follow-up): ```text - > LinearB: — effort (rework X% ; target : ~N lines rewritten/90d) — . + > LinearB: — effort (; rework X% ; target : ~N lines rewritten/90d) — . ``` This is the reliable path: it's part of the same up-front measurement, not a "remember later" step. - **Only if the file cannot be known without exploring** (vague prompt) — emit the repo verdict first, explore to find the files, then before your first `Edit`/`Write` report phase 2. **Which line you print depends on whether phase 2 changes the effort** (recall `effort = max(repo-health, change-area, task)`, so a fix/revert hotspot can raise it above the repo level): - **If phase 2 CHANGES the combined effort** — re-emit the **full verdict line** with the new level, so it stands as the corrected decision (this is the line that gets recorded; without it, telemetry keeps the stale phase-1 level): ```text - > LinearB: — effort (repo rework X% — but target : ~N lines rewritten/90d) — . + > LinearB: — effort (; repo rework X% — but target : ~N lines rewritten/90d) — . ``` - **If phase 2 does NOT change the level** — it's pure targeting context; print the short follow-up: ```text @@ -264,7 +275,7 @@ Phase 2 was only ever deferred because it needs the target file paths. So: The effort level governs **code changes only**: it never lowers the care, accuracy or thoroughness of reading, analysis, answers or writing. -Use the effort level to guide the implementation silently. The aim is to **spend effort proportional to repo health** — go light (saving tokens and time) on calm repos, dig in on fragile ones. Don't waste effort. +Work to the definition of the level you landed on; Raised / Kept / Reduced only says how it compares with the model's setting. `Kept effort (xhigh)` or `(max)` follows HIGH. Use the effort level to guide the implementation silently. The aim is to **spend effort proportional to repo health** — go light (saving tokens and time) on calm repos, dig in on fragile ones. Don't waste effort. LOW effort — repo confirmed healthy, go light and token-efficient (an active repo can still be LOW — busy ≠ risky): @@ -294,24 +305,26 @@ HIGH effort — repo looks fragile, be careful: The moment the **repo sweep** (including the ticket lookup, if any) returns — and **before any other tool call, and before spawning any Explore/subagent, reading, searching, `ls`/`find`, or editing code** — your next output MUST be this exact verdict line, **formatted as a markdown blockquote** (start it with `> ` so the terminal renders it with the left bar / emphasis and it stands out). Emit it as assistant text — do NOT wrap it in an `echo`/shell command or code fence (that hides it in a collapsed tool block). The one exception is the exact `printf '%s\n' '> LinearB: …'` Bash call from the Output contract, when you aren't writing visible text or the hook asks for it. Do not paraphrase it into prose like "LOW effort signals confirmed". This is the **repo-level verdict**; it is fully knowable from the sweep alone, so nothing else may happen first. Format: ```text -> LinearB: — effort () — +> LinearB: — effort ( → ; ) — +> LinearB: — Kept effort (; ) — ``` Rules: -- The line begins with `> ` (blockquote), then literally `LinearB: `, the real repo name, the CAPS effort level, and the word `effort`. +- The line begins with `> ` (blockquote), then literally `LinearB: `, the real repo name, then `Raised effort`, `Reduced effort` or `Kept effort` and the levels in the parentheses (see Effort Change). - `` = the actual figures (rework %, unreviewed merges, incident note), not a summary. -- The `— ` clause is required for MEDIUM/HIGH, optional for LOW. +- The `— ` clause is required for MEDIUM/HIGH grades, optional for LOW. - Emit it once, as the first line, before any exploration. -- **Fold phase 2 into this line when the target file is already known** (named in the prompt or obvious from cwd): run its probe alongside the sweep and include the file finding in this same verdict, e.g. `… HIGH effort (rework 9.5% NEEDS FOCUS; target PaymentForm.tsx: ~140 lines rewritten/90d over 6 commits) — …`. See "Prefer running phase 2 with phase 1" above. Only when the file can't be known without exploring do you emit this repo-level line first and add the phase-2 line after exploration (before the first edit). Never delay this repo-level verdict to wait on exploration. +- **Fold phase 2 into this line when the target file is already known** (named in the prompt or obvious from cwd): run its probe alongside the sweep and include the file finding in this same verdict, e.g. `… Raised effort (medium → high; rework 9.5% NEEDS FOCUS; target PaymentForm.tsx: ~140 lines rewritten/90d over 6 commits) — …`. See "Prefer running phase 2 with phase 1" above. Only when the file can't be known without exploring do you emit this repo-level line first and add the phase-2 line after exploration (before the first edit). Never delay this repo-level verdict to wait on exploration. Concrete examples (fill in the real repo name and actual figures): ```text -> LinearB: api-service — LOW effort (healthy: rework 0.4%, no incidents, 0 unreviewed merges). -> LinearB: payments-service — MEDIUM effort (rework 2.2%, prod OOM incident ~3wk ago, resolved) — keeping the change focused with edge-case tests. -> LinearB: data-pipeline — HIGH effort (rework 7.9%, NEEDS FOCUS tier) — smallest viable change, defensive guards + failure-mode tests. -> LinearB: — MEDIUM effort (signals unavailable) — defaulting to regular coding. -> LinearB: — MEDIUM effort (LINEARB_API_TOKEN not set — LinearB signals and usage reporting are off) — defaulting to regular coding. +> LinearB: api-service — Reduced effort (high → low; healthy: rework 0.4%, no incidents, 0 unreviewed merges). +> LinearB: payments-service — Kept effort (medium; rework 2.2%, prod OOM incident ~3wk ago, resolved) — keeping the change focused with edge-case tests. +> LinearB: data-pipeline — Raised effort (medium → high; rework 7.9%, NEEDS FOCUS tier) — smallest viable change, defensive guards + failure-mode tests. +> LinearB: data-pipeline — Kept effort (max; rework 7.9%, NEEDS FOCUS tier) — smallest viable change, defensive guards + failure-mode tests. +> LinearB: — Kept effort (medium; signals unavailable) — defaulting to regular coding. +> LinearB: — Reduced effort (high → medium; LINEARB_API_TOKEN not set — LinearB signals and usage reporting are off) — defaulting to regular coding. ``` Keep this skill lightweight. Do not print a full report unless the user asks for one. diff --git a/tests/agentic-advisor/hooks_test.sh b/tests/agentic-advisor/hooks_test.sh index cc5e637..5a4fabc 100755 --- a/tests/agentic-advisor/hooks_test.sh +++ b/tests/agentic-advisor/hooks_test.sh @@ -52,7 +52,7 @@ nudged() { [ "$rc" -eq 0 ] && jq -e '.hookSpecificOutput.additionalContext | co reminded() { [ "$rc" -eq 0 ] && jq -e '(.hookSpecificOutput.permissionDecision == null) and (.hookSpecificOutput.additionalContext | contains("verdict line"))' <<<"$out" >/dev/null; } denied() { [ "$rc" -eq 0 ] && jq -e '.hookSpecificOutput.permissionDecision == "deny"' <<<"$out" >/dev/null; } add_skill() { printf '%s\n' '{"timestamp":"2026-01-01T00:00:00Z","message":{"role":"assistant","model":"m","usage":{"output_tokens":10},"content":[{"type":"tool_use","name":"Skill","input":{"skill":"agentic-advisor:agentic-advisor"}}]}}' >> "$TRANSCRIPT"; } -add_verdict() { printf '%s\n' '{"timestamp":"2026-01-01T00:00:05Z","message":{"role":"assistant","model":"m","usage":{"output_tokens":20},"content":[{"type":"text","text":"> LinearB: widgets — MEDIUM effort (rework 6.5%) — focused diff."}]}}' >> "$TRANSCRIPT"; } +add_verdict() { printf '%s\n' '{"timestamp":"2026-01-01T00:00:05Z","message":{"role":"assistant","model":"m","usage":{"output_tokens":20},"content":[{"type":"text","text":"> LinearB: widgets — Kept effort (medium; rework 6.5%) — focused diff."}]}}' >> "$TRANSCRIPT"; } # The loaded SKILL.md lands in the transcript as a user message, example verdict line included. add_skill_body() { printf '%s\n' '{"type":"user","isMeta":true,"message":{"role":"user","content":[{"type":"text","text":"Example: > LinearB: api-service — HIGH effort (rework 3.4%) — small steps."}]}}' >> "$TRANSCRIPT"; } # The fallback: the agent reads the skill's SKILL.md instead of calling the Skill tool. @@ -63,12 +63,21 @@ add_verdict_for() { printf '{"message":{"role":"assistant","content":[{"type":"t ncalls() { find "$T/curl" -type f | wc -l | tr -d ' '; } await_calls() { local _; for _ in 1 2 3 4 5 6 7 8 9 10; do [ "$(ncalls)" -ge "$1" ] && return; sleep 0.2; done; } call_json() { cat "$T/curl"/call.* 2>/dev/null; } +one_call() { [ "$(ncalls)" = 1 ]; } no_calls() { [ "$rc" -eq 0 ] && [ "$(ncalls)" = 0 ]; } one_decision() { [ "$rc" -eq 0 ] && [ "$(ncalls)" = 1 ] && call_json | jq -e '.value == 2 and .tags.effort_level == "MEDIUM" and .tags.phase == "decision" and .entity.repo_url == "https://github.com/acme/widgets.git"' >/dev/null } backfilled() { call_json | jq -se 'any(.[]; .tags.phase == "decision" and .tags.backfill == "session_end")' >/dev/null; } +raised_decision() { + [ "$rc" -eq 0 ] && [ "$(ncalls)" = 1 ] && call_json | jq -e '.value == 3 and .tags.effort_level == "HIGH" and .tags.effort_change == "raised" and .tags.target_effort == "high" and .tags.evidence == "rework 9%"' >/dev/null +} +old_shape_decision() { + [ "$rc" -eq 0 ] && [ "$(ncalls)" = 1 ] && call_json | jq -e '.tags.effort_level == "LOW" and (.tags | has("effort_change") | not)' >/dev/null +} +add_line() { printf '{"message":{"role":"assistant","content":[{"type":"text","text":"%s"}]}}\n' "$1" >> "$TRANSCRIPT"; } + # --- trigger: UserPromptSubmit --- setup run "$TRIGGER" UserPromptSubmit "$(prompt 'fix the login bug LINBEE-123')" @@ -293,5 +302,167 @@ LINEARB_AGENTIC_ADVISOR_ENABLED=false run "$REPORT" SessionEnd check "report: LINEARB_AGENTIC_ADVISOR_ENABLED=false -> nothing reported" no_calls teardown +# --- report: Reduced/Kept/Raised shape --- +kept_max() { + [ "$rc" -eq 0 ] && [ "$(ncalls)" = 1 ] && call_json | jq -e '.value == 3 and .tags.effort_change == "kept" and .tags.target_effort == "max" and .tags.evidence == "rework 7.9%"' >/dev/null +} +setup +add_skill; add_line "> LinearB: widgets — Kept effort (max; rework 7.9%) — small steps." +run "$REPORT" Stop; await_calls 1 +check "report: Kept effort (max) -> HIGH, kept, target max" kept_max +teardown + +setup +add_skill; add_line "> LinearB: widgets — Raised effort (medium → high; rework 9%) — small steps." +run "$REPORT" Stop; await_calls 1 +check "report: Raised effort (medium → high) -> HIGH, raised, target high, evidence without levels" raised_decision +teardown +setup +add_skill; add_line "> LinearB: widgets — LOW effort (healthy)." +run "$REPORT" Stop; await_calls 1 +check "report: the old LOW effort shape still reports, with no effort_change" old_shape_decision +teardown +setup +add_skill; add_line "> LinearB: widgets — Reduced effort (high → low; healthy)." +run "$TRIGGER" PreToolUse "$(tool Edit app.js)" +check "trigger: a Reduced effort verdict releases the hold" silent +teardown + +# --- report: matrix of Reduced/Kept/Raised lines --- +matrix_ok() { # $1 change $2 target $3 evidence $4 level $5 value + [ "$rc" -eq 0 ] && [ "$(ncalls)" = 1 ] && call_json | jq -e --arg c "$1" --arg t "$2" --arg e "$3" --arg l "$4" --argjson v "$5" \ + '.value == $v and .tags.effort_level == $l and .tags.effort_change == $c and .tags.target_effort == $t and ((.tags.evidence // "") == $e) and .tags.repo == "widgets"' >/dev/null +} +setup +add_skill; add_line "> LinearB: widgets — Kept effort (low; rework 1%) — go." +run "$REPORT" Stop; await_calls 1 +check "matrix: ept effort (low; rework 1%) — go. -> kept/low/LOW" matrix_ok 'kept' 'low' 'rework 1%' 'LOW' 1 +teardown +setup +add_skill; add_line "> LinearB: widgets — Kept effort (medium; rework 1%) — go." +run "$REPORT" Stop; await_calls 1 +check "matrix: ept effort (medium; rework 1%) — go. -> kept/medium/MEDIUM" matrix_ok 'kept' 'medium' 'rework 1%' 'MEDIUM' 2 +teardown +setup +add_skill; add_line "> LinearB: widgets — Kept effort (high; rework 1%) — go." +run "$REPORT" Stop; await_calls 1 +check "matrix: ept effort (high; rework 1%) — go. -> kept/high/HIGH" matrix_ok 'kept' 'high' 'rework 1%' 'HIGH' 3 +teardown +setup +add_skill; add_line "> LinearB: widgets — Kept effort (xhigh; rework 1%) — go." +run "$REPORT" Stop; await_calls 1 +check "matrix: ept effort (xhigh; rework 1%) — go. -> kept/xhigh/HIGH" matrix_ok 'kept' 'xhigh' 'rework 1%' 'HIGH' 3 +teardown +setup +add_skill; add_line "> LinearB: widgets — Kept effort (max; rework 1%) — go." +run "$REPORT" Stop; await_calls 1 +check "matrix: ept effort (max; rework 1%) — go. -> kept/max/HIGH" matrix_ok 'kept' 'max' 'rework 1%' 'HIGH' 3 +teardown +setup +add_skill; add_line "> LinearB: widgets — Raised effort (low → medium; rework 9%) — careful." +run "$REPORT" Stop; await_calls 1 +check "matrix: aised effort (low → medium; rework 9%) — careful -> raised/medium/MEDIUM" matrix_ok 'raised' 'medium' 'rework 9%' 'MEDIUM' 2 +teardown +setup +add_skill; add_line "> LinearB: widgets — Raised effort (low → high; rework 9%) — careful." +run "$REPORT" Stop; await_calls 1 +check "matrix: aised effort (low → high; rework 9%) — careful. -> raised/high/HIGH" matrix_ok 'raised' 'high' 'rework 9%' 'HIGH' 3 +teardown +setup +add_skill; add_line "> LinearB: widgets — Reduced effort (medium → low; healthy) — lean." +run "$REPORT" Stop; await_calls 1 +check "matrix: educed effort (medium → low; healthy) — lean. -> reduced/low/LOW" matrix_ok 'reduced' 'low' 'healthy' 'LOW' 1 +teardown +setup +add_skill; add_line "> LinearB: widgets — Raised effort (medium → high; rework 9%) — careful." +run "$REPORT" Stop; await_calls 1 +check "matrix: aised effort (medium → high; rework 9%) — carefu -> raised/high/HIGH" matrix_ok 'raised' 'high' 'rework 9%' 'HIGH' 3 +teardown +setup +add_skill; add_line "> LinearB: widgets — Reduced effort (high → low; healthy) — lean." +run "$REPORT" Stop; await_calls 1 +check "matrix: educed effort (high → low; healthy) — lean. -> reduced/low/LOW" matrix_ok 'reduced' 'low' 'healthy' 'LOW' 1 +teardown +setup +add_skill; add_line "> LinearB: widgets — Reduced effort (high → medium; healthy) — lean." +run "$REPORT" Stop; await_calls 1 +check "matrix: educed effort (high → medium; healthy) — lean. -> reduced/medium/MEDIUM" matrix_ok 'reduced' 'medium' 'healthy' 'MEDIUM' 2 +teardown +setup +add_skill; add_line "> LinearB: widgets — Reduced effort (xhigh → low; healthy) — lean." +run "$REPORT" Stop; await_calls 1 +check "matrix: educed effort (xhigh → low; healthy) — lean. -> reduced/low/LOW" matrix_ok 'reduced' 'low' 'healthy' 'LOW' 1 +teardown +setup +add_skill; add_line "> LinearB: widgets — Reduced effort (xhigh → medium; healthy) — lean." +run "$REPORT" Stop; await_calls 1 +check "matrix: educed effort (xhigh → medium; healthy) — lean. -> reduced/medium/MEDIUM" matrix_ok 'reduced' 'medium' 'healthy' 'MEDIUM' 2 +teardown +setup +add_skill; add_line "> LinearB: widgets — Reduced effort (xhigh → high; healthy) — lean." +run "$REPORT" Stop; await_calls 1 +check "matrix: educed effort (xhigh → high; healthy) — lean. -> reduced/high/HIGH" matrix_ok 'reduced' 'high' 'healthy' 'HIGH' 3 +teardown +setup +add_skill; add_line "> LinearB: widgets — Reduced effort (max → low; healthy) — lean." +run "$REPORT" Stop; await_calls 1 +check "matrix: educed effort (max → low; healthy) — lean. -> reduced/low/LOW" matrix_ok 'reduced' 'low' 'healthy' 'LOW' 1 +teardown +setup +add_skill; add_line "> LinearB: widgets — Reduced effort (max → medium; healthy) — lean." +run "$REPORT" Stop; await_calls 1 +check "matrix: educed effort (max → medium; healthy) — lean. -> reduced/medium/MEDIUM" matrix_ok 'reduced' 'medium' 'healthy' 'MEDIUM' 2 +teardown +setup +add_skill; add_line "> LinearB: widgets — Reduced effort (max → high; healthy) — lean." +run "$REPORT" Stop; await_calls 1 +check "matrix: educed effort (max → high; healthy) — lean. -> reduced/high/HIGH" matrix_ok 'reduced' 'high' 'healthy' 'HIGH' 3 +teardown +setup +add_skill; add_line "> LinearB: widgets — Raised effort (medium -> high; rework 9%) — careful." +run "$REPORT" Stop; await_calls 1 +check "matrix: aised effort (medium -> high; rework 9%) — caref -> raised/high/HIGH" matrix_ok 'raised' 'high' 'rework 9%' 'HIGH' 3 +teardown +setup +add_skill; add_line "> LinearB: widgets — Kept effort (medium) — go." +run "$REPORT" Stop; await_calls 1 +check "matrix: ept effort (medium) — go. -> kept/medium/MEDIUM" matrix_ok 'kept' 'medium' '' 'MEDIUM' 2 +teardown +setup +add_skill; add_line "> LinearB: widgets — Raised effort (low → medium; ticket ABC-1: Highest bug; rework 2%)" +run "$REPORT" Stop; await_calls 1 +check "matrix: aised effort (low → medium; ticket ABC-1: Highes -> raised/medium/MEDIUM" matrix_ok 'raised' 'medium' 'ticket ABC-1: Highest bug; rework 2%' 'MEDIUM' 2 +teardown +setup +add_skill; add_line "Looked at the signals.\n\n> LinearB: widgets — Reduced effort (max → low; healthy) — lean.\n\nNext I will edit." +run "$REPORT" Stop; await_calls 1 +check "matrix: \n\n> LinearB: widgets — Reduced effort (max → l -> reduced/low/LOW" matrix_ok 'reduced' 'low' 'healthy' 'LOW' 1 +teardown +setup +add_skill; printf '%s\n' '{"message":{"role":"assistant","content":[{"type":"tool_use","name":"Bash","input":{"command":"printf '"'"'%s\\n'"'"' '"'"'> LinearB: widgets — Raised effort (low → high; rework 9%) — careful.'"'"'"}}]}}' >> "$TRANSCRIPT" +run "$REPORT" Stop; await_calls 1 +check "matrix: printf-form Raised line is reported" matrix_ok raised high 'rework 9%' HIGH 3 +teardown +setup +add_skill; add_line "> LinearB: widgets — Raised effort (medium → high; x) — y." +run "$TRIGGER" PreToolUse "$(tool Edit app.js)" +check "matrix: trigger releases hold on Raised" silent +teardown +setup +add_skill; add_line "> LinearB: widgets — Kept effort (max; x) — y." +run "$TRIGGER" PreToolUse "$(tool Edit app.js)" +check "matrix: trigger releases hold on Kept (max)" silent +teardown +setup +add_skill; add_line "> LinearB: widgets — Raised the effort because rework is high." +run "$TRIGGER" PreToolUse "$(tool Edit app.js)" +check "matrix: prose without the shape still holds" denied +teardown +setup +add_skill; add_line "> LinearB: widgets — Raised effort (medium → high; x) — y."; add_line "> LinearB: widgets — Raised effort (medium → high; x) — y." +run "$REPORT" Stop; await_calls 1; run "$REPORT" Stop; sleep 0.5 +check "matrix: the same decision is sent once" one_call +teardown + printf '\n%d passed, %d failed\n' "$pass" "$fail" [ "$fail" -eq 0 ] From 1f35db13f93765833caf1f4573aa263d03fe6e50 Mon Sep 17 00:00:00 2001 From: nivSwisa1 Date: Wed, 4 Nov 2026 15:07:29 +0200 Subject: [PATCH 2/2] fix: require complete verdict levels before releasing the hold or reporting (round 1) Co-Authored-By: Claude Opus 5.5 --- .../hooks/agentic-advisor-report.sh | 15 ++++++++++----- .../hooks/agentic-advisor-trigger.sh | 4 ++-- tests/agentic-advisor/hooks_test.sh | 10 ++++++++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/plugins/agentic-advisor/hooks/agentic-advisor-report.sh b/plugins/agentic-advisor/hooks/agentic-advisor-report.sh index 2f6d733..e9fe2eb 100755 --- a/plugins/agentic-advisor/hooks/agentic-advisor-report.sh +++ b/plugins/agentic-advisor/hooks/agentic-advisor-report.sh @@ -118,7 +118,7 @@ state="$marker_dir/${session_id:-default}.reported" touch "$state" 2>/dev/null || true # A graded verdict line: the Reduced/Kept/Raised shape, or the older LOW/MEDIUM/HIGH one. -GRADED_RE='LinearB: .+ ((Reduced|Kept|Raised) effort \((low|medium|high|xhigh|max)|(LOW|MEDIUM|HIGH) effort)' +GRADED_RE='LinearB: .+ ((Reduced|Raised) effort \((low|medium|high|xhigh|max) (→|->) (low|medium|high|xhigh|max)[;)]|Kept effort \((low|medium|high|xhigh|max)[;)]|(LOW|MEDIUM|HIGH) effort)' # Cheap fast-exits for the graded path, BEFORE the git + jq repo-resolution work — so a # per-turn Stop that has nothing new to report costs only a grep or two, not a full @@ -256,7 +256,7 @@ for line in open(sys.argv[1]): # verdict must be ASSISTANT-authored text AFTER the Skill call — not the # SKILL.md examples that load on invocation (they also match the pattern). elif role == "assistant" and skill is not None and verdict is None \ - and re.search(r"LinearB: .+ ((Reduced|Kept|Raised) effort \((low|medium|high|xhigh|max)|(LOW|MEDIUM|HIGH) effort)", (x.get("text") or "") if typ == "text" else + and re.search(r"LinearB: .+ ((Reduced|Raised) effort \((low|medium|high|xhigh|max) (→|->) (low|medium|high|xhigh|max)[;)]|Kept effort \((low|medium|high|xhigh|max)[;)]|(LOW|MEDIUM|HIGH) effort)", (x.get("text") or "") if typ == "text" else str((x.get("input") or {}).get("command", "")) if typ == "tool_use" and x.get("name") == "Bash" and re.search(r"(^|\n|;|&&)\s*printf\s[^\n]*LinearB: ", str((x.get("input") or {}).get("command", ""))) else ""): verdict = te @@ -337,15 +337,20 @@ verdicts="$(jq -r 'select(.message.role? == "assistant") | .message.content? // re='LinearB: (.+) (LOW|MEDIUM|HIGH) effort' # Current shape: the change against the model's effort, e.g. "Raised effort (medium → high; …)" or "Kept effort (high; …)". -re_change='LinearB: (.+) (Reduced|Kept|Raised) effort \((low|medium|high|xhigh|max)( (→|->) (low|medium|high|xhigh|max))?' +re_change='LinearB: (.+) (Reduced|Raised) effort \((low|medium|high|xhigh|max) (→|->) (low|medium|high|xhigh|max)[;)]' +re_kept='LinearB: (.+) Kept effort \((low|medium|high|xhigh|max)[;)]' er='\(([^\)]*)\)' # Parse a verdict line into repo/effort/change/target/evidence globals; effort stays LOW/MEDIUM/HIGH. parse_verdict() { # $1 = line vrepo=""; veffort=""; vchange=""; vtarget=""; vev="" if [[ "$1" =~ $re_change ]]; then - vrepo="${BASH_REMATCH[1]}"; vtarget="${BASH_REMATCH[6]:-${BASH_REMATCH[3]}}" - case "${BASH_REMATCH[2]}" in Reduced) vchange=reduced ;; Kept) vchange=kept ;; *) vchange=raised ;; esac + vrepo="${BASH_REMATCH[1]}"; vtarget="${BASH_REMATCH[5]}" + if [ "${BASH_REMATCH[2]}" = Reduced ]; then vchange=reduced; else vchange=raised; fi + elif [[ "$1" =~ $re_kept ]]; then + vrepo="${BASH_REMATCH[1]}"; vtarget="${BASH_REMATCH[2]}"; vchange=kept + fi + if [ -n "$vchange" ]; then case "$vtarget" in low) veffort=LOW ;; medium) veffort=MEDIUM ;; *) veffort=HIGH ;; esac elif [[ "$1" =~ $re ]]; then vrepo="${BASH_REMATCH[1]}"; veffort="${BASH_REMATCH[2]}" diff --git a/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh b/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh index 4df71e5..f4d2d20 100755 --- a/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh +++ b/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh @@ -58,8 +58,8 @@ skill_ran() { # Is the LATEST advisor Skill call still missing its verdict (assistant text, or the # hold's `printf` Bash call)? Prints "pending" or "ok"; prints nothing on parse failure. -VERDICT_RE='LinearB: .+ (((Reduced|Kept|Raised) effort \((low|medium|high|xhigh|max)|(LOW|MEDIUM|HIGH) effort)|skipped \(not a code task\))' -GRADED_RE='LinearB: .+ ((Reduced|Kept|Raised) effort \((low|medium|high|xhigh|max)|(LOW|MEDIUM|HIGH) effort)' +VERDICT_RE='LinearB: .+ (((Reduced|Raised) effort \((low|medium|high|xhigh|max) (→|->) (low|medium|high|xhigh|max)[;)]|Kept effort \((low|medium|high|xhigh|max)[;)]|(LOW|MEDIUM|HIGH) effort)|skipped \(not a code task\))' +GRADED_RE='LinearB: .+ ((Reduced|Raised) effort \((low|medium|high|xhigh|max) (→|->) (low|medium|high|xhigh|max)[;)]|Kept effort \((low|medium|high|xhigh|max)[;)]|(LOW|MEDIUM|HIGH) effort)' verdict_state() { # $1 = the verdict regex required (default: a grade or a skip) jq -rs --arg n "agentic-advisor" --arg re "${1:-$VERDICT_RE}" ' [.[] | select(.message.role? == "assistant") | .message.content? // [] | .[]?] as $c diff --git a/tests/agentic-advisor/hooks_test.sh b/tests/agentic-advisor/hooks_test.sh index 5a4fabc..f9b5c7b 100755 --- a/tests/agentic-advisor/hooks_test.sh +++ b/tests/agentic-advisor/hooks_test.sh @@ -463,6 +463,16 @@ add_skill; add_line "> LinearB: widgets — Raised effort (medium → high; x) run "$REPORT" Stop; await_calls 1; run "$REPORT" Stop; sleep 0.5 check "matrix: the same decision is sent once" one_call teardown +for bad in "> LinearB: widgets — Raised effort (medium" "> LinearB: widgets — Kept effort (high-level summary only)" "> LinearB: widgets — Raised effort (high; no target)"; do + setup + add_skill; add_line "$bad" + run "$TRIGGER" PreToolUse "$(tool Edit app.js)" + check "matrix: incomplete verdict still holds: $bad" denied + run "$REPORT" Stop; sleep 0.5 + check "matrix: incomplete verdict is not reported: $bad" no_calls + teardown +done + printf '\n%d passed, %d failed\n' "$pass" "$fail" [ "$fail" -eq 0 ]