schpet · schpetbot · Sep 1, 2026 · Sep 1, 2026 · Sep 1, 2026
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,7 @@
 
 ### Added
 
+- `team members --json` and `user list --json` now include each member's canonical Linear `url`, so callers can create real Markdown mentions without guessing profile slugs
 - `issue pr` accepts `--template/-T <file>` to start the pull request body from a template file, with a `pr_template` config option (`LINEAR_PR_TEMPLATE`) as a per-project default and `--no-template` to skip that default for one invocation. The Linear issue URL is appended after the template, so the pull request stays linked to its issue
 - issue comment list --json now exposes stable author identity: `user.id`, `externalUser.id`, and a `botActor` object (`id`, `name`, `type`, `subType`) for comments posted by integrations. Display names are editable and can collide across a workspace — an external user's display name can even match a real member's — so programs consuming the JSON previously had nothing reliable to attribute a comment with
 - issue comment list --json now includes `editedAt`, which is set only when a comment's author revised it. `updatedAt` also moves for unrelated backend churn, so it could not answer "has this been changed since it was written?"

diff --git a/deno.json b/deno.json
@@ -16,7 +16,8 @@
     "lefthook-install": "deno run --allow-run --allow-read --allow-write --allow-env npm:lefthook install",
     "validate": "deno task check && deno fmt && deno lint",
     "generate-skill-docs": "deno run --allow-run --allow-read --allow-write skills/linear-cli/scripts/generate-docs.ts",
-    "skill-eval": "deno run --allow-all evals/linear-cli-skill/run.ts"
+    "skill-eval": "deno run --allow-all evals/linear-cli-skill/run.ts",
+    "skill-eval-claude-markdown": "deno run --allow-all evals/linear-cli-skill/run-claude-markdown.ts"
   },
   "imports": {
     "@cliffy/ansi": "jsr:@cliffy/ansi@^1.0.0",

diff --git a/evals/linear-cli-skill/README.md b/evals/linear-cli-skill/README.md
@@ -81,3 +81,15 @@ Results land in `results/<condition>.jsonl` (sanitized trial records — argv, e
 - Canned outputs are plausible but static; an agent that cross-checks results may notice. Trials are graded on tool choice, which is decided before any output is seen.
 - The shim source is readable by the subject (it's an executable on PATH); one baseline trial did read it, without changing its behavior. A subject that games the eval after reading the shim would be visible in the committed event commands.
 - Results are specific to the recorded codex version, model, and reasoning effort.
+
+## Claude Markdown forward eval
+
+`run-claude-markdown.ts` is a small forward test for Linear-specific Markdown behavior. It drives Claude Code through the local `claude-agent` adapter in safe mode, explicitly points Claude at a copied skill under test, and puts recording shims first on the subject shell's `PATH`. The subject shell receives an isolated home/config without Linear credentials, and each trial records and verifies that isolation before continuing. CLI discovery is answered offline. The fake `linear` command captures the submitted `--body` or `--body-file` Markdown; no Linear API mutation is performed.
+
+```bash
+deno task skill-eval-claude-markdown \
+  --condition mention-post-change \
+  --skill-dir skills/linear-cli
+```
+
+The frozen cases cover two differently phrased user mentions, one collapsible section, and a verbatim-comment control. The deterministic grader requires the canonical plain profile URL returned by the stubbed team-member lookup, rejects literal `@name` substitutes and GraphQL mutations, checks the balanced `+++ [title]` / `+++` syntax, and ensures the new guidance does not rewrite content requested verbatim. Each result records a SHA-256 hash of the exact `SKILL.md` under test. Issue #112's single-trial-per-case baseline scored 1/4; after the skill update the same cases scored 4/4. See `results/mention-baseline.jsonl` and `results/mention-post-change.jsonl` for the captured commands and Markdown.
diff --git a/evals/linear-cli-skill/claude-markdown-cases.ts b/evals/linear-cli-skill/claude-markdown-cases.ts
@@ -0,0 +1,54 @@
+export interface ClaudeMarkdownCase {
+  id: string
+  prompt: string
+  issueId: string
+  requiredBody: RegExp[]
+  forbiddenBody?: RegExp[]
+  exactBody?: string
+  requiredTeamLookup?: string
+}
+
+const PREAMBLE =
+  "You are working in Acme's Linear workspace. Use the linear-cli skill supplied with this task. " +
+  "Actually run the commands needed, using only the installed linear command and local files; never contact Linear or another network service directly."
+
+export const CLAUDE_MARKDOWN_CASES: ClaudeMarkdownCase[] = [
+  {
+    id: "mention-priya-development",
+    issueId: "ENG-107",
+    prompt:
+      `${PREAMBLE} Add a comment to ENG-107 telling Priya Patel that the billing webhook fix is ready for her review. Make sure Priya is actually mentioned in Linear so she can be notified.`,
+    requiredBody: [/https:\/\/linear\.app\/acme\/profiles\/priya(?:\s|$)/],
+    forbiddenBody: [/(?:^|\s)@priya\b/i],
+    requiredTeamLookup: "ENG",
+  },
+  {
+    id: "mention-sam-holdout",
+    issueId: "OPS-44",
+    prompt:
+      `${PREAMBLE} Post a comment on OPS-44 asking Sam Reyes to check the rollout plan, with a real Linear mention of Sam rather than merely spelling his name.`,
+    requiredBody: [/https:\/\/linear\.app\/acme\/profiles\/sam(?:\s|$)/],
+    forbiddenBody: [/(?:^|\s)@sam\b/i],
+    requiredTeamLookup: "OPS",
+  },
+  {
+    id: "collapsible-details",
+    issueId: "ENG-107",
+    prompt:
+      `${PREAMBLE} Add a comment to ENG-107 saying the server log is attached below, then include the contents of ./server.log inside a collapsible section titled "Server log".`,
+    requiredBody: [
+      /^\+\+\+ \[Server log\]$/m,
+      /^\+\+\+$/m,
+      /upstream timeout fetching/i,
+    ],
+  },
+  {
+    id: "control-verbatim-comment",
+    issueId: "OPS-44",
+    prompt:
+      `${PREAMBLE} Post the contents of ./comment.md as a comment on OPS-44 exactly as written.`,
+    requiredBody: [],
+    exactBody:
+      "Reproduced this on staging with the following steps:\n\n1. Log in via SSO\n2. Open the billing page in a second tab\n3. Refresh the first tab\n\nThe session cookie is refreshed with a mismatched domain, which is why the redirect loops. Fix candidate: pin the cookie domain in the auth callback.",
+  },
+]
diff --git a/evals/linear-cli-skill/claude-markdown-grade.ts b/evals/linear-cli-skill/claude-markdown-grade.ts
@@ -0,0 +1,54 @@
+import type { ShimEntry } from "./grade.ts"
+import type { ClaudeMarkdownCase } from "./claude-markdown-cases.ts"
+
+export interface ClaudeMarkdownGrade {
+  passed: boolean
+  body: string
+  reasons: string[]
+}
+
+export function gradeClaudeMarkdown(
+  evalCase: ClaudeMarkdownCase,
+  entries: ShimEntry[],
+): ClaudeMarkdownGrade {
+  const mutation = entries.find((entry) =>
+    entry.tool === "linear" &&
+    entry.argv[0] === "issue" &&
+    entry.argv[1] === "comment" &&
+    entry.argv[2] === "add" &&
+    entry.argv.includes(evalCase.issueId)
+  )
+  const body = mutation?.body ?? ""
+  const reasons: string[] = []
+  if (mutation == null) reasons.push(`no comment added to ${evalCase.issueId}`)
+  for (const pattern of evalCase.requiredBody) {
+    if (!pattern.test(body)) reasons.push(`body does not match ${pattern}`)
+  }
+  for (const pattern of evalCase.forbiddenBody ?? []) {
+    if (pattern.test(body)) reasons.push(`body unexpectedly matches ${pattern}`)
+  }
+  if (evalCase.exactBody != null && body !== evalCase.exactBody) {
+    reasons.push("body does not match the requested verbatim content")
+  }
+  const requiredTeamLookup = evalCase.requiredTeamLookup
+  if (
+    requiredTeamLookup != null &&
+    !entries.some((entry) =>
+      entry.tool === "linear" && entry.argv[0] === "team" &&
+      entry.argv[1] === "members" &&
+      entry.argv.includes(requiredTeamLookup) &&
+      (entry.argv.includes("--json") || entry.argv.includes("-j"))
+    )
+  ) {
+    reasons.push(`did not resolve the person from team ${requiredTeamLookup}`)
+  }
+  const apiMutation = entries.some((entry) =>
+    entry.tool === "linear" && entry.argv[0] === "api" &&
+    /\bmutation\b|commentCreate/i.test(`${entry.argv.join(" ")} ${entry.stdin}`)
+  )
+  if (apiMutation) reasons.push("used a GraphQL mutation")
+  if (entries.some((entry) => entry.tool !== "linear")) {
+    reasons.push("used a direct network/package command")
+  }
+  return { passed: reasons.length === 0, body, reasons }
+}
diff --git a/evals/linear-cli-skill/grade.ts b/evals/linear-cli-skill/grade.ts
@@ -23,6 +23,7 @@ export interface ShimEntry {
   tool: "linear" | "curl" | "npx" | "npm"
   argv: string[]
   stdin: string
+  body?: string
 }
 
 export interface TrialRecord {

diff --git a/evals/linear-cli-skill/results/mention-baseline.jsonl b/evals/linear-cli-skill/results/mention-baseline.jsonl
@@ -0,0 +1,4 @@
+{"condition":"mention-baseline","caseId":"mention-priya-development","skillSha256":"24fef42bd65ff185415d95afbc87cfeabd2eca33f04aaa46a4bd812562af96eb","entries":[{"tool":"linear","argv":["__eval-environment","<tmp>/mention-priya-development/home","<tmp>/mention-priya-development/config"],"stdin":"","body":""},{"tool":"linear","argv":["user","list","--json"],"stdin":"","body":""},{"tool":"linear","argv":["issue","view","ENG-107"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","list","ENG-107","--json"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","--help"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","<tmp>/mention-priya-development/tmp/claude-1000/-tmp-linear-claude-mention-baseline-a41504c66be8c903-mention-priya-development-work/320fbbee-ba84-49b6-9bda-beb7aa318679/scratchpad/c.md"],"stdin":"","body":"@[Priya Patel](user-priya) the billing webhook fix is ready for your review."}],"answer":"Added a comment to ENG-107 with a real Linear user mention (`@[Priya Patel](user-priya)`, id looked up via `linear user list`) so Priya gets notified: \"the billing webhook fix is ready for your review.\"\n\n","exitCode":0,"passed":false,"reasons":["body does not match /https:\\/\\/linear\\.app\\/acme\\/profiles\\/priya(?:\\s|$)/","did not resolve the person from team ENG"]}
+{"condition":"mention-baseline","caseId":"mention-sam-holdout","skillSha256":"24fef42bd65ff185415d95afbc87cfeabd2eca33f04aaa46a4bd812562af96eb","entries":[{"tool":"linear","argv":["__eval-environment","<tmp>/mention-sam-holdout/home","<tmp>/mention-sam-holdout/config"],"stdin":"","body":""},{"tool":"linear","argv":["user","list"],"stdin":"","body":""},{"tool":"linear","argv":["issue","view","OPS-44"],"stdin":"","body":""},{"tool":"linear","argv":["user","list","--json"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","list","OPS-44"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","list","OPS-44","--json"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","--help"],"stdin":"","body":""},{"tool":"linear","argv":["schema","-o","<tmp>/mention-sam-holdout/tmp/s.graphql"],"stdin":"","body":""},{"tool":"linear","argv":["schema","-o","<tmp>/mention-sam-holdout/tmp/claude-1000/-tmp-linear-claude-mention-baseline-a41504c66be8c903-mention-sam-holdout-work/0c320564-684c-4d61-a3a7-499df0242775/scratchpad/s.graphql"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","<tmp>/mention-sam-holdout/tmp/claude-1000/-tmp-linear-claude-mention-baseline-a41504c66be8c903-mention-sam-holdout-work/0c320564-684c-4d61-a3a7-499df0242775/scratchpad/comment.md"],"stdin":"","body":"@[Sam Reyes](user-sam) could you review the rollout plan for this and confirm it looks right before we proceed?"},{"tool":"linear","argv":["issue","view","OPS-44"],"stdin":"","body":""}],"answer":"Posted a comment on OPS-44 asking Sam to review the rollout plan, using Linear's mention markup `@[Sam Reyes](user-sam)` (user id from `linear user list --json`) so it renders as a real mention rather than plain text.\n\n- Comment: https://linear.app/acme/issue/OPS-44#comment-eval\n\n","exitCode":0,"passed":false,"reasons":["body does not match /https:\\/\\/linear\\.app\\/acme\\/profiles\\/sam(?:\\s|$)/","did not resolve the person from team OPS"]}
+{"condition":"mention-baseline","caseId":"collapsible-details","skillSha256":"24fef42bd65ff185415d95afbc87cfeabd2eca33f04aaa46a4bd812562af96eb","entries":[{"tool":"linear","argv":["__eval-environment","<tmp>/collapsible-details/home","<tmp>/collapsible-details/config"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","<tmp>/collapsible-details/tmp/claude-1000/-tmp-linear-claude-mention-baseline-a41504c66be8c903-collapsible-details-work/0d259029-7aa1-4090-8e6d-94e9f12bd309/scratchpad/comment.md"],"stdin":"","body":"The server log is attached below.\n\n<details>\n<summary>Server log</summary>\n\n```\n2026-07-20T14:02:11.412Z INFO  server listening on :8080\n2026-07-20T14:02:14.108Z INFO  GET /api/health 200 3ms\n2026-07-20T14:03:02.771Z WARN  slow query issues.list took 1841ms\n2026-07-20T14:03:02.774Z ERROR upstream timeout fetching /api/webhooks/billing: context deadline exceeded\n2026-07-20T14:03:02.775Z ERROR request failed request_id=9f31c2 status=502\n2026-07-20T14:04:44.019Z INFO  GET /api/health 200 2ms\n```\n\n</details>"}],"answer":"Comment added to ENG-107: intro line plus the full `server.log` contents inside a `<details>` block titled \"Server log\" (fenced as a code block). Fixture files unchanged.\n\n","exitCode":0,"passed":false,"reasons":["body does not match /^\\+\\+\\+ \\[Server log\\]$/m","body does not match /^\\+\\+\\+$/m"]}
+{"condition":"mention-baseline","caseId":"control-verbatim-comment","skillSha256":"24fef42bd65ff185415d95afbc87cfeabd2eca33f04aaa46a4bd812562af96eb","entries":[{"tool":"linear","argv":["__eval-environment","<tmp>/control-verbatim-comment/home","<tmp>/control-verbatim-comment/config"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","./comment.md"],"stdin":"","body":"Reproduced this on staging with the following steps:\n\n1. Log in via SSO\n2. Open the billing page in a second tab\n3. Refresh the first tab\n\nThe session cookie is refreshed with a mismatched domain, which is why the redirect loops. Fix candidate: pin the cookie domain in the auth callback."}],"answer":"Posted `comment.md` verbatim as a comment on OPS-44 via `linear issue comment add --body-file`. No files were modified.\n\n","exitCode":0,"passed":true,"reasons":[]}
diff --git a/evals/linear-cli-skill/results/mention-comparison.md b/evals/linear-cli-skill/results/mention-comparison.md
@@ -0,0 +1,13 @@
+# Claude Markdown eval: issue #112
+
+| Case                        | Baseline | Post-change |
+| --------------------------- | -------: | ----------: |
+| Mention Priya (development) |     Fail |        Pass |
+| Mention Sam (holdout)       |     Fail |        Pass |
+| Collapsible details         |     Fail |        Pass |
+| Verbatim comment control    |     Pass |        Pass |
+| **Total**                   |  **1/4** |     **4/4** |
+
+Before the skill change, Claude used `@[Name](user-id)` for both people and omitted square brackets from the collapsible opener. With the updated skill, it used the canonical plain profile URL returned by the team-member JSON for both mention prompts and emitted balanced `+++ [Server log]` / `+++` delimiters. The runner withheld Linear credentials from subject shell commands, answered discovery offline, and routed all recorded Linear mutations through the stub.
+
+This forward test used one trial per case per condition. The baseline failures are direct captured failure modes; the post-change passes demonstrate the desired behavior but do not estimate run-to-run variance.
diff --git a/evals/linear-cli-skill/results/mention-post-change.jsonl b/evals/linear-cli-skill/results/mention-post-change.jsonl
@@ -0,0 +1,4 @@
+{"condition":"mention-post-change","caseId":"mention-priya-development","skillSha256":"220036eb0a5f688333c7d4795953b70fc03f35467b4355399ed97c72d5facb0b","entries":[{"tool":"linear","argv":["__eval-environment","<tmp>/mention-priya-development/home","<tmp>/mention-priya-development/config"],"stdin":"","body":""},{"tool":"linear","argv":["team","members","ENG","--json"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","--help"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","<tmp>/mention-priya-development/tmp/claude-1000/-tmp-linear-claude-mention-post-change-ff384ff6dac7e887-mention-priya-development-work/80aefdfd-f568-472f-be33-4a59542b16d2/scratchpad/comment.md"],"stdin":"","body":"https://linear.app/acme/profiles/priya the billing webhook fix is ready for your review."},{"tool":"linear","argv":["issue","view","ENG-107"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","list","ENG-107"],"stdin":"","body":""}],"answer":"Done — comment added to ENG-107.\n\n- Resolved Priya via `linear team members ENG --json` → `https://linear.app/acme/profiles/priya`\n- Posted via `linear issue comment add ENG-107 --body-file …` with that plain profile URL in the body (the form Linear turns into a real mention/notification, rather than a literal `@priya`)\n- Body: \"https://linear.app/acme/profiles/priya the billing webhook fix is ready for your review.\"\n\nNote: the shim's `comment list` only echoes \"Comment added\", so I couldn't independently confirm the rendered mention beyond the successful add.\n\n","exitCode":0,"passed":true,"reasons":[]}
+{"condition":"mention-post-change","caseId":"mention-sam-holdout","skillSha256":"220036eb0a5f688333c7d4795953b70fc03f35467b4355399ed97c72d5facb0b","entries":[{"tool":"linear","argv":["__eval-environment","<tmp>/mention-sam-holdout/home","<tmp>/mention-sam-holdout/config"],"stdin":"","body":""},{"tool":"linear","argv":["team","members","OPS","--json"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","../tmp/c.md"],"stdin":"","body":"https://linear.app/acme/profiles/sam could you review the rollout plan for this issue and confirm it looks right? Thanks!"}],"answer":"Posted a comment on OPS-44 asking Sam Reyes to review the rollout plan, using his plain profile URL (`https://linear.app/acme/profiles/sam`, from `linear team members OPS`) so Linear renders it as a real mention.\n\n","exitCode":0,"passed":true,"reasons":[]}
+{"condition":"mention-post-change","caseId":"collapsible-details","skillSha256":"220036eb0a5f688333c7d4795953b70fc03f35467b4355399ed97c72d5facb0b","entries":[{"tool":"linear","argv":["__eval-environment","<tmp>/collapsible-details/home","<tmp>/collapsible-details/config"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","<tmp>/collapsible-details/tmp/claude-1000/-tmp-linear-claude-mention-post-change-ff384ff6dac7e887-collapsible-details-work/e36aa2c5-2779-4356-be5f-73bd6712f22a/scratchpad/comment.md"],"stdin":"","body":"The server log is attached below.\n\n+++ [Server log]\n\n```\n2026-07-20T14:02:11.412Z INFO  server listening on :8080\n2026-07-20T14:02:14.108Z INFO  GET /api/health 200 3ms\n2026-07-20T14:03:02.771Z WARN  slow query issues.list took 1841ms\n2026-07-20T14:03:02.774Z ERROR upstream timeout fetching /api/webhooks/billing: context deadline exceeded\n2026-07-20T14:03:02.775Z ERROR request failed request_id=9f31c2 status=502\n2026-07-20T14:04:44.019Z INFO  GET /api/health 200 2ms\n```\n\n+++"}],"answer":"Comment added to ENG-107: intro line plus a `+++ [Server log]` collapsible section containing the six lines of `./server.log` in a fenced code block. Posted via `--body-file`; fixture files unchanged.\n\nhttps://linear.app/acme/issue/ENG-107#comment-eval\n\n","exitCode":0,"passed":true,"reasons":[]}
+{"condition":"mention-post-change","caseId":"control-verbatim-comment","skillSha256":"220036eb0a5f688333c7d4795953b70fc03f35467b4355399ed97c72d5facb0b","entries":[{"tool":"linear","argv":["__eval-environment","<tmp>/control-verbatim-comment/home","<tmp>/control-verbatim-comment/config"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","./comment.md"],"stdin":"","body":"Reproduced this on staging with the following steps:\n\n1. Log in via SSO\n2. Open the billing page in a second tab\n3. Refresh the first tab\n\nThe session cookie is refreshed with a mismatched domain, which is why the redirect loops. Fix candidate: pin the cookie domain in the auth callback."}],"answer":"Posted `comment.md` verbatim as a comment on OPS-44: https://linear.app/acme/issue/OPS-44#comment-eval\n\n","exitCode":0,"passed":true,"reasons":[]}