From 3f322301044ce4f56d07fc8b3b62c4a44a404e6c Mon Sep 17 00:00:00 2001 From: Peter Schilling Date: Mon, 31 Aug 2026 21:14:51 -0700 Subject: [PATCH 1/2] Teach agents to create real Linear mentions Expose canonical user URLs in team and workspace member JSON so the skill can resolve mentions without guessing profile slugs. Document team-first URL mentions and Linear collapsible syntax. Add a stubbed Claude forward eval that captures submitted Markdown. The frozen cases improved from 1/4 to 4/4 while preserving the verbatim-body control. Fixes #112 --- CHANGELOG.md | 1 + deno.json | 3 +- evals/linear-cli-skill/README.md | 12 + .../linear-cli-skill/claude-markdown-cases.ts | 54 +++++ .../linear-cli-skill/claude-markdown-grade.ts | 54 +++++ evals/linear-cli-skill/grade.ts | 1 + .../results/mention-baseline.jsonl | 4 + .../results/mention-comparison.md | 13 ++ .../results/mention-post-change.jsonl | 4 + evals/linear-cli-skill/run-claude-markdown.ts | 215 ++++++++++++++++++ evals/linear-cli-skill/shims/linear | 96 +++++++- skills/linear-cli/SKILL.md | 32 +++ skills/linear-cli/SKILL.template.md | 32 +++ src/utils/linear.ts | 2 + .../__snapshots__/team-members.test.ts.snap | 7 + test/commands/team/team-members.test.ts | 1 + .../user/__snapshots__/user-list.test.ts.snap | 7 + test/commands/user/user-list.test.ts | 1 + ...ar-cli-skill-claude-markdown-grade.test.ts | 77 +++++++ 19 files changed, 604 insertions(+), 12 deletions(-) create mode 100644 evals/linear-cli-skill/claude-markdown-cases.ts create mode 100644 evals/linear-cli-skill/claude-markdown-grade.ts create mode 100644 evals/linear-cli-skill/results/mention-baseline.jsonl create mode 100644 evals/linear-cli-skill/results/mention-comparison.md create mode 100644 evals/linear-cli-skill/results/mention-post-change.jsonl create mode 100644 evals/linear-cli-skill/run-claude-markdown.ts create mode 100644 test/evals/linear-cli-skill-claude-markdown-grade.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index d9801515..ccdd2f59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added +- `team members --json` and `user list --json` now include each member's canonical Linear `url`, so callers can create real Markdown mentions without guessing profile slugs - `issue pr` accepts `--template/-T ` to start the pull request body from a template file, with a `pr_template` config option (`LINEAR_PR_TEMPLATE`) as a per-project default and `--no-template` to skip that default for one invocation. The Linear issue URL is appended after the template, so the pull request stays linked to its issue - issue comment list --json now exposes stable author identity: `user.id`, `externalUser.id`, and a `botActor` object (`id`, `name`, `type`, `subType`) for comments posted by integrations. Display names are editable and can collide across a workspace — an external user's display name can even match a real member's — so programs consuming the JSON previously had nothing reliable to attribute a comment with - issue comment list --json now includes `editedAt`, which is set only when a comment's author revised it. `updatedAt` also moves for unrelated backend churn, so it could not answer "has this been changed since it was written?" diff --git a/deno.json b/deno.json index 08697eda..d26097bd 100644 --- a/deno.json +++ b/deno.json @@ -16,7 +16,8 @@ "lefthook-install": "deno run --allow-run --allow-read --allow-write --allow-env npm:lefthook install", "validate": "deno task check && deno fmt && deno lint", "generate-skill-docs": "deno run --allow-run --allow-read --allow-write skills/linear-cli/scripts/generate-docs.ts", - "skill-eval": "deno run --allow-all evals/linear-cli-skill/run.ts" + "skill-eval": "deno run --allow-all evals/linear-cli-skill/run.ts", + "skill-eval-claude-markdown": "deno run --allow-all evals/linear-cli-skill/run-claude-markdown.ts" }, "imports": { "@cliffy/ansi": "jsr:@cliffy/ansi@^1.0.0", diff --git a/evals/linear-cli-skill/README.md b/evals/linear-cli-skill/README.md index 9e8f32a2..55dc7046 100644 --- a/evals/linear-cli-skill/README.md +++ b/evals/linear-cli-skill/README.md @@ -81,3 +81,15 @@ Results land in `results/.jsonl` (sanitized trial records — argv, e - Canned outputs are plausible but static; an agent that cross-checks results may notice. Trials are graded on tool choice, which is decided before any output is seen. - The shim source is readable by the subject (it's an executable on PATH); one baseline trial did read it, without changing its behavior. A subject that games the eval after reading the shim would be visible in the committed event commands. - Results are specific to the recorded codex version, model, and reasoning effort. + +## Claude Markdown forward eval + +`run-claude-markdown.ts` is a small forward test for Linear-specific Markdown behavior. It drives Claude Code through the local `claude-agent` adapter in safe mode, explicitly points Claude at a copied skill under test, and puts recording shims first on the subject shell's `PATH`. The subject shell receives an isolated home/config without Linear credentials, and each trial records and verifies that isolation before continuing. CLI discovery is answered offline. The fake `linear` command captures the submitted `--body` or `--body-file` Markdown; no Linear API mutation is performed. + +```bash +deno task skill-eval-claude-markdown \ + --condition mention-post-change \ + --skill-dir skills/linear-cli +``` + +The frozen cases cover two differently phrased user mentions, one collapsible section, and a verbatim-comment control. The deterministic grader requires the canonical plain profile URL returned by the stubbed team-member lookup, rejects literal `@name` substitutes and GraphQL mutations, checks the balanced `+++ [title]` / `+++` syntax, and ensures the new guidance does not rewrite content requested verbatim. Each result records a SHA-256 hash of the exact `SKILL.md` under test. Issue #112's single-trial-per-case baseline scored 1/4; after the skill update the same cases scored 4/4. See `results/mention-baseline.jsonl` and `results/mention-post-change.jsonl` for the captured commands and Markdown. diff --git a/evals/linear-cli-skill/claude-markdown-cases.ts b/evals/linear-cli-skill/claude-markdown-cases.ts new file mode 100644 index 00000000..be77b7c6 --- /dev/null +++ b/evals/linear-cli-skill/claude-markdown-cases.ts @@ -0,0 +1,54 @@ +export interface ClaudeMarkdownCase { + id: string + prompt: string + issueId: string + requiredBody: RegExp[] + forbiddenBody?: RegExp[] + exactBody?: string + requiredTeamLookup?: string +} + +const PREAMBLE = + "You are working in Acme's Linear workspace. Use the linear-cli skill supplied with this task. " + + "Actually run the commands needed, using only the installed linear command and local files; never contact Linear or another network service directly." + +export const CLAUDE_MARKDOWN_CASES: ClaudeMarkdownCase[] = [ + { + id: "mention-priya-development", + issueId: "ENG-107", + prompt: + `${PREAMBLE} Add a comment to ENG-107 telling Priya Patel that the billing webhook fix is ready for her review. Make sure Priya is actually mentioned in Linear so she can be notified.`, + requiredBody: [/https:\/\/linear\.app\/acme\/profiles\/priya(?:\s|$)/], + forbiddenBody: [/(?:^|\s)@priya\b/i], + requiredTeamLookup: "ENG", + }, + { + id: "mention-sam-holdout", + issueId: "OPS-44", + prompt: + `${PREAMBLE} Post a comment on OPS-44 asking Sam Reyes to check the rollout plan, with a real Linear mention of Sam rather than merely spelling his name.`, + requiredBody: [/https:\/\/linear\.app\/acme\/profiles\/sam(?:\s|$)/], + forbiddenBody: [/(?:^|\s)@sam\b/i], + requiredTeamLookup: "OPS", + }, + { + id: "collapsible-details", + issueId: "ENG-107", + prompt: + `${PREAMBLE} Add a comment to ENG-107 saying the server log is attached below, then include the contents of ./server.log inside a collapsible section titled "Server log".`, + requiredBody: [ + /^\+\+\+ \[Server log\]$/m, + /^\+\+\+$/m, + /upstream timeout fetching/i, + ], + }, + { + id: "control-verbatim-comment", + issueId: "OPS-44", + prompt: + `${PREAMBLE} Post the contents of ./comment.md as a comment on OPS-44 exactly as written.`, + requiredBody: [], + exactBody: + "Reproduced this on staging with the following steps:\n\n1. Log in via SSO\n2. Open the billing page in a second tab\n3. Refresh the first tab\n\nThe session cookie is refreshed with a mismatched domain, which is why the redirect loops. Fix candidate: pin the cookie domain in the auth callback.", + }, +] diff --git a/evals/linear-cli-skill/claude-markdown-grade.ts b/evals/linear-cli-skill/claude-markdown-grade.ts new file mode 100644 index 00000000..ae4bea85 --- /dev/null +++ b/evals/linear-cli-skill/claude-markdown-grade.ts @@ -0,0 +1,54 @@ +import type { ShimEntry } from "./grade.ts" +import type { ClaudeMarkdownCase } from "./claude-markdown-cases.ts" + +export interface ClaudeMarkdownGrade { + passed: boolean + body: string + reasons: string[] +} + +export function gradeClaudeMarkdown( + evalCase: ClaudeMarkdownCase, + entries: ShimEntry[], +): ClaudeMarkdownGrade { + const mutation = entries.find((entry) => + entry.tool === "linear" && + entry.argv[0] === "issue" && + entry.argv[1] === "comment" && + entry.argv[2] === "add" && + entry.argv.includes(evalCase.issueId) + ) + const body = mutation?.body ?? "" + const reasons: string[] = [] + if (mutation == null) reasons.push(`no comment added to ${evalCase.issueId}`) + for (const pattern of evalCase.requiredBody) { + if (!pattern.test(body)) reasons.push(`body does not match ${pattern}`) + } + for (const pattern of evalCase.forbiddenBody ?? []) { + if (pattern.test(body)) reasons.push(`body unexpectedly matches ${pattern}`) + } + if (evalCase.exactBody != null && body !== evalCase.exactBody) { + reasons.push("body does not match the requested verbatim content") + } + const requiredTeamLookup = evalCase.requiredTeamLookup + if ( + requiredTeamLookup != null && + !entries.some((entry) => + entry.tool === "linear" && entry.argv[0] === "team" && + entry.argv[1] === "members" && + entry.argv.includes(requiredTeamLookup) && + (entry.argv.includes("--json") || entry.argv.includes("-j")) + ) + ) { + reasons.push(`did not resolve the person from team ${requiredTeamLookup}`) + } + const apiMutation = entries.some((entry) => + entry.tool === "linear" && entry.argv[0] === "api" && + /\bmutation\b|commentCreate/i.test(`${entry.argv.join(" ")} ${entry.stdin}`) + ) + if (apiMutation) reasons.push("used a GraphQL mutation") + if (entries.some((entry) => entry.tool !== "linear")) { + reasons.push("used a direct network/package command") + } + return { passed: reasons.length === 0, body, reasons } +} diff --git a/evals/linear-cli-skill/grade.ts b/evals/linear-cli-skill/grade.ts index 6165c73c..6b103a13 100644 --- a/evals/linear-cli-skill/grade.ts +++ b/evals/linear-cli-skill/grade.ts @@ -23,6 +23,7 @@ export interface ShimEntry { tool: "linear" | "curl" | "npx" | "npm" argv: string[] stdin: string + body?: string } export interface TrialRecord { diff --git a/evals/linear-cli-skill/results/mention-baseline.jsonl b/evals/linear-cli-skill/results/mention-baseline.jsonl new file mode 100644 index 00000000..a6e1db89 --- /dev/null +++ b/evals/linear-cli-skill/results/mention-baseline.jsonl @@ -0,0 +1,4 @@ +{"condition":"mention-baseline","caseId":"mention-priya-development","skillSha256":"24fef42bd65ff185415d95afbc87cfeabd2eca33f04aaa46a4bd812562af96eb","entries":[{"tool":"linear","argv":["__eval-environment","/mention-priya-development/home","/mention-priya-development/config"],"stdin":"","body":""},{"tool":"linear","argv":["user","list","--json"],"stdin":"","body":""},{"tool":"linear","argv":["issue","view","ENG-107"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","list","ENG-107","--json"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","--help"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","/mention-priya-development/tmp/claude-1000/-tmp-linear-claude-mention-baseline-a41504c66be8c903-mention-priya-development-work/320fbbee-ba84-49b6-9bda-beb7aa318679/scratchpad/c.md"],"stdin":"","body":"@[Priya Patel](user-priya) the billing webhook fix is ready for your review."}],"answer":"Added a comment to ENG-107 with a real Linear user mention (`@[Priya Patel](user-priya)`, id looked up via `linear user list`) so Priya gets notified: \"the billing webhook fix is ready for your review.\"\n\n","exitCode":0,"passed":false,"reasons":["body does not match /https:\\/\\/linear\\.app\\/acme\\/profiles\\/priya(?:\\s|$)/","did not resolve the person from team ENG"]} +{"condition":"mention-baseline","caseId":"mention-sam-holdout","skillSha256":"24fef42bd65ff185415d95afbc87cfeabd2eca33f04aaa46a4bd812562af96eb","entries":[{"tool":"linear","argv":["__eval-environment","/mention-sam-holdout/home","/mention-sam-holdout/config"],"stdin":"","body":""},{"tool":"linear","argv":["user","list"],"stdin":"","body":""},{"tool":"linear","argv":["issue","view","OPS-44"],"stdin":"","body":""},{"tool":"linear","argv":["user","list","--json"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","list","OPS-44"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","list","OPS-44","--json"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","--help"],"stdin":"","body":""},{"tool":"linear","argv":["schema","-o","/mention-sam-holdout/tmp/s.graphql"],"stdin":"","body":""},{"tool":"linear","argv":["schema","-o","/mention-sam-holdout/tmp/claude-1000/-tmp-linear-claude-mention-baseline-a41504c66be8c903-mention-sam-holdout-work/0c320564-684c-4d61-a3a7-499df0242775/scratchpad/s.graphql"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","/mention-sam-holdout/tmp/claude-1000/-tmp-linear-claude-mention-baseline-a41504c66be8c903-mention-sam-holdout-work/0c320564-684c-4d61-a3a7-499df0242775/scratchpad/comment.md"],"stdin":"","body":"@[Sam Reyes](user-sam) could you review the rollout plan for this and confirm it looks right before we proceed?"},{"tool":"linear","argv":["issue","view","OPS-44"],"stdin":"","body":""}],"answer":"Posted a comment on OPS-44 asking Sam to review the rollout plan, using Linear's mention markup `@[Sam Reyes](user-sam)` (user id from `linear user list --json`) so it renders as a real mention rather than plain text.\n\n- Comment: https://linear.app/acme/issue/OPS-44#comment-eval\n\n","exitCode":0,"passed":false,"reasons":["body does not match /https:\\/\\/linear\\.app\\/acme\\/profiles\\/sam(?:\\s|$)/","did not resolve the person from team OPS"]} +{"condition":"mention-baseline","caseId":"collapsible-details","skillSha256":"24fef42bd65ff185415d95afbc87cfeabd2eca33f04aaa46a4bd812562af96eb","entries":[{"tool":"linear","argv":["__eval-environment","/collapsible-details/home","/collapsible-details/config"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","/collapsible-details/tmp/claude-1000/-tmp-linear-claude-mention-baseline-a41504c66be8c903-collapsible-details-work/0d259029-7aa1-4090-8e6d-94e9f12bd309/scratchpad/comment.md"],"stdin":"","body":"The server log is attached below.\n\n
\nServer log\n\n```\n2026-07-20T14:02:11.412Z INFO server listening on :8080\n2026-07-20T14:02:14.108Z INFO GET /api/health 200 3ms\n2026-07-20T14:03:02.771Z WARN slow query issues.list took 1841ms\n2026-07-20T14:03:02.774Z ERROR upstream timeout fetching /api/webhooks/billing: context deadline exceeded\n2026-07-20T14:03:02.775Z ERROR request failed request_id=9f31c2 status=502\n2026-07-20T14:04:44.019Z INFO GET /api/health 200 2ms\n```\n\n
"}],"answer":"Comment added to ENG-107: intro line plus the full `server.log` contents inside a `
` block titled \"Server log\" (fenced as a code block). Fixture files unchanged.\n\n","exitCode":0,"passed":false,"reasons":["body does not match /^\\+\\+\\+ \\[Server log\\]$/m","body does not match /^\\+\\+\\+$/m"]} +{"condition":"mention-baseline","caseId":"control-verbatim-comment","skillSha256":"24fef42bd65ff185415d95afbc87cfeabd2eca33f04aaa46a4bd812562af96eb","entries":[{"tool":"linear","argv":["__eval-environment","/control-verbatim-comment/home","/control-verbatim-comment/config"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","./comment.md"],"stdin":"","body":"Reproduced this on staging with the following steps:\n\n1. Log in via SSO\n2. Open the billing page in a second tab\n3. Refresh the first tab\n\nThe session cookie is refreshed with a mismatched domain, which is why the redirect loops. Fix candidate: pin the cookie domain in the auth callback."}],"answer":"Posted `comment.md` verbatim as a comment on OPS-44 via `linear issue comment add --body-file`. No files were modified.\n\n","exitCode":0,"passed":true,"reasons":[]} diff --git a/evals/linear-cli-skill/results/mention-comparison.md b/evals/linear-cli-skill/results/mention-comparison.md new file mode 100644 index 00000000..b73dc573 --- /dev/null +++ b/evals/linear-cli-skill/results/mention-comparison.md @@ -0,0 +1,13 @@ +# Claude Markdown eval: issue #112 + +| Case | Baseline | Post-change | +| --------------------------- | -------: | ----------: | +| Mention Priya (development) | Fail | Pass | +| Mention Sam (holdout) | Fail | Pass | +| Collapsible details | Fail | Pass | +| Verbatim comment control | Pass | Pass | +| **Total** | **1/4** | **4/4** | + +Before the skill change, Claude used `@[Name](user-id)` for both people and omitted square brackets from the collapsible opener. With the updated skill, it used the canonical plain profile URL returned by the team-member JSON for both mention prompts and emitted balanced `+++ [Server log]` / `+++` delimiters. The runner withheld Linear credentials from subject shell commands, answered discovery offline, and routed all recorded Linear mutations through the stub. + +This forward test used one trial per case per condition. The baseline failures are direct captured failure modes; the post-change passes demonstrate the desired behavior but do not estimate run-to-run variance. diff --git a/evals/linear-cli-skill/results/mention-post-change.jsonl b/evals/linear-cli-skill/results/mention-post-change.jsonl new file mode 100644 index 00000000..2421f99a --- /dev/null +++ b/evals/linear-cli-skill/results/mention-post-change.jsonl @@ -0,0 +1,4 @@ +{"condition":"mention-post-change","caseId":"mention-priya-development","skillSha256":"220036eb0a5f688333c7d4795953b70fc03f35467b4355399ed97c72d5facb0b","entries":[{"tool":"linear","argv":["__eval-environment","/mention-priya-development/home","/mention-priya-development/config"],"stdin":"","body":""},{"tool":"linear","argv":["team","members","ENG","--json"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","--help"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","/mention-priya-development/tmp/claude-1000/-tmp-linear-claude-mention-post-change-ff384ff6dac7e887-mention-priya-development-work/80aefdfd-f568-472f-be33-4a59542b16d2/scratchpad/comment.md"],"stdin":"","body":"https://linear.app/acme/profiles/priya the billing webhook fix is ready for your review."},{"tool":"linear","argv":["issue","view","ENG-107"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","list","ENG-107"],"stdin":"","body":""}],"answer":"Done — comment added to ENG-107.\n\n- Resolved Priya via `linear team members ENG --json` → `https://linear.app/acme/profiles/priya`\n- Posted via `linear issue comment add ENG-107 --body-file …` with that plain profile URL in the body (the form Linear turns into a real mention/notification, rather than a literal `@priya`)\n- Body: \"https://linear.app/acme/profiles/priya the billing webhook fix is ready for your review.\"\n\nNote: the shim's `comment list` only echoes \"Comment added\", so I couldn't independently confirm the rendered mention beyond the successful add.\n\n","exitCode":0,"passed":true,"reasons":[]} +{"condition":"mention-post-change","caseId":"mention-sam-holdout","skillSha256":"220036eb0a5f688333c7d4795953b70fc03f35467b4355399ed97c72d5facb0b","entries":[{"tool":"linear","argv":["__eval-environment","/mention-sam-holdout/home","/mention-sam-holdout/config"],"stdin":"","body":""},{"tool":"linear","argv":["team","members","OPS","--json"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","../tmp/c.md"],"stdin":"","body":"https://linear.app/acme/profiles/sam could you review the rollout plan for this issue and confirm it looks right? Thanks!"}],"answer":"Posted a comment on OPS-44 asking Sam Reyes to review the rollout plan, using his plain profile URL (`https://linear.app/acme/profiles/sam`, from `linear team members OPS`) so Linear renders it as a real mention.\n\n","exitCode":0,"passed":true,"reasons":[]} +{"condition":"mention-post-change","caseId":"collapsible-details","skillSha256":"220036eb0a5f688333c7d4795953b70fc03f35467b4355399ed97c72d5facb0b","entries":[{"tool":"linear","argv":["__eval-environment","/collapsible-details/home","/collapsible-details/config"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","/collapsible-details/tmp/claude-1000/-tmp-linear-claude-mention-post-change-ff384ff6dac7e887-collapsible-details-work/e36aa2c5-2779-4356-be5f-73bd6712f22a/scratchpad/comment.md"],"stdin":"","body":"The server log is attached below.\n\n+++ [Server log]\n\n```\n2026-07-20T14:02:11.412Z INFO server listening on :8080\n2026-07-20T14:02:14.108Z INFO GET /api/health 200 3ms\n2026-07-20T14:03:02.771Z WARN slow query issues.list took 1841ms\n2026-07-20T14:03:02.774Z ERROR upstream timeout fetching /api/webhooks/billing: context deadline exceeded\n2026-07-20T14:03:02.775Z ERROR request failed request_id=9f31c2 status=502\n2026-07-20T14:04:44.019Z INFO GET /api/health 200 2ms\n```\n\n+++"}],"answer":"Comment added to ENG-107: intro line plus a `+++ [Server log]` collapsible section containing the six lines of `./server.log` in a fenced code block. Posted via `--body-file`; fixture files unchanged.\n\nhttps://linear.app/acme/issue/ENG-107#comment-eval\n\n","exitCode":0,"passed":true,"reasons":[]} +{"condition":"mention-post-change","caseId":"control-verbatim-comment","skillSha256":"220036eb0a5f688333c7d4795953b70fc03f35467b4355399ed97c72d5facb0b","entries":[{"tool":"linear","argv":["__eval-environment","/control-verbatim-comment/home","/control-verbatim-comment/config"],"stdin":"","body":""},{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","./comment.md"],"stdin":"","body":"Reproduced this on staging with the following steps:\n\n1. Log in via SSO\n2. Open the billing page in a second tab\n3. Refresh the first tab\n\nThe session cookie is refreshed with a mismatched domain, which is why the redirect loops. Fix candidate: pin the cookie domain in the auth callback."}],"answer":"Posted `comment.md` verbatim as a comment on OPS-44: https://linear.app/acme/issue/OPS-44#comment-eval\n\n","exitCode":0,"passed":true,"reasons":[]} diff --git a/evals/linear-cli-skill/run-claude-markdown.ts b/evals/linear-cli-skill/run-claude-markdown.ts new file mode 100644 index 00000000..95934208 --- /dev/null +++ b/evals/linear-cli-skill/run-claude-markdown.ts @@ -0,0 +1,215 @@ +import { parseArgs } from "@std/cli" +import { copy } from "@std/fs" +import { dirname, fromFileUrl, join, resolve } from "@std/path" +import { CLAUDE_MARKDOWN_CASES } from "./claude-markdown-cases.ts" +import { gradeClaudeMarkdown } from "./claude-markdown-grade.ts" +import type { ShimEntry } from "./grade.ts" + +const EVAL_DIR = dirname(fromFileUrl(import.meta.url)) + +interface EvalRecord { + condition: string + caseId: string + skillSha256: string + entries: ShimEntry[] + answer: string + exitCode: number + passed: boolean + reasons: string[] +} + +function sanitize(text: string, root: string): string { + return text.replaceAll(root, "") +} + +async function sha256(path: string): Promise { + const digest = await crypto.subtle.digest( + "SHA-256", + await Deno.readFile(path), + ) + return [...new Uint8Array(digest)].map((byte) => + byte.toString(16).padStart(2, "0") + ).join("") +} + +async function executableDir(name: string): Promise { + const output = await new Deno.Command("which", { + args: [name], + stdout: "piped", + stderr: "piped", + }).output() + if (!output.success) throw new Error(`required executable not found: ${name}`) + return dirname(new TextDecoder().decode(output.stdout).trim()) +} + +async function main(): Promise { + const flags = parseArgs(Deno.args, { + string: ["condition", "skill-dir", "cases", "out", "claude-agent-script"], + default: { effort: "low", "timeout-seconds": 600 }, + }) + if (flags.condition == null || flags["skill-dir"] == null) { + throw new Error( + "usage: run-claude-markdown.ts --condition NAME --skill-dir DIR [--cases id,id] [--out FILE]", + ) + } + const skillDir = resolve(flags["skill-dir"]) + const script = resolve( + flags["claude-agent-script"] ?? + join( + Deno.env.get("HOME") ?? "", + "repos/routines/skills/claude-agent/scripts/run.sh", + ), + ) + const selected = flags.cases == null + ? CLAUDE_MARKDOWN_CASES + : flags.cases.split(",").map((id) => { + const found = CLAUDE_MARKDOWN_CASES.find((candidate) => + candidate.id === id + ) + if (found == null) throw new Error(`unknown case: ${id}`) + return found + }) + const root = await Deno.makeTempDir({ + prefix: `linear-claude-${flags.condition}-`, + }) + const claudeDir = await executableDir("claude") + const jqDir = await executableDir("jq") + const skillSha256 = await sha256(join(skillDir, "SKILL.md")) + const records: EvalRecord[] = [] + const realHome = Deno.env.get("HOME") ?? "" + try { + for (const evalCase of selected) { + const trial = join(root, evalCase.id) + const work = join(trial, "work") + const tmp = join(trial, "tmp") + const fakeConfigHome = join(trial, "config") + const fakeHome = join(trial, "home") + const evalBin = join(trial, "bin") + const skillCopy = join(trial, "linear-cli-skill") + await Deno.mkdir(work, { recursive: true }) + await Deno.mkdir(tmp, { recursive: true }) + await Deno.mkdir(fakeConfigHome, { recursive: true }) + await Deno.mkdir(fakeHome, { recursive: true }) + await Deno.mkdir(evalBin, { recursive: true }) + await copy(join(EVAL_DIR, "fixtures"), work, { overwrite: true }) + await copy(skillDir, skillCopy, { overwrite: true }) + // Claude Code only accepts SHELL paths whose filename identifies a + // supported shell. Keep "bash" in this wrapper name deliberately. + const subjectShell = join(evalBin, "subject-bash") + await Deno.writeTextFile( + subjectShell, + `#!/bin/bash\nexport HOME=${fakeHome}\nexport XDG_CONFIG_HOME=${fakeConfigHome}\nexec /bin/bash "$@"\n`, + ) + await Deno.writeTextFile( + join(evalBin, "claude"), + `#!/bin/bash\nexport SHELL=${subjectShell}\nexec ${ + join(claudeDir, "claude") + } "$@"\n`, + ) + await Deno.chmod(subjectShell, 0o700) + await Deno.chmod(join(evalBin, "claude"), 0o700) + const shimLog = join(trial, "shim.jsonl") + const promptFile = join(trial, "prompt.txt") + await Deno.writeTextFile( + promptFile, + `First run: linear __eval-environment "$HOME" "$XDG_CONFIG_HOME"\nThis is a required isolation probe; stop if it fails.\n\nRead and follow the skill at ${ + join(skillCopy, "SKILL.md") + }.\n\n${evalCase.prompt}\n\nDo not edit the supplied fixture files. Report the result briefly.`, + ) + const command = new Deno.Command(script, { + args: [ + "--cwd", + work, + "--prompt-file", + promptFile, + "--effort", + String(flags.effort), + "--timeout", + String(flags["timeout-seconds"]), + "--trust-workspace", + "--safe-mode", + "--tools", + "Read,Write,Edit,Bash,Glob,Grep", + ], + clearEnv: true, + env: { + HOME: realHome, + PATH: `${join(EVAL_DIR, "shims")}:${evalBin}:${jqDir}:/usr/bin:/bin`, + SHELL: "/bin/bash", + TMPDIR: tmp, + CLAUDE_TMUX_SOCKET_DIR: join(trial, "tmux"), + LINEAR_SHIM_LOG: shimLog, + LINEAR_SHIM_OFFLINE: "1", + NO_COLOR: "1", + TERM: "dumb", + LANG: "C.UTF-8", + }, + cwd: work, + stdout: "piped", + stderr: "piped", + }) + const output = await command.output() + let entries: ShimEntry[] = [] + try { + entries = (await Deno.readTextFile(shimLog)).trim().split("\n") + .filter(Boolean).map((line) => JSON.parse(line) as ShimEntry) + } catch (error) { + if (!(error instanceof Deno.errors.NotFound)) throw error + } + const grade = gradeClaudeMarkdown(evalCase, entries) + const isolationProbe = entries.find((entry) => + entry.tool === "linear" && entry.argv[0] === "__eval-environment" + ) + entries = entries.map((entry) => ({ + ...entry, + argv: entry.argv.map((arg) => sanitize(arg, root)), + stdin: sanitize(entry.stdin, root), + body: entry.body == null ? undefined : sanitize(entry.body, root), + })) + const answer = sanitize(new TextDecoder().decode(output.stdout), root) + const stderr = new TextDecoder().decode(output.stderr) + const reasons = [...grade.reasons] + if ( + isolationProbe?.argv[1] !== fakeHome || + isolationProbe.argv[2] !== fakeConfigHome + ) { + reasons.push("subject shell did not use the isolated home/config") + } + if (!output.success) { + reasons.push( + `claude-agent exited ${output.code}: ${ + sanitize(stderr.slice(-300), root) + }`, + ) + } + records.push({ + condition: flags.condition, + caseId: evalCase.id, + skillSha256, + entries, + answer, + exitCode: output.code, + passed: reasons.length === 0, + reasons, + }) + console.log( + `${evalCase.id}: ${ + reasons.length === 0 ? "PASS" : `FAIL (${reasons.join("; ")})` + }`, + ) + } + const out = resolve( + flags.out ?? join(EVAL_DIR, "results", `${flags.condition}.jsonl`), + ) + await Deno.mkdir(dirname(out), { recursive: true }) + await Deno.writeTextFile( + out, + records.map((record) => JSON.stringify(record)).join("\n") + "\n", + ) + if (records.some((record) => !record.passed)) Deno.exitCode = 1 + } finally { + await Deno.remove(root, { recursive: true }) + } +} + +if (import.meta.main) await main() diff --git a/evals/linear-cli-skill/shims/linear b/evals/linear-cli-skill/shims/linear index f1089356..f01e28d3 100755 --- a/evals/linear-cli-skill/shims/linear +++ b/evals/linear-cli-skill/shims/linear @@ -9,7 +9,7 @@ set -u LOG="${LINEAR_SHIM_LOG:?LINEAR_SHIM_LOG must be set}" -REPO="${LINEAR_SHIM_REPO:?LINEAR_SHIM_REPO must be set}" +REPO="${LINEAR_SHIM_REPO:-}" # Capture GraphQL heredocs piped to `linear api` so grading can inspect the # query text. Other commands never read stdin. @@ -18,15 +18,74 @@ if [ "${1:-}" = "api" ] && [ ! -t 0 ]; then STDIN_DATA=$(cat) fi +# Capture Markdown at the CLI boundary so behavioral evals can grade what an +# agent actually submitted without letting the fake mutation touch Linear. +BODY="" +PREV="" +for ARG in "$@"; do + case "$PREV" in + --body | -b | --description | -d) + BODY="$ARG" + break + ;; + --body-file | --description-file) + if [ -r "$ARG" ]; then BODY=$(cat -- "$ARG"); fi + break + ;; + esac + case "$ARG" in + --body=* | --description=*) + BODY="${ARG#*=}" + break + ;; + --body-file=* | --description-file=*) + BODY_FILE="${ARG#*=}" + if [ -r "$BODY_FILE" ]; then BODY=$(cat -- "$BODY_FILE"); fi + break + ;; + esac + PREV="$ARG" +done + # Argv goes to jq on stdin, delimited by the ASCII unit separator (0x1f): # jq's --args treats dash-prefixed values (e.g. --team) as its own options, # and jq strings cannot represent NUL, so neither works for arbitrary argv. if [ "$#" -eq 0 ]; then - jq -cn --arg stdin "$STDIN_DATA" \ - '{tool: "linear", argv: [], stdin: $stdin}' >> "$LOG" + jq -cn --arg stdin "$STDIN_DATA" --arg body "$BODY" \ + '{tool: "linear", argv: [], stdin: $stdin, body: $body}' >> "$LOG" else - printf '%s\x1f' "$@" | jq -Rsc --arg stdin "$STDIN_DATA" \ - '{tool: "linear", argv: split("\u001f")[:-1], stdin: $stdin}' >> "$LOG" + printf '%s\x1f' "$@" | jq -Rsc --arg stdin "$STDIN_DATA" --arg body "$BODY" \ + '{tool: "linear", argv: split("\u001f")[:-1], stdin: $stdin, body: $body}' >> "$LOG" +fi + +# The Claude Markdown eval must be hermetic. Its subject can ask for discovery, +# but discovery is answered locally instead of executing the repository CLI. +# Keep this after logging so every attempted command remains auditable. +if [ "${LINEAR_SHIM_OFFLINE:-}" = "1" ]; then + if [ "${1:-}" = "__eval-environment" ]; then exit 0; fi + for arg in "$@"; do + case "$arg" in + --version) + printf 'linear 2.5.0 (eval shim)\n' + exit 0 + ;; + --help | -h) + printf 'Linear CLI eval shim. Dedicated commands accept the documented arguments.\n' + exit 0 + ;; + esac + done + if [ "${1:-}" = "schema" ]; then + OUTPUT="" + PREV="" + for arg in "$@"; do + if [ "$PREV" = "-o" ] || [ "$PREV" = "--output" ]; then OUTPUT="$arg"; fi + PREV="$arg" + done + SCHEMA='type User { id: ID! name: String! displayName: String! url: String! }' + if [ -n "$OUTPUT" ]; then printf '%s\n' "$SCHEMA" > "$OUTPUT"; else printf '%s\n' "$SCHEMA"; fi + exit 0 + fi fi # Discovery goes to the real CLI so help output is accurate for the skill @@ -34,11 +93,13 @@ fi for arg in "$@"; do case "$arg" in --help | -h | --version) + [ -n "$REPO" ] || { printf 'LINEAR_SHIM_REPO must be set outside offline mode\n' >&2; exit 1; } NO_COLOR=1 exec deno run --quiet --allow-all "$REPO/src/main.ts" "$@" ;; esac done if [ "${1:-}" = "schema" ]; then + [ -n "$REPO" ] || { printf 'LINEAR_SHIM_REPO must be set outside offline mode\n' >&2; exit 1; } NO_COLOR=1 exec deno run --quiet --allow-all "$REPO/src/main.ts" "$@" fi @@ -204,6 +265,25 @@ case "${1:-} ${2:-}" in "issue title") printf 'Fix login redirect loop\n' ;; + "user list") + if has_flag --json "$@" || has_flag -j "$@"; then + printf '%s\n' '{"nodes":[{"id":"user-priya","name":"Priya Patel","displayName":"priya","email":"priya@acme.test","active":true,"url":"https://linear.app/acme/profiles/priya"},{"id":"user-sam","name":"Sam Reyes","displayName":"sam","email":"sam@acme.test","active":true,"url":"https://linear.app/acme/profiles/sam"}],"pageInfo":{"hasNextPage":false,"endCursor":null}}' + else + printf 'Workspace Members (2):\n\npriya [PP]\n Email: priya@acme.test\n\nsam [SR]\n Email: sam@acme.test\n\n' + fi + ;; + "team members") + TEAM="${3:-ENG}" + if has_flag --json "$@" || has_flag -j "$@"; then + if [ "$TEAM" = "OPS" ]; then + printf '%s\n' '{"nodes":[{"id":"user-sam","name":"Sam Reyes","displayName":"sam","email":"sam@acme.test","active":true,"url":"https://linear.app/acme/profiles/sam"}],"pageInfo":{"hasNextPage":false,"endCursor":null}}' + else + printf '%s\n' '{"nodes":[{"id":"user-priya","name":"Priya Patel","displayName":"priya","email":"priya@acme.test","active":true,"url":"https://linear.app/acme/profiles/priya"}],"pageInfo":{"hasNextPage":false,"endCursor":null}}' + fi + else + printf 'Team Members (1):\n\n%s\n' "$TEAM" + fi + ;; "issue create") TITLE=$(flag_value "--title -t" "$@") || TITLE="New issue" PREFIX=$(flag_value "--team" "$@") || PREFIX="ENG" @@ -288,12 +368,6 @@ case "${1:-} ${2:-}" in "team states") printf 'Triage\nBacklog\nTodo\nIn Progress\nIn Review\nDone\nCanceled\n' ;; - "team members") - printf 'sam Sam Reyes\npriya Priya Patel\n' - ;; - "user list") - printf 'sam Sam Reyes sam@acme.test\npriya Priya Patel priya@acme.test\n' - ;; "label list") printf 'bug\ndocs\napi\ninfra\nsecurity\ntech-debt\n' ;; diff --git a/skills/linear-cli/SKILL.md b/skills/linear-cli/SKILL.md index eadf0ee9..e35af0ea 100644 --- a/skills/linear-cli/SKILL.md +++ b/skills/linear-cli/SKILL.md @@ -127,6 +127,38 @@ linear issue comment add ENG-123 --body-file /tmp/comment.md **Only use inline flags** (`--description`, `--body`) for simple, single-line content. +## Linear Markdown Features + +### Mention people and resources with plain URLs + +Linear turns a resource's **plain Linear URL** in Markdown into a linked mention. A literal `@name`, `@[Name](id)`, or a Markdown link such as `[Name](url)` does not create the same mention. Put the plain URL directly in the comment or description: + +```markdown +https://linear.app/yourworkspace/profiles/someuser could you review this? https://linear.app/yourworkspace/issue/ENG-123 is related. +``` + +Resolve people within the relevant team first. The team can usually be inferred from the issue identifier or current directory: + +```bash +linear team members ENG --json +``` + +Use the selected member's `url` field verbatim in the Markdown. If the intended person is not a member of that team, stop and confirm before searching the whole workspace with `linear user list --json`; mentioning someone outside the team is likely accidental. For issues, use `linear issue url ENG-123` and include that plain URL. + +### Add collapsible sections + +Open a collapsible section with `+++ [title]` and close it with `+++`: + +```markdown ++++ [Server log] + +Markdown content that is initially hidden. + ++++ +``` + +The square brackets around the title and the closing `+++` are required. + ## Available Commands Compact command list, generated from `linear --help`: diff --git a/skills/linear-cli/SKILL.template.md b/skills/linear-cli/SKILL.template.md index dc69dc4d..c3fc1f84 100644 --- a/skills/linear-cli/SKILL.template.md +++ b/skills/linear-cli/SKILL.template.md @@ -127,6 +127,38 @@ linear issue comment add ENG-123 --body-file /tmp/comment.md **Only use inline flags** (`--description`, `--body`) for simple, single-line content. +## Linear Markdown Features + +### Mention people and resources with plain URLs + +Linear turns a resource's **plain Linear URL** in Markdown into a linked mention. A literal `@name`, `@[Name](id)`, or a Markdown link such as `[Name](url)` does not create the same mention. Put the plain URL directly in the comment or description: + +```markdown +https://linear.app/yourworkspace/profiles/someuser could you review this? https://linear.app/yourworkspace/issue/ENG-123 is related. +``` + +Resolve people within the relevant team first. The team can usually be inferred from the issue identifier or current directory: + +```bash +linear team members ENG --json +``` + +Use the selected member's `url` field verbatim in the Markdown. If the intended person is not a member of that team, stop and confirm before searching the whole workspace with `linear user list --json`; mentioning someone outside the team is likely accidental. For issues, use `linear issue url ENG-123` and include that plain URL. + +### Add collapsible sections + +Open a collapsible section with `+++ [title]` and close it with `+++`: + +```markdown ++++ [Server log] + +Markdown content that is initially hidden. + ++++ +``` + +The square brackets around the title and the closing `+++` are required. + ## Available Commands Compact command list, generated from `linear --help`: diff --git a/src/utils/linear.ts b/src/utils/linear.ts index 3adcca24..5e472ca2 100644 --- a/src/utils/linear.ts +++ b/src/utils/linear.ts @@ -1801,6 +1801,7 @@ export async function getTeamMembers( admin owner isMe + url } pageInfo { hasNextPage @@ -1890,6 +1891,7 @@ export async function getOrganizationMembers( admin owner isMe + url } pageInfo { hasNextPage diff --git a/test/commands/team/__snapshots__/team-members.test.ts.snap b/test/commands/team/__snapshots__/team-members.test.ts.snap index 458ccec6..81fa3280 100644 --- a/test/commands/team/__snapshots__/team-members.test.ts.snap +++ b/test/commands/team/__snapshots__/team-members.test.ts.snap @@ -72,6 +72,7 @@ stdout: "admin": false, "owner": false, "isMe": true, + "url": "https://linear.app/acme/profiles/pat", "id": "u-pat", "displayName": "pat" }, @@ -90,6 +91,7 @@ stdout: "admin": true, "owner": true, "isMe": false, + "url": "https://linear.app/acme/profiles/zoe", "id": "u-zoe", "displayName": "zoe" } @@ -123,6 +125,7 @@ stdout: "admin": false, "owner": false, "isMe": false, + "url": "https://linear.app/acme/profiles/olduser", "id": "u-old", "displayName": "olduser" }, @@ -141,6 +144,7 @@ stdout: "admin": false, "owner": false, "isMe": true, + "url": "https://linear.app/acme/profiles/pat", "id": "u-pat", "displayName": "pat" }, @@ -159,6 +163,7 @@ stdout: "admin": true, "owner": true, "isMe": false, + "url": "https://linear.app/acme/profiles/zoe", "id": "u-zoe", "displayName": "zoe" } @@ -206,6 +211,7 @@ stdout: "admin": false, "owner": false, "isMe": false, + "url": "https://linear.app/acme/profiles/aaron", "id": "u-first", "displayName": "aaron" }, @@ -224,6 +230,7 @@ stdout: "admin": false, "owner": false, "isMe": false, + "url": "https://linear.app/acme/profiles/mona", "id": "u-mid", "displayName": "mona" } diff --git a/test/commands/team/team-members.test.ts b/test/commands/team/team-members.test.ts index 0d2cc9ef..dd93626b 100644 --- a/test/commands/team/team-members.test.ts +++ b/test/commands/team/team-members.test.ts @@ -26,6 +26,7 @@ function member( admin: false, owner: false, isMe: false, + url: `https://linear.app/acme/profiles/${overrides.displayName}`, ...overrides, } } diff --git a/test/commands/user/__snapshots__/user-list.test.ts.snap b/test/commands/user/__snapshots__/user-list.test.ts.snap index 85995c2c..ae5f211c 100644 --- a/test/commands/user/__snapshots__/user-list.test.ts.snap +++ b/test/commands/user/__snapshots__/user-list.test.ts.snap @@ -72,6 +72,7 @@ stdout: "admin": true, "owner": false, "isMe": true, + "url": "https://linear.app/acme/profiles/pat", "id": "u-pat", "displayName": "pat" }, @@ -90,6 +91,7 @@ stdout: "admin": false, "owner": true, "isMe": false, + "url": "https://linear.app/acme/profiles/zoe", "id": "u-zoe", "displayName": "zoe" } @@ -123,6 +125,7 @@ stdout: "admin": false, "owner": false, "isMe": false, + "url": "https://linear.app/acme/profiles/olduser", "id": "u-old", "displayName": "olduser" }, @@ -141,6 +144,7 @@ stdout: "admin": true, "owner": false, "isMe": true, + "url": "https://linear.app/acme/profiles/pat", "id": "u-pat", "displayName": "pat" }, @@ -159,6 +163,7 @@ stdout: "admin": false, "owner": true, "isMe": false, + "url": "https://linear.app/acme/profiles/zoe", "id": "u-zoe", "displayName": "zoe" } @@ -206,6 +211,7 @@ stdout: "admin": false, "owner": false, "isMe": false, + "url": "https://linear.app/acme/profiles/aaron", "id": "u-first", "displayName": "aaron" }, @@ -224,6 +230,7 @@ stdout: "admin": false, "owner": false, "isMe": false, + "url": "https://linear.app/acme/profiles/mona", "id": "u-mid", "displayName": "mona" } diff --git a/test/commands/user/user-list.test.ts b/test/commands/user/user-list.test.ts index 769e0d63..3e35d8c8 100644 --- a/test/commands/user/user-list.test.ts +++ b/test/commands/user/user-list.test.ts @@ -24,6 +24,7 @@ function member( admin: false, owner: false, isMe: false, + url: `https://linear.app/acme/profiles/${overrides.displayName}`, ...overrides, } } diff --git a/test/evals/linear-cli-skill-claude-markdown-grade.test.ts b/test/evals/linear-cli-skill-claude-markdown-grade.test.ts new file mode 100644 index 00000000..85a484bb --- /dev/null +++ b/test/evals/linear-cli-skill-claude-markdown-grade.test.ts @@ -0,0 +1,77 @@ +import { assertEquals } from "@std/assert" +import { CLAUDE_MARKDOWN_CASES } from "../../evals/linear-cli-skill/claude-markdown-cases.ts" +import { gradeClaudeMarkdown } from "../../evals/linear-cli-skill/claude-markdown-grade.ts" +import type { ShimEntry } from "../../evals/linear-cli-skill/grade.ts" + +const priya = CLAUDE_MARKDOWN_CASES[0] +const priyaLookup: ShimEntry = { + tool: "linear", + argv: ["team", "members", "ENG", "--json"], + stdin: "", + body: "", +} + +function comment(body: string): ShimEntry { + return { + tool: "linear", + argv: ["issue", "comment", "add", "ENG-107", "--body-file", "./comment.md"], + stdin: "", + body, + } +} + +Deno.test("Claude Markdown grade requires a plain profile URL mention", () => { + assertEquals( + gradeClaudeMarkdown(priya, [ + priyaLookup, + comment("https://linear.app/acme/profiles/priya please review"), + ]).passed, + true, + ) + assertEquals( + gradeClaudeMarkdown(priya, [priyaLookup, comment("@priya please review")]) + .passed, + false, + ) + assertEquals( + gradeClaudeMarkdown(priya, [ + priyaLookup, + comment("[Priya](https://linear.app/acme/profiles/priya) please review"), + ]).passed, + false, + ) +}) + +Deno.test("Claude Markdown grade rejects GraphQL mutations", () => { + assertEquals( + gradeClaudeMarkdown(priya, [ + priyaLookup, + comment("https://linear.app/acme/profiles/priya please review"), + { + tool: "linear", + argv: ["api"], + stdin: "mutation { commentCreate { success } }", + body: "", + }, + ]).passed, + false, + ) +}) + +Deno.test("Claude Markdown grade preserves a verbatim comment", () => { + const control = CLAUDE_MARKDOWN_CASES[3] + const entry: ShimEntry = { + tool: "linear", + argv: ["issue", "comment", "add", "OPS-44", "--body-file", "./comment.md"], + stdin: "", + body: control.exactBody, + } + assertEquals(gradeClaudeMarkdown(control, [entry]).passed, true) + assertEquals( + gradeClaudeMarkdown(control, [{ + ...entry, + body: `${control.exactBody}\n+++`, + }]).passed, + false, + ) +}) From 7c7f3e24bf20acfdd4c1b83c2e54f7874246fa3f Mon Sep 17 00:00:00 2001 From: Peter Schilling Date: Tue, 1 Sep 2026 08:01:37 -0700 Subject: [PATCH 2/2] Stabilize issue-mine ordering snapshot Use a fixed future update timestamp so this ordering-focused snapshot does not fail whenever the calendar day changes. --- .../issue/__snapshots__/issue-mine.test.ts.snap | 16 ++++++++-------- test/commands/issue/issue-mine.test.ts | 16 +++++++++------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/test/commands/issue/__snapshots__/issue-mine.test.ts.snap b/test/commands/issue/__snapshots__/issue-mine.test.ts.snap index 911fb7e9..6578e83f 100644 --- a/test/commands/issue/__snapshots__/issue-mine.test.ts.snap +++ b/test/commands/issue/__snapshots__/issue-mine.test.ts.snap @@ -37,14 +37,14 @@ stderr: snapshot[`Issue Mine Command - Groups Statuses In Linear's Order 1`] = ` stdout: -"◌ ID TITLE LABELS B E STATE UPDATED ---- ENG-1 Someday - Backlog 155 days ago ---- ENG-5 Next up - Todo 155 days ago ---- ENG-3 Doing it - In Progress 155 days ago ---- ENG-2 Reviewing - In Review 155 days ago ---- ENG-4 Shipped - Done 155 days ago ---- ENG-6 Rejected work - Rejected 155 days ago ---- ENG-7 From a future status - Paused 155 days ago +"◌ ID TITLE LABELS B E STATE UPDATED +--- ENG-1 Someday - Backlog just now +--- ENG-5 Next up - Todo just now +--- ENG-3 Doing it - In Progress just now +--- ENG-2 Reviewing - In Review just now +--- ENG-4 Shipped - Done just now +--- ENG-6 Rejected work - Rejected just now +--- ENG-7 From a future status - Paused just now " stderr: "" diff --git a/test/commands/issue/issue-mine.test.ts b/test/commands/issue/issue-mine.test.ts index e7d44309..862c7de3 100644 --- a/test/commands/issue/issue-mine.test.ts +++ b/test/commands/issue/issue-mine.test.ts @@ -808,6 +808,8 @@ Deno.test("Issue Mine Command - Shows Cycle Column", async () => { // // `queryIncludes` pins the GraphQL selection: the mock echoes fixtures verbatim, // so without it this would still pass if the query stopped requesting position. +// Use a fixed future update time because this snapshot tests ordering, not the +// current date; formatRelativeTime renders future values deterministically. await cliffySnapshotTest({ name: "Issue Mine Command - Groups Statuses In Linear's Order", meta: import.meta, @@ -846,7 +848,7 @@ await cliffySnapshotTest({ activeCycle: null, }, inverseRelations: { nodes: [] }, - updatedAt: "2026-03-29T10:00:00.000Z", + updatedAt: "2099-03-29T10:00:00.000Z", }, { id: "issue-eng-2", @@ -871,7 +873,7 @@ await cliffySnapshotTest({ activeCycle: null, }, inverseRelations: { nodes: [] }, - updatedAt: "2026-03-29T10:00:00.000Z", + updatedAt: "2099-03-29T10:00:00.000Z", }, { id: "issue-eng-4", @@ -896,7 +898,7 @@ await cliffySnapshotTest({ activeCycle: null, }, inverseRelations: { nodes: [] }, - updatedAt: "2026-03-29T10:00:00.000Z", + updatedAt: "2099-03-29T10:00:00.000Z", }, { id: "issue-eng-1", @@ -921,7 +923,7 @@ await cliffySnapshotTest({ activeCycle: null, }, inverseRelations: { nodes: [] }, - updatedAt: "2026-03-29T10:00:00.000Z", + updatedAt: "2099-03-29T10:00:00.000Z", }, { id: "issue-eng-7", @@ -946,7 +948,7 @@ await cliffySnapshotTest({ activeCycle: null, }, inverseRelations: { nodes: [] }, - updatedAt: "2026-03-29T10:00:00.000Z", + updatedAt: "2099-03-29T10:00:00.000Z", }, { id: "issue-eng-3", @@ -971,7 +973,7 @@ await cliffySnapshotTest({ activeCycle: null, }, inverseRelations: { nodes: [] }, - updatedAt: "2026-03-29T10:00:00.000Z", + updatedAt: "2099-03-29T10:00:00.000Z", }, { id: "issue-eng-5", @@ -996,7 +998,7 @@ await cliffySnapshotTest({ activeCycle: null, }, inverseRelations: { nodes: [] }, - updatedAt: "2026-03-29T10:00:00.000Z", + updatedAt: "2099-03-29T10:00:00.000Z", }, ], pageInfo: { hasNextPage: false, endCursor: null },