perf(dashboard-agent): share one 1h-TTL prompt cache between head start and the agent

Head start passed `system` as a bare string, so Anthropic neither wrote nor read
the cache: the ~18.4k-token prefix was billed in full and the agent's step 2 then
paid for a fresh write. Both breakpoints now carry a 1-hour TTL.

The two prefixes were also not identical — `get_run` sat in a different position in
the agent's tool set than in the canonical schema-only one, so they could never have
shared a cache. Reordered, and a fingerprint over system text + tool definitions is
logged per model call alongside the provider's cache-write, cache-read and uncached
input token counts, so a future drift is visible instead of silent.

Author: kathiekiwi

apps/webapp/app/services/dashboardAgentHeadStart.server.ts

@@ -6,6 +6,11 @@ import {
   dashboardAgentCodeToolSchemas,
   dashboardAgentToolSchemas,
 } from "@internal/dashboard-agent/tool-schemas";
+import {
+  describePromptPrefix,
+  PROMPT_CACHE_CONTROL,
+  promptCacheAttributes,
+} from "@internal/dashboard-agent/prompt-prefix";
 import { ApiClient, SessionStreamInstance, writeTurnCompleteRecord } from "@trigger.dev/core/v3";
 import { chat as chatServer } from "@trigger.dev/sdk/chat-server";
 import { streamText, type UIMessage, type UIMessageChunk } from "ai";
@@ -109,7 +114,26 @@ export async function startDashboardAgentHeadStart(params: {
       streamText({
         ...helper.toStreamTextOptions({ tools }),
         model: anthropic(DASHBOARD_AGENT_MODEL),
-        system,
+        // A structured system message, not a bare string: without provider options
+        // Anthropic neither writes nor reads the cache, so this call paid full price
+        // for the prefix and the agent's step 2 then paid for a fresh write. The tool
+        // key order is frozen (see `tool-schemas.ts`) so both prefixes are identical
+        // — the logged fingerprint is how a drift becomes visible.
+        system: {
+          role: "system",
+          content: system,
+          providerOptions: { anthropic: { cacheControl: PROMPT_CACHE_CONTROL } },
+        },
+        onStepFinish: (step) => {
+          logger.info(
+            "Dashboard agent prompt cache",
+            promptCacheAttributes({
+              source: "head-start",
+              usage: step.usage,
+              prefix: describePromptPrefix({ system, tools }),
+            })
+          );
+        },
       }),
   });
 

internal-packages/dashboard-agent/package.json

@@ -7,7 +7,8 @@
   "type": "module",
   "exports": {
     ".": "./src/index.ts",
-    "./tool-schemas": "./src/tool-schemas.ts"
+    "./tool-schemas": "./src/tool-schemas.ts",
+    "./prompt-prefix": "./src/prompt-prefix.ts"
   },
   "dependencies": {
     "@ai-sdk/anthropic": "^3.0.0",

internal-packages/dashboard-agent/src/dashboard-agent.ts

@@ -27,6 +27,12 @@ import {
 import { z } from "zod";
 import type { EvalTurnPayload, evalTurn } from "./eval-turn";
 import { codeSystemPrompt, systemPrompt, titlePrompt } from "./prompts";
+import {
+  describePromptPrefix,
+  PROMPT_CACHE_CONTROL,
+  promptCacheAttributes,
+  type PromptCacheUsage,
+} from "./prompt-prefix";
 import { buildDashboardAgentTools } from "./tools";
 
 /**
@@ -587,12 +593,39 @@ function withCacheBreakpointOnLast(messages: ModelMessage[]): ModelMessage[] {
       ...last,
       providerOptions: {
         ...last.providerOptions,
-        anthropic: { cacheControl: { type: "ephemeral" } },
+        anthropic: { cacheControl: PROMPT_CACHE_CONTROL },
       },
     },
   ];
 }
 
+/**
+ * One line per model call: what the provider billed as a cache write, a cache read
+ * and uncached input, against the prefix we expect to be cached. The estimate and
+ * the fingerprint are ours; the token counts are the provider's, and are logged as
+ * `null` when it reported none.
+ */
+function recordPromptCacheUsage(args: {
+  source: string;
+  usage: PromptCacheUsage | undefined;
+  system: string;
+  tools: ToolSet;
+}): void {
+  try {
+    logger.info(
+      "dashboard-agent prompt cache",
+      promptCacheAttributes({
+        source: args.source,
+        usage: args.usage,
+        prefix: describePromptPrefix({ system: args.system, tools: args.tools }),
+      })
+    );
+  } catch (error) {
+    // Measurement must never fail a turn.
+    logger.debug("dashboard-agent prompt cache measurement failed", { error });
+  }
+}
+
 /**
  * How the watch ended. The narration speaks resolution and observed outcome,
  * never "fired"/"expired" — those are the wire encoding, and a watch that ran its
@@ -1192,7 +1225,7 @@ export const dashboardAgent = chat.agent({
     // prompt; the resolve is cached per process. The cache breakpoint on the system
     // block carries through toStreamTextOptions() and survives suspend/resume.
     chat.prompt.set(await getSystemPrompt(modeFor(clientData)), {
-      providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } },
+      providerOptions: { anthropic: { cacheControl: PROMPT_CACHE_CONTROL } },
     });
   },
 
@@ -1306,7 +1339,7 @@ export const dashboardAgent = chat.agent({
         ...last,
         providerOptions: {
           ...last.providerOptions,
-          anthropic: { cacheControl: { type: "ephemeral" } },
+          anthropic: { cacheControl: PROMPT_CACHE_CONTROL },
         },
       },
     ];
@@ -1327,6 +1360,14 @@ export const dashboardAgent = chat.agent({
         ),
       messages,
       abortSignal: signal,
+      // Per model call, so the head-start prefix and this one can be compared.
+      onStepFinish: (step) =>
+        recordPromptCacheUsage({
+          source: "agent-turn",
+          usage: step.usage,
+          system: resolved.text,
+          tools: tools ?? {},
+        }),
       // toStreamTextOptions() defaults to a single step; override so the model can
       // call a tool and then answer from its result in the same turn.
       stopWhen: stepCountIs(10),

internal-packages/dashboard-agent/src/prompt-prefix.test.ts

@@ -0,0 +1,113 @@
+import { describe, expect, it } from "vitest";
+import { describePromptPrefix, PROMPT_CACHE_CONTROL, promptCacheAttributes } from "./prompt-prefix";
+import {
+  DASHBOARD_AGENT_CODE_SYSTEM_PROMPT,
+  DASHBOARD_AGENT_SYSTEM_PROMPT,
+  dashboardAgentCodeToolSchemas,
+  dashboardAgentToolSchemas,
+} from "./tool-schemas";
+import { buildDashboardAgentTools } from "./tools";
+import type { RepoSnapshot } from "./repo-tools";
+
+const SCOPE = { projectRef: "proj_abc", environmentId: "env_abc" };
+
+const snapshot: RepoSnapshot = {
+  tarballUrl: "http://unused.invalid/never-fetched",
+  owner: "acme",
+  repo: "demo",
+  sha: "deadbeefdeadbeef",
+};
+
+/**
+ * The whole point of the frozen tool key order: the webapp's warm first call and the
+ * agent task's every-later call must hand Anthropic a byte-identical prefix, or they
+ * cache separately and every call pays a fresh write.
+ */
+describe("the head-start and agent prefixes are the same prefix", () => {
+  it("matches in assistant mode", () => {
+    const headStart = describePromptPrefix({
+      system: DASHBOARD_AGENT_SYSTEM_PROMPT,
+      tools: dashboardAgentToolSchemas,
+    });
+    const agent = describePromptPrefix({
+      system: DASHBOARD_AGENT_SYSTEM_PROMPT,
+      tools: buildDashboardAgentTools(SCOPE),
+    });
+
+    expect(agent.fingerprint).toBe(headStart.fingerprint);
+    expect(agent.chars).toBe(headStart.chars);
+  });
+
+  it("matches in code mode", () => {
+    const headStart = describePromptPrefix({
+      system: DASHBOARD_AGENT_CODE_SYSTEM_PROMPT,
+      tools: dashboardAgentCodeToolSchemas,
+    });
+    const agent = describePromptPrefix({
+      system: DASHBOARD_AGENT_CODE_SYSTEM_PROMPT,
+      tools: buildDashboardAgentTools({ ...SCOPE, repoSnapshot: snapshot }),
+    });
+
+    expect(agent.fingerprint).toBe(headStart.fingerprint);
+  });
+
+  it("notices a reordered or changed tool set", () => {
+    const base = describePromptPrefix({
+      system: DASHBOARD_AGENT_SYSTEM_PROMPT,
+      tools: dashboardAgentToolSchemas,
+    });
+
+    const { list_projects, ...rest } = dashboardAgentToolSchemas;
+    const reordered = describePromptPrefix({
+      system: DASHBOARD_AGENT_SYSTEM_PROMPT,
+      tools: { ...rest, list_projects },
+    });
+    expect(reordered.fingerprint).not.toBe(base.fingerprint);
+
+    const editedPrompt = describePromptPrefix({
+      system: `${DASHBOARD_AGENT_SYSTEM_PROMPT}\nOne more rule.`,
+      tools: dashboardAgentToolSchemas,
+    });
+    expect(editedPrompt.fingerprint).not.toBe(base.fingerprint);
+  });
+
+  it("caches on the 1-hour breakpoint", () => {
+    expect(PROMPT_CACHE_CONTROL).toEqual({ type: "ephemeral", ttl: "1h" });
+  });
+});
+
+describe("promptCacheAttributes", () => {
+  const prefix = { chars: 80_000, estimatedTokens: 20_000, fingerprint: "abcd1234" };
+
+  it("records the four token counts the provider reports", () => {
+    expect(
+      promptCacheAttributes({
+        source: "agent-turn",
+        usage: {
+          inputTokens: 21_000,
+          inputTokenDetails: { noCacheTokens: 100, cacheReadTokens: 20_900, cacheWriteTokens: 0 },
+        },
+        prefix,
+      })
+    ).toEqual({
+      "dashboard_agent.prompt_cache.source": "agent-turn",
+      "gen_ai.usage.input_tokens": 21_000,
+      "gen_ai.usage.cache_write_input_tokens": 0,
+      "gen_ai.usage.cache_read_input_tokens": 20_900,
+      "gen_ai.usage.uncached_input_tokens": 100,
+      "dashboard_agent.prefix.estimated_tokens": 20_000,
+      "dashboard_agent.prefix.chars": 80_000,
+      "dashboard_agent.prefix.fingerprint": "abcd1234",
+    });
+  });
+
+  it("reports a value the provider didn't give as null rather than zero", () => {
+    const attributes = promptCacheAttributes({ source: "head-start", usage: undefined, prefix });
+    expect(attributes["gen_ai.usage.cache_write_input_tokens"]).toBeNull();
+    expect(attributes["gen_ai.usage.cache_read_input_tokens"]).toBeNull();
+    expect(attributes["gen_ai.usage.uncached_input_tokens"]).toBeNull();
+    expect(attributes["gen_ai.usage.input_tokens"]).toBeNull();
+    // Ours, so always present.
+    expect(attributes["dashboard_agent.prefix.estimated_tokens"]).toBe(20_000);
+  });
+});

internal-packages/dashboard-agent/src/prompt-prefix.ts

@@ -1076,16 +1076,6 @@ export function buildDashboardAgentTools(ctx: DashboardAgentToolContext): ToolSe
       },
     }),
 
-    get_run: tool({
-      ...getRunSchema,
-      execute: async ({ runId }) => {
-        const result = await envApiGet(`/api/v3/runs/${runId}`);
-        if (!result) return { error: "No current environment is available to read runs from." };
-        if (!result.ok) return { error: `Couldn't get run ${runId} (status ${result.status}).` };
-        return curateRun(result.data);
-      },
-    }),
-
     list_tasks: tool({
       ...listTasksSchema,
       execute: async () => {
@@ -1121,6 +1111,19 @@ export function buildDashboardAgentTools(ctx: DashboardAgentToolContext): ToolSe
       },
     }),
 
+    // Order matters up to here: `dashboardAgentToolSchemas` is the canonical key
+    // order (head start builds its prefix from it), and a different order is a
+    // different cached prefix.
+    get_run: tool({
+      ...getRunSchema,
+      execute: async ({ runId }) => {
+        const result = await envApiGet(`/api/v3/runs/${runId}`);
+        if (!result) return { error: "No current environment is available to read runs from." };
+        if (!result.ok) return { error: `Couldn't get run ${runId} (status ${result.status}).` };
+        return curateRun(result.data);
+      },
+    }),
+
     get_run_trace: tool({
       ...getRunTraceSchema,
       execute: async ({ runId }) => {