feat(vision): @coderadar/vision screenshot adapter, ephemeral (A10/E3/A13)

TRACKER.md

@@ -5,8 +5,8 @@
 ## Status
 
 - **Current phase:** 4 — Matching engine
-- **Next step:** 4.4 — Screenshot adapter (@coderadar/vision)
-- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.5, 4.1–4.3
+- **Next step:** 4.5 — Confidence calibration & ambiguity protocol
+- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.5, 4.1–4.4
 - **Gates passed:** Gate 0 (CI + red-path, #5/#6) · Gate 1 (precision 1.000, recall 0.895, zero poison) · Gate 2 (C1 instance attribution 1.000 · B1 4-level handler chains · C6 store writers↔readers · A9 portals — scorecard 137/0/0, precision & recall 1.000) · Gate 3 (B3 action effects · B4 routers · B6 cyclic journeys terminate · B7/B8 form & non-JSX events · G5 flag/role conditions — precision & recall 1.000)
 
 ## What CodeRadar is
@@ -236,7 +236,7 @@ The heart of the project. C1 and B1 live here.
 **Build:** when matches nest (Page > Section > Card all match), return the deepest instance covering the matched term/structure set; ancestors listed as `context`, not competing candidates.
 **Accept:** fixture `a6-composed-page` green: full-page term set → page node; card-specific terms → card instance with page as context.
 
-### [ ] 4.4 Screenshot adapter
+### [x] 4.4 Screenshot adapter
 **Failure modes:** A10, E3, A13
 **Build:** `@coderadar/vision` package:
 - `VisionAdapter` interface: `extract(image) → { terms: string[], structure: StructureDescriptor, annotations: Region[] }`. Ships with a Claude-vision implementation; OCR-only fallback stub for tests.

eval/fixtures/e3-annotated-screenshot/app/CostPanel.tsx

@@ -0,0 +1,8 @@
+export function CostPanel() {
+  return (
+    <section>
+      <h2>Cost breakdown</h2>
+      <span>Total</span>
+    </section>
+  );
+}

eval/fixtures/e3-annotated-screenshot/app/RevenuePanel.tsx

@@ -0,0 +1,8 @@
+export function RevenuePanel() {
+  return (
+    <section>
+      <h2>Revenue breakdown</h2>
+      <span>Total</span>
+    </section>
+  );
+}

eval/fixtures/e3-annotated-screenshot/extraction.json

@@ -0,0 +1,13 @@
+{
+  "note": "Pre-extracted vision output (checked in — no live API). The user circled 'Revenue breakdown'. Without the annotation this is ambiguous (RevenuePanel and CostPanel both share 'Total' and each own one 'breakdown' term); the 3x annotation boost resolves it to RevenuePanel.",
+  "terms": ["Revenue breakdown", "Cost breakdown", "Total"],
+  "structure": {},
+  "annotations": [
+    {
+      "kind": "circle",
+      "bounds": { "x": 20, "y": 40, "w": 180, "h": 32 },
+      "terms": ["Revenue breakdown"]
+    }
+  ],
+  "looksLikeApp": true
+}

eval/fixtures/e3-annotated-screenshot/golden.json

@@ -0,0 +1,10 @@
+{
+  "failureMode": "E3",
+  "note": "Annotation-region weighting is verified in a unit test (packages/parser-react/src/vision.test.ts) against the checked-in extraction.json — the user circled 'Revenue breakdown', and the 3x boost resolves an otherwise-ambiguous match to RevenuePanel. This golden just asserts the two panels parse.",
+  "expect": {
+    "components": [
+      { "name": "RevenuePanel", "instances": 0 },
+      { "name": "CostPanel", "instances": 0 }
+    ]
+  }
+}

packages/cli/package.json

@@ -14,6 +14,10 @@
     ".": {
       "types": "./dist/lib.d.ts",
       "default": "./dist/lib.js"
+    },
+    "./vision": {
+      "types": "./dist/vision.d.ts",
+      "default": "./dist/vision.js"
     }
   },
   "files": [
@@ -47,6 +51,7 @@
   "devDependencies": {
     "@coderadar/core": "workspace:*",
     "@coderadar/parser-react": "workspace:*",
+    "@coderadar/vision": "workspace:*",
     "@types/node": "^22.20.1",
     "tsup": "^8.5.1",
     "typescript": "^5.7.0"

packages/cli/src/vision.ts

@@ -0,0 +1,6 @@
+/**
+ * ui-lineage/vision — screenshot → { terms, structure, annotations } adapters.
+ * Live Claude extraction needs `@anthropic-ai/sdk` installed separately; the
+ * stub adapter and matching helpers work with no extra dependency.
+ */
+export * from "@coderadar/vision";

packages/cli/tsup.config.ts

@@ -7,6 +7,7 @@ export default defineConfig({
   entry: {
     index: "src/index.ts", // CLI bin (keeps its #!/usr/bin/env node shebang)
     lib: "src/lib.ts", // library entry
+    vision: "src/vision.ts", // ui-lineage/vision subpath
   },
   format: ["esm"],
   target: "node20",
@@ -16,5 +17,6 @@ export default defineConfig({
   clean: true,
   sourcemap: true,
   noExternal: [/^@coderadar\//],
-  external: ["ts-morph", "yaml", "commander"],
+  // @anthropic-ai/sdk is a lazy, optional runtime dep of the vision adapter.
+  external: ["ts-morph", "yaml", "commander", "@anthropic-ai/sdk"],
 });

packages/core/src/query.ts

@@ -67,6 +67,12 @@ export interface ComponentMatch {
 export interface MatchQuery {
   terms?: string[];
   structure?: StructureDescriptor;
+  /**
+   * Per-term weight multipliers (Phase 4.4): terms a vision adapter found
+   * inside an annotation (a circle/arrow the user drew) are boosted, e.g. 3×,
+   * so the emphasized element outranks incidental text.
+   */
+  boosts?: Record<string, number>;
 }
 
 /** Weight of a full structural match relative to a rare matched term. */
@@ -130,8 +136,14 @@ export function matchComponents(
     return null;
   };
 
-  const termWeight = (term: string): number =>
-    tokenize(term).reduce((sum, t) => sum + idf(t), 0);
+  // Boost keys are normalized so they match the normalized query terms below.
+  const boosts = new Map(
+    Object.entries(query.boosts ?? {}).map(([term, weight]) => [normalizeText(term), weight]),
+  );
+  const termWeight = (term: string): number => {
+    const base = tokenize(term).reduce((sum, t) => sum + idf(t), 0);
+    return base * (boosts.get(term) ?? 1);
+  };
 
   // Render tree (definition level): A renders B when A has a `renders` edge to
   // an instance whose `instance-of` points at B. Used to collapse nested

packages/parser-react/package.json

@@ -27,6 +27,7 @@
     "yaml": "^2.9.0"
   },
   "devDependencies": {
+    "@coderadar/vision": "workspace:*",
     "@types/node": "^22.20.1",
     "typescript": "^5.7.0",
     "vitest": "^3.2.7"

packages/parser-react/src/vision.test.ts

@@ -0,0 +1,31 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { matchComponents } from "@coderadar/core";
+import { extractionToQuery, matchFromVision, type VisionExtraction } from "@coderadar/vision";
+import { describe, expect, it } from "vitest";
+
+import { resolveHookEdges, scanReact } from "./scan.js";
+
+const fixtureDir = path.resolve(
+  path.dirname(fileURLToPath(import.meta.url)),
+  "../../../eval/fixtures/e3-annotated-screenshot",
+);
+const graph = resolveHookEdges(scanReact({ root: path.join(fixtureDir, "app") }));
+const extraction = JSON.parse(
+  fs.readFileSync(path.join(fixtureDir, "extraction.json"), "utf-8"),
+) as VisionExtraction;
+
+describe("annotation weighting (e3 fixture, TRACKER 4.4, E3)", () => {
+  it("without the annotation the screenshot is ambiguous", () => {
+    const query = extractionToQuery({ ...extraction, annotations: [] });
+    expect(matchComponents(graph, query).status).toBe("ambiguous");
+  });
+
+  it("the circled term (3x boost) resolves the match to its panel", () => {
+    const result = matchFromVision(graph, extraction);
+    expect(result.status).toBe("ok");
+    expect(result.candidates[0]?.value.component.name).toBe("RevenuePanel");
+  });
+});

packages/vision/README.md

@@ -0,0 +1,27 @@
+# @coderadar/vision
+
+Screenshot → `{ terms, structure, annotations }` for the ui-lineage matcher. Bundled into the published `ui-lineage` package.
+
+## Ephemeral by policy (G7)
+
+Images are processed **in memory only**. This package never writes image bytes to disk, to the lineage graph, or to any cache. Adapters must uphold this — do not add persistence of screenshots. The corrections store (Phase 4.6) keeps *terms*, never images.
+
+## Usage
+
+```ts
+import { StubVisionAdapter, matchFromVision } from "ui-lineage/vision";
+
+// Deterministic adapter for tests / recorded extractions:
+const adapter = new StubVisionAdapter(recordedExtraction);
+const extraction = await adapter.extract(image);
+const result = matchFromVision(graph, extraction);
+```
+
+Live extraction uses Claude vision and requires you to install the SDK yourself (kept out of the default bundle):
+
+```ts
+import { ClaudeVisionAdapter } from "ui-lineage/vision"; // needs `npm i @anthropic-ai/sdk`
+const adapter = new ClaudeVisionAdapter({ apiKey: process.env.ANTHROPIC_API_KEY });
+```
+
+Terms found inside an annotation (a circle or arrow the user drew) are weighted 3× (`ANNOTATION_BOOST`), so the emphasized element outranks incidental text. A screenshot that isn't this app (`looksLikeApp: false`) declines rather than guessing.

packages/vision/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@coderadar/vision",
+  "version": "0.1.0",
+  "private": true,
+  "description": "Screenshot → { terms, structure, annotations } vision adapter for ui-lineage. Images are processed in memory and never persisted (G7). Bundled into the published ui-lineage package.",
+  "license": "MIT",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "test": "vitest run"
+  },
+  "dependencies": {
+    "@coderadar/core": "workspace:*"
+  },
+  "devDependencies": {
+    "@types/node": "^22.20.1",
+    "typescript": "^5.7.0",
+    "vitest": "^3.2.7"
+  }
+}

packages/vision/src/claude.ts

@@ -0,0 +1,114 @@
+/**
+ * Claude-vision implementation of VisionAdapter (TRACKER step 4.4).
+ *
+ * Not exercised in CI (no live API). The `@anthropic-ai/sdk` dependency is
+ * imported lazily so neither the rest of ui-lineage nor the test suite requires
+ * it — install it yourself to use live extraction:  `npm i @anthropic-ai/sdk`.
+ *
+ * G7 (ephemeral): the image is sent in-memory and never written to disk or the
+ * graph. Do not add caching that persists image bytes.
+ */
+import type { VisionAdapter, VisionExtraction, VisionImage } from "./types.js";
+
+export interface ClaudeVisionOptions {
+  /** Falls back to the ANTHROPIC_API_KEY environment variable. */
+  apiKey?: string;
+  /** Defaults to the latest Opus. */
+  model?: string;
+}
+
+/** JSON shape the model is asked to return — mirrors VisionExtraction. */
+const EXTRACTION_SCHEMA = {
+  type: "object",
+  additionalProperties: false,
+  properties: {
+    terms: { type: "array", items: { type: "string" } },
+    structure: {
+      type: "object",
+      additionalProperties: false,
+      properties: {
+        table: { type: "boolean" },
+        columns: { type: "integer" },
+        form: { type: "boolean" },
+        inputs: { type: "integer" },
+        buttons: { type: "integer" },
+        images: { type: "integer" },
+        list: { type: "boolean" },
+        cards: { type: "integer" },
+      },
+      required: [],
+    },
+    annotations: {
+      type: "array",
+      items: {
+        type: "object",
+        additionalProperties: false,
+        properties: {
+          kind: { type: "string", enum: ["circle", "arrow", "box", "highlight"] },
+          bounds: {
+            type: "object",
+            additionalProperties: false,
+            properties: {
+              x: { type: "number" },
+              y: { type: "number" },
+              w: { type: "number" },
+              h: { type: "number" },
+            },
+            required: ["x", "y", "w", "h"],
+          },
+          terms: { type: "array", items: { type: "string" } },
+        },
+        required: ["kind", "bounds", "terms"],
+      },
+    },
+    looksLikeApp: { type: "boolean" },
+  },
+  required: ["terms", "structure", "annotations", "looksLikeApp"],
+} as const;
+
+const EXTRACTION_PROMPT =
+  "Read this UI screenshot. Return the visible text fragments, a structural " +
+  "descriptor (is there a table and how many columns, a form and how many " +
+  "inputs/buttons, images, a list, repeated cards), and any hand-drawn " +
+  "annotations (circles/arrows/highlights) with the text they point at. Set " +
+  "looksLikeApp=false for Figma frames, marketing pages, or anything that is " +
+  "not a running application UI.";
+
+export class ClaudeVisionAdapter implements VisionAdapter {
+  constructor(private readonly options: ClaudeVisionOptions = {}) {}
+
+  async extract(image: VisionImage): Promise<VisionExtraction> {
+    // Non-literal specifier keeps the SDK out of the type graph and the bundle;
+    // it is resolved at runtime only when live extraction is actually used.
+    const specifier = "@anthropic-ai/sdk";
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const mod: any = await import(specifier as string);
+    const Anthropic = mod.default ?? mod.Anthropic;
+    const client = new Anthropic(this.options.apiKey ? { apiKey: this.options.apiKey } : {});
+    const data =
+      typeof image.data === "string"
+        ? image.data
+        : Buffer.from(image.data).toString("base64");
+
+    const response = await client.messages.create({
+      model: this.options.model ?? "claude-opus-4-8",
+      max_tokens: 2000,
+      output_config: { format: { type: "json_schema", schema: EXTRACTION_SCHEMA } },
+      messages: [
+        {
+          role: "user",
+          content: [
+            { type: "image", source: { type: "base64", media_type: image.mediaType, data } },
+            { type: "text", text: EXTRACTION_PROMPT },
+          ],
+        },
+      ],
+    });
+
+    const text = (response.content ?? [])
+      .filter((b: { type: string }) => b.type === "text")
+      .map((b: { text: string }) => b.text)
+      .join("");
+    return JSON.parse(text) as VisionExtraction;
+  }
+}

packages/vision/src/index.ts

@@ -0,0 +1,4 @@
+export type { Region, VisionAdapter, VisionExtraction, VisionImage } from "./types.js";
+export { ANNOTATION_BOOST, extractionToQuery, matchFromVision } from "./query.js";
+export { StubVisionAdapter } from "./stub.js";
+export { ClaudeVisionAdapter, type ClaudeVisionOptions } from "./claude.js";

packages/vision/src/query.ts

@@ -0,0 +1,42 @@
+/**
+ * Turn a VisionExtraction into a graph match (TRACKER step 4.4).
+ *
+ * Terms inside an annotation (E3) are weighted 3× so the element the user
+ * circled outranks incidental text. A screenshot that doesn't look like this
+ * app (`looksLikeApp: false`) declines rather than guessing (A13).
+ */
+import {
+  type ComponentMatch,
+  declined,
+  type LineageGraph,
+  matchComponents,
+  type MatchQuery,
+  type QueryResult,
+} from "@coderadar/core";
+
+import type { VisionExtraction } from "./types.js";
+
+/** Weight multiplier applied to terms found inside an annotation region. */
+export const ANNOTATION_BOOST = 3;
+
+/** Build a matcher query from an extraction, boosting annotated terms. */
+export function extractionToQuery(extraction: VisionExtraction): MatchQuery {
+  const boosts: Record<string, number> = {};
+  for (const region of extraction.annotations) {
+    for (const term of region.terms) boosts[term] = ANNOTATION_BOOST;
+  }
+  return {
+    terms: extraction.terms,
+    structure: extraction.structure,
+    ...(Object.keys(boosts).length > 0 ? { boosts } : {}),
+  };
+}
+
+/** Match a screenshot extraction against a graph, or decline a non-app image. */
+export function matchFromVision(
+  graph: LineageGraph,
+  extraction: VisionExtraction,
+): QueryResult<ComponentMatch> {
+  if (!extraction.looksLikeApp) return declined("not-our-app");
+  return matchComponents(graph, extractionToQuery(extraction));
+}