From 68d26c6474bd5fe44ed1e401d4403622ac0370f7 Mon Sep 17 00:00:00 2001 From: Deep Kumar Singh Kushwah Date: Wed, 15 Jul 2026 02:21:20 +0530 Subject: [PATCH] feat(vision): @coderadar/vision screenshot adapter, ephemeral (A10/E3/A13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New internal @coderadar/vision package (TRACKER step 4.4), bundled into the published ui-lineage as the `ui-lineage/vision` subpath: - VisionAdapter interface: extract(image) → { terms, structure, annotations, looksLikeApp }. Images are processed in memory, never persisted (G7). - StubVisionAdapter — deterministic, replays a recorded extraction (the OCR-only fallback; no network in CI). - ClaudeVisionAdapter — Claude-vision implementation (model claude-opus-4-8, base64 image + json_schema output). The @anthropic-ai/sdk import is lazy and optional, so neither CI nor the default bundle depends on it. - extractionToQuery / matchFromVision: annotation-region terms (a circle/arrow the user drew, E3) are weighted 3×; a non-app screenshot (looksLikeApp:false) declines rather than guessing (A13). Core: MatchQuery gains `boosts` (per-term weight multipliers, normalized keys). Fixture e3-annotated-screenshot with a checked-in extraction.json: the circled "Revenue breakdown" resolves an otherwise-ambiguous match to RevenuePanel. 36 core + 3 vision + 102 parser tests pass; eval green. Co-Authored-By: Claude Opus 4.8 --- TRACKER.md | 6 +- .../e3-annotated-screenshot/app/CostPanel.tsx | 8 ++ .../app/RevenuePanel.tsx | 8 ++ .../e3-annotated-screenshot/extraction.json | 13 ++ .../e3-annotated-screenshot/golden.json | 10 ++ packages/cli/package.json | 5 + packages/cli/src/vision.ts | 6 + packages/cli/tsup.config.ts | 4 +- packages/core/src/query.ts | 16 ++- packages/parser-react/package.json | 1 + packages/parser-react/src/vision.test.ts | 31 +++++ packages/vision/README.md | 27 +++++ packages/vision/package.json | 32 +++++ packages/vision/src/claude.ts | 114 ++++++++++++++++++ packages/vision/src/index.ts | 4 + packages/vision/src/query.ts | 42 +++++++ packages/vision/src/stub.ts | 14 +++ packages/vision/src/types.ts | 44 +++++++ packages/vision/src/vision.test.ts | 42 +++++++ packages/vision/tsconfig.json | 8 ++ pnpm-lock.yaml | 22 ++++ 21 files changed, 451 insertions(+), 6 deletions(-) create mode 100644 eval/fixtures/e3-annotated-screenshot/app/CostPanel.tsx create mode 100644 eval/fixtures/e3-annotated-screenshot/app/RevenuePanel.tsx create mode 100644 eval/fixtures/e3-annotated-screenshot/extraction.json create mode 100644 eval/fixtures/e3-annotated-screenshot/golden.json create mode 100644 packages/cli/src/vision.ts create mode 100644 packages/parser-react/src/vision.test.ts create mode 100644 packages/vision/README.md create mode 100644 packages/vision/package.json create mode 100644 packages/vision/src/claude.ts create mode 100644 packages/vision/src/index.ts create mode 100644 packages/vision/src/query.ts create mode 100644 packages/vision/src/stub.ts create mode 100644 packages/vision/src/types.ts create mode 100644 packages/vision/src/vision.test.ts create mode 100644 packages/vision/tsconfig.json diff --git a/TRACKER.md b/TRACKER.md index 749edfe..f9bf66e 100644 --- a/TRACKER.md +++ b/TRACKER.md @@ -5,8 +5,8 @@ ## Status - **Current phase:** 4 — Matching engine -- **Next step:** 4.4 — Screenshot adapter (@coderadar/vision) -- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.5, 4.1–4.3 +- **Next step:** 4.5 — Confidence calibration & ambiguity protocol +- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.5, 4.1–4.4 - **Gates passed:** Gate 0 (CI + red-path, #5/#6) · Gate 1 (precision 1.000, recall 0.895, zero poison) · Gate 2 (C1 instance attribution 1.000 · B1 4-level handler chains · C6 store writers↔readers · A9 portals — scorecard 137/0/0, precision & recall 1.000) · Gate 3 (B3 action effects · B4 routers · B6 cyclic journeys terminate · B7/B8 form & non-JSX events · G5 flag/role conditions — precision & recall 1.000) ## What CodeRadar is @@ -236,7 +236,7 @@ The heart of the project. C1 and B1 live here. **Build:** when matches nest (Page > Section > Card all match), return the deepest instance covering the matched term/structure set; ancestors listed as `context`, not competing candidates. **Accept:** fixture `a6-composed-page` green: full-page term set → page node; card-specific terms → card instance with page as context. -### [ ] 4.4 Screenshot adapter +### [x] 4.4 Screenshot adapter **Failure modes:** A10, E3, A13 **Build:** `@coderadar/vision` package: - `VisionAdapter` interface: `extract(image) → { terms: string[], structure: StructureDescriptor, annotations: Region[] }`. Ships with a Claude-vision implementation; OCR-only fallback stub for tests. diff --git a/eval/fixtures/e3-annotated-screenshot/app/CostPanel.tsx b/eval/fixtures/e3-annotated-screenshot/app/CostPanel.tsx new file mode 100644 index 0000000..9a1bbaf --- /dev/null +++ b/eval/fixtures/e3-annotated-screenshot/app/CostPanel.tsx @@ -0,0 +1,8 @@ +export function CostPanel() { + return ( +
+

Cost breakdown

+ Total +
+ ); +} diff --git a/eval/fixtures/e3-annotated-screenshot/app/RevenuePanel.tsx b/eval/fixtures/e3-annotated-screenshot/app/RevenuePanel.tsx new file mode 100644 index 0000000..93113bb --- /dev/null +++ b/eval/fixtures/e3-annotated-screenshot/app/RevenuePanel.tsx @@ -0,0 +1,8 @@ +export function RevenuePanel() { + return ( +
+

Revenue breakdown

+ Total +
+ ); +} diff --git a/eval/fixtures/e3-annotated-screenshot/extraction.json b/eval/fixtures/e3-annotated-screenshot/extraction.json new file mode 100644 index 0000000..1ae3e2f --- /dev/null +++ b/eval/fixtures/e3-annotated-screenshot/extraction.json @@ -0,0 +1,13 @@ +{ + "note": "Pre-extracted vision output (checked in — no live API). The user circled 'Revenue breakdown'. Without the annotation this is ambiguous (RevenuePanel and CostPanel both share 'Total' and each own one 'breakdown' term); the 3x annotation boost resolves it to RevenuePanel.", + "terms": ["Revenue breakdown", "Cost breakdown", "Total"], + "structure": {}, + "annotations": [ + { + "kind": "circle", + "bounds": { "x": 20, "y": 40, "w": 180, "h": 32 }, + "terms": ["Revenue breakdown"] + } + ], + "looksLikeApp": true +} diff --git a/eval/fixtures/e3-annotated-screenshot/golden.json b/eval/fixtures/e3-annotated-screenshot/golden.json new file mode 100644 index 0000000..2bd93ae --- /dev/null +++ b/eval/fixtures/e3-annotated-screenshot/golden.json @@ -0,0 +1,10 @@ +{ + "failureMode": "E3", + "note": "Annotation-region weighting is verified in a unit test (packages/parser-react/src/vision.test.ts) against the checked-in extraction.json — the user circled 'Revenue breakdown', and the 3x boost resolves an otherwise-ambiguous match to RevenuePanel. This golden just asserts the two panels parse.", + "expect": { + "components": [ + { "name": "RevenuePanel", "instances": 0 }, + { "name": "CostPanel", "instances": 0 } + ] + } +} diff --git a/packages/cli/package.json b/packages/cli/package.json index ebe02dc..c868b1a 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -14,6 +14,10 @@ ".": { "types": "./dist/lib.d.ts", "default": "./dist/lib.js" + }, + "./vision": { + "types": "./dist/vision.d.ts", + "default": "./dist/vision.js" } }, "files": [ @@ -47,6 +51,7 @@ "devDependencies": { "@coderadar/core": "workspace:*", "@coderadar/parser-react": "workspace:*", + "@coderadar/vision": "workspace:*", "@types/node": "^22.20.1", "tsup": "^8.5.1", "typescript": "^5.7.0" diff --git a/packages/cli/src/vision.ts b/packages/cli/src/vision.ts new file mode 100644 index 0000000..8ba6621 --- /dev/null +++ b/packages/cli/src/vision.ts @@ -0,0 +1,6 @@ +/** + * ui-lineage/vision — screenshot → { terms, structure, annotations } adapters. + * Live Claude extraction needs `@anthropic-ai/sdk` installed separately; the + * stub adapter and matching helpers work with no extra dependency. + */ +export * from "@coderadar/vision"; diff --git a/packages/cli/tsup.config.ts b/packages/cli/tsup.config.ts index 3ed16da..43cd1bc 100644 --- a/packages/cli/tsup.config.ts +++ b/packages/cli/tsup.config.ts @@ -7,6 +7,7 @@ export default defineConfig({ entry: { index: "src/index.ts", // CLI bin (keeps its #!/usr/bin/env node shebang) lib: "src/lib.ts", // library entry + vision: "src/vision.ts", // ui-lineage/vision subpath }, format: ["esm"], target: "node20", @@ -16,5 +17,6 @@ export default defineConfig({ clean: true, sourcemap: true, noExternal: [/^@coderadar\//], - external: ["ts-morph", "yaml", "commander"], + // @anthropic-ai/sdk is a lazy, optional runtime dep of the vision adapter. + external: ["ts-morph", "yaml", "commander", "@anthropic-ai/sdk"], }); diff --git a/packages/core/src/query.ts b/packages/core/src/query.ts index 076606e..f931f6f 100644 --- a/packages/core/src/query.ts +++ b/packages/core/src/query.ts @@ -67,6 +67,12 @@ export interface ComponentMatch { export interface MatchQuery { terms?: string[]; structure?: StructureDescriptor; + /** + * Per-term weight multipliers (Phase 4.4): terms a vision adapter found + * inside an annotation (a circle/arrow the user drew) are boosted, e.g. 3×, + * so the emphasized element outranks incidental text. + */ + boosts?: Record; } /** Weight of a full structural match relative to a rare matched term. */ @@ -130,8 +136,14 @@ export function matchComponents( return null; }; - const termWeight = (term: string): number => - tokenize(term).reduce((sum, t) => sum + idf(t), 0); + // Boost keys are normalized so they match the normalized query terms below. + const boosts = new Map( + Object.entries(query.boosts ?? {}).map(([term, weight]) => [normalizeText(term), weight]), + ); + const termWeight = (term: string): number => { + const base = tokenize(term).reduce((sum, t) => sum + idf(t), 0); + return base * (boosts.get(term) ?? 1); + }; // Render tree (definition level): A renders B when A has a `renders` edge to // an instance whose `instance-of` points at B. Used to collapse nested diff --git a/packages/parser-react/package.json b/packages/parser-react/package.json index c2f2e2a..6f6b753 100644 --- a/packages/parser-react/package.json +++ b/packages/parser-react/package.json @@ -27,6 +27,7 @@ "yaml": "^2.9.0" }, "devDependencies": { + "@coderadar/vision": "workspace:*", "@types/node": "^22.20.1", "typescript": "^5.7.0", "vitest": "^3.2.7" diff --git a/packages/parser-react/src/vision.test.ts b/packages/parser-react/src/vision.test.ts new file mode 100644 index 0000000..3c779d1 --- /dev/null +++ b/packages/parser-react/src/vision.test.ts @@ -0,0 +1,31 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { matchComponents } from "@coderadar/core"; +import { extractionToQuery, matchFromVision, type VisionExtraction } from "@coderadar/vision"; +import { describe, expect, it } from "vitest"; + +import { resolveHookEdges, scanReact } from "./scan.js"; + +const fixtureDir = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../../eval/fixtures/e3-annotated-screenshot", +); +const graph = resolveHookEdges(scanReact({ root: path.join(fixtureDir, "app") })); +const extraction = JSON.parse( + fs.readFileSync(path.join(fixtureDir, "extraction.json"), "utf-8"), +) as VisionExtraction; + +describe("annotation weighting (e3 fixture, TRACKER 4.4, E3)", () => { + it("without the annotation the screenshot is ambiguous", () => { + const query = extractionToQuery({ ...extraction, annotations: [] }); + expect(matchComponents(graph, query).status).toBe("ambiguous"); + }); + + it("the circled term (3x boost) resolves the match to its panel", () => { + const result = matchFromVision(graph, extraction); + expect(result.status).toBe("ok"); + expect(result.candidates[0]?.value.component.name).toBe("RevenuePanel"); + }); +}); diff --git a/packages/vision/README.md b/packages/vision/README.md new file mode 100644 index 0000000..9b202f9 --- /dev/null +++ b/packages/vision/README.md @@ -0,0 +1,27 @@ +# @coderadar/vision + +Screenshot → `{ terms, structure, annotations }` for the ui-lineage matcher. Bundled into the published `ui-lineage` package. + +## Ephemeral by policy (G7) + +Images are processed **in memory only**. This package never writes image bytes to disk, to the lineage graph, or to any cache. Adapters must uphold this — do not add persistence of screenshots. The corrections store (Phase 4.6) keeps *terms*, never images. + +## Usage + +```ts +import { StubVisionAdapter, matchFromVision } from "ui-lineage/vision"; + +// Deterministic adapter for tests / recorded extractions: +const adapter = new StubVisionAdapter(recordedExtraction); +const extraction = await adapter.extract(image); +const result = matchFromVision(graph, extraction); +``` + +Live extraction uses Claude vision and requires you to install the SDK yourself (kept out of the default bundle): + +```ts +import { ClaudeVisionAdapter } from "ui-lineage/vision"; // needs `npm i @anthropic-ai/sdk` +const adapter = new ClaudeVisionAdapter({ apiKey: process.env.ANTHROPIC_API_KEY }); +``` + +Terms found inside an annotation (a circle or arrow the user drew) are weighted 3× (`ANNOTATION_BOOST`), so the emphasized element outranks incidental text. A screenshot that isn't this app (`looksLikeApp: false`) declines rather than guessing. diff --git a/packages/vision/package.json b/packages/vision/package.json new file mode 100644 index 0000000..3814588 --- /dev/null +++ b/packages/vision/package.json @@ -0,0 +1,32 @@ +{ + "name": "@coderadar/vision", + "version": "0.1.0", + "private": true, + "description": "Screenshot → { terms, structure, annotations } vision adapter for ui-lineage. Images are processed in memory and never persisted (G7). Bundled into the published ui-lineage package.", + "license": "MIT", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "files": [ + "dist" + ], + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "vitest run" + }, + "dependencies": { + "@coderadar/core": "workspace:*" + }, + "devDependencies": { + "@types/node": "^22.20.1", + "typescript": "^5.7.0", + "vitest": "^3.2.7" + } +} diff --git a/packages/vision/src/claude.ts b/packages/vision/src/claude.ts new file mode 100644 index 0000000..bf635e8 --- /dev/null +++ b/packages/vision/src/claude.ts @@ -0,0 +1,114 @@ +/** + * Claude-vision implementation of VisionAdapter (TRACKER step 4.4). + * + * Not exercised in CI (no live API). The `@anthropic-ai/sdk` dependency is + * imported lazily so neither the rest of ui-lineage nor the test suite requires + * it — install it yourself to use live extraction: `npm i @anthropic-ai/sdk`. + * + * G7 (ephemeral): the image is sent in-memory and never written to disk or the + * graph. Do not add caching that persists image bytes. + */ +import type { VisionAdapter, VisionExtraction, VisionImage } from "./types.js"; + +export interface ClaudeVisionOptions { + /** Falls back to the ANTHROPIC_API_KEY environment variable. */ + apiKey?: string; + /** Defaults to the latest Opus. */ + model?: string; +} + +/** JSON shape the model is asked to return — mirrors VisionExtraction. */ +const EXTRACTION_SCHEMA = { + type: "object", + additionalProperties: false, + properties: { + terms: { type: "array", items: { type: "string" } }, + structure: { + type: "object", + additionalProperties: false, + properties: { + table: { type: "boolean" }, + columns: { type: "integer" }, + form: { type: "boolean" }, + inputs: { type: "integer" }, + buttons: { type: "integer" }, + images: { type: "integer" }, + list: { type: "boolean" }, + cards: { type: "integer" }, + }, + required: [], + }, + annotations: { + type: "array", + items: { + type: "object", + additionalProperties: false, + properties: { + kind: { type: "string", enum: ["circle", "arrow", "box", "highlight"] }, + bounds: { + type: "object", + additionalProperties: false, + properties: { + x: { type: "number" }, + y: { type: "number" }, + w: { type: "number" }, + h: { type: "number" }, + }, + required: ["x", "y", "w", "h"], + }, + terms: { type: "array", items: { type: "string" } }, + }, + required: ["kind", "bounds", "terms"], + }, + }, + looksLikeApp: { type: "boolean" }, + }, + required: ["terms", "structure", "annotations", "looksLikeApp"], +} as const; + +const EXTRACTION_PROMPT = + "Read this UI screenshot. Return the visible text fragments, a structural " + + "descriptor (is there a table and how many columns, a form and how many " + + "inputs/buttons, images, a list, repeated cards), and any hand-drawn " + + "annotations (circles/arrows/highlights) with the text they point at. Set " + + "looksLikeApp=false for Figma frames, marketing pages, or anything that is " + + "not a running application UI."; + +export class ClaudeVisionAdapter implements VisionAdapter { + constructor(private readonly options: ClaudeVisionOptions = {}) {} + + async extract(image: VisionImage): Promise { + // Non-literal specifier keeps the SDK out of the type graph and the bundle; + // it is resolved at runtime only when live extraction is actually used. + const specifier = "@anthropic-ai/sdk"; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const mod: any = await import(specifier as string); + const Anthropic = mod.default ?? mod.Anthropic; + const client = new Anthropic(this.options.apiKey ? { apiKey: this.options.apiKey } : {}); + const data = + typeof image.data === "string" + ? image.data + : Buffer.from(image.data).toString("base64"); + + const response = await client.messages.create({ + model: this.options.model ?? "claude-opus-4-8", + max_tokens: 2000, + output_config: { format: { type: "json_schema", schema: EXTRACTION_SCHEMA } }, + messages: [ + { + role: "user", + content: [ + { type: "image", source: { type: "base64", media_type: image.mediaType, data } }, + { type: "text", text: EXTRACTION_PROMPT }, + ], + }, + ], + }); + + const text = (response.content ?? []) + .filter((b: { type: string }) => b.type === "text") + .map((b: { text: string }) => b.text) + .join(""); + return JSON.parse(text) as VisionExtraction; + } +} diff --git a/packages/vision/src/index.ts b/packages/vision/src/index.ts new file mode 100644 index 0000000..b4ccf03 --- /dev/null +++ b/packages/vision/src/index.ts @@ -0,0 +1,4 @@ +export type { Region, VisionAdapter, VisionExtraction, VisionImage } from "./types.js"; +export { ANNOTATION_BOOST, extractionToQuery, matchFromVision } from "./query.js"; +export { StubVisionAdapter } from "./stub.js"; +export { ClaudeVisionAdapter, type ClaudeVisionOptions } from "./claude.js"; diff --git a/packages/vision/src/query.ts b/packages/vision/src/query.ts new file mode 100644 index 0000000..100a98d --- /dev/null +++ b/packages/vision/src/query.ts @@ -0,0 +1,42 @@ +/** + * Turn a VisionExtraction into a graph match (TRACKER step 4.4). + * + * Terms inside an annotation (E3) are weighted 3× so the element the user + * circled outranks incidental text. A screenshot that doesn't look like this + * app (`looksLikeApp: false`) declines rather than guessing (A13). + */ +import { + type ComponentMatch, + declined, + type LineageGraph, + matchComponents, + type MatchQuery, + type QueryResult, +} from "@coderadar/core"; + +import type { VisionExtraction } from "./types.js"; + +/** Weight multiplier applied to terms found inside an annotation region. */ +export const ANNOTATION_BOOST = 3; + +/** Build a matcher query from an extraction, boosting annotated terms. */ +export function extractionToQuery(extraction: VisionExtraction): MatchQuery { + const boosts: Record = {}; + for (const region of extraction.annotations) { + for (const term of region.terms) boosts[term] = ANNOTATION_BOOST; + } + return { + terms: extraction.terms, + structure: extraction.structure, + ...(Object.keys(boosts).length > 0 ? { boosts } : {}), + }; +} + +/** Match a screenshot extraction against a graph, or decline a non-app image. */ +export function matchFromVision( + graph: LineageGraph, + extraction: VisionExtraction, +): QueryResult { + if (!extraction.looksLikeApp) return declined("not-our-app"); + return matchComponents(graph, extractionToQuery(extraction)); +} diff --git a/packages/vision/src/stub.ts b/packages/vision/src/stub.ts new file mode 100644 index 0000000..6c00474 --- /dev/null +++ b/packages/vision/src/stub.ts @@ -0,0 +1,14 @@ +import type { VisionAdapter, VisionExtraction } from "./types.js"; + +/** + * A deterministic adapter that returns a pre-recorded extraction. Used to test + * the matching path against checked-in extraction outputs without a live model + * (the tracker's "OCR-only fallback stub" — no network in CI). + */ +export class StubVisionAdapter implements VisionAdapter { + constructor(private readonly extraction: VisionExtraction) {} + + extract(): Promise { + return Promise.resolve(this.extraction); + } +} diff --git a/packages/vision/src/types.ts b/packages/vision/src/types.ts new file mode 100644 index 0000000..66c0bc3 --- /dev/null +++ b/packages/vision/src/types.ts @@ -0,0 +1,44 @@ +/** + * Vision adapter contract (TRACKER step 4.4, failure modes A10/E3/A13). + * + * An adapter turns a screenshot into the same signals the matcher already + * understands — visible text, a structure descriptor, and annotation regions + * (circles/arrows a user drew to point at something). Images are **ephemeral**: + * processed in memory, never written to the graph or disk (G7). + */ +import type { StructureDescriptor } from "@coderadar/core"; + +/** A screenshot passed to an adapter. `data` is raw bytes or a base64 string. */ +export interface VisionImage { + data: Uint8Array | string; + mediaType: "image/png" | "image/jpeg" | "image/gif" | "image/webp"; +} + +/** A user annotation on the screenshot and the terms it emphasizes (E3). */ +export interface Region { + kind: "circle" | "arrow" | "box" | "highlight"; + /** Bounding box in image pixels. */ + bounds: { x: number; y: number; w: number; h: number }; + /** Text inside/near the annotation — weighted higher by the matcher. */ + terms: string[]; +} + +/** The structured result of reading a screenshot. */ +export interface VisionExtraction { + /** All visible text fragments. */ + terms: string[]; + /** Structural shape of the UI (a table with N columns, a form, a card grid). */ + structure: StructureDescriptor; + /** Annotation regions (circles/arrows) and the terms they point at. */ + annotations: Region[]; + /** + * False for Figma frames, marketing pages, and other non-app images — the + * matcher declines rather than guessing (A13). + */ + looksLikeApp: boolean; +} + +export interface VisionAdapter { + /** Extract signals from an image. Implementations must not persist the image. */ + extract(image: VisionImage): Promise; +} diff --git a/packages/vision/src/vision.test.ts b/packages/vision/src/vision.test.ts new file mode 100644 index 0000000..e0da20e --- /dev/null +++ b/packages/vision/src/vision.test.ts @@ -0,0 +1,42 @@ +import type { LineageGraph } from "@coderadar/core"; +import { describe, expect, it } from "vitest"; + +import { ANNOTATION_BOOST, extractionToQuery, matchFromVision } from "./query.js"; +import { StubVisionAdapter } from "./stub.js"; +import type { VisionExtraction } from "./types.js"; + +const emptyGraph: LineageGraph = { + version: 2, + root: "/app", + generatedAt: "2026-01-01T00:00:00.000Z", + generator: "test", + nodes: [], + edges: [], +}; + +const extraction: VisionExtraction = { + terms: ["Revenue breakdown", "Cost breakdown", "Total"], + structure: {}, + annotations: [{ kind: "circle", bounds: { x: 0, y: 0, w: 10, h: 10 }, terms: ["Revenue breakdown"] }], + looksLikeApp: true, +}; + +describe("vision adapter plumbing (TRACKER 4.4)", () => { + it("the stub returns its recorded extraction", async () => { + const adapter = new StubVisionAdapter(extraction); + expect(await adapter.extract()).toEqual(extraction); + }); + + it("boosts annotated terms and passes structure + terms through", () => { + const query = extractionToQuery(extraction); + expect(query.terms).toEqual(extraction.terms); + expect(query.boosts?.["Revenue breakdown"]).toBe(ANNOTATION_BOOST); + expect(query.boosts?.["Total"]).toBeUndefined(); + }); + + it("declines a screenshot that isn't this app (A13)", () => { + const result = matchFromVision(emptyGraph, { ...extraction, looksLikeApp: false }); + expect(result.status).toBe("declined"); + expect(result.declineReason).toBe("not-our-app"); + }); +}); diff --git a/packages/vision/tsconfig.json b/packages/vision/tsconfig.json new file mode 100644 index 0000000..5285d28 --- /dev/null +++ b/packages/vision/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist" + }, + "include": ["src"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fb26991..b61e8ef 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -42,6 +42,9 @@ importers: '@coderadar/parser-react': specifier: workspace:* version: link:../parser-react + '@coderadar/vision': + specifier: workspace:* + version: link:../vision '@types/node': specifier: ^22.20.1 version: 22.20.1 @@ -78,6 +81,25 @@ importers: yaml: specifier: ^2.9.0 version: 2.9.0 + devDependencies: + '@coderadar/vision': + specifier: workspace:* + version: link:../vision + '@types/node': + specifier: ^22.20.1 + version: 22.20.1 + typescript: + specifier: ^5.7.0 + version: 5.9.3 + vitest: + specifier: ^3.2.7 + version: 3.2.7(@types/node@22.20.1)(yaml@2.9.0) + + packages/vision: + dependencies: + '@coderadar/core': + specifier: workspace:* + version: link:../core devDependencies: '@types/node': specifier: ^22.20.1