diff --git a/TRACKER.md b/TRACKER.md index 2e3be31..a6426e9 100644 --- a/TRACKER.md +++ b/TRACKER.md @@ -227,7 +227,7 @@ The heart of the project. C1 and B1 live here. **Accept:** `a4-generic-text` green ("Save" alone → `ambiguous`; "Save" + "invoice details" → correct top-1); noisy-term fixture `a10-ocr-noise` (misspelled terms) top-3 correct. ### [x] 4.2 Structural matching -**Failure modes:** A1, A3, A12 +**Failure modes:** A1, A3, A12 — fixtures: `a1-no-static-text`, `a3-api-text`, `a12-non-text` (structure-only matches capped at medium confidence — honest graceful degradation) **Build:** structural signature per instance subtree (child element kinds/counts: table with N columns, form with M inputs, card grid). Query side accepts a structure descriptor (from vision output: "a table with columns Name, Email, Actions") and scores against signatures. Text and structure scores combine into one ranking. **Accept:** fixture `a1-no-static-text` (dashboard, zero literals) top-3 correct via structure alone. diff --git a/eval/fixtures/a12-non-text/app/FilterBar.tsx b/eval/fixtures/a12-non-text/app/FilterBar.tsx new file mode 100644 index 0000000..7bd0ac8 --- /dev/null +++ b/eval/fixtures/a12-non-text/app/FilterBar.tsx @@ -0,0 +1,8 @@ +export function FilterBar() { + return ( +
+ + +
+ ); +} diff --git a/eval/fixtures/a12-non-text/app/MetricsChart.tsx b/eval/fixtures/a12-non-text/app/MetricsChart.tsx new file mode 100644 index 0000000..d6f5280 --- /dev/null +++ b/eval/fixtures/a12-non-text/app/MetricsChart.tsx @@ -0,0 +1,11 @@ +// A chart with icon-only controls — a canvas and two unlabeled buttons. There +// is nothing to text-match; only structure gives a (low-confidence) candidate. +export function MetricsChart() { + return ( +
+ +
+ ); +} diff --git a/eval/fixtures/a12-non-text/golden.json b/eval/fixtures/a12-non-text/golden.json new file mode 100644 index 0000000..b5b88fd --- /dev/null +++ b/eval/fixtures/a12-non-text/golden.json @@ -0,0 +1,14 @@ +{ + "failureMode": "A12", + "note": "Non-text UI (charts, canvases, icon buttons): nothing to text-match. A text query declines rather than guessing; a structure descriptor returns a candidate at medium confidence — graceful degradation, not a confident wrong answer.", + "expect": { + "components": [ + { "name": "MetricsChart", "instances": 0 }, + { "name": "FilterBar", "instances": 0 } + ], + "queries": [ + { "terms": ["Revenue over time"], "status": "declined" }, + { "terms": [], "structure": { "buttons": 2 }, "status": "ok", "top": "MetricsChart", "confidence": "medium" } + ] + } +} diff --git a/eval/fixtures/a3-api-text/app/ApiForm.tsx b/eval/fixtures/a3-api-text/app/ApiForm.tsx new file mode 100644 index 0000000..fe56937 --- /dev/null +++ b/eval/fixtures/a3-api-text/app/ApiForm.tsx @@ -0,0 +1,10 @@ +export function ApiForm({ onSubmit }: { onSubmit: () => void }) { + // Labels/placeholders are all CMS-driven — no literals in source. + return ( +
+ + +