From 0d153ebcf8fab651bd0bd0384f89e84c0217c6d4 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 14:50:58 +0200 Subject: [PATCH 01/15] fix(desktop): preserve browser snapshot scroll context --- .github/workflows/preview-snapshot-tests.yml | 44 ++ apps/desktop/src/preview/Manager.ts | 69 +-- apps/desktop/src/preview/SnapshotPage.test.ts | 423 ++++++++++++++++++ apps/desktop/src/preview/SnapshotPage.ts | 312 +++++++++++++ apps/server/src/mcp/McpHttpServer.test.ts | 217 ++++++++- apps/server/src/mcp/McpHttpServer.ts | 138 ++++-- apps/server/src/mcp/toolkits/preview/tools.ts | 2 +- docs/user/browser.md | 14 + packages/contracts/src/previewAutomation.ts | 27 ++ .../src/previewAutomationSnapshot.test.ts | 65 +++ 10 files changed, 1216 insertions(+), 95 deletions(-) create mode 100644 .github/workflows/preview-snapshot-tests.yml create mode 100644 apps/desktop/src/preview/SnapshotPage.test.ts create mode 100644 apps/desktop/src/preview/SnapshotPage.ts create mode 100644 docs/user/browser.md create mode 100644 packages/contracts/src/previewAutomationSnapshot.test.ts diff --git a/.github/workflows/preview-snapshot-tests.yml b/.github/workflows/preview-snapshot-tests.yml new file mode 100644 index 000000000000..286eb6aae147 --- /dev/null +++ b/.github/workflows/preview-snapshot-tests.yml @@ -0,0 +1,44 @@ +name: Preview snapshot tests + +on: + pull_request: + paths: + - .github/workflows/preview-snapshot-tests.yml + - apps/desktop/src/preview/** + - apps/server/src/mcp/** + - packages/contracts/src/preview*.ts + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + name: Snapshot tests (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, windows-2025, macos-15] + steps: + - uses: actions/checkout@v6 + with: + sparse-checkout: | + /* + !/.repos/ + sparse-checkout-cone-mode: false + - uses: voidzero-dev/setup-vp@v1 + with: + node-version-file: package.json + cache: false + run-install: false + - run: vp install + - run: >- + vp test run + apps/desktop/src/preview/SnapshotPage.test.ts + apps/desktop/src/preview/Manager.test.ts + apps/server/src/mcp/McpHttpServer.test.ts + apps/server/src/mcp/PreviewAutomationBroker.test.ts + packages/contracts/src/previewAutomationSnapshot.test.ts + packages/contracts/src/preview.test.ts diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index f7b5046a7fed..e3734df849da 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -90,6 +90,7 @@ import { previewAutomationEditingCommandExpression, } from "./PreviewKeyboard.ts"; import { captureFavicon, safeHttpOrigin, selectFaviconCandidates } from "./FaviconCapture.ts"; +import { snapshotPageExpression, type SnapshotPage } from "./SnapshotPage.ts"; import { DEFAULT_RECORDING_INPUT_OPTIONS, type RecordingInputOptions } from "./RecordingInput.ts"; export type PreviewNavStatus = @@ -130,14 +131,6 @@ const ZOOM_LEVELS: ReadonlyArray = [ const DEFAULT_ZOOM_FACTOR = 1.0; const ZOOM_EPSILON = 0.001; const MAX_EVALUATION_BYTES = 64_000; -const MAX_VISIBLE_TEXT_LENGTH = 20_000; -const MAX_INTERACTIVE_ELEMENTS = 200; -/** - * A `[role]` container's innerText is its whole subtree, which turned one - * snapshot's element list into 60 KB of repeated page text. Names are labels, - * not content, so cap them where they are read. - */ -const MAX_INTERACTIVE_ELEMENT_NAME_LENGTH = 200; const MAX_SCREENSHOT_WIDTH = 1280; /** How long an armed tab keeps the exclusive display-media slot before another tab may take it. */ const RECORDING_ARM_GRACE_MS = 10_000; @@ -3744,66 +3737,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function concurrency: 2, discard: true, }); - const page = yield* evaluateWithDebugger<{ - url: string; - title: string; - loading: boolean; - visibleText: string; - interactiveElements: PreviewAutomationSnapshot["interactiveElements"]; - }>( + const page = yield* evaluateWithDebugger( tabId, send, - `(() => { - const selectorFor = (element) => { - if (element.id) return "#" + CSS.escape(element.id); - for (const attribute of ["data-testid", "name"]) { - const value = element.getAttribute(attribute); - if (value) return element.tagName.toLowerCase() + "[" + attribute + "=" + JSON.stringify(value) + "]"; - } - const buildParts = (current, parts = []) => { - if (!current || current.nodeType !== Node.ELEMENT_NODE || parts.length >= 8) { - return parts; - } - const parent = current.parentElement; - const siblings = parent - ? Array.from(parent.children).filter((child) => child.tagName === current.tagName) - : []; - const base = current.tagName.toLowerCase(); - const part = siblings.length > 1 - ? base + ":nth-of-type(" + (siblings.indexOf(current) + 1) + ")" - : base; - return buildParts(parent, [part, ...parts]); - }; - return buildParts(element).join(" > "); - }; - const visible = (element) => { - const style = getComputedStyle(element); - const rect = element.getBoundingClientRect(); - return style.visibility !== "hidden" && style.display !== "none" && rect.width > 0 && rect.height > 0; - }; - const elements = Array.from(document.querySelectorAll( - "a[href],button,input,textarea,select,[role],[tabindex]" - )).filter(visible).slice(0, ${MAX_INTERACTIVE_ELEMENTS}).map((element) => { - const rect = element.getBoundingClientRect(); - return { - tag: element.tagName.toLowerCase(), - role: element.getAttribute("role"), - name: (element.getAttribute("aria-label") || element.innerText || element.getAttribute("name") || "").slice(0, ${MAX_INTERACTIVE_ELEMENT_NAME_LENGTH}), - selector: selectorFor(element), - x: rect.x, - y: rect.y, - width: rect.width, - height: rect.height - }; - }); - return { - url: location.href, - title: document.title, - loading: document.readyState !== "complete", - visibleText: (document.body?.innerText || "").slice(0, ${MAX_VISIBLE_TEXT_LENGTH}), - interactiveElements: elements - }; - })()`, + snapshotPageExpression(), true, ); const [accessibility, sourceImage, diagnostics, timelines] = yield* Effect.all([ diff --git a/apps/desktop/src/preview/SnapshotPage.test.ts b/apps/desktop/src/preview/SnapshotPage.test.ts new file mode 100644 index 000000000000..9fdf0355cd20 --- /dev/null +++ b/apps/desktop/src/preview/SnapshotPage.test.ts @@ -0,0 +1,423 @@ +import * as NodeVM from "node:vm"; +import { describe, expect, it } from "vite-plus/test"; + +import { snapshotPageExpression } from "./SnapshotPage.ts"; + +const rect = (x: number, y: number, width = 100, height = 20) => ({ + x, + y, + left: x, + top: y, + right: x + width, + bottom: y + height, + width, + height, +}); + +class PageElement { + tagName = "DIV"; + id = ""; + innerText = ""; + parentElement: PageElement | null = null; + offsetParent: PageElement | null = null; + children: PageElement[] = []; + clientLeft = 0; + clientTop = 0; + clientWidth = 100; + clientHeight = 20; + offsetWidth = 100; + offsetHeight = 20; + scrollWidth = 100; + scrollHeight = 20; + scrollLeft = 0; + scrollTop = 0; + bounds = rect(0, 0); + style = { + display: "block", + visibility: "visible", + opacity: "1", + contentVisibility: "visible", + overflowX: "visible", + overflowY: "visible", + position: "static", + transform: "none", + perspective: "none", + filter: "none", + backdropFilter: "none", + contain: "none", + willChange: "auto", + borderRightWidth: "0px", + borderBottomWidth: "0px", + }; + getAttribute() { + return null; + } + getBoundingClientRect() { + return this.bounds; + } + checkVisibility(options: { checkVisibilityCSS?: boolean } = {}) { + const ancestors: PageElement[] = [this]; + if (this.parentElement) ancestors.push(this.parentElement); + for (let index = 0; index < ancestors.length; index++) { + const current = ancestors[index]!; + if (index > 0 && current.parentElement) ancestors.push(current.parentElement); + if ( + current.style.display === "none" || + current.style.opacity === "0" || + current.style.contentVisibility === "hidden" + ) + return false; + } + return ( + (!options.checkVisibilityCSS || this.style.visibility === "visible") && + this.style.display !== "contents" + ); + } +} + +type PageText = { + data: string; + length: number; + parentElement: PageElement; + rectangles: (start: number, end: number) => ReturnType[]; +}; + +const fixture = () => { + const root = new PageElement(); + const body = new PageElement(); + body.parentElement = root; + root.children.push(body); + root.scrollHeight = 10_000; + const elements: PageElement[] = []; + const texts: PageText[] = []; + const element = (id: string, bounds: ReturnType, parent = body) => { + const value = new PageElement(); + value.id = id; + value.bounds = bounds; + value.parentElement = parent; + parent.children.push(value); + return value; + }; + const text = ( + data: string, + parent: PageElement, + rectangles: PageText["rectangles"] = () => [parent.bounds], + ) => { + texts.push({ data, length: data.length, parentElement: parent, rectangles }); + }; + let start = 0; + let end = 0; + let active: PageText; + const range = { + setStart(node: PageText, offset: number) { + active = node; + start = offset; + }, + setEnd(_node: PageText, offset: number) { + end = offset; + }, + getClientRects() { + return active.rectangles(start, end); + }, + getBoundingClientRect() { + const rectangles = this.getClientRects(); + const left = Math.min(...rectangles.map((value) => value.left)); + const top = Math.min(...rectangles.map((value) => value.top)); + return rect( + left, + top, + Math.max(...rectangles.map((value) => value.right)) - left, + Math.max(...rectangles.map((value) => value.bottom)) - top, + ); + }, + }; + const context = { + innerWidth: 300, + innerHeight: 200, + scrollX: 0, + scrollY: 4_000, + location: { href: "https://example.test" }, + CSS: { escape: (value: string) => value }, + HTMLElement: PageElement, + NodeFilter: { SHOW_TEXT: 4 }, + getComputedStyle: (value: PageElement) => value.style, + document: { + documentElement: root, + body, + scrollingElement: root as PageElement | null, + title: "Page", + readyState: "complete", + compatMode: "CSS1Compat", + querySelectorAll: () => elements, + createRange: () => range, + createTreeWalker() { + let index = 0; + return { nextNode: () => texts[index++] ?? null }; + }, + }, + }; + const capture = (scrollingElement: PageElement | null = root, compatMode = "CSS1Compat") => { + context.document.scrollingElement = scrollingElement; + context.document.compatMode = compatMode; + return NodeVM.runInNewContext(snapshotPageExpression(), context) as ReturnType< + typeof import("./SnapshotPage.ts").collectSnapshotPage + >; + }; + return { root, body, elements, texts, element, text, capture }; +}; + +describe("snapshot page collector", () => { + it("keeps bottom text after more than 4096 offscreen paragraphs", () => { + const page = fixture(); + for (let index = 0; index < 5_000; index++) { + page.text("offscreen", page.element(`paragraph-${index}`, rect(0, -100))); + } + page.text("bottom marker", page.element("bottom", rect(0, 100))); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("bottom marker"); + expect(snapshot.truncated?.viewportText).toBe(false); + }); + + it("reads direct display:contents text while respecting hidden ancestors", () => { + const page = fixture(); + const contents = page.element("contents", rect(0, 0, 0, 0)); + contents.style.display = "contents"; + page.text("direct contents text", contents, () => [rect(0, 20)]); + const hidden = page.element("hidden", rect(0, 40)); + hidden.style.opacity = "0"; + const hiddenContents = page.element("hidden-contents", rect(0, 0, 0, 0), hidden); + hiddenContents.style.display = "contents"; + page.text("hidden contents text", hiddenContents, () => [rect(0, 40)]); + const visibility = page.element("visibility", rect(0, 60)); + visibility.style.visibility = "hidden"; + const override = page.element("override", rect(0, 0, 0, 0), visibility); + override.style.display = "contents"; + page.text("visible override", override, () => [rect(0, 60)]); + expect(page.capture().viewportText).toBe("direct contents text visible override"); + }); + + it("captures text under deeply nested ancestors without recursive stack growth", () => { + const page = fixture(); + let parent = page.body; + for (let index = 0; index < 5_000; index++) + parent = page.element(`nested-${index}`, rect(0, 100), parent); + page.text("deep text", parent); + expect(page.capture().viewportText).toBe("deep text"); + }); + + it("clips an independently scrolling body in standard and quirks documents", () => { + const page = fixture(); + page.root.style.overflowY = "hidden"; + page.body.style.overflowY = "auto"; + page.body.bounds = rect(0, 0, 300, 100); + page.body.clientWidth = page.body.offsetWidth = 300; + page.body.clientHeight = page.body.offsetHeight = 100; + page.body.scrollHeight = 1_000; + page.body.scrollTop = 200; + const clipped = page.element("body-clipped", rect(0, 140)); + page.elements.push(clipped); + page.text("clipped body text", clipped); + for (const scrollingElement of [page.root, null]) { + const snapshot = page.capture(scrollingElement); + expect(snapshot.viewportText).toBe(""); + expect(snapshot.interactiveElements[0]?.inViewport).toBe(false); + expect(snapshot.scroll?.containers[0]).toMatchObject({ + selector: "div > div", + y: 200, + height: 100, + scrollHeight: 1_000, + }); + } + }); + + it("uses the actual body box when quirks client dimensions report the viewport", () => { + const page = fixture(); + page.root.style.overflowX = page.root.style.overflowY = "hidden"; + page.body.style.overflowX = page.body.style.overflowY = "auto"; + page.body.bounds = rect(0, 0, 200, 200); + page.body.clientWidth = 300; + page.body.clientHeight = 800; + page.body.offsetWidth = page.body.offsetHeight = 100; + page.body.clientLeft = page.body.clientTop = 2; + page.body.style.borderRightWidth = page.body.style.borderBottomWidth = "2px"; + page.body.scrollWidth = 300; + page.body.scrollHeight = 245; + const vertical = page.element("vertical", rect(10, 198, 50, 2)); + const horizontal = page.element("horizontal", rect(198, 10, 2, 50)); + page.elements.push(vertical, horizontal); + page.text("vertical text", vertical); + page.text("horizontal text", horizontal); + const snapshot = page.capture(null, "BackCompat"); + expect(snapshot.viewportText).toBe(""); + expect(snapshot.interactiveElements.every((element) => element.inViewport === false)).toBe( + true, + ); + expect(snapshot.scroll?.containers[0]).toMatchObject({ + width: 96, + height: 96, + scrollWidth: 300, + scrollHeight: 245, + }); + page.body.offsetHeight = 1_000; + page.body.bounds = rect(0, 0, 200, 2_000); + page.body.scrollHeight = 2_000; + expect(page.capture(null, "BackCompat").scroll?.containers[0]?.height).toBe(996); + }); + + it("keeps current controls ahead of more than 200 offscreen controls", () => { + const page = fixture(); + for (let index = 0; index < 240; index++) { + const control = page.element(`button-${index}`, rect(0, index < 239 ? -1_000 : 100)); + control.tagName = "BUTTON"; + page.elements.push(control); + } + page.body.innerText = "whole page ".repeat(3_000); + page.text("on screen", page.elements[239]!); + const snapshot = page.capture(); + expect(snapshot.interactiveElements).toHaveLength(200); + expect(snapshot.interactiveElements[0]).toMatchObject({ + selector: "#button-239", + inViewport: true, + }); + expect(snapshot.viewportText).toBe("on screen"); + expect(snapshot.visibleText).toHaveLength(20_000); + expect(snapshot.truncated).toEqual({ + visibleText: true, + viewportText: false, + interactiveElements: true, + }); + expect(snapshot.scroll).toMatchObject({ y: 4_000, height: 200, scrollHeight: 10_000 }); + }); + + it("clips nested scroll text and controls without changing scroll offsets", () => { + const page = fixture(); + const outer = page.element("outer", rect(10, 10, 100, 100)); + outer.clientHeight = 100; + outer.offsetHeight = 100; + outer.style.overflowY = "auto"; + outer.scrollHeight = 1_000; + outer.scrollTop = 600; + const inner = page.element("inner", rect(20, 30, 80, 40), outer); + inner.clientWidth = inner.offsetWidth = 80; + inner.clientHeight = inner.offsetHeight = 40; + inner.style.overflowX = "hidden"; + inner.style.overflowY = "clip"; + const visible = page.element("visible", rect(20, 35, 70, 20), inner); + const clipped = page.element("clipped", rect(20, 80, 70, 20), inner); + page.elements.push(clipped, visible); + page.text("visible words", visible); + page.text("hidden under scroll", clipped); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("visible words"); + expect(snapshot.interactiveElements[0]).toMatchObject({ + selector: "#visible", + inViewport: true, + }); + expect(snapshot.interactiveElements[1]).toMatchObject({ + selector: "#clipped", + inViewport: false, + }); + expect(snapshot.scroll?.containers).toEqual([ + { + selector: "#outer", + x: 0, + y: 600, + width: 100, + height: 100, + scrollWidth: 100, + scrollHeight: 1_000, + }, + ]); + expect(outer.scrollTop).toBe(600); + }); + + it("reads only visible characters from a huge wrapped text node", () => { + const page = fixture(); + const parent = page.element("long", rect(0, -100_000, 100, 200_000)); + const data = "x".repeat(100_000) + "VISIBLE" + "z".repeat(100_000); + page.text(data, parent, (start: number, end: number) => { + const rectangles = []; + if (start < 100_000) rectangles.push(rect(0, -100, 100, 20)); + if (end > 100_000 && start < 100_007) rectangles.push(rect(0, 100, 100, 20)); + if (end > 100_007) rectangles.push(rect(0, 500, 100, 20)); + return rectangles; + }); + expect(page.capture().viewportText).toBe("VISIBLE"); + }); + + it("clips positioned descendants at their containing block", () => { + const page = fixture(); + const scroller = page.element("scroller", rect(0, 0)); + scroller.style.overflowY = "auto"; + scroller.scrollHeight = 1_000; + const fixed = page.element("fixed", rect(180, 100), scroller); + fixed.style.position = "fixed"; + const absolute = page.element("absolute", rect(0, 140), scroller); + absolute.style.position = "absolute"; + absolute.offsetParent = page.body; + const transformed = page.element("transformed", rect(0, 0)); + transformed.style.overflowY = "hidden"; + transformed.style.transform = "translateX(0px)"; + const clipped = page.element("clipped", rect(0, 160), transformed); + clipped.style.position = "fixed"; + clipped.offsetParent = transformed; + page.elements.push(fixed, absolute, clipped); + page.text("fixed label", fixed); + page.text("absolute label", absolute); + page.text("clipped label", clipped); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("fixed label absolute label"); + expect( + snapshot.interactiveElements.map((element) => [element.selector, element.inViewport]), + ).toEqual([ + ["#fixed", true], + ["#absolute", true], + ["#clipped", false], + ]); + }); + + it("omits hidden styles and clips horizontal text", () => { + const page = fixture(); + const transparent = page.element("transparent", rect(0, 0)); + transparent.style.opacity = "0"; + const hidden = page.element("hidden", rect(0, 20)); + hidden.style.contentVisibility = "hidden"; + const visibility = page.element("visibility", rect(0, 40)); + visibility.style.visibility = "hidden"; + for (const parent of [transparent, hidden, visibility]) page.text("secret", parent); + const clipped = page.element("horizontal", rect(-100, 60, 500, 20)); + page.text("abcdefghijklmnopqrst", clipped, (start: number, end: number) => [ + rect(-100 + start * 25, 60, (end - start) * 25, 20), + ]); + expect(page.capture().viewportText).toBe("efghijklmnop"); + }); + + it("reports capped viewport text and scroll container metadata", () => { + const page = fixture(); + for (let index = 0; index < 21; index++) { + const container = page.element(`scroll-${index}`, rect(0, 0)); + container.style.overflowY = "scroll"; + container.scrollHeight = 100; + page.text("text", container); + } + const large = page.element("large", rect(0, 100)); + page.text("x".repeat(25_000), large); + const snapshot = page.capture(); + expect(snapshot.scroll?.containers).toHaveLength(20); + expect(snapshot.scroll?.containersTruncated).toBe(true); + expect(snapshot.viewportText).toHaveLength(20_000); + expect(snapshot.truncated?.viewportText).toBe(true); + }); + + it("omits oversized scroll selectors while preserving current text", () => { + const page = fixture(); + const container = page.element("x".repeat(2_000), rect(0, 0)); + container.style.overflowY = "scroll"; + container.scrollHeight = 1_000; + page.text("current text", container); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("current text"); + expect(snapshot.scroll?.containers).toEqual([]); + expect(snapshot.scroll?.containersTruncated).toBe(true); + }); +}); diff --git a/apps/desktop/src/preview/SnapshotPage.ts b/apps/desktop/src/preview/SnapshotPage.ts new file mode 100644 index 000000000000..a5bb94aba301 --- /dev/null +++ b/apps/desktop/src/preview/SnapshotPage.ts @@ -0,0 +1,312 @@ +import type { PreviewAutomationSnapshot } from "@t3tools/contracts"; + +export type SnapshotPage = Pick< + PreviewAutomationSnapshot, + | "url" + | "title" + | "loading" + | "visibleText" + | "viewportText" + | "scroll" + | "truncated" + | "interactiveElements" +>; + +export function collectSnapshotPage(): SnapshotPage { + const maxTextLength = 20_000; + const maxElements = 200; + const viewport = { left: 0, top: 0, right: innerWidth, bottom: innerHeight }; + type Bounds = typeof viewport; + const intersects = (rect: Bounds, clip: Bounds) => + rect.right > clip.left && + rect.left < clip.right && + rect.bottom > clip.top && + rect.top < clip.bottom; + const selectorFor = (element: Element): string => { + if (element.id) return "#" + CSS.escape(element.id); + for (const attribute of ["data-testid", "name"]) { + const value = element.getAttribute(attribute); + if (value) + return element.tagName.toLowerCase() + "[" + attribute + "=" + JSON.stringify(value) + "]"; + } + const parts: string[] = []; + for ( + let current: Element | null = element; + current && parts.length < 8; + current = current.parentElement + ) { + const siblings = current.parentElement + ? Array.from(current.parentElement.children).filter( + (child) => child.tagName === current.tagName, + ) + : []; + parts.unshift( + current.tagName.toLowerCase() + + (siblings.length > 1 ? ":nth-of-type(" + (siblings.indexOf(current) + 1) + ")" : ""), + ); + } + return parts.join(" > "); + }; + const containers: Array["containers"][number]> = []; + let containersTruncated = false; + const clips = new Map(); + const styles = new Map(); + const styleFor = (element: Element) => { + let style = styles.get(element); + if (!style) { + style = getComputedStyle(element); + styles.set(element, style); + } + return style; + }; + const clipFor = (element: Element): Bounds | null => { + const pending: Element[] = []; + let ancestor: Element | null = element; + while (ancestor && !clips.has(ancestor)) { + pending.push(ancestor); + ancestor = + ancestor instanceof HTMLElement && /^(absolute|fixed)$/.test(styleFor(ancestor).position) + ? ancestor.offsetParent + : ancestor.parentElement; + } + let parentClip: Bounds | null = ancestor ? (clips.get(ancestor) ?? null) : viewport; + for (let index = pending.length - 1; index >= 0; index--) { + const current = pending[index]!; + const style = styleFor(current); + if ( + !parentClip || + style.display === "none" || + style.opacity === "0" || + style.contentVisibility === "hidden" + ) { + clips.set(current, null); + parentClip = null; + continue; + } + const clip = { ...parentClip }; + const rootStyle = styleFor(document.documentElement); + const viewportBody = + current === document.body && + rootStyle.overflowX === "visible" && + rootStyle.overflowY === "visible"; + if (current !== document.documentElement && !viewportBody && style.display !== "contents") { + const clipsX = /^(auto|scroll|hidden|clip)$/.test(style.overflowX); + const clipsY = /^(auto|scroll|hidden|clip)$/.test(style.overflowY); + if (clipsX || clipsY) { + const rect = current.getBoundingClientRect(); + const quirksBody = + current === document.body && + current instanceof HTMLElement && + document.compatMode === "BackCompat"; + const clientWidth = quirksBody + ? Math.max( + 0, + current.offsetWidth - + current.clientLeft - + Number.parseFloat(style.borderRightWidth), + ) + : current.clientWidth; + const clientHeight = quirksBody + ? Math.max( + 0, + current.offsetHeight - + current.clientTop - + Number.parseFloat(style.borderBottomWidth), + ) + : current.clientHeight; + const scaleX = + current instanceof HTMLElement && current.offsetWidth + ? rect.width / current.offsetWidth + : 1; + const scaleY = + current instanceof HTMLElement && current.offsetHeight + ? rect.height / current.offsetHeight + : 1; + if (clipsX) { + clip.left = Math.max(clip.left, rect.left + current.clientLeft * scaleX); + clip.right = Math.min( + clip.right, + rect.left + (current.clientLeft + clientWidth) * scaleX, + ); + } + if (clipsY) { + clip.top = Math.max(clip.top, rect.top + current.clientTop * scaleY); + clip.bottom = Math.min( + clip.bottom, + rect.top + (current.clientTop + clientHeight) * scaleY, + ); + } + const scrollable = + (/^(auto|scroll)$/.test(style.overflowX) && current.scrollWidth > clientWidth) || + (/^(auto|scroll)$/.test(style.overflowY) && current.scrollHeight > clientHeight); + if (scrollable && intersects(rect, parentClip)) { + const selector = selectorFor(current); + if (containers.length < 20 && selector.length <= 1_000) { + containers.push({ + selector, + x: current.scrollLeft, + y: current.scrollTop, + width: clientWidth, + height: clientHeight, + scrollWidth: current.scrollWidth, + scrollHeight: current.scrollHeight, + }); + } else containersTruncated = true; + } + } + } + parentClip = clip.right > clip.left && clip.bottom > clip.top ? clip : null; + clips.set(current, parentClip); + } + return clips.get(element)!; + }; + const rendered = (element: Element) => { + const style = styleFor(element); + let box = element; + while (styleFor(box).display === "contents" && box.parentElement) box = box.parentElement; + return ( + style.visibility !== "hidden" && + style.visibility !== "collapse" && + style.display !== "none" && + box.checkVisibility({ + checkOpacity: true, + contentVisibilityAuto: true, + }) + ); + }; + const inViewport = (element: Element, rect: DOMRect) => { + if (!intersects(rect, viewport)) return false; + const clip = clipFor(element); + return clip !== null && intersects(rect, clip); + }; + const currentElements: Element[] = []; + const otherElements: Element[] = []; + let elementCount = 0; + for (const element of document.querySelectorAll( + "a[href],button,input,textarea,select,[role],[tabindex]", + )) { + if (!rendered(element)) continue; + const rect = element.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) continue; + const current = inViewport(element, rect); + elementCount++; + const target = current ? currentElements : otherElements; + if (target.length < maxElements) target.push(element); + } + const interactiveElements = [...currentElements, ...otherElements] + .slice(0, maxElements) + .map((element) => { + const rect = element.getBoundingClientRect(); + return { + tag: element.tagName.toLowerCase(), + role: element.getAttribute("role"), + name: ( + element.getAttribute("aria-label") || + (element instanceof HTMLElement ? element.innerText : "") || + element.getAttribute("name") || + "" + ).slice(0, 200), + selector: selectorFor(element), + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + inViewport: currentElements.includes(element), + }; + }); + let viewportText = ""; + let viewportTextTruncated = false; + let rangeReads = 0; + let nodeText = ""; + const range = document.createRange(); + const appendText = (value: string) => { + const normalized = value.replace(/\s+/g, " "); + if (!normalized.trim()) return; + const separator = + viewportText && !viewportText.endsWith(" ") && !normalized.startsWith(" ") ? " " : ""; + const remaining = maxTextLength - viewportText.length; + const addition = separator + normalized; + viewportText += addition.slice(0, remaining); + if (addition.length > remaining) viewportTextTruncated = true; + }; + const readText = (node: Text, start: number, end: number, clip: Bounds) => { + if (viewportTextTruncated || start === end) return; + range.setStart(node, start); + range.setEnd(node, end); + const bounds = range.getBoundingClientRect(); + if (!intersects(bounds, clip)) return; + if (++rangeReads > 4_096) { + viewportTextTruncated = true; + return; + } + if ( + bounds.left >= clip.left && + bounds.right <= clip.right && + bounds.top >= clip.top && + bounds.bottom <= clip.bottom + ) { + nodeText += node.data.slice(start, end); + return; + } + if (!Array.from(range.getClientRects()).some((rect) => intersects(rect, clip))) return; + if (end - start === 1) { + nodeText += node.data.slice(start, end); + return; + } + const middle = start + Math.floor((end - start) / 2); + readText(node, start, middle, clip); + readText(node, middle, end, clip); + }; + if (document.body) { + const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT); + let node = walker.nextNode(); + let visited = 0; + while (node && !viewportTextTruncated) { + if (++visited > 50_000) { + viewportTextTruncated = true; + break; + } + const parent = node.parentElement; + if ( + parent && + !/^(SCRIPT|STYLE|NOSCRIPT|TEMPLATE)$/.test(parent.tagName) && + rendered(parent) + ) { + const clip = clipFor(parent); + if (clip) { + nodeText = ""; + readText(node as Text, 0, (node as Text).length, clip); + appendText(nodeText); + } + } + node = walker.nextNode(); + } + } + const visibleText = document.body?.innerText || ""; + const root = document.scrollingElement ?? document.documentElement; + return { + url: location.href, + title: document.title, + loading: document.readyState !== "complete", + visibleText: visibleText.slice(0, maxTextLength), + viewportText: viewportText.trim(), + interactiveElements, + scroll: { + x: scrollX, + y: scrollY, + width: innerWidth, + height: innerHeight, + scrollWidth: root.scrollWidth, + scrollHeight: root.scrollHeight, + containers, + containersTruncated, + }, + truncated: { + visibleText: visibleText.length > maxTextLength, + viewportText: viewportTextTruncated, + interactiveElements: elementCount > maxElements, + }, + }; +} + +export const snapshotPageExpression = () => `(${collectSnapshotPage.toString()})()`; diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index 69801ee06100..cce09515a8ac 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -1,3 +1,5 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off +import * as NodeHttp from "node:http"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import { expect, it } from "@effect/vitest"; @@ -12,7 +14,14 @@ import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import { McpProtocol, McpSchema, McpServer } from "effect/unstable/ai"; -import { HttpBody, HttpClient, HttpRouter, HttpServerResponse } from "effect/unstable/http"; +import { + FetchHttpClient, + HttpBody, + HttpClient, + HttpRouter, + HttpServer, + HttpServerResponse, +} from "effect/unstable/http"; import * as ProjectService from "../project/ProjectService.ts"; import * as ServerConfig from "../config.ts"; @@ -671,6 +680,201 @@ it.effect("sheds log entries before locators when every list is full", () => ).pipe(Effect.provide(TestLayer)), ); +it.effect("bounds JSON-escaped page identifiers after other snapshot fields are empty", () => + Effect.scoped( + Effect.gen(function* () { + yield* serveSnapshots("mcp-escaped-identifiers-client", { + ...snapshotResult, + url: `http://example.test/${"\u0000".repeat(3_000)}`, + title: "\u0000".repeat(3_000), + visibleText: "", + }); + const snapshot = yield* callSnapshot({ includeImage: false }); + expect(snapshot.isError).toBe(false); + const [, text, notice] = snapshot.content; + const body = text?.type === "text" ? text.text : ""; + expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual( + McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES, + ); + expect(notice?.type === "text" ? notice.text : "").toContain( + "url or title after 1024 characters", + ); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect( + "keeps current-view text and controls when offscreen controls and logs fill the budget", + () => + Effect.scoped( + Effect.gen(function* () { + const scroll = { + x: 0, + y: 12_000, + width: 1_000, + height: 800, + scrollWidth: 1_000, + scrollHeight: 20_000, + containers: [], + containersTruncated: false, + }; + yield* serveSnapshots("mcp-current-view-client", { + ...snapshotResult, + visibleText: "Page start ".repeat(4_000), + viewportText: "Bottom of page: choose Continue to finish.", + scroll, + truncated: { visibleText: true, viewportText: false, interactiveElements: true }, + interactiveElements: Array.from({ length: 240 }, (_, index) => ({ + tag: "button", + role: "button", + name: `Button ${index}`, + selector: `#button-${index}-${"x".repeat(500)}`, + inViewport: index >= 237, + x: 0, + y: index >= 237 ? 20 : -1_000, + width: 10, + height: 10, + })), + consoleEntries: Array.from({ length: 100 }, (_, index) => ({ + level: "log", + text: `Log ${index}: ${"x".repeat(2_000)}`, + timestamp: "t", + })), + }); + + const snapshot = yield* callSnapshot({ includeImage: false }); + expect(snapshot.isError).toBe(false); + const [, text, notice] = snapshot.content; + const body = text?.type === "text" ? text.text : ""; + expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual( + McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES, + ); + const parsed = decodeJsonText(body) as { + readonly viewportText: string; + readonly scroll: unknown; + readonly truncated: unknown; + readonly interactiveElements: ReadonlyArray<{ readonly name: string }>; + }; + expect(parsed.viewportText).toBe("Bottom of page: choose Continue to finish."); + expect(parsed.scroll).toEqual(scroll); + expect(parsed.truncated).toEqual({ + visibleText: true, + viewportText: false, + interactiveElements: true, + }); + expect(parsed.interactiveElements.slice(0, 3).map((element) => element.name)).toEqual([ + "Button 237", + "Button 238", + "Button 239", + ]); + expect(notice?.type === "text" ? notice.text : "").toContain("interactiveElements"); + expect(snapshot.structuredContent).toEqual({ ...parsed, omitted: expect.any(Array) }); + expect(snapshot.content.some((content) => content.type === "image")).toBe(false); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect.each(["η•ŒπŸ˜€".repeat(5_000), `${"a".repeat(7_999)}${"πŸ˜€".repeat(4_000)}`])( + "bounds current-view Unicode text without splitting a character %#", + (viewportText) => + Effect.scoped( + Effect.gen(function* () { + yield* serveSnapshots("mcp-viewport-unicode-client", { + ...snapshotResult, + viewportText, + visibleText: "Offscreen ".repeat(4_000), + truncated: { visibleText: false, viewportText: false, interactiveElements: false }, + interactiveElements: Array.from({ length: 20 }, (_, index) => ({ + tag: "button", + role: "button", + name: `Button ${index}`, + selector: `#button-${index}-${"x".repeat(500)}`, + inViewport: true, + x: 0, + y: 0, + width: 10, + height: 10, + })), + }); + const snapshot = yield* callSnapshot({ includeImage: false }); + expect(snapshot.isError).toBe(false); + const [, text, notice] = snapshot.content; + const body = text?.type === "text" ? text.text : ""; + expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual( + McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES, + ); + const parsed = decodeJsonText(body) as { + readonly viewportText: string; + readonly interactiveElements: ReadonlyArray; + }; + expect(parsed.interactiveElements.length).toBeGreaterThan(0); + expect(parsed.viewportText.length).toBeGreaterThan(1_000); + expect(parsed.viewportText.endsWith("…")).toBe(true); + expect(parsed.viewportText.isWellFormed()).toBe(true); + expect(parsed).toMatchObject({ truncated: { viewportText: true } }); + expect(notice?.type === "text" ? notice.text : "").toContain("viewportText after"); + expect(snapshot.structuredContent).toEqual({ ...parsed, omitted: expect.any(Array) }); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect("keeps current-view Unicode text in full when it fits the snapshot budget", () => + Effect.scoped( + Effect.gen(function* () { + const viewportText = "η•Œ".repeat(5_000); + yield* serveSnapshots("mcp-viewport-full-unicode-client", { + ...snapshotResult, + visibleText: "", + viewportText, + }); + const snapshot = yield* callSnapshot({ includeImage: false }); + expect(snapshot.isError).toBe(false); + expect(snapshot.structuredContent).toMatchObject({ viewportText }); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect("bounds scroll-container locators while preserving page scroll position", () => + Effect.scoped( + Effect.gen(function* () { + yield* serveSnapshots("mcp-scroll-budget-client", { + ...snapshotResult, + viewportText: "Current content", + scroll: { + x: 20, + y: 8_000, + width: 1_000, + height: 800, + scrollWidth: 2_000, + scrollHeight: 10_000, + containers: Array.from({ length: 20 }, () => ({ + selector: `#${"x".repeat(30_000)}`, + x: 0, + y: 200, + width: 100, + height: 100, + scrollWidth: 100, + scrollHeight: 1_000, + })), + containersTruncated: false, + }, + }); + const snapshot = yield* callSnapshot({ includeImage: false }); + expect(snapshot.isError).toBe(false); + const [, text, notice] = snapshot.content; + const body = text?.type === "text" ? text.text : ""; + expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual( + McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES, + ); + expect(decodeJsonText(body)).toMatchObject({ + viewportText: "Current content", + scroll: { x: 20, y: 8_000, containers: [], containersTruncated: true }, + }); + expect(notice?.type === "text" ? notice.text : "").toContain("20 of 20 scrollContainers"); + }), + ).pipe(Effect.provide(TestLayer)), +); + it.effect("terminates HTTP MCP sessions with DELETE", () => Effect.scoped( Effect.gen(function* () { @@ -722,7 +926,16 @@ it.effect("terminates HTTP MCP sessions with DELETE", () => }); expect(reusedSessionResponse.status).toBe(404); }), - ).pipe(Effect.provide(NodeHttpServer.layerTest)), + ).pipe( + Effect.provide( + HttpServer.layerTestClient.pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provideMerge( + NodeHttpServer.layer(NodeHttp.createServer, { host: "127.0.0.1", port: 0 }), + ), + ), + ), + ), ); it.effect("registers annotated tools and preserves authenticated request context", () => diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 5c8f146bcadf..80c2190d1949 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -149,8 +149,12 @@ const MAX_SNAPSHOT_IDENTIFIER_CHARS = 2_048; const encodeJsonText = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); const utf8Length = (text: string) => Buffer.byteLength(text, "utf8"); -const cutText = (text: string, max: number) => - text.length > max ? `${text.slice(0, max)}…` : text; +const cutText = (text: string, max: number) => { + if (text.length <= max) return text; + const last = text.charCodeAt(max - 1); + const end = last >= 0xd800 && last <= 0xdbff ? max - 1 : max; + return `${text.slice(0, end)}…`; +}; /** Shortens every string field of a log entry; other fields pass through. */ const cutEntryStrings = (entry: A): A => @@ -172,8 +176,20 @@ type SnapshotMetadata = { readonly url: string; readonly title: string; readonly visibleText: string; + readonly viewportText?: string; + readonly truncated?: { + readonly visibleText: boolean; + readonly viewportText: boolean; + readonly interactiveElements: boolean; + }; + readonly scroll?: { + readonly containers: ReadonlyArray; + readonly containersTruncated: boolean; + readonly [key: string]: unknown; + }; readonly interactiveElements: ReadonlyArray<{ readonly name: string; + readonly inViewport?: boolean; readonly [key: string]: unknown; }>; readonly consoleEntries: ReadonlyArray; @@ -222,24 +238,22 @@ const boundSnapshotMetadata = (metadata: SnapshotMetadata) => { ...withoutTree, url: cutText(metadata.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), title: cutText(metadata.title, MAX_SNAPSHOT_IDENTIFIER_CHARS), - interactiveElements: metadata.interactiveElements.map((element) => ({ - ...element, - name: cutText(element.name, MAX_SNAPSHOT_ELEMENT_NAME_CHARS), - })), + interactiveElements: metadata.interactiveElements + .map((element) => ({ + ...element, + name: cutText(element.name, MAX_SNAPSHOT_ELEMENT_NAME_CHARS), + })) + .sort((left, right) => Number(right.inViewport === true) - Number(left.inViewport === true)), consoleEntries: tail(metadata.consoleEntries, "console entries"), networkEntries: tail(metadata.networkEntries, "network entries"), actionTimeline: tail(metadata.actionTimeline, "action timeline entries"), }; - // Per-field caps do not sum below the ceiling: three log arrays of 40 capped - // entries alone can pass 60 KB, and the caps count characters, not bytes. - // Halve one thing per round until the JSON fits: logs first, then page - // text, then the locators. The identifier caps bound the rest, so this - // terminates. const shedOrder = [ "actionTimeline", "networkEntries", "consoleEntries", + "scrollContainers", "interactiveElements", ] as const; const lists: Record<(typeof shedOrder)[number], ReadonlyArray> = { @@ -247,42 +261,104 @@ const boundSnapshotMetadata = (metadata: SnapshotMetadata) => { consoleEntries: bounded.consoleEntries, networkEntries: bounded.networkEntries, actionTimeline: bounded.actionTimeline, + scrollContainers: metadata.scroll?.containers ?? [], }; const dropped: Record<(typeof shedOrder)[number], number> = { interactiveElements: 0, consoleEntries: 0, networkEntries: 0, actionTimeline: 0, + scrollContainers: 0, }; let visibleTextChars = Math.min(metadata.visibleText.length, MAX_SNAPSHOT_VISIBLE_TEXT_CHARS); + let viewportTextChars = Math.min( + metadata.viewportText?.length ?? 0, + MAX_SNAPSHOT_VISIBLE_TEXT_CHARS, + ); + let identifierChars = MAX_SNAPSHOT_IDENTIFIER_CHARS; const value = () => ({ ...bounded, + url: cutText(metadata.url, identifierChars), + title: cutText(metadata.title, identifierChars), visibleText: cutText(metadata.visibleText, visibleTextChars), - ...lists, + ...(metadata.viewportText === undefined + ? {} + : { viewportText: cutText(metadata.viewportText, viewportTextChars) }), + interactiveElements: lists.interactiveElements, + consoleEntries: lists.consoleEntries, + networkEntries: lists.networkEntries, + actionTimeline: lists.actionTimeline, + ...(metadata.truncated === undefined + ? {} + : { + truncated: { + visibleText: + metadata.truncated.visibleText || visibleTextChars < metadata.visibleText.length, + viewportText: + metadata.truncated.viewportText || + viewportTextChars < (metadata.viewportText?.length ?? 0), + interactiveElements: + metadata.truncated.interactiveElements || dropped.interactiveElements > 0, + }, + }), + ...(metadata.scroll === undefined + ? {} + : { + scroll: { + ...metadata.scroll, + containers: lists.scrollContainers, + containersTruncated: + metadata.scroll.containersTruncated || dropped.scrollContainers > 0, + }, + }), }); + const inViewportCount = bounded.interactiveElements.filter( + (element) => element.inViewport === true, + ).length; let text = encodeJsonText(value()); while (utf8Length(text) > MAX_SNAPSHOT_TEXT_BYTES) { - // Elements carry the locators, so they go last; logs shed newest-last. - const key = - shedOrder.find( - (candidate) => candidate !== "interactiveElements" && lists[candidate].length > 0, - ) ?? - (visibleTextChars > 0 - ? "visibleText" - : lists.interactiveElements.length > 0 - ? "interactiveElements" - : undefined); + let key: + | (typeof shedOrder)[number] + | "visibleText" + | "viewportText" + | "identifiers" + | undefined = shedOrder.find( + (candidate) => + candidate !== "interactiveElements" && + candidate !== "scrollContainers" && + lists[candidate].length > 0, + ); + if (key === undefined) { + if (visibleTextChars > 0) key = "visibleText"; + else if (lists.interactiveElements.length > inViewportCount) key = "interactiveElements"; + else if (lists.scrollContainers.length > 0) key = "scrollContainers"; + else if ( + viewportTextChars > 0 && + utf8Length(encodeJsonText(cutText(metadata.viewportText ?? "", viewportTextChars))) > + MAX_SNAPSHOT_TEXT_BYTES / 2 + ) + key = "viewportText"; + else if (lists.interactiveElements.length > 0) key = "interactiveElements"; + else if (viewportTextChars > 0) key = "viewportText"; + else if (identifierChars > 0) key = "identifiers"; + } if (key === undefined) break; if (key === "visibleText") { visibleTextChars = Math.floor(visibleTextChars / 2); + } else if (key === "viewportText") { + viewportTextChars = Math.floor(viewportTextChars / 2); + } else if (key === "identifiers") { + identifierChars = Math.floor(identifierChars / 2); } else { - const keep = Math.floor(lists[key].length / 2); + const keep = + key === "interactiveElements" && lists[key].length > inViewportCount + ? inViewportCount + Math.floor((lists[key].length - inViewportCount) / 2) + : Math.floor(lists[key].length / 2); dropped[key] += lists[key].length - keep; - // slice(-0) keeps everything, so spell out the empty case. lists[key] = keep === 0 ? [] - : key === "interactiveElements" + : key === "interactiveElements" || key === "scrollContainers" ? lists[key].slice(0, keep) : lists[key].slice(-keep); } @@ -293,9 +369,19 @@ const boundSnapshotMetadata = (metadata: SnapshotMetadata) => { `visibleText after ${visibleTextChars} characters (use preview_evaluate for more)`, ); } + if (viewportTextChars < (metadata.viewportText?.length ?? 0)) { + omitted.push( + `viewportText after ${viewportTextChars} characters (use preview_evaluate for more)`, + ); + } + if (identifierChars < MAX_SNAPSHOT_IDENTIFIER_CHARS) { + omitted.push(`url or title after ${identifierChars} characters`); + } for (const key of shedOrder) { if (dropped[key] > 0) { - omitted.push(`${dropped[key]} of ${bounded[key].length} ${key}`); + const originalLength = + key === "scrollContainers" ? metadata.scroll?.containers.length : bounded[key].length; + omitted.push(`${dropped[key]} of ${originalLength} ${key}`); } } return { value: value(), text, omitted }; diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 1c790e8b643a..278187aed0c8 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -119,7 +119,7 @@ const PreviewSetAppearanceTool = safeBrowserTool( export const PreviewSnapshotTool = readonlyBrowserTool( Tool.make("preview_snapshot", { description: - "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; with includeImage=false, save=true returns only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", + "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. When the desktop host supports it, viewportText describes the current view, scroll reports page and visible container scroll positions, and inViewport elements come first. visibleText also includes rendered text outside the view; content not yet loaded requires scrolling. The text is capped near 20 KB, keeps current-view text ahead of offscreen page text, and lists what it omitted; use preview_evaluate to read more. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; with includeImage=false, save=true returns only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", parameters: Schema.Struct({ ...PreviewAutomationTabTargetInput.fields, includeImage: Schema.optional( diff --git a/docs/user/browser.md b/docs/user/browser.md new file mode 100644 index 000000000000..39975110d725 --- /dev/null +++ b/docs/user/browser.md @@ -0,0 +1,14 @@ +# Agent browser snapshots + +When an agent inspects a page in the desktop browser preview, its snapshot +includes text and controls in the current view, page text, and scroll positions. +Controls in the current view come first. Scroll details also include visible +scroll boxes found around text or controls. + +Snapshots have a size limit. Current-view text takes priority over page text and +logs. The result reports omitted content so the agent can request more detail. +The screenshot shows the current view. + +Content that loads during scrolling is available after it loads. Embedded frames +and shadow DOM can require a separate inspection. A snapshot does not scroll the +page or change its content. diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index f87739670f75..04c5bf595339 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -496,6 +496,7 @@ export const PreviewAutomationElement = Schema.Struct({ y: Schema.Number, width: Schema.Number, height: Schema.Number, + inViewport: Schema.optional(Schema.Boolean), }); export type PreviewAutomationElement = typeof PreviewAutomationElement.Type; @@ -527,11 +528,37 @@ export const PreviewAutomationActionEvent = Schema.Struct({ }); export type PreviewAutomationActionEvent = typeof PreviewAutomationActionEvent.Type; +const PreviewAutomationScrollPosition = { + x: Schema.Number, + y: Schema.Number, + width: Schema.Number, + height: Schema.Number, + scrollWidth: Schema.Number, + scrollHeight: Schema.Number, +}; + export const PreviewAutomationSnapshot = Schema.Struct({ url: Schema.String, title: Schema.String, loading: Schema.Boolean, visibleText: Schema.String, + viewportText: Schema.optional(Schema.String), + scroll: Schema.optional( + Schema.Struct({ + ...PreviewAutomationScrollPosition, + containers: Schema.Array( + Schema.Struct({ selector: Schema.String, ...PreviewAutomationScrollPosition }), + ), + containersTruncated: Schema.Boolean, + }), + ), + truncated: Schema.optional( + Schema.Struct({ + visibleText: Schema.Boolean, + viewportText: Schema.Boolean, + interactiveElements: Schema.Boolean, + }), + ), interactiveElements: Schema.Array(PreviewAutomationElement), accessibilityTree: Schema.Unknown, consoleEntries: Schema.Array(PreviewAutomationConsoleEntry), diff --git a/packages/contracts/src/previewAutomationSnapshot.test.ts b/packages/contracts/src/previewAutomationSnapshot.test.ts new file mode 100644 index 000000000000..74245583744e --- /dev/null +++ b/packages/contracts/src/previewAutomationSnapshot.test.ts @@ -0,0 +1,65 @@ +import { Schema } from "effect"; +import { describe, expect, it } from "vite-plus/test"; + +import { PreviewAutomationSnapshot } from "./previewAutomation.ts"; + +const legacySnapshot = { + url: "https://example.test/", + title: "Page", + loading: false, + visibleText: "Whole page", + interactiveElements: [ + { + tag: "button", + role: null, + name: "Open", + selector: "#open", + x: 0, + y: 0, + width: 80, + height: 20, + }, + ], + accessibilityTree: null, + consoleEntries: [], + networkEntries: [], + actionTimeline: [], + screenshot: { mimeType: "image/png", data: "image", width: 300, height: 200 }, +}; +const decode = Schema.decodeUnknownSync(PreviewAutomationSnapshot); + +describe("PreviewAutomationSnapshot", () => { + it("accepts snapshots from desktop hosts without scroll context", () => { + expect(decode(legacySnapshot)).toEqual(legacySnapshot); + }); + + it("preserves viewport text, scroll context, and omission flags across the wire", () => { + const snapshot = { + ...legacySnapshot, + viewportText: "Current view", + interactiveElements: [{ ...legacySnapshot.interactiveElements[0], inViewport: true }], + scroll: { + x: 0, + y: 2_000, + width: 300, + height: 200, + scrollWidth: 300, + scrollHeight: 5_000, + containers: [ + { + selector: "#list", + x: 10, + y: 600, + width: 100, + height: 100, + scrollWidth: 200, + scrollHeight: 1_000, + }, + ], + containersTruncated: false, + }, + truncated: { visibleText: true, viewportText: false, interactiveElements: true }, + }; + expect(decode(snapshot)).toEqual(snapshot); + }); +}); From 81a15460ad41fbd100fdcf315e941294c3747751 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 14:56:25 +0200 Subject: [PATCH 02/15] fix(desktop): keep snapshot collector private --- apps/desktop/src/preview/SnapshotPage.test.ts | 6 ++---- apps/desktop/src/preview/SnapshotPage.ts | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/apps/desktop/src/preview/SnapshotPage.test.ts b/apps/desktop/src/preview/SnapshotPage.test.ts index 9fdf0355cd20..ca00d2c438dc 100644 --- a/apps/desktop/src/preview/SnapshotPage.test.ts +++ b/apps/desktop/src/preview/SnapshotPage.test.ts @@ -1,7 +1,7 @@ import * as NodeVM from "node:vm"; import { describe, expect, it } from "vite-plus/test"; -import { snapshotPageExpression } from "./SnapshotPage.ts"; +import { snapshotPageExpression, type SnapshotPage } from "./SnapshotPage.ts"; const rect = (x: number, y: number, width = 100, height = 20) => ({ x, @@ -159,9 +159,7 @@ const fixture = () => { const capture = (scrollingElement: PageElement | null = root, compatMode = "CSS1Compat") => { context.document.scrollingElement = scrollingElement; context.document.compatMode = compatMode; - return NodeVM.runInNewContext(snapshotPageExpression(), context) as ReturnType< - typeof import("./SnapshotPage.ts").collectSnapshotPage - >; + return NodeVM.runInNewContext(snapshotPageExpression(), context) as SnapshotPage; }; return { root, body, elements, texts, element, text, capture }; }; diff --git a/apps/desktop/src/preview/SnapshotPage.ts b/apps/desktop/src/preview/SnapshotPage.ts index a5bb94aba301..61fd89bee778 100644 --- a/apps/desktop/src/preview/SnapshotPage.ts +++ b/apps/desktop/src/preview/SnapshotPage.ts @@ -12,7 +12,7 @@ export type SnapshotPage = Pick< | "interactiveElements" >; -export function collectSnapshotPage(): SnapshotPage { +function collectSnapshotPage(): SnapshotPage { const maxTextLength = 20_000; const maxElements = 200; const viewport = { left: 0, top: 0, right: innerWidth, bottom: innerHeight }; From dc6b8e6757ba8190403805476fe372ae4869a9bf Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 15:14:05 +0200 Subject: [PATCH 03/15] test(preview): fix Windows snapshot path assertion --- .github/workflows/preview-snapshot-tests.yml | 1 + apps/server/src/mcp/McpHttpServer.test.ts | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/preview-snapshot-tests.yml b/.github/workflows/preview-snapshot-tests.yml index 286eb6aae147..bd44f235f5e4 100644 --- a/.github/workflows/preview-snapshot-tests.yml +++ b/.github/workflows/preview-snapshot-tests.yml @@ -24,6 +24,7 @@ jobs: steps: - uses: actions/checkout@v6 with: + persist-credentials: false sparse-checkout: | /* !/.repos/ diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index cce09515a8ac..edaacdff079b 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -402,7 +402,9 @@ it.effect("saves the snapshot PNG on request and reports its path", () => ); expect(Buffer.from(yield* fileSystem.readFile(screenshotPath!)).toString()).toBe("png"); const [, text] = snapshot.content; - expect(text?.type === "text" ? text.text : "").toContain(screenshotPath); + expect(text?.type === "text" ? decodeJsonText(text.text) : null).toMatchObject({ + screenshotPath, + }); const unsaved = yield* callSnapshot({}); expect(unsaved.structuredContent).not.toHaveProperty("screenshotPath"); From c5c5b4872f7957051fb11d64c7176be3bb784e95 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 15:21:20 +0200 Subject: [PATCH 04/15] fix(desktop): include image-only snapshot scroll areas --- apps/desktop/src/preview/SnapshotPage.test.ts | 96 +++++++++++++++++-- apps/desktop/src/preview/SnapshotPage.ts | 28 ++++-- docs/user/browser.md | 5 +- 3 files changed, 112 insertions(+), 17 deletions(-) diff --git a/apps/desktop/src/preview/SnapshotPage.test.ts b/apps/desktop/src/preview/SnapshotPage.test.ts index ca00d2c438dc..14f9a72e1ff0 100644 --- a/apps/desktop/src/preview/SnapshotPage.test.ts +++ b/apps/desktop/src/preview/SnapshotPage.test.ts @@ -15,12 +15,14 @@ const rect = (x: number, y: number, width = 100, height = 20) => ({ }); class PageElement { + nodeType = 1; tagName = "DIV"; id = ""; innerText = ""; parentElement: PageElement | null = null; offsetParent: PageElement | null = null; children: PageElement[] = []; + attributes = new Map(); clientLeft = 0; clientTop = 0; clientWidth = 100; @@ -49,8 +51,8 @@ class PageElement { borderRightWidth: "0px", borderBottomWidth: "0px", }; - getAttribute() { - return null; + getAttribute(attribute: string) { + return this.attributes.get(attribute) ?? null; } getBoundingClientRect() { return this.bounds; @@ -76,26 +78,28 @@ class PageElement { } type PageText = { + nodeType: number; data: string; length: number; parentElement: PageElement; rectangles: (start: number, end: number) => ReturnType[]; }; -const fixture = () => { +const fixture = (escape = (value: string) => value) => { const root = new PageElement(); const body = new PageElement(); body.parentElement = root; root.children.push(body); root.scrollHeight = 10_000; const elements: PageElement[] = []; - const texts: PageText[] = []; + const nodes: Array = []; const element = (id: string, bounds: ReturnType, parent = body) => { const value = new PageElement(); value.id = id; value.bounds = bounds; value.parentElement = parent; parent.children.push(value); + nodes.push(value); return value; }; const text = ( @@ -103,7 +107,8 @@ const fixture = () => { parent: PageElement, rectangles: PageText["rectangles"] = () => [parent.bounds], ) => { - texts.push({ data, length: data.length, parentElement: parent, rectangles }); + const value = { nodeType: 3, data, length: data.length, parentElement: parent, rectangles }; + nodes.push(value); }; let start = 0; let end = 0; @@ -137,9 +142,10 @@ const fixture = () => { scrollX: 0, scrollY: 4_000, location: { href: "https://example.test" }, - CSS: { escape: (value: string) => value }, + CSS: { escape }, HTMLElement: PageElement, - NodeFilter: { SHOW_TEXT: 4 }, + Node: { ELEMENT_NODE: 1 }, + NodeFilter: { SHOW_TEXT: 4, SHOW_ELEMENT: 1 }, getComputedStyle: (value: PageElement) => value.style, document: { documentElement: root, @@ -152,7 +158,7 @@ const fixture = () => { createRange: () => range, createTreeWalker() { let index = 0; - return { nextNode: () => texts[index++] ?? null }; + return { nextNode: () => nodes[index++] ?? null }; }, }, }; @@ -161,10 +167,75 @@ const fixture = () => { context.document.compatMode = compatMode; return NodeVM.runInNewContext(snapshotPageExpression(), context) as SnapshotPage; }; - return { root, body, elements, texts, element, text, capture }; + return { root, body, elements, nodes, element, text, capture }; }; describe("snapshot page collector", () => { + it.each(["name", "data-testid"])("escapes line breaks in %s selectors", (attribute) => { + const page = fixture((value) => value.replaceAll("\n", "\\a ")); + const control = page.element("", rect(0, 0)); + control.tagName = "BUTTON"; + control.attributes.set(attribute, "line\nnext"); + page.elements.push(control); + expect(page.capture().interactiveElements[0]?.selector).toBe( + `button[${attribute}=line\\a next]`, + ); + }); + + it("finds image and canvas scroll areas without text or controls", () => { + const page = fixture(); + for (const tag of ["IMG", "CANVAS"]) { + const scroller = page.element(`scroll-${tag}`, rect(0, 0)); + scroller.style.overflowY = "auto"; + scroller.scrollHeight = 1_000; + scroller.scrollTop = 200; + page.element("", rect(0, 0), scroller).tagName = tag; + } + const hidden = page.element("hidden", rect(0, 0)); + hidden.style.opacity = "0"; + const hiddenScroller = page.element("hidden-scroll", rect(0, 0), hidden); + hiddenScroller.style.overflowY = "auto"; + hiddenScroller.scrollHeight = 1_000; + const clipped = page.element("clipping", rect(0, 0)); + clipped.style.overflowY = "hidden"; + const clippedScroller = page.element("clipped-scroll", rect(0, 100), clipped); + clippedScroller.style.overflowY = "auto"; + clippedScroller.scrollHeight = 1_000; + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe(""); + expect(snapshot.interactiveElements).toEqual([]); + expect( + snapshot.scroll?.containers.map((container) => [container.selector, container.y]), + ).toEqual([ + ["#scroll-IMG", 200], + ["#scroll-CANVAS", 200], + ]); + }); + + it("finds an independently scrolling image-only body", () => { + const page = fixture(); + page.root.style.overflowY = "hidden"; + page.body.style.overflowY = "auto"; + page.body.scrollHeight = 1_000; + page.element("image", rect(0, 0)).tagName = "IMG"; + expect(page.capture().scroll?.containers[0]?.height).toBe(20); + }); + + it("keeps scanning scroll areas after text is capped and reports an incomplete scan", () => { + const page = fixture(); + page.text("x".repeat(25_000), page.element("text", rect(0, 0))); + const scroller = page.element("later-scroll", rect(0, 0)); + scroller.style.overflowY = "auto"; + scroller.scrollHeight = 1_000; + expect(page.capture().scroll?.containers[0]?.selector).toBe("#later-scroll"); + const empty = page.element("empty", rect(0, 0)); + page.nodes.length = 100_001; + page.nodes.fill(empty); + const snapshot = page.capture(); + expect(snapshot.truncated?.viewportText).toBe(true); + expect(snapshot.scroll?.containersTruncated).toBe(true); + }); + it("keeps bottom text after more than 4096 offscreen paragraphs", () => { const page = fixture(); for (let index = 0; index < 5_000; index++) { @@ -391,7 +462,11 @@ describe("snapshot page collector", () => { }); it("reports capped viewport text and scroll container metadata", () => { - const page = fixture(); + let selectorsRead = 0; + const page = fixture((value) => { + selectorsRead++; + return value; + }); for (let index = 0; index < 21; index++) { const container = page.element(`scroll-${index}`, rect(0, 0)); container.style.overflowY = "scroll"; @@ -403,6 +478,7 @@ describe("snapshot page collector", () => { const snapshot = page.capture(); expect(snapshot.scroll?.containers).toHaveLength(20); expect(snapshot.scroll?.containersTruncated).toBe(true); + expect(selectorsRead).toBe(20); expect(snapshot.viewportText).toHaveLength(20_000); expect(snapshot.truncated?.viewportText).toBe(true); }); diff --git a/apps/desktop/src/preview/SnapshotPage.ts b/apps/desktop/src/preview/SnapshotPage.ts index 61fd89bee778..84a71131f8f0 100644 --- a/apps/desktop/src/preview/SnapshotPage.ts +++ b/apps/desktop/src/preview/SnapshotPage.ts @@ -27,7 +27,7 @@ function collectSnapshotPage(): SnapshotPage { for (const attribute of ["data-testid", "name"]) { const value = element.getAttribute(attribute); if (value) - return element.tagName.toLowerCase() + "[" + attribute + "=" + JSON.stringify(value) + "]"; + return element.tagName.toLowerCase() + "[" + attribute + "=" + CSS.escape(value) + "]"; } const parts: string[] = []; for ( @@ -140,8 +140,8 @@ function collectSnapshotPage(): SnapshotPage { (/^(auto|scroll)$/.test(style.overflowX) && current.scrollWidth > clientWidth) || (/^(auto|scroll)$/.test(style.overflowY) && current.scrollHeight > clientHeight); if (scrollable && intersects(rect, parentClip)) { - const selector = selectorFor(current); - if (containers.length < 20 && selector.length <= 1_000) { + const selector = containers.length < 20 ? selectorFor(current) : null; + if (selector !== null && selector.length <= 1_000) { containers.push({ selector, x: current.scrollLeft, @@ -258,16 +258,32 @@ function collectSnapshotPage(): SnapshotPage { readText(node, middle, end, clip); }; if (document.body) { - const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT); + if (rendered(document.body)) clipFor(document.body); + const walker = document.createTreeWalker( + document.body, + NodeFilter.SHOW_ELEMENT | NodeFilter.SHOW_TEXT, + ); let node = walker.nextNode(); let visited = 0; - while (node && !viewportTextTruncated) { - if (++visited > 50_000) { + while (node) { + if (++visited > 100_000) { viewportTextTruncated = true; + containersTruncated = true; break; } + if (node.nodeType === Node.ELEMENT_NODE) { + const element = node as Element; + const style = styleFor(element); + if (/^(auto|scroll)$/.test(style.overflowX) || /^(auto|scroll)$/.test(style.overflowY)) { + if (rendered(element) && intersects(element.getBoundingClientRect(), viewport)) + clipFor(element); + } + node = walker.nextNode(); + continue; + } const parent = node.parentElement; if ( + !viewportTextTruncated && parent && !/^(SCRIPT|STYLE|NOSCRIPT|TEMPLATE)$/.test(parent.tagName) && rendered(parent) diff --git a/docs/user/browser.md b/docs/user/browser.md index 39975110d725..55288ebecdce 100644 --- a/docs/user/browser.md +++ b/docs/user/browser.md @@ -3,12 +3,15 @@ When an agent inspects a page in the desktop browser preview, its snapshot includes text and controls in the current view, page text, and scroll positions. Controls in the current view come first. Scroll details also include visible -scroll boxes found around text or controls. +scroll boxes, including boxes that contain only images or canvas content. Snapshots have a size limit. Current-view text takes priority over page text and logs. The result reports omitted content so the agent can request more detail. The screenshot shows the current view. +On old HTML pages, a scroll box on the page body can report a size that includes +scrollbar space. Use the screenshot when precise edge positions matter. + Content that loads during scrolling is available after it loads. Embedded frames and shadow DOM can require a separate inspection. A snapshot does not scroll the page or change its content. From 2ef814fb79708dc78801b45a237f5d60f9826416 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 18:03:21 +0200 Subject: [PATCH 05/15] fix(desktop): respect snapshot paint clipping --- apps/desktop/src/preview/SnapshotPage.test.ts | 177 ++++++++++++++++++ apps/desktop/src/preview/SnapshotPage.ts | 124 +++++++++++- docs/user/browser.md | 3 + 3 files changed, 301 insertions(+), 3 deletions(-) diff --git a/apps/desktop/src/preview/SnapshotPage.test.ts b/apps/desktop/src/preview/SnapshotPage.test.ts index 14f9a72e1ff0..66a965372cd1 100644 --- a/apps/desktop/src/preview/SnapshotPage.test.ts +++ b/apps/desktop/src/preview/SnapshotPage.test.ts @@ -47,6 +47,8 @@ class PageElement { filter: "none", backdropFilter: "none", contain: "none", + clipPath: "none", + overflowClipMargin: "0px", willChange: "auto", borderRightWidth: "0px", borderBottomWidth: "0px", @@ -171,6 +173,181 @@ const fixture = (escape = (value: string) => value) => { }; describe("snapshot page collector", () => { + it.each(["paint", "content", "strict", "layout"])( + "respects %s containment on boxes and the body", + (contain) => { + for (const onBody of [false, true]) { + const page = fixture(); + const parent = onBody ? page.body : page.element("clip", rect(0, 0, 150, 60)); + parent.bounds = rect(0, 0, 150, 60); + parent.clientWidth = parent.offsetWidth = 150; + parent.clientHeight = parent.offsetHeight = 60; + parent.style.contain = contain; + const hidden = page.element("hidden", rect(0, 120), parent); + page.elements.push(hidden); + page.text("hidden label", hidden); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe(contain === "layout" ? "hidden label" : ""); + expect(snapshot.interactiveElements[0]?.inViewport).toBe(contain === "layout"); + } + }, + ); + + it.each(["inset(0px)", "inset(0%)", "inset(10px 10% 20px 5%)"])( + "clips text and controls with %s on boxes and the body", + (clipPath) => { + for (const onBody of [false, true]) { + const page = fixture(); + const parent = onBody ? page.body : page.element("clip", rect(0, 0, 150, 60)); + parent.bounds = rect(0, 0, 150, 60); + parent.offsetWidth = 150; + parent.offsetHeight = 60; + parent.style.clipPath = clipPath; + const hidden = page.element("hidden", rect(0, 120), parent); + const visible = page.element("visible", rect(20, 20, 50, 10), parent); + page.elements.push(hidden, visible); + page.text("hidden label", hidden); + page.text("visible label", visible); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("visible label"); + expect(snapshot.interactiveElements.map((element) => element.inViewport)).toEqual([ + true, + false, + ]); + expect(snapshot.truncated?.viewportText).toBe(false); + } + }, + ); + + it("clips positioned descendants at static clip-path ancestors", () => { + const page = fixture(); + const parent = page.element("clip", rect(0, 0, 150, 60)); + parent.style.clipPath = "inset(0px)"; + for (const position of ["fixed", "absolute"]) { + const hidden = page.element(position, rect(0, 120), parent); + hidden.style.position = position; + hidden.offsetParent = position === "absolute" ? page.body : null; + page.elements.push(hidden); + page.text("hidden label", hidden); + } + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe(""); + expect(snapshot.interactiveElements.every((element) => !element.inViewport)).toBe(true); + }); + + it.each(["inline", "table-row", "ruby", "ruby-text"])( + "does not apply paint containment to %s boxes", + (display) => { + const page = fixture(); + const parent = page.element("clip", rect(0, 0)); + parent.style.display = display; + parent.style.contain = "paint"; + const visible = page.element("visible", rect(0, 120), parent); + visible.style.position = "absolute"; + visible.offsetParent = page.body; + page.elements.push(visible); + page.text("visible label", visible); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("visible label"); + expect(snapshot.interactiveElements[0]?.inViewport).toBe(true); + }, + ); + + it("respects paint containment overflow clip margins", () => { + const page = fixture(); + const parent = page.element("clip", rect(0, 0, 100, 100)); + parent.offsetHeight = 100; + parent.style.contain = "paint"; + parent.style.overflowClipMargin = "padding-box 20px"; + const visible = page.element("visible", rect(0, 105, 50, 10), parent); + const hidden = page.element("hidden", rect(0, 140), parent); + page.elements.push(visible, hidden); + page.text("visible label", visible); + page.text("hidden label", hidden); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("visible label"); + expect(snapshot.interactiveElements.map((element) => element.inViewport)).toEqual([ + true, + false, + ]); + parent.style.overflowClipMargin = "content-box 20px"; + const omitted = page.capture(); + expect(omitted.viewportText).toBe(""); + expect(omitted.interactiveElements).toEqual([]); + expect(omitted.truncated?.viewportText).toBe(true); + expect(omitted.truncated?.interactiveElements).toBe(true); + }); + + it.each(["paint", "layout"])( + "finds independent body scrolling when %s containment prevents overflow propagation", + (contain) => { + for (const onRoot of [false, true]) { + const page = fixture(); + (onRoot ? page.root : page.body).style.contain = contain; + page.root.bounds = rect(0, 0, 300, 200); + page.body.bounds = rect(0, 0, 150, 60); + page.body.clientWidth = page.body.offsetWidth = 150; + page.body.clientHeight = page.body.offsetHeight = 60; + page.body.style.overflowY = "auto"; + page.body.scrollHeight = 500; + page.body.scrollTop = 200; + const hidden = page.element("hidden", rect(0, 120)); + page.elements.push(hidden); + page.text("hidden label", hidden); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe(""); + expect(snapshot.interactiveElements[0]?.inViewport).toBe(false); + expect(snapshot.scroll?.containers[0]).toMatchObject({ + y: 200, + height: 60, + scrollHeight: 500, + }); + } + }, + ); + + it("uses border-box inset percentages and permits negative insets", () => { + const page = fixture(); + const parent = page.element("clip", rect(0, 0, 100, 100)); + parent.clientLeft = parent.clientTop = 10; + parent.style.borderRightWidth = parent.style.borderBottomWidth = "10px"; + parent.style.clipPath = "inset(0px)"; + const label = page.element("label", rect(2, 2, 5, 5), parent); + page.text("border label", label); + expect(page.capture().viewportText).toBe("border label"); + parent.style.contain = "paint"; + expect(page.capture().viewportText).toBe(""); + parent.style.contain = "none"; + parent.style.clipPath = "inset(-50px)"; + label.bounds = rect(0, 120); + expect(page.capture().viewportText).toBe("border label"); + parent.style.clipPath = "inset(0 0 50%)"; + label.bounds = rect(0, 60); + expect(page.capture().viewportText).toBe(""); + }); + + it.each(["circle(50%)", "inset(0px round 20px)", "inset(calc(10px + 5%))"])( + "reports omitted text and controls for unsupported %s geometry", + (clipPath) => { + const page = fixture(); + const parent = page.element("clip", rect(0, 0, 100, 100)); + parent.style.clipPath = clipPath; + const uncertain = page.element("uncertain", rect(20, 20), parent); + uncertain.style.position = "fixed"; + const visible = page.element("visible", rect(0, 140)); + page.elements.push(uncertain, visible); + page.text("uncertain label", uncertain); + page.text("visible label", visible); + page.body.innerText = "uncertain label visible label"; + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("visible label"); + expect(snapshot.visibleText).toBe("uncertain label visible label"); + expect(snapshot.interactiveElements.map((element) => element.selector)).toEqual(["#visible"]); + expect(snapshot.truncated?.viewportText).toBe(true); + expect(snapshot.truncated?.interactiveElements).toBe(true); + }, + ); + it.each(["name", "data-testid"])("escapes line breaks in %s selectors", (attribute) => { const page = fixture((value) => value.replaceAll("\n", "\\a ")); const control = page.element("", rect(0, 0)); diff --git a/apps/desktop/src/preview/SnapshotPage.ts b/apps/desktop/src/preview/SnapshotPage.ts index 84a71131f8f0..a8e009d2e884 100644 --- a/apps/desktop/src/preview/SnapshotPage.ts +++ b/apps/desktop/src/preview/SnapshotPage.ts @@ -50,6 +50,9 @@ function collectSnapshotPage(): SnapshotPage { const containers: Array["containers"][number]> = []; let containersTruncated = false; const clips = new Map(); + const paintClips = new Map(); + const unsupportedClips = new Set(); + let unsupportedClip = false; const styles = new Map(); const styleFor = (element: Element) => { let style = styles.get(element); @@ -59,6 +62,104 @@ function collectSnapshotPage(): SnapshotPage { } return style; }; + const paintClipFor = (element: Element): Bounds | null => { + const pending: Element[] = []; + let ancestor: Element | null = element; + while (ancestor && !paintClips.has(ancestor)) { + pending.push(ancestor); + ancestor = ancestor.parentElement; + } + let clip = ancestor ? (paintClips.get(ancestor) ?? null) : viewport; + let unsupported = ancestor ? unsupportedClips.has(ancestor) : false; + for (let index = pending.length - 1; index >= 0; index--) { + const current = pending[index]!; + const style = styleFor(current); + if (clip && style.display !== "contents") { + const paint = + /\b(paint|content|strict)\b/.test(style.contain) && + !/^(inline$|table-(?!cell$|caption$)|ruby(?:$|-))/.test(style.display); + const path = style.clipPath && style.clipPath !== "none"; + if (paint || path) { + const rect = current.getBoundingClientRect(); + const scaleX = + current instanceof HTMLElement && current.offsetWidth + ? rect.width / current.offsetWidth + : 1; + const scaleY = + current instanceof HTMLElement && current.offsetHeight + ? rect.height / current.offsetHeight + : 1; + let left = rect.left; + let top = rect.top; + let right = rect.right; + let bottom = rect.bottom; + if (paint) { + const margin = /^(?:padding-box )?(\d+(?:\.\d+)?|\.\d+)px$/.exec( + style.overflowClipMargin || "0px", + ); + if (!margin) { + unsupportedClip = true; + unsupported = true; + unsupportedClips.add(current); + clip = null; + paintClips.set(current, clip); + continue; + } + const outset = Number.parseFloat(margin[1]!); + left += (current.clientLeft - outset) * scaleX; + top += (current.clientTop - outset) * scaleY; + right -= (Number.parseFloat(style.borderRightWidth) - outset) * scaleX; + bottom -= (Number.parseFloat(style.borderBottomWidth) - outset) * scaleY; + } + if (path) { + const values = /^inset\(([^()]+)\)$/.exec(style.clipPath)?.[1]?.trim().split(/\s+/); + if ( + !values || + values.length > 4 || + values.some((value) => !/^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:px|%)?$/.test(value)) + ) { + unsupportedClip = true; + unsupported = true; + unsupportedClips.add(current); + clip = null; + paintClips.set(current, clip); + continue; + } + const offsets = [ + values[0]!, + values[1] ?? values[0]!, + values[2] ?? values[0]!, + values[3] ?? values[1] ?? values[0]!, + ].map( + (value, side) => + Number.parseFloat(value) * + (value.endsWith("%") + ? (side % 2 === 0 ? rect.height : rect.width) / 100 + : side % 2 === 0 + ? scaleY + : scaleX), + ); + left = paint ? Math.max(left, rect.left + offsets[3]!) : rect.left + offsets[3]!; + top = paint ? Math.max(top, rect.top + offsets[0]!) : rect.top + offsets[0]!; + right = paint ? Math.min(right, rect.right - offsets[1]!) : rect.right - offsets[1]!; + bottom = paint + ? Math.min(bottom, rect.bottom - offsets[2]!) + : rect.bottom - offsets[2]!; + } + clip = { + left: Math.max(clip.left, left), + top: Math.max(clip.top, top), + right: Math.min(clip.right, right), + bottom: Math.min(clip.bottom, bottom), + }; + if (clip.right <= clip.left || clip.bottom <= clip.top) clip = null; + } + } + if (unsupported) unsupportedClips.add(current); + paintClips.set(current, clip); + } + return paintClips.get(element)!; + }; const clipFor = (element: Element): Bounds | null => { const pending: Element[] = []; let ancestor: Element | null = element; @@ -87,6 +188,10 @@ function collectSnapshotPage(): SnapshotPage { const rootStyle = styleFor(document.documentElement); const viewportBody = current === document.body && + style.contain === "none" && + rootStyle.contain === "none" && + style.contentVisibility === "visible" && + rootStyle.contentVisibility === "visible" && rootStyle.overflowX === "visible" && rootStyle.overflowY === "visible"; if (current !== document.documentElement && !viewportBody && style.display !== "contents") { @@ -155,7 +260,14 @@ function collectSnapshotPage(): SnapshotPage { } } } - parentClip = clip.right > clip.left && clip.bottom > clip.top ? clip : null; + const paintClip = paintClipFor(current); + if (paintClip) { + clip.left = Math.max(clip.left, paintClip.left); + clip.top = Math.max(clip.top, paintClip.top); + clip.right = Math.min(clip.right, paintClip.right); + clip.bottom = Math.min(clip.bottom, paintClip.bottom); + } + parentClip = paintClip && clip.right > clip.left && clip.bottom > clip.top ? clip : null; clips.set(current, parentClip); } return clips.get(element)!; @@ -182,12 +294,18 @@ function collectSnapshotPage(): SnapshotPage { const currentElements: Element[] = []; const otherElements: Element[] = []; let elementCount = 0; + let controlsOmitted = false; for (const element of document.querySelectorAll( "a[href],button,input,textarea,select,[role],[tabindex]", )) { if (!rendered(element)) continue; const rect = element.getBoundingClientRect(); if (rect.width <= 0 || rect.height <= 0) continue; + paintClipFor(element); + if (unsupportedClips.has(element)) { + controlsOmitted = true; + continue; + } const current = inViewport(element, rect); elementCount++; const target = current ? currentElements : otherElements; @@ -319,8 +437,8 @@ function collectSnapshotPage(): SnapshotPage { }, truncated: { visibleText: visibleText.length > maxTextLength, - viewportText: viewportTextTruncated, - interactiveElements: elementCount > maxElements, + viewportText: viewportTextTruncated || unsupportedClip, + interactiveElements: elementCount > maxElements || controlsOmitted, }, }; } diff --git a/docs/user/browser.md b/docs/user/browser.md index 55288ebecdce..80022d1eec4f 100644 --- a/docs/user/browser.md +++ b/docs/user/browser.md @@ -9,6 +9,9 @@ Snapshots have a size limit. Current-view text takes priority over page text and logs. The result reports omitted content so the agent can request more detail. The screenshot shows the current view. +Complex CSS clip shapes can cause current-view text and controls to be omitted. +The result reports these omissions. Use the screenshot to inspect those areas. + On old HTML pages, a scroll box on the page body can report a size that includes scrollbar space. Use the screenshot when precise edge positions matter. From a5e9eb1499bba3f2c607a61211a4cdbd48cdd634 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 18:32:11 +0200 Subject: [PATCH 06/15] feat(preview): export all loaded snapshot text --- .github/workflows/preview-snapshot-tests.yml | 2 + apps/server/src/mcp/McpHttpServer.test.ts | 296 ++++++++++++++++-- apps/server/src/mcp/McpHttpServer.ts | 172 +++++----- apps/server/src/mcp/PreviewTextExport.test.ts | 276 ++++++++++++++++ apps/server/src/mcp/PreviewTextExport.ts | 133 ++++++++ .../src/mcp/toolkits/preview/handlers.ts | 7 +- apps/server/src/mcp/toolkits/preview/tools.ts | 10 +- docs/user/browser.md | 5 + 8 files changed, 806 insertions(+), 95 deletions(-) create mode 100644 apps/server/src/mcp/PreviewTextExport.test.ts create mode 100644 apps/server/src/mcp/PreviewTextExport.ts diff --git a/.github/workflows/preview-snapshot-tests.yml b/.github/workflows/preview-snapshot-tests.yml index bd44f235f5e4..bb958a196998 100644 --- a/.github/workflows/preview-snapshot-tests.yml +++ b/.github/workflows/preview-snapshot-tests.yml @@ -41,5 +41,7 @@ jobs: apps/desktop/src/preview/Manager.test.ts apps/server/src/mcp/McpHttpServer.test.ts apps/server/src/mcp/PreviewAutomationBroker.test.ts + apps/server/src/mcp/PreviewTextExport.test.ts + apps/server/src/mcp/toolkits/preview/tools.test.ts packages/contracts/src/previewAutomationSnapshot.test.ts packages/contracts/src/preview.test.ts diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index edaacdff079b..7e61933f579f 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -1,11 +1,18 @@ // @effect-diagnostics-next-line nodeBuiltinImport:off import * as NodeHttp from "node:http"; +import * as NodeVM from "node:vm"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import { expect, it } from "@effect/vitest"; import { NodeHttpServer } from "@effect/platform-node"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { EnvironmentId, PreviewTabId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import { + EnvironmentId, + PreviewTabId, + ProviderInstanceId, + ThreadId, + type PreviewAutomationRequest, +} from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; @@ -34,6 +41,7 @@ const threadId = ThreadId.make("thread-mcp-test"); const tabId = PreviewTabId.make("tab-mcp-test"); const alternateTabId = PreviewTabId.make("tab-mcp-alternate"); const decodeJsonText = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown)); +const encodeJsonText = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); const invocation = { environmentId, threadId, @@ -123,6 +131,99 @@ const callSnapshot = (args: Record) => ); }); +const serveTextExports = ( + clientId: string, + text: string, + options: { switchTab?: boolean; navigate?: boolean } = {}, +) => + Effect.gen(function* () { + const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker; + const connected = yield* Deferred.make(); + const switched = yield* Deferred.make(); + const requests: PreviewAutomationRequest[] = []; + const timers = new Set<() => void>(); + let textReads = 0; + let chunks = 0; + const context = NodeVM.createContext({ + document: { + body: { + get innerText() { + textReads++; + return text; + }, + }, + }, + location: { href: snapshotResult.url }, + setTimeout: (callback: () => void) => { + timers.add(callback); + return callback; + }, + clearTimeout: (callback: () => void) => timers.delete(callback), + }); + const events = yield* broker.connect({ clientId, environmentId }); + yield* Stream.runForEach(events, (event) => { + if (event.type === "connected") return Deferred.succeed(connected, undefined); + requests.push(event.request); + return Effect.gen(function* () { + const request = event.request; + let result: unknown = snapshotResult; + if (request.operation === "status") { + result = { + available: true, + visible: true, + tabId: request.tabId === alternateTabId ? alternateTabId : tabId, + url: snapshotResult.url, + title: snapshotResult.title, + loading: false, + }; + } else if (request.operation === "evaluate") { + if (options.switchTab) yield* Deferred.await(switched); + const expression = (request.input as { expression: string }).expression; + if (options.navigate && expression.includes("let end") && ++chunks === 2) { + NodeVM.runInContext("location.href = 'https://other.test/'", context); + } + try { + result = NodeVM.runInContext(expression, context); + } catch { + yield* broker.respond({ + clientId, + connectionId: event.connectionId, + requestId: request.requestId, + ok: false, + error: { + _tag: "PreviewAutomationExecutionError", + message: "private renderer failure", + }, + }); + return; + } + } + yield* broker.respond({ + clientId, + connectionId: event.connectionId, + requestId: request.requestId, + ok: true, + result, + }); + if ( + options.switchTab && + request.operation === "status" && + request.tabId !== alternateTabId + ) { + yield* broker.invoke({ + scope: invocation, + operation: "status", + tabId: alternateTabId, + input: {}, + }); + yield* Deferred.succeed(switched, undefined); + } + }).pipe(Effect.forkScoped, Effect.asVoid); + }).pipe(Effect.forkScoped); + yield* Deferred.await(connected); + return { requests, context, timers, textReads: () => textReads }; + }); + it("normalizes empty successful notification responses to accepted", () => { const notificationResponse = McpHttpServer.normalizeMcpHttpResponse( HttpServerResponse.text("", { status: 200, contentType: "application/json" }), @@ -358,26 +459,30 @@ it.effect.each([ ).pipe(Effect.provide(TestLayer)), ); -it.effect("rejects non-boolean snapshot image options before selecting a browser host", () => - Effect.gen(function* () { - const server = yield* McpServer.McpServer; - for (const includeImage of ["false", 0, null]) { - const result = yield* server - .callTool({ - name: "preview_snapshot", - arguments: { includeImage }, - }) - .pipe( - Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), - Effect.provideService(McpSchema.McpServerClient, client), - ); - expect(result.isError).toBe(true); - expect(result.content).toEqual([{ type: "text", text: "Preview snapshot failed: AiError." }]); - expect(result.structuredContent).toEqual({ - error: { _tag: "AiError", operation: "snapshot", failureCount: 1 }, - }); - } - }).pipe(Effect.provide(TestLayer)), +it.effect.each(["includeImage", "saveText"])( + "rejects non-boolean snapshot %s options before selecting a browser host", + (option) => + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + for (const value of ["false", 0, null]) { + const result = yield* server + .callTool({ + name: "preview_snapshot", + arguments: { [option]: value }, + }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + Effect.provideService(McpSchema.McpServerClient, client), + ); + expect(result.isError).toBe(true); + expect(result.content).toEqual([ + { type: "text", text: "Preview snapshot failed: AiError." }, + ]); + expect(result.structuredContent).toEqual({ + error: { _tag: "AiError", operation: "snapshot", failureCount: 1 }, + }); + } + }).pipe(Effect.provide(TestLayer)), ); it.effect("saves the snapshot PNG on request and reports its path", () => @@ -443,6 +548,155 @@ it.effect("reports a tagged error when the screenshot cannot be saved", () => ).pipe(Effect.provide(TestLayer)), ); +it.effect.each([ + { save: false, includeImage: false }, + { save: true, includeImage: false }, + { save: true, includeImage: true }, +])("exports complete loaded text with snapshot options %j", (options) => + Effect.scoped( + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const loadedText = "a".repeat(4095) + "πŸ˜€δΈ­ζ–‡\n".repeat(12_000) + "Offscreen loaded end"; + const host = yield* serveTextExports("mcp-text-export-client", loadedText, { + switchTab: true, + }); + + const snapshot = yield* callSnapshot({ saveText: true, ...options }); + + expect(snapshot.isError).toBe(false); + const saved = snapshot.structuredContent as { + readonly textPath: string; + readonly screenshotPath?: string; + }; + expect(saved).toMatchObject({ + textPath: expect.any(String), + textChars: loadedText.length, + textBytes: Buffer.byteLength(loadedText, "utf8"), + textUrl: snapshotResult.url, + }); + expect(path.dirname(saved.textPath)).toBe(config.browserArtifactsDir); + expect(path.basename(saved.textPath)).toMatch(/^browser-text-[0-9a-f-]+\.txt$/); + expect(yield* fs.readFileString(saved.textPath)).toBe(loadedText); + expect(host.textReads()).toBe(1); + expect(host.requests.filter((request) => request.operation === "status")).toHaveLength(2); + const pageRequests = host.requests.filter((request) => request.operation !== "status"); + expect(pageRequests.length).toBeGreaterThan(10); + expect(pageRequests.every((request) => request.tabId === tabId)).toBe(true); + expect(pageRequests.at(-1)).toMatchObject({ operation: "snapshot", input: {} }); + const texts = snapshot.content.filter((content) => content.type === "text"); + const metadata = texts[options.save && !options.includeImage ? 0 : 1]; + expect(metadata?.type === "text" ? decodeJsonText(metadata.text) : null).toMatchObject({ + textPath: saved.textPath, + textChars: loadedText.length, + textBytes: Buffer.byteLength(loadedText, "utf8"), + textUrl: snapshotResult.url, + }); + expect( + metadata?.type === "text" ? Buffer.byteLength(metadata.text, "utf8") : Infinity, + ).toBeLessThanOrEqual(McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES); + expect(encodeJsonText(saved)).not.toContain("Offscreen loaded end"); + expect(snapshot.content.some((content) => content.type === "image")).toBe( + options.includeImage, + ); + if (options.save) { + expect(path.dirname(saved.screenshotPath!)).toBe(config.browserArtifactsDir); + expect(Buffer.from(yield* fs.readFile(saved.screenshotPath!)).toString()).toBe("png"); + } else expect(saved).not.toHaveProperty("screenshotPath"); + if (options.save && !options.includeImage) { + expect(Object.keys(saved).sort()).toEqual([ + "screenshotPath", + "textBytes", + "textChars", + "textPath", + "textUrl", + "url", + ]); + expect(snapshot.content).toHaveLength(1); + } + expect(host.timers.size).toBe(0); + expect( + Object.getOwnPropertyNames(host.context).filter((key) => + key.startsWith("__t3_text_export_"), + ), + ).toEqual([]); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect.each([{}, { saveText: false }])("keeps text export opt-in %j", (options) => + Effect.scoped( + Effect.gen(function* () { + const host = yield* serveTextExports("mcp-text-not-requested-client", "Loaded text"); + + const snapshot = yield* callSnapshot({ ...options, includeImage: false }); + + expect(snapshot.isError).toBe(false); + expect(host.requests).toHaveLength(1); + expect(host.requests[0]).toMatchObject({ operation: "snapshot", input: {} }); + expect(host.textReads()).toBe(0); + expect(snapshot.structuredContent).not.toHaveProperty("textPath"); + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + expect(yield* fs.exists(config.browserArtifactsDir)).toBe(false); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect("rejects an invalid snapshot before starting a requested text export", () => + Effect.scoped( + Effect.gen(function* () { + const host = yield* serveTextExports("mcp-invalid-text-export-client", "Loaded text"); + + const snapshot = yield* callSnapshot({ saveText: true, includeImage: "wrong" }); + + expect(snapshot.isError).toBe(true); + expect(host.requests).toEqual([]); + expect(host.textReads()).toBe(0); + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + expect(yield* fs.exists(config.browserArtifactsDir)).toBe(false); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect("reports text export failure without a successful snapshot or partial files", () => + Effect.scoped( + Effect.gen(function* () { + const host = yield* serveTextExports( + "mcp-text-failure-client", + "private text ".repeat(10_000), + { + navigate: true, + }, + ); + + const snapshot = yield* callSnapshot({ saveText: true, save: true }); + + const message = + "Could not save the loaded page text. The page may have changed or the file could not be written."; + expect(snapshot.isError).toBe(true); + expect(snapshot.content).toEqual([ + { type: "text", text: `Preview snapshot failed: ${message}` }, + ]); + expect(snapshot.structuredContent).toEqual({ + error: { _tag: "PreviewTextExportError", operation: "snapshot", failureCount: 1, message }, + }); + expect(host.requests.some((request) => request.operation === "snapshot")).toBe(false); + expect(host.timers.size).toBe(0); + expect( + Object.getOwnPropertyNames(host.context).filter((key) => + key.startsWith("__t3_text_export_"), + ), + ).toEqual([]); + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([]); + }), + ).pipe(Effect.provide(TestLayer)), +); + it.effect( "registers the pull request toolkit and surfaces a missing capability as a tool error", () => diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 80c2190d1949..0e5f3ebd9a63 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -33,6 +33,7 @@ import { ThreadToolkitHandlersLive } from "./toolkits/thread/handlers.ts"; import * as ThreadMetadataMcpService from "./ThreadMetadataMcpService.ts"; import * as McpSessionRegistry from "./McpSessionRegistry.ts"; import * as PreviewAutomationBroker from "./PreviewAutomationBroker.ts"; +import { PreviewTextExportError, savePreviewText } from "./PreviewTextExport.ts"; import { OrchestratorToolkitHandlersLive } from "./toolkits/orchestrator/handlers.ts"; import { OrchestratorToolkit } from "./toolkits/orchestrator/tools.ts"; import { @@ -433,6 +434,8 @@ const saveScreenshot = Effect.fn("McpHttpServer.saveScreenshot")(function* ( }); const isPreviewAutomationError = Schema.is(PreviewAutomationError); +const isPreviewTextExportError = Schema.is(PreviewTextExportError); +const isPreviewSnapshotInput = Schema.is(PreviewSnapshotTool.parametersSchema); const previewSnapshotFailure = (cause: Cause.Cause) => { if (Cause.hasInterrupts(cause) || cause.reasons.some(Cause.isDieReason)) { @@ -449,7 +452,10 @@ const previewSnapshotFailure = (cause: Cause.Cause) => { : "PreviewSnapshotError"; // Preview errors build their message on the server, never from page output, // and it tells the agent what to do next, such as falling back to a shell browser. - const message = isPreviewAutomationError(firstFailure) ? firstFailure.message : undefined; + const message = + isPreviewAutomationError(firstFailure) || isPreviewTextExportError(firstFailure) + ? firstFailure.message + : undefined; const result = new McpSchema.CallToolResult({ isError: true, structuredContent: { @@ -502,79 +508,103 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot fiber.context, McpInvocationContext.McpInvocationContext, ); - return built.handle("preview_snapshot", payload).pipe( - Stream.unwrap, - Stream.run(Sink.last()), - Effect.flatMap(Effect.fromOption), - Effect.provideService(PreviewAutomationBroker.PreviewAutomationBroker, broker), - Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), - Effect.flatMap(({ encodedResult }) => - Effect.gen(function* () { - const snapshot = encodedResult as SnapshotMetadata & { - readonly url: string; - readonly screenshot: { - readonly mimeType: "image/png"; - readonly data: string; - readonly width: number; - readonly height: number; - }; - }; - const { screenshot, ...page } = snapshot; - const png = new Uint8Array(Buffer.from(screenshot.data, "base64")); - const screenshotPath = - payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; - if (screenshotPath !== undefined && payload?.includeImage === false) { - // The agent only wants a file to show the user. The url keeps the site icon on the tool row. - const saved = { - url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), - screenshotPath, - }; - return new McpSchema.CallToolResult({ - isError: false, - structuredContent: saved, - content: [{ type: "text", text: encodeJsonText(saved) }], - }); - } - const metadata = { - ...page, - screenshot: { - mimeType: screenshot.mimeType, - width: screenshot.width, - height: screenshot.height, - }, - ...(screenshotPath === undefined ? {} : { screenshotPath }), - }; - const bounded = boundSnapshotMetadata(metadata); - return new McpSchema.CallToolResult({ - isError: false, - structuredContent: - bounded.omitted.length === 0 - ? bounded.value - : { ...bounded.value, omitted: bounded.omitted }, - content: [ - // Keep the page identity readable even if a provider truncates the snapshot. - { - type: "text", - text: encodeJsonText({ - url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), - }), - }, - { type: "text", text: bounded.text }, - ...(bounded.omitted.length === 0 - ? [] - : [ + const exportText = Effect.gen(function* () { + if (payload?.saveText !== true || !isPreviewSnapshotInput(payload)) return undefined; + return yield* savePreviewText(payload.tabId); + }); + return exportText.pipe( + Effect.flatMap((textExport) => + built + .handle("preview_snapshot", { + ...payload, + ...(textExport === undefined ? {} : { tabId: textExport.tabId }), + }) + .pipe( + Stream.unwrap, + Stream.run(Sink.last()), + Effect.flatMap(Effect.fromOption), + Effect.flatMap(({ encodedResult }) => + Effect.gen(function* () { + const snapshot = encodedResult as SnapshotMetadata & { + readonly url: string; + readonly screenshot: { + readonly mimeType: "image/png"; + readonly data: string; + readonly width: number; + readonly height: number; + }; + }; + const { screenshot, ...page } = snapshot; + const png = new Uint8Array(Buffer.from(screenshot.data, "base64")); + const screenshotPath = + payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; + const textArtifact = + textExport === undefined + ? {} + : { + textPath: textExport.textPath, + textChars: textExport.totalChars, + textBytes: textExport.sizeBytes, + textUrl: textExport.url, + }; + if (screenshotPath !== undefined && payload?.includeImage === false) { + // The agent only wants a file to show the user. The url keeps the site icon on the tool row. + const saved = { + url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), + screenshotPath, + ...textArtifact, + }; + return new McpSchema.CallToolResult({ + isError: false, + structuredContent: saved, + content: [{ type: "text", text: encodeJsonText(saved) }], + }); + } + const metadata = { + ...page, + screenshot: { + mimeType: screenshot.mimeType, + width: screenshot.width, + height: screenshot.height, + }, + ...(screenshotPath === undefined ? {} : { screenshotPath }), + ...textArtifact, + }; + const bounded = boundSnapshotMetadata(metadata); + return new McpSchema.CallToolResult({ + isError: false, + structuredContent: + bounded.omitted.length === 0 + ? bounded.value + : { ...bounded.value, omitted: bounded.omitted }, + content: [ + // Keep the page identity readable even if a provider truncates the snapshot. { - type: "text" as const, - text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`, + type: "text", + text: encodeJsonText({ + url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), + }), }, - ]), - ...(payload?.includeImage === false - ? [] - : [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]), - ], - }); - }), + { type: "text", text: bounded.text }, + ...(bounded.omitted.length === 0 + ? [] + : [ + { + type: "text" as const, + text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`, + }, + ]), + ...(payload?.includeImage === false + ? [] + : [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]), + ], + }); + }), + ), + ), ), + Effect.provideService(PreviewAutomationBroker.PreviewAutomationBroker, broker), + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), Effect.provide(saveServices), Effect.matchCauseEffect({ onFailure: previewSnapshotFailure, diff --git a/apps/server/src/mcp/PreviewTextExport.test.ts b/apps/server/src/mcp/PreviewTextExport.test.ts new file mode 100644 index 000000000000..a7a3f9b6c088 --- /dev/null +++ b/apps/server/src/mcp/PreviewTextExport.test.ts @@ -0,0 +1,276 @@ +import * as NodeVM from "node:vm"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import { + EnvironmentId, + PreviewAutomationNoAvailableHostError, + PreviewTabId, + ProviderInstanceId, + ThreadId, +} from "@t3tools/contracts"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +import * as ServerConfig from "../config.ts"; +import { McpInvocationContext } from "./McpInvocationContext.ts"; +import { + PreviewAutomationBroker, + type PreviewAutomationInvokeInput, +} from "./PreviewAutomationBroker.ts"; +import { savePreviewText } from "./PreviewTextExport.ts"; + +const tabId = PreviewTabId.make("text-tab"); +const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); +const scope = { + environmentId: EnvironmentId.make("text-environment"), + threadId: ThreadId.make("text-thread"), + providerSessionId: "text-session", + providerInstanceId: ProviderInstanceId.make("codex"), + capabilities: new Set(["preview"] as const), + issuedAt: 1, +}; +const TestLayer = ServerConfig.layerTest(process.cwd(), { prefix: "t3-preview-text-" }).pipe( + Layer.provideMerge(NodeServices.layer), +); + +const makePage = (initialText: string) => { + let text = initialText; + let reads = 0; + const timers = new Set<() => void>(); + const requests: PreviewAutomationInvokeInput[] = []; + const context = NodeVM.createContext({ + document: { + body: { + get innerText() { + reads++; + return text; + }, + }, + }, + location: { href: "https://example.test/page" }, + setTimeout: (callback: () => void) => { + timers.add(callback); + return callback; + }, + clearTimeout: (callback: () => void) => timers.delete(callback), + }); + const page = { + context, + timers, + requests, + reads: () => reads, + available: true, + setText: (value: string) => { + text = value; + }, + beforeChunk: undefined as (() => void) | undefined, + chunkGate: undefined as Effect.Effect | undefined, + broker: undefined as PreviewAutomationBroker["Service"] | undefined, + }; + const invoke = (request: PreviewAutomationInvokeInput) => + Effect.gen(function* () { + requests.push(request); + if (request.operation === "status") { + return { + available: page.available, + visible: true, + tabId, + url: "https://example.test/page", + title: "Page", + loading: false, + } as A; + } + expect(request.operation).toBe("evaluate"); + expect(request.tabId).toBe(tabId); + expect(request.timeoutMs).toBeUndefined(); + const expression = (request.input as { expression: string }).expression; + if (expression.includes("let end")) { + page.beforeChunk?.(); + if (page.chunkGate) yield* page.chunkGate; + } + return yield* Effect.try({ + try: () => { + const value = NodeVM.runInContext(expression, context) as A; + expect(Buffer.byteLength(encodeJson(value), "utf8")).toBeLessThan(32_000); + return value; + }, + catch: () => new PreviewAutomationNoAvailableHostError({ ...scope, operation: "evaluate" }), + }); + }); + page.broker = PreviewAutomationBroker.of({ + invoke, + connect: () => Effect.die("unused"), + focusHost: () => Effect.void, + respond: () => Effect.void, + }); + return page; +}; + +const runExport = (page: ReturnType, target?: PreviewTabId) => + savePreviewText(target).pipe( + Effect.provideService(McpInvocationContext, scope), + Effect.provideService(PreviewAutomationBroker, page.broker!), + ); +const assertClean = (page: ReturnType) => { + expect(page.timers.size).toBe(0); + expect( + Object.getOwnPropertyNames(page.context).filter((name) => name.startsWith("__t3_text_export_")), + ).toEqual([]); +}; + +it.effect("exports all loaded text through bounded V1 evaluate chunks and pins the tab", () => + Effect.gen(function* () { + const text = "\u0001".repeat(70_000) + "loaded end"; + const page = makePage(text); + const saved = yield* runExport(page); + const fs = yield* FileSystem.FileSystem; + expect(yield* fs.readFileString(saved.textPath)).toBe(text); + expect(saved).toMatchObject({ + totalChars: text.length, + sizeBytes: Buffer.byteLength(text), + tabId, + url: "https://example.test/page", + }); + expect(page.reads()).toBe(1); + expect(page.requests[0]?.operation).toBe("status"); + assertClean(page); + }).pipe(Effect.provide(TestLayer)), +); + +it.effect("keeps Unicode pairs intact at chunk boundaries", () => + Effect.gen(function* () { + const text = "a".repeat(4095) + "πŸ˜€δΈ­ζ–‡" + "b".repeat(4093) + "πŸš€"; + const page = makePage(text); + const saved = yield* runExport(page, tabId); + const fs = yield* FileSystem.FileSystem; + expect(yield* fs.readFileString(saved.textPath)).toBe(text); + expect(saved.sizeBytes).toBe(Buffer.byteLength(text)); + expect(page.requests.every((request) => request.operation === "evaluate")).toBe(true); + assertClean(page); + }).pipe(Effect.provide(TestLayer)), +); + +it.effect("writes an empty loaded page without reading chunks", () => + Effect.gen(function* () { + const page = makePage(""); + const saved = yield* runExport(page); + const fs = yield* FileSystem.FileSystem; + expect(yield* fs.readFileString(saved.textPath)).toBe(""); + expect(saved).toMatchObject({ totalChars: 0, sizeBytes: 0 }); + assertClean(page); + }).pipe(Effect.provide(TestLayer)), +); + +it.effect("freezes captured text while the page changes", () => + Effect.gen(function* () { + const text = "original ".repeat(9000); + const page = makePage(text); + page.beforeChunk = () => { + page.setText("changed page"); + const key = Object.getOwnPropertyNames(page.context).find((name) => + name.startsWith("__t3_text_export_"), + )!; + expect( + NodeVM.runInContext(`Object.isFrozen(globalThis[${encodeJson(key)}])`, page.context), + ).toBe(true); + }; + const saved = yield* runExport(page); + const fs = yield* FileSystem.FileSystem; + expect(yield* fs.readFileString(saved.textPath)).toBe(text); + expect(page.reads()).toBe(1); + assertClean(page); + }).pipe(Effect.provide(TestLayer)), +); + +it.effect.each(["navigation", "expiry"] as const)("removes partial text after %s", (failure) => + Effect.gen(function* () { + const page = makePage("text ".repeat(20_000)); + let chunks = 0; + page.beforeChunk = () => { + if (++chunks !== 2) return; + if (failure === "navigation") + NodeVM.runInContext("location.href = 'https://other.test/'", page.context); + else [...page.timers][0]!(); + }; + const result = yield* Effect.result(runExport(page)); + expect(result._tag).toBe("Failure"); + const fs = yield* FileSystem.FileSystem; + const config = yield* ServerConfig.ServerConfig; + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([]); + assertClean(page); + }).pipe(Effect.provide(TestLayer)), +); + +it.effect("cleans the browser capture when the artifacts directory cannot be created", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const config = yield* ServerConfig.ServerConfig; + yield* fs.makeDirectory(config.stateDir, { recursive: true }); + yield* fs.writeFileString(config.browserArtifactsDir, "existing file"); + const page = makePage("text"); + const result = yield* Effect.result(runExport(page)); + expect(result._tag).toBe("Failure"); + expect(yield* fs.readFileString(config.browserArtifactsDir)).toBe("existing file"); + assertClean(page); + }).pipe(Effect.provide(TestLayer)), +); + +it.effect("removes a partial file when writing the next chunk fails", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const config = yield* ServerConfig.ServerConfig; + const page = makePage("text ".repeat(3000)); + let writes = 0; + const failingFs = FileSystem.FileSystem.of({ + ...fs, + open: (filePath, options) => + fs.open(filePath, options).pipe( + Effect.map((file) => ({ + ...file, + writeAll: (bytes) => + ++writes === 2 + ? fs.writeFileString(config.browserArtifactsDir, "fails on a directory") + : file.writeAll(bytes), + })), + ), + }); + const result = yield* Effect.result( + runExport(page).pipe(Effect.provideService(FileSystem.FileSystem, failingFs)), + ); + expect(result._tag).toBe("Failure"); + expect(writes).toBe(2); + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([]); + assertClean(page); + }).pipe(Effect.provide(TestLayer)), +); + +it.effect("does not create a capture when the resolved tab is unavailable", () => + Effect.gen(function* () { + const page = makePage("text"); + page.available = false; + const result = yield* Effect.result(runExport(page)); + expect(result._tag).toBe("Failure"); + expect(page.requests).toHaveLength(1); + expect(page.reads()).toBe(0); + assertClean(page); + }).pipe(Effect.provide(TestLayer)), +); + +it.effect("removes partial files and the browser capture on cancellation", () => + Effect.gen(function* () { + const enteredChunk = yield* Deferred.make(); + const page = makePage("text ".repeat(2000)); + page.chunkGate = Deferred.succeed(enteredChunk, undefined).pipe(Effect.andThen(Effect.never)); + const fiber = yield* Effect.forkChild(runExport(page)); + yield* Deferred.await(enteredChunk); + yield* Fiber.interrupt(fiber); + const fs = yield* FileSystem.FileSystem; + const config = yield* ServerConfig.ServerConfig; + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([]); + assertClean(page); + }).pipe(Effect.provide(TestLayer)), +); diff --git a/apps/server/src/mcp/PreviewTextExport.ts b/apps/server/src/mcp/PreviewTextExport.ts new file mode 100644 index 000000000000..4e6aeca69e01 --- /dev/null +++ b/apps/server/src/mcp/PreviewTextExport.ts @@ -0,0 +1,133 @@ +import { PreviewAutomationStatus, type PreviewTabId } from "@t3tools/contracts"; +import * as NodeCrypto from "node:crypto"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; + +import * as ServerConfig from "../config.ts"; +import { requireMcpCapability } from "./McpInvocationContext.ts"; +import { PreviewAutomationBroker } from "./PreviewAutomationBroker.ts"; + +export class PreviewTextExportError extends Schema.TaggedError()( + "PreviewTextExportError", + { cause: Schema.Defect() }, +) { + override get message(): string { + return "Could not save the loaded page text. The page may have changed or the file could not be written."; + } +} + +const Capture = Schema.Struct({ + totalChars: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + url: Schema.String.check(Schema.isMaxLength(2048)), +}); +const TextChunk = Schema.Struct({ + text: Schema.String.check(Schema.isMaxLength(4096)), + next: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), +}); +const encodeKey = Schema.encodeEffect(Schema.fromJsonString(Schema.String)); + +export const savePreviewText = Effect.fn("PreviewTextExport.savePreviewText")(function* ( + requestedTabId?: PreviewTabId, +) { + const scope = yield* requireMcpCapability("preview"); + const broker = yield* PreviewAutomationBroker; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const config = yield* ServerConfig.ServerConfig; + const id = NodeCrypto.randomUUID(); + let tabId = requestedTabId; + if (tabId === undefined) { + const status = yield* broker.invoke({ scope, operation: "status", input: {} }).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(PreviewAutomationStatus)), + Effect.mapError((cause) => new PreviewTextExportError({ cause })), + ); + if (!status.available || status.tabId === null) { + return yield* new PreviewTextExportError({ cause: "No available preview tab." }); + } + tabId = status.tabId; + } + const key = yield* encodeKey(`__t3_text_export_${id}`).pipe(Effect.orDie); + const textPath = path.join(config.browserArtifactsDir, `browser-text-${id}.txt`); + const evaluate = (expression: string) => + broker.invoke({ + scope, + operation: "evaluate", + tabId, + input: { expression, returnByValue: true }, + updateCurrentTab: false, + }); + return yield* Effect.scoped( + Effect.gen(function* () { + let created = false; + let complete = false; + yield* Effect.addFinalizer(() => + evaluate(`(() => { + const capture = globalThis[${key}]; + capture?.dispose(); + return true; + })()`).pipe(Effect.interruptible, Effect.timeoutOption(5000), Effect.ignore), + ); + yield* Effect.addFinalizer(() => + created && !complete ? fileSystem.remove(textPath).pipe(Effect.ignore) : Effect.void, + ); + const capture = yield* evaluate(`(() => { + const text = document.body?.innerText ?? ""; + let timer; + const dispose = () => { + clearTimeout(timer); + if (globalThis[${key}] === capture) delete globalThis[${key}]; + }; + const refresh = () => { + clearTimeout(timer); + timer = setTimeout(dispose, 300000); + }; + const capture = Object.freeze({ text, url: location.href, dispose, refresh }); + Object.defineProperty(globalThis, ${key}, { value: capture, configurable: true }); + refresh(); + return { totalChars: text.length, url: capture.url.slice(0, 2048) }; + })()`).pipe(Effect.flatMap(Schema.decodeUnknownEffect(Capture))); + yield* fileSystem.makeDirectory(config.browserArtifactsDir, { recursive: true }); + const file = yield* Effect.uninterruptible( + fileSystem.open(textPath, { flag: "wx", mode: 0o600 }).pipe( + Effect.tap(() => + Effect.sync(() => { + created = true; + }), + ), + ), + ); + let offset = 0; + let sizeBytes = 0; + while (offset < capture.totalChars) { + const chunk = yield* evaluate(`(() => { + const capture = globalThis[${key}]; + if (!capture || capture.url !== location.href) throw new Error("Text capture lost."); + let end = Math.min(${offset} + 4096, capture.text.length); + if (end < capture.text.length) { + const last = capture.text.charCodeAt(end - 1); + const next = capture.text.charCodeAt(end); + if (last >= 0xd800 && last <= 0xdbff && next >= 0xdc00 && next <= 0xdfff) end--; + } + capture.refresh(); + return { text: capture.text.slice(${offset}, end), next: end }; + })()`).pipe(Effect.flatMap(Schema.decodeUnknownEffect(TextChunk))); + if ( + chunk.next !== offset + chunk.text.length || + chunk.next <= offset || + chunk.next > capture.totalChars + ) { + return yield* new PreviewTextExportError({ cause: "Invalid text capture chunk." }); + } + const bytes = new TextEncoder().encode(chunk.text); + yield* file.writeAll(bytes); + sizeBytes += bytes.byteLength; + offset = chunk.next; + } + yield* file.sync; + complete = true; + return { textPath, totalChars: capture.totalChars, sizeBytes, url: capture.url, tabId }; + }), + ).pipe(Effect.mapError((cause) => new PreviewTextExportError({ cause }))); +}); diff --git a/apps/server/src/mcp/toolkits/preview/handlers.ts b/apps/server/src/mcp/toolkits/preview/handlers.ts index b70e68862683..59c2cc14b4d8 100644 --- a/apps/server/src/mcp/toolkits/preview/handlers.ts +++ b/apps/server/src/mcp/toolkits/preview/handlers.ts @@ -199,7 +199,12 @@ const handlers = { invokeTargeted("setColorScheme", input), preview_snapshot: (input) => { // Output selection and saving are MCP-only; the browser still produces a complete snapshot. - const { includeImage: _includeImage, save: _save, ...operationInput } = input ?? {}; + const { + includeImage: _includeImage, + save: _save, + saveText: _saveText, + ...operationInput + } = input ?? {}; return invokeTargeted("snapshot", operationInput); }, preview_click: (input) => invokeTargeted("click", input, input.timeoutMs), diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 278187aed0c8..46d816a0e11d 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -119,7 +119,7 @@ const PreviewSetAppearanceTool = safeBrowserTool( export const PreviewSnapshotTool = readonlyBrowserTool( Tool.make("preview_snapshot", { description: - "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. When the desktop host supports it, viewportText describes the current view, scroll reports page and visible container scroll positions, and inViewport elements come first. visibleText also includes rendered text outside the view; content not yet loaded requires scrolling. The text is capped near 20 KB, keeps current-view text ahead of offscreen page text, and lists what it omitted; use preview_evaluate to read more. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; with includeImage=false, save=true returns only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", + "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use the current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. When the desktop host supports it, viewportText describes the current view, scroll reports page and visible container scroll positions, and inViewport elements come first. visibleText also includes rendered text outside the view; content not yet loaded requires scrolling. The text is capped near 20 KB, keeps current-view text ahead of offscreen page text, and lists what it omitted. Set saveText=true to save all loaded, rendered main-page text to a UTF-8 file without a total character cap; textPath points to the file in the environment. This does not scroll or load missing text, and excludes embedded frames and shadow DOM. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; with includeImage=false, save=true returns only the url and saved artifact paths. Embed screenshotPath in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", parameters: Schema.Struct({ ...PreviewAutomationTabTargetInput.fields, includeImage: Schema.optional( @@ -131,7 +131,13 @@ export const PreviewSnapshotTool = readonlyBrowserTool( save: Schema.optional( Schema.Boolean.annotate({ description: - "Write the screenshot PNG to disk and return its absolute path as screenshotPath. With includeImage=false, return only the url and screenshotPath. Defaults to false.", + "Write the screenshot PNG to disk and return its absolute path as screenshotPath. With includeImage=false, return only the url and saved artifact details. Defaults to false.", + }), + ), + saveText: Schema.optional( + Schema.Boolean.annotate({ + description: + "Save all loaded, rendered main-page text to a UTF-8 file and return textPath, textChars, and textBytes. No total character cap; snapshot output stays bounded. Does not scroll or load missing content. Defaults to false.", }), ), }), diff --git a/docs/user/browser.md b/docs/user/browser.md index 80022d1eec4f..de84ca54065c 100644 --- a/docs/user/browser.md +++ b/docs/user/browser.md @@ -9,6 +9,11 @@ Snapshots have a size limit. Current-view text takes priority over page text and logs. The result reports omitted content so the agent can request more detail. The screenshot shows the current view. +An agent can set `saveText=true` when taking a snapshot to save all loaded, +rendered main-page text to a UTF-8 file. The file has no total character cap. The +snapshot returns `textPath` in the environment so the agent can read the file +in parts. This export does not scroll or load missing content. + Complex CSS clip shapes can cause current-view text and controls to be omitted. The result reports these omissions. Use the screenshot to inspect those areas. From 1963e68d23d683194ea1a08f06cb098e78ef46df Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 18:43:04 +0200 Subject: [PATCH 07/15] fix(preview): clean up undelivered text exports --- apps/server/src/mcp/McpHttpServer.test.ts | 140 ++++++++++++- apps/server/src/mcp/McpHttpServer.ts | 194 ++++++++++-------- apps/server/src/mcp/PreviewTextExport.test.ts | 29 ++- apps/server/src/mcp/PreviewTextExport.ts | 16 +- 4 files changed, 282 insertions(+), 97 deletions(-) diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index 7e61933f579f..0a04741611b2 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -15,6 +15,7 @@ import { } from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; @@ -62,12 +63,12 @@ const client = McpSchema.McpServerClient.of({ }, getClient: Effect.die("unused"), }); -const TestLayer = McpHttpServer.PreviewToolkitRegistrationLive.pipe( +const PreviewTestLayer = McpHttpServer.PreviewToolkitRegistrationLive.pipe( Layer.provideMerge(McpServer.McpServer.layer), Layer.provideMerge(PreviewAutomationBroker.layer), Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-mcp-http-server-test-" })), - Layer.provideMerge(NodeServices.layer), ); +const TestLayer = PreviewTestLayer.pipe(Layer.provideMerge(NodeServices.layer)); const PullRequestsTestLayer = McpHttpServer.PullRequestsToolkitRegistrationLive.pipe( Layer.provideMerge(McpServer.McpServer.layer), Layer.provide( @@ -134,7 +135,12 @@ const callSnapshot = (args: Record) => const serveTextExports = ( clientId: string, text: string, - options: { switchTab?: boolean; navigate?: boolean } = {}, + options: { + switchTab?: boolean; + navigate?: boolean; + snapshotFailure?: boolean; + beforeSnapshot?: Effect.Effect; + } = {}, ) => Effect.gen(function* () { const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker; @@ -167,6 +173,22 @@ const serveTextExports = ( return Effect.gen(function* () { const request = event.request; let result: unknown = snapshotResult; + if (request.operation === "snapshot") { + if (options.beforeSnapshot) yield* options.beforeSnapshot; + if (options.snapshotFailure) { + yield* broker.respond({ + clientId, + connectionId: event.connectionId, + requestId: request.requestId, + ok: false, + error: { + _tag: "PreviewAutomationExecutionError", + message: "private snapshot failure", + }, + }); + return; + } + } if (request.operation === "status") { result = { available: true, @@ -697,6 +719,118 @@ it.effect("reports text export failure without a successful snapshot or partial ).pipe(Effect.provide(TestLayer)), ); +it.effect("removes the completed text export when the following snapshot fails", () => + Effect.scoped( + Effect.gen(function* () { + const host = yield* serveTextExports("mcp-export-snapshot-failure-client", "loaded text", { + snapshotFailure: true, + }); + + const snapshot = yield* callSnapshot({ saveText: true }); + + expect(snapshot.isError).toBe(true); + expect(snapshot.structuredContent).toMatchObject({ + error: { _tag: "PreviewAutomationExecutionError", operation: "snapshot" }, + }); + expect(snapshot.content.every((content) => content.type === "text")).toBe(true); + expect(host.requests.at(-1)?.operation).toBe("snapshot"); + expect(host.textReads()).toBe(1); + expect(host.timers.size).toBe(0); + expect( + Object.getOwnPropertyNames(host.context).filter((key) => + key.startsWith("__t3_text_export_"), + ), + ).toEqual([]); + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([]); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect("removes only its completed text export when saving the PNG fails", () => + Effect.scoped( + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* fs.makeDirectory(config.browserArtifactsDir, { recursive: true }); + const existing = path.join(config.browserArtifactsDir, "existing.txt"); + yield* fs.writeFileString(existing, "keep this file"); + const host = yield* serveTextExports("mcp-export-png-failure-client", "loaded text"); + + const snapshot = yield* callSnapshot({ saveText: true, save: true }); + + expect(snapshot.isError).toBe(true); + expect(snapshot.structuredContent).toMatchObject({ + error: { _tag: "PreviewScreenshotSaveError", operation: "snapshot" }, + }); + expect(snapshot.content.every((content) => content.type === "text")).toBe(true); + expect(host.textReads()).toBe(1); + expect(host.timers.size).toBe(0); + expect( + Object.getOwnPropertyNames(host.context).filter((key) => + key.startsWith("__t3_text_export_"), + ), + ).toEqual([]); + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual(["existing.txt"]); + expect(yield* fs.readFileString(existing)).toBe("keep this file"); + }), + ).pipe( + Effect.provide( + PreviewTestLayer.pipe( + Layer.provideMerge( + Layer.effect( + FileSystem.FileSystem, + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + return FileSystem.FileSystem.of({ + ...fs, + writeFile: (filePath, bytes, options) => + filePath.endsWith(".png") + ? fs.writeFileString(path.dirname(filePath), "fails on a directory") + : fs.writeFile(filePath, bytes, options), + }); + }), + ), + ), + Layer.provideMerge(NodeServices.layer), + ), + ), + ), +); + +it.effect("removes the completed text export when the following snapshot is cancelled", () => + Effect.scoped( + Effect.gen(function* () { + const enteredSnapshot = yield* Deferred.make(); + const host = yield* serveTextExports("mcp-export-snapshot-cancel-client", "loaded text", { + beforeSnapshot: Deferred.succeed(enteredSnapshot, undefined).pipe( + Effect.andThen(Effect.never), + ), + }); + const fiber = yield* Effect.forkChild(callSnapshot({ saveText: true })); + yield* Deferred.await(enteredSnapshot); + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toHaveLength(1); + + yield* Fiber.interrupt(fiber); + + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([]); + expect(host.requests.at(-1)?.operation).toBe("snapshot"); + expect(host.textReads()).toBe(1); + expect(host.timers.size).toBe(0); + expect( + Object.getOwnPropertyNames(host.context).filter((key) => + key.startsWith("__t3_text_export_"), + ), + ).toEqual([]); + }), + ).pipe(Effect.provide(TestLayer)), +); + it.effect( "registers the pull request toolkit and surfaces a missing capability as a tool error", () => diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 0e5f3ebd9a63..b0145d91168e 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -512,97 +512,119 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot if (payload?.saveText !== true || !isPreviewSnapshotInput(payload)) return undefined; return yield* savePreviewText(payload.tabId); }); - return exportText.pipe( - Effect.flatMap((textExport) => - built - .handle("preview_snapshot", { - ...payload, - ...(textExport === undefined ? {} : { tabId: textExport.tabId }), - }) - .pipe( - Stream.unwrap, - Stream.run(Sink.last()), - Effect.flatMap(Effect.fromOption), - Effect.flatMap(({ encodedResult }) => - Effect.gen(function* () { - const snapshot = encodedResult as SnapshotMetadata & { - readonly url: string; - readonly screenshot: { - readonly mimeType: "image/png"; - readonly data: string; - readonly width: number; - readonly height: number; - }; - }; - const { screenshot, ...page } = snapshot; - const png = new Uint8Array(Buffer.from(screenshot.data, "base64")); - const screenshotPath = - payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; - const textArtifact = - textExport === undefined - ? {} - : { - textPath: textExport.textPath, - textChars: textExport.totalChars, - textBytes: textExport.sizeBytes, - textUrl: textExport.url, + return Effect.uninterruptibleMask((restore) => + restore(exportText).pipe( + Effect.flatMap((textExport) => + restore( + built + .handle("preview_snapshot", { + ...payload, + ...(textExport === undefined ? {} : { tabId: textExport.tabId }), + }) + .pipe( + Stream.unwrap, + Stream.run(Sink.last()), + Effect.flatMap(Effect.fromOption), + Effect.flatMap(({ encodedResult }) => + Effect.gen(function* () { + const snapshot = encodedResult as SnapshotMetadata & { + readonly url: string; + readonly screenshot: { + readonly mimeType: "image/png"; + readonly data: string; + readonly width: number; + readonly height: number; }; - if (screenshotPath !== undefined && payload?.includeImage === false) { - // The agent only wants a file to show the user. The url keeps the site icon on the tool row. - const saved = { - url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), - screenshotPath, - ...textArtifact, - }; - return new McpSchema.CallToolResult({ - isError: false, - structuredContent: saved, - content: [{ type: "text", text: encodeJsonText(saved) }], - }); - } - const metadata = { - ...page, - screenshot: { - mimeType: screenshot.mimeType, - width: screenshot.width, - height: screenshot.height, - }, - ...(screenshotPath === undefined ? {} : { screenshotPath }), - ...textArtifact, - }; - const bounded = boundSnapshotMetadata(metadata); - return new McpSchema.CallToolResult({ - isError: false, - structuredContent: - bounded.omitted.length === 0 - ? bounded.value - : { ...bounded.value, omitted: bounded.omitted }, - content: [ - // Keep the page identity readable even if a provider truncates the snapshot. - { - type: "text", - text: encodeJsonText({ + }; + const { screenshot, ...page } = snapshot; + const png = new Uint8Array(Buffer.from(screenshot.data, "base64")); + const screenshotPath = + payload?.save === true + ? yield* saveScreenshot(snapshot.url, png) + : undefined; + const textArtifact = + textExport === undefined + ? {} + : { + textPath: textExport.textPath, + textChars: textExport.totalChars, + textBytes: textExport.sizeBytes, + textUrl: textExport.url, + }; + if (screenshotPath !== undefined && payload?.includeImage === false) { + // The agent only wants a file to show the user. The url keeps the site icon on the tool row. + const saved = { url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), - }), - }, - { type: "text", text: bounded.text }, - ...(bounded.omitted.length === 0 - ? [] - : [ - { - type: "text" as const, - text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`, - }, - ]), - ...(payload?.includeImage === false - ? [] - : [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]), - ], - }); - }), + screenshotPath, + ...textArtifact, + }; + return new McpSchema.CallToolResult({ + isError: false, + structuredContent: saved, + content: [{ type: "text", text: encodeJsonText(saved) }], + }); + } + const metadata = { + ...page, + screenshot: { + mimeType: screenshot.mimeType, + width: screenshot.width, + height: screenshot.height, + }, + ...(screenshotPath === undefined ? {} : { screenshotPath }), + ...textArtifact, + }; + const bounded = boundSnapshotMetadata(metadata); + return new McpSchema.CallToolResult({ + isError: false, + structuredContent: + bounded.omitted.length === 0 + ? bounded.value + : { ...bounded.value, omitted: bounded.omitted }, + content: [ + // Keep the page identity readable even if a provider truncates the snapshot. + { + type: "text", + text: encodeJsonText({ + url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), + }), + }, + { type: "text", text: bounded.text }, + ...(bounded.omitted.length === 0 + ? [] + : [ + { + type: "text" as const, + text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`, + }, + ]), + ...(payload?.includeImage === false + ? [] + : [ + { + type: "image" as const, + data: png, + mimeType: screenshot.mimeType, + }, + ]), + ], + }); + }), + ), + ), + ).pipe( + Effect.onExit((exit) => + textExport !== undefined && exit._tag === "Failure" + ? FileSystem.FileSystem.pipe( + Effect.flatMap((fileSystem) => fileSystem.remove(textExport.textPath)), + Effect.ignore, + ) + : Effect.void, ), ), + ), ), + ).pipe( Effect.provideService(PreviewAutomationBroker.PreviewAutomationBroker, broker), Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), Effect.provide(saveServices), diff --git a/apps/server/src/mcp/PreviewTextExport.test.ts b/apps/server/src/mcp/PreviewTextExport.test.ts index a7a3f9b6c088..f10127120d51 100644 --- a/apps/server/src/mcp/PreviewTextExport.test.ts +++ b/apps/server/src/mcp/PreviewTextExport.test.ts @@ -69,6 +69,7 @@ const makePage = (initialText: string) => { }, beforeChunk: undefined as (() => void) | undefined, chunkGate: undefined as Effect.Effect | undefined, + disposeGate: undefined as Effect.Effect | undefined, broker: undefined as PreviewAutomationBroker["Service"] | undefined, }; const invoke = (request: PreviewAutomationInvokeInput) => @@ -92,7 +93,7 @@ const makePage = (initialText: string) => { page.beforeChunk?.(); if (page.chunkGate) yield* page.chunkGate; } - return yield* Effect.try({ + const value = yield* Effect.try({ try: () => { const value = NodeVM.runInContext(expression, context) as A; expect(Buffer.byteLength(encodeJson(value), "utf8")).toBeLessThan(32_000); @@ -100,6 +101,10 @@ const makePage = (initialText: string) => { }, catch: () => new PreviewAutomationNoAvailableHostError({ ...scope, operation: "evaluate" }), }); + if (expression.includes("capture?.dispose()") && page.disposeGate) { + yield* page.disposeGate; + } + return value; }); page.broker = PreviewAutomationBroker.of({ invoke, @@ -274,3 +279,25 @@ it.effect("removes partial files and the browser capture on cancellation", () => assertClean(page); }).pipe(Effect.provide(TestLayer)), ); + +it.effect("removes the completed file when cancellation occurs during browser cleanup", () => + Effect.gen(function* () { + const enteredDispose = yield* Deferred.make(); + const releaseDispose = yield* Deferred.make(); + const page = makePage("completed text"); + page.disposeGate = Deferred.succeed(enteredDispose, undefined).pipe( + Effect.andThen(Deferred.await(releaseDispose)), + ); + const fiber = yield* Effect.forkChild(runExport(page)); + yield* Deferred.await(enteredDispose); + const fs = yield* FileSystem.FileSystem; + const config = yield* ServerConfig.ServerConfig; + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toHaveLength(1); + const interruption = yield* Effect.forkChild(Fiber.interrupt(fiber)); + yield* Effect.yieldNow; + yield* Deferred.succeed(releaseDispose, undefined); + yield* Fiber.join(interruption); + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([]); + assertClean(page); + }).pipe(Effect.provide(TestLayer)), +); diff --git a/apps/server/src/mcp/PreviewTextExport.ts b/apps/server/src/mcp/PreviewTextExport.ts index 4e6aeca69e01..19477963f39f 100644 --- a/apps/server/src/mcp/PreviewTextExport.ts +++ b/apps/server/src/mcp/PreviewTextExport.ts @@ -58,10 +58,9 @@ export const savePreviewText = Effect.fn("PreviewTextExport.savePreviewText")(fu input: { expression, returnByValue: true }, updateCurrentTab: false, }); + let created = false; return yield* Effect.scoped( Effect.gen(function* () { - let created = false; - let complete = false; yield* Effect.addFinalizer(() => evaluate(`(() => { const capture = globalThis[${key}]; @@ -69,9 +68,6 @@ export const savePreviewText = Effect.fn("PreviewTextExport.savePreviewText")(fu return true; })()`).pipe(Effect.interruptible, Effect.timeoutOption(5000), Effect.ignore), ); - yield* Effect.addFinalizer(() => - created && !complete ? fileSystem.remove(textPath).pipe(Effect.ignore) : Effect.void, - ); const capture = yield* evaluate(`(() => { const text = document.body?.innerText ?? ""; let timer; @@ -126,8 +122,14 @@ export const savePreviewText = Effect.fn("PreviewTextExport.savePreviewText")(fu offset = chunk.next; } yield* file.sync; - complete = true; return { textPath, totalChars: capture.totalChars, sizeBytes, url: capture.url, tabId }; }), - ).pipe(Effect.mapError((cause) => new PreviewTextExportError({ cause }))); + ).pipe( + Effect.onExit((exit) => + created && exit._tag === "Failure" + ? fileSystem.remove(textPath).pipe(Effect.ignore) + : Effect.void, + ), + Effect.mapError((cause) => new PreviewTextExportError({ cause })), + ); }); From b0abd4b9df8f09ae83f2b2e5ff499c6827c76055 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 18:53:31 +0200 Subject: [PATCH 08/15] fix(preview): omit fully clipped scroll containers --- apps/desktop/src/preview/SnapshotPage.test.ts | 34 +++++++++++++++++++ apps/desktop/src/preview/SnapshotPage.ts | 30 ++++++++++------ 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/apps/desktop/src/preview/SnapshotPage.test.ts b/apps/desktop/src/preview/SnapshotPage.test.ts index 66a965372cd1..d67cb172c33f 100644 --- a/apps/desktop/src/preview/SnapshotPage.test.ts +++ b/apps/desktop/src/preview/SnapshotPage.test.ts @@ -173,6 +173,40 @@ const fixture = (escape = (value: string) => value) => { }; describe("snapshot page collector", () => { + it.each(["inset(100%)", "inset(0px 0px 100%)", "circle(50%)"])( + "omits scroll containers clipped by %s", + (clipPath) => { + for (const onAncestor of [false, true]) { + const page = fixture(); + const parent = page.element("parent", rect(0, 0)); + const scroller = page.element("scroll", rect(0, 0), parent); + (onAncestor ? parent : scroller).style.clipPath = clipPath; + scroller.style.overflowY = "auto"; + scroller.scrollHeight = 1_000; + const snapshot = page.capture(); + expect(snapshot.scroll?.containers).toEqual([]); + expect(snapshot.scroll?.containersTruncated).toBe(clipPath === "circle(50%)"); + } + }, + ); + + it("keeps partially clipped scroll containers and omits known offscreen areas", () => { + const page = fixture(); + const visible = page.element("partial", rect(0, 0)); + visible.style.clipPath = "inset(50% 0px 0px)"; + visible.style.overflowY = "auto"; + visible.scrollHeight = 1_000; + const offscreen = page.element("offscreen", rect(0, 250)); + offscreen.style.clipPath = "inset(0px)"; + offscreen.style.overflowY = "auto"; + offscreen.scrollHeight = 1_000; + const snapshot = page.capture(); + expect(snapshot.scroll?.containers.map((container) => container.selector)).toEqual([ + "#partial", + ]); + expect(snapshot.scroll?.containersTruncated).toBe(false); + }); + it.each(["paint", "content", "strict", "layout"])( "respects %s containment on boxes and the body", (contain) => { diff --git a/apps/desktop/src/preview/SnapshotPage.ts b/apps/desktop/src/preview/SnapshotPage.ts index a8e009d2e884..edd42ea7ebc4 100644 --- a/apps/desktop/src/preview/SnapshotPage.ts +++ b/apps/desktop/src/preview/SnapshotPage.ts @@ -175,7 +175,6 @@ function collectSnapshotPage(): SnapshotPage { const current = pending[index]!; const style = styleFor(current); if ( - !parentClip || style.display === "none" || style.opacity === "0" || style.contentVisibility === "hidden" @@ -184,7 +183,14 @@ function collectSnapshotPage(): SnapshotPage { parentClip = null; continue; } - const clip = { ...parentClip }; + const clip = { ...(parentClip ?? viewport) }; + const paintClip = paintClipFor(current); + if (paintClip) { + clip.left = Math.max(clip.left, paintClip.left); + clip.top = Math.max(clip.top, paintClip.top); + clip.right = Math.min(clip.right, paintClip.right); + clip.bottom = Math.min(clip.bottom, paintClip.bottom); + } const rootStyle = styleFor(document.documentElement); const viewportBody = current === document.body && @@ -244,7 +250,15 @@ function collectSnapshotPage(): SnapshotPage { const scrollable = (/^(auto|scroll)$/.test(style.overflowX) && current.scrollWidth > clientWidth) || (/^(auto|scroll)$/.test(style.overflowY) && current.scrollHeight > clientHeight); - if (scrollable && intersects(rect, parentClip)) { + if (scrollable && unsupportedClips.has(current)) containersTruncated = true; + if ( + scrollable && + parentClip && + paintClip && + clip.right > clip.left && + clip.bottom > clip.top && + intersects(rect, clip) + ) { const selector = containers.length < 20 ? selectorFor(current) : null; if (selector !== null && selector.length <= 1_000) { containers.push({ @@ -260,14 +274,8 @@ function collectSnapshotPage(): SnapshotPage { } } } - const paintClip = paintClipFor(current); - if (paintClip) { - clip.left = Math.max(clip.left, paintClip.left); - clip.top = Math.max(clip.top, paintClip.top); - clip.right = Math.min(clip.right, paintClip.right); - clip.bottom = Math.min(clip.bottom, paintClip.bottom); - } - parentClip = paintClip && clip.right > clip.left && clip.bottom > clip.top ? clip : null; + parentClip = + parentClip && paintClip && clip.right > clip.left && clip.bottom > clip.top ? clip : null; clips.set(current, parentClip); } return clips.get(element)!; From 03dfef243c9df4a75ae42200d4ac71d463828b84 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:00:41 +0200 Subject: [PATCH 09/15] refactor(preview): let snapshot service own saved artifacts --- .github/workflows/preview-snapshot-tests.yml | 3 +- apps/server/src/mcp/McpHttpServer.test.ts | 25 ++ apps/server/src/mcp/McpHttpServer.ts | 304 ++++++------------ apps/server/src/mcp/PreviewTextExport.ts | 135 -------- .../src/mcp/toolkits/preview/handlers.ts | 23 +- apps/server/src/mcp/toolkits/preview/tools.ts | 2 - .../Snapshot.test.ts} | 75 ++++- apps/server/src/preview/Snapshot.ts | 271 ++++++++++++++++ 8 files changed, 477 insertions(+), 361 deletions(-) delete mode 100644 apps/server/src/mcp/PreviewTextExport.ts rename apps/server/src/{mcp/PreviewTextExport.test.ts => preview/Snapshot.test.ts} (78%) create mode 100644 apps/server/src/preview/Snapshot.ts diff --git a/.github/workflows/preview-snapshot-tests.yml b/.github/workflows/preview-snapshot-tests.yml index bb958a196998..ef40032fdff7 100644 --- a/.github/workflows/preview-snapshot-tests.yml +++ b/.github/workflows/preview-snapshot-tests.yml @@ -6,6 +6,7 @@ on: - .github/workflows/preview-snapshot-tests.yml - apps/desktop/src/preview/** - apps/server/src/mcp/** + - apps/server/src/preview/Snapshot*.ts - packages/contracts/src/preview*.ts workflow_dispatch: @@ -41,7 +42,7 @@ jobs: apps/desktop/src/preview/Manager.test.ts apps/server/src/mcp/McpHttpServer.test.ts apps/server/src/mcp/PreviewAutomationBroker.test.ts - apps/server/src/mcp/PreviewTextExport.test.ts + apps/server/src/preview/Snapshot.test.ts apps/server/src/mcp/toolkits/preview/tools.test.ts packages/contracts/src/previewAutomationSnapshot.test.ts packages/contracts/src/preview.test.ts diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index 0a04741611b2..c617008f80b3 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -507,6 +507,31 @@ it.effect.each(["includeImage", "saveText"])( }).pipe(Effect.provide(TestLayer)), ); +it.effect("accepts a snapshot call with the arguments field omitted", () => + Effect.scoped( + Effect.gen(function* () { + const inputs = yield* serveSnapshots("mcp-omitted-snapshot-arguments-client", snapshotResult); + const server = yield* McpServer.McpServer; + + const snapshot = yield* server + .callTool({ name: "preview_snapshot" }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + Effect.provideService(McpSchema.McpServerClient, client), + ); + + expect(snapshot.isError).toBe(false); + expect(inputs).toEqual([{}]); + expect(snapshot.structuredContent).toMatchObject({ + url: snapshotResult.url, + title: snapshotResult.title, + visibleText: snapshotResult.visibleText, + }); + expect(snapshot.content.some((content) => content.type === "image")).toBe(true); + }), + ).pipe(Effect.provide(TestLayer)), +); + it.effect("saves the snapshot PNG on request and reports its path", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index b0145d91168e..aba0032fcd6f 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -1,22 +1,17 @@ -import * as NodeCrypto from "node:crypto"; import * as Cause from "effect/Cause"; -import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; import type * as Types from "effect/Types"; -import { McpProtocol, McpSchema, McpServer, Tool } from "effect/unstable/ai"; +import { AiError, McpProtocol, McpSchema, McpServer, Tool } from "effect/unstable/ai"; import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; import { PreviewAutomationError } from "@t3tools/contracts"; import packageJson from "../../package.json" with { type: "json" }; -import * as ServerConfig from "../config.ts"; import * as DeviceService from "../device/DeviceService.ts"; import * as McpInvocationContext from "./McpInvocationContext.ts"; import * as OrchestratorMcpService from "./OrchestratorMcpService.ts"; @@ -32,19 +27,11 @@ import { ThreadToolkit } from "./toolkits/thread/tools.ts"; import { ThreadToolkitHandlersLive } from "./toolkits/thread/handlers.ts"; import * as ThreadMetadataMcpService from "./ThreadMetadataMcpService.ts"; import * as McpSessionRegistry from "./McpSessionRegistry.ts"; -import * as PreviewAutomationBroker from "./PreviewAutomationBroker.ts"; -import { PreviewTextExportError, savePreviewText } from "./PreviewTextExport.ts"; +import * as PreviewSnapshot from "../preview/Snapshot.ts"; import { OrchestratorToolkitHandlersLive } from "./toolkits/orchestrator/handlers.ts"; import { OrchestratorToolkit } from "./toolkits/orchestrator/tools.ts"; -import { - PreviewSnapshotToolkitHandlersLive, - PreviewStandardToolkitHandlersLive, -} from "./toolkits/preview/handlers.ts"; -import { - PreviewSnapshotTool, - PreviewSnapshotToolkit, - PreviewStandardToolkit, -} from "./toolkits/preview/tools.ts"; +import { PreviewStandardToolkitHandlersLive } from "./toolkits/preview/handlers.ts"; +import { PreviewSnapshotTool, PreviewStandardToolkit } from "./toolkits/preview/tools.ts"; import { WorktreeToolkitHandlersLive } from "./toolkits/worktree/handlers.ts"; import { WorktreeToolkit } from "./toolkits/worktree/tools.ts"; import * as WorktreeMcpService from "./WorktreeMcpService.ts"; @@ -177,20 +164,24 @@ type SnapshotMetadata = { readonly url: string; readonly title: string; readonly visibleText: string; - readonly viewportText?: string; - readonly truncated?: { - readonly visibleText: boolean; - readonly viewportText: boolean; - readonly interactiveElements: boolean; - }; - readonly scroll?: { - readonly containers: ReadonlyArray; - readonly containersTruncated: boolean; - readonly [key: string]: unknown; - }; + readonly viewportText?: string | undefined; + readonly truncated?: + | { + readonly visibleText: boolean; + readonly viewportText: boolean; + readonly interactiveElements: boolean; + } + | undefined; + readonly scroll?: + | { + readonly containers: ReadonlyArray; + readonly containersTruncated: boolean; + readonly [key: string]: unknown; + } + | undefined; readonly interactiveElements: ReadonlyArray<{ readonly name: string; - readonly inViewport?: boolean; + readonly inViewport?: boolean | undefined; readonly [key: string]: unknown; }>; readonly consoleEntries: ReadonlyArray; @@ -388,54 +379,9 @@ const boundSnapshotMetadata = (metadata: SnapshotMetadata) => { return { value: value(), text, omitted }; }; -export class PreviewScreenshotSaveError extends Schema.TaggedError()( - "PreviewScreenshotSaveError", - { screenshotPath: Schema.String, cause: Schema.Defect() }, -) { - override get message(): string { - return `Could not save preview screenshot to ${this.screenshotPath}.`; - } -} - -const MAX_SCREENSHOT_SITE_SLUG_LENGTH = 40; - -/** Hostname reduced to a filename-safe slug, matching the desktop's own screenshot names. */ -const screenshotSiteSlug = (rawUrl: string): string => { - try { - const slug = new URL(rawUrl).hostname - .toLowerCase() - .replace(/[^a-z0-9]+/g, "-") - .replace(/^-+|-+$/g, "") - .slice(0, MAX_SCREENSHOT_SITE_SLUG_LENGTH) - .replace(/-+$/g, ""); - return slug || "site"; - } catch { - return "site"; - } -}; - -/** Writes the snapshot PNG under the browser artifacts directory and returns its path. */ -const saveScreenshot = Effect.fn("McpHttpServer.saveScreenshot")(function* ( - pageUrl: string, - data: Uint8Array, -) { - const config = yield* ServerConfig.ServerConfig; - const fileSystem = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const millis = yield* Clock.currentTimeMillis; - // Two saves in the same millisecond must not overwrite each other. - const fileName = `browser-screenshot-${screenshotSiteSlug(pageUrl)}-${millis.toString(36)}-${NodeCrypto.randomUUID().slice(0, 8)}.png`; - const screenshotPath = path.join(config.browserArtifactsDir, fileName); - yield* fileSystem.makeDirectory(config.browserArtifactsDir, { recursive: true }).pipe( - Effect.andThen(fileSystem.writeFile(screenshotPath, data)), - Effect.mapError((cause) => new PreviewScreenshotSaveError({ screenshotPath, cause })), - ); - return screenshotPath; -}); - const isPreviewAutomationError = Schema.is(PreviewAutomationError); -const isPreviewTextExportError = Schema.is(PreviewTextExportError); -const isPreviewSnapshotInput = Schema.is(PreviewSnapshotTool.parametersSchema); +const isPreviewTextExportError = Schema.is(PreviewSnapshot.PreviewTextExportError); +const decodePreviewSnapshotInput = Schema.decodeUnknownEffect(PreviewSnapshotTool.parametersSchema); const previewSnapshotFailure = (cause: Cause.Cause) => { if (Cause.hasInterrupts(cause) || cause.reasons.some(Cause.isDieReason)) { @@ -478,12 +424,7 @@ const previewSnapshotFailure = (cause: Cause.Cause) => { const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot")(function* () { const server = yield* McpServer.McpServer; - const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker; - // The MCP tool runner only supplies the client, so hand the save path its services here. - const saveServices = yield* Effect.context< - ServerConfig.ServerConfig | FileSystem.FileSystem | Path.Path - >(); - const built = yield* PreviewSnapshotToolkit; + const snapshots = yield* PreviewSnapshot.PreviewSnapshot; const tool = PreviewSnapshotTool; yield* server.addTool({ tool: new McpSchema.Tool({ @@ -508,130 +449,99 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot fiber.context, McpInvocationContext.McpInvocationContext, ); - const exportText = Effect.gen(function* () { - if (payload?.saveText !== true || !isPreviewSnapshotInput(payload)) return undefined; - return yield* savePreviewText(payload.tabId); - }); - return Effect.uninterruptibleMask((restore) => - restore(exportText).pipe( - Effect.flatMap((textExport) => - restore( - built - .handle("preview_snapshot", { - ...payload, - ...(textExport === undefined ? {} : { tabId: textExport.tabId }), - }) - .pipe( - Stream.unwrap, - Stream.run(Sink.last()), - Effect.flatMap(Effect.fromOption), - Effect.flatMap(({ encodedResult }) => - Effect.gen(function* () { - const snapshot = encodedResult as SnapshotMetadata & { - readonly url: string; - readonly screenshot: { - readonly mimeType: "image/png"; - readonly data: string; - readonly width: number; - readonly height: number; - }; - }; - const { screenshot, ...page } = snapshot; - const png = new Uint8Array(Buffer.from(screenshot.data, "base64")); - const screenshotPath = - payload?.save === true - ? yield* saveScreenshot(snapshot.url, png) - : undefined; - const textArtifact = - textExport === undefined - ? {} - : { - textPath: textExport.textPath, - textChars: textExport.totalChars, - textBytes: textExport.sizeBytes, - textUrl: textExport.url, - }; - if (screenshotPath !== undefined && payload?.includeImage === false) { - // The agent only wants a file to show the user. The url keeps the site icon on the tool row. - const saved = { - url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), - screenshotPath, - ...textArtifact, - }; - return new McpSchema.CallToolResult({ - isError: false, - structuredContent: saved, - content: [{ type: "text", text: encodeJsonText(saved) }], - }); - } - const metadata = { - ...page, - screenshot: { - mimeType: screenshot.mimeType, - width: screenshot.width, - height: screenshot.height, - }, - ...(screenshotPath === undefined ? {} : { screenshotPath }), + return decodePreviewSnapshotInput(payload ?? {}).pipe( + Effect.mapError((cause) => + AiError.make({ + module: "Toolkit", + method: "preview_snapshot.handle", + reason: new AiError.ToolParameterValidationError({ + toolName: "preview_snapshot", + description: cause.message, + }), + }), + ), + Effect.flatMap((input) => + McpInvocationContext.requireMcpCapability("preview").pipe( + Effect.flatMap((scope) => + snapshots.withSnapshot( + { scope, ...input }, + ({ snapshot, png, textExport, screenshotPath }) => + Effect.sync(() => { + const { screenshot, ...page } = snapshot; + const textArtifact = + textExport === undefined + ? {} + : { + textPath: textExport.textPath, + textChars: textExport.totalChars, + textBytes: textExport.sizeBytes, + textUrl: textExport.url, + }; + if (screenshotPath !== undefined && input.includeImage === false) { + // The agent only wants a file to show the user. The url keeps the site icon on the tool row. + const saved = { + url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), + screenshotPath, ...textArtifact, }; - const bounded = boundSnapshotMetadata(metadata); return new McpSchema.CallToolResult({ isError: false, - structuredContent: - bounded.omitted.length === 0 - ? bounded.value - : { ...bounded.value, omitted: bounded.omitted }, - content: [ - // Keep the page identity readable even if a provider truncates the snapshot. - { - type: "text", - text: encodeJsonText({ - url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), - }), - }, - { type: "text", text: bounded.text }, - ...(bounded.omitted.length === 0 - ? [] - : [ - { - type: "text" as const, - text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`, - }, - ]), - ...(payload?.includeImage === false - ? [] - : [ - { - type: "image" as const, - data: png, - mimeType: screenshot.mimeType, - }, - ]), - ], + structuredContent: saved, + content: [{ type: "text", text: encodeJsonText(saved) }], }); - }), - ), - ), - ).pipe( - Effect.onExit((exit) => - textExport !== undefined && exit._tag === "Failure" - ? FileSystem.FileSystem.pipe( - Effect.flatMap((fileSystem) => fileSystem.remove(textExport.textPath)), - Effect.ignore, - ) - : Effect.void, + } + const metadata = { + ...page, + screenshot: { + mimeType: screenshot.mimeType, + width: screenshot.width, + height: screenshot.height, + }, + ...(screenshotPath === undefined ? {} : { screenshotPath }), + ...textArtifact, + }; + const bounded = boundSnapshotMetadata(metadata); + return new McpSchema.CallToolResult({ + isError: false, + structuredContent: + bounded.omitted.length === 0 + ? bounded.value + : { ...bounded.value, omitted: bounded.omitted }, + content: [ + // Keep the page identity readable even if a provider truncates the snapshot. + { + type: "text", + text: encodeJsonText({ + url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), + }), + }, + { type: "text", text: bounded.text }, + ...(bounded.omitted.length === 0 + ? [] + : [ + { + type: "text" as const, + text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`, + }, + ]), + ...(input.includeImage === false + ? [] + : [ + { + type: "image" as const, + data: png, + mimeType: screenshot.mimeType, + }, + ]), + ], + }); + }), ), ), ), ), - ).pipe( - Effect.provideService(PreviewAutomationBroker.PreviewAutomationBroker, broker), Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), - Effect.provide(saveServices), - Effect.matchCauseEffect({ - onFailure: previewSnapshotFailure, - onSuccess: Effect.succeed, - }), + Effect.matchCauseEffect({ onFailure: previewSnapshotFailure, onSuccess: Effect.succeed }), ); }), }); @@ -790,7 +700,7 @@ const PreviewStandardToolkitRegistrationLive = McpServer.toolkit(PreviewStandard ); const PreviewSnapshotRegistrationLive = Layer.effectDiscard(registerPreviewSnapshot()).pipe( - Layer.provide(PreviewSnapshotToolkitHandlersLive), + Layer.provide(PreviewSnapshot.layer), ); export const PreviewToolkitRegistrationLive = Layer.mergeAll( diff --git a/apps/server/src/mcp/PreviewTextExport.ts b/apps/server/src/mcp/PreviewTextExport.ts deleted file mode 100644 index 19477963f39f..000000000000 --- a/apps/server/src/mcp/PreviewTextExport.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { PreviewAutomationStatus, type PreviewTabId } from "@t3tools/contracts"; -import * as NodeCrypto from "node:crypto"; -import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; -import * as Path from "effect/Path"; -import * as Schema from "effect/Schema"; - -import * as ServerConfig from "../config.ts"; -import { requireMcpCapability } from "./McpInvocationContext.ts"; -import { PreviewAutomationBroker } from "./PreviewAutomationBroker.ts"; - -export class PreviewTextExportError extends Schema.TaggedError()( - "PreviewTextExportError", - { cause: Schema.Defect() }, -) { - override get message(): string { - return "Could not save the loaded page text. The page may have changed or the file could not be written."; - } -} - -const Capture = Schema.Struct({ - totalChars: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), - url: Schema.String.check(Schema.isMaxLength(2048)), -}); -const TextChunk = Schema.Struct({ - text: Schema.String.check(Schema.isMaxLength(4096)), - next: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), -}); -const encodeKey = Schema.encodeEffect(Schema.fromJsonString(Schema.String)); - -export const savePreviewText = Effect.fn("PreviewTextExport.savePreviewText")(function* ( - requestedTabId?: PreviewTabId, -) { - const scope = yield* requireMcpCapability("preview"); - const broker = yield* PreviewAutomationBroker; - const fileSystem = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const config = yield* ServerConfig.ServerConfig; - const id = NodeCrypto.randomUUID(); - let tabId = requestedTabId; - if (tabId === undefined) { - const status = yield* broker.invoke({ scope, operation: "status", input: {} }).pipe( - Effect.flatMap(Schema.decodeUnknownEffect(PreviewAutomationStatus)), - Effect.mapError((cause) => new PreviewTextExportError({ cause })), - ); - if (!status.available || status.tabId === null) { - return yield* new PreviewTextExportError({ cause: "No available preview tab." }); - } - tabId = status.tabId; - } - const key = yield* encodeKey(`__t3_text_export_${id}`).pipe(Effect.orDie); - const textPath = path.join(config.browserArtifactsDir, `browser-text-${id}.txt`); - const evaluate = (expression: string) => - broker.invoke({ - scope, - operation: "evaluate", - tabId, - input: { expression, returnByValue: true }, - updateCurrentTab: false, - }); - let created = false; - return yield* Effect.scoped( - Effect.gen(function* () { - yield* Effect.addFinalizer(() => - evaluate(`(() => { - const capture = globalThis[${key}]; - capture?.dispose(); - return true; - })()`).pipe(Effect.interruptible, Effect.timeoutOption(5000), Effect.ignore), - ); - const capture = yield* evaluate(`(() => { - const text = document.body?.innerText ?? ""; - let timer; - const dispose = () => { - clearTimeout(timer); - if (globalThis[${key}] === capture) delete globalThis[${key}]; - }; - const refresh = () => { - clearTimeout(timer); - timer = setTimeout(dispose, 300000); - }; - const capture = Object.freeze({ text, url: location.href, dispose, refresh }); - Object.defineProperty(globalThis, ${key}, { value: capture, configurable: true }); - refresh(); - return { totalChars: text.length, url: capture.url.slice(0, 2048) }; - })()`).pipe(Effect.flatMap(Schema.decodeUnknownEffect(Capture))); - yield* fileSystem.makeDirectory(config.browserArtifactsDir, { recursive: true }); - const file = yield* Effect.uninterruptible( - fileSystem.open(textPath, { flag: "wx", mode: 0o600 }).pipe( - Effect.tap(() => - Effect.sync(() => { - created = true; - }), - ), - ), - ); - let offset = 0; - let sizeBytes = 0; - while (offset < capture.totalChars) { - const chunk = yield* evaluate(`(() => { - const capture = globalThis[${key}]; - if (!capture || capture.url !== location.href) throw new Error("Text capture lost."); - let end = Math.min(${offset} + 4096, capture.text.length); - if (end < capture.text.length) { - const last = capture.text.charCodeAt(end - 1); - const next = capture.text.charCodeAt(end); - if (last >= 0xd800 && last <= 0xdbff && next >= 0xdc00 && next <= 0xdfff) end--; - } - capture.refresh(); - return { text: capture.text.slice(${offset}, end), next: end }; - })()`).pipe(Effect.flatMap(Schema.decodeUnknownEffect(TextChunk))); - if ( - chunk.next !== offset + chunk.text.length || - chunk.next <= offset || - chunk.next > capture.totalChars - ) { - return yield* new PreviewTextExportError({ cause: "Invalid text capture chunk." }); - } - const bytes = new TextEncoder().encode(chunk.text); - yield* file.writeAll(bytes); - sizeBytes += bytes.byteLength; - offset = chunk.next; - } - yield* file.sync; - return { textPath, totalChars: capture.totalChars, sizeBytes, url: capture.url, tabId }; - }), - ).pipe( - Effect.onExit((exit) => - created && exit._tag === "Failure" - ? fileSystem.remove(textPath).pipe(Effect.ignore) - : Effect.void, - ), - Effect.mapError((cause) => new PreviewTextExportError({ cause })), - ); -}); diff --git a/apps/server/src/mcp/toolkits/preview/handlers.ts b/apps/server/src/mcp/toolkits/preview/handlers.ts index 59c2cc14b4d8..9c801214a9d8 100644 --- a/apps/server/src/mcp/toolkits/preview/handlers.ts +++ b/apps/server/src/mcp/toolkits/preview/handlers.ts @@ -14,7 +14,6 @@ import { type PreviewAutomationRecordingStatus, type PreviewAutomationResizeResult, type PreviewAutomationSetColorSchemeResult, - type PreviewAutomationSnapshot, type PreviewAutomationStatus, type PreviewTabId, } from "@t3tools/contracts"; @@ -29,7 +28,7 @@ import { resolveAttachmentRelativePath } from "../../../attachmentPaths.ts"; import * as ServerConfig from "../../../config.ts"; import * as McpInvocationContext from "../../McpInvocationContext.ts"; import * as PreviewAutomationBroker from "../../PreviewAutomationBroker.ts"; -import { PreviewSnapshotToolkit, PreviewStandardToolkit, PreviewToolkit } from "./tools.ts"; +import { PreviewStandardToolkit } from "./tools.ts"; /** * Collapses the `show` alias onto `open` and defaults tab reuse. @@ -197,16 +196,6 @@ const handlers = { invokeTargeted("resize", input, input.timeoutMs), preview_set_appearance: (input) => invokeTargeted("setColorScheme", input), - preview_snapshot: (input) => { - // Output selection and saving are MCP-only; the browser still produces a complete snapshot. - const { - includeImage: _includeImage, - save: _save, - saveText: _saveText, - ...operationInput - } = input ?? {}; - return invokeTargeted("snapshot", operationInput); - }, preview_click: (input) => invokeTargeted("click", input, input.timeoutMs), preview_type: (input) => invokeTargeted("type", input, input.timeoutMs), preview_press: (input) => invokeTargeted("press", input), @@ -234,12 +223,6 @@ const handlers = { const artifact = yield* claimPreviewRecording(scope.threadId, response.result); return { ...artifact, ...(response.toolIcon ? { toolIcon: response.toolIcon } : {}) }; }), -} satisfies Parameters[0]; - -const { preview_snapshot, ...standardHandlers } = handlers; +} satisfies Parameters[0]; -export const PreviewStandardToolkitHandlersLive = PreviewStandardToolkit.toLayer(standardHandlers); - -export const PreviewSnapshotToolkitHandlersLive = PreviewSnapshotToolkit.toLayer({ - preview_snapshot, -}); +export const PreviewStandardToolkitHandlersLive = PreviewStandardToolkit.toLayer(handlers); diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 46d816a0e11d..5acb2599d299 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -279,5 +279,3 @@ export const PreviewStandardToolkit = Toolkit.make( PreviewRecordingStartTool, PreviewRecordingStopTool, ); - -export const PreviewSnapshotToolkit = Toolkit.make(PreviewSnapshotTool); diff --git a/apps/server/src/mcp/PreviewTextExport.test.ts b/apps/server/src/preview/Snapshot.test.ts similarity index 78% rename from apps/server/src/mcp/PreviewTextExport.test.ts rename to apps/server/src/preview/Snapshot.test.ts index f10127120d51..663c346ff0a5 100644 --- a/apps/server/src/mcp/PreviewTextExport.test.ts +++ b/apps/server/src/preview/Snapshot.test.ts @@ -13,15 +13,15 @@ import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; import * as ServerConfig from "../config.ts"; -import { McpInvocationContext } from "./McpInvocationContext.ts"; import { PreviewAutomationBroker, type PreviewAutomationInvokeInput, -} from "./PreviewAutomationBroker.ts"; -import { savePreviewText } from "./PreviewTextExport.ts"; +} from "../mcp/PreviewAutomationBroker.ts"; +import * as Snapshot from "./Snapshot.ts"; const tabId = PreviewTabId.make("text-tab"); const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); @@ -85,6 +85,26 @@ const makePage = (initialText: string) => { loading: false, } as A; } + if (request.operation === "snapshot") { + expect(request.tabId).toBe(tabId); + return { + url: "https://example.test/page", + title: "Page", + loading: false, + visibleText: "Page", + interactiveElements: [], + accessibilityTree: {}, + consoleEntries: [], + networkEntries: [], + actionTimeline: [], + screenshot: { + mimeType: "image/png", + data: Buffer.from("png").toString("base64"), + width: 10, + height: 5, + }, + } as A; + } expect(request.operation).toBe("evaluate"); expect(request.tabId).toBe(tabId); expect(request.timeoutMs).toBeUndefined(); @@ -116,8 +136,14 @@ const makePage = (initialText: string) => { }; const runExport = (page: ReturnType, target?: PreviewTabId) => - savePreviewText(target).pipe( - Effect.provideService(McpInvocationContext, scope), + Effect.gen(function* () { + const snapshot = yield* Snapshot.PreviewSnapshot; + return yield* snapshot.withSnapshot( + { scope, saveText: true, ...(target === undefined ? {} : { tabId: target }) }, + ({ textExport }) => Effect.succeed(textExport!), + ); + }).pipe( + Effect.provide(Snapshot.layer), Effect.provideService(PreviewAutomationBroker, page.broker!), ); const assertClean = (page: ReturnType) => { @@ -154,7 +180,8 @@ it.effect("keeps Unicode pairs intact at chunk boundaries", () => const fs = yield* FileSystem.FileSystem; expect(yield* fs.readFileString(saved.textPath)).toBe(text); expect(saved.sizeBytes).toBe(Buffer.byteLength(text)); - expect(page.requests.every((request) => request.operation === "evaluate")).toBe(true); + expect(page.requests.some((request) => request.operation === "status")).toBe(false); + expect(page.requests.at(-1)?.operation).toBe("snapshot"); assertClean(page); }).pipe(Effect.provide(TestLayer)), ); @@ -301,3 +328,39 @@ it.effect("removes the completed file when cancellation occurs during browser cl assertClean(page); }).pipe(Effect.provide(TestLayer)), ); + +it.effect.each(["failure", "cancellation"] as const)( + "removes its export after snapshot result delivery %s", + (outcome) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const config = yield* ServerConfig.ServerConfig; + yield* fs.makeDirectory(config.browserArtifactsDir, { recursive: true }); + const existing = path.join(config.browserArtifactsDir, "existing.txt"); + yield* fs.writeFileString(existing, "keep this file"); + const enteredUse = yield* Deferred.make(); + const page = makePage("completed text"); + const delivery = Effect.gen(function* () { + const snapshots = yield* Snapshot.PreviewSnapshot; + return yield* snapshots.withSnapshot({ scope, saveText: true }, ({ textExport }) => + Effect.gen(function* () { + expect(yield* fs.readFileString(textExport!.textPath)).toBe("completed text"); + yield* Deferred.succeed(enteredUse, undefined); + return yield* outcome === "failure" ? Effect.fail("delivery failed") : Effect.never; + }), + ); + }).pipe( + Effect.provide(Snapshot.layer), + Effect.provideService(PreviewAutomationBroker, page.broker!), + ); + const fiber = yield* Effect.forkChild(delivery); + yield* Deferred.await(enteredUse); + if (outcome === "cancellation") yield* Fiber.interrupt(fiber); + const result = yield* Fiber.await(fiber); + expect(result._tag).toBe("Failure"); + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual(["existing.txt"]); + expect(yield* fs.readFileString(existing)).toBe("keep this file"); + assertClean(page); + }).pipe(Effect.provide(TestLayer)), +); diff --git a/apps/server/src/preview/Snapshot.ts b/apps/server/src/preview/Snapshot.ts new file mode 100644 index 000000000000..0e1a68df4fe1 --- /dev/null +++ b/apps/server/src/preview/Snapshot.ts @@ -0,0 +1,271 @@ +import { + PreviewAutomationSnapshot, + PreviewAutomationStatus, + type PreviewAutomationError, + type PreviewTabId, +} from "@t3tools/contracts"; +import * as NodeCrypto from "node:crypto"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; + +import * as ServerConfig from "../config.ts"; +import * as PreviewAutomationBroker from "../mcp/PreviewAutomationBroker.ts"; + +export class PreviewTextExportError extends Schema.TaggedError()( + "PreviewTextExportError", + { cause: Schema.Defect() }, +) { + override get message(): string { + return "Could not save the loaded page text. The page may have changed or the file could not be written."; + } +} + +const Capture = Schema.Struct({ + totalChars: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + url: Schema.String.check(Schema.isMaxLength(2048)), +}); +const TextChunk = Schema.Struct({ + text: Schema.String.check(Schema.isMaxLength(4096)), + next: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), +}); +const encodeKey = Schema.encodeEffect(Schema.fromJsonString(Schema.String)); +const decodeCapture = Schema.decodeUnknownEffect(Capture); +const decodeTextChunk = Schema.decodeUnknownEffect(TextChunk); +const decodeStatus = Schema.decodeUnknownEffect(PreviewAutomationStatus); +const decodeSnapshot = Schema.decodeUnknownEffect(PreviewAutomationSnapshot); + +export class PreviewScreenshotSaveError extends Schema.TaggedError()( + "PreviewScreenshotSaveError", + { screenshotPath: Schema.String, cause: Schema.Defect() }, +) { + override get message(): string { + return `Could not save preview screenshot to ${this.screenshotPath}.`; + } +} + +interface SnapshotInput { + readonly scope: PreviewAutomationBroker.PreviewAutomationInvokeInput["scope"]; + readonly tabId?: PreviewTabId | undefined; + readonly saveText?: boolean | undefined; + readonly save?: boolean | undefined; +} + +interface SnapshotCapture { + readonly snapshot: PreviewAutomationSnapshot; + readonly png: Uint8Array; + readonly textExport?: { + readonly textPath: string; + readonly totalChars: number; + readonly sizeBytes: number; + readonly url: string; + readonly tabId: PreviewTabId; + }; + readonly screenshotPath?: string; +} + +export class PreviewSnapshot extends Context.Service< + PreviewSnapshot, + { + readonly withSnapshot: ( + input: SnapshotInput, + use: (capture: SnapshotCapture) => Effect.Effect, + ) => Effect.Effect< + A, + | E + | PreviewAutomationError + | PreviewTextExportError + | PreviewScreenshotSaveError + | Schema.SchemaError, + R + >; + } +>()("t3/preview/Snapshot/PreviewSnapshot") {} + +const screenshotSiteSlug = (rawUrl: string): string => { + try { + const slug = new URL(rawUrl).hostname + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 40) + .replace(/-+$/g, ""); + return slug || "site"; + } catch { + return "site"; + } +}; + +const make = Effect.gen(function* () { + const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const config = yield* ServerConfig.ServerConfig; + + const saveText = Effect.fn("PreviewSnapshot.saveText")(function* ( + scope: SnapshotInput["scope"], + requestedTabId?: PreviewTabId, + ) { + const id = NodeCrypto.randomUUID(); + let tabId = requestedTabId; + if (tabId === undefined) { + const status = yield* broker.invoke({ scope, operation: "status", input: {} }).pipe( + Effect.flatMap(decodeStatus), + Effect.mapError((cause) => new PreviewTextExportError({ cause })), + ); + if (!status.available || status.tabId === null) { + return yield* new PreviewTextExportError({ cause: "No available preview tab." }); + } + tabId = status.tabId; + } + const key = yield* encodeKey(`__t3_text_export_${id}`).pipe(Effect.orDie); + const textPath = path.join(config.browserArtifactsDir, `browser-text-${id}.txt`); + const evaluate = (expression: string) => + broker.invoke({ + scope, + operation: "evaluate", + tabId, + input: { expression, returnByValue: true }, + updateCurrentTab: false, + }); + let created = false; + return yield* Effect.scoped( + Effect.gen(function* () { + yield* Effect.addFinalizer(() => + evaluate(`(() => { + const capture = globalThis[${key}]; + capture?.dispose(); + return true; + })()`).pipe(Effect.interruptible, Effect.timeoutOption(5000), Effect.ignore), + ); + const capture = yield* evaluate(`(() => { + const text = document.body?.innerText ?? ""; + let timer; + const dispose = () => { + clearTimeout(timer); + if (globalThis[${key}] === capture) delete globalThis[${key}]; + }; + const refresh = () => { + clearTimeout(timer); + timer = setTimeout(dispose, 300000); + }; + const capture = Object.freeze({ text, url: location.href, dispose, refresh }); + Object.defineProperty(globalThis, ${key}, { value: capture, configurable: true }); + refresh(); + return { totalChars: text.length, url: capture.url.slice(0, 2048) }; + })()`).pipe(Effect.flatMap(decodeCapture)); + yield* fileSystem.makeDirectory(config.browserArtifactsDir, { recursive: true }); + const file = yield* Effect.uninterruptible( + fileSystem.open(textPath, { flag: "wx", mode: 0o600 }).pipe( + Effect.tap(() => + Effect.sync(() => { + created = true; + }), + ), + ), + ); + let offset = 0; + let sizeBytes = 0; + while (offset < capture.totalChars) { + const chunk = yield* evaluate(`(() => { + const capture = globalThis[${key}]; + if (!capture || capture.url !== location.href) throw new Error("Text capture lost."); + let end = Math.min(${offset} + 4096, capture.text.length); + if (end < capture.text.length) { + const last = capture.text.charCodeAt(end - 1); + const next = capture.text.charCodeAt(end); + if (last >= 0xd800 && last <= 0xdbff && next >= 0xdc00 && next <= 0xdfff) end--; + } + capture.refresh(); + return { text: capture.text.slice(${offset}, end), next: end }; + })()`).pipe(Effect.flatMap(decodeTextChunk)); + if ( + chunk.next !== offset + chunk.text.length || + chunk.next <= offset || + chunk.next > capture.totalChars + ) { + return yield* new PreviewTextExportError({ cause: "Invalid text capture chunk." }); + } + const bytes = new TextEncoder().encode(chunk.text); + yield* file.writeAll(bytes); + sizeBytes += bytes.byteLength; + offset = chunk.next; + } + yield* file.sync; + return { textPath, totalChars: capture.totalChars, sizeBytes, url: capture.url, tabId }; + }), + ).pipe( + Effect.onExit((exit) => + created && exit._tag === "Failure" + ? fileSystem.remove(textPath).pipe(Effect.ignore) + : Effect.void, + ), + Effect.mapError((cause) => new PreviewTextExportError({ cause })), + ); + }); + + const saveScreenshot = Effect.fn("PreviewSnapshot.saveScreenshot")(function* ( + pageUrl: string, + data: Uint8Array, + ) { + const millis = yield* Clock.currentTimeMillis; + const fileName = `browser-screenshot-${screenshotSiteSlug(pageUrl)}-${millis.toString(36)}-${NodeCrypto.randomUUID().slice(0, 8)}.png`; + const screenshotPath = path.join(config.browserArtifactsDir, fileName); + yield* fileSystem.makeDirectory(config.browserArtifactsDir, { recursive: true }).pipe( + Effect.andThen(fileSystem.writeFile(screenshotPath, data)), + Effect.mapError((cause) => new PreviewScreenshotSaveError({ screenshotPath, cause })), + ); + return screenshotPath; + }); + + const withSnapshot = Effect.fn("PreviewSnapshot.withSnapshot")(function* ( + input: SnapshotInput, + use: (capture: SnapshotCapture) => Effect.Effect, + ) { + return yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + let textExport: SnapshotCapture["textExport"]; + if (input.saveText === true) { + textExport = yield* restore(saveText(input.scope, input.tabId)); + } + const exportedText = textExport; + return yield* restore( + Effect.gen(function* () { + const tabId = exportedText?.tabId ?? input.tabId; + const snapshot = yield* broker + .invoke({ + scope: input.scope, + operation: "snapshot", + input: {}, + ...(tabId === undefined ? {} : { tabId }), + }) + .pipe(Effect.flatMap(decodeSnapshot)); + const png = new Uint8Array(Buffer.from(snapshot.screenshot.data, "base64")); + const screenshotPath = + input.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; + return yield* use({ + snapshot, + png, + ...(exportedText === undefined ? {} : { textExport: exportedText }), + ...(screenshotPath === undefined ? {} : { screenshotPath }), + }); + }), + ).pipe( + Effect.onExit((exit) => + exportedText !== undefined && exit._tag === "Failure" + ? fileSystem.remove(exportedText.textPath).pipe(Effect.ignore) + : Effect.void, + ), + ); + }), + ); + }); + + return PreviewSnapshot.of({ withSnapshot }); +}); + +export const layer = Layer.effect(PreviewSnapshot, make); From 4c6cfc1a3f16e339f41c960c387f9113b92410e5 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:07:07 +0200 Subject: [PATCH 10/15] fix(preview): preserve text export recovery guidance --- apps/server/src/mcp/McpHttpServer.test.ts | 38 ++++++++++-------- apps/server/src/preview/Snapshot.test.ts | 48 +++++++++++++++++++++++ apps/server/src/preview/Snapshot.ts | 18 ++++++--- 3 files changed, 82 insertions(+), 22 deletions(-) diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index c617008f80b3..1b3d7f7b69d3 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -345,19 +345,21 @@ it.effect.each([ ).pipe(Effect.provide(TestLayer)), ); -it.effect("tells the agent how to fall back when no desktop app can run the snapshot", () => - Effect.gen(function* () { - const snapshot = yield* callSnapshot({}); - - expect(snapshot.isError).toBe(true); - const [text] = snapshot.content; - expect(text?.type === "text" ? text.text : "").toContain( - "use a headless browser from the shell", - ); - expect(snapshot.structuredContent).toMatchObject({ - error: { _tag: "PreviewAutomationNoAvailableHostError" }, - }); - }).pipe(Effect.provide(TestLayer)), +it.effect.each([{}, { saveText: true }])( + "tells the agent how to fall back when no desktop app can run the snapshot %j", + (args) => + Effect.gen(function* () { + const snapshot = yield* callSnapshot(args); + + expect(snapshot.isError).toBe(true); + const [text] = snapshot.content; + expect(text?.type === "text" ? text.text : "").toContain( + "use a headless browser from the shell", + ); + expect(snapshot.structuredContent).toMatchObject({ + error: { _tag: "PreviewAutomationNoAvailableHostError" }, + }); + }).pipe(Effect.provide(TestLayer)), ); it.effect.each([ @@ -721,14 +723,18 @@ it.effect("reports text export failure without a successful snapshot or partial const snapshot = yield* callSnapshot({ saveText: true, save: true }); - const message = - "Could not save the loaded page text. The page may have changed or the file could not be written."; + const message = "Preview automation evaluate failed on client mcp-text-failure-client."; expect(snapshot.isError).toBe(true); expect(snapshot.content).toEqual([ { type: "text", text: `Preview snapshot failed: ${message}` }, ]); expect(snapshot.structuredContent).toEqual({ - error: { _tag: "PreviewTextExportError", operation: "snapshot", failureCount: 1, message }, + error: { + _tag: "PreviewAutomationExecutionError", + operation: "snapshot", + failureCount: 1, + message, + }, }); expect(host.requests.some((request) => request.operation === "snapshot")).toBe(false); expect(host.timers.size).toBe(0); diff --git a/apps/server/src/preview/Snapshot.test.ts b/apps/server/src/preview/Snapshot.test.ts index 663c346ff0a5..b4237694b775 100644 --- a/apps/server/src/preview/Snapshot.test.ts +++ b/apps/server/src/preview/Snapshot.test.ts @@ -4,6 +4,8 @@ import { expect, it } from "@effect/vitest"; import { EnvironmentId, PreviewAutomationNoAvailableHostError, + PreviewAutomationRequestQueueClosedError, + PreviewAutomationTimeoutError, PreviewTabId, ProviderInstanceId, ThreadId, @@ -292,6 +294,52 @@ it.effect("does not create a capture when the resolved tab is unavailable", () = }).pipe(Effect.provide(TestLayer)), ); +it.effect.each(["status", "evaluate"] as const)( + "preserves browser recovery errors during text export %s", + (operation) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const config = yield* ServerConfig.ServerConfig; + yield* fs.makeDirectory(config.browserArtifactsDir, { recursive: true }); + const existing = path.join(config.browserArtifactsDir, "existing.txt"); + yield* fs.writeFileString(existing, "keep this file"); + for (const ErrorClass of [ + PreviewAutomationNoAvailableHostError, + PreviewAutomationTimeoutError, + PreviewAutomationRequestQueueClosedError, + ]) { + const error = new ErrorClass({ + ...scope, + operation, + clientId: "text-client", + connectionId: "text-connection", + requestId: "text-request", + timeoutMs: 15_000, + }); + const page = makePage("loaded text"); + const original = page.broker!; + page.broker = PreviewAutomationBroker.of({ + ...original, + invoke: (request) => + request.operation === operation && + (operation === "status" || + (request.input as { expression: string }).expression.includes("let end")) + ? Effect.fail(error) + : original.invoke(request), + }); + + const failed = yield* Effect.flip(runExport(page)); + + expect(failed).toBe(error); + expect(failed.message).toBe(error.message); + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual(["existing.txt"]); + expect(yield* fs.readFileString(existing)).toBe("keep this file"); + assertClean(page); + } + }).pipe(Effect.provide(TestLayer)), +); + it.effect("removes partial files and the browser capture on cancellation", () => Effect.gen(function* () { const enteredChunk = yield* Deferred.make(); diff --git a/apps/server/src/preview/Snapshot.ts b/apps/server/src/preview/Snapshot.ts index 0e1a68df4fe1..de36fc7eb888 100644 --- a/apps/server/src/preview/Snapshot.ts +++ b/apps/server/src/preview/Snapshot.ts @@ -1,7 +1,7 @@ import { PreviewAutomationSnapshot, PreviewAutomationStatus, - type PreviewAutomationError, + PreviewAutomationError, type PreviewTabId, } from "@t3tools/contracts"; import * as NodeCrypto from "node:crypto"; @@ -25,6 +25,13 @@ export class PreviewTextExportError extends Schema.TaggedError + isPreviewAutomationError(cause) || isPreviewTextExportError(cause) + ? cause + : new PreviewTextExportError({ cause }); + const Capture = Schema.Struct({ totalChars: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), url: Schema.String.check(Schema.isMaxLength(2048)), @@ -113,10 +120,9 @@ const make = Effect.gen(function* () { const id = NodeCrypto.randomUUID(); let tabId = requestedTabId; if (tabId === undefined) { - const status = yield* broker.invoke({ scope, operation: "status", input: {} }).pipe( - Effect.flatMap(decodeStatus), - Effect.mapError((cause) => new PreviewTextExportError({ cause })), - ); + const status = yield* broker + .invoke({ scope, operation: "status", input: {} }) + .pipe(Effect.flatMap(decodeStatus), Effect.mapError(textExportError)); if (!status.available || status.tabId === null) { return yield* new PreviewTextExportError({ cause: "No available preview tab." }); } @@ -204,7 +210,7 @@ const make = Effect.gen(function* () { ? fileSystem.remove(textPath).pipe(Effect.ignore) : Effect.void, ), - Effect.mapError((cause) => new PreviewTextExportError({ cause })), + Effect.mapError(textExportError), ); }); From a4edfe0809d6367a00b1ff295b8059d72f643878 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:20:40 +0200 Subject: [PATCH 11/15] fix(preview): report uncertain transformed clipping --- apps/desktop/src/preview/SnapshotPage.test.ts | 140 +++++++++++++++++- apps/desktop/src/preview/SnapshotPage.ts | 66 ++++++++- docs/user/browser.md | 5 +- 3 files changed, 204 insertions(+), 7 deletions(-) diff --git a/apps/desktop/src/preview/SnapshotPage.test.ts b/apps/desktop/src/preview/SnapshotPage.test.ts index d67cb172c33f..8170cf636ee0 100644 --- a/apps/desktop/src/preview/SnapshotPage.test.ts +++ b/apps/desktop/src/preview/SnapshotPage.test.ts @@ -43,6 +43,8 @@ class PageElement { overflowY: "visible", position: "static", transform: "none", + rotate: "none", + scale: "none", perspective: "none", filter: "none", backdropFilter: "none", @@ -145,6 +147,40 @@ const fixture = (escape = (value: string) => value) => { scrollY: 4_000, location: { href: "https://example.test" }, CSS: { escape }, + DOMMatrixReadOnly: class { + transform: string; + constructor(transform: string) { + this.transform = transform; + } + toFloat64Array() { + const values = this.transform + .slice(this.transform.indexOf("(") + 1, -1) + .split(",") + .map(Number); + return new Float64Array( + values.length === 16 + ? values + : [ + values[0]!, + values[1]!, + 0, + 0, + values[2]!, + values[3]!, + 0, + 0, + 0, + 0, + 1, + 0, + values[4]!, + values[5]!, + 0, + 1, + ], + ); + } + }, HTMLElement: PageElement, Node: { ELEMENT_NODE: 1 }, NodeFilter: { SHOW_TEXT: 4, SHOW_ELEMENT: 1 }, @@ -173,6 +209,108 @@ const fixture = (escape = (value: string) => value) => { }; describe("snapshot page collector", () => { + it.each([ + ["transform", "matrix(0.707107, 0.707107, -0.707107, 0.707107, 0, 0)"], + ["transform", "matrix(1, 0, 0.5, 1, 0, 0)"], + ["transform", "matrix(-1, 0, 0, 1, 0, 0)"], + ["transform", "matrix3d(1,0,0,0,0,1,0,0,0,0,1,-0.002,0,0,0,1)"], + ["rotate", "45deg"], + ["perspective", "500px"], + ["scale", "-1 1"], + ] as const)("omits unsupported %s=%s clipping geometry", (property, value) => { + for (const effect of ["overflow", "contain", "path"]) { + for (const onAncestor of [false, true]) { + const page = fixture(); + const parent = page.element("parent", rect(0, 0)); + const scroller = page.element("scroll", rect(0, 0), parent); + (onAncestor ? parent : scroller).style[property] = value; + scroller.style.overflowY = "auto"; + scroller.scrollHeight = 1_000; + if (effect === "contain") scroller.style.contain = "paint"; + if (effect === "path") scroller.style.clipPath = "inset(0px)"; + const uncertain = page.element("uncertain", rect(10, 5, 20, 10), scroller); + page.elements.push(uncertain); + page.text("uncertain label", uncertain); + const visible = page.element("visible", rect(0, 100)); + page.elements.push(visible); + page.text("visible label", visible); + page.body.innerText = "uncertain label visible label"; + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("visible label"); + expect(snapshot.visibleText).toBe("uncertain label visible label"); + expect(snapshot.interactiveElements.map((element) => element.selector)).toEqual([ + "#visible", + ]); + expect(snapshot.scroll?.containers).toEqual([]); + expect(snapshot.scroll?.containersTruncated).toBe(true); + expect(snapshot.truncated?.viewportText).toBe(true); + expect(snapshot.truncated?.interactiveElements).toBe(true); + } + } + }); + + it.each([ + ["transform", "matrix(2, 0, 0, 3, 10, 20)"], + ["transform", "matrix(1, 0, 0, 1, 30, -10)"], + ["transform", "matrix3d(2,0,0,0,0,3,0,0,0,0,1,0,10,20,30,1)"], + ["rotate", "360deg"], + ["scale", "2 3"], + ] as const)("keeps axis-aligned %s=%s clipping", (property, value) => { + const page = fixture(); + const scroller = page.element("scroll", rect(0, 0, 200, 60)); + scroller.style[property] = value; + scroller.style.overflowY = "auto"; + scroller.style.contain = "paint"; + scroller.style.clipPath = "inset(0px)"; + scroller.scrollHeight = 1_000; + const visible = page.element("visible", rect(20, 10), scroller); + page.elements.push(visible); + page.text("visible label", visible); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("visible label"); + expect(snapshot.interactiveElements[0]?.inViewport).toBe(true); + expect(snapshot.scroll?.containers[0]?.selector).toBe("#scroll"); + expect(snapshot.scroll?.containersTruncated).toBe(false); + expect(snapshot.truncated?.viewportText).toBe(false); + expect(snapshot.truncated?.interactiveElements).toBe(false); + }); + + it("preserves positioned overflow escapes but respects transformed paint clipping", () => { + const page = fixture(); + const rotated = page.element("rotated", rect(0, 0)); + rotated.style.rotate = "45deg"; + const scroller = page.element("scroll", rect(0, 0), rotated); + scroller.style.overflowY = "auto"; + scroller.scrollHeight = 1_000; + const escaped = page.element("escaped", rect(0, 100), scroller); + escaped.style.position = "absolute"; + escaped.offsetParent = rotated; + page.elements.push(escaped); + page.text("escaped label", escaped); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("escaped label"); + expect(snapshot.interactiveElements[0]?.selector).toBe("#escaped"); + expect(snapshot.interactiveElements[0]?.inViewport).toBe(true); + scroller.style.contain = "paint"; + const painted = page.capture(); + expect(painted.viewportText).toBe(""); + expect(painted.interactiveElements).toEqual([]); + expect(painted.truncated?.interactiveElements).toBe(true); + }); + + it("keeps transformed text and controls when no geometric clipping applies", () => { + const page = fixture(); + const rotated = page.element("rotated", rect(0, 0)); + rotated.style.rotate = "45deg"; + page.elements.push(rotated); + page.text("rotated label", rotated); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("rotated label"); + expect(snapshot.interactiveElements[0]?.inViewport).toBe(true); + expect(snapshot.truncated?.viewportText).toBe(false); + expect(snapshot.truncated?.interactiveElements).toBe(false); + }); + it.each(["inset(100%)", "inset(0px 0px 100%)", "circle(50%)"])( "omits scroll containers clipped by %s", (clipPath) => { @@ -637,7 +775,7 @@ describe("snapshot page collector", () => { absolute.offsetParent = page.body; const transformed = page.element("transformed", rect(0, 0)); transformed.style.overflowY = "hidden"; - transformed.style.transform = "translateX(0px)"; + transformed.style.transform = "matrix(1, 0, 0, 1, 0, 0)"; const clipped = page.element("clipped", rect(0, 160), transformed); clipped.style.position = "fixed"; clipped.offsetParent = transformed; diff --git a/apps/desktop/src/preview/SnapshotPage.ts b/apps/desktop/src/preview/SnapshotPage.ts index edd42ea7ebc4..86fcf777fbc9 100644 --- a/apps/desktop/src/preview/SnapshotPage.ts +++ b/apps/desktop/src/preview/SnapshotPage.ts @@ -52,6 +52,8 @@ function collectSnapshotPage(): SnapshotPage { const clips = new Map(); const paintClips = new Map(); const unsupportedClips = new Set(); + const unsupportedOverflowClips = new Set(); + const transformedClips = new Map(); let unsupportedClip = false; const styles = new Map(); const styleFor = (element: Element) => { @@ -62,6 +64,37 @@ function collectSnapshotPage(): SnapshotPage { } return style; }; + const unsupportedTransform = (style: CSSStyleDeclaration) => { + if (style.perspective && style.perspective !== "none") return true; + if ( + style.scale && + style.scale !== "none" && + style.scale.split(/\s+/).some((value) => Number.parseFloat(value) <= 0) + ) + return true; + if (style.rotate && style.rotate !== "none") { + const angle = /([+-]?(?:\d+(?:\.\d*)?|\.\d+))(deg|rad|grad|turn)$/.exec(style.rotate); + const period = + angle?.[2] === "deg" + ? 360 + : angle?.[2] === "rad" + ? 2 * Math.PI + : angle?.[2] === "grad" + ? 400 + : 1; + if (!angle || Math.abs(Number(angle[1]) % period) > 1e-8) return true; + } + if (!style.transform || style.transform === "none") return false; + const values = new DOMMatrixReadOnly(style.transform).toFloat64Array(); + return ( + values[0]! <= 0 || + values[5]! <= 0 || + values[15] !== 1 || + values.some( + (value, index) => ![0, 5, 10, 12, 13, 14, 15].includes(index) && Math.abs(value) > 1e-8, + ) + ); + }; const paintClipFor = (element: Element): Bounds | null => { const pending: Element[] = []; let ancestor: Element | null = element; @@ -71,15 +104,26 @@ function collectSnapshotPage(): SnapshotPage { } let clip = ancestor ? (paintClips.get(ancestor) ?? null) : viewport; let unsupported = ancestor ? unsupportedClips.has(ancestor) : false; + let transformed = ancestor ? (transformedClips.get(ancestor) ?? false) : false; for (let index = pending.length - 1; index >= 0; index--) { const current = pending[index]!; const style = styleFor(current); + transformed ||= unsupportedTransform(style); + transformedClips.set(current, transformed); if (clip && style.display !== "contents") { const paint = /\b(paint|content|strict)\b/.test(style.contain) && !/^(inline$|table-(?!cell$|caption$)|ruby(?:$|-))/.test(style.display); const path = style.clipPath && style.clipPath !== "none"; if (paint || path) { + if (transformed) { + unsupportedClip = true; + unsupported = true; + unsupportedClips.add(current); + clip = null; + paintClips.set(current, clip); + continue; + } const rect = current.getBoundingClientRect(); const scaleX = current instanceof HTMLElement && current.offsetWidth @@ -171,6 +215,7 @@ function collectSnapshotPage(): SnapshotPage { : ancestor.parentElement; } let parentClip: Bounds | null = ancestor ? (clips.get(ancestor) ?? null) : viewport; + let unsupportedOverflow = ancestor ? unsupportedOverflowClips.has(ancestor) : false; for (let index = pending.length - 1; index >= 0; index--) { const current = pending[index]!; const style = styleFor(current); @@ -250,9 +295,15 @@ function collectSnapshotPage(): SnapshotPage { const scrollable = (/^(auto|scroll)$/.test(style.overflowX) && current.scrollWidth > clientWidth) || (/^(auto|scroll)$/.test(style.overflowY) && current.scrollHeight > clientHeight); - if (scrollable && unsupportedClips.has(current)) containersTruncated = true; + if (transformedClips.get(current)) { + unsupportedClip = true; + unsupportedOverflow = true; + } + if (scrollable && (unsupportedClips.has(current) || unsupportedOverflow)) + containersTruncated = true; if ( scrollable && + !unsupportedOverflow && parentClip && paintClip && clip.right > clip.left && @@ -274,8 +325,15 @@ function collectSnapshotPage(): SnapshotPage { } } } + if (unsupportedOverflow) unsupportedOverflowClips.add(current); parentClip = - parentClip && paintClip && clip.right > clip.left && clip.bottom > clip.top ? clip : null; + !unsupportedOverflow && + parentClip && + paintClip && + clip.right > clip.left && + clip.bottom > clip.top + ? clip + : null; clips.set(current, parentClip); } return clips.get(element)!; @@ -310,11 +368,11 @@ function collectSnapshotPage(): SnapshotPage { const rect = element.getBoundingClientRect(); if (rect.width <= 0 || rect.height <= 0) continue; paintClipFor(element); - if (unsupportedClips.has(element)) { + const current = inViewport(element, rect); + if (unsupportedClips.has(element) || unsupportedOverflowClips.has(element)) { controlsOmitted = true; continue; } - const current = inViewport(element, rect); elementCount++; const target = current ? currentElements : otherElements; if (target.length < maxElements) target.push(element); diff --git a/docs/user/browser.md b/docs/user/browser.md index de84ca54065c..52251a90f5ef 100644 --- a/docs/user/browser.md +++ b/docs/user/browser.md @@ -14,8 +14,9 @@ rendered main-page text to a UTF-8 file. The file has no total character cap. Th snapshot returns `textPath` in the environment so the agent can read the file in parts. This export does not scroll or load missing content. -Complex CSS clip shapes can cause current-view text and controls to be omitted. -The result reports these omissions. Use the screenshot to inspect those areas. +Complex CSS clip shapes and transformed clips can cause current-view text and +controls to be omitted. The result reports these omissions. Use the screenshot +to inspect those areas. On old HTML pages, a scroll box on the page body can report a size that includes scrollbar space. Use the screenshot when precise edge positions matter. From 8bf11c4d6df6c198f4b9a1ada2cae40f516b8cb0 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:21:41 +0200 Subject: [PATCH 12/15] fix(preview): keep text exports tied to their page --- apps/server/src/mcp/McpHttpServer.test.ts | 53 ++++++++- apps/server/src/preview/Snapshot.test.ts | 76 ++++++++++++- apps/server/src/preview/Snapshot.ts | 133 ++++++++++++++-------- 3 files changed, 209 insertions(+), 53 deletions(-) diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index 1b3d7f7b69d3..728f26623e0a 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -148,6 +148,7 @@ const serveTextExports = ( const switched = yield* Deferred.make(); const requests: PreviewAutomationRequest[] = []; const timers = new Set<() => void>(); + const listeners = new Map void>(); let textReads = 0; let chunks = 0; const context = NodeVM.createContext({ @@ -165,6 +166,10 @@ const serveTextExports = ( return callback; }, clearTimeout: (callback: () => void) => timers.delete(callback), + addEventListener: (event: string, callback: () => void) => listeners.set(event, callback), + removeEventListener: (event: string, callback: () => void) => { + if (listeners.get(event) === callback) listeners.delete(event); + }, }); const events = yield* broker.connect({ clientId, environmentId }); yield* Stream.runForEach(events, (event) => { @@ -243,7 +248,7 @@ const serveTextExports = ( }).pipe(Effect.forkScoped, Effect.asVoid); }).pipe(Effect.forkScoped); yield* Deferred.await(connected); - return { requests, context, timers, textReads: () => textReads }; + return { requests, context, timers, listeners, textReads: () => textReads }; }); it("normalizes empty successful notification responses to accepted", () => { @@ -633,7 +638,9 @@ it.effect.each([ const pageRequests = host.requests.filter((request) => request.operation !== "status"); expect(pageRequests.length).toBeGreaterThan(10); expect(pageRequests.every((request) => request.tabId === tabId)).toBe(true); - expect(pageRequests.at(-1)).toMatchObject({ operation: "snapshot", input: {} }); + expect(pageRequests.filter((request) => request.operation === "snapshot")).toEqual([ + expect.objectContaining({ operation: "snapshot", input: {} }), + ]); const texts = snapshot.content.filter((content) => content.type === "text"); const metadata = texts[options.save && !options.includeImage ? 0 : 1]; expect(metadata?.type === "text" ? decodeJsonText(metadata.text) : null).toMatchObject({ @@ -750,6 +757,44 @@ it.effect("reports text export failure without a successful snapshot or partial ).pipe(Effect.provide(TestLayer)), ); +it.effect.each(["navigation", "reload"] as const)( + "does not return saved text from the old page after %s before snapshot", + (change) => + Effect.scoped( + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const options: { beforeSnapshot?: Effect.Effect } = {}; + const host = yield* serveTextExports( + "mcp-export-page-change-client", + "loaded text", + options, + ); + options.beforeSnapshot = Effect.gen(function* () { + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toHaveLength(1); + if (change === "navigation") + NodeVM.runInContext("location.href = 'https://other.test/'", host.context); + else host.listeners.get("pagehide")!(); + }).pipe(Effect.orDie); + + const snapshot = yield* callSnapshot({ saveText: true }); + + expect(snapshot.isError).toBe(true); + expect(snapshot.content.every((content) => content.type === "text")).toBe(true); + expect(snapshot.structuredContent).not.toHaveProperty("textPath"); + expect(host.requests.filter((request) => request.operation === "snapshot")).toHaveLength(1); + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([]); + expect(host.timers.size).toBe(0); + expect(host.listeners.size).toBe(0); + expect( + Object.getOwnPropertyNames(host.context).filter((key) => + key.startsWith("__t3_text_export_"), + ), + ).toEqual([]); + }), + ).pipe(Effect.provide(TestLayer)), +); + it.effect("removes the completed text export when the following snapshot fails", () => Effect.scoped( Effect.gen(function* () { @@ -764,7 +809,7 @@ it.effect("removes the completed text export when the following snapshot fails", error: { _tag: "PreviewAutomationExecutionError", operation: "snapshot" }, }); expect(snapshot.content.every((content) => content.type === "text")).toBe(true); - expect(host.requests.at(-1)?.operation).toBe("snapshot"); + expect(host.requests.filter((request) => request.operation === "snapshot")).toHaveLength(1); expect(host.textReads()).toBe(1); expect(host.timers.size).toBe(0); expect( @@ -850,7 +895,7 @@ it.effect("removes the completed text export when the following snapshot is canc yield* Fiber.interrupt(fiber); expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([]); - expect(host.requests.at(-1)?.operation).toBe("snapshot"); + expect(host.requests.filter((request) => request.operation === "snapshot")).toHaveLength(1); expect(host.textReads()).toBe(1); expect(host.timers.size).toBe(0); expect( diff --git a/apps/server/src/preview/Snapshot.test.ts b/apps/server/src/preview/Snapshot.test.ts index b4237694b775..3163e7dc2f90 100644 --- a/apps/server/src/preview/Snapshot.test.ts +++ b/apps/server/src/preview/Snapshot.test.ts @@ -43,6 +43,7 @@ const makePage = (initialText: string) => { let text = initialText; let reads = 0; const timers = new Set<() => void>(); + const listeners = new Map void>(); const requests: PreviewAutomationInvokeInput[] = []; const context = NodeVM.createContext({ document: { @@ -59,10 +60,15 @@ const makePage = (initialText: string) => { return callback; }, clearTimeout: (callback: () => void) => timers.delete(callback), + addEventListener: (event: string, callback: () => void) => listeners.set(event, callback), + removeEventListener: (event: string, callback: () => void) => { + if (listeners.get(event) === callback) listeners.delete(event); + }, }); const page = { context, timers, + listeners, requests, reads: () => reads, available: true, @@ -70,6 +76,7 @@ const makePage = (initialText: string) => { text = value; }, beforeChunk: undefined as (() => void) | undefined, + beforeSnapshot: undefined as (() => void) | undefined, chunkGate: undefined as Effect.Effect | undefined, disposeGate: undefined as Effect.Effect | undefined, broker: undefined as PreviewAutomationBroker["Service"] | undefined, @@ -89,8 +96,9 @@ const makePage = (initialText: string) => { } if (request.operation === "snapshot") { expect(request.tabId).toBe(tabId); + page.beforeSnapshot?.(); return { - url: "https://example.test/page", + url: NodeVM.runInContext("location.href", context), title: "Page", loading: false, visibleText: "Page", @@ -150,6 +158,7 @@ const runExport = (page: ReturnType, target?: PreviewTabId) => ); const assertClean = (page: ReturnType) => { expect(page.timers.size).toBe(0); + expect(page.listeners.size).toBe(0); expect( Object.getOwnPropertyNames(page.context).filter((name) => name.startsWith("__t3_text_export_")), ).toEqual([]); @@ -183,7 +192,7 @@ it.effect("keeps Unicode pairs intact at chunk boundaries", () => expect(yield* fs.readFileString(saved.textPath)).toBe(text); expect(saved.sizeBytes).toBe(Buffer.byteLength(text)); expect(page.requests.some((request) => request.operation === "status")).toBe(false); - expect(page.requests.at(-1)?.operation).toBe("snapshot"); + expect(page.requests.filter((request) => request.operation === "snapshot")).toHaveLength(1); assertClean(page); }).pipe(Effect.provide(TestLayer)), ); @@ -220,6 +229,69 @@ it.effect("freezes captured text while the page changes", () => }).pipe(Effect.provide(TestLayer)), ); +it.effect.each(["navigation", "reload", "new document"] as const)( + "removes the completed text export after %s before the snapshot", + (change) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const config = yield* ServerConfig.ServerConfig; + yield* fs.makeDirectory(config.browserArtifactsDir, { recursive: true }); + const existing = path.join(config.browserArtifactsDir, "existing.txt"); + yield* fs.writeFileString(existing, "keep this file"); + const page = makePage("original loaded text"); + page.beforeSnapshot = () => { + if (change === "navigation") { + NodeVM.runInContext("location.href = 'https://other.test/'", page.context); + } else { + if (change === "reload") page.listeners.get("pagehide")!(); + NodeVM.runInContext( + "document = { body: { innerText: 'different document' } }", + page.context, + ); + } + }; + + const result = yield* Effect.result(runExport(page)); + + expect(result._tag).toBe("Failure"); + expect(page.reads()).toBe(1); + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual(["existing.txt"]); + expect(yield* fs.readFileString(existing)).toBe("keep this file"); + assertClean(page); + }).pipe(Effect.provide(TestLayer)), +); + +it.effect("validates an unchanged page with a full URL longer than the output limit", () => + Effect.gen(function* () { + const page = makePage("loaded text"); + const url = `https://example.test/page?q=${"x".repeat(70_000)}`; + NodeVM.runInContext(`location.href = ${encodeJson(url)}`, page.context); + + const saved = yield* runExport(page); + + const fs = yield* FileSystem.FileSystem; + expect(yield* fs.readFileString(saved.textPath)).toBe("loaded text"); + expect(saved.url).toBe(url.slice(0, 2048)); + expect(page.reads()).toBe(1); + assertClean(page); + }).pipe(Effect.provide(TestLayer)), +); + +it.effect("allows a text edit in the same document before the snapshot", () => + Effect.gen(function* () { + const page = makePage("original loaded text"); + page.beforeSnapshot = () => page.setText("edited loaded text"); + + const saved = yield* runExport(page); + + const fs = yield* FileSystem.FileSystem; + expect(yield* fs.readFileString(saved.textPath)).toBe("original loaded text"); + expect(page.reads()).toBe(1); + assertClean(page); + }).pipe(Effect.provide(TestLayer)), +); + it.effect.each(["navigation", "expiry"] as const)("removes partial text after %s", (failure) => Effect.gen(function* () { const page = makePage("text ".repeat(20_000)); diff --git a/apps/server/src/preview/Snapshot.ts b/apps/server/src/preview/Snapshot.ts index de36fc7eb888..632bb70a3984 100644 --- a/apps/server/src/preview/Snapshot.ts +++ b/apps/server/src/preview/Snapshot.ts @@ -40,11 +40,12 @@ const TextChunk = Schema.Struct({ text: Schema.String.check(Schema.isMaxLength(4096)), next: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), }); -const encodeKey = Schema.encodeEffect(Schema.fromJsonString(Schema.String)); +const encodePageString = Schema.encodeEffect(Schema.fromJsonString(Schema.String)); const decodeCapture = Schema.decodeUnknownEffect(Capture); const decodeTextChunk = Schema.decodeUnknownEffect(TextChunk); const decodeStatus = Schema.decodeUnknownEffect(PreviewAutomationStatus); const decodeSnapshot = Schema.decodeUnknownEffect(PreviewAutomationSnapshot); +const decodeCaptureMatch = Schema.decodeUnknownEffect(Schema.Literal(true)); export class PreviewScreenshotSaveError extends Schema.TaggedError()( "PreviewScreenshotSaveError", @@ -128,7 +129,7 @@ const make = Effect.gen(function* () { } tabId = status.tabId; } - const key = yield* encodeKey(`__t3_text_export_${id}`).pipe(Effect.orDie); + const key = yield* encodePageString(`__t3_text_export_${id}`).pipe(Effect.orDie); const textPath = path.join(config.browserArtifactsDir, `browser-text-${id}.txt`); const evaluate = (expression: string) => broker.invoke({ @@ -138,29 +139,31 @@ const make = Effect.gen(function* () { input: { expression, returnByValue: true }, updateCurrentTab: false, }); - let created = false; - return yield* Effect.scoped( - Effect.gen(function* () { - yield* Effect.addFinalizer(() => - evaluate(`(() => { + yield* Effect.addFinalizer(() => + evaluate(`(() => { const capture = globalThis[${key}]; capture?.dispose(); return true; })()`).pipe(Effect.interruptible, Effect.timeoutOption(5000), Effect.ignore), - ); + ); + let created = false; + return yield* Effect.scoped( + Effect.gen(function* () { const capture = yield* evaluate(`(() => { const text = document.body?.innerText ?? ""; let timer; const dispose = () => { clearTimeout(timer); + removeEventListener("pagehide", dispose); if (globalThis[${key}] === capture) delete globalThis[${key}]; }; const refresh = () => { clearTimeout(timer); timer = setTimeout(dispose, 300000); }; - const capture = Object.freeze({ text, url: location.href, dispose, refresh }); + const capture = Object.freeze({ text, url: location.href, document, dispose, refresh }); Object.defineProperty(globalThis, ${key}, { value: capture, configurable: true }); + addEventListener("pagehide", dispose, { once: true }); refresh(); return { totalChars: text.length, url: capture.url.slice(0, 2048) }; })()`).pipe(Effect.flatMap(decodeCapture)); @@ -179,7 +182,7 @@ const make = Effect.gen(function* () { while (offset < capture.totalChars) { const chunk = yield* evaluate(`(() => { const capture = globalThis[${key}]; - if (!capture || capture.url !== location.href) throw new Error("Text capture lost."); + if (!capture || capture.document !== document || capture.url !== location.href) throw new Error("Text capture lost."); let end = Math.min(${offset} + 4096, capture.text.length); if (end < capture.text.length) { const last = capture.text.charCodeAt(end - 1); @@ -202,7 +205,14 @@ const make = Effect.gen(function* () { offset = chunk.next; } yield* file.sync; - return { textPath, totalChars: capture.totalChars, sizeBytes, url: capture.url, tabId }; + return { + textPath, + totalChars: capture.totalChars, + sizeBytes, + url: capture.url, + tabId, + captureKey: key, + }; }), ).pipe( Effect.onExit((exit) => @@ -232,42 +242,71 @@ const make = Effect.gen(function* () { input: SnapshotInput, use: (capture: SnapshotCapture) => Effect.Effect, ) { - return yield* Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - let textExport: SnapshotCapture["textExport"]; - if (input.saveText === true) { - textExport = yield* restore(saveText(input.scope, input.tabId)); - } - const exportedText = textExport; - return yield* restore( - Effect.gen(function* () { - const tabId = exportedText?.tabId ?? input.tabId; - const snapshot = yield* broker - .invoke({ - scope: input.scope, - operation: "snapshot", - input: {}, - ...(tabId === undefined ? {} : { tabId }), - }) - .pipe(Effect.flatMap(decodeSnapshot)); - const png = new Uint8Array(Buffer.from(snapshot.screenshot.data, "base64")); - const screenshotPath = - input.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; - return yield* use({ - snapshot, - png, - ...(exportedText === undefined ? {} : { textExport: exportedText }), - ...(screenshotPath === undefined ? {} : { screenshotPath }), - }); - }), - ).pipe( - Effect.onExit((exit) => - exportedText !== undefined && exit._tag === "Failure" - ? fileSystem.remove(exportedText.textPath).pipe(Effect.ignore) - : Effect.void, - ), - ); - }), + let exportedText: (SnapshotCapture["textExport"] & { readonly captureKey: string }) | undefined; + return yield* Effect.scoped( + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + if (input.saveText === true) { + exportedText = yield* restore(saveText(input.scope, input.tabId)); + } + const capturedText = exportedText; + return yield* restore( + Effect.gen(function* () { + const tabId = capturedText?.tabId ?? input.tabId; + const snapshot = yield* broker + .invoke({ + scope: input.scope, + operation: "snapshot", + input: {}, + ...(tabId === undefined ? {} : { tabId }), + }) + .pipe(Effect.flatMap(decodeSnapshot)); + let textExport: SnapshotCapture["textExport"]; + if (capturedText !== undefined) { + const { captureKey, ...artifact } = capturedText; + for (let offset = 0; offset < Math.max(snapshot.url.length, 1); offset += 4096) { + const urlChunk = yield* encodePageString( + snapshot.url.slice(offset, offset + 4096), + ).pipe(Effect.orDie); + yield* broker + .invoke({ + scope: input.scope, + operation: "evaluate", + tabId: capturedText.tabId, + updateCurrentTab: false, + input: { + expression: `(() => { + const capture = globalThis[${captureKey}]; + if (!capture || capture.document !== document || capture.url !== location.href) return false; + capture.refresh(); + return capture.url.length === ${snapshot.url.length} && capture.url.slice(${offset}, ${offset + 4096}) === ${urlChunk}; + })()`, + returnByValue: true, + }, + }) + .pipe(Effect.flatMap(decodeCaptureMatch), Effect.mapError(textExportError)); + } + textExport = artifact; + } + const png = new Uint8Array(Buffer.from(snapshot.screenshot.data, "base64")); + const screenshotPath = + input.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; + return yield* use({ + snapshot, + png, + ...(textExport === undefined ? {} : { textExport }), + ...(screenshotPath === undefined ? {} : { screenshotPath }), + }); + }), + ); + }), + ), + ).pipe( + Effect.onExit((exit) => + exportedText !== undefined && exit._tag === "Failure" + ? fileSystem.remove(exportedText.textPath).pipe(Effect.ignore) + : Effect.void, + ), ); }); From af4cb6394dfa1f4b20ed12e4ba479eee4fdc84fe Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:31:52 +0200 Subject: [PATCH 13/15] fix(desktop): make snapshot scroll targets unique --- apps/desktop/src/preview/SnapshotPage.test.ts | 97 ++++++++++++++++++- apps/desktop/src/preview/SnapshotPage.ts | 32 +++--- 2 files changed, 115 insertions(+), 14 deletions(-) diff --git a/apps/desktop/src/preview/SnapshotPage.test.ts b/apps/desktop/src/preview/SnapshotPage.test.ts index 8170cf636ee0..0a052d7f2423 100644 --- a/apps/desktop/src/preview/SnapshotPage.test.ts +++ b/apps/desktop/src/preview/SnapshotPage.test.ts @@ -96,6 +96,7 @@ const fixture = (escape = (value: string) => value) => { root.children.push(body); root.scrollHeight = 10_000; const elements: PageElement[] = []; + const selectorMatches = new Map(); const nodes: Array = []; const element = (id: string, bounds: ReturnType, parent = body) => { const value = new PageElement(); @@ -192,7 +193,26 @@ const fixture = (escape = (value: string) => value) => { title: "Page", readyState: "complete", compatMode: "CSS1Compat", - querySelectorAll: () => elements, + querySelectorAll(selector: string) { + if (selector === "a[href],button,input,textarea,select,[role],[tabindex]") return elements; + if (selectorMatches.has(selector)) return selectorMatches.get(selector)!; + return Array.from( + new Set([root, body, ...nodes.filter((node) => node instanceof PageElement)]), + ).filter( + (element) => + (element.id && selector === "#" + element.id) || + ["name", "data-testid"].some( + (attribute) => + selector === + element.tagName.toLowerCase() + + "[" + + attribute + + "=" + + element.getAttribute(attribute) + + "]", + ), + ); + }, createRange: () => range, createTreeWalker() { let index = 0; @@ -205,10 +225,80 @@ const fixture = (escape = (value: string) => value) => { context.document.compatMode = compatMode; return NodeVM.runInNewContext(snapshotPageExpression(), context) as SnapshotPage; }; - return { root, body, elements, nodes, element, text, capture }; + return { root, body, elements, nodes, selectorMatches, element, text, capture }; }; describe("snapshot page collector", () => { + it.each(["id", "name", "data-testid"])( + "uses unique document paths for duplicate %s targets in deep branches", + (attribute) => { + const page = fixture(); + for (let branch = 0; branch < 2; branch++) { + let parent = page.element("", rect(0, 0)); + parent.tagName = "SECTION"; + for (let depth = 0; depth < 9; depth++) parent = page.element("", rect(0, 0), parent); + const scroller = page.element("", rect(0, 0), parent); + scroller.style.overflowY = "auto"; + scroller.scrollHeight = 1_000; + const control = page.element("", rect(0, 0), scroller); + control.tagName = "BUTTON"; + if (attribute === "id") scroller.id = control.id = "duplicate"; + else { + scroller.attributes.set(attribute, "duplicate"); + control.attributes.set(attribute, "duplicate"); + } + page.elements.push(control); + } + const snapshot = page.capture(); + const paths = [1, 2].map( + (branch) => + ":root > div > section:nth-of-type(" + + branch + + ") > " + + Array.from({ length: 10 }, () => "div").join(" > "), + ); + expect(snapshot.scroll?.containers.map((container) => container.selector)).toEqual(paths); + expect(snapshot.interactiveElements.map((element) => element.selector)).toEqual( + paths.map((path) => path + " > button"), + ); + }, + ); + + it.each(["id", "name", "data-testid"])("keeps short unique %s selectors", (attribute) => { + const page = fixture(); + const control = page.element("", rect(0, 0)); + control.tagName = "BUTTON"; + if (attribute === "id") control.id = "unique"; + else control.attributes.set(attribute, "unique"); + page.elements.push(control); + expect(page.capture().interactiveElements[0]?.selector).toBe( + attribute === "id" ? "#unique" : "button[" + attribute + "=unique]", + ); + }); + + it("uses a unique attribute when an id is duplicated", () => { + const page = fixture(); + page.element("duplicate", rect(0, 0)); + const control = page.element("duplicate", rect(0, 0)); + control.tagName = "BUTTON"; + control.attributes.set("data-testid", "unique"); + page.elements.push(control); + expect(page.capture().interactiveElements[0]?.selector).toBe("button[data-testid=unique]"); + }); + + it("reports omitted scroll containers when the unique ancestor path exceeds its budget", () => { + const page = fixture(); + let scroller = page.body; + for (let depth = 0; depth < 200; depth++) scroller = page.element("", rect(0, 0), scroller); + scroller.style.overflowY = "auto"; + scroller.scrollHeight = 1_000; + page.text("current label", scroller); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("current label"); + expect(snapshot.scroll?.containers).toEqual([]); + expect(snapshot.scroll?.containersTruncated).toBe(true); + }); + it.each([ ["transform", "matrix(0.707107, 0.707107, -0.707107, 0.707107, 0, 0)"], ["transform", "matrix(1, 0, 0.5, 1, 0, 0)"], @@ -525,6 +615,7 @@ describe("snapshot page collector", () => { const control = page.element("", rect(0, 0)); control.tagName = "BUTTON"; control.attributes.set(attribute, "line\nnext"); + page.selectorMatches.set(`button[${attribute}=line\\a next]`, [control]); page.elements.push(control); expect(page.capture().interactiveElements[0]?.selector).toBe( `button[${attribute}=line\\a next]`, @@ -640,7 +731,7 @@ describe("snapshot page collector", () => { expect(snapshot.viewportText).toBe(""); expect(snapshot.interactiveElements[0]?.inViewport).toBe(false); expect(snapshot.scroll?.containers[0]).toMatchObject({ - selector: "div > div", + selector: ":root > div", y: 200, height: 100, scrollHeight: 1_000, diff --git a/apps/desktop/src/preview/SnapshotPage.ts b/apps/desktop/src/preview/SnapshotPage.ts index 86fcf777fbc9..fc36863da7cd 100644 --- a/apps/desktop/src/preview/SnapshotPage.ts +++ b/apps/desktop/src/preview/SnapshotPage.ts @@ -22,30 +22,40 @@ function collectSnapshotPage(): SnapshotPage { rect.left < clip.right && rect.bottom > clip.top && rect.top < clip.bottom; + const selectorTargets = new Map(); + const uniquelyTargets = (selector: string, element: Element) => { + if (!selectorTargets.has(selector)) { + const matches = document.querySelectorAll(selector); + selectorTargets.set(selector, matches.length === 1 ? matches[0]! : null); + } + return selectorTargets.get(selector) === element; + }; const selectorFor = (element: Element): string => { - if (element.id) return "#" + CSS.escape(element.id); + if (element.id) { + const selector = "#" + CSS.escape(element.id); + if (uniquelyTargets(selector, element)) return selector; + } for (const attribute of ["data-testid", "name"]) { const value = element.getAttribute(attribute); - if (value) - return element.tagName.toLowerCase() + "[" + attribute + "=" + CSS.escape(value) + "]"; + if (value) { + const selector = + element.tagName.toLowerCase() + "[" + attribute + "=" + CSS.escape(value) + "]"; + if (uniquelyTargets(selector, element)) return selector; + } } const parts: string[] = []; - for ( - let current: Element | null = element; - current && parts.length < 8; - current = current.parentElement - ) { + for (let current: Element | null = element; current; current = current.parentElement) { const siblings = current.parentElement ? Array.from(current.parentElement.children).filter( (child) => child.tagName === current.tagName, ) : []; - parts.unshift( - current.tagName.toLowerCase() + + parts.push( + (current === document.documentElement ? ":root" : current.tagName.toLowerCase()) + (siblings.length > 1 ? ":nth-of-type(" + (siblings.indexOf(current) + 1) + ")" : ""), ); } - return parts.join(" > "); + return parts.toReversed().join(" > "); }; const containers: Array["containers"][number]> = []; let containersTruncated = false; From 3651b058338c97af608b361a40e540f665657009 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:24:35 +0200 Subject: [PATCH 14/15] fix(preview): read loaded page text from memory --- apps/server/src/mcp/McpHttpServer.test.ts | 390 ++++++---- apps/server/src/mcp/McpHttpServer.ts | 164 ++-- .../src/mcp/toolkits/preview/tools.test.ts | 29 + apps/server/src/mcp/toolkits/preview/tools.ts | 54 +- apps/server/src/preview/Snapshot.test.ts | 705 +++++++++++------- apps/server/src/preview/Snapshot.ts | 372 ++++----- docs/user/browser.md | 13 +- 7 files changed, 1069 insertions(+), 658 deletions(-) diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index 728f26623e0a..a9f34d9229d2 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -132,12 +132,22 @@ const callSnapshot = (args: Record) => ); }); -const serveTextExports = ( +const callReadText = (args: Record, scope = invocation) => + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + return yield* server + .callTool({ name: "preview_read_text", arguments: args }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, scope), + Effect.provideService(McpSchema.McpServerClient, client), + ); + }); + +const serveTextCaptures = ( clientId: string, text: string, options: { switchTab?: boolean; - navigate?: boolean; snapshotFailure?: boolean; beforeSnapshot?: Effect.Effect; } = {}, @@ -150,8 +160,8 @@ const serveTextExports = ( const timers = new Set<() => void>(); const listeners = new Map void>(); let textReads = 0; - let chunks = 0; const context = NodeVM.createContext({ + TextEncoder, document: { body: { get innerText() { @@ -206,9 +216,6 @@ const serveTextExports = ( } else if (request.operation === "evaluate") { if (options.switchTab) yield* Deferred.await(switched); const expression = (request.input as { expression: string }).expression; - if (options.navigate && expression.includes("let end") && ++chunks === 2) { - NodeVM.runInContext("location.href = 'https://other.test/'", context); - } try { result = NodeVM.runInContext(expression, context); } catch { @@ -350,7 +357,7 @@ it.effect.each([ ).pipe(Effect.provide(TestLayer)), ); -it.effect.each([{}, { saveText: true }])( +it.effect.each([{}, { captureText: true }])( "tells the agent how to fall back when no desktop app can run the snapshot %j", (args) => Effect.gen(function* () { @@ -488,7 +495,7 @@ it.effect.each([ ).pipe(Effect.provide(TestLayer)), ); -it.effect.each(["includeImage", "saveText"])( +it.effect.each(["includeImage", "captureText"])( "rejects non-boolean snapshot %s options before selecting a browser host", (option) => Effect.gen(function* () { @@ -606,85 +613,140 @@ it.effect.each([ { save: false, includeImage: false }, { save: true, includeImage: false }, { save: true, includeImage: true }, -])("exports complete loaded text with snapshot options %j", (options) => +])("reads complete loaded text from memory with snapshot options %j", (options) => Effect.scoped( Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const loadedText = "a".repeat(4095) + "πŸ˜€δΈ­ζ–‡\n".repeat(12_000) + "Offscreen loaded end"; - const host = yield* serveTextExports("mcp-text-export-client", loadedText, { + const host = yield* serveTextCaptures("mcp-text-capture-client", loadedText, { switchTab: true, }); - const snapshot = yield* callSnapshot({ saveText: true, ...options }); + const snapshot = yield* callSnapshot({ captureText: true, ...options }); expect(snapshot.isError).toBe(false); - const saved = snapshot.structuredContent as { - readonly textPath: string; + const captured = snapshot.structuredContent as { + readonly textCaptureId: string; + readonly textTabId: string; readonly screenshotPath?: string; }; - expect(saved).toMatchObject({ - textPath: expect.any(String), + expect(captured).toMatchObject({ + textCaptureId: expect.any(String), textChars: loadedText.length, - textBytes: Buffer.byteLength(loadedText, "utf8"), textUrl: snapshotResult.url, + textTabId: tabId, }); - expect(path.dirname(saved.textPath)).toBe(config.browserArtifactsDir); - expect(path.basename(saved.textPath)).toMatch(/^browser-text-[0-9a-f-]+\.txt$/); - expect(yield* fs.readFileString(saved.textPath)).toBe(loadedText); + expect(captured).not.toHaveProperty("textPath"); + expect(captured).not.toHaveProperty("textBytes"); expect(host.textReads()).toBe(1); - expect(host.requests.filter((request) => request.operation === "status")).toHaveLength(2); - const pageRequests = host.requests.filter((request) => request.operation !== "status"); - expect(pageRequests.length).toBeGreaterThan(10); + const beforeRead = [...host.requests]; + expect(beforeRead.filter((request) => request.operation === "status")).toHaveLength(2); + const pageRequests = beforeRead.filter((request) => request.operation !== "status"); expect(pageRequests.every((request) => request.tabId === tabId)).toBe(true); expect(pageRequests.filter((request) => request.operation === "snapshot")).toEqual([ expect.objectContaining({ operation: "snapshot", input: {} }), ]); + expect(beforeRead.length).toBeLessThan(8); const texts = snapshot.content.filter((content) => content.type === "text"); const metadata = texts[options.save && !options.includeImage ? 0 : 1]; expect(metadata?.type === "text" ? decodeJsonText(metadata.text) : null).toMatchObject({ - textPath: saved.textPath, + textCaptureId: captured.textCaptureId, textChars: loadedText.length, - textBytes: Buffer.byteLength(loadedText, "utf8"), textUrl: snapshotResult.url, + textTabId: tabId, }); expect( metadata?.type === "text" ? Buffer.byteLength(metadata.text, "utf8") : Infinity, ).toBeLessThanOrEqual(McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES); - expect(encodeJsonText(saved)).not.toContain("Offscreen loaded end"); + expect(encodeJsonText(captured)).not.toContain("Offscreen loaded end"); expect(snapshot.content.some((content) => content.type === "image")).toBe( options.includeImage, ); if (options.save) { - expect(path.dirname(saved.screenshotPath!)).toBe(config.browserArtifactsDir); - expect(Buffer.from(yield* fs.readFile(saved.screenshotPath!)).toString()).toBe("png"); - } else expect(saved).not.toHaveProperty("screenshotPath"); + expect(path.dirname(captured.screenshotPath!)).toBe(config.browserArtifactsDir); + expect(Buffer.from(yield* fs.readFile(captured.screenshotPath!)).toString()).toBe("png"); + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([ + path.basename(captured.screenshotPath!), + ]); + } else { + expect(captured).not.toHaveProperty("screenshotPath"); + expect(yield* fs.exists(config.browserArtifactsDir)).toBe(false); + } if (options.save && !options.includeImage) { - expect(Object.keys(saved).sort()).toEqual([ + expect(Object.keys(captured).sort()).toEqual([ "screenshotPath", - "textBytes", + "textCaptureId", "textChars", - "textPath", + "textTabId", "textUrl", "url", ]); expect(snapshot.content).toHaveLength(1); } - expect(host.timers.size).toBe(0); + + let offset = 0; + let complete = ""; + while (offset < loadedText.length) { + const result = yield* callReadText({ + captureId: captured.textCaptureId, + tabId: captured.textTabId, + offset, + }); + expect(result.isError).toBe(false); + const chunk = result.structuredContent as { + readonly text: string; + readonly nextOffset: number; + readonly totalChars: number; + readonly done: boolean; + readonly released: boolean; + }; + expect(chunk.text.length).toBeLessThanOrEqual(4096); + expect(chunk.nextOffset).toBe(offset + chunk.text.length); + expect(chunk.nextOffset).toBeGreaterThan(offset); + expect(chunk.totalChars).toBe(loadedText.length); + expect(chunk.done).toBe(chunk.nextOffset === loadedText.length); + expect(chunk.released).toBe(false); + expect(chunk.text.isWellFormed()).toBe(true); + const [content] = result.content; + expect(content?.type === "text" ? decodeJsonText(content.text) : null).toEqual(chunk); + expect(Buffer.byteLength(encodeJsonText(chunk))).toBeLessThan(25_000); + complete += chunk.text; + offset = chunk.nextOffset; + } + expect(complete).toBe(loadedText); + expect(host.textReads()).toBe(1); expect( - Object.getOwnPropertyNames(host.context).filter((key) => - key.startsWith("__t3_text_export_"), - ), - ).toEqual([]); + host.requests + .slice(beforeRead.length) + .every((request) => request.operation === "evaluate" && request.tabId === tabId), + ).toBe(true); + expect(host.timers.size).toBe(1); + + const released = yield* callReadText({ + captureId: captured.textCaptureId, + tabId: captured.textTabId, + release: true, + }); + expect(released.isError).toBe(false); + expect(released.structuredContent).toEqual({ + text: "", + nextOffset: 0, + totalChars: loadedText.length, + done: true, + released: true, + }); + expect(host.timers.size).toBe(0); + expect(host.listeners.size).toBe(0); }), ).pipe(Effect.provide(TestLayer)), ); -it.effect.each([{}, { saveText: false }])("keeps text export opt-in %j", (options) => +it.effect.each([{}, { captureText: false }])("keeps full-text capture opt-in %j", (options) => Effect.scoped( Effect.gen(function* () { - const host = yield* serveTextExports("mcp-text-not-requested-client", "Loaded text"); + const host = yield* serveTextCaptures("mcp-text-not-requested-client", "Loaded text"); const snapshot = yield* callSnapshot({ ...options, includeImage: false }); @@ -692,7 +754,7 @@ it.effect.each([{}, { saveText: false }])("keeps text export opt-in %j", (option expect(host.requests).toHaveLength(1); expect(host.requests[0]).toMatchObject({ operation: "snapshot", input: {} }); expect(host.textReads()).toBe(0); - expect(snapshot.structuredContent).not.toHaveProperty("textPath"); + expect(snapshot.structuredContent).not.toHaveProperty("textCaptureId"); const config = yield* ServerConfig.ServerConfig; const fs = yield* FileSystem.FileSystem; expect(yield* fs.exists(config.browserArtifactsDir)).toBe(false); @@ -700,12 +762,12 @@ it.effect.each([{}, { saveText: false }])("keeps text export opt-in %j", (option ).pipe(Effect.provide(TestLayer)), ); -it.effect("rejects an invalid snapshot before starting a requested text export", () => +it.effect("rejects an invalid snapshot before starting a requested text capture", () => Effect.scoped( Effect.gen(function* () { - const host = yield* serveTextExports("mcp-invalid-text-export-client", "Loaded text"); + const host = yield* serveTextCaptures("mcp-invalid-text-capture-client", "Loaded text"); - const snapshot = yield* callSnapshot({ saveText: true, includeImage: "wrong" }); + const snapshot = yield* callSnapshot({ captureText: true, includeImage: "wrong" }); expect(snapshot.isError).toBe(true); expect(host.requests).toEqual([]); @@ -717,92 +779,47 @@ it.effect("rejects an invalid snapshot before starting a requested text export", ).pipe(Effect.provide(TestLayer)), ); -it.effect("reports text export failure without a successful snapshot or partial files", () => - Effect.scoped( - Effect.gen(function* () { - const host = yield* serveTextExports( - "mcp-text-failure-client", - "private text ".repeat(10_000), - { - navigate: true, - }, - ); - - const snapshot = yield* callSnapshot({ saveText: true, save: true }); - - const message = "Preview automation evaluate failed on client mcp-text-failure-client."; - expect(snapshot.isError).toBe(true); - expect(snapshot.content).toEqual([ - { type: "text", text: `Preview snapshot failed: ${message}` }, - ]); - expect(snapshot.structuredContent).toEqual({ - error: { - _tag: "PreviewAutomationExecutionError", - operation: "snapshot", - failureCount: 1, - message, - }, - }); - expect(host.requests.some((request) => request.operation === "snapshot")).toBe(false); - expect(host.timers.size).toBe(0); - expect( - Object.getOwnPropertyNames(host.context).filter((key) => - key.startsWith("__t3_text_export_"), - ), - ).toEqual([]); - const config = yield* ServerConfig.ServerConfig; - const fs = yield* FileSystem.FileSystem; - expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([]); - }), - ).pipe(Effect.provide(TestLayer)), -); - it.effect.each(["navigation", "reload"] as const)( - "does not return saved text from the old page after %s before snapshot", + "does not return captured text from the old page after %s before snapshot", (change) => Effect.scoped( Effect.gen(function* () { - const config = yield* ServerConfig.ServerConfig; - const fs = yield* FileSystem.FileSystem; const options: { beforeSnapshot?: Effect.Effect } = {}; - const host = yield* serveTextExports( - "mcp-export-page-change-client", + const host = yield* serveTextCaptures( + "mcp-capture-page-change-client", "loaded text", options, ); - options.beforeSnapshot = Effect.gen(function* () { - expect(yield* fs.readDirectory(config.browserArtifactsDir)).toHaveLength(1); + options.beforeSnapshot = Effect.sync(() => { + expect(host.timers.size).toBe(1); if (change === "navigation") NodeVM.runInContext("location.href = 'https://other.test/'", host.context); else host.listeners.get("pagehide")!(); - }).pipe(Effect.orDie); + }); - const snapshot = yield* callSnapshot({ saveText: true }); + const snapshot = yield* callSnapshot({ captureText: true }); expect(snapshot.isError).toBe(true); expect(snapshot.content.every((content) => content.type === "text")).toBe(true); - expect(snapshot.structuredContent).not.toHaveProperty("textPath"); + expect(snapshot.structuredContent).not.toHaveProperty("textCaptureId"); expect(host.requests.filter((request) => request.operation === "snapshot")).toHaveLength(1); - expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([]); expect(host.timers.size).toBe(0); expect(host.listeners.size).toBe(0); - expect( - Object.getOwnPropertyNames(host.context).filter((key) => - key.startsWith("__t3_text_export_"), - ), - ).toEqual([]); + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + expect(yield* fs.exists(config.browserArtifactsDir)).toBe(false); }), ).pipe(Effect.provide(TestLayer)), ); -it.effect("removes the completed text export when the following snapshot fails", () => +it.effect("releases the text capture when the following snapshot fails", () => Effect.scoped( Effect.gen(function* () { - const host = yield* serveTextExports("mcp-export-snapshot-failure-client", "loaded text", { + const host = yield* serveTextCaptures("mcp-capture-snapshot-failure-client", "loaded text", { snapshotFailure: true, }); - const snapshot = yield* callSnapshot({ saveText: true }); + const snapshot = yield* callSnapshot({ captureText: true }); expect(snapshot.isError).toBe(true); expect(snapshot.structuredContent).toMatchObject({ @@ -812,30 +829,23 @@ it.effect("removes the completed text export when the following snapshot fails", expect(host.requests.filter((request) => request.operation === "snapshot")).toHaveLength(1); expect(host.textReads()).toBe(1); expect(host.timers.size).toBe(0); - expect( - Object.getOwnPropertyNames(host.context).filter((key) => - key.startsWith("__t3_text_export_"), - ), - ).toEqual([]); + expect(host.listeners.size).toBe(0); const config = yield* ServerConfig.ServerConfig; const fs = yield* FileSystem.FileSystem; - expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([]); + expect(yield* fs.exists(config.browserArtifactsDir)).toBe(false); }), ).pipe(Effect.provide(TestLayer)), ); -it.effect("removes only its completed text export when saving the PNG fails", () => +it.effect("releases its text capture when saving the PNG fails", () => Effect.scoped( Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - yield* fs.makeDirectory(config.browserArtifactsDir, { recursive: true }); - const existing = path.join(config.browserArtifactsDir, "existing.txt"); - yield* fs.writeFileString(existing, "keep this file"); - const host = yield* serveTextExports("mcp-export-png-failure-client", "loaded text"); + yield* fs.writeFileString(config.browserArtifactsDir, "keep this file"); + const host = yield* serveTextCaptures("mcp-capture-png-failure-client", "loaded text"); - const snapshot = yield* callSnapshot({ saveText: true, save: true }); + const snapshot = yield* callSnapshot({ captureText: true, save: true }); expect(snapshot.isError).toBe(true); expect(snapshot.structuredContent).toMatchObject({ @@ -844,65 +854,133 @@ it.effect("removes only its completed text export when saving the PNG fails", () expect(snapshot.content.every((content) => content.type === "text")).toBe(true); expect(host.textReads()).toBe(1); expect(host.timers.size).toBe(0); - expect( - Object.getOwnPropertyNames(host.context).filter((key) => - key.startsWith("__t3_text_export_"), - ), - ).toEqual([]); - expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual(["existing.txt"]); - expect(yield* fs.readFileString(existing)).toBe("keep this file"); + expect(host.listeners.size).toBe(0); + expect(yield* fs.readFileString(config.browserArtifactsDir)).toBe("keep this file"); }), - ).pipe( - Effect.provide( - PreviewTestLayer.pipe( - Layer.provideMerge( - Layer.effect( - FileSystem.FileSystem, - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - return FileSystem.FileSystem.of({ - ...fs, - writeFile: (filePath, bytes, options) => - filePath.endsWith(".png") - ? fs.writeFileString(path.dirname(filePath), "fails on a directory") - : fs.writeFile(filePath, bytes, options), - }); - }), - ), - ), - Layer.provideMerge(NodeServices.layer), - ), - ), - ), + ).pipe(Effect.provide(TestLayer)), ); -it.effect("removes the completed text export when the following snapshot is cancelled", () => +it.effect("releases the text capture when the following snapshot is cancelled", () => Effect.scoped( Effect.gen(function* () { const enteredSnapshot = yield* Deferred.make(); - const host = yield* serveTextExports("mcp-export-snapshot-cancel-client", "loaded text", { + const host = yield* serveTextCaptures("mcp-capture-snapshot-cancel-client", "loaded text", { beforeSnapshot: Deferred.succeed(enteredSnapshot, undefined).pipe( Effect.andThen(Effect.never), ), }); - const fiber = yield* Effect.forkChild(callSnapshot({ saveText: true })); + const fiber = yield* Effect.forkChild(callSnapshot({ captureText: true })); yield* Deferred.await(enteredSnapshot); - const config = yield* ServerConfig.ServerConfig; - const fs = yield* FileSystem.FileSystem; - expect(yield* fs.readDirectory(config.browserArtifactsDir)).toHaveLength(1); + expect(host.timers.size).toBe(1); yield* Fiber.interrupt(fiber); - expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([]); expect(host.requests.filter((request) => request.operation === "snapshot")).toHaveLength(1); expect(host.textReads()).toBe(1); expect(host.timers.size).toBe(0); - expect( - Object.getOwnPropertyNames(host.context).filter((key) => - key.startsWith("__t3_text_export_"), - ), - ).toEqual([]); + expect(host.listeners.size).toBe(0); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect.each([ + {}, + { tabId }, + { captureId: "capture" }, + { tabId, captureId: "" }, + { tabId, captureId: 1 }, + { tabId, captureId: "capture", offset: -1 }, + { tabId, captureId: "capture", offset: 0.5 }, + { tabId, captureId: "capture", offset: "0" }, + { tabId, captureId: "capture", release: "true" }, +])("rejects invalid read-text parameters before browser dispatch %j", (args) => + Effect.scoped( + Effect.gen(function* () { + const host = yield* serveTextCaptures("mcp-invalid-text-read-client", "Loaded text"); + + const result = yield* callReadText(args); + + expect(result.isError).toBe(true); + expect(host.requests).toEqual([]); + expect(encodeJsonText(result)).not.toContain("Loaded text"); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect("requires the preview capability before reading captured text", () => + Effect.scoped( + Effect.gen(function* () { + const host = yield* serveTextCaptures("mcp-denied-text-read-client", "Loaded text"); + const server = yield* McpServer.McpServer; + + const result = yield* server + .callTool({ name: "preview_read_text", arguments: { tabId, captureId: "capture" } }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, { + ...invocation, + capabilities: new Set(), + }), + Effect.provideService(McpSchema.McpServerClient, client), + ); + + expect(result.isError).toBe(true); + expect(encodeJsonText(result)).toContain("preview capability"); + expect(host.requests).toEqual([]); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect.each(["navigation", "reload", "expiry", "replacement", "release"] as const)( + "rejects a stale text capture after %s with useful guidance and no page text", + (change) => + Effect.scoped( + Effect.gen(function* () { + const host = yield* serveTextCaptures("mcp-stale-text-read-client", "private loaded text"); + const snapshot = yield* callSnapshot({ captureText: true }); + const captured = snapshot.structuredContent as { readonly textCaptureId: string }; + const args = { tabId, captureId: captured.textCaptureId }; + if (change === "navigation") + NodeVM.runInContext("location.href = 'https://other.test/'", host.context); + else if (change === "reload") host.listeners.get("pagehide")!(); + else if (change === "expiry") [...host.timers][0]!(); + else if (change === "replacement") { + const newer = yield* callSnapshot({ captureText: true }); + expect(newer.isError).toBe(false); + expect(host.timers.size).toBe(1); + } else yield* callReadText({ ...args, release: true }); + + const result = yield* callReadText(args); + + expect(result.isError).toBe(true); + expect(encodeJsonText(result)).toContain("captureText"); + expect(encodeJsonText(result)).not.toContain("private loaded text"); + if (change !== "replacement") { + expect(host.timers.size).toBe(0); + expect(host.listeners.size).toBe(0); + } + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect("does not expose another provider session's text capture", () => + Effect.scoped( + Effect.gen(function* () { + const host = yield* serveTextCaptures("mcp-isolated-text-read-client", "private loaded text"); + const snapshot = yield* callSnapshot({ captureText: true }); + const captured = snapshot.structuredContent as { readonly textCaptureId: string }; + const args = { tabId, captureId: captured.textCaptureId }; + + const foreign = yield* callReadText(args, { + ...invocation, + providerSessionId: "different-session", + }); + + expect(foreign.isError).toBe(true); + expect(encodeJsonText(foreign)).not.toContain("private loaded text"); + const own = yield* callReadText(args); + expect(own.isError).toBe(false); + expect(own.structuredContent).toMatchObject({ text: "private loaded text" }); + expect(host.timers.size).toBe(1); }), ).pipe(Effect.provide(TestLayer)), ); diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index aba0032fcd6f..bf3f87570b56 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -31,7 +31,11 @@ import * as PreviewSnapshot from "../preview/Snapshot.ts"; import { OrchestratorToolkitHandlersLive } from "./toolkits/orchestrator/handlers.ts"; import { OrchestratorToolkit } from "./toolkits/orchestrator/tools.ts"; import { PreviewStandardToolkitHandlersLive } from "./toolkits/preview/handlers.ts"; -import { PreviewSnapshotTool, PreviewStandardToolkit } from "./toolkits/preview/tools.ts"; +import { + PreviewReadTextTool, + PreviewSnapshotTool, + PreviewStandardToolkit, +} from "./toolkits/preview/tools.ts"; import { WorktreeToolkitHandlersLive } from "./toolkits/worktree/handlers.ts"; import { WorktreeToolkit } from "./toolkits/worktree/tools.ts"; import * as WorktreeMcpService from "./WorktreeMcpService.ts"; @@ -380,47 +384,55 @@ const boundSnapshotMetadata = (metadata: SnapshotMetadata) => { }; const isPreviewAutomationError = Schema.is(PreviewAutomationError); -const isPreviewTextExportError = Schema.is(PreviewSnapshot.PreviewTextExportError); +const isPreviewTextCaptureError = Schema.is(PreviewSnapshot.PreviewTextCaptureError); const decodePreviewSnapshotInput = Schema.decodeUnknownEffect(PreviewSnapshotTool.parametersSchema); +const decodePreviewReadTextInput = Schema.decodeUnknownEffect(PreviewReadTextTool.parametersSchema); -const previewSnapshotFailure = (cause: Cause.Cause) => { - if (Cause.hasInterrupts(cause) || cause.reasons.some(Cause.isDieReason)) { - return Effect.failCause(cause).pipe(Effect.orDie); - } - const failures = cause.reasons.filter(Cause.isFailReason); - const firstFailure = failures[0]?.error; - const errorTag = - typeof firstFailure === "object" && - firstFailure !== null && - "_tag" in firstFailure && - typeof firstFailure._tag === "string" - ? firstFailure._tag - : "PreviewSnapshotError"; - // Preview errors build their message on the server, never from page output, - // and it tells the agent what to do next, such as falling back to a shell browser. - const message = - isPreviewAutomationError(firstFailure) || isPreviewTextExportError(firstFailure) - ? firstFailure.message - : undefined; - const result = new McpSchema.CallToolResult({ - isError: true, - structuredContent: { - error: { - _tag: errorTag, - operation: "snapshot", - failureCount: failures.length, - ...(message === undefined ? {} : { message }), +const previewFailure = + (operation: "snapshot" | "readText") => + (cause: Cause.Cause) => { + if (Cause.hasInterrupts(cause) || cause.reasons.some(Cause.isDieReason)) { + return Effect.failCause(cause).pipe(Effect.orDie); + } + const failures = cause.reasons.filter(Cause.isFailReason); + const firstFailure = failures[0]?.error; + const errorTag = + typeof firstFailure === "object" && + firstFailure !== null && + "_tag" in firstFailure && + typeof firstFailure._tag === "string" + ? firstFailure._tag + : "PreviewSnapshotError"; + // Preview errors build their message on the server, never from page output, + // and it tells the agent what to do next, such as falling back to a shell browser. + const message = + isPreviewAutomationError(firstFailure) || isPreviewTextCaptureError(firstFailure) + ? firstFailure.message + : undefined; + const result = new McpSchema.CallToolResult({ + isError: true, + structuredContent: { + error: { + _tag: errorTag, + operation, + failureCount: failures.length, + ...(message === undefined ? {} : { message }), + }, }, - }, - // Some clients show only the text content and others only structuredContent, so both carry it. - content: [{ type: "text", text: `Preview snapshot failed: ${message ?? `${errorTag}.`}` }], - }); - return Effect.logWarning("preview snapshot failed", { - operation: "snapshot", - errorTag, - failureCount: failures.length, - }).pipe(Effect.as(result)); -}; + // Some clients show only the text content and others only structuredContent, so both carry it. + content: [ + { + type: "text", + text: `Preview ${operation === "snapshot" ? "snapshot" : "text read"} failed: ${message ?? `${errorTag}.`}`, + }, + ], + }); + return Effect.logWarning(`preview ${operation} failed`, { + operation, + errorTag, + failureCount: failures.length, + }).pipe(Effect.as(result)); + }; const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot")(function* () { const server = yield* McpServer.McpServer; @@ -465,17 +477,17 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot Effect.flatMap((scope) => snapshots.withSnapshot( { scope, ...input }, - ({ snapshot, png, textExport, screenshotPath }) => + ({ snapshot, png, textCapture, screenshotPath }) => Effect.sync(() => { const { screenshot, ...page } = snapshot; const textArtifact = - textExport === undefined + textCapture === undefined ? {} : { - textPath: textExport.textPath, - textChars: textExport.totalChars, - textBytes: textExport.sizeBytes, - textUrl: textExport.url, + textCaptureId: textCapture.captureId, + textChars: textCapture.totalChars, + textUrl: textCapture.url, + textTabId: textCapture.tabId, }; if (screenshotPath !== undefined && input.includeImage === false) { // The agent only wants a file to show the user. The url keeps the site icon on the tool row. @@ -541,7 +553,67 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot ), ), Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), - Effect.matchCauseEffect({ onFailure: previewSnapshotFailure, onSuccess: Effect.succeed }), + Effect.matchCauseEffect({ + onFailure: previewFailure("snapshot"), + onSuccess: Effect.succeed, + }), + ); + }), + }); + const readTool = PreviewReadTextTool; + yield* server.addTool({ + tool: new McpSchema.Tool({ + name: readTool.name, + description: Tool.getDescription(readTool), + inputSchema: Tool.getJsonSchema(readTool), + outputSchema: Tool.getJsonSchemaFromSchema(readTool.successSchema), + annotations: { + ...Context.getOption(readTool.annotations, Tool.Title).pipe( + Option.map((title) => ({ title })), + Option.getOrUndefined, + ), + readOnlyHint: Context.get(readTool.annotations, Tool.Readonly), + destructiveHint: Context.get(readTool.annotations, Tool.Destructive), + idempotentHint: Context.get(readTool.annotations, Tool.Idempotent), + openWorldHint: Context.get(readTool.annotations, Tool.OpenWorld), + }, + }), + annotations: readTool.annotations, + handle: (payload) => + Effect.withFiber((fiber) => { + const invocation = Context.getUnsafe( + fiber.context, + McpInvocationContext.McpInvocationContext, + ); + return decodePreviewReadTextInput(payload ?? {}).pipe( + Effect.mapError((cause) => + AiError.make({ + module: "Toolkit", + method: "preview_read_text.handle", + reason: new AiError.ToolParameterValidationError({ + toolName: "preview_read_text", + description: cause.message, + }), + }), + ), + Effect.flatMap((input) => + McpInvocationContext.requireMcpCapability("preview").pipe( + Effect.flatMap((scope) => snapshots.readText({ scope, ...input })), + ), + ), + Effect.map( + (result) => + new McpSchema.CallToolResult({ + isError: false, + structuredContent: result, + content: [{ type: "text", text: encodeJsonText(result) }], + }), + ), + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + Effect.matchCauseEffect({ + onFailure: previewFailure("readText"), + onSuccess: Effect.succeed, + }), ); }), }); diff --git a/apps/server/src/mcp/toolkits/preview/tools.test.ts b/apps/server/src/mcp/toolkits/preview/tools.test.ts index 0937fba72c60..aa77cc177f62 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.test.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.test.ts @@ -1,4 +1,5 @@ import { expect, it } from "@effect/vitest"; +import * as Context from "effect/Context"; import { Tool } from "effect/unstable/ai"; import { PreviewToolkit } from "./tools.ts"; @@ -75,3 +76,31 @@ it("exports exact object result schemas for preview actions", () => { }); } }); + +it("keeps text capture opt-in and requires its tab and capture identifiers for reads", () => { + expect(Context.get(PreviewToolkit.tools.preview_snapshot.annotations, Tool.Idempotent)).toBe( + false, + ); + const snapshotSchema = Tool.getJsonSchema(PreviewToolkit.tools.preview_snapshot) as { + readonly properties: Readonly>; + readonly required?: ReadonlyArray; + }; + expect(snapshotSchema.properties.captureText).toBeDefined(); + expect(snapshotSchema.properties.saveText).toBeUndefined(); + expect(snapshotSchema.required ?? []).not.toContain("captureText"); + const readSchema = Tool.getJsonSchema(PreviewToolkit.tools.preview_read_text) as { + readonly required: ReadonlyArray; + }; + expect(readSchema.required.toSorted()).toEqual(["captureId", "tabId"]); + const resultSchema = Tool.getJsonSchemaFromSchema( + PreviewToolkit.tools.preview_read_text.successSchema, + ) as { readonly type: string; readonly required: ReadonlyArray }; + expect(resultSchema.type).toBe("object"); + expect(resultSchema.required.toSorted()).toEqual([ + "done", + "nextOffset", + "released", + "text", + "totalChars", + ]); +}); diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 5acb2599d299..7585a648d80d 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -18,6 +18,7 @@ import { PreviewAutomationTabTargetInput, PreviewAutomationTypeInput, PreviewAutomationWaitForInput, + PreviewTabId, } from "@t3tools/contracts"; import * as Schema from "effect/Schema"; import * as FileSystem from "effect/FileSystem"; @@ -116,10 +117,10 @@ const PreviewSetAppearanceTool = safeBrowserTool( .annotate(Tool.Idempotent, true), ); -export const PreviewSnapshotTool = readonlyBrowserTool( +export const PreviewSnapshotTool = safeBrowserTool( Tool.make("preview_snapshot", { description: - "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use the current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. When the desktop host supports it, viewportText describes the current view, scroll reports page and visible container scroll positions, and inViewport elements come first. visibleText also includes rendered text outside the view; content not yet loaded requires scrolling. The text is capped near 20 KB, keeps current-view text ahead of offscreen page text, and lists what it omitted. Set saveText=true to save all loaded, rendered main-page text to a UTF-8 file without a total character cap; textPath points to the file in the environment. This does not scroll or load missing text, and excludes embedded frames and shadow DOM. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; with includeImage=false, save=true returns only the url and saved artifact paths. Embed screenshotPath in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", + "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use the current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. When the desktop host supports it, viewportText describes the current view, scroll reports page and visible container scroll positions, and inViewport elements come first. visibleText also includes rendered text outside the view; content not yet loaded requires scrolling. The text is capped near 20 KB, keeps current-view text ahead of offscreen page text, and lists what it omitted. Set captureText=true to hold all loaded, rendered main-page text in temporary browser memory without a total character cap; use textCaptureId and textTabId with preview_read_text to read it in small parts. No text file is created. The capture expires after five idle minutes, a page change, or a replacement capture. This does not scroll or load missing text, and excludes embedded frames and shadow DOM. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; with includeImage=false, save=true returns only the url and saved artifact details. Embed screenshotPath in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", parameters: Schema.Struct({ ...PreviewAutomationTabTargetInput.fields, includeImage: Schema.optional( @@ -134,17 +135,59 @@ export const PreviewSnapshotTool = readonlyBrowserTool( "Write the screenshot PNG to disk and return its absolute path as screenshotPath. With includeImage=false, return only the url and saved artifact details. Defaults to false.", }), ), - saveText: Schema.optional( + captureText: Schema.optional( Schema.Boolean.annotate({ description: - "Save all loaded, rendered main-page text to a UTF-8 file and return textPath, textChars, and textBytes. No total character cap; snapshot output stays bounded. Does not scroll or load missing content. Defaults to false.", + "Keep all loaded, rendered main-page text in temporary browser memory and return textCaptureId, textTabId, textChars, and textUrl. Read it with preview_read_text. No text file or total character cap; snapshot output stays bounded. Does not scroll or load missing content. Defaults to false.", }), ), }), success: PreviewAutomationSnapshot, failure: PreviewAutomationError, dependencies, - }).annotate(Tool.Title, "Inspect browser page"), + }) + .annotate(Tool.Title, "Inspect browser page") + .annotate(Tool.Readonly, true) + .annotate(Tool.Idempotent, false), +); + +export const PreviewReadTextTool = safeBrowserTool( + Tool.make("preview_read_text", { + description: + "Read one small part of the loaded main-page text captured by preview_snapshot with captureText=true. Pass its textCaptureId as captureId and textTabId as tabId. Start with offset=0, then use nextOffset for the next part until done=true. Each response contains at most 4096 UTF-16 characters and preserves whole character pairs. Reading refreshes the five-minute idle expiry. Set release=true to discard the capture when finished; it returns no text. A page change or replacement capture also discards it. Text is read from temporary browser memory without creating a text file; normal chat and tool history can still store the text you read.", + parameters: Schema.Struct({ + tabId: PreviewTabId.pipe( + Schema.annotateEncoded({ + description: "The textTabId returned by the snapshot that captured this text.", + }), + ), + captureId: Schema.String.check(Schema.isNonEmpty(), Schema.isMaxLength(128)).annotate({ + description: "The textCaptureId returned by preview_snapshot with captureText=true.", + }), + offset: Schema.optional( + Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)).annotate({ + description: "UTF-16 character offset. Defaults to zero; use nextOffset to continue.", + }), + ), + release: Schema.optional( + Schema.Boolean.annotate({ + description: "Discard the captured text without reading another part. Defaults to false.", + }), + ), + }), + success: Schema.Struct({ + text: Schema.String.check(Schema.isMaxLength(4096)), + nextOffset: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + totalChars: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + done: Schema.Boolean, + released: Schema.Boolean, + }), + failure: PreviewAutomationError, + dependencies, + }) + .annotate(Tool.Title, "Read captured browser text") + .annotate(Tool.Readonly, true) + .annotate(Tool.Idempotent, false), ); const PreviewClickTool = browserTool( @@ -254,6 +297,7 @@ export const PreviewToolkit = Toolkit.make( PreviewResizeTool, PreviewSetAppearanceTool, PreviewSnapshotTool, + PreviewReadTextTool, PreviewClickTool, PreviewTypeTool, PreviewPressTool, diff --git a/apps/server/src/preview/Snapshot.test.ts b/apps/server/src/preview/Snapshot.test.ts index 3163e7dc2f90..15ee08da4458 100644 --- a/apps/server/src/preview/Snapshot.test.ts +++ b/apps/server/src/preview/Snapshot.test.ts @@ -15,7 +15,6 @@ import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; -import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; import * as ServerConfig from "../config.ts"; @@ -42,10 +41,11 @@ const TestLayer = ServerConfig.layerTest(process.cwd(), { prefix: "t3-preview-te const makePage = (initialText: string) => { let text = initialText; let reads = 0; - const timers = new Set<() => void>(); - const listeners = new Map void>(); + const timers = new Map<() => void, number>(); + const listeners = new Set<() => void>(); const requests: PreviewAutomationInvokeInput[] = []; const context = NodeVM.createContext({ + TextEncoder, document: { body: { get innerText() { @@ -55,14 +55,18 @@ const makePage = (initialText: string) => { }, }, location: { href: "https://example.test/page" }, - setTimeout: (callback: () => void) => { - timers.add(callback); + setTimeout: (callback: () => void, delay: number) => { + timers.set(callback, delay); return callback; }, clearTimeout: (callback: () => void) => timers.delete(callback), - addEventListener: (event: string, callback: () => void) => listeners.set(event, callback), + addEventListener: (event: string, callback: () => void) => { + expect(event).toBe("pagehide"); + listeners.add(callback); + }, removeEventListener: (event: string, callback: () => void) => { - if (listeners.get(event) === callback) listeners.delete(event); + expect(event).toBe("pagehide"); + listeners.delete(callback); }, }); const page = { @@ -75,10 +79,10 @@ const makePage = (initialText: string) => { setText: (value: string) => { text = value; }, - beforeChunk: undefined as (() => void) | undefined, beforeSnapshot: undefined as (() => void) | undefined, + snapshotGate: undefined as Effect.Effect | undefined, + captureGate: undefined as Effect.Effect | undefined, chunkGate: undefined as Effect.Effect | undefined, - disposeGate: undefined as Effect.Effect | undefined, broker: undefined as PreviewAutomationBroker["Service"] | undefined, }; const invoke = (request: PreviewAutomationInvokeInput) => @@ -94,9 +98,15 @@ const makePage = (initialText: string) => { loading: false, } as A; } + if (request.tabId !== undefined && request.tabId !== tabId) { + return yield* new PreviewAutomationNoAvailableHostError({ + ...scope, + operation: request.operation, + }); + } if (request.operation === "snapshot") { - expect(request.tabId).toBe(tabId); page.beforeSnapshot?.(); + if (page.snapshotGate) yield* page.snapshotGate; return { url: NodeVM.runInContext("location.href", context), title: "Page", @@ -118,22 +128,18 @@ const makePage = (initialText: string) => { expect(request.operation).toBe("evaluate"); expect(request.tabId).toBe(tabId); expect(request.timeoutMs).toBeUndefined(); + expect(request.updateCurrentTab).toBe(false); const expression = (request.input as { expression: string }).expression; - if (expression.includes("let end")) { - page.beforeChunk?.(); - if (page.chunkGate) yield* page.chunkGate; - } + if (expression.includes("let end") && page.chunkGate) yield* page.chunkGate; const value = yield* Effect.try({ try: () => { const value = NodeVM.runInContext(expression, context) as A; - expect(Buffer.byteLength(encodeJson(value), "utf8")).toBeLessThan(32_000); + expect(Buffer.byteLength(encodeJson(value), "utf8")).toBeLessThan(16_512); return value; }, catch: () => new PreviewAutomationNoAvailableHostError({ ...scope, operation: "evaluate" }), }); - if (expression.includes("capture?.dispose()") && page.disposeGate) { - yield* page.disposeGate; - } + if (expression.includes("Object.defineProperty") && page.captureGate) yield* page.captureGate; return value; }); page.broker = PreviewAutomationBroker.of({ @@ -145,242 +151,394 @@ const makePage = (initialText: string) => { return page; }; -const runExport = (page: ReturnType, target?: PreviewTabId) => - Effect.gen(function* () { - const snapshot = yield* Snapshot.PreviewSnapshot; - return yield* snapshot.withSnapshot( - { scope, saveText: true, ...(target === undefined ? {} : { tabId: target }) }, - ({ textExport }) => Effect.succeed(textExport!), - ); - }).pipe( - Effect.provide(Snapshot.layer), - Effect.provideService(PreviewAutomationBroker, page.broker!), +const captureText = Effect.fnUntraced(function* (requestedTabId?: PreviewTabId) { + const snapshot = yield* Snapshot.PreviewSnapshot; + return yield* snapshot.withSnapshot( + { + scope, + captureText: true, + ...(requestedTabId === undefined ? {} : { tabId: requestedTabId }), + }, + ({ textCapture }) => Effect.succeed(textCapture!), + ); +}); +const readText = Effect.fnUntraced(function* ( + captureId: string, + offset = 0, + release = false, + requestScope = scope, + requestedTabId = tabId, +) { + const snapshot = yield* Snapshot.PreviewSnapshot; + return yield* snapshot.readText({ + scope: requestScope, + tabId: requestedTabId, + captureId, + offset, + release, + }); +}); +const providePage = (page: ReturnType) => + Effect.provide( + Snapshot.layer.pipe(Layer.provide(Layer.succeed(PreviewAutomationBroker, page.broker!))), ); const assertClean = (page: ReturnType) => { expect(page.timers.size).toBe(0); expect(page.listeners.size).toBe(0); expect( - Object.getOwnPropertyNames(page.context).filter((name) => name.startsWith("__t3_text_export_")), + Object.getOwnPropertyNames(page.context).filter((name) => + name.startsWith("__t3_text_capture_"), + ), ).toEqual([]); }; -it.effect("exports all loaded text through bounded V1 evaluate chunks and pins the tab", () => - Effect.gen(function* () { - const text = "\u0001".repeat(70_000) + "loaded end"; - const page = makePage(text); - const saved = yield* runExport(page); +it.effect("leaves default snapshots unchanged and does not create browser or disk text", () => { + const page = makePage("loaded text"); + return Effect.gen(function* () { + const snapshots = yield* Snapshot.PreviewSnapshot; + const result = yield* snapshots.withSnapshot({ scope }, Effect.succeed); const fs = yield* FileSystem.FileSystem; - expect(yield* fs.readFileString(saved.textPath)).toBe(text); - expect(saved).toMatchObject({ - totalChars: text.length, - sizeBytes: Buffer.byteLength(text), - tabId, - url: "https://example.test/page", - }); - expect(page.reads()).toBe(1); - expect(page.requests[0]?.operation).toBe("status"); + const config = yield* ServerConfig.ServerConfig; + expect(result.textCapture).toBeUndefined(); + expect(result.snapshot.visibleText).toBe("Page"); + expect(Buffer.from(result.png).toString()).toBe("png"); + expect(page.reads()).toBe(0); + expect(page.requests.map((request) => request.operation)).toEqual(["snapshot"]); + expect(yield* fs.exists(config.browserArtifactsDir)).toBe(false); assertClean(page); - }).pipe(Effect.provide(TestLayer)), -); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); -it.effect("keeps Unicode pairs intact at chunk boundaries", () => - Effect.gen(function* () { - const text = "a".repeat(4095) + "πŸ˜€δΈ­ζ–‡" + "b".repeat(4093) + "πŸš€"; +it.effect( + "reads all loaded text in bounded parts without disk writes or repeated DOM reads", + () => { + const text = "\u0001".repeat(70_000) + "loaded end"; const page = makePage(text); - const saved = yield* runExport(page, tabId); - const fs = yield* FileSystem.FileSystem; - expect(yield* fs.readFileString(saved.textPath)).toBe(text); - expect(saved.sizeBytes).toBe(Buffer.byteLength(text)); + return Effect.gen(function* () { + const captured = yield* captureText(); + expect(captured).toMatchObject({ + totalChars: text.length, + tabId, + url: "https://example.test/page", + }); + expect(page.requests[0]?.operation).toBe("status"); + expect(page.requests.filter((request) => request.operation === "evaluate")).toHaveLength(2); + let offset = 0; + let collected = ""; + while (offset < captured.totalChars) { + const part = yield* readText(captured.captureId, offset); + expect(part.text.length).toBeLessThanOrEqual(4096); + expect(part.nextOffset).toBe(offset + part.text.length); + expect(part.totalChars).toBe(text.length); + expect(part.done).toBe(part.nextOffset === text.length); + expect(part.released).toBe(false); + collected += part.text; + offset = part.nextOffset; + } + expect(collected).toBe(text); + expect(page.reads()).toBe(1); + const fs = yield* FileSystem.FileSystem; + const config = yield* ServerConfig.ServerConfig; + expect(yield* fs.exists(config.browserArtifactsDir)).toBe(false); + yield* readText(captured.captureId, 0, true); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); + }, +); + +it.effect("preserves Unicode pairs at read boundaries and rejects an offset inside a pair", () => { + const text = "a".repeat(4095) + "πŸ˜€δΈ­ζ–‡" + "b".repeat(4093) + "πŸš€"; + const page = makePage(text); + return Effect.gen(function* () { + const captured = yield* captureText(tabId); + const first = yield* readText(captured.captureId); + expect(first.text).toBe("a".repeat(4095)); + const second = yield* readText(captured.captureId, first.nextOffset); + expect(second.text.startsWith("πŸ˜€δΈ­ζ–‡")).toBe(true); + const third = yield* readText(captured.captureId, second.nextOffset); + expect(first.text + second.text + third.text).toBe(text); + expect(yield* Effect.flip(readText(captured.captureId, 4096))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); expect(page.requests.some((request) => request.operation === "status")).toBe(false); - expect(page.requests.filter((request) => request.operation === "snapshot")).toHaveLength(1); + yield* readText(captured.captureId, 0, true); assertClean(page); - }).pipe(Effect.provide(TestLayer)), -); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); -it.effect("writes an empty loaded page without reading chunks", () => - Effect.gen(function* () { - const page = makePage(""); - const saved = yield* runExport(page); - const fs = yield* FileSystem.FileSystem; - expect(yield* fs.readFileString(saved.textPath)).toBe(""); - expect(saved).toMatchObject({ totalChars: 0, sizeBytes: 0 }); +it.effect.each(["", "short text"])("retains completed reads for retries: %j", (text) => { + const page = makePage(text); + return Effect.gen(function* () { + const captured = yield* captureText(); + const first = yield* readText(captured.captureId); + expect(first).toEqual({ + text, + nextOffset: text.length, + totalChars: text.length, + done: true, + released: false, + }); + expect(yield* readText(captured.captureId)).toEqual(first); + expect(yield* readText(captured.captureId, text.length)).toEqual({ + text: "", + nextOffset: text.length, + totalChars: text.length, + done: true, + released: false, + }); + const released = yield* readText(captured.captureId, 0, true); + expect(released).toEqual({ + text: "", + nextOffset: 0, + totalChars: text.length, + done: true, + released: true, + }); + expect(yield* Effect.flip(readText(captured.captureId))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); assertClean(page); - }).pipe(Effect.provide(TestLayer)), -); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); -it.effect("freezes captured text while the page changes", () => - Effect.gen(function* () { - const text = "original ".repeat(9000); - const page = makePage(text); - page.beforeChunk = () => { - page.setText("changed page"); - const key = Object.getOwnPropertyNames(page.context).find((name) => - name.startsWith("__t3_text_export_"), - )!; - expect( - NodeVM.runInContext(`Object.isFrozen(globalThis[${encodeJson(key)}])`, page.context), - ).toBe(true); - }; - const saved = yield* runExport(page); - const fs = yield* FileSystem.FileSystem; - expect(yield* fs.readFileString(saved.textPath)).toBe(text); +it.effect("freezes loaded text while the same document changes", () => { + const page = makePage("original ".repeat(9000)); + return Effect.gen(function* () { + page.beforeSnapshot = () => page.setText("changed before snapshot"); + const captured = yield* captureText(); + const key = Object.getOwnPropertyNames(page.context).find((name) => + name.startsWith("__t3_text_capture_"), + )!; + expect( + NodeVM.runInContext(`Object.isFrozen(globalThis[${encodeJson(key)}])`, page.context), + ).toBe(true); + page.setText("changed before read"); + expect((yield* readText(captured.captureId)).text).toBe( + "original ".repeat(9000).slice(0, 4096), + ); expect(page.reads()).toBe(1); + yield* readText(captured.captureId, 0, true); assertClean(page); - }).pipe(Effect.provide(TestLayer)), -); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); it.effect.each(["navigation", "reload", "new document"] as const)( - "removes the completed text export after %s before the snapshot", - (change) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const config = yield* ServerConfig.ServerConfig; - yield* fs.makeDirectory(config.browserArtifactsDir, { recursive: true }); - const existing = path.join(config.browserArtifactsDir, "existing.txt"); - yield* fs.writeFileString(existing, "keep this file"); - const page = makePage("original loaded text"); + "rejects %s before snapshot delivery and removes its capture", + (change) => { + const page = makePage("original loaded text"); + return Effect.gen(function* () { page.beforeSnapshot = () => { - if (change === "navigation") { + if (change === "navigation") NodeVM.runInContext("location.href = 'https://other.test/'", page.context); - } else { - if (change === "reload") page.listeners.get("pagehide")!(); + else { + if (change === "reload") [...page.listeners][0]!(); NodeVM.runInContext( "document = { body: { innerText: 'different document' } }", page.context, ); } }; - - const result = yield* Effect.result(runExport(page)); - - expect(result._tag).toBe("Failure"); - expect(page.reads()).toBe(1); - expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual(["existing.txt"]); - expect(yield* fs.readFileString(existing)).toBe("keep this file"); + expect(yield* Effect.flip(captureText())).toBeInstanceOf(Snapshot.PreviewTextCaptureError); assertClean(page); - }).pipe(Effect.provide(TestLayer)), + }).pipe(providePage(page), Effect.provide(TestLayer)); + }, ); -it.effect("validates an unchanged page with a full URL longer than the output limit", () => - Effect.gen(function* () { - const page = makePage("loaded text"); +it.effect("validates a full page URL longer than the evaluate output limit", () => { + const page = makePage("loaded text"); + return Effect.gen(function* () { const url = `https://example.test/page?q=${"x".repeat(70_000)}`; NodeVM.runInContext(`location.href = ${encodeJson(url)}`, page.context); - - const saved = yield* runExport(page); - - const fs = yield* FileSystem.FileSystem; - expect(yield* fs.readFileString(saved.textPath)).toBe("loaded text"); - expect(saved.url).toBe(url.slice(0, 2048)); + const captured = yield* captureText(); + expect(captured.url).toBe(url.slice(0, 2048)); + expect((yield* readText(captured.captureId)).text).toBe("loaded text"); expect(page.reads()).toBe(1); + yield* readText(captured.captureId, 0, true); assertClean(page); - }).pipe(Effect.provide(TestLayer)), -); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); -it.effect("allows a text edit in the same document before the snapshot", () => - Effect.gen(function* () { - const page = makePage("original loaded text"); - page.beforeSnapshot = () => page.setText("edited loaded text"); +it.effect.each(["navigation", "expiry", "pagehide", "new document"] as const)( + "rejects further reads after %s", + (failure) => { + const page = makePage("text ".repeat(2000)); + return Effect.gen(function* () { + const captured = yield* captureText(); + yield* readText(captured.captureId); + if (failure === "navigation") + NodeVM.runInContext("location.href = 'https://other.test/'", page.context); + else if (failure === "expiry") { + expect([...page.timers.values()]).toEqual([300_000]); + [...page.timers.keys()][0]!(); + } else if (failure === "pagehide") [...page.listeners][0]!(); + else NodeVM.runInContext("document = { body: { innerText: 'different' } }", page.context); + expect(yield* Effect.flip(readText(captured.captureId, 4096))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); + if (page.timers.size > 0) [...page.timers.keys()][0]!(); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); + }, +); - const saved = yield* runExport(page); +it.effect.each(["environmentId", "threadId", "providerSessionId", "providerInstanceId"] as const)( + "does not disclose or release another %s's capture", + (field) => { + const page = makePage("private text"); + return Effect.gen(function* () { + const captured = yield* captureText(); + const otherScope = { ...scope, [field]: "different-owner" }; + expect(yield* Effect.flip(readText(captured.captureId, 0, false, otherScope))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); + expect(yield* Effect.flip(readText(captured.captureId, 0, true, otherScope))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); + expect((yield* readText(captured.captureId)).text).toBe("private text"); + yield* readText(captured.captureId, 0, true); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); + }, +); - const fs = yield* FileSystem.FileSystem; - expect(yield* fs.readFileString(saved.textPath)).toBe("original loaded text"); - expect(page.reads()).toBe(1); +it.effect("rejects a different tab or capture version without releasing the valid capture", () => { + const page = makePage("private text"); + return Effect.gen(function* () { + const captured = yield* captureText(); + expect( + yield* Effect.flip( + readText(captured.captureId, 0, false, scope, PreviewTabId.make("other-tab")), + ), + ).toBeInstanceOf(PreviewAutomationNoAvailableHostError); + expect(yield* Effect.flip(readText("wrong-capture"))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); + expect(yield* Effect.flip(readText("wrong-capture", 0, true))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); + expect((yield* readText(captured.captureId)).text).toBe("private text"); + yield* readText(captured.captureId, 0, true); assertClean(page); - }).pipe(Effect.provide(TestLayer)), + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect.each([-1, 0.5, 5, Number.NaN, Number.POSITIVE_INFINITY])( + "rejects invalid offset %s without damaging the capture", + (offset) => { + const page = makePage("text"); + return Effect.gen(function* () { + const captured = yield* captureText(); + expect(yield* Effect.flip(readText(captured.captureId, offset))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); + expect((yield* readText(captured.captureId)).text).toBe("text"); + yield* readText(captured.captureId, 0, true); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); + }, ); -it.effect.each(["navigation", "expiry"] as const)("removes partial text after %s", (failure) => - Effect.gen(function* () { - const page = makePage("text ".repeat(20_000)); - let chunks = 0; - page.beforeChunk = () => { - if (++chunks !== 2) return; - if (failure === "navigation") - NodeVM.runInContext("location.href = 'https://other.test/'", page.context); - else [...page.timers][0]!(); - }; - const result = yield* Effect.result(runExport(page)); - expect(result._tag).toBe("Failure"); - const fs = yield* FileSystem.FileSystem; - const config = yield* ServerConfig.ServerConfig; - expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([]); +it.effect("replaces the prior capture and keeps one timer and page listener", () => { + const page = makePage("first text"); + return Effect.gen(function* () { + const first = yield* captureText(); + page.setText("second text"); + const second = yield* captureText(); + expect(second.captureId).not.toBe(first.captureId); + expect(page.timers.size).toBe(1); + expect(page.listeners.size).toBe(1); + expect(yield* Effect.flip(readText(first.captureId))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); + expect(yield* Effect.flip(readText(first.captureId, 0, true))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); + expect((yield* readText(second.captureId)).text).toBe("second text"); + yield* readText(second.captureId, 0, true); assertClean(page); - }).pipe(Effect.provide(TestLayer)), + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect.each(["failure", "cancellation"] as const)( + "removes an undelivered capture after result delivery %s", + (outcome) => { + const page = makePage("completed text"); + return Effect.gen(function* () { + const enteredUse = yield* Deferred.make(); + const snapshots = yield* Snapshot.PreviewSnapshot; + const fiber = yield* Effect.forkChild( + snapshots.withSnapshot({ scope, captureText: true }, ({ textCapture }) => + Effect.gen(function* () { + expect(textCapture?.totalChars).toBe(14); + yield* Deferred.succeed(enteredUse, undefined); + return yield* outcome === "failure" ? Effect.fail("delivery failed") : Effect.never; + }), + ), + ); + yield* Deferred.await(enteredUse); + if (outcome === "cancellation") yield* Fiber.interrupt(fiber); + expect((yield* Fiber.await(fiber))._tag).toBe("Failure"); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); + }, ); -it.effect("cleans the browser capture when the artifacts directory cannot be created", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const config = yield* ServerConfig.ServerConfig; - yield* fs.makeDirectory(config.stateDir, { recursive: true }); - yield* fs.writeFileString(config.browserArtifactsDir, "existing file"); - const page = makePage("text"); - const result = yield* Effect.result(runExport(page)); - expect(result._tag).toBe("Failure"); - expect(yield* fs.readFileString(config.browserArtifactsDir)).toBe("existing file"); +it.effect("does not let cancelled old delivery remove a newer capture", () => { + const page = makePage("first text"); + return Effect.gen(function* () { + const enteredUse = yield* Deferred.make(); + const snapshots = yield* Snapshot.PreviewSnapshot; + const fiber = yield* Effect.forkChild( + snapshots.withSnapshot({ scope, captureText: true }, () => + Deferred.succeed(enteredUse, undefined).pipe(Effect.andThen(Effect.never)), + ), + ); + yield* Deferred.await(enteredUse); + page.setText("new text"); + const captured = yield* captureText(); + yield* Fiber.interrupt(fiber); + expect((yield* readText(captured.captureId)).text).toBe("new text"); + expect(page.timers.size).toBe(1); + yield* readText(captured.captureId, 0, true); assertClean(page); - }).pipe(Effect.provide(TestLayer)), -); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); -it.effect("removes a partial file when writing the next chunk fails", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const config = yield* ServerConfig.ServerConfig; - const page = makePage("text ".repeat(3000)); - let writes = 0; - const failingFs = FileSystem.FileSystem.of({ - ...fs, - open: (filePath, options) => - fs.open(filePath, options).pipe( - Effect.map((file) => ({ - ...file, - writeAll: (bytes) => - ++writes === 2 - ? fs.writeFileString(config.browserArtifactsDir, "fails on a directory") - : file.writeAll(bytes), - })), - ), - }); - const result = yield* Effect.result( - runExport(page).pipe(Effect.provideService(FileSystem.FileSystem, failingFs)), +it.effect("removes the capture when snapshot collection is interrupted", () => { + const page = makePage("loaded text"); + return Effect.gen(function* () { + const enteredSnapshot = yield* Deferred.make(); + page.snapshotGate = Deferred.succeed(enteredSnapshot, undefined).pipe( + Effect.andThen(Effect.never), ); - expect(result._tag).toBe("Failure"); - expect(writes).toBe(2); - expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([]); + const fiber = yield* Effect.forkChild(captureText()); + yield* Deferred.await(enteredSnapshot); + yield* Fiber.interrupt(fiber); assertClean(page); - }).pipe(Effect.provide(TestLayer)), -); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); -it.effect("does not create a capture when the resolved tab is unavailable", () => - Effect.gen(function* () { - const page = makePage("text"); - page.available = false; - const result = yield* Effect.result(runExport(page)); - expect(result._tag).toBe("Failure"); +it.effect("does not capture text when the resolved tab is unavailable", () => { + const page = makePage("text"); + page.available = false; + return Effect.gen(function* () { + expect(yield* Effect.flip(captureText())).toBeInstanceOf(Snapshot.PreviewTextCaptureError); expect(page.requests).toHaveLength(1); expect(page.reads()).toBe(0); assertClean(page); - }).pipe(Effect.provide(TestLayer)), -); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); it.effect.each(["status", "evaluate"] as const)( - "preserves browser recovery errors during text export %s", + "preserves browser recovery errors during text %s", (operation) => Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const config = yield* ServerConfig.ServerConfig; - yield* fs.makeDirectory(config.browserArtifactsDir, { recursive: true }); - const existing = path.join(config.browserArtifactsDir, "existing.txt"); - yield* fs.writeFileString(existing, "keep this file"); for (const ErrorClass of [ PreviewAutomationNoAvailableHostError, PreviewAutomationTimeoutError, PreviewAutomationRequestQueueClosedError, ]) { + const page = makePage("loaded text"); const error = new ErrorClass({ ...scope, operation, @@ -389,98 +547,109 @@ it.effect.each(["status", "evaluate"] as const)( requestId: "text-request", timeoutMs: 15_000, }); - const page = makePage("loaded text"); const original = page.broker!; page.broker = PreviewAutomationBroker.of({ ...original, invoke: (request) => - request.operation === operation && - (operation === "status" || - (request.input as { expression: string }).expression.includes("let end")) - ? Effect.fail(error) - : original.invoke(request), + request.operation === operation ? Effect.fail(error) : original.invoke(request), }); - - const failed = yield* Effect.flip(runExport(page)); - - expect(failed).toBe(error); - expect(failed.message).toBe(error.message); - expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual(["existing.txt"]); - expect(yield* fs.readFileString(existing)).toBe("keep this file"); + expect(yield* Effect.flip(captureText().pipe(providePage(page)))).toBe(error); assertClean(page); } }).pipe(Effect.provide(TestLayer)), ); -it.effect("removes partial files and the browser capture on cancellation", () => - Effect.gen(function* () { - const enteredChunk = yield* Deferred.make(); - const page = makePage("text ".repeat(2000)); - page.chunkGate = Deferred.succeed(enteredChunk, undefined).pipe(Effect.andThen(Effect.never)); - const fiber = yield* Effect.forkChild(runExport(page)); - yield* Deferred.await(enteredChunk); - yield* Fiber.interrupt(fiber); +it.effect("preserves opt-in PNG saving without writing a text file", () => { + const page = makePage("loaded text"); + return Effect.gen(function* () { + const snapshots = yield* Snapshot.PreviewSnapshot; + const result = yield* snapshots.withSnapshot( + { scope, captureText: true, save: true }, + Effect.succeed, + ); const fs = yield* FileSystem.FileSystem; const config = yield* ServerConfig.ServerConfig; - expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([]); + expect(Buffer.from(yield* fs.readFile(result.screenshotPath!)).toString()).toBe("png"); + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toHaveLength(1); + expect(result.screenshotPath).toMatch(/browser-screenshot-example-test-.*\.png$/); + yield* readText(result.textCapture!.captureId, 0, true); assertClean(page); - }).pipe(Effect.provide(TestLayer)), -); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); -it.effect("removes the completed file when cancellation occurs during browser cleanup", () => - Effect.gen(function* () { - const enteredDispose = yield* Deferred.make(); - const releaseDispose = yield* Deferred.make(); - const page = makePage("completed text"); - page.disposeGate = Deferred.succeed(enteredDispose, undefined).pipe( - Effect.andThen(Deferred.await(releaseDispose)), +it.effect("keeps another owner's live capture when this owner captures or releases text", () => { + const page = makePage("first owner's text"); + return Effect.gen(function* () { + const snapshots = yield* Snapshot.PreviewSnapshot; + const first = yield* captureText(); + const otherScope = { ...scope, threadId: ThreadId.make("other-thread") }; + page.setText("second owner's text"); + const second = yield* snapshots.withSnapshot( + { scope: otherScope, tabId, captureText: true }, + ({ textCapture }) => Effect.succeed(textCapture!), + ); + expect(page.timers.size).toBe(2); + expect((yield* readText(first.captureId)).text).toBe("first owner's text"); + expect((yield* readText(second.captureId, 0, false, otherScope)).text).toBe( + "second owner's text", + ); + yield* readText(first.captureId, 0, true); + expect(page.timers.size).toBe(1); + expect((yield* readText(second.captureId, 0, false, otherScope)).text).toBe( + "second owner's text", ); - const fiber = yield* Effect.forkChild(runExport(page)); - yield* Deferred.await(enteredDispose); + yield* readText(second.captureId, 0, true, otherScope); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect("allows the same offset to be retried after an interrupted read", () => { + const page = makePage("loaded ".repeat(2000)); + return Effect.gen(function* () { + const captured = yield* captureText(); + const enteredRead = yield* Deferred.make(); + page.chunkGate = Deferred.succeed(enteredRead, undefined).pipe(Effect.andThen(Effect.never)); + const fiber = yield* Effect.forkChild(readText(captured.captureId)); + yield* Deferred.await(enteredRead); + yield* Fiber.interrupt(fiber); + page.chunkGate = undefined; + const retried = yield* readText(captured.captureId); + expect(retried.text).toBe("loaded ".repeat(2000).slice(0, 4096)); + expect(retried.nextOffset).toBe(4096); + expect(page.reads()).toBe(1); + yield* readText(captured.captureId, 0, true); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect("releases its memory capture when the requested PNG cannot be saved", () => { + const page = makePage("loaded text"); + return Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const config = yield* ServerConfig.ServerConfig; - expect(yield* fs.readDirectory(config.browserArtifactsDir)).toHaveLength(1); - const interruption = yield* Effect.forkChild(Fiber.interrupt(fiber)); - yield* Effect.yieldNow; - yield* Deferred.succeed(releaseDispose, undefined); - yield* Fiber.join(interruption); - expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([]); + yield* fs.makeDirectory(config.stateDir, { recursive: true }); + yield* fs.writeFileString(config.browserArtifactsDir, "existing file"); + const snapshots = yield* Snapshot.PreviewSnapshot; + const failed = yield* Effect.flip( + snapshots.withSnapshot({ scope, captureText: true, save: true }, Effect.succeed), + ); + expect(failed).toBeInstanceOf(Snapshot.PreviewScreenshotSaveError); + expect(yield* fs.readFileString(config.browserArtifactsDir)).toBe("existing file"); assertClean(page); - }).pipe(Effect.provide(TestLayer)), -); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); -it.effect.each(["failure", "cancellation"] as const)( - "removes its export after snapshot result delivery %s", - (outcome) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const config = yield* ServerConfig.ServerConfig; - yield* fs.makeDirectory(config.browserArtifactsDir, { recursive: true }); - const existing = path.join(config.browserArtifactsDir, "existing.txt"); - yield* fs.writeFileString(existing, "keep this file"); - const enteredUse = yield* Deferred.make(); - const page = makePage("completed text"); - const delivery = Effect.gen(function* () { - const snapshots = yield* Snapshot.PreviewSnapshot; - return yield* snapshots.withSnapshot({ scope, saveText: true }, ({ textExport }) => - Effect.gen(function* () { - expect(yield* fs.readFileString(textExport!.textPath)).toBe("completed text"); - yield* Deferred.succeed(enteredUse, undefined); - return yield* outcome === "failure" ? Effect.fail("delivery failed") : Effect.never; - }), - ); - }).pipe( - Effect.provide(Snapshot.layer), - Effect.provideService(PreviewAutomationBroker, page.broker!), - ); - const fiber = yield* Effect.forkChild(delivery); - yield* Deferred.await(enteredUse); - if (outcome === "cancellation") yield* Fiber.interrupt(fiber); - const result = yield* Fiber.await(fiber); - expect(result._tag).toBe("Failure"); - expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual(["existing.txt"]); - expect(yield* fs.readFileString(existing)).toBe("keep this file"); - assertClean(page); - }).pipe(Effect.provide(TestLayer)), -); +it.effect("cleans an installed capture when its initial response is interrupted", () => { + const page = makePage("loaded text"); + return Effect.gen(function* () { + const enteredCapture = yield* Deferred.make(); + page.captureGate = Deferred.succeed(enteredCapture, undefined).pipe( + Effect.andThen(Effect.never), + ); + const fiber = yield* Effect.forkChild(captureText()); + yield* Deferred.await(enteredCapture); + expect(page.timers.size).toBe(1); + yield* Fiber.interrupt(fiber); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); diff --git a/apps/server/src/preview/Snapshot.ts b/apps/server/src/preview/Snapshot.ts index 632bb70a3984..aeca36ad9517 100644 --- a/apps/server/src/preview/Snapshot.ts +++ b/apps/server/src/preview/Snapshot.ts @@ -16,21 +16,21 @@ import * as Schema from "effect/Schema"; import * as ServerConfig from "../config.ts"; import * as PreviewAutomationBroker from "../mcp/PreviewAutomationBroker.ts"; -export class PreviewTextExportError extends Schema.TaggedError()( - "PreviewTextExportError", +export class PreviewTextCaptureError extends Schema.TaggedError()( + "PreviewTextCaptureError", { cause: Schema.Defect() }, ) { override get message(): string { - return "Could not save the loaded page text. The page may have changed or the file could not be written."; + return "Could not read the loaded page text. Take another snapshot with captureText=true if the page changed or the capture expired."; } } const isPreviewAutomationError = Schema.is(PreviewAutomationError); -const isPreviewTextExportError = Schema.is(PreviewTextExportError); -const textExportError = (cause: unknown) => - isPreviewAutomationError(cause) || isPreviewTextExportError(cause) +const isPreviewTextCaptureError = Schema.is(PreviewTextCaptureError); +const textCaptureError = (cause: unknown) => + isPreviewAutomationError(cause) || isPreviewTextCaptureError(cause) ? cause - : new PreviewTextExportError({ cause }); + : new PreviewTextCaptureError({ cause }); const Capture = Schema.Struct({ totalChars: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), @@ -38,7 +38,10 @@ const Capture = Schema.Struct({ }); const TextChunk = Schema.Struct({ text: Schema.String.check(Schema.isMaxLength(4096)), - next: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + nextOffset: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + totalChars: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + done: Schema.Boolean, + released: Schema.Boolean, }); const encodePageString = Schema.encodeEffect(Schema.fromJsonString(Schema.String)); const decodeCapture = Schema.decodeUnknownEffect(Capture); @@ -59,17 +62,16 @@ export class PreviewScreenshotSaveError extends Schema.TaggedError Effect.Effect; readonly withSnapshot: ( input: SnapshotInput, use: (capture: SnapshotCapture) => Effect.Effect, @@ -86,7 +95,7 @@ export class PreviewSnapshot extends Context.Service< A, | E | PreviewAutomationError - | PreviewTextExportError + | PreviewTextCaptureError | PreviewScreenshotSaveError | Schema.SchemaError, R @@ -114,114 +123,82 @@ const make = Effect.gen(function* () { const path = yield* Path.Path; const config = yield* ServerConfig.ServerConfig; - const saveText = Effect.fn("PreviewSnapshot.saveText")(function* ( - scope: SnapshotInput["scope"], - requestedTabId?: PreviewTabId, - ) { - const id = NodeCrypto.randomUUID(); - let tabId = requestedTabId; - if (tabId === undefined) { - const status = yield* broker - .invoke({ scope, operation: "status", input: {} }) - .pipe(Effect.flatMap(decodeStatus), Effect.mapError(textExportError)); - if (!status.available || status.tabId === null) { - return yield* new PreviewTextExportError({ cause: "No available preview tab." }); + const captureKey = (scope: SnapshotInput["scope"]) => + `__t3_text_capture_${NodeCrypto.createHash("sha256") + .update( + JSON.stringify([ + scope.environmentId, + scope.threadId, + scope.providerSessionId, + scope.providerInstanceId, + ]), + ) + .digest("hex")}`; + const evaluate = (scope: SnapshotInput["scope"], tabId: PreviewTabId, expression: string) => + broker.invoke({ + scope, + operation: "evaluate", + tabId, + input: { expression, returnByValue: true }, + updateCurrentTab: false, + }); + + const readText = Effect.fn("PreviewSnapshot.readText")(function* (input: { + readonly scope: SnapshotInput["scope"]; + readonly tabId: PreviewTabId; + readonly captureId: string; + readonly offset?: number | undefined; + readonly release?: boolean | undefined; + }) { + const offset = input.offset ?? 0; + if (!Number.isSafeInteger(offset) || offset < 0 || input.captureId.length > 128) { + return yield* new PreviewTextCaptureError({ cause: "Invalid capture or offset." }); + } + const key = yield* encodePageString(captureKey(input.scope)).pipe(Effect.orDie); + const id = yield* encodePageString(input.captureId).pipe(Effect.orDie); + const chunk = yield* evaluate( + input.scope, + input.tabId, + `(() => { + const capture = globalThis[${key}]; + if (!capture || capture.captureId !== ${id}) return null; + if (capture.document !== document || capture.url !== location.href) { + capture.dispose(); + return null; } - tabId = status.tabId; + const totalChars = capture.text.length; + if (${input.release === true}) { + capture.dispose(); + return { text: "", nextOffset: 0, totalChars, done: true, released: true }; + } + const offset = ${offset}; + if (offset > totalChars) return null; + const first = capture.text.charCodeAt(offset); + const previous = capture.text.charCodeAt(offset - 1); + if (first >= 0xdc00 && first <= 0xdfff && previous >= 0xd800 && previous <= 0xdbff) return null; + let end = Math.min(offset + 4096, totalChars); + if (new TextEncoder().encode(JSON.stringify({ text: capture.text.slice(offset, end) })).byteLength > 16000) { + end = offset + Math.floor((end - offset) / 2); + } + if (end < totalChars) { + const last = capture.text.charCodeAt(end - 1); + const next = capture.text.charCodeAt(end); + if (last >= 0xd800 && last <= 0xdbff && next >= 0xdc00 && next <= 0xdfff) end--; + } + capture.refresh(); + return { text: capture.text.slice(offset, end), nextOffset: end, totalChars, done: end === totalChars, released: false }; + })()`, + ).pipe(Effect.flatMap(decodeTextChunk), Effect.mapError(textCaptureError)); + if ( + !chunk.released && + (chunk.nextOffset !== offset + chunk.text.length || + chunk.nextOffset > chunk.totalChars || + (offset < chunk.totalChars && chunk.nextOffset <= offset) || + chunk.done !== (chunk.nextOffset === chunk.totalChars)) + ) { + return yield* new PreviewTextCaptureError({ cause: "Invalid text capture chunk." }); } - const key = yield* encodePageString(`__t3_text_export_${id}`).pipe(Effect.orDie); - const textPath = path.join(config.browserArtifactsDir, `browser-text-${id}.txt`); - const evaluate = (expression: string) => - broker.invoke({ - scope, - operation: "evaluate", - tabId, - input: { expression, returnByValue: true }, - updateCurrentTab: false, - }); - yield* Effect.addFinalizer(() => - evaluate(`(() => { - const capture = globalThis[${key}]; - capture?.dispose(); - return true; - })()`).pipe(Effect.interruptible, Effect.timeoutOption(5000), Effect.ignore), - ); - let created = false; - return yield* Effect.scoped( - Effect.gen(function* () { - const capture = yield* evaluate(`(() => { - const text = document.body?.innerText ?? ""; - let timer; - const dispose = () => { - clearTimeout(timer); - removeEventListener("pagehide", dispose); - if (globalThis[${key}] === capture) delete globalThis[${key}]; - }; - const refresh = () => { - clearTimeout(timer); - timer = setTimeout(dispose, 300000); - }; - const capture = Object.freeze({ text, url: location.href, document, dispose, refresh }); - Object.defineProperty(globalThis, ${key}, { value: capture, configurable: true }); - addEventListener("pagehide", dispose, { once: true }); - refresh(); - return { totalChars: text.length, url: capture.url.slice(0, 2048) }; - })()`).pipe(Effect.flatMap(decodeCapture)); - yield* fileSystem.makeDirectory(config.browserArtifactsDir, { recursive: true }); - const file = yield* Effect.uninterruptible( - fileSystem.open(textPath, { flag: "wx", mode: 0o600 }).pipe( - Effect.tap(() => - Effect.sync(() => { - created = true; - }), - ), - ), - ); - let offset = 0; - let sizeBytes = 0; - while (offset < capture.totalChars) { - const chunk = yield* evaluate(`(() => { - const capture = globalThis[${key}]; - if (!capture || capture.document !== document || capture.url !== location.href) throw new Error("Text capture lost."); - let end = Math.min(${offset} + 4096, capture.text.length); - if (end < capture.text.length) { - const last = capture.text.charCodeAt(end - 1); - const next = capture.text.charCodeAt(end); - if (last >= 0xd800 && last <= 0xdbff && next >= 0xdc00 && next <= 0xdfff) end--; - } - capture.refresh(); - return { text: capture.text.slice(${offset}, end), next: end }; - })()`).pipe(Effect.flatMap(decodeTextChunk)); - if ( - chunk.next !== offset + chunk.text.length || - chunk.next <= offset || - chunk.next > capture.totalChars - ) { - return yield* new PreviewTextExportError({ cause: "Invalid text capture chunk." }); - } - const bytes = new TextEncoder().encode(chunk.text); - yield* file.writeAll(bytes); - sizeBytes += bytes.byteLength; - offset = chunk.next; - } - yield* file.sync; - return { - textPath, - totalChars: capture.totalChars, - sizeBytes, - url: capture.url, - tabId, - captureKey: key, - }; - }), - ).pipe( - Effect.onExit((exit) => - created && exit._tag === "Failure" - ? fileSystem.remove(textPath).pipe(Effect.ignore) - : Effect.void, - ), - Effect.mapError(textExportError), - ); + return chunk; }); const saveScreenshot = Effect.fn("PreviewSnapshot.saveScreenshot")(function* ( @@ -242,75 +219,112 @@ const make = Effect.gen(function* () { input: SnapshotInput, use: (capture: SnapshotCapture) => Effect.Effect, ) { - let exportedText: (SnapshotCapture["textExport"] & { readonly captureKey: string }) | undefined; - return yield* Effect.scoped( - Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - if (input.saveText === true) { - exportedText = yield* restore(saveText(input.scope, input.tabId)); + const captureId = NodeCrypto.randomUUID(); + const key = yield* encodePageString(captureKey(input.scope)).pipe(Effect.orDie); + const id = yield* encodePageString(captureId).pipe(Effect.orDie); + let captureTabId: PreviewTabId | undefined; + return yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + let textCapture: SnapshotCapture["textCapture"]; + if (input.captureText === true) { + let tabId = input.tabId; + if (tabId === undefined) { + const status = yield* restore( + broker + .invoke({ scope: input.scope, operation: "status", input: {} }) + .pipe(Effect.flatMap(decodeStatus), Effect.mapError(textCaptureError)), + ); + if (!status.available || status.tabId === null) { + return yield* new PreviewTextCaptureError({ cause: "No available preview tab." }); + } + tabId = status.tabId; } - const capturedText = exportedText; - return yield* restore( - Effect.gen(function* () { - const tabId = capturedText?.tabId ?? input.tabId; - const snapshot = yield* broker - .invoke({ - scope: input.scope, - operation: "snapshot", - input: {}, - ...(tabId === undefined ? {} : { tabId }), - }) - .pipe(Effect.flatMap(decodeSnapshot)); - let textExport: SnapshotCapture["textExport"]; - if (capturedText !== undefined) { - const { captureKey, ...artifact } = capturedText; - for (let offset = 0; offset < Math.max(snapshot.url.length, 1); offset += 4096) { - const urlChunk = yield* encodePageString( - snapshot.url.slice(offset, offset + 4096), - ).pipe(Effect.orDie); - yield* broker - .invoke({ - scope: input.scope, - operation: "evaluate", - tabId: capturedText.tabId, - updateCurrentTab: false, - input: { - expression: `(() => { - const capture = globalThis[${captureKey}]; - if (!capture || capture.document !== document || capture.url !== location.href) return false; - capture.refresh(); - return capture.url.length === ${snapshot.url.length} && capture.url.slice(${offset}, ${offset + 4096}) === ${urlChunk}; - })()`, - returnByValue: true, - }, - }) - .pipe(Effect.flatMap(decodeCaptureMatch), Effect.mapError(textExportError)); - } - textExport = artifact; - } - const png = new Uint8Array(Buffer.from(snapshot.screenshot.data, "base64")); - const screenshotPath = - input.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; - return yield* use({ - snapshot, - png, - ...(textExport === undefined ? {} : { textExport }), - ...(screenshotPath === undefined ? {} : { screenshotPath }), - }); - }), + captureTabId = tabId; + const capture = yield* restore( + evaluate( + input.scope, + tabId, + `(() => { + globalThis[${key}]?.dispose(); + const text = document.body?.innerText ?? ""; + let timer; + const dispose = () => { + clearTimeout(timer); + removeEventListener("pagehide", dispose); + if (globalThis[${key}] === capture) delete globalThis[${key}]; + }; + const refresh = () => { + clearTimeout(timer); + timer = setTimeout(dispose, 300000); + }; + const capture = Object.freeze({ captureId: ${id}, text, url: location.href, document, dispose, refresh }); + Object.defineProperty(globalThis, ${key}, { value: capture, configurable: true }); + addEventListener("pagehide", dispose, { once: true }); + refresh(); + return { totalChars: text.length, url: capture.url.slice(0, 2048) }; + })()`, + ).pipe(Effect.flatMap(decodeCapture), Effect.mapError(textCaptureError)), ); - }), - ), + textCapture = { captureId, ...capture, tabId }; + } + return yield* restore( + Effect.gen(function* () { + const tabId = textCapture?.tabId ?? input.tabId; + const snapshot = yield* broker + .invoke({ + scope: input.scope, + operation: "snapshot", + input: {}, + ...(tabId === undefined ? {} : { tabId }), + }) + .pipe(Effect.flatMap(decodeSnapshot)); + if (textCapture !== undefined) { + for (let offset = 0; offset < Math.max(snapshot.url.length, 1); offset += 4096) { + const urlChunk = yield* encodePageString( + snapshot.url.slice(offset, offset + 4096), + ).pipe(Effect.orDie); + yield* evaluate( + input.scope, + textCapture.tabId, + `(() => { + const capture = globalThis[${key}]; + if (!capture || capture.captureId !== ${id} || capture.document !== document || capture.url !== location.href) return false; + capture.refresh(); + return capture.url.length === ${snapshot.url.length} && capture.url.slice(${offset}, ${offset + 4096}) === ${urlChunk}; + })()`, + ).pipe(Effect.flatMap(decodeCaptureMatch), Effect.mapError(textCaptureError)); + } + } + const png = new Uint8Array(Buffer.from(snapshot.screenshot.data, "base64")); + const screenshotPath = + input.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; + return yield* use({ + snapshot, + png, + ...(textCapture === undefined ? {} : { textCapture }), + ...(screenshotPath === undefined ? {} : { screenshotPath }), + }); + }), + ); + }), ).pipe( Effect.onExit((exit) => - exportedText !== undefined && exit._tag === "Failure" - ? fileSystem.remove(exportedText.textPath).pipe(Effect.ignore) + captureTabId !== undefined && exit._tag === "Failure" + ? evaluate( + input.scope, + captureTabId, + `(() => { + const capture = globalThis[${key}]; + if (capture?.captureId === ${id}) capture.dispose(); + return true; + })()`, + ).pipe(Effect.interruptible, Effect.timeoutOption(5000), Effect.ignore) : Effect.void, ), ); }); - return PreviewSnapshot.of({ withSnapshot }); + return PreviewSnapshot.of({ withSnapshot, readText }); }); export const layer = Layer.effect(PreviewSnapshot, make); diff --git a/docs/user/browser.md b/docs/user/browser.md index 52251a90f5ef..bc84a3f472a1 100644 --- a/docs/user/browser.md +++ b/docs/user/browser.md @@ -9,10 +9,15 @@ Snapshots have a size limit. Current-view text takes priority over page text and logs. The result reports omitted content so the agent can request more detail. The screenshot shows the current view. -An agent can set `saveText=true` when taking a snapshot to save all loaded, -rendered main-page text to a UTF-8 file. The file has no total character cap. The -snapshot returns `textPath` in the environment so the agent can read the file -in parts. This export does not scroll or load missing content. +An agent can set `captureText=true` when taking a snapshot to keep all loaded, +rendered main-page text in temporary browser memory. The capture has no total +character cap. The agent uses the returned `textCaptureId` and `textTabId` with +`preview_read_text` to read small parts, starting at offset zero and following +`nextOffset` until `done=true`. This does not scroll or load missing content. + +No text file is created. Normal chat and tool history can still store text the +agent reads. The capture expires after five idle minutes, a page change, or a +replacement capture. The agent can also release it with `release=true`. Complex CSS clip shapes and transformed clips can cause current-view text and controls to be omitted. The result reports these omissions. Use the screenshot From c2eaa2ce30af1726a58e5b697032d447fa54c90b Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:37:31 +0200 Subject: [PATCH 15/15] fix(preview): register captured text tool presentation --- .github/workflows/preview-snapshot-tests.yml | 3 +++ packages/shared/src/t3McpToolPresentation.test.ts | 4 ++++ packages/shared/src/t3McpToolPresentation.ts | 5 +++++ 3 files changed, 12 insertions(+) diff --git a/.github/workflows/preview-snapshot-tests.yml b/.github/workflows/preview-snapshot-tests.yml index ef40032fdff7..d6e79d83c159 100644 --- a/.github/workflows/preview-snapshot-tests.yml +++ b/.github/workflows/preview-snapshot-tests.yml @@ -8,6 +8,7 @@ on: - apps/server/src/mcp/** - apps/server/src/preview/Snapshot*.ts - packages/contracts/src/preview*.ts + - packages/shared/src/t3McpToolPresentation*.ts workflow_dispatch: permissions: @@ -42,7 +43,9 @@ jobs: apps/desktop/src/preview/Manager.test.ts apps/server/src/mcp/McpHttpServer.test.ts apps/server/src/mcp/PreviewAutomationBroker.test.ts + apps/server/src/mcp/toolkits/core.test.ts apps/server/src/preview/Snapshot.test.ts apps/server/src/mcp/toolkits/preview/tools.test.ts packages/contracts/src/previewAutomationSnapshot.test.ts packages/contracts/src/preview.test.ts + packages/shared/src/t3McpToolPresentation.test.ts diff --git a/packages/shared/src/t3McpToolPresentation.test.ts b/packages/shared/src/t3McpToolPresentation.test.ts index 43b2d9c2de15..dc2bbb58f8b5 100644 --- a/packages/shared/src/t3McpToolPresentation.test.ts +++ b/packages/shared/src/t3McpToolPresentation.test.ts @@ -72,6 +72,10 @@ describe("resolveT3McpToolPresentation", () => { displayName: "Get preview browser status", logo: "t3-code", }); + expect(resolveT3McpToolPresentation("mcp__t3_code__preview_read_text")).toEqual({ + displayName: "Read captured browser text", + logo: "t3-code", + }); }); it("matches the separator variants ACP registry agents emit", () => { diff --git a/packages/shared/src/t3McpToolPresentation.ts b/packages/shared/src/t3McpToolPresentation.ts index d85104cf4bc1..c7bf52c7e972 100644 --- a/packages/shared/src/t3McpToolPresentation.ts +++ b/packages/shared/src/t3McpToolPresentation.ts @@ -160,6 +160,11 @@ const T3_MCP_TOOLS: Readonly> = { "browser", "Snapshot the preview page", ), + preview_read_text: tool( + ["Read", "Reading", "Read", "captured browser text"], + "browser", + "browser", + ), preview_click: tool( ["Click", "Clicking", "Clicked", "in the preview browser"], "browser",