diff --git a/.github/workflows/preview-snapshot-tests.yml b/.github/workflows/preview-snapshot-tests.yml new file mode 100644 index 000000000000..d6e79d83c159 --- /dev/null +++ b/.github/workflows/preview-snapshot-tests.yml @@ -0,0 +1,51 @@ +name: Preview snapshot tests + +on: + pull_request: + paths: + - .github/workflows/preview-snapshot-tests.yml + - apps/desktop/src/preview/** + - apps/server/src/mcp/** + - apps/server/src/preview/Snapshot*.ts + - packages/contracts/src/preview*.ts + - packages/shared/src/t3McpToolPresentation*.ts + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + name: Snapshot tests (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, windows-2025, macos-15] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + sparse-checkout: | + /* + !/.repos/ + sparse-checkout-cone-mode: false + - uses: voidzero-dev/setup-vp@v1 + with: + node-version-file: package.json + cache: false + run-install: false + - run: vp install + - run: >- + vp test run + apps/desktop/src/preview/SnapshotPage.test.ts + apps/desktop/src/preview/Manager.test.ts + apps/server/src/mcp/McpHttpServer.test.ts + apps/server/src/mcp/PreviewAutomationBroker.test.ts + apps/server/src/mcp/toolkits/core.test.ts + apps/server/src/preview/Snapshot.test.ts + apps/server/src/mcp/toolkits/preview/tools.test.ts + packages/contracts/src/previewAutomationSnapshot.test.ts + packages/contracts/src/preview.test.ts + packages/shared/src/t3McpToolPresentation.test.ts diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 3593ae5f5df6..70655f142636 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -92,6 +92,7 @@ import { previewAutomationEditingCommandExpression, } from "./PreviewKeyboard.ts"; import { captureFavicon, safeHttpOrigin, selectFaviconCandidates } from "./FaviconCapture.ts"; +import { snapshotPageExpression, type SnapshotPage } from "./SnapshotPage.ts"; import { DEFAULT_RECORDING_INPUT_OPTIONS, type RecordingInputOptions } from "./RecordingInput.ts"; export type PreviewNavStatus = @@ -132,14 +133,6 @@ const ZOOM_LEVELS: ReadonlyArray = [ const DEFAULT_ZOOM_FACTOR = 1.0; const ZOOM_EPSILON = 0.001; const MAX_EVALUATION_BYTES = 64_000; -const MAX_VISIBLE_TEXT_LENGTH = 20_000; -const MAX_INTERACTIVE_ELEMENTS = 200; -/** - * A `[role]` container's innerText is its whole subtree, which turned one - * snapshot's element list into 60 KB of repeated page text. Names are labels, - * not content, so cap them where they are read. - */ -const MAX_INTERACTIVE_ELEMENT_NAME_LENGTH = 200; const MAX_SCREENSHOT_WIDTH = 1280; /** How long an armed tab keeps the exclusive display-media slot before another tab may take it. */ const RECORDING_ARM_GRACE_MS = 10_000; @@ -3800,66 +3793,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function concurrency: 2, discard: true, }); - const page = yield* evaluateWithDebugger<{ - url: string; - title: string; - loading: boolean; - visibleText: string; - interactiveElements: PreviewAutomationSnapshot["interactiveElements"]; - }>( + const page = yield* evaluateWithDebugger( tabId, send, - `(() => { - const selectorFor = (element) => { - if (element.id) return "#" + CSS.escape(element.id); - for (const attribute of ["data-testid", "name"]) { - const value = element.getAttribute(attribute); - if (value) return element.tagName.toLowerCase() + "[" + attribute + "=" + JSON.stringify(value) + "]"; - } - const buildParts = (current, parts = []) => { - if (!current || current.nodeType !== Node.ELEMENT_NODE || parts.length >= 8) { - return parts; - } - const parent = current.parentElement; - const siblings = parent - ? Array.from(parent.children).filter((child) => child.tagName === current.tagName) - : []; - const base = current.tagName.toLowerCase(); - const part = siblings.length > 1 - ? base + ":nth-of-type(" + (siblings.indexOf(current) + 1) + ")" - : base; - return buildParts(parent, [part, ...parts]); - }; - return buildParts(element).join(" > "); - }; - const visible = (element) => { - const style = getComputedStyle(element); - const rect = element.getBoundingClientRect(); - return style.visibility !== "hidden" && style.display !== "none" && rect.width > 0 && rect.height > 0; - }; - const elements = Array.from(document.querySelectorAll( - "a[href],button,input,textarea,select,[role],[tabindex]" - )).filter(visible).slice(0, ${MAX_INTERACTIVE_ELEMENTS}).map((element) => { - const rect = element.getBoundingClientRect(); - return { - tag: element.tagName.toLowerCase(), - role: element.getAttribute("role"), - name: (element.getAttribute("aria-label") || element.innerText || element.getAttribute("name") || "").slice(0, ${MAX_INTERACTIVE_ELEMENT_NAME_LENGTH}), - selector: selectorFor(element), - x: rect.x, - y: rect.y, - width: rect.width, - height: rect.height - }; - }); - return { - url: location.href, - title: document.title, - loading: document.readyState !== "complete", - visibleText: (document.body?.innerText || "").slice(0, ${MAX_VISIBLE_TEXT_LENGTH}), - interactiveElements: elements - }; - })()`, + snapshotPageExpression(), true, ); const [accessibility, sourceImage, diagnostics, timelines] = yield* Effect.all([ diff --git a/apps/desktop/src/preview/SnapshotPage.test.ts b/apps/desktop/src/preview/SnapshotPage.test.ts new file mode 100644 index 000000000000..0a052d7f2423 --- /dev/null +++ b/apps/desktop/src/preview/SnapshotPage.test.ts @@ -0,0 +1,937 @@ +import * as NodeVM from "node:vm"; +import { describe, expect, it } from "vite-plus/test"; + +import { snapshotPageExpression, type SnapshotPage } from "./SnapshotPage.ts"; + +const rect = (x: number, y: number, width = 100, height = 20) => ({ + x, + y, + left: x, + top: y, + right: x + width, + bottom: y + height, + width, + height, +}); + +class PageElement { + nodeType = 1; + tagName = "DIV"; + id = ""; + innerText = ""; + parentElement: PageElement | null = null; + offsetParent: PageElement | null = null; + children: PageElement[] = []; + attributes = new Map(); + clientLeft = 0; + clientTop = 0; + clientWidth = 100; + clientHeight = 20; + offsetWidth = 100; + offsetHeight = 20; + scrollWidth = 100; + scrollHeight = 20; + scrollLeft = 0; + scrollTop = 0; + bounds = rect(0, 0); + style = { + display: "block", + visibility: "visible", + opacity: "1", + contentVisibility: "visible", + overflowX: "visible", + overflowY: "visible", + position: "static", + transform: "none", + rotate: "none", + scale: "none", + perspective: "none", + filter: "none", + backdropFilter: "none", + contain: "none", + clipPath: "none", + overflowClipMargin: "0px", + willChange: "auto", + borderRightWidth: "0px", + borderBottomWidth: "0px", + }; + getAttribute(attribute: string) { + return this.attributes.get(attribute) ?? null; + } + getBoundingClientRect() { + return this.bounds; + } + checkVisibility(options: { checkVisibilityCSS?: boolean } = {}) { + const ancestors: PageElement[] = [this]; + if (this.parentElement) ancestors.push(this.parentElement); + for (let index = 0; index < ancestors.length; index++) { + const current = ancestors[index]!; + if (index > 0 && current.parentElement) ancestors.push(current.parentElement); + if ( + current.style.display === "none" || + current.style.opacity === "0" || + current.style.contentVisibility === "hidden" + ) + return false; + } + return ( + (!options.checkVisibilityCSS || this.style.visibility === "visible") && + this.style.display !== "contents" + ); + } +} + +type PageText = { + nodeType: number; + data: string; + length: number; + parentElement: PageElement; + rectangles: (start: number, end: number) => ReturnType[]; +}; + +const fixture = (escape = (value: string) => value) => { + const root = new PageElement(); + const body = new PageElement(); + body.parentElement = root; + root.children.push(body); + root.scrollHeight = 10_000; + const elements: PageElement[] = []; + const selectorMatches = new Map(); + const nodes: Array = []; + const element = (id: string, bounds: ReturnType, parent = body) => { + const value = new PageElement(); + value.id = id; + value.bounds = bounds; + value.parentElement = parent; + parent.children.push(value); + nodes.push(value); + return value; + }; + const text = ( + data: string, + parent: PageElement, + rectangles: PageText["rectangles"] = () => [parent.bounds], + ) => { + const value = { nodeType: 3, data, length: data.length, parentElement: parent, rectangles }; + nodes.push(value); + }; + let start = 0; + let end = 0; + let active: PageText; + const range = { + setStart(node: PageText, offset: number) { + active = node; + start = offset; + }, + setEnd(_node: PageText, offset: number) { + end = offset; + }, + getClientRects() { + return active.rectangles(start, end); + }, + getBoundingClientRect() { + const rectangles = this.getClientRects(); + const left = Math.min(...rectangles.map((value) => value.left)); + const top = Math.min(...rectangles.map((value) => value.top)); + return rect( + left, + top, + Math.max(...rectangles.map((value) => value.right)) - left, + Math.max(...rectangles.map((value) => value.bottom)) - top, + ); + }, + }; + const context = { + innerWidth: 300, + innerHeight: 200, + scrollX: 0, + scrollY: 4_000, + location: { href: "https://example.test" }, + CSS: { escape }, + DOMMatrixReadOnly: class { + transform: string; + constructor(transform: string) { + this.transform = transform; + } + toFloat64Array() { + const values = this.transform + .slice(this.transform.indexOf("(") + 1, -1) + .split(",") + .map(Number); + return new Float64Array( + values.length === 16 + ? values + : [ + values[0]!, + values[1]!, + 0, + 0, + values[2]!, + values[3]!, + 0, + 0, + 0, + 0, + 1, + 0, + values[4]!, + values[5]!, + 0, + 1, + ], + ); + } + }, + HTMLElement: PageElement, + Node: { ELEMENT_NODE: 1 }, + NodeFilter: { SHOW_TEXT: 4, SHOW_ELEMENT: 1 }, + getComputedStyle: (value: PageElement) => value.style, + document: { + documentElement: root, + body, + scrollingElement: root as PageElement | null, + title: "Page", + readyState: "complete", + compatMode: "CSS1Compat", + querySelectorAll(selector: string) { + if (selector === "a[href],button,input,textarea,select,[role],[tabindex]") return elements; + if (selectorMatches.has(selector)) return selectorMatches.get(selector)!; + return Array.from( + new Set([root, body, ...nodes.filter((node) => node instanceof PageElement)]), + ).filter( + (element) => + (element.id && selector === "#" + element.id) || + ["name", "data-testid"].some( + (attribute) => + selector === + element.tagName.toLowerCase() + + "[" + + attribute + + "=" + + element.getAttribute(attribute) + + "]", + ), + ); + }, + createRange: () => range, + createTreeWalker() { + let index = 0; + return { nextNode: () => nodes[index++] ?? null }; + }, + }, + }; + const capture = (scrollingElement: PageElement | null = root, compatMode = "CSS1Compat") => { + context.document.scrollingElement = scrollingElement; + context.document.compatMode = compatMode; + return NodeVM.runInNewContext(snapshotPageExpression(), context) as SnapshotPage; + }; + return { root, body, elements, nodes, selectorMatches, element, text, capture }; +}; + +describe("snapshot page collector", () => { + it.each(["id", "name", "data-testid"])( + "uses unique document paths for duplicate %s targets in deep branches", + (attribute) => { + const page = fixture(); + for (let branch = 0; branch < 2; branch++) { + let parent = page.element("", rect(0, 0)); + parent.tagName = "SECTION"; + for (let depth = 0; depth < 9; depth++) parent = page.element("", rect(0, 0), parent); + const scroller = page.element("", rect(0, 0), parent); + scroller.style.overflowY = "auto"; + scroller.scrollHeight = 1_000; + const control = page.element("", rect(0, 0), scroller); + control.tagName = "BUTTON"; + if (attribute === "id") scroller.id = control.id = "duplicate"; + else { + scroller.attributes.set(attribute, "duplicate"); + control.attributes.set(attribute, "duplicate"); + } + page.elements.push(control); + } + const snapshot = page.capture(); + const paths = [1, 2].map( + (branch) => + ":root > div > section:nth-of-type(" + + branch + + ") > " + + Array.from({ length: 10 }, () => "div").join(" > "), + ); + expect(snapshot.scroll?.containers.map((container) => container.selector)).toEqual(paths); + expect(snapshot.interactiveElements.map((element) => element.selector)).toEqual( + paths.map((path) => path + " > button"), + ); + }, + ); + + it.each(["id", "name", "data-testid"])("keeps short unique %s selectors", (attribute) => { + const page = fixture(); + const control = page.element("", rect(0, 0)); + control.tagName = "BUTTON"; + if (attribute === "id") control.id = "unique"; + else control.attributes.set(attribute, "unique"); + page.elements.push(control); + expect(page.capture().interactiveElements[0]?.selector).toBe( + attribute === "id" ? "#unique" : "button[" + attribute + "=unique]", + ); + }); + + it("uses a unique attribute when an id is duplicated", () => { + const page = fixture(); + page.element("duplicate", rect(0, 0)); + const control = page.element("duplicate", rect(0, 0)); + control.tagName = "BUTTON"; + control.attributes.set("data-testid", "unique"); + page.elements.push(control); + expect(page.capture().interactiveElements[0]?.selector).toBe("button[data-testid=unique]"); + }); + + it("reports omitted scroll containers when the unique ancestor path exceeds its budget", () => { + const page = fixture(); + let scroller = page.body; + for (let depth = 0; depth < 200; depth++) scroller = page.element("", rect(0, 0), scroller); + scroller.style.overflowY = "auto"; + scroller.scrollHeight = 1_000; + page.text("current label", scroller); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("current label"); + expect(snapshot.scroll?.containers).toEqual([]); + expect(snapshot.scroll?.containersTruncated).toBe(true); + }); + + it.each([ + ["transform", "matrix(0.707107, 0.707107, -0.707107, 0.707107, 0, 0)"], + ["transform", "matrix(1, 0, 0.5, 1, 0, 0)"], + ["transform", "matrix(-1, 0, 0, 1, 0, 0)"], + ["transform", "matrix3d(1,0,0,0,0,1,0,0,0,0,1,-0.002,0,0,0,1)"], + ["rotate", "45deg"], + ["perspective", "500px"], + ["scale", "-1 1"], + ] as const)("omits unsupported %s=%s clipping geometry", (property, value) => { + for (const effect of ["overflow", "contain", "path"]) { + for (const onAncestor of [false, true]) { + const page = fixture(); + const parent = page.element("parent", rect(0, 0)); + const scroller = page.element("scroll", rect(0, 0), parent); + (onAncestor ? parent : scroller).style[property] = value; + scroller.style.overflowY = "auto"; + scroller.scrollHeight = 1_000; + if (effect === "contain") scroller.style.contain = "paint"; + if (effect === "path") scroller.style.clipPath = "inset(0px)"; + const uncertain = page.element("uncertain", rect(10, 5, 20, 10), scroller); + page.elements.push(uncertain); + page.text("uncertain label", uncertain); + const visible = page.element("visible", rect(0, 100)); + page.elements.push(visible); + page.text("visible label", visible); + page.body.innerText = "uncertain label visible label"; + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("visible label"); + expect(snapshot.visibleText).toBe("uncertain label visible label"); + expect(snapshot.interactiveElements.map((element) => element.selector)).toEqual([ + "#visible", + ]); + expect(snapshot.scroll?.containers).toEqual([]); + expect(snapshot.scroll?.containersTruncated).toBe(true); + expect(snapshot.truncated?.viewportText).toBe(true); + expect(snapshot.truncated?.interactiveElements).toBe(true); + } + } + }); + + it.each([ + ["transform", "matrix(2, 0, 0, 3, 10, 20)"], + ["transform", "matrix(1, 0, 0, 1, 30, -10)"], + ["transform", "matrix3d(2,0,0,0,0,3,0,0,0,0,1,0,10,20,30,1)"], + ["rotate", "360deg"], + ["scale", "2 3"], + ] as const)("keeps axis-aligned %s=%s clipping", (property, value) => { + const page = fixture(); + const scroller = page.element("scroll", rect(0, 0, 200, 60)); + scroller.style[property] = value; + scroller.style.overflowY = "auto"; + scroller.style.contain = "paint"; + scroller.style.clipPath = "inset(0px)"; + scroller.scrollHeight = 1_000; + const visible = page.element("visible", rect(20, 10), scroller); + page.elements.push(visible); + page.text("visible label", visible); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("visible label"); + expect(snapshot.interactiveElements[0]?.inViewport).toBe(true); + expect(snapshot.scroll?.containers[0]?.selector).toBe("#scroll"); + expect(snapshot.scroll?.containersTruncated).toBe(false); + expect(snapshot.truncated?.viewportText).toBe(false); + expect(snapshot.truncated?.interactiveElements).toBe(false); + }); + + it("preserves positioned overflow escapes but respects transformed paint clipping", () => { + const page = fixture(); + const rotated = page.element("rotated", rect(0, 0)); + rotated.style.rotate = "45deg"; + const scroller = page.element("scroll", rect(0, 0), rotated); + scroller.style.overflowY = "auto"; + scroller.scrollHeight = 1_000; + const escaped = page.element("escaped", rect(0, 100), scroller); + escaped.style.position = "absolute"; + escaped.offsetParent = rotated; + page.elements.push(escaped); + page.text("escaped label", escaped); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("escaped label"); + expect(snapshot.interactiveElements[0]?.selector).toBe("#escaped"); + expect(snapshot.interactiveElements[0]?.inViewport).toBe(true); + scroller.style.contain = "paint"; + const painted = page.capture(); + expect(painted.viewportText).toBe(""); + expect(painted.interactiveElements).toEqual([]); + expect(painted.truncated?.interactiveElements).toBe(true); + }); + + it("keeps transformed text and controls when no geometric clipping applies", () => { + const page = fixture(); + const rotated = page.element("rotated", rect(0, 0)); + rotated.style.rotate = "45deg"; + page.elements.push(rotated); + page.text("rotated label", rotated); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("rotated label"); + expect(snapshot.interactiveElements[0]?.inViewport).toBe(true); + expect(snapshot.truncated?.viewportText).toBe(false); + expect(snapshot.truncated?.interactiveElements).toBe(false); + }); + + it.each(["inset(100%)", "inset(0px 0px 100%)", "circle(50%)"])( + "omits scroll containers clipped by %s", + (clipPath) => { + for (const onAncestor of [false, true]) { + const page = fixture(); + const parent = page.element("parent", rect(0, 0)); + const scroller = page.element("scroll", rect(0, 0), parent); + (onAncestor ? parent : scroller).style.clipPath = clipPath; + scroller.style.overflowY = "auto"; + scroller.scrollHeight = 1_000; + const snapshot = page.capture(); + expect(snapshot.scroll?.containers).toEqual([]); + expect(snapshot.scroll?.containersTruncated).toBe(clipPath === "circle(50%)"); + } + }, + ); + + it("keeps partially clipped scroll containers and omits known offscreen areas", () => { + const page = fixture(); + const visible = page.element("partial", rect(0, 0)); + visible.style.clipPath = "inset(50% 0px 0px)"; + visible.style.overflowY = "auto"; + visible.scrollHeight = 1_000; + const offscreen = page.element("offscreen", rect(0, 250)); + offscreen.style.clipPath = "inset(0px)"; + offscreen.style.overflowY = "auto"; + offscreen.scrollHeight = 1_000; + const snapshot = page.capture(); + expect(snapshot.scroll?.containers.map((container) => container.selector)).toEqual([ + "#partial", + ]); + expect(snapshot.scroll?.containersTruncated).toBe(false); + }); + + it.each(["paint", "content", "strict", "layout"])( + "respects %s containment on boxes and the body", + (contain) => { + for (const onBody of [false, true]) { + const page = fixture(); + const parent = onBody ? page.body : page.element("clip", rect(0, 0, 150, 60)); + parent.bounds = rect(0, 0, 150, 60); + parent.clientWidth = parent.offsetWidth = 150; + parent.clientHeight = parent.offsetHeight = 60; + parent.style.contain = contain; + const hidden = page.element("hidden", rect(0, 120), parent); + page.elements.push(hidden); + page.text("hidden label", hidden); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe(contain === "layout" ? "hidden label" : ""); + expect(snapshot.interactiveElements[0]?.inViewport).toBe(contain === "layout"); + } + }, + ); + + it.each(["inset(0px)", "inset(0%)", "inset(10px 10% 20px 5%)"])( + "clips text and controls with %s on boxes and the body", + (clipPath) => { + for (const onBody of [false, true]) { + const page = fixture(); + const parent = onBody ? page.body : page.element("clip", rect(0, 0, 150, 60)); + parent.bounds = rect(0, 0, 150, 60); + parent.offsetWidth = 150; + parent.offsetHeight = 60; + parent.style.clipPath = clipPath; + const hidden = page.element("hidden", rect(0, 120), parent); + const visible = page.element("visible", rect(20, 20, 50, 10), parent); + page.elements.push(hidden, visible); + page.text("hidden label", hidden); + page.text("visible label", visible); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("visible label"); + expect(snapshot.interactiveElements.map((element) => element.inViewport)).toEqual([ + true, + false, + ]); + expect(snapshot.truncated?.viewportText).toBe(false); + } + }, + ); + + it("clips positioned descendants at static clip-path ancestors", () => { + const page = fixture(); + const parent = page.element("clip", rect(0, 0, 150, 60)); + parent.style.clipPath = "inset(0px)"; + for (const position of ["fixed", "absolute"]) { + const hidden = page.element(position, rect(0, 120), parent); + hidden.style.position = position; + hidden.offsetParent = position === "absolute" ? page.body : null; + page.elements.push(hidden); + page.text("hidden label", hidden); + } + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe(""); + expect(snapshot.interactiveElements.every((element) => !element.inViewport)).toBe(true); + }); + + it.each(["inline", "table-row", "ruby", "ruby-text"])( + "does not apply paint containment to %s boxes", + (display) => { + const page = fixture(); + const parent = page.element("clip", rect(0, 0)); + parent.style.display = display; + parent.style.contain = "paint"; + const visible = page.element("visible", rect(0, 120), parent); + visible.style.position = "absolute"; + visible.offsetParent = page.body; + page.elements.push(visible); + page.text("visible label", visible); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("visible label"); + expect(snapshot.interactiveElements[0]?.inViewport).toBe(true); + }, + ); + + it("respects paint containment overflow clip margins", () => { + const page = fixture(); + const parent = page.element("clip", rect(0, 0, 100, 100)); + parent.offsetHeight = 100; + parent.style.contain = "paint"; + parent.style.overflowClipMargin = "padding-box 20px"; + const visible = page.element("visible", rect(0, 105, 50, 10), parent); + const hidden = page.element("hidden", rect(0, 140), parent); + page.elements.push(visible, hidden); + page.text("visible label", visible); + page.text("hidden label", hidden); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("visible label"); + expect(snapshot.interactiveElements.map((element) => element.inViewport)).toEqual([ + true, + false, + ]); + parent.style.overflowClipMargin = "content-box 20px"; + const omitted = page.capture(); + expect(omitted.viewportText).toBe(""); + expect(omitted.interactiveElements).toEqual([]); + expect(omitted.truncated?.viewportText).toBe(true); + expect(omitted.truncated?.interactiveElements).toBe(true); + }); + + it.each(["paint", "layout"])( + "finds independent body scrolling when %s containment prevents overflow propagation", + (contain) => { + for (const onRoot of [false, true]) { + const page = fixture(); + (onRoot ? page.root : page.body).style.contain = contain; + page.root.bounds = rect(0, 0, 300, 200); + page.body.bounds = rect(0, 0, 150, 60); + page.body.clientWidth = page.body.offsetWidth = 150; + page.body.clientHeight = page.body.offsetHeight = 60; + page.body.style.overflowY = "auto"; + page.body.scrollHeight = 500; + page.body.scrollTop = 200; + const hidden = page.element("hidden", rect(0, 120)); + page.elements.push(hidden); + page.text("hidden label", hidden); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe(""); + expect(snapshot.interactiveElements[0]?.inViewport).toBe(false); + expect(snapshot.scroll?.containers[0]).toMatchObject({ + y: 200, + height: 60, + scrollHeight: 500, + }); + } + }, + ); + + it("uses border-box inset percentages and permits negative insets", () => { + const page = fixture(); + const parent = page.element("clip", rect(0, 0, 100, 100)); + parent.clientLeft = parent.clientTop = 10; + parent.style.borderRightWidth = parent.style.borderBottomWidth = "10px"; + parent.style.clipPath = "inset(0px)"; + const label = page.element("label", rect(2, 2, 5, 5), parent); + page.text("border label", label); + expect(page.capture().viewportText).toBe("border label"); + parent.style.contain = "paint"; + expect(page.capture().viewportText).toBe(""); + parent.style.contain = "none"; + parent.style.clipPath = "inset(-50px)"; + label.bounds = rect(0, 120); + expect(page.capture().viewportText).toBe("border label"); + parent.style.clipPath = "inset(0 0 50%)"; + label.bounds = rect(0, 60); + expect(page.capture().viewportText).toBe(""); + }); + + it.each(["circle(50%)", "inset(0px round 20px)", "inset(calc(10px + 5%))"])( + "reports omitted text and controls for unsupported %s geometry", + (clipPath) => { + const page = fixture(); + const parent = page.element("clip", rect(0, 0, 100, 100)); + parent.style.clipPath = clipPath; + const uncertain = page.element("uncertain", rect(20, 20), parent); + uncertain.style.position = "fixed"; + const visible = page.element("visible", rect(0, 140)); + page.elements.push(uncertain, visible); + page.text("uncertain label", uncertain); + page.text("visible label", visible); + page.body.innerText = "uncertain label visible label"; + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("visible label"); + expect(snapshot.visibleText).toBe("uncertain label visible label"); + expect(snapshot.interactiveElements.map((element) => element.selector)).toEqual(["#visible"]); + expect(snapshot.truncated?.viewportText).toBe(true); + expect(snapshot.truncated?.interactiveElements).toBe(true); + }, + ); + + it.each(["name", "data-testid"])("escapes line breaks in %s selectors", (attribute) => { + const page = fixture((value) => value.replaceAll("\n", "\\a ")); + const control = page.element("", rect(0, 0)); + control.tagName = "BUTTON"; + control.attributes.set(attribute, "line\nnext"); + page.selectorMatches.set(`button[${attribute}=line\\a next]`, [control]); + page.elements.push(control); + expect(page.capture().interactiveElements[0]?.selector).toBe( + `button[${attribute}=line\\a next]`, + ); + }); + + it("finds image and canvas scroll areas without text or controls", () => { + const page = fixture(); + for (const tag of ["IMG", "CANVAS"]) { + const scroller = page.element(`scroll-${tag}`, rect(0, 0)); + scroller.style.overflowY = "auto"; + scroller.scrollHeight = 1_000; + scroller.scrollTop = 200; + page.element("", rect(0, 0), scroller).tagName = tag; + } + const hidden = page.element("hidden", rect(0, 0)); + hidden.style.opacity = "0"; + const hiddenScroller = page.element("hidden-scroll", rect(0, 0), hidden); + hiddenScroller.style.overflowY = "auto"; + hiddenScroller.scrollHeight = 1_000; + const clipped = page.element("clipping", rect(0, 0)); + clipped.style.overflowY = "hidden"; + const clippedScroller = page.element("clipped-scroll", rect(0, 100), clipped); + clippedScroller.style.overflowY = "auto"; + clippedScroller.scrollHeight = 1_000; + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe(""); + expect(snapshot.interactiveElements).toEqual([]); + expect( + snapshot.scroll?.containers.map((container) => [container.selector, container.y]), + ).toEqual([ + ["#scroll-IMG", 200], + ["#scroll-CANVAS", 200], + ]); + }); + + it("finds an independently scrolling image-only body", () => { + const page = fixture(); + page.root.style.overflowY = "hidden"; + page.body.style.overflowY = "auto"; + page.body.scrollHeight = 1_000; + page.element("image", rect(0, 0)).tagName = "IMG"; + expect(page.capture().scroll?.containers[0]?.height).toBe(20); + }); + + it("keeps scanning scroll areas after text is capped and reports an incomplete scan", () => { + const page = fixture(); + page.text("x".repeat(25_000), page.element("text", rect(0, 0))); + const scroller = page.element("later-scroll", rect(0, 0)); + scroller.style.overflowY = "auto"; + scroller.scrollHeight = 1_000; + expect(page.capture().scroll?.containers[0]?.selector).toBe("#later-scroll"); + const empty = page.element("empty", rect(0, 0)); + page.nodes.length = 100_001; + page.nodes.fill(empty); + const snapshot = page.capture(); + expect(snapshot.truncated?.viewportText).toBe(true); + expect(snapshot.scroll?.containersTruncated).toBe(true); + }); + + it("keeps bottom text after more than 4096 offscreen paragraphs", () => { + const page = fixture(); + for (let index = 0; index < 5_000; index++) { + page.text("offscreen", page.element(`paragraph-${index}`, rect(0, -100))); + } + page.text("bottom marker", page.element("bottom", rect(0, 100))); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("bottom marker"); + expect(snapshot.truncated?.viewportText).toBe(false); + }); + + it("reads direct display:contents text while respecting hidden ancestors", () => { + const page = fixture(); + const contents = page.element("contents", rect(0, 0, 0, 0)); + contents.style.display = "contents"; + page.text("direct contents text", contents, () => [rect(0, 20)]); + const hidden = page.element("hidden", rect(0, 40)); + hidden.style.opacity = "0"; + const hiddenContents = page.element("hidden-contents", rect(0, 0, 0, 0), hidden); + hiddenContents.style.display = "contents"; + page.text("hidden contents text", hiddenContents, () => [rect(0, 40)]); + const visibility = page.element("visibility", rect(0, 60)); + visibility.style.visibility = "hidden"; + const override = page.element("override", rect(0, 0, 0, 0), visibility); + override.style.display = "contents"; + page.text("visible override", override, () => [rect(0, 60)]); + expect(page.capture().viewportText).toBe("direct contents text visible override"); + }); + + it("captures text under deeply nested ancestors without recursive stack growth", () => { + const page = fixture(); + let parent = page.body; + for (let index = 0; index < 5_000; index++) + parent = page.element(`nested-${index}`, rect(0, 100), parent); + page.text("deep text", parent); + expect(page.capture().viewportText).toBe("deep text"); + }); + + it("clips an independently scrolling body in standard and quirks documents", () => { + const page = fixture(); + page.root.style.overflowY = "hidden"; + page.body.style.overflowY = "auto"; + page.body.bounds = rect(0, 0, 300, 100); + page.body.clientWidth = page.body.offsetWidth = 300; + page.body.clientHeight = page.body.offsetHeight = 100; + page.body.scrollHeight = 1_000; + page.body.scrollTop = 200; + const clipped = page.element("body-clipped", rect(0, 140)); + page.elements.push(clipped); + page.text("clipped body text", clipped); + for (const scrollingElement of [page.root, null]) { + const snapshot = page.capture(scrollingElement); + expect(snapshot.viewportText).toBe(""); + expect(snapshot.interactiveElements[0]?.inViewport).toBe(false); + expect(snapshot.scroll?.containers[0]).toMatchObject({ + selector: ":root > div", + y: 200, + height: 100, + scrollHeight: 1_000, + }); + } + }); + + it("uses the actual body box when quirks client dimensions report the viewport", () => { + const page = fixture(); + page.root.style.overflowX = page.root.style.overflowY = "hidden"; + page.body.style.overflowX = page.body.style.overflowY = "auto"; + page.body.bounds = rect(0, 0, 200, 200); + page.body.clientWidth = 300; + page.body.clientHeight = 800; + page.body.offsetWidth = page.body.offsetHeight = 100; + page.body.clientLeft = page.body.clientTop = 2; + page.body.style.borderRightWidth = page.body.style.borderBottomWidth = "2px"; + page.body.scrollWidth = 300; + page.body.scrollHeight = 245; + const vertical = page.element("vertical", rect(10, 198, 50, 2)); + const horizontal = page.element("horizontal", rect(198, 10, 2, 50)); + page.elements.push(vertical, horizontal); + page.text("vertical text", vertical); + page.text("horizontal text", horizontal); + const snapshot = page.capture(null, "BackCompat"); + expect(snapshot.viewportText).toBe(""); + expect(snapshot.interactiveElements.every((element) => element.inViewport === false)).toBe( + true, + ); + expect(snapshot.scroll?.containers[0]).toMatchObject({ + width: 96, + height: 96, + scrollWidth: 300, + scrollHeight: 245, + }); + page.body.offsetHeight = 1_000; + page.body.bounds = rect(0, 0, 200, 2_000); + page.body.scrollHeight = 2_000; + expect(page.capture(null, "BackCompat").scroll?.containers[0]?.height).toBe(996); + }); + + it("keeps current controls ahead of more than 200 offscreen controls", () => { + const page = fixture(); + for (let index = 0; index < 240; index++) { + const control = page.element(`button-${index}`, rect(0, index < 239 ? -1_000 : 100)); + control.tagName = "BUTTON"; + page.elements.push(control); + } + page.body.innerText = "whole page ".repeat(3_000); + page.text("on screen", page.elements[239]!); + const snapshot = page.capture(); + expect(snapshot.interactiveElements).toHaveLength(200); + expect(snapshot.interactiveElements[0]).toMatchObject({ + selector: "#button-239", + inViewport: true, + }); + expect(snapshot.viewportText).toBe("on screen"); + expect(snapshot.visibleText).toHaveLength(20_000); + expect(snapshot.truncated).toEqual({ + visibleText: true, + viewportText: false, + interactiveElements: true, + }); + expect(snapshot.scroll).toMatchObject({ y: 4_000, height: 200, scrollHeight: 10_000 }); + }); + + it("clips nested scroll text and controls without changing scroll offsets", () => { + const page = fixture(); + const outer = page.element("outer", rect(10, 10, 100, 100)); + outer.clientHeight = 100; + outer.offsetHeight = 100; + outer.style.overflowY = "auto"; + outer.scrollHeight = 1_000; + outer.scrollTop = 600; + const inner = page.element("inner", rect(20, 30, 80, 40), outer); + inner.clientWidth = inner.offsetWidth = 80; + inner.clientHeight = inner.offsetHeight = 40; + inner.style.overflowX = "hidden"; + inner.style.overflowY = "clip"; + const visible = page.element("visible", rect(20, 35, 70, 20), inner); + const clipped = page.element("clipped", rect(20, 80, 70, 20), inner); + page.elements.push(clipped, visible); + page.text("visible words", visible); + page.text("hidden under scroll", clipped); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("visible words"); + expect(snapshot.interactiveElements[0]).toMatchObject({ + selector: "#visible", + inViewport: true, + }); + expect(snapshot.interactiveElements[1]).toMatchObject({ + selector: "#clipped", + inViewport: false, + }); + expect(snapshot.scroll?.containers).toEqual([ + { + selector: "#outer", + x: 0, + y: 600, + width: 100, + height: 100, + scrollWidth: 100, + scrollHeight: 1_000, + }, + ]); + expect(outer.scrollTop).toBe(600); + }); + + it("reads only visible characters from a huge wrapped text node", () => { + const page = fixture(); + const parent = page.element("long", rect(0, -100_000, 100, 200_000)); + const data = "x".repeat(100_000) + "VISIBLE" + "z".repeat(100_000); + page.text(data, parent, (start: number, end: number) => { + const rectangles = []; + if (start < 100_000) rectangles.push(rect(0, -100, 100, 20)); + if (end > 100_000 && start < 100_007) rectangles.push(rect(0, 100, 100, 20)); + if (end > 100_007) rectangles.push(rect(0, 500, 100, 20)); + return rectangles; + }); + expect(page.capture().viewportText).toBe("VISIBLE"); + }); + + it("clips positioned descendants at their containing block", () => { + const page = fixture(); + const scroller = page.element("scroller", rect(0, 0)); + scroller.style.overflowY = "auto"; + scroller.scrollHeight = 1_000; + const fixed = page.element("fixed", rect(180, 100), scroller); + fixed.style.position = "fixed"; + const absolute = page.element("absolute", rect(0, 140), scroller); + absolute.style.position = "absolute"; + absolute.offsetParent = page.body; + const transformed = page.element("transformed", rect(0, 0)); + transformed.style.overflowY = "hidden"; + transformed.style.transform = "matrix(1, 0, 0, 1, 0, 0)"; + const clipped = page.element("clipped", rect(0, 160), transformed); + clipped.style.position = "fixed"; + clipped.offsetParent = transformed; + page.elements.push(fixed, absolute, clipped); + page.text("fixed label", fixed); + page.text("absolute label", absolute); + page.text("clipped label", clipped); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("fixed label absolute label"); + expect( + snapshot.interactiveElements.map((element) => [element.selector, element.inViewport]), + ).toEqual([ + ["#fixed", true], + ["#absolute", true], + ["#clipped", false], + ]); + }); + + it("omits hidden styles and clips horizontal text", () => { + const page = fixture(); + const transparent = page.element("transparent", rect(0, 0)); + transparent.style.opacity = "0"; + const hidden = page.element("hidden", rect(0, 20)); + hidden.style.contentVisibility = "hidden"; + const visibility = page.element("visibility", rect(0, 40)); + visibility.style.visibility = "hidden"; + for (const parent of [transparent, hidden, visibility]) page.text("secret", parent); + const clipped = page.element("horizontal", rect(-100, 60, 500, 20)); + page.text("abcdefghijklmnopqrst", clipped, (start: number, end: number) => [ + rect(-100 + start * 25, 60, (end - start) * 25, 20), + ]); + expect(page.capture().viewportText).toBe("efghijklmnop"); + }); + + it("reports capped viewport text and scroll container metadata", () => { + let selectorsRead = 0; + const page = fixture((value) => { + selectorsRead++; + return value; + }); + for (let index = 0; index < 21; index++) { + const container = page.element(`scroll-${index}`, rect(0, 0)); + container.style.overflowY = "scroll"; + container.scrollHeight = 100; + page.text("text", container); + } + const large = page.element("large", rect(0, 100)); + page.text("x".repeat(25_000), large); + const snapshot = page.capture(); + expect(snapshot.scroll?.containers).toHaveLength(20); + expect(snapshot.scroll?.containersTruncated).toBe(true); + expect(selectorsRead).toBe(20); + expect(snapshot.viewportText).toHaveLength(20_000); + expect(snapshot.truncated?.viewportText).toBe(true); + }); + + it("omits oversized scroll selectors while preserving current text", () => { + const page = fixture(); + const container = page.element("x".repeat(2_000), rect(0, 0)); + container.style.overflowY = "scroll"; + container.scrollHeight = 1_000; + page.text("current text", container); + const snapshot = page.capture(); + expect(snapshot.viewportText).toBe("current text"); + expect(snapshot.scroll?.containers).toEqual([]); + expect(snapshot.scroll?.containersTruncated).toBe(true); + }); +}); diff --git a/apps/desktop/src/preview/SnapshotPage.ts b/apps/desktop/src/preview/SnapshotPage.ts new file mode 100644 index 000000000000..fc36863da7cd --- /dev/null +++ b/apps/desktop/src/preview/SnapshotPage.ts @@ -0,0 +1,522 @@ +import type { PreviewAutomationSnapshot } from "@t3tools/contracts"; + +export type SnapshotPage = Pick< + PreviewAutomationSnapshot, + | "url" + | "title" + | "loading" + | "visibleText" + | "viewportText" + | "scroll" + | "truncated" + | "interactiveElements" +>; + +function collectSnapshotPage(): SnapshotPage { + const maxTextLength = 20_000; + const maxElements = 200; + const viewport = { left: 0, top: 0, right: innerWidth, bottom: innerHeight }; + type Bounds = typeof viewport; + const intersects = (rect: Bounds, clip: Bounds) => + rect.right > clip.left && + rect.left < clip.right && + rect.bottom > clip.top && + rect.top < clip.bottom; + const selectorTargets = new Map(); + const uniquelyTargets = (selector: string, element: Element) => { + if (!selectorTargets.has(selector)) { + const matches = document.querySelectorAll(selector); + selectorTargets.set(selector, matches.length === 1 ? matches[0]! : null); + } + return selectorTargets.get(selector) === element; + }; + const selectorFor = (element: Element): string => { + if (element.id) { + const selector = "#" + CSS.escape(element.id); + if (uniquelyTargets(selector, element)) return selector; + } + for (const attribute of ["data-testid", "name"]) { + const value = element.getAttribute(attribute); + if (value) { + const selector = + element.tagName.toLowerCase() + "[" + attribute + "=" + CSS.escape(value) + "]"; + if (uniquelyTargets(selector, element)) return selector; + } + } + const parts: string[] = []; + for (let current: Element | null = element; current; current = current.parentElement) { + const siblings = current.parentElement + ? Array.from(current.parentElement.children).filter( + (child) => child.tagName === current.tagName, + ) + : []; + parts.push( + (current === document.documentElement ? ":root" : current.tagName.toLowerCase()) + + (siblings.length > 1 ? ":nth-of-type(" + (siblings.indexOf(current) + 1) + ")" : ""), + ); + } + return parts.toReversed().join(" > "); + }; + const containers: Array["containers"][number]> = []; + let containersTruncated = false; + const clips = new Map(); + const paintClips = new Map(); + const unsupportedClips = new Set(); + const unsupportedOverflowClips = new Set(); + const transformedClips = new Map(); + let unsupportedClip = false; + const styles = new Map(); + const styleFor = (element: Element) => { + let style = styles.get(element); + if (!style) { + style = getComputedStyle(element); + styles.set(element, style); + } + return style; + }; + const unsupportedTransform = (style: CSSStyleDeclaration) => { + if (style.perspective && style.perspective !== "none") return true; + if ( + style.scale && + style.scale !== "none" && + style.scale.split(/\s+/).some((value) => Number.parseFloat(value) <= 0) + ) + return true; + if (style.rotate && style.rotate !== "none") { + const angle = /([+-]?(?:\d+(?:\.\d*)?|\.\d+))(deg|rad|grad|turn)$/.exec(style.rotate); + const period = + angle?.[2] === "deg" + ? 360 + : angle?.[2] === "rad" + ? 2 * Math.PI + : angle?.[2] === "grad" + ? 400 + : 1; + if (!angle || Math.abs(Number(angle[1]) % period) > 1e-8) return true; + } + if (!style.transform || style.transform === "none") return false; + const values = new DOMMatrixReadOnly(style.transform).toFloat64Array(); + return ( + values[0]! <= 0 || + values[5]! <= 0 || + values[15] !== 1 || + values.some( + (value, index) => ![0, 5, 10, 12, 13, 14, 15].includes(index) && Math.abs(value) > 1e-8, + ) + ); + }; + const paintClipFor = (element: Element): Bounds | null => { + const pending: Element[] = []; + let ancestor: Element | null = element; + while (ancestor && !paintClips.has(ancestor)) { + pending.push(ancestor); + ancestor = ancestor.parentElement; + } + let clip = ancestor ? (paintClips.get(ancestor) ?? null) : viewport; + let unsupported = ancestor ? unsupportedClips.has(ancestor) : false; + let transformed = ancestor ? (transformedClips.get(ancestor) ?? false) : false; + for (let index = pending.length - 1; index >= 0; index--) { + const current = pending[index]!; + const style = styleFor(current); + transformed ||= unsupportedTransform(style); + transformedClips.set(current, transformed); + if (clip && style.display !== "contents") { + const paint = + /\b(paint|content|strict)\b/.test(style.contain) && + !/^(inline$|table-(?!cell$|caption$)|ruby(?:$|-))/.test(style.display); + const path = style.clipPath && style.clipPath !== "none"; + if (paint || path) { + if (transformed) { + unsupportedClip = true; + unsupported = true; + unsupportedClips.add(current); + clip = null; + paintClips.set(current, clip); + continue; + } + const rect = current.getBoundingClientRect(); + const scaleX = + current instanceof HTMLElement && current.offsetWidth + ? rect.width / current.offsetWidth + : 1; + const scaleY = + current instanceof HTMLElement && current.offsetHeight + ? rect.height / current.offsetHeight + : 1; + let left = rect.left; + let top = rect.top; + let right = rect.right; + let bottom = rect.bottom; + if (paint) { + const margin = /^(?:padding-box )?(\d+(?:\.\d+)?|\.\d+)px$/.exec( + style.overflowClipMargin || "0px", + ); + if (!margin) { + unsupportedClip = true; + unsupported = true; + unsupportedClips.add(current); + clip = null; + paintClips.set(current, clip); + continue; + } + const outset = Number.parseFloat(margin[1]!); + left += (current.clientLeft - outset) * scaleX; + top += (current.clientTop - outset) * scaleY; + right -= (Number.parseFloat(style.borderRightWidth) - outset) * scaleX; + bottom -= (Number.parseFloat(style.borderBottomWidth) - outset) * scaleY; + } + if (path) { + const values = /^inset\(([^()]+)\)$/.exec(style.clipPath)?.[1]?.trim().split(/\s+/); + if ( + !values || + values.length > 4 || + values.some((value) => !/^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:px|%)?$/.test(value)) + ) { + unsupportedClip = true; + unsupported = true; + unsupportedClips.add(current); + clip = null; + paintClips.set(current, clip); + continue; + } + const offsets = [ + values[0]!, + values[1] ?? values[0]!, + values[2] ?? values[0]!, + values[3] ?? values[1] ?? values[0]!, + ].map( + (value, side) => + Number.parseFloat(value) * + (value.endsWith("%") + ? (side % 2 === 0 ? rect.height : rect.width) / 100 + : side % 2 === 0 + ? scaleY + : scaleX), + ); + left = paint ? Math.max(left, rect.left + offsets[3]!) : rect.left + offsets[3]!; + top = paint ? Math.max(top, rect.top + offsets[0]!) : rect.top + offsets[0]!; + right = paint ? Math.min(right, rect.right - offsets[1]!) : rect.right - offsets[1]!; + bottom = paint + ? Math.min(bottom, rect.bottom - offsets[2]!) + : rect.bottom - offsets[2]!; + } + clip = { + left: Math.max(clip.left, left), + top: Math.max(clip.top, top), + right: Math.min(clip.right, right), + bottom: Math.min(clip.bottom, bottom), + }; + if (clip.right <= clip.left || clip.bottom <= clip.top) clip = null; + } + } + if (unsupported) unsupportedClips.add(current); + paintClips.set(current, clip); + } + return paintClips.get(element)!; + }; + const clipFor = (element: Element): Bounds | null => { + const pending: Element[] = []; + let ancestor: Element | null = element; + while (ancestor && !clips.has(ancestor)) { + pending.push(ancestor); + ancestor = + ancestor instanceof HTMLElement && /^(absolute|fixed)$/.test(styleFor(ancestor).position) + ? ancestor.offsetParent + : ancestor.parentElement; + } + let parentClip: Bounds | null = ancestor ? (clips.get(ancestor) ?? null) : viewport; + let unsupportedOverflow = ancestor ? unsupportedOverflowClips.has(ancestor) : false; + for (let index = pending.length - 1; index >= 0; index--) { + const current = pending[index]!; + const style = styleFor(current); + if ( + style.display === "none" || + style.opacity === "0" || + style.contentVisibility === "hidden" + ) { + clips.set(current, null); + parentClip = null; + continue; + } + const clip = { ...(parentClip ?? viewport) }; + const paintClip = paintClipFor(current); + if (paintClip) { + clip.left = Math.max(clip.left, paintClip.left); + clip.top = Math.max(clip.top, paintClip.top); + clip.right = Math.min(clip.right, paintClip.right); + clip.bottom = Math.min(clip.bottom, paintClip.bottom); + } + const rootStyle = styleFor(document.documentElement); + const viewportBody = + current === document.body && + style.contain === "none" && + rootStyle.contain === "none" && + style.contentVisibility === "visible" && + rootStyle.contentVisibility === "visible" && + rootStyle.overflowX === "visible" && + rootStyle.overflowY === "visible"; + if (current !== document.documentElement && !viewportBody && style.display !== "contents") { + const clipsX = /^(auto|scroll|hidden|clip)$/.test(style.overflowX); + const clipsY = /^(auto|scroll|hidden|clip)$/.test(style.overflowY); + if (clipsX || clipsY) { + const rect = current.getBoundingClientRect(); + const quirksBody = + current === document.body && + current instanceof HTMLElement && + document.compatMode === "BackCompat"; + const clientWidth = quirksBody + ? Math.max( + 0, + current.offsetWidth - + current.clientLeft - + Number.parseFloat(style.borderRightWidth), + ) + : current.clientWidth; + const clientHeight = quirksBody + ? Math.max( + 0, + current.offsetHeight - + current.clientTop - + Number.parseFloat(style.borderBottomWidth), + ) + : current.clientHeight; + const scaleX = + current instanceof HTMLElement && current.offsetWidth + ? rect.width / current.offsetWidth + : 1; + const scaleY = + current instanceof HTMLElement && current.offsetHeight + ? rect.height / current.offsetHeight + : 1; + if (clipsX) { + clip.left = Math.max(clip.left, rect.left + current.clientLeft * scaleX); + clip.right = Math.min( + clip.right, + rect.left + (current.clientLeft + clientWidth) * scaleX, + ); + } + if (clipsY) { + clip.top = Math.max(clip.top, rect.top + current.clientTop * scaleY); + clip.bottom = Math.min( + clip.bottom, + rect.top + (current.clientTop + clientHeight) * scaleY, + ); + } + const scrollable = + (/^(auto|scroll)$/.test(style.overflowX) && current.scrollWidth > clientWidth) || + (/^(auto|scroll)$/.test(style.overflowY) && current.scrollHeight > clientHeight); + if (transformedClips.get(current)) { + unsupportedClip = true; + unsupportedOverflow = true; + } + if (scrollable && (unsupportedClips.has(current) || unsupportedOverflow)) + containersTruncated = true; + if ( + scrollable && + !unsupportedOverflow && + parentClip && + paintClip && + clip.right > clip.left && + clip.bottom > clip.top && + intersects(rect, clip) + ) { + const selector = containers.length < 20 ? selectorFor(current) : null; + if (selector !== null && selector.length <= 1_000) { + containers.push({ + selector, + x: current.scrollLeft, + y: current.scrollTop, + width: clientWidth, + height: clientHeight, + scrollWidth: current.scrollWidth, + scrollHeight: current.scrollHeight, + }); + } else containersTruncated = true; + } + } + } + if (unsupportedOverflow) unsupportedOverflowClips.add(current); + parentClip = + !unsupportedOverflow && + parentClip && + paintClip && + clip.right > clip.left && + clip.bottom > clip.top + ? clip + : null; + clips.set(current, parentClip); + } + return clips.get(element)!; + }; + const rendered = (element: Element) => { + const style = styleFor(element); + let box = element; + while (styleFor(box).display === "contents" && box.parentElement) box = box.parentElement; + return ( + style.visibility !== "hidden" && + style.visibility !== "collapse" && + style.display !== "none" && + box.checkVisibility({ + checkOpacity: true, + contentVisibilityAuto: true, + }) + ); + }; + const inViewport = (element: Element, rect: DOMRect) => { + if (!intersects(rect, viewport)) return false; + const clip = clipFor(element); + return clip !== null && intersects(rect, clip); + }; + const currentElements: Element[] = []; + const otherElements: Element[] = []; + let elementCount = 0; + let controlsOmitted = false; + for (const element of document.querySelectorAll( + "a[href],button,input,textarea,select,[role],[tabindex]", + )) { + if (!rendered(element)) continue; + const rect = element.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) continue; + paintClipFor(element); + const current = inViewport(element, rect); + if (unsupportedClips.has(element) || unsupportedOverflowClips.has(element)) { + controlsOmitted = true; + continue; + } + elementCount++; + const target = current ? currentElements : otherElements; + if (target.length < maxElements) target.push(element); + } + const interactiveElements = [...currentElements, ...otherElements] + .slice(0, maxElements) + .map((element) => { + const rect = element.getBoundingClientRect(); + return { + tag: element.tagName.toLowerCase(), + role: element.getAttribute("role"), + name: ( + element.getAttribute("aria-label") || + (element instanceof HTMLElement ? element.innerText : "") || + element.getAttribute("name") || + "" + ).slice(0, 200), + selector: selectorFor(element), + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + inViewport: currentElements.includes(element), + }; + }); + let viewportText = ""; + let viewportTextTruncated = false; + let rangeReads = 0; + let nodeText = ""; + const range = document.createRange(); + const appendText = (value: string) => { + const normalized = value.replace(/\s+/g, " "); + if (!normalized.trim()) return; + const separator = + viewportText && !viewportText.endsWith(" ") && !normalized.startsWith(" ") ? " " : ""; + const remaining = maxTextLength - viewportText.length; + const addition = separator + normalized; + viewportText += addition.slice(0, remaining); + if (addition.length > remaining) viewportTextTruncated = true; + }; + const readText = (node: Text, start: number, end: number, clip: Bounds) => { + if (viewportTextTruncated || start === end) return; + range.setStart(node, start); + range.setEnd(node, end); + const bounds = range.getBoundingClientRect(); + if (!intersects(bounds, clip)) return; + if (++rangeReads > 4_096) { + viewportTextTruncated = true; + return; + } + if ( + bounds.left >= clip.left && + bounds.right <= clip.right && + bounds.top >= clip.top && + bounds.bottom <= clip.bottom + ) { + nodeText += node.data.slice(start, end); + return; + } + if (!Array.from(range.getClientRects()).some((rect) => intersects(rect, clip))) return; + if (end - start === 1) { + nodeText += node.data.slice(start, end); + return; + } + const middle = start + Math.floor((end - start) / 2); + readText(node, start, middle, clip); + readText(node, middle, end, clip); + }; + if (document.body) { + if (rendered(document.body)) clipFor(document.body); + const walker = document.createTreeWalker( + document.body, + NodeFilter.SHOW_ELEMENT | NodeFilter.SHOW_TEXT, + ); + let node = walker.nextNode(); + let visited = 0; + while (node) { + if (++visited > 100_000) { + viewportTextTruncated = true; + containersTruncated = true; + break; + } + if (node.nodeType === Node.ELEMENT_NODE) { + const element = node as Element; + const style = styleFor(element); + if (/^(auto|scroll)$/.test(style.overflowX) || /^(auto|scroll)$/.test(style.overflowY)) { + if (rendered(element) && intersects(element.getBoundingClientRect(), viewport)) + clipFor(element); + } + node = walker.nextNode(); + continue; + } + const parent = node.parentElement; + if ( + !viewportTextTruncated && + parent && + !/^(SCRIPT|STYLE|NOSCRIPT|TEMPLATE)$/.test(parent.tagName) && + rendered(parent) + ) { + const clip = clipFor(parent); + if (clip) { + nodeText = ""; + readText(node as Text, 0, (node as Text).length, clip); + appendText(nodeText); + } + } + node = walker.nextNode(); + } + } + const visibleText = document.body?.innerText || ""; + const root = document.scrollingElement ?? document.documentElement; + return { + url: location.href, + title: document.title, + loading: document.readyState !== "complete", + visibleText: visibleText.slice(0, maxTextLength), + viewportText: viewportText.trim(), + interactiveElements, + scroll: { + x: scrollX, + y: scrollY, + width: innerWidth, + height: innerHeight, + scrollWidth: root.scrollWidth, + scrollHeight: root.scrollHeight, + containers, + containersTruncated, + }, + truncated: { + visibleText: visibleText.length > maxTextLength, + viewportText: viewportTextTruncated || unsupportedClip, + interactiveElements: elementCount > maxElements || controlsOmitted, + }, + }; +} + +export const snapshotPageExpression = () => `(${collectSnapshotPage.toString()})()`; diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index 96a0de29f3d0..bc4f4c4bba0b 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -1,18 +1,35 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off +import * as NodeHttp from "node:http"; +import * as NodeVM from "node:vm"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import { expect, it } from "@effect/vitest"; import { NodeHttpServer } from "@effect/platform-node"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { EnvironmentId, PreviewTabId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import { + EnvironmentId, + PreviewTabId, + ProviderInstanceId, + ThreadId, + type PreviewAutomationRequest, +} from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import { McpProtocol, McpSchema, McpServer } from "effect/unstable/ai"; -import { HttpBody, HttpClient, HttpRouter, HttpServerResponse } from "effect/unstable/http"; +import { + FetchHttpClient, + HttpBody, + HttpClient, + HttpRouter, + HttpServer, + HttpServerResponse, +} from "effect/unstable/http"; import * as ProjectService from "../project/ProjectService.ts"; import * as ServerConfig from "../config.ts"; @@ -25,6 +42,7 @@ const threadId = ThreadId.make("thread-mcp-test"); const tabId = PreviewTabId.make("tab-mcp-test"); const alternateTabId = PreviewTabId.make("tab-mcp-alternate"); const decodeJsonText = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown)); +const encodeJsonText = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); const invocation = { environmentId, requestNamespace: "provider-session-mcp-test", @@ -49,12 +67,12 @@ const client = McpSchema.McpServerClient.of({ }, getClient: Effect.die("unused"), }); -const TestLayer = McpHttpServer.PreviewToolkitRegistrationLive.pipe( +const PreviewTestLayer = McpHttpServer.PreviewToolkitRegistrationLive.pipe( Layer.provideMerge(McpServer.McpServer.layer), Layer.provideMerge(PreviewAutomationBroker.layer), Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-mcp-http-server-test-" })), - Layer.provideMerge(NodeServices.layer), ); +const TestLayer = PreviewTestLayer.pipe(Layer.provideMerge(NodeServices.layer)); const PullRequestsTestLayer = McpHttpServer.PullRequestsToolkitRegistrationLive.pipe( Layer.provideMerge(McpServer.McpServer.layer), Layer.provide( @@ -118,6 +136,132 @@ const callSnapshot = (args: Record) => ); }); +const callReadText = (args: Record, scope = invocation) => + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + return yield* server + .callTool({ name: "preview_read_text", arguments: args }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, scope), + Effect.provideService(McpSchema.McpServerClient, client), + ); + }); + +const serveTextCaptures = ( + clientId: string, + text: string, + options: { + switchTab?: boolean; + snapshotFailure?: boolean; + beforeSnapshot?: Effect.Effect; + } = {}, +) => + Effect.gen(function* () { + const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker; + const connected = yield* Deferred.make(); + const switched = yield* Deferred.make(); + const requests: PreviewAutomationRequest[] = []; + const timers = new Set<() => void>(); + const listeners = new Map void>(); + let textReads = 0; + const context = NodeVM.createContext({ + TextEncoder, + document: { + body: { + get innerText() { + textReads++; + return text; + }, + }, + }, + location: { href: snapshotResult.url }, + setTimeout: (callback: () => void) => { + timers.add(callback); + return callback; + }, + clearTimeout: (callback: () => void) => timers.delete(callback), + addEventListener: (event: string, callback: () => void) => listeners.set(event, callback), + removeEventListener: (event: string, callback: () => void) => { + if (listeners.get(event) === callback) listeners.delete(event); + }, + }); + const events = yield* broker.connect({ clientId, environmentId }); + yield* Stream.runForEach(events, (event) => { + if (event.type === "connected") return Deferred.succeed(connected, undefined); + requests.push(event.request); + return Effect.gen(function* () { + const request = event.request; + let result: unknown = snapshotResult; + if (request.operation === "snapshot") { + if (options.beforeSnapshot) yield* options.beforeSnapshot; + if (options.snapshotFailure) { + yield* broker.respond({ + clientId, + connectionId: event.connectionId, + requestId: request.requestId, + ok: false, + error: { + _tag: "PreviewAutomationExecutionError", + message: "private snapshot failure", + }, + }); + return; + } + } + if (request.operation === "status") { + result = { + available: true, + visible: true, + tabId: request.tabId === alternateTabId ? alternateTabId : tabId, + url: snapshotResult.url, + title: snapshotResult.title, + loading: false, + }; + } else if (request.operation === "evaluate") { + if (options.switchTab) yield* Deferred.await(switched); + const expression = (request.input as { expression: string }).expression; + try { + result = NodeVM.runInContext(expression, context); + } catch { + yield* broker.respond({ + clientId, + connectionId: event.connectionId, + requestId: request.requestId, + ok: false, + error: { + _tag: "PreviewAutomationExecutionError", + message: "private renderer failure", + }, + }); + return; + } + } + yield* broker.respond({ + clientId, + connectionId: event.connectionId, + requestId: request.requestId, + ok: true, + result, + }); + if ( + options.switchTab && + request.operation === "status" && + request.tabId !== alternateTabId + ) { + yield* broker.invoke({ + scope: invocation, + operation: "status", + tabId: alternateTabId, + input: {}, + }); + yield* Deferred.succeed(switched, undefined); + } + }).pipe(Effect.forkScoped, Effect.asVoid); + }).pipe(Effect.forkScoped); + yield* Deferred.await(connected); + return { requests, context, timers, listeners, textReads: () => textReads }; + }); + it("normalizes empty successful notification responses to accepted", () => { const notificationResponse = McpHttpServer.normalizeMcpHttpResponse( HttpServerResponse.text("", { status: 200, contentType: "application/json" }), @@ -217,19 +361,21 @@ it.effect.each([ ).pipe(Effect.provide(TestLayer)), ); -it.effect("tells the agent how to fall back when no desktop app can run the snapshot", () => - Effect.gen(function* () { - const snapshot = yield* callSnapshot({}); - - expect(snapshot.isError).toBe(true); - const [text] = snapshot.content; - expect(text?.type === "text" ? text.text : "").toContain( - "use a headless browser from the shell", - ); - expect(snapshot.structuredContent).toMatchObject({ - error: { _tag: "PreviewAutomationNoAvailableHostError" }, - }); - }).pipe(Effect.provide(TestLayer)), +it.effect.each([{}, { captureText: true }])( + "tells the agent how to fall back when no desktop app can run the snapshot %j", + (args) => + Effect.gen(function* () { + const snapshot = yield* callSnapshot(args); + + expect(snapshot.isError).toBe(true); + const [text] = snapshot.content; + expect(text?.type === "text" ? text.text : "").toContain( + "use a headless browser from the shell", + ); + expect(snapshot.structuredContent).toMatchObject({ + error: { _tag: "PreviewAutomationNoAvailableHostError" }, + }); + }).pipe(Effect.provide(TestLayer)), ); it.effect.each([ @@ -353,26 +499,55 @@ it.effect.each([ ).pipe(Effect.provide(TestLayer)), ); -it.effect("rejects non-boolean snapshot image options before selecting a browser host", () => - Effect.gen(function* () { - const server = yield* McpServer.McpServer; - for (const includeImage of ["false", 0, null]) { - const result = yield* server - .callTool({ - name: "preview_snapshot", - arguments: { includeImage }, - }) +it.effect.each(["includeImage", "captureText"])( + "rejects non-boolean snapshot %s options before selecting a browser host", + (option) => + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + for (const value of ["false", 0, null]) { + const result = yield* server + .callTool({ + name: "preview_snapshot", + arguments: { [option]: value }, + }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + Effect.provideService(McpSchema.McpServerClient, client), + ); + expect(result.isError).toBe(true); + expect(result.content).toEqual([ + { type: "text", text: "Preview snapshot failed: AiError." }, + ]); + expect(result.structuredContent).toEqual({ + error: { _tag: "AiError", operation: "snapshot", failureCount: 1 }, + }); + } + }).pipe(Effect.provide(TestLayer)), +); + +it.effect("accepts a snapshot call with the arguments field omitted", () => + Effect.scoped( + Effect.gen(function* () { + const inputs = yield* serveSnapshots("mcp-omitted-snapshot-arguments-client", snapshotResult); + const server = yield* McpServer.McpServer; + + const snapshot = yield* server + .callTool({ name: "preview_snapshot" }) .pipe( Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), Effect.provideService(McpSchema.McpServerClient, client), ); - expect(result.isError).toBe(true); - expect(result.content).toEqual([{ type: "text", text: "Preview snapshot failed: AiError." }]); - expect(result.structuredContent).toEqual({ - error: { _tag: "AiError", operation: "snapshot", failureCount: 1 }, + + expect(snapshot.isError).toBe(false); + expect(inputs).toEqual([{}]); + expect(snapshot.structuredContent).toMatchObject({ + url: snapshotResult.url, + title: snapshotResult.title, + visibleText: snapshotResult.visibleText, }); - } - }).pipe(Effect.provide(TestLayer)), + expect(snapshot.content.some((content) => content.type === "image")).toBe(true); + }), + ).pipe(Effect.provide(TestLayer)), ); it.effect("saves the snapshot PNG on request and reports its path", () => @@ -397,7 +572,9 @@ it.effect("saves the snapshot PNG on request and reports its path", () => ); expect(Buffer.from(yield* fileSystem.readFile(screenshotPath!)).toString()).toBe("png"); const [, text] = snapshot.content; - expect(text?.type === "text" ? text.text : "").toContain(screenshotPath); + expect(text?.type === "text" ? decodeJsonText(text.text) : null).toMatchObject({ + screenshotPath, + }); const unsaved = yield* callSnapshot({}); expect(unsaved.structuredContent).not.toHaveProperty("screenshotPath"); @@ -436,6 +613,414 @@ it.effect("reports a tagged error when the screenshot cannot be saved", () => ).pipe(Effect.provide(TestLayer)), ); +it.effect.each([ + { save: false, includeImage: false }, + { save: true, includeImage: false }, + { save: true, includeImage: true }, +])("reads complete loaded text from memory with snapshot options %j", (options) => + Effect.scoped( + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const loadedText = "a".repeat(4095) + "πŸ˜€δΈ­ζ–‡\n".repeat(12_000) + "Offscreen loaded end"; + const host = yield* serveTextCaptures("mcp-text-capture-client", loadedText, { + switchTab: true, + }); + + const snapshot = yield* callSnapshot({ captureText: true, ...options }); + + expect(snapshot.isError).toBe(false); + const captured = snapshot.structuredContent as { + readonly textCaptureId: string; + readonly textTabId: string; + readonly screenshotPath?: string; + }; + expect(captured).toMatchObject({ + textCaptureId: expect.any(String), + textChars: loadedText.length, + textUrl: snapshotResult.url, + textTabId: tabId, + }); + expect(captured).not.toHaveProperty("textPath"); + expect(captured).not.toHaveProperty("textBytes"); + expect(host.textReads()).toBe(1); + const beforeRead = [...host.requests]; + expect(beforeRead.filter((request) => request.operation === "status")).toHaveLength(2); + const pageRequests = beforeRead.filter((request) => request.operation !== "status"); + expect(pageRequests.every((request) => request.tabId === tabId)).toBe(true); + expect(pageRequests.filter((request) => request.operation === "snapshot")).toEqual([ + expect.objectContaining({ operation: "snapshot", input: {} }), + ]); + expect(beforeRead.length).toBeLessThan(8); + const texts = snapshot.content.filter((content) => content.type === "text"); + const metadata = texts[options.save && !options.includeImage ? 0 : 1]; + expect(metadata?.type === "text" ? decodeJsonText(metadata.text) : null).toMatchObject({ + textCaptureId: captured.textCaptureId, + textChars: loadedText.length, + textUrl: snapshotResult.url, + textTabId: tabId, + }); + expect( + metadata?.type === "text" ? Buffer.byteLength(metadata.text, "utf8") : Infinity, + ).toBeLessThanOrEqual(McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES); + expect(encodeJsonText(captured)).not.toContain("Offscreen loaded end"); + expect(snapshot.content.some((content) => content.type === "image")).toBe( + options.includeImage, + ); + if (options.save) { + expect(path.dirname(captured.screenshotPath!)).toBe(config.browserArtifactsDir); + expect(Buffer.from(yield* fs.readFile(captured.screenshotPath!)).toString()).toBe("png"); + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toEqual([ + path.basename(captured.screenshotPath!), + ]); + } else { + expect(captured).not.toHaveProperty("screenshotPath"); + expect(yield* fs.exists(config.browserArtifactsDir)).toBe(false); + } + if (options.save && !options.includeImage) { + expect(Object.keys(captured).sort()).toEqual([ + "screenshotPath", + "textCaptureId", + "textChars", + "textTabId", + "textUrl", + "url", + ]); + expect(snapshot.content).toHaveLength(1); + } + + let offset = 0; + let complete = ""; + while (offset < loadedText.length) { + const result = yield* callReadText({ + captureId: captured.textCaptureId, + tabId: captured.textTabId, + offset, + }); + expect(result.isError).toBe(false); + const chunk = result.structuredContent as { + readonly text: string; + readonly nextOffset: number; + readonly totalChars: number; + readonly done: boolean; + readonly released: boolean; + }; + expect(chunk.text.length).toBeLessThanOrEqual(4096); + expect(chunk.nextOffset).toBe(offset + chunk.text.length); + expect(chunk.nextOffset).toBeGreaterThan(offset); + expect(chunk.totalChars).toBe(loadedText.length); + expect(chunk.done).toBe(chunk.nextOffset === loadedText.length); + expect(chunk.released).toBe(false); + expect(chunk.text.isWellFormed()).toBe(true); + const [content] = result.content; + expect(content?.type === "text" ? decodeJsonText(content.text) : null).toEqual(chunk); + expect(Buffer.byteLength(encodeJsonText(chunk))).toBeLessThan(25_000); + complete += chunk.text; + offset = chunk.nextOffset; + } + expect(complete).toBe(loadedText); + expect(host.textReads()).toBe(1); + expect( + host.requests + .slice(beforeRead.length) + .every((request) => request.operation === "evaluate" && request.tabId === tabId), + ).toBe(true); + expect(host.timers.size).toBe(1); + + const released = yield* callReadText({ + captureId: captured.textCaptureId, + tabId: captured.textTabId, + release: true, + }); + expect(released.isError).toBe(false); + expect(released.structuredContent).toEqual({ + text: "", + nextOffset: 0, + totalChars: loadedText.length, + done: true, + released: true, + }); + expect(host.timers.size).toBe(0); + expect(host.listeners.size).toBe(0); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect.each([{}, { captureText: false }])("keeps full-text capture opt-in %j", (options) => + Effect.scoped( + Effect.gen(function* () { + const host = yield* serveTextCaptures("mcp-text-not-requested-client", "Loaded text"); + + const snapshot = yield* callSnapshot({ ...options, includeImage: false }); + + expect(snapshot.isError).toBe(false); + expect(host.requests).toHaveLength(1); + expect(host.requests[0]).toMatchObject({ operation: "snapshot", input: {} }); + expect(host.textReads()).toBe(0); + expect(snapshot.structuredContent).not.toHaveProperty("textCaptureId"); + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + expect(yield* fs.exists(config.browserArtifactsDir)).toBe(false); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect("rejects an invalid snapshot before starting a requested text capture", () => + Effect.scoped( + Effect.gen(function* () { + const host = yield* serveTextCaptures("mcp-invalid-text-capture-client", "Loaded text"); + + const snapshot = yield* callSnapshot({ captureText: true, includeImage: "wrong" }); + + expect(snapshot.isError).toBe(true); + expect(host.requests).toEqual([]); + expect(host.textReads()).toBe(0); + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + expect(yield* fs.exists(config.browserArtifactsDir)).toBe(false); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect.each(["navigation", "reload"] as const)( + "does not return captured text from the old page after %s before snapshot", + (change) => + Effect.scoped( + Effect.gen(function* () { + const options: { beforeSnapshot?: Effect.Effect } = {}; + const host = yield* serveTextCaptures( + "mcp-capture-page-change-client", + "loaded text", + options, + ); + options.beforeSnapshot = Effect.sync(() => { + expect(host.timers.size).toBe(1); + if (change === "navigation") + NodeVM.runInContext("location.href = 'https://other.test/'", host.context); + else host.listeners.get("pagehide")!(); + }); + + const snapshot = yield* callSnapshot({ captureText: true }); + + expect(snapshot.isError).toBe(true); + expect(snapshot.content.every((content) => content.type === "text")).toBe(true); + expect(snapshot.structuredContent).not.toHaveProperty("textCaptureId"); + expect(host.requests.filter((request) => request.operation === "snapshot")).toHaveLength(1); + expect(host.timers.size).toBe(0); + expect(host.listeners.size).toBe(0); + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + expect(yield* fs.exists(config.browserArtifactsDir)).toBe(false); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect("releases the text capture when the following snapshot fails", () => + Effect.scoped( + Effect.gen(function* () { + const host = yield* serveTextCaptures("mcp-capture-snapshot-failure-client", "loaded text", { + snapshotFailure: true, + }); + + const snapshot = yield* callSnapshot({ captureText: true }); + + expect(snapshot.isError).toBe(true); + expect(snapshot.structuredContent).toMatchObject({ + error: { _tag: "PreviewAutomationExecutionError", operation: "snapshot" }, + }); + expect(snapshot.content.every((content) => content.type === "text")).toBe(true); + expect(host.requests.filter((request) => request.operation === "snapshot")).toHaveLength(1); + expect(host.textReads()).toBe(1); + expect(host.timers.size).toBe(0); + expect(host.listeners.size).toBe(0); + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + expect(yield* fs.exists(config.browserArtifactsDir)).toBe(false); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect("releases its text capture when saving the PNG fails", () => + Effect.scoped( + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + yield* fs.writeFileString(config.browserArtifactsDir, "keep this file"); + const host = yield* serveTextCaptures("mcp-capture-png-failure-client", "loaded text"); + + const snapshot = yield* callSnapshot({ captureText: true, save: true }); + + expect(snapshot.isError).toBe(true); + expect(snapshot.structuredContent).toMatchObject({ + error: { _tag: "PreviewScreenshotSaveError", operation: "snapshot" }, + }); + expect(snapshot.content.every((content) => content.type === "text")).toBe(true); + expect(host.textReads()).toBe(1); + expect(host.timers.size).toBe(0); + expect(host.listeners.size).toBe(0); + expect(yield* fs.readFileString(config.browserArtifactsDir)).toBe("keep this file"); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect("releases the text capture when the following snapshot is cancelled", () => + Effect.scoped( + Effect.gen(function* () { + const enteredSnapshot = yield* Deferred.make(); + const host = yield* serveTextCaptures("mcp-capture-snapshot-cancel-client", "loaded text", { + beforeSnapshot: Deferred.succeed(enteredSnapshot, undefined).pipe( + Effect.andThen(Effect.never), + ), + }); + const fiber = yield* Effect.forkChild(callSnapshot({ captureText: true })); + yield* Deferred.await(enteredSnapshot); + expect(host.timers.size).toBe(1); + + yield* Fiber.interrupt(fiber); + + expect(host.requests.filter((request) => request.operation === "snapshot")).toHaveLength(1); + expect(host.textReads()).toBe(1); + expect(host.timers.size).toBe(0); + expect(host.listeners.size).toBe(0); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect.each([ + {}, + { tabId }, + { captureId: "capture" }, + { tabId, captureId: "" }, + { tabId, captureId: 1 }, + { tabId, captureId: "capture", offset: -1 }, + { tabId, captureId: "capture", offset: 0.5 }, + { tabId, captureId: "capture", offset: "0" }, + { tabId, captureId: "capture", release: "true" }, +])("rejects invalid read-text parameters before browser dispatch %j", (args) => + Effect.scoped( + Effect.gen(function* () { + const host = yield* serveTextCaptures("mcp-invalid-text-read-client", "Loaded text"); + + const result = yield* callReadText(args); + + expect(result.isError).toBe(true); + expect(host.requests).toEqual([]); + expect(encodeJsonText(result)).not.toContain("Loaded text"); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect("requires the preview capability before reading captured text", () => + Effect.scoped( + Effect.gen(function* () { + const host = yield* serveTextCaptures("mcp-denied-text-read-client", "Loaded text"); + const server = yield* McpServer.McpServer; + + const result = yield* server + .callTool({ name: "preview_read_text", arguments: { tabId, captureId: "capture" } }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, { + ...invocation, + capabilities: new Set(), + }), + Effect.provideService(McpSchema.McpServerClient, client), + ); + + expect(result.isError).toBe(true); + expect(encodeJsonText(result)).toContain("preview capability"); + expect(host.requests).toEqual([]); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect.each(["navigation", "reload", "expiry", "replacement", "release"] as const)( + "rejects a stale text capture after %s with useful guidance and no page text", + (change) => + Effect.scoped( + Effect.gen(function* () { + const host = yield* serveTextCaptures("mcp-stale-text-read-client", "private loaded text"); + const snapshot = yield* callSnapshot({ captureText: true }); + const captured = snapshot.structuredContent as { readonly textCaptureId: string }; + const args = { tabId, captureId: captured.textCaptureId }; + if (change === "navigation") + NodeVM.runInContext("location.href = 'https://other.test/'", host.context); + else if (change === "reload") host.listeners.get("pagehide")!(); + else if (change === "expiry") [...host.timers][0]!(); + else if (change === "replacement") { + const newer = yield* callSnapshot({ captureText: true }); + expect(newer.isError).toBe(false); + expect(host.timers.size).toBe(1); + } else yield* callReadText({ ...args, release: true }); + + const result = yield* callReadText(args); + + expect(result.isError).toBe(true); + expect(encodeJsonText(result)).toContain("captureText"); + expect(encodeJsonText(result)).not.toContain("private loaded text"); + if (change !== "replacement") { + expect(host.timers.size).toBe(0); + expect(host.listeners.size).toBe(0); + } + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect("does not expose another provider session's text capture", () => + Effect.scoped( + Effect.gen(function* () { + const host = yield* serveTextCaptures("mcp-isolated-text-read-client", "private loaded text"); + const snapshot = yield* callSnapshot({ captureText: true }); + const captured = snapshot.structuredContent as { readonly textCaptureId: string }; + const args = { tabId, captureId: captured.textCaptureId }; + + const foreign = yield* callReadText(args, { + ...invocation, + thread: { ...invocation.thread, providerSessionId: "different-session" }, + }); + + expect(foreign.isError).toBe(true); + expect(encodeJsonText(foreign)).not.toContain("private loaded text"); + const own = yield* callReadText(args); + expect(own.isError).toBe(false); + expect(own.structuredContent).toMatchObject({ text: "private loaded text" }); + expect(host.timers.size).toBe(1); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect.each([ + { name: "preview_snapshot", args: { captureText: true } }, + { name: "preview_read_text", args: { tabId, captureId: "capture" } }, +])("requires a thread caller for $name even with the preview capability", ({ name, args }) => + Effect.scoped( + Effect.gen(function* () { + const host = yield* serveTextCaptures("mcp-client-text-capture-denied", "Loaded text"); + const server = yield* McpServer.McpServer; + + const denied = yield* server.callTool({ name, arguments: args }).pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, { + ...invocation, + thread: undefined, + client: { + sessionId: "outside-thread-session", + label: "MCP client", + runtimeModeCeiling: "auto", + }, + }), + Effect.provideService(McpSchema.McpServerClient, client), + ); + + expect(denied.isError).toBe(true); + expect(denied.structuredContent).toMatchObject({ + error: { _tag: "PreviewAutomationUnavailableError" }, + }); + expect(host.requests).toEqual([]); + expect(host.textReads()).toBe(0); + }), + ).pipe(Effect.provide(TestLayer)), +); + it.effect( "registers the pull request toolkit and surfaces a missing capability as a tool error", () => @@ -675,6 +1260,201 @@ it.effect("sheds log entries before locators when every list is full", () => ).pipe(Effect.provide(TestLayer)), ); +it.effect("bounds JSON-escaped page identifiers after other snapshot fields are empty", () => + Effect.scoped( + Effect.gen(function* () { + yield* serveSnapshots("mcp-escaped-identifiers-client", { + ...snapshotResult, + url: `http://example.test/${"\u0000".repeat(3_000)}`, + title: "\u0000".repeat(3_000), + visibleText: "", + }); + const snapshot = yield* callSnapshot({ includeImage: false }); + expect(snapshot.isError).toBe(false); + const [, text, notice] = snapshot.content; + const body = text?.type === "text" ? text.text : ""; + expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual( + McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES, + ); + expect(notice?.type === "text" ? notice.text : "").toContain( + "url or title after 1024 characters", + ); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect( + "keeps current-view text and controls when offscreen controls and logs fill the budget", + () => + Effect.scoped( + Effect.gen(function* () { + const scroll = { + x: 0, + y: 12_000, + width: 1_000, + height: 800, + scrollWidth: 1_000, + scrollHeight: 20_000, + containers: [], + containersTruncated: false, + }; + yield* serveSnapshots("mcp-current-view-client", { + ...snapshotResult, + visibleText: "Page start ".repeat(4_000), + viewportText: "Bottom of page: choose Continue to finish.", + scroll, + truncated: { visibleText: true, viewportText: false, interactiveElements: true }, + interactiveElements: Array.from({ length: 240 }, (_, index) => ({ + tag: "button", + role: "button", + name: `Button ${index}`, + selector: `#button-${index}-${"x".repeat(500)}`, + inViewport: index >= 237, + x: 0, + y: index >= 237 ? 20 : -1_000, + width: 10, + height: 10, + })), + consoleEntries: Array.from({ length: 100 }, (_, index) => ({ + level: "log", + text: `Log ${index}: ${"x".repeat(2_000)}`, + timestamp: "t", + })), + }); + + const snapshot = yield* callSnapshot({ includeImage: false }); + expect(snapshot.isError).toBe(false); + const [, text, notice] = snapshot.content; + const body = text?.type === "text" ? text.text : ""; + expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual( + McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES, + ); + const parsed = decodeJsonText(body) as { + readonly viewportText: string; + readonly scroll: unknown; + readonly truncated: unknown; + readonly interactiveElements: ReadonlyArray<{ readonly name: string }>; + }; + expect(parsed.viewportText).toBe("Bottom of page: choose Continue to finish."); + expect(parsed.scroll).toEqual(scroll); + expect(parsed.truncated).toEqual({ + visibleText: true, + viewportText: false, + interactiveElements: true, + }); + expect(parsed.interactiveElements.slice(0, 3).map((element) => element.name)).toEqual([ + "Button 237", + "Button 238", + "Button 239", + ]); + expect(notice?.type === "text" ? notice.text : "").toContain("interactiveElements"); + expect(snapshot.structuredContent).toEqual({ ...parsed, omitted: expect.any(Array) }); + expect(snapshot.content.some((content) => content.type === "image")).toBe(false); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect.each(["η•ŒπŸ˜€".repeat(5_000), `${"a".repeat(7_999)}${"πŸ˜€".repeat(4_000)}`])( + "bounds current-view Unicode text without splitting a character %#", + (viewportText) => + Effect.scoped( + Effect.gen(function* () { + yield* serveSnapshots("mcp-viewport-unicode-client", { + ...snapshotResult, + viewportText, + visibleText: "Offscreen ".repeat(4_000), + truncated: { visibleText: false, viewportText: false, interactiveElements: false }, + interactiveElements: Array.from({ length: 20 }, (_, index) => ({ + tag: "button", + role: "button", + name: `Button ${index}`, + selector: `#button-${index}-${"x".repeat(500)}`, + inViewport: true, + x: 0, + y: 0, + width: 10, + height: 10, + })), + }); + const snapshot = yield* callSnapshot({ includeImage: false }); + expect(snapshot.isError).toBe(false); + const [, text, notice] = snapshot.content; + const body = text?.type === "text" ? text.text : ""; + expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual( + McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES, + ); + const parsed = decodeJsonText(body) as { + readonly viewportText: string; + readonly interactiveElements: ReadonlyArray; + }; + expect(parsed.interactiveElements.length).toBeGreaterThan(0); + expect(parsed.viewportText.length).toBeGreaterThan(1_000); + expect(parsed.viewportText.endsWith("…")).toBe(true); + expect(parsed.viewportText.isWellFormed()).toBe(true); + expect(parsed).toMatchObject({ truncated: { viewportText: true } }); + expect(notice?.type === "text" ? notice.text : "").toContain("viewportText after"); + expect(snapshot.structuredContent).toEqual({ ...parsed, omitted: expect.any(Array) }); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect("keeps current-view Unicode text in full when it fits the snapshot budget", () => + Effect.scoped( + Effect.gen(function* () { + const viewportText = "η•Œ".repeat(5_000); + yield* serveSnapshots("mcp-viewport-full-unicode-client", { + ...snapshotResult, + visibleText: "", + viewportText, + }); + const snapshot = yield* callSnapshot({ includeImage: false }); + expect(snapshot.isError).toBe(false); + expect(snapshot.structuredContent).toMatchObject({ viewportText }); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect("bounds scroll-container locators while preserving page scroll position", () => + Effect.scoped( + Effect.gen(function* () { + yield* serveSnapshots("mcp-scroll-budget-client", { + ...snapshotResult, + viewportText: "Current content", + scroll: { + x: 20, + y: 8_000, + width: 1_000, + height: 800, + scrollWidth: 2_000, + scrollHeight: 10_000, + containers: Array.from({ length: 20 }, () => ({ + selector: `#${"x".repeat(30_000)}`, + x: 0, + y: 200, + width: 100, + height: 100, + scrollWidth: 100, + scrollHeight: 1_000, + })), + containersTruncated: false, + }, + }); + const snapshot = yield* callSnapshot({ includeImage: false }); + expect(snapshot.isError).toBe(false); + const [, text, notice] = snapshot.content; + const body = text?.type === "text" ? text.text : ""; + expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual( + McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES, + ); + expect(decodeJsonText(body)).toMatchObject({ + viewportText: "Current content", + scroll: { x: 20, y: 8_000, containers: [], containersTruncated: true }, + }); + expect(notice?.type === "text" ? notice.text : "").toContain("20 of 20 scrollContainers"); + }), + ).pipe(Effect.provide(TestLayer)), +); + it.effect("terminates HTTP MCP sessions with DELETE", () => Effect.scoped( Effect.gen(function* () { @@ -726,7 +1506,16 @@ it.effect("terminates HTTP MCP sessions with DELETE", () => }); expect(reusedSessionResponse.status).toBe(404); }), - ).pipe(Effect.provide(NodeHttpServer.layerTest)), + ).pipe( + Effect.provide( + HttpServer.layerTestClient.pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provideMerge( + NodeHttpServer.layer(NodeHttp.createServer, { host: "127.0.0.1", port: 0 }), + ), + ), + ), + ), ); it.effect("registers annotated tools and preserves authenticated request context", () => @@ -780,9 +1569,19 @@ it.effect("registers annotated tools and preserves authenticated request context const snapshotTool = server.tools.find(({ tool }) => tool.name === "preview_snapshot"); expect(snapshotTool?.tool.annotations?.readOnlyHint).toBe(true); - expect(snapshotTool?.tool.annotations?.idempotentHint).toBe(true); + expect(snapshotTool?.tool.annotations?.idempotentHint).toBe(false); expect(snapshotTool?.tool.annotations?.openWorldHint).toBe(true); + const readTextTool = server.tools.find(({ tool }) => tool.name === "preview_read_text"); + expect(readTextTool?.tool.annotations?.readOnlyHint).toBe(true); + expect(readTextTool?.tool.annotations?.idempotentHint).toBe(false); + expect(readTextTool?.tool.annotations?.destructiveHint).toBe(false); + expect(readTextTool?.tool.annotations?.openWorldHint).toBe(true); + expect(readTextTool?.tool.outputSchema).toMatchObject({ + type: "object", + required: expect.arrayContaining(["text", "nextOffset", "totalChars", "done", "released"]), + }); + const clickTool = server.tools.find(({ tool }) => tool.name === "preview_click"); expect(clickTool?.tool.annotations?.readOnlyHint).toBe(false); expect(clickTool?.tool.annotations?.destructiveHint).toBe(true); diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 5c8f146bcadf..76d3ddbb6812 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -1,22 +1,17 @@ -import * as NodeCrypto from "node:crypto"; import * as Cause from "effect/Cause"; -import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; import type * as Types from "effect/Types"; -import { McpProtocol, McpSchema, McpServer, Tool } from "effect/unstable/ai"; +import { AiError, McpProtocol, McpSchema, McpServer, Tool } from "effect/unstable/ai"; import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; import { PreviewAutomationError } from "@t3tools/contracts"; import packageJson from "../../package.json" with { type: "json" }; -import * as ServerConfig from "../config.ts"; import * as DeviceService from "../device/DeviceService.ts"; import * as McpInvocationContext from "./McpInvocationContext.ts"; import * as OrchestratorMcpService from "./OrchestratorMcpService.ts"; @@ -32,16 +27,13 @@ import { ThreadToolkit } from "./toolkits/thread/tools.ts"; import { ThreadToolkitHandlersLive } from "./toolkits/thread/handlers.ts"; import * as ThreadMetadataMcpService from "./ThreadMetadataMcpService.ts"; import * as McpSessionRegistry from "./McpSessionRegistry.ts"; -import * as PreviewAutomationBroker from "./PreviewAutomationBroker.ts"; +import * as PreviewSnapshot from "../preview/Snapshot.ts"; import { OrchestratorToolkitHandlersLive } from "./toolkits/orchestrator/handlers.ts"; import { OrchestratorToolkit } from "./toolkits/orchestrator/tools.ts"; +import { PreviewStandardToolkitHandlersLive } from "./toolkits/preview/handlers.ts"; import { - PreviewSnapshotToolkitHandlersLive, - PreviewStandardToolkitHandlersLive, -} from "./toolkits/preview/handlers.ts"; -import { + PreviewReadTextTool, PreviewSnapshotTool, - PreviewSnapshotToolkit, PreviewStandardToolkit, } from "./toolkits/preview/tools.ts"; import { WorktreeToolkitHandlersLive } from "./toolkits/worktree/handlers.ts"; @@ -149,8 +141,12 @@ const MAX_SNAPSHOT_IDENTIFIER_CHARS = 2_048; const encodeJsonText = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); const utf8Length = (text: string) => Buffer.byteLength(text, "utf8"); -const cutText = (text: string, max: number) => - text.length > max ? `${text.slice(0, max)}…` : text; +const cutText = (text: string, max: number) => { + if (text.length <= max) return text; + const last = text.charCodeAt(max - 1); + const end = last >= 0xd800 && last <= 0xdbff ? max - 1 : max; + return `${text.slice(0, end)}…`; +}; /** Shortens every string field of a log entry; other fields pass through. */ const cutEntryStrings = (entry: A): A => @@ -172,8 +168,24 @@ type SnapshotMetadata = { readonly url: string; readonly title: string; readonly visibleText: string; + readonly viewportText?: string | undefined; + readonly truncated?: + | { + readonly visibleText: boolean; + readonly viewportText: boolean; + readonly interactiveElements: boolean; + } + | undefined; + readonly scroll?: + | { + readonly containers: ReadonlyArray; + readonly containersTruncated: boolean; + readonly [key: string]: unknown; + } + | undefined; readonly interactiveElements: ReadonlyArray<{ readonly name: string; + readonly inViewport?: boolean | undefined; readonly [key: string]: unknown; }>; readonly consoleEntries: ReadonlyArray; @@ -222,24 +234,22 @@ const boundSnapshotMetadata = (metadata: SnapshotMetadata) => { ...withoutTree, url: cutText(metadata.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), title: cutText(metadata.title, MAX_SNAPSHOT_IDENTIFIER_CHARS), - interactiveElements: metadata.interactiveElements.map((element) => ({ - ...element, - name: cutText(element.name, MAX_SNAPSHOT_ELEMENT_NAME_CHARS), - })), + interactiveElements: metadata.interactiveElements + .map((element) => ({ + ...element, + name: cutText(element.name, MAX_SNAPSHOT_ELEMENT_NAME_CHARS), + })) + .sort((left, right) => Number(right.inViewport === true) - Number(left.inViewport === true)), consoleEntries: tail(metadata.consoleEntries, "console entries"), networkEntries: tail(metadata.networkEntries, "network entries"), actionTimeline: tail(metadata.actionTimeline, "action timeline entries"), }; - // Per-field caps do not sum below the ceiling: three log arrays of 40 capped - // entries alone can pass 60 KB, and the caps count characters, not bytes. - // Halve one thing per round until the JSON fits: logs first, then page - // text, then the locators. The identifier caps bound the rest, so this - // terminates. const shedOrder = [ "actionTimeline", "networkEntries", "consoleEntries", + "scrollContainers", "interactiveElements", ] as const; const lists: Record<(typeof shedOrder)[number], ReadonlyArray> = { @@ -247,42 +257,104 @@ const boundSnapshotMetadata = (metadata: SnapshotMetadata) => { consoleEntries: bounded.consoleEntries, networkEntries: bounded.networkEntries, actionTimeline: bounded.actionTimeline, + scrollContainers: metadata.scroll?.containers ?? [], }; const dropped: Record<(typeof shedOrder)[number], number> = { interactiveElements: 0, consoleEntries: 0, networkEntries: 0, actionTimeline: 0, + scrollContainers: 0, }; let visibleTextChars = Math.min(metadata.visibleText.length, MAX_SNAPSHOT_VISIBLE_TEXT_CHARS); + let viewportTextChars = Math.min( + metadata.viewportText?.length ?? 0, + MAX_SNAPSHOT_VISIBLE_TEXT_CHARS, + ); + let identifierChars = MAX_SNAPSHOT_IDENTIFIER_CHARS; const value = () => ({ ...bounded, + url: cutText(metadata.url, identifierChars), + title: cutText(metadata.title, identifierChars), visibleText: cutText(metadata.visibleText, visibleTextChars), - ...lists, + ...(metadata.viewportText === undefined + ? {} + : { viewportText: cutText(metadata.viewportText, viewportTextChars) }), + interactiveElements: lists.interactiveElements, + consoleEntries: lists.consoleEntries, + networkEntries: lists.networkEntries, + actionTimeline: lists.actionTimeline, + ...(metadata.truncated === undefined + ? {} + : { + truncated: { + visibleText: + metadata.truncated.visibleText || visibleTextChars < metadata.visibleText.length, + viewportText: + metadata.truncated.viewportText || + viewportTextChars < (metadata.viewportText?.length ?? 0), + interactiveElements: + metadata.truncated.interactiveElements || dropped.interactiveElements > 0, + }, + }), + ...(metadata.scroll === undefined + ? {} + : { + scroll: { + ...metadata.scroll, + containers: lists.scrollContainers, + containersTruncated: + metadata.scroll.containersTruncated || dropped.scrollContainers > 0, + }, + }), }); + const inViewportCount = bounded.interactiveElements.filter( + (element) => element.inViewport === true, + ).length; let text = encodeJsonText(value()); while (utf8Length(text) > MAX_SNAPSHOT_TEXT_BYTES) { - // Elements carry the locators, so they go last; logs shed newest-last. - const key = - shedOrder.find( - (candidate) => candidate !== "interactiveElements" && lists[candidate].length > 0, - ) ?? - (visibleTextChars > 0 - ? "visibleText" - : lists.interactiveElements.length > 0 - ? "interactiveElements" - : undefined); + let key: + | (typeof shedOrder)[number] + | "visibleText" + | "viewportText" + | "identifiers" + | undefined = shedOrder.find( + (candidate) => + candidate !== "interactiveElements" && + candidate !== "scrollContainers" && + lists[candidate].length > 0, + ); + if (key === undefined) { + if (visibleTextChars > 0) key = "visibleText"; + else if (lists.interactiveElements.length > inViewportCount) key = "interactiveElements"; + else if (lists.scrollContainers.length > 0) key = "scrollContainers"; + else if ( + viewportTextChars > 0 && + utf8Length(encodeJsonText(cutText(metadata.viewportText ?? "", viewportTextChars))) > + MAX_SNAPSHOT_TEXT_BYTES / 2 + ) + key = "viewportText"; + else if (lists.interactiveElements.length > 0) key = "interactiveElements"; + else if (viewportTextChars > 0) key = "viewportText"; + else if (identifierChars > 0) key = "identifiers"; + } if (key === undefined) break; if (key === "visibleText") { visibleTextChars = Math.floor(visibleTextChars / 2); + } else if (key === "viewportText") { + viewportTextChars = Math.floor(viewportTextChars / 2); + } else if (key === "identifiers") { + identifierChars = Math.floor(identifierChars / 2); } else { - const keep = Math.floor(lists[key].length / 2); + const keep = + key === "interactiveElements" && lists[key].length > inViewportCount + ? inViewportCount + Math.floor((lists[key].length - inViewportCount) / 2) + : Math.floor(lists[key].length / 2); dropped[key] += lists[key].length - keep; - // slice(-0) keeps everything, so spell out the empty case. lists[key] = keep === 0 ? [] - : key === "interactiveElements" + : key === "interactiveElements" || key === "scrollContainers" ? lists[key].slice(0, keep) : lists[key].slice(-keep); } @@ -293,105 +365,78 @@ const boundSnapshotMetadata = (metadata: SnapshotMetadata) => { `visibleText after ${visibleTextChars} characters (use preview_evaluate for more)`, ); } + if (viewportTextChars < (metadata.viewportText?.length ?? 0)) { + omitted.push( + `viewportText after ${viewportTextChars} characters (use preview_evaluate for more)`, + ); + } + if (identifierChars < MAX_SNAPSHOT_IDENTIFIER_CHARS) { + omitted.push(`url or title after ${identifierChars} characters`); + } for (const key of shedOrder) { if (dropped[key] > 0) { - omitted.push(`${dropped[key]} of ${bounded[key].length} ${key}`); + const originalLength = + key === "scrollContainers" ? metadata.scroll?.containers.length : bounded[key].length; + omitted.push(`${dropped[key]} of ${originalLength} ${key}`); } } return { value: value(), text, omitted }; }; -export class PreviewScreenshotSaveError extends Schema.TaggedError()( - "PreviewScreenshotSaveError", - { screenshotPath: Schema.String, cause: Schema.Defect() }, -) { - override get message(): string { - return `Could not save preview screenshot to ${this.screenshotPath}.`; - } -} - -const MAX_SCREENSHOT_SITE_SLUG_LENGTH = 40; - -/** Hostname reduced to a filename-safe slug, matching the desktop's own screenshot names. */ -const screenshotSiteSlug = (rawUrl: string): string => { - try { - const slug = new URL(rawUrl).hostname - .toLowerCase() - .replace(/[^a-z0-9]+/g, "-") - .replace(/^-+|-+$/g, "") - .slice(0, MAX_SCREENSHOT_SITE_SLUG_LENGTH) - .replace(/-+$/g, ""); - return slug || "site"; - } catch { - return "site"; - } -}; - -/** Writes the snapshot PNG under the browser artifacts directory and returns its path. */ -const saveScreenshot = Effect.fn("McpHttpServer.saveScreenshot")(function* ( - pageUrl: string, - data: Uint8Array, -) { - const config = yield* ServerConfig.ServerConfig; - const fileSystem = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const millis = yield* Clock.currentTimeMillis; - // Two saves in the same millisecond must not overwrite each other. - const fileName = `browser-screenshot-${screenshotSiteSlug(pageUrl)}-${millis.toString(36)}-${NodeCrypto.randomUUID().slice(0, 8)}.png`; - const screenshotPath = path.join(config.browserArtifactsDir, fileName); - yield* fileSystem.makeDirectory(config.browserArtifactsDir, { recursive: true }).pipe( - Effect.andThen(fileSystem.writeFile(screenshotPath, data)), - Effect.mapError((cause) => new PreviewScreenshotSaveError({ screenshotPath, cause })), - ); - return screenshotPath; -}); - const isPreviewAutomationError = Schema.is(PreviewAutomationError); +const isPreviewTextCaptureError = Schema.is(PreviewSnapshot.PreviewTextCaptureError); +const decodePreviewSnapshotInput = Schema.decodeUnknownEffect(PreviewSnapshotTool.parametersSchema); +const decodePreviewReadTextInput = Schema.decodeUnknownEffect(PreviewReadTextTool.parametersSchema); -const previewSnapshotFailure = (cause: Cause.Cause) => { - if (Cause.hasInterrupts(cause) || cause.reasons.some(Cause.isDieReason)) { - return Effect.failCause(cause).pipe(Effect.orDie); - } - const failures = cause.reasons.filter(Cause.isFailReason); - const firstFailure = failures[0]?.error; - const errorTag = - typeof firstFailure === "object" && - firstFailure !== null && - "_tag" in firstFailure && - typeof firstFailure._tag === "string" - ? firstFailure._tag - : "PreviewSnapshotError"; - // Preview errors build their message on the server, never from page output, - // and it tells the agent what to do next, such as falling back to a shell browser. - const message = isPreviewAutomationError(firstFailure) ? firstFailure.message : undefined; - const result = new McpSchema.CallToolResult({ - isError: true, - structuredContent: { - error: { - _tag: errorTag, - operation: "snapshot", - failureCount: failures.length, - ...(message === undefined ? {} : { message }), +const previewFailure = + (operation: "snapshot" | "readText") => + (cause: Cause.Cause) => { + if (Cause.hasInterrupts(cause) || cause.reasons.some(Cause.isDieReason)) { + return Effect.failCause(cause).pipe(Effect.orDie); + } + const failures = cause.reasons.filter(Cause.isFailReason); + const firstFailure = failures[0]?.error; + const errorTag = + typeof firstFailure === "object" && + firstFailure !== null && + "_tag" in firstFailure && + typeof firstFailure._tag === "string" + ? firstFailure._tag + : "PreviewSnapshotError"; + // Preview errors build their message on the server, never from page output, + // and it tells the agent what to do next, such as falling back to a shell browser. + const message = + isPreviewAutomationError(firstFailure) || isPreviewTextCaptureError(firstFailure) + ? firstFailure.message + : undefined; + const result = new McpSchema.CallToolResult({ + isError: true, + structuredContent: { + error: { + _tag: errorTag, + operation, + failureCount: failures.length, + ...(message === undefined ? {} : { message }), + }, }, - }, - // Some clients show only the text content and others only structuredContent, so both carry it. - content: [{ type: "text", text: `Preview snapshot failed: ${message ?? `${errorTag}.`}` }], - }); - return Effect.logWarning("preview snapshot failed", { - operation: "snapshot", - errorTag, - failureCount: failures.length, - }).pipe(Effect.as(result)); -}; + // Some clients show only the text content and others only structuredContent, so both carry it. + content: [ + { + type: "text", + text: `Preview ${operation === "snapshot" ? "snapshot" : "text read"} failed: ${message ?? `${errorTag}.`}`, + }, + ], + }); + return Effect.logWarning(`preview ${operation} failed`, { + operation, + errorTag, + failureCount: failures.length, + }).pipe(Effect.as(result)); + }; const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot")(function* () { const server = yield* McpServer.McpServer; - const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker; - // The MCP tool runner only supplies the client, so hand the save path its services here. - const saveServices = yield* Effect.context< - ServerConfig.ServerConfig | FileSystem.FileSystem | Path.Path - >(); - const built = yield* PreviewSnapshotToolkit; + const snapshots = yield* PreviewSnapshot.PreviewSnapshot; const tool = PreviewSnapshotTool; yield* server.addTool({ tool: new McpSchema.Tool({ @@ -416,82 +461,157 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot fiber.context, McpInvocationContext.McpInvocationContext, ); - return built.handle("preview_snapshot", payload).pipe( - Stream.unwrap, - Stream.run(Sink.last()), - Effect.flatMap(Effect.fromOption), - Effect.provideService(PreviewAutomationBroker.PreviewAutomationBroker, broker), - Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), - Effect.flatMap(({ encodedResult }) => - Effect.gen(function* () { - const snapshot = encodedResult as SnapshotMetadata & { - readonly url: string; - readonly screenshot: { - readonly mimeType: "image/png"; - readonly data: string; - readonly width: number; - readonly height: number; - }; - }; - const { screenshot, ...page } = snapshot; - const png = new Uint8Array(Buffer.from(screenshot.data, "base64")); - const screenshotPath = - payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; - if (screenshotPath !== undefined && payload?.includeImage === false) { - // The agent only wants a file to show the user. The url keeps the site icon on the tool row. - const saved = { - url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), - screenshotPath, - }; - return new McpSchema.CallToolResult({ - isError: false, - structuredContent: saved, - content: [{ type: "text", text: encodeJsonText(saved) }], - }); - } - const metadata = { - ...page, - screenshot: { - mimeType: screenshot.mimeType, - width: screenshot.width, - height: screenshot.height, - }, - ...(screenshotPath === undefined ? {} : { screenshotPath }), - }; - const bounded = boundSnapshotMetadata(metadata); - return new McpSchema.CallToolResult({ - isError: false, - structuredContent: - bounded.omitted.length === 0 - ? bounded.value - : { ...bounded.value, omitted: bounded.omitted }, - content: [ - // Keep the page identity readable even if a provider truncates the snapshot. - { - type: "text", - text: encodeJsonText({ - url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), - }), - }, - { type: "text", text: bounded.text }, - ...(bounded.omitted.length === 0 - ? [] - : [ - { - type: "text" as const, - text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`, + return decodePreviewSnapshotInput(payload ?? {}).pipe( + Effect.mapError((cause) => + AiError.make({ + module: "Toolkit", + method: "preview_snapshot.handle", + reason: new AiError.ToolParameterValidationError({ + toolName: "preview_snapshot", + description: cause.message, + }), + }), + ), + Effect.flatMap((input) => + McpInvocationContext.requireThreadMcpCapability("preview").pipe( + Effect.flatMap((scope) => + snapshots.withSnapshot( + { scope, ...input }, + ({ snapshot, png, textCapture, screenshotPath }) => + Effect.sync(() => { + const { screenshot, ...page } = snapshot; + const textArtifact = + textCapture === undefined + ? {} + : { + textCaptureId: textCapture.captureId, + textChars: textCapture.totalChars, + textUrl: textCapture.url, + textTabId: textCapture.tabId, + }; + if (screenshotPath !== undefined && input.includeImage === false) { + // The agent only wants a file to show the user. The url keeps the site icon on the tool row. + const saved = { + url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), + screenshotPath, + ...textArtifact, + }; + return new McpSchema.CallToolResult({ + isError: false, + structuredContent: saved, + content: [{ type: "text", text: encodeJsonText(saved) }], + }); + } + const metadata = { + ...page, + screenshot: { + mimeType: screenshot.mimeType, + width: screenshot.width, + height: screenshot.height, }, - ]), - ...(payload?.includeImage === false - ? [] - : [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]), - ], - }); + ...(screenshotPath === undefined ? {} : { screenshotPath }), + ...textArtifact, + }; + const bounded = boundSnapshotMetadata(metadata); + return new McpSchema.CallToolResult({ + isError: false, + structuredContent: + bounded.omitted.length === 0 + ? bounded.value + : { ...bounded.value, omitted: bounded.omitted }, + content: [ + // Keep the page identity readable even if a provider truncates the snapshot. + { + type: "text", + text: encodeJsonText({ + url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS), + }), + }, + { type: "text", text: bounded.text }, + ...(bounded.omitted.length === 0 + ? [] + : [ + { + type: "text" as const, + text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`, + }, + ]), + ...(input.includeImage === false + ? [] + : [ + { + type: "image" as const, + data: png, + mimeType: screenshot.mimeType, + }, + ]), + ], + }); + }), + ), + ), + ), + ), + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + Effect.matchCauseEffect({ + onFailure: previewFailure("snapshot"), + onSuccess: Effect.succeed, + }), + ); + }), + }); + const readTool = PreviewReadTextTool; + yield* server.addTool({ + tool: new McpSchema.Tool({ + name: readTool.name, + description: Tool.getDescription(readTool), + inputSchema: Tool.getJsonSchema(readTool), + outputSchema: Tool.getJsonSchemaFromSchema(readTool.successSchema), + annotations: { + ...Context.getOption(readTool.annotations, Tool.Title).pipe( + Option.map((title) => ({ title })), + Option.getOrUndefined, + ), + readOnlyHint: Context.get(readTool.annotations, Tool.Readonly), + destructiveHint: Context.get(readTool.annotations, Tool.Destructive), + idempotentHint: Context.get(readTool.annotations, Tool.Idempotent), + openWorldHint: Context.get(readTool.annotations, Tool.OpenWorld), + }, + }), + annotations: readTool.annotations, + handle: (payload) => + Effect.withFiber((fiber) => { + const invocation = Context.getUnsafe( + fiber.context, + McpInvocationContext.McpInvocationContext, + ); + return decodePreviewReadTextInput(payload ?? {}).pipe( + Effect.mapError((cause) => + AiError.make({ + module: "Toolkit", + method: "preview_read_text.handle", + reason: new AiError.ToolParameterValidationError({ + toolName: "preview_read_text", + description: cause.message, + }), }), ), - Effect.provide(saveServices), + Effect.flatMap((input) => + McpInvocationContext.requireThreadMcpCapability("preview").pipe( + Effect.flatMap((scope) => snapshots.readText({ scope, ...input })), + ), + ), + Effect.map( + (result) => + new McpSchema.CallToolResult({ + isError: false, + structuredContent: result, + content: [{ type: "text", text: encodeJsonText(result) }], + }), + ), + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), Effect.matchCauseEffect({ - onFailure: previewSnapshotFailure, + onFailure: previewFailure("readText"), onSuccess: Effect.succeed, }), ); @@ -652,7 +772,7 @@ const PreviewStandardToolkitRegistrationLive = McpServer.toolkit(PreviewStandard ); const PreviewSnapshotRegistrationLive = Layer.effectDiscard(registerPreviewSnapshot()).pipe( - Layer.provide(PreviewSnapshotToolkitHandlersLive), + Layer.provide(PreviewSnapshot.layer), ); export const PreviewToolkitRegistrationLive = Layer.mergeAll( diff --git a/apps/server/src/mcp/toolkits/preview/handlers.ts b/apps/server/src/mcp/toolkits/preview/handlers.ts index 9bbbc13a7109..fbb58b241259 100644 --- a/apps/server/src/mcp/toolkits/preview/handlers.ts +++ b/apps/server/src/mcp/toolkits/preview/handlers.ts @@ -14,7 +14,6 @@ import { type PreviewAutomationRecordingStatus, type PreviewAutomationResizeResult, type PreviewAutomationSetColorSchemeResult, - type PreviewAutomationSnapshot, type PreviewAutomationStatus, type PreviewTabId, } from "@t3tools/contracts"; @@ -29,7 +28,7 @@ import { resolveAttachmentRelativePath } from "../../../attachmentPaths.ts"; import * as ServerConfig from "../../../config.ts"; import * as McpInvocationContext from "../../McpInvocationContext.ts"; import * as PreviewAutomationBroker from "../../PreviewAutomationBroker.ts"; -import { PreviewSnapshotToolkit, PreviewStandardToolkit, PreviewToolkit } from "./tools.ts"; +import { PreviewStandardToolkit } from "./tools.ts"; /** * Collapses the `show` alias onto `open` and defaults tab reuse. @@ -197,11 +196,6 @@ const handlers = { invokeTargeted("resize", input, input.timeoutMs), preview_set_appearance: (input) => invokeTargeted("setColorScheme", input), - preview_snapshot: (input) => { - // Output selection and saving are MCP-only; the browser still produces a complete snapshot. - const { includeImage: _includeImage, save: _save, ...operationInput } = input ?? {}; - return invokeTargeted("snapshot", operationInput); - }, preview_click: (input) => invokeTargeted("click", input, input.timeoutMs), preview_type: (input) => invokeTargeted("type", input, input.timeoutMs), preview_press: (input) => invokeTargeted("press", input), @@ -229,12 +223,6 @@ const handlers = { const artifact = yield* claimPreviewRecording(scope.thread.threadId, response.result); return { ...artifact, ...(response.toolIcon ? { toolIcon: response.toolIcon } : {}) }; }), -} satisfies Parameters[0]; - -const { preview_snapshot, ...standardHandlers } = handlers; +} satisfies Parameters[0]; -export const PreviewStandardToolkitHandlersLive = PreviewStandardToolkit.toLayer(standardHandlers); - -export const PreviewSnapshotToolkitHandlersLive = PreviewSnapshotToolkit.toLayer({ - preview_snapshot, -}); +export const PreviewStandardToolkitHandlersLive = PreviewStandardToolkit.toLayer(handlers); diff --git a/apps/server/src/mcp/toolkits/preview/tools.test.ts b/apps/server/src/mcp/toolkits/preview/tools.test.ts index 0937fba72c60..aa77cc177f62 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.test.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.test.ts @@ -1,4 +1,5 @@ import { expect, it } from "@effect/vitest"; +import * as Context from "effect/Context"; import { Tool } from "effect/unstable/ai"; import { PreviewToolkit } from "./tools.ts"; @@ -75,3 +76,31 @@ it("exports exact object result schemas for preview actions", () => { }); } }); + +it("keeps text capture opt-in and requires its tab and capture identifiers for reads", () => { + expect(Context.get(PreviewToolkit.tools.preview_snapshot.annotations, Tool.Idempotent)).toBe( + false, + ); + const snapshotSchema = Tool.getJsonSchema(PreviewToolkit.tools.preview_snapshot) as { + readonly properties: Readonly>; + readonly required?: ReadonlyArray; + }; + expect(snapshotSchema.properties.captureText).toBeDefined(); + expect(snapshotSchema.properties.saveText).toBeUndefined(); + expect(snapshotSchema.required ?? []).not.toContain("captureText"); + const readSchema = Tool.getJsonSchema(PreviewToolkit.tools.preview_read_text) as { + readonly required: ReadonlyArray; + }; + expect(readSchema.required.toSorted()).toEqual(["captureId", "tabId"]); + const resultSchema = Tool.getJsonSchemaFromSchema( + PreviewToolkit.tools.preview_read_text.successSchema, + ) as { readonly type: string; readonly required: ReadonlyArray }; + expect(resultSchema.type).toBe("object"); + expect(resultSchema.required.toSorted()).toEqual([ + "done", + "nextOffset", + "released", + "text", + "totalChars", + ]); +}); diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 1c790e8b643a..7585a648d80d 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -18,6 +18,7 @@ import { PreviewAutomationTabTargetInput, PreviewAutomationTypeInput, PreviewAutomationWaitForInput, + PreviewTabId, } from "@t3tools/contracts"; import * as Schema from "effect/Schema"; import * as FileSystem from "effect/FileSystem"; @@ -116,10 +117,10 @@ const PreviewSetAppearanceTool = safeBrowserTool( .annotate(Tool.Idempotent, true), ); -export const PreviewSnapshotTool = readonlyBrowserTool( +export const PreviewSnapshotTool = safeBrowserTool( Tool.make("preview_snapshot", { description: - "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; with includeImage=false, save=true returns only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", + "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use the current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. When the desktop host supports it, viewportText describes the current view, scroll reports page and visible container scroll positions, and inViewport elements come first. visibleText also includes rendered text outside the view; content not yet loaded requires scrolling. The text is capped near 20 KB, keeps current-view text ahead of offscreen page text, and lists what it omitted. Set captureText=true to hold all loaded, rendered main-page text in temporary browser memory without a total character cap; use textCaptureId and textTabId with preview_read_text to read it in small parts. No text file is created. The capture expires after five idle minutes, a page change, or a replacement capture. This does not scroll or load missing text, and excludes embedded frames and shadow DOM. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; with includeImage=false, save=true returns only the url and saved artifact details. Embed screenshotPath in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.", parameters: Schema.Struct({ ...PreviewAutomationTabTargetInput.fields, includeImage: Schema.optional( @@ -131,14 +132,62 @@ export const PreviewSnapshotTool = readonlyBrowserTool( save: Schema.optional( Schema.Boolean.annotate({ description: - "Write the screenshot PNG to disk and return its absolute path as screenshotPath. With includeImage=false, return only the url and screenshotPath. Defaults to false.", + "Write the screenshot PNG to disk and return its absolute path as screenshotPath. With includeImage=false, return only the url and saved artifact details. Defaults to false.", + }), + ), + captureText: Schema.optional( + Schema.Boolean.annotate({ + description: + "Keep all loaded, rendered main-page text in temporary browser memory and return textCaptureId, textTabId, textChars, and textUrl. Read it with preview_read_text. No text file or total character cap; snapshot output stays bounded. Does not scroll or load missing content. Defaults to false.", }), ), }), success: PreviewAutomationSnapshot, failure: PreviewAutomationError, dependencies, - }).annotate(Tool.Title, "Inspect browser page"), + }) + .annotate(Tool.Title, "Inspect browser page") + .annotate(Tool.Readonly, true) + .annotate(Tool.Idempotent, false), +); + +export const PreviewReadTextTool = safeBrowserTool( + Tool.make("preview_read_text", { + description: + "Read one small part of the loaded main-page text captured by preview_snapshot with captureText=true. Pass its textCaptureId as captureId and textTabId as tabId. Start with offset=0, then use nextOffset for the next part until done=true. Each response contains at most 4096 UTF-16 characters and preserves whole character pairs. Reading refreshes the five-minute idle expiry. Set release=true to discard the capture when finished; it returns no text. A page change or replacement capture also discards it. Text is read from temporary browser memory without creating a text file; normal chat and tool history can still store the text you read.", + parameters: Schema.Struct({ + tabId: PreviewTabId.pipe( + Schema.annotateEncoded({ + description: "The textTabId returned by the snapshot that captured this text.", + }), + ), + captureId: Schema.String.check(Schema.isNonEmpty(), Schema.isMaxLength(128)).annotate({ + description: "The textCaptureId returned by preview_snapshot with captureText=true.", + }), + offset: Schema.optional( + Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)).annotate({ + description: "UTF-16 character offset. Defaults to zero; use nextOffset to continue.", + }), + ), + release: Schema.optional( + Schema.Boolean.annotate({ + description: "Discard the captured text without reading another part. Defaults to false.", + }), + ), + }), + success: Schema.Struct({ + text: Schema.String.check(Schema.isMaxLength(4096)), + nextOffset: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + totalChars: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + done: Schema.Boolean, + released: Schema.Boolean, + }), + failure: PreviewAutomationError, + dependencies, + }) + .annotate(Tool.Title, "Read captured browser text") + .annotate(Tool.Readonly, true) + .annotate(Tool.Idempotent, false), ); const PreviewClickTool = browserTool( @@ -248,6 +297,7 @@ export const PreviewToolkit = Toolkit.make( PreviewResizeTool, PreviewSetAppearanceTool, PreviewSnapshotTool, + PreviewReadTextTool, PreviewClickTool, PreviewTypeTool, PreviewPressTool, @@ -273,5 +323,3 @@ export const PreviewStandardToolkit = Toolkit.make( PreviewRecordingStartTool, PreviewRecordingStopTool, ); - -export const PreviewSnapshotToolkit = Toolkit.make(PreviewSnapshotTool); diff --git a/apps/server/src/preview/Snapshot.test.ts b/apps/server/src/preview/Snapshot.test.ts new file mode 100644 index 000000000000..050b07732c85 --- /dev/null +++ b/apps/server/src/preview/Snapshot.test.ts @@ -0,0 +1,693 @@ +import * as NodeVM from "node:vm"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import { + EnvironmentId, + PreviewAutomationNoAvailableHostError, + PreviewAutomationRequestQueueClosedError, + PreviewAutomationTimeoutError, + PreviewTabId, + ProviderInstanceId, + ThreadId, +} from "@t3tools/contracts"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +import * as ServerConfig from "../config.ts"; +import { + PreviewAutomationBroker, + type PreviewAutomationInvokeInput, +} from "../mcp/PreviewAutomationBroker.ts"; +import * as Snapshot from "./Snapshot.ts"; + +const tabId = PreviewTabId.make("text-tab"); +const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); +const scope = { + environmentId: EnvironmentId.make("text-environment"), + requestNamespace: "text-request-namespace", + thread: { + threadId: ThreadId.make("text-thread"), + providerSessionId: "text-session", + providerInstanceId: ProviderInstanceId.make("codex"), + }, + client: undefined, + capabilities: new Set(["preview"] as const), + issuedAt: 1, +}; +const TestLayer = ServerConfig.layerTest(process.cwd(), { prefix: "t3-preview-text-" }).pipe( + Layer.provideMerge(NodeServices.layer), +); + +const makePage = (initialText: string) => { + let text = initialText; + let reads = 0; + const timers = new Map<() => void, number>(); + const listeners = new Set<() => void>(); + const requests: PreviewAutomationInvokeInput[] = []; + const context = NodeVM.createContext({ + TextEncoder, + document: { + body: { + get innerText() { + reads++; + return text; + }, + }, + }, + location: { href: "https://example.test/page" }, + setTimeout: (callback: () => void, delay: number) => { + timers.set(callback, delay); + return callback; + }, + clearTimeout: (callback: () => void) => timers.delete(callback), + addEventListener: (event: string, callback: () => void) => { + expect(event).toBe("pagehide"); + listeners.add(callback); + }, + removeEventListener: (event: string, callback: () => void) => { + expect(event).toBe("pagehide"); + listeners.delete(callback); + }, + }); + const page = { + context, + timers, + listeners, + requests, + reads: () => reads, + available: true, + setText: (value: string) => { + text = value; + }, + beforeSnapshot: undefined as (() => void) | undefined, + snapshotGate: undefined as Effect.Effect | undefined, + captureGate: undefined as Effect.Effect | undefined, + chunkGate: undefined as Effect.Effect | undefined, + broker: undefined as PreviewAutomationBroker["Service"] | undefined, + }; + const invoke = (request: PreviewAutomationInvokeInput) => + Effect.gen(function* () { + requests.push(request); + if (request.operation === "status") { + return { + available: page.available, + visible: true, + tabId, + url: "https://example.test/page", + title: "Page", + loading: false, + } as A; + } + if (request.tabId !== undefined && request.tabId !== tabId) { + return yield* new PreviewAutomationNoAvailableHostError({ + environmentId: scope.environmentId, + ...scope.thread, + operation: request.operation, + }); + } + if (request.operation === "snapshot") { + page.beforeSnapshot?.(); + if (page.snapshotGate) yield* page.snapshotGate; + return { + url: NodeVM.runInContext("location.href", context), + title: "Page", + loading: false, + visibleText: "Page", + interactiveElements: [], + accessibilityTree: {}, + consoleEntries: [], + networkEntries: [], + actionTimeline: [], + screenshot: { + mimeType: "image/png", + data: Buffer.from("png").toString("base64"), + width: 10, + height: 5, + }, + } as A; + } + expect(request.operation).toBe("evaluate"); + expect(request.tabId).toBe(tabId); + expect(request.timeoutMs).toBeUndefined(); + expect(request.updateCurrentTab).toBe(false); + const expression = (request.input as { expression: string }).expression; + if (expression.includes("let end") && page.chunkGate) yield* page.chunkGate; + const value = yield* Effect.try({ + try: () => { + const value = NodeVM.runInContext(expression, context) as A; + expect(Buffer.byteLength(encodeJson(value), "utf8")).toBeLessThan(16_512); + return value; + }, + catch: () => + new PreviewAutomationNoAvailableHostError({ + environmentId: scope.environmentId, + ...scope.thread, + operation: "evaluate", + }), + }); + if (expression.includes("Object.defineProperty") && page.captureGate) yield* page.captureGate; + return value; + }); + page.broker = PreviewAutomationBroker.of({ + invoke, + connect: () => Effect.die("unused"), + focusHost: () => Effect.void, + respond: () => Effect.void, + }); + return page; +}; + +const captureText = Effect.fnUntraced(function* (requestedTabId?: PreviewTabId) { + const snapshot = yield* Snapshot.PreviewSnapshot; + return yield* snapshot.withSnapshot( + { + scope, + captureText: true, + ...(requestedTabId === undefined ? {} : { tabId: requestedTabId }), + }, + ({ textCapture }) => Effect.succeed(textCapture!), + ); +}); +const readText = Effect.fnUntraced(function* ( + captureId: string, + offset = 0, + release = false, + requestScope = scope, + requestedTabId = tabId, +) { + const snapshot = yield* Snapshot.PreviewSnapshot; + return yield* snapshot.readText({ + scope: requestScope, + tabId: requestedTabId, + captureId, + offset, + release, + }); +}); +const providePage = (page: ReturnType) => + Effect.provide( + Snapshot.layer.pipe(Layer.provide(Layer.succeed(PreviewAutomationBroker, page.broker!))), + ); +const assertClean = (page: ReturnType) => { + expect(page.timers.size).toBe(0); + expect(page.listeners.size).toBe(0); + expect( + Object.getOwnPropertyNames(page.context).filter((name) => + name.startsWith("__t3_text_capture_"), + ), + ).toEqual([]); +}; + +it.effect("leaves default snapshots unchanged and does not create browser or disk text", () => { + const page = makePage("loaded text"); + return Effect.gen(function* () { + const snapshots = yield* Snapshot.PreviewSnapshot; + const result = yield* snapshots.withSnapshot({ scope }, Effect.succeed); + const fs = yield* FileSystem.FileSystem; + const config = yield* ServerConfig.ServerConfig; + expect(result.textCapture).toBeUndefined(); + expect(result.snapshot.visibleText).toBe("Page"); + expect(Buffer.from(result.png).toString()).toBe("png"); + expect(page.reads()).toBe(0); + expect(page.requests.map((request) => request.operation)).toEqual(["snapshot"]); + expect(yield* fs.exists(config.browserArtifactsDir)).toBe(false); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect( + "reads all loaded text in bounded parts without disk writes or repeated DOM reads", + () => { + const text = "\u0001".repeat(70_000) + "loaded end"; + const page = makePage(text); + return Effect.gen(function* () { + const captured = yield* captureText(); + expect(captured).toMatchObject({ + totalChars: text.length, + tabId, + url: "https://example.test/page", + }); + expect(page.requests[0]?.operation).toBe("status"); + expect(page.requests.filter((request) => request.operation === "evaluate")).toHaveLength(2); + let offset = 0; + let collected = ""; + while (offset < captured.totalChars) { + const part = yield* readText(captured.captureId, offset); + expect(part.text.length).toBeLessThanOrEqual(4096); + expect(part.nextOffset).toBe(offset + part.text.length); + expect(part.totalChars).toBe(text.length); + expect(part.done).toBe(part.nextOffset === text.length); + expect(part.released).toBe(false); + collected += part.text; + offset = part.nextOffset; + } + expect(collected).toBe(text); + expect(page.reads()).toBe(1); + const fs = yield* FileSystem.FileSystem; + const config = yield* ServerConfig.ServerConfig; + expect(yield* fs.exists(config.browserArtifactsDir)).toBe(false); + yield* readText(captured.captureId, 0, true); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); + }, +); + +it.effect("preserves Unicode pairs at read boundaries and rejects an offset inside a pair", () => { + const text = "a".repeat(4095) + "πŸ˜€δΈ­ζ–‡" + "b".repeat(4093) + "πŸš€"; + const page = makePage(text); + return Effect.gen(function* () { + const captured = yield* captureText(tabId); + const first = yield* readText(captured.captureId); + expect(first.text).toBe("a".repeat(4095)); + const second = yield* readText(captured.captureId, first.nextOffset); + expect(second.text.startsWith("πŸ˜€δΈ­ζ–‡")).toBe(true); + const third = yield* readText(captured.captureId, second.nextOffset); + expect(first.text + second.text + third.text).toBe(text); + expect(yield* Effect.flip(readText(captured.captureId, 4096))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); + expect(page.requests.some((request) => request.operation === "status")).toBe(false); + yield* readText(captured.captureId, 0, true); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect.each(["", "short text"])("retains completed reads for retries: %j", (text) => { + const page = makePage(text); + return Effect.gen(function* () { + const captured = yield* captureText(); + const first = yield* readText(captured.captureId); + expect(first).toEqual({ + text, + nextOffset: text.length, + totalChars: text.length, + done: true, + released: false, + }); + expect(yield* readText(captured.captureId)).toEqual(first); + expect(yield* readText(captured.captureId, text.length)).toEqual({ + text: "", + nextOffset: text.length, + totalChars: text.length, + done: true, + released: false, + }); + const released = yield* readText(captured.captureId, 0, true); + expect(released).toEqual({ + text: "", + nextOffset: 0, + totalChars: text.length, + done: true, + released: true, + }); + expect(yield* Effect.flip(readText(captured.captureId))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect("freezes loaded text while the same document changes", () => { + const page = makePage("original ".repeat(9000)); + return Effect.gen(function* () { + page.beforeSnapshot = () => page.setText("changed before snapshot"); + const captured = yield* captureText(); + const key = Object.getOwnPropertyNames(page.context).find((name) => + name.startsWith("__t3_text_capture_"), + )!; + expect( + NodeVM.runInContext(`Object.isFrozen(globalThis[${encodeJson(key)}])`, page.context), + ).toBe(true); + page.setText("changed before read"); + expect((yield* readText(captured.captureId)).text).toBe( + "original ".repeat(9000).slice(0, 4096), + ); + expect(page.reads()).toBe(1); + yield* readText(captured.captureId, 0, true); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect.each(["navigation", "reload", "new document"] as const)( + "rejects %s before snapshot delivery and removes its capture", + (change) => { + const page = makePage("original loaded text"); + return Effect.gen(function* () { + page.beforeSnapshot = () => { + if (change === "navigation") + NodeVM.runInContext("location.href = 'https://other.test/'", page.context); + else { + if (change === "reload") [...page.listeners][0]!(); + NodeVM.runInContext( + "document = { body: { innerText: 'different document' } }", + page.context, + ); + } + }; + expect(yield* Effect.flip(captureText())).toBeInstanceOf(Snapshot.PreviewTextCaptureError); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); + }, +); + +it.effect("validates a full page URL longer than the evaluate output limit", () => { + const page = makePage("loaded text"); + return Effect.gen(function* () { + const url = `https://example.test/page?q=${"x".repeat(70_000)}`; + NodeVM.runInContext(`location.href = ${encodeJson(url)}`, page.context); + const captured = yield* captureText(); + expect(captured.url).toBe(url.slice(0, 2048)); + expect((yield* readText(captured.captureId)).text).toBe("loaded text"); + expect(page.reads()).toBe(1); + yield* readText(captured.captureId, 0, true); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect.each(["navigation", "expiry", "pagehide", "new document"] as const)( + "rejects further reads after %s", + (failure) => { + const page = makePage("text ".repeat(2000)); + return Effect.gen(function* () { + const captured = yield* captureText(); + yield* readText(captured.captureId); + if (failure === "navigation") + NodeVM.runInContext("location.href = 'https://other.test/'", page.context); + else if (failure === "expiry") { + expect([...page.timers.values()]).toEqual([300_000]); + [...page.timers.keys()][0]!(); + } else if (failure === "pagehide") [...page.listeners][0]!(); + else NodeVM.runInContext("document = { body: { innerText: 'different' } }", page.context); + expect(yield* Effect.flip(readText(captured.captureId, 4096))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); + if (page.timers.size > 0) [...page.timers.keys()][0]!(); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); + }, +); + +it.effect.each(["environmentId", "threadId", "providerSessionId", "providerInstanceId"] as const)( + "does not disclose or release another %s's capture", + (field) => { + const page = makePage("private text"); + return Effect.gen(function* () { + const captured = yield* captureText(); + const otherScope = { + ...scope, + environmentId: + field === "environmentId" ? EnvironmentId.make("other-environment") : scope.environmentId, + thread: { + threadId: field === "threadId" ? ThreadId.make("other-thread") : scope.thread.threadId, + providerSessionId: + field === "providerSessionId" ? "other-session" : scope.thread.providerSessionId, + providerInstanceId: + field === "providerInstanceId" + ? ProviderInstanceId.make("other-provider") + : scope.thread.providerInstanceId, + }, + }; + expect(yield* Effect.flip(readText(captured.captureId, 0, false, otherScope))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); + expect(yield* Effect.flip(readText(captured.captureId, 0, true, otherScope))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); + expect((yield* readText(captured.captureId)).text).toBe("private text"); + yield* readText(captured.captureId, 0, true); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); + }, +); + +it.effect("rejects a different tab or capture version without releasing the valid capture", () => { + const page = makePage("private text"); + return Effect.gen(function* () { + const captured = yield* captureText(); + expect( + yield* Effect.flip( + readText(captured.captureId, 0, false, scope, PreviewTabId.make("other-tab")), + ), + ).toBeInstanceOf(PreviewAutomationNoAvailableHostError); + expect(yield* Effect.flip(readText("wrong-capture"))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); + expect(yield* Effect.flip(readText("wrong-capture", 0, true))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); + expect((yield* readText(captured.captureId)).text).toBe("private text"); + yield* readText(captured.captureId, 0, true); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect.each([-1, 0.5, 5, Number.NaN, Number.POSITIVE_INFINITY])( + "rejects invalid offset %s without damaging the capture", + (offset) => { + const page = makePage("text"); + return Effect.gen(function* () { + const captured = yield* captureText(); + expect(yield* Effect.flip(readText(captured.captureId, offset))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); + expect((yield* readText(captured.captureId)).text).toBe("text"); + yield* readText(captured.captureId, 0, true); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); + }, +); + +it.effect("replaces the prior capture and keeps one timer and page listener", () => { + const page = makePage("first text"); + return Effect.gen(function* () { + const first = yield* captureText(); + page.setText("second text"); + const second = yield* captureText(); + expect(second.captureId).not.toBe(first.captureId); + expect(page.timers.size).toBe(1); + expect(page.listeners.size).toBe(1); + expect(yield* Effect.flip(readText(first.captureId))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); + expect(yield* Effect.flip(readText(first.captureId, 0, true))).toBeInstanceOf( + Snapshot.PreviewTextCaptureError, + ); + expect((yield* readText(second.captureId)).text).toBe("second text"); + yield* readText(second.captureId, 0, true); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect.each(["failure", "cancellation"] as const)( + "removes an undelivered capture after result delivery %s", + (outcome) => { + const page = makePage("completed text"); + return Effect.gen(function* () { + const enteredUse = yield* Deferred.make(); + const snapshots = yield* Snapshot.PreviewSnapshot; + const fiber = yield* Effect.forkChild( + snapshots.withSnapshot({ scope, captureText: true }, ({ textCapture }) => + Effect.gen(function* () { + expect(textCapture?.totalChars).toBe(14); + yield* Deferred.succeed(enteredUse, undefined); + return yield* outcome === "failure" ? Effect.fail("delivery failed") : Effect.never; + }), + ), + ); + yield* Deferred.await(enteredUse); + if (outcome === "cancellation") yield* Fiber.interrupt(fiber); + expect((yield* Fiber.await(fiber))._tag).toBe("Failure"); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); + }, +); + +it.effect("does not let cancelled old delivery remove a newer capture", () => { + const page = makePage("first text"); + return Effect.gen(function* () { + const enteredUse = yield* Deferred.make(); + const snapshots = yield* Snapshot.PreviewSnapshot; + const fiber = yield* Effect.forkChild( + snapshots.withSnapshot({ scope, captureText: true }, () => + Deferred.succeed(enteredUse, undefined).pipe(Effect.andThen(Effect.never)), + ), + ); + yield* Deferred.await(enteredUse); + page.setText("new text"); + const captured = yield* captureText(); + yield* Fiber.interrupt(fiber); + expect((yield* readText(captured.captureId)).text).toBe("new text"); + expect(page.timers.size).toBe(1); + yield* readText(captured.captureId, 0, true); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect("removes the capture when snapshot collection is interrupted", () => { + const page = makePage("loaded text"); + return Effect.gen(function* () { + const enteredSnapshot = yield* Deferred.make(); + page.snapshotGate = Deferred.succeed(enteredSnapshot, undefined).pipe( + Effect.andThen(Effect.never), + ); + const fiber = yield* Effect.forkChild(captureText()); + yield* Deferred.await(enteredSnapshot); + yield* Fiber.interrupt(fiber); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect("does not capture text when the resolved tab is unavailable", () => { + const page = makePage("text"); + page.available = false; + return Effect.gen(function* () { + expect(yield* Effect.flip(captureText())).toBeInstanceOf(Snapshot.PreviewTextCaptureError); + expect(page.requests).toHaveLength(1); + expect(page.reads()).toBe(0); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect.each(["status", "evaluate"] as const)( + "preserves browser recovery errors during text %s", + (operation) => + Effect.gen(function* () { + for (const ErrorClass of [ + PreviewAutomationNoAvailableHostError, + PreviewAutomationTimeoutError, + PreviewAutomationRequestQueueClosedError, + ]) { + const page = makePage("loaded text"); + const error = new ErrorClass({ + environmentId: scope.environmentId, + ...scope.thread, + operation, + clientId: "text-client", + connectionId: "text-connection", + requestId: "text-request", + timeoutMs: 15_000, + }); + const original = page.broker!; + page.broker = PreviewAutomationBroker.of({ + ...original, + invoke: (request) => + request.operation === operation ? Effect.fail(error) : original.invoke(request), + }); + expect(yield* Effect.flip(captureText().pipe(providePage(page)))).toBe(error); + assertClean(page); + } + }).pipe(Effect.provide(TestLayer)), +); + +it.effect("preserves opt-in PNG saving without writing a text file", () => { + const page = makePage("loaded text"); + return Effect.gen(function* () { + const snapshots = yield* Snapshot.PreviewSnapshot; + const result = yield* snapshots.withSnapshot( + { scope, captureText: true, save: true }, + Effect.succeed, + ); + const fs = yield* FileSystem.FileSystem; + const config = yield* ServerConfig.ServerConfig; + expect(Buffer.from(yield* fs.readFile(result.screenshotPath!)).toString()).toBe("png"); + expect(yield* fs.readDirectory(config.browserArtifactsDir)).toHaveLength(1); + expect(result.screenshotPath).toMatch(/browser-screenshot-example-test-.*\.png$/); + yield* readText(result.textCapture!.captureId, 0, true); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect("keeps another owner's live capture when this owner captures or releases text", () => { + const page = makePage("first owner's text"); + return Effect.gen(function* () { + const snapshots = yield* Snapshot.PreviewSnapshot; + const first = yield* captureText(); + const otherScope = { + ...scope, + thread: { ...scope.thread, threadId: ThreadId.make("other-thread") }, + }; + page.setText("second owner's text"); + const second = yield* snapshots.withSnapshot( + { scope: otherScope, tabId, captureText: true }, + ({ textCapture }) => Effect.succeed(textCapture!), + ); + expect(page.timers.size).toBe(2); + expect((yield* readText(first.captureId)).text).toBe("first owner's text"); + expect((yield* readText(second.captureId, 0, false, otherScope)).text).toBe( + "second owner's text", + ); + yield* readText(first.captureId, 0, true); + expect(page.timers.size).toBe(1); + expect((yield* readText(second.captureId, 0, false, otherScope)).text).toBe( + "second owner's text", + ); + yield* readText(second.captureId, 0, true, otherScope); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect("allows the same offset to be retried after an interrupted read", () => { + const page = makePage("loaded ".repeat(2000)); + return Effect.gen(function* () { + const captured = yield* captureText(); + const enteredRead = yield* Deferred.make(); + page.chunkGate = Deferred.succeed(enteredRead, undefined).pipe(Effect.andThen(Effect.never)); + const fiber = yield* Effect.forkChild(readText(captured.captureId)); + yield* Deferred.await(enteredRead); + yield* Fiber.interrupt(fiber); + page.chunkGate = undefined; + const retried = yield* readText(captured.captureId); + expect(retried.text).toBe("loaded ".repeat(2000).slice(0, 4096)); + expect(retried.nextOffset).toBe(4096); + expect(page.reads()).toBe(1); + yield* readText(captured.captureId, 0, true); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect("releases its memory capture when the requested PNG cannot be saved", () => { + const page = makePage("loaded text"); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const config = yield* ServerConfig.ServerConfig; + yield* fs.makeDirectory(config.stateDir, { recursive: true }); + yield* fs.writeFileString(config.browserArtifactsDir, "existing file"); + const snapshots = yield* Snapshot.PreviewSnapshot; + const failed = yield* Effect.flip( + snapshots.withSnapshot({ scope, captureText: true, save: true }, Effect.succeed), + ); + expect(failed).toBeInstanceOf(Snapshot.PreviewScreenshotSaveError); + expect(yield* fs.readFileString(config.browserArtifactsDir)).toBe("existing file"); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect("cleans an installed capture when its initial response is interrupted", () => { + const page = makePage("loaded text"); + return Effect.gen(function* () { + const enteredCapture = yield* Deferred.make(); + page.captureGate = Deferred.succeed(enteredCapture, undefined).pipe( + Effect.andThen(Effect.never), + ); + const fiber = yield* Effect.forkChild(captureText()); + yield* Deferred.await(enteredCapture); + expect(page.timers.size).toBe(1); + yield* Fiber.interrupt(fiber); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); + +it.effect("keeps the same owner's capture when its credential is renewed", () => { + const page = makePage("loaded text"); + return Effect.gen(function* () { + const captured = yield* captureText(); + const renewedScope = { ...scope, issuedAt: 2, requestNamespace: "renewed-request-namespace" }; + expect((yield* readText(captured.captureId, 0, false, renewedScope)).text).toBe("loaded text"); + yield* readText(captured.captureId, 0, true, renewedScope); + assertClean(page); + }).pipe(providePage(page), Effect.provide(TestLayer)); +}); diff --git a/apps/server/src/preview/Snapshot.ts b/apps/server/src/preview/Snapshot.ts new file mode 100644 index 000000000000..e4f9bdc89d20 --- /dev/null +++ b/apps/server/src/preview/Snapshot.ts @@ -0,0 +1,330 @@ +import { + PreviewAutomationSnapshot, + PreviewAutomationStatus, + PreviewAutomationError, + type PreviewTabId, +} from "@t3tools/contracts"; +import * as NodeCrypto from "node:crypto"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; + +import * as ServerConfig from "../config.ts"; +import * as PreviewAutomationBroker from "../mcp/PreviewAutomationBroker.ts"; + +export class PreviewTextCaptureError extends Schema.TaggedError()( + "PreviewTextCaptureError", + { cause: Schema.Defect() }, +) { + override get message(): string { + return "Could not read the loaded page text. Take another snapshot with captureText=true if the page changed or the capture expired."; + } +} + +const isPreviewAutomationError = Schema.is(PreviewAutomationError); +const isPreviewTextCaptureError = Schema.is(PreviewTextCaptureError); +const textCaptureError = (cause: unknown) => + isPreviewAutomationError(cause) || isPreviewTextCaptureError(cause) + ? cause + : new PreviewTextCaptureError({ cause }); + +const Capture = Schema.Struct({ + totalChars: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + url: Schema.String.check(Schema.isMaxLength(2048)), +}); +const TextChunk = Schema.Struct({ + text: Schema.String.check(Schema.isMaxLength(4096)), + nextOffset: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + totalChars: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + done: Schema.Boolean, + released: Schema.Boolean, +}); +const encodePageString = Schema.encodeEffect(Schema.fromJsonString(Schema.String)); +const decodeCapture = Schema.decodeUnknownEffect(Capture); +const decodeTextChunk = Schema.decodeUnknownEffect(TextChunk); +const decodeStatus = Schema.decodeUnknownEffect(PreviewAutomationStatus); +const decodeSnapshot = Schema.decodeUnknownEffect(PreviewAutomationSnapshot); +const decodeCaptureMatch = Schema.decodeUnknownEffect(Schema.Literal(true)); + +export class PreviewScreenshotSaveError extends Schema.TaggedError()( + "PreviewScreenshotSaveError", + { screenshotPath: Schema.String, cause: Schema.Defect() }, +) { + override get message(): string { + return `Could not save preview screenshot to ${this.screenshotPath}.`; + } +} + +interface SnapshotInput { + readonly scope: PreviewAutomationBroker.PreviewAutomationInvokeInput["scope"]; + readonly tabId?: PreviewTabId | undefined; + readonly captureText?: boolean | undefined; + readonly save?: boolean | undefined; +} + +interface SnapshotCapture { + readonly snapshot: PreviewAutomationSnapshot; + readonly png: Uint8Array; + readonly textCapture?: { + readonly captureId: string; + readonly totalChars: number; + readonly url: string; + readonly tabId: PreviewTabId; + }; + readonly screenshotPath?: string; +} + +export class PreviewSnapshot extends Context.Service< + PreviewSnapshot, + { + readonly readText: (input: { + readonly scope: SnapshotInput["scope"]; + readonly tabId: PreviewTabId; + readonly captureId: string; + readonly offset?: number | undefined; + readonly release?: boolean | undefined; + }) => Effect.Effect; + readonly withSnapshot: ( + input: SnapshotInput, + use: (capture: SnapshotCapture) => Effect.Effect, + ) => Effect.Effect< + A, + | E + | PreviewAutomationError + | PreviewTextCaptureError + | PreviewScreenshotSaveError + | Schema.SchemaError, + R + >; + } +>()("t3/preview/Snapshot/PreviewSnapshot") {} + +const screenshotSiteSlug = (rawUrl: string): string => { + try { + const slug = new URL(rawUrl).hostname + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 40) + .replace(/-+$/g, ""); + return slug || "site"; + } catch { + return "site"; + } +}; + +const make = Effect.gen(function* () { + const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const config = yield* ServerConfig.ServerConfig; + + const captureKey = (scope: SnapshotInput["scope"]) => + `__t3_text_capture_${NodeCrypto.createHash("sha256") + .update( + JSON.stringify([ + scope.environmentId, + scope.thread.threadId, + scope.thread.providerSessionId, + scope.thread.providerInstanceId, + ]), + ) + .digest("hex")}`; + const evaluate = (scope: SnapshotInput["scope"], tabId: PreviewTabId, expression: string) => + broker.invoke({ + scope, + operation: "evaluate", + tabId, + input: { expression, returnByValue: true }, + updateCurrentTab: false, + }); + + const readText = Effect.fn("PreviewSnapshot.readText")(function* (input: { + readonly scope: SnapshotInput["scope"]; + readonly tabId: PreviewTabId; + readonly captureId: string; + readonly offset?: number | undefined; + readonly release?: boolean | undefined; + }) { + const offset = input.offset ?? 0; + if (!Number.isSafeInteger(offset) || offset < 0 || input.captureId.length > 128) { + return yield* new PreviewTextCaptureError({ cause: "Invalid capture or offset." }); + } + const key = yield* encodePageString(captureKey(input.scope)).pipe(Effect.orDie); + const id = yield* encodePageString(input.captureId).pipe(Effect.orDie); + const chunk = yield* evaluate( + input.scope, + input.tabId, + `(() => { + const capture = globalThis[${key}]; + if (!capture || capture.captureId !== ${id}) return null; + if (capture.document !== document || capture.url !== location.href) { + capture.dispose(); + return null; + } + const totalChars = capture.text.length; + if (${input.release === true}) { + capture.dispose(); + return { text: "", nextOffset: 0, totalChars, done: true, released: true }; + } + const offset = ${offset}; + if (offset > totalChars) return null; + const first = capture.text.charCodeAt(offset); + const previous = capture.text.charCodeAt(offset - 1); + if (first >= 0xdc00 && first <= 0xdfff && previous >= 0xd800 && previous <= 0xdbff) return null; + let end = Math.min(offset + 4096, totalChars); + if (new TextEncoder().encode(JSON.stringify({ text: capture.text.slice(offset, end) })).byteLength > 16000) { + end = offset + Math.floor((end - offset) / 2); + } + if (end < totalChars) { + const last = capture.text.charCodeAt(end - 1); + const next = capture.text.charCodeAt(end); + if (last >= 0xd800 && last <= 0xdbff && next >= 0xdc00 && next <= 0xdfff) end--; + } + capture.refresh(); + return { text: capture.text.slice(offset, end), nextOffset: end, totalChars, done: end === totalChars, released: false }; + })()`, + ).pipe(Effect.flatMap(decodeTextChunk), Effect.mapError(textCaptureError)); + if ( + !chunk.released && + (chunk.nextOffset !== offset + chunk.text.length || + chunk.nextOffset > chunk.totalChars || + (offset < chunk.totalChars && chunk.nextOffset <= offset) || + chunk.done !== (chunk.nextOffset === chunk.totalChars)) + ) { + return yield* new PreviewTextCaptureError({ cause: "Invalid text capture chunk." }); + } + return chunk; + }); + + const saveScreenshot = Effect.fn("PreviewSnapshot.saveScreenshot")(function* ( + pageUrl: string, + data: Uint8Array, + ) { + const millis = yield* Clock.currentTimeMillis; + const fileName = `browser-screenshot-${screenshotSiteSlug(pageUrl)}-${millis.toString(36)}-${NodeCrypto.randomUUID().slice(0, 8)}.png`; + const screenshotPath = path.join(config.browserArtifactsDir, fileName); + yield* fileSystem.makeDirectory(config.browserArtifactsDir, { recursive: true }).pipe( + Effect.andThen(fileSystem.writeFile(screenshotPath, data)), + Effect.mapError((cause) => new PreviewScreenshotSaveError({ screenshotPath, cause })), + ); + return screenshotPath; + }); + + const withSnapshot = Effect.fn("PreviewSnapshot.withSnapshot")(function* ( + input: SnapshotInput, + use: (capture: SnapshotCapture) => Effect.Effect, + ) { + const captureId = NodeCrypto.randomUUID(); + const key = yield* encodePageString(captureKey(input.scope)).pipe(Effect.orDie); + const id = yield* encodePageString(captureId).pipe(Effect.orDie); + let captureTabId: PreviewTabId | undefined; + return yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + let textCapture: SnapshotCapture["textCapture"]; + if (input.captureText === true) { + let tabId = input.tabId; + if (tabId === undefined) { + const status = yield* restore( + broker + .invoke({ scope: input.scope, operation: "status", input: {} }) + .pipe(Effect.flatMap(decodeStatus), Effect.mapError(textCaptureError)), + ); + if (!status.available || status.tabId === null) { + return yield* new PreviewTextCaptureError({ cause: "No available preview tab." }); + } + tabId = status.tabId; + } + captureTabId = tabId; + const capture = yield* restore( + evaluate( + input.scope, + tabId, + `(() => { + globalThis[${key}]?.dispose(); + const text = document.body?.innerText ?? ""; + let timer; + const dispose = () => { + clearTimeout(timer); + removeEventListener("pagehide", dispose); + if (globalThis[${key}] === capture) delete globalThis[${key}]; + }; + const refresh = () => { + clearTimeout(timer); + timer = setTimeout(dispose, 300000); + }; + const capture = Object.freeze({ captureId: ${id}, text, url: location.href, document, dispose, refresh }); + Object.defineProperty(globalThis, ${key}, { value: capture, configurable: true }); + addEventListener("pagehide", dispose, { once: true }); + refresh(); + return { totalChars: text.length, url: capture.url.slice(0, 2048) }; + })()`, + ).pipe(Effect.flatMap(decodeCapture), Effect.mapError(textCaptureError)), + ); + textCapture = { captureId, ...capture, tabId }; + } + return yield* restore( + Effect.gen(function* () { + const tabId = textCapture?.tabId ?? input.tabId; + const snapshot = yield* broker + .invoke({ + scope: input.scope, + operation: "snapshot", + input: {}, + ...(tabId === undefined ? {} : { tabId }), + }) + .pipe(Effect.flatMap(decodeSnapshot)); + if (textCapture !== undefined) { + for (let offset = 0; offset < Math.max(snapshot.url.length, 1); offset += 4096) { + const urlChunk = yield* encodePageString( + snapshot.url.slice(offset, offset + 4096), + ).pipe(Effect.orDie); + yield* evaluate( + input.scope, + textCapture.tabId, + `(() => { + const capture = globalThis[${key}]; + if (!capture || capture.captureId !== ${id} || capture.document !== document || capture.url !== location.href) return false; + capture.refresh(); + return capture.url.length === ${snapshot.url.length} && capture.url.slice(${offset}, ${offset + 4096}) === ${urlChunk}; + })()`, + ).pipe(Effect.flatMap(decodeCaptureMatch), Effect.mapError(textCaptureError)); + } + } + const png = new Uint8Array(Buffer.from(snapshot.screenshot.data, "base64")); + const screenshotPath = + input.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; + return yield* use({ + snapshot, + png, + ...(textCapture === undefined ? {} : { textCapture }), + ...(screenshotPath === undefined ? {} : { screenshotPath }), + }); + }), + ); + }), + ).pipe( + Effect.onExit((exit) => + captureTabId !== undefined && exit._tag === "Failure" + ? evaluate( + input.scope, + captureTabId, + `(() => { + const capture = globalThis[${key}]; + if (capture?.captureId === ${id}) capture.dispose(); + return true; + })()`, + ).pipe(Effect.interruptible, Effect.timeoutOption(5000), Effect.ignore) + : Effect.void, + ), + ); + }); + + return PreviewSnapshot.of({ withSnapshot, readText }); +}); + +export const layer = Layer.effect(PreviewSnapshot, make); diff --git a/docs/user/browser.md b/docs/user/browser.md new file mode 100644 index 000000000000..bc84a3f472a1 --- /dev/null +++ b/docs/user/browser.md @@ -0,0 +1,31 @@ +# Agent browser snapshots + +When an agent inspects a page in the desktop browser preview, its snapshot +includes text and controls in the current view, page text, and scroll positions. +Controls in the current view come first. Scroll details also include visible +scroll boxes, including boxes that contain only images or canvas content. + +Snapshots have a size limit. Current-view text takes priority over page text and +logs. The result reports omitted content so the agent can request more detail. +The screenshot shows the current view. + +An agent can set `captureText=true` when taking a snapshot to keep all loaded, +rendered main-page text in temporary browser memory. The capture has no total +character cap. The agent uses the returned `textCaptureId` and `textTabId` with +`preview_read_text` to read small parts, starting at offset zero and following +`nextOffset` until `done=true`. This does not scroll or load missing content. + +No text file is created. Normal chat and tool history can still store text the +agent reads. The capture expires after five idle minutes, a page change, or a +replacement capture. The agent can also release it with `release=true`. + +Complex CSS clip shapes and transformed clips can cause current-view text and +controls to be omitted. The result reports these omissions. Use the screenshot +to inspect those areas. + +On old HTML pages, a scroll box on the page body can report a size that includes +scrollbar space. Use the screenshot when precise edge positions matter. + +Content that loads during scrolling is available after it loads. Embedded frames +and shadow DOM can require a separate inspection. A snapshot does not scroll the +page or change its content. diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index 734aa1bb53df..efc1d9cbfeb5 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -496,6 +496,7 @@ export const PreviewAutomationElement = Schema.Struct({ y: Schema.Number, width: Schema.Number, height: Schema.Number, + inViewport: Schema.optional(Schema.Boolean), }); export type PreviewAutomationElement = typeof PreviewAutomationElement.Type; @@ -527,11 +528,37 @@ export const PreviewAutomationActionEvent = Schema.Struct({ }); export type PreviewAutomationActionEvent = typeof PreviewAutomationActionEvent.Type; +const PreviewAutomationScrollPosition = { + x: Schema.Number, + y: Schema.Number, + width: Schema.Number, + height: Schema.Number, + scrollWidth: Schema.Number, + scrollHeight: Schema.Number, +}; + export const PreviewAutomationSnapshot = Schema.Struct({ url: Schema.String, title: Schema.String, loading: Schema.Boolean, visibleText: Schema.String, + viewportText: Schema.optional(Schema.String), + scroll: Schema.optional( + Schema.Struct({ + ...PreviewAutomationScrollPosition, + containers: Schema.Array( + Schema.Struct({ selector: Schema.String, ...PreviewAutomationScrollPosition }), + ), + containersTruncated: Schema.Boolean, + }), + ), + truncated: Schema.optional( + Schema.Struct({ + visibleText: Schema.Boolean, + viewportText: Schema.Boolean, + interactiveElements: Schema.Boolean, + }), + ), interactiveElements: Schema.Array(PreviewAutomationElement), accessibilityTree: Schema.Unknown, consoleEntries: Schema.Array(PreviewAutomationConsoleEntry), diff --git a/packages/contracts/src/previewAutomationSnapshot.test.ts b/packages/contracts/src/previewAutomationSnapshot.test.ts new file mode 100644 index 000000000000..74245583744e --- /dev/null +++ b/packages/contracts/src/previewAutomationSnapshot.test.ts @@ -0,0 +1,65 @@ +import { Schema } from "effect"; +import { describe, expect, it } from "vite-plus/test"; + +import { PreviewAutomationSnapshot } from "./previewAutomation.ts"; + +const legacySnapshot = { + url: "https://example.test/", + title: "Page", + loading: false, + visibleText: "Whole page", + interactiveElements: [ + { + tag: "button", + role: null, + name: "Open", + selector: "#open", + x: 0, + y: 0, + width: 80, + height: 20, + }, + ], + accessibilityTree: null, + consoleEntries: [], + networkEntries: [], + actionTimeline: [], + screenshot: { mimeType: "image/png", data: "image", width: 300, height: 200 }, +}; +const decode = Schema.decodeUnknownSync(PreviewAutomationSnapshot); + +describe("PreviewAutomationSnapshot", () => { + it("accepts snapshots from desktop hosts without scroll context", () => { + expect(decode(legacySnapshot)).toEqual(legacySnapshot); + }); + + it("preserves viewport text, scroll context, and omission flags across the wire", () => { + const snapshot = { + ...legacySnapshot, + viewportText: "Current view", + interactiveElements: [{ ...legacySnapshot.interactiveElements[0], inViewport: true }], + scroll: { + x: 0, + y: 2_000, + width: 300, + height: 200, + scrollWidth: 300, + scrollHeight: 5_000, + containers: [ + { + selector: "#list", + x: 10, + y: 600, + width: 100, + height: 100, + scrollWidth: 200, + scrollHeight: 1_000, + }, + ], + containersTruncated: false, + }, + truncated: { visibleText: true, viewportText: false, interactiveElements: true }, + }; + expect(decode(snapshot)).toEqual(snapshot); + }); +}); diff --git a/packages/shared/src/t3McpToolPresentation.test.ts b/packages/shared/src/t3McpToolPresentation.test.ts index 43b2d9c2de15..dc2bbb58f8b5 100644 --- a/packages/shared/src/t3McpToolPresentation.test.ts +++ b/packages/shared/src/t3McpToolPresentation.test.ts @@ -72,6 +72,10 @@ describe("resolveT3McpToolPresentation", () => { displayName: "Get preview browser status", logo: "t3-code", }); + expect(resolveT3McpToolPresentation("mcp__t3_code__preview_read_text")).toEqual({ + displayName: "Read captured browser text", + logo: "t3-code", + }); }); it("matches the separator variants ACP registry agents emit", () => { diff --git a/packages/shared/src/t3McpToolPresentation.ts b/packages/shared/src/t3McpToolPresentation.ts index d85104cf4bc1..c7bf52c7e972 100644 --- a/packages/shared/src/t3McpToolPresentation.ts +++ b/packages/shared/src/t3McpToolPresentation.ts @@ -160,6 +160,11 @@ const T3_MCP_TOOLS: Readonly> = { "browser", "Snapshot the preview page", ), + preview_read_text: tool( + ["Read", "Reading", "Read", "captured browser text"], + "browser", + "browser", + ), preview_click: tool( ["Click", "Clicking", "Clicked", "in the preview browser"], "browser",