From f487d9418fc2cd0f184433468a3cec936d8a7b23 Mon Sep 17 00:00:00 2001 From: Shanu Date: Fri, 16 Oct 2026 01:08:21 +0530 Subject: [PATCH 1/4] Share the pointer press between the surface and sessions BrowserSurface::press_point built the raw mouse events of a press inline. They move to convert::press_at, with convert::move_to for the move alone, so every press the crate sends by position is the same trusted move, button down, and button up. --- .../src/convert/interaction.rs | 36 +++++++++++++++++++ .../tinycomputer-browser/src/convert/mod.rs | 2 +- .../tinycomputer-browser/src/surface/cover.rs | 16 ++------- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/crates/tinycomputer-browser/src/convert/interaction.rs b/crates/tinycomputer-browser/src/convert/interaction.rs index 862004f1..70cf7501 100644 --- a/crates/tinycomputer-browser/src/convert/interaction.rs +++ b/crates/tinycomputer-browser/src/convert/interaction.rs @@ -164,6 +164,42 @@ fn wait_for( } } +/// The raw mouse event that moves the pointer to the window's point (`x`, +/// `y`), as a person's hand does: whatever is on top there sees it arrive. +#[must_use] +pub(crate) fn move_to(x: f64, y: f64) -> Value { + json!({ + "action": "mouse", + "eventType": "mouseMoved", + "x": x, + "y": y, + "button": "none", + "clickCount": 0, + }) +} + +/// The raw mouse events of a person's press at the window's point (`x`, +/// `y`): the pointer moved there, then its left button pressed and +/// released. Whatever is on top there takes the press as trusted input. +#[must_use] +pub(crate) fn press_at(x: f64, y: f64) -> [Value; 3] { + let button = |event: &str| { + json!({ + "action": "mouse", + "eventType": event, + "x": x, + "y": y, + "button": "left", + "clickCount": 1, + }) + }; + [ + move_to(x, y), + button("mousePressed"), + button("mouseReleased"), + ] +} + /// The engine selector for a ref or CSS target. /// /// # Errors diff --git a/crates/tinycomputer-browser/src/convert/mod.rs b/crates/tinycomputer-browser/src/convert/mod.rs index 1d152837..2f809e49 100644 --- a/crates/tinycomputer-browser/src/convert/mod.rs +++ b/crates/tinycomputer-browser/src/convert/mod.rs @@ -13,7 +13,7 @@ mod interaction; mod page; mod session; -pub(crate) use interaction::{action, selector}; +pub(crate) use interaction::{action, press_at, selector}; pub(crate) use page::{evaluate, navigate, read, screenshot, snapshot}; pub(crate) use session::{launch, viewport}; diff --git a/crates/tinycomputer-browser/src/surface/cover.rs b/crates/tinycomputer-browser/src/surface/cover.rs index 622945d2..f7137e5e 100644 --- a/crates/tinycomputer-browser/src/surface/cover.rs +++ b/crates/tinycomputer-browser/src/surface/cover.rs @@ -261,20 +261,8 @@ impl BrowserSurface { /// Presses the window at (`x`, `y`) as a person's pointer does, with /// raw mouse events: whatever is on top there takes it. pub(super) fn press_point(&self, id: &SessionId, x: f64, y: f64) -> Option<()> { - for event in ["mouseMoved", "mousePressed", "mouseReleased"] { - let pressed = event != "mouseMoved"; - self.block(self.browser.command( - id, - json!({ - "action": "mouse", - "eventType": event, - "x": x, - "y": y, - "button": if pressed { "left" } else { "none" }, - "clickCount": i32::from(pressed), - }), - )) - .ok()?; + for command in crate::convert::press_at(x, y) { + self.block(self.browser.command(id, command)).ok()?; } Some(()) } From aad6ab18d8176c917642c7a696e302e31963f9a2 Mon Sep 17 00:00:00 2001 From: Shanu Date: Fri, 16 Oct 2026 01:09:30 +0530 Subject: [PATCH 2/4] List and act on controls inside shadow roots as d refs Pages built from web components keep their controls inside open shadow roots, often as plain spans and divs drawn with a pointer and wired to a click by the component's script. agent-browser looks for such elements in the light DOM only, so they came out as bare text, or, in a compact snapshot, not at all: there was nothing to press. Browser::snapshot now follows agent-browser's snapshot with one look through the open shadow roots (deep/deep.js) for what a person can press or type into that the tree has no ref for: an own pointer, an onclick, a tab stop, or an editable region, drawn and outermost, at most 40. Each is marked data-tc-deep and listed as dN, on the line that shows its words or in a group after the tree that a max_chars cut keeps whole. A number is never reused in a session, and an element keeps it across snapshots. A page with none comes out unchanged. Browser::perform acts on a d ref itself: the element is found by its mark (gone: StaleRef), brought into view, hit-tested through open shadow roots (covered: NotActionable naming the cover), and pressed with trusted mouse events. Type and Fill press into a field and keep the focus check; Focus, Press, Hover, GetText, and IsVisible work too, and any other action is InvalidInput. --- crates/tinycomputer-browser/README.md | 1 + .../tinycomputer-browser/src/convert/mod.rs | 2 +- crates/tinycomputer-browser/src/deep/deep.js | 324 +++++++++++++++ .../src/deep/deep_tests.rs | 316 +++++++++++++++ .../src/deep/deep_tests/live_tests.rs | 380 ++++++++++++++++++ .../src/deep/deep_tests/merge_tests.rs | 187 +++++++++ crates/tinycomputer-browser/src/deep/merge.rs | 199 +++++++++ crates/tinycomputer-browser/src/deep/mod.rs | 326 +++++++++++++++ crates/tinycomputer-browser/src/lib.rs | 1 + .../tinycomputer-browser/src/sessions/deep.rs | 270 +++++++++++++ .../src/sessions/focus.rs | 2 +- .../tinycomputer-browser/src/sessions/mod.rs | 4 + .../tinycomputer-browser/src/sessions/page.rs | 69 +++- .../src/sessions/sessions_tests.rs | 3 + .../sessions_tests/deep_keys_tests.rs | 249 ++++++++++++ .../src/sessions/sessions_tests/deep_page.rs | 83 ++++ .../src/sessions/sessions_tests/deep_tests.rs | 228 +++++++++++ .../surface/surface_tests/perception_tests.rs | 9 +- docs/crates/tinycomputer-browser/README.md | 1 + .../tinycomputer-browser/interacting.md | 64 ++- docs/crates/tinycomputer-bus/browser.md | 5 +- 21 files changed, 2702 insertions(+), 21 deletions(-) create mode 100644 crates/tinycomputer-browser/src/deep/deep.js create mode 100644 crates/tinycomputer-browser/src/deep/deep_tests.rs create mode 100644 crates/tinycomputer-browser/src/deep/deep_tests/live_tests.rs create mode 100644 crates/tinycomputer-browser/src/deep/deep_tests/merge_tests.rs create mode 100644 crates/tinycomputer-browser/src/deep/merge.rs create mode 100644 crates/tinycomputer-browser/src/deep/mod.rs create mode 100644 crates/tinycomputer-browser/src/sessions/deep.rs create mode 100644 crates/tinycomputer-browser/src/sessions/sessions_tests/deep_keys_tests.rs create mode 100644 crates/tinycomputer-browser/src/sessions/sessions_tests/deep_page.rs create mode 100644 crates/tinycomputer-browser/src/sessions/sessions_tests/deep_tests.rs diff --git a/crates/tinycomputer-browser/README.md b/crates/tinycomputer-browser/README.md index fc1fb982..a31fe4fa 100644 --- a/crates/tinycomputer-browser/README.md +++ b/crates/tinycomputer-browser/README.md @@ -19,6 +19,7 @@ it as a surface next to the desktop, and `tinycomputer` serves it over TinyBus. | `src/linked/` | `AgentBrowser` (feature `agent-browser`): agent-browser's dispatcher linked in-process, one `DaemonState` per session, built without reading the host's `AGENT_BROWSER_*` environment | | `src/convert/` | contract requests to agent-browser commands, as pure functions | | `src/reply/` | agent-browser replies to typed results | +| `src/deep/` | controls a page keeps inside open shadow roots that the accessibility tree gives no ref: looked for after every snapshot (`deep.js`), listed with refs `d1`, `d2`, …, and pressed, typed into, or read where they show (`src/sessions/deep.rs`) | | `src/surface/` | `BrowserSurface`: one session as a `tinycomputer-core` `Surface`; `sight/`, which reads the rendered page the way a person looks at it (the default, `docs/technical/specs/browser-sight.md`), and `tree.rs`, which parses snapshot text into a `Screen` when sight cannot reach the page or `Perception::Tree` is chosen | | `src/error/` | `Error`: what a caller should do next, one published wire name per variant | | `src/outputs/` | held screenshots and PDFs: bounded count, size and lifetime, chunked reads | diff --git a/crates/tinycomputer-browser/src/convert/mod.rs b/crates/tinycomputer-browser/src/convert/mod.rs index 2f809e49..aaeb78e8 100644 --- a/crates/tinycomputer-browser/src/convert/mod.rs +++ b/crates/tinycomputer-browser/src/convert/mod.rs @@ -13,7 +13,7 @@ mod interaction; mod page; mod session; -pub(crate) use interaction::{action, press_at, selector}; +pub(crate) use interaction::{action, move_to, press_at, selector}; pub(crate) use page::{evaluate, navigate, read, screenshot, snapshot}; pub(crate) use session::{launch, viewport}; diff --git a/crates/tinycomputer-browser/src/deep/deep.js b/crates/tinycomputer-browser/src/deep/deep.js new file mode 100644 index 00000000..2adb7df1 --- /dev/null +++ b/crates/tinycomputer-browser/src/deep/deep.js @@ -0,0 +1,324 @@ +// Controls a page keeps inside open shadow roots, where the accessibility +// tree gives them no ref: a plain `span` or `div` a component wires to a +// click and draws with a pointer, or an editable region. See `mod.rs` for +// why, and for the shapes of the replies. +// +// Called as `(op, options) => reply`. `find` looks for the controls under +// `options.scope` (a CSS selector, or null for the whole page) and marks +// each with `data-tc-deep`. Every other `op` finds the one element carrying +// the mark `options.number` and acts on it or reads it, replying +// `{ gone: true }` when no element carries that mark, or more than one does. +((op, options) => { + const MARK = 'data-tc-deep'; + // The most elements one look walks: a page larger than this is read no + // further. + const BUDGET = 50000; + const squash = (text) => (text || '').replace(/[​-‍⁠]/g, '') + .replace(/\s+/g, ' ').trim(); + // Up the tree the page is drawn from: a slotted element to the slot that + // shows it, a shadow root's child to its host. + const up = (node) => node.assignedSlot || node.parentElement + || (node.parentNode && node.parentNode.host) || null; + // The elements drawn inside `node`: an open shadow root's children in + // place of a host's own, and what a slot shows in place of the slot. A + // closed root cannot be read, and its host's own children are walked. + const kids = (node) => { + if (node.shadowRoot) return node.shadowRoot.children; + if (node.localName === 'slot') { + const shown = node.assignedElements(); + return shown.length ? shown : node.children; + } + return node.children; + }; + const drawn = (element) => { + let area = false; + for (const rect of element.getClientRects()) { + if (rect.width > 0 && rect.height > 0) { + area = true; + break; + } + } + return area && (!element.checkVisibility + || element.checkVisibility({ opacityProperty: true, visibilityProperty: true })); + }; + // Hidden from people as the tree hides it: inside `hidden`, `inert`, or + // `aria-hidden` content. + const veiled = (element) => element.hidden || element.inert + || element.getAttribute('aria-hidden') === 'true'; + const hidden = (element) => { + for (let at = element; at; at = up(at)) if (veiled(at)) return true; + return false; + }; + // The text drawn inside `element`, through the shadow roots its own + // `innerText` never enters: up to `most` characters, a space around each + // block. + const SILENT = new Set(['style', 'script', 'template', 'noscript']); + const drawnText = (element, most) => { + let said = ''; + const stack = [element]; + while (stack.length && said.length < most) { + const node = stack.pop(); + if (node === ' ') { + said += ' '; + } else if (node.nodeType === Node.TEXT_NODE) { + said += node.data; + } else if (node.nodeType === Node.ELEMENT_NODE && !SILENT.has(node.localName)) { + const display = node === element ? 'inline' : getComputedStyle(node).display; + if (display === 'none') continue; + const block = !['inline', 'contents'].includes(display); + const assigned = node.localName === 'slot' ? node.assignedNodes() : []; + const children = node.shadowRoot ? node.shadowRoot.childNodes + : assigned.length ? assigned : node.childNodes; + if (block) stack.push(' '); + for (let index = children.length - 1; index >= 0; index -= 1) stack.push(children[index]); + if (block) stack.push(' '); + } + } + return said; + }; + // The words an element shows: its rendered text; for one whose words are + // all drawn by components inside it, the text drawn there; else its label + // or title. + const words = (element, most) => squash(element.innerText) + || squash(drawnText(element, most * 2)) + || squash(element.getAttribute('aria-label')) + || squash(element.getAttribute('title')); + + // Marks and lists the controls, in the order the page draws them. + const find = ({ scope, next, keep, most, longest }) => { + // What agent-browser's snapshot gives a ref of its own: a native + // control, a role that is one, and a role it refs once named. Such an + // element is in the tree already, and so is whatever a person presses + // inside a link or a button. + const INTERACTIVE = new Set(['button', 'link', 'textbox', 'checkbox', 'radio', 'combobox', + 'listbox', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'searchbox', 'slider', + 'spinbutton', 'switch', 'tab', 'treeitem']); + const NAMED = new Set(['heading', 'cell', 'gridcell', 'columnheader', 'rowheader', 'listitem', + 'article', 'region', 'main', 'navigation']); + const BY_CONTENT = new Set(['heading', 'cell', 'gridcell', 'columnheader', 'rowheader']); + const IMPLICIT = { + h1: 'heading', h2: 'heading', h3: 'heading', h4: 'heading', h5: 'heading', h6: 'heading', + td: 'cell', th: 'columnheader', li: 'listitem', article: 'article', main: 'main', nav: 'navigation', + }; + const PRESSED_WITH = 'a[href], button, [role=button], [role=link], [role=menuitem], ' + + '[role=option], [role=tab], [role=checkbox], [role=radio], [role=switch]'; + const role = (element) => (element.getAttribute('role') || '').trim().toLowerCase().split(/\s+/)[0] + || IMPLICIT[element.localName] || ''; + const inTree = (element) => { + const tag = element.localName; + if (['button', 'select', 'textarea', 'iframe'].includes(tag)) return true; + if (tag === 'a') return element.hasAttribute('href'); + if (tag === 'input') return element.type !== 'hidden'; + const named = role(element); + if (INTERACTIVE.has(named)) return true; + return NAMED.has(named) && Boolean(squash(element.getAttribute('aria-label')) + || element.hasAttribute('aria-labelledby') + || (BY_CONTENT.has(named) && squash(element.textContent))); + }; + const pointers = new Map(); + const pointer = (element) => { + if (!pointers.has(element)) pointers.set(element, getComputedStyle(element).cursor === 'pointer'); + return pointers.get(element); + }; + // How an element responds: `editable`, the outermost element of an + // editable region; `clickable`, a click handler of its own, or a + // pointer its parent does not show (a pointer inherited from a row is + // the row's); `focusable`, a tab stop. Null for none of these: a click + // listener added by script, with no pointer, cannot be seen from here. + const kindOf = (element) => { + const parent = up(element); + if (element.isContentEditable) return parent && parent.isContentEditable ? null : 'editable'; + if (element.hasAttribute('onclick') || typeof element.onclick === 'function') return 'clickable'; + if (pointer(element) && !(parent && pointer(parent))) return 'clickable'; + if (element.hasAttribute('tabindex') && element.tabIndex >= 0) return 'focusable'; + return null; + }; + let base = document.documentElement; + if (scope) { + try { + base = document.querySelector(scope); + } catch (error) { + base = null; + } + } + if (!base) return { found: [], next }; + const picks = []; + const picked = new Set(); + // Every element still marked: a mark not handed out again comes off. + const marked = []; + const stack = [[document.documentElement, false]]; + let walked = 0; + while (stack.length && walked < BUDGET) { + const [element, outer] = stack.pop(); + walked += 1; + if (element.hasAttribute(MARK)) marked.push(element); + const inside = outer || element === base; + const children = kids(element); + for (let index = children.length - 1; index >= 0; index -= 1) stack.push([children[index], inside]); + // Only what a shadow root holds: the tree reaches the rest itself. + if (!inside || !(element.getRootNode() instanceof ShadowRoot) || inTree(element)) continue; + const kind = kindOf(element); + if (!kind || veiled(element) || !drawn(element)) continue; + // The outermost of nested controls, though a field inside a pressable + // row is a field of its own; nothing inside a control the tree has, + // or hidden from people. + let kept = true; + for (let at = up(element); at && kept; at = up(at)) { + kept = !veiled(at) && !at.matches(PRESSED_WITH) + && !(picked.has(at) && (kind !== 'editable' || at.isContentEditable)); + } + if (!kept) continue; + picks.push({ element, kind }); + picked.add(element); + } + // A control still marked with a number the last snapshot handed out + // keeps it; any other gets the next number, never handed out before. + const held = new Set(keep); + const used = new Set(); + const shown = picks.slice(0, most); + const found = shown.map(({ element, kind }) => { + let number = Number(element.getAttribute(MARK)); + if (!held.has(number) || used.has(number)) { + number = next; + next += 1; + } + used.add(number); + // Written only when it changes: a page's own observers see no + // attribute set again to what it was. + if (element.getAttribute(MARK) !== String(number)) element.setAttribute(MARK, String(number)); + const text = words(element, longest); + return { n: number, kind, text: text.length > longest ? `${text.slice(0, longest - 1)}…` : text }; + }); + const listed = new Set(shown.map(({ element }) => element)); + for (const element of marked) { + if (!listed.has(element)) element.removeAttribute(MARK); + } + return { found, next }; + }; + + // One thing done with, or read from, the element marked `number`; with + // `field`, an aim also says whether the element takes text or holds + // something that does. + const act = (what, { number, field }) => { + const matches = []; + const roots = [document]; + while (roots.length) { + const root = roots.pop(); + for (const element of root.querySelectorAll('*')) { + if (element.shadowRoot) roots.push(element.shadowRoot); + if (element.getAttribute(MARK) === String(number)) matches.push(element); + } + } + if (matches.length !== 1) return { gone: true }; + const element = matches[0]; + const holds = (node) => { + for (let at = node; at; at = up(at)) if (at === element) return true; + return false; + }; + // What holds the keyboard focus, inside open shadow roots. + const focused = () => { + let at = document.activeElement; + while (at && at.shadowRoot && at.shadowRoot.activeElement) at = at.shadowRoot.activeElement; + return at === document.body || at === document.documentElement ? null : at; + }; + const where = (at) => ({ inside: Boolean(at) && holds(at), on: at ? at.localName : 'nothing' }); + const TEXT_TYPES = ['text', 'search', 'email', 'tel', 'url', 'number', 'password']; + const takesText = (at) => { + if (at.localName === 'textarea') return !at.readOnly && !at.disabled; + if (at.localName === 'input') { + return TEXT_TYPES.includes((at.getAttribute('type') || 'text').toLowerCase()) + && !at.readOnly && !at.disabled; + } + return at.isContentEditable === true; + }; + switch (what) { + case 'aim': { + // Into the window when any of it lies outside, then the middle of + // its largest box the window shows, and what a press there reaches. + if (!drawn(element) || hidden(element)) return { unseen: true }; + const whole = element.getBoundingClientRect(); + if (whole.top < 0 || whole.left < 0 || whole.bottom > innerHeight || whole.right > innerWidth) { + element.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' }); + } + let best = null; + for (const rect of element.getClientRects()) { + const left = Math.max(rect.left, 0); + const top = Math.max(rect.top, 0); + const right = Math.min(rect.right, innerWidth); + const bottom = Math.min(rect.bottom, innerHeight); + const area = Math.max(0, right - left) * Math.max(0, bottom - top); + if (area > 0 && (!best || area > best.area)) { + best = { area, x: (left + right) / 2, y: (top + bottom) / 2 }; + } + } + if (!best) return { unseen: true }; + let hit = document.elementFromPoint(best.x, best.y); + while (hit && hit.shadowRoot) { + const inner = hit.shadowRoot.elementFromPoint(best.x, best.y); + if (!inner || inner === hit) break; + hit = inner; + } + let takes = false; + if (field) { + const stack = [element]; + for (let walked = 0; stack.length && !takes && walked < BUDGET; walked += 1) { + const at = stack.pop(); + takes = takesText(at); + const children = kids(at); + for (let index = children.length - 1; index >= 0; index -= 1) stack.push(children[index]); + } + } + if (hit && holds(hit)) return { x: best.x, y: best.y, field: takes }; + let cover = hit ? hit.localName : 'nothing'; + if (hit && hit.id) cover += `#${hit.id}`; + else if (hit && typeof hit.className === 'string' && hit.className.trim()) { + cover += `.${hit.className.trim().split(/\s+/).slice(0, 2).join('.')}`; + } + return { x: best.x, y: best.y, field: takes, cover }; + } + case 'focus': + element.focus(); + return where(focused()); + case 'end': + case 'select': { + // The field the press focused, with its caret put after what it + // holds (`end`, to type) or all it holds selected (`select`, to + // replace it). + const at = focused(); + const place = where(at); + if (!place.inside) return place; + if (!takesText(at)) return { ...place, takes: false }; + if (at.localName === 'input' || at.localName === 'textarea') { + if (what === 'select') { + at.select(); + } else { + try { + at.setSelectionRange(at.value.length, at.value.length); + } catch (error) { + // A field with no caret of its own (`email`, `number`) keeps + // it where the press put it. + } + } + } else { + let region = at; + while (up(region) && up(region).isContentEditable) region = up(region); + const range = document.createRange(); + range.selectNodeContents(region); + if (what === 'end') range.collapse(false); + const selection = getSelection(); + selection.removeAllRanges(); + selection.addRange(range); + } + return { ...place, takes: true }; + } + case 'text': + return { text: element.innerText || squash(drawnText(element, 100000)) }; + case 'visible': + return { visible: drawn(element) && !hidden(element) }; + default: + return { gone: true }; + } + }; + + return op === 'find' ? find(options) : act(op, options); +}) diff --git a/crates/tinycomputer-browser/src/deep/deep_tests.rs b/crates/tinycomputer-browser/src/deep/deep_tests.rs new file mode 100644 index 00000000..ebd5d7c5 --- /dev/null +++ b/crates/tinycomputer-browser/src/deep/deep_tests.rs @@ -0,0 +1,316 @@ +//! Tests for deep controls: reading a `d` ref, what one takes, the refs a +//! session hands out, and placing the controls in a snapshot's tree. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +mod live_tests; +mod merge_tests; + +use serde_json::{Value, json}; +use tinycomputer_bus::browser::{Action, Target}; + +use tinycomputer_bus::browser::{ScrollDirection, WaitState}; + +use super::{Control, Kind, Minted, aimed_at, number, supports, unsupported}; +use crate::error::Error; + +#[test] +fn a_deep_ref_is_d_and_digits_with_or_without_its_at() { + assert_eq!(number(&Target::reference("d3")), Some(3)); + assert_eq!(number(&Target::reference("@d12")), Some(12)); + assert_eq!(number(&Target::parse("@d7")), Some(7)); + for other in ["e3", "d", "dx", "d3a", "seen:4", ""] { + assert_eq!(number(&Target::reference(other)), None, "{other}"); + } + assert_eq!( + number(&Target::selector("d3")), + None, + "a selector is no ref" + ); +} + +#[test] +fn an_action_is_aimed_at_the_deep_ref_it_targets() { + let deep = || Target::reference("d2"); + assert_eq!( + aimed_at(&Action::Click { + target: deep(), + new_tab: false + }), + Some(2) + ); + assert_eq!( + aimed_at(&Action::Press { + key: "Enter".to_owned(), + target: Some(deep()), + }), + Some(2) + ); + assert_eq!( + aimed_at(&Action::Type { + target: None, + text: "x".to_owned(), + delay_ms: None, + }), + None + ); + assert_eq!( + aimed_at(&Action::Click { + target: Target::reference("e2"), + new_tab: false + }), + None + ); + assert_eq!(aimed_at(&Action::Back), None); +} + +#[test] +fn every_action_is_aimed_by_its_target_and_named_when_refused() { + let deep = || Target::reference("d5"); + let aimed = [ + ( + Action::Click { + target: deep(), + new_tab: false, + }, + "click", + ), + (Action::DoubleClick { target: deep() }, "double_click"), + (Action::Hover { target: deep() }, "hover"), + (Action::Focus { target: deep() }, "focus"), + ( + Action::Fill { + target: deep(), + value: String::new(), + }, + "fill", + ), + ( + Action::Type { + target: Some(deep()), + text: String::new(), + delay_ms: None, + }, + "type", + ), + ( + Action::Press { + key: "Enter".to_owned(), + target: Some(deep()), + }, + "press", + ), + ( + Action::Select { + target: deep(), + values: Vec::new(), + }, + "select", + ), + ( + Action::Check { + target: deep(), + checked: true, + }, + "check", + ), + ( + Action::Scroll { + direction: ScrollDirection::Down, + pixels: None, + target: Some(deep()), + }, + "scroll", + ), + (Action::GetText { target: deep() }, "get_text"), + ( + Action::GetAttribute { + target: deep(), + attribute: "href".to_owned(), + }, + "get_attribute", + ), + (Action::IsVisible { target: deep() }, "is_visible"), + ( + Action::WaitFor { + target: Some(deep()), + text: None, + state: WaitState::Visible, + ms: None, + timeout_ms: None, + }, + "wait_for", + ), + ]; + for (action, verb) in &aimed { + assert_eq!(aimed_at(action), Some(5), "{verb}"); + assert!( + unsupported(action) + .to_string() + .contains(&format!("{verb} does not take")), + "{verb}" + ); + } + for (action, verb) in [ + (Action::Back, "back"), + (Action::Forward, "forward"), + (Action::Reload, "reload"), + ] { + assert_eq!(aimed_at(&action), None); + assert!( + unsupported(&action) + .to_string() + .contains(&format!("{verb} does not take")) + ); + } +} + +#[test] +fn a_deep_ref_takes_presses_text_and_reads_only() { + let deep = || Target::reference("d1"); + for action in [ + Action::Click { + target: deep(), + new_tab: false, + }, + Action::Hover { target: deep() }, + Action::Focus { target: deep() }, + Action::Fill { + target: deep(), + value: "x".to_owned(), + }, + Action::GetText { target: deep() }, + Action::IsVisible { target: deep() }, + ] { + assert!(supports(&action).is_ok(), "{action:?}"); + } + let refused = supports(&Action::Select { + target: deep(), + values: vec!["a".to_owned()], + }) + .unwrap_err(); + assert!( + matches!(&refused, Error::InvalidInput { message } if message.starts_with("select does not take a deep ref")), + "{refused}" + ); + assert!(matches!( + supports(&Action::Click { + target: deep(), + new_tab: true + }), + Err(Error::InvalidInput { .. }) + )); + assert!(matches!( + supports(&Action::Press { + key: " ".to_owned(), + target: Some(deep()), + }), + Err(Error::InvalidInput { .. }) + )); +} + +#[test] +fn a_control_reads_as_a_line_of_the_tree() { + let named = Control { + number: 4, + kind: Kind::Clickable, + text: "Say \"hi\"".to_owned(), + }; + assert_eq!(named.line(2), " - clickable \"Say \\\"hi\\\"\" [ref=d4]"); + let nameless = Control { + number: 9, + kind: Kind::Editable, + text: String::new(), + }; + assert_eq!(nameless.line(0), "- editable [ref=d9]"); + let tab_stop = Control { + number: 10, + kind: Kind::Focusable, + text: "Ride options".to_owned(), + }; + assert_eq!(tab_stop.line(1), " - focusable \"Ride options\" [ref=d10]"); +} + +/// A look's reply listing `found`, minting from `next`. +fn look(found: &Value, next: u64) -> Value { + json!({"found": found, "next": next}) +} + +/// The options a look's script is called with. +fn options(script: &str) -> Value { + let call = script + .rfind(")(\"find\", ") + .expect("a look calls the script with find"); + let options = &script[call + ")(\"find\", ".len()..script.len() - 1]; + serde_json::from_str(options).expect("the options are JSON") +} + +#[test] +fn a_session_hands_out_what_its_latest_look_found() { + let mut minted = Minted::default(); + assert!(!minted.holds(1)); + assert_eq!( + options(&minted.look(None)), + json!({"keep": [], "longest": 100, "most": 40, "next": 1, "scope": null}) + ); + let controls = minted.take(Some(&look( + &json!([ + {"n": 1, "kind": "clickable", "text": " LOG\n IN "}, + {"n": 2, "kind": "editable", "text": ""}, + ]), + 3, + ))); + assert_eq!( + controls, + vec![ + Control { + number: 1, + kind: Kind::Clickable, + text: "LOG IN".to_owned() + }, + Control { + number: 2, + kind: Kind::Editable, + text: String::new() + }, + ] + ); + assert!(minted.holds(1) && minted.holds(2)); + assert_eq!( + options(&minted.look(Some("#app"))), + json!({"keep": [1, 2], "longest": 100, "most": 40, "next": 3, "scope": "#app"}) + ); + // A look that does not reply hands out none: the refs before are stale. + assert!(minted.take(None).is_empty()); + assert!(!minted.holds(1)); + assert!(minted.take(Some(&json!(42))).is_empty()); + // Numbers never go back, even when the page reports a lower next. + minted.take(Some(&look(&json!([]), 1))); + assert_eq!(options(&minted.look(None))["next"], 3); +} + +#[test] +fn a_look_reply_is_read_defensively() { + let mut minted = Minted::default(); + let long = "w".repeat(150); + let controls = minted.take(Some(&look( + &json!([ + {"n": 0, "kind": "clickable", "text": "zero"}, + {"n": 5, "kind": "button", "text": "unknown kind"}, + {"n": 6, "kind": "focusable", "text": long}, + {"n": 6, "kind": "clickable", "text": "twice"}, + {"kind": "clickable", "text": "no number"}, + ]), + 2, + ))); + assert_eq!(controls.len(), 1); + assert_eq!(controls[0].number, 6); + assert_eq!(controls[0].kind, Kind::Focusable); + assert_eq!(controls[0].text.chars().count(), 100); + // Past every number found, whatever the page said. + assert_eq!(options(&minted.look(None))["next"], 7); + let many = (1..=50) + .map(|n| json!({"n": n, "kind": "clickable", "text": format!("row {n}")})) + .collect::>(); + assert_eq!(minted.take(Some(&look(&json!(many), 51))).len(), 40); +} diff --git a/crates/tinycomputer-browser/src/deep/deep_tests/live_tests.rs b/crates/tinycomputer-browser/src/deep/deep_tests/live_tests.rs new file mode 100644 index 00000000..e4491b29 --- /dev/null +++ b/crates/tinycomputer-browser/src/deep/deep_tests/live_tests.rs @@ -0,0 +1,380 @@ +//! Live tests of deep controls in a real browser, gated on the +//! `agent-browser` feature and `TINYCOMPUTER_LIVE_BROWSER=1`, since CI has +//! no browser to launch: a page built from web components, whose controls +//! agent-browser leaves without refs, snapshotted and acted on through +//! [`crate::Browser`]. The browser launched is the one +//! `TINYCOMPUTER_BROWSER_EXECUTABLE` names, else the one the engine finds. + +#[cfg(feature = "agent-browser")] +use serde_json::{Value, json}; +#[cfg(feature = "agent-browser")] +use tinycomputer_bus::browser::{ + Action, SessionId, SessionOptions, Snapshot, SnapshotRequest, Target, +}; + +#[cfg(feature = "agent-browser")] +use crate::Browser; +#[cfg(feature = "agent-browser")] +use crate::error::Error; + +/// A page built from web components: a pointer `span` to log in one shadow +/// root down, rows two down (one with no words, a chevron), an editable +/// region, a native button and field (refs of agent-browser's own), words +/// after it all, and a closed shadow root that cannot be read. Every press +/// and hover is recorded in `window.calls`, with whether it was trusted. +#[cfg(feature = "agent-browser")] +const COMPONENTS: &str = r#" + + + +

Please log in

+ + +"#; + +/// A page with no shadow root: controls, a pointer row agent-browser refs +/// itself, and words. +#[cfg(feature = "agent-browser")] +const LIGHT: &str = r#" + +

Rides

+ +Help +
Mini 3 min
+

Please log in

+"#; + +/// A real browser with `html` written into a blank tab: `None` unless +/// `TINYCOMPUTER_LIVE_BROWSER=1`. +#[cfg(feature = "agent-browser")] +async fn live_page(html: &str) -> Option<(Browser, SessionId)> { + use std::sync::Arc; + + if std::env::var("TINYCOMPUTER_LIVE_BROWSER").as_deref() != Ok("1") { + return None; + } + let browser = Browser::new(Arc::new(crate::AgentBrowser)); + let options = SessionOptions { + executable: std::env::var("TINYCOMPUTER_BROWSER_EXECUTABLE").ok(), + ..SessionOptions::default() + }; + let info = browser + .open_session(options) + .await + .expect("a browser launches when live runs are asked for"); + let write = format!( + "document.open(); document.write({}); document.close(); true", + Value::String(html.to_owned()) + ); + browser + .command(&info.id, json!({"action": "evaluate", "script": write})) + .await + .expect("the fixture is written"); + Some((browser, info.id)) +} + +/// A snapshot as an agent's host reads a page: compact, not only the +/// interactive lines, cut at 24,000 characters. +#[cfg(feature = "agent-browser")] +fn compact() -> SnapshotRequest { + SnapshotRequest { + compact: true, + max_chars: 24_000, + ..SnapshotRequest::default() + } +} + +/// The ref `snapshot` lists for the element named `name` with `role`. +#[cfg(feature = "agent-browser")] +fn ref_of(snapshot: &Snapshot, role: &str, name: &str) -> String { + snapshot + .refs + .iter() + .find(|element| element.role == role && element.name == name) + .unwrap_or_else(|| panic!("no {role} {name:?} in {:#?}", snapshot.refs)) + .id + .clone() +} + +/// What the page's script evaluates `expression` to. +#[cfg(feature = "agent-browser")] +async fn page_value(browser: &Browser, id: &SessionId, expression: &str) -> Value { + browser + .command(id, json!({"action": "evaluate", "script": expression})) + .await + .expect("the expression runs")["result"] + .clone() +} + +/// `action` on `target`, as `Browser::perform` does it. +#[cfg(feature = "agent-browser")] +async fn on( + browser: &Browser, + id: &SessionId, + target: &str, + action: fn(Target) -> Action, +) -> crate::error::Result { + browser + .perform(id, action(Target::reference(target))) + .await + .map(|outcome| outcome.value) +} + +#[cfg(feature = "agent-browser")] +fn click(target: Target) -> Action { + Action::Click { + target, + new_tab: false, + } +} + +#[cfg(feature = "agent-browser")] +const EDITOR_TEXT: &str = + "document.querySelector('x-app').shadowRoot.querySelector('.editor').innerText"; + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_controls_inside_shadow_roots_get_refs_of_their_own() { + let Some((browser, id)) = live_page(COMPONENTS).await else { + return; + }; + let shot = browser.snapshot(&id, compact()).await.unwrap(); + eprintln!("{}", shot.tree); + let login = ref_of(&shot, "clickable", "LOG IN"); + let mini = ref_of(&shot, "clickable", "Mini 3 min"); + let chevron = ref_of(&shot, "clickable", ""); + let editor = ref_of(&shot, "editable", ""); + for deep in [&login, &mini, &chevron, &editor] { + assert!(deep.starts_with('d'), "{deep}"); + assert!( + shot.tree.contains(&format!("[ref={deep}]")), + "{deep} is in the tree" + ); + } + assert!( + shot.tree + .contains(&format!("- clickable \"LOG IN\" [ref={login}]")) + ); + // agent-browser's own refs stand, and are not listed twice; the closed + // root's pointer cannot be reached, and costs nothing. + assert!(ref_of(&shot, "button", "Native").starts_with('e')); + assert!(ref_of(&shot, "textbox", "Where to").starts_with('e')); + assert!(!shot.refs.iter().any(|element| element.id.starts_with('d') + && (element.name == "Native" || element.name == "Hidden away"))); + // Read in full, the tree shows their words where the page draws them, + // and that is where they go; read again, they keep their numbers. + let full = browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + eprintln!("{}", full.tree); + let group = full + .tree + .find("- group \"Controls inside web components\"") + .map_or("", |at| &full.tree[at..]); + assert!( + !group.contains("LOG IN") && !group.contains("Mini 3 min"), + "{}", + full.tree + ); + assert_eq!(ref_of(&full, "clickable", "LOG IN"), login); + assert_eq!(ref_of(&full, "clickable", "Mini 3 min"), mini); + browser.close_session(&id).await.unwrap(); +} + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_deep_press_is_a_persons_and_a_redrawn_row_is_stale() { + let Some((browser, id)) = live_page(COMPONENTS).await else { + return; + }; + let shot = browser.snapshot(&id, compact()).await.unwrap(); + let login = ref_of(&shot, "clickable", "LOG IN"); + let mini = ref_of(&shot, "clickable", "Mini 3 min"); + assert_eq!(on(&browser, &id, &mini, click).await.unwrap(), Value::Null); + assert_eq!( + page_value(&browser, &id, "window.calls").await, + json!([{"what": "mini", "trusted": true}]) + ); + on(&browser, &id, &login, |target| Action::Hover { target }) + .await + .unwrap(); + assert_eq!( + page_value(&browser, &id, "window.calls.at(-1)").await, + json!({"what": "hover login", "trusted": true}) + ); + let text = on(&browser, &id, &login, |target| Action::GetText { target }); + assert_eq!(text.await.unwrap(), json!("LOG IN")); + let shows = on(&browser, &id, &login, |target| Action::IsVisible { target }); + assert_eq!(shows.await.unwrap(), json!(true)); + // What lies over the element is named, and nothing is pressed. + page_value( + &browser, + &id, + "const mask = document.createElement('div'); mask.className = 'mask dim'; \ + mask.style.cssText = 'position: fixed; inset: 0'; document.body.append(mask); true", + ) + .await; + let covered = on(&browser, &id, &login, click).await.unwrap_err(); + assert!( + matches!(&covered, Error::NotActionable { reason } if reason.contains("is covered by ")), + "{covered}" + ); + page_value( + &browser, + &id, + "document.querySelector('.mask').remove(); true", + ) + .await; + // Re-drawn, the row is a new element: its old ref is stale, and presses + // nothing. + page_value( + &browser, + &id, + "document.querySelector('x-app').shadowRoot.querySelector('x-list').render(); true", + ) + .await; + let stale = on(&browser, &id, &mini, click).await.unwrap_err(); + assert!( + matches!(&stale, Error::StaleRef { reference } if *reference == mini), + "{stale}" + ); + assert_eq!( + page_value( + &browser, + &id, + "window.calls.filter((call) => call.what !== 'hover login')" + ) + .await, + json!([{"what": "mini", "trusted": true}]) + ); + // A fresh snapshot keeps the number of what still lives, and names the + // re-drawn row anew. + let again = browser.snapshot(&id, compact()).await.unwrap(); + assert_eq!(ref_of(&again, "clickable", "LOG IN"), login); + assert_ne!(ref_of(&again, "clickable", "Mini 3 min"), mini); + browser.close_session(&id).await.unwrap(); +} + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_text_goes_into_the_editable_region_and_nowhere_else() { + let Some((browser, id)) = live_page(COMPONENTS).await else { + return; + }; + let shot = browser.snapshot(&id, compact()).await.unwrap(); + let login = ref_of(&shot, "clickable", "LOG IN"); + let editor = ref_of(&shot, "editable", ""); + let typed = on(&browser, &id, &editor, |target| Action::Type { + target: Some(target), + text: "Kora".to_owned(), + delay_ms: None, + }); + typed.await.unwrap(); + assert_eq!(page_value(&browser, &id, EDITOR_TEXT).await, json!("Kora")); + let filled = on(&browser, &id, &editor, |target| Action::Fill { + target, + value: "Airport".to_owned(), + }); + filled.await.unwrap(); + assert_eq!( + page_value(&browser, &id, EDITOR_TEXT).await, + json!("Airport") + ); + // Text never goes to what takes none, which is not even pressed, and + // the focus is never left where it was: the log-in span takes none. + let typed = on(&browser, &id, &login, |target| Action::Type { + target: Some(target), + text: "x".to_owned(), + delay_ms: None, + }); + assert!(matches!( + typed.await, + Err(Error::NotActionable { reason }) if reason.contains("takes no typed text") + )); + let focused = on(&browser, &id, &login, |target| Action::Focus { target }); + assert!(matches!( + focused.await, + Err(Error::FocusMissed { focused }) if focused == "nothing" + )); + assert_eq!(page_value(&browser, &id, "window.calls").await, json!([])); + assert_eq!( + page_value(&browser, &id, EDITOR_TEXT).await, + json!("Airport") + ); + browser.close_session(&id).await.unwrap(); +} + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_page_without_shadow_roots_is_snapshotted_as_the_engine_gives_it() { + let Some((browser, id)) = live_page(LIGHT).await else { + return; + }; + for request in [SnapshotRequest::default(), compact()] { + let shot = browser.snapshot(&id, request.clone()).await.unwrap(); + let engine = browser + .command( + &id, + json!({"action": "snapshot", "compact": request.compact, "interactive": false, "urls": false}), + ) + .await + .unwrap(); + assert_eq!(shot.tree, engine["snapshot"].as_str().unwrap()); + assert_eq!(shot.refs.len(), engine["refs"].as_object().unwrap().len()); + assert!(!shot.refs.iter().any(|element| element.id.starts_with('d'))); + } + assert_eq!( + page_value( + &browser, + &id, + "document.querySelectorAll('[data-tc-deep]').length" + ) + .await, + json!(0) + ); + browser.close_session(&id).await.unwrap(); +} diff --git a/crates/tinycomputer-browser/src/deep/deep_tests/merge_tests.rs b/crates/tinycomputer-browser/src/deep/deep_tests/merge_tests.rs new file mode 100644 index 00000000..b2015b41 --- /dev/null +++ b/crates/tinycomputer-browser/src/deep/deep_tests/merge_tests.rs @@ -0,0 +1,187 @@ +//! Placing deep controls in a snapshot's tree: on the line that shows their +//! words, or in a group after the tree, which a cut never drops. + +use tinycomputer_bus::browser::{ElementRef, Snapshot}; + +use crate::deep::{Control, Kind, place}; + +fn clickable(number: u64, text: &str) -> Control { + Control { + number, + kind: Kind::Clickable, + text: text.to_owned(), + } +} + +/// A snapshot of `tree` as agent-browser gave it, with one ref of its own. +fn snapshot(tree: &str) -> Snapshot { + Snapshot { + url: "https://cabs.test/".to_owned(), + title: "Cabs".to_owned(), + sequence: 1, + tree: tree.to_owned(), + refs: vec![ElementRef { + id: "e1".to_owned(), + role: "button".to_owned(), + name: "Book".to_owned(), + }], + truncated: false, + } +} + +/// `tree` with `controls` placed in it, uncut. +fn placed(tree: &str, controls: &[Control]) -> String { + let mut shot = snapshot(tree); + place(&mut shot, tree, controls, 200_000); + shot.tree +} + +#[test] +fn no_controls_leave_the_snapshot_as_it_was() { + let tree = "- banner\n - StaticText \"LOG IN\"\n- button \"Book\" [ref=e1]"; + let mut shot = snapshot(tree); + place(&mut shot, tree, &[], 10); + assert_eq!(shot, snapshot(tree)); +} + +#[test] +fn a_text_line_with_the_controls_words_becomes_the_control() { + let tree = "- banner\n - StaticText \"LOG IN\"\n- button \"Book\" [ref=e1]\n- StaticText \"Please log in\""; + let mut shot = snapshot(tree); + place(&mut shot, tree, &[clickable(1, "LOG IN")], 200_000); + assert_eq!( + shot.tree, + "- banner\n - clickable \"LOG IN\" [ref=d1]\n- button \"Book\" [ref=e1]\n- StaticText \"Please log in\"" + ); + assert_eq!( + shot.refs.last(), + Some(&ElementRef { + id: "d1".to_owned(), + role: "clickable".to_owned(), + name: "LOG IN".to_owned(), + }) + ); + assert_eq!(shot.refs[0].id, "e1", "agent-browser's refs come first"); + assert!(!shot.truncated); +} + +#[test] +fn a_container_of_the_words_pieces_becomes_the_control_and_keeps_them() { + // The tree drops the space between two pieces of a row's text. + let tree = + "- list\n - generic\n - StaticText \"Mini\"\n - StaticText \"3 min\"\n - image"; + assert_eq!( + placed(tree, &[clickable(2, "Mini 3 min")]), + "- list\n - clickable \"Mini 3 min\" [ref=d2]\n - StaticText \"Mini\"\n - StaticText \"3 min\"\n - image" + ); + assert_eq!( + placed(tree, &[clickable(2, "Mini3 min")]), + placed(tree, &[clickable(2, "Mini 3 min")]).replace("Mini 3 min", "Mini3 min"), + "words compare without their spaces" + ); +} + +#[test] +fn a_line_with_a_name_or_a_ref_of_its_own_is_never_taken() { + let tree = "- group \"Mini\"\n- generic\n - StaticText \"Mini\"\n - button \"Book\" [ref=e1]"; + // The named group and the container holding a button stay; the text + // line itself is the control's. + assert_eq!( + placed(tree, &[clickable(1, "Mini")]), + "- group \"Mini\"\n- generic\n - clickable \"Mini\" [ref=d1]\n - button \"Book\" [ref=e1]" + ); + let named = "- group \"Mini\""; + assert_eq!( + placed(named, &[clickable(1, "Mini")]), + "- group \"Mini\"\n- group \"Controls inside web components\"\n - clickable \"Mini\" [ref=d1]" + ); +} + +#[test] +fn a_container_showing_other_words_is_passed_over() { + let tree = + "- generic\n - StaticText \"Prime\"\n - StaticText \"Mini\"\n- StaticText \"Mini\""; + assert_eq!( + placed(tree, &[clickable(1, "Mini")]), + "- generic\n - StaticText \"Prime\"\n - clickable \"Mini\" [ref=d1]\n- StaticText \"Mini\"" + ); +} + +#[test] +fn words_shown_twice_go_to_the_line_in_the_controls_place() { + let tree = "- StaticText \"Mini\"\n- StaticText \"Prime\"\n- StaticText \"Mini\""; + // The first control takes the first "Prime" there is; the one after it + // can only be a "Mini" after that. + assert_eq!( + placed(tree, &[clickable(1, "Prime"), clickable(2, "Mini")]), + "- StaticText \"Mini\"\n- clickable \"Prime\" [ref=d1]\n- clickable \"Mini\" [ref=d2]" + ); +} + +#[test] +fn controls_no_line_shows_follow_the_tree_in_a_group() { + // A compact tree keeps no text after its last ref. + let tree = "- banner\n - StaticText \"LOG IN\"\n- button \"Book\" [ref=e1]"; + let controls = [ + clickable(1, "LOG IN"), + clickable(2, "Mini 3 min"), + clickable(3, ""), + Control { + number: 4, + kind: Kind::Editable, + text: String::new(), + }, + ]; + let mut shot = snapshot(tree); + place(&mut shot, tree, &controls, 200_000); + assert_eq!( + shot.tree, + "- banner\n - clickable \"LOG IN\" [ref=d1]\n- button \"Book\" [ref=e1]\n\ + - group \"Controls inside web components\"\n - clickable \"Mini 3 min\" [ref=d2]\n \ + - clickable [ref=d3]\n - editable [ref=d4]" + ); + let ids: Vec<_> = shot + .refs + .iter() + .map(|element| element.id.as_str()) + .collect(); + assert_eq!(ids, ["e1", "d1", "d2", "d3", "d4"]); +} + +#[test] +fn an_empty_tree_is_replaced_by_the_group() { + for empty in ["(empty page)", "(no interactive elements)", ""] { + assert_eq!( + placed(empty, &[clickable(1, "Mini")]), + "- group \"Controls inside web components\"\n - clickable \"Mini\" [ref=d1]", + "{empty:?}" + ); + } +} + +#[test] +fn a_cut_keeps_the_group_whole_and_cuts_the_tree_shorter() { + let tree = format!( + "- button \"Book\" [ref=e1]\n{}", + "- StaticText \"filler\"\n".repeat(40) + ); + let tree = tree.trim_end(); + let controls = [clickable(1, "Mini 3 min")]; + let group = "- group \"Controls inside web components\"\n - clickable \"Mini 3 min\" [ref=d1]"; + let mut shot = snapshot(tree); + place(&mut shot, tree, &controls, 200); + assert!(shot.truncated); + assert_eq!(shot.tree.chars().count(), 200); + assert!(shot.tree.ends_with(&format!("\n{group}")), "{}", shot.tree); + assert!(shot.tree.starts_with("- button \"Book\" [ref=e1]")); + // Too small for the group: cut as agent-browser's tree is. + let mut tiny = snapshot(tree); + place(&mut tiny, tree, &controls, 20); + assert!(tiny.truncated); + assert_eq!(tiny.tree, "- button \"Book\" [ref"); + // Room for all of it: whole, and not cut. + let mut roomy = snapshot(tree); + place(&mut roomy, tree, &controls, 10_000); + assert!(!roomy.truncated); + assert!(roomy.tree.ends_with(group)); +} diff --git a/crates/tinycomputer-browser/src/deep/merge.rs b/crates/tinycomputer-browser/src/deep/merge.rs new file mode 100644 index 00000000..e2dbad49 --- /dev/null +++ b/crates/tinycomputer-browser/src/deep/merge.rs @@ -0,0 +1,199 @@ +//! Placing deep controls in a snapshot's tree. +//! +//! agent-browser shows the words of a shadow root's control as plain text: +//! `- StaticText "LOG IN"`, or a row's pieces under an unnamed container +//! (`- generic`). A control is placed on the innermost line whose whole text +//! is the control's words: a text line, or a container holding only text +//! and other such containers (no ref, no name of its own). Words compare +//! without their spaces, since the tree splits a row's text into pieces and +//! drops the spaces between them. Lines are searched in the order the page +//! draws them, each control after the last one placed, so a word the page +//! shows twice goes to the one in its place. The line becomes the control +//! at the same depth, `- clickable "LOG IN" [ref=d1]`, and keeps what it +//! holds below it. +//! +//! A control that shows no words, or whose words no line holds (a compact +//! snapshot drops the text after its last ref), is listed after the tree in +//! a group of its own. That group is the one place those controls show, so +//! when the snapshot is cut to its size it is kept whole and the tree before +//! it is cut shorter. + +use tinycomputer_bus::browser::{ElementRef, Snapshot}; + +use super::Control; + +/// The line the controls no line holds are listed under. +const GROUP: &str = "- group \"Controls inside web components\""; + +/// What agent-browser puts in place of a tree with nothing in it. +const PLACEHOLDERS: &[&str] = &["(empty page)", "(no interactive elements)"]; + +/// One line of the tree, as placing reads it. +struct Row { + depth: usize, + shape: Shape, +} + +enum Shape { + /// A text line, its words without their spaces. + Text(String), + /// A line with no ref, no name, and no value: it shows only what it holds. + Container, + /// Anything else: a control, or a line with a name of its own. + Fixed, +} + +fn row(line: &str) -> Row { + let body = line.trim_start_matches(' '); + let depth = (line.len() - body.len()) / 2; + let text = body + .strip_prefix("- StaticText ") + .and_then(|quoted| serde_json::from_str::(quoted).ok()); + let shape = match text { + Some(text) => Shape::Text(spaceless(&text)), + None if body.starts_with("- ") + && !body.contains("ref=") + && !body.contains(" \"") + && !body.contains(':') => + { + Shape::Container + } + None => Shape::Fixed, + }; + Row { depth, shape } +} + +fn spaceless(text: &str) -> String { + text.split_whitespace().collect() +} + +/// Whether the whole text of the line at `at` is `words`. +fn shows(rows: &[Row], at: usize, words: &str) -> bool { + match &rows[at].shape { + Shape::Text(text) => text == words, + Shape::Fixed => false, + Shape::Container => { + let mut held = String::new(); + for row in rows[at + 1..] + .iter() + .take_while(|row| row.depth > rows[at].depth) + { + match &row.shape { + Shape::Fixed => return false, + Shape::Container => {} + Shape::Text(text) => { + held.push_str(text); + if !words.starts_with(&held) { + return false; + } + } + } + } + !held.is_empty() && held == words + } + } +} + +/// The index just past what the line at `at` holds. +fn end_of(rows: &[Row], at: usize) -> usize { + at + 1 + + rows[at + 1..] + .iter() + .take_while(|row| row.depth > rows[at].depth) + .count() +} + +/// Where `words` go, searching from `from`: the innermost line showing them, +/// and the index past the outermost one's lines. +fn spot(rows: &[Row], from: usize, words: &str) -> Option<(usize, usize)> { + let outer = (from..rows.len()).find(|&at| shows(rows, at, words))?; + let end = end_of(rows, outer); + let inner = (outer..end) + .rev() + .find(|&at| shows(rows, at, words)) + .unwrap_or(outer); + Some((inner, end)) +} + +/// `tree` with `controls` placed in it, as the tree itself and the group of +/// those no line holds; either may be empty. +fn merge(tree: &str, controls: &[Control]) -> (String, String) { + let lines: Vec<&str> = tree.lines().collect(); + let rows: Vec = lines.iter().map(|line| row(line)).collect(); + let mut placed: Vec> = vec![None; rows.len()]; + let mut from = 0; + let mut left = Vec::new(); + for control in controls { + let words = spaceless(&control.text); + let found = if words.is_empty() { + None + } else { + spot(&rows, from, &words) + }; + match found { + Some((at, end)) => { + placed[at] = Some(control); + from = end; + } + None => left.push(control), + } + } + let body = if PLACEHOLDERS.contains(&tree.trim()) { + String::new() + } else { + lines + .iter() + .zip(&rows) + .zip(&placed) + .map(|((line, row), placed)| { + placed.map_or_else(|| (*line).to_owned(), |control| control.line(row.depth)) + }) + .collect::>() + .join("\n") + }; + let group = if left.is_empty() { + String::new() + } else { + std::iter::once(GROUP.to_owned()) + .chain(left.iter().map(|control| control.line(1))) + .collect::>() + .join("\n") + }; + (body, group) +} + +/// Puts `controls` in `snapshot`, whose tree agent-browser gave as `tree` +/// before it was cut to `max_chars`, and lists their refs after +/// agent-browser's. With no controls, nothing changes. +pub(crate) fn place(snapshot: &mut Snapshot, tree: &str, controls: &[Control], max_chars: usize) { + if controls.is_empty() { + return; + } + let (body, group) = merge(tree, controls); + let whole = match (body.is_empty(), group.is_empty()) { + (true, _) => group.clone(), + (_, true) => body.clone(), + _ => format!("{body}\n{group}"), + }; + let group_size = group.chars().count(); + snapshot.truncated = whole.chars().count() > max_chars; + snapshot.tree = if snapshot.truncated { + if body.is_empty() || group.is_empty() || group_size + 1 >= max_chars { + whole.chars().take(max_chars).collect() + } else { + let mut cut: String = body.chars().take(max_chars - group_size - 1).collect(); + cut.push('\n'); + cut.push_str(&group); + cut + } + } else { + whole + }; + snapshot + .refs + .extend(controls.iter().map(|control| ElementRef { + id: control.reference(), + role: control.kind.as_str().to_owned(), + name: control.text.clone(), + })); +} diff --git a/crates/tinycomputer-browser/src/deep/mod.rs b/crates/tinycomputer-browser/src/deep/mod.rs new file mode 100644 index 00000000..e67034a2 --- /dev/null +++ b/crates/tinycomputer-browser/src/deep/mod.rs @@ -0,0 +1,326 @@ +//! Controls inside open shadow roots that the accessibility tree gives no +//! ref: found after every snapshot, handed out as `d` refs, and acted on by +//! position. +//! +//! Pages built from web components keep their controls inside shadow roots. +//! Chrome's accessibility tree reaches into them, and agent-browser refs +//! what has a control's role there as anywhere: a native button or field +//! inside a component gets its `eN`. But a component's own control is often +//! a plain `span` or `div` drawn with a pointer and wired to a click by the +//! component's script: no role, no tab stop, no `onclick` attribute. +//! agent-browser finds such elements by their pointer in the page's light +//! DOM only, so inside a component they come out as bare text, or, in a +//! compact snapshot, not at all. Live, a booking page built this way showed +//! its log-in link as plain text and dropped its list of results entirely, +//! and an agent had nothing to press. +//! +//! So every snapshot is followed by one look through the page's open shadow +//! roots (`deep.js`, through the engine's `evaluate`) for the elements a +//! person can press or type into that the tree holds no ref for: a pointer +//! of their own (not one inherited from a row around them), a click handler +//! set as an attribute or property, a tab stop, or an editable region. Only +//! what is drawn and not hidden from people counts, and of nested ones only +//! the outermost, though a field inside a pressable row stays a field. At +//! most [`MOST`] are kept, in the order the page draws them, each marked +//! with a `data-tc-deep` attribute holding its number. A closed shadow root +//! cannot be read and is passed over. +//! +//! Each is handed out as a ref `d1`, `d2`, …, which agent-browser's `eN` +//! cannot collide with, and placed in the snapshot's tree (`merge.rs`). A +//! number is handed out once per session, and an element keeps its number +//! from one snapshot to the next while it lives and is still listed, so a +//! ref never comes to name another element. A page with nothing to list +//! comes out exactly as agent-browser gave it. +//! +//! An action on a `d` ref is the session's own (`sessions/deep.rs`): the +//! element is found by its mark, and pressed where it shows with the +//! pointer's own events. + +mod merge; + +pub(crate) use merge::place; + +use serde_json::{Value, json}; +use tinycomputer_bus::browser::{Action, Target}; + +use crate::error::Error; + +/// The script, called as `(op, options)`. +const DEEP_JS: &str = include_str!("deep.js"); + +/// The most deep controls one snapshot lists. +const MOST: usize = 40; + +/// The longest name a deep control is given, in characters. +const LONGEST: usize = 100; + +/// What a deep control's ref starts with, before its number. +const PREFIX: char = 'd'; + +/// How a deep control responds, which is the role its line is given. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Kind { + /// It takes a press: a pointer of its own, or a click handler. + Clickable, + /// The outermost element of an editable region. + Editable, + /// A tab stop and nothing more. + Focusable, +} + +impl Kind { + fn parse(kind: &str) -> Option { + match kind { + "clickable" => Some(Self::Clickable), + "editable" => Some(Self::Editable), + "focusable" => Some(Self::Focusable), + _ => None, + } + } + + /// The role a snapshot names it by. + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Clickable => "clickable", + Self::Editable => "editable", + Self::Focusable => "focusable", + } + } +} + +/// One control a look found. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Control { + /// Its number: its ref is `d` and this number. + pub(crate) number: u64, + /// How it responds. + pub(crate) kind: Kind, + /// The words it shows, whitespace collapsed, at most [`LONGEST`] + /// characters; empty when it shows none. + pub(crate) text: String, +} + +impl Control { + /// Its ref, as a snapshot lists it. + pub(crate) fn reference(&self) -> String { + reference(self.number) + } + + /// Its line in a snapshot's tree, `depth` levels in: + /// `- clickable "LOG IN" [ref=d1]`. + pub(crate) fn line(&self, depth: usize) -> String { + let mut line = format!("{}- {}", " ".repeat(depth), self.kind.as_str()); + if !self.text.is_empty() { + line.push(' '); + line.push_str(&Value::String(self.text.clone()).to_string()); + } + line.push_str(" [ref="); + line.push_str(&self.reference()); + line.push(']'); + line + } +} + +/// The ref of deep control `number`. +pub(crate) fn reference(number: u64) -> String { + format!("{PREFIX}{number}") +} + +/// The deep control `target` names, by its number: a ref `d` and digits, +/// with or without its `@`. `None` for any other target. +pub(crate) fn number(target: &Target) -> Option { + let Target::Ref { value } = target else { + return None; + }; + let digits = value.trim().trim_start_matches('@').strip_prefix(PREFIX)?; + if digits.is_empty() || !digits.chars().all(|digit| digit.is_ascii_digit()) { + return None; + } + digits.parse().ok() +} + +/// The deep control `action` is aimed at, if its target is one. +pub(crate) fn aimed_at(action: &Action) -> Option { + let target = match action { + Action::Click { target, .. } + | Action::DoubleClick { target } + | Action::Hover { target } + | Action::Focus { target } + | Action::Fill { target, .. } + | Action::Select { target, .. } + | Action::Check { target, .. } + | Action::GetText { target } + | Action::GetAttribute { target, .. } + | Action::IsVisible { target } => target, + Action::Type { target, .. } + | Action::Press { target, .. } + | Action::Scroll { target, .. } + | Action::WaitFor { target, .. } => target.as_ref()?, + Action::Back | Action::Forward | Action::Reload => return None, + }; + number(target) +} + +/// Refuses an action a deep control cannot take, before anything reaches +/// the page: what is pressed, typed into, focused, or read takes one; +/// whatever needs agent-browser to resolve the element does not. +/// +/// # Errors +/// +/// [`Error::InvalidInput`] naming what a deep ref takes, or for a click in a +/// new tab or a press with no key. +pub(crate) fn supports(action: &Action) -> crate::error::Result<()> { + match action { + Action::Click { new_tab: true, .. } => Err(Error::invalid_input( + "new_tab opens a link's page in a new tab, and a deep ref (dN) names no link; click it without new_tab", + )), + Action::Press { key, .. } if key.trim().is_empty() => { + Err(Error::invalid_input("press needs a key")) + } + Action::Click { .. } + | Action::Hover { .. } + | Action::Focus { .. } + | Action::Fill { .. } + | Action::Type { .. } + | Action::Press { .. } + | Action::GetText { .. } + | Action::IsVisible { .. } => Ok(()), + other => Err(unsupported(other)), + } +} + +/// The refusal of `action` on a deep ref. +pub(crate) fn unsupported(action: &Action) -> Error { + let verb = match action { + Action::Click { .. } => "click", + Action::DoubleClick { .. } => "double_click", + Action::Hover { .. } => "hover", + Action::Focus { .. } => "focus", + Action::Fill { .. } => "fill", + Action::Type { .. } => "type", + Action::Press { .. } => "press", + Action::Select { .. } => "select", + Action::Check { .. } => "check", + Action::Scroll { .. } => "scroll", + Action::GetText { .. } => "get_text", + Action::GetAttribute { .. } => "get_attribute", + Action::IsVisible { .. } => "is_visible", + Action::WaitFor { .. } => "wait_for", + Action::Back => "back", + Action::Forward => "forward", + Action::Reload => "reload", + }; + Error::invalid_input(format!( + "{verb} does not take a deep ref (dN); a deep ref takes click, hover, focus, type, fill, press, get_text, and is_visible" + )) +} + +/// The deep refs a session's latest snapshot handed out, and the next +/// number to mint, which no ref of the session has had. +#[derive(Debug)] +pub(crate) struct Minted { + live: Vec, + next: u64, +} + +impl Default for Minted { + fn default() -> Self { + Self { + live: Vec::new(), + next: 1, + } + } +} + +impl Minted { + /// Whether the latest snapshot handed out deep control `number`. + pub(crate) fn holds(&self, number: u64) -> bool { + self.live.contains(&number) + } + + /// The look for deep controls under `scope`, a CSS selector, or all + /// the page when `None`: an element still marked with a ref this table + /// holds keeps it, and any other is given a number from `next`. + pub(crate) fn look(&self, scope: Option<&str>) -> String { + call( + "find", + &json!({ + "scope": scope, + "next": self.next, + "keep": self.live, + "most": MOST, + "longest": LONGEST, + }), + ) + } + + /// Takes what a look replied, `None` when it did not reply, and returns + /// the controls it found, which are now the ones this table holds. A + /// reply that is not a look's holds none. + pub(crate) fn take(&mut self, result: Option<&Value>) -> Vec { + let Some((controls, next)) = result.and_then(|result| read(result, self.next)) else { + self.live.clear(); + return Vec::new(); + }; + self.live = controls.iter().map(|control| control.number).collect(); + self.next = next; + controls + } +} + +/// `op` on the element marked `number`: `aim`, `focus`, `end`, `select`, +/// `text`, or `visible` (`deep.js`). With `field`, an aim also says whether +/// the element takes text, or holds something that does. +pub(crate) fn act(op: &str, number: u64, field: bool) -> String { + call(op, &json!({ "number": number, "field": field })) +} + +fn call(op: &str, options: &Value) -> String { + format!( + "({})({}, {options})", + DEEP_JS.trim().trim_end_matches(';'), + Value::String(op.to_owned()) + ) +} + +/// The controls in a look's `result`, and the next number to mint, never +/// below `next` nor any number found; `None` when `result` is not a look's +/// reply. At most [`MOST`] controls, each number once, each name at most +/// [`LONGEST`] characters. +fn read(result: &Value, next: u64) -> Option<(Vec, u64)> { + let found = result.get("found")?.as_array()?; + let mut controls: Vec = Vec::new(); + for item in found.iter().take(MOST) { + let number = item.get("n").and_then(Value::as_u64).unwrap_or_default(); + let kind = item + .get("kind") + .and_then(Value::as_str) + .and_then(Kind::parse); + let Some(kind) = kind.filter(|_| number > 0) else { + continue; + }; + if controls.iter().any(|control| control.number == number) { + continue; + } + let text = item + .get("text") + .and_then(Value::as_str) + .unwrap_or_default() + .split_whitespace() + .collect::>() + .join(" ") + .chars() + .take(LONGEST) + .collect(); + controls.push(Control { number, kind, text }); + } + let after = controls + .iter() + .map(|control| control.number.saturating_add(1)) + .chain(result.get("next").and_then(Value::as_u64)) + .fold(next, u64::max); + Some((controls, after)) +} + +#[cfg(test)] +mod deep_tests; diff --git a/crates/tinycomputer-browser/src/lib.rs b/crates/tinycomputer-browser/src/lib.rs index 0f8f58e1..f42cc376 100644 --- a/crates/tinycomputer-browser/src/lib.rs +++ b/crates/tinycomputer-browser/src/lib.rs @@ -30,6 +30,7 @@ //! is the same type as `tinycomputer_bus::browser::Action`. mod convert; +mod deep; mod engine; mod error; #[cfg(test)] diff --git a/crates/tinycomputer-browser/src/sessions/deep.rs b/crates/tinycomputer-browser/src/sessions/deep.rs new file mode 100644 index 00000000..1c73cdb5 --- /dev/null +++ b/crates/tinycomputer-browser/src/sessions/deep.rs @@ -0,0 +1,270 @@ +//! A session's deep controls (`crate::deep`): the look that follows each +//! snapshot, and actions on the `d` refs it hands out. +//! +//! agent-browser cannot reach a deep control, so the session acts on it by +//! position, as a person's hand does. The element is found by its mark +//! through the open shadow roots; gone, its ref is stale, so the caller takes +//! a fresh snapshot. It is brought into the window, and the middle of what +//! the window shows of it is checked to be what a press there reaches: the +//! element, or something inside it, through open shadow roots. Then the +//! pointer moves there and presses with the browser's own input events +//! (`convert::press_at`, as the surface presses a point), never a script's +//! `click()`, which a page can tell from a person's and which sends no +//! pointer down. +//! +//! Keys go only where the action points, as `focus.rs` keeps them: before a +//! focus, a key, or text, the focus comes off whatever held it, the element +//! takes it (pressed, for text, as a person clicks into a field), and the +//! action is refused unless the focus is on the element or inside it. Text +//! goes only into what takes it: a field or an editable region. + +use std::time::Duration; + +use serde_json::{Value, json}; +use tinycomputer_bus::browser::Action; + +use super::Session; +use super::focus::missed; +use crate::convert; +use crate::deep::{self, Control}; +use crate::error::{Error, Result}; + +/// The longest the look after a snapshot may take: past it, the snapshot is +/// handed over as agent-browser gave it. +const LOOK: Duration = Duration::from_secs(3); + +impl Session { + /// Looks for the deep controls under `scope`, a CSS selector, or all the + /// page when `None`, after a snapshot, and hands out their refs. Never + /// fails: a look that cannot run, or takes longer than [`LOOK`], hands + /// out none, and the refs of the snapshot before are stale. + pub(super) async fn find_deep(&mut self, scope: Option<&str>) -> Vec { + let script = self.deep.look(scope); + let looking = self.run(json!({"action": "evaluate", "script": script})); + let reply = tokio::time::timeout(LOOK, looking) + .await + .ok() + .and_then(Result::ok); + self.deep + .take(reply.as_ref().and_then(|data| data.get("result"))) + } + + /// Performs `action`, one [`deep::supports`] admits, on deep control + /// `number`, and returns what it read (`null` for an action that reads + /// nothing). + /// + /// # Errors + /// + /// [`Error::StaleRef`] when the latest snapshot did not hand the ref out + /// or its element is gone, [`Error::NotActionable`] when it shows nowhere + /// in the window, something else lies over its middle, or it takes no + /// text meant for it, [`Error::FocusMissed`] when the focus does not land + /// on it, and whatever the engine reports. + pub(super) async fn act_deep(&mut self, number: u64, action: &Action) -> Result { + if !self.deep.holds(number) { + return Err(stale(number)); + } + match action { + Action::Click { .. } => { + let (x, y) = self.aim(number, false).await?; + self.press_at(x, y).await?; + } + Action::Hover { .. } => { + let (x, y) = self.aim(number, false).await?; + self.run(convert::move_to(x, y)).await?; + } + Action::Focus { .. } => self.focus_deep(number).await?, + Action::Press { key, .. } => { + self.focus_deep(number).await?; + self.run(json!({"action": "press", "key": key})).await?; + } + Action::Type { text, delay_ms, .. } => { + self.press_into(number, "end").await?; + self.type_keys(text, *delay_ms).await?; + } + Action::Fill { value, .. } => { + self.press_into(number, "select").await?; + // The text replaces what the field held, selected; no text + // deletes it. + let command = if value.is_empty() { + json!({"action": "press", "key": "Backspace"}) + } else { + json!({"action": "inserttext", "text": value}) + }; + self.run(command).await?; + } + Action::GetText { .. } => { + let read = self.find_mark(number, "text").await?; + return Ok(read.get("text").cloned().unwrap_or(Value::Null)); + } + Action::IsVisible { .. } => { + let read = self.find_mark(number, "visible").await?; + return Ok(read.get("visible").cloned().unwrap_or(Value::Bool(false))); + } + other => return Err(deep::unsupported(other)), + } + Ok(Value::Null) + } + + /// Runs `op` (`deep.js`) on the element marked `number`: its reply, or + /// `None` when no one element carries the mark. + async fn probe(&mut self, number: u64, op: &str) -> Result> { + self.probe_for(number, op, false).await + } + + /// [`Session::probe`], asking an aim whether the element takes text + /// when `field`. + async fn probe_for(&mut self, number: u64, op: &str, field: bool) -> Result> { + let data = self + .run(json!({"action": "evaluate", "script": deep::act(op, number, field)})) + .await?; + let reply = data.get("result").cloned().unwrap_or(Value::Null); + Ok((reply.is_object() && reply.get("gone") != Some(&Value::Bool(true))).then_some(reply)) + } + + /// [`Session::probe`], with an element gone as a stale ref: for a probe + /// before anything was sent to the element. + async fn find_mark(&mut self, number: u64, op: &str) -> Result { + self.probe(number, op).await?.ok_or_else(|| stale(number)) + } + + /// Where a press on deep control `number` goes: the middle of what the + /// window shows of it, brought into the window first, once that point is + /// checked to reach the element. For `text`, only an element that takes + /// text or holds something that does is pressed at all. + async fn aim(&mut self, number: u64, text: bool) -> Result<(f64, f64)> { + let aimed = self + .probe_for(number, "aim", text) + .await? + .ok_or_else(|| stale(number))?; + let reference = deep::reference(number); + if aimed.get("unseen") == Some(&Value::Bool(true)) { + return Err(Error::not_actionable(format!( + "@{reference} shows nowhere in the window: it draws no box, or the page hides it" + ))); + } + let (Some(x), Some(y)) = ( + aimed.get("x").and_then(Value::as_f64), + aimed.get("y").and_then(Value::as_f64), + ) else { + return Err(Error::not_actionable(format!( + "@{reference} could not be measured" + ))); + }; + if let Some(cover) = aimed.get("cover").and_then(Value::as_str) { + // Named by tag, id, and class, as agent-browser names a cover: + // nothing more of the page's own text goes into the error. + let cover: String = cover + .chars() + .filter(|character| { + character.is_ascii_alphanumeric() || "-_#.".contains(*character) + }) + .take(80) + .collect(); + let cover = if cover.is_empty() { + "an element" + } else { + &cover + }; + return Err(Error::not_actionable(format!( + "@{reference} is covered by <{cover}> at its middle, so a press would land on that instead; dismiss or close what covers it first" + ))); + } + if text && aimed.get("field") != Some(&Value::Bool(true)) { + return Err(no_text( + &reference, + "there is no field or editable region in it", + )); + } + Ok((x, y)) + } + + /// Presses the window at (`x`, `y`) as a person's pointer does. + async fn press_at(&mut self, x: f64, y: f64) -> Result<()> { + for command in convert::press_at(x, y) { + self.run(command).await?; + } + Ok(()) + } + + /// Gives deep control `number` the focus, and only it. + async fn focus_deep(&mut self, number: u64) -> Result<()> { + self.blur().await?; + let found = self.find_mark(number, "focus").await?; + on_it(&found) + } + + /// Presses deep control `number` to put the focus in it, as a person + /// clicks into a field, then readies the field it focused for text: + /// with its caret after what it holds (`end`) or all it holds selected + /// (`select`). + async fn press_into(&mut self, number: u64, how: &str) -> Result<()> { + // Refused before the focus moves: what held it (a box whose + // suggestions are open, say) keeps it. + self.aim(number, true).await?; + self.blur().await?; + // Measured again: letting go of the focus can close what it held open. + let (x, y) = self.aim(number, true).await?; + self.press_at(x, y).await?; + // The press reached the page: an element gone now went with it, which + // is no ref to retry blindly. + let reference = deep::reference(number); + let Some(found) = self.probe(number, how).await? else { + return Err(Error::not_actionable(format!( + "@{reference} went away when it was pressed, so no text was sent" + ))); + }; + on_it(&found)?; + if found.get("takes") == Some(&Value::Bool(true)) { + Ok(()) + } else { + Err(no_text( + &reference, + "pressing it focused no field or editable region in it", + )) + } + } + + /// Types `text` into the focused element a character at a time, as + /// agent-browser types into a field, `delay_ms` apart. + async fn type_keys(&mut self, text: &str, delay_ms: Option) -> Result<()> { + let Some(delay) = delay_ms.filter(|delay| *delay > 0) else { + return self + .run(json!({"action": "keyboard", "subaction": "type", "text": text})) + .await + .map(drop); + }; + for (index, character) in text.chars().enumerate() { + if index > 0 { + tokio::time::sleep(Duration::from_millis(delay)).await; + } + self.run( + json!({"action": "keyboard", "subaction": "type", "text": character.to_string()}), + ) + .await?; + } + Ok(()) + } +} + +/// Whether the focus, as a probe found it, is on the element or inside it. +fn on_it(found: &Value) -> Result<()> { + if found.get("inside") == Some(&Value::Bool(true)) { + Ok(()) + } else { + Err(missed(found)) + } +} + +/// The refusal of text for `reference`, which takes none, and why. +fn no_text(reference: &str, why: &str) -> Error { + Error::not_actionable(format!( + "@{reference} takes no typed text: {why}, so no text was sent" + )) +} + +fn stale(number: u64) -> Error { + Error::StaleRef { + reference: deep::reference(number), + } +} diff --git a/crates/tinycomputer-browser/src/sessions/focus.rs b/crates/tinycomputer-browser/src/sessions/focus.rs index b6d25a3a..20f3cc80 100644 --- a/crates/tinycomputer-browser/src/sessions/focus.rs +++ b/crates/tinycomputer-browser/src/sessions/focus.rs @@ -152,7 +152,7 @@ impl Session { /// The refusal for a focus that landed elsewhere, naming where by tag only: /// anything more would be page text in an error a model reads. -fn missed(found: &Value) -> Error { +pub(super) fn missed(found: &Value) -> Error { let on = found["on"] .as_str() .filter(|tag| { diff --git a/crates/tinycomputer-browser/src/sessions/mod.rs b/crates/tinycomputer-browser/src/sessions/mod.rs index 669e7fcc..cae18cad 100644 --- a/crates/tinycomputer-browser/src/sessions/mod.rs +++ b/crates/tinycomputer-browser/src/sessions/mod.rs @@ -20,6 +20,7 @@ use crate::outputs::OutputStore; use crate::reply; mod artifacts; +mod deep; mod focus; mod page; @@ -76,6 +77,8 @@ struct Session { /// The pages `options.allowed_origins` admits. origins: Origins, sequence: u64, + /// The deep refs the latest snapshot handed out (`crate::deep`). + deep: crate::deep::Minted, downloads: Vec, dir: PathBuf, } @@ -269,6 +272,7 @@ impl Browser { origins: Origins::new(&options.allowed_origins), options, sequence: 0, + deep: crate::deep::Minted::default(), downloads: Vec::new(), dir: self.scratch.join(id.as_str()), }; diff --git a/crates/tinycomputer-browser/src/sessions/page.rs b/crates/tinycomputer-browser/src/sessions/page.rs index ad49c062..9150dc03 100644 --- a/crates/tinycomputer-browser/src/sessions/page.rs +++ b/crates/tinycomputer-browser/src/sessions/page.rs @@ -7,8 +7,9 @@ use tinycomputer_bus::browser::{ ReadFormat, ReadRequest, ScreenshotRequest, SessionId, Snapshot, SnapshotRequest, Target, }; -use super::Browser; +use super::{Browser, Session}; use crate::convert; +use crate::deep; use crate::error::{Error, Result}; use crate::outputs::within_cap; use crate::reply; @@ -79,6 +80,13 @@ impl Browser { /// Captures the page's accessibility tree with element refs. /// + /// The controls a page keeps inside open shadow roots that the tree + /// gives no ref (a component's own clickable `span` or `div`, an + /// editable region) are looked for after agent-browser replies, and + /// listed with refs of their own, `d1`, `d2`, …: on the line that shows + /// their words, or after the tree (`crate::deep`). A page with none + /// comes out as agent-browser gave it. + /// /// # Errors /// /// [`Error::NoSuchSession`], and whatever the snapshot reports. @@ -87,16 +95,24 @@ impl Browser { let mut session = session.lock().await; session.check_if_readable(false).await?; let data = session.run(convert::snapshot(&request)).await?; + let controls = session.find_deep(request.selector.as_deref()).await; let page = session.page().await?; let page = session.admit(page).await?; session.sequence += 1; - Ok(reply::snapshot( + let mut snapshot = reply::snapshot( &data, page.url, page.title, session.sequence, request.max_chars, - )) + ); + deep::place( + &mut snapshot, + &reply::text(&data, "snapshot"), + &controls, + request.max_chars, + ); + Ok(snapshot) } /// Performs one interaction and reports the page it left. @@ -107,17 +123,34 @@ impl Browser { /// locator's fill, which only the engine resolves, has the focus taken /// off whatever held it first, so its text lands in the field or nowhere. /// + /// An action on a deep ref (`d1`, from [`Browser::snapshot`]) is the + /// session's own, as agent-browser cannot reach the element: it is found + /// by its mark, and pressed, hovered, typed into, or read where it + /// shows, with the pointer's and keyboard's own input events + /// (`sessions/deep.rs`). `Click`, `Hover`, `Focus`, `Type`, `Fill`, + /// `Press`, `GetText`, and `IsVisible` take one. + /// /// # Errors /// /// [`Error::NoSuchSession`], [`Error::InvalidInput`] for an action the - /// engine cannot express, [`Error::StaleRef`] for a ref from an earlier - /// snapshot, [`Error::FocusMissed`] when the focus does not land on the + /// engine cannot express or a deep ref cannot take, [`Error::StaleRef`] + /// for a ref from an earlier snapshot or a deep ref whose element is + /// gone, [`Error::NotActionable`] for a deep ref's element covered or + /// not shown, [`Error::FocusMissed`] when the focus does not land on the /// element keys were meant for, [`Error::BlockedByPolicy`] when the page /// it would act on, or the page it leads to, is outside the allowed /// origins, and whatever else the action reports. pub async fn perform(&self, id: &SessionId, action: Action) -> Result { let session = self.session(id)?; let mut session = session.lock().await; + if let Some(number) = deep::aimed_at(&action) { + deep::supports(&action)?; + // The page may have moved on its own since the last call: an + // action is never sent to a refused one. + session.check_if_readable(false).await?; + let value = session.act_deep(number, &action).await?; + return session.outcome(value, None).await; + } let command = convert::action(&action, session.options.default_timeout_ms)?; let keys = keys_go_to(&action)?; // The page may have moved on its own since the last call (a redirect, @@ -146,15 +179,7 @@ impl Browser { } => Some(format!("{:?} {:?}", value.by, value.value)), _ => None, }; - let page = session.page().await?; - let page = session.admit(page).await?; - session.info.url.clone_from(&page.url); - session.info.title.clone_from(&page.title); - Ok(ActionOutcome { - value, - page, - matched, - }) + session.outcome(value, matched).await } /// Extracts the active page as text, markdown, or HTML. @@ -308,6 +333,22 @@ impl Browser { } } +impl Session { + /// What an action read, with the page it left the session on: refused, + /// and left, when the allowed origins refuse it. + async fn outcome(&mut self, value: Value, matched: Option) -> Result { + let page = self.page().await?; + let page = self.admit(page).await?; + self.info.url.clone_from(&page.url); + self.info.title.clone_from(&page.title); + Ok(ActionOutcome { + value, + page, + matched, + }) + } +} + /// How an action's keys, or focus, are kept on its target. enum Keys { /// The target's selector: focus it and check the focus landed there. diff --git a/crates/tinycomputer-browser/src/sessions/sessions_tests.rs b/crates/tinycomputer-browser/src/sessions/sessions_tests.rs index a5992f53..e9275a0f 100644 --- a/crates/tinycomputer-browser/src/sessions/sessions_tests.rs +++ b/crates/tinycomputer-browser/src/sessions/sessions_tests.rs @@ -15,6 +15,9 @@ use super::Browser; use crate::fake::Fake; mod artifacts_tests; +mod deep_keys_tests; +mod deep_page; +mod deep_tests; mod focus_tests; mod lifecycle_tests; mod origins_tests; diff --git a/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_keys_tests.rs b/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_keys_tests.rs new file mode 100644 index 00000000..6f2d1f6d --- /dev/null +++ b/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_keys_tests.rs @@ -0,0 +1,249 @@ +//! Tests for keys and reads on deep controls: text typed and filled where a +//! press put the focus, keys refused anywhere else, and the marked element +//! focused and read, over a scripted engine. + +use serde_json::json; +use tinycomputer_bus::browser::{Action, SnapshotRequest, Target}; + +use super::deep_page::{mouse, ops, page, shown}; +use super::open; +use crate::error::Error; + +#[tokio::test] +async fn text_is_typed_where_the_press_put_the_focus() { + let fake = page(shown); + let (browser, id) = open(&fake, "deep-type").await; + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + browser + .perform( + &id, + Action::Type { + target: Some(Target::reference("d2")), + text: "Kora".to_owned(), + delay_ms: None, + }, + ) + .await + .unwrap(); + assert_eq!(ops(&fake), ["find", "aim", "aim", "end"]); + assert_eq!(mouse(&fake).len(), 3, "pressed into the field"); + assert_eq!( + fake.last("keyboard"), + json!({"action": "keyboard", "subaction": "type", "text": "Kora"}) + ); + // The focus came off whatever held it before the press. + let actions = fake.actions(); + let blurred = fake + .sent() + .iter() + .position(|command| { + command["action"] == "evaluate" + && command["script"] + .as_str() + .is_some_and(|script| script.contains("element.blur()")) + }) + .expect("the focus was taken off"); + let pressed = actions.iter().position(|action| action == "mouse").unwrap(); + assert!(blurred < pressed); +} + +#[tokio::test] +async fn typing_with_a_delay_sends_a_character_at_a_time() { + let fake = page(shown); + let (browser, id) = open(&fake, "deep-type-slow").await; + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + browser + .perform( + &id, + Action::Type { + target: Some(Target::reference("d2")), + text: "ab".to_owned(), + delay_ms: Some(1), + }, + ) + .await + .unwrap(); + let typed: Vec<_> = fake + .sent() + .iter() + .filter(|command| command["action"] == "keyboard") + .map(|command| command["text"].as_str().unwrap().to_owned()) + .collect(); + assert_eq!(typed, ["a", "b"]); +} + +#[tokio::test] +async fn a_fill_replaces_what_the_field_held() { + let fake = page(shown); + let (browser, id) = open(&fake, "deep-fill").await; + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + let fill = |value: &str| Action::Fill { + target: Target::reference("d2"), + value: value.to_owned(), + }; + browser.perform(&id, fill("Airport")).await.unwrap(); + assert_eq!(ops(&fake), ["find", "aim", "aim", "select"]); + assert_eq!( + fake.last("inserttext"), + json!({"action": "inserttext", "text": "Airport"}) + ); + browser.perform(&id, fill("")).await.unwrap(); + assert_eq!( + fake.last("press"), + json!({"action": "press", "key": "Backspace"}) + ); +} + +#[tokio::test] +async fn keys_never_go_anywhere_the_action_does_not_point() { + // The press left the focus on a field outside the element. + let elsewhere = page(|op| match op { + "aim" => json!({"x": 40.0, "y": 12.0, "field": true}), + "end" | "select" | "focus" => json!({"inside": false, "on": "input"}), + _ => json!({"gone": true}), + }); + let (browser, id) = open(&elsewhere, "deep-focus-missed").await; + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + let typed = Action::Type { + target: Some(Target::reference("d1")), + text: "secret".to_owned(), + delay_ms: None, + }; + assert!(matches!( + browser.perform(&id, typed).await, + Err(Error::FocusMissed { focused }) if focused == "input" + )); + let pressed = Action::Press { + key: "Enter".to_owned(), + target: Some(Target::reference("d1")), + }; + assert!(matches!( + browser.perform(&id, pressed).await, + Err(Error::FocusMissed { .. }) + )); + assert!( + !elsewhere + .actions() + .iter() + .any(|action| action == "keyboard" || action == "press"), + "no key was sent" + ); + // Nothing in the element takes text: it is not even pressed. + let no_field = page(|op| match op { + "aim" => json!({"x": 40.0, "y": 12.0, "field": false}), + _ => json!({"gone": true}), + }); + let (browser, id) = open(&no_field, "deep-no-field").await; + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + let filled = || Action::Fill { + target: Target::reference("d1"), + value: "x".to_owned(), + }; + assert!(matches!( + browser.perform(&id, filled()).await, + Err(Error::NotActionable { reason }) if reason == "@d1 takes no typed text: there is no field or editable region in it, so no text was sent" + )); + assert!(mouse(&no_field).is_empty(), "nothing was pressed"); + let blurred = no_field.sent().iter().any(|command| { + command["script"] + .as_str() + .is_some_and(|script| script.contains("element.blur()")) + }); + assert!(!blurred, "the focus stayed where it was"); + // It holds one, but the press focused something in it that takes none. + let no_text = page(|op| match op { + "aim" => json!({"x": 40.0, "y": 12.0, "field": true}), + "select" => json!({"inside": true, "on": "div", "takes": false}), + _ => json!({"gone": true}), + }); + let (browser, id) = open(&no_text, "deep-no-text").await; + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + assert!(matches!( + browser.perform(&id, filled()).await, + Err(Error::NotActionable { reason }) if reason.contains("pressing it focused no field") + )); + assert!( + !no_text + .actions() + .iter() + .any(|action| action == "inserttext") + ); + // Gone once pressed: no stale ref, since the press reached the page. + let vanished = page(|op| match op { + "aim" => json!({"x": 40.0, "y": 12.0, "field": true}), + _ => json!({"gone": true}), + }); + let (browser, id) = open(&vanished, "deep-vanished").await; + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + let typed = Action::Type { + target: Some(Target::reference("d1")), + text: "x".to_owned(), + delay_ms: None, + }; + assert!(matches!( + browser.perform(&id, typed).await, + Err(Error::NotActionable { reason }) if reason.contains("went away") + )); +} + +#[tokio::test] +async fn focus_press_and_reads_reach_the_marked_element() { + let fake = page(shown); + let (browser, id) = open(&fake, "deep-reads").await; + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + let deep = || Target::reference("d1"); + browser + .perform(&id, Action::Focus { target: deep() }) + .await + .unwrap(); + browser + .perform( + &id, + Action::Press { + key: "Enter".to_owned(), + target: Some(deep()), + }, + ) + .await + .unwrap(); + assert_eq!( + fake.last("press"), + json!({"action": "press", "key": "Enter"}) + ); + let text = browser + .perform(&id, Action::GetText { target: deep() }) + .await + .unwrap(); + assert_eq!(text.value, json!("LOG IN")); + let visible = browser + .perform(&id, Action::IsVisible { target: deep() }) + .await + .unwrap(); + assert_eq!(visible.value, json!(true)); + assert_eq!(ops(&fake), ["find", "focus", "focus", "text", "visible"]); + assert!(mouse(&fake).is_empty(), "a focus is no press"); +} diff --git a/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_page.rs b/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_page.rs new file mode 100644 index 00000000..86a06a9e --- /dev/null +++ b/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_page.rs @@ -0,0 +1,83 @@ +//! A scripted page with controls inside shadow roots, for the deep tests: +//! its snapshot, the look that finds its deep controls, and the answers its +//! marked elements give. + +use serde_json::{Value, json}; +use tinycomputer_bus::browser::{Action, Target}; + +use crate::fake::{Fake, ok}; + +/// The `deep.js` operation `command` runs, if it runs one. +pub(super) fn op(command: &Value) -> Option { + let script = command["script"].as_str()?; + if command["action"] != "evaluate" || !script.contains("data-tc-deep") { + return None; + } + let call = script.rfind(")(\"")? + 3; + let end = call + script[call..].find('"')?; + Some(script[call..end].to_owned()) +} + +/// The ops run, in order. +pub(super) fn ops(fake: &Fake) -> Vec { + fake.sent().iter().filter_map(op).collect() +} + +/// The mouse events sent, as `(event, x, y)`. +pub(super) fn mouse(fake: &Fake) -> Vec<(String, f64, f64)> { + fake.sent() + .iter() + .filter(|command| command["action"] == "mouse") + .map(|command| { + ( + command["eventType"].as_str().unwrap().to_owned(), + command["x"].as_f64().unwrap(), + command["y"].as_f64().unwrap(), + ) + }) + .collect() +} + +pub(super) const TREE: &str = "- banner\n - StaticText \"LOG IN\"\n- button \"Book\" [ref=e1]"; + +/// A page whose snapshot is [`TREE`] and whose look finds "LOG IN" and a +/// nameless row as `d1` and `d2`; every other op answers `answer(op)`. +pub(super) fn page(answer: impl Fn(&str) -> Value + Send + Sync + 'static) -> Fake { + Fake::scripted(move |command| { + if command["action"] == "snapshot" { + return Some(ok(&json!({ + "snapshot": TREE, + "refs": {"e1": {"role": "button", "name": "Book"}}, + }))); + } + let op = op(command)?; + let result = if op == "find" { + json!({"found": [ + {"n": 1, "kind": "clickable", "text": "LOG IN"}, + {"n": 2, "kind": "clickable", "text": ""}, + ], "next": 3}) + } else { + answer(&op) + }; + Some(ok(&json!({ "result": result }))) + }) +} + +/// The answers of a page whose deep elements show at (40, 12), take the +/// focus, and take text. +pub(super) fn shown(op: &str) -> Value { + match op { + "aim" => json!({"x": 40.0, "y": 12.0, "field": true}), + "focus" | "end" | "select" => json!({"inside": true, "on": "div", "takes": true}), + "text" => json!({"text": "LOG IN"}), + "visible" => json!({"visible": true}), + _ => json!({"gone": true}), + } +} + +pub(super) fn click(reference: &str) -> Action { + Action::Click { + target: Target::reference(reference), + new_tab: false, + } +} diff --git a/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_tests.rs b/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_tests.rs new file mode 100644 index 00000000..5dd18b86 --- /dev/null +++ b/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_tests.rs @@ -0,0 +1,228 @@ +//! Tests for deep controls on a session: the look after each snapshot, and +//! pressing, hovering, and refusing the `d` refs it hands out, over a +//! scripted engine. + +use serde_json::{Value, json}; +use tinycomputer_bus::browser::{Action, SnapshotRequest, Target}; + +use super::deep_page::{TREE, click, mouse, op, ops, page, shown}; +use super::open; +use crate::error::Error; +use crate::fake::{Fake, failure, ok}; + +#[tokio::test] +async fn a_snapshot_lists_the_deep_controls_with_refs_of_their_own() { + let fake = page(shown); + let (browser, id) = open(&fake, "deep-snapshot").await; + let shot = browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + assert_eq!( + shot.tree, + "- banner\n - clickable \"LOG IN\" [ref=d1]\n- button \"Book\" [ref=e1]\n\ + - group \"Controls inside web components\"\n - clickable [ref=d2]" + ); + let ids: Vec<_> = shot + .refs + .iter() + .map(|element| element.id.as_str()) + .collect(); + assert_eq!(ids, ["e1", "d1", "d2"]); + // The next look keeps the numbers handed out, and mints past them. + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + let looks: Vec<_> = fake + .sent() + .into_iter() + .filter(|command| op(command).as_deref() == Some("find")) + .collect(); + let options = |look: &Value| { + let script = look["script"].as_str().unwrap(); + let call = script.rfind(")(\"find\", ").unwrap() + ")(\"find\", ".len(); + serde_json::from_str::(&script[call..script.len() - 1]).unwrap() + }; + assert_eq!(options(&looks[0])["keep"], json!([])); + assert_eq!(options(&looks[1])["keep"], json!([1, 2])); + assert_eq!(options(&looks[1])["next"], json!(3)); +} + +#[tokio::test] +async fn a_page_with_no_deep_controls_is_snapshotted_as_the_engine_gave_it() { + // The default engine answers every script with 42: no look's reply. + let fake = Fake::scripted(|command| { + (command["action"] == "snapshot").then(|| { + ok(&json!({ + "snapshot": TREE, + "refs": {"e1": {"role": "button", "name": "Book"}}, + })) + }) + }); + let (browser, id) = open(&fake, "deep-none").await; + let shot = browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + assert_eq!(shot.tree, TREE); + assert_eq!(shot.refs.len(), 1); + // A look that fails leaves it as it was too. + let failing = Fake::scripted(|command| { + op(command).map(|_| failure("Evaluation error: the page is gone")) + }); + let (browser, id) = open(&failing, "deep-failing").await; + let shot = browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + assert_eq!(shot.tree, "- button \"Search\" [ref=e1]"); + assert_eq!(ops(&failing), ["find"]); +} + +#[tokio::test] +async fn a_click_presses_the_middle_of_the_element_with_the_pointer() { + let fake = page(shown); + let (browser, id) = open(&fake, "deep-click").await; + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + let outcome = browser.perform(&id, click("@d1")).await.unwrap(); + assert!(outcome.value.is_null()); + assert_eq!(outcome.page.url, "https://flights.test/"); + assert_eq!(ops(&fake), ["find", "aim"]); + assert_eq!( + mouse(&fake), + [ + ("mouseMoved".to_owned(), 40.0, 12.0), + ("mousePressed".to_owned(), 40.0, 12.0), + ("mouseReleased".to_owned(), 40.0, 12.0), + ] + ); + // Never through agent-browser, which cannot resolve it. + assert!(!fake.actions().iter().any(|action| action == "click")); +} + +#[tokio::test] +async fn a_hover_only_moves_the_pointer_there() { + let fake = page(shown); + let (browser, id) = open(&fake, "deep-hover").await; + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + browser + .perform( + &id, + Action::Hover { + target: Target::reference("d2"), + }, + ) + .await + .unwrap(); + assert_eq!(mouse(&fake), [("mouseMoved".to_owned(), 40.0, 12.0)]); +} + +#[tokio::test] +async fn a_deep_ref_not_handed_out_or_gone_is_stale() { + let fake = page(|_| json!({"gone": true})); + let (browser, id) = open(&fake, "deep-stale").await; + // Before any snapshot, no deep ref is handed out: nothing is sent. + let before = browser.perform(&id, click("d1")).await.unwrap_err(); + assert!(matches!(&before, Error::StaleRef { reference } if reference == "d1")); + assert!(ops(&fake).is_empty()); + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + assert!(matches!( + browser.perform(&id, click("d9")).await, + Err(Error::StaleRef { reference }) if reference == "d9" + )); + // Handed out, but the page re-drew it. + assert!(matches!( + browser.perform(&id, click("d1")).await, + Err(Error::StaleRef { reference }) if reference == "d1" + )); + assert!(mouse(&fake).is_empty(), "nothing was pressed"); +} + +#[tokio::test] +async fn a_covered_or_unseen_element_is_not_pressed() { + let fake = page(|op| match op { + "aim" => json!({"x": 40.0, "y": 12.0, "cover": "div.mask.dim +"#; + +/// The ref of the first `clickable` line named `name` after the line that +/// shows `marker` in `tree`. +#[cfg(feature = "agent-browser")] +fn ref_after(tree: &str, marker: &str, name: &str) -> String { + let wanted = format!("- clickable \"{name}\" [ref="); + let line = tree + .lines() + .skip_while(|line| !line.contains(marker)) + .find(|line| line.contains(&wanted)) + .unwrap_or_else(|| panic!("no {name:?} after {marker:?} in\n{tree}")); + let at = line.find("[ref=").expect("a ref") + "[ref=".len(); + line[at..line[at..].find(']').expect("a ref's end") + at].to_owned() +} + +/// What the page recorded as pressed, by name. +#[cfg(feature = "agent-browser")] +async fn pressed( + browser: &crate::Browser, + id: &tinycomputer_bus::browser::SessionId, +) -> Vec { + let calls = page_value(browser, id, "window.calls.map((call) => call.what)").await; + serde_json::from_value(calls).expect("the calls are names") +} + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_each_select_goes_on_its_own_flight_whatever_comes_before_it() { + let Some((browser, id)) = live_page(RESULTS).await else { + return; + }; + let full = browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + eprintln!("{}", full.tree); + // Flight A's "Select" is no control, and stays as it was. + let flight_a = full + .tree + .lines() + .skip_while(|line| !line.contains("\"Flight A\"")) + .nth(1) + .unwrap(); + assert_eq!(flight_a.trim(), "- StaticText \"Select\""); + let flight_b = ref_after(&full.tree, "\"Flight B\"", "Select"); + let flight_c = ref_after(&full.tree, "\"Flight C\"", "Select"); + assert_ne!(flight_b, flight_c); + on(&browser, &id, &flight_b, click).await.unwrap(); + assert_eq!(pressed(&browser, &id).await, ["B"]); + on(&browser, &id, &flight_c, click).await.unwrap(); + assert_eq!(pressed(&browser, &id).await, ["B", "C"]); + browser.close_session(&id).await.unwrap(); +} + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_row_redrawn_in_place_to_show_another_item_is_stale() { + let Some((browser, id)) = live_page(RESULTS).await else { + return; + }; + let shot = browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + let mini = ref_of(&shot, "clickable", "Mini"); + // The page reuses the row's own node for another item, as a list that + // recycles its rows does: the mark stays, the words change. + page_value( + &browser, + &id, + "document.querySelector('x-results').shadowRoot.getElementById('cab').textContent = 'Auto'; true", + ) + .await; + let stale = on(&browser, &id, &mini, click).await.unwrap_err(); + assert!( + matches!(&stale, Error::StaleRef { reference } if *reference == mini), + "{stale}" + ); + let read = on(&browser, &id, &mini, |target| Action::GetText { target }); + assert!(matches!(read.await, Err(Error::StaleRef { .. }))); + assert_eq!(pressed(&browser, &id).await.len(), 0, "nothing was pressed"); + // A fresh snapshot names it anew. + let again = browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + assert_ne!(ref_of(&again, "clickable", "Auto"), mini); + browser.close_session(&id).await.unwrap(); +} + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_card_is_pressed_beside_its_own_button_never_on_it() { + let Some((browser, id)) = live_page(RESULTS).await else { + return; + }; + let shot = browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + let card = ref_of(&shot, "clickable", "Mumbai Info Delhi"); + on(&browser, &id, &card, click).await.unwrap(); + assert_eq!(pressed(&browser, &id).await, ["card"]); + browser.close_session(&id).await.unwrap(); +} + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_a_press_something_comes_over_on_its_way_is_stopped() { + let Some((browser, id)) = live_page(RESULTS).await else { + return; + }; + let shot = browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + // The pointer's arrival raises a layer over "Prime", as a hover card + // can: the press would land on the layer. + let prime = ref_of(&shot, "clickable", "Prime"); + let stopped = on(&browser, &id, &prime, click).await.unwrap_err(); + assert!( + matches!(&stopped, Error::NotActionable { reason } if reason.contains(" came over it")), + "{stopped}" + ); + assert_eq!(pressed(&browser, &id).await.len(), 0, "neither was pressed"); + browser.close_session(&id).await.unwrap(); +} + +#[cfg(feature = "agent-browser")] +#[tokio::test] +async fn live_text_goes_after_what_a_field_with_no_caret_to_set_holds() { + let Some((browser, id)) = live_page(RESULTS).await else { + return; + }; + let shot = browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + // The box around the field shows no words of its own. + let wrap = ref_of(&shot, "clickable", ""); + let typed = on(&browser, &id, &wrap, |target| Action::Type { + target: Some(target), + text: "example.com".to_owned(), + delay_ms: None, + }); + typed.await.unwrap(); + assert_eq!( + page_value( + &browser, + &id, + "document.querySelector('x-results').shadowRoot.getElementById('mail').value" + ) + .await, + json!("ana.maria.silva@example.com") + ); + browser.close_session(&id).await.unwrap(); +} diff --git a/crates/tinycomputer-browser/src/deep/deep_tests/live_tests.rs b/crates/tinycomputer-browser/src/deep/deep_tests/live_tests.rs index e4491b29..691eb02c 100644 --- a/crates/tinycomputer-browser/src/deep/deep_tests/live_tests.rs +++ b/crates/tinycomputer-browser/src/deep/deep_tests/live_tests.rs @@ -90,7 +90,7 @@ const LIGHT: &str = r#" /// A real browser with `html` written into a blank tab: `None` unless /// `TINYCOMPUTER_LIVE_BROWSER=1`. #[cfg(feature = "agent-browser")] -async fn live_page(html: &str) -> Option<(Browser, SessionId)> { +pub(super) async fn live_page(html: &str) -> Option<(Browser, SessionId)> { use std::sync::Arc; if std::env::var("TINYCOMPUTER_LIVE_BROWSER").as_deref() != Ok("1") { @@ -119,7 +119,7 @@ async fn live_page(html: &str) -> Option<(Browser, SessionId)> { /// A snapshot as an agent's host reads a page: compact, not only the /// interactive lines, cut at 24,000 characters. #[cfg(feature = "agent-browser")] -fn compact() -> SnapshotRequest { +pub(super) fn compact() -> SnapshotRequest { SnapshotRequest { compact: true, max_chars: 24_000, @@ -129,7 +129,7 @@ fn compact() -> SnapshotRequest { /// The ref `snapshot` lists for the element named `name` with `role`. #[cfg(feature = "agent-browser")] -fn ref_of(snapshot: &Snapshot, role: &str, name: &str) -> String { +pub(super) fn ref_of(snapshot: &Snapshot, role: &str, name: &str) -> String { snapshot .refs .iter() @@ -141,7 +141,7 @@ fn ref_of(snapshot: &Snapshot, role: &str, name: &str) -> String { /// What the page's script evaluates `expression` to. #[cfg(feature = "agent-browser")] -async fn page_value(browser: &Browser, id: &SessionId, expression: &str) -> Value { +pub(super) async fn page_value(browser: &Browser, id: &SessionId, expression: &str) -> Value { browser .command(id, json!({"action": "evaluate", "script": expression})) .await @@ -151,7 +151,7 @@ async fn page_value(browser: &Browser, id: &SessionId, expression: &str) -> Valu /// `action` on `target`, as `Browser::perform` does it. #[cfg(feature = "agent-browser")] -async fn on( +pub(super) async fn on( browser: &Browser, id: &SessionId, target: &str, @@ -164,7 +164,7 @@ async fn on( } #[cfg(feature = "agent-browser")] -fn click(target: Target) -> Action { +pub(super) fn click(target: Target) -> Action { Action::Click { target, new_tab: false, diff --git a/crates/tinycomputer-browser/src/deep/deep_tests/merge_tests.rs b/crates/tinycomputer-browser/src/deep/deep_tests/merge_tests.rs index b2015b41..99c68e50 100644 --- a/crates/tinycomputer-browser/src/deep/deep_tests/merge_tests.rs +++ b/crates/tinycomputer-browser/src/deep/deep_tests/merge_tests.rs @@ -1,18 +1,26 @@ //! Placing deep controls in a snapshot's tree: on the line that shows their -//! words, or in a group after the tree, which a cut never drops. +//! words in their place, or in a group after the tree, which a cut never +//! drops. use tinycomputer_bus::browser::{ElementRef, Snapshot}; -use crate::deep::{Control, Kind, place}; +use crate::deep::{Control, Kind, Spot, place}; -fn clickable(number: u64, text: &str) -> Control { +/// A control whose words the page shows `of` times, its own the `seen`th. +fn nth(number: u64, text: &str, seen: usize, of: usize) -> Control { Control { number, kind: Kind::Clickable, text: text.to_owned(), + spot: Some(Spot { seen, of }), } } +/// A control the only one on the page to show its words. +fn clickable(number: u64, text: &str) -> Control { + nth(number, text, 0, 1) +} + /// A snapshot of `tree` as agent-browser gave it, with one ref of its own. fn snapshot(tree: &str) -> Snapshot { Snapshot { @@ -102,7 +110,7 @@ fn a_container_showing_other_words_is_passed_over() { let tree = "- generic\n - StaticText \"Prime\"\n - StaticText \"Mini\"\n- StaticText \"Mini\""; assert_eq!( - placed(tree, &[clickable(1, "Mini")]), + placed(tree, &[nth(1, "Mini", 0, 2)]), "- generic\n - StaticText \"Prime\"\n - clickable \"Mini\" [ref=d1]\n- StaticText \"Mini\"" ); } @@ -110,10 +118,9 @@ fn a_container_showing_other_words_is_passed_over() { #[test] fn words_shown_twice_go_to_the_line_in_the_controls_place() { let tree = "- StaticText \"Mini\"\n- StaticText \"Prime\"\n- StaticText \"Mini\""; - // The first control takes the first "Prime" there is; the one after it - // can only be a "Mini" after that. + // The page's second "Mini" is the control: the second line showing it. assert_eq!( - placed(tree, &[clickable(1, "Prime"), clickable(2, "Mini")]), + placed(tree, &[clickable(1, "Prime"), nth(2, "Mini", 1, 2)]), "- StaticText \"Mini\"\n- clickable \"Prime\" [ref=d1]\n- clickable \"Mini\" [ref=d2]" ); } @@ -130,6 +137,7 @@ fn controls_no_line_shows_follow_the_tree_in_a_group() { number: 4, kind: Kind::Editable, text: String::new(), + spot: None, }, ]; let mut shot = snapshot(tree); @@ -185,3 +193,54 @@ fn a_cut_keeps_the_group_whole_and_cuts_the_tree_shorter() { assert!(!roomy.truncated); assert!(roomy.tree.ends_with(group)); } + +/// Three result rows, each with its own "Select". +const ROWS: &str = "- generic\n - StaticText \"Flight A\"\n - StaticText \"Select\"\n\ + - generic\n - StaticText \"Flight B\"\n - StaticText \"Select\"\n\ + - generic\n - StaticText \"Flight C\"\n - StaticText \"Select\""; + +#[test] +fn a_row_the_page_does_not_list_moves_no_control_to_another_row() { + // Flight A is sold out: its "Select" shows, but takes no press, and is + // not listed. B's and C's are the second and third of three. + assert_eq!( + placed(ROWS, &[nth(1, "Select", 1, 3), nth(2, "Select", 2, 3)]), + "- generic\n - StaticText \"Flight A\"\n - StaticText \"Select\"\n\ + - generic\n - StaticText \"Flight B\"\n - clickable \"Select\" [ref=d1]\n\ + - generic\n - StaticText \"Flight C\"\n - clickable \"Select\" [ref=d2]" + ); +} + +#[test] +fn words_the_tree_shows_another_number_of_times_go_to_the_group() { + // The page shows "Select" four times, the tree three: which line is + // whose cannot be told, so none is guessed. + let group = "- group \"Controls inside web components\"\n - clickable \"Select\" [ref=d1]"; + assert_eq!( + placed(ROWS, &[nth(1, "Select", 1, 4)]), + format!("{ROWS}\n{group}") + ); + assert_eq!( + placed(ROWS, &[nth(1, "Select", 0, 2)]), + format!("{ROWS}\n{group}") + ); +} + +#[test] +fn words_compare_in_lower_case_as_the_tree_shows_them_drawn() { + assert_eq!( + placed("- StaticText \"LOG IN\"", &[clickable(1, "Log in")]), + "- clickable \"Log in\" [ref=d1]" + ); +} + +#[test] +fn a_line_already_taken_is_never_taken_twice() { + assert_eq!( + placed( + "- StaticText \"Mini\"", + &[clickable(1, "Mini"), clickable(2, "Mini")] + ), + "- clickable \"Mini\" [ref=d1]\n- group \"Controls inside web components\"\n - clickable \"Mini\" [ref=d2]" + ); +} diff --git a/crates/tinycomputer-browser/src/deep/merge.rs b/crates/tinycomputer-browser/src/deep/merge.rs index e2dbad49..bfcffa81 100644 --- a/crates/tinycomputer-browser/src/deep/merge.rs +++ b/crates/tinycomputer-browser/src/deep/merge.rs @@ -2,25 +2,36 @@ //! //! agent-browser shows the words of a shadow root's control as plain text: //! `- StaticText "LOG IN"`, or a row's pieces under an unnamed container -//! (`- generic`). A control is placed on the innermost line whose whole text -//! is the control's words: a text line, or a container holding only text -//! and other such containers (no ref, no name of its own). Words compare -//! without their spaces, since the tree splits a row's text into pieces and -//! drops the spaces between them. Lines are searched in the order the page -//! draws them, each control after the last one placed, so a word the page -//! shows twice goes to the one in its place. The line becomes the control -//! at the same depth, `- clickable "LOG IN" [ref=d1]`, and keeps what it -//! holds below it. +//! (`- generic`). A line shows a control's words when its whole text is +//! those words: a text line, or a container holding only text and other +//! such containers (no ref, no name of its own). Words compare without +//! their spaces, since the tree splits a row's text into pieces and drops +//! the spaces between them, and in lower case, as the tree shows text as +//! it is drawn. //! -//! A control that shows no words, or whose words no line holds (a compact -//! snapshot drops the text after its last ref), is listed after the tree in -//! a group of its own. That group is the one place those controls show, so -//! when the snapshot is cut to its size it is kept whole and the tree before -//! it is cut shorter. +//! The same words often show more than once ("Select" on every result), +//! and not every place is a listed control: a sold-out row, a heading, a +//! button hidden until hovered. So the look counts where each control's +//! words show among the page's text, as the tree shows its lines +//! ([`Spot`]), and a control goes on that place among the outermost lines +//! showing its words, the innermost line there, only when the tree shows +//! them as many times as the page does. That line becomes the control at +//! the same depth, `- clickable "LOG IN" [ref=d1]`, and keeps what it holds +//! below it. +//! +//! A control that shows no words, or whose place cannot be told for +//! certain (a compact snapshot drops the text after its last ref), is +//! listed after the tree in a group of its own. That group is the one place +//! those controls show, so when the snapshot is cut to its size it is kept +//! whole and the tree before it is cut shorter. + +use std::collections::HashMap; use tinycomputer_bus::browser::{ElementRef, Snapshot}; use super::Control; +#[cfg(doc)] +use super::Spot; /// The line the controls no line holds are listed under. const GROUP: &str = "- group \"Controls inside web components\""; @@ -35,7 +46,7 @@ struct Row { } enum Shape { - /// A text line, its words without their spaces. + /// A text line, its words as [`key`] makes them. Text(String), /// A line with no ref, no name, and no value: it shows only what it holds. Container, @@ -50,7 +61,7 @@ fn row(line: &str) -> Row { .strip_prefix("- StaticText ") .and_then(|quoted| serde_json::from_str::(quoted).ok()); let shape = match text { - Some(text) => Shape::Text(spaceless(&text)), + Some(text) => Shape::Text(key(&text)), None if body.starts_with("- ") && !body.contains("ref=") && !body.contains(" \"") @@ -63,8 +74,9 @@ fn row(line: &str) -> Row { Row { depth, shape } } -fn spaceless(text: &str) -> String { - text.split_whitespace().collect() +/// Words as they are compared: without their spaces, in lower case. +fn key(text: &str) -> String { + text.split_whitespace().collect::().to_lowercase() } /// Whether the whole text of the line at `at` is `words`. @@ -103,38 +115,48 @@ fn end_of(rows: &[Row], at: usize) -> usize { .count() } -/// Where `words` go, searching from `from`: the innermost line showing them, -/// and the index past the outermost one's lines. -fn spot(rows: &[Row], from: usize, words: &str) -> Option<(usize, usize)> { - let outer = (from..rows.len()).find(|&at| shows(rows, at, words))?; - let end = end_of(rows, outer); - let inner = (outer..end) - .rev() - .find(|&at| shows(rows, at, words)) - .unwrap_or(outer); - Some((inner, end)) +/// The places the tree shows `words`, in its order: of each outermost line +/// showing them, the innermost line among its own that does. +fn places(rows: &[Row], words: &str) -> Vec { + let mut found = Vec::new(); + let mut at = 0; + while at < rows.len() { + if shows(rows, at, words) { + let end = end_of(rows, at); + found.push( + (at..end) + .rev() + .find(|&line| shows(rows, line, words)) + .unwrap_or(at), + ); + at = end; + } else { + at += 1; + } + } + found } /// `tree` with `controls` placed in it, as the tree itself and the group of -/// those no line holds; either may be empty. +/// those no line holds for certain; either may be empty. fn merge(tree: &str, controls: &[Control]) -> (String, String) { let lines: Vec<&str> = tree.lines().collect(); let rows: Vec = lines.iter().map(|line| row(line)).collect(); let mut placed: Vec> = vec![None; rows.len()]; - let mut from = 0; + let mut shown: HashMap> = HashMap::new(); let mut left = Vec::new(); for control in controls { - let words = spaceless(&control.text); - let found = if words.is_empty() { - None - } else { - spot(&rows, from, &words) - }; - match found { - Some((at, end)) => { - placed[at] = Some(control); - from = end; - } + let words = key(&control.text); + let line = control.spot.filter(|_| !words.is_empty()).and_then(|spot| { + let found = shown + .entry(words.clone()) + .or_insert_with(|| places(&rows, &words)); + (found.len() == spot.of) + .then(|| found[spot.seen]) + .filter(|&line| placed[line].is_none()) + }); + match line { + Some(line) => placed[line] = Some(control), None => left.push(control), } } diff --git a/crates/tinycomputer-browser/src/deep/mod.rs b/crates/tinycomputer-browser/src/deep/mod.rs index e67034a2..1ddfb76c 100644 --- a/crates/tinycomputer-browser/src/deep/mod.rs +++ b/crates/tinycomputer-browser/src/deep/mod.rs @@ -27,14 +27,18 @@ //! //! Each is handed out as a ref `d1`, `d2`, …, which agent-browser's `eN` //! cannot collide with, and placed in the snapshot's tree (`merge.rs`). A -//! number is handed out once per session, and an element keeps its number -//! from one snapshot to the next while it lives and is still listed, so a -//! ref never comes to name another element. A page with nothing to list -//! comes out exactly as agent-browser gave it. +//! number is handed out once per session, a look that went unread +//! included, and an element keeps its number from one snapshot to the next +//! only while it lives, is still listed, and still shows the words it was +//! handed out with: a node a page re-draws in place to show another item +//! gets a new one, so a ref never comes to name another element. A page +//! with nothing to list comes out exactly as agent-browser gave it. While a +//! raw `frame` command has agent-browser read a frame, there is no look: +//! the script sees only the top document. //! //! An action on a `d` ref is the session's own (`sessions/deep.rs`): the -//! element is found by its mark, and pressed where it shows with the -//! pointer's own events. +//! element is found by its mark and its words, and pressed where it shows +//! with the pointer's own events. mod merge; @@ -96,8 +100,21 @@ pub(crate) struct Control { /// How it responds. pub(crate) kind: Kind, /// The words it shows, whitespace collapsed, at most [`LONGEST`] - /// characters; empty when it shows none. + /// characters; empty when it shows none. An editable region shows its + /// label, never what was typed into it. pub(crate) text: String, + /// Where its words show among the page's text, when they show there. + pub(crate) spot: Option, +} + +/// Where a control's words show among the page's text, counted as the tree +/// shows its lines (`merge.rs`): the `seen`th place, from 0, of `of`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Spot { + /// How many places showing the same words come before it. + pub(crate) seen: usize, + /// How many places show them in all. + pub(crate) of: usize, } impl Control { @@ -215,12 +232,14 @@ pub(crate) fn unsupported(action: &Action) -> Error { )) } -/// The deep refs a session's latest snapshot handed out, and the next -/// number to mint, which no ref of the session has had. +/// The deep refs a session's latest snapshot handed out with the words each +/// showed, the next number to mint, which no ref of the session has had, +/// and whether a raw `frame` command has agent-browser read a frame. #[derive(Debug)] pub(crate) struct Minted { - live: Vec, + live: Vec<(u64, String)>, next: u64, + framed: bool, } impl Default for Minted { @@ -228,51 +247,118 @@ impl Default for Minted { Self { live: Vec::new(), next: 1, + framed: false, } } } impl Minted { - /// Whether the latest snapshot handed out deep control `number`. - pub(crate) fn holds(&self, number: u64) -> bool { - self.live.contains(&number) + /// The words deep control `number` showed when the latest snapshot + /// handed it out; `None` when it did not. + pub(crate) fn words(&self, number: u64) -> Option<&str> { + self.live + .iter() + .find(|(held, _)| *held == number) + .map(|(_, words)| words.as_str()) + } + + /// Whether to look at all: not while agent-browser reads a frame, whose + /// document the script, run in the top document, never sees. + pub(crate) fn looks(&self) -> bool { + !self.framed } /// The look for deep controls under `scope`, a CSS selector, or all /// the page when `None`: an element still marked with a ref this table - /// holds keeps it, and any other is given a number from `next`. + /// holds keeps it while it shows the same words, and any other is given + /// a number from `next`. pub(crate) fn look(&self, scope: Option<&str>) -> String { + let keep: Vec = self + .live + .iter() + .map(|(number, words)| json!({"n": number, "text": words})) + .collect(); call( "find", &json!({ "scope": scope, "next": self.next, - "keep": self.live, + "keep": keep, "most": MOST, "longest": LONGEST, }), ) } - /// Takes what a look replied, `None` when it did not reply, and returns - /// the controls it found, which are now the ones this table holds. A - /// reply that is not a look's holds none. + /// Takes what a look replied, `None` when its reply never came, and + /// returns the controls it found, which are now the ones this table + /// holds. A look whose reply is lost, or is no look's, holds none, and + /// the numbers it may have marked elements with are never handed out: + /// a page that comes back from the back-forward cache still carries + /// them. pub(crate) fn take(&mut self, result: Option<&Value>) -> Vec { let Some((controls, next)) = result.and_then(|result| read(result, self.next)) else { self.live.clear(); + self.next = self.next.saturating_add(MOST as u64); return Vec::new(); }; - self.live = controls.iter().map(|control| control.number).collect(); + self.live = controls + .iter() + .map(|control| (control.number, control.text.clone())) + .collect(); self.next = next; controls } + + /// Hands out nothing for a snapshot that had no look. + pub(crate) fn skip(&mut self) { + self.live.clear(); + } + + /// Follows agent-browser's choice of the frame it reads, after the raw + /// command `action` ran: a frame once one is picked, the top document + /// once the main frame is, or the page or tab changes. + pub(crate) fn follow(&mut self, action: &str) { + match action { + "frame" => self.framed = true, + "mainframe" | "navigate" | "tab_new" | "tab_switch" | "tab_close" => { + self.framed = false; + } + _ => {} + } + } } -/// `op` on the element marked `number`: `aim`, `focus`, `end`, `select`, -/// `text`, or `visible` (`deep.js`). With `field`, an aim also says whether -/// the element takes text, or holds something that does. -pub(crate) fn act(op: &str, number: u64, field: bool) -> String { - call(op, &json!({ "number": number, "field": field })) +/// `op` on the element marked `number`: `focus`, `end`, `select`, `text`, or +/// `visible` (`deep.js`). With `words`, the element must still show the +/// words its ref was handed out with. +pub(crate) fn act(op: &str, number: u64, words: Option<&str>) -> String { + call( + op, + &json!({ "number": number, "words": words, "longest": LONGEST }), + ) +} + +/// The aim at the element marked `number`, which must still show `words`: +/// where a press on it goes. With `field`, also whether it takes text or +/// holds something that does; with `guard`, the press that follows is +/// stopped if it would reach anything else (read by [`pressed`]). +pub(crate) fn aim(number: u64, words: &str, field: bool, guard: bool) -> String { + call( + "aim", + &json!({ + "number": number, + "words": words, + "longest": LONGEST, + "field": field, + "guard": guard, + }), + ) +} + +/// Whether the press an aim guarded went where it was aimed. +pub(crate) fn pressed() -> String { + call("pressed", &json!({})) } fn call(op: &str, options: &Value) -> String { @@ -302,17 +388,30 @@ fn read(result: &Value, next: u64) -> Option<(Vec, u64)> { if controls.iter().any(|control| control.number == number) { continue; } + // As the page gave them: an action compares them with what the + // element shows then, in the same words. let text = item .get("text") .and_then(Value::as_str) .unwrap_or_default() - .split_whitespace() - .collect::>() - .join(" ") .chars() .take(LONGEST) .collect(); - controls.push(Control { number, kind, text }); + let index = |key: &str| { + item.get(key) + .and_then(Value::as_u64) + .and_then(|value| usize::try_from(value).ok()) + }; + let spot = index("seen") + .zip(index("of")) + .filter(|(seen, of)| seen < of) + .map(|(seen, of)| Spot { seen, of }); + controls.push(Control { + number, + kind, + text, + spot, + }); } let after = controls .iter() diff --git a/crates/tinycomputer-browser/src/sessions/deep.rs b/crates/tinycomputer-browser/src/sessions/deep.rs index 1c73cdb5..ebe51c3a 100644 --- a/crates/tinycomputer-browser/src/sessions/deep.rs +++ b/crates/tinycomputer-browser/src/sessions/deep.rs @@ -3,20 +3,24 @@ //! //! agent-browser cannot reach a deep control, so the session acts on it by //! position, as a person's hand does. The element is found by its mark -//! through the open shadow roots; gone, its ref is stale, so the caller takes -//! a fresh snapshot. It is brought into the window, and the middle of what -//! the window shows of it is checked to be what a press there reaches: the -//! element, or something inside it, through open shadow roots. Then the -//! pointer moves there and presses with the browser's own input events -//! (`convert::press_at`, as the surface presses a point), never a script's -//! `click()`, which a page can tell from a person's and which sends no -//! pointer down. +//! through the open shadow roots, and must still show the words its ref was +//! handed out with; gone, or showing another item, its ref is stale, so the +//! caller takes a fresh snapshot. It is brought into the window, and once it +//! has held still for two frames, a point on it is chosen where what a +//! press reaches is the element, or something inside it that is no control +//! of its own. Then the pointer moves there and presses with the browser's +//! own input events (`convert::press_at`, as the surface presses a point), +//! never a script's `click()`, which a page can tell from a person's and +//! which sends no pointer down. Between the aim and the press the page can +//! still move: a guard set with the aim stops any part of the press whose +//! path misses the element, and the press is reported refused. //! //! Keys go only where the action points, as `focus.rs` keeps them: before a //! focus, a key, or text, the focus comes off whatever held it, the element //! takes it (pressed, for text, as a person clicks into a field), and the -//! action is refused unless the focus is on the element or inside it. Text -//! goes only into what takes it: a field or an editable region. +//! action is refused unless the focus, two frames later, is on the element +//! or inside it. Text goes only into what takes it: a field or an editable +//! region. use std::time::Duration; @@ -33,12 +37,31 @@ use crate::error::{Error, Result}; /// handed over as agent-browser gave it. const LOOK: Duration = Duration::from_secs(3); +/// Whether an aim readies a press, and for what. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Aim { + /// The pointer only moves there. + Hover, + /// A press, guarded. + Press, + /// A check, before the focus comes off anything, that the element takes + /// text: nothing is pressed yet. + Text, + /// A press into a field, guarded. + Type, +} + impl Session { /// Looks for the deep controls under `scope`, a CSS selector, or all the /// page when `None`, after a snapshot, and hands out their refs. Never /// fails: a look that cannot run, or takes longer than [`LOOK`], hands - /// out none, and the refs of the snapshot before are stale. + /// out none, and the refs of the snapshot before are stale. While + /// agent-browser reads a frame, there is no look. pub(super) async fn find_deep(&mut self, scope: Option<&str>) -> Vec { + if !self.deep.looks() { + self.deep.skip(); + return Vec::new(); + } let script = self.deep.look(scope); let looking = self.run(json!({"action": "evaluate", "script": script})); let reply = tokio::time::timeout(LOOK, looking) @@ -56,34 +79,42 @@ impl Session { /// # Errors /// /// [`Error::StaleRef`] when the latest snapshot did not hand the ref out - /// or its element is gone, [`Error::NotActionable`] when it shows nowhere - /// in the window, something else lies over its middle, or it takes no - /// text meant for it, [`Error::FocusMissed`] when the focus does not land - /// on it, and whatever the engine reports. + /// or its element is gone or shows something else, [`Error::NotActionable`] + /// when it shows nowhere in the window, will not hold still, is covered, + /// holds another control where it shows, a press would have reached + /// something else, or it takes no text meant for it, + /// [`Error::FocusMissed`] when the focus does not land on it, and + /// whatever the engine reports. pub(super) async fn act_deep(&mut self, number: u64, action: &Action) -> Result { - if !self.deep.holds(number) { + let Some(words) = self.deep.words(number).map(str::to_owned) else { return Err(stale(number)); - } + }; match action { Action::Click { .. } => { - let (x, y) = self.aim(number, false).await?; + let (x, y) = self.aim(number, &words, Aim::Press).await?; self.press_at(x, y).await?; + self.pressed(number).await?; } Action::Hover { .. } => { - let (x, y) = self.aim(number, false).await?; + let (x, y) = self.aim(number, &words, Aim::Hover).await?; self.run(convert::move_to(x, y)).await?; } - Action::Focus { .. } => self.focus_deep(number).await?, + Action::Focus { .. } => self.focus_deep(number, &words).await?, Action::Press { key, .. } => { - self.focus_deep(number).await?; + self.focus_deep(number, &words).await?; self.run(json!({"action": "press", "key": key})).await?; } Action::Type { text, delay_ms, .. } => { - self.press_into(number, "end").await?; + let found = self.press_into(number, &words, "end").await?; + // A field with no caret to set (`email`, `number`) is typed + // into after what it holds all the same. + if found.get("caret") == Some(&Value::Bool(false)) { + self.run(json!({"action": "press", "key": "End"})).await?; + } self.type_keys(text, *delay_ms).await?; } Action::Fill { value, .. } => { - self.press_into(number, "select").await?; + self.press_into(number, &words, "select").await?; // The text replaces what the field held, selected; no text // deletes it. let command = if value.is_empty() { @@ -94,11 +125,11 @@ impl Session { self.run(command).await?; } Action::GetText { .. } => { - let read = self.find_mark(number, "text").await?; + let read = self.find_mark(number, "text", Some(&words)).await?; return Ok(read.get("text").cloned().unwrap_or(Value::Null)); } Action::IsVisible { .. } => { - let read = self.find_mark(number, "visible").await?; + let read = self.find_mark(number, "visible", None).await?; return Ok(read.get("visible").cloned().unwrap_or(Value::Bool(false))); } other => return Err(deep::unsupported(other)), @@ -106,43 +137,54 @@ impl Session { Ok(Value::Null) } - /// Runs `op` (`deep.js`) on the element marked `number`: its reply, or - /// `None` when no one element carries the mark. - async fn probe(&mut self, number: u64, op: &str) -> Result> { - self.probe_for(number, op, false).await - } - - /// [`Session::probe`], asking an aim whether the element takes text - /// when `field`. - async fn probe_for(&mut self, number: u64, op: &str, field: bool) -> Result> { + /// Runs `script`, a probe of the element marked `number` (`deep.js`): + /// its reply, or `None` when no one element carries the mark, or it no + /// longer shows the words it was asked to. + async fn probe(&mut self, script: String) -> Result> { let data = self - .run(json!({"action": "evaluate", "script": deep::act(op, number, field)})) + .run(json!({"action": "evaluate", "script": script})) .await?; let reply = data.get("result").cloned().unwrap_or(Value::Null); Ok((reply.is_object() && reply.get("gone") != Some(&Value::Bool(true))).then_some(reply)) } - /// [`Session::probe`], with an element gone as a stale ref: for a probe + /// `op` on the element marked `number`, which must still show `words` + /// when they are given, with an element gone as a stale ref: for a probe /// before anything was sent to the element. - async fn find_mark(&mut self, number: u64, op: &str) -> Result { - self.probe(number, op).await?.ok_or_else(|| stale(number)) + async fn find_mark(&mut self, number: u64, op: &str, words: Option<&str>) -> Result { + self.probe(deep::act(op, number, words)) + .await? + .ok_or_else(|| stale(number)) } - /// Where a press on deep control `number` goes: the middle of what the - /// window shows of it, brought into the window first, once that point is - /// checked to reach the element. For `text`, only an element that takes - /// text or holds something that does is pressed at all. - async fn aim(&mut self, number: u64, text: bool) -> Result<(f64, f64)> { + /// Where a press on deep control `number`, which must still show + /// `words`, goes: a point on it, brought into the window and held still, + /// that reaches the element itself. For text, only an element that + /// takes text or holds something that does goes ahead. + async fn aim(&mut self, number: u64, words: &str, aim: Aim) -> Result<(f64, f64)> { + let text = matches!(aim, Aim::Text | Aim::Type); + let guard = matches!(aim, Aim::Press | Aim::Type); let aimed = self - .probe_for(number, "aim", text) + .probe(deep::aim(number, words, text, guard)) .await? .ok_or_else(|| stale(number))?; let reference = deep::reference(number); + let said = |key: &str| aimed.get(key).and_then(Value::as_str).map(named); if aimed.get("unseen") == Some(&Value::Bool(true)) { return Err(Error::not_actionable(format!( "@{reference} shows nowhere in the window: it draws no box, or the page hides it" ))); } + if aimed.get("moving") == Some(&Value::Bool(true)) { + return Err(Error::not_actionable(format!( + "@{reference} would not hold still in the window, so it was not pressed; let the page settle and retry" + ))); + } + if let Some(control) = said("nested") { + return Err(Error::not_actionable(format!( + "@{reference} shows another control everywhere a press could reach it (<{control}>), so it was not pressed; press that control by its own ref" + ))); + } let (Some(x), Some(y)) = ( aimed.get("x").and_then(Value::as_f64), aimed.get("y").and_then(Value::as_f64), @@ -151,21 +193,7 @@ impl Session { "@{reference} could not be measured" ))); }; - if let Some(cover) = aimed.get("cover").and_then(Value::as_str) { - // Named by tag, id, and class, as agent-browser names a cover: - // nothing more of the page's own text goes into the error. - let cover: String = cover - .chars() - .filter(|character| { - character.is_ascii_alphanumeric() || "-_#.".contains(*character) - }) - .take(80) - .collect(); - let cover = if cover.is_empty() { - "an element" - } else { - &cover - }; + if let Some(cover) = said("cover") { return Err(Error::not_actionable(format!( "@{reference} is covered by <{cover}> at its middle, so a press would land on that instead; dismiss or close what covers it first" ))); @@ -187,36 +215,64 @@ impl Session { Ok(()) } - /// Gives deep control `number` the focus, and only it. - async fn focus_deep(&mut self, number: u64) -> Result<()> { + /// Whether the press on deep control `number` went where it was aimed. + /// A press that led to another page did; one the guard stopped, because + /// something else was on top by then, did not, and nothing of it reached + /// the page's own handlers. + async fn pressed(&mut self, number: u64) -> Result<()> { + let Ok(data) = self + .run(json!({"action": "evaluate", "script": deep::pressed()})) + .await + else { + return Ok(()); + }; + let missed = data + .get("result") + .and_then(|result| result.get("missed")) + .and_then(Value::as_str); + match missed { + Some(other) => Err(Error::not_actionable(format!( + "@{} moved, or <{}> came over it, as it was pressed, so the press was stopped; take a fresh snapshot and retry", + deep::reference(number), + named(other) + ))), + None => Ok(()), + } + } + + /// Gives deep control `number`, which must still show `words`, the + /// focus, and only it. + async fn focus_deep(&mut self, number: u64, words: &str) -> Result<()> { self.blur().await?; - let found = self.find_mark(number, "focus").await?; + let found = self.find_mark(number, "focus", Some(words)).await?; on_it(&found) } - /// Presses deep control `number` to put the focus in it, as a person - /// clicks into a field, then readies the field it focused for text: - /// with its caret after what it holds (`end`) or all it holds selected - /// (`select`). - async fn press_into(&mut self, number: u64, how: &str) -> Result<()> { + /// Presses deep control `number`, which must still show `words`, to put + /// the focus in it, as a person clicks into a field, then readies the + /// field it focused for text: with its caret after what it holds + /// (`end`) or all it holds selected (`select`). Returns what the field + /// said of itself. + async fn press_into(&mut self, number: u64, words: &str, how: &str) -> Result { // Refused before the focus moves: what held it (a box whose // suggestions are open, say) keeps it. - self.aim(number, true).await?; + self.aim(number, words, Aim::Text).await?; self.blur().await?; - // Measured again: letting go of the focus can close what it held open. - let (x, y) = self.aim(number, true).await?; + // Aimed again: letting go of the focus can close what it held open. + let (x, y) = self.aim(number, words, Aim::Type).await?; self.press_at(x, y).await?; + self.pressed(number).await?; // The press reached the page: an element gone now went with it, which // is no ref to retry blindly. let reference = deep::reference(number); - let Some(found) = self.probe(number, how).await? else { + let Some(found) = self.probe(deep::act(how, number, None)).await? else { return Err(Error::not_actionable(format!( "@{reference} went away when it was pressed, so no text was sent" ))); }; on_it(&found)?; if found.get("takes") == Some(&Value::Bool(true)) { - Ok(()) + Ok(found) } else { Err(no_text( &reference, @@ -256,6 +312,21 @@ fn on_it(found: &Value) -> Result<()> { } } +/// An element the page described by tag, id, and class, as agent-browser +/// names a cover: nothing more of the page's own text goes into an error. +fn named(described: &str) -> String { + let named: String = described + .chars() + .filter(|character| character.is_ascii_alphanumeric() || "-_#.".contains(*character)) + .take(80) + .collect(); + if named.is_empty() { + "an element".to_owned() + } else { + named + } +} + /// The refusal of text for `reference`, which takes none, and why. fn no_text(reference: &str, why: &str) -> Error { Error::not_actionable(format!( diff --git a/crates/tinycomputer-browser/src/sessions/page.rs b/crates/tinycomputer-browser/src/sessions/page.rs index 9150dc03..4fa628cd 100644 --- a/crates/tinycomputer-browser/src/sessions/page.rs +++ b/crates/tinycomputer-browser/src/sessions/page.rs @@ -61,6 +61,7 @@ impl Browser { return Err(Error::BlockedByPolicy { url: request.url }); } let data = session.run(command).await?; + session.deep.follow("navigate"); // Under a list the page is read afresh: the engine reports the // address it was asked for when the one it reached cannot be read. let page = if session.origins.restricts() { @@ -266,8 +267,12 @@ impl Browser { if NEITHER_ACTS_NOR_READS.contains(&action) { return session.run(command).await; } + let action = action.to_owned(); session.check_if_readable(false).await?; let data = session.run(command).await?; + // A frame agent-browser now reads, or reads no more: the look for + // deep controls follows it. + session.deep.follow(&action); // What a command read or did on a page it then left for a refused // one is never returned. session.check_if_readable(true).await?; diff --git a/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_keys_tests.rs b/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_keys_tests.rs index 6f2d1f6d..274c4ed9 100644 --- a/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_keys_tests.rs +++ b/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_keys_tests.rs @@ -5,7 +5,7 @@ use serde_json::json; use tinycomputer_bus::browser::{Action, SnapshotRequest, Target}; -use super::deep_page::{mouse, ops, page, shown}; +use super::deep_page::{mouse, ops, options, page, shown}; use super::open; use crate::error::Error; @@ -28,7 +28,19 @@ async fn text_is_typed_where_the_press_put_the_focus() { ) .await .unwrap(); - assert_eq!(ops(&fake), ["find", "aim", "aim", "end"]); + // Checked to take text, before the focus moves; aimed again, guarded; + // pressed; the guard read; the caret put at the end. + assert_eq!(ops(&fake), ["find", "aim", "aim", "pressed", "end"]); + let aims = options(&fake, "aim"); + assert_eq!( + (&aims[0]["field"], &aims[0]["guard"]), + (&json!(true), &json!(false)) + ); + assert_eq!( + (&aims[1]["field"], &aims[1]["guard"]), + (&json!(true), &json!(true)) + ); + assert_eq!(aims[1]["words"], ""); assert_eq!(mouse(&fake).len(), 3, "pressed into the field"); assert_eq!( fake.last("keyboard"), @@ -91,7 +103,7 @@ async fn a_fill_replaces_what_the_field_held() { value: value.to_owned(), }; browser.perform(&id, fill("Airport")).await.unwrap(); - assert_eq!(ops(&fake), ["find", "aim", "aim", "select"]); + assert_eq!(ops(&fake), ["find", "aim", "aim", "pressed", "select"]); assert_eq!( fake.last("inserttext"), json!({"action": "inserttext", "text": "Airport"}) @@ -245,5 +257,47 @@ async fn focus_press_and_reads_reach_the_marked_element() { .unwrap(); assert_eq!(visible.value, json!(true)); assert_eq!(ops(&fake), ["find", "focus", "focus", "text", "visible"]); + // The element must still show the words its ref was handed out with, + // to be focused or read; whether it shows at all asks for none. + assert_eq!(options(&fake, "focus")[0]["words"], "LOG IN"); + assert_eq!(options(&fake, "text")[0]["words"], "LOG IN"); + assert_eq!(options(&fake, "visible")[0]["words"], json!(null)); assert!(mouse(&fake).is_empty(), "a focus is no press"); } + +#[tokio::test] +async fn a_field_with_no_caret_to_set_is_typed_into_after_its_end_key() { + // An `email` or `number` field cannot have its caret set by script. + let fake = page(|op| match op { + "end" => json!({"inside": true, "on": "input", "takes": true, "caret": false}), + other => shown(other), + }); + let (browser, id) = open(&fake, "deep-no-caret").await; + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + browser + .perform( + &id, + Action::Type { + target: Some(Target::reference("d2")), + text: "4".to_owned(), + delay_ms: None, + }, + ) + .await + .unwrap(); + let keys: Vec<_> = fake + .sent() + .into_iter() + .filter(|command| command["action"] == "press" || command["action"] == "keyboard") + .collect(); + assert_eq!( + keys, + [ + json!({"action": "press", "key": "End"}), + json!({"action": "keyboard", "subaction": "type", "text": "4"}), + ] + ); +} diff --git a/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_page.rs b/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_page.rs index 86a06a9e..fce318e3 100644 --- a/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_page.rs +++ b/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_page.rs @@ -23,6 +23,19 @@ pub(super) fn ops(fake: &Fake) -> Vec { fake.sent().iter().filter_map(op).collect() } +/// The options each `op` was run with, in order. +pub(super) fn options(fake: &Fake, op_named: &str) -> Vec { + fake.sent() + .iter() + .filter(|command| op(command).as_deref() == Some(op_named)) + .map(|command| { + let script = command["script"].as_str().unwrap(); + let at = script.rfind(&format!(")(\"{op_named}\", ")).unwrap() + op_named.len() + 5; + serde_json::from_str(&script[at..script.len() - 1]).unwrap() + }) + .collect() +} + /// The mouse events sent, as `(event, x, y)`. pub(super) fn mouse(fake: &Fake) -> Vec<(String, f64, f64)> { fake.sent() @@ -40,8 +53,9 @@ pub(super) fn mouse(fake: &Fake) -> Vec<(String, f64, f64)> { pub(super) const TREE: &str = "- banner\n - StaticText \"LOG IN\"\n- button \"Book\" [ref=e1]"; -/// A page whose snapshot is [`TREE`] and whose look finds "LOG IN" and a -/// nameless row as `d1` and `d2`; every other op answers `answer(op)`. +/// A page whose snapshot is [`TREE`] and whose look finds "LOG IN", the +/// only words of its kind on the page, and a nameless row as `d1` and `d2`; +/// every other op answers `answer(op)`. pub(super) fn page(answer: impl Fn(&str) -> Value + Send + Sync + 'static) -> Fake { Fake::scripted(move |command| { if command["action"] == "snapshot" { @@ -53,7 +67,7 @@ pub(super) fn page(answer: impl Fn(&str) -> Value + Send + Sync + 'static) -> Fa let op = op(command)?; let result = if op == "find" { json!({"found": [ - {"n": 1, "kind": "clickable", "text": "LOG IN"}, + {"n": 1, "kind": "clickable", "text": "LOG IN", "seen": 0, "of": 1}, {"n": 2, "kind": "clickable", "text": ""}, ], "next": 3}) } else { @@ -63,11 +77,12 @@ pub(super) fn page(answer: impl Fn(&str) -> Value + Send + Sync + 'static) -> Fa }) } -/// The answers of a page whose deep elements show at (40, 12), take the -/// focus, and take text. +/// The answers of a page whose deep elements show still at (40, 12), take +/// the focus, take text, and take every press aimed at them. pub(super) fn shown(op: &str) -> Value { match op { "aim" => json!({"x": 40.0, "y": 12.0, "field": true}), + "pressed" => json!({"missed": null}), "focus" | "end" | "select" => json!({"inside": true, "on": "div", "takes": true}), "text" => json!({"text": "LOG IN"}), "visible" => json!({"visible": true}), diff --git a/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_tests.rs b/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_tests.rs index 9fc9746f..ec0bda00 100644 --- a/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_tests.rs +++ b/crates/tinycomputer-browser/src/sessions/sessions_tests/deep_tests.rs @@ -2,10 +2,13 @@ //! pressing, hovering, and refusing the `d` refs it hands out, over a //! scripted engine. -use serde_json::{Value, json}; -use tinycomputer_bus::browser::{Action, SnapshotRequest, Target}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; -use super::deep_page::{TREE, click, mouse, op, ops, page, shown}; +use serde_json::json; +use tinycomputer_bus::browser::{Action, NavigateRequest, SnapshotRequest, Target}; + +use super::deep_page::{TREE, click, mouse, op, ops, options, page, shown}; use super::open; use crate::error::Error; use crate::fake::{Fake, failure, ok}; @@ -29,24 +32,19 @@ async fn a_snapshot_lists_the_deep_controls_with_refs_of_their_own() { .map(|element| element.id.as_str()) .collect(); assert_eq!(ids, ["e1", "d1", "d2"]); - // The next look keeps the numbers handed out, and mints past them. + // The next look keeps the numbers handed out on what still shows the + // same words, and mints past them. browser .snapshot(&id, SnapshotRequest::default()) .await .unwrap(); - let looks: Vec<_> = fake - .sent() - .into_iter() - .filter(|command| op(command).as_deref() == Some("find")) - .collect(); - let options = |look: &Value| { - let script = look["script"].as_str().unwrap(); - let call = script.rfind(")(\"find\", ").unwrap() + ")(\"find\", ".len(); - serde_json::from_str::(&script[call..script.len() - 1]).unwrap() - }; - assert_eq!(options(&looks[0])["keep"], json!([])); - assert_eq!(options(&looks[1])["keep"], json!([1, 2])); - assert_eq!(options(&looks[1])["next"], json!(3)); + let looks = options(&fake, "find"); + assert_eq!(looks[0]["keep"], json!([])); + assert_eq!( + looks[1]["keep"], + json!([{"n": 1, "text": "LOG IN"}, {"n": 2, "text": ""}]) + ); + assert_eq!(looks[1]["next"], json!(3)); } #[tokio::test] @@ -91,7 +89,15 @@ async fn a_click_presses_the_middle_of_the_element_with_the_pointer() { let outcome = browser.perform(&id, click("@d1")).await.unwrap(); assert!(outcome.value.is_null()); assert_eq!(outcome.page.url, "https://flights.test/"); - assert_eq!(ops(&fake), ["find", "aim"]); + // Aimed at the element still showing its words, guarded, pressed, and + // the guard read. + assert_eq!(ops(&fake), ["find", "aim", "pressed"]); + let aimed = &options(&fake, "aim")[0]; + assert_eq!(aimed["words"], "LOG IN"); + assert_eq!( + (&aimed["guard"], &aimed["field"]), + (&json!(true), &json!(false)) + ); assert_eq!( mouse(&fake), [ @@ -122,6 +128,12 @@ async fn a_hover_only_moves_the_pointer_there() { .await .unwrap(); assert_eq!(mouse(&fake), [("mouseMoved".to_owned(), 40.0, 12.0)]); + assert_eq!( + options(&fake, "aim")[0]["guard"], + false, + "a hover is no press" + ); + assert_eq!(ops(&fake), ["find", "aim"]); } #[tokio::test] @@ -226,3 +238,143 @@ async fn what_a_deep_ref_cannot_take_is_refused_before_the_page_is_touched() { assert!(matches!(refused, Error::InvalidInput { .. })); assert_eq!(fake.sent().len(), sent, "nothing was sent"); } + +#[tokio::test] +async fn a_press_stopped_on_its_way_is_refused_and_one_that_left_the_page_is_not() { + let moved = page(|op| match op { + "pressed" => json!({"missed": "div.popup"}), + other => shown(other), + }); + let (browser, id) = open(&moved, "deep-missed").await; + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + let refused = browser.perform(&id, click("d1")).await.unwrap_err(); + assert!( + matches!(&refused, Error::NotActionable { reason } + if reason == "@d1 moved, or came over it, as it was pressed, so the press was stopped; take a fresh snapshot and retry"), + "{refused}" + ); + // A press whose page went away with it pressed something: it went. + let left = Fake::scripted(|command| { + if command["action"] == "snapshot" { + return Some(ok(&json!({"snapshot": TREE, "refs": {}}))); + } + Some(match op(command)?.as_str() { + "pressed" => failure("Execution context was destroyed."), + "find" => ok(&json!({"result": { + "found": [{"n": 1, "kind": "clickable", "text": "LOG IN"}], + "next": 2, + }})), + other => ok(&json!({ "result": shown(other) })), + }) + }); + let (browser, id) = open(&left, "deep-left").await; + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + assert!(browser.perform(&id, click("d1")).await.is_ok()); +} + +#[tokio::test] +async fn an_element_that_will_not_hold_still_or_shows_another_control_is_not_pressed() { + for (aimed, said) in [ + (json!({"moving": true}), "would not hold still"), + ( + json!({"nested": "button.details"}), + "shows another control everywhere a press could reach it ()", + ), + ] { + let fake = page(move |op| { + if op == "aim" { + aimed.clone() + } else { + shown(op) + } + }); + let (browser, id) = open(&fake, "deep-unsteady").await; + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + let refused = browser.perform(&id, click("d1")).await.unwrap_err(); + assert!( + matches!(&refused, Error::NotActionable { reason } if reason.contains(said)), + "{refused}" + ); + assert_eq!(mouse(&fake).len(), 0, "nothing was pressed"); + } +} + +#[tokio::test] +async fn while_agent_browser_reads_a_frame_a_snapshot_has_no_look() { + let fake = page(shown); + let (browser, id) = open(&fake, "deep-frame").await; + browser + .command(&id, json!({"action": "frame", "selector": "#payment"})) + .await + .unwrap(); + let framed = browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + assert_eq!(framed.tree, TREE); + assert_eq!( + ops(&fake).len(), + 0, + "the script never sees a frame's document" + ); + browser + .command(&id, json!({"action": "mainframe"})) + .await + .unwrap(); + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + assert_eq!(ops(&fake), ["find"]); + // A page loaded anew is read from its top document too. + browser + .command(&id, json!({"action": "frame", "selector": "#payment"})) + .await + .unwrap(); + browser + .navigate(&id, NavigateRequest::new("https://flights.test/pay")) + .await + .unwrap(); + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + assert_eq!(ops(&fake), ["find", "find"]); +} + +#[tokio::test] +async fn a_look_that_never_replies_hands_out_none_and_its_numbers_never() { + // The first look is never answered, as on a page being replaced; it may + // still have marked elements before its time ran out. + let stalled = Arc::new(AtomicBool::new(false)); + let fake = page(shown).stalling(move |command| { + op(command).as_deref() == Some("find") && !stalled.swap(true, Ordering::SeqCst) + }); + let (browser, id) = open(&fake, "deep-lost").await; + let lost = browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + assert_eq!(lost.tree, TREE); + assert_eq!(lost.refs.len(), 1); + browser + .snapshot(&id, SnapshotRequest::default()) + .await + .unwrap(); + let looks = options(&fake, "find"); + assert_eq!(looks[0]["next"], 1); + assert_eq!( + looks[1]["next"], 41, + "no number the lost look had is handed out" + ); + assert_eq!(looks[1]["keep"], json!([])); +} diff --git a/docs/crates/tinycomputer-browser/interacting.md b/docs/crates/tinycomputer-browser/interacting.md index 2d8f3db0..569181e0 100644 --- a/docs/crates/tinycomputer-browser/interacting.md +++ b/docs/crates/tinycomputer-browser/interacting.md @@ -70,34 +70,56 @@ region. Only what is drawn and not hidden from people counts, only the outermost of nested ones (a field inside a pressable row stays a field), and at most 40, in the order the page draws them. Each is marked `data-tc-deep="N"` and listed as ref `dN`, with the role `clickable`, -`editable`, or `focusable`: +`editable`, or `focusable`, named by its words: its rendered text, or for +an editable region its label, never what was typed into it. -- on the innermost line whose whole text is its words, which becomes it: +The same words often show more than once ("Select" on every result), and +not every place is a listed control: a sold-out row, a heading, a button +shown only on hover. So the look also counts where each control's words +show among the page's text, the way the tree shows its lines, and a control +goes: + +- on that place among the lines showing its words, when the tree shows them + as many times as the page does: its line becomes the control, as `- StaticText "LOG IN"` becomes `- clickable "LOG IN" [ref=d1]`; - otherwise after the tree, under `- group "Controls inside web components"`, which a cut to `max_chars` never drops: the tree before it is cut shorter. + A compact snapshot, which drops most text lines, puts most controls here. `Snapshot::refs` lists them after agent-browser's. A page with nothing to list comes out exactly as agent-browser gave it. A closed shadow root cannot -be read and is passed over, as are frames. A number is never handed out -twice in a session, and an element keeps its number from one snapshot to -the next while it lives. +be read and is passed over, as are frames; while a raw `frame` command has +agent-browser read a frame, there is no look at all, as the script sees only +the top document. A number is never handed out twice in a session, not even +one a look marked elements with before its reply was lost, and an element +keeps its number from one snapshot to the next only while it lives and +shows the same words: a list that re-draws a row's own node to show another +item has that row named anew. `Browser::perform` acts on a `d` ref itself, never through agent-browser, -which cannot resolve it. The element is found by its mark (gone, or carried -by two elements: `StaleRef`), brought into the window, and the middle of -what the window shows of it checked, through open shadow roots, to be the -element or inside it (otherwise `NotActionable`, naming what covers it by -tag, id, and class). Then: +which cannot resolve it. The element is found by its mark and must still +show the words it was handed out with (gone, carried by two elements, or +showing something else: `StaleRef`). It is brought into the window, and +once it has held still for two frames (`NotActionable` if it will not), a +point on it is chosen, through open shadow roots, where what a press reaches +is the element or something inside it that is no control of its own: a card +holding its own "Info" button is pressed beside the button. When nothing on +it can be reached, the action is `NotActionable`, naming what covers it, or +the control it holds, by tag, id, and class. Then: - `Click` presses there with raw mouse events, moved, pressed, released (`convert::press_at`, the press the surface uses on a point), so the page sees a person's trusted press with the button going down, never a - script's `click()`; `Hover` moves the pointer there; + script's `click()`. A guard set with the aim stops any part of the press + whose path misses the element (something came over it, as a layer a hover + raises can), and the action is `NotActionable`; `Hover` moves the pointer + there; - `Type` and `Fill`, only on an element that takes text or holds something - that does, press it as a person clicks into a field, and go ahead only if - the focus landed in it (`FocusMissed` otherwise, as below): `Type` types - a character at a time after what it holds, `Fill` replaces all it holds; + that does (checked before the focus moves), press it as a person clicks + into a field, and go ahead only if the focus is in it two frames later + (`FocusMissed` otherwise, as below): `Type` types a character at a time + after what it holds (after the End key, in a field whose caret a script + cannot set), `Fill` replaces all it holds; - `Focus`, and `Press` with a key, give it the focus without a press, checked the same way; - `GetText` and `IsVisible` read it.