From bb8ef6260159777155cf726330bab5e85263fb26 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Tue, 23 Jun 2026 13:00:06 +0800 Subject: [PATCH] Add token-budgeted a11y text observation (indexed, viewport-pruned) --- README/WHATS_NEW_zh-CN.md | 6 ++ README/WHATS_NEW_zh-TW.md | 6 ++ WHATS_NEW.md | 6 ++ .../doc/new_features/v152_features_doc.rst | 44 ++++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/v152_features_doc.rst | 38 ++++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 7 ++ .../gui/script_builder/command_schema.py | 22 +++++ .../utils/executor/action_executor.py | 33 +++++++ .../utils/mcp_server/tools/_factories.py | 35 +++++++- .../utils/mcp_server/tools/_handlers.py | 11 +++ je_auto_control/utils/observation/__init__.py | 6 ++ .../utils/observation/observation.py | 86 +++++++++++++++++++ .../headless/test_observation_batch.py | 65 ++++++++++++++ 15 files changed, 366 insertions(+), 1 deletion(-) create mode 100644 docs/source/Eng/doc/new_features/v152_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v152_features_doc.rst create mode 100644 je_auto_control/utils/observation/__init__.py create mode 100644 je_auto_control/utils/observation/observation.py create mode 100644 test/unit_test/headless/test_observation_batch.py diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index aaeeffc5..86f8c4d0 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 符记预算内的无障碍文字观测 + +把无障碍树转成 VLM 可操作的已编号文字区块。完整参考:[`docs/source/Zh/doc/new_features/v152_features_doc.rst`](../docs/source/Zh/doc/new_features/v152_features_doc.rst)。 + +- **`serialize_observation` / `observation_index` / `flatten_tree`**(`AC_serialize_observation`、`AC_observation_index`):`describe_screen` 给角色*数量* + 平面标签列表——没有稳定索引、没有 `[12] button "Submit" @(x,y)` 行、没有视口裁切、没有符记预算。本功能把(嵌套)元素树扁平化为仅互动项、裁切到视口、依阅读顺序排序、上限 `max_elements`、指派稳定 `index`,并渲染模型可操作的行(「click [12]」)。纯标准库,作用于元素字典;与 `fuse_elements`/`set_of_marks` 搭配。可无头测试。 + ## 本次更新 (2026-06-23) — 标准化 Computer-Use 动作结构 把 Anthropic / OpenAI agent 动作桥接到 AutoControl 命令。完整参考:[`docs/source/Zh/doc/new_features/v151_features_doc.rst`](../docs/source/Zh/doc/new_features/v151_features_doc.rst)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index 59675f4d..c86f1155 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,11 @@ # 本次更新 — AutoControl +## 本次更新 (2026-06-23) — 符記預算內的無障礙文字觀測 + +把無障礙樹轉成 VLM 可操作的已編號文字區塊。完整參考:[`docs/source/Zh/doc/new_features/v152_features_doc.rst`](../docs/source/Zh/doc/new_features/v152_features_doc.rst)。 + +- **`serialize_observation` / `observation_index` / `flatten_tree`**(`AC_serialize_observation`、`AC_observation_index`):`describe_screen` 給角色*數量* + 平面標籤清單——沒有穩定索引、沒有 `[12] button "Submit" @(x,y)` 行、沒有視口裁切、沒有符記預算。本功能把(巢狀)元素樹扁平化為僅互動項、裁切到視口、依閱讀順序排序、上限 `max_elements`、指派穩定 `index`,並渲染模型可操作的行(「click [12]」)。純標準函式庫,作用於元素字典;與 `fuse_elements`/`set_of_marks` 搭配。可無頭測試。 + ## 本次更新 (2026-06-23) — 標準化 Computer-Use 動作結構 把 Anthropic / OpenAI agent 動作橋接到 AutoControl 命令。完整參考:[`docs/source/Zh/doc/new_features/v151_features_doc.rst`](../docs/source/Zh/doc/new_features/v151_features_doc.rst)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index fcb19626..5180b364 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-23) — Token-Budgeted A11y Text Observation + +Turn the a11y tree into an indexed text block a VLM can act on. Full reference: [`docs/source/Eng/doc/new_features/v152_features_doc.rst`](docs/source/Eng/doc/new_features/v152_features_doc.rst). + +- **`serialize_observation` / `observation_index` / `flatten_tree`** (`AC_serialize_observation`, `AC_observation_index`): `describe_screen` gives role *counts* + a flat label list — no stable index, no `[12] button "Submit" @(x,y)` lines, no viewport clip, no token budget. This flattens a (nested) element tree to interactive-only, clips to the viewport, orders reading-style, caps at `max_elements`, assigns a stable `index`, and renders the lines a model acts on ("click [12]"). Pure-stdlib over element dicts; pairs with `fuse_elements`/`set_of_marks`. Headless-testable. + ## What's new (2026-06-23) — Canonical Computer-Use Action Schema Bridge Anthropic / OpenAI agent actions to AutoControl commands. Full reference: [`docs/source/Eng/doc/new_features/v151_features_doc.rst`](docs/source/Eng/doc/new_features/v151_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v152_features_doc.rst b/docs/source/Eng/doc/new_features/v152_features_doc.rst new file mode 100644 index 00000000..e44af518 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v152_features_doc.rst @@ -0,0 +1,44 @@ +Token-Budgeted A11y Text Observation +==================================== + +``screen_state.describe_screen`` returns role *counts* plus a flat list of control +labels — but no stable per-element index, no ``[12] button "Submit" @(x,y)`` lines, no +viewport clipping, and no element cap / token budget. Modern desktop and web agents +feed a *flattened, indexed, viewport-pruned* text block (the "accessibility tree as the +text observation" pattern) and then act by index ("click [12]"). This builds that +observation and the index behind it, pairing with :doc:`v138_features_doc` and +``set_of_marks``. + +Pure-stdlib over plain element dicts (``role`` / ``name`` / ``x`` / ``y`` / ``width`` / +``height``, optionally nested ``children``), so it is fully unit-testable. Imports no +``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import (serialize_observation, observation_index, + flatten_tree) + + text = serialize_observation(a11y_tree, viewport=(0, 0, 1920, 1080), + max_elements=60) + # [0] button "Save" @(30,20) + # [1] textbox "Search" @(140,20) + # ... feed `text` to the model; it replies "click [1]" + + target = observation_index(a11y_tree)[1] # the structured element behind [1] + click(*[target["x"] + target["width"] // 2, target["y"] + target["height"] // 2]) + +``flatten_tree`` flattens a nested element tree, keeping only interactive roles by +default. ``observation_index`` clips to the ``viewport``, orders top-to-bottom / +left-to-right, caps at ``max_elements`` and assigns a stable ``index``. +``serialize_observation`` renders those as ``[i] role "name" @(cx,cy)`` lines. + +Executor commands +----------------- + +``AC_serialize_observation`` (``elements`` / ``viewport`` / ``max_elements`` → +``{observation, count}``) and ``AC_observation_index`` (same inputs → +``{count, elements}``). They are exposed as the MCP tools ``ac_serialize_observation`` +/ ``ac_observation_index`` and as Script Builder commands under **Native UI**. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 832f3a1f..65134512 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -174,6 +174,7 @@ Comprehensive guides for all AutoControl features. doc/new_features/v149_features_doc doc/new_features/v150_features_doc doc/new_features/v151_features_doc + doc/new_features/v152_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/docs/source/Zh/doc/new_features/v152_features_doc.rst b/docs/source/Zh/doc/new_features/v152_features_doc.rst new file mode 100644 index 00000000..7362963a --- /dev/null +++ b/docs/source/Zh/doc/new_features/v152_features_doc.rst @@ -0,0 +1,38 @@ +符記預算內的無障礙文字觀測 +============================ + +``screen_state.describe_screen`` 回傳角色*數量*加上控制項標籤的平面清單——但沒有穩定的逐元素索引、沒有 +``[12] button "Submit" @(x,y)`` 行、沒有視口裁切,也沒有元素上限 / 符記預算。現代桌面與網頁 agent 餵入*扁平化、 +已編號、依視口修剪*的文字區塊(「無障礙樹作為文字觀測」模式),再依索引操作(「click [12]」)。本功能建立該觀測 +與其背後的索引,與 :doc:`v138_features_doc` 及 ``set_of_marks`` 搭配。 + +純標準函式庫,作用於純元素字典(``role`` / ``name`` / ``x`` / ``y`` / ``width`` / ``height``,可含巢狀 +``children``),因此完全可單元測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import (serialize_observation, observation_index, + flatten_tree) + + text = serialize_observation(a11y_tree, viewport=(0, 0, 1920, 1080), + max_elements=60) + # [0] button "Save" @(30,20) + # [1] textbox "Search" @(140,20) + # ... 把 `text` 餵給模型;它回覆「click [1]」 + + target = observation_index(a11y_tree)[1] # [1] 背後的結構化元素 + click(*[target["x"] + target["width"] // 2, target["y"] + target["height"] // 2]) + +``flatten_tree`` 扁平化巢狀元素樹,預設只保留互動角色。``observation_index`` 裁切到 ``viewport``、由上到下 / +由左到右排序、上限 ``max_elements`` 並指派穩定 ``index``。``serialize_observation`` 將其渲染為 +``[i] role "name" @(cx,cy)`` 行。 + +執行器命令 +---------- + +``AC_serialize_observation``(``elements`` / ``viewport`` / ``max_elements`` → ``{observation, count}``)與 +``AC_observation_index``(相同輸入 → ``{count, elements}``)。它們以 MCP 工具 ``ac_serialize_observation`` / +``ac_observation_index`` 以及 Script Builder 中 **Native UI** 分類下的命令提供。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index d43485b8..b574c82b 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -174,6 +174,7 @@ AutoControl 所有功能的完整使用指南。 doc/new_features/v149_features_doc doc/new_features/v150_features_doc doc/new_features/v151_features_doc + doc/new_features/v152_features_doc doc/ocr_backends/ocr_backends_doc doc/observability/observability_doc doc/operations_layer/operations_layer_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 5e44e1d3..93685ca2 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -365,6 +365,10 @@ from je_auto_control.utils.cua_action import ( canonical_action, from_anthropic, from_openai_cua, to_ac_command, ) +# Token-budgeted, indexed a11y text observation for VLM/agent grounding +from je_auto_control.utils.observation import ( + flatten_tree, observation_index, serialize_observation, +) # CI workflow annotations (GitHub Actions) from je_auto_control.utils.ci_annotations import ( emit_annotations, format_annotation, @@ -1243,6 +1247,9 @@ def start_autocontrol_gui(*args, **kwargs): "from_anthropic", "from_openai_cua", "to_ac_command", + "flatten_tree", + "observation_index", + "serialize_observation", "emit_annotations", "format_annotation", "ClipboardHistory", "default_clipboard_history", "analyze_heal_log", "heal_stats", "scan_secrets", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 6b17f704..81d515a2 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -2920,6 +2920,28 @@ def _add_set_of_marks_specs(specs: List[CommandSpec]) -> None: ), description="Map an Anthropic / OpenAI computer-use action to an AC command.", )) + specs.append(CommandSpec( + "AC_serialize_observation", "Native UI", "Observation: Serialize Elements", + fields=( + FieldSpec("elements", FieldType.STRING, + placeholder='[{"role":"button","name":"OK","x":..,"y":..}]'), + FieldSpec("viewport", FieldType.STRING, optional=True, + placeholder="[x, y, w, h]"), + FieldSpec("max_elements", FieldType.INT, optional=True, default=80), + ), + description="Indexed text observation of UI elements for a VLM (act by index).", + )) + specs.append(CommandSpec( + "AC_observation_index", "Native UI", "Observation: Index Elements", + fields=( + FieldSpec("elements", FieldType.STRING, + placeholder='[{"role":"button","name":"OK","x":..,"y":..}]'), + FieldSpec("viewport", FieldType.STRING, optional=True, + placeholder="[x, y, w, h]"), + FieldSpec("max_elements", FieldType.INT, optional=True, default=80), + ), + description="Reading-ordered, viewport-clipped, indexed element list.", + )) specs.append(CommandSpec( "AC_mark_screen", "Native UI", "Set-of-Marks: Number Elements", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 44c147d5..b41d64de 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -3829,6 +3829,37 @@ def _cua_command(payload: Any, source: str = "canonical") -> Dict[str, Any]: return {"canonical": canonical, "command": to_ac_command(canonical)} +def _serialize_observation(elements: Any, viewport: Any = None, + max_elements: Any = 80) -> Dict[str, Any]: + """Adapter: render an indexed a11y text observation from element dicts.""" + import json + from je_auto_control.utils.observation import (observation_index, + serialize_observation) + if isinstance(elements, str): + elements = json.loads(elements) + if isinstance(viewport, str): + viewport = json.loads(viewport) if viewport.strip() else None + text = serialize_observation(list(elements), viewport=viewport, + max_elements=int(max_elements)) + indexed = observation_index(list(elements), viewport=viewport, + max_elements=int(max_elements)) + return {"observation": text, "count": len(indexed)} + + +def _observation_index(elements: Any, viewport: Any = None, + max_elements: Any = 80) -> Dict[str, Any]: + """Adapter: the on-screen elements in reading order, capped, each indexed.""" + import json + from je_auto_control.utils.observation import observation_index + if isinstance(elements, str): + elements = json.loads(elements) + if isinstance(viewport, str): + viewport = json.loads(viewport) if viewport.strip() else None + indexed = observation_index(list(elements), viewport=viewport, + max_elements=int(max_elements)) + return {"count": len(indexed), "elements": indexed} + + def _with_modifiers(modifiers: Any, actions: Any) -> Dict[str, Any]: """Adapter: run nested actions while modifier keys are held down.""" import json @@ -5584,6 +5615,8 @@ def __init__(self): "AC_get_client_rect": _get_client_rect, "AC_client_point": _client_point, "AC_cua_command": _cua_command, + "AC_serialize_observation": _serialize_observation, + "AC_observation_index": _observation_index, "AC_tile_rect": _tile_rect, "AC_grid_rects": _grid_rects, "AC_cascade_rects": _cascade_rects, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index fe2b2f6b..538df121 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -3276,6 +3276,39 @@ def cua_action_tools() -> List[MCPTool]: ] +def observation_tools() -> List[MCPTool]: + return [ + MCPTool( + name="ac_serialize_observation", + description=("Render an indexed a11y text observation from 'elements' " + "(role/name/x/y/width/height dicts, optionally nested): " + "'[i] role \"name\" @(cx,cy)' lines, interactive-only, " + "viewport-clipped, capped at 'max_elements'. Returns " + "{observation, count} — feed it to a VLM, act by index."), + input_schema=schema({ + "elements": {"type": "array", "items": {"type": "object"}}, + "viewport": {"type": "array", "items": {"type": "integer"}}, + "max_elements": {"type": "integer"}}, + required=["elements"]), + handler=h.serialize_observation, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_observation_index", + description=("The on-screen elements in reading order, viewport-clipped " + "and capped, each with a stable 'index'. Returns {count, " + "elements} — the structured form behind the observation."), + input_schema=schema({ + "elements": {"type": "array", "items": {"type": "object"}}, + "viewport": {"type": "array", "items": {"type": "integer"}}, + "max_elements": {"type": "integer"}}, + required=["elements"]), + handler=h.observation_index, + annotations=READ_ONLY, + ), + ] + + def ssim_tools() -> List[MCPTool]: return [ MCPTool( @@ -6784,7 +6817,7 @@ def media_assert_tools() -> List[MCPTool]: locator_chain_tools, rich_clipboard_tools, img_histogram_tools, motion_regions_tools, window_zorder_tools, soft_assert_tools, perceptual_diff_tools, window_geometry_tools, cua_action_tools, - plugin_sdk_tools, governance_tools, + observation_tools, plugin_sdk_tools, governance_tools, credential_lease_tools, egress_tools, approval_testing_tools, trajectory_eval_tools, compliance_tools, agent_trace_tools, video_report_tools, fuzzy_tools, artifact_store_tools, image_dedup_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 1425e1ef..5a0a25e0 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -2309,6 +2309,17 @@ def cua_command(payload, source="canonical"): return _cua_command(payload, source) +def serialize_observation(elements, viewport=None, max_elements=80): + from je_auto_control.utils.executor.action_executor import ( + _serialize_observation) + return _serialize_observation(elements, viewport, max_elements) + + +def observation_index(elements, viewport=None, max_elements=80): + from je_auto_control.utils.executor.action_executor import _observation_index + return _observation_index(elements, viewport, max_elements) + + def detect_drift(reference, current, threshold=0.25, bins=10): from je_auto_control.utils.executor.action_executor import _detect_drift return _detect_drift(reference, current, threshold, bins) diff --git a/je_auto_control/utils/observation/__init__.py b/je_auto_control/utils/observation/__init__.py new file mode 100644 index 00000000..21ea39e8 --- /dev/null +++ b/je_auto_control/utils/observation/__init__.py @@ -0,0 +1,6 @@ +"""Token-budgeted, indexed a11y text observation for VLM/agent grounding.""" +from je_auto_control.utils.observation.observation import ( + flatten_tree, observation_index, serialize_observation, +) + +__all__ = ["flatten_tree", "observation_index", "serialize_observation"] diff --git a/je_auto_control/utils/observation/observation.py b/je_auto_control/utils/observation/observation.py new file mode 100644 index 00000000..5f844443 --- /dev/null +++ b/je_auto_control/utils/observation/observation.py @@ -0,0 +1,86 @@ +"""Token-budgeted, indexed a11y text observation — what to feed a VLM/agent. + +``screen_state.describe_screen`` returns role *counts* plus a flat list of control +labels — but no stable per-element index, no ``[12] button "Submit" @(x,y)`` lines, no +viewport clipping, and no element cap / token budget. Modern desktop/web agents feed a +*flattened, indexed, viewport-pruned* text block (the "accessibility tree as the text +observation" pattern), then act by index ("click [12]"). This builds that observation +and the index behind it, pairing with :doc:`element_parse` and ``set_of_marks``. + +Pure-stdlib over plain element dicts (``role`` / ``name`` / ``x`` / ``y`` / ``width`` / +``height``, optionally nested ``children``), so it is fully unit-testable. Imports no +``PySide6``. +""" +from typing import Any, Dict, List, Optional, Sequence + +Element = Dict[str, Any] +_INTERACTIVE = {"button", "link", "textbox", "edit", "textfield", "checkbox", + "radio", "menuitem", "tab", "combobox", "listitem", "switch", + "slider", "menu", "option"} + + +def _center(element: Element) -> List[int]: + return [int(element.get("x", 0)) + int(element.get("width", 0)) // 2, + int(element.get("y", 0)) + int(element.get("height", 0)) // 2] + + +def flatten_tree(elements: Sequence[Element], *, + interactive_only: bool = True) -> List[Element]: + """Flatten a (possibly nested ``children``) element tree to a flat list. + + With ``interactive_only`` (default) only actionable roles (button, link, textbox, + …) survive. Each returned node drops its ``children`` key. + """ + flat: List[Element] = [] + + def walk(items: Sequence[Element]) -> None: + for element in items: + flat.append({key: value for key, value in element.items() + if key != "children"}) + if element.get("children"): + walk(element["children"]) + + walk(elements) + if interactive_only: + flat = [e for e in flat if str(e.get("role", "")).lower() in _INTERACTIVE] + return flat + + +def _in_viewport(element: Element, viewport: Optional[Sequence[int]]) -> bool: + if not viewport: + return True + vx, vy, vw, vh = (int(v) for v in viewport[:4]) + cx, cy = _center(element) + return vx <= cx <= vx + vw and vy <= cy <= vy + vh + + +def observation_index(elements: Sequence[Element], *, + viewport: Optional[Sequence[int]] = None, + max_elements: int = 80, + interactive_only: bool = True) -> List[Element]: + """Return the on-screen elements in reading order, capped, each with an ``index``. + + Flattens the tree, keeps only elements whose centre is inside ``viewport`` (if + given), orders them top-to-bottom / left-to-right, caps at ``max_elements`` and + assigns a stable ``index`` an agent can refer to. + """ + from je_auto_control.utils.element_parse import reading_order + flat = flatten_tree(elements, interactive_only=interactive_only) + visible = [e for e in flat if _in_viewport(e, viewport)] + ordered = reading_order(visible)[:int(max_elements)] + return [dict(element, index=index) for index, element in enumerate(ordered)] + + +def serialize_observation(elements: Sequence[Element], *, + viewport: Optional[Sequence[int]] = None, + max_elements: int = 80, + interactive_only: bool = True) -> str: + """Render the indexed observation as ``[i] role "name" @(cx,cy)`` lines.""" + lines = [] + for element in observation_index(elements, viewport=viewport, + max_elements=max_elements, + interactive_only=interactive_only): + cx, cy = _center(element) + lines.append(f'[{element["index"]}] {element.get("role", "element")} ' + f'"{element.get("name", "")}" @({cx},{cy})') + return "\n".join(lines) diff --git a/test/unit_test/headless/test_observation_batch.py b/test/unit_test/headless/test_observation_batch.py new file mode 100644 index 00000000..9aa4f8cf --- /dev/null +++ b/test/unit_test/headless/test_observation_batch.py @@ -0,0 +1,65 @@ +"""Headless tests for the indexed a11y text observation. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.observation import ( + flatten_tree, observation_index, serialize_observation, +) + + +def _tree(): + return [{"role": "window", "name": "App", "children": [ + {"role": "button", "name": "Save", "x": 10, "y": 10, "width": 40, + "height": 20}, + {"role": "textbox", "name": "Search", "x": 100, "y": 10, "width": 80, + "height": 20}, + {"role": "label", "name": "static", "x": 10, "y": 50, "width": 60, + "height": 20}, + {"role": "button", "name": "Offscreen", "x": 10, "y": 5000, "width": 40, + "height": 20}, + ]}] + + +def test_flatten_keeps_only_interactive(): + roles = [(e["role"], e["name"]) for e in flatten_tree(_tree())] + assert ("button", "Save") in roles and ("textbox", "Search") in roles + assert ("label", "static") not in roles # non-interactive dropped + assert ("window", "App") not in roles + + +def test_flatten_all_when_not_interactive_only(): + roles = {e["role"] for e in flatten_tree(_tree(), interactive_only=False)} + assert {"window", "button", "textbox", "label"} <= roles + + +def test_observation_index_clips_viewport_and_indexes(): + indexed = observation_index(_tree(), viewport=(0, 0, 1920, 1080)) + assert [(e["index"], e["name"]) for e in indexed] == [(0, "Save"), + (1, "Search")] + # the y=5000 button is clipped out + + +def test_observation_index_cap(): + assert len(observation_index(_tree(), max_elements=1)) == 1 + + +def test_serialize_observation_lines(): + text = serialize_observation(_tree(), viewport=(0, 0, 1920, 1080)) + assert text.splitlines() == ['[0] button "Save" @(30,20)', + '[1] textbox "Search" @(140,20)'] + + +# --- wiring --------------------------------------------------------------- + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_serialize_observation", "AC_observation_index"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_serialize_observation", "ac_observation_index"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_serialize_observation", "AC_observation_index"} <= specs + + +def test_facade_exports(): + for attr in ("flatten_tree", "observation_index", "serialize_observation"): + assert hasattr(ac, attr) and attr in ac.__all__