From 59d8e285179252589a48cd44433c24851c550b5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hanno=20J=2E=20G=C3=B6decke?= Date: Sat, 21 Nov 2026 14:34:28 +0100 Subject: [PATCH] fix(android): refresh the accessibility tree for element selection --- packages/core/src/serve-emu.ts | 3 ++- packages/runtimes/serve-emu/src/api/routes/inspection.ts | 5 +++-- .../runtimes/serve-emu/src/device-session-context.ts | 2 +- packages/runtimes/test/serve-emu-node.test.ts | 9 +++++++++ 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/core/src/serve-emu.ts b/packages/core/src/serve-emu.ts index b82899a9..84e1d32c 100644 --- a/packages/core/src/serve-emu.ts +++ b/packages/core/src/serve-emu.ts @@ -400,7 +400,8 @@ export class ServeEmu { const backend = await this.start(id); const adb = await adbPath(); const [tree, health, display] = await Promise.all([ - this.json(backend.url, "/api/accessibility"), + // The tree must line up with the current screen even immediately after an input. + this.json(backend.url, "/api/accessibility", { headers: { "Cache-Control": "no-cache" } }), this.health(backend.url), execute(adb, ["-s", id, "shell", "wm", "size"], { timeout: 3000, maxBuffer: 4096 }), ]); diff --git a/packages/runtimes/serve-emu/src/api/routes/inspection.ts b/packages/runtimes/serve-emu/src/api/routes/inspection.ts index 4aa7de0d..29951b4b 100644 --- a/packages/runtimes/serve-emu/src/api/routes/inspection.ts +++ b/packages/runtimes/serve-emu/src/api/routes/inspection.ts @@ -88,11 +88,12 @@ export function inspectionRoutes(): ApiRoute[] { { method: "GET", path: "/api/accessibility", - handler: async ({ deps }) => { + handler: async ({ deps, request }) => { const { readAccessibilitySnapshot, requestContext, errorResponse } = deps; try { - return Response.json(await readAccessibilitySnapshot(requestContext)); + const cacheMs = request.headers.get("cache-control") === "no-cache" ? 0 : undefined; + return Response.json(await readAccessibilitySnapshot(requestContext, cacheMs)); } catch (err) { return errorResponse(err); } diff --git a/packages/runtimes/serve-emu/src/device-session-context.ts b/packages/runtimes/serve-emu/src/device-session-context.ts index 8935abb8..2b68014f 100644 --- a/packages/runtimes/serve-emu/src/device-session-context.ts +++ b/packages/runtimes/serve-emu/src/device-session-context.ts @@ -171,7 +171,7 @@ export class ActiveDeviceSession< this.assertUsable(); const now = this.#now(); if ( - this.#accessibilitySnapshotCache && + cacheMs > 0 && this.#accessibilitySnapshotCache && this.#accessibilitySnapshotCache.expiresMs > now ) { return Promise.resolve(this.#accessibilitySnapshotCache.snapshot); diff --git a/packages/runtimes/test/serve-emu-node.test.ts b/packages/runtimes/test/serve-emu-node.test.ts index dd90fb6a..f27860c2 100644 --- a/packages/runtimes/test/serve-emu-node.test.ts +++ b/packages/runtimes/test/serve-emu-node.test.ts @@ -82,9 +82,11 @@ function fakeScrcpy() { test("Node Android server preserves HTTP, WebSocket video, input acknowledgements, and cleanup", async t => { const scrcpy = fakeScrcpy(); + let captures = 0; const backend = await runtime.startServer({ serial: "emulator-5554", port: 0 }, { openScrcpy: async () => scrcpy.session, createInputQueue: () => scrcpy.inputQueue, + loadAccessibility: async () => ({ ok: true, capturedAt: String(++captures), nodes: [] }), }); t.after(() => backend.stop()); const base = `http://127.0.0.1:${backend.server.port}`; @@ -96,6 +98,12 @@ test("Node Android server preserves HTTP, WebSocket video, input acknowledgement const page = await fetch(base); assert.match(page.headers.get("content-type") ?? "", /text\/html/); assert.equal(await page.text(), "Android"); + const tree = async (fresh = false) => { + const response = await fetch(`${base}/api/accessibility`, { headers: fresh ? { "Cache-Control": "no-cache" } : {} }); + return response.json(); + }; + assert.equal((await tree()).capturedAt, "1"); + assert.equal((await tree()).capturedAt, "1"); const socket = new WebSocket(`ws://127.0.0.1:${backend.server.port}/ws?frame-meta=1`); t.after(() => socket.terminate()); @@ -108,6 +116,7 @@ test("Node Android server preserves HTTP, WebSocket video, input acknowledgement assert.equal(parsedReply.ok, true); assert.equal(parsedReply.requestId, "input-1"); assert.ok(scrcpy.packets.length > 0); + assert.equal((await tree(true)).capturedAt, "2", "element selection reads the screen after input"); const video = once(socket, "message"); const data = Buffer.from([0, 0, 0, 1, 0x65, 0x80]); scrcpy.frame({ type: "frame", data, pts: 1000n, isKey: true, isConfig: false });