Let a Bot choose from a dropdown, and carry on after a browser step fails

.github/workflows/ci.yml

@@ -139,7 +139,7 @@ jobs:
       - run: >-
           bun test tests/sign-in.test.ts tests/session-cookies.test.ts tests/follows-popup.test.ts
           tests/demonstration-live.test.ts tests/webrtc-proxy.test.ts tests/web-bot-auth-browser.test.ts
-          tests/handoff-page-restore.test.ts
+          tests/handoff-page-restore.test.ts tests/enter-text.test.ts
         working-directory: agent-computer
         env:
           OPENBOT_COMPUTER_BROWSER: '1'

CHANGELOG.md

@@ -35,6 +35,23 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
   request for an administrator to switch on a work app, grant it to a Bot, or connect a Shared app's
   team account is kept, with who answered it and when. It starts empty and nothing is backfilled.
 
+### A Bot can choose from a dropdown
+
+A Bot filling in a form in its browser could not set a dropdown (a native `<select>`, such as a CRM's
+Industry or Stage). Typing into one waited until it timed out and was reported as a field that had
+gone from the page, so the Bot took a fresh snapshot and tried the same thing again. Typing into a
+dropdown now chooses the option it names, by the option's label or its value, and the page sees the
+same change a person's choice makes. Text that names no option is refused with a message saying so.
+A page snapshot now lists each dropdown's options and the one it shows, so the Bot can see what it
+may choose and what is already chosen.
+
+### A Bot carries on after a browser step fails
+
+When a click, typed entry or key press the Bot made in its browser failed, the Bot's turn ended on
+the spot: the step showed "Didn't work" and the Bot said nothing more. The failure now reaches the
+Bot like any other result, so it can take a fresh snapshot, try another way, or tell you what
+stopped it. A refused step reads as a refusal the Bot can explain.
+
 ### A Bot can send, draft and reply in Gmail
 
 OpenBot left out every Composio action that takes a file, because a Bot has no way to supply one.

agent-computer/src/aria-snapshot.ts

@@ -73,8 +73,16 @@ export type SnapshotElement = {
   value?: string;
   disabled?: boolean;
   checked?: boolean;
+  /**
+   * A native dropdown's choices, by what each says. Its options carry no refs, so without this a Bot
+   * saw `combobox "Industry"` and nothing it could choose; it chooses by sending one of these as text.
+   */
+  options?: string[];
 };
 
+/** The most choices one dropdown reports, so a list of every country cannot crowd out the form. */
+const OPTION_LIMIT = 50;
+
 /** The descriptor half of an entry: everything before the colon. */
 type Descriptor = {
   role: string;
@@ -199,6 +207,17 @@ function toElement(
 
   if (descriptor.flags.has("disabled")) element.disabled = true;
 
+  // A native `<select>`: its value in the YAML is the list of its options, not text. The option marked
+  // `[selected]` is what the field shows, which is the value a person reads off it.
+  if (descriptor.role === "combobox" || descriptor.role === "listbox") {
+    const list = optionsOf(value);
+    if (list) {
+      element.options = list.options;
+      if (list.selected !== undefined && element.value === undefined)
+        element.value = list.selected;
+    }
+  }
+
   // Playwright emits `[checked]` only when something is checked, so absence is ambiguous on its own: a
   // Bot cannot tell an unticked box from a control that does not tick. Reported as false for the roles
   // that can be checked, and left off entirely for the ones that cannot.
@@ -218,6 +237,35 @@ function toElement(
   return element;
 }
 
+/**
+ * The options listed under a dropdown, and which one is selected.
+ *
+ * Only options with no ref of their own: one with a ref is a custom listbox's option, already an
+ * element the Bot clicks, and listing it twice would offer two ways to do one thing.
+ */
+function optionsOf(
+  value: unknown,
+): { options: string[]; selected?: string } | null {
+  if (!Array.isArray(value)) return null;
+  const options: string[] = [];
+  let selected: string | undefined;
+  for (const child of value) {
+    const key =
+      typeof child === "string"
+        ? child
+        : child && typeof child === "object"
+          ? Object.keys(child)[0]
+          : undefined;
+    const descriptor = key ? parseDescriptor(key) : null;
+    if (descriptor?.role !== "option" || descriptor.flags.has("ref")) continue;
+    const name = cutAtCodeUnits(descriptor.name, 200);
+    if (descriptor.flags.has("selected") && selected === undefined)
+      selected = name;
+    if (options.length < OPTION_LIMIT) options.push(name);
+  }
+  return options.length ? { options, selected } : null;
+}
+
 /** Turn an aria snapshot into the flat element list the tool contract publishes. */
 export function parseAriaSnapshot(yaml: string): {
   elements: SnapshotElement[];

agent-computer/src/enter-text.ts

@@ -0,0 +1,62 @@
+import type { Locator } from "playwright";
+
+/** What every Playwright call here is given: the action timeout, and Stop when there is one. */
+type Acting = { timeout: number; signal?: AbortSignal };
+
+/**
+ * The text a Bot sent cannot go in this field: a list with no option by that name.
+ *
+ * The caller's to fix, like a stale ref, so the route answers 400 rather than 502. The message names
+ * neither the text (a typed value is never echoed) nor the options (they are page content, and page
+ * content reaches the model only inside the untrusted envelope a snapshot puts it in).
+ */
+export class FieldValueError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "FieldValueError";
+  }
+}
+
+/**
+ * Put `text` in a field the way a person would.
+ *
+ * A text box is filled: `fill` clears it first, which is what "put this value in this box" means.
+ * A native `<select>` takes no typing at all, and `fill` waits on one until it times out, which the
+ * server then reported as a ref that had gone stale. A person picks the option by what it says, so
+ * this does too: the option whose label is the text, ignoring case and surrounding spaces, and
+ * failing that the option whose value is. Picking it fires the same input and change events a
+ * person's choice does, so the page reacts as it would to them.
+ */
+export async function enterText(
+  field: Locator,
+  text: string,
+  acting: Acting,
+): Promise<void> {
+  const options = await field.evaluate(
+    (element) =>
+      element instanceof HTMLSelectElement
+        ? Array.from(element.options, (option) => ({
+            label: option.label,
+            value: option.value,
+            disabled: option.disabled,
+          }))
+        : null,
+    undefined,
+    { timeout: acting.timeout },
+  );
+  if (!options) {
+    await field.fill(text, acting);
+    return;
+  }
+  const wanted = text.trim().toLowerCase();
+  const choosable = options.filter((option) => !option.disabled);
+  const chosen =
+    choosable.find((option) => option.label.trim().toLowerCase() === wanted) ??
+    choosable.find((option) => option.value.trim().toLowerCase() === wanted);
+  if (!chosen) {
+    throw new FieldValueError(
+      "That field is a list, and none of its options matches that text. Take a snapshot to see its options and send one of them exactly.",
+    );
+  }
+  await field.selectOption({ value: chosen.value }, acting);
+}

agent-computer/src/index.ts

@@ -27,6 +27,7 @@ import {
   TAKE_CONTROL_FIRST,
 } from "./control";
 import { describeHumanGesture } from "./demonstration";
+import { enterText, FieldValueError } from "./enter-text";
 import { handleEgressPolicyRequest, startEgressFilter } from "./egress";
 import { identity } from "./identity";
 import {
@@ -1423,6 +1424,10 @@ serve<StreamData>({
           if (error instanceof StaleSnapshotError) {
             return json({ error: error.message, stale: true }, 409);
           }
+          // The same reasoning: the computer is fine, and the Bot fixes it by sending another value.
+          if (error instanceof FieldValueError) {
+            return json({ error: error.message }, 400);
+          }
           return json({ error: describe(error, "The action failed.") }, 502);
         }
       }
@@ -1585,8 +1590,8 @@ async function performAction(
     const field = await resolveRef(session, target, ref, expected);
     // `fill` rather than keystrokes: it clears the field first, which is what "put this value in
     // this box" means. Typing into a field a previous attempt half-filled otherwise appends, and the
-    // form ends up with "AlicAlice" in it.
-    await field.fill(body.text, acting);
+    // form ends up with "AlicAlice" in it. A dropdown is chosen from instead (see enterText).
+    await enterText(field, body.text, acting);
     if (body.submit === true) {
       await field.press("Enter", acting);
     }

agent-computer/tests/aria-snapshot.test.ts

@@ -325,3 +325,64 @@ describe("parseDescriptor", () => {
     expect(parseDescriptor("   ")).toBeNull();
   });
 });
+
+/**
+ * Captured from `ariaSnapshot({ mode: "ai" })` in this package's Playwright: two native `<select>`s
+ * and a custom listbox. A native select's options carry no refs, so they used to be dropped and a Bot
+ * saw `combobox "Industry"` with nothing it could choose and no way to tell what it was set to.
+ */
+const DROPDOWNS = `- generic [active] [ref=e1]:
+  - generic [ref=e2]:
+    - text: Industry
+    - combobox "Industry" [ref=e3]:
+      - option "Select an industry" [selected]
+      - option "Logistics"
+      - option "Retail"
+  - generic [ref=e4]:
+    - text: Stage
+    - combobox "Stage" [ref=e5]:
+      - option "Prospect" [selected]
+      - option "Qualified"
+  - listbox "Owner" [ref=e6]:
+    - option "Alex" [ref=e7]`;
+
+describe("parseAriaSnapshot, dropdowns", () => {
+  test("a native select reports its options and the one it shows", () => {
+    const { elements } = parseAriaSnapshot(DROPDOWNS);
+    expect(elements.find((element) => element.ref === "e3")).toEqual({
+      ref: "e3",
+      role: "combobox",
+      name: "Industry",
+      options: ["Select an industry", "Logistics", "Retail"],
+      value: "Select an industry",
+    });
+    expect(elements.find((element) => element.ref === "e5")?.value).toBe(
+      "Prospect",
+    );
+  });
+
+  test("a custom listbox's options stay elements of their own, listed once", () => {
+    const { elements } = parseAriaSnapshot(DROPDOWNS);
+    expect(elements.find((element) => element.ref === "e6")).toEqual({
+      ref: "e6",
+      role: "listbox",
+      name: "Owner",
+    });
+    expect(elements.find((element) => element.ref === "e7")).toEqual({
+      ref: "e7",
+      role: "option",
+      name: "Alex",
+    });
+  });
+
+  test("a very long list is cut to fifty choices", () => {
+    const many = Array.from(
+      { length: 120 },
+      (_, index) => `      - option "Country ${index}"`,
+    ).join("\n");
+    const { elements } = parseAriaSnapshot(
+      `- combobox "Country" [ref=e9]:\n${many}`,
+    );
+    expect(elements[0]?.options).toHaveLength(50);
+  });
+});

agent-computer/tests/enter-text.test.ts

@@ -0,0 +1,136 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import type { Browser, Page } from "playwright";
+import { parseAriaSnapshot } from "../src/aria-snapshot";
+import { enterText, FieldValueError } from "../src/enter-text";
+
+/**
+ * Typing into a form the way a Bot does: an AI snapshot, then the field by its `aria-ref`.
+ *
+ * A CRM's New account form has text boxes and dropdowns side by side. Typing "Logistics" into the
+ * Industry dropdown used `fill`, which waits on a `<select>` until it times out; the server reported
+ * that as a ref gone stale, the Bot's turn ended, and no account was ever saved.
+ *
+ *   OPENBOT_COMPUTER_BROWSER=1 bun test tests/enter-text.test.ts
+ */
+const asked = process.env.OPENBOT_COMPUTER_BROWSER === "1";
+const ACTING = { timeout: 2_000 };
+
+const FORM = `<!doctype html><title>New account</title>
+<form>
+  <label>Account name <input name="name" value="Draft"></label>
+  <label>Industry
+    <select name="industry">
+      <option value="">Select an industry</option>
+      <option value="fin">Financial services</option>
+      <option value="logistics">Logistics</option>
+      <option value="retired" disabled>Retired</option>
+    </select>
+  </label>
+</form>
+<output id="changes">0</output>
+<script>
+  const changes = document.getElementById("changes");
+  document.querySelector("select").addEventListener("change", () => {
+    changes.textContent = String(Number(changes.textContent) + 1);
+  });
+</script>`;
+
+let browser: Browser;
+
+beforeAll(async () => {
+  if (!asked) return;
+  const { chromium } = await import("playwright");
+  browser = await chromium.launch();
+});
+
+afterAll(async () => {
+  await browser?.close();
+});
+
+/** The form, and the refs its AI snapshot gives each field, as the computer reads them. */
+async function openForm(): Promise<{
+  tab: Page;
+  refOf: (name: string) => string;
+  industry: { options?: string[]; value?: string } | undefined;
+}> {
+  const tab = await browser.newPage();
+  await tab.setContent(FORM);
+  const snapshot = await tab.ariaSnapshot({ mode: "ai" });
+  const { elements } = parseAriaSnapshot(snapshot);
+  return {
+    tab,
+    industry: elements.find((element) => element.name === "Industry"),
+    refOf: (name) => {
+      const found = elements.find((element) => element.name === name);
+      if (!found) throw new Error(`No ${name} in the snapshot.`);
+      return found.ref;
+    },
+  };
+}
+
+describe.skipIf(!asked)("entering text in a form", () => {
+  test("a dropdown is chosen from by the option's label, and the page sees a change", async () => {
+    const { tab, refOf } = await openForm();
+    await enterText(
+      tab.locator(`aria-ref=${refOf("Industry")}`),
+      " logistics ",
+      ACTING,
+    );
+    expect(await tab.locator("select").inputValue()).toBe("logistics");
+    expect(await tab.locator("#changes").textContent()).toBe("1");
+    await tab.close();
+  });
+
+  test("a dropdown falls back to the option's value", async () => {
+    const { tab, refOf } = await openForm();
+    await enterText(
+      tab.locator(`aria-ref=${refOf("Industry")}`),
+      "fin",
+      ACTING,
+    );
+    expect(await tab.locator("select").inputValue()).toBe("fin");
+    await tab.close();
+  });
+
+  test("text that is not an option, or names a disabled one, is refused and changes nothing", async () => {
+    const { tab, refOf } = await openForm();
+    const industry = tab.locator(`aria-ref=${refOf("Industry")}`);
+    for (const text of ["Aerospace", "Retired"]) {
+      const refused = await enterText(industry, text, ACTING).catch(
+        (error: unknown) => error,
+      );
+      expect(refused).toBeInstanceOf(FieldValueError);
+      // Never the text it was sent, never the page's own option labels.
+      expect((refused as Error).message).not.toContain(text);
+      expect((refused as Error).message).not.toContain("Logistics");
+    }
+    expect(await tab.locator("select").inputValue()).toBe("");
+    expect(await tab.locator("#changes").textContent()).toBe("0");
+    await tab.close();
+  });
+
+  test("the snapshot the refusal points to lists the dropdown's options and its choice", async () => {
+    const { tab, industry } = await openForm();
+    expect(industry?.options).toEqual([
+      "Select an industry",
+      "Financial services",
+      "Logistics",
+      "Retired",
+    ]);
+    expect(industry?.value).toBe("Select an industry");
+    await tab.close();
+  });
+
+  test("a text box is still filled, replacing what was there", async () => {
+    const { tab, refOf } = await openForm();
+    await enterText(
+      tab.locator(`aria-ref=${refOf("Account name")}`),
+      "Navia Freight",
+      ACTING,
+    );
+    expect(await tab.locator("input[name=name]").inputValue()).toBe(
+      "Navia Freight",
+    );
+    await tab.close();
+  });
+});