From f91eea704621406716d5b11d5ed4b180208182a7 Mon Sep 17 00:00:00 2001
From: David McKay <david@copilotkit.ai>
Date: Fri, 23 Oct 2026 16:46:23 -0700
Subject: [PATCH 1/6] Choose a dropdown's option when a Bot types into it

The computer's /type filled every field with Playwright's fill, which a native <select> does not take: it waited until the action timed out, and the server reported the timeout as an element gone from the page. The Bot took a fresh snapshot and tried the same thing again, so a form with a dropdown (a CRM's Industry or Stage) could never be completed. enterText now picks the option whose label is the text, ignoring case and surrounding spaces, and failing that the one whose value is, which fires the change a person's choice does. Text that names no enabled option is refused with a 400 that names neither the text nor the page's options.
---
 agent-computer/src/enter-text.ts | 62 ++++++++++++++++++++++++++++++++
 agent-computer/src/index.ts      |  9 +++--
 2 files changed, 69 insertions(+), 2 deletions(-)
 create mode 100644 agent-computer/src/enter-text.ts

diff --git a/agent-computer/src/enter-text.ts b/agent-computer/src/enter-text.ts
new file mode 100644
index 0000000000..5db55ffaf7
--- /dev/null
+++ b/agent-computer/src/enter-text.ts
@@ -0,0 +1,62 @@
+import type { Locator } from "playwright";
+
+/** What every Playwright call here is given: the action timeout, and Stop when there is one. */
+type Acting = { timeout: number; signal?: AbortSignal };
+
+/**
+ * The text a Bot sent cannot go in this field: a list with no option by that name.
+ *
+ * The caller's to fix, like a stale ref, so the route answers 400 rather than 502. The message names
+ * neither the text (a typed value is never echoed) nor the options (they are page content, and page
+ * content reaches the model only inside the untrusted envelope a snapshot puts it in).
+ */
+export class FieldValueError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "FieldValueError";
+  }
+}
+
+/**
+ * Put `text` in a field the way a person would.
+ *
+ * A text box is filled: `fill` clears it first, which is what "put this value in this box" means.
+ * A native `<select>` takes no typing at all, and `fill` waits on one until it times out, which the
+ * server then reported as a ref that had gone stale. A person picks the option by what it says, so
+ * this does too: the option whose label is the text, ignoring case and surrounding spaces, and
+ * failing that the option whose value is. Picking it fires the same input and change events a
+ * person's choice does, so the page reacts as it would to them.
+ */
+export async function enterText(
+  field: Locator,
+  text: string,
+  acting: Acting,
+): Promise<void> {
+  const options = await field.evaluate(
+    (element) =>
+      element instanceof HTMLSelectElement
+        ? Array.from(element.options, (option) => ({
+            label: option.label,
+            value: option.value,
+            disabled: option.disabled,
+          }))
+        : null,
+    undefined,
+    { timeout: acting.timeout },
+  );
+  if (!options) {
+    await field.fill(text, acting);
+    return;
+  }
+  const wanted = text.trim().toLowerCase();
+  const choosable = options.filter((option) => !option.disabled);
+  const chosen =
+    choosable.find((option) => option.label.trim().toLowerCase() === wanted) ??
+    choosable.find((option) => option.value.trim().toLowerCase() === wanted);
+  if (!chosen) {
+    throw new FieldValueError(
+      "That field is a list, and none of its options matches that text. Take a snapshot to see its options and send one of them exactly.",
+    );
+  }
+  await field.selectOption({ value: chosen.value }, acting);
+}
diff --git a/agent-computer/src/index.ts b/agent-computer/src/index.ts
index e05165acbb..0848640259 100644
--- a/agent-computer/src/index.ts
+++ b/agent-computer/src/index.ts
@@ -27,6 +27,7 @@ import {
   TAKE_CONTROL_FIRST,
 } from "./control";
 import { describeHumanGesture } from "./demonstration";
+import { enterText, FieldValueError } from "./enter-text";
 import { handleEgressPolicyRequest, startEgressFilter } from "./egress";
 import { identity } from "./identity";
 import {
@@ -1423,6 +1424,10 @@ serve<StreamData>({
           if (error instanceof StaleSnapshotError) {
             return json({ error: error.message, stale: true }, 409);
           }
+          // The same reasoning: the computer is fine, and the Bot fixes it by sending another value.
+          if (error instanceof FieldValueError) {
+            return json({ error: error.message }, 400);
+          }
           return json({ error: describe(error, "The action failed.") }, 502);
         }
       }
@@ -1585,8 +1590,8 @@ async function performAction(
     const field = await resolveRef(session, target, ref, expected);
     // `fill` rather than keystrokes: it clears the field first, which is what "put this value in
     // this box" means. Typing into a field a previous attempt half-filled otherwise appends, and the
-    // form ends up with "AlicAlice" in it.
-    await field.fill(body.text, acting);
+    // form ends up with "AlicAlice" in it. A dropdown is chosen from instead (see enterText).
+    await enterText(field, body.text, acting);
     if (body.submit === true) {
       await field.press("Enter", acting);
     }

From acf829ccab66df091517ccf835ef810ade72ec56 Mon Sep 17 00:00:00 2001
From: David McKay <david@copilotkit.ai>
Date: Fri, 23 Oct 2026 16:46:23 -0700
Subject: [PATCH 2/6] List each dropdown's options and choice in a page
 snapshot

A native <select>'s options carry no refs, so the snapshot dropped them, and its value is the list of options rather than text, so the combobox came through with no value either. A Bot saw combobox "Industry" with nothing it could choose and no way to tell what it was set to. The element now carries its options (at most 50) and, as its value, the one marked selected. A custom listbox's options keep their own refs and stay elements of their own, listed once.
---
 agent-computer/src/aria-snapshot.ts | 48 +++++++++++++++++++++++++++++
 server/src/computer/schema.ts       |  2 ++
 2 files changed, 50 insertions(+)

diff --git a/agent-computer/src/aria-snapshot.ts b/agent-computer/src/aria-snapshot.ts
index 88db6ad7e6..037ba14035 100644
--- a/agent-computer/src/aria-snapshot.ts
+++ b/agent-computer/src/aria-snapshot.ts
@@ -73,8 +73,16 @@ export type SnapshotElement = {
   value?: string;
   disabled?: boolean;
   checked?: boolean;
+  /**
+   * A native dropdown's choices, by what each says. Its options carry no refs, so without this a Bot
+   * saw `combobox "Industry"` and nothing it could choose; it chooses by sending one of these as text.
+   */
+  options?: string[];
 };
 
+/** The most choices one dropdown reports, so a list of every country cannot crowd out the form. */
+const OPTION_LIMIT = 50;
+
 /** The descriptor half of an entry: everything before the colon. */
 type Descriptor = {
   role: string;
@@ -199,6 +207,17 @@ function toElement(
 
   if (descriptor.flags.has("disabled")) element.disabled = true;
 
+  // A native `<select>`: its value in the YAML is the list of its options, not text. The option marked
+  // `[selected]` is what the field shows, which is the value a person reads off it.
+  if (descriptor.role === "combobox" || descriptor.role === "listbox") {
+    const list = optionsOf(value);
+    if (list) {
+      element.options = list.options;
+      if (list.selected !== undefined && element.value === undefined)
+        element.value = list.selected;
+    }
+  }
+
   // Playwright emits `[checked]` only when something is checked, so absence is ambiguous on its own: a
   // Bot cannot tell an unticked box from a control that does not tick. Reported as false for the roles
   // that can be checked, and left off entirely for the ones that cannot.
@@ -218,6 +237,35 @@ function toElement(
   return element;
 }
 
+/**
+ * The options listed under a dropdown, and which one is selected.
+ *
+ * Only options with no ref of their own: one with a ref is a custom listbox's option, already an
+ * element the Bot clicks, and listing it twice would offer two ways to do one thing.
+ */
+function optionsOf(
+  value: unknown,
+): { options: string[]; selected?: string } | null {
+  if (!Array.isArray(value)) return null;
+  const options: string[] = [];
+  let selected: string | undefined;
+  for (const child of value) {
+    const key =
+      typeof child === "string"
+        ? child
+        : child && typeof child === "object"
+          ? Object.keys(child)[0]
+          : undefined;
+    const descriptor = key ? parseDescriptor(key) : null;
+    if (descriptor?.role !== "option" || descriptor.flags.has("ref")) continue;
+    const name = cutAtCodeUnits(descriptor.name, 200);
+    if (descriptor.flags.has("selected") && selected === undefined)
+      selected = name;
+    if (options.length < OPTION_LIMIT) options.push(name);
+  }
+  return options.length ? { options, selected } : null;
+}
+
 /** Turn an aria snapshot into the flat element list the tool contract publishes. */
 export function parseAriaSnapshot(yaml: string): {
   elements: SnapshotElement[];
diff --git a/server/src/computer/schema.ts b/server/src/computer/schema.ts
index 4836eaed94..62da7af707 100644
--- a/server/src/computer/schema.ts
+++ b/server/src/computer/schema.ts
@@ -133,6 +133,8 @@ export type SnapshotElement = {
   type?: string;
   disabled?: boolean;
   checked?: boolean;
+  /** A native dropdown's choices, by label. A Bot chooses one by typing it into the dropdown. */
+  options?: string[];
 };
 
 export type SnapshotResult = {

From 70e316909c8321122dfe185b3dd35d9f035415bb Mon Sep 17 00:00:00 2001
From: David McKay <david@copilotkit.ai>
Date: Fri, 23 Oct 2026 16:46:23 -0700
Subject: [PATCH 3/6] Test choosing from a dropdown, and run it with the
 computer's browser tests

Drives a real form through an AI snapshot and the aria-ref locator, as the computer does: a label and a value are chosen, an unknown or disabled option is refused without changing the field, a text box is still filled, and the snapshot lists the dropdown's options. The parser tests pin the captured snapshot of two selects and a custom listbox. The browser test joins the computer job's list, which is the only CI job that sets OPENBOT_COMPUTER_BROWSER.
---
 .github/workflows/ci.yml                   |   2 +-
 agent-computer/tests/aria-snapshot.test.ts |  61 +++++++++
 agent-computer/tests/enter-text.test.ts    | 136 +++++++++++++++++++++
 3 files changed, 198 insertions(+), 1 deletion(-)
 create mode 100644 agent-computer/tests/enter-text.test.ts

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8c488b2f25..301973e270 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -139,7 +139,7 @@ jobs:
       - run: >-
           bun test tests/sign-in.test.ts tests/session-cookies.test.ts tests/follows-popup.test.ts
           tests/demonstration-live.test.ts tests/webrtc-proxy.test.ts tests/web-bot-auth-browser.test.ts
-          tests/handoff-page-restore.test.ts
+          tests/handoff-page-restore.test.ts tests/enter-text.test.ts
         working-directory: agent-computer
         env:
           OPENBOT_COMPUTER_BROWSER: '1'
diff --git a/agent-computer/tests/aria-snapshot.test.ts b/agent-computer/tests/aria-snapshot.test.ts
index 4052b47ee2..79d93d5037 100644
--- a/agent-computer/tests/aria-snapshot.test.ts
+++ b/agent-computer/tests/aria-snapshot.test.ts
@@ -325,3 +325,64 @@ describe("parseDescriptor", () => {
     expect(parseDescriptor("   ")).toBeNull();
   });
 });
+
+/**
+ * Captured from `ariaSnapshot({ mode: "ai" })` in this package's Playwright: two native `<select>`s
+ * and a custom listbox. A native select's options carry no refs, so they used to be dropped and a Bot
+ * saw `combobox "Industry"` with nothing it could choose and no way to tell what it was set to.
+ */
+const DROPDOWNS = `- generic [active] [ref=e1]:
+  - generic [ref=e2]:
+    - text: Industry
+    - combobox "Industry" [ref=e3]:
+      - option "Select an industry" [selected]
+      - option "Logistics"
+      - option "Retail"
+  - generic [ref=e4]:
+    - text: Stage
+    - combobox "Stage" [ref=e5]:
+      - option "Prospect" [selected]
+      - option "Qualified"
+  - listbox "Owner" [ref=e6]:
+    - option "Alex" [ref=e7]`;
+
+describe("parseAriaSnapshot, dropdowns", () => {
+  test("a native select reports its options and the one it shows", () => {
+    const { elements } = parseAriaSnapshot(DROPDOWNS);
+    expect(elements.find((element) => element.ref === "e3")).toEqual({
+      ref: "e3",
+      role: "combobox",
+      name: "Industry",
+      options: ["Select an industry", "Logistics", "Retail"],
+      value: "Select an industry",
+    });
+    expect(elements.find((element) => element.ref === "e5")?.value).toBe(
+      "Prospect",
+    );
+  });
+
+  test("a custom listbox's options stay elements of their own, listed once", () => {
+    const { elements } = parseAriaSnapshot(DROPDOWNS);
+    expect(elements.find((element) => element.ref === "e6")).toEqual({
+      ref: "e6",
+      role: "listbox",
+      name: "Owner",
+    });
+    expect(elements.find((element) => element.ref === "e7")).toEqual({
+      ref: "e7",
+      role: "option",
+      name: "Alex",
+    });
+  });
+
+  test("a very long list is cut to fifty choices", () => {
+    const many = Array.from(
+      { length: 120 },
+      (_, index) => `      - option "Country ${index}"`,
+    ).join("\n");
+    const { elements } = parseAriaSnapshot(
+      `- combobox "Country" [ref=e9]:\n${many}`,
+    );
+    expect(elements[0]?.options).toHaveLength(50);
+  });
+});
diff --git a/agent-computer/tests/enter-text.test.ts b/agent-computer/tests/enter-text.test.ts
new file mode 100644
index 0000000000..ac9510cfb0
--- /dev/null
+++ b/agent-computer/tests/enter-text.test.ts
@@ -0,0 +1,136 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import type { Browser, Page } from "playwright";
+import { parseAriaSnapshot } from "../src/aria-snapshot";
+import { enterText, FieldValueError } from "../src/enter-text";
+
+/**
+ * Typing into a form the way a Bot does: an AI snapshot, then the field by its `aria-ref`.
+ *
+ * A CRM's New account form has text boxes and dropdowns side by side. Typing "Logistics" into the
+ * Industry dropdown used `fill`, which waits on a `<select>` until it times out; the server reported
+ * that as a ref gone stale, the Bot's turn ended, and no account was ever saved.
+ *
+ *   OPENBOT_COMPUTER_BROWSER=1 bun test tests/enter-text.test.ts
+ */
+const asked = process.env.OPENBOT_COMPUTER_BROWSER === "1";
+const ACTING = { timeout: 2_000 };
+
+const FORM = `<!doctype html><title>New account</title>
+<form>
+  <label>Account name <input name="name" value="Draft"></label>
+  <label>Industry
+    <select name="industry">
+      <option value="">Select an industry</option>
+      <option value="fin">Financial services</option>
+      <option value="logistics">Logistics</option>
+      <option value="retired" disabled>Retired</option>
+    </select>
+  </label>
+</form>
+<output id="changes">0</output>
+<script>
+  const changes = document.getElementById("changes");
+  document.querySelector("select").addEventListener("change", () => {
+    changes.textContent = String(Number(changes.textContent) + 1);
+  });
+</script>`;
+
+let browser: Browser;
+
+beforeAll(async () => {
+  if (!asked) return;
+  const { chromium } = await import("playwright");
+  browser = await chromium.launch();
+});
+
+afterAll(async () => {
+  await browser?.close();
+});
+
+/** The form, and the refs its AI snapshot gives each field, as the computer reads them. */
+async function openForm(): Promise<{
+  tab: Page;
+  refOf: (name: string) => string;
+  industry: { options?: string[]; value?: string } | undefined;
+}> {
+  const tab = await browser.newPage();
+  await tab.setContent(FORM);
+  const snapshot = await tab.ariaSnapshot({ mode: "ai" });
+  const { elements } = parseAriaSnapshot(snapshot);
+  return {
+    tab,
+    industry: elements.find((element) => element.name === "Industry"),
+    refOf: (name) => {
+      const found = elements.find((element) => element.name === name);
+      if (!found) throw new Error(`No ${name} in the snapshot.`);
+      return found.ref;
+    },
+  };
+}
+
+describe.skipIf(!asked)("entering text in a form", () => {
+  test("a dropdown is chosen from by the option's label, and the page sees a change", async () => {
+    const { tab, refOf } = await openForm();
+    await enterText(
+      tab.locator(`aria-ref=${refOf("Industry")}`),
+      " logistics ",
+      ACTING,
+    );
+    expect(await tab.locator("select").inputValue()).toBe("logistics");
+    expect(await tab.locator("#changes").textContent()).toBe("1");
+    await tab.close();
+  });
+
+  test("a dropdown falls back to the option's value", async () => {
+    const { tab, refOf } = await openForm();
+    await enterText(
+      tab.locator(`aria-ref=${refOf("Industry")}`),
+      "fin",
+      ACTING,
+    );
+    expect(await tab.locator("select").inputValue()).toBe("fin");
+    await tab.close();
+  });
+
+  test("text that is not an option, or names a disabled one, is refused and changes nothing", async () => {
+    const { tab, refOf } = await openForm();
+    const industry = tab.locator(`aria-ref=${refOf("Industry")}`);
+    for (const text of ["Aerospace", "Retired"]) {
+      const refused = await enterText(industry, text, ACTING).catch(
+        (error: unknown) => error,
+      );
+      expect(refused).toBeInstanceOf(FieldValueError);
+      // Never the text it was sent, never the page's own option labels.
+      expect((refused as Error).message).not.toContain(text);
+      expect((refused as Error).message).not.toContain("Logistics");
+    }
+    expect(await tab.locator("select").inputValue()).toBe("");
+    expect(await tab.locator("#changes").textContent()).toBe("0");
+    await tab.close();
+  });
+
+  test("the snapshot the refusal points to lists the dropdown's options and its choice", async () => {
+    const { tab, industry } = await openForm();
+    expect(industry?.options).toEqual([
+      "Select an industry",
+      "Financial services",
+      "Logistics",
+      "Retired",
+    ]);
+    expect(industry?.value).toBe("Select an industry");
+    await tab.close();
+  });
+
+  test("a text box is still filled, replacing what was there", async () => {
+    const { tab, refOf } = await openForm();
+    await enterText(
+      tab.locator(`aria-ref=${refOf("Account name")}`),
+      "Navia Freight",
+      ACTING,
+    );
+    expect(await tab.locator("input[name=name]").inputValue()).toBe(
+      "Navia Freight",
+    );
+    await tab.close();
+  });
+});

From 77a649dfed1fb2b017818380348f66526d8e54fb Mon Sep 17 00:00:00 2001
From: David McKay <david@copilotkit.ai>
Date: Fri, 23 Oct 2026 16:46:23 -0700
Subject: [PATCH 4/6] Answer a failed approved browser step to the Bot instead
 of throwing

A click, typed entry or key press goes through the approval gate, and when the gate answered with a failure the wrapper threw. CopilotKit starts no follow-up run after a frontend tool throws, so the Bot's turn ended on the spot: the step read "Didn't work" and the Bot never spoke again. The wrapper now answers the way callComputer already did for the steps that skip the gate, through the same failedComputerCall: a failed result with the reason, plus the refusal, stale-ref and person-has-control flags the model reads to choose its next step. A request that cannot reach the gate, or an answer with no result, is a failed result too.
---
 app/src/lib/computers/call.ts          | 51 ++++++++++++++++----------
 app/src/lib/copilot/approval-tools.tsx | 40 +++++++++++++-------
 2 files changed, 59 insertions(+), 32 deletions(-)

diff --git a/app/src/lib/computers/call.ts b/app/src/lib/computers/call.ts
index 44ce4f1bca..38357e5696 100644
--- a/app/src/lib/computers/call.ts
+++ b/app/src/lib/computers/call.ts
@@ -76,26 +76,39 @@ export async function callComputer(
      */
     if (response.status === 403 && run !== null)
       withdrawComputerActivity(botId, run);
-    return {
-      ok: false,
-      reason: (body?.error as string) ?? "That did not work.",
-      // Preserve refusal/stale-ref/control distinctions for the model's next step.
-      ...(response.status === 403
-        ? { refused: true, rule: body?.rule ?? null }
-        : {}),
-      ...(response.status === 409
-        ? body?.humanHasControl === true
-          ? {
-              humanHasControl: true,
-              requestId: body?.requestId,
-              handoff: body?.handoff,
-            }
-          : body?.snapshotRequired === true
-            ? { staleRefs: true, snapshotRequired: true }
-            : { staleRefs: true }
-        : {}),
-    };
+    return failedComputerCall(response.status, body);
   }
 
   return { ok: true, ...(body ?? {}) };
 }
+
+/**
+ * What a Bot is told when a computer call came back refused or failed.
+ *
+ * A result, never a thrown error. The SDK starts no follow-up run after a frontend tool throws, so a
+ * throw ends the Bot's turn on the spot with nothing said; a result reaches the model, which can take
+ * a fresh snapshot, pick another field, or tell the person what stopped it. Shared with the approval
+ * gate's wrapper, which reaches the same computer by another route and must answer the same way.
+ */
+export function failedComputerCall(
+  status: number,
+  body: Record<string, unknown> | null,
+): ToolOutcome {
+  return {
+    ok: false,
+    reason: typeof body?.error === "string" ? body.error : "That did not work.",
+    // Preserve refusal/stale-ref/control distinctions for the model's next step.
+    ...(status === 403 ? { refused: true, rule: body?.rule ?? null } : {}),
+    ...(status === 409
+      ? body?.humanHasControl === true
+        ? {
+            humanHasControl: true,
+            requestId: body?.requestId,
+            handoff: body?.handoff,
+          }
+        : body?.snapshotRequired === true
+          ? { staleRefs: true, snapshotRequired: true }
+          : { staleRefs: true }
+      : {}),
+  };
+}
diff --git a/app/src/lib/copilot/approval-tools.tsx b/app/src/lib/copilot/approval-tools.tsx
index c6eeb111e8..ca7014b84e 100644
--- a/app/src/lib/copilot/approval-tools.tsx
+++ b/app/src/lib/copilot/approval-tools.tsx
@@ -7,6 +7,7 @@ import { useQueryClient } from "@tanstack/react-query";
 import type { ReactNode } from "react";
 import { InlineApproval } from "@/components/approvals/inline-approval";
 import { tryClient } from "@/lib/client";
+import { failedComputerCall } from "@/lib/computers/call";
 import { useActiveBotHolder } from "./active-bot";
 import { frontendApprovalSnapshot } from "./approval-context";
 
@@ -69,11 +70,22 @@ export function useFrontendTool<T extends Record<string, unknown>>(
             // run it when nothing asks, and says why when something does.
             return tool.handler?.(args, context);
           }
-          const response = await tryClient(
-            `/api/approvals/computer/${encodeURIComponent(bot.current)}`,
-            { method: "POST", body: snapshot, signal: context.signal },
-          );
-          const payload: unknown = await response.json();
+          let response: Response;
+          try {
+            response = await tryClient(
+              `/api/approvals/computer/${encodeURIComponent(bot.current)}`,
+              { method: "POST", body: snapshot, signal: context.signal },
+            );
+          } catch (error) {
+            // The same two answers callComputer gives when the request itself does not complete.
+            if (error instanceof DOMException && error.name === "AbortError")
+              return { ok: false, reason: "Stopped.", stopped: true };
+            return {
+              ok: false,
+              reason: "The assistant's computer could not be reached.",
+            };
+          }
+          const payload: unknown = await response.json().catch(() => null);
           if (response.status === 202) {
             if (!context.agent)
               throw new Error("The interrupted conversation is unavailable.");
@@ -84,17 +96,19 @@ export function useFrontendTool<T extends Record<string, unknown>>(
             void queryClient.invalidateQueries({ queryKey: ["approvals"] });
             return { waiting: true };
           }
+          // Answered, not thrown: a thrown handler ends the Bot's turn with nothing said (see
+          // failedComputerCall), where a failed result lets it snapshot again or try another way.
           if (!response.ok)
-            throw new Error(
-              payload &&
-                typeof payload === "object" &&
-                "error" in payload &&
-                typeof payload.error === "string"
-                ? payload.error
-                : "The computer action could not be approved.",
+            return failedComputerCall(
+              response.status,
+              payload && typeof payload === "object"
+                ? (payload as Record<string, unknown>)
+                : null,
             );
           if (!payload || typeof payload !== "object" || !("result" in payload))
-            throw new Error("The computer action returned no result.");
+            return failedComputerCall(response.status, {
+              error: "The computer action returned no result.",
+            });
           return payload.result;
         }
       : undefined,

From d95bfcc079e6baa57cdc1e08eb321a741efcdf77 Mon Sep 17 00:00:00 2001
From: David McKay <david@copilotkit.ai>
Date: Fri, 23 Oct 2026 16:46:23 -0700
Subject: [PATCH 5/6] Test that a failed approved step reaches the Bot as a
 result

Drives the handler the SDK keeps against a gate answering 409, 403, a request that cannot connect, and a 200 with no result. Each comes back as a failed result rather than a throw.
---
 app/tests/approval-tools.test.tsx | 118 ++++++++++++++++++++++++++++++
 1 file changed, 118 insertions(+)

diff --git a/app/tests/approval-tools.test.tsx b/app/tests/approval-tools.test.tsx
index 8945763f53..77c881efbc 100644
--- a/app/tests/approval-tools.test.tsx
+++ b/app/tests/approval-tools.test.tsx
@@ -155,3 +155,121 @@ test("a change goes to the server's gate even when nothing about approvals has l
     global.fetch = originalFetch;
   }
 });
+
+/**
+ * Runs the kept computer_click handler against a gate that answers with `answer`.
+ *
+ * The SDK starts no follow-up run after a frontend tool throws (core's executeSpecificTool returns
+ * false on a handler error), so a handler that throws ends the Bot's turn on the spot with nothing
+ * said. A form field the computer could not fill did exactly that: the Bot opened a CRM form, failed
+ * on one dropdown, and the conversation simply stopped.
+ */
+async function clickAgainstGate(
+  answer: () => Promise<Response>,
+): Promise<unknown> {
+  const originalFetch = global.fetch;
+  global.fetch = Object.assign(async () => answer(), {
+    preconnect: originalFetch.preconnect,
+  });
+  try {
+    render(
+      <QueryClientProvider client={new QueryClient()}>
+        <Tools />
+      </QueryClientProvider>,
+    );
+    const tool = registered.find((entry) => entry.name === "computer_click");
+    const toolCall = {
+      id: "click-call",
+      type: "function" as const,
+      function: {
+        name: "computer_click",
+        arguments: '{"ref":"e7","snapshotId":3}',
+      },
+    };
+    const subscribers: {
+      onRunInitialized?: (input: { input: unknown }) => void;
+    }[] = [];
+    const agent = {
+      threadId: "thread",
+      state: {},
+      messages: [
+        { id: "u1", role: "user", content: "Submit the form" },
+        { id: "a1", role: "assistant", content: "", toolCalls: [toolCall] },
+      ],
+      subscribe: (subscriber: (typeof subscribers)[number]) => {
+        subscribers.push(subscriber);
+        return { unsubscribe: () => undefined };
+      },
+    };
+    const stop = observeApprovalAgent(agent as never);
+    for (const subscriber of subscribers)
+      subscriber.onRunInitialized?.({
+        input: {
+          runId: "run-1",
+          threadId: "thread",
+          messages: agent.messages,
+          tools: [],
+          context: [],
+          state: {},
+          forwardedProps: {},
+        },
+      });
+    try {
+      return await tool?.handler?.({ ref: "e7", snapshotId: 3 }, {
+        agent,
+        toolCall,
+      } as never);
+    } finally {
+      stop();
+    }
+  } finally {
+    global.fetch = originalFetch;
+  }
+}
+
+test("a change the computer could not make comes back to the Bot as a result, not a throw", async () => {
+  const result = await clickAgainstGate(async () =>
+    Response.json(
+      {
+        error:
+          "Element e7 is not on the page any more. Take a fresh snapshot and use the refs from it.",
+      },
+      { status: 409 },
+    ),
+  );
+  expect(result).toEqual({
+    ok: false,
+    reason:
+      "Element e7 is not on the page any more. Take a fresh snapshot and use the refs from it.",
+    staleRefs: true,
+  });
+});
+
+test("a refused change comes back as a refusal the Bot can explain", async () => {
+  const result = await clickAgainstGate(async () =>
+    Response.json(
+      { error: "Your team does not allow this.", rule: "team-rule-1" },
+      { status: 403 },
+    ),
+  );
+  expect(result).toEqual({
+    ok: false,
+    reason: "Your team does not allow this.",
+    refused: true,
+    rule: "team-rule-1",
+  });
+});
+
+test("a gate that cannot be reached, or answers with no result, is still a result", async () => {
+  expect(
+    await clickAgainstGate(async () => {
+      throw new TypeError("fetch failed");
+    }),
+  ).toEqual({
+    ok: false,
+    reason: "The assistant's computer could not be reached.",
+  });
+  expect(
+    await clickAgainstGate(async () => Response.json({}, { status: 200 })),
+  ).toEqual({ ok: false, reason: "The computer action returned no result." });
+});

From 20d060f95d849ae1918e81121bec150e830255c1 Mon Sep 17 00:00:00 2001
From: David McKay <david@copilotkit.ai>
Date: Fri, 23 Oct 2026 16:46:23 -0700
Subject: [PATCH 6/6] Note that a Bot can choose from a dropdown and carries on
 after a failed step

---
 CHANGELOG.md | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2cd51b8e0c..3f9bc54a70 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -35,6 +35,23 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
   request for an administrator to switch on a work app, grant it to a Bot, or connect a Shared app's
   team account is kept, with who answered it and when. It starts empty and nothing is backfilled.
 
+### A Bot can choose from a dropdown
+
+A Bot filling in a form in its browser could not set a dropdown (a native `<select>`, such as a CRM's
+Industry or Stage). Typing into one waited until it timed out and was reported as a field that had
+gone from the page, so the Bot took a fresh snapshot and tried the same thing again. Typing into a
+dropdown now chooses the option it names, by the option's label or its value, and the page sees the
+same change a person's choice makes. Text that names no option is refused with a message saying so.
+A page snapshot now lists each dropdown's options and the one it shows, so the Bot can see what it
+may choose and what is already chosen.
+
+### A Bot carries on after a browser step fails
+
+When a click, typed entry or key press the Bot made in its browser failed, the Bot's turn ended on
+the spot: the step showed "Didn't work" and the Bot said nothing more. The failure now reaches the
+Bot like any other result, so it can take a fresh snapshot, try another way, or tell you what
+stopped it. A refused step reads as a refusal the Bot can explain.
+
 ### A Bot can send, draft and reply in Gmail
 
 OpenBot left out every Composio action that takes a file, because a Bot has no way to supply one.