feat(ai-components): bring the AI agent work to V10

.prettierignore

@@ -41,6 +41,9 @@ package/src/theme/generated/
 perf/gauntlet/out/
 perf/gauntlet/runtime/generated/
 
+# A2UI's conformance cases, kept as upstream wrote them
+package/__fixtures__/a2ui-conformance/
+
 # Workflow / tooling configs not owned by this project's Prettier
 .github/
 .husky/

.yarnrc.yml

@@ -22,6 +22,8 @@ npmPreapprovedPackages:
   - '@stream-io/state-store'
   - '@stream-io/worker-timer'
   - '@stream-io/ws'
+  - '@stream-io/calypso'
+  - '@a2ui/web_core'
 
 npmPublishProvenance: true
 

AGENTS.md

@@ -219,6 +219,8 @@ Native code shared by both wrappers lives in `package/shared-native/{ios,android
 - the main entry points must never reach `src/ai-components` or the AI-only packages (`react-syntax-highlighter`, `@khanacademy/simple-markdown`, `zod`, `victory-native`, Skia). `@stream-io/state-store` and `lodash` are shared: the main entry imports them too (the stores in `state-store/`, the Poll contexts);
 - the AI entry points may only reach `src/ai-components` plus an explicit allowlist (`src/native.ts`, the native shimmer). The wrappers' `ai-components/index.js` registers the native pieces the AI code needs itself and must never import the wrappers' `src/index.js`.
 
+`AIConversation` and its list sit behind a third entry point, `ai-components/conversation` (`package/ai-components/conversation/`, `package/{native,expo}-package/ai-components/conversation/`), because `@legendapp/list/keyboard` requires `react-native-keyboard-controller` unguarded. The isolation check keeps the `ai-components` and `a2ui` entry points from reaching `src/ai-components/conversation` or `@legendapp/list/keyboard`, and lets them load keyboard-controller only through the guarded `src/ai-components/internal/keyboard.tsx`. The conversation's state layer (`AIConversationController`, its types, `useAIConversationState`) lives in `src/ai-components/conversation-state` and stays in the plain entry.
+
 Do not add AI slots to `DEFAULT_COMPONENTS`; core `require`s the defaults, which would break isolation. Skia and `victory-native` are optional peers (native libraries must be owned by the app); the JS-only AI dependencies are regular dependencies.
 
 Native pieces live in `package/shared-native/`: the Android-only `PerfText` Fabric view (Kotlin plus a C++ shadow node in `shared-native/android/jni`, built through each wrapper's `android/src/main/jni/CMakeLists.txt`, registered with `cmakeListsPath`/`componentDescriptors` in `react-native.config.js`) and the `AIDictation` TurboModule. **The iOS dictation sources are excluded from the pod unless the app opts in** (`$StreamChatEnableDictation = true` in the Podfile, or the `stream-chat-expo` config plugin in `package/expo-package/plugin`), because referencing the Speech framework makes App Store Connect demand a speech recognition usage description from every app. Never declare `RECORD_AUDIO` in a library manifest. The dictation codegen spec is duplicated into each wrapper's `src/native/` (codegen only reads a package's own `jsSrcsDir`); the isolation check keeps the copies identical.

examples/ExpoChatGPTSample/.env.example

@@ -0,0 +1,15 @@
+# Copy to .env.local (gitignored) and fill in. Public values only: Expo inlines every EXPO_PUBLIC_
+# variable into the app bundle. The Stream secret stays with the token backend.
+
+# The Stream app the Vision-Agents router runs in.
+EXPO_PUBLIC_STREAM_API_KEY=YOUR_STREAM_API_KEY
+
+# The router and its token backend (Vision-Agents' `go run ./backend`). On an Android emulator,
+# `adb reverse tcp:8080 tcp:8080 && adb reverse tcp:8098 tcp:8098` makes localhost reach them.
+EXPO_PUBLIC_AGENT_ROUTER_URL=http://localhost:8080
+EXPO_PUBLIC_AGENT_TOKEN_URL=http://localhost:8098
+
+# The customer and agent `go run ./configure` stored, and who this app is.
+EXPO_PUBLIC_AGENT_CUSTOMER_ID=examples
+EXPO_PUBLIC_AGENT_NAME=swift_assistant
+EXPO_PUBLIC_AGENT_USER_ID=rn-ai-sample-user

examples/ExpoChatGPTSample/README.md

@@ -9,17 +9,31 @@ Ported from the `ExpoChatGPTSample` of the deprecated `@stream-io/chat-react-nat
 
 ## How it works
 
-- The drawer lists the user's conversations; `+` starts a new one.
-- A new conversation is created on its first message. Its title is summarized from that message by the sample backend.
-- Before sending, the app asks the sample backend to add an AI agent to the channel (`http/requests.ts`). The agent
-  replies with streamed `ai_generated` messages, rendered by `StreamingMessageView`.
-
-The Stream app, user and backend in `chatConfig.ts` and `http/requests.ts` are Stream's public demo setup. To use your
-own, replace them and point `http/requests.ts` at your own AI agent server.
+The agent runs on a [Vision-Agents](https://github.com/GetStream/Vision-Agents) router, which the app talks to through
+[`@stream-io/calypso`](https://www.npmjs.com/package/@stream-io/calypso) (`agent/`).
+
+- Each conversation is a router session, kept in an `agent` channel. The drawer lists the user's `agent` channels. `+`
+  starts a new session, and one that is never asked anything is deleted when you leave it.
+- The conversation is `AIConversation`, fed from `agent/agentChat.ts`: a small store that watches the channel, copies
+  its messages on every `message.*` event, loads older pages, and keeps each reply's full reasoning from its live
+  updates. There's no `<Channel>`, so the store also watches the channel again after `connection.recovered`, to pick up
+  what was written while the connection was down.
+- Asking goes through the session, not the chat composer. The router writes the question and the streamed reply into
+  the channel, the reply's steps as `ai_reasoning` and `ai_tool_call` attachments, and titles the conversation itself.
+  A plain question goes to `respond` with this install's id, so the agent can call the phone's tools
+  (`agent/deviceTools.ts`). A question with photos or "Think harder" goes to `responses.create`, which takes neither.
+- The keyboard is handled by the AI components through `react-native-keyboard-controller`; the app only wraps itself
+  in `KeyboardProvider`. The starters and the voice button sit in the composer's `header`, so they ride the keyboard.
 
 ## Running
 
-From the repository root:
+1. Run a Vision-Agents router locally, with its `swift_assistant` agent and token backend, as in
+   [`examples/voice_agents/swift_assistant`](https://github.com/GetStream/Vision-Agents/tree/accelerate/examples/voice_agents/swift_assistant):
+   `docker compose up` (with `ROUTER_AUTH_MODE=proxy`), `go run ./configure`, then `go run ./backend`. The chat model
+   is Grok, so the router needs `XAI_API_KEY`.
+2. Copy `.env.example` to `.env.local` and set `EXPO_PUBLIC_STREAM_API_KEY` to the Stream app the router runs in.
+3. On an Android emulator, forward the ports: `adb reverse tcp:8080 tcp:8080 && adb reverse tcp:8098 tcp:8098`.
+4. From the repository root:
 
 ```bash
 yarn install
@@ -28,3 +42,6 @@ yarn workspace expochatgptsample ios      # or: android
 
 `ios` and `android` sync the SDK's shared native code, run `expo prebuild` and build the app. The native `ios` and
 `android` folders are generated and not committed.
+
+On iOS, if the app crashes at launch in `dyld` with a missing symbol, reinstall the pods without Expo's
+precompiled modules: `cd ios && EXPO_USE_PRECOMPILED_MODULES=0 pod install`, then build again.

examples/ExpoChatGPTSample/agent/UIComponents.json

@@ -0,0 +1,76 @@
+[
+  {
+    "name": "place_card",
+    "description": "One place of a plan, with its photo and a short summary, which the phone loads from Wikipedia, so it needs no image URL. Use one per sight, beach, monastery, village or day trip.",
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "title": {
+          "type": "string",
+          "description": "The place's exact English Wikipedia article title, such as \"Church of St. John at Kaneo\""
+        },
+        "note": {
+          "type": "string",
+          "description": "One short line about it for this plan, such as \"Best at sunset\" or \"2 hours\""
+        }
+      },
+      "required": ["title"],
+      "x-order": ["title", "note"]
+    },
+    "events": [
+      {
+        "name": "ask_about",
+        "description": "Tapping the card asks you about the place."
+      }
+    ]
+  },
+  {
+    "name": "trip_map",
+    "description": "A map of a plan with a pin for each place, drawn on the phone. Use one near the top of every trip plan.",
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "places": {
+          "type": "array",
+          "description": "Every place of the plan, with its coordinates",
+          "items": {
+            "type": "object",
+            "properties": {
+              "name": { "type": "string" },
+              "latitude": { "type": "number" },
+              "longitude": { "type": "number" }
+            },
+            "required": ["name", "latitude", "longitude"],
+            "x-order": ["name", "latitude", "longitude"]
+          }
+        }
+      },
+      "required": ["places"],
+      "x-order": ["places"]
+    },
+    "events": [
+      {
+        "name": "select_place",
+        "description": "Tapping a pin asks you about that place."
+      }
+    ]
+  },
+  {
+    "name": "calendar_check",
+    "description": "The person's own calendar for the days of a plan, read on the phone. You never see it: use it whenever a plan has dates, so they can see what the trip would clash with.",
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "start": {
+          "type": "string",
+          "format": "date",
+          "description": "The first day, as YYYY-MM-DD"
+        },
+        "end": { "type": "string", "format": "date", "description": "The last day, as YYYY-MM-DD" }
+      },
+      "required": ["start", "end"],
+      "x-order": ["start", "end"]
+    },
+    "events": []
+  }
+]

examples/ExpoChatGPTSample/agent/agentChat.ts

@@ -0,0 +1,154 @@
+import { LiveReasoning, type Session } from '@stream-io/calypso';
+import { StateStore } from '@stream-io/state-store';
+import type { Channel, Event, LocalMessage } from 'stream-chat';
+
+/** How many messages are read at a time. */
+const PAGE_SIZE = 30;
+
+export type AgentChatState = {
+  /**
+   * The channel the router keeps the conversation in, once it is followed. Its `messagePaginator`
+   * holds the messages.
+   */
+  channel: Channel | undefined;
+  /** The session the conversation is held in, once something here needed it. */
+  session: Session | undefined;
+};
+
+/**
+ * Opens a conversation's session: carries on `sessionId`, or starts a new conversation without one.
+ */
+export type OpenSession = (sessionId: string | undefined) => Promise<Session>;
+
+/**
+ * A written conversation as the AI components want it: the channel the router keeps it in, and the
+ * whole of each reply's thinking.
+ *
+ * The router writes everything there. Asking goes through the session, which records the question
+ * in the channel and answers it there: the reply's text, and its steps (each round of reasoning and
+ * each tool call) as attachments.
+ *
+ * The channel and the session open apart. Reading a conversation needs only its channel, which
+ * Stream serves in a fraction of the time the session takes to come back, and carrying a session
+ * on marks the conversation active, which moves it up the router's list. So the session opens only
+ * once something needs it, such as a question, and a new conversation has none until it is asked
+ * something: the router never lists one that was only looked at.
+ */
+export class AgentChat {
+  /** Each reply's thinking in full, from its live updates. A stored step only keeps its opening. */
+  readonly reasoning = new LiveReasoning();
+
+  /** Whether the person asked something here, which may not have arrived in `messages` yet. */
+  wasAsked = false;
+
+  readonly state = new StateStore<AgentChatState>({ channel: undefined, session: undefined });
+
+  private stops: (() => void)[] = [];
+  private opening: Promise<Session> | undefined;
+  private closed = false;
+
+  /**
+   * @param key Tells this conversation apart from the one shown before it, such as for a React key.
+   * @param sessionId The conversation's session, or `undefined` for a new conversation.
+   */
+  constructor(
+    readonly key: string,
+    private readonly sessionId: string | undefined,
+    private readonly openSession: OpenSession,
+  ) {}
+
+  /** The conversation's session id: the one it was opened with, or the one asking created. */
+  get id() {
+    return this.state.getLatestValue().session?.id ?? this.sessionId;
+  }
+
+  /** Whether this is a new conversation, which has no messages to wait for. */
+  get isNew() {
+    return this.sessionId === undefined;
+  }
+
+  /** The messages loaded, oldest first. */
+  get messages(): readonly LocalMessage[] {
+    return this.state.getLatestValue().channel?.messagePaginator.state.getLatestValue().items ?? [];
+  }
+
+  /**
+   * Watches the channel and loads its latest messages, as the chat SDK's `<Channel>` does. While it
+   * is followed the channel is active, so the client reloads it after a reconnect.
+   */
+  async follow(channel: Channel) {
+    if (this.state.getLatestValue().channel || this.closed) return;
+    // Every event rather than the paginator's messages: each live update carries its own window of
+    // the reply's thinking, and folding them into one would lose all but the last.
+    const subscription = channel.on((event: Event) => {
+      if (event.type === 'message.updated' && event.message) this.reasoning.read(event.message);
+    });
+    this.stops.push(() => subscription.unsubscribe(), channel.activate());
+    this.state.partialNext({ channel });
+
+    const paginator = channel.messagePaginator;
+    paginator.pageSize = PAGE_SIZE;
+    await channel.watch();
+    if (!paginator.state.getLatestValue().items?.length) await paginator.reload();
+  }
+
+  /**
+   * The conversation's session, opened the first time it is needed. A new conversation is created
+   * then, and its channel followed. Rejects when it can't open, to be asked again.
+   */
+  session(): Promise<Session> {
+    this.opening ??= this.open().catch((error: unknown) => {
+      this.opening = undefined;
+      throw error;
+    });
+    return this.opening;
+  }
+
+  /**
+   * Deletes the session a new conversation was given when the question that created it failed
+   * before anything was written, so the router doesn't list an empty conversation. The next
+   * question creates another.
+   */
+  async discardUnusedSession() {
+    const { session } = this.state.getLatestValue();
+    if (!this.isNew || !session || this.messages.length > 0) return;
+    this.opening = undefined;
+    this.state.partialNext({ session: undefined });
+    await session.delete().catch(() => {});
+  }
+
+  /** Loads the messages before the oldest one loaded. Throws when it can't, to be asked again. */
+  loadOlderMessages = async () => {
+    const paginator = this.state.getLatestValue().channel?.messagePaginator;
+    if (!paginator?.hasMoreTail) return;
+    await paginator.toTail();
+    // The paginator keeps a failure rather than throwing it.
+    if (paginator.lastQueryError) throw paginator.lastQueryError;
+  };
+
+  /** Stops following the channel, and lets the session go. */
+  async close() {
+    this.closed = true;
+    for (const stop of this.stops.splice(0)) stop();
+    const opened = await this.opening?.catch(() => undefined);
+    await opened?.close();
+  }
+
+  private async open() {
+    const session = await this.openSession(this.sessionId);
+    if (this.closed) {
+      // The person moved on while it opened. A conversation created for them was never used.
+      await (this.isNew ? session.delete() : session.close()).catch(() => {});
+      throw new Error('The conversation was closed');
+    }
+    if (!this.state.getLatestValue().channel) {
+      const { channel } = await session.chat();
+      await this.follow(channel as Channel);
+    }
+    this.state.partialNext({ session });
+    return session;
+  }
+}
+
+/** A reply the agent wrote. */
+export const isAgentReply = (message: LocalMessage) => message.custom?.ai_generated === true;