From 3049237799c7e31f85c03b9fe821d570ef3aa3db Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 14:43:01 +0100 Subject: [PATCH 01/16] fix(ai-components): wait before loading older messages again when they fail AIConversationController.loadOlderMessages swallowed the transport's error, so the conversation's list took the load for a success and asked again every 300 ms while the person stayed at the top. It now rejects, and the list waits 3 s before it asks again. --- .../ai-components/__tests__/conversationController.test.ts | 6 +++--- .../controller/AIConversationController.ts | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/package/src/ai-components/__tests__/conversationController.test.ts b/package/src/ai-components/__tests__/conversationController.test.ts index 2139f2bda6..f8608070de 100644 --- a/package/src/ai-components/__tests__/conversationController.test.ts +++ b/package/src/ai-components/__tests__/conversationController.test.ts @@ -1298,14 +1298,14 @@ describe('AIConversationController older messages', () => { expect(transport.loads).toBe(2); }); - it('can load again when the transport fails', async () => { + it('rejects when the transport fails, and can load again', async () => { const transport = new ConversationTransport(); transport.loadError = new TransportError(); const controller = makeController(transport); transport.feed?.updateHasOlderMessages(true); - await controller.loadOlderMessages(); - await controller.loadOlderMessages(); + await expect(controller.loadOlderMessages()).rejects.toBe(transport.loadError); + await expect(controller.loadOlderMessages()).rejects.toBe(transport.loadError); expect(transport.loads).toBe(2); // Not a request of the person's. diff --git a/package/src/ai-components/conversation-state/controller/AIConversationController.ts b/package/src/ai-components/conversation-state/controller/AIConversationController.ts index 3edb4c1d1f..0100067fe9 100644 --- a/package/src/ai-components/conversation-state/controller/AIConversationController.ts +++ b/package/src/ai-components/conversation-state/controller/AIConversationController.ts @@ -313,14 +313,15 @@ export class AIConversationController { this.refresh(); } - /** Loads the messages before the first one, while there are any. */ + /** + * Loads the messages before the first one, while there are any. Rejects when they couldn't load, + * so the conversation waits before it asks again. + */ loadOlderMessages = async () => { if (!this.hasOlderMessages || this.isLoadingOlderMessages) return; this.isLoadingOlderMessages = true; try { await this.transport.loadOlderMessages?.(); - } catch { - // The person scrolls to the start again to try again. } finally { this.isLoadingOlderMessages = false; } From 7c74ef1ba1abd964b5a58f420516cda1210bf95f Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 14:43:16 +0100 Subject: [PATCH 02/16] fix(ai-components): keep turns steady and show a question once A pending question whose server copy arrived in the same update it was first seen in showed twice: the copy counted as a message the question couldn't be. It's now matched against the previous update's messages. A page of older messages that started with a reply took that reply into the turn above it, which changed the turn's key and re-pinned the latest one. A row that started a turn now keeps starting one. --- .../__tests__/conversation.test.tsx | 28 +++++++++++++++++++ .../src/ai-components/conversation/entries.ts | 24 +++++++++++----- .../conversation/useConversationRows.ts | 3 +- 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/package/src/ai-components/__tests__/conversation.test.tsx b/package/src/ai-components/__tests__/conversation.test.tsx index 27c37ee595..1b9e9320f9 100644 --- a/package/src/ai-components/__tests__/conversation.test.tsx +++ b/package/src/ai-components/__tests__/conversation.test.tsx @@ -150,6 +150,34 @@ describe('AIConversation', () => { expect(screen.getAllByText('footer')).toHaveLength(1); }); + it('keeps a turn that starts with a reply when older messages load above it', () => { + const starts = new Set(); + const tracker = new AIPendingMessageTracker(); + const page = [ + reply('a1', 'It is noon.'), + user('u2', 'And the weather?'), + reply('a2', 'Sunny.'), + ]; + expect(turnsOf(tracker.entries(page), starts).map((turn) => turn.id)).toEqual(['a1', 'u2']); + + const older = [user('u1', 'What time is it?'), ...page]; + expect(turnsOf(tracker.entries(older), starts).map((turn) => turn.id)).toEqual([ + 'u1', + 'a1', + 'u2', + ]); + }); + + it('takes the copy of a message on its way that arrives in the update that first passes it', () => { + const tracker = new AIPendingMessageTracker(); + const history = [user('u1', 'Hello'), reply('a1', 'Hi!')]; + tracker.entries(history); + const pending = { id: 'local-1', text: 'Plan my day' }; + const entries = tracker.entries([...history, user('stream-u2', 'Plan my day')], pending); + expect(entries.map((entry) => entry.id)).toEqual(['u1', 'a1', 'local-1']); + expect(entries.filter((entry) => entry.pending)).toHaveLength(0); + }); + it('shows a pending message with the agent thinking, until its reply starts', async () => { const mounts: string[] = []; const UserMessage = ({ text }: AIConversationUserMessageProps) => { diff --git a/package/src/ai-components/conversation/entries.ts b/package/src/ai-components/conversation/entries.ts index 2fa540606e..0157ca943d 100644 --- a/package/src/ai-components/conversation/entries.ts +++ b/package/src/ai-components/conversation/entries.ts @@ -62,12 +62,18 @@ export const hasFinishedSteps = ( message.parts.every(isStepOver) && !(message.uiDocument && canShowUI(message.uiDocument)); -/** Groups the rows into turns, each starting with one of the person's messages. */ -export const turnsOf = (entries: AIConversationEntry[]) => +/** + * Groups the rows into turns, each starting with one of the person's messages. A row that + * started a turn, kept in `starts`, keeps starting one: a reply that opened the first page loaded + * stays the start of its turn when older messages load above it, so the turn keeps its key. + */ +export const turnsOf = (entries: AIConversationEntry[], starts = new Set()) => entries.reduce((turns, entry) => { const last = turns[turns.length - 1]; - if (!last || entry.isFromUser) turns.push({ entries: [entry], id: entry.id }); - else last.entries.push(entry); + if (!last || entry.isFromUser || starts.has(entry.id)) { + starts.add(entry.id); + turns.push({ entries: [entry], id: entry.id }); + } else last.entries.push(entry); return turns; }, []); @@ -85,6 +91,9 @@ export class AIPendingMessageTracker { // The pending message of the last update, whose copy can arrive in the update that stops // passing it. private lastPending?: AIPendingMessage; + // The person's messages of the last update. A pending message first seen in an update that + // also brings its copy can't take that update's messages as ones its copy isn't. + private lastSentIds?: Set; // The rows of a conversation: its messages, apart from replies with nothing to show yet, and // a message on its way until its copy arrives. A reply's UI shows when `canShowUI` says. @@ -94,13 +103,13 @@ export class AIPendingMessageTracker { canShowUI: AIConversationCanShowUI = showsNoUI, ): AIConversationEntry[] { let showsPending = !!pending; + const sent = messages.filter((message) => message.role === 'user'); + const ids = new Set(sent.map((message) => message.id)); // A copy that arrives in the update that stops passing its pending message still takes over // its row. Only then: a message sent later is not the copy of one that failed. const awaited = pending ?? this.lastPending; if (awaited) { - const sent = messages.filter((message) => message.role === 'user'); - const ids = new Set(sent.map((message) => message.id)); - const known = this.seen.get(awaited.id) ?? ids; + const known = this.seen.get(awaited.id) ?? this.lastSentIds ?? ids; const matched = [...this.copies].find( ([copyId, original]) => original.id === awaited.id && ids.has(copyId), )?.[0]; @@ -113,6 +122,7 @@ export class AIPendingMessageTracker { } } this.lastPending = pending ?? undefined; + this.lastSentIds = ids; const entries: AIConversationEntry[] = messages .filter((message) => !showsNothing(message, canShowUI)) diff --git a/package/src/ai-components/conversation/useConversationRows.ts b/package/src/ai-components/conversation/useConversationRows.ts index 61762c879d..fbe1326307 100644 --- a/package/src/ai-components/conversation/useConversationRows.ts +++ b/package/src/ai-components/conversation/useConversationRows.ts @@ -165,7 +165,8 @@ export const useConversationRows = ({ } const thinkingSince = !thinking && stepsKey !== undefined ? gap?.since : undefined; const thinkingReplyId = thinkingSince !== undefined ? lastMessage?.id : undefined; - const turns = useMemo(() => turnsOf(entries), [entries]); + const turnStarts = useRef(new Set()); + const turns = useMemo(() => turnsOf(entries, turnStarts.current), [entries]); const latest = turns[turns.length - 1]; const latestHasReplies = (latest?.entries.length ?? 0) > 1; const latestFirstStepId = firstStepOf(latest); From b05eda92b8228fc08eb0cc521e7bab13ebd4756e Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 14:43:20 +0100 Subject: [PATCH 03/16] fix(ai-components): type replies out as fast as the agent writes them StreamingMessageView typed a fixed 2 letters a frame, so a fast model left it behind, and the reply's actions, which wait for it, came late. Without letterInterval or renderingLetterCount it now types each new part within about 0.4 seconds, at least a letter a frame, as the web SDK does. Passing either keeps a fixed pace. The view and useStreamingMessage also offer skipAnimation, to show the rest at once. --- package/src/ai-components/README.md | 4 +- .../ai-components/StreamingMessageView.tsx | 168 ++++++++++++------ .../__tests__/StreamingMessageView.test.tsx | 45 +++++ .../__tests__/conversation.test.tsx | 2 +- 4 files changed, 160 insertions(+), 59 deletions(-) diff --git a/package/src/ai-components/README.md b/package/src/ai-components/README.md index 044d6e1829..75c6e24d6e 100644 --- a/package/src/ai-components/README.md +++ b/package/src/ai-components/README.md @@ -152,8 +152,8 @@ Under the hood, it implements a letter-by-letter typewriter animation with a cha | `paragraphNumberOfLines` | `number` | no | The `numberOfLines` applied to markdown `Paragraph` and `Text` components. Particularly useful if we want to display the same message, but in a "cut" fashion (for example when replying to someone). | | `rules` | [`MarkdownRules`](./markdown/types.ts) | no | An object of `MarkdownRules` that is then going to be deeply merged with our default rules, based on the [`SimpleMarkdown` parsing engine](https://github.com/Khan/perseus/tree/main/packages/simple-markdown#adding-a-simple-extension). Can be used to add custom rules or change existing rules. You can disable a rule by passing `{ [ruleName]: { match: () => null }}`. | | `onLink` | `(url: string) => void` | no | A function that is going to be invoked whenever a link is pressed within markdown parsed text. | -| `letterInterval` | `number` | no | A number signifying the interval at which the typewriter animation is going to render characters. Defaults to `0` | -| `renderingLetterCount` | `number` | no | A number signifying the number of letters that are going to be rendered per tick of the interval during the typewriter animation. Defaults to `2` | +| `letterInterval` | `number` | no | The milliseconds between steps, for a fixed pace. Without it and `renderingLetterCount`, the text keeps up with what arrives: each new part is typed out within about 0.4 seconds. | +| `renderingLetterCount` | `number` | no | The letters typed each step, for a fixed pace. Defaults to `2` when `letterInterval` is given. | #### Example diff --git a/package/src/ai-components/StreamingMessageView.tsx b/package/src/ai-components/StreamingMessageView.tsx index 24c5b1e37e..65a3df28fb 100644 --- a/package/src/ai-components/StreamingMessageView.tsx +++ b/package/src/ai-components/StreamingMessageView.tsx @@ -1,4 +1,11 @@ -import React, { useEffect, useMemo, useRef, useState } from 'react'; +import React, { + forwardRef, + useEffect, + useImperativeHandle, + useMemo, + useRef, + useState, +} from 'react'; import { Linking } from 'react-native'; @@ -9,15 +16,30 @@ import type { MarkdownRules } from './markdown'; export type UseStreamingMessageProps = { text: string; + /** + * The milliseconds between steps, for a fixed pace. Without it and `renderingLetterCount`, the + * text keeps up with what arrives: each new part is typed out within about 0.4 seconds. + */ letterInterval?: number; + /** The letters typed each step, for a fixed pace. */ renderingLetterCount?: number; }; +// How long text that arrives takes to type out, when no fixed pace is given. +const CATCH_UP_MS = 400; +// How often the text steps on when it keeps up with what arrives: about one frame. +const FRAME_MS = 16; + export const useStreamingMessage = ({ - letterInterval = 0, - renderingLetterCount = 2, + letterInterval, + renderingLetterCount, text, -}: UseStreamingMessageProps): { isWriting: boolean; streamedMessageText: string } => { +}: UseStreamingMessageProps): { + isWriting: boolean; + /** Shows the whole text at once, such as when the person doesn't want to wait for it. */ + skipAnimation: () => void; + streamedMessageText: string; +} => { const [streamedMessageText, setStreamedMessageText] = useState(text); const textCursor = useRef(text.length); // How much of the text is typed out, to say whether it still is. @@ -34,76 +56,110 @@ export const useStreamingMessage = ({ } } const typedToward = useRef(text); + const isFixedPace = letterInterval !== undefined || renderingLetterCount !== undefined; useEffect(() => { const textLength = text.length; if (!text.startsWith(typedToward.current)) textCursor.current = textLength; typedToward.current = text; - const interval = setInterval(() => { - if (!text || textCursor.current >= textLength) { - clearInterval(interval); - } - const newCursorValue = textCursor.current + renderingLetterCount; - const newText = text.substring(0, newCursorValue); - textCursor.current += newText.length - textCursor.current; - const codeBlockCounts = (newText.match(/```/g) || []).length; - const shouldOptimisticallyCloseCodeBlock = codeBlockCounts > 0 && codeBlockCounts % 2 > 0; - setStreamedMessageText(shouldOptimisticallyCloseCodeBlock ? newText + '```' : newText); - setTypedLength(textCursor.current); - }, letterInterval); + const step = isFixedPace + ? (renderingLetterCount ?? 2) + : Math.max(1, Math.ceil(((textLength - textCursor.current) * FRAME_MS) / CATCH_UP_MS)); + const interval = setInterval( + () => { + if (!text || textCursor.current >= textLength) { + clearInterval(interval); + } + const newCursorValue = textCursor.current + step; + const newText = text.substring(0, newCursorValue); + textCursor.current += newText.length - textCursor.current; + const codeBlockCounts = (newText.match(/```/g) || []).length; + const shouldOptimisticallyCloseCodeBlock = codeBlockCounts > 0 && codeBlockCounts % 2 > 0; + setStreamedMessageText(shouldOptimisticallyCloseCodeBlock ? newText + '```' : newText); + setTypedLength(textCursor.current); + }, + isFixedPace ? (letterInterval ?? 0) : FRAME_MS, + ); return () => { clearInterval(interval); }; - }, [letterInterval, renderingLetterCount, text]); + }, [isFixedPace, letterInterval, renderingLetterCount, text]); + + const skipAnimation = useStableCallback(() => { + textCursor.current = text.length; + setStreamedMessageText(text); + setTypedLength(text.length); + }); - return { isWriting: typedLength < text.length, streamedMessageText }; + return { isWriting: typedLength < text.length, skipAnimation, streamedMessageText }; }; -export const StreamingMessageView = ({ - text, - paragraphNumberOfLines, - rules, - onLink: onLinkParam, - letterInterval, - onWritingChange, - renderingLetterCount, -}: { +/** What a `StreamingMessageView`'s ref offers. */ +export type StreamingMessageViewRef = { + /** Shows the whole text at once, such as when the person doesn't want to wait for it. */ + skipAnimation: () => void; +}; + +export type StreamingMessageViewProps = { text: string; paragraphNumberOfLines?: number; rules?: MarkdownRules; onLink?: (url: string) => void; + /** + * The milliseconds between steps, for a fixed pace. Without it and `renderingLetterCount`, the + * text keeps up with what arrives: each new part is typed out within about 0.4 seconds. + */ letterInterval?: number; /** Called when the text starts or stops being typed out, such as to show what follows it once it is. */ onWritingChange?: (isWriting: boolean) => void; + /** The letters typed each step, for a fixed pace. */ renderingLetterCount?: number; -}) => { - const markdownText = useMemo(() => generateMarkdownText(text), [text]); - - const { isWriting, streamedMessageText } = useStreamingMessage({ - text: markdownText ?? '', - letterInterval, - renderingLetterCount, - }); - const reportWriting = useStableCallback((writing: boolean) => onWritingChange?.(writing)); - useEffect(() => { - reportWriting(isWriting); - }, [isWriting, reportWriting]); - - const onLink = useStableCallback((url: string): void | Promise => - onLinkParam - ? onLinkParam(url) - : Linking.canOpenURL(url).then((canOpenUrl) => { - if (canOpenUrl) { - return Linking.openURL(url); - } - return undefined; - }), - ); - - return ( - - {streamedMessageText} - - ); }; + +export const StreamingMessageView = forwardRef( + ( + { + text, + paragraphNumberOfLines, + rules, + onLink: onLinkParam, + letterInterval, + onWritingChange, + renderingLetterCount, + }, + ref, + ) => { + const markdownText = useMemo(() => generateMarkdownText(text), [text]); + + const { isWriting, skipAnimation, streamedMessageText } = useStreamingMessage({ + text: markdownText ?? '', + letterInterval, + renderingLetterCount, + }); + useImperativeHandle(ref, () => ({ skipAnimation }), [skipAnimation]); + const reportWriting = useStableCallback((writing: boolean) => onWritingChange?.(writing)); + useEffect(() => { + reportWriting(isWriting); + }, [isWriting, reportWriting]); + + const onLink = useStableCallback((url: string): void | Promise => + onLinkParam + ? onLinkParam(url) + : Linking.canOpenURL(url).then((canOpenUrl) => { + if (canOpenUrl) { + return Linking.openURL(url); + } + return undefined; + }), + ); + + return ( + + {streamedMessageText} + + ); + }, +); + +StreamingMessageView.displayName = 'StreamingMessageView'; diff --git a/package/src/ai-components/__tests__/StreamingMessageView.test.tsx b/package/src/ai-components/__tests__/StreamingMessageView.test.tsx index eb8be0fd01..86def4d0b0 100644 --- a/package/src/ai-components/__tests__/StreamingMessageView.test.tsx +++ b/package/src/ai-components/__tests__/StreamingMessageView.test.tsx @@ -21,6 +21,51 @@ describe('ai-components streaming', () => { }); describe('useStreamingMessage', () => { + it('keeps up with what arrives when no pace is given', () => { + const { result, rerender } = renderHook( + ({ text }: { text: string }) => useStreamingMessage({ text }), + { initialProps: { text: '' } }, + ); + const chunk = 'x'.repeat(250); + rerender({ text: chunk }); + act(() => { + jest.advanceTimersByTime(16); + }); + // 250 letters in 0.4 seconds: 10 a frame. + expect(result.current.streamedMessageText).toHaveLength(10); + expect(result.current.isWriting).toBe(true); + + act(() => { + jest.advanceTimersByTime(420); + }); + expect(result.current.streamedMessageText).toBe(chunk); + expect(result.current.isWriting).toBe(false); + }); + + it('types at least a letter a frame when little arrives', () => { + const { result, rerender } = renderHook( + ({ text }: { text: string }) => useStreamingMessage({ text }), + { initialProps: { text: 'Hi' } }, + ); + rerender({ text: 'Hi there' }); + act(() => { + jest.advanceTimersByTime(16); + }); + expect(result.current.streamedMessageText).toBe('Hi '); + }); + + it('shows the whole text at once when the animation is skipped', () => { + const { result, rerender } = renderHook( + ({ text }: { text: string }) => + useStreamingMessage({ letterInterval: 50, renderingLetterCount: 1, text }), + { initialProps: { text: '' } }, + ); + rerender({ text: 'A long answer' }); + act(() => result.current.skipAnimation()); + expect(result.current.streamedMessageText).toBe('A long answer'); + expect(result.current.isWriting).toBe(false); + }); + it('renders the initial text immediately and animates text appended later', () => { const { result, rerender } = renderHook( ({ text }: { text: string }) => diff --git a/package/src/ai-components/__tests__/conversation.test.tsx b/package/src/ai-components/__tests__/conversation.test.tsx index 1b9e9320f9..2a4af40124 100644 --- a/package/src/ai-components/__tests__/conversation.test.tsx +++ b/package/src/ai-components/__tests__/conversation.test.tsx @@ -718,7 +718,7 @@ describe('AIConversation with a controller', () => { for (const text of ['Here is', 'Here is your', 'Here is your plan.']) { await update([...history, reply('a2', text, { isGenerating: true })]); } - expect(screen.getByText('Here is your plan.')).toBeTruthy(); + expect(await screen.findByText('Here is your plan.')).toBeTruthy(); expect(renders).toBe(before); controller.dispose(); }); From ab25cdbd9cb37edbabc3245e863492eeb2274cef Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 14:43:35 +0100 Subject: [PATCH 04/16] fix(ai-components): leave client tool calls without their own id alone A step without an id is told apart by its place in the reply, such as ai_tool_call-0, which another reply repeats, so a later call looked already sent and never ran. The runner now skips such calls, as Android's does. A maxAttempts below 1, which silently never ran anything, now throws. --- .../__tests__/clientToolRunner.test.ts | 19 ++++++++++++++++++ .../tool-calls/AIClientToolRunner.ts | 20 +++++++++++++++---- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/package/src/ai-components/__tests__/clientToolRunner.test.ts b/package/src/ai-components/__tests__/clientToolRunner.test.ts index 0d4efa0fe0..14b4309f67 100644 --- a/package/src/ai-components/__tests__/clientToolRunner.test.ts +++ b/package/src/ai-components/__tests__/clientToolRunner.test.ts @@ -68,6 +68,25 @@ describe('AIClientToolRunner', () => { }); }); + it('leaves a call without an id of its own alone', async () => { + const tool = countingTool(); + const runner = runnerWith([tool]); + const send = jest.fn, Parameters>(() => + Promise.resolve(), + ); + + runner.run([step({ id: undefined })], send); + await settle(); + + expect(tool.run).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + }); + + it('needs at least one attempt', () => { + expect(() => runnerWith([countingTool()], 0)).toThrow(RangeError); + expect(() => runnerWith([countingTool()], 1)).not.toThrow(); + }); + it('sends a result that could not be sent again, without running the tool again', async () => { const tool = countingTool(); const runner = runnerWith([tool]); diff --git a/package/src/ai-components/tool-calls/AIClientToolRunner.ts b/package/src/ai-components/tool-calls/AIClientToolRunner.ts index 98c8599d1c..46ed37d247 100644 --- a/package/src/ai-components/tool-calls/AIClientToolRunner.ts +++ b/package/src/ai-components/tool-calls/AIClientToolRunner.ts @@ -98,10 +98,16 @@ export type AIClientToolRunnerOptions = { tools: AIClientTool[]; /** The person signed in, matched against a call's `target_user_id`. */ userId: string; - /** How many times a result is offered to `send` before the runner gives up on it. Defaults to 3. */ + /** + * How many times a result is offered to `send` before the runner gives up on it. At least 1; + * defaults to 3. + */ maxAttempts?: number; }; +const hasOwnId = (payload: Record) => + typeof payload.id === 'string' && payload.id !== ''; + type CallState = { attempts: number; result?: AIClientToolResult; @@ -114,7 +120,8 @@ type CallState = { * as they update. It runs a call when the call awaits this person and this install, the runner * has a tool of that name and the call has not run here before, then hands the result to `send`. * A result that could not be sent is sent again on a later update, without running the tool - * again. + * again. A call without an `id` of its own is left alone: its step is told apart by its place in + * the reply, which another reply repeats. * * ```ts * const runner = new AIClientToolRunner({ clientId: AIClientIdentity.installId(), tools: [localTime], userId }); @@ -129,6 +136,9 @@ export class AIClientToolRunner { private readonly tools = new Map(); constructor({ clientId, maxAttempts = 3, tools, userId }: AIClientToolRunnerOptions) { + if (!Number.isInteger(maxAttempts) || maxAttempts < 1) { + throw new RangeError(`maxAttempts must be at least 1, was ${maxAttempts}`); + } this.clientId = clientId; this.maxAttempts = maxAttempts; this.userId = userId; @@ -162,8 +172,10 @@ export class AIClientToolRunner { /** Runs the calls among `parts` that await this device, and sends their results. */ run(parts: readonly AIMessagePartData[], send: AIClientToolSend) { - for (const { toolCall: call } of parts) { - if (!call || !isAwaitingClient(call, this.userId, this.clientId)) continue; + for (const { payload, toolCall: call } of parts) { + if (!call || !hasOwnId(payload) || !isAwaitingClient(call, this.userId, this.clientId)) { + continue; + } const tool = this.tools.get(call.name); if (!tool) continue; const state = this.calls.get(call.id) ?? { attempts: 0, sending: false, sent: false }; From 6a211efe439fe780b9eee2b01a81443ff1f2a545 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 14:43:40 +0100 Subject: [PATCH 05/16] fix(ai-components): describe the app's tools to the backend as MCP tools AIUIHost.capabilities() sent the tools' definitions as given, and an empty list without tools. It now sends them as the web SDK does, as Model Context Protocol tools with an inputSchema ({ type: 'object' } when there is none) and a boolean readOnlyHint, and leaves tools out when there are none. aiUIClientToolJSON is exported for backends that write the capabilities themselves. --- .../__tests__/generativeUI.test.ts | 35 ++++++++++++++++++- .../ai-components/generative-ui/AIUIHost.ts | 20 +++++++++-- .../src/ai-components/generative-ui/index.ts | 8 ++++- 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/package/src/ai-components/__tests__/generativeUI.test.ts b/package/src/ai-components/__tests__/generativeUI.test.ts index 3c9960e69c..bb3b7642a0 100644 --- a/package/src/ai-components/__tests__/generativeUI.test.ts +++ b/package/src/ai-components/__tests__/generativeUI.test.ts @@ -20,6 +20,7 @@ import { parseAIUIComponentDefinition, parseAIUIDocument, requiredAIUIParameter, + aiUIClientToolJSON, } from '../index'; import type { AIMessage } from '../index'; @@ -602,11 +603,43 @@ describe('AIUIHost', () => { { catalog_id: 'app', components: 1, format: 'a2ui' }, { format: 'openui', instructions: 'Write OpenUI.' }, ], - tools: [], v: 1, }); expect(ui.component('place_card')?.definition).toBe(definition); }); + + it('tells the backend the app’s tools as Model Context Protocol tools', async () => { + const ui = new AIUIHost({ + renderers: [], + send: jest.fn(), + tools: [ + { definition: { name: 'local_time' }, run: () => ({ output: '12:00' }) }, + { + definition: { + annotations: { readOnlyHint: true }, + description: 'The weather in a city.', + inputSchema: { properties: { city: { type: 'string' } }, type: 'object' }, + name: 'weather', + }, + run: () => ({ output: 'Sunny' }), + }, + ], + }); + expect((await ui.capabilities()).tools).toEqual([ + { annotations: { readOnlyHint: false }, inputSchema: { type: 'object' }, name: 'local_time' }, + { + annotations: { readOnlyHint: true }, + description: 'The weather in a city.', + inputSchema: { properties: { city: { type: 'string' } }, type: 'object' }, + name: 'weather', + }, + ]); + expect(aiUIClientToolJSON({ name: 'x' })).toEqual({ + annotations: { readOnlyHint: false }, + inputSchema: { type: 'object' }, + name: 'x', + }); + }); }); describe('utf8ByteLength', () => { diff --git a/package/src/ai-components/generative-ui/AIUIHost.ts b/package/src/ai-components/generative-ui/AIUIHost.ts index c257521dcd..fe779bc1d1 100644 --- a/package/src/ai-components/generative-ui/AIUIHost.ts +++ b/package/src/ai-components/generative-ui/AIUIHost.ts @@ -74,6 +74,23 @@ const openWebOrEmailLink = (url: string) => { Linking.openURL(url.trim()).catch(() => undefined); }; +/** + * A tool of the app's as a Model Context Protocol tool, as the agent's backend reads it: its + * `name`, `description`, the `inputSchema` of its arguments (an object with no properties when it + * has none) and whether it only reads. + */ +export const aiUIClientToolJSON = ({ + annotations, + description, + inputSchema, + name, +}: AIClientToolDefinition) => ({ + annotations: { readOnlyHint: annotations?.readOnlyHint === true }, + ...(description ? { description } : {}), + inputSchema: inputSchema ?? { type: 'object' }, + name, +}); + /** * The generated UI of one conversation: the renderers of the formats the agent writes, the * app's components and tools, what each reply's UI was left in, and where what the person does @@ -334,8 +351,7 @@ export class AIUIHost { async capabilities() { return { components: this.components.map(({ definition }) => aiUIComponentDefinitionJSON(definition)), - // The app's tools as Model Context Protocol tools. - tools: this.toolDefinitions, + ...(this.tools.size > 0 ? { tools: this.toolDefinitions.map(aiUIClientToolJSON) } : {}), renderers: await Promise.all( this.renderers.map(async (renderer) => ({ ...(await renderer.capabilities?.(this.components, this.toolDefinitions)), diff --git a/package/src/ai-components/generative-ui/index.ts b/package/src/ai-components/generative-ui/index.ts index 9cd8a502ee..c974e3831e 100644 --- a/package/src/ai-components/generative-ui/index.ts +++ b/package/src/ai-components/generative-ui/index.ts @@ -37,7 +37,13 @@ export { parseAIUIDocument, } from './AIUIDocument'; export { AIUICascade, AIUIEntrance, AIUIEntranceStage, useAIUIEntrance } from './AIUIEntrance'; -export { AIUIHost, type AIUIHostOptions, AIUIToolError, isWebOrEmailLink } from './AIUIHost'; +export { + AIUIHost, + type AIUIHostOptions, + AIUIToolError, + aiUIClientToolJSON, + isWebOrEmailLink, +} from './AIUIHost'; export { aiUIMediaSource, type AIUIMediaURL, From 1289dab6980928d050606c9df883fb589d467fc1 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 14:43:45 +0100 Subject: [PATCH 06/16] fix(ai-components): tell a custom message whether its reply shows UI or failed to A custom AssistantMessage couldn't tell a reply that answers with UI, or whose UI failed to show, from one that answered with nothing. It now gets showsUI and uiFailed, as on the web, and AIUIHost has hasFailed(messageId). --- .../__tests__/conversationUI.test.tsx | 32 ++++++++++++++++++- .../conversation/AIConversation.tsx | 7 ++++ .../ai-components/generative-ui/AIUIHost.ts | 5 +++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/package/src/ai-components/__tests__/conversationUI.test.tsx b/package/src/ai-components/__tests__/conversationUI.test.tsx index 9c1a65a691..f832674c59 100644 --- a/package/src/ai-components/__tests__/conversationUI.test.tsx +++ b/package/src/ai-components/__tests__/conversationUI.test.tsx @@ -3,7 +3,7 @@ import { Pressable, StyleSheet, Text, View } from 'react-native'; import { act, fireEvent, render, screen } from '@testing-library/react-native'; -import { AIConversation } from '../conversation'; +import { AIConversation, type AIConversationAssistantMessageProps } from '../conversation'; import { AIConversationController, type AIMessage, @@ -327,9 +327,39 @@ describe('AIConversation with generated UI', () => { expect(screen.getByTestId('card')).toHaveTextContent('Ohrid plan'); expect(onRenderError).toHaveBeenCalledTimes(1); expect(onRenderError).toHaveBeenCalledWith(expect.any(Error), 'a1'); + expect(ui.hasFailed('a1')).toBe(true); + expect(ui.hasFailed('a2')).toBe(false); logged.mockRestore(); }); + it('tells a custom message whether its reply shows UI, or whose UI failed', async () => { + const seen = new Map(); + const AssistantMessage = ({ + message, + showsUI, + text, + uiFailed, + }: AIConversationAssistantMessageProps) => { + seen.set(message.id, { showsUI, uiFailed }); + return {text}; + }; + const ui = makeHost(); + await renderConversation( + , + ); + expect(seen.get('a1')).toEqual({ showsUI: true, uiFailed: false }); + expect(seen.get('a2')).toEqual({ showsUI: false, uiFailed: false }); + }); + it('tries a reply’s UI that failed while it was written again once it is written', async () => { const Throws = () => { throw new Error('Half a map'); diff --git a/package/src/ai-components/conversation/AIConversation.tsx b/package/src/ai-components/conversation/AIConversation.tsx index 58ea977925..3a5971586f 100644 --- a/package/src/ai-components/conversation/AIConversation.tsx +++ b/package/src/ai-components/conversation/AIConversation.tsx @@ -64,6 +64,10 @@ export type AIConversationAssistantMessageProps = { */ onWritingChange?: (isWriting: boolean) => void; reasoningText?: (stepId: string) => string | undefined; + /** Whether the reply answers with UI the agent generated, which shows under it in place of its text. */ + showsUI?: boolean; + /** Whether the reply's generated UI failed to show, so its text shows instead. */ + uiFailed?: boolean; }; export type AIConversationReplyActionsProps = { @@ -283,6 +287,7 @@ const EntryView = ({ const { uiDocument } = message; const generated = ui && uiDocument ? ui.content(uiDocument, message.id) : null; const showsUI = !!uiDocument && generated !== null && generated !== undefined; + const uiFailed = !!uiDocument && !!ui?.hasFailed(message.id); // A finished reply has its actions under it whether it answers in text or in UI, once the // UI is written. const hasAnswer = message.text !== '' || showsUI; @@ -297,7 +302,9 @@ const EntryView = ({ onWritingChange={onWritingChange} parts={message.parts ?? []} reasoningText={message.reasoningText} + showsUI={showsUI} text={showsUI ? '' : message.text} + uiFailed={uiFailed} /> {showsUI ? ( Date: Wed, 11 Nov 2026 14:44:00 +0100 Subject: [PATCH 07/16] fix(ai-components): show what a tool call sends before the user allows it The approval card asked only the agent's question, so the user allowed a call without seeing what it would send. As on the web, it now shows the call's arguments as indented JSON under "What it sends", folded when long. The approver's argumentsPreview(call) gives a view of its own, or null for none, AIToolApprovalCard takes the preview as argumentsPreview, and AIToolApproval's render function gets it as its 4th argument. AIToolArguments is the default view. --- .../__tests__/messagePartViews.test.tsx | 53 ++++++++- .../message-parts/AIToolApproval.tsx | 109 +++++++++++++++++- .../message-parts/toolArguments.ts | 22 ++++ package/src/ai-components/theme.ts | 6 + .../tool-calls/AIToolApprover.tsx | 7 +- 5 files changed, 190 insertions(+), 7 deletions(-) create mode 100644 package/src/ai-components/message-parts/toolArguments.ts diff --git a/package/src/ai-components/__tests__/messagePartViews.test.tsx b/package/src/ai-components/__tests__/messagePartViews.test.tsx index 773baee37a..5601a9788b 100644 --- a/package/src/ai-components/__tests__/messagePartViews.test.tsx +++ b/package/src/ai-components/__tests__/messagePartViews.test.tsx @@ -1,5 +1,5 @@ import React, { type ReactElement } from 'react'; -import { I18nManager } from 'react-native'; +import { I18nManager, Text } from 'react-native'; import { act, fireEvent, render, screen } from '@testing-library/react-native'; @@ -124,6 +124,57 @@ describe('AIToolApproval', () => { userId: 'alex', }); + it('shows what the call sends, folding long arguments', () => { + const decide = jest.fn(() => Promise.resolve()); + const { rerender } = renderWithTheme( + , + ); + expect(screen.getByText('What it sends')).toBeTruthy(); + expect(screen.getByText(/"precision": "city"/)).toBeTruthy(); + + const long = { body: 'x'.repeat(300) }; + rerender(); + expect(screen.queryByText(/"body"/)).toBeNull(); + fireEvent.press(screen.getByRole('button', { name: 'What it sends' })); + expect(screen.getByText(/"body"/)).toBeTruthy(); + + rerender(); + expect(screen.queryByText('What it sends')).toBeNull(); + }); + + it('shows the approver’s own preview, or none, and hands it to a card of the app’s', () => { + const decide = jest.fn(() => Promise.resolve()); + const part = { ...asking, arguments: { precision: 'city' } }; + const { rerender } = renderWithTheme( + Your city only }} + part={part} + />, + ); + expect(screen.getByText('Your city only')).toBeTruthy(); + expect(screen.queryByText('What it sends')).toBeNull(); + + rerender( + null }} + part={part} + />, + ); + expect(screen.queryByText('What it sends')).toBeNull(); + + const children = jest.fn((_approval, _state, _decide, preview) => <>{preview}); + rerender( + + {children} + , + ); + expect(screen.getByText('What it sends')).toBeTruthy(); + expect(children.mock.calls[0]).toHaveLength(4); + }); + it('asks only the targeted person on the targeted install', () => { const decide = jest.fn(() => Promise.resolve()); const { rerender } = renderWithTheme(); diff --git a/package/src/ai-components/message-parts/AIToolApproval.tsx b/package/src/ai-components/message-parts/AIToolApproval.tsx index 1df70ff3f9..ee4df26adb 100644 --- a/package/src/ai-components/message-parts/AIToolApproval.tsx +++ b/package/src/ai-components/message-parts/AIToolApproval.tsx @@ -1,11 +1,13 @@ -import React, { type ReactNode, useEffect } from 'react'; -import { ActivityIndicator, Pressable, StyleSheet, Text, View } from 'react-native'; +import React, { type ReactNode, useEffect, useState } from 'react'; +import { ActivityIndicator, Platform, Pressable, StyleSheet, Text, View } from 'react-native'; import { callStateOf, setCallState, useCallState } from './callStates'; import { isAwaitingApproval } from './parseMessageParts'; +import { formatToolArguments } from './toolArguments'; import type { AIToolApprovalData, AIToolCallPart } from './types'; import { useAITranslation, useTheme } from '../contexts'; +import { useAIIcons } from '../icons'; import { primitives } from '../internal/tokens'; import { type AIToolApprover, useAIToolApprover } from '../tool-calls/AIToolApprover'; @@ -31,10 +33,77 @@ export const approvalLines = (approval: AIToolApprovalData) => { return lines; }; +// Arguments longer than this start folded, so the question and its buttons stay in view. +const LONG_ARGUMENTS_CHARACTERS = 280; +const LONG_ARGUMENTS_LINES = 6; + +export type AIToolArgumentsProps = { + /** The call's arguments, such as `part.arguments`. */ + args: unknown; +}; + +/** + * What a tool call sends, so the person sees it before they allow the call: its arguments as + * indented JSON, under "What it sends", folded when they are long. Nothing for a call that + * sends nothing. + */ +export const AIToolArguments = ({ args }: AIToolArgumentsProps) => { + const { t } = useAITranslation(); + const { + theme: { colors, messageParts }, + } = useTheme(); + const { Chevron } = useAIIcons(); + const card = messageParts?.approvalCard; + const text = formatToolArguments(args); + const isLong = + !!text && + (text.length > LONG_ARGUMENTS_CHARACTERS || text.split('\n').length > LONG_ARGUMENTS_LINES); + const [isOpen, setIsOpen] = useState(false); + if (!text) return null; + + const label = t('What it sends'); + const code = ( + + {text} + + ); + return ( + + {isLong ? ( + setIsOpen((open) => !open)} + style={styles.argumentsHeader} + > + + {label} + + + + + + ) : ( + + {label} + + )} + {!isLong || isOpen ? code : null} + + ); +}; + export type AIToolApprovalCardProps = { approval: AIToolApprovalData; /** Called with the person's answer. */ onDecide: (allowed: boolean) => void; + /** What the call sends, shown under the question, such as an `AIToolArguments`. */ + argumentsPreview?: ReactNode; state?: AIToolApprovalState; }; @@ -45,6 +114,7 @@ export type AIToolApprovalCardProps = { */ export const AIToolApprovalCard = ({ approval, + argumentsPreview, onDecide, state = idle, }: AIToolApprovalCardProps) => { @@ -72,6 +142,7 @@ export const AIToolApprovalCard = ({ ))} + {argumentsPreview ?? null} void, + argumentsPreview: ReactNode, ) => ReactNode; }; @@ -158,18 +233,42 @@ export const AIToolApproval = ({ approver: approverProp, children, part }: AIToo .catch(() => setCallState(key, { failed: true, isSending: false }, idle)); }; + const argumentsPreview = toolArgumentsPreview(part, approver); + return ( <> {children ? ( - children(part.approval, state, decide) + children(part.approval, state, decide, argumentsPreview) ) : ( - + )} ); }; +// What the call sends, as the approver previews it, else its arguments. +const toolArgumentsPreview = (call: AIToolCallPart, approver: AIToolApprover): ReactNode => { + const own = approver.argumentsPreview?.(call); + if (own !== undefined) return own; + return formatToolArguments(call.arguments) === undefined ? null : ( + + ); +}; + const styles = StyleSheet.create({ + arguments: { borderRadius: 12, gap: 6, paddingHorizontal: 10, paddingVertical: 8 }, + argumentsCode: { + fontFamily: Platform.OS === 'ios' ? 'Courier' : 'monospace', + fontSize: primitives.typographyFontSizeXs, + }, + argumentsHeader: { alignItems: 'center', flexDirection: 'row', gap: 6 }, + argumentsLabel: { fontSize: primitives.typographyFontSizeXs, fontWeight: '600' }, + chevronOpen: { transform: [{ rotate: '90deg' }] }, actions: { alignItems: 'center', flexDirection: 'row', flexWrap: 'wrap', gap: 8 }, button: { alignItems: 'center', diff --git a/package/src/ai-components/message-parts/toolArguments.ts b/package/src/ai-components/message-parts/toolArguments.ts new file mode 100644 index 0000000000..338219ec76 --- /dev/null +++ b/package/src/ai-components/message-parts/toolArguments.ts @@ -0,0 +1,22 @@ +/** + * A tool call's arguments as the person reads them before they allow the call: JSON, indented, + * or the text as it came when it isn't JSON. Nothing for a call that sends nothing. + */ +export const formatToolArguments = (args: unknown): string | undefined => { + if (args === undefined || args === null || args === '') return undefined; + if (typeof args === 'string') { + try { + return formatToolArguments(JSON.parse(args)); + } catch { + return args; + } + } + if (typeof args === 'object' && !Array.isArray(args) && Object.keys(args).length === 0) { + return undefined; + } + try { + return JSON.stringify(args, null, 2); + } catch { + return String(args); + } +}; diff --git a/package/src/ai-components/theme.ts b/package/src/ai-components/theme.ts index 33e7bc372c..bed34f32d2 100644 --- a/package/src/ai-components/theme.ts +++ b/package/src/ai-components/theme.ts @@ -103,6 +103,9 @@ export type Theme = { declineButton?: ViewStyle; declineButtonText?: TextStyle; failure?: TextStyle; + arguments?: ViewStyle; + argumentsLabel?: TextStyle; + argumentsCode?: TextStyle; }; authorizationCard?: { container?: ViewStyle; @@ -388,6 +391,9 @@ export const defaultTheme: Theme = { declineButton: {}, declineButtonText: {}, failure: {}, + arguments: {}, + argumentsLabel: {}, + argumentsCode: {}, }, authorizationCard: { container: {}, diff --git a/package/src/ai-components/tool-calls/AIToolApprover.tsx b/package/src/ai-components/tool-calls/AIToolApprover.tsx index d8a28f9110..7aacdbbcb4 100644 --- a/package/src/ai-components/tool-calls/AIToolApprover.tsx +++ b/package/src/ai-components/tool-calls/AIToolApprover.tsx @@ -1,4 +1,4 @@ -import React, { createContext, type PropsWithChildren, useContext } from 'react'; +import React, { createContext, type PropsWithChildren, type ReactNode, useContext } from 'react'; import type { AIToolCallPart } from '../message-parts/types'; @@ -41,6 +41,11 @@ export type AIToolApprover = { userId: string; /** How this person signs in when a call waits for their account. Without one, no sign-in is offered here. */ authorizer?: AIToolAuthorizer; + /** + * What a call sends, shown under its question, such as a formatted email instead of its JSON. + * `null` shows nothing; `undefined` shows the call's arguments. + */ + argumentsPreview?: (call: AIToolCallPart) => ReactNode | null | undefined; }; const AIToolApproverContext = createContext(undefined); From 6c9724a4b3d04541cf1891572997442c5dad3b7b Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 14:44:04 +0100 Subject: [PATCH 08/16] fix(ai-components): offer copy and share only under replies with text A reply that answers only with UI or attachments showed copy and share, and sharing it shared an empty string. Both now show only when the reply has text, as on iOS, and the actions render nothing when there's nothing to offer. --- .../__tests__/conversationUI.test.tsx | 22 ++++++++++++++++-- .../__tests__/messageViews.test.tsx | 14 +++++++++++ .../ai-components/message/AIReplyActions.tsx | 23 +++++++++++-------- 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/package/src/ai-components/__tests__/conversationUI.test.tsx b/package/src/ai-components/__tests__/conversationUI.test.tsx index f832674c59..fa8f7ab828 100644 --- a/package/src/ai-components/__tests__/conversationUI.test.tsx +++ b/package/src/ai-components/__tests__/conversationUI.test.tsx @@ -191,16 +191,18 @@ describe('AIConversation with generated UI', () => { it('has the reply’s actions under a finished UI-only reply, not while its UI is written', async () => { const ui = makeHost(); const question = user('u1', 'Plan'); + const ReplyActions = () => Rate this plan; const { rerender } = await renderConversation( , ); - expect(screen.queryByRole('button', { name: /copy/i })).toBeNull(); + expect(screen.queryByText('Rate this plan')).toBeNull(); await rerender( { question, reply('a1', '', { content: 'Ohrid', format: 'a2ui', status: 'complete' }), ]} + ReplyActions={ReplyActions} ui={ui} />, ); - expect(screen.getByRole('button', { name: /copy/i })).toBeTruthy(); + expect(screen.getByText('Rate this plan')).toBeTruthy(); + }); + + it('offers no Copy or Share under a reply that answers only in UI', async () => { + await renderConversation( + , + ); + expect(screen.getByTestId('card')).toBeTruthy(); + expect(screen.queryByRole('button', { name: /copy/i })).toBeNull(); + expect(screen.queryByRole('button', { name: /share/i })).toBeNull(); }); it('keeps the UI views of rows it reuses for other replies', async () => { diff --git a/package/src/ai-components/__tests__/messageViews.test.tsx b/package/src/ai-components/__tests__/messageViews.test.tsx index bfe9d12f73..1931b7e692 100644 --- a/package/src/ai-components/__tests__/messageViews.test.tsx +++ b/package/src/ai-components/__tests__/messageViews.test.tsx @@ -208,6 +208,20 @@ describe('AIReplyActions', () => { expect(screen.getByRole('button', { name: 'Copy' })).toBeTruthy(); }); + it('offers no Copy or Share for a reply without text', () => { + const onRegenerate = jest.fn(); + const { rerender } = renderWithTheme(); + expect(screen.queryByRole('button', { name: 'Copy' })).toBeNull(); + expect(screen.queryByRole('button', { name: 'Share' })).toBeNull(); + expect(screen.getByRole('button', { name: 'Regenerate' })).toBeTruthy(); + rerender( + + + , + ); + expect(screen.queryByTestId('ai-reply-actions')).toBeNull(); + }); + it('shares the text through the system share sheet', () => { const share = jest .spyOn(Share, 'share') diff --git a/package/src/ai-components/message/AIReplyActions.tsx b/package/src/ai-components/message/AIReplyActions.tsx index 4b44df4527..ce032f17a4 100644 --- a/package/src/ai-components/message/AIReplyActions.tsx +++ b/package/src/ai-components/message/AIReplyActions.tsx @@ -71,15 +71,18 @@ export const AIReplyActions = ({ onRate, onRegenerate, rating, text }: AIReplyAc setIsCopied(false); } + // A reply that answers only in UI or attachments has no text to copy or share. + const hasText = text.trim() !== ''; const setString = setClipboardString; - const copy = setString - ? () => { - setString(text); - setIsCopied(true); - clearTimeout(timer.current); - timer.current = setTimeout(() => setIsCopied(false), COPIED_MS); - } - : undefined; + const copy = + setString && hasText + ? () => { + setString(text); + setIsCopied(true); + clearTimeout(timer.current); + timer.current = setTimeout(() => setIsCopied(false), COPIED_MS); + } + : undefined; // The person closing the share sheet resolves too; there's nothing to do either way. const share = () => { @@ -89,6 +92,8 @@ export const AIReplyActions = ({ onRate, onRegenerate, rating, text }: AIReplyAc const positive = rating === 'positive'; const negative = rating === 'negative'; + if (!hasText && !onRegenerate && !onRate) return null; + return ( ) : null} - + {hasText ? : null} {onRegenerate ? ( ) : null} From a7502816adb6826d1619d3d4a68be1b1dceeba7f Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 14:44:09 +0100 Subject: [PATCH 09/16] fix(ai-components): offer Connect again when a sign-in never settles Once a sign-in came back connected, the card said "Connecting..." until the agent changed the call's step, which it may never do. After 30 seconds it now offers Connect again, as Android does. --- .../src/ai-components/__tests__/signIn.test.tsx | 12 +++++++++++- .../message-parts/AIToolAuthorization.tsx | 17 ++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/package/src/ai-components/__tests__/signIn.test.tsx b/package/src/ai-components/__tests__/signIn.test.tsx index 1902715d0e..b839a7c09d 100644 --- a/package/src/ai-components/__tests__/signIn.test.tsx +++ b/package/src/ai-components/__tests__/signIn.test.tsx @@ -125,7 +125,8 @@ describe('AIToolAuthorization', () => { expect(screen.getByRole('button', { name: 'Not now' })).toBeEnabled(); }); - it('keeps connecting once connected, until the step changes', async () => { + it('keeps connecting once connected, until the step changes or 30 seconds pass', async () => { + jest.useFakeTimers(); renderWithTheme( Promise.resolve(true), decline: jest.fn() })} @@ -135,6 +136,15 @@ describe('AIToolAuthorization', () => { fireEvent.press(screen.getByRole('button', { name: 'Connect' })); await act(() => Promise.resolve()); expect(screen.getByRole('button', { name: 'Connecting...' })).toBeDisabled(); + act(() => { + jest.advanceTimersByTime(29_000); + }); + expect(screen.getByRole('button', { name: 'Connecting...' })).toBeDisabled(); + act(() => { + jest.advanceTimersByTime(1_000); + }); + expect(screen.getByRole('button', { name: 'Connect' })).toBeEnabled(); + jest.useRealTimers(); }); it('says when the sign-in could not open, and lets the person try again', async () => { diff --git a/package/src/ai-components/message-parts/AIToolAuthorization.tsx b/package/src/ai-components/message-parts/AIToolAuthorization.tsx index 6835aa12ec..8a096560bb 100644 --- a/package/src/ai-components/message-parts/AIToolAuthorization.tsx +++ b/package/src/ai-components/message-parts/AIToolAuthorization.tsx @@ -157,6 +157,13 @@ export const AIToolAuthorizationCard = ({ ); }; +// How long a sign-in that came back connected reads as connecting before Connect is offered +// again, when the agent never moves the call on. +const CONNECTED_SETTLE_MS = 30_000; + +// The latest sign-in opened for each call, so a late answer to an earlier one changes nothing. +const attempts = new Map(); + /** * The sign-in this person is asked for on `call` here: one the call waits for them to finish, * of a kind this device opens, and only when they can sign in on it. @@ -214,13 +221,21 @@ export const AIToolAuthorization = ({ const now = callStateOf(key, idle); if (now.isConnecting || now.isDeclining) return; update({ ...idle, isConnecting: true }); + const attempt = (attempts.get(key) ?? 0) + 1; + attempts.set(key, attempt); signIn.connect(part).then( // Your backend has the provider's answer before the sign-in comes back, and its change to // the step hides this. Until then it says it is connecting, unless the provider did not - // connect them. A sign-in the person closed can't be opened again, so it declines. + // connect them, or the step never changes. A sign-in the person closed can't be opened + // again, so it declines. (connected) => { if (connected === 'closed') sendDecline(signIn); else if (!connected) update(idle); + else { + setTimeout(() => { + if (attempts.get(key) === attempt && callStateOf(key, idle).isConnecting) update(idle); + }, CONNECTED_SETTLE_MS); + } }, () => update({ ...idle, connectFailed: true }), ); From dc8a5347ad3131ca1450bcf98bea3d7e74d650fc Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 14:44:22 +0100 Subject: [PATCH 10/16] fix(ai-components): dictate and label in the user's language Dictation always recognized en-US; it now uses the device's language, as every other SDK does, unless a language is given. The composer's placeholder, the voice visualizations' name and state labels, and the default text of AIAgentError were English only; they now go through the Chat SDK's translations, with keys in every locale. AI_AGENT_ERROR_MESSAGE is the error's key, so the samples show t(error.message). --- .../ExpoChatGPTSample/screens/ChatContent.tsx | 4 ++- .../ExpoCrewAssistant/screens/ChatContent.tsx | 4 ++- package/src/ai-components/README.md | 2 +- .../__tests__/conversationController.test.ts | 2 +- .../__tests__/dictation.test.tsx | 1 - .../controller/AIAgentError.ts | 8 +++++- .../message-composer/ComposerView.tsx | 10 ++++++-- .../transcription/useDictation.ts | 2 +- .../VoiceVisualizationView.tsx | 25 +++++++++++-------- package/src/i18n/ar.json | 11 +++++++- package/src/i18n/en.json | 11 +++++++- package/src/i18n/es.json | 11 +++++++- package/src/i18n/fr.json | 11 +++++++- package/src/i18n/he.json | 11 +++++++- package/src/i18n/hi.json | 11 +++++++- package/src/i18n/it.json | 11 +++++++- package/src/i18n/ja.json | 11 +++++++- package/src/i18n/ko.json | 11 +++++++- package/src/i18n/nl.json | 11 +++++++- package/src/i18n/pt-br.json | 11 +++++++- package/src/i18n/ru.json | 11 +++++++- package/src/i18n/tr.json | 11 +++++++- 22 files changed, 168 insertions(+), 33 deletions(-) diff --git a/examples/ExpoChatGPTSample/screens/ChatContent.tsx b/examples/ExpoChatGPTSample/screens/ChatContent.tsx index 2041540bf9..efcea65fff 100644 --- a/examples/ExpoChatGPTSample/screens/ChatContent.tsx +++ b/examples/ExpoChatGPTSample/screens/ChatContent.tsx @@ -24,6 +24,7 @@ import { ComposerView, isAIConversationAnswering, useAIConversationState, + useAITranslation, } from 'stream-chat-expo/ai-components'; import { AIConversation } from 'stream-chat-expo/ai-components/conversation'; @@ -51,6 +52,7 @@ const ComposerArea = ({ failure: string | undefined; }) => { const { error } = useAIConversationState(conversation, errorSelector); + const { t } = useAITranslation(); const safeAreaInsets = useSafeAreaInsets(); const insets = useMemo( @@ -73,7 +75,7 @@ const ComposerArea = ({ {failure ? {failure} : null} {error ? ( conversation.retry()}> - {error.message} + {t(error.message)} Try again ) : null} diff --git a/examples/ExpoCrewAssistant/screens/ChatContent.tsx b/examples/ExpoCrewAssistant/screens/ChatContent.tsx index a53284771f..51de24037a 100644 --- a/examples/ExpoCrewAssistant/screens/ChatContent.tsx +++ b/examples/ExpoCrewAssistant/screens/ChatContent.tsx @@ -46,6 +46,7 @@ import { ComposerView, isAIConversationAnswering, useAIConversationState, + useAITranslation, } from 'stream-chat-expo/ai-components'; import { AIConversation, @@ -79,6 +80,7 @@ const ComposerArea = ({ failure: string | undefined; }) => { const { error } = useAIConversationState(conversation, errorSelector); + const { t } = useAITranslation(); const safeAreaInsets = useSafeAreaInsets(); const insets = useMemo( @@ -102,7 +104,7 @@ const ComposerArea = ({ {failure ? {failure} : null} {error ? ( conversation.retry()}> - {error.message} + {t(error.message)} Try again ) : null} diff --git a/package/src/ai-components/README.md b/package/src/ai-components/README.md index 75c6e24d6e..85b166a507 100644 --- a/package/src/ai-components/README.md +++ b/package/src/ai-components/README.md @@ -238,7 +238,7 @@ It takes a single property named `options` that has the following keys: | Name | Type | Required | Description | | ------------------ | --------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `language` | `string` | no | The language we want to transcribe from. It will default to `en-US`. | +| `language` | `string` | no | The language we want to transcribe from. Defaults to the device's language. | | `interimResults` | `boolean` | no | A boolean signifying whether we want to receive the intermediate results from the transcription or just the final result when the transcription is deemed done. Defaults to `true` | | `silenceTimeoutMs` | `number` | no | A number signifying the number of milliseconds of silence until transcription is deemed finished. Defaults to `2500`. | diff --git a/package/src/ai-components/__tests__/conversationController.test.ts b/package/src/ai-components/__tests__/conversationController.test.ts index f8608070de..64ab58b458 100644 --- a/package/src/ai-components/__tests__/conversationController.test.ts +++ b/package/src/ai-components/__tests__/conversationController.test.ts @@ -1260,7 +1260,7 @@ describe('AIConversationController errors', () => { it('describes an agent error in the agent’s words, else its own', () => { expect(new AIAgentError('Rate limited').message).toBe('Rate limited'); - expect(new AIAgentError().message).toBe('The agent couldn’t answer. Try again.'); + expect(new AIAgentError().message).toBe("The agent couldn't answer. Try again."); expect(new AIAgentError('').agentMessage).toBeUndefined(); }); }); diff --git a/package/src/ai-components/__tests__/dictation.test.tsx b/package/src/ai-components/__tests__/dictation.test.tsx index 4755ea5e5d..fdb26d7093 100644 --- a/package/src/ai-components/__tests__/dictation.test.tsx +++ b/package/src/ai-components/__tests__/dictation.test.tsx @@ -80,7 +80,6 @@ describe('ai-components dictation', () => { }); expect(nativeModule.start).toHaveBeenCalledWith({ interimResults: true, - language: 'en-US', silenceTimeoutMs: 2500, }); diff --git a/package/src/ai-components/conversation-state/controller/AIAgentError.ts b/package/src/ai-components/conversation-state/controller/AIAgentError.ts index 840ef76013..818cb8dd2e 100644 --- a/package/src/ai-components/conversation-state/controller/AIAgentError.ts +++ b/package/src/ai-components/conversation-state/controller/AIAgentError.ts @@ -1,3 +1,9 @@ +/** + * What an `AIAgentError` says when the agent said nothing of its own, a translation key: show it + * with `t(error.message)`. + */ +export const AI_AGENT_ERROR_MESSAGE = "The agent couldn't answer. Try again."; + /** * The agent couldn't answer: it said so, with what it said as `agentMessage`, or it didn't start * its reply in time. @@ -7,7 +13,7 @@ export class AIAgentError extends Error { readonly agentMessage?: string; constructor(agentMessage?: string) { - super(agentMessage || 'The agent couldn’t answer. Try again.'); + super(agentMessage || AI_AGENT_ERROR_MESSAGE); this.name = 'AIAgentError'; this.agentMessage = agentMessage || undefined; } diff --git a/package/src/ai-components/message-composer/ComposerView.tsx b/package/src/ai-components/message-composer/ComposerView.tsx index 6752ac3084..ece13be0c7 100644 --- a/package/src/ai-components/message-composer/ComposerView.tsx +++ b/package/src/ai-components/message-composer/ComposerView.tsx @@ -16,7 +16,12 @@ import { MediaPreviewList } from './MediaPreviewList'; import { BottomSheet } from '../components/BottomSheet'; -import { ComposerProvider, useMessageComposerContext, useTheme } from '../contexts'; +import { + ComposerProvider, + useAITranslation, + useMessageComposerContext, + useTheme, +} from '../contexts'; import type { AIConversationController, AIConversationControllerState, @@ -206,6 +211,7 @@ export const AIMessageInput = () => { }, } = useTheme(); const styles = useStyles(); + const { t } = useAITranslation(); const { error, isRecording, source } = useDictationState(); const { text } = useComposerText(); @@ -244,7 +250,7 @@ export const AIMessageInput = () => { = { - connecting: 'Connecting', - idle: 'Ready', - listening: 'Listening', - speaking: 'Speaking', - thinking: 'Thinking', -}; - /** * What VoiceWave and VoiceGlow share: the motion, the canvas kept at the view's size and at most * the style's density, reduced motion, taps and hover, and accessibility. @@ -40,7 +35,7 @@ const DEFAULT_STATE_LABELS: Record = { * only the shape takes them: around it they reach whatever is behind. */ export const VoiceVisualizationView = ({ - accessibilityLabel = 'Voice agent', + accessibilityLabel, accessibilityValue, audioLevel, colors, @@ -54,6 +49,14 @@ export const VoiceVisualizationView = ({ visualStyle, ...restProps }: ViewProps) => { + const { t } = useAITranslation(); + const defaultStateLabels: Record = { + connecting: t('Connecting'), + idle: t('Ready'), + listening: t('Listening'), + speaking: t('Speaking'), + thinking: t('Thinking'), + }; const visualization = useVoiceVisualization( { audioLevel, colors, expressivity, state }, visualStyle, @@ -121,10 +124,10 @@ export const VoiceVisualizationView = ({ return ( Date: Wed, 11 Nov 2026 14:44:39 +0100 Subject: [PATCH 11/16] fix(a2ui): draw surfaces that name the web SDK's catalog id The web SDK names the app's catalog stream-app-catalog, so an agent set up from a web app's capabilities wrote surfaces this renderer refused. With the default catalog id, getstream.io:app-catalog, which iOS and Android also use, the renderer now accepts that id too, and so does a2uiProblems. It's exported as A2UI_WEB_CATALOG_ID. --- package/src/a2ui/A2UIRenderer.tsx | 4 ++-- .../a2ui/__tests__/A2UIInstructions.test.ts | 14 +++++++++++++ .../src/a2ui/__tests__/A2UIRenderer.test.tsx | 21 ++++++++++++++++++- package/src/a2ui/catalogDocument.ts | 10 +++++++++ package/src/a2ui/catalogs.ts | 10 +++------ package/src/a2ui/index.ts | 1 + package/src/a2ui/instructions.ts | 11 ++++++++-- 7 files changed, 59 insertions(+), 12 deletions(-) diff --git a/package/src/a2ui/A2UIRenderer.tsx b/package/src/a2ui/A2UIRenderer.tsx index e50d8b7bae..f5a8106828 100644 --- a/package/src/a2ui/A2UIRenderer.tsx +++ b/package/src/a2ui/A2UIRenderer.tsx @@ -19,7 +19,7 @@ import { supportsA2UIVersion, } from './A2UIStream'; import { A2UISurfaceView } from './A2UISurface'; -import { A2UI_DEFAULT_CATALOG_ID, a2uiTypeName } from './catalogDocument'; +import { A2UI_DEFAULT_CATALOG_ID, A2UI_WEB_CATALOG_ID, a2uiTypeName } from './catalogDocument'; import { a2uiCatalogsOf } from './catalogs'; import { A2UI_BASIC_COMPONENTS } from './components/basicComponents'; import type { A2UIComponents, A2UIIcons } from './components/types'; @@ -44,7 +44,7 @@ import { import type { AIUIEngine, AIUIRenderer } from '../ai-components/generative-ui/AIUIRenderer'; import { primitives } from '../ai-components/internal/tokens'; -export { A2UI_DEFAULT_CATALOG_ID }; +export { A2UI_DEFAULT_CATALOG_ID, A2UI_WEB_CATALOG_ID }; /** What went wrong in a reply's A2UI, as the A2UI specification reports it to an agent. */ export type A2UIError = { diff --git a/package/src/a2ui/__tests__/A2UIInstructions.test.ts b/package/src/a2ui/__tests__/A2UIInstructions.test.ts index 62130e400f..6a3813cb8f 100644 --- a/package/src/a2ui/__tests__/A2UIInstructions.test.ts +++ b/package/src/a2ui/__tests__/A2UIInstructions.test.ts @@ -3,6 +3,7 @@ import { parseAIUIComponentDefinition } from '../../ai-components'; import { ohridLines, uiComponents } from '../../mock-builders/a2ui/ohrid'; import { A2UI_DEFAULT_CATALOG_ID, + A2UI_WEB_CATALOG_ID, a2uiInstructions, A2UIInstructionsError, a2uiProblems, @@ -217,6 +218,19 @@ describe('a2uiProblems', () => { ].join('\n'), ), ).toEqual([expect.stringMatching(/^surface s is created again/)]); + expect( + await a2uiProblems( + [ + line({ createSurface: { catalogId: A2UI_WEB_CATALOG_ID, surfaceId: 's' } }), + line({ + updateComponents: { + components: [{ component: 'Text', id: 'root', text: 'One' }], + surfaceId: 's', + }, + }), + ].join('\n'), + ), + ).toEqual([]); expect( await a2uiProblems( [ diff --git a/package/src/a2ui/__tests__/A2UIRenderer.test.tsx b/package/src/a2ui/__tests__/A2UIRenderer.test.tsx index fe9269d136..695e033b15 100644 --- a/package/src/a2ui/__tests__/A2UIRenderer.test.tsx +++ b/package/src/a2ui/__tests__/A2UIRenderer.test.tsx @@ -24,7 +24,12 @@ import basicCatalogJSON from '../basicCatalog.json'; import { a2uiAppCatalogDocument, a2uiTypeName } from '../catalogDocument'; import { formatMoment, parseMoment } from '../components/DateTimeInput'; import { A2UI_ICON_PATHS } from '../components/iconPaths'; -import { A2UI_DEFAULT_CATALOG_ID, type A2UIRendererOptions, createA2UIRenderer } from '../index'; +import { + A2UI_DEFAULT_CATALOG_ID, + A2UI_WEB_CATALOG_ID, + type A2UIRendererOptions, + createA2UIRenderer, +} from '../index'; import { withoutPaths, withoutSecrets } from '../secrets'; const definitionOf = (name: string) => { @@ -322,6 +327,20 @@ describe('the A2UI versions and catalog it reads', () => { }); describe('a reply written in A2UI', () => { + it('draws a surface that names the web SDK’s catalog id', async () => { + const { errors, ui } = setup(); + await renderReply(ui, [ + line({ createSurface: { catalogId: A2UI_WEB_CATALOG_ID, surfaceId: 's' } }), + line({ + updateComponents: { + components: [{ component: 'Text', id: 'root', text: 'From the web agent' }], + surfaceId: 's', + }, + }), + ]); + await waitFor(() => expect(screen.getByText(/From the web agent/)).toBeTruthy()); + expect(errors).toEqual([]); + }); it('shows the Ohrid plan with the basic catalog and the app’s components', async () => { const { errors, ui } = setup(); await renderReply(ui, ohridLines); diff --git a/package/src/a2ui/catalogDocument.ts b/package/src/a2ui/catalogDocument.ts index 2512f3758c..9d3bb94931 100644 --- a/package/src/a2ui/catalogDocument.ts +++ b/package/src/a2ui/catalogDocument.ts @@ -13,6 +13,16 @@ import type { AIClientToolDefinition } from '../ai-components/tool-calls/AIClien /** The id of the catalog with the app's components, unless the renderer is given another. */ export const A2UI_DEFAULT_CATALOG_ID = 'getstream.io:app-catalog'; +/** + * The id the web SDK names the same catalog with. A renderer on the default id draws surfaces + * that name it too, so an agent set up for a web app shows its UI here as well. + */ +export const A2UI_WEB_CATALOG_ID = 'stream-app-catalog'; + +/** The ids a surface may name for a renderer whose catalog is `catalogId`. */ +export const a2uiAppCatalogIds = (catalogId: string) => + catalogId === A2UI_DEFAULT_CATALOG_ID ? [catalogId, A2UI_WEB_CATALOG_ID] : [catalogId]; + const definitionOf = (component: AIUIComponent | AIUIComponentDefinition) => 'definition' in component ? component.definition : component; diff --git a/package/src/a2ui/catalogs.ts b/package/src/a2ui/catalogs.ts index 6b94bcb276..53a2121f0e 100644 --- a/package/src/a2ui/catalogs.ts +++ b/package/src/a2ui/catalogs.ts @@ -10,7 +10,7 @@ import { import { A2UI_VERSION } from './A2UIStream'; import basicCatalogDocument from './basicCatalog.json'; import { basicFunctionsOf } from './basicFunctions'; -import { a2uiAppCatalogDocument, a2uiTypeName } from './catalogDocument'; +import { a2uiAppCatalogDocument, a2uiAppCatalogIds, a2uiTypeName } from './catalogDocument'; import { secretValuesOf, withoutSecrets } from './secrets'; import type { AIUIComponent } from '../ai-components/generative-ui/AIUIComponent'; @@ -176,10 +176,6 @@ export const a2uiCatalogsOf = ({ } const basic = [...definitions.components.values()].filter(({ name }) => !removed.has(name)); const all = [...basic, ...appApis] as ComponentApi[]; - return [ - new Catalog(catalogId, A2UI_VERSION, all, functions), - ...(catalogId === A2UI_BASIC_CATALOG_ID - ? [] - : [new Catalog(A2UI_BASIC_CATALOG_ID, A2UI_VERSION, all, functions)]), - ]; + const ids = new Set([...a2uiAppCatalogIds(catalogId), A2UI_BASIC_CATALOG_ID]); + return [...ids].map((id) => new Catalog(id, A2UI_VERSION, all, functions)); }; diff --git a/package/src/a2ui/index.ts b/package/src/a2ui/index.ts index da9263d6c6..4a57a03c6d 100644 --- a/package/src/a2ui/index.ts +++ b/package/src/a2ui/index.ts @@ -2,6 +2,7 @@ // agent never writes A2UI doesn't bundle the A2UI core. export { A2UI_DEFAULT_CATALOG_ID, + A2UI_WEB_CATALOG_ID, type A2UIError, type A2UIRendererOptions, createA2UIRenderer, diff --git a/package/src/a2ui/instructions.ts b/package/src/a2ui/instructions.ts index e62571ea70..c5b8be1403 100644 --- a/package/src/a2ui/instructions.ts +++ b/package/src/a2ui/instructions.ts @@ -20,7 +20,11 @@ import { writing, } from './A2UIInstructions'; import { A2UI_VERSION, a2uiMessagesIn, normalizeA2UIMessage } from './A2UIStream'; -import { A2UI_DEFAULT_CATALOG_ID, a2uiAppCatalogDocument } from './catalogDocument'; +import { + A2UI_DEFAULT_CATALOG_ID, + a2uiAppCatalogDocument, + a2uiAppCatalogIds, +} from './catalogDocument'; import { basicCatalogDefinitions } from './catalogs'; import type { AIUIComponent } from '../ai-components/generative-ui/AIUIComponent'; @@ -183,7 +187,10 @@ export const a2uiProblemsWith = (text: string, catalog: Catalog) => { created = true; const components = componentsOf(body.components); problems.push(...duplicates(components, surfaceId)); - if (typeof body.catalogId === 'string' && body.catalogId !== catalog.id) { + if ( + typeof body.catalogId === 'string' && + !a2uiAppCatalogIds(catalog.id).includes(body.catalogId) + ) { problems.push( `surface ${surfaceId} names the catalog ${body.catalogId}, not ${catalog.id}`, ); From d659a3957a813804bc7b9a48c6483511d363b00d Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 14:44:51 +0100 Subject: [PATCH 12/16] fix(a2ui): tell the model which of the app's tools only read The A2UI instructions gave the app's tools only as functions of the catalog, so the model could bind a tool that changes data to a display, which the host then refuses to run. They now list the tools, mark the read-only ones, and say that the others run only from a button the user taps, as Android's do. --- .../src/a2ui/__tests__/A2UIRenderer.test.tsx | 3 +++ package/src/a2ui/instructions.ts | 26 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/package/src/a2ui/__tests__/A2UIRenderer.test.tsx b/package/src/a2ui/__tests__/A2UIRenderer.test.tsx index 695e033b15..9614860f33 100644 --- a/package/src/a2ui/__tests__/A2UIRenderer.test.tsx +++ b/package/src/a2ui/__tests__/A2UIRenderer.test.tsx @@ -899,6 +899,9 @@ describe('the app’s tools in a reply’s UI', () => { expect(instructions).toContain('"weather"'); expect(instructions).toContain('The weather in a city today.'); expect(instructions).toContain('"city"'); + expect(instructions).toContain("## The app's tools"); + expect(instructions).toContain('- `weather` (read-only): The weather in a city today.'); + expect(instructions).toContain('Any other runs only from a Button'); }); it('shows the rest of the UI when a tool fails, and says so', async () => { diff --git a/package/src/a2ui/instructions.ts b/package/src/a2ui/instructions.ts index c5b8be1403..7859c94880 100644 --- a/package/src/a2ui/instructions.ts +++ b/package/src/a2ui/instructions.ts @@ -30,6 +30,7 @@ import { basicCatalogDefinitions } from './catalogs'; import type { AIUIComponent } from '../ai-components/generative-ui/AIUIComponent'; import type { AIUIComponentDefinition } from '../ai-components/generative-ui/AIUIComponentDefinition'; import { isRecord } from '../ai-components/generative-ui/AIUIHelpers'; +import type { AIClientToolDefinition } from '../ai-components/tool-calls/AIClientToolRunner'; type Entry = ComponentApi; @@ -90,6 +91,28 @@ export const a2uiCatalogFor = ({ }); }; +// The app's tools as the model is told them, each marked when it only reads, with the rule for +// where they run: the host runs a tool that changes something only when the person taps for it. +const toolsSection = (tools: AIClientToolDefinition[]) => { + if (!tools.length) return []; + const lines = tools + .slice() + .sort((a, b) => a.name.localeCompare(b.name)) + .map(({ annotations, description, name }) => { + const readOnly = annotations?.readOnlyHint === true ? ' (read-only)' : ''; + return `- \`${name}\`${readOnly}${description ? `: ${description}` : ''}`; + }); + return [ + [ + "## The app's tools", + '', + 'These run on the person\'s device. Call one with `{"@call": "", "args": {...}}`. A read-only tool runs wherever you call it, such as to show its result in a Text. Any other runs only from a Button\'s `functionCall` action, when the person taps it.', + '', + ...lines, + ].join('\n'), + ]; +}; + /** * The instructions that teach an agent's model to write A2UI the app's renderer shows, in * Markdown: how a reply carries its UI, the protocol, the catalog's guidelines and the catalog @@ -113,6 +136,9 @@ export const a2uiInstructionsText = (config: A2UIInstructionsConfig = {}) => { A2UI_PROTOCOL_OVERVIEW, ...(typeof instructions === 'string' && instructions ? [instructions] : []), ...(notes.length ? [`## The app's components\n\n${notes.join('\n')}`] : []), + ...toolsSection( + (config.tools ?? []).filter(({ name }) => !basicCatalogDefinitions().functions.has(name)), + ), `## The catalog \`${catalog.id}\`\n\n\`\`\`json\n${JSON.stringify(sorted(document))}\n\`\`\``, ...(rules.length ? [`## Rules\n\n${rules.map((rule, index) => `${index + 1}. ${rule}`).join('\n')}`] From 26287f6512053fa832b13032457af82fb33f8f21 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 14:44:55 +0100 Subject: [PATCH 13/16] fix(a2ui): format numbers and dates as the device does A2UI's formatting always used en-US, formatCurrency always used 2 decimals, and formatDate returned nothing for epoch timestamps and times of day. Numbers and dates now follow the device's locale unless the renderer is given one, each currency gets its own decimals unless decimals is given, and formatDate reads epoch seconds and milliseconds and HH:mm, as Android's does. --- package/src/a2ui/A2UIRenderer.tsx | 2 +- .../src/a2ui/__tests__/A2UIRenderer.test.tsx | 23 +++++++++ package/src/a2ui/basicFunctions.ts | 51 +++++++++++++++---- package/src/ai-components/README.md | 2 +- 4 files changed, 66 insertions(+), 12 deletions(-) diff --git a/package/src/a2ui/A2UIRenderer.tsx b/package/src/a2ui/A2UIRenderer.tsx index f5a8106828..7ec153f93f 100644 --- a/package/src/a2ui/A2UIRenderer.tsx +++ b/package/src/a2ui/A2UIRenderer.tsx @@ -78,7 +78,7 @@ export type A2UIRendererOptions = { * answers to write as UI. */ instructions?: string; - /** The locale numbers and dates are formatted in. `en-US` by default. */ + /** The locale numbers and dates are formatted in. The device's by default. */ locale?: string; /** * Decides what an Image, Video or AudioPlayer loads for the link the agent wrote: the address diff --git a/package/src/a2ui/__tests__/A2UIRenderer.test.tsx b/package/src/a2ui/__tests__/A2UIRenderer.test.tsx index 9614860f33..58b60579a6 100644 --- a/package/src/a2ui/__tests__/A2UIRenderer.test.tsx +++ b/package/src/a2ui/__tests__/A2UIRenderer.test.tsx @@ -1318,6 +1318,29 @@ describe('the basic catalog', () => { expect(screen.getByText(/Saturday 14 November/)).toBeTruthy(); }); + it('formats yen without decimals, and epoch timestamps and times of day', async () => { + const { ui } = setup({ locale: 'en-US' }); + const text = (id: string, call: string, args: Record) => ({ + component: 'Text', + id, + text: { '@call': call, args }, + }); + await renderReply( + ui, + surfaceOf([ + { children: ['yen', 'seconds', 'millis', 'time'], component: 'Column', id: 'root' }, + text('yen', 'formatCurrency', { currency: 'JPY', value: 1500 }), + text('seconds', 'formatDate', { format: 'yyyy-MM-dd', value: 1_763_078_400 }), + text('millis', 'formatDate', { format: 'd MMMM', value: 1_763_078_400_000 }), + text('time', 'formatDate', { format: 'h:mm a', value: '18:30' }), + ]), + ); + await waitFor(() => expect(screen.getByText('¥1,500')).toBeTruthy()); + expect(screen.getByText('2025-11-14')).toBeTruthy(); + expect(screen.getByText('14 November')).toBeTruthy(); + expect(screen.getByText('6:30 PM')).toBeTruthy(); + }); + it('keeps what the person types in the data model, and holds a button until its checks pass', async () => { const { sent, ui } = setup(); await renderReply( diff --git a/package/src/a2ui/basicFunctions.ts b/package/src/a2ui/basicFunctions.ts index 42ef72dc91..e0a3927a92 100644 --- a/package/src/a2ui/basicFunctions.ts +++ b/package/src/a2ui/basicFunctions.ts @@ -33,11 +33,20 @@ import { isSignal, } from '@a2ui/web_core'; -/** The locale formatting falls back to, so it never depends on the host's. */ +/** The locale formatting falls back to when neither the app nor the device names one it has. */ const DEFAULT_LOCALE = 'en-US'; -const resolveLocale = (locale?: string) => { - if (!locale) return DEFAULT_LOCALE; +// The device's locale, as the person reads numbers and dates elsewhere. +const deviceLocale = () => { + try { + return Intl.DateTimeFormat().resolvedOptions().locale || DEFAULT_LOCALE; + } catch { + return DEFAULT_LOCALE; + } +}; + +const resolveLocale = (asked?: string) => { + const locale = asked || deviceLocale(); try { return Intl.NumberFormat.supportedLocalesOf(locale).length > 0 ? locale : DEFAULT_LOCALE; } catch { @@ -146,18 +155,21 @@ const formatNumber = ( useGrouping: grouping, }).format(value); +// A currency's amount with `decimals` decimals, or as many as the currency has, such as none for +// yen. const formatCurrency = ( value: number, code: unknown, - decimals = 2, + decimals?: number, grouping?: boolean, locale = DEFAULT_LOCALE, ) => { if (isNaN(value)) return ''; const currency = String(code).toUpperCase(); const options = { - maximumFractionDigits: decimals, - minimumFractionDigits: decimals, + ...(decimals === undefined + ? {} + : { maximumFractionDigits: decimals, minimumFractionDigits: decimals }), useGrouping: grouping, }; try { @@ -242,9 +254,28 @@ const parseTimestamp = (value: string) => { return fieldsRoundTrip(value, shifted) ? { instant, shifted } : null; }; +// Epoch milliseconds start beyond this; a smaller number is epoch seconds. +const EPOCH_MILLISECONDS_FROM = 100_000_000_000; +const TIME_OF_DAY = /^\d{2}:\d{2}(:\d{2})?$/; + +// An epoch timestamp, in seconds or milliseconds, as the device's clock reads it. +const parseEpoch = (value: number) => { + const instant = new Date(Math.abs(value) > EPOCH_MILLISECONDS_FROM ? value : value * 1000); + if (isNaN(instant.getTime())) return null; + return { + instant, + shifted: new Date(instant.getTime() - instant.getTimezoneOffset() * 60_000), + }; +}; + const formatDate = (value: unknown, pattern = 'yyyy-MM-dd', locale = DEFAULT_LOCALE) => { - if (!value) return ''; - const parsed = parseTimestamp(String(value)); + if (value === undefined || value === null || value === '') return ''; + const parsed = + typeof value === 'number' + ? parseEpoch(value) + : TIME_OF_DAY.test(String(value)) + ? parseTimestamp(`1970-01-01T${String(value)}`) + : parseTimestamp(String(value)); if (!parsed) return ''; const { instant, shifted } = parsed; const format = pattern || 'yyyy-MM-dd'; @@ -372,7 +403,7 @@ type Execute = (args: Args, context: DataContext) => unknown; /** * The basic catalog's functions, for the function definitions `apis` of its document, with - * `openUrl` run by `openUrl`. Formatting follows `locale`, or `en-US`. + * `openUrl` run by `openUrl`. Formatting follows `locale`, or the device's. */ export const basicFunctionsOf = ( apis: Map, @@ -388,7 +419,7 @@ export const basicFunctionsOf = ( formatCurrency( number(value), currency, - optionalNumber(decimals) ?? 2, + optionalNumber(decimals), grouping as boolean | undefined, resolved, ), diff --git a/package/src/ai-components/README.md b/package/src/ai-components/README.md index 85b166a507..4db0c07116 100644 --- a/package/src/ai-components/README.md +++ b/package/src/ai-components/README.md @@ -335,7 +335,7 @@ The label of the message an action sends is the text of the control the person t | `components` | Your own views for basic components, by type, such as `{ Card: MyCard }`. Each gets the node and its resolved props. | | `icons` | Your own icons for the names `Icon` takes, over the Material icons drawn by default. | | `removing` | Basic components or functions to leave out, such as `['Video']`. The agent's instructions leave them out too. | -| `locale` | The locale numbers and dates are formatted in. `en-US` by default. | +| `locale` | The locale numbers and dates are formatted in. The device's by default. | | `mediaUrl` | Decides what an `Image`, `Video` or `AudioPlayer` loads for a link. Public web links only by default. | | `instructions` | What you tell the model before A2UI's own instructions, such as which answers to write as UI. | | `catalogId` | The id of the catalog with your own components. | From e4dbef6c8df8d918308a762c25db0a9c86cebb11 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 14:45:08 +0100 Subject: [PATCH 14/16] fix: render code in a monospace font on Android Inline code in messages, Markdown code in the AI components and the syntax highlighter's default asked Android for the font 'Monospace'. Android knows it only as 'monospace', and falls back to the default font without a warning, so code was drawn in a proportional font. iOS is unchanged. --- package/src/ai-components/markdown/styles.ts | 8 ++++---- .../syntax-highlighting/SyntaxHighlighter.tsx | 2 +- .../Message/MessageItemView/utils/renderText.tsx | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package/src/ai-components/markdown/styles.ts b/package/src/ai-components/markdown/styles.ts index 9c18fb506b..f0604d9256 100644 --- a/package/src/ai-components/markdown/styles.ts +++ b/package/src/ai-components/markdown/styles.ts @@ -47,7 +47,7 @@ export const getLocalStyles = ({ theme }: { theme: Theme }) => { }, codeBlock: { backgroundColor: colors.code_block, - fontFamily: Platform.OS === 'ios' ? 'Courier' : 'Monospace', + fontFamily: Platform.OS === 'ios' ? 'Courier' : 'monospace', fontWeight: '500', color: colors.black, paddingLeft: 8, @@ -68,7 +68,7 @@ export const getLocalStyles = ({ theme }: { theme: Theme }) => { color: colors.black, }, codeBlockLineNumberCell: { - fontFamily: Platform.OS === 'ios' ? 'Courier' : 'Monospace', + fontFamily: Platform.OS === 'ios' ? 'Courier' : 'monospace', color: colors.grey, fontSize: primitives.typographyFontSizeXs, lineHeight: primitives.typographyLineHeightTight, @@ -81,7 +81,7 @@ export const getLocalStyles = ({ theme }: { theme: Theme }) => { codeBlockHeaderTitle: { fontSize: primitives.typographyFontSizeXs, color: colors.grey, - fontFamily: Platform.OS === 'ios' ? 'Courier' : 'Monospace', + fontFamily: Platform.OS === 'ios' ? 'Courier' : 'monospace', }, del: { textDecorationLine: 'line-through', @@ -117,7 +117,7 @@ export const getLocalStyles = ({ theme }: { theme: Theme }) => { borderRadius: 3, borderWidth: 1, color: colors.accent_red, - fontFamily: Platform.OS === 'ios' ? 'Courier' : 'Monospace', + fontFamily: Platform.OS === 'ios' ? 'Courier' : 'monospace', fontWeight: '500', fontSize: primitives.typographyFontSizeSm, padding: 3, diff --git a/package/src/ai-components/syntax-highlighting/SyntaxHighlighter.tsx b/package/src/ai-components/syntax-highlighting/SyntaxHighlighter.tsx index 9ca9131d6c..556872e8a4 100644 --- a/package/src/ai-components/syntax-highlighting/SyntaxHighlighter.tsx +++ b/package/src/ai-components/syntax-highlighting/SyntaxHighlighter.tsx @@ -22,7 +22,7 @@ export type ColorRange = { }; export const SyntaxHighlighter = ({ - fontFamily = Platform.OS === 'ios' ? 'Courier' : 'Monospace', + fontFamily = Platform.OS === 'ios' ? 'Courier' : 'monospace', fontSize = DEFAULT_FONT_SIZE, children, highlighter = 'highlightjs', diff --git a/package/src/components/Message/MessageItemView/utils/renderText.tsx b/package/src/components/Message/MessageItemView/utils/renderText.tsx index 12e8d3e56f..e9558f4a49 100644 --- a/package/src/components/Message/MessageItemView/utils/renderText.tsx +++ b/package/src/components/Message/MessageItemView/utils/renderText.tsx @@ -95,7 +95,7 @@ export const MarkdownReactiveScrollView = ({ children }: { children: ReactNode } const defaultMarkdownStyles: MarkdownStyle = { codeBlock: { - fontFamily: Platform.OS === 'ios' ? 'Courier' : 'Monospace', + fontFamily: Platform.OS === 'ios' ? 'Courier' : 'monospace', fontWeight: '500', marginVertical: primitives.spacingXs, fontSize: primitives.typographyFontSizeMd, From 54ae152398dbd380b4a1b57f1abf87113c573907 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 14:45:12 +0100 Subject: [PATCH 15/16] fix(ai-components): measure Android code blocks in the font they're drawn in The fast syntax highlighter's native text measured its width without its fontFamily, in Android's default font, then drew in monospace, which is wider. The width was padded by a flat 1.6 to make up for it, which left empty space after most code and still clipped and wrapped lines of narrow characters, such as i and l. The measurement now uses the font the text is drawn in, and the padding is gone, so a block is as wide as its longest line. --- .../shared-native/android/PerfTextViewManager.kt | 14 +++++++++----- .../android/jni/PerfTextMeasurementManager.cpp | 1 + 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/package/shared-native/android/PerfTextViewManager.kt b/package/shared-native/android/PerfTextViewManager.kt index 76a03d5df8..6aa7c7963d 100644 --- a/package/shared-native/android/PerfTextViewManager.kt +++ b/package/shared-native/android/PerfTextViewManager.kt @@ -83,7 +83,7 @@ class PerfTextViewManager : SimpleViewManager() { ): Long { val view = PerfTextView(context) - // Apply props that influence size (text, fontSize, lineHeight) + // Apply props that influence size (text, fontSize, fontFamily, lineHeight) props?.let { p -> if (p.hasKey("fontSize") && !p.isNull("fontSize")) { val fs = p.getDouble("fontSize") @@ -92,9 +92,13 @@ class PerfTextViewManager : SimpleViewManager() { } } - if (p.hasKey("fontFamily") && !p.isNull("fontFamily")) { - view.setFontFamilyCompat(p.getString("fontFamily")) - } + view.setFontFamilyCompat( + if (p.hasKey("fontFamily") && !p.isNull("fontFamily")) { + p.getString("fontFamily")?.takeIf { it.isNotEmpty() } + } else { + null + }, + ) if (p.hasKey("lineHeight") && !p.isNull("lineHeight")) { val lh = p.getDouble("lineHeight") @@ -137,7 +141,7 @@ class PerfTextViewManager : SimpleViewManager() { val measuredWidthDp = ceil(PixelUtil.toDIPFromPixel(view.measuredWidth.toFloat())) val measuredHeightDp = ceil(PixelUtil.toDIPFromPixel(view.measuredHeight.toFloat())) - return YogaMeasureOutput.make(measuredWidthDp * 1.6f, measuredHeightDp * 1.0f) + return YogaMeasureOutput.make(measuredWidthDp, measuredHeightDp) } } diff --git a/package/shared-native/android/jni/PerfTextMeasurementManager.cpp b/package/shared-native/android/jni/PerfTextMeasurementManager.cpp index d0a5fb5324..f79a44d9a1 100644 --- a/package/shared-native/android/jni/PerfTextMeasurementManager.cpp +++ b/package/shared-native/android/jni/PerfTextMeasurementManager.cpp @@ -42,6 +42,7 @@ Size PerfTextMeasurementManager::measure( serializedProps["text"] = props.text; serializedProps["fontSize"] = props.fontSize; serializedProps["lineHeight"] = props.lineHeight; + serializedProps["fontFamily"] = props.fontFamily; auto propsRNM = ReadableNativeMap::newObjectCxxArgs(serializedProps); local_ref propsRM = From 0a44b328af960a572c27e61f58b49d73274b89d6 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 15:30:34 +0100 Subject: [PATCH 16/16] fix(ai-components): find a pending question's copy without going over every message Matching a pending question's copy that arrives in the same update the question is first seen in kept the ids of all the person's messages from every update, so every streamed chunk of a reply went over the whole list. The copy of a question just sent is always the person's newest message, so the tracker now remembers only that one, found by stepping back from the end, and takes it for the copy when it arrived with the update. --- .../__tests__/conversation.test.tsx | 20 ++++++++++++ .../src/ai-components/conversation/entries.ts | 32 +++++++++++++++---- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/package/src/ai-components/__tests__/conversation.test.tsx b/package/src/ai-components/__tests__/conversation.test.tsx index 2a4af40124..72c1beb2b1 100644 --- a/package/src/ai-components/__tests__/conversation.test.tsx +++ b/package/src/ai-components/__tests__/conversation.test.tsx @@ -178,6 +178,26 @@ describe('AIConversation', () => { expect(entries.filter((entry) => entry.pending)).toHaveLength(0); }); + it('takes the copy of a message on its way from a window of the same length', () => { + const tracker = new AIPendingMessageTracker(); + tracker.entries([user('u1', 'Hello'), reply('a1', 'Hi!')]); + const pending = { id: 'local-1', text: 'Plan my day' }; + // A reconnect reloads the latest page: as many messages, the newest one the copy. + const entries = tracker.entries( + [reply('a1', 'Hi!'), user('stream-u2', 'Plan my day')], + pending, + ); + expect(entries.map((entry) => entry.id)).toEqual(['a1', 'local-1']); + }); + + it('takes no older question for the copy of a message on its way when it mounts with one', () => { + const history = [user('u1', 'Hello'), reply('a1', 'Hi!'), user('u2', 'Plan my day')]; + const pending = { id: 'local-3', text: 'Plan my week' }; + const entries = new AIPendingMessageTracker().entries(history, pending); + expect(entries.map((entry) => entry.id)).toEqual(['u1', 'a1', 'u2', 'local-3']); + expect(entries[3].pending).toBe(pending); + }); + it('shows a pending message with the agent thinking, until its reply starts', async () => { const mounts: string[] = []; const UserMessage = ({ text }: AIConversationUserMessageProps) => { diff --git a/package/src/ai-components/conversation/entries.ts b/package/src/ai-components/conversation/entries.ts index 0157ca943d..a9b6bb3af3 100644 --- a/package/src/ai-components/conversation/entries.ts +++ b/package/src/ai-components/conversation/entries.ts @@ -27,6 +27,15 @@ export type AIConversationCanShowUI = (document: AIUIDocument) => boolean; const showsNoUI: AIConversationCanShowUI = () => false; +// The id of the person's newest message, looked for from the end: the copy of one they just sent +// is always it, and while a reply is written it is a step or two from the end. +const newestFromUserOf = (messages: AIMessage[]) => { + for (let index = messages.length - 1; index >= 0; index -= 1) { + if (messages[index].role === 'user') return messages[index].id; + } + return undefined; +}; + // A reply the agent has started but that has nothing to show yet: no text, steps or attachments, // and no UI that `canShowUI` says shows. const showsNothing = (message: AIMessage, canShowUI: AIConversationCanShowUI) => @@ -91,9 +100,10 @@ export class AIPendingMessageTracker { // The pending message of the last update, whose copy can arrive in the update that stops // passing it. private lastPending?: AIPendingMessage; - // The person's messages of the last update. A pending message first seen in an update that - // also brings its copy can't take that update's messages as ones its copy isn't. - private lastSentIds?: Set; + // The person's newest message as of the last update, `null` before the first one. A message on + // its way first seen in an update that also brings its copy finds it as the person's newest + // message, when that one is new. + private lastNewestFromUser: string | null | undefined = null; // The rows of a conversation: its messages, apart from replies with nothing to show yet, and // a message on its way until its copy arrives. A reply's UI shows when `canShowUI` says. @@ -103,13 +113,21 @@ export class AIPendingMessageTracker { canShowUI: AIConversationCanShowUI = showsNoUI, ): AIConversationEntry[] { let showsPending = !!pending; - const sent = messages.filter((message) => message.role === 'user'); - const ids = new Set(sent.map((message) => message.id)); + const newestFromUser = newestFromUserOf(messages); // A copy that arrives in the update that stops passing its pending message still takes over // its row. Only then: a message sent later is not the copy of one that failed. const awaited = pending ?? this.lastPending; if (awaited) { - const known = this.seen.get(awaited.id) ?? this.lastSentIds ?? ids; + const sent = messages.filter((message) => message.role === 'user'); + const ids = new Set(sent.map((message) => message.id)); + let known = this.seen.get(awaited.id); + if (!known) { + known = new Set(ids); + // The person's message that arrived with this update can be the copy. + const arrived = + this.lastNewestFromUser !== null && newestFromUser !== this.lastNewestFromUser; + if (arrived && newestFromUser !== undefined) known.delete(newestFromUser); + } const matched = [...this.copies].find( ([copyId, original]) => original.id === awaited.id && ids.has(copyId), )?.[0]; @@ -122,7 +140,7 @@ export class AIPendingMessageTracker { } } this.lastPending = pending ?? undefined; - this.lastSentIds = ids; + this.lastNewestFromUser = newestFromUser; const entries: AIConversationEntry[] = messages .filter((message) => !showsNothing(message, canShowUI))