diff --git a/examples/ExpoChatGPTSample/screens/ChatContent.tsx b/examples/ExpoChatGPTSample/screens/ChatContent.tsx index 2041540bf9..efcea65fff 100644 --- a/examples/ExpoChatGPTSample/screens/ChatContent.tsx +++ b/examples/ExpoChatGPTSample/screens/ChatContent.tsx @@ -24,6 +24,7 @@ import { ComposerView, isAIConversationAnswering, useAIConversationState, + useAITranslation, } from 'stream-chat-expo/ai-components'; import { AIConversation } from 'stream-chat-expo/ai-components/conversation'; @@ -51,6 +52,7 @@ const ComposerArea = ({ failure: string | undefined; }) => { const { error } = useAIConversationState(conversation, errorSelector); + const { t } = useAITranslation(); const safeAreaInsets = useSafeAreaInsets(); const insets = useMemo( @@ -73,7 +75,7 @@ const ComposerArea = ({ {failure ? {failure} : null} {error ? ( conversation.retry()}> - {error.message} + {t(error.message)} Try again ) : null} diff --git a/examples/ExpoCrewAssistant/screens/ChatContent.tsx b/examples/ExpoCrewAssistant/screens/ChatContent.tsx index a53284771f..51de24037a 100644 --- a/examples/ExpoCrewAssistant/screens/ChatContent.tsx +++ b/examples/ExpoCrewAssistant/screens/ChatContent.tsx @@ -46,6 +46,7 @@ import { ComposerView, isAIConversationAnswering, useAIConversationState, + useAITranslation, } from 'stream-chat-expo/ai-components'; import { AIConversation, @@ -79,6 +80,7 @@ const ComposerArea = ({ failure: string | undefined; }) => { const { error } = useAIConversationState(conversation, errorSelector); + const { t } = useAITranslation(); const safeAreaInsets = useSafeAreaInsets(); const insets = useMemo( @@ -102,7 +104,7 @@ const ComposerArea = ({ {failure ? {failure} : null} {error ? ( conversation.retry()}> - {error.message} + {t(error.message)} Try again ) : null} diff --git a/package/shared-native/android/PerfTextViewManager.kt b/package/shared-native/android/PerfTextViewManager.kt index 76a03d5df8..6aa7c7963d 100644 --- a/package/shared-native/android/PerfTextViewManager.kt +++ b/package/shared-native/android/PerfTextViewManager.kt @@ -83,7 +83,7 @@ class PerfTextViewManager : SimpleViewManager() { ): Long { val view = PerfTextView(context) - // Apply props that influence size (text, fontSize, lineHeight) + // Apply props that influence size (text, fontSize, fontFamily, lineHeight) props?.let { p -> if (p.hasKey("fontSize") && !p.isNull("fontSize")) { val fs = p.getDouble("fontSize") @@ -92,9 +92,13 @@ class PerfTextViewManager : SimpleViewManager() { } } - if (p.hasKey("fontFamily") && !p.isNull("fontFamily")) { - view.setFontFamilyCompat(p.getString("fontFamily")) - } + view.setFontFamilyCompat( + if (p.hasKey("fontFamily") && !p.isNull("fontFamily")) { + p.getString("fontFamily")?.takeIf { it.isNotEmpty() } + } else { + null + }, + ) if (p.hasKey("lineHeight") && !p.isNull("lineHeight")) { val lh = p.getDouble("lineHeight") @@ -137,7 +141,7 @@ class PerfTextViewManager : SimpleViewManager() { val measuredWidthDp = ceil(PixelUtil.toDIPFromPixel(view.measuredWidth.toFloat())) val measuredHeightDp = ceil(PixelUtil.toDIPFromPixel(view.measuredHeight.toFloat())) - return YogaMeasureOutput.make(measuredWidthDp * 1.6f, measuredHeightDp * 1.0f) + return YogaMeasureOutput.make(measuredWidthDp, measuredHeightDp) } } diff --git a/package/shared-native/android/jni/PerfTextMeasurementManager.cpp b/package/shared-native/android/jni/PerfTextMeasurementManager.cpp index d0a5fb5324..f79a44d9a1 100644 --- a/package/shared-native/android/jni/PerfTextMeasurementManager.cpp +++ b/package/shared-native/android/jni/PerfTextMeasurementManager.cpp @@ -42,6 +42,7 @@ Size PerfTextMeasurementManager::measure( serializedProps["text"] = props.text; serializedProps["fontSize"] = props.fontSize; serializedProps["lineHeight"] = props.lineHeight; + serializedProps["fontFamily"] = props.fontFamily; auto propsRNM = ReadableNativeMap::newObjectCxxArgs(serializedProps); local_ref propsRM = diff --git a/package/src/a2ui/A2UIRenderer.tsx b/package/src/a2ui/A2UIRenderer.tsx index e50d8b7bae..7ec153f93f 100644 --- a/package/src/a2ui/A2UIRenderer.tsx +++ b/package/src/a2ui/A2UIRenderer.tsx @@ -19,7 +19,7 @@ import { supportsA2UIVersion, } from './A2UIStream'; import { A2UISurfaceView } from './A2UISurface'; -import { A2UI_DEFAULT_CATALOG_ID, a2uiTypeName } from './catalogDocument'; +import { A2UI_DEFAULT_CATALOG_ID, A2UI_WEB_CATALOG_ID, a2uiTypeName } from './catalogDocument'; import { a2uiCatalogsOf } from './catalogs'; import { A2UI_BASIC_COMPONENTS } from './components/basicComponents'; import type { A2UIComponents, A2UIIcons } from './components/types'; @@ -44,7 +44,7 @@ import { import type { AIUIEngine, AIUIRenderer } from '../ai-components/generative-ui/AIUIRenderer'; import { primitives } from '../ai-components/internal/tokens'; -export { A2UI_DEFAULT_CATALOG_ID }; +export { A2UI_DEFAULT_CATALOG_ID, A2UI_WEB_CATALOG_ID }; /** What went wrong in a reply's A2UI, as the A2UI specification reports it to an agent. */ export type A2UIError = { @@ -78,7 +78,7 @@ export type A2UIRendererOptions = { * answers to write as UI. */ instructions?: string; - /** The locale numbers and dates are formatted in. `en-US` by default. */ + /** The locale numbers and dates are formatted in. The device's by default. */ locale?: string; /** * Decides what an Image, Video or AudioPlayer loads for the link the agent wrote: the address diff --git a/package/src/a2ui/__tests__/A2UIInstructions.test.ts b/package/src/a2ui/__tests__/A2UIInstructions.test.ts index 62130e400f..6a3813cb8f 100644 --- a/package/src/a2ui/__tests__/A2UIInstructions.test.ts +++ b/package/src/a2ui/__tests__/A2UIInstructions.test.ts @@ -3,6 +3,7 @@ import { parseAIUIComponentDefinition } from '../../ai-components'; import { ohridLines, uiComponents } from '../../mock-builders/a2ui/ohrid'; import { A2UI_DEFAULT_CATALOG_ID, + A2UI_WEB_CATALOG_ID, a2uiInstructions, A2UIInstructionsError, a2uiProblems, @@ -217,6 +218,19 @@ describe('a2uiProblems', () => { ].join('\n'), ), ).toEqual([expect.stringMatching(/^surface s is created again/)]); + expect( + await a2uiProblems( + [ + line({ createSurface: { catalogId: A2UI_WEB_CATALOG_ID, surfaceId: 's' } }), + line({ + updateComponents: { + components: [{ component: 'Text', id: 'root', text: 'One' }], + surfaceId: 's', + }, + }), + ].join('\n'), + ), + ).toEqual([]); expect( await a2uiProblems( [ diff --git a/package/src/a2ui/__tests__/A2UIRenderer.test.tsx b/package/src/a2ui/__tests__/A2UIRenderer.test.tsx index fe9269d136..58b60579a6 100644 --- a/package/src/a2ui/__tests__/A2UIRenderer.test.tsx +++ b/package/src/a2ui/__tests__/A2UIRenderer.test.tsx @@ -24,7 +24,12 @@ import basicCatalogJSON from '../basicCatalog.json'; import { a2uiAppCatalogDocument, a2uiTypeName } from '../catalogDocument'; import { formatMoment, parseMoment } from '../components/DateTimeInput'; import { A2UI_ICON_PATHS } from '../components/iconPaths'; -import { A2UI_DEFAULT_CATALOG_ID, type A2UIRendererOptions, createA2UIRenderer } from '../index'; +import { + A2UI_DEFAULT_CATALOG_ID, + A2UI_WEB_CATALOG_ID, + type A2UIRendererOptions, + createA2UIRenderer, +} from '../index'; import { withoutPaths, withoutSecrets } from '../secrets'; const definitionOf = (name: string) => { @@ -322,6 +327,20 @@ describe('the A2UI versions and catalog it reads', () => { }); describe('a reply written in A2UI', () => { + it('draws a surface that names the web SDK’s catalog id', async () => { + const { errors, ui } = setup(); + await renderReply(ui, [ + line({ createSurface: { catalogId: A2UI_WEB_CATALOG_ID, surfaceId: 's' } }), + line({ + updateComponents: { + components: [{ component: 'Text', id: 'root', text: 'From the web agent' }], + surfaceId: 's', + }, + }), + ]); + await waitFor(() => expect(screen.getByText(/From the web agent/)).toBeTruthy()); + expect(errors).toEqual([]); + }); it('shows the Ohrid plan with the basic catalog and the app’s components', async () => { const { errors, ui } = setup(); await renderReply(ui, ohridLines); @@ -880,6 +899,9 @@ describe('the app’s tools in a reply’s UI', () => { expect(instructions).toContain('"weather"'); expect(instructions).toContain('The weather in a city today.'); expect(instructions).toContain('"city"'); + expect(instructions).toContain("## The app's tools"); + expect(instructions).toContain('- `weather` (read-only): The weather in a city today.'); + expect(instructions).toContain('Any other runs only from a Button'); }); it('shows the rest of the UI when a tool fails, and says so', async () => { @@ -1296,6 +1318,29 @@ describe('the basic catalog', () => { expect(screen.getByText(/Saturday 14 November/)).toBeTruthy(); }); + it('formats yen without decimals, and epoch timestamps and times of day', async () => { + const { ui } = setup({ locale: 'en-US' }); + const text = (id: string, call: string, args: Record) => ({ + component: 'Text', + id, + text: { '@call': call, args }, + }); + await renderReply( + ui, + surfaceOf([ + { children: ['yen', 'seconds', 'millis', 'time'], component: 'Column', id: 'root' }, + text('yen', 'formatCurrency', { currency: 'JPY', value: 1500 }), + text('seconds', 'formatDate', { format: 'yyyy-MM-dd', value: 1_763_078_400 }), + text('millis', 'formatDate', { format: 'd MMMM', value: 1_763_078_400_000 }), + text('time', 'formatDate', { format: 'h:mm a', value: '18:30' }), + ]), + ); + await waitFor(() => expect(screen.getByText('¥1,500')).toBeTruthy()); + expect(screen.getByText('2025-11-14')).toBeTruthy(); + expect(screen.getByText('14 November')).toBeTruthy(); + expect(screen.getByText('6:30 PM')).toBeTruthy(); + }); + it('keeps what the person types in the data model, and holds a button until its checks pass', async () => { const { sent, ui } = setup(); await renderReply( diff --git a/package/src/a2ui/basicFunctions.ts b/package/src/a2ui/basicFunctions.ts index 42ef72dc91..e0a3927a92 100644 --- a/package/src/a2ui/basicFunctions.ts +++ b/package/src/a2ui/basicFunctions.ts @@ -33,11 +33,20 @@ import { isSignal, } from '@a2ui/web_core'; -/** The locale formatting falls back to, so it never depends on the host's. */ +/** The locale formatting falls back to when neither the app nor the device names one it has. */ const DEFAULT_LOCALE = 'en-US'; -const resolveLocale = (locale?: string) => { - if (!locale) return DEFAULT_LOCALE; +// The device's locale, as the person reads numbers and dates elsewhere. +const deviceLocale = () => { + try { + return Intl.DateTimeFormat().resolvedOptions().locale || DEFAULT_LOCALE; + } catch { + return DEFAULT_LOCALE; + } +}; + +const resolveLocale = (asked?: string) => { + const locale = asked || deviceLocale(); try { return Intl.NumberFormat.supportedLocalesOf(locale).length > 0 ? locale : DEFAULT_LOCALE; } catch { @@ -146,18 +155,21 @@ const formatNumber = ( useGrouping: grouping, }).format(value); +// A currency's amount with `decimals` decimals, or as many as the currency has, such as none for +// yen. const formatCurrency = ( value: number, code: unknown, - decimals = 2, + decimals?: number, grouping?: boolean, locale = DEFAULT_LOCALE, ) => { if (isNaN(value)) return ''; const currency = String(code).toUpperCase(); const options = { - maximumFractionDigits: decimals, - minimumFractionDigits: decimals, + ...(decimals === undefined + ? {} + : { maximumFractionDigits: decimals, minimumFractionDigits: decimals }), useGrouping: grouping, }; try { @@ -242,9 +254,28 @@ const parseTimestamp = (value: string) => { return fieldsRoundTrip(value, shifted) ? { instant, shifted } : null; }; +// Epoch milliseconds start beyond this; a smaller number is epoch seconds. +const EPOCH_MILLISECONDS_FROM = 100_000_000_000; +const TIME_OF_DAY = /^\d{2}:\d{2}(:\d{2})?$/; + +// An epoch timestamp, in seconds or milliseconds, as the device's clock reads it. +const parseEpoch = (value: number) => { + const instant = new Date(Math.abs(value) > EPOCH_MILLISECONDS_FROM ? value : value * 1000); + if (isNaN(instant.getTime())) return null; + return { + instant, + shifted: new Date(instant.getTime() - instant.getTimezoneOffset() * 60_000), + }; +}; + const formatDate = (value: unknown, pattern = 'yyyy-MM-dd', locale = DEFAULT_LOCALE) => { - if (!value) return ''; - const parsed = parseTimestamp(String(value)); + if (value === undefined || value === null || value === '') return ''; + const parsed = + typeof value === 'number' + ? parseEpoch(value) + : TIME_OF_DAY.test(String(value)) + ? parseTimestamp(`1970-01-01T${String(value)}`) + : parseTimestamp(String(value)); if (!parsed) return ''; const { instant, shifted } = parsed; const format = pattern || 'yyyy-MM-dd'; @@ -372,7 +403,7 @@ type Execute = (args: Args, context: DataContext) => unknown; /** * The basic catalog's functions, for the function definitions `apis` of its document, with - * `openUrl` run by `openUrl`. Formatting follows `locale`, or `en-US`. + * `openUrl` run by `openUrl`. Formatting follows `locale`, or the device's. */ export const basicFunctionsOf = ( apis: Map, @@ -388,7 +419,7 @@ export const basicFunctionsOf = ( formatCurrency( number(value), currency, - optionalNumber(decimals) ?? 2, + optionalNumber(decimals), grouping as boolean | undefined, resolved, ), diff --git a/package/src/a2ui/catalogDocument.ts b/package/src/a2ui/catalogDocument.ts index 2512f3758c..9d3bb94931 100644 --- a/package/src/a2ui/catalogDocument.ts +++ b/package/src/a2ui/catalogDocument.ts @@ -13,6 +13,16 @@ import type { AIClientToolDefinition } from '../ai-components/tool-calls/AIClien /** The id of the catalog with the app's components, unless the renderer is given another. */ export const A2UI_DEFAULT_CATALOG_ID = 'getstream.io:app-catalog'; +/** + * The id the web SDK names the same catalog with. A renderer on the default id draws surfaces + * that name it too, so an agent set up for a web app shows its UI here as well. + */ +export const A2UI_WEB_CATALOG_ID = 'stream-app-catalog'; + +/** The ids a surface may name for a renderer whose catalog is `catalogId`. */ +export const a2uiAppCatalogIds = (catalogId: string) => + catalogId === A2UI_DEFAULT_CATALOG_ID ? [catalogId, A2UI_WEB_CATALOG_ID] : [catalogId]; + const definitionOf = (component: AIUIComponent | AIUIComponentDefinition) => 'definition' in component ? component.definition : component; diff --git a/package/src/a2ui/catalogs.ts b/package/src/a2ui/catalogs.ts index 6b94bcb276..53a2121f0e 100644 --- a/package/src/a2ui/catalogs.ts +++ b/package/src/a2ui/catalogs.ts @@ -10,7 +10,7 @@ import { import { A2UI_VERSION } from './A2UIStream'; import basicCatalogDocument from './basicCatalog.json'; import { basicFunctionsOf } from './basicFunctions'; -import { a2uiAppCatalogDocument, a2uiTypeName } from './catalogDocument'; +import { a2uiAppCatalogDocument, a2uiAppCatalogIds, a2uiTypeName } from './catalogDocument'; import { secretValuesOf, withoutSecrets } from './secrets'; import type { AIUIComponent } from '../ai-components/generative-ui/AIUIComponent'; @@ -176,10 +176,6 @@ export const a2uiCatalogsOf = ({ } const basic = [...definitions.components.values()].filter(({ name }) => !removed.has(name)); const all = [...basic, ...appApis] as ComponentApi[]; - return [ - new Catalog(catalogId, A2UI_VERSION, all, functions), - ...(catalogId === A2UI_BASIC_CATALOG_ID - ? [] - : [new Catalog(A2UI_BASIC_CATALOG_ID, A2UI_VERSION, all, functions)]), - ]; + const ids = new Set([...a2uiAppCatalogIds(catalogId), A2UI_BASIC_CATALOG_ID]); + return [...ids].map((id) => new Catalog(id, A2UI_VERSION, all, functions)); }; diff --git a/package/src/a2ui/index.ts b/package/src/a2ui/index.ts index da9263d6c6..4a57a03c6d 100644 --- a/package/src/a2ui/index.ts +++ b/package/src/a2ui/index.ts @@ -2,6 +2,7 @@ // agent never writes A2UI doesn't bundle the A2UI core. export { A2UI_DEFAULT_CATALOG_ID, + A2UI_WEB_CATALOG_ID, type A2UIError, type A2UIRendererOptions, createA2UIRenderer, diff --git a/package/src/a2ui/instructions.ts b/package/src/a2ui/instructions.ts index e62571ea70..7859c94880 100644 --- a/package/src/a2ui/instructions.ts +++ b/package/src/a2ui/instructions.ts @@ -20,12 +20,17 @@ import { writing, } from './A2UIInstructions'; import { A2UI_VERSION, a2uiMessagesIn, normalizeA2UIMessage } from './A2UIStream'; -import { A2UI_DEFAULT_CATALOG_ID, a2uiAppCatalogDocument } from './catalogDocument'; +import { + A2UI_DEFAULT_CATALOG_ID, + a2uiAppCatalogDocument, + a2uiAppCatalogIds, +} from './catalogDocument'; import { basicCatalogDefinitions } from './catalogs'; import type { AIUIComponent } from '../ai-components/generative-ui/AIUIComponent'; import type { AIUIComponentDefinition } from '../ai-components/generative-ui/AIUIComponentDefinition'; import { isRecord } from '../ai-components/generative-ui/AIUIHelpers'; +import type { AIClientToolDefinition } from '../ai-components/tool-calls/AIClientToolRunner'; type Entry = ComponentApi; @@ -86,6 +91,28 @@ export const a2uiCatalogFor = ({ }); }; +// The app's tools as the model is told them, each marked when it only reads, with the rule for +// where they run: the host runs a tool that changes something only when the person taps for it. +const toolsSection = (tools: AIClientToolDefinition[]) => { + if (!tools.length) return []; + const lines = tools + .slice() + .sort((a, b) => a.name.localeCompare(b.name)) + .map(({ annotations, description, name }) => { + const readOnly = annotations?.readOnlyHint === true ? ' (read-only)' : ''; + return `- \`${name}\`${readOnly}${description ? `: ${description}` : ''}`; + }); + return [ + [ + "## The app's tools", + '', + 'These run on the person\'s device. Call one with `{"@call": "", "args": {...}}`. A read-only tool runs wherever you call it, such as to show its result in a Text. Any other runs only from a Button\'s `functionCall` action, when the person taps it.', + '', + ...lines, + ].join('\n'), + ]; +}; + /** * The instructions that teach an agent's model to write A2UI the app's renderer shows, in * Markdown: how a reply carries its UI, the protocol, the catalog's guidelines and the catalog @@ -109,6 +136,9 @@ export const a2uiInstructionsText = (config: A2UIInstructionsConfig = {}) => { A2UI_PROTOCOL_OVERVIEW, ...(typeof instructions === 'string' && instructions ? [instructions] : []), ...(notes.length ? [`## The app's components\n\n${notes.join('\n')}`] : []), + ...toolsSection( + (config.tools ?? []).filter(({ name }) => !basicCatalogDefinitions().functions.has(name)), + ), `## The catalog \`${catalog.id}\`\n\n\`\`\`json\n${JSON.stringify(sorted(document))}\n\`\`\``, ...(rules.length ? [`## Rules\n\n${rules.map((rule, index) => `${index + 1}. ${rule}`).join('\n')}`] @@ -183,7 +213,10 @@ export const a2uiProblemsWith = (text: string, catalog: Catalog) => { created = true; const components = componentsOf(body.components); problems.push(...duplicates(components, surfaceId)); - if (typeof body.catalogId === 'string' && body.catalogId !== catalog.id) { + if ( + typeof body.catalogId === 'string' && + !a2uiAppCatalogIds(catalog.id).includes(body.catalogId) + ) { problems.push( `surface ${surfaceId} names the catalog ${body.catalogId}, not ${catalog.id}`, ); diff --git a/package/src/ai-components/README.md b/package/src/ai-components/README.md index 044d6e1829..4db0c07116 100644 --- a/package/src/ai-components/README.md +++ b/package/src/ai-components/README.md @@ -152,8 +152,8 @@ Under the hood, it implements a letter-by-letter typewriter animation with a cha | `paragraphNumberOfLines` | `number` | no | The `numberOfLines` applied to markdown `Paragraph` and `Text` components. Particularly useful if we want to display the same message, but in a "cut" fashion (for example when replying to someone). | | `rules` | [`MarkdownRules`](./markdown/types.ts) | no | An object of `MarkdownRules` that is then going to be deeply merged with our default rules, based on the [`SimpleMarkdown` parsing engine](https://github.com/Khan/perseus/tree/main/packages/simple-markdown#adding-a-simple-extension). Can be used to add custom rules or change existing rules. You can disable a rule by passing `{ [ruleName]: { match: () => null }}`. | | `onLink` | `(url: string) => void` | no | A function that is going to be invoked whenever a link is pressed within markdown parsed text. | -| `letterInterval` | `number` | no | A number signifying the interval at which the typewriter animation is going to render characters. Defaults to `0` | -| `renderingLetterCount` | `number` | no | A number signifying the number of letters that are going to be rendered per tick of the interval during the typewriter animation. Defaults to `2` | +| `letterInterval` | `number` | no | The milliseconds between steps, for a fixed pace. Without it and `renderingLetterCount`, the text keeps up with what arrives: each new part is typed out within about 0.4 seconds. | +| `renderingLetterCount` | `number` | no | The letters typed each step, for a fixed pace. Defaults to `2` when `letterInterval` is given. | #### Example @@ -238,7 +238,7 @@ It takes a single property named `options` that has the following keys: | Name | Type | Required | Description | | ------------------ | --------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `language` | `string` | no | The language we want to transcribe from. It will default to `en-US`. | +| `language` | `string` | no | The language we want to transcribe from. Defaults to the device's language. | | `interimResults` | `boolean` | no | A boolean signifying whether we want to receive the intermediate results from the transcription or just the final result when the transcription is deemed done. Defaults to `true` | | `silenceTimeoutMs` | `number` | no | A number signifying the number of milliseconds of silence until transcription is deemed finished. Defaults to `2500`. | @@ -335,7 +335,7 @@ The label of the message an action sends is the text of the control the person t | `components` | Your own views for basic components, by type, such as `{ Card: MyCard }`. Each gets the node and its resolved props. | | `icons` | Your own icons for the names `Icon` takes, over the Material icons drawn by default. | | `removing` | Basic components or functions to leave out, such as `['Video']`. The agent's instructions leave them out too. | -| `locale` | The locale numbers and dates are formatted in. `en-US` by default. | +| `locale` | The locale numbers and dates are formatted in. The device's by default. | | `mediaUrl` | Decides what an `Image`, `Video` or `AudioPlayer` loads for a link. Public web links only by default. | | `instructions` | What you tell the model before A2UI's own instructions, such as which answers to write as UI. | | `catalogId` | The id of the catalog with your own components. | diff --git a/package/src/ai-components/StreamingMessageView.tsx b/package/src/ai-components/StreamingMessageView.tsx index 24c5b1e37e..65a3df28fb 100644 --- a/package/src/ai-components/StreamingMessageView.tsx +++ b/package/src/ai-components/StreamingMessageView.tsx @@ -1,4 +1,11 @@ -import React, { useEffect, useMemo, useRef, useState } from 'react'; +import React, { + forwardRef, + useEffect, + useImperativeHandle, + useMemo, + useRef, + useState, +} from 'react'; import { Linking } from 'react-native'; @@ -9,15 +16,30 @@ import type { MarkdownRules } from './markdown'; export type UseStreamingMessageProps = { text: string; + /** + * The milliseconds between steps, for a fixed pace. Without it and `renderingLetterCount`, the + * text keeps up with what arrives: each new part is typed out within about 0.4 seconds. + */ letterInterval?: number; + /** The letters typed each step, for a fixed pace. */ renderingLetterCount?: number; }; +// How long text that arrives takes to type out, when no fixed pace is given. +const CATCH_UP_MS = 400; +// How often the text steps on when it keeps up with what arrives: about one frame. +const FRAME_MS = 16; + export const useStreamingMessage = ({ - letterInterval = 0, - renderingLetterCount = 2, + letterInterval, + renderingLetterCount, text, -}: UseStreamingMessageProps): { isWriting: boolean; streamedMessageText: string } => { +}: UseStreamingMessageProps): { + isWriting: boolean; + /** Shows the whole text at once, such as when the person doesn't want to wait for it. */ + skipAnimation: () => void; + streamedMessageText: string; +} => { const [streamedMessageText, setStreamedMessageText] = useState(text); const textCursor = useRef(text.length); // How much of the text is typed out, to say whether it still is. @@ -34,76 +56,110 @@ export const useStreamingMessage = ({ } } const typedToward = useRef(text); + const isFixedPace = letterInterval !== undefined || renderingLetterCount !== undefined; useEffect(() => { const textLength = text.length; if (!text.startsWith(typedToward.current)) textCursor.current = textLength; typedToward.current = text; - const interval = setInterval(() => { - if (!text || textCursor.current >= textLength) { - clearInterval(interval); - } - const newCursorValue = textCursor.current + renderingLetterCount; - const newText = text.substring(0, newCursorValue); - textCursor.current += newText.length - textCursor.current; - const codeBlockCounts = (newText.match(/```/g) || []).length; - const shouldOptimisticallyCloseCodeBlock = codeBlockCounts > 0 && codeBlockCounts % 2 > 0; - setStreamedMessageText(shouldOptimisticallyCloseCodeBlock ? newText + '```' : newText); - setTypedLength(textCursor.current); - }, letterInterval); + const step = isFixedPace + ? (renderingLetterCount ?? 2) + : Math.max(1, Math.ceil(((textLength - textCursor.current) * FRAME_MS) / CATCH_UP_MS)); + const interval = setInterval( + () => { + if (!text || textCursor.current >= textLength) { + clearInterval(interval); + } + const newCursorValue = textCursor.current + step; + const newText = text.substring(0, newCursorValue); + textCursor.current += newText.length - textCursor.current; + const codeBlockCounts = (newText.match(/```/g) || []).length; + const shouldOptimisticallyCloseCodeBlock = codeBlockCounts > 0 && codeBlockCounts % 2 > 0; + setStreamedMessageText(shouldOptimisticallyCloseCodeBlock ? newText + '```' : newText); + setTypedLength(textCursor.current); + }, + isFixedPace ? (letterInterval ?? 0) : FRAME_MS, + ); return () => { clearInterval(interval); }; - }, [letterInterval, renderingLetterCount, text]); + }, [isFixedPace, letterInterval, renderingLetterCount, text]); + + const skipAnimation = useStableCallback(() => { + textCursor.current = text.length; + setStreamedMessageText(text); + setTypedLength(text.length); + }); - return { isWriting: typedLength < text.length, streamedMessageText }; + return { isWriting: typedLength < text.length, skipAnimation, streamedMessageText }; }; -export const StreamingMessageView = ({ - text, - paragraphNumberOfLines, - rules, - onLink: onLinkParam, - letterInterval, - onWritingChange, - renderingLetterCount, -}: { +/** What a `StreamingMessageView`'s ref offers. */ +export type StreamingMessageViewRef = { + /** Shows the whole text at once, such as when the person doesn't want to wait for it. */ + skipAnimation: () => void; +}; + +export type StreamingMessageViewProps = { text: string; paragraphNumberOfLines?: number; rules?: MarkdownRules; onLink?: (url: string) => void; + /** + * The milliseconds between steps, for a fixed pace. Without it and `renderingLetterCount`, the + * text keeps up with what arrives: each new part is typed out within about 0.4 seconds. + */ letterInterval?: number; /** Called when the text starts or stops being typed out, such as to show what follows it once it is. */ onWritingChange?: (isWriting: boolean) => void; + /** The letters typed each step, for a fixed pace. */ renderingLetterCount?: number; -}) => { - const markdownText = useMemo(() => generateMarkdownText(text), [text]); - - const { isWriting, streamedMessageText } = useStreamingMessage({ - text: markdownText ?? '', - letterInterval, - renderingLetterCount, - }); - const reportWriting = useStableCallback((writing: boolean) => onWritingChange?.(writing)); - useEffect(() => { - reportWriting(isWriting); - }, [isWriting, reportWriting]); - - const onLink = useStableCallback((url: string): void | Promise => - onLinkParam - ? onLinkParam(url) - : Linking.canOpenURL(url).then((canOpenUrl) => { - if (canOpenUrl) { - return Linking.openURL(url); - } - return undefined; - }), - ); - - return ( - - {streamedMessageText} - - ); }; + +export const StreamingMessageView = forwardRef( + ( + { + text, + paragraphNumberOfLines, + rules, + onLink: onLinkParam, + letterInterval, + onWritingChange, + renderingLetterCount, + }, + ref, + ) => { + const markdownText = useMemo(() => generateMarkdownText(text), [text]); + + const { isWriting, skipAnimation, streamedMessageText } = useStreamingMessage({ + text: markdownText ?? '', + letterInterval, + renderingLetterCount, + }); + useImperativeHandle(ref, () => ({ skipAnimation }), [skipAnimation]); + const reportWriting = useStableCallback((writing: boolean) => onWritingChange?.(writing)); + useEffect(() => { + reportWriting(isWriting); + }, [isWriting, reportWriting]); + + const onLink = useStableCallback((url: string): void | Promise => + onLinkParam + ? onLinkParam(url) + : Linking.canOpenURL(url).then((canOpenUrl) => { + if (canOpenUrl) { + return Linking.openURL(url); + } + return undefined; + }), + ); + + return ( + + {streamedMessageText} + + ); + }, +); + +StreamingMessageView.displayName = 'StreamingMessageView'; diff --git a/package/src/ai-components/__tests__/StreamingMessageView.test.tsx b/package/src/ai-components/__tests__/StreamingMessageView.test.tsx index eb8be0fd01..86def4d0b0 100644 --- a/package/src/ai-components/__tests__/StreamingMessageView.test.tsx +++ b/package/src/ai-components/__tests__/StreamingMessageView.test.tsx @@ -21,6 +21,51 @@ describe('ai-components streaming', () => { }); describe('useStreamingMessage', () => { + it('keeps up with what arrives when no pace is given', () => { + const { result, rerender } = renderHook( + ({ text }: { text: string }) => useStreamingMessage({ text }), + { initialProps: { text: '' } }, + ); + const chunk = 'x'.repeat(250); + rerender({ text: chunk }); + act(() => { + jest.advanceTimersByTime(16); + }); + // 250 letters in 0.4 seconds: 10 a frame. + expect(result.current.streamedMessageText).toHaveLength(10); + expect(result.current.isWriting).toBe(true); + + act(() => { + jest.advanceTimersByTime(420); + }); + expect(result.current.streamedMessageText).toBe(chunk); + expect(result.current.isWriting).toBe(false); + }); + + it('types at least a letter a frame when little arrives', () => { + const { result, rerender } = renderHook( + ({ text }: { text: string }) => useStreamingMessage({ text }), + { initialProps: { text: 'Hi' } }, + ); + rerender({ text: 'Hi there' }); + act(() => { + jest.advanceTimersByTime(16); + }); + expect(result.current.streamedMessageText).toBe('Hi '); + }); + + it('shows the whole text at once when the animation is skipped', () => { + const { result, rerender } = renderHook( + ({ text }: { text: string }) => + useStreamingMessage({ letterInterval: 50, renderingLetterCount: 1, text }), + { initialProps: { text: '' } }, + ); + rerender({ text: 'A long answer' }); + act(() => result.current.skipAnimation()); + expect(result.current.streamedMessageText).toBe('A long answer'); + expect(result.current.isWriting).toBe(false); + }); + it('renders the initial text immediately and animates text appended later', () => { const { result, rerender } = renderHook( ({ text }: { text: string }) => diff --git a/package/src/ai-components/__tests__/clientToolRunner.test.ts b/package/src/ai-components/__tests__/clientToolRunner.test.ts index 0d4efa0fe0..14b4309f67 100644 --- a/package/src/ai-components/__tests__/clientToolRunner.test.ts +++ b/package/src/ai-components/__tests__/clientToolRunner.test.ts @@ -68,6 +68,25 @@ describe('AIClientToolRunner', () => { }); }); + it('leaves a call without an id of its own alone', async () => { + const tool = countingTool(); + const runner = runnerWith([tool]); + const send = jest.fn, Parameters>(() => + Promise.resolve(), + ); + + runner.run([step({ id: undefined })], send); + await settle(); + + expect(tool.run).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + }); + + it('needs at least one attempt', () => { + expect(() => runnerWith([countingTool()], 0)).toThrow(RangeError); + expect(() => runnerWith([countingTool()], 1)).not.toThrow(); + }); + it('sends a result that could not be sent again, without running the tool again', async () => { const tool = countingTool(); const runner = runnerWith([tool]); diff --git a/package/src/ai-components/__tests__/conversation.test.tsx b/package/src/ai-components/__tests__/conversation.test.tsx index 27c37ee595..72c1beb2b1 100644 --- a/package/src/ai-components/__tests__/conversation.test.tsx +++ b/package/src/ai-components/__tests__/conversation.test.tsx @@ -150,6 +150,54 @@ describe('AIConversation', () => { expect(screen.getAllByText('footer')).toHaveLength(1); }); + it('keeps a turn that starts with a reply when older messages load above it', () => { + const starts = new Set(); + const tracker = new AIPendingMessageTracker(); + const page = [ + reply('a1', 'It is noon.'), + user('u2', 'And the weather?'), + reply('a2', 'Sunny.'), + ]; + expect(turnsOf(tracker.entries(page), starts).map((turn) => turn.id)).toEqual(['a1', 'u2']); + + const older = [user('u1', 'What time is it?'), ...page]; + expect(turnsOf(tracker.entries(older), starts).map((turn) => turn.id)).toEqual([ + 'u1', + 'a1', + 'u2', + ]); + }); + + it('takes the copy of a message on its way that arrives in the update that first passes it', () => { + const tracker = new AIPendingMessageTracker(); + const history = [user('u1', 'Hello'), reply('a1', 'Hi!')]; + tracker.entries(history); + const pending = { id: 'local-1', text: 'Plan my day' }; + const entries = tracker.entries([...history, user('stream-u2', 'Plan my day')], pending); + expect(entries.map((entry) => entry.id)).toEqual(['u1', 'a1', 'local-1']); + expect(entries.filter((entry) => entry.pending)).toHaveLength(0); + }); + + it('takes the copy of a message on its way from a window of the same length', () => { + const tracker = new AIPendingMessageTracker(); + tracker.entries([user('u1', 'Hello'), reply('a1', 'Hi!')]); + const pending = { id: 'local-1', text: 'Plan my day' }; + // A reconnect reloads the latest page: as many messages, the newest one the copy. + const entries = tracker.entries( + [reply('a1', 'Hi!'), user('stream-u2', 'Plan my day')], + pending, + ); + expect(entries.map((entry) => entry.id)).toEqual(['a1', 'local-1']); + }); + + it('takes no older question for the copy of a message on its way when it mounts with one', () => { + const history = [user('u1', 'Hello'), reply('a1', 'Hi!'), user('u2', 'Plan my day')]; + const pending = { id: 'local-3', text: 'Plan my week' }; + const entries = new AIPendingMessageTracker().entries(history, pending); + expect(entries.map((entry) => entry.id)).toEqual(['u1', 'a1', 'u2', 'local-3']); + expect(entries[3].pending).toBe(pending); + }); + it('shows a pending message with the agent thinking, until its reply starts', async () => { const mounts: string[] = []; const UserMessage = ({ text }: AIConversationUserMessageProps) => { @@ -690,7 +738,7 @@ describe('AIConversation with a controller', () => { for (const text of ['Here is', 'Here is your', 'Here is your plan.']) { await update([...history, reply('a2', text, { isGenerating: true })]); } - expect(screen.getByText('Here is your plan.')).toBeTruthy(); + expect(await screen.findByText('Here is your plan.')).toBeTruthy(); expect(renders).toBe(before); controller.dispose(); }); diff --git a/package/src/ai-components/__tests__/conversationController.test.ts b/package/src/ai-components/__tests__/conversationController.test.ts index 2139f2bda6..64ab58b458 100644 --- a/package/src/ai-components/__tests__/conversationController.test.ts +++ b/package/src/ai-components/__tests__/conversationController.test.ts @@ -1260,7 +1260,7 @@ describe('AIConversationController errors', () => { it('describes an agent error in the agent’s words, else its own', () => { expect(new AIAgentError('Rate limited').message).toBe('Rate limited'); - expect(new AIAgentError().message).toBe('The agent couldn’t answer. Try again.'); + expect(new AIAgentError().message).toBe("The agent couldn't answer. Try again."); expect(new AIAgentError('').agentMessage).toBeUndefined(); }); }); @@ -1298,14 +1298,14 @@ describe('AIConversationController older messages', () => { expect(transport.loads).toBe(2); }); - it('can load again when the transport fails', async () => { + it('rejects when the transport fails, and can load again', async () => { const transport = new ConversationTransport(); transport.loadError = new TransportError(); const controller = makeController(transport); transport.feed?.updateHasOlderMessages(true); - await controller.loadOlderMessages(); - await controller.loadOlderMessages(); + await expect(controller.loadOlderMessages()).rejects.toBe(transport.loadError); + await expect(controller.loadOlderMessages()).rejects.toBe(transport.loadError); expect(transport.loads).toBe(2); // Not a request of the person's. diff --git a/package/src/ai-components/__tests__/conversationUI.test.tsx b/package/src/ai-components/__tests__/conversationUI.test.tsx index 9c1a65a691..fa8f7ab828 100644 --- a/package/src/ai-components/__tests__/conversationUI.test.tsx +++ b/package/src/ai-components/__tests__/conversationUI.test.tsx @@ -3,7 +3,7 @@ import { Pressable, StyleSheet, Text, View } from 'react-native'; import { act, fireEvent, render, screen } from '@testing-library/react-native'; -import { AIConversation } from '../conversation'; +import { AIConversation, type AIConversationAssistantMessageProps } from '../conversation'; import { AIConversationController, type AIMessage, @@ -191,16 +191,18 @@ describe('AIConversation with generated UI', () => { it('has the reply’s actions under a finished UI-only reply, not while its UI is written', async () => { const ui = makeHost(); const question = user('u1', 'Plan'); + const ReplyActions = () => Rate this plan; const { rerender } = await renderConversation( , ); - expect(screen.queryByRole('button', { name: /copy/i })).toBeNull(); + expect(screen.queryByText('Rate this plan')).toBeNull(); await rerender( { question, reply('a1', '', { content: 'Ohrid', format: 'a2ui', status: 'complete' }), ]} + ReplyActions={ReplyActions} ui={ui} />, ); - expect(screen.getByRole('button', { name: /copy/i })).toBeTruthy(); + expect(screen.getByText('Rate this plan')).toBeTruthy(); + }); + + it('offers no Copy or Share under a reply that answers only in UI', async () => { + await renderConversation( + , + ); + expect(screen.getByTestId('card')).toBeTruthy(); + expect(screen.queryByRole('button', { name: /copy/i })).toBeNull(); + expect(screen.queryByRole('button', { name: /share/i })).toBeNull(); }); it('keeps the UI views of rows it reuses for other replies', async () => { @@ -327,9 +345,39 @@ describe('AIConversation with generated UI', () => { expect(screen.getByTestId('card')).toHaveTextContent('Ohrid plan'); expect(onRenderError).toHaveBeenCalledTimes(1); expect(onRenderError).toHaveBeenCalledWith(expect.any(Error), 'a1'); + expect(ui.hasFailed('a1')).toBe(true); + expect(ui.hasFailed('a2')).toBe(false); logged.mockRestore(); }); + it('tells a custom message whether its reply shows UI, or whose UI failed', async () => { + const seen = new Map(); + const AssistantMessage = ({ + message, + showsUI, + text, + uiFailed, + }: AIConversationAssistantMessageProps) => { + seen.set(message.id, { showsUI, uiFailed }); + return {text}; + }; + const ui = makeHost(); + await renderConversation( + , + ); + expect(seen.get('a1')).toEqual({ showsUI: true, uiFailed: false }); + expect(seen.get('a2')).toEqual({ showsUI: false, uiFailed: false }); + }); + it('tries a reply’s UI that failed while it was written again once it is written', async () => { const Throws = () => { throw new Error('Half a map'); diff --git a/package/src/ai-components/__tests__/dictation.test.tsx b/package/src/ai-components/__tests__/dictation.test.tsx index 4755ea5e5d..fdb26d7093 100644 --- a/package/src/ai-components/__tests__/dictation.test.tsx +++ b/package/src/ai-components/__tests__/dictation.test.tsx @@ -80,7 +80,6 @@ describe('ai-components dictation', () => { }); expect(nativeModule.start).toHaveBeenCalledWith({ interimResults: true, - language: 'en-US', silenceTimeoutMs: 2500, }); diff --git a/package/src/ai-components/__tests__/generativeUI.test.ts b/package/src/ai-components/__tests__/generativeUI.test.ts index 3c9960e69c..bb3b7642a0 100644 --- a/package/src/ai-components/__tests__/generativeUI.test.ts +++ b/package/src/ai-components/__tests__/generativeUI.test.ts @@ -20,6 +20,7 @@ import { parseAIUIComponentDefinition, parseAIUIDocument, requiredAIUIParameter, + aiUIClientToolJSON, } from '../index'; import type { AIMessage } from '../index'; @@ -602,11 +603,43 @@ describe('AIUIHost', () => { { catalog_id: 'app', components: 1, format: 'a2ui' }, { format: 'openui', instructions: 'Write OpenUI.' }, ], - tools: [], v: 1, }); expect(ui.component('place_card')?.definition).toBe(definition); }); + + it('tells the backend the app’s tools as Model Context Protocol tools', async () => { + const ui = new AIUIHost({ + renderers: [], + send: jest.fn(), + tools: [ + { definition: { name: 'local_time' }, run: () => ({ output: '12:00' }) }, + { + definition: { + annotations: { readOnlyHint: true }, + description: 'The weather in a city.', + inputSchema: { properties: { city: { type: 'string' } }, type: 'object' }, + name: 'weather', + }, + run: () => ({ output: 'Sunny' }), + }, + ], + }); + expect((await ui.capabilities()).tools).toEqual([ + { annotations: { readOnlyHint: false }, inputSchema: { type: 'object' }, name: 'local_time' }, + { + annotations: { readOnlyHint: true }, + description: 'The weather in a city.', + inputSchema: { properties: { city: { type: 'string' } }, type: 'object' }, + name: 'weather', + }, + ]); + expect(aiUIClientToolJSON({ name: 'x' })).toEqual({ + annotations: { readOnlyHint: false }, + inputSchema: { type: 'object' }, + name: 'x', + }); + }); }); describe('utf8ByteLength', () => { diff --git a/package/src/ai-components/__tests__/messagePartViews.test.tsx b/package/src/ai-components/__tests__/messagePartViews.test.tsx index 773baee37a..5601a9788b 100644 --- a/package/src/ai-components/__tests__/messagePartViews.test.tsx +++ b/package/src/ai-components/__tests__/messagePartViews.test.tsx @@ -1,5 +1,5 @@ import React, { type ReactElement } from 'react'; -import { I18nManager } from 'react-native'; +import { I18nManager, Text } from 'react-native'; import { act, fireEvent, render, screen } from '@testing-library/react-native'; @@ -124,6 +124,57 @@ describe('AIToolApproval', () => { userId: 'alex', }); + it('shows what the call sends, folding long arguments', () => { + const decide = jest.fn(() => Promise.resolve()); + const { rerender } = renderWithTheme( + , + ); + expect(screen.getByText('What it sends')).toBeTruthy(); + expect(screen.getByText(/"precision": "city"/)).toBeTruthy(); + + const long = { body: 'x'.repeat(300) }; + rerender(); + expect(screen.queryByText(/"body"/)).toBeNull(); + fireEvent.press(screen.getByRole('button', { name: 'What it sends' })); + expect(screen.getByText(/"body"/)).toBeTruthy(); + + rerender(); + expect(screen.queryByText('What it sends')).toBeNull(); + }); + + it('shows the approver’s own preview, or none, and hands it to a card of the app’s', () => { + const decide = jest.fn(() => Promise.resolve()); + const part = { ...asking, arguments: { precision: 'city' } }; + const { rerender } = renderWithTheme( + Your city only }} + part={part} + />, + ); + expect(screen.getByText('Your city only')).toBeTruthy(); + expect(screen.queryByText('What it sends')).toBeNull(); + + rerender( + null }} + part={part} + />, + ); + expect(screen.queryByText('What it sends')).toBeNull(); + + const children = jest.fn((_approval, _state, _decide, preview) => <>{preview}); + rerender( + + {children} + , + ); + expect(screen.getByText('What it sends')).toBeTruthy(); + expect(children.mock.calls[0]).toHaveLength(4); + }); + it('asks only the targeted person on the targeted install', () => { const decide = jest.fn(() => Promise.resolve()); const { rerender } = renderWithTheme(); diff --git a/package/src/ai-components/__tests__/messageViews.test.tsx b/package/src/ai-components/__tests__/messageViews.test.tsx index bfe9d12f73..1931b7e692 100644 --- a/package/src/ai-components/__tests__/messageViews.test.tsx +++ b/package/src/ai-components/__tests__/messageViews.test.tsx @@ -208,6 +208,20 @@ describe('AIReplyActions', () => { expect(screen.getByRole('button', { name: 'Copy' })).toBeTruthy(); }); + it('offers no Copy or Share for a reply without text', () => { + const onRegenerate = jest.fn(); + const { rerender } = renderWithTheme(); + expect(screen.queryByRole('button', { name: 'Copy' })).toBeNull(); + expect(screen.queryByRole('button', { name: 'Share' })).toBeNull(); + expect(screen.getByRole('button', { name: 'Regenerate' })).toBeTruthy(); + rerender( + + + , + ); + expect(screen.queryByTestId('ai-reply-actions')).toBeNull(); + }); + it('shares the text through the system share sheet', () => { const share = jest .spyOn(Share, 'share') diff --git a/package/src/ai-components/__tests__/signIn.test.tsx b/package/src/ai-components/__tests__/signIn.test.tsx index 1902715d0e..b839a7c09d 100644 --- a/package/src/ai-components/__tests__/signIn.test.tsx +++ b/package/src/ai-components/__tests__/signIn.test.tsx @@ -125,7 +125,8 @@ describe('AIToolAuthorization', () => { expect(screen.getByRole('button', { name: 'Not now' })).toBeEnabled(); }); - it('keeps connecting once connected, until the step changes', async () => { + it('keeps connecting once connected, until the step changes or 30 seconds pass', async () => { + jest.useFakeTimers(); renderWithTheme( Promise.resolve(true), decline: jest.fn() })} @@ -135,6 +136,15 @@ describe('AIToolAuthorization', () => { fireEvent.press(screen.getByRole('button', { name: 'Connect' })); await act(() => Promise.resolve()); expect(screen.getByRole('button', { name: 'Connecting...' })).toBeDisabled(); + act(() => { + jest.advanceTimersByTime(29_000); + }); + expect(screen.getByRole('button', { name: 'Connecting...' })).toBeDisabled(); + act(() => { + jest.advanceTimersByTime(1_000); + }); + expect(screen.getByRole('button', { name: 'Connect' })).toBeEnabled(); + jest.useRealTimers(); }); it('says when the sign-in could not open, and lets the person try again', async () => { diff --git a/package/src/ai-components/conversation-state/controller/AIAgentError.ts b/package/src/ai-components/conversation-state/controller/AIAgentError.ts index 840ef76013..818cb8dd2e 100644 --- a/package/src/ai-components/conversation-state/controller/AIAgentError.ts +++ b/package/src/ai-components/conversation-state/controller/AIAgentError.ts @@ -1,3 +1,9 @@ +/** + * What an `AIAgentError` says when the agent said nothing of its own, a translation key: show it + * with `t(error.message)`. + */ +export const AI_AGENT_ERROR_MESSAGE = "The agent couldn't answer. Try again."; + /** * The agent couldn't answer: it said so, with what it said as `agentMessage`, or it didn't start * its reply in time. @@ -7,7 +13,7 @@ export class AIAgentError extends Error { readonly agentMessage?: string; constructor(agentMessage?: string) { - super(agentMessage || 'The agent couldn’t answer. Try again.'); + super(agentMessage || AI_AGENT_ERROR_MESSAGE); this.name = 'AIAgentError'; this.agentMessage = agentMessage || undefined; } diff --git a/package/src/ai-components/conversation-state/controller/AIConversationController.ts b/package/src/ai-components/conversation-state/controller/AIConversationController.ts index 3edb4c1d1f..0100067fe9 100644 --- a/package/src/ai-components/conversation-state/controller/AIConversationController.ts +++ b/package/src/ai-components/conversation-state/controller/AIConversationController.ts @@ -313,14 +313,15 @@ export class AIConversationController { this.refresh(); } - /** Loads the messages before the first one, while there are any. */ + /** + * Loads the messages before the first one, while there are any. Rejects when they couldn't load, + * so the conversation waits before it asks again. + */ loadOlderMessages = async () => { if (!this.hasOlderMessages || this.isLoadingOlderMessages) return; this.isLoadingOlderMessages = true; try { await this.transport.loadOlderMessages?.(); - } catch { - // The person scrolls to the start again to try again. } finally { this.isLoadingOlderMessages = false; } diff --git a/package/src/ai-components/conversation/AIConversation.tsx b/package/src/ai-components/conversation/AIConversation.tsx index 58ea977925..3a5971586f 100644 --- a/package/src/ai-components/conversation/AIConversation.tsx +++ b/package/src/ai-components/conversation/AIConversation.tsx @@ -64,6 +64,10 @@ export type AIConversationAssistantMessageProps = { */ onWritingChange?: (isWriting: boolean) => void; reasoningText?: (stepId: string) => string | undefined; + /** Whether the reply answers with UI the agent generated, which shows under it in place of its text. */ + showsUI?: boolean; + /** Whether the reply's generated UI failed to show, so its text shows instead. */ + uiFailed?: boolean; }; export type AIConversationReplyActionsProps = { @@ -283,6 +287,7 @@ const EntryView = ({ const { uiDocument } = message; const generated = ui && uiDocument ? ui.content(uiDocument, message.id) : null; const showsUI = !!uiDocument && generated !== null && generated !== undefined; + const uiFailed = !!uiDocument && !!ui?.hasFailed(message.id); // A finished reply has its actions under it whether it answers in text or in UI, once the // UI is written. const hasAnswer = message.text !== '' || showsUI; @@ -297,7 +302,9 @@ const EntryView = ({ onWritingChange={onWritingChange} parts={message.parts ?? []} reasoningText={message.reasoningText} + showsUI={showsUI} text={showsUI ? '' : message.text} + uiFailed={uiFailed} /> {showsUI ? ( boolean; const showsNoUI: AIConversationCanShowUI = () => false; +// The id of the person's newest message, looked for from the end: the copy of one they just sent +// is always it, and while a reply is written it is a step or two from the end. +const newestFromUserOf = (messages: AIMessage[]) => { + for (let index = messages.length - 1; index >= 0; index -= 1) { + if (messages[index].role === 'user') return messages[index].id; + } + return undefined; +}; + // A reply the agent has started but that has nothing to show yet: no text, steps or attachments, // and no UI that `canShowUI` says shows. const showsNothing = (message: AIMessage, canShowUI: AIConversationCanShowUI) => @@ -62,12 +71,18 @@ export const hasFinishedSteps = ( message.parts.every(isStepOver) && !(message.uiDocument && canShowUI(message.uiDocument)); -/** Groups the rows into turns, each starting with one of the person's messages. */ -export const turnsOf = (entries: AIConversationEntry[]) => +/** + * Groups the rows into turns, each starting with one of the person's messages. A row that + * started a turn, kept in `starts`, keeps starting one: a reply that opened the first page loaded + * stays the start of its turn when older messages load above it, so the turn keeps its key. + */ +export const turnsOf = (entries: AIConversationEntry[], starts = new Set()) => entries.reduce((turns, entry) => { const last = turns[turns.length - 1]; - if (!last || entry.isFromUser) turns.push({ entries: [entry], id: entry.id }); - else last.entries.push(entry); + if (!last || entry.isFromUser || starts.has(entry.id)) { + starts.add(entry.id); + turns.push({ entries: [entry], id: entry.id }); + } else last.entries.push(entry); return turns; }, []); @@ -85,6 +100,10 @@ export class AIPendingMessageTracker { // The pending message of the last update, whose copy can arrive in the update that stops // passing it. private lastPending?: AIPendingMessage; + // The person's newest message as of the last update, `null` before the first one. A message on + // its way first seen in an update that also brings its copy finds it as the person's newest + // message, when that one is new. + private lastNewestFromUser: string | null | undefined = null; // The rows of a conversation: its messages, apart from replies with nothing to show yet, and // a message on its way until its copy arrives. A reply's UI shows when `canShowUI` says. @@ -94,13 +113,21 @@ export class AIPendingMessageTracker { canShowUI: AIConversationCanShowUI = showsNoUI, ): AIConversationEntry[] { let showsPending = !!pending; + const newestFromUser = newestFromUserOf(messages); // A copy that arrives in the update that stops passing its pending message still takes over // its row. Only then: a message sent later is not the copy of one that failed. const awaited = pending ?? this.lastPending; if (awaited) { const sent = messages.filter((message) => message.role === 'user'); const ids = new Set(sent.map((message) => message.id)); - const known = this.seen.get(awaited.id) ?? ids; + let known = this.seen.get(awaited.id); + if (!known) { + known = new Set(ids); + // The person's message that arrived with this update can be the copy. + const arrived = + this.lastNewestFromUser !== null && newestFromUser !== this.lastNewestFromUser; + if (arrived && newestFromUser !== undefined) known.delete(newestFromUser); + } const matched = [...this.copies].find( ([copyId, original]) => original.id === awaited.id && ids.has(copyId), )?.[0]; @@ -113,6 +140,7 @@ export class AIPendingMessageTracker { } } this.lastPending = pending ?? undefined; + this.lastNewestFromUser = newestFromUser; const entries: AIConversationEntry[] = messages .filter((message) => !showsNothing(message, canShowUI)) diff --git a/package/src/ai-components/conversation/useConversationRows.ts b/package/src/ai-components/conversation/useConversationRows.ts index 61762c879d..fbe1326307 100644 --- a/package/src/ai-components/conversation/useConversationRows.ts +++ b/package/src/ai-components/conversation/useConversationRows.ts @@ -165,7 +165,8 @@ export const useConversationRows = ({ } const thinkingSince = !thinking && stepsKey !== undefined ? gap?.since : undefined; const thinkingReplyId = thinkingSince !== undefined ? lastMessage?.id : undefined; - const turns = useMemo(() => turnsOf(entries), [entries]); + const turnStarts = useRef(new Set()); + const turns = useMemo(() => turnsOf(entries, turnStarts.current), [entries]); const latest = turns[turns.length - 1]; const latestHasReplies = (latest?.entries.length ?? 0) > 1; const latestFirstStepId = firstStepOf(latest); diff --git a/package/src/ai-components/generative-ui/AIUIHost.ts b/package/src/ai-components/generative-ui/AIUIHost.ts index c257521dcd..9e9f5d3e56 100644 --- a/package/src/ai-components/generative-ui/AIUIHost.ts +++ b/package/src/ai-components/generative-ui/AIUIHost.ts @@ -74,6 +74,23 @@ const openWebOrEmailLink = (url: string) => { Linking.openURL(url.trim()).catch(() => undefined); }; +/** + * A tool of the app's as a Model Context Protocol tool, as the agent's backend reads it: its + * `name`, `description`, the `inputSchema` of its arguments (an object with no properties when it + * has none) and whether it only reads. + */ +export const aiUIClientToolJSON = ({ + annotations, + description, + inputSchema, + name, +}: AIClientToolDefinition) => ({ + annotations: { readOnlyHint: annotations?.readOnlyHint === true }, + ...(description ? { description } : {}), + inputSchema: inputSchema ?? { type: 'object' }, + name, +}); + /** * The generated UI of one conversation: the renderers of the formats the agent writes, the * app's components and tools, what each reply's UI was left in, and where what the person does @@ -214,6 +231,11 @@ export class AIUIHost { return sent; } + /** Whether a reply's UI failed to show, so its text shows in its place. */ + hasFailed(messageId: string) { + return this.failed.has(messageId); + } + /** * Shows a reply's text in place of its UI, which failed to show, and tells the app. It lasts * while the reply is in the conversation, or, for a reply still being written, until it is @@ -334,8 +356,7 @@ export class AIUIHost { async capabilities() { return { components: this.components.map(({ definition }) => aiUIComponentDefinitionJSON(definition)), - // The app's tools as Model Context Protocol tools. - tools: this.toolDefinitions, + ...(this.tools.size > 0 ? { tools: this.toolDefinitions.map(aiUIClientToolJSON) } : {}), renderers: await Promise.all( this.renderers.map(async (renderer) => ({ ...(await renderer.capabilities?.(this.components, this.toolDefinitions)), diff --git a/package/src/ai-components/generative-ui/index.ts b/package/src/ai-components/generative-ui/index.ts index 9cd8a502ee..c974e3831e 100644 --- a/package/src/ai-components/generative-ui/index.ts +++ b/package/src/ai-components/generative-ui/index.ts @@ -37,7 +37,13 @@ export { parseAIUIDocument, } from './AIUIDocument'; export { AIUICascade, AIUIEntrance, AIUIEntranceStage, useAIUIEntrance } from './AIUIEntrance'; -export { AIUIHost, type AIUIHostOptions, AIUIToolError, isWebOrEmailLink } from './AIUIHost'; +export { + AIUIHost, + type AIUIHostOptions, + AIUIToolError, + aiUIClientToolJSON, + isWebOrEmailLink, +} from './AIUIHost'; export { aiUIMediaSource, type AIUIMediaURL, diff --git a/package/src/ai-components/markdown/styles.ts b/package/src/ai-components/markdown/styles.ts index 9c18fb506b..f0604d9256 100644 --- a/package/src/ai-components/markdown/styles.ts +++ b/package/src/ai-components/markdown/styles.ts @@ -47,7 +47,7 @@ export const getLocalStyles = ({ theme }: { theme: Theme }) => { }, codeBlock: { backgroundColor: colors.code_block, - fontFamily: Platform.OS === 'ios' ? 'Courier' : 'Monospace', + fontFamily: Platform.OS === 'ios' ? 'Courier' : 'monospace', fontWeight: '500', color: colors.black, paddingLeft: 8, @@ -68,7 +68,7 @@ export const getLocalStyles = ({ theme }: { theme: Theme }) => { color: colors.black, }, codeBlockLineNumberCell: { - fontFamily: Platform.OS === 'ios' ? 'Courier' : 'Monospace', + fontFamily: Platform.OS === 'ios' ? 'Courier' : 'monospace', color: colors.grey, fontSize: primitives.typographyFontSizeXs, lineHeight: primitives.typographyLineHeightTight, @@ -81,7 +81,7 @@ export const getLocalStyles = ({ theme }: { theme: Theme }) => { codeBlockHeaderTitle: { fontSize: primitives.typographyFontSizeXs, color: colors.grey, - fontFamily: Platform.OS === 'ios' ? 'Courier' : 'Monospace', + fontFamily: Platform.OS === 'ios' ? 'Courier' : 'monospace', }, del: { textDecorationLine: 'line-through', @@ -117,7 +117,7 @@ export const getLocalStyles = ({ theme }: { theme: Theme }) => { borderRadius: 3, borderWidth: 1, color: colors.accent_red, - fontFamily: Platform.OS === 'ios' ? 'Courier' : 'Monospace', + fontFamily: Platform.OS === 'ios' ? 'Courier' : 'monospace', fontWeight: '500', fontSize: primitives.typographyFontSizeSm, padding: 3, diff --git a/package/src/ai-components/message-composer/ComposerView.tsx b/package/src/ai-components/message-composer/ComposerView.tsx index 6752ac3084..ece13be0c7 100644 --- a/package/src/ai-components/message-composer/ComposerView.tsx +++ b/package/src/ai-components/message-composer/ComposerView.tsx @@ -16,7 +16,12 @@ import { MediaPreviewList } from './MediaPreviewList'; import { BottomSheet } from '../components/BottomSheet'; -import { ComposerProvider, useMessageComposerContext, useTheme } from '../contexts'; +import { + ComposerProvider, + useAITranslation, + useMessageComposerContext, + useTheme, +} from '../contexts'; import type { AIConversationController, AIConversationControllerState, @@ -206,6 +211,7 @@ export const AIMessageInput = () => { }, } = useTheme(); const styles = useStyles(); + const { t } = useAITranslation(); const { error, isRecording, source } = useDictationState(); const { text } = useComposerText(); @@ -244,7 +250,7 @@ export const AIMessageInput = () => { { return lines; }; +// Arguments longer than this start folded, so the question and its buttons stay in view. +const LONG_ARGUMENTS_CHARACTERS = 280; +const LONG_ARGUMENTS_LINES = 6; + +export type AIToolArgumentsProps = { + /** The call's arguments, such as `part.arguments`. */ + args: unknown; +}; + +/** + * What a tool call sends, so the person sees it before they allow the call: its arguments as + * indented JSON, under "What it sends", folded when they are long. Nothing for a call that + * sends nothing. + */ +export const AIToolArguments = ({ args }: AIToolArgumentsProps) => { + const { t } = useAITranslation(); + const { + theme: { colors, messageParts }, + } = useTheme(); + const { Chevron } = useAIIcons(); + const card = messageParts?.approvalCard; + const text = formatToolArguments(args); + const isLong = + !!text && + (text.length > LONG_ARGUMENTS_CHARACTERS || text.split('\n').length > LONG_ARGUMENTS_LINES); + const [isOpen, setIsOpen] = useState(false); + if (!text) return null; + + const label = t('What it sends'); + const code = ( + + {text} + + ); + return ( + + {isLong ? ( + setIsOpen((open) => !open)} + style={styles.argumentsHeader} + > + + {label} + + + + + + ) : ( + + {label} + + )} + {!isLong || isOpen ? code : null} + + ); +}; + export type AIToolApprovalCardProps = { approval: AIToolApprovalData; /** Called with the person's answer. */ onDecide: (allowed: boolean) => void; + /** What the call sends, shown under the question, such as an `AIToolArguments`. */ + argumentsPreview?: ReactNode; state?: AIToolApprovalState; }; @@ -45,6 +114,7 @@ export type AIToolApprovalCardProps = { */ export const AIToolApprovalCard = ({ approval, + argumentsPreview, onDecide, state = idle, }: AIToolApprovalCardProps) => { @@ -72,6 +142,7 @@ export const AIToolApprovalCard = ({ ))} + {argumentsPreview ?? null} void, + argumentsPreview: ReactNode, ) => ReactNode; }; @@ -158,18 +233,42 @@ export const AIToolApproval = ({ approver: approverProp, children, part }: AIToo .catch(() => setCallState(key, { failed: true, isSending: false }, idle)); }; + const argumentsPreview = toolArgumentsPreview(part, approver); + return ( <> {children ? ( - children(part.approval, state, decide) + children(part.approval, state, decide, argumentsPreview) ) : ( - + )} ); }; +// What the call sends, as the approver previews it, else its arguments. +const toolArgumentsPreview = (call: AIToolCallPart, approver: AIToolApprover): ReactNode => { + const own = approver.argumentsPreview?.(call); + if (own !== undefined) return own; + return formatToolArguments(call.arguments) === undefined ? null : ( + + ); +}; + const styles = StyleSheet.create({ + arguments: { borderRadius: 12, gap: 6, paddingHorizontal: 10, paddingVertical: 8 }, + argumentsCode: { + fontFamily: Platform.OS === 'ios' ? 'Courier' : 'monospace', + fontSize: primitives.typographyFontSizeXs, + }, + argumentsHeader: { alignItems: 'center', flexDirection: 'row', gap: 6 }, + argumentsLabel: { fontSize: primitives.typographyFontSizeXs, fontWeight: '600' }, + chevronOpen: { transform: [{ rotate: '90deg' }] }, actions: { alignItems: 'center', flexDirection: 'row', flexWrap: 'wrap', gap: 8 }, button: { alignItems: 'center', diff --git a/package/src/ai-components/message-parts/AIToolAuthorization.tsx b/package/src/ai-components/message-parts/AIToolAuthorization.tsx index 6835aa12ec..8a096560bb 100644 --- a/package/src/ai-components/message-parts/AIToolAuthorization.tsx +++ b/package/src/ai-components/message-parts/AIToolAuthorization.tsx @@ -157,6 +157,13 @@ export const AIToolAuthorizationCard = ({ ); }; +// How long a sign-in that came back connected reads as connecting before Connect is offered +// again, when the agent never moves the call on. +const CONNECTED_SETTLE_MS = 30_000; + +// The latest sign-in opened for each call, so a late answer to an earlier one changes nothing. +const attempts = new Map(); + /** * The sign-in this person is asked for on `call` here: one the call waits for them to finish, * of a kind this device opens, and only when they can sign in on it. @@ -214,13 +221,21 @@ export const AIToolAuthorization = ({ const now = callStateOf(key, idle); if (now.isConnecting || now.isDeclining) return; update({ ...idle, isConnecting: true }); + const attempt = (attempts.get(key) ?? 0) + 1; + attempts.set(key, attempt); signIn.connect(part).then( // Your backend has the provider's answer before the sign-in comes back, and its change to // the step hides this. Until then it says it is connecting, unless the provider did not - // connect them. A sign-in the person closed can't be opened again, so it declines. + // connect them, or the step never changes. A sign-in the person closed can't be opened + // again, so it declines. (connected) => { if (connected === 'closed') sendDecline(signIn); else if (!connected) update(idle); + else { + setTimeout(() => { + if (attempts.get(key) === attempt && callStateOf(key, idle).isConnecting) update(idle); + }, CONNECTED_SETTLE_MS); + } }, () => update({ ...idle, connectFailed: true }), ); diff --git a/package/src/ai-components/message-parts/toolArguments.ts b/package/src/ai-components/message-parts/toolArguments.ts new file mode 100644 index 0000000000..338219ec76 --- /dev/null +++ b/package/src/ai-components/message-parts/toolArguments.ts @@ -0,0 +1,22 @@ +/** + * A tool call's arguments as the person reads them before they allow the call: JSON, indented, + * or the text as it came when it isn't JSON. Nothing for a call that sends nothing. + */ +export const formatToolArguments = (args: unknown): string | undefined => { + if (args === undefined || args === null || args === '') return undefined; + if (typeof args === 'string') { + try { + return formatToolArguments(JSON.parse(args)); + } catch { + return args; + } + } + if (typeof args === 'object' && !Array.isArray(args) && Object.keys(args).length === 0) { + return undefined; + } + try { + return JSON.stringify(args, null, 2); + } catch { + return String(args); + } +}; diff --git a/package/src/ai-components/message/AIReplyActions.tsx b/package/src/ai-components/message/AIReplyActions.tsx index 4b44df4527..ce032f17a4 100644 --- a/package/src/ai-components/message/AIReplyActions.tsx +++ b/package/src/ai-components/message/AIReplyActions.tsx @@ -71,15 +71,18 @@ export const AIReplyActions = ({ onRate, onRegenerate, rating, text }: AIReplyAc setIsCopied(false); } + // A reply that answers only in UI or attachments has no text to copy or share. + const hasText = text.trim() !== ''; const setString = setClipboardString; - const copy = setString - ? () => { - setString(text); - setIsCopied(true); - clearTimeout(timer.current); - timer.current = setTimeout(() => setIsCopied(false), COPIED_MS); - } - : undefined; + const copy = + setString && hasText + ? () => { + setString(text); + setIsCopied(true); + clearTimeout(timer.current); + timer.current = setTimeout(() => setIsCopied(false), COPIED_MS); + } + : undefined; // The person closing the share sheet resolves too; there's nothing to do either way. const share = () => { @@ -89,6 +92,8 @@ export const AIReplyActions = ({ onRate, onRegenerate, rating, text }: AIReplyAc const positive = rating === 'positive'; const negative = rating === 'negative'; + if (!hasText && !onRegenerate && !onRate) return null; + return ( ) : null} - + {hasText ? : null} {onRegenerate ? ( ) : null} diff --git a/package/src/ai-components/syntax-highlighting/SyntaxHighlighter.tsx b/package/src/ai-components/syntax-highlighting/SyntaxHighlighter.tsx index 9ca9131d6c..556872e8a4 100644 --- a/package/src/ai-components/syntax-highlighting/SyntaxHighlighter.tsx +++ b/package/src/ai-components/syntax-highlighting/SyntaxHighlighter.tsx @@ -22,7 +22,7 @@ export type ColorRange = { }; export const SyntaxHighlighter = ({ - fontFamily = Platform.OS === 'ios' ? 'Courier' : 'Monospace', + fontFamily = Platform.OS === 'ios' ? 'Courier' : 'monospace', fontSize = DEFAULT_FONT_SIZE, children, highlighter = 'highlightjs', diff --git a/package/src/ai-components/theme.ts b/package/src/ai-components/theme.ts index 33e7bc372c..bed34f32d2 100644 --- a/package/src/ai-components/theme.ts +++ b/package/src/ai-components/theme.ts @@ -103,6 +103,9 @@ export type Theme = { declineButton?: ViewStyle; declineButtonText?: TextStyle; failure?: TextStyle; + arguments?: ViewStyle; + argumentsLabel?: TextStyle; + argumentsCode?: TextStyle; }; authorizationCard?: { container?: ViewStyle; @@ -388,6 +391,9 @@ export const defaultTheme: Theme = { declineButton: {}, declineButtonText: {}, failure: {}, + arguments: {}, + argumentsLabel: {}, + argumentsCode: {}, }, authorizationCard: { container: {}, diff --git a/package/src/ai-components/tool-calls/AIClientToolRunner.ts b/package/src/ai-components/tool-calls/AIClientToolRunner.ts index 98c8599d1c..46ed37d247 100644 --- a/package/src/ai-components/tool-calls/AIClientToolRunner.ts +++ b/package/src/ai-components/tool-calls/AIClientToolRunner.ts @@ -98,10 +98,16 @@ export type AIClientToolRunnerOptions = { tools: AIClientTool[]; /** The person signed in, matched against a call's `target_user_id`. */ userId: string; - /** How many times a result is offered to `send` before the runner gives up on it. Defaults to 3. */ + /** + * How many times a result is offered to `send` before the runner gives up on it. At least 1; + * defaults to 3. + */ maxAttempts?: number; }; +const hasOwnId = (payload: Record) => + typeof payload.id === 'string' && payload.id !== ''; + type CallState = { attempts: number; result?: AIClientToolResult; @@ -114,7 +120,8 @@ type CallState = { * as they update. It runs a call when the call awaits this person and this install, the runner * has a tool of that name and the call has not run here before, then hands the result to `send`. * A result that could not be sent is sent again on a later update, without running the tool - * again. + * again. A call without an `id` of its own is left alone: its step is told apart by its place in + * the reply, which another reply repeats. * * ```ts * const runner = new AIClientToolRunner({ clientId: AIClientIdentity.installId(), tools: [localTime], userId }); @@ -129,6 +136,9 @@ export class AIClientToolRunner { private readonly tools = new Map(); constructor({ clientId, maxAttempts = 3, tools, userId }: AIClientToolRunnerOptions) { + if (!Number.isInteger(maxAttempts) || maxAttempts < 1) { + throw new RangeError(`maxAttempts must be at least 1, was ${maxAttempts}`); + } this.clientId = clientId; this.maxAttempts = maxAttempts; this.userId = userId; @@ -162,8 +172,10 @@ export class AIClientToolRunner { /** Runs the calls among `parts` that await this device, and sends their results. */ run(parts: readonly AIMessagePartData[], send: AIClientToolSend) { - for (const { toolCall: call } of parts) { - if (!call || !isAwaitingClient(call, this.userId, this.clientId)) continue; + for (const { payload, toolCall: call } of parts) { + if (!call || !hasOwnId(payload) || !isAwaitingClient(call, this.userId, this.clientId)) { + continue; + } const tool = this.tools.get(call.name); if (!tool) continue; const state = this.calls.get(call.id) ?? { attempts: 0, sending: false, sent: false }; diff --git a/package/src/ai-components/tool-calls/AIToolApprover.tsx b/package/src/ai-components/tool-calls/AIToolApprover.tsx index d8a28f9110..7aacdbbcb4 100644 --- a/package/src/ai-components/tool-calls/AIToolApprover.tsx +++ b/package/src/ai-components/tool-calls/AIToolApprover.tsx @@ -1,4 +1,4 @@ -import React, { createContext, type PropsWithChildren, useContext } from 'react'; +import React, { createContext, type PropsWithChildren, type ReactNode, useContext } from 'react'; import type { AIToolCallPart } from '../message-parts/types'; @@ -41,6 +41,11 @@ export type AIToolApprover = { userId: string; /** How this person signs in when a call waits for their account. Without one, no sign-in is offered here. */ authorizer?: AIToolAuthorizer; + /** + * What a call sends, shown under its question, such as a formatted email instead of its JSON. + * `null` shows nothing; `undefined` shows the call's arguments. + */ + argumentsPreview?: (call: AIToolCallPart) => ReactNode | null | undefined; }; const AIToolApproverContext = createContext(undefined); diff --git a/package/src/ai-components/transcription/useDictation.ts b/package/src/ai-components/transcription/useDictation.ts index bfbf050675..99e05c2502 100644 --- a/package/src/ai-components/transcription/useDictation.ts +++ b/package/src/ai-components/transcription/useDictation.ts @@ -8,8 +8,8 @@ import { AIDictation } from '../native/AIDictation'; import type { DictationResult, DictationStartOptions } from '../native-specs/NativeAIDictation'; import { dictationStore, type DictationStoreState } from '../store'; +// No `language`: the device's own language is used, as the person speaks it. const DEFAULT_OPTIONS = { - language: 'en-US', interimResults: true, silenceTimeoutMs: 2500, }; diff --git a/package/src/ai-components/voice-visualization/VoiceVisualizationView.tsx b/package/src/ai-components/voice-visualization/VoiceVisualizationView.tsx index 8276245d84..361ca5ac54 100644 --- a/package/src/ai-components/voice-visualization/VoiceVisualizationView.tsx +++ b/package/src/ai-components/voice-visualization/VoiceVisualizationView.tsx @@ -11,9 +11,12 @@ import { import Animated from 'react-native-reanimated'; import type { VoiceVisualizationStyle } from './styles'; + import type { VoiceAgentState, VoiceVisualizationProps } from './types'; import { useVoiceVisualization, type VoiceVisualization } from './useVoiceVisualization'; +import { useAITranslation } from '../contexts'; + /** What the renderer draws in: the canvas size, in its own units, and the screen's pixels per unit. */ export type VoiceCanvasSize = { height: number; pixelRatio: number; width: number }; @@ -23,14 +26,6 @@ type ViewProps = VoiceVisualizationProps & { visualStyle: VoiceVisualizationStyle; }; -const DEFAULT_STATE_LABELS: Record = { - connecting: 'Connecting', - idle: 'Ready', - listening: 'Listening', - speaking: 'Speaking', - thinking: 'Thinking', -}; - /** * What VoiceWave and VoiceGlow share: the motion, the canvas kept at the view's size and at most * the style's density, reduced motion, taps and hover, and accessibility. @@ -40,7 +35,7 @@ const DEFAULT_STATE_LABELS: Record = { * only the shape takes them: around it they reach whatever is behind. */ export const VoiceVisualizationView = ({ - accessibilityLabel = 'Voice agent', + accessibilityLabel, accessibilityValue, audioLevel, colors, @@ -54,6 +49,14 @@ export const VoiceVisualizationView = ({ visualStyle, ...restProps }: ViewProps) => { + const { t } = useAITranslation(); + const defaultStateLabels: Record = { + connecting: t('Connecting'), + idle: t('Ready'), + listening: t('Listening'), + speaking: t('Speaking'), + thinking: t('Thinking'), + }; const visualization = useVoiceVisualization( { audioLevel, colors, expressivity, state }, visualStyle, @@ -121,10 +124,10 @@ export const VoiceVisualizationView = ({ return (