From 6519e4bd922e7062367126b0221cef849bc77d06 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 23:57:08 +0100 Subject: [PATCH 01/23] feat(ai-components): parse agent reply steps from message attachments Ported from #3920 on v9-release. --- .../__tests__/messageParts.test.ts | 336 ++++++++++++++++++ package/src/ai-components/index.ts | 2 + .../src/ai-components/message-parts/index.ts | 2 + .../message-parts/parseMessageParts.ts | 195 ++++++++++ .../src/ai-components/message-parts/types.ts | 159 +++++++++ 5 files changed, 694 insertions(+) create mode 100644 package/src/ai-components/__tests__/messageParts.test.ts create mode 100644 package/src/ai-components/message-parts/index.ts create mode 100644 package/src/ai-components/message-parts/parseMessageParts.ts create mode 100644 package/src/ai-components/message-parts/types.ts diff --git a/package/src/ai-components/__tests__/messageParts.test.ts b/package/src/ai-components/__tests__/messageParts.test.ts new file mode 100644 index 0000000000..731be7ecc8 --- /dev/null +++ b/package/src/ai-components/__tests__/messageParts.test.ts @@ -0,0 +1,336 @@ +import { + isAuthorizationExpired, + isAwaitingApproval, + isAwaitingAuthorization, + isAwaitingClient, + isDeclined, + isSupportedPart, + isToolCallFinished, + parseMessagePart, + parseMessageParts, +} from '../message-parts'; +import type { AIToolCallPart } from '../message-parts'; + +// Shaped as the router writes them (acceleration/internal/conversation/parts.go). +const reasoning = { + duration_ms: 4200, + finished_at: '2026-11-02T10:00:04Z', + id: 'r1', + preview: 'Let me check the calendar first.', + started_at: '2026-11-02T10:00:00Z', + status: 'completed', + summary: 'Checked the calendar', + type: 'ai_reasoning', + v: 1, +}; + +const approvalCall = { + approval: { + allow_title: 'Share', + decline_title: 'Not now', + message: 'Only your city is shared.', + reason: 'to check the local weather', + title: 'Share your location?', + }, + arguments: { reason: 'to check the local weather' }, + display_title: 'Checking your location', + executor: 'client', + id: 'call_1', + name: 'get_location', + status: 'awaiting_approval', + target_client_id: 'ios-1', + target_user_id: 'alex', + type: 'ai_tool_call', + v: 1, +}; + +const authorizationCall = { + authorization: { + expires_at: '2026-11-02T10:10:00.123Z', + handoff_token: 'tok', + message: 'Juno needs your Linear account to file the issue.', + provider: 'linear', + provider_name: 'Linear', + title: 'Connect Linear', + type: 'oauth', + url: 'https://router.example.com/v1/agents/connectors/oauth/launch/abc', + }, + display_title: 'Filing a Linear issue', + executor: 'server', + id: 'call_2', + name: 'linear_create_issue', + status: 'awaiting_authorization', + target_user_id: 'alex', + type: 'ai_tool_call', + v: 1, +}; + +const signInWithUrl = (url: string) => + parseMessagePart({ + ...authorizationCall, + authorization: { ...authorizationCall.authorization, url }, + })?.toolCall?.authorization; + +describe('parseMessageParts', () => { + it('reads the steps in order and skips other attachments', () => { + const parts = parseMessageParts([ + reasoning, + { image_url: 'https://example.com/a.png', type: 'image' }, + approvalCall, + authorizationCall, + ]); + expect(parts.map((part) => part.id)).toEqual(['r1', 'call_1', 'call_2']); + expect(parts.map((part) => part.kind)).toEqual([ + 'ai_reasoning', + 'ai_tool_call', + 'ai_tool_call', + ]); + expect(parts.every(isSupportedPart)).toBe(true); + }); + + it('reads a reply as a local Vision-Agents router wrote it', () => { + // The attachments of one real reply, copied from the channel (2026-11-03, xai/grok-4.7). + const parts = parseMessageParts([ + { + duration_ms: 2387, + finished_at: '2026-11-03T14:20:45.15695058Z', + id: 'r1', + preview: "Searching for today's top tech headline.", + started_at: '2026-11-03T14:20:45.156092748Z', + status: 'completed', + summary: "Searching for today's top tech headline.", + type: 'ai_reasoning', + v: 1, + }, + { + display_title: 'Searching the web', + duration_ms: 687, + executor: 'server', + finished_at: '2026-11-03T14:20:45.844631174Z', + id: 'call-b1352079-bc55-4229-ab07-8f9c2fc5d4d7-0', + name: 'search', + started_at: '2026-11-03T14:20:45.15688333Z', + status: 'completed', + type: 'ai_tool_call', + v: 1, + }, + { + duration_ms: 1884, + finished_at: '2026-11-03T14:20:47.729154697Z', + id: 'r2', + preview: "I will search for today's top tech headline and summarize it in one sentence.", + started_at: '2026-11-03T14:20:47.72908728Z', + status: 'completed', + summary: "I will search for today's top tech headline and summarize it in one sentence.", + type: 'ai_reasoning', + v: 1, + }, + ]); + expect(parts.map((part) => part.id)).toEqual([ + 'r1', + 'call-b1352079-bc55-4229-ab07-8f9c2fc5d4d7-0', + 'r2', + ]); + expect(parts[0].reasoning).toEqual({ + durationMs: 2387, + id: 'r1', + preview: "Searching for today's top tech headline.", + status: 'completed', + summary: "Searching for today's top tech headline.", + }); + expect(parts[1].toolCall).toMatchObject({ + displayTitle: 'Searching the web', + durationMs: 687, + executor: 'server', + name: 'search', + status: 'completed', + }); + expect(parts.every(isSupportedPart)).toBe(true); + }); + + it('reads a reasoning step', () => { + expect(parseMessagePart(reasoning)?.reasoning).toEqual({ + durationMs: 4200, + id: 'r1', + preview: 'Let me check the calendar first.', + status: 'completed', + summary: 'Checked the calendar', + }); + }); + + it('reads a tool call with its approval', () => { + const call = parseMessagePart(approvalCall)?.toolCall; + expect(call).toMatchObject({ + approval: { + allowTitle: 'Share', + declineTitle: 'Not now', + message: 'Only your city is shared.', + reason: 'to check the local weather', + title: 'Share your location?', + }, + arguments: { reason: 'to check the local weather' }, + displayTitle: 'Checking your location', + executor: 'client', + id: 'call_1', + name: 'get_location', + status: 'awaiting_approval', + targetClientId: 'ios-1', + targetUserId: 'alex', + }); + }); + + it('reads a tool call with its authorization', () => { + const call = parseMessagePart(authorizationCall)?.toolCall; + expect(call?.authorization).toEqual({ + expiresAt: Date.parse('2026-11-02T10:10:00.123Z'), + handoffToken: 'tok', + message: 'Juno needs your Linear account to file the issue.', + provider: 'linear', + providerName: 'Linear', + status: undefined, + title: 'Connect Linear', + type: 'oauth', + url: 'https://router.example.com/v1/agents/connectors/oauth/launch/abc', + }); + }); + + it('defaults missing fields', () => { + const part = parseMessagePart({ type: 'ai_tool_call' }, 3); + expect(part?.id).toBe('ai_tool_call-3'); + expect(part?.version).toBe(1); + expect(part?.toolCall).toMatchObject({ + executor: 'server', + name: '', + status: 'running', + }); + expect(parseMessagePart({ type: 'ai_reasoning' })?.reasoning?.status).toBe('completed'); + }); + + it('reads fields of the wrong type as missing', () => { + const call = parseMessagePart({ + approval: { message: 'no title' }, + authorization: { title: 'Connect', url: 'ftp://example.com/launch' }, + duration_ms: '12', + id: 7, + name: ['x'], + status: '', + type: 'ai_tool_call', + })?.toolCall; + expect(call).toMatchObject({ durationMs: undefined, id: 'ai_tool_call-0', name: '' }); + expect(call?.status).toBe('running'); + // a question with no title asks nothing, and a sign-in needs a web page + expect(call?.approval).toBeUndefined(); + expect(call?.authorization).toBeUndefined(); + }); + + it('opens a sign-in only on a web page with a host', () => { + expect( + signInWithUrl('http://localhost:8080/v1/agents/connectors/oauth/launch/abc'), + ).toBeDefined(); + expect(signInWithUrl('HTTPS://router.example.com')).toBeDefined(); + expect(signInWithUrl('https://user@router.example.com/launch')).toBeDefined(); + expect(signInWithUrl('https://')).toBeUndefined(); + expect(signInWithUrl('https:///launch')).toBeUndefined(); + expect(signInWithUrl('javascript:alert(1)')).toBeUndefined(); + expect(signInWithUrl('router.example.com/launch')).toBeUndefined(); + }); + + it('reads a sign-in where URL does not implement protocol or host', () => { + // React Native before 0.80 throws "not implemented" from these getters. + const OriginalURL = globalThis.URL; + class LimitedURL { + get host(): string { + throw new Error('URL.host is not implemented'); + } + get protocol(): string { + throw new Error('URL.protocol is not implemented'); + } + } + globalThis.URL = LimitedURL as unknown as typeof URL; + try { + expect(parseMessagePart(authorizationCall)?.toolCall?.authorization).toBeDefined(); + } finally { + globalThis.URL = OriginalURL; + } + }); + + it('keeps steps it cannot read, without a typed view', () => { + const newer = parseMessagePart({ id: 'x', type: 'ai_tool_call', v: 2 }); + expect(newer?.toolCall).toBeUndefined(); + expect(newer && isSupportedPart(newer)).toBe(false); + const custom = parseMessagePart({ id: 'c', type: 'ai_citation', url: 'u' }); + expect(custom?.kind).toBe('ai_citation'); + expect(custom?.payload).toMatchObject({ url: 'u' }); + expect(custom && isSupportedPart(custom)).toBe(false); + }); + + it('ignores what is not an AI step', () => { + expect(parseMessageParts(undefined)).toEqual([]); + expect(parseMessageParts([null, 'x', { type: 'file' }, {}])).toEqual([]); + }); +}); + +describe('tool call addressing', () => { + const call = (overrides: Partial): AIToolCallPart => ({ + ...(parseMessagePart(approvalCall)?.toolCall as AIToolCallPart), + ...overrides, + }); + + it('asks the targeted person on the targeted install', () => { + expect(isAwaitingApproval(call({}), 'alex', 'ios-1')).toBe(true); + expect(isAwaitingApproval(call({}), 'alex', 'android-2')).toBe(false); + expect(isAwaitingApproval(call({}), 'sam', 'ios-1')).toBe(false); + }); + + it('asks a server tool on any of the person’s installs', () => { + const server = call({ executor: 'server', targetClientId: undefined }); + expect(isAwaitingApproval(server, 'alex', 'ios-1')).toBe(true); + expect(isAwaitingApproval(server, 'alex', 'android-9')).toBe(true); + }); + + it('asks nothing once answered or without a question', () => { + expect(isAwaitingApproval(call({ status: 'running' }), 'alex', 'ios-1')).toBe(false); + expect(isAwaitingApproval(call({ approval: undefined }), 'alex', 'ios-1')).toBe(false); + }); + + it('waits for this device to run a client tool', () => { + const device = call({ status: 'awaiting_client' }); + expect(isAwaitingClient(device, 'alex', 'ios-1')).toBe(true); + expect(isAwaitingClient(device, 'alex', 'ios-2')).toBe(false); + expect(isAwaitingClient(call({ executor: 'server' }), 'alex', 'ios-1')).toBe(false); + }); + + it('asks the targeted person to sign in from any install', () => { + const signIn = parseMessagePart(authorizationCall)?.toolCall as AIToolCallPart; + expect(isAwaitingAuthorization(signIn, 'alex')).toBe(true); + expect(isAwaitingAuthorization(signIn, 'sam')).toBe(false); + expect(isAwaitingAuthorization({ ...signIn, status: 'completed' }, 'alex')).toBe(false); + }); + + it('knows a declined call', () => { + const declinedApproval = call({ + approval: { decision: 'declined', title: 'Share?' }, + status: 'cancelled', + }); + expect(isDeclined(declinedApproval)).toBe(true); + const signIn = parseMessagePart(authorizationCall)?.toolCall as AIToolCallPart; + expect( + isDeclined({ + ...signIn, + authorization: { ...signIn.authorization!, status: 'declined' }, + }), + ).toBe(true); + expect(isDeclined(call({}))).toBe(false); + }); + + it('knows finished statuses and expired sign-ins', () => { + expect(['completed', 'failed', 'cancelled'].every(isToolCallFinished)).toBe(true); + expect(isToolCallFinished('awaiting_client')).toBe(false); + const signIn = parseMessagePart(authorizationCall)?.toolCall?.authorization as NonNullable< + AIToolCallPart['authorization'] + >; + expect(isAuthorizationExpired(signIn, signIn.expiresAt! - 1)).toBe(false); + expect(isAuthorizationExpired(signIn, signIn.expiresAt!)).toBe(true); + expect(isAuthorizationExpired({ ...signIn, status: 'connected' }, Infinity)).toBe(false); + }); +}); diff --git a/package/src/ai-components/index.ts b/package/src/ai-components/index.ts index 9712c5ce8e..9494ef01de 100644 --- a/package/src/ai-components/index.ts +++ b/package/src/ai-components/index.ts @@ -6,6 +6,8 @@ export * from './StreamingMessageView'; export * from './AITypingIndicatorView'; export * from './voice-visualization'; +export * from './message-parts'; + export * from './components'; export * from './contexts'; diff --git a/package/src/ai-components/message-parts/index.ts b/package/src/ai-components/message-parts/index.ts new file mode 100644 index 0000000000..9ade54059a --- /dev/null +++ b/package/src/ai-components/message-parts/index.ts @@ -0,0 +1,2 @@ +export * from './types'; +export * from './parseMessageParts'; diff --git a/package/src/ai-components/message-parts/parseMessageParts.ts b/package/src/ai-components/message-parts/parseMessageParts.ts new file mode 100644 index 0000000000..999412e4c1 --- /dev/null +++ b/package/src/ai-components/message-parts/parseMessageParts.ts @@ -0,0 +1,195 @@ +import type { + AIMessagePartData, + AIReasoningPart, + AIToolApprovalData, + AIToolAuthorizationData, + AIToolCallPart, + AIToolCallStatus, +} from './types'; + +/** The attachment type prefix every AI step uses. */ +export const AI_MESSAGE_PART_TYPE_PREFIX = 'ai_'; +/** The newest format version of the built-in kinds this SDK understands. */ +export const AI_MESSAGE_PART_SUPPORTED_VERSION = 1; + +type JSONObject = Record; + +// Lenient reads: a field of the wrong type reads as missing, and an empty string as absent. +const isObject = (value: unknown): value is JSONObject => + typeof value === 'object' && value !== null && !Array.isArray(value); + +const string = (object: JSONObject, key: string) => { + const value = object[key]; + return typeof value === 'string' && value !== '' ? value : undefined; +}; + +const int = (object: JSONObject, key: string) => { + const value = object[key]; + return typeof value === 'number' && Number.isFinite(value) ? Math.trunc(value) : undefined; +}; + +const date = (object: JSONObject, key: string) => { + const value = string(object, key); + if (value === undefined) return undefined; + const time = Date.parse(value); + return Number.isNaN(time) ? undefined : time; +}; + +const parseApproval = (object: JSONObject): AIToolApprovalData | undefined => { + // A question with no title asks nothing. + const title = string(object, 'title'); + if (!title) return undefined; + return { + allowTitle: string(object, 'allow_title'), + decision: string(object, 'decision'), + declineTitle: string(object, 'decline_title'), + message: string(object, 'message'), + reason: string(object, 'reason'), + title, + }; +}; + +// An http(s) URL with a host. Matched rather than read through `URL`, whose `protocol` and +// `host` throw "not implemented" on React Native before 0.80. +const WEB_URL = /^https?:\/\/[^\s/?#@]+(?:@[^\s/?#@]+)?(?:[/?#]|$)/i; + +const isWebURL = (value: string) => WEB_URL.test(value); + +const parseAuthorization = (object: JSONObject): AIToolAuthorizationData | undefined => { + // A sign-in with no title, or no web page to open, asks nothing. + const title = string(object, 'title'); + const url = string(object, 'url'); + if (!title || !url || !isWebURL(url)) return undefined; + return { + expiresAt: date(object, 'expires_at'), + handoffToken: string(object, 'handoff_token'), + message: string(object, 'message'), + provider: string(object, 'provider') ?? '', + providerName: string(object, 'provider_name'), + status: string(object, 'status'), + title, + type: string(object, 'type') ?? 'oauth', + url, + }; +}; + +const parseReasoning = (id: string, object: JSONObject): AIReasoningPart => ({ + durationMs: int(object, 'duration_ms'), + id, + preview: string(object, 'preview'), + status: string(object, 'status') ?? 'completed', + summary: string(object, 'summary'), +}); + +const parseToolCall = (id: string, object: JSONObject): AIToolCallPart => { + const approval = object.approval; + const authorization = object.authorization; + const args = object.arguments; + return { + approval: isObject(approval) ? parseApproval(approval) : undefined, + arguments: args === null ? undefined : args, + authorization: isObject(authorization) ? parseAuthorization(authorization) : undefined, + displayTitle: string(object, 'display_title'), + durationMs: int(object, 'duration_ms'), + executor: string(object, 'executor') ?? 'server', + id, + name: string(object, 'name') ?? '', + status: string(object, 'status') ?? 'running', + summary: string(object, 'summary'), + targetClientId: string(object, 'target_client_id'), + targetUserId: string(object, 'target_user_id'), + }; +}; + +/** + * Decodes one attachment, or returns `undefined` when it is not an AI step. + * + * @param attachment The attachment, as it is on the message (with its `type`). + * @param position The attachment's index, used as the identity of a step with no ID of its own. + */ +export const parseMessagePart = ( + attachment: unknown, + position = 0, +): AIMessagePartData | undefined => { + if (!isObject(attachment)) return undefined; + const type = attachment.type; + if (typeof type !== 'string' || !type.startsWith(AI_MESSAGE_PART_TYPE_PREFIX)) { + return undefined; + } + const id = string(attachment, 'id') ?? `${type}-${position}`; + const version = int(attachment, 'v') ?? 1; + const known = version <= AI_MESSAGE_PART_SUPPORTED_VERSION; + return { + id, + kind: type, + payload: attachment, + reasoning: known && type === 'ai_reasoning' ? parseReasoning(id, attachment) : undefined, + toolCall: known && type === 'ai_tool_call' ? parseToolCall(id, attachment) : undefined, + version, + }; +}; + +/** + * Decodes the AI steps among a message's attachments, in order. Attachments that are not AI + * steps (images, files and so on) are skipped. Decoding is lenient: missing fields get + * defaults, and a step newer than this SDK understands keeps its payload but has no typed + * view. + * + * ```ts + * const parts = parseMessageParts(message.attachments); + * ``` + */ +export const parseMessageParts = (attachments: readonly unknown[] | null | undefined) => + (attachments ?? []).reduce((parts, attachment, position) => { + const part = parseMessagePart(attachment, position); + if (part) parts.push(part); + return parts; + }, []); + +/** Whether this SDK has a typed view of the step. */ +export const isSupportedPart = (part: AIMessagePartData) => + part.reasoning !== undefined || part.toolCall !== undefined; + +/** Whether the call has finished, one way or another. */ +export const isToolCallFinished = (status: AIToolCallStatus) => + status === 'completed' || status === 'failed' || status === 'cancelled'; + +/** + * Whether the person declined the call, or the sign-in it waited for, so it never ran. + */ +export const isDeclined = (call: AIToolCallPart) => + call.approval?.decision === 'declined' || call.authorization?.status === 'declined'; + +/** + * Whether this call is waiting for this device: a client tool, still awaiting its result, + * targeted at this person and this install. + */ +export const isAwaitingClient = (call: AIToolCallPart, userId: string, clientId: string) => + call.executor === 'client' && + call.status === 'awaiting_client' && + call.targetUserId === userId && + call.targetClientId === clientId; + +/** + * Whether this call is waiting for this person to allow it, on this device: still awaiting + * approval, targeted at this person, and at this install when it names one (a client tool + * does; a server tool's question may be answered from any of their devices). + */ +export const isAwaitingApproval = (call: AIToolCallPart, userId: string, clientId: string) => + call.status === 'awaiting_approval' && + call.approval !== undefined && + call.targetUserId === userId && + (call.targetClientId === undefined || call.targetClientId === clientId); + +/** Whether this call is waiting for this person to sign in, from any of their devices. */ +export const isAwaitingAuthorization = (call: AIToolCallPart, userId: string) => + call.status === 'awaiting_authorization' && + call.authorization !== undefined && + call.targetUserId === userId; + +/** Whether a sign-in that is still asked for has passed its `expiresAt`. */ +export const isAuthorizationExpired = (authorization: AIToolAuthorizationData, now = Date.now()) => + authorization.status === 'expired' || + (authorization.status === undefined && + authorization.expiresAt !== undefined && + authorization.expiresAt <= now); diff --git a/package/src/ai-components/message-parts/types.ts b/package/src/ai-components/message-parts/types.ts new file mode 100644 index 0000000000..9d17ea2d2b --- /dev/null +++ b/package/src/ai-components/message-parts/types.ts @@ -0,0 +1,159 @@ +/** + * The steps an AI agent took while replying, such as a round of reasoning or a tool call. + * + * The agent writes each step as a custom attachment on its reply (`ai_reasoning`, + * `ai_tool_call`), and the order of the attachments is the order of the steps. The final + * answer stays in the message text and is shown after them. The React Native counterpart of + * `AIMessagePart` in StreamChatAI (iOS), stream-chat-android-ai-compose and + * `stream-chat-react/ai-components`. + * + * The kinds, statuses and decisions are open sets: compare against the values you know and + * give the rest a fallback. A step newer than this SDK understands keeps its payload but has + * no typed view. + */ + +// Keeps autocompletion for the known values while accepting any string. +type Open = T | (string & {}); + +export type AIMessagePartKind = Open<'ai_reasoning' | 'ai_tool_call'>; + +/** Where a round of reasoning is. Defaults to `completed` when the agent sends none. */ +export type AIReasoningStatus = Open<'streaming' | 'completed'>; + +/** A round of the model's reasoning (`ai_reasoning`). */ +export type AIReasoningPart = { + id: string; + status: AIReasoningStatus; + /** How long the model thought, in milliseconds. */ + durationMs?: number; + /** + * A capped excerpt: the latest thoughts while streaming, the opening once done. The full + * reasoning, when an app has it, arrives separately. + */ + preview?: string; + /** A one-line summary of the reasoning, once it is done. */ + summary?: string; +}; + +/** + * Where a tool call is. Treat a status this SDK doesn't know as still in progress. + * + * - `running`: the agent's backend or a device is running it. + * - `awaiting_client`: waiting for the targeted device to run the tool and send its result. + * - `awaiting_approval`: waiting for the targeted person to allow or decline it. + * - `awaiting_authorization`: waiting for the targeted person to sign in at a provider. + * - `completed`, `failed`, `cancelled`: finished, one way or another. + */ +export type AIToolCallStatus = Open< + | 'running' + | 'awaiting_client' + | 'awaiting_approval' + | 'awaiting_authorization' + | 'completed' + | 'failed' + | 'cancelled' +>; + +/** Who runs a tool: the agent's backend (`server`) or a person's device (`client`). */ +export type AIToolExecutor = Open<'server' | 'client'>; + +export type AIToolApprovalDecision = Open<'allowed' | 'declined'>; + +/** + * What a tool call asks the person it waits for before it runs, such as "Share your + * location?", and how they answered. Allowed, the call goes on; declined, it is cancelled + * and never runs. + */ +export type AIToolApprovalData = { + /** The question, such as "Share your location?". */ + title: string; + /** The label of the button that allows the call. Defaults to "Allow" when the agent sends none. */ + allowTitle?: string; + /** How the person answered, once they did. */ + decision?: AIToolApprovalDecision; + /** The label of the button that declines the call. Defaults to "Don't Allow". */ + declineTitle?: string; + /** What allowing it shares or does, such as "Only your city is shared." */ + message?: string; + /** The agent's own words for why it wants the call, such as "to check the local weather". */ + reason?: string; +}; + +/** How a person signs in: `oauth` is a consent at the provider, in a browser. */ +export type AIToolAuthorizationType = Open<'oauth'>; + +/** + * How a sign-in ended. `failed` is one the provider or the agent's backend ended without + * connecting, such as when the person came back as another account. + */ +export type AIToolAuthorizationStatus = Open<'connected' | 'declined' | 'expired' | 'failed'>; + +/** + * A sign-in a tool call waits for, such as "Connect Linear", and how it ended. It holds no + * credential: `url` is the agent's own sign-in page, and the provider hands what it grants to + * the agent's backend, never to the app. + */ +export type AIToolAuthorizationData = { + /** The provider, such as `linear`. */ + provider: string; + /** The question, such as "Connect Linear". */ + title: string; + type: AIToolAuthorizationType; + /** The sign-in page to open (the router's launch page). An `http(s)` URL. */ + url: string; + /** When the sign-in ends, in milliseconds since the epoch. */ + expiresAt?: number; + /** + * What binds the sign-in to the first browser that opens it. The launch page reads it from its + * URL's fragment. Gone once the sign-in is over. + */ + handoffToken?: string; + /** What signing in means for the person. */ + message?: string; + /** What people call the provider, such as "Linear". */ + providerName?: string; + /** How it ended, once it did. */ + status?: AIToolAuthorizationStatus; +}; + +/** A tool the agent called, run by the agent's backend or by a person's device. */ +export type AIToolCallPart = { + /** The model provider's tool-call ID. A device's result is matched against it. */ + id: string; + name: string; + /** Defaults to `server` when the agent sends none. */ + executor: AIToolExecutor; + /** Defaults to `running` when the agent sends none. */ + status: AIToolCallStatus; + /** What the call asks the person it waits for, and how they answered, when its tool asks first. */ + approval?: AIToolApprovalData; + /** The call's arguments, present for client tools, which need them to run. */ + arguments?: unknown; + /** The sign-in the call waits for, and how it ended, when it needs the person's account at a provider. */ + authorization?: AIToolAuthorizationData; + /** What the call is doing, in words for people, such as "Checking your location". */ + displayTitle?: string; + durationMs?: number; + /** A short, shareable outcome, such as "Found your location". */ + summary?: string; + /** The install that must run a client tool. */ + targetClientId?: string; + /** The person the call waits for. */ + targetUserId?: string; +}; + +/** One step of a reply, decoded from one `ai_*` attachment. */ +export type AIMessagePartData = { + /** The step's stable identity, for keys and animation. Tool calls use the provider's tool-call ID. */ + id: string; + /** The attachment type, such as `ai_reasoning`. */ + kind: AIMessagePartKind; + /** The attachment as it came, for kinds of your own or fields this SDK does not read. */ + payload: Record; + /** The step's format version (`v`). It changes only when a kind changes incompatibly. */ + version: number; + /** The step as a round of reasoning, when it is one this SDK can read. */ + reasoning?: AIReasoningPart; + /** The step as a tool call, when it is one this SDK can read. */ + toolCall?: AIToolCallPart; +}; From 5902c0e25855bd12c1dfbd91fb69017acadaad45 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 23:57:08 +0100 Subject: [PATCH 02/23] feat(ai-components): translate the AI components via the chat SDK, sample on the agents router Ported from #3921 on v9-release. The AI components read the chat SDK's V10 translation context, which the entry isolation check now allows. --- examples/ExpoChatGPTSample/.env.example | 15 ++ examples/ExpoChatGPTSample/README.md | 23 ++- examples/ExpoChatGPTSample/agent/agents.ts | 32 ++++ examples/ExpoChatGPTSample/agent/config.ts | 32 ++++ examples/ExpoChatGPTSample/agent/polyfills.ts | 53 +++++++ examples/ExpoChatGPTSample/app/_layout.tsx | 58 ++----- examples/ExpoChatGPTSample/chatConfig.ts | 5 - .../ExpoChatGPTSample/contexts/AppContext.tsx | 146 ++++++++++++------ .../hooks/useSendAIMessage.ts | 50 +++--- examples/ExpoChatGPTSample/http/api.ts | 19 --- examples/ExpoChatGPTSample/http/requests.ts | 19 --- examples/ExpoChatGPTSample/package.json | 1 + .../ExpoChatGPTSample/screens/ChatContent.tsx | 48 ++++-- .../ExpoChatGPTSample/screens/MenuDrawer.tsx | 31 ++-- package/bin/validate-entry-isolation.js | 5 + .../__tests__/translation.test.tsx | 82 ++++++++++ package/src/ai-components/contexts/index.ts | 1 + .../contexts/translation-context.tsx | 41 +++++ 18 files changed, 471 insertions(+), 190 deletions(-) create mode 100644 examples/ExpoChatGPTSample/.env.example create mode 100644 examples/ExpoChatGPTSample/agent/agents.ts create mode 100644 examples/ExpoChatGPTSample/agent/config.ts create mode 100644 examples/ExpoChatGPTSample/agent/polyfills.ts delete mode 100644 examples/ExpoChatGPTSample/chatConfig.ts delete mode 100644 examples/ExpoChatGPTSample/http/api.ts delete mode 100644 examples/ExpoChatGPTSample/http/requests.ts create mode 100644 package/src/ai-components/__tests__/translation.test.tsx create mode 100644 package/src/ai-components/contexts/translation-context.tsx diff --git a/examples/ExpoChatGPTSample/.env.example b/examples/ExpoChatGPTSample/.env.example new file mode 100644 index 0000000000..0a0a5fcc3a --- /dev/null +++ b/examples/ExpoChatGPTSample/.env.example @@ -0,0 +1,15 @@ +# Copy to .env.local (gitignored) and fill in. Public values only: Expo inlines every EXPO_PUBLIC_ +# variable into the app bundle. The Stream secret stays with the token backend. + +# The Stream app the Vision-Agents router runs in. +EXPO_PUBLIC_STREAM_API_KEY=YOUR_STREAM_API_KEY + +# The router and its token backend (Vision-Agents' `go run ./backend`). On an Android emulator, +# `adb reverse tcp:8080 tcp:8080 && adb reverse tcp:8098 tcp:8098` makes localhost reach them. +EXPO_PUBLIC_AGENT_ROUTER_URL=http://localhost:8080 +EXPO_PUBLIC_AGENT_TOKEN_URL=http://localhost:8098 + +# The customer and agent `go run ./configure` stored, and who this app is. +EXPO_PUBLIC_AGENT_CUSTOMER_ID=examples +EXPO_PUBLIC_AGENT_NAME=swift_assistant +EXPO_PUBLIC_AGENT_USER_ID=rn-ai-sample-user diff --git a/examples/ExpoChatGPTSample/README.md b/examples/ExpoChatGPTSample/README.md index 7f68eeacef..4da20e2836 100644 --- a/examples/ExpoChatGPTSample/README.md +++ b/examples/ExpoChatGPTSample/README.md @@ -9,17 +9,26 @@ Ported from the `ExpoChatGPTSample` of the deprecated `@stream-io/chat-react-nat ## How it works -- The drawer lists the user's conversations; `+` starts a new one. -- A new conversation is created on its first message. Its title is summarized from that message by the sample backend. -- Before sending, the app asks the sample backend to add an AI agent to the channel (`http/requests.ts`). The agent - replies with streamed `ai_generated` messages, rendered by `StreamingMessageView`. +The agent runs on a [Vision-Agents](https://github.com/GetStream/Vision-Agents) router, which the app talks to through +[`@stream-io/calypso`](https://www.npmjs.com/package/@stream-io/calypso) (`agent/`). -The Stream app, user and backend in `chatConfig.ts` and `http/requests.ts` are Stream's public demo setup. To use your -own, replace them and point `http/requests.ts` at your own AI agent server. +- Each conversation is a router session, kept in an `agent` channel. `` follows that channel as usual. +- Asking goes through `session.responses.create`, not the chat composer. The router writes the question and the agent's + streamed reply into the channel, and titles the conversation itself. +- The drawer lists the user's `agent` channels. `+` starts a new session, and one that is never asked anything is + deleted when you leave it. +- The reply's steps (`ai_reasoning`, `ai_tool_call` attachments) are hidden for now. They get their own views in a + later change. ## Running -From the repository root: +1. Run a Vision-Agents router locally, with its `swift_assistant` agent and token backend, as in + [`examples/voice_agents/swift_assistant`](https://github.com/GetStream/Vision-Agents/tree/accelerate/examples/voice_agents/swift_assistant): + `docker compose up` (with `ROUTER_AUTH_MODE=proxy`), `go run ./configure`, then `go run ./backend`. The chat model + is Grok, so the router needs `XAI_API_KEY`. +2. Copy `.env.example` to `.env.local` and set `EXPO_PUBLIC_STREAM_API_KEY` to the Stream app the router runs in. +3. On an Android emulator, forward the ports: `adb reverse tcp:8080 tcp:8080 && adb reverse tcp:8098 tcp:8098`. +4. From the repository root: ```bash yarn install diff --git a/examples/ExpoChatGPTSample/agent/agents.ts b/examples/ExpoChatGPTSample/agent/agents.ts new file mode 100644 index 0000000000..b7b6c55ea9 --- /dev/null +++ b/examples/ExpoChatGPTSample/agent/agents.ts @@ -0,0 +1,32 @@ +import { Client } from '@stream-io/calypso'; +// Lets `session.chat()` open the conversation's channel on the app's Stream Chat client. +import '@stream-io/calypso/chat'; + +import { AgentConfig } from './config'; + +/** The router, as this app's user. */ +export const agents = new Client({ + apiKey: AgentConfig.streamApiKey, + customerId: AgentConfig.customerId, + url: AgentConfig.routerUrl, +}); + +/** The agent's conversations: opening a new one and carrying an old one on. */ +export const agentSessions = () => agents.agent(AgentConfig.agentName).sessions; + +/** Asks the token backend for this user's Stream token, which both the chat and the router take. */ +export const streamToken = async (): Promise => { + const response = await fetch(`${AgentConfig.tokenUrl}/stream-token`, { + body: JSON.stringify({ user_id: AgentConfig.user.id }), + headers: { 'Content-Type': 'application/json' }, + method: 'POST', + }); + if (!response.ok) { + throw new Error(`The token backend answered ${response.status}: is \`go run ./backend\` up?`); + } + const { token } = (await response.json()) as { token?: string }; + if (!token) { + throw new Error('The token backend answered without a token.'); + } + return token; +}; diff --git a/examples/ExpoChatGPTSample/agent/config.ts b/examples/ExpoChatGPTSample/agent/config.ts new file mode 100644 index 0000000000..772084b3c6 --- /dev/null +++ b/examples/ExpoChatGPTSample/agent/config.ts @@ -0,0 +1,32 @@ +/** + * Where the Vision-Agents router and its token backend are, and who this app is. Set in + * `.env.local`, which `.env.example` describes. Only public values belong there: Expo inlines every + * `EXPO_PUBLIC_` variable into the bundle, so the Stream secret stays with the token backend. + */ + +const streamApiKey = process.env.EXPO_PUBLIC_STREAM_API_KEY; + +if (!streamApiKey) { + throw new Error( + 'EXPO_PUBLIC_STREAM_API_KEY is not set: copy .env.example to .env.local and fill it in.', + ); +} + +export const AgentConfig = { + /** The Stream app the router runs in. Its channels hold the conversations. */ + streamApiKey, + /** The router. On an Android emulator, `adb reverse tcp:8080 tcp:8080` makes localhost reach it. */ + routerUrl: process.env.EXPO_PUBLIC_AGENT_ROUTER_URL || 'http://localhost:8080', + /** Mints this user's Stream token (Vision-Agents' `go run ./backend`). */ + tokenUrl: process.env.EXPO_PUBLIC_AGENT_TOKEN_URL || 'http://localhost:8098', + /** The customer `go run ./configure` stored the agent under. */ + customerId: process.env.EXPO_PUBLIC_AGENT_CUSTOMER_ID || 'examples', + /** The agent, by the name in its `agent.yaml`. */ + agentName: process.env.EXPO_PUBLIC_AGENT_NAME || 'swift_assistant', + /** A model that streams its reasoning, so replies have reasoning steps. */ + chatModel: 'xai/grok-4.7', + user: { + id: process.env.EXPO_PUBLIC_AGENT_USER_ID || 'rn-ai-sample-user', + name: 'Alex', + }, +} as const; diff --git a/examples/ExpoChatGPTSample/agent/polyfills.ts b/examples/ExpoChatGPTSample/agent/polyfills.ts new file mode 100644 index 0000000000..988a998f4f --- /dev/null +++ b/examples/ExpoChatGPTSample/agent/polyfills.ts @@ -0,0 +1,53 @@ +// Hermes has no `crypto.randomUUID`. Ported from stream-chat-js's `generateUUIDv4`. +/* eslint-disable no-bitwise -- RFC 4122 version and variant bits */ + +type CryptoLike = { + getRandomValues?: (bytes: Uint8Array) => Uint8Array; + randomUUID?: () => string; +}; + +const globalWithCrypto = globalThis as { crypto?: CryptoLike }; + +const getRandomValuesWithMathRandom = (bytes: Uint8Array) => { + const max = Math.pow(2, (8 * bytes.byteLength) / bytes.length); + for (let i = 0; i < bytes.length; i++) { + bytes[i] = Math.random() * max; + } + return bytes; +}; + +const hex = (bytes: Uint8Array) => { + let s = ''; + for (let i = 0; i < bytes.length; i++) { + s += bytes[i].toString(16).padStart(2, '0'); + } + return s; +}; + +// https://tools.ietf.org/html/rfc4122 +const generateUUIDv4 = () => { + const getRandomValues = + globalWithCrypto.crypto?.getRandomValues?.bind(globalWithCrypto.crypto) ?? + getRandomValuesWithMathRandom; + const bytes = getRandomValues(new Uint8Array(16)); + bytes[6] = (bytes[6] & 0x0f) | 0x40; // version + bytes[8] = (bytes[8] & 0xbf) | 0x80; // variant + + return ( + hex(bytes.subarray(0, 4)) + + '-' + + hex(bytes.subarray(4, 6)) + + '-' + + hex(bytes.subarray(6, 8)) + + '-' + + hex(bytes.subarray(8, 10)) + + '-' + + hex(bytes.subarray(10, 16)) + ); +}; + +if (typeof globalWithCrypto.crypto?.randomUUID !== 'function') { + globalWithCrypto.crypto = Object.assign(globalWithCrypto.crypto ?? {}, { + randomUUID: generateUUIDv4, + }); +} diff --git a/examples/ExpoChatGPTSample/app/_layout.tsx b/examples/ExpoChatGPTSample/app/_layout.tsx index f08b3c34d6..a83444ff2a 100644 --- a/examples/ExpoChatGPTSample/app/_layout.tsx +++ b/examples/ExpoChatGPTSample/app/_layout.tsx @@ -1,4 +1,6 @@ -import React, { useEffect } from 'react'; +import '../agent/polyfills'; + +import React from 'react'; import { StyleSheet } from 'react-native'; @@ -7,66 +9,26 @@ import { SafeAreaProvider } from 'react-native-safe-area-context'; import { Drawer } from 'expo-router/drawer'; -import type { LocalMessage, OperationRequestFn, StreamChat } from 'stream-chat'; +import type { LocalMessage } from 'stream-chat'; import { Chat, OverlayProvider, useCreateChatClient } from 'stream-chat-expo'; import { StreamTheme } from 'stream-chat-expo/ai-components'; -import { chatApiKey, chatUserId, chatUserName, chatUserToken } from '../chatConfig'; +import { streamToken } from '../agent/agents'; +import { AgentConfig } from '../agent/config'; import { AppProvider } from '../contexts/AppContext'; -import { startAI, summarize } from '../http/requests'; import { MenuDrawer } from '../screens/MenuDrawer'; -const isMessageAIGenerated = (message: LocalMessage) => !!message.custom?.ai_generated; - -// Creates the channel on its first message, names it after a summary of that message, and makes -// sure the AI bot is watching before the message goes out. -const createSendMessageRequest = - (client: StreamChat): OperationRequestFn<'send'> => - async (params, defaultRequest) => { - const { localMessage } = params; - const [type, id] = localMessage.cid.split(':'); - const channel = client.channel(type, id); - - if (!channel.initialized) { - await channel.watch(); - summarize(localMessage.text ?? '').then((response) => { - // `post` resolves to undefined when the request fails; the channel then keeps its id. - const summary = (response as { summary?: string } | undefined)?.summary; - if (summary) { - channel.updatePartial({ set: { name: summary } }); - } - }); - } - - if ( - !Object.keys(channel.state.watchers).some((watcher) => watcher.startsWith('ai-bot')) && - channel.id - ) { - await startAI(channel.id); - } - - return defaultRequest(params); - }; +const isMessageAIGenerated = (message: LocalMessage) => !!message.ai_generated; export default function Layout() { const chatClient = useCreateChatClient({ - apiKey: chatApiKey, - tokenOrProvider: chatUserToken, - userData: { id: chatUserId, name: chatUserName }, + apiKey: AgentConfig.streamApiKey, + tokenOrProvider: streamToken, + userData: AgentConfig.user, }); - useEffect(() => { - if (!chatClient) { - return; - } - chatClient.config.set({ - channel: { requestHandlers: { sendMessageRequest: createSendMessageRequest(chatClient) } }, - messageComposer: { attachments: { pendingUploadsEnabled: true } }, - }); - }, [chatClient]); - if (!chatClient) { return null; } diff --git a/examples/ExpoChatGPTSample/chatConfig.ts b/examples/ExpoChatGPTSample/chatConfig.ts deleted file mode 100644 index e385f5f427..0000000000 --- a/examples/ExpoChatGPTSample/chatConfig.ts +++ /dev/null @@ -1,5 +0,0 @@ -export const chatApiKey = 'zcgvnykxsfm8'; -export const chatUserId = 'rn-ai-test-user-3'; -export const chatUserToken = - 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyX2lkIjoicm4tYWktdGVzdC11c2VyLTMifQ.kOg3zLkzHc-b_5IyckQr-RixbebN-GDs7N3c570oA5A'; -export const chatUserName = 'Bob'; diff --git a/examples/ExpoChatGPTSample/contexts/AppContext.tsx b/examples/ExpoChatGPTSample/contexts/AppContext.tsx index f838b13ada..eeef3e9846 100644 --- a/examples/ExpoChatGPTSample/contexts/AppContext.tsx +++ b/examples/ExpoChatGPTSample/contexts/AppContext.tsx @@ -1,70 +1,128 @@ import type { PropsWithChildren } from 'react'; -import React, { useEffect, useMemo, useState } from 'react'; +import React, { useEffect, useMemo, useRef, useState } from 'react'; -import { ChannelPaginator } from 'stream-chat'; -import type { Channel, SortParamRequest, StreamChat } from 'stream-chat'; +import type { ImageSource, Session } from '@stream-io/calypso'; +import type { Channel, StreamChat } from 'stream-chat'; +import { useStableCallback } from 'stream-chat-expo'; -import { chatUserId } from '../chatConfig'; +import { agents, agentSessions, streamToken } from '../agent/agents'; +import { AgentConfig } from '../agent/config'; -const channelListFilters = { members: { $in: [chatUserId] } }; -const channelListSort: SortParamRequest[] = [{ direction: -1, field: 'last_updated' }]; +type Conversation = { channel: Channel; session: Session }; export type AppContextValue = { + /** The channel the router writes the open conversation into. */ channel: Channel | undefined; - channelListPaginator: ChannelPaginator | undefined; - setChannel: (channel: Channel) => void; + /** Why the last conversation couldn't be opened, or what asking failed with. */ + failure: string | undefined; + /** Asks the agent, with photos as data URIs. Its reply arrives in the channel. */ + ask: (text: string, images?: ImageSource[]) => Promise; + /** Carries on a conversation from the drawer. */ + openConversation: (channel: Channel) => void; + /** Starts a new conversation. */ + startConversation: () => void; + /** Stops the reply being written. What it wrote so far stays. */ + stop: () => void; }; export const AppContext = React.createContext({ - setChannel: () => {}, + ask: async () => {}, channel: undefined, - channelListPaginator: undefined, + failure: undefined, + openConversation: () => {}, + startConversation: () => {}, + stop: () => {}, }); -const ALPHABET = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz-'; +const messageOf = (error: unknown) => (error instanceof Error ? error.message : String(error)); -export const nanoid = (size: number = 21): string => { - let id = ''; - for (let i = 0; i < size; i++) { - const r = Math.floor(Math.random() * ALPHABET.length); - id += ALPHABET[r]; - } - return id; -}; +export const AppProvider = ({ client, children }: PropsWithChildren<{ client: StreamChat }>) => { + const [conversation, setConversation] = useState(); + const [failure, setFailure] = useState(); + const current = useRef(undefined); -export const createChannel = (client: StreamChat) => - client.channel('messaging', nanoid(), { - members: [{ user_id: chatUserId }], + // Opening one conversation after another: the newest wins, and one that was never asked + // anything is deleted rather than left in the drawer. + const open = useStableCallback(async (opening: () => Promise) => { + setFailure(undefined); + try { + const session = await opening(); + const { channel } = await session.chat(); + const previous = current.current; + const next = { channel: channel as Channel, session }; + current.current = next; + setConversation(next); + if (previous && previous.session.id !== session.id) { + const unused = previous.channel.state.messages.length === 0; + await (unused ? previous.session.delete() : previous.session.close()).catch(() => {}); + } + } catch (error) { + setFailure(messageOf(error)); + } }); -export const AppProvider = ({ client, children }: PropsWithChildren<{ client: StreamChat }>) => { - const [channel, setChannel] = useState(() => createChannel(client)); - - // The app owns the channel list's paginator. It lives as long as the client, so the list keeps its - // channels when the drawer unmounts. `disconnectUser` empties the registered lists but keeps them - // registered, so it is removed when the client goes away. - const channelListPaginator = useMemo( - () => - new ChannelPaginator({ - client, - filters: channelListFilters, - id: 'channel-list', - sort: channelListSort, - }), - [client], - ); + const startConversation = useStableCallback(() => { + void open(() => agentSessions().create({ modelOverwrites: { llm: AgentConfig.chatModel } })); + }); + + const openConversation = useStableCallback((channel: Channel) => { + if (!channel.id || channel.id === current.current?.session.id) { + return; + } + // A conversation's session id is its channel's id. + void open(() => agentSessions().resume(channel.id as string)); + }); + + const ask = useStableCallback(async (text: string, images: ImageSource[] = []) => { + const session = current.current?.session; + if (!session) { + return; + } + setFailure(undefined); + try { + await session.responses.create(text, images.length > 0 ? { images } : undefined); + } catch (error) { + setFailure(messageOf(error)); + } + }); + + const stop = useStableCallback(() => { + try { + current.current?.session.interrupt(); + } catch { + // The reply finished first, which is what stopping it was for. + } + }); useEffect(() => { - client.channelManager.insertPaginator({ paginator: channelListPaginator }); + let cancelled = false; + (async () => { + try { + // The router and Stream Chat as the same user, on the app's own chat client. + await agents.setUser(AgentConfig.user, streamToken); + agents.use({ chat: client }); + if (!cancelled) { + startConversation(); + } + } catch (error) { + setFailure(messageOf(error)); + } + })(); return () => { - client.channelManager.removePaginator(channelListPaginator); - channelListPaginator.dispose(); + cancelled = true; }; - }, [channelListPaginator, client]); + }, [client, startConversation]); const contextValue = useMemo( - () => ({ channel, channelListPaginator, setChannel }), - [channel, channelListPaginator], + () => ({ + ask, + channel: conversation?.channel, + failure, + openConversation, + startConversation, + stop, + }), + [ask, conversation, failure, openConversation, startConversation, stop], ); return {children}; diff --git a/examples/ExpoChatGPTSample/hooks/useSendAIMessage.ts b/examples/ExpoChatGPTSample/hooks/useSendAIMessage.ts index c60df86cb8..efd6c8de9b 100644 --- a/examples/ExpoChatGPTSample/hooks/useSendAIMessage.ts +++ b/examples/ExpoChatGPTSample/hooks/useSendAIMessage.ts @@ -1,35 +1,39 @@ -import { useMessageComposer, useMessageInputContext, useStableCallback } from 'stream-chat-expo'; +import { manipulateAsync, SaveFormat } from 'expo-image-manipulator'; + +import { useStableCallback } from 'stream-chat-expo'; import type { ComposerViewProps } from 'stream-chat-expo/ai-components'; +import { useAppContext } from '../contexts/AppContext'; + export type AIMessage = Parameters[0]; +// The router takes photos inline, so they are shrunk first. +const MAX_IMAGE_WIDTH = 1280; + +const toDataUri = async (uri: string) => { + const { base64 } = await manipulateAsync(uri, [{ resize: { width: MAX_IMAGE_WIDTH } }], { + base64: true, + compress: 0.7, + format: SaveFormat.JPEG, + }); + return `data:image/jpeg;base64,${base64}`; +}; + /** - * Sends a message to the AI channel through the channel's composer, with its attachments. Shared - * by the composer and voice mode, so both go through the client's `sendMessageRequest` handler - * (which starts the agent on the first message). + * Asks the agent, with the photos picked in the composer. The router writes the question and its + * reply into the conversation's channel. Shared by the composer and voice mode. */ export const useSendAIMessage = () => { - const messageComposer = useMessageComposer(); - const { sendMessage } = useMessageInputContext(); + const { ask } = useAppContext(); return useStableCallback(async ({ text, attachments }: AIMessage) => { - messageComposer.textComposer.setText(text); - if (attachments && attachments.length > 0) { - const localAttachments = await Promise.all( - attachments.map((asset) => - messageComposer.attachmentManager.fileToLocalUploadAttachment({ - ...asset, - // The picker can leave these out (e.g. some iOS photos have no file name). - name: asset.name ?? asset.uri.split('/').pop() ?? 'image', - size: asset.size ?? 0, - type: asset.type ?? 'image/jpeg', - }), - ), - ); - messageComposer.attachmentManager.upsertAttachments(localAttachments); - } - - await sendMessage(); + const photos = (attachments ?? []).filter( + (asset) => !asset.type || asset.type.startsWith('image'), + ); + const images = await Promise.all( + photos.map(async (asset) => ({ url: await toDataUri(asset.uri) })), + ); + await ask(text, images); }); }; diff --git a/examples/ExpoChatGPTSample/http/api.ts b/examples/ExpoChatGPTSample/http/api.ts deleted file mode 100644 index 4ff8fba92f..0000000000 --- a/examples/ExpoChatGPTSample/http/api.ts +++ /dev/null @@ -1,19 +0,0 @@ -export const post = async (url: string, data: unknown = {}) => { - try { - const response = await fetch(url, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(data), - }); - - if (!response.ok) { - throw new Error(`An HTTP Error has occurred. Status: ${response.status}`); - } - - return await response.json(); - } catch (error) { - console.error('Error:', error); - } -}; diff --git a/examples/ExpoChatGPTSample/http/requests.ts b/examples/ExpoChatGPTSample/http/requests.ts deleted file mode 100644 index fce4cd3941..0000000000 --- a/examples/ExpoChatGPTSample/http/requests.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { post } from './api'; - -// The AI provider the sample backend uses, as in the React SDK's ai-chatbot example. -const PLATFORM = 'openai'; - -export const startAI = (channelId: string) => - post('https://ai-sdk-server-0f347d455e2e.herokuapp.com/start-ai-agent', { - channel_id: channelId, - platform: PLATFORM, - }); -export const stopAI = (channelId: string) => - post('https://ai-sdk-server-0f347d455e2e.herokuapp.com/stop-ai-agent', { - channel_id: channelId, - }); -export const summarize = (text: string) => - post('https://ai-sdk-server-0f347d455e2e.herokuapp.com/summarize', { - platform: PLATFORM, - text, - }); diff --git a/examples/ExpoChatGPTSample/package.json b/examples/ExpoChatGPTSample/package.json index 9ec164fe7f..6b13018c84 100644 --- a/examples/ExpoChatGPTSample/package.json +++ b/examples/ExpoChatGPTSample/package.json @@ -12,6 +12,7 @@ "dependencies": { "@react-native-community/netinfo": "12.0.1", "@shopify/react-native-skia": "^2.14.0", + "@stream-io/calypso": "^0.2.3", "expo": "^58.0.8", "expo-clipboard": "~58.0.3", "expo-constants": "~58.0.11", diff --git a/examples/ExpoChatGPTSample/screens/ChatContent.tsx b/examples/ExpoChatGPTSample/screens/ChatContent.tsx index 7a98628935..36b796c9f7 100644 --- a/examples/ExpoChatGPTSample/screens/ChatContent.tsx +++ b/examples/ExpoChatGPTSample/screens/ChatContent.tsx @@ -1,11 +1,11 @@ -import { useCallback, useMemo } from 'react'; -import { Platform, StatusBar, StyleSheet, Text, View } from 'react-native'; +import { useMemo, useState } from 'react'; +import { ActivityIndicator, Platform, StatusBar, StyleSheet, Text, View } from 'react-native'; import Animated, { FadeIn, FadeOut } from 'react-native-reanimated'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { useHeaderHeight } from 'expo-router/react-navigation'; -import type { MessageProps } from 'stream-chat-expo'; +import type { MessageProps, ResolvedComponents } from 'stream-chat-expo'; import { AIStates, Channel, @@ -16,12 +16,14 @@ import { ThemeProvider, useAIState, useChannelContext, + useComponentsContext, useMessageContext, useTheme, WithComponents, } from 'stream-chat-expo'; import { + AI_MESSAGE_PART_TYPE_PREFIX, AITypingIndicatorView, ComposerView, StreamingMessageView, @@ -35,7 +37,7 @@ import { useSendAIMessage } from '../hooks/useSendAIMessage'; const CustomMessage = (props: MessageProps) => { const { theme } = useTheme(); const { message } = props; - const isFromBot = message.custom?.ai_generated; + const isFromBot = message.ai_generated; const hasPendingAttachments = useMemo( () => (message.attachments ?? []).some( @@ -79,6 +81,17 @@ const CustomMessage = (props: MessageProps) => { ); }; +type UnsupportedAttachmentComponent = ResolvedComponents['UnsupportedAttachment']; + +// A reply's steps (`ai_reasoning`, `ai_tool_call`) are attachments the chat SDK doesn't know. They +// get their own views; until then they're left out rather than shown as unsupported files, and any +// other unknown attachment keeps the SDK's own view. +const hidingAISteps = (Fallback: UnsupportedAttachmentComponent) => { + const HideAISteps: UnsupportedAttachmentComponent = (props) => + props.attachment.type?.startsWith(AI_MESSAGE_PART_TYPE_PREFIX) ? null : ; + return HideAISteps; +}; + const CustomStreamingMessageView = () => { const { message } = useMessageContext(); return ( @@ -90,12 +103,14 @@ const CustomStreamingMessageView = () => { const CustomComposerView = () => { const { channel } = useChannelContext(); + const { stop } = useAppContext(); const { aiState } = useAIState(channel); - const stopGenerating = useCallback(() => channel?.stopAIResponse(), [channel]); + // The router stops through its own session, not with an AI indicator event. + const stopGenerating = useMemo(() => () => Promise.resolve(stop()), [stop]); - const isGenerating = aiState === AIStates.Thinking || aiState === AIStates.Generating; + const isGenerating = [AIStates.Thinking, AIStates.Generating].includes(aiState); const safeAreaInsets = useSafeAreaInsets(); const insets = useMemo( @@ -161,14 +176,21 @@ const additionalFlatListProps = { }; export const ChatContent = () => { - const { channel } = useAppContext(); + const { channel, failure } = useAppContext(); const { bottom } = useSafeAreaInsets(); // The channel sits below the drawer header, so the keyboard offset has to account for it // (without it, Android keeps the composer behind the keyboard). const headerHeight = useHeaderHeight(); + // Read above this screen's own overrides, so it is the SDK's view. + const { UnsupportedAttachment } = useComponentsContext(); + const [CustomUnsupportedAttachment] = useState(() => hidingAISteps(UnsupportedAttachment)); if (!channel) { - return null; + return ( + + {failure ? {failure} : } + + ); } return ( @@ -186,16 +208,13 @@ export const ChatContent = () => { MessageFooter: RenderNull, NetworkDownIndicator: RenderNull, StreamingMessageView: CustomStreamingMessageView, + UnsupportedAttachment: CustomUnsupportedAttachment, }} > - + + {failure ? {failure} : null} @@ -214,6 +233,7 @@ const styles = StyleSheet.create({ alignItems: 'center', }, emptyContainerText: { fontSize: 24, fontWeight: 'bold' }, + failureText: { color: '#B00020', fontSize: 15, paddingHorizontal: 24, textAlign: 'center' }, streamingMessageViewWrapper: { width: '100%', paddingHorizontal: 16 }, aiTypingIndicatorWrapper: { paddingHorizontal: 24, diff --git a/examples/ExpoChatGPTSample/screens/MenuDrawer.tsx b/examples/ExpoChatGPTSample/screens/MenuDrawer.tsx index 12783c765f..3157a9dd42 100644 --- a/examples/ExpoChatGPTSample/screens/MenuDrawer.tsx +++ b/examples/ExpoChatGPTSample/screens/MenuDrawer.tsx @@ -4,18 +4,30 @@ import { Pressable, StyleSheet, Text, View } from 'react-native'; import { SafeAreaView } from 'react-native-safe-area-context'; import type { DrawerContentComponentProps } from 'expo-router/drawer'; -import type { Channel as ChannelClass } from 'stream-chat'; +import type { ChannelSort } from 'stream-chat'; + +import type { Channel as ChannelClass, ChannelFilters } from 'stream-chat'; import { ChannelList, ChannelPreviewProps, Plus, useChannelsContext, - useChatContext, useStableCallback, WithComponents, } from 'stream-chat-expo'; -import { createChannel, useAppContext } from '../contexts/AppContext'; +import { AgentConfig } from '../agent/config'; +import { useAppContext } from '../contexts/AppContext'; + +// The router keeps each conversation in an `agent` channel the user is a member of. +const filters: ChannelFilters = { + members: { + $in: [AgentConfig.user.id], + }, + type: 'agent', +}; + +const sort: ChannelSort = { last_updated: -1 }; const ChannelPreview = (props: ChannelPreviewProps) => { const channel = props.channel; @@ -33,23 +45,22 @@ const ChannelPreview = (props: ChannelPreviewProps) => { onPress={onPress} > - {channel.data?.custom?.name ?? channel.cid} + {channel.data?.name ?? channel.cid} ); }; export const MenuDrawer = ({ navigation }: DrawerContentComponentProps) => { - const { client } = useChatContext(); - const { channelListPaginator, setChannel } = useAppContext(); + const { openConversation, startConversation } = useAppContext(); const onSelect = useStableCallback((channel: ChannelClass) => { - setChannel(channel); + openConversation(channel); navigation.closeDrawer(); }); const onCreateNewChat = useStableCallback(() => { - setChannel(createChannel(client)); + startConversation(); navigation.closeDrawer(); }); @@ -62,9 +73,7 @@ export const MenuDrawer = ({ navigation }: DrawerContentComponentProps) => { - {channelListPaginator ? ( - - ) : null} + ); diff --git a/package/bin/validate-entry-isolation.js b/package/bin/validate-entry-isolation.js index d69e885890..b747c3435f 100644 --- a/package/bin/validate-entry-isolation.js +++ b/package/bin/validate-entry-isolation.js @@ -39,6 +39,11 @@ const AI_ONLY_PACKAGES = [ const AI_ALLOWED_FILES = [ path.join(CORE_SRC, 'native.ts'), path.join(CORE_SRC, 'components/UIComponents/NativeShimmerView.tsx'), + // The chat SDK's translation context, which the AI components translate with (not Streami18n). + path.join(CORE_SRC, 'contexts/translationContext/TranslationContext.tsx'), + path.join(CORE_SRC, 'contexts/translationContext/types.ts'), + path.join(CORE_SRC, 'contexts/utils/isTestEnvironment.ts'), + path.join(CORE_SRC, 'i18n/utils.ts'), ...WRAPPERS.flatMap((wrapper) => [ path.join(wrapper, 'ai-components/index.js'), path.join(wrapper, 'src/optionalDependencies/NativeShimmerView.ts'), diff --git a/package/src/ai-components/__tests__/translation.test.tsx b/package/src/ai-components/__tests__/translation.test.tsx new file mode 100644 index 0000000000..097677c86e --- /dev/null +++ b/package/src/ai-components/__tests__/translation.test.tsx @@ -0,0 +1,82 @@ +import React, { type PropsWithChildren } from 'react'; + +import { renderHook } from '@testing-library/react-native'; + +import { + TranslationProvider as ChatTranslationProvider, + type TranslationContextValue, +} from '../../contexts/translationContext'; +import { Streami18n } from '../../utils/i18n/Streami18n'; +import { + type AITranslate, + defaultAITranslate, + interpolateAIString, + TranslationProvider, + useAITranslation, +} from '../contexts'; + +const withValue = + (value: TranslationContextValue) => + ({ children }: PropsWithChildren) => ( + {children} + ); + +describe('AI translation', () => { + it('fills placeholders and leaves unknown ones', () => { + expect(interpolateAIString('Thought for {{ duration }}', { duration: '12s' })).toBe( + 'Thought for 12s', + ); + expect(interpolateAIString('Connected to {{provider}}', { provider: 'Linear' })).toBe( + 'Connected to Linear', + ); + expect(interpolateAIString('Thought for {{ duration }}', {})).toBe( + 'Thought for {{ duration }}', + ); + expect(interpolateAIString('Thinking…')).toBe('Thinking…'); + expect(interpolateAIString('{{ count }} files', { count: 0 })).toBe('0 files'); + }); + + it('is English without ', () => { + const { result } = renderHook(() => useAITranslation()); + expect(result.current.t).toBe(defaultAITranslate); + expect(result.current.t('Thinking… {{ duration }}', { duration: '4s' })).toBe('Thinking… 4s'); + }); + + it('follows the chat SDK translator, as inside ', async () => { + const i18n = new Streami18n({ language: 'fr' }); + // `registerTranslation` only takes keys in `en.json`; the AI keys join it with the views. + i18n.registerTranslation('fr', { + 'Thought for {{ duration }}': 'Réfléchi pendant {{ duration }}', + } as Parameters[1]); + const { t, tDateTimeParser } = await i18n.init(); + const translators = { t, tDateTimeParser } as unknown as TranslationContextValue; + + const { result } = renderHook(() => useAITranslation(), { + wrapper: withValue({ ...translators, userLanguage: 'fr' }), + }); + expect(result.current.t('Thought for {{ duration }}', { duration: '12s' })).toBe( + 'Réfléchi pendant 12s', + ); + }); + + it('is the same provider as the chat SDK, for apps without ', () => { + expect(TranslationProvider).toBe(ChatTranslationProvider); + const t = ((key: string) => `[es] ${key}`) as unknown as TranslationContextValue['t']; + const { result } = renderHook(() => useAITranslation(), { + wrapper: ({ children }: PropsWithChildren) => ( + input ?? 0, userLanguage: 'es' }} + > + {children} + + ), + }); + expect(result.current.t('Allow')).toBe('[es] Allow'); + }); + + it('takes the chat SDK translator as it is', () => { + const coreT = ((key: string) => key) as unknown as TranslationContextValue['t']; + const t = coreT as unknown as AITranslate; + expect(t('Allow')).toBe('Allow'); + }); +}); diff --git a/package/src/ai-components/contexts/index.ts b/package/src/ai-components/contexts/index.ts index d84d8bb745..f0e08d8aa0 100644 --- a/package/src/ai-components/contexts/index.ts +++ b/package/src/ai-components/contexts/index.ts @@ -1,2 +1,3 @@ export * from './composer-context'; export * from './theme-context'; +export * from './translation-context'; diff --git a/package/src/ai-components/contexts/translation-context.tsx b/package/src/ai-components/contexts/translation-context.tsx new file mode 100644 index 0000000000..a78dab78f6 --- /dev/null +++ b/package/src/ai-components/contexts/translation-context.tsx @@ -0,0 +1,41 @@ +import { useContext, useMemo } from 'react'; + +import { TranslationContext } from '../../contexts/translationContext/TranslationContext'; +import { defaultTranslatorFunction } from '../../i18n/utils'; + +// The chat SDK's own provider, for apps that use the AI components without ``. +export { + TranslationProvider, + type TranslationContextValue, +} from '../../contexts/translationContext/TranslationContext'; + +/** + * Translates one of the AI components' strings. The key is the English string itself, with + * `{{ name }}` placeholders filled from `options`, as in the chat SDK's i18n files. + */ +export type AITranslate = (key: string, options?: Record) => string; + +const PLACEHOLDER = /{{\s*([\w.]+)\s*}}/g; + +/** Fills `{{ name }}` placeholders in `text` from `options`, leaving unknown ones as they are. */ +export const interpolateAIString = (text: string, options?: Record) => + options + ? text.replace(PLACEHOLDER, (placeholder, name: string) => + options[name] === undefined || options[name] === null ? placeholder : String(options[name]), + ) + : text; + +/** English: the key itself, with its placeholders filled. */ +export const defaultAITranslate: AITranslate = (key, options) => interpolateAIString(key, options); + +/** + * The chat SDK's translator, so the AI components follow the chat's language inside ``. With + * no translator set up (no ``, or before it has one) they are in English. + */ +export const useAITranslation = () => { + const { t } = useContext(TranslationContext); + return useMemo( + () => ({ t: t === defaultTranslatorFunction ? defaultAITranslate : (t as AITranslate) }), + [t], + ); +}; From db66e1caf8a0b26e4118bfde29ce476bda3479a2 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 23:57:08 +0100 Subject: [PATCH 03/23] feat(ai-components): agent reply steps (reasoning, tool calls, approvals) Ported from #3924 on v9-release. --- .../ExpoChatGPTSample/contexts/AppContext.tsx | 119 +++++- .../ExpoChatGPTSample/screens/ChatContent.tsx | 109 +++-- .../ExpoChatGPTSample/screens/MenuDrawer.tsx | 98 ++--- .../ai-components/AITypingIndicatorView.tsx | 24 +- .../ai-components/__tests__/format.test.ts | 32 ++ .../__tests__/messagePartViews.test.tsx | 228 +++++++++++ .../__tests__/translation.test.tsx | 6 +- .../contexts/translation-context.tsx | 9 +- package/src/ai-components/index.ts | 2 + .../src/ai-components/internal/Shimmer.tsx | 40 ++ package/src/ai-components/internal/format.ts | 56 +++ .../internal/icons/AgentIcons.tsx | 60 +++ .../message-parts/AIMessageParts.tsx | 94 +++++ .../message-parts/AIToolApproval.tsx | 193 +++++++++ .../message-parts/AIToolCall.tsx | 112 ++++++ .../src/ai-components/message-parts/index.ts | 3 + .../ai-components/reasoning/AIReasoning.tsx | 376 ++++++++++++++++++ package/src/ai-components/reasoning/index.ts | 2 + .../ai-components/reasoning/thinkingClock.ts | 109 +++++ package/src/ai-components/theme.ts | 54 +++ .../tool-calls/AIToolApprover.tsx | 58 +++ package/src/ai-components/tool-calls/index.ts | 1 + 22 files changed, 1655 insertions(+), 130 deletions(-) create mode 100644 package/src/ai-components/__tests__/format.test.ts create mode 100644 package/src/ai-components/__tests__/messagePartViews.test.tsx create mode 100644 package/src/ai-components/internal/Shimmer.tsx create mode 100644 package/src/ai-components/internal/format.ts create mode 100644 package/src/ai-components/internal/icons/AgentIcons.tsx create mode 100644 package/src/ai-components/message-parts/AIMessageParts.tsx create mode 100644 package/src/ai-components/message-parts/AIToolApproval.tsx create mode 100644 package/src/ai-components/message-parts/AIToolCall.tsx create mode 100644 package/src/ai-components/reasoning/AIReasoning.tsx create mode 100644 package/src/ai-components/reasoning/index.ts create mode 100644 package/src/ai-components/reasoning/thinkingClock.ts create mode 100644 package/src/ai-components/tool-calls/AIToolApprover.tsx create mode 100644 package/src/ai-components/tool-calls/index.ts diff --git a/examples/ExpoChatGPTSample/contexts/AppContext.tsx b/examples/ExpoChatGPTSample/contexts/AppContext.tsx index eeef3e9846..14ee62f117 100644 --- a/examples/ExpoChatGPTSample/contexts/AppContext.tsx +++ b/examples/ExpoChatGPTSample/contexts/AppContext.tsx @@ -1,24 +1,47 @@ import type { PropsWithChildren } from 'react'; import React, { useEffect, useMemo, useRef, useState } from 'react'; -import type { ImageSource, Session } from '@stream-io/calypso'; -import type { Channel, StreamChat } from 'stream-chat'; +import { + type ConversationSnapshot, + type ImageSource, + LiveReasoning, + type Session, +} from '@stream-io/calypso'; +import type { Channel, Event, StreamChat } from 'stream-chat'; import { useStableCallback } from 'stream-chat-expo'; +import type { AIToolApprover } from 'stream-chat-expo/ai-components'; import { agents, agentSessions, streamToken } from '../agent/agents'; import { AgentConfig } from '../agent/config'; -type Conversation = { channel: Channel; session: Session }; +type Conversation = { channel: Channel; reasoning: LiveReasoning; session: Session }; + +const IDLE: ConversationSnapshot = { kind: 'idle', skills: [] }; + +// This install, for the tool calls addressed to it. Kept for the app's lifetime. +const clientId = `rn-sample-${(globalThis as { crypto: { randomUUID: () => string } }).crypto.randomUUID()}`; export type AppContextValue = { + /** What the agent is doing in the open conversation, from the session's socket. */ + agentState: ConversationSnapshot; + /** Answers the open conversation's tool calls that ask first. */ + approver: AIToolApprover | undefined; /** The channel the router writes the open conversation into. */ channel: Channel | undefined; + /** Bumped when the drawer's list of conversations may have changed. */ + conversationsVersion: number; /** Why the last conversation couldn't be opened, or what asking failed with. */ failure: string | undefined; + /** The open conversation's session id. */ + sessionId: string | undefined; /** Asks the agent, with photos as data URIs. Its reply arrives in the channel. */ ask: (text: string, images?: ImageSource[]) => Promise; - /** Carries on a conversation from the drawer. */ - openConversation: (channel: Channel) => void; + /** Carries on a conversation from the drawer, by its session id. */ + openConversation: (sessionId: string) => void; + /** The whole of a reply's reasoning step so far, from its live updates. */ + reasoningText: (messageId: string, stepId: string) => string | undefined; + /** Bumped as reasoning arrives, so the steps showing it render again. */ + reasoningVersion: number; /** Starts a new conversation. */ startConversation: () => void; /** Stops the reply being written. What it wrote so far stays. */ @@ -26,10 +49,16 @@ export type AppContextValue = { }; export const AppContext = React.createContext({ + agentState: IDLE, + approver: undefined, ask: async () => {}, channel: undefined, + conversationsVersion: 0, failure: undefined, openConversation: () => {}, + reasoningText: () => undefined, + reasoningVersion: 0, + sessionId: undefined, startConversation: () => {}, stop: () => {}, }); @@ -39,6 +68,9 @@ const messageOf = (error: unknown) => (error instanceof Error ? error.message : export const AppProvider = ({ client, children }: PropsWithChildren<{ client: StreamChat }>) => { const [conversation, setConversation] = useState(); const [failure, setFailure] = useState(); + const [agentState, setAgentState] = useState(IDLE); + const [reasoningVersion, setReasoningVersion] = useState(0); + const [conversationsVersion, setConversationsVersion] = useState(0); const current = useRef(undefined); // Opening one conversation after another: the newest wins, and one that was never asked @@ -49,13 +81,14 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St const session = await opening(); const { channel } = await session.chat(); const previous = current.current; - const next = { channel: channel as Channel, session }; + const next = { channel: channel as Channel, reasoning: new LiveReasoning(), session }; current.current = next; setConversation(next); if (previous && previous.session.id !== session.id) { const unused = previous.channel.state.messages.length === 0; await (unused ? previous.session.delete() : previous.session.close()).catch(() => {}); } + setConversationsVersion((version) => version + 1); } catch (error) { setFailure(messageOf(error)); } @@ -65,12 +98,11 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St void open(() => agentSessions().create({ modelOverwrites: { llm: AgentConfig.chatModel } })); }); - const openConversation = useStableCallback((channel: Channel) => { - if (!channel.id || channel.id === current.current?.session.id) { + const openConversation = useStableCallback((sessionId: string) => { + if (sessionId === current.current?.session.id) { return; } - // A conversation's session id is its channel's id. - void open(() => agentSessions().resume(channel.id as string)); + void open(() => agentSessions().resume(sessionId)); }); const ask = useStableCallback(async (text: string, images: ImageSource[] = []) => { @@ -94,6 +126,53 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St } }); + const reasoningText = useStableCallback((messageId: string, stepId: string) => + current.current?.reasoning.get(messageId, stepId), + ); + + // The open conversation: what the agent is doing, and every live update's slice of thinking, + // read from each event rather than the message list, which can fold several into one. + useEffect(() => { + if (!conversation) { + return; + } + const { channel, reasoning, session } = conversation; + setAgentState(session.state.get()); + const stopState = session.state.subscribe(() => { + const state = session.state.get(); + setAgentState(state); + // The router titles a conversation as it goes. + if (state.kind === 'idle') setConversationsVersion((version) => version + 1); + }); + const subscription = channel.on('message.updated', (event: Event) => { + if (event.message) { + reasoning.read(event.message as { id: string; reasoning?: unknown }); + setReasoningVersion((version) => version + 1); + } + }); + return () => { + stopState(); + subscription.unsubscribe(); + }; + }, [conversation]); + + const approver = useMemo( + () => + conversation + ? { + clientId, + // `decide` sends over the session's socket and throws when it can't. + decide: (call, allowed) => + new Promise((resolve) => { + conversation.session.decide(call.id, { allowed }); + resolve(); + }), + userId: AgentConfig.user.id, + } + : undefined, + [conversation], + ); + useEffect(() => { let cancelled = false; (async () => { @@ -115,14 +194,32 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St const contextValue = useMemo( () => ({ + agentState, + approver, ask, channel: conversation?.channel, + conversationsVersion, failure, openConversation, + reasoningText, + reasoningVersion, + sessionId: conversation?.session.id, startConversation, stop, }), - [ask, conversation, failure, openConversation, startConversation, stop], + [ + agentState, + approver, + ask, + conversation, + conversationsVersion, + failure, + openConversation, + reasoningText, + reasoningVersion, + startConversation, + stop, + ], ); return {children}; diff --git a/examples/ExpoChatGPTSample/screens/ChatContent.tsx b/examples/ExpoChatGPTSample/screens/ChatContent.tsx index 36b796c9f7..6e6b978571 100644 --- a/examples/ExpoChatGPTSample/screens/ChatContent.tsx +++ b/examples/ExpoChatGPTSample/screens/ChatContent.tsx @@ -1,5 +1,13 @@ import { useMemo, useState } from 'react'; -import { ActivityIndicator, Platform, StatusBar, StyleSheet, Text, View } from 'react-native'; +import { + ActivityIndicator, + Platform, + StatusBar, + StyleSheet, + Text, + useWindowDimensions, + View, +} from 'react-native'; import Animated, { FadeIn, FadeOut } from 'react-native-reanimated'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; @@ -7,25 +15,27 @@ import { useHeaderHeight } from 'expo-router/react-navigation'; import type { MessageProps, ResolvedComponents } from 'stream-chat-expo'; import { - AIStates, Channel, isLocalUrl, mergeThemes, Message, MessageList, ThemeProvider, - useAIState, - useChannelContext, useComponentsContext, useMessageContext, + usePaginatedMessageListContext, useTheme, WithComponents, } from 'stream-chat-expo'; import { AI_MESSAGE_PART_TYPE_PREFIX, + AIMessageParts, + AIThinking, + AIToolApproverProvider, AITypingIndicatorView, ComposerView, + parseMessageParts, StreamingMessageView, } from 'stream-chat-expo/ai-components'; @@ -84,33 +94,43 @@ const CustomMessage = (props: MessageProps) => { type UnsupportedAttachmentComponent = ResolvedComponents['UnsupportedAttachment']; // A reply's steps (`ai_reasoning`, `ai_tool_call`) are attachments the chat SDK doesn't know. They -// get their own views; until then they're left out rather than shown as unsupported files, and any -// other unknown attachment keeps the SDK's own view. +// show in the reply's own view, so they're left out here; any other unknown attachment keeps the +// SDK's own view. const hidingAISteps = (Fallback: UnsupportedAttachmentComponent) => { const HideAISteps: UnsupportedAttachmentComponent = (props) => props.attachment.type?.startsWith(AI_MESSAGE_PART_TYPE_PREFIX) ? null : ; return HideAISteps; }; +// A reply: its steps (reasoning, tool calls and their questions), then its answer. const CustomStreamingMessageView = () => { const { message } = useMessageContext(); + // Read so the steps render again as more of their reasoning arrives. + const { reasoningText, reasoningVersion } = useAppContext(); + const parts = useMemo(() => parseMessageParts(message.attachments), [message.attachments]); + const stepText = useMemo( + () => (stepId: string) => reasoningText(message.id, stepId), + // eslint-disable-next-line react-hooks/exhaustive-deps -- a new function per slice of reasoning + [message.id, reasoningText, reasoningVersion], + ); + // The message list sizes a bubble to its content; a reply's steps take the row's whole width, + // the screen less the list's 16 on each side. + const { width } = useWindowDimensions(); return ( - - + + {parts.length > 0 ? : null} + {message.text ? : null} ); }; const CustomComposerView = () => { - const { channel } = useChannelContext(); - const { stop } = useAppContext(); - - const { aiState } = useAIState(channel); + const { agentState, stop } = useAppContext(); // The router stops through its own session, not with an AI indicator event. const stopGenerating = useMemo(() => () => Promise.resolve(stop()), [stop]); - const isGenerating = [AIStates.Thinking, AIStates.Generating].includes(aiState); + const isGenerating = agentState.kind === 'responding' || agentState.kind === 'working'; const safeAreaInsets = useSafeAreaInsets(); const insets = useMemo( @@ -142,27 +162,30 @@ const EmptyStateIndicator = () => ( ); +// What the agent is doing while its reply shows nothing yet: thinking, or running skills. const CustomAITypingIndicatorView = () => { - const { channel } = useChannelContext(); - const { aiState } = useAIState(channel); - - const allowedStates = { - [AIStates.Thinking]: 'Thinking about the question...', - [AIStates.Generating]: 'Generating a response...', - [AIStates.ExternalSources]: 'Checking external sources...', - }; + const { agentState } = useAppContext(); + const { messages } = usePaginatedMessageListContext(); + const latest = messages[messages.length - 1]; + const replyHasBegun = + !!latest?.ai_generated && (!!latest.text || parseMessageParts(latest.attachments).length > 0); - const indicatorText = allowedStates[aiState as keyof typeof allowedStates]; - - if (!indicatorText) { - return null; + if (agentState.kind === 'working') { + const skills = agentState.skills.map((skill) => skill.replace(/_/g, ' ')).join(' and '); + return ( + + + + ); } - - return ( - - - - ); + if (agentState.kind === 'responding' && !replyHasBegun) { + return ( + + + + ); + } + return null; }; const RenderNull = () => null; @@ -176,7 +199,7 @@ const additionalFlatListProps = { }; export const ChatContent = () => { - const { channel, failure } = useAppContext(); + const { approver, channel, failure } = useAppContext(); const { bottom } = useSafeAreaInsets(); // The channel sits below the drawer header, so the keyboard offset has to account for it // (without it, Android keeps the composer behind the keyboard). @@ -211,13 +234,19 @@ export const ChatContent = () => { UnsupportedAttachment: CustomUnsupportedAttachment, }} > - - - - {failure ? {failure} : null} - - - + + + + + {failure ? {failure} : null} + + + + ); @@ -234,7 +263,7 @@ const styles = StyleSheet.create({ }, emptyContainerText: { fontSize: 24, fontWeight: 'bold' }, failureText: { color: '#B00020', fontSize: 15, paddingHorizontal: 24, textAlign: 'center' }, - streamingMessageViewWrapper: { width: '100%', paddingHorizontal: 16 }, + streamingMessageViewWrapper: { paddingHorizontal: 16 }, aiTypingIndicatorWrapper: { paddingHorizontal: 24, paddingVertical: 12, diff --git a/examples/ExpoChatGPTSample/screens/MenuDrawer.tsx b/examples/ExpoChatGPTSample/screens/MenuDrawer.tsx index 3157a9dd42..7a518cf3a9 100644 --- a/examples/ExpoChatGPTSample/screens/MenuDrawer.tsx +++ b/examples/ExpoChatGPTSample/screens/MenuDrawer.tsx @@ -1,61 +1,40 @@ -import React from 'react'; -import { Pressable, StyleSheet, Text, View } from 'react-native'; +import React, { useEffect, useState } from 'react'; +import { FlatList, Pressable, StyleSheet, Text, View } from 'react-native'; import { SafeAreaView } from 'react-native-safe-area-context'; +import type { Schemas } from '@stream-io/calypso'; import type { DrawerContentComponentProps } from 'expo-router/drawer'; -import type { ChannelSort } from 'stream-chat'; -import type { Channel as ChannelClass, ChannelFilters } from 'stream-chat'; -import { - ChannelList, - ChannelPreviewProps, - Plus, - useChannelsContext, - useStableCallback, - WithComponents, -} from 'stream-chat-expo'; +import { Plus, useStableCallback } from 'stream-chat-expo'; -import { AgentConfig } from '../agent/config'; +import { agentSessions } from '../agent/agents'; import { useAppContext } from '../contexts/AppContext'; -// The router keeps each conversation in an `agent` channel the user is a member of. -const filters: ChannelFilters = { - members: { - $in: [AgentConfig.user.id], - }, - type: 'agent', -}; - -const sort: ChannelSort = { last_updated: -1 }; +type SessionRow = Schemas['Session']; -const ChannelPreview = (props: ChannelPreviewProps) => { - const channel = props.channel; - const { onSelect } = useChannelsContext(); - const onPress = useStableCallback(() => { - onSelect?.(channel); - }); - return ( - ({ - paddingVertical: 8, - paddingHorizontal: 12, - opacity: pressed ? 0.6 : 1, - })} - onPress={onPress} - > - - {channel.data?.name ?? channel.cid} - - - ); -}; +const keyExtractor = (session: SessionRow) => session.id; +/** The agent's conversations, newest first, as the router lists them. */ export const MenuDrawer = ({ navigation }: DrawerContentComponentProps) => { - const { openConversation, startConversation } = useAppContext(); + const { conversationsVersion, openConversation, sessionId, startConversation } = useAppContext(); + const [sessions, setSessions] = useState([]); + + useEffect(() => { + let cancelled = false; + agentSessions() + .query({ limit: 30 }) + .then((page) => { + if (!cancelled) setSessions(page.items); + }) + .catch(() => undefined); + return () => { + cancelled = true; + }; + }, [conversationsVersion]); - const onSelect = useStableCallback((channel: ChannelClass) => { - openConversation(channel); + const onSelect = useStableCallback((id: string) => { + openConversation(id); navigation.closeDrawer(); }); @@ -64,31 +43,42 @@ export const MenuDrawer = ({ navigation }: DrawerContentComponentProps) => { navigation.closeDrawer(); }); + const renderItem = ({ item }: { item: SessionRow }) => ( + onSelect(item.id)} + style={({ pressed }) => [styles.row, { opacity: pressed ? 0.6 : 1 }]} + > + + {item.title || 'New conversation'} + + + ); + return ( Conversations - + - - - + ); }; const styles = StyleSheet.create({ - wrapper: { flex: 1, backgroundColor: '#FFFFFF' }, container: { + borderBottomColor: 'grey', + borderBottomWidth: 1, flexDirection: 'row', justifyContent: 'space-between', marginHorizontal: 12, paddingVertical: 16, - borderBottomWidth: 1, - borderBottomColor: 'grey', }, - title: { fontSize: 15, fontWeight: 'bold' }, + current: { color: '#005FFF' }, previewText: { fontSize: 15, fontWeight: 'bold' }, + row: { paddingHorizontal: 12, paddingVertical: 8 }, + title: { fontSize: 15, fontWeight: 'bold' }, + wrapper: { backgroundColor: '#FFFFFF', flex: 1 }, }); diff --git a/package/src/ai-components/AITypingIndicatorView.tsx b/package/src/ai-components/AITypingIndicatorView.tsx index f9bc401d10..b648e33699 100644 --- a/package/src/ai-components/AITypingIndicatorView.tsx +++ b/package/src/ai-components/AITypingIndicatorView.tsx @@ -1,30 +1,16 @@ import { useMemo } from 'react'; -import { StyleSheet, Text, View } from 'react-native'; +import { StyleSheet, Text } from 'react-native'; import { useTheme } from './contexts'; - -import { NativeShimmerView } from '../components/UIComponents/NativeShimmerView'; +import { Shimmer } from './internal/Shimmer'; export const AITypingIndicatorView = ({ text }: { text: string }) => { - const { - theme: { - colors: { shimmer }, - }, - } = useTheme(); const styles = useStyles(); - // The native shimmer paints behind its children, so it overlays the text. return ( - + {text} - - + ); }; @@ -34,8 +20,6 @@ const useStyles = () => { return useMemo( () => StyleSheet.create({ - // The native highlight band is soft and wide, so the opacity stays high. - shimmer: { opacity: 0.8 }, text: { fontSize: 16, color: theme.colors.grey_dark, diff --git a/package/src/ai-components/__tests__/format.test.ts b/package/src/ai-components/__tests__/format.test.ts new file mode 100644 index 0000000000..894fc64d97 --- /dev/null +++ b/package/src/ai-components/__tests__/format.test.ts @@ -0,0 +1,32 @@ +type Format = typeof import('../internal/format'); + +const loadFormat = () => { + let format: Format | undefined; + jest.isolateModules(() => { + format = require('../internal/format'); + }); + return format as Format; +}; + +describe('formatCallDuration', () => { + afterEach(() => jest.restoreAllMocks()); + + it('writes narrow units', () => { + const { formatCallDuration, formatMinutesAndSeconds } = loadFormat(); + expect(formatCallDuration(0.42)).toBe('0.4s'); + expect(formatCallDuration(65)).toBe('1m 5s'); + expect(formatMinutesAndSeconds(120)).toBe('2m'); + }); + + it('writes the same where Intl ignores the narrow width, as Hermes on iOS does', () => { + jest + .spyOn(Intl, 'NumberFormat') + .mockImplementation( + () => ({ format: (value: number) => `${value} sec` }) as unknown as Intl.NumberFormat, + ); + const { formatCallDuration, formatMinutesAndSeconds } = loadFormat(); + expect(formatCallDuration(0.42)).toBe('0.4s'); + expect(formatCallDuration(65)).toBe('1m 5s'); + expect(formatMinutesAndSeconds(7)).toBe('7s'); + }); +}); diff --git a/package/src/ai-components/__tests__/messagePartViews.test.tsx b/package/src/ai-components/__tests__/messagePartViews.test.tsx new file mode 100644 index 0000000000..7f64b6207c --- /dev/null +++ b/package/src/ai-components/__tests__/messagePartViews.test.tsx @@ -0,0 +1,228 @@ +import React, { type ReactElement } from 'react'; +import { I18nManager } from 'react-native'; + +import { act, fireEvent, render, screen } from '@testing-library/react-native'; + +import { + AIMessagePart, + AIReasoning, + AIThinking, + AIToolApproval, + AIToolApprovalCard, + type AIToolApprover, + AIToolCall, + type AIToolCallPart, + parseMessagePart, + StreamTheme, +} from '../index'; + +const renderWithTheme = (ui: ReactElement) => { + const result = render({ui}); + return { + ...result, + rerender: (next: ReactElement) => result.rerender({next}), + }; +}; + +const toolCall = (overrides: Partial = {}): AIToolCallPart => ({ + executor: 'server', + id: 'call_1', + name: 'get_weather', + status: 'running', + ...overrides, +}); + +describe('AIToolCall', () => { + it('names the call from its display title, else its name', () => { + const { rerender } = renderWithTheme( + , + ); + expect(screen.getByText('Checking the weather')).toBeTruthy(); + rerender(); + expect(screen.getByText('get weather')).toBeTruthy(); + }); + + it.each([ + ['awaiting_approval', 'Waiting for approval'], + ['awaiting_authorization', 'Waiting for you to sign in'], + ['awaiting_client', 'Waiting for the device'], + ['failed', "Didn't complete"], + ['cancelled', 'Cancelled'], + ])('says what a %s call waits for', (status, detail) => { + renderWithTheme(); + expect(screen.getByText(detail)).toBeTruthy(); + }); + + it('prefers the summary, and says a declined call was declined', () => { + const { rerender } = renderWithTheme( + , + ); + expect(screen.getByText('No network')).toBeTruthy(); + rerender( + , + ); + expect(screen.getByText('Declined')).toBeTruthy(); + }); + + it('shows how long a finished call took', () => { + const { rerender } = renderWithTheme( + , + ); + expect(screen.getByText('0.4s')).toBeTruthy(); + rerender(); + expect(screen.getByText('1m 5s')).toBeTruthy(); + rerender(); + expect(screen.queryByText('1m 5s')).toBeNull(); + }); +}); + +describe('AIToolApprovalCard', () => { + it('asks with the reason as a sentence and the agent’s own button titles', () => { + const onDecide = jest.fn(); + renderWithTheme( + , + ); + expect(screen.getByText('To check the local weather.')).toBeTruthy(); + expect(screen.getByText('Only your city is shared.')).toBeTruthy(); + fireEvent.press(screen.getByRole('button', { name: 'Share' })); + fireEvent.press(screen.getByRole('button', { name: "Don't Allow" })); + expect(onDecide.mock.calls).toEqual([[true], [false]]); + }); +}); + +describe('AIToolApproval', () => { + const asking = toolCall({ + approval: { title: 'Share your location?' }, + executor: 'client', + status: 'awaiting_approval', + targetClientId: 'ios-1', + targetUserId: 'alex', + }); + const approver = (decide: AIToolApprover['decide']): AIToolApprover => ({ + clientId: 'ios-1', + decide, + userId: 'alex', + }); + + it('asks only the targeted person on the targeted install', () => { + const decide = jest.fn(() => Promise.resolve()); + const { rerender } = renderWithTheme(); + expect(screen.queryByText('Share your location?')).toBeNull(); + rerender(); + expect(screen.queryByText('Share your location?')).toBeNull(); + rerender(); + expect(screen.getByText('Share your location?')).toBeTruthy(); + }); + + it('sends the answer once, and lets the person answer again when it fails', async () => { + let fail: () => void = () => undefined; + const decide = jest.fn( + () => + new Promise((_, reject) => { + fail = () => reject(new Error('offline')); + }), + ); + renderWithTheme(); + + fireEvent.press(screen.getByRole('button', { name: 'Allow' })); + fireEvent.press(screen.getByRole('button', { name: 'Allow' })); + expect(decide).toHaveBeenCalledTimes(1); + expect(decide).toHaveBeenCalledWith(asking, true); + expect(screen.getByRole('button', { name: 'Allow' })).toBeDisabled(); + + await act(async () => { + fail(); + await Promise.resolve(); + }); + expect(screen.getByText("Your answer couldn't be sent. Try again.")).toBeTruthy(); + expect(screen.getByRole('button', { name: 'Allow' })).toBeEnabled(); + }); +}); + +describe('AIMessagePart', () => { + it('shows a placeholder for a step it cannot show', () => { + renderWithTheme(); + expect(screen.getByText("A step this app version can't show")).toBeTruthy(); + }); +}); + +describe('AIReasoning', () => { + afterEach(() => { + jest.useRealTimers(); + jest.restoreAllMocks(); + }); + + it('counts the seconds while thinking, and folds when done', () => { + jest.useFakeTimers(); + const text = 'First thought.\n\n## Plan\nSecond thought.'; + const { rerender } = renderWithTheme(); + expect(screen.getByRole('button', { expanded: true, name: /Thinking\.\.\./ })).toBeTruthy(); + expect(screen.getByText('Plan')).toBeTruthy(); + + act(() => jest.advanceTimersByTime(7_000)); + expect(screen.getByText('Thinking... 7s')).toBeTruthy(); + + rerender( + , + ); + expect(screen.getByRole('button', { expanded: false, name: /Thought for 12s/ })).toBeTruthy(); + expect(screen.getByText('Planned the day')).toBeTruthy(); + expect(screen.queryByText('Plan')).toBeNull(); + + fireEvent.press(screen.getByRole('button', { name: /Thought for 12s/ })); + expect(screen.getByRole('button', { expanded: true, name: /Thought for 12s/ })).toBeTruthy(); + expect(screen.getByText('Plan')).toBeTruthy(); + }); + + it('shows a step’s live text, else its preview', () => { + const part = { durationMs: 2000, id: 'r1', preview: 'The opening', status: 'completed' }; + const { rerender } = renderWithTheme( + 'The whole thought'} />, + ); + expect(screen.getByText('The whole thought')).toBeTruthy(); + expect(screen.getByRole('button', { name: /Thought for 2s/ })).toBeTruthy(); + rerender(); + expect(screen.getByText('The opening')).toBeTruthy(); + }); + + it('says Thinking... for thinking that shows nothing yet', () => { + renderWithTheme(); + expect(screen.getByRole('button', { expanded: false, name: 'Thinking...' })).toBeTruthy(); + }); + + it('turns the open chevron the other way in RTL', () => { + const rotatedBy = (degrees: string) => + screen.UNSAFE_root.findAll((node) => + JSON.stringify(node.props?.style ?? '').includes(`"rotate":"${degrees}"`), + ).length > 0; + const originalIsRTL = I18nManager.isRTL; + const setRTL = (value: boolean) => + Object.defineProperty(I18nManager, 'isRTL', { configurable: true, value }); + + try { + setRTL(false); + const { unmount } = renderWithTheme(); + expect(rotatedBy('90deg')).toBe(true); + unmount(); + + setRTL(true); + renderWithTheme(); + expect(rotatedBy('-90deg')).toBe(true); + } finally { + setRTL(originalIsRTL); + } + }); +}); diff --git a/package/src/ai-components/__tests__/translation.test.tsx b/package/src/ai-components/__tests__/translation.test.tsx index 097677c86e..2e6dea43f1 100644 --- a/package/src/ai-components/__tests__/translation.test.tsx +++ b/package/src/ai-components/__tests__/translation.test.tsx @@ -32,14 +32,16 @@ describe('AI translation', () => { expect(interpolateAIString('Thought for {{ duration }}', {})).toBe( 'Thought for {{ duration }}', ); - expect(interpolateAIString('Thinking…')).toBe('Thinking…'); + expect(interpolateAIString('Thinking...')).toBe('Thinking...'); expect(interpolateAIString('{{ count }} files', { count: 0 })).toBe('0 files'); }); it('is English without ', () => { const { result } = renderHook(() => useAITranslation()); expect(result.current.t).toBe(defaultAITranslate); - expect(result.current.t('Thinking… {{ duration }}', { duration: '4s' })).toBe('Thinking… 4s'); + expect(result.current.t('Thinking... {{ duration }}', { duration: '4s' })).toBe( + 'Thinking... 4s', + ); }); it('follows the chat SDK translator, as inside ', async () => { diff --git a/package/src/ai-components/contexts/translation-context.tsx b/package/src/ai-components/contexts/translation-context.tsx index a78dab78f6..6d6bce7af5 100644 --- a/package/src/ai-components/contexts/translation-context.tsx +++ b/package/src/ai-components/contexts/translation-context.tsx @@ -33,9 +33,12 @@ export const defaultAITranslate: AITranslate = (key, options) => interpolateAISt * no translator set up (no ``, or before it has one) they are in English. */ export const useAITranslation = () => { - const { t } = useContext(TranslationContext); + const { t, userLanguage } = useContext(TranslationContext); return useMemo( - () => ({ t: t === defaultTranslatorFunction ? defaultAITranslate : (t as AITranslate) }), - [t], + () => ({ + t: t === defaultTranslatorFunction ? defaultAITranslate : (t as AITranslate), + userLanguage: userLanguage ?? 'en', + }), + [t, userLanguage], ); }; diff --git a/package/src/ai-components/index.ts b/package/src/ai-components/index.ts index 9494ef01de..23d3042976 100644 --- a/package/src/ai-components/index.ts +++ b/package/src/ai-components/index.ts @@ -7,6 +7,8 @@ export * from './AITypingIndicatorView'; export * from './voice-visualization'; export * from './message-parts'; +export * from './reasoning'; +export * from './tool-calls'; export * from './components'; diff --git a/package/src/ai-components/internal/Shimmer.tsx b/package/src/ai-components/internal/Shimmer.tsx new file mode 100644 index 0000000000..aef0925571 --- /dev/null +++ b/package/src/ai-components/internal/Shimmer.tsx @@ -0,0 +1,40 @@ +import React, { type PropsWithChildren } from 'react'; +import { type StyleProp, StyleSheet, View, type ViewStyle } from 'react-native'; + +import { NativeShimmerView } from '../../components/UIComponents/NativeShimmerView'; +import { useTheme } from '../contexts'; + +type ShimmerProps = PropsWithChildren<{ active?: boolean; style?: StyleProp }>; + +/** + * Sweeps the native shimmer across its content while `active`, for something still in progress. + * The native shimmer paints behind its children, so it is laid over the content instead, with a + * transparent base: the highlight has to cross the glyphs, which would otherwise hide it. + */ +export const Shimmer = ({ active = true, children, style }: ShimmerProps) => { + const { + theme: { + colors: { shimmer }, + }, + } = useTheme(); + + return ( + + {children} + {active ? ( + + ) : null} + + ); +}; + +const styles = StyleSheet.create({ + // The native highlight band is soft and wide, so it is kept fairly strong over the text. + shimmer: { opacity: 0.8 }, +}); diff --git a/package/src/ai-components/internal/format.ts b/package/src/ai-components/internal/format.ts new file mode 100644 index 0000000000..026f55280d --- /dev/null +++ b/package/src/ai-components/internal/format.ts @@ -0,0 +1,56 @@ +import { useEffect, useState } from 'react'; + +// Hermes on iOS formats units with Foundation, which ignores the narrow width ("1 sec"), so +// there the units are written the way they are without Intl. +const honoursNarrowUnits = (() => { + try { + return ( + new Intl.NumberFormat('en', { style: 'unit', unit: 'second', unitDisplay: 'narrow' }).format( + 1, + ) === '1s' + ); + } catch { + return false; + } +})(); + +const unit = (locale: string, value: number, name: 'minute' | 'second') => { + if (!honoursNarrowUnits) return `${value}${name === 'minute' ? 'm' : 's'}`; + try { + return new Intl.NumberFormat(locale, { + style: 'unit', + unit: name, + unitDisplay: 'narrow', + }).format(value); + } catch { + return `${value}${name === 'minute' ? 'm' : 's'}`; + } +}; + +/** A whole number of seconds as minutes and seconds, in narrow units: "7s", "1m 5s". */ +export const formatMinutesAndSeconds = (totalSeconds: number, locale = 'en') => { + const seconds = Math.max(0, Math.floor(totalSeconds)); + const minutes = Math.floor(seconds / 60); + const rest = seconds % 60; + if (minutes === 0) return unit(locale, rest, 'second'); + if (rest === 0) return unit(locale, minutes, 'minute'); + return `${unit(locale, minutes, 'minute')} ${unit(locale, rest, 'second')}`; +}; + +/** A tool call's duration: tenths below a second ("0.4s"), then minutes and seconds. */ +export const formatCallDuration = (durationSeconds: number, locale = 'en') => + durationSeconds < 1 + ? unit(locale, Math.round(durationSeconds * 10) / 10, 'second') + : formatMinutesAndSeconds(Math.round(durationSeconds), locale); + +/** `Date.now()`, refreshed every `intervalMs` while `active`. */ +export const useNow = (active: boolean, intervalMs = 1000) => { + const [now, setNow] = useState(() => Date.now()); + useEffect(() => { + if (!active) return; + setNow(Date.now()); + const timer = setInterval(() => setNow(Date.now()), intervalMs); + return () => clearInterval(timer); + }, [active, intervalMs]); + return now; +}; diff --git a/package/src/ai-components/internal/icons/AgentIcons.tsx b/package/src/ai-components/internal/icons/AgentIcons.tsx new file mode 100644 index 0000000000..188bd0fe9b --- /dev/null +++ b/package/src/ai-components/internal/icons/AgentIcons.tsx @@ -0,0 +1,60 @@ +import React from 'react'; +import { I18nManager } from 'react-native'; +import Svg, { Path } from 'react-native-svg'; + +// The agent reply steps' icons, on a 20 × 20 grid with a 1.5 stroke, as in stream-chat-react's +// AI components. Decorative: the text beside each one says what it means. + +export type AgentIconProps = { color: string; size?: number }; + +const circle = + 'M17.5 10C17.5 14.14 14.14 17.5 10 17.5C5.86 17.5 2.5 14.14 2.5 10C2.5 5.86 5.86 2.5 10 2.5C14.14 2.5 17.5 5.86 17.5 10Z'; + +const strokeIcon = (d: string, mirrorsInRTL = false) => { + const Icon = ({ color, size = 16 }: AgentIconProps) => ( + + + + ); + return Icon; +}; + +export const ReasoningIcon = strokeIcon( + 'M7.5 15.625H12.5M8.125 18.125H11.875M10 1.875C6.89 1.875 4.375 4.39 4.375 7.5C4.375 9.53 5.45 11.31 7.06 12.3C7.33 12.47 7.5 12.76 7.5 13.08V13.75H12.5V13.08C12.5 12.76 12.67 12.47 12.94 12.3C14.55 11.31 15.625 9.53 15.625 7.5C15.625 4.39 13.11 1.875 10 1.875Z', +); + +export const ChevronIcon = strokeIcon('M7.5 4.375L13.125 10L7.5 15.625', true); + +export const CheckIcon = strokeIcon('M4.375 10.625L8.125 14.375L15.625 5.625'); + +export const FailedIcon = strokeIcon(`M10 6.25V10.625M10 13.75H10.006${circle}`); + +export const CancelledIcon = strokeIcon(`M6.875 10H13.125${circle}`); + +export const InfoIcon = strokeIcon(`M10 9.375V13.75M10 6.25H10.006${circle}`); + +export const HandRaisedIcon = strokeIcon( + 'M10 5V9.375M10 5C10 4.31 10.56 3.75 11.25 3.75C11.94 3.75 12.5 4.31 12.5 5V9.375M10 5V3.75C10 3.06 9.44 2.5 8.75 2.5C8.06 2.5 7.5 3.06 7.5 3.75V5M12.5 6.25C12.5 5.56 13.06 5 13.75 5C14.44 5 15 5.56 15 6.25V11.875C15 15.03 12.84 17.5 10 17.5C7.82 17.5 6.43 16.37 5.42 14.43L3.92 11.53C3.6 10.92 3.84 10.17 4.45 9.86C5 9.58 5.67 9.73 6.04 10.23L7.5 12.19V5M7.5 5V9.375', +); + +export const KeyIcon = strokeIcon( + 'M12.5 5.625C13.19 5.625 13.75 6.185 13.75 6.875M16.875 6.875C16.875 9.635 14.635 11.875 11.875 11.875C11.37 11.875 10.88 11.8 10.42 11.66L8.75 13.125H7.5V14.375H6.25V15.625H5V16.875H3.125V14.88L8.34 9.66C8.2 9.2 8.125 8.71 8.125 8.125C8.125 4.94 10.37 1.875 13.75 2.5C15.63 2.85 16.875 4.69 16.875 6.875Z', +); + +export const DeviceIcon = strokeIcon( + 'M8.125 15H11.875M6.25 18.125H13.75C14.44 18.125 15 17.565 15 16.875V3.125C15 2.435 14.44 1.875 13.75 1.875H6.25C5.56 1.875 5 2.435 5 3.125V16.875C5 17.565 5.56 18.125 6.25 18.125Z', +); diff --git a/package/src/ai-components/message-parts/AIMessageParts.tsx b/package/src/ai-components/message-parts/AIMessageParts.tsx new file mode 100644 index 0000000000..9c99694349 --- /dev/null +++ b/package/src/ai-components/message-parts/AIMessageParts.tsx @@ -0,0 +1,94 @@ +import React, { type ReactNode } from 'react'; +import { StyleSheet, Text, View } from 'react-native'; + +import { AIToolApproval } from './AIToolApproval'; +import { AIToolCall } from './AIToolCall'; +import type { AIMessagePartData } from './types'; + +import { useAITranslation, useTheme } from '../contexts'; +import { InfoIcon } from '../internal/icons/AgentIcons'; +import { AIReasoning } from '../reasoning/AIReasoning'; +import type { AIToolApprover } from '../tool-calls/AIToolApprover'; + +export type AIMessagePartProps = { + part: AIMessagePartData; + /** + * Who answers tool calls' questions on this device, to ask them. Defaults to the one of the + * nearest `AIToolApproverProvider`. + */ + approver?: AIToolApprover; + /** The whole of a reasoning step's thinking, when the app has it. */ + reasoningText?: (stepId: string) => string | undefined; +}; + +/** + * One step of a reply: a round of reasoning (`AIReasoning`), a tool call (`AIToolCall`, with its + * question when it waits for this person here), or a neutral line for a step this SDK can't + * show. The counterpart of `AIMessagePartView`. + */ +export const AIMessagePart = ({ approver, part, reasoningText }: AIMessagePartProps) => { + const { t } = useAITranslation(); + const { + theme: { colors, messageParts }, + } = useTheme(); + + if (part.reasoning) { + return ; + } + if (part.toolCall) { + return ( + + + + + ); + } + return ( + + + + {t("A step this app version can't show")} + + + ); +}; + +export type AIMessagePartsProps = Omit & { + parts: AIMessagePartData[]; + /** Renders a step, instead of `AIMessagePart`. */ + children?: (part: AIMessagePartData) => ReactNode; +}; + +/** The steps of a reply, in order. The counterpart of `AIMessagePartsView`. */ +export const AIMessageParts = ({ + approver, + children, + parts, + reasoningText, +}: AIMessagePartsProps) => { + const { + theme: { messageParts }, + } = useTheme(); + return ( + + {parts.map((part) => ( + + {children ? ( + children(part) + ) : ( + + )} + + ))} + + ); +}; + +const styles = StyleSheet.create({ + parts: { gap: 8, width: '100%' }, + tool: { gap: 8 }, + unsupported: { alignItems: 'center', flexDirection: 'row', gap: 8 }, + unsupportedText: { flexShrink: 1, fontSize: 14 }, +}); diff --git a/package/src/ai-components/message-parts/AIToolApproval.tsx b/package/src/ai-components/message-parts/AIToolApproval.tsx new file mode 100644 index 0000000000..73e3d2efa6 --- /dev/null +++ b/package/src/ai-components/message-parts/AIToolApproval.tsx @@ -0,0 +1,193 @@ +import React, { type ReactNode, useEffect, useRef, useState } from 'react'; +import { ActivityIndicator, Pressable, StyleSheet, Text, View } from 'react-native'; + +import { isAwaitingApproval } from './parseMessageParts'; +import type { AIToolApprovalData, AIToolCallPart } from './types'; + +import { useAITranslation, useTheme } from '../contexts'; +import { type AIToolApprover, useAIToolApprover } from '../tool-calls/AIToolApprover'; + +/** Where the person's answer to a tool call's question is. */ +export type AIToolApprovalState = { + /** The answer couldn't be sent, and the person can answer again. */ + failed: boolean; + /** The answer is on its way, or arrived and the call's step hasn't changed yet. */ + isSending: boolean; +}; + +const idle: AIToolApprovalState = { failed: false, isSending: false }; + +/** The agent's reason, as a sentence, then what allowing it shares. */ +export const approvalLines = (approval: AIToolApprovalData) => { + const lines: string[] = []; + const reason = approval.reason?.trim(); + if (reason) { + const sentence = reason[0].toUpperCase() + reason.slice(1); + lines.push(/[.!?…]$/.test(sentence) ? sentence : `${sentence}.`); + } + if (approval.message) lines.push(approval.message); + return lines; +}; + +export type AIToolApprovalCardProps = { + approval: AIToolApprovalData; + /** Called with the person's answer. */ + onDecide: (allowed: boolean) => void; + state?: AIToolApprovalState; +}; + +/** + * Asks the person a tool call's question, such as "Share your location?", with the agent's + * reason and what allowing it shares, and an Allow and a Don't Allow button. Used inline under + * a step by `AIToolApproval`, and on its own, such as on a voice call. The counterpart of + * `AIToolApprovalCard`. + */ +export const AIToolApprovalCard = ({ + approval, + onDecide, + state = idle, +}: AIToolApprovalCardProps) => { + const { t } = useAITranslation(); + const { + theme: { colors, messageParts }, + } = useTheme(); + const card = messageParts?.approvalCard; + + return ( + + + {approval.title} + {approvalLines(approval).map((line) => ( + + {line} + + ))} + + + onDecide(true)} + style={({ pressed }) => [ + styles.button, + { backgroundColor: colors.accent_blue, opacity: pressed ? 0.7 : 1 }, + card?.allowButton, + ]} + > + + {approval.allowTitle ?? t('Allow')} + + + onDecide(false)} + style={({ pressed }) => [ + styles.button, + styles.declineButton, + { borderColor: colors.grey_gainsboro, opacity: pressed ? 0.7 : 1 }, + card?.declineButton, + ]} + > + + {approval.declineTitle ?? t("Don't Allow")} + + + {state.isSending ? : null} + + {state.failed ? ( + + {t("Your answer couldn't be sent. Try again.")} + + ) : null} + + ); +}; + +export type AIToolApprovalProps = { + part: AIToolCallPart; + /** Who answers on this device. Defaults to the one of the nearest `AIToolApproverProvider`. */ + approver?: AIToolApprover; + /** Renders the question, instead of `AIToolApprovalCard`. */ + children?: ( + approval: AIToolApprovalData, + state: AIToolApprovalState, + decide: (allowed: boolean) => void, + ) => ReactNode; +}; + +/** + * Asks the question of a tool call that waits for this person on this device, and sends their + * answer with `approver.decide`. Until the call's step changes the buttons wait; a failed send + * lets them answer again. Renders nothing for a call that does not wait for them here. The + * counterpart of `AIToolApprovalView`. + */ +export const AIToolApproval = ({ approver: approverProp, children, part }: AIToolApprovalProps) => { + const approver = useAIToolApprover(approverProp); + const [state, setState] = useState(idle); + const mounted = useRef(true); + useEffect(() => { + mounted.current = true; + return () => { + mounted.current = false; + }; + }, []); + + if ( + !approver || + !part.approval || + !isAwaitingApproval(part, approver.userId, approver.clientId) + ) { + return null; + } + + const decide = (allowed: boolean) => { + if (state.isSending) return; + setState({ failed: false, isSending: true }); + approver.decide(part, allowed).catch(() => { + if (mounted.current) setState({ failed: true, isSending: false }); + }); + }; + + return ( + <> + {children ? ( + children(part.approval, state, decide) + ) : ( + + )} + + ); +}; + +const styles = StyleSheet.create({ + actions: { alignItems: 'center', flexDirection: 'row', flexWrap: 'wrap', gap: 8 }, + button: { + alignItems: 'center', + borderRadius: 9999, + justifyContent: 'center', + minHeight: 36, + paddingHorizontal: 14, + paddingVertical: 6, + }, + buttonText: { fontSize: 14, fontWeight: '600' }, + container: { borderRadius: 16, borderWidth: 1, gap: 12, padding: 12 }, + declineButton: { borderWidth: 1 }, + failure: { fontSize: 13 }, + message: { fontSize: 14, lineHeight: 20 }, + text: { gap: 4 }, + title: { fontSize: 15, fontWeight: '600', lineHeight: 20 }, +}); diff --git a/package/src/ai-components/message-parts/AIToolCall.tsx b/package/src/ai-components/message-parts/AIToolCall.tsx new file mode 100644 index 0000000000..1a1422ed21 --- /dev/null +++ b/package/src/ai-components/message-parts/AIToolCall.tsx @@ -0,0 +1,112 @@ +import React from 'react'; +import { ActivityIndicator, StyleSheet, Text, View } from 'react-native'; + +import { isDeclined, isToolCallFinished } from './parseMessageParts'; +import type { AIToolCallPart } from './types'; + +import { useAITranslation, useTheme } from '../contexts'; +import { formatCallDuration } from '../internal/format'; +import { + CancelledIcon, + CheckIcon, + DeviceIcon, + FailedIcon, + HandRaisedIcon, + KeyIcon, +} from '../internal/icons/AgentIcons'; +import { Shimmer } from '../internal/Shimmer'; + +export type AIToolCallProps = { + part: AIToolCallPart; +}; + +/** + * One tool call of a reply: what it is doing (its `displayTitle`, else its name), how it is + * going, and how long it took once it is done. The counterpart of `AIToolCallView`. + */ +export const AIToolCall = ({ part }: AIToolCallProps) => { + const { t, userLanguage } = useAITranslation(); + const { + theme: { colors, messageParts }, + } = useTheme(); + + const title = part.displayTitle ?? part.name.replace(/_/g, ' '); + const detail = (() => { + if (isDeclined(part)) return part.summary ?? t('Declined'); + switch (part.status) { + case 'awaiting_approval': + return part.summary ?? t('Waiting for approval'); + case 'awaiting_authorization': + return part.summary ?? t('Waiting for you to sign in'); + case 'awaiting_client': + return part.summary ?? t('Waiting for the device'); + case 'failed': + return part.summary ?? t("Didn't complete"); + case 'cancelled': + return part.summary ?? t('Cancelled'); + default: + return part.summary; + } + })(); + + // A status this SDK doesn't know reads as still in progress. + const icon = (() => { + switch (part.status) { + case 'awaiting_approval': + return ; + case 'awaiting_authorization': + return ; + case 'awaiting_client': + return ; + case 'completed': + return ; + case 'failed': + return ; + case 'cancelled': + return ; + default: + return ; + } + })(); + + const finished = isToolCallFinished(part.status); + const waiting = part.status.startsWith('awaiting_'); + + return ( + + {icon} + + + + {title} + + + {detail ? ( + + {detail} + + ) : null} + + {finished && part.durationMs !== undefined ? ( + + {formatCallDuration(part.durationMs / 1000, userLanguage)} + + ) : null} + + ); +}; + +const styles = StyleSheet.create({ + container: { alignItems: 'flex-start', flexDirection: 'row', gap: 8 }, + detail: { fontSize: 13, lineHeight: 18 }, + duration: { fontSize: 13, fontVariant: ['tabular-nums'], lineHeight: 20 }, + icon: { marginTop: 2 }, + // The platform spinner, sized to the 16 the icons are drawn at. + spinner: { height: 16, transform: [{ scale: 0.8 }], width: 16 }, + text: { flex: 1, gap: 2, minWidth: 0 }, + title: { fontSize: 15, lineHeight: 20 }, + titleShimmer: { alignSelf: 'flex-start' }, +}); diff --git a/package/src/ai-components/message-parts/index.ts b/package/src/ai-components/message-parts/index.ts index 9ade54059a..c34894c2f6 100644 --- a/package/src/ai-components/message-parts/index.ts +++ b/package/src/ai-components/message-parts/index.ts @@ -1,2 +1,5 @@ export * from './types'; export * from './parseMessageParts'; +export * from './AIMessageParts'; +export * from './AIToolApproval'; +export * from './AIToolCall'; diff --git a/package/src/ai-components/reasoning/AIReasoning.tsx b/package/src/ai-components/reasoning/AIReasoning.tsx new file mode 100644 index 0000000000..9b4bae937a --- /dev/null +++ b/package/src/ai-components/reasoning/AIReasoning.tsx @@ -0,0 +1,376 @@ +import React, { memo, type ReactNode, useEffect, useMemo, useRef, useState } from 'react'; +import { + I18nManager, + type NativeScrollEvent, + type NativeSyntheticEvent, + Pressable, + ScrollView, + StyleSheet, + Text, + View, +} from 'react-native'; + +import { FINISHED_REASONING_SHOWN_MS, useAIThinkingTurn } from './thinkingClock'; + +import type { ScrollViewRef } from '../../types/react-native-compat'; +import { type DeepPartial, StreamTheme, useAITranslation, useTheme } from '../contexts'; +import { formatMinutesAndSeconds, useNow } from '../internal/format'; +import { ChevronIcon, ReasoningIcon } from '../internal/icons/AgentIcons'; +import { Shimmer } from '../internal/Shimmer'; +import { Markdown } from '../markdown'; +import type { AIReasoningPart } from '../message-parts/types'; +import type { Theme } from '../theme'; + +/** + * Splits reasoning at blank lines. Reasoning only grows at its end, so a paragraph's position + * is a stable identity. + */ +export const splitReasoningParagraphs = (text: string) => + text + .split('\n\n') + .map((paragraph) => paragraph.trim()) + .filter(Boolean); + +// A heading reads as a bold line, since a paragraph of thinking has no document to structure. +const headingsAsBold = (paragraph: string) => + paragraph + .split('\n') + .map((line) => { + const heading = /^ *#+\s*(.*)$/.exec(line)?.[1]?.trim(); + return heading ? `**${heading}**` : line; + }) + .join('\n'); + +const ReasoningParagraph = memo(({ text }: { text: string }) => ( + {headingsAsBold(text)} +)); +ReasoningParagraph.displayName = 'ReasoningParagraph'; + +// Reveals text that grows a little at a time, a sixth of the backlog every frame at 30 fps, as +// new thoughts arrive about that often. Text that is replaced, or not animated, shows at once. +const useTextReveal = (text: string, animated: boolean) => { + const [shown, setShown] = useState(text); + const shownRef = useRef(text); + const targetRef = useRef(text); + + useEffect(() => { + const grows = animated && text.startsWith(shownRef.current); + targetRef.current = text; + if (!grows) { + shownRef.current = text; + setShown(text); + return; + } + const timer = setInterval(() => { + const target = targetRef.current; + const current = shownRef.current; + if (current.length >= target.length) { + clearInterval(timer); + return; + } + let end = current.length + Math.max(1, Math.floor((target.length - current.length) / 6)); + // never split a surrogate pair + const code = target.charCodeAt(end - 1); + if (code >= 0xd800 && code <= 0xdbff) end += 1; + shownRef.current = target.slice(0, end); + setShown(shownRef.current); + }, 1000 / 30); + return () => clearInterval(timer); + }, [animated, text]); + + return shown; +}; + +const ReasoningPanel = ({ + isLive, + maxHeight, + text, +}: { + isLive: boolean; + maxHeight: number; + text: string; +}) => { + const shown = useTextReveal(text, isLive); + const scrollerRef = useRef(null); + // Whether the reader is at the end, so new thoughts keep it there. A finished trace opens at + // its start. + const following = useRef(isLive); + const { + theme: { messageParts }, + } = useTheme(); + + const onScroll = ({ nativeEvent }: NativeSyntheticEvent) => { + const { contentOffset, contentSize, layoutMeasurement } = nativeEvent; + following.current = contentOffset.y + layoutMeasurement.height >= contentSize.height - 24; + }; + + return ( + { + if (following.current) scrollerRef.current?.scrollToEnd({ animated: false }); + }} + onScroll={onScroll} + ref={scrollerRef} + scrollEventThrottle={32} + style={[{ maxHeight }, messageParts?.reasoning?.panel]} + > + {splitReasoningParagraphs(shown).map((paragraph, index) => ( + + ))} + + ); +}; + +export type AIReasoningProps = { + /** How long the model has thought, in seconds, shown as "Thought for 12s" once it is done. */ + duration?: number; + /** A note under the open reasoning, such as how long it is kept. */ + footnote?: ReactNode; + /** Whether the reasoning is open once the model is done. Defaults to `false`. */ + initiallyExpanded?: boolean; + /** Whether the model is still thinking. Defaults to whether `part` is streaming. */ + isThinking?: boolean; + /** How tall the reasoning grows, in points, before it scrolls. Defaults to 260. */ + maxExpandedHeight?: number; + /** A reasoning step, whose preview, summary, status and duration it shows. */ + part?: AIReasoningPart; + /** + * The whole of a reasoning step's thinking when the app has it, such as from live updates. + * Read on every render; without it, the step shows its own preview. + */ + reasoningText?: (stepId: string) => string | undefined; + /** Whether the reasoning is open while the model thinks. Defaults to `true`. */ + showsLiveReasoning?: boolean; + /** When the thinking began (`Date.now()`), for a header that counts from before it appeared. */ + since?: number; + /** A one-line summary shown beside the header once the model is done. */ + summary?: string; + /** + * The reasoning so far. Blank lines separate paragraphs, and Markdown is rendered. Defaults to + * `reasoningText(part.id)`, else the step's preview or summary. + */ + text?: string; +}; + +const opensByDefault = ( + isThinking: boolean, + showsLiveReasoning: boolean, + initiallyExpanded: boolean, +) => (isThinking ? showsLiveReasoning : initiallyExpanded); + +/** + * A round of the model's reasoning: a header that counts the seconds while it thinks ("Thinking... + * 7s") and says how long it thought once done ("Thought for 12s"), over the reasoning itself, + * which streams into view while the model thinks and folds away when it is done. Pressing the + * header opens or folds it. The counterpart of `StreamingReasoningView`. + * + * Inside an `AIConversation` that showed the agent thinking, the reply's first reasoning step + * takes over that thinking: its seconds count from when the question was asked. + */ +export const AIReasoning = ({ + duration: durationProp, + footnote, + initiallyExpanded = false, + isThinking: isThinkingProp, + maxExpandedHeight = 260, + part, + reasoningText, + showsLiveReasoning = true, + since, + summary: summaryProp, + text: textProp, +}: AIReasoningProps) => { + const { t, userLanguage } = useAITranslation(); + const { theme } = useTheme(); + const { colors, messageParts } = theme; + const reasoningStyle = messageParts?.reasoning; + const turn = useAIThinkingTurn(); + + const text = + textProp ?? + (part ? (reasoningText?.(part.id) ?? part.preview ?? part.summary) : undefined) ?? + ''; + const isThinking = isThinkingProp ?? part?.status === 'streaming'; + const ownDuration = + durationProp ?? (part?.durationMs !== undefined ? part.durationMs / 1000 : undefined); + const summary = summaryProp ?? part?.summary; + + // The turn whose thinking this step takes over, when it is the first step of a turn that was + // seen thinking. + const thinkingTurn = + part && turn && turn.firstStepId === part.id && turn.clock.start(turn.key) !== undefined + ? turn + : undefined; + const clock = thinkingTurn?.clock; + const turnKey = thinkingTurn?.key; + useEffect(() => { + if (isThinking && clock && turnKey !== undefined) clock.firstStepStreams(turnKey); + }, [clock, isThinking, turnKey]); + + // What the reader chose by pressing the header. Until they do, the reasoning is open while + // the model thinks and folded once it is done. + const [choice, setChoice] = useState(null); + const [open, setOpen] = useState(() => + opensByDefault(isThinking, showsLiveReasoning, initiallyExpanded), + ); + const [thinkingSince, setThinkingSince] = useState(() => since ?? Date.now()); + const [thoughtFor, setThoughtFor] = useState(); + + // A conversation's first step that arrived done, with the answer, shows its reasoning before + // it folds: the model sent none of it while it thought. + const [showsFinished] = useState( + () => + !isThinking && + text !== '' && + !!thinkingTurn && + thinkingTurn.clock.showsFirstStep(thinkingTurn.key), + ); + useEffect(() => { + if (!showsFinished) return; + setOpen(true); + const timer = setTimeout(() => setOpen(false), FINISHED_REASONING_SHOWN_MS); + return () => clearTimeout(timer); + }, [showsFinished]); + + const wasThinking = useRef(isThinking); + useEffect(() => { + if (wasThinking.current === isThinking) return; + wasThinking.current = isThinking; + if (isThinking) { + setThinkingSince(Date.now()); + } else { + if (clock && turnKey !== undefined) clock.firstStepDone(turnKey, ownDuration); + setThoughtFor(Math.max(ownDuration ?? 0, (Date.now() - thinkingSince) / 1000)); + } + setOpen(choice ?? opensByDefault(isThinking, showsLiveReasoning, initiallyExpanded)); + }, [ + choice, + clock, + initiallyExpanded, + isThinking, + ownDuration, + showsLiveReasoning, + thinkingSince, + turnKey, + ]); + + const start = since ?? (thinkingTurn ? thinkingTurn.clock.start(thinkingTurn.key) : undefined); + const now = useNow(isThinking); + const elapsed = Math.max(ownDuration ?? 0, (now - (start ?? thinkingSince)) / 1000); + const measured = + !isThinking && thinkingTurn + ? thinkingTurn.clock.durationOfFirstStep(thinkingTurn.key, ownDuration) + : undefined; + const thought = measured ?? ownDuration ?? thoughtFor; + + const title = isThinking + ? elapsed >= 1 + ? t('Thinking... {{ duration }}', { + duration: formatMinutesAndSeconds(elapsed, userLanguage), + }) + : t('Thinking...') + : thought !== undefined + ? t('Thought for {{ duration }}', { + duration: formatMinutesAndSeconds(Math.max(1, thought), userLanguage), + }) + : t('Thought process'); + + // Live reasoning is open before any of it arrives, but there is nothing to show until then. + const expanded = open && text !== ''; + + // The reasoning reads in the secondary colour, a little smaller than an answer. + const panelStyle = useMemo>( + () => ({ + markdown: { + paragraph: { marginBottom: 4, marginTop: 4 }, + text: { color: colors.grey, fontSize: 14, lineHeight: 20 }, + }, + }), + [colors.grey], + ); + + return ( + + { + setChoice(!expanded); + setOpen(!expanded); + }} + style={[styles.header, reasoningStyle?.header]} + > + + + + {title} + + + {summary && !isThinking && !expanded ? ( + + {summary} + + ) : null} + + + + + {expanded ? ( + + + {footnote && !isThinking && (choice === true || initiallyExpanded) ? footnote : null} + + ) : null} + + ); +}; + +export type AIThinkingProps = { + /** When the thinking began (`Date.now()`). Inside an `AIConversation`, when the question was asked. */ + since?: number; +}; + +/** + * The agent thinking before its reply shows anything: a header that counts the seconds. In an + * `AIConversation`, the reply's first reasoning step takes over its seconds. The counterpart of + * `AIThinkingView`. + */ +export const AIThinking = ({ since }: AIThinkingProps) => { + const turn = useAIThinkingTurn(); + return ( + + ); +}; + +const styles = StyleSheet.create({ + container: { borderStartWidth: 2, gap: 4, paddingStart: 12 }, + header: { + alignItems: 'center', + alignSelf: 'flex-start', + flexDirection: 'row', + gap: 6, + maxWidth: '100%', + }, + lead: { alignItems: 'center', flexDirection: 'row', gap: 6 }, + summary: { flexShrink: 1, fontSize: 14 }, + title: { fontSize: 14, fontVariant: ['tabular-nums'], fontWeight: '500' }, +}); diff --git a/package/src/ai-components/reasoning/index.ts b/package/src/ai-components/reasoning/index.ts new file mode 100644 index 0000000000..221d5d7052 --- /dev/null +++ b/package/src/ai-components/reasoning/index.ts @@ -0,0 +1,2 @@ +export * from './AIReasoning'; +export * from './thinkingClock'; diff --git a/package/src/ai-components/reasoning/thinkingClock.ts b/package/src/ai-components/reasoning/thinkingClock.ts new file mode 100644 index 0000000000..7c40c0fab4 --- /dev/null +++ b/package/src/ai-components/reasoning/thinkingClock.ts @@ -0,0 +1,109 @@ +import { createContext, useContext } from 'react'; + +/** How long a first step that arrived done stays open, showing its reasoning, before it folds. */ +export const FINISHED_REASONING_SHOWN_MS = 1600; +/** How long the reasoning takes to fold. */ +export const REASONING_FOLD_MS = 450; +// How soon after a turn stops thinking its first step has to arrive, done, to open and show its +// reasoning: it came with the reply that ended the thinking. +const ARRIVAL_WINDOW_MS = 1000; + +type Turn = { + start: number; + end?: number; + // How long the first step thought, once it was done, in seconds. + firstStepDuration?: number; + firstStepStreamed: boolean; + // Whether a reply of the turn showed. + hadReplies: boolean; +}; + +/** + * Times each turn's thinking, so the reply's first reasoning step takes over the seconds the + * conversation counted while it showed the agent thinking. The counterpart of `AIThinkingClock`. + * + * It is written only from effects (`update`, `firstStepStreams`, `firstStepDone`) and read + * during render, so a render React throws away changes nothing. + */ +export class AIThinkingClock { + private turns = new Map(); + + // A turn's thinking starts the first time it is the latest turn and thinking, and ends when it + // stops. Thinking that resumes before the first step shows carries on. A turn whose replies were + // taken away starts thinking afresh. + update( + latest: string | undefined, + isThinking: boolean, + hasReplies: boolean, + hasFirstStep: boolean, + now = Date.now(), + ) { + if (latest === undefined) return; + const timed = this.turns.get(latest); + if (!timed || (isThinking && timed.hadReplies && !hasReplies)) { + if (isThinking) { + this.turns.set(latest, { + firstStepStreamed: false, + hadReplies: false, + start: now, + }); + } + return; + } + if (hasReplies) timed.hadReplies = true; + if (isThinking) { + if (!hasFirstStep) timed.end = undefined; + } else if (timed.end === undefined) { + timed.end = now; + } + } + + start(turn: string) { + return this.turns.get(turn)?.start; + } + + firstStepStreams(turn: string) { + const timed = this.turns.get(turn); + if (timed) timed.firstStepStreamed = true; + } + + // The first step stopped streaming: it thought from when the turn started thinking until now, + // or for its own time when that is longer. + firstStepDone(turn: string, own: number | undefined, now = Date.now()) { + const timed = this.turns.get(turn); + if (timed && timed.firstStepDuration === undefined) { + timed.firstStepDuration = Math.max(own ?? 0, (now - timed.start) / 1000); + } + } + + // How long a turn's first step thought, in seconds. One that arrived done thought until the + // turn stopped thinking, which with no end yet is the render it arrives in; one that streamed, + // until `firstStepDone`. + durationOfFirstStep(turn: string, own: number | undefined, now = Date.now()) { + const timed = this.turns.get(turn); + if (!timed) return own; + if (timed.firstStepDuration !== undefined) return timed.firstStepDuration; + if (timed.firstStepStreamed) return own; + return Math.max(own ?? 0, ((timed.end ?? now) - timed.start) / 1000); + } + + // Whether a turn's first step, which arrived done with the reply that ends the thinking, opens + // to show its reasoning before it folds and the answer starts. The thinking's end is noted + // after the render the step arrives in, so no end yet counts as ending now. + showsFirstStep(turn: string, now = Date.now()) { + const timed = this.turns.get(turn); + if (!timed || timed.firstStepStreamed) return false; + return timed.end === undefined || now - timed.end < ARRIVAL_WINDOW_MS; + } +} + +/** A turn's clock, and the id of its first reasoning step: the one that takes over its thinking. */ +export type AIThinkingTurn = { + clock: AIThinkingClock; + key: string; + firstStepId?: string; +}; + +export const AIThinkingTurnContext = createContext(undefined); + +export const useAIThinkingTurn = () => useContext(AIThinkingTurnContext); diff --git a/package/src/ai-components/theme.ts b/package/src/ai-components/theme.ts index c2047a2d7b..74dc069346 100644 --- a/package/src/ai-components/theme.ts +++ b/package/src/ai-components/theme.ts @@ -69,6 +69,43 @@ export type Theme = { */ backdropColor?: string; }; + /** + * The steps of an agent's reply: reasoning, tool calls and their questions. Optional, so a + * hand-built `Theme` without it keeps type-checking. + */ + messageParts?: { + /** The colour of a completed tool call's check. */ + successColor?: string; + container?: ViewStyle; + reasoning?: { + container?: ViewStyle; + rule?: ViewStyle; + header?: ViewStyle; + title?: TextStyle; + summary?: TextStyle; + panel?: ViewStyle; + }; + toolCall?: { + container?: ViewStyle; + title?: TextStyle; + detail?: TextStyle; + duration?: TextStyle; + }; + approvalCard?: { + container?: ViewStyle; + title?: TextStyle; + message?: TextStyle; + allowButton?: ViewStyle; + allowButtonText?: TextStyle; + declineButton?: ViewStyle; + declineButtonText?: TextStyle; + failure?: TextStyle; + }; + unsupported?: { + container?: ViewStyle; + text?: TextStyle; + }; + }; }; export const defaultTheme: Theme = { @@ -151,4 +188,21 @@ export const defaultTheme: Theme = { voiceVisualization: { colors: [colors.accent_blue], }, + messageParts: { + successColor: '#20E070', + container: {}, + reasoning: { container: {}, rule: {}, header: {}, title: {}, summary: {}, panel: {} }, + toolCall: { container: {}, title: {}, detail: {}, duration: {} }, + approvalCard: { + container: {}, + title: {}, + message: {}, + allowButton: {}, + allowButtonText: {}, + declineButton: {}, + declineButtonText: {}, + failure: {}, + }, + unsupported: { container: {}, text: {} }, + }, }; diff --git a/package/src/ai-components/tool-calls/AIToolApprover.tsx b/package/src/ai-components/tool-calls/AIToolApprover.tsx new file mode 100644 index 0000000000..73709e81e2 --- /dev/null +++ b/package/src/ai-components/tool-calls/AIToolApprover.tsx @@ -0,0 +1,58 @@ +import React, { createContext, type PropsWithChildren, useContext } from 'react'; + +import type { AIToolCallPart } from '../message-parts/types'; + +/** + * How the person signs in on this device when a tool call waits for their account at a + * provider, and how their "not now" reaches your backend, which holds the call until one or + * the other. + */ +export type AIToolAuthorizer = { + /** + * Opens the call's sign-in. Resolves `true` once the account is connected: your backend then + * updates the call's step, and until it does the sign-in reads as connecting. Resolves `false` + * when the person closed the sign-in or it ended without connecting, so they can try again. + * Rejects when the sign-in could not be opened. + */ + connect: (call: AIToolCallPart) => Promise; + /** + * Tells your backend the person won't sign in now. Your backend ends the call, which never + * runs. A rejection lets the person answer again. + */ + decline: (call: AIToolCallPart) => Promise; +}; + +/** + * The person answering tool calls' questions on this device, and how their answer reaches + * your backend, which holds each call until it gets one. + */ +export type AIToolApprover = { + /** This install, matched against a client tool call's `target_client_id`. */ + clientId: string; + /** + * Sends the answer. Your backend checks it is this person's (and this install's, for a + * client tool), then updates the call's step; until then the question stays, with its + * buttons disabled. A rejection lets the person answer again. + */ + decide: (call: AIToolCallPart, allowed: boolean) => Promise; + /** The person signed in, matched against a call's `target_user_id`. */ + userId: string; + /** How this person signs in when a call waits for their account. Without one, no sign-in is offered here. */ + authorizer?: AIToolAuthorizer; +}; + +const AIToolApproverContext = createContext(undefined); + +/** Provides the approver to the AI components below it, for those not given one as a prop. */ +export const AIToolApproverProvider = ({ + approver, + children, +}: PropsWithChildren<{ approver?: AIToolApprover }>) => ( + {children} +); + +/** The approver given as a prop, else the one of the nearest `AIToolApproverProvider`. */ +export const useAIToolApprover = (approver?: AIToolApprover) => { + const fromContext = useContext(AIToolApproverContext); + return approver ?? fromContext; +}; diff --git a/package/src/ai-components/tool-calls/index.ts b/package/src/ai-components/tool-calls/index.ts new file mode 100644 index 0000000000..ad73d0bd40 --- /dev/null +++ b/package/src/ai-components/tool-calls/index.ts @@ -0,0 +1 @@ +export * from './AIToolApprover'; From 9ec0ef834dd200defe1573b7090ff7d1fece32c9 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 23:57:08 +0100 Subject: [PATCH 04/23] feat(ai-components): native AIAgent module for install id and tool-call sign-in Ported from #3926 on v9-release. --- examples/ExpoChatGPTSample/app.json | 2 +- .../ExpoChatGPTSample/app/+native-intent.tsx | 5 + .../ExpoChatGPTSample/contexts/AppContext.tsx | 37 ++- .../ExpoChatGPTSample/screens/ChatContent.tsx | 20 +- package/bin/validate-entry-isolation.js | 4 + package/expo-package/.gitignore | 1 + package/expo-package/android/build.gradle | 24 ++ .../android/src/main/AndroidManifest.xml | 8 + .../streamchatexpo/StreamChatExpoPackage.java | 16 + .../com/streamchatexpo/AIAgentModule.kt | 37 +++ package/expo-package/package.json | 1 + .../expo-package/src/native/NativeAIAgent.ts | 25 ++ package/expo-package/stream-chat-expo.podspec | 5 +- package/native-package/.gitignore | 1 + package/native-package/android/build.gradle | 24 ++ .../android/src/main/AndroidManifest.xml | 10 +- .../StreamChatReactNativePackage.java | 17 ++ .../streamchatreactnative/AIAgentModule.kt | 37 +++ package/native-package/package.json | 1 + .../src/native/NativeAIAgent.ts | 25 ++ .../stream-chat-react-native.podspec | 5 +- package/scripts/clean-shared-native-copies.sh | 3 +- package/scripts/sync-shared-native.sh | 10 + .../android-authtab/AgentAuthTab.kt | 24 ++ .../android-customtabs/AgentAuthTab.kt | 20 ++ .../android/agent/AIAgentController.kt | 173 +++++++++++ package/shared-native/ios/AIAgent.h | 16 + package/shared-native/ios/AIAgent.mm | 65 ++++ package/shared-native/ios/AIAgentModule.swift | 84 ++++++ .../ai-components/__tests__/signIn.test.tsx | 279 ++++++++++++++++++ .../message-parts/AIMessageParts.tsx | 4 +- .../message-parts/AIToolAuthorization.tsx | 266 +++++++++++++++++ .../src/ai-components/message-parts/index.ts | 1 + .../native-specs/NativeAIAgent.ts | 25 ++ package/src/ai-components/theme.ts | 22 ++ .../tool-calls/AIClientIdentity.ts | 33 +++ .../tool-calls/authSessionAuthorizer.ts | 125 ++++++++ package/src/ai-components/tool-calls/index.ts | 2 + 38 files changed, 1441 insertions(+), 16 deletions(-) create mode 100644 examples/ExpoChatGPTSample/app/+native-intent.tsx create mode 100644 package/expo-package/android/src/newarch/com/streamchatexpo/AIAgentModule.kt create mode 100644 package/expo-package/src/native/NativeAIAgent.ts create mode 100644 package/native-package/android/src/newarch/com/streamchatreactnative/AIAgentModule.kt create mode 100644 package/native-package/src/native/NativeAIAgent.ts create mode 100644 package/shared-native/android-authtab/AgentAuthTab.kt create mode 100644 package/shared-native/android-customtabs/AgentAuthTab.kt create mode 100644 package/shared-native/android/agent/AIAgentController.kt create mode 100644 package/shared-native/ios/AIAgent.h create mode 100644 package/shared-native/ios/AIAgent.mm create mode 100644 package/shared-native/ios/AIAgentModule.swift create mode 100644 package/src/ai-components/__tests__/signIn.test.tsx create mode 100644 package/src/ai-components/message-parts/AIToolAuthorization.tsx create mode 100644 package/src/ai-components/native-specs/NativeAIAgent.ts create mode 100644 package/src/ai-components/tool-calls/AIClientIdentity.ts create mode 100644 package/src/ai-components/tool-calls/authSessionAuthorizer.ts diff --git a/examples/ExpoChatGPTSample/app.json b/examples/ExpoChatGPTSample/app.json index 7310f41c67..6607bf3c99 100644 --- a/examples/ExpoChatGPTSample/app.json +++ b/examples/ExpoChatGPTSample/app.json @@ -2,7 +2,7 @@ "expo": { "name": "ExpoChatGPTSample", "slug": "ExpoChatGPTSample", - "scheme": "expochatgptsample", + "scheme": ["expochatgptsample", "io.getstream.expochatgptsample"], "ios": { "bundleIdentifier": "io.getstream.expochatgptsample", "appleTeamId": "EHV7XZLAHA" diff --git a/examples/ExpoChatGPTSample/app/+native-intent.tsx b/examples/ExpoChatGPTSample/app/+native-intent.tsx new file mode 100644 index 0000000000..a54ac3b988 --- /dev/null +++ b/examples/ExpoChatGPTSample/app/+native-intent.tsx @@ -0,0 +1,5 @@ +// A sign-in opened in a Custom Tab comes back to the app on its scheme (`io.getstream.expochatgptsample:// +// sign-in?…`), which the SDK's sign-in reads itself. Stay on the screen it was opened from rather +// than routing to a page that doesn't exist. +export const redirectSystemPath = ({ path }: { initial: boolean; path: string | null }) => + path?.includes('sign-in') ? null : path; diff --git a/examples/ExpoChatGPTSample/contexts/AppContext.tsx b/examples/ExpoChatGPTSample/contexts/AppContext.tsx index 14ee62f117..73636148ae 100644 --- a/examples/ExpoChatGPTSample/contexts/AppContext.tsx +++ b/examples/ExpoChatGPTSample/contexts/AppContext.tsx @@ -9,7 +9,11 @@ import { } from '@stream-io/calypso'; import type { Channel, Event, StreamChat } from 'stream-chat'; import { useStableCallback } from 'stream-chat-expo'; -import type { AIToolApprover } from 'stream-chat-expo/ai-components'; +import { + AIClientIdentity, + type AIToolApprover, + createAuthSessionAuthorizer, +} from 'stream-chat-expo/ai-components'; import { agents, agentSessions, streamToken } from '../agent/agents'; import { AgentConfig } from '../agent/config'; @@ -18,8 +22,20 @@ type Conversation = { channel: Channel; reasoning: LiveReasoning; session: Sessi const IDLE: ConversationSnapshot = { kind: 'idle', skills: [] }; -// This install, for the tool calls addressed to it. Kept for the app's lifetime. -const clientId = `rn-sample-${(globalThis as { crypto: { randomUUID: () => string } }).crypto.randomUUID()}`; +// This install, for the tool calls addressed to it. +const clientId = AIClientIdentity.installId(); + +// Where a sign-in comes back to: the app's own scheme, which the web auth session hands back. +// The router takes only https or a native app's own scheme in reverse domain form (RFC 8252). +const SIGN_IN_SCHEME = 'io.getstream.expochatgptsample'; +const SIGN_IN_RETURN_URL = `${SIGN_IN_SCHEME}://sign-in`; + +// The person's "not now" on a sign-in. calypso 0.2.3 has no `declineLogin` yet, so this sends +// the frame Vision-Agents' JS SDK sends for it over the session's socket. +const declineLogin = (session: Session, toolCallId: string) => + (session as unknown as { held(): { send(frame: Record): void } }) + .held() + .send({ allowed: false, tool_call_id: toolCallId, type: 'tool_approval' }); export type AppContextValue = { /** What the agent is doing in the open conversation, from the session's socket. */ @@ -95,7 +111,12 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St }); const startConversation = useStableCallback(() => { - void open(() => agentSessions().create({ modelOverwrites: { llm: AgentConfig.chatModel } })); + void open(() => + agentSessions().create({ + modelOverwrites: { llm: AgentConfig.chatModel }, + return_url: SIGN_IN_RETURN_URL, + }), + ); }); const openConversation = useStableCallback((sessionId: string) => { @@ -160,6 +181,14 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St () => conversation ? { + authorizer: createAuthSessionAuthorizer({ + callbackScheme: SIGN_IN_SCHEME, + decline: (call) => + new Promise((resolve) => { + declineLogin(conversation.session, call.id); + resolve(); + }), + }), clientId, // `decide` sends over the session's socket and throws when it can't. decide: (call, allowed) => diff --git a/examples/ExpoChatGPTSample/screens/ChatContent.tsx b/examples/ExpoChatGPTSample/screens/ChatContent.tsx index 6e6b978571..722e24721b 100644 --- a/examples/ExpoChatGPTSample/screens/ChatContent.tsx +++ b/examples/ExpoChatGPTSample/screens/ChatContent.tsx @@ -94,19 +94,27 @@ const CustomMessage = (props: MessageProps) => { type UnsupportedAttachmentComponent = ResolvedComponents['UnsupportedAttachment']; // A reply's steps (`ai_reasoning`, `ai_tool_call`) are attachments the chat SDK doesn't know. They -// show in the reply's own view, so they're left out here; any other unknown attachment keeps the -// SDK's own view. +// show in the reply's own view, so they're left out here, as is the router's `connector_authorization`, +// a second copy of a sign-in its tool call's step already asks for. Any other unknown attachment +// keeps the SDK's own view. const hidingAISteps = (Fallback: UnsupportedAttachmentComponent) => { const HideAISteps: UnsupportedAttachmentComponent = (props) => - props.attachment.type?.startsWith(AI_MESSAGE_PART_TYPE_PREFIX) ? null : ; + props.attachment.type?.startsWith(AI_MESSAGE_PART_TYPE_PREFIX) || + props.attachment.type === 'connector_authorization' ? null : ( + + ); return HideAISteps; }; // A reply: its steps (reasoning, tool calls and their questions), then its answer. const CustomStreamingMessageView = () => { - const { message } = useMessageContext(); - // Read so the steps render again as more of their reasoning arrives. - const { reasoningText, reasoningVersion } = useAppContext(); + const { message: listed } = useMessageContext(); + // Read so the steps render again as each live update arrives. + const { channel, reasoningText, reasoningVersion } = useAppContext(); + // The list passes a row its message again only when it changes in ways it compares, which a + // step changing on its own, such as a tool call starting to wait for a sign-in, isn't. The + // channel's state has every update. + const message = channel?.state.findMessage(listed.id) ?? listed; const parts = useMemo(() => parseMessageParts(message.attachments), [message.attachments]); const stepText = useMemo( () => (stepId: string) => reasoningText(message.id, stepId), diff --git a/package/bin/validate-entry-isolation.js b/package/bin/validate-entry-isolation.js index b747c3435f..480df61224 100644 --- a/package/bin/validate-entry-isolation.js +++ b/package/bin/validate-entry-isolation.js @@ -208,6 +208,10 @@ const copies = [ path.join(AI_SRC, 'native-specs/NativeAIDictation.ts'), ...WRAPPERS.map((w) => path.join(w, 'src/native/NativeAIDictation.ts')), ], + [ + path.join(AI_SRC, 'native-specs/NativeAIAgent.ts'), + ...WRAPPERS.map((w) => path.join(w, 'src/native/NativeAIAgent.ts')), + ], WRAPPERS.map((w) => path.join(w, 'src/native/PerfTextNativeComponent.ts')), ]; for (const [source, ...targets] of copies) { diff --git a/package/expo-package/.gitignore b/package/expo-package/.gitignore index 81e0235dce..c70b7e72ca 100644 --- a/package/expo-package/.gitignore +++ b/package/expo-package/.gitignore @@ -1,6 +1,7 @@ # android android/build android/src/main/java/com/streamchatreactnative/shared +android/src/agent-sign-in # ios ios/build diff --git a/package/expo-package/android/build.gradle b/package/expo-package/android/build.gradle index 5e231ac5dc..a63828b6cd 100644 --- a/package/expo-package/android/build.gradle +++ b/package/expo-package/android/build.gradle @@ -42,6 +42,26 @@ def hasNativeSources = { File dir -> dir.exists() && !fileTree(dir).matching { include "**/*.kt"; include "**/*.java" }.files.isEmpty() } +// The agent components' sign-ins open in the browser's Auth Tab, which needs androidx.browser +// 1.9.0, and that needs the app to build with compileSdk 36 and AGP 8.9.1. An app below either +// gets 1.8.0 and opens them in a Custom Tab instead, so the SDK asks neither of anyone. +def versionAtLeast = { String version, List wanted -> + def parts = version.tokenize('.-').collect { it.replaceAll(/\D.*$/, '') }.findAll { it }.collect { it.toInteger() } + for (int i = 0; i < wanted.size(); i++) { + def part = i < parts.size() ? parts[i] : 0 + if (part != wanted[i]) return part > wanted[i] + } + return true +} +def agentSignInWithAuthTab = + getExtOrIntegerDefault("compileSdkVersion").toString().toInteger() >= 36 && + versionAtLeast(com.android.Version.ANDROID_GRADLE_PLUGIN_VERSION, [8, 9, 1]) +def agentSignInVariant = agentSignInWithAuthTab ? "authtab" : "customtabs" +def sharedAgentSignInDir = new File(canonicalProjectDir, "../../shared-native/android-${agentSignInVariant}") +def agentSignInDir = hasNativeSources(sharedAgentSignInDir) + ? sharedAgentSignInDir + : new File(projectDir, "src/agent-sign-in/${agentSignInVariant}") + android { compileSdkVersion getExtOrIntegerDefault("compileSdkVersion") def agpVersion = com.android.Version.ANDROID_GRADLE_PLUGIN_VERSION @@ -82,6 +102,7 @@ android { java.srcDirs += [ "src/newarch", "src/main/java/com/streamchatreactnative/shared", + agentSignInDir, "${project.buildDir}/generated/source/codegen/java" ] } @@ -140,6 +161,9 @@ repositories { dependencies { implementation "com.facebook.react:react-native:+" + // The agent components' tool-call sign-ins: an Auth Tab where the app's build allows 1.9.0, + // else a Custom Tab (see `agentSignInWithAuthTab`). + implementation "androidx.browser:browser:${agentSignInWithAuthTab ? '1.9.0' : '1.8.0'}" } if (isNewArchitectureEnabled()) { diff --git a/package/expo-package/android/src/main/AndroidManifest.xml b/package/expo-package/android/src/main/AndroidManifest.xml index 17df9bf89c..b71eebea63 100644 --- a/package/expo-package/android/src/main/AndroidManifest.xml +++ b/package/expo-package/android/src/main/AndroidManifest.xml @@ -1,4 +1,12 @@ + + + + + + + diff --git a/package/expo-package/android/src/main/java/com/streamchatexpo/StreamChatExpoPackage.java b/package/expo-package/android/src/main/java/com/streamchatexpo/StreamChatExpoPackage.java index 4442d860eb..9336dd4546 100644 --- a/package/expo-package/android/src/main/java/com/streamchatexpo/StreamChatExpoPackage.java +++ b/package/expo-package/android/src/main/java/com/streamchatexpo/StreamChatExpoPackage.java @@ -18,6 +18,7 @@ public class StreamChatExpoPackage extends TurboReactPackage { private static final String STREAM_MULTIPART_UPLOADER_MODULE = "StreamMultipartUploader"; private static final String STREAM_VIDEO_THUMBNAIL_MODULE = "StreamVideoThumbnail"; private static final String AI_DICTATION_MODULE = "AIDictation"; + private static final String AI_AGENT_MODULE = "AIAgent"; @Nullable @Override @@ -34,6 +35,10 @@ public NativeModule getModule(String name, ReactApplicationContext reactContext) return createNewArchModule("com.streamchatexpo.AIDictationModule", reactContext); } + if (name.equals(AI_AGENT_MODULE)) { + return createNewArchModule("com.streamchatexpo.AIAgentModule", reactContext); + } + return null; } @@ -75,6 +80,17 @@ public ReactModuleInfoProvider getReactModuleInfoProvider() { false, // isCxxModule true // isTurboModule )); + moduleInfos.put( + AI_AGENT_MODULE, + new ReactModuleInfo( + AI_AGENT_MODULE, + AI_AGENT_MODULE, + false, // canOverrideExistingModule + false, // needsEagerInit + false, // hasConstants + false, // isCxxModule + true // isTurboModule + )); return moduleInfos; }; } diff --git a/package/expo-package/android/src/newarch/com/streamchatexpo/AIAgentModule.kt b/package/expo-package/android/src/newarch/com/streamchatexpo/AIAgentModule.kt new file mode 100644 index 0000000000..3a3490e723 --- /dev/null +++ b/package/expo-package/android/src/newarch/com/streamchatexpo/AIAgentModule.kt @@ -0,0 +1,37 @@ +package com.streamchatexpo + +import com.facebook.react.bridge.Promise +import com.facebook.react.bridge.ReactApplicationContext +import com.streamchatreactnative.shared.agent.AIAgentController + +/** + * The `AIAgent` TurboModule used by the `ai-components` agent views: this install's identifier + * and the sign-ins of tool calls. The implementation is shared with the other SDK package; see + * [AIAgentController]. + */ +class AIAgentModule( + reactContext: ReactApplicationContext, +) : NativeAIAgentSpec(reactContext) { + private val controller = AIAgentController(reactContext) + + override fun getName(): String = NAME + + override fun initialize() { + super.initialize() + controller.initialize() + } + + override fun invalidate() { + super.invalidate() + controller.invalidate() + } + + override fun installId(): String = controller.installId() + + override fun openAuthSession(url: String, callbackScheme: String, promise: Promise) = + controller.openAuthSession(url, callbackScheme, promise) + + companion object { + const val NAME = AIAgentController.NAME + } +} diff --git a/package/expo-package/package.json b/package/expo-package/package.json index c78b74a28b..0f9ad7db9e 100644 --- a/package/expo-package/package.json +++ b/package/expo-package/package.json @@ -111,6 +111,7 @@ }, "ios": { "modulesProvider": { + "AIAgent": "AIAgent", "StreamMultipartUploader": "StreamMultipartUploader", "StreamVideoThumbnail": "StreamVideoThumbnail" }, diff --git a/package/expo-package/src/native/NativeAIAgent.ts b/package/expo-package/src/native/NativeAIAgent.ts new file mode 100644 index 0000000000..7e22ca992e --- /dev/null +++ b/package/expo-package/src/native/NativeAIAgent.ts @@ -0,0 +1,25 @@ +import type { TurboModule } from 'react-native'; +import { TurboModuleRegistry } from 'react-native'; + +/** + * Codegen spec of the native module behind the agent components: this install's identifier and + * the sign-ins of tool calls. + * + * This file is duplicated (byte for byte) in `package/native-package/src/native/NativeAIAgent.ts` + * and `package/expo-package/src/native/NativeAIAgent.ts`, which are the copies codegen reads. + * `yarn validate-entry-isolation` fails if they drift apart. + */ +export interface Spec extends TurboModule { + /** This install's identifier, created on first use and kept in storage that isn't backed up. */ + installId(): string; + /** + * Opens `url` in the platform's web authentication session and resolves with the URL the page + * handed back to `callbackScheme`, or an empty string when the person closed it. Rejects when + * the session couldn't be opened. + */ + openAuthSession(url: string, callbackScheme: string): Promise; +} + +// `get` rather than `getEnforcing`, so importing the AI entry point never throws in an app built +// without the module, such as one on an older version of the SDK's native code. +export default TurboModuleRegistry.get('AIAgent'); diff --git a/package/expo-package/stream-chat-expo.podspec b/package/expo-package/stream-chat-expo.podspec index ead4ddac26..b9e842bef4 100644 --- a/package/expo-package/stream-chat-expo.podspec +++ b/package/expo-package/stream-chat-expo.podspec @@ -42,9 +42,12 @@ Pod::Spec.new do |s| false end + # `AIAgent` (the agent components' install id and tool-call sign-ins) opens sign-ins in + # `ASWebAuthenticationSession`, which needs no usage description, so it is always compiled. if dictation_enabled - s.frameworks = "Speech" + s.frameworks = ["AuthenticationServices", "Speech"] else + s.frameworks = "AuthenticationServices" s.exclude_files = ["ios/shared/AIDictation.h", "ios/shared/AIDictation.mm", "ios/shared/AIDictationModule.swift"] end diff --git a/package/native-package/.gitignore b/package/native-package/.gitignore index 81e0235dce..c70b7e72ca 100644 --- a/package/native-package/.gitignore +++ b/package/native-package/.gitignore @@ -1,6 +1,7 @@ # android android/build android/src/main/java/com/streamchatreactnative/shared +android/src/agent-sign-in # ios ios/build diff --git a/package/native-package/android/build.gradle b/package/native-package/android/build.gradle index 11d3af6763..187eef47e3 100644 --- a/package/native-package/android/build.gradle +++ b/package/native-package/android/build.gradle @@ -49,6 +49,26 @@ def hasNativeSources = { File dir -> dir.exists() && !fileTree(dir).matching { include "**/*.kt"; include "**/*.java" }.files.isEmpty() } +// The agent components' sign-ins open in the browser's Auth Tab, which needs androidx.browser +// 1.9.0, and that needs the app to build with compileSdk 36 and AGP 8.9.1. An app below either +// gets 1.8.0 and opens them in a Custom Tab instead, so the SDK asks neither of anyone. +def versionAtLeast = { String version, List wanted -> + def parts = version.tokenize('.-').collect { it.replaceAll(/\D.*$/, '') }.findAll { it }.collect { it.toInteger() } + for (int i = 0; i < wanted.size(); i++) { + def part = i < parts.size() ? parts[i] : 0 + if (part != wanted[i]) return part > wanted[i] + } + return true +} +def agentSignInWithAuthTab = + getExtOrIntegerDefault("compileSdkVersion").toString().toInteger() >= 36 && + versionAtLeast(com.android.Version.ANDROID_GRADLE_PLUGIN_VERSION, [8, 9, 1]) +def agentSignInVariant = agentSignInWithAuthTab ? "authtab" : "customtabs" +def sharedAgentSignInDir = new File(canonicalProjectDir, "../../shared-native/android-${agentSignInVariant}") +def agentSignInDir = hasNativeSources(sharedAgentSignInDir) + ? sharedAgentSignInDir + : new File(projectDir, "src/agent-sign-in/${agentSignInVariant}") + android { compileSdkVersion getExtOrIntegerDefault("compileSdkVersion") def agpVersion = com.android.Version.ANDROID_GRADLE_PLUGIN_VERSION @@ -92,6 +112,7 @@ android { java.srcDirs += [ "src/newarch", "src/main/java/com/streamchatreactnative/shared", + agentSignInDir, // This is needed to build Kotlin project with NewArch enabled "${project.buildDir}/generated/source/codegen/java" ] @@ -158,6 +179,9 @@ dependencies { //noinspection GradleDynamicVersion implementation "com.facebook.react:react-native:+" implementation "androidx.exifinterface:exifinterface:1.3.2" + // The agent components' tool-call sign-ins: an Auth Tab where the app's build allows 1.9.0, + // else a Custom Tab (see `agentSignInWithAuthTab`). + implementation "androidx.browser:browser:${agentSignInWithAuthTab ? '1.9.0' : '1.8.0'}" } if (isNewArchitectureEnabled()) { diff --git a/package/native-package/android/src/main/AndroidManifest.xml b/package/native-package/android/src/main/AndroidManifest.xml index 95bb2ff52c..df471ba82c 100644 --- a/package/native-package/android/src/main/AndroidManifest.xml +++ b/package/native-package/android/src/main/AndroidManifest.xml @@ -1,4 +1,12 @@ - \ No newline at end of file + + + + + + + + diff --git a/package/native-package/android/src/main/java/com/streamchatreactnative/StreamChatReactNativePackage.java b/package/native-package/android/src/main/java/com/streamchatreactnative/StreamChatReactNativePackage.java index 0a07a2c2b4..aa8e7be391 100644 --- a/package/native-package/android/src/main/java/com/streamchatreactnative/StreamChatReactNativePackage.java +++ b/package/native-package/android/src/main/java/com/streamchatreactnative/StreamChatReactNativePackage.java @@ -17,6 +17,7 @@ public class StreamChatReactNativePackage extends TurboReactPackage { private static final String STREAM_MULTIPART_UPLOADER_MODULE = "StreamMultipartUploader"; private static final String STREAM_VIDEO_THUMBNAIL_MODULE = "StreamVideoThumbnail"; private static final String AI_DICTATION_MODULE = "AIDictation"; + private static final String AI_AGENT_MODULE = "AIAgent"; @Nullable @Override @@ -38,6 +39,11 @@ public NativeModule getModule(String name, ReactApplicationContext reactContext) "com.streamchatreactnative.AIDictationModule", reactContext ); + } else if (name.equals(AI_AGENT_MODULE)) { + return createNewArchModule( + "com.streamchatreactnative.AIAgentModule", + reactContext + ); } else { return null; } @@ -92,6 +98,17 @@ public ReactModuleInfoProvider getReactModuleInfoProvider() { false, // isCxxModule true // isTurboModule )); + moduleInfos.put( + AI_AGENT_MODULE, + new ReactModuleInfo( + AI_AGENT_MODULE, + AI_AGENT_MODULE, + false, // canOverrideExistingModule + false, // needsEagerInit + false, // hasConstants + false, // isCxxModule + true // isTurboModule + )); return moduleInfos; }; } diff --git a/package/native-package/android/src/newarch/com/streamchatreactnative/AIAgentModule.kt b/package/native-package/android/src/newarch/com/streamchatreactnative/AIAgentModule.kt new file mode 100644 index 0000000000..35ac1c99d6 --- /dev/null +++ b/package/native-package/android/src/newarch/com/streamchatreactnative/AIAgentModule.kt @@ -0,0 +1,37 @@ +package com.streamchatreactnative + +import com.facebook.react.bridge.Promise +import com.facebook.react.bridge.ReactApplicationContext +import com.streamchatreactnative.shared.agent.AIAgentController + +/** + * The `AIAgent` TurboModule used by the `ai-components` agent views: this install's identifier + * and the sign-ins of tool calls. The implementation is shared with the other SDK package; see + * [AIAgentController]. + */ +class AIAgentModule( + reactContext: ReactApplicationContext, +) : NativeAIAgentSpec(reactContext) { + private val controller = AIAgentController(reactContext) + + override fun getName(): String = NAME + + override fun initialize() { + super.initialize() + controller.initialize() + } + + override fun invalidate() { + super.invalidate() + controller.invalidate() + } + + override fun installId(): String = controller.installId() + + override fun openAuthSession(url: String, callbackScheme: String, promise: Promise) = + controller.openAuthSession(url, callbackScheme, promise) + + companion object { + const val NAME = AIAgentController.NAME + } +} diff --git a/package/native-package/package.json b/package/native-package/package.json index c415e036a8..b9b3f9e7dd 100644 --- a/package/native-package/package.json +++ b/package/native-package/package.json @@ -105,6 +105,7 @@ }, "ios": { "modulesProvider": { + "AIAgent": "AIAgent", "StreamChatReactNative": "StreamChatReactNative", "StreamMultipartUploader": "StreamMultipartUploader", "StreamVideoThumbnail": "StreamVideoThumbnail" diff --git a/package/native-package/src/native/NativeAIAgent.ts b/package/native-package/src/native/NativeAIAgent.ts new file mode 100644 index 0000000000..7e22ca992e --- /dev/null +++ b/package/native-package/src/native/NativeAIAgent.ts @@ -0,0 +1,25 @@ +import type { TurboModule } from 'react-native'; +import { TurboModuleRegistry } from 'react-native'; + +/** + * Codegen spec of the native module behind the agent components: this install's identifier and + * the sign-ins of tool calls. + * + * This file is duplicated (byte for byte) in `package/native-package/src/native/NativeAIAgent.ts` + * and `package/expo-package/src/native/NativeAIAgent.ts`, which are the copies codegen reads. + * `yarn validate-entry-isolation` fails if they drift apart. + */ +export interface Spec extends TurboModule { + /** This install's identifier, created on first use and kept in storage that isn't backed up. */ + installId(): string; + /** + * Opens `url` in the platform's web authentication session and resolves with the URL the page + * handed back to `callbackScheme`, or an empty string when the person closed it. Rejects when + * the session couldn't be opened. + */ + openAuthSession(url: string, callbackScheme: string): Promise; +} + +// `get` rather than `getEnforcing`, so importing the AI entry point never throws in an app built +// without the module, such as one on an older version of the SDK's native code. +export default TurboModuleRegistry.get('AIAgent'); diff --git a/package/native-package/stream-chat-react-native.podspec b/package/native-package/stream-chat-react-native.podspec index aef09c54f1..c584dcd81e 100644 --- a/package/native-package/stream-chat-react-native.podspec +++ b/package/native-package/stream-chat-react-native.podspec @@ -42,9 +42,12 @@ Pod::Spec.new do |s| false end + # `AIAgent` (the agent components' install id and tool-call sign-ins) opens sign-ins in + # `ASWebAuthenticationSession`, which needs no usage description, so it is always compiled. if dictation_enabled - s.frameworks = "Speech" + s.frameworks = ["AuthenticationServices", "Speech"] else + s.frameworks = "AuthenticationServices" s.exclude_files = ["ios/shared/AIDictation.h", "ios/shared/AIDictation.mm", "ios/shared/AIDictationModule.swift"] end diff --git a/package/scripts/clean-shared-native-copies.sh b/package/scripts/clean-shared-native-copies.sh index 4b183454ce..1b96e1fc3b 100644 --- a/package/scripts/clean-shared-native-copies.sh +++ b/package/scripts/clean-shared-native-copies.sh @@ -11,8 +11,9 @@ clean_package() { local package_name="$1" local android_dir="$ROOT_DIR/$package_name/android/src/main/java/com/streamchatreactnative/shared" local ios_dir="$ROOT_DIR/$package_name/ios/shared" + local agent_sign_in_dir="$ROOT_DIR/$package_name/android/src/agent-sign-in" - rm -rf "$android_dir" "$ios_dir" + rm -rf "$android_dir" "$ios_dir" "$agent_sign_in_dir" rm -f \ "$STATE_DIR/${package_name}_android.manifest" \ "$STATE_DIR/${package_name}_ios.manifest" diff --git a/package/scripts/sync-shared-native.sh b/package/scripts/sync-shared-native.sh index 8d09ded9d5..06d0acb396 100644 --- a/package/scripts/sync-shared-native.sh +++ b/package/scripts/sync-shared-native.sh @@ -7,6 +7,9 @@ TARGET="${1:-all}" ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" SHARED_ANDROID_DIR="$ROOT_DIR/shared-native/android" SHARED_IOS_DIR="$ROOT_DIR/shared-native/ios" +# The agent sign-in's two Android builds: with Auth Tab (androidx.browser 1.9.0) and without. +# Each package's build.gradle compiles the one the app's build allows. +AGENT_SIGN_IN_VARIANTS=(authtab customtabs) sync_dir_contents() { local src_dir="$1" @@ -33,6 +36,13 @@ copy_to_package() { echo "Skipping Android sync: missing $SHARED_ANDROID_DIR" fi + for variant in "${AGENT_SIGN_IN_VARIANTS[@]}"; do + local variant_src="$ROOT_DIR/shared-native/android-$variant" + if [ -d "$variant_src" ]; then + sync_dir_contents "$variant_src" "$ROOT_DIR/$package_name/android/src/agent-sign-in/$variant" + fi + done + if [ -d "$SHARED_IOS_DIR" ]; then sync_dir_contents "$SHARED_IOS_DIR" "$ios_dst_dir" else diff --git a/package/shared-native/android-authtab/AgentAuthTab.kt b/package/shared-native/android-authtab/AgentAuthTab.kt new file mode 100644 index 0000000000..2092f9d21c --- /dev/null +++ b/package/shared-native/android-authtab/AgentAuthTab.kt @@ -0,0 +1,24 @@ +package com.streamchatreactnative.shared.agent + +import android.content.Context +import android.content.Intent +import android.net.Uri +import androidx.activity.result.ActivityResultLauncher +import androidx.browser.auth.AuthTabIntent +import androidx.browser.customtabs.CustomTabsClient + +/** + * The browser's Auth Tab, from `androidx.browser` 1.9.0. Compiled in when the app builds with + * compileSdk 36 and AGP 8.9.1 or newer, which that version needs; otherwise the build takes the + * `android-customtabs` copy of this file, and sign-ins open in a Custom Tab. + */ +internal object AgentAuthTab { + fun isSupported(context: Context, browser: String): Boolean = + CustomTabsClient.isAuthTabSupported(context, browser) + + fun launch(launcher: ActivityResultLauncher, browser: String, uri: Uri, callbackScheme: String) { + AuthTabIntent.Builder().build() + .apply { intent.setPackage(browser) } + .launch(launcher, uri, callbackScheme) + } +} diff --git a/package/shared-native/android-customtabs/AgentAuthTab.kt b/package/shared-native/android-customtabs/AgentAuthTab.kt new file mode 100644 index 0000000000..4e6d9fa0bc --- /dev/null +++ b/package/shared-native/android-customtabs/AgentAuthTab.kt @@ -0,0 +1,20 @@ +package com.streamchatreactnative.shared.agent + +import android.content.Context +import android.content.Intent +import android.net.Uri +import androidx.activity.result.ActivityResultLauncher + +/** + * No Auth Tab: the app builds below compileSdk 36 or AGP 8.9.1, which `androidx.browser` 1.9.0 + * needs, so sign-ins open in a Custom Tab. The `android-authtab` copy of this file has the Auth Tab. + */ +internal object AgentAuthTab { + @Suppress("UNUSED_PARAMETER") + fun isSupported(context: Context, browser: String): Boolean = false + + @Suppress("UNUSED_PARAMETER") + fun launch(launcher: ActivityResultLauncher, browser: String, uri: Uri, callbackScheme: String) { + throw UnsupportedOperationException("Auth Tabs need androidx.browser 1.9.0.") + } +} diff --git a/package/shared-native/android/agent/AIAgentController.kt b/package/shared-native/android/agent/AIAgentController.kt new file mode 100644 index 0000000000..af4dbd41e5 --- /dev/null +++ b/package/shared-native/android/agent/AIAgentController.kt @@ -0,0 +1,173 @@ +package com.streamchatreactnative.shared.agent + +import android.app.Activity +import android.content.Intent +import android.net.Uri +import androidx.activity.ComponentActivity +import androidx.activity.result.ActivityResultLauncher +import androidx.activity.result.contract.ActivityResultContracts +import androidx.browser.customtabs.CustomTabsClient +import androidx.browser.customtabs.CustomTabsIntent +import com.facebook.react.bridge.ActivityEventListener +import com.facebook.react.bridge.LifecycleEventListener +import com.facebook.react.bridge.Promise +import com.facebook.react.bridge.ReactApplicationContext +import java.io.File +import java.util.Locale +import java.util.UUID + +/** + * This install's identifier and the sign-ins of tool calls, for the `AIAgent` TurboModule of both + * SDK packages (each package's codegen generates its own `NativeAIAgentSpec`), which forward + * their calls here. + * + * A sign-in opens in the browser's Auth Tab where the app is built with one (see + * `AgentAuthTab`) and the browser has it: the tab hands the page's redirect to the callback scheme + * straight back. Otherwise it opens in a Custom Tab: the redirect then reaches the app as a new + * intent, and coming back to the app without one means the person closed the tab. + */ +class AIAgentController(private val reactContext: ReactApplicationContext) : + ActivityEventListener, + LifecycleEventListener { + companion object { + const val NAME = "AIAgent" + private const val INSTALL_ID_FILE = "io.getstream.ai.client-id" + private const val ERROR = "ai_agent_auth_session" + } + + /** The sign-in open now. One runs at a time; opening another ends this one. */ + private class SignIn( + val callbackScheme: String, + val promise: Promise, + val launcher: ActivityResultLauncher?, + ) { + /** Whether the person left the app for the Custom Tab, so coming back means it was closed. */ + var left = false + } + + private var signIn: SignIn? = null + private var launches = 0 + + fun initialize() { + reactContext.addActivityEventListener(this) + reactContext.addLifecycleEventListener(this) + } + + fun invalidate() { + reactContext.removeActivityEventListener(this) + reactContext.removeLifecycleEventListener(this) + finish("") + } + + /** + * This install's identifier, created on first use and kept until the app's data is cleared. + * The same file and form as the Android AI SDK's `AIClientIdentity`. It is not backed up, so a + * device restored from a backup gets its own. + */ + @Synchronized + fun installId(): String { + // noBackupFilesDir: a restored device must not answer calls meant for the old one. + val file = File(reactContext.applicationContext.noBackupFilesDir, INSTALL_ID_FILE) + if (file.exists()) file.readText().takeIf { it.isNotBlank() }?.let { return it } + val created = "android-" + UUID.randomUUID().toString().uppercase(Locale.ROOT) + file.writeText(created) + return created + } + + fun openAuthSession(url: String, callbackScheme: String, promise: Promise) { + val activity = reactContext.currentActivity + if (activity == null) { + promise.reject(ERROR, "There's no activity to open the sign-in from.") + return + } + activity.runOnUiThread { + finish("") + val uri = Uri.parse(url) + val browser = CustomTabsClient.getPackageName(activity, null) + try { + if (activity is ComponentActivity && + browser != null && + AgentAuthTab.isSupported(activity, browser) + ) { + openAuthTab(activity, browser, uri, callbackScheme, promise) + } else { + signIn = SignIn(callbackScheme, promise, launcher = null) + CustomTabsIntent.Builder().build().launchUrl(activity, uri) + } + } catch (error: Exception) { + signIn = null + promise.reject(ERROR, error.message ?: "The sign-in couldn't be opened.", error) + } + } + } + + private fun openAuthTab( + activity: ComponentActivity, + browser: String, + uri: Uri, + callbackScheme: String, + promise: Promise, + ) { + // Registered for this sign-in alone and dropped once it answers, since a TurboModule can't + // register before the activity starts, as `AuthTabIntent.registerActivityResultLauncher` + // requires. Auth Tab's own contract only passes the intent through and reads the result's + // data, which is what `StartActivityForResult` does. + lateinit var launcher: ActivityResultLauncher + launcher = activity.activityResultRegistry.register( + "stream-chat-ai-sign-in-${launches++}", + ActivityResultContracts.StartActivityForResult(), + ) { result -> + if (signIn?.launcher !== launcher) return@register + when (result.resultCode) { + // Auth Tab's RESULT_OK and RESULT_CANCELED are the activity's own. + Activity.RESULT_OK -> finish(result.data?.data?.toString() ?: "") + Activity.RESULT_CANCELED -> finish("") + else -> fail("The sign-in didn't complete.") + } + } + signIn = SignIn(callbackScheme, promise, launcher) + AgentAuthTab.launch(launcher, browser, uri, callbackScheme) + } + + private fun finish(callback: String) { + val current = signIn ?: return + signIn = null + current.launcher?.unregister() + current.promise.resolve(callback) + } + + private fun fail(message: String) { + val current = signIn ?: return + signIn = null + current.launcher?.unregister() + current.promise.reject(ERROR, message) + } + + // The Custom Tab's redirect to the callback scheme reaches the app as a new intent, which + // arrives before the activity resumes. + override fun onNewIntent(intent: Intent) { + val current = signIn ?: return + val data = intent.data ?: return + if (current.launcher == null && data.scheme.equals(current.callbackScheme, ignoreCase = true)) { + finish(data.toString()) + } + } + + override fun onActivityResult( + activity: Activity, + requestCode: Int, + resultCode: Int, + data: Intent?, + ) = Unit + + override fun onHostPause() { + signIn?.takeIf { it.launcher == null }?.left = true + } + + override fun onHostResume() { + val current = signIn ?: return + if (current.launcher == null && current.left) finish("") + } + + override fun onHostDestroy() = Unit +} diff --git a/package/shared-native/ios/AIAgent.h b/package/shared-native/ios/AIAgent.h new file mode 100644 index 0000000000..e9c50ba2b9 --- /dev/null +++ b/package/shared-native/ios/AIAgent.h @@ -0,0 +1,16 @@ +#ifdef RCT_NEW_ARCH_ENABLED + +#if __has_include("StreamChatReactNativeSpec.h") +#import "StreamChatReactNativeSpec.h" +#elif __has_include("StreamChatExpoSpec.h") +#import "StreamChatExpoSpec.h" +#else +#error "Unable to find generated codegen spec header for AIAgent." +#endif + +// This install's identifier and the sign-ins of tool calls, for the `ai-components` agent views. +// The implementation lives in AIAgentModule.swift. +@interface AIAgent : NSObject +@end + +#endif diff --git a/package/shared-native/ios/AIAgent.mm b/package/shared-native/ios/AIAgent.mm new file mode 100644 index 0000000000..1ae5578491 --- /dev/null +++ b/package/shared-native/ios/AIAgent.mm @@ -0,0 +1,65 @@ +#import "AIAgent.h" + +#ifdef RCT_NEW_ARCH_ENABLED + +#if __has_include() +#import +#elif __has_include() +#import +#elif __has_include("stream_chat_react_native-Swift.h") +#import "stream_chat_react_native-Swift.h" +#elif __has_include("stream_chat_expo-Swift.h") +#import "stream_chat_expo-Swift.h" +#else +#error "Unable to import generated Swift header for AIAgent." +#endif + +@implementation AIAgent { + AIAgentModule *moduleInstance; +} + +RCT_EXPORT_MODULE(AIAgent) + ++ (BOOL)requiresMainQueueSetup +{ + return NO; +} + +- (instancetype)init +{ + if (self = [super init]) { + moduleInstance = [AIAgentModule new]; + } + return self; +} + +- (NSString *)installId +{ + return [AIAgentModule installId]; +} + +- (void)openAuthSession:(NSString *)url + callbackScheme:(NSString *)callbackScheme + resolve:(RCTPromiseResolveBlock)resolve + reject:(RCTPromiseRejectBlock)reject +{ + [moduleInstance openAuthSessionWithUrl:url + callbackScheme:callbackScheme + completion:^(NSString *_Nullable callback, NSString *_Nullable error) { + if (error) { + reject(@"ai_agent_auth_session", error, nil); + } else { + resolve(callback ?: @""); + } + }]; +} + +- (std::shared_ptr)getTurboModule: + (const facebook::react::ObjCTurboModule::InitParams &)params +{ + return std::make_shared(params); +} + +@end + +#endif diff --git a/package/shared-native/ios/AIAgentModule.swift b/package/shared-native/ios/AIAgentModule.swift new file mode 100644 index 0000000000..e7e4eac086 --- /dev/null +++ b/package/shared-native/ios/AIAgentModule.swift @@ -0,0 +1,84 @@ +import AuthenticationServices +import Foundation +import UIKit + +/// This install's identifier and the sign-ins of tool calls, for the `AIAgent` TurboModule +/// (`AIAgent.mm`). Ported from StreamChatAI's `AIClientIdentity` and `AIWebAuthentication`. +@objc(AIAgentModule) +public class AIAgentModule: NSObject { + private static let installIdKey = "io.getstream.ai.client-id" + + /// The session open now. One sign-in runs at a time; opening another ends this one. + private var session: ASWebAuthenticationSession? + private var completion: ((String?, String?) -> Void)? + private let anchor = PresentationAnchor() + + /// This install's identifier, created on first use. The same key and form as StreamChatAI's, + /// so a native app moving to React Native keeps its id. + @objc public static func installId() -> String { + if let saved = UserDefaults.standard.string(forKey: installIdKey) { return saved } + let created = "ios-" + UUID().uuidString + UserDefaults.standard.set(created, forKey: installIdKey) + return created + } + + /// Opens `url` in a web authentication session. `completion` gets the URL the page handed + /// back to `callbackScheme`, nothing when the person closed the sheet, or an error when the + /// session couldn't be opened. + @objc public func openAuthSession( + url: String, + callbackScheme: String, + completion: @escaping (String?, String?) -> Void + ) { + DispatchQueue.main.async { + guard let target = URL(string: url) else { + completion(nil, "The sign-in URL isn't valid.") + return + } + // A sign-in still open is ended as closed; its session's own answer is then ignored. + let previous = self.session + self.finish(nil, error: nil) + previous?.cancel() + + // Weak, so the session and its own callback don't hold each other. + weak var opened: ASWebAuthenticationSession? + let session = ASWebAuthenticationSession(url: target, callbackURLScheme: callbackScheme) { callback, error in + let cancelled = (error as? ASWebAuthenticationSessionError)?.code == .canceledLogin + DispatchQueue.main.async { + guard let opened, self.session === opened else { return } + if let callback { + self.finish(callback.absoluteString, error: nil) + } else if cancelled { + self.finish(nil, error: nil) + } else { + self.finish(nil, error: error?.localizedDescription ?? "The sign-in couldn't be opened.") + } + } + } + opened = session + session.presentationContextProvider = self.anchor + self.session = session + self.completion = completion + if !session.start() { + self.finish(nil, error: "The sign-in couldn't be opened.") + } + } + } + + // The session may answer after a start that failed, so each sign-in is answered once. + private func finish(_ callback: String?, error: String?) { + guard let completion else { return } + self.completion = nil + session = nil + completion(callback, error) + } +} + +// Kept off `AIAgentModule`, whose conformances go in the generated Swift header that every +// Objective-C file of the pod imports without AuthenticationServices. +private final class PresentationAnchor: NSObject, ASWebAuthenticationPresentationContextProviding { + func presentationAnchor(for session: ASWebAuthenticationSession) -> ASPresentationAnchor { + let scenes = UIApplication.shared.connectedScenes.compactMap { $0 as? UIWindowScene } + return scenes.flatMap(\.windows).first(where: \.isKeyWindow) ?? scenes.first?.windows.first ?? ASPresentationAnchor() + } +} diff --git a/package/src/ai-components/__tests__/signIn.test.tsx b/package/src/ai-components/__tests__/signIn.test.tsx new file mode 100644 index 0000000000..7dca68fd39 --- /dev/null +++ b/package/src/ai-components/__tests__/signIn.test.tsx @@ -0,0 +1,279 @@ +import React, { type ReactElement } from 'react'; + +import { act, fireEvent, render, screen } from '@testing-library/react-native'; + +import { + AIClientIdentity, + AIConnectorSignInError, + AIMessagePart, + type AIToolApprover, + AIToolApproverProvider, + AIToolAuthorization, + AIToolAuthorizationCard, + type AIToolAuthorizer, + type AIToolCallPart, + createAuthSessionAuthorizer, + parseMessagePart, + readConnectorSignInResult, + signInLaunchUrl, + StreamTheme, +} from '../index'; +import NativeAIAgent from '../native-specs/NativeAIAgent'; + +jest.mock('../native-specs/NativeAIAgent', () => ({ + __esModule: true, + default: { installId: jest.fn(), openAuthSession: jest.fn() }, +})); + +const native = NativeAIAgent as unknown as { + installId: jest.Mock; + openAuthSession: jest.Mock; +}; + +const renderWithTheme = (ui: ReactElement) => { + const result = render({ui}); + return { + ...result, + rerender: (next: ReactElement) => result.rerender({next}), + }; +}; + +const signIn = parseMessagePart({ + authorization: { + handoff_token: 'tok', + provider: 'linear', + provider_name: 'Linear', + title: 'Connect Linear', + type: 'oauth', + url: 'https://router.example.com/launch/abc', + }, + id: 'call_2', + name: 'linear_create_issue', + status: 'awaiting_authorization', + target_user_id: 'alex', + type: 'ai_tool_call', +})?.toolCall as AIToolCallPart; + +const approverWith = (authorizer?: AIToolAuthorizer): AIToolApprover => ({ + authorizer, + clientId: 'ios-1', + decide: jest.fn(() => Promise.resolve()), + userId: 'alex', +}); + +describe('AIToolAuthorization', () => { + it('asks nothing without an authorizer, or of another person', () => { + const { rerender } = renderWithTheme( + , + ); + expect(screen.queryByText('Connect Linear')).toBeNull(); + rerender( + , + ); + expect(screen.queryByText('Connect Linear')).toBeNull(); + }); + + it('connects, and lets the person try again when the sign-in closes', async () => { + let finish: (connected: boolean) => void = () => undefined; + const connect = jest.fn( + () => + new Promise((resolve) => { + finish = resolve; + }), + ); + renderWithTheme( + , + ); + fireEvent.press(screen.getByRole('button', { name: 'Connect' })); + expect(connect).toHaveBeenCalledWith(signIn); + expect(screen.getByRole('button', { name: 'Connecting...' })).toBeDisabled(); + // Not now stays available while the sign-in is open. + expect(screen.getByRole('button', { name: 'Not now' })).toBeEnabled(); + + await act(async () => { + finish(false); + await Promise.resolve(); + }); + expect(screen.getByRole('button', { name: 'Connect' })).toBeEnabled(); + }); + + it('keeps connecting once connected, until the step changes', async () => { + renderWithTheme( + Promise.resolve(true), decline: jest.fn() })} + part={signIn} + />, + ); + fireEvent.press(screen.getByRole('button', { name: 'Connect' })); + await act(() => Promise.resolve()); + expect(screen.getByRole('button', { name: 'Connecting...' })).toBeDisabled(); + }); + + it('says when the sign-in could not open, and lets the person try again', async () => { + const connect = jest.fn(() => Promise.reject(new AIConnectorSignInError('failed'))); + renderWithTheme( + , + ); + fireEvent.press(screen.getByRole('button', { name: 'Connect' })); + await act(() => Promise.resolve()); + expect(screen.getByText("The sign-in couldn't be opened. Try again.")).toBeTruthy(); + expect(screen.getByRole('button', { name: 'Connect' })).toBeEnabled(); + }); + + it('declines, and lets the person answer again when that fails', async () => { + const decline = jest.fn(() => Promise.reject(new Error('offline'))); + renderWithTheme( + , + ); + fireEvent.press(screen.getByRole('button', { name: 'Not now' })); + expect(decline).toHaveBeenCalledWith(signIn); + expect(screen.getByRole('button', { name: 'Not now' })).toBeDisabled(); + await act(() => Promise.resolve()); + expect(screen.getByText("Your answer couldn't be sent. Try again.")).toBeTruthy(); + expect(screen.getByRole('button', { name: 'Not now' })).toBeEnabled(); + }); + + it.each([ + ['connected', 'Connected to Linear'], + ['declined', 'Not connected'], + ['failed', "The sign-in didn't complete."], + ['expired', 'This sign-in has expired.'], + ])('says how a %s sign-in ended, without buttons', (status, outcome) => { + renderWithTheme( + , + ); + expect(screen.getByText(outcome)).toBeTruthy(); + expect(screen.queryByRole('button', { name: 'Connect' })).toBeNull(); + }); + + it('can no longer be opened past its expiry', () => { + renderWithTheme( + , + ); + expect(screen.getByText('This sign-in has expired.')).toBeTruthy(); + expect(screen.queryByRole('button', { name: 'Connect' })).toBeNull(); + expect(screen.getByRole('button', { name: 'Not now' })).toBeEnabled(); + }); + + it('is asked under its tool call, with the approver of the provider', () => { + renderWithTheme( + + + , + ); + expect(screen.getByText('Connect Linear')).toBeTruthy(); + expect(screen.getByText('Waiting for you to sign in')).toBeTruthy(); + }); +}); + +const signInAttachment = { + authorization: { + handoff_token: 'tok', + provider: 'linear', + title: 'Connect Linear', + type: 'oauth', + url: 'https://router.example.com/launch/abc', + }, + id: 'call_3', + name: 'linear_list_issues', + status: 'awaiting_authorization', + target_user_id: 'alex', +}; + +describe('createAuthSessionAuthorizer', () => { + beforeEach(() => jest.clearAllMocks()); + + it('puts the handoff token in the launch page’s fragment, encoded', () => { + expect(signInLaunchUrl({ ...signIn.authorization!, handoffToken: 'a+b&c=d/e' })).toBe( + 'https://router.example.com/launch/abc#handoff_token=a%2Bb%26c%3Dd%2Fe', + ); + expect(signInLaunchUrl({ ...signIn.authorization!, handoffToken: undefined })).toBe( + 'https://router.example.com/launch/abc', + ); + }); + + it('opens the sign-in for the app’s scheme, and reads how it ended', async () => { + const decline = jest.fn(() => Promise.resolve()); + const authorizer = createAuthSessionAuthorizer({ callbackScheme: 'myapp', decline }); + + native.openAuthSession.mockResolvedValueOnce( + 'myapp://signed-in?connection_id=c1&status=connected', + ); + await expect(authorizer.connect(signIn)).resolves.toBe(true); + expect(native.openAuthSession).toHaveBeenCalledWith( + 'https://router.example.com/launch/abc#handoff_token=tok', + 'myapp', + ); + + native.openAuthSession.mockResolvedValueOnce( + 'myapp://signed-in?connection_id=c1&status=denied', + ); + await expect(authorizer.connect(signIn)).resolves.toBe(false); + + // The person closed the sheet. + native.openAuthSession.mockResolvedValueOnce(''); + await expect(authorizer.connect(signIn)).resolves.toBe(false); + + await authorizer.decline(signIn); + expect(decline).toHaveBeenCalledWith(signIn); + }); + + it('rejects when the sign-in could not be opened', async () => { + const authorizer = createAuthSessionAuthorizer({ callbackScheme: 'myapp', decline: jest.fn() }); + + native.openAuthSession.mockRejectedValueOnce(new Error('No browser')); + await expect(authorizer.connect(signIn)).rejects.toMatchObject({ + message: 'No browser', + reason: 'failed', + }); + await expect(authorizer.connect({ ...signIn, authorization: undefined })).rejects.toMatchObject( + { reason: 'unavailable' }, + ); + }); + + it('reads only an outcome the router sends', () => { + expect(readConnectorSignInResult('myapp://x?status=connected&connection_id=conn-1')).toEqual({ + connectionId: 'conn-1', + status: 'connected', + }); + expect(readConnectorSignInResult('myapp://x?status=maybe&connection_id=conn-1')).toBeNull(); + expect(readConnectorSignInResult('myapp://x?status=connected')).toBeNull(); + expect(readConnectorSignInResult('myapp://x')).toBeNull(); + }); +}); + +describe('AIClientIdentity', () => { + it('is the install id the native module keeps', () => { + native.installId.mockReturnValue('ios-ABC'); + expect(AIClientIdentity.installId()).toBe('ios-ABC'); + }); + + it('lives in memory, unchanged, when the native module has none', () => { + native.installId.mockImplementation(() => { + throw new Error('not linked'); + }); + const first = AIClientIdentity.installId(); + expect(first).toMatch(/^(ios|android)-[0-9a-f]{32}$/); + expect(AIClientIdentity.installId()).toBe(first); + }); +}); diff --git a/package/src/ai-components/message-parts/AIMessageParts.tsx b/package/src/ai-components/message-parts/AIMessageParts.tsx index 9c99694349..707fd727ee 100644 --- a/package/src/ai-components/message-parts/AIMessageParts.tsx +++ b/package/src/ai-components/message-parts/AIMessageParts.tsx @@ -2,6 +2,7 @@ import React, { type ReactNode } from 'react'; import { StyleSheet, Text, View } from 'react-native'; import { AIToolApproval } from './AIToolApproval'; +import { AIToolAuthorization } from './AIToolAuthorization'; import { AIToolCall } from './AIToolCall'; import type { AIMessagePartData } from './types'; @@ -23,7 +24,7 @@ export type AIMessagePartProps = { /** * One step of a reply: a round of reasoning (`AIReasoning`), a tool call (`AIToolCall`, with its - * question when it waits for this person here), or a neutral line for a step this SDK can't + * question or sign-in when it waits for this person here), or a neutral line for a step this SDK can't * show. The counterpart of `AIMessagePartView`. */ export const AIMessagePart = ({ approver, part, reasoningText }: AIMessagePartProps) => { @@ -40,6 +41,7 @@ export const AIMessagePart = ({ approver, part, reasoningText }: AIMessagePartPr + ); } diff --git a/package/src/ai-components/message-parts/AIToolAuthorization.tsx b/package/src/ai-components/message-parts/AIToolAuthorization.tsx new file mode 100644 index 0000000000..599960f8be --- /dev/null +++ b/package/src/ai-components/message-parts/AIToolAuthorization.tsx @@ -0,0 +1,266 @@ +import React, { type ReactNode, useEffect, useRef, useState } from 'react'; +import { ActivityIndicator, Pressable, StyleSheet, Text, View } from 'react-native'; + +import { isAuthorizationExpired, isAwaitingAuthorization } from './parseMessageParts'; +import type { AIToolAuthorizationData, AIToolCallPart } from './types'; + +import { useAITranslation, useTheme } from '../contexts'; +import { useNow } from '../internal/format'; +import { + type AIToolApprover, + type AIToolAuthorizer, + useAIToolApprover, +} from '../tool-calls/AIToolApprover'; + +/** Where the person's answer to a sign-in is. */ +export type AIToolAuthorizationState = { + /** The sign-in couldn't be opened, and the person can try again. */ + connectFailed: boolean; + /** Their "not now" couldn't be sent, and the person can answer again. */ + declineFailed: boolean; + /** The sign-in is open, or done and the call's step hasn't changed yet. */ + isConnecting: boolean; + /** Their "not now" is on its way, or arrived and the call's step hasn't changed yet. */ + isDeclining: boolean; +}; + +const idle: AIToolAuthorizationState = { + connectFailed: false, + declineFailed: false, + isConnecting: false, + isDeclining: false, +}; + +export type AIToolAuthorizationCardProps = { + authorization: AIToolAuthorizationData; + onConnect: () => void; + onDecline: () => void; + state?: AIToolAuthorizationState; +}; + +/** + * Asks the person to sign in at a provider a tool call needs, such as "Connect Linear", with a + * Connect and a Not now button. Once the sign-in is over it says how it ended (connected, + * declined, expired or failed), and a sign-in past its expiry can no longer be opened. The + * counterpart of `AIToolAuthorizationCard`. + */ +export const AIToolAuthorizationCard = ({ + authorization, + onConnect, + onDecline, + state = idle, +}: AIToolAuthorizationCardProps) => { + const { t } = useAITranslation(); + const { + theme: { colors, messageParts }, + } = useTheme(); + const card = messageParts?.authorizationCard; + const open = authorization.status === undefined; + // Re-checked every few seconds while it is open, so it turns expired on its own. + const now = useNow(open && authorization.expiresAt !== undefined, 5000); + const expired = isAuthorizationExpired(authorization, now); + const provider = authorization.providerName ?? authorization.provider; + const busy = state.isConnecting || state.isDeclining; + + const outcome = (() => { + if (expired) return t('This sign-in has expired.'); + switch (authorization.status) { + case undefined: + return undefined; + case 'connected': + return t('Connected to {{ provider }}', { provider }); + case 'declined': + return t('Not connected'); + default: + return t("The sign-in didn't complete."); + } + })(); + + const failure = (() => { + if (state.connectFailed) return t("The sign-in couldn't be opened. Try again."); + if (state.declineFailed) return t("Your answer couldn't be sent. Try again."); + return undefined; + })(); + + return ( + + + + {authorization.title} + + {authorization.message ? ( + + {authorization.message} + + ) : null} + + {outcome ? ( + {outcome} + ) : null} + {open ? ( + + {!expired ? ( + [ + styles.button, + { backgroundColor: colors.accent_blue, opacity: pressed ? 0.7 : 1 }, + card?.connectButton, + ]} + > + + {state.isConnecting ? t('Connecting...') : t('Connect')} + + + ) : null} + {/* Left enabled while the sign-in is open, so one that never comes back leaves a way out. */} + [ + styles.button, + styles.declineButton, + { borderColor: colors.grey_gainsboro, opacity: pressed ? 0.7 : 1 }, + card?.declineButton, + ]} + > + + {t('Not now')} + + + {busy ? : null} + + ) : null} + {failure ? ( + + {failure} + + ) : null} + + ); +}; + +/** + * The sign-in this person is asked for on `call` here: one the call waits for them to finish, + * of a kind this device opens, and only when they can sign in on it. + */ +export const signInFor = (call: AIToolCallPart, approver: AIToolApprover | undefined) => { + if (!approver?.authorizer || !isAwaitingAuthorization(call, approver.userId)) return; + return call.authorization?.type === 'oauth' ? call.authorization : undefined; +}; + +export type AIToolAuthorizationProps = { + part: AIToolCallPart; + /** Who answers on this device, with the `authorizer` the sign-in needs. Defaults to the one of the nearest `AIToolApproverProvider`. */ + approver?: AIToolApprover; + /** Renders the sign-in, instead of `AIToolAuthorizationCard`. */ + children?: ( + authorization: AIToolAuthorizationData, + state: AIToolAuthorizationState, + actions: { connect: () => void; decline: () => void }, + ) => ReactNode; +}; + +/** + * Asks for the sign-in of a tool call that waits for this person, and runs it with + * `approver.authorizer`. Renders nothing for a call that does not wait for them, or without an + * authorizer. The counterpart of `AIToolAuthorizationView`. + */ +export const AIToolAuthorization = ({ + approver: approverProp, + children, + part, +}: AIToolAuthorizationProps) => { + const approver = useAIToolApprover(approverProp); + const [state, setState] = useState(idle); + const mounted = useRef(true); + useEffect(() => { + mounted.current = true; + return () => { + mounted.current = false; + }; + }, []); + + const authorization = signInFor(part, approver); + const authorizer = approver?.authorizer; + if (!authorization || !authorizer) return null; + + const update = (next: AIToolAuthorizationState) => { + if (mounted.current) setState(next); + }; + + const connect = (signIn: AIToolAuthorizer) => { + if (state.isConnecting || state.isDeclining) return; + setState({ ...idle, isConnecting: true }); + signIn.connect(part).then( + // Your backend has the provider's answer before the sign-in comes back, and its change to + // the step hides this. Until then it says it is connecting, unless the person closed the + // sign-in or the provider did not connect them. + (connected) => { + if (!connected) update(idle); + }, + () => update({ ...idle, connectFailed: true }), + ); + }; + + const decline = (signIn: AIToolAuthorizer) => { + if (state.isDeclining) return; + setState({ ...idle, isDeclining: true }); + signIn.decline(part).catch(() => update({ ...idle, declineFailed: true })); + }; + + const actions = { + connect: () => connect(authorizer), + decline: () => decline(authorizer), + }; + + return ( + <> + {children ? ( + children(authorization, state, actions) + ) : ( + + )} + + ); +}; + +const styles = StyleSheet.create({ + actions: { alignItems: 'center', flexDirection: 'row', flexWrap: 'wrap', gap: 8 }, + button: { + alignItems: 'center', + borderRadius: 9999, + justifyContent: 'center', + minHeight: 36, + paddingHorizontal: 14, + paddingVertical: 6, + }, + buttonText: { fontSize: 14, fontWeight: '600' }, + container: { borderRadius: 16, borderWidth: 1, gap: 12, padding: 12 }, + declineButton: { borderWidth: 1 }, + failure: { fontSize: 13 }, + message: { fontSize: 14, lineHeight: 20 }, + text: { gap: 4 }, + title: { fontSize: 15, fontWeight: '600', lineHeight: 20 }, +}); diff --git a/package/src/ai-components/message-parts/index.ts b/package/src/ai-components/message-parts/index.ts index c34894c2f6..23b9b28e4b 100644 --- a/package/src/ai-components/message-parts/index.ts +++ b/package/src/ai-components/message-parts/index.ts @@ -3,3 +3,4 @@ export * from './parseMessageParts'; export * from './AIMessageParts'; export * from './AIToolApproval'; export * from './AIToolCall'; +export * from './AIToolAuthorization'; diff --git a/package/src/ai-components/native-specs/NativeAIAgent.ts b/package/src/ai-components/native-specs/NativeAIAgent.ts new file mode 100644 index 0000000000..7e22ca992e --- /dev/null +++ b/package/src/ai-components/native-specs/NativeAIAgent.ts @@ -0,0 +1,25 @@ +import type { TurboModule } from 'react-native'; +import { TurboModuleRegistry } from 'react-native'; + +/** + * Codegen spec of the native module behind the agent components: this install's identifier and + * the sign-ins of tool calls. + * + * This file is duplicated (byte for byte) in `package/native-package/src/native/NativeAIAgent.ts` + * and `package/expo-package/src/native/NativeAIAgent.ts`, which are the copies codegen reads. + * `yarn validate-entry-isolation` fails if they drift apart. + */ +export interface Spec extends TurboModule { + /** This install's identifier, created on first use and kept in storage that isn't backed up. */ + installId(): string; + /** + * Opens `url` in the platform's web authentication session and resolves with the URL the page + * handed back to `callbackScheme`, or an empty string when the person closed it. Rejects when + * the session couldn't be opened. + */ + openAuthSession(url: string, callbackScheme: string): Promise; +} + +// `get` rather than `getEnforcing`, so importing the AI entry point never throws in an app built +// without the module, such as one on an older version of the SDK's native code. +export default TurboModuleRegistry.get('AIAgent'); diff --git a/package/src/ai-components/theme.ts b/package/src/ai-components/theme.ts index 74dc069346..af50497239 100644 --- a/package/src/ai-components/theme.ts +++ b/package/src/ai-components/theme.ts @@ -101,6 +101,17 @@ export type Theme = { declineButtonText?: TextStyle; failure?: TextStyle; }; + authorizationCard?: { + container?: ViewStyle; + title?: TextStyle; + message?: TextStyle; + outcome?: TextStyle; + connectButton?: ViewStyle; + connectButtonText?: TextStyle; + declineButton?: ViewStyle; + declineButtonText?: TextStyle; + failure?: TextStyle; + }; unsupported?: { container?: ViewStyle; text?: TextStyle; @@ -203,6 +214,17 @@ export const defaultTheme: Theme = { declineButtonText: {}, failure: {}, }, + authorizationCard: { + container: {}, + title: {}, + message: {}, + outcome: {}, + connectButton: {}, + connectButtonText: {}, + declineButton: {}, + declineButtonText: {}, + failure: {}, + }, unsupported: { container: {}, text: {} }, }, }; diff --git a/package/src/ai-components/tool-calls/AIClientIdentity.ts b/package/src/ai-components/tool-calls/AIClientIdentity.ts new file mode 100644 index 0000000000..d47504e004 --- /dev/null +++ b/package/src/ai-components/tool-calls/AIClientIdentity.ts @@ -0,0 +1,33 @@ +import { Platform } from 'react-native'; + +import NativeAIAgent from '../native-specs/NativeAIAgent'; + +// Used when the app is built without the SDK's native code, so the id still holds for as long +// as the app runs. +let inMemoryId: string | undefined; + +const randomHex = (length: number) => + Array.from({ length }, () => Math.floor(Math.random() * 16).toString(16)).join(''); + +/** + * A stable identifier for this install, for addressing client tool calls to it. Send it with the + * person's questions (`client_id`) so the agent addresses the calls it makes while answering to + * this install, and pass it to `AIToolApprover` as `clientId`. + */ +export const AIClientIdentity = { + /** + * This install's identifier, created on first use and kept until the app's data is cleared: + * `ios-…` in `UserDefaults`, `android-…` in a file that isn't backed up, as in the native AI + * SDKs. Without the SDK's native code it lives in memory, for as long as the app runs. + */ + installId(): string { + try { + const saved = NativeAIAgent?.installId(); + if (saved) return saved; + } catch { + // Falls back to an id kept in memory. + } + inMemoryId ??= `${Platform.OS}-${randomHex(32)}`; + return inMemoryId; + }, +}; diff --git a/package/src/ai-components/tool-calls/authSessionAuthorizer.ts b/package/src/ai-components/tool-calls/authSessionAuthorizer.ts new file mode 100644 index 0000000000..8fe64f716e --- /dev/null +++ b/package/src/ai-components/tool-calls/authSessionAuthorizer.ts @@ -0,0 +1,125 @@ +import type { AIToolAuthorizer } from './AIToolApprover'; + +import type { AIToolAuthorizationData, AIToolCallPart } from '../message-parts/types'; +import NativeAIAgent from '../native-specs/NativeAIAgent'; + +// The sign-in runs the router's launch page in the platform's web authentication session +// (`ASWebAuthenticationSession` on iOS, an Auth Tab or a Custom Tab on Android), with the +// handoff token in the page's fragment, which no server sees. The page goes to the provider, +// which sends the person back through the router to the session's return URL, on this app's +// scheme, with how it ended in the query. The counterpart of `AIWebAuthentication` in +// StreamChatAI and of `createPopupAuthorizer` on the web. + +/** How the router says a consent ended, in the return URL's `status` query parameter. */ +export type ConnectorSignInStatus = 'connected' | 'denied' | 'failed' | 'account_mismatch'; + +const OUTCOMES: readonly string[] = [ + 'connected', + 'denied', + 'failed', + 'account_mismatch', +] satisfies ConnectorSignInStatus[]; + +const CONNECTION_ID = /^[\w-]{1,128}$/; + +/** The sign-in's outcome, as the return URL says it. */ +export type ConnectorSignInResult = { + connectionId: string; + status: ConnectorSignInStatus; +}; + +/** + * Why a sign-in could not be opened: the call carries no sign-in, or the app was built without + * the SDK's native code (`unavailable`), or the platform's session failed to open (`failed`). + */ +export class AIConnectorSignInError extends Error { + readonly reason: 'failed' | 'unavailable'; + + constructor(reason: AIConnectorSignInError['reason'], message?: string) { + super(message ?? `The sign-in couldn't be opened (${reason}).`); + this.name = 'AIConnectorSignInError'; + this.reason = reason; + } +} + +// The page reads its fragment as form parameters, where `+` is a space and `&` and `=` separate +// them, so a token carrying one of them is percent-encoded, as is everything else unsafe. +const fragmentValue = (value: string) => encodeURIComponent(value); + +/** + * The page to open for a sign-in: its `url` with the handoff token in the fragment. Read without + * `URL`, whose parts throw on React Native before 0.80. + */ +export const signInLaunchUrl = ({ handoffToken, url }: AIToolAuthorizationData) => { + if (!handoffToken) return url; + const base = url.split('#')[0]; + return `${base}#handoff_token=${fragmentValue(handoffToken)}`; +}; + +/** Reads the router's redirect to the return URL: `…?connection_id=…&status=…`. */ +export const readConnectorSignInResult = (returned: string): ConnectorSignInResult | null => { + const query = returned.split('#')[0].split('?')[1] ?? ''; + const params: Record = {}; + for (const pair of query.split('&')) { + const [name, value = ''] = pair.split('='); + if (!name) continue; + try { + params[decodeURIComponent(name)] = decodeURIComponent(value.replace(/\+/g, ' ')); + } catch { + // A parameter that isn't valid percent-encoding is left out. + } + } + const connectionId = params.connection_id ?? ''; + const status = params.status ?? ''; + if (!CONNECTION_ID.test(connectionId) || !OUTCOMES.includes(status)) return null; + return { connectionId, status: status as ConnectorSignInStatus }; +}; + +export type AuthSessionAuthorizerOptions = { + /** + * Your app's own URL scheme, such as `com.example.app`. It is the scheme of the return URL your + * sessions give their sign-ins (`return_url`), which the sign-in comes back to. + */ + callbackScheme: string; + /** + * Tells your backend the person won't sign in now. Your backend ends the call. A rejection lets + * the person answer again. + */ + decline: (call: AIToolCallPart) => Promise; +}; + +export type AuthSessionAuthorizer = AIToolAuthorizer & { callbackScheme: string }; + +/** + * An `AIToolAuthorizer` that opens a call's sign-in in the platform's web authentication session: + * `ASWebAuthenticationSession` on iOS, and on Android the browser's Auth Tab, or a Custom Tab when + * the browser has none. + * + * `connect` resolves `true` when the return URL says the account is connected, and `false` when + * the person closed the sign-in or it ended otherwise. It rejects with an `AIConnectorSignInError` + * when the sign-in couldn't be opened. + */ +export const createAuthSessionAuthorizer = ({ + callbackScheme, + decline, +}: AuthSessionAuthorizerOptions): AuthSessionAuthorizer => { + const connect = async (call: AIToolCallPart) => { + const authorization = call.authorization; + if (!authorization || !NativeAIAgent) throw new AIConnectorSignInError('unavailable'); + let returned: string; + try { + returned = await NativeAIAgent.openAuthSession( + signInLaunchUrl(authorization), + callbackScheme, + ); + } catch (error) { + throw new AIConnectorSignInError( + 'failed', + error instanceof Error ? error.message : undefined, + ); + } + return returned !== '' && readConnectorSignInResult(returned)?.status === 'connected'; + }; + + return { callbackScheme, connect, decline }; +}; diff --git a/package/src/ai-components/tool-calls/index.ts b/package/src/ai-components/tool-calls/index.ts index ad73d0bd40..98cd79cca7 100644 --- a/package/src/ai-components/tool-calls/index.ts +++ b/package/src/ai-components/tool-calls/index.ts @@ -1 +1,3 @@ +export * from './AIClientIdentity'; export * from './AIToolApprover'; +export * from './authSessionAuthorizer'; From 8b1d6c8023395da0ed377b710ce12ca53322e88e Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 23:57:08 +0100 Subject: [PATCH 05/23] feat(ai-components): message views, reply actions and icon overrides Ported from #3928 on v9-release. --- .../ExpoChatGPTSample/contexts/AppContext.tsx | 24 ++ examples/ExpoChatGPTSample/custom-types.d.ts | 2 + .../ExpoChatGPTSample/screens/ChatContent.tsx | 150 +++++---- .../__tests__/messageViews.test.tsx | 289 ++++++++++++++++++ .../ai-components/contexts/theme-context.tsx | 14 +- package/src/ai-components/icons.ts | 98 ++++++ package/src/ai-components/index.ts | 2 + .../internal/icons/AgentIcons.tsx | 88 +++++- .../message-parts/AIMessageParts.tsx | 5 +- .../message-parts/AIToolCall.tsx | 22 +- .../message/AIAssistantMessage.tsx | 82 +++++ .../message/AIMessageAttachment.tsx | 247 +++++++++++++++ .../ai-components/message/AIReplyActions.tsx | 134 ++++++++ .../ai-components/message/AIUserMessage.tsx | 121 ++++++++ package/src/ai-components/message/index.ts | 4 + .../ai-components/reasoning/AIReasoning.tsx | 7 +- .../services/clipboard-service/expo.ts | 22 +- package/src/ai-components/theme.ts | 65 ++++ 18 files changed, 1265 insertions(+), 111 deletions(-) create mode 100644 package/src/ai-components/__tests__/messageViews.test.tsx create mode 100644 package/src/ai-components/icons.ts create mode 100644 package/src/ai-components/message/AIAssistantMessage.tsx create mode 100644 package/src/ai-components/message/AIMessageAttachment.tsx create mode 100644 package/src/ai-components/message/AIReplyActions.tsx create mode 100644 package/src/ai-components/message/AIUserMessage.tsx create mode 100644 package/src/ai-components/message/index.ts diff --git a/examples/ExpoChatGPTSample/contexts/AppContext.tsx b/examples/ExpoChatGPTSample/contexts/AppContext.tsx index 73636148ae..78addabde3 100644 --- a/examples/ExpoChatGPTSample/contexts/AppContext.tsx +++ b/examples/ExpoChatGPTSample/contexts/AppContext.tsx @@ -11,6 +11,7 @@ import type { Channel, Event, StreamChat } from 'stream-chat'; import { useStableCallback } from 'stream-chat-expo'; import { AIClientIdentity, + type AIMessageRating, type AIToolApprover, createAuthSessionAuthorizer, } from 'stream-chat-expo/ai-components'; @@ -54,6 +55,10 @@ export type AppContextValue = { ask: (text: string, images?: ImageSource[]) => Promise; /** Carries on a conversation from the drawer, by its session id. */ openConversation: (sessionId: string) => void; + /** Rates a reply, or takes the rating back with null. */ + rate: (messageId: string, rating: AIMessageRating | null) => void; + /** How the person rated replies, by message. Kept for this run only: the router takes no ratings yet. */ + ratings: Readonly>; /** The whole of a reply's reasoning step so far, from its live updates. */ reasoningText: (messageId: string, stepId: string) => string | undefined; /** Bumped as reasoning arrives, so the steps showing it render again. */ @@ -72,6 +77,8 @@ export const AppContext = React.createContext({ conversationsVersion: 0, failure: undefined, openConversation: () => {}, + rate: () => {}, + ratings: {}, reasoningText: () => undefined, reasoningVersion: 0, sessionId: undefined, @@ -87,6 +94,7 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St const [agentState, setAgentState] = useState(IDLE); const [reasoningVersion, setReasoningVersion] = useState(0); const [conversationsVersion, setConversationsVersion] = useState(0); + const [ratings, setRatings] = useState>>({}); const current = useRef(undefined); // Opening one conversation after another: the newest wins, and one that was never asked @@ -147,6 +155,18 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St } }); + const rate = useStableCallback((messageId: string, rating: AIMessageRating | null) => + setRatings((previous) => { + const next = { ...previous }; + if (rating) { + next[messageId] = rating; + } else { + delete next[messageId]; + } + return next; + }), + ); + const reasoningText = useStableCallback((messageId: string, stepId: string) => current.current?.reasoning.get(messageId, stepId), ); @@ -230,6 +250,8 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St conversationsVersion, failure, openConversation, + rate, + ratings, reasoningText, reasoningVersion, sessionId: conversation?.session.id, @@ -244,6 +266,8 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St conversationsVersion, failure, openConversation, + rate, + ratings, reasoningText, reasoningVersion, startConversation, diff --git a/examples/ExpoChatGPTSample/custom-types.d.ts b/examples/ExpoChatGPTSample/custom-types.d.ts index af1dc3dcdb..4af867514c 100644 --- a/examples/ExpoChatGPTSample/custom-types.d.ts +++ b/examples/ExpoChatGPTSample/custom-types.d.ts @@ -27,6 +27,8 @@ declare module 'stream-chat' { interface CustomMessageData extends DefaultMessageData { ai_generated?: boolean; + /** Set by the router while the agent is still writing a reply. */ + generating?: boolean; } interface CustomPollOptionData extends DefaultPollOptionData {} diff --git a/examples/ExpoChatGPTSample/screens/ChatContent.tsx b/examples/ExpoChatGPTSample/screens/ChatContent.tsx index 722e24721b..e5eb2e9118 100644 --- a/examples/ExpoChatGPTSample/screens/ChatContent.tsx +++ b/examples/ExpoChatGPTSample/screens/ChatContent.tsx @@ -1,4 +1,4 @@ -import { useMemo, useState } from 'react'; +import { useMemo } from 'react'; import { ActivityIndicator, Platform, @@ -13,30 +13,26 @@ import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { useHeaderHeight } from 'expo-router/react-navigation'; -import type { MessageProps, ResolvedComponents } from 'stream-chat-expo'; +import type { Attachment, LocalMessage } from 'stream-chat'; +import type { MessageProps } from 'stream-chat-expo'; import { Channel, isLocalUrl, - mergeThemes, - Message, MessageList, - ThemeProvider, - useComponentsContext, - useMessageContext, usePaginatedMessageListContext, - useTheme, WithComponents, } from 'stream-chat-expo'; import { - AI_MESSAGE_PART_TYPE_PREFIX, - AIMessageParts, + AIAssistantMessage, + type AIMessageAttachmentData, + AIReplyActions, AIThinking, AIToolApproverProvider, AITypingIndicatorView, + AIUserMessage, ComposerView, parseMessageParts, - StreamingMessageView, } from 'stream-chat-expo/ai-components'; import { bottomSheetOptions } from '../bottomSheetOptions'; @@ -44,11 +40,39 @@ import { VoiceMode } from '../components/VoiceMode'; import { useAppContext } from '../contexts/AppContext'; import { useSendAIMessage } from '../hooks/useSendAIMessage'; -const CustomMessage = (props: MessageProps) => { - const { theme } = useTheme(); - const { message } = props; - const isFromBot = message.ai_generated; - const hasPendingAttachments = useMemo( +// Images and files only: a reply's steps (`ai_*`) are read as its parts, and other kinds, such as +// the router's `connector_authorization`, a second copy of a sign-in its tool call's step asks for, +// aren't shown. +const attachmentOf = (attachment: Attachment): AIMessageAttachmentData | undefined => { + const name = attachment.title ?? attachment.fallback ?? undefined; + if (attachment.type === 'image') { + const url = attachment.image_url ?? attachment.asset_url ?? attachment.thumb_url; + return url ? { kind: 'image', mimeType: attachment.mime_type, name, url } : undefined; + } + if (attachment.type === 'file') { + const size = Number(attachment.file_size); + return attachment.asset_url + ? { + kind: 'file', + mimeType: attachment.mime_type, + name, + size: Number.isFinite(size) && size > 0 ? size : undefined, + url: attachment.asset_url, + } + : undefined; + } + return undefined; +}; + +const attachmentsOf = (attachments: Attachment[] | undefined) => + (attachments ?? []) + .map(attachmentOf) + .filter((attachment): attachment is AIMessageAttachmentData => !!attachment); + +// The person's message, dimmed while its photos upload. +const UserMessage = ({ message }: { message: LocalMessage }) => { + const attachments = useMemo(() => attachmentsOf(message.attachments), [message.attachments]); + const isUploading = useMemo( () => (message.attachments ?? []).some( (attachment) => @@ -57,81 +81,55 @@ const CustomMessage = (props: MessageProps) => { ), [message.attachments], ); - - const modifiedTheme = useMemo( - () => - mergeThemes({ - theme, - style: { - messageItemView: isFromBot - ? { - content: { - containerInner: { - backgroundColor: 'transparent', - borderRadius: 0, - borderColor: 'transparent', - }, - textContainer: { maxWidth: '100%' }, - }, - } - : { - container: { - opacity: hasPendingAttachments ? 0.5 : 1, - }, - }, - }, - }), - [theme, isFromBot, hasPendingAttachments], - ); - return ( - - - + + + ); }; -type UnsupportedAttachmentComponent = ResolvedComponents['UnsupportedAttachment']; - -// A reply's steps (`ai_reasoning`, `ai_tool_call`) are attachments the chat SDK doesn't know. They -// show in the reply's own view, so they're left out here, as is the router's `connector_authorization`, -// a second copy of a sign-in its tool call's step already asks for. Any other unknown attachment -// keeps the SDK's own view. -const hidingAISteps = (Fallback: UnsupportedAttachmentComponent) => { - const HideAISteps: UnsupportedAttachmentComponent = (props) => - props.attachment.type?.startsWith(AI_MESSAGE_PART_TYPE_PREFIX) || - props.attachment.type === 'connector_authorization' ? null : ( - - ); - return HideAISteps; -}; - -// A reply: its steps (reasoning, tool calls and their questions), then its answer. -const CustomStreamingMessageView = () => { - const { message: listed } = useMessageContext(); - // Read so the steps render again as each live update arrives. - const { channel, reasoningText, reasoningVersion } = useAppContext(); +// A reply: its steps, its answer and what it sent, then its actions once it's written. +const AssistantMessage = ({ message: listed }: { message: LocalMessage }) => { + // Read so the reply renders again as each live update arrives. + const { channel, rate, ratings, reasoningText, reasoningVersion } = useAppContext(); // The list passes a row its message again only when it changes in ways it compares, which a // step changing on its own, such as a tool call starting to wait for a sign-in, isn't. The // channel's state has every update. const message = channel?.state.findMessage(listed.id) ?? listed; const parts = useMemo(() => parseMessageParts(message.attachments), [message.attachments]); + const attachments = useMemo(() => attachmentsOf(message.attachments), [message.attachments]); const stepText = useMemo( () => (stepId: string) => reasoningText(message.id, stepId), // eslint-disable-next-line react-hooks/exhaustive-deps -- a new function per slice of reasoning [message.id, reasoningText, reasoningVersion], ); - // The message list sizes a bubble to its content; a reply's steps take the row's whole width, - // the screen less the list's 16 on each side. + const isGenerating = message.generating === true; + const text = message.text ?? ''; + // The list sizes a row to its content; a reply takes the row's whole width. const { width } = useWindowDimensions(); return ( - - {parts.length > 0 ? : null} - {message.text ? : null} + + + {!isGenerating && text ? ( + rate(message.id, rating)} + rating={ratings[message.id] ?? null} + text={text} + /> + ) : null} ); }; +const CustomMessage = ({ message }: MessageProps) => + message.ai_generated ? : ; + const CustomComposerView = () => { const { agentState, stop } = useAppContext(); @@ -212,9 +210,6 @@ export const ChatContent = () => { // The channel sits below the drawer header, so the keyboard offset has to account for it // (without it, Android keeps the composer behind the keyboard). const headerHeight = useHeaderHeight(); - // Read above this screen's own overrides, so it is the SDK's view. - const { UnsupportedAttachment } = useComponentsContext(); - const [CustomUnsupportedAttachment] = useState(() => hidingAISteps(UnsupportedAttachment)); if (!channel) { return ( @@ -235,11 +230,7 @@ export const ChatContent = () => { overrides={{ EmptyStateIndicator, Message: CustomMessage, - MessageAuthor: RenderNull, - MessageFooter: RenderNull, NetworkDownIndicator: RenderNull, - StreamingMessageView: CustomStreamingMessageView, - UnsupportedAttachment: CustomUnsupportedAttachment, }} > @@ -271,7 +262,8 @@ const styles = StyleSheet.create({ }, emptyContainerText: { fontSize: 24, fontWeight: 'bold' }, failureText: { color: '#B00020', fontSize: 15, paddingHorizontal: 24, textAlign: 'center' }, - streamingMessageViewWrapper: { paddingHorizontal: 16 }, + assistantMessage: { gap: 4 }, + message: { paddingHorizontal: 16, paddingVertical: 8 }, aiTypingIndicatorWrapper: { paddingHorizontal: 24, paddingVertical: 12, diff --git a/package/src/ai-components/__tests__/messageViews.test.tsx b/package/src/ai-components/__tests__/messageViews.test.tsx new file mode 100644 index 0000000000..c91ea64079 --- /dev/null +++ b/package/src/ai-components/__tests__/messageViews.test.tsx @@ -0,0 +1,289 @@ +import React, { type ReactElement } from 'react'; +import { Linking, Share, Text } from 'react-native'; + +import { act, fireEvent, render, screen } from '@testing-library/react-native'; + +import { + AIAssistantMessage, + type AIIcon, + AIMessageAttachment, + type AIMessageAttachmentProps, + AIReplyActions, + AIUserMessage, + attachmentDetails, + defaultAIIcons, + formatFileSize, + mergeThemes, + parseMessageParts, + StreamTheme, +} from '../index'; +import { setClipboardString } from '../services/clipboard-service'; + +jest.mock('../services/clipboard-service', () => ({ setClipboardString: jest.fn() })); + +const renderWithTheme = (ui: ReactElement) => render({ui}); + +afterEach(() => { + jest.restoreAllMocks(); + jest.useRealTimers(); +}); + +describe('formatFileSize', () => { + it('writes three significant digits in the largest unit that fits', () => { + expect(formatFileSize(0)).toBe('0.00 B'); + expect(formatFileSize(512)).toBe('512 B'); + expect(formatFileSize(2_453_000)).toBe('2.34 MB'); + }); + + it('rounds a size just below the next unit instead of turning exponential', () => { + expect(formatFileSize(1000)).toBe('1000 B'); + expect(formatFileSize(1023 * 1024)).toBe('1023 kB'); + }); +}); + +describe('attachmentDetails', () => { + it('joins the type and the size', () => { + expect(attachmentDetails({ kind: 'file', name: 'report.pdf', size: 2_453_000 })).toBe( + 'PDF · 2.34 MB', + ); + }); + + it('takes the type from the URL, then the media type', () => { + expect(attachmentDetails({ kind: 'file', url: 'https://x.io/a/notes.txt?v=1' })).toBe('TXT'); + expect(attachmentDetails({ kind: 'file', mimeType: 'image/svg+xml' })).toBe('SVG'); + }); + + it('is empty with nothing to say', () => { + expect(attachmentDetails({ kind: 'file' })).toBe(''); + }); +}); + +describe('AIMessageAttachment', () => { + it('shows an image as a thumbnail named for it, which does not open', () => { + const openURL = jest.spyOn(Linking, 'openURL').mockResolvedValue(undefined); + renderWithTheme( + , + ); + expect(screen.getByLabelText('cat.png')).toBeTruthy(); + expect(screen.queryByRole('imagebutton')).toBeNull(); + expect(screen.queryByRole('link')).toBeNull(); + expect(openURL).not.toHaveBeenCalled(); + }); + + it('shows an image without a URL as a file', () => { + renderWithTheme(); + expect(screen.getByText('photo.png')).toBeTruthy(); + expect(screen.getByText('PNG')).toBeTruthy(); + }); + + it('shows a file with its name, type and size, and opens one at a web URL', () => { + const openURL = jest.spyOn(Linking, 'openURL').mockResolvedValue(undefined); + renderWithTheme( + , + ); + expect(screen.getByText('report.pdf')).toBeTruthy(); + expect(screen.getByText('PDF · 2.34 MB')).toBeTruthy(); + fireEvent.press(screen.getByRole('link')); + expect(openURL).toHaveBeenCalledWith('https://x.io/r'); + }); + + it("doesn't open anything but a web URL", () => { + renderWithTheme( + , + ); + expect(screen.queryByRole('link')).toBeNull(); + }); + + it('names one without a name by its kind, and shows a kind it does not know as a file', () => { + renderWithTheme(); + expect(screen.getByText('Image')).toBeTruthy(); + renderWithTheme(); + expect(screen.getByText('File')).toBeTruthy(); + }); +}); + +describe('AIUserMessage', () => { + it('shows the text and the attachments', () => { + renderWithTheme( + , + ); + expect(screen.getByText('Summarise this')).toBeTruthy(); + expect(screen.getByText('a.pdf')).toBeTruthy(); + }); + + it('offers editing only with onEdit', () => { + const onEdit = jest.fn(); + const { rerender } = renderWithTheme(); + expect(screen.queryByRole('button', { name: 'Edit' })).toBeNull(); + rerender( + + + , + ); + fireEvent.press(screen.getByRole('button', { name: 'Edit' })); + expect(onEdit).toHaveBeenCalledTimes(1); + }); + + it('shows no bubble for a message of attachments alone', () => { + renderWithTheme(); + expect(screen.queryByRole('button', { name: 'Edit' })).toBeNull(); + expect(screen.getByText('a.pdf')).toBeTruthy(); + }); +}); + +describe('AIAssistantMessage', () => { + it('shows the steps, then the answer, then the attachments', () => { + const parts = parseMessageParts([ + { id: 'r1', status: 'completed', summary: 'Looked it up', type: 'ai_reasoning' }, + ]); + renderWithTheme( + , + ); + expect(screen.getByTestId('ai-message-parts')).toBeTruthy(); + expect(screen.getByText('The answer')).toBeTruthy(); + expect(screen.getByText('chart.csv')).toBeTruthy(); + }); + + it('is busy while the agent writes it', () => { + renderWithTheme(); + expect(screen.getByTestId('ai-assistant-message').props.accessibilityState).toEqual({ + busy: true, + }); + }); + + it('shows nothing of the answer before it has text', () => { + renderWithTheme(); + expect(screen.queryByTestId('ai-message-parts')).toBeNull(); + }); +}); + +describe('AIReplyActions', () => { + it('copies the text and shows it was copied for a moment', () => { + jest.useFakeTimers(); + renderWithTheme(); + fireEvent.press(screen.getByRole('button', { name: 'Copy' })); + expect(setClipboardString).toHaveBeenCalledWith('The answer'); + expect(screen.getByRole('button', { name: 'Copied' })).toBeTruthy(); + act(() => { + jest.advanceTimersByTime(1500); + }); + expect(screen.getByRole('button', { name: 'Copy' })).toBeTruthy(); + }); + + it('shares the text through the system share sheet', () => { + const share = jest + .spyOn(Share, 'share') + .mockResolvedValue({ action: 'sharedAction', activityType: undefined }); + renderWithTheme(); + fireEvent.press(screen.getByRole('button', { name: 'Share' })); + expect(share).toHaveBeenCalledWith({ message: 'The answer' }); + }); + + it('offers regenerating and rating only when given what they do', () => { + renderWithTheme(); + expect(screen.queryByRole('button', { name: 'Regenerate' })).toBeNull(); + expect(screen.queryByRole('button', { name: 'Good response' })).toBeNull(); + }); + + it('regenerates', () => { + const onRegenerate = jest.fn(); + renderWithTheme(); + fireEvent.press(screen.getByRole('button', { name: 'Regenerate' })); + expect(onRegenerate).toHaveBeenCalledTimes(1); + }); + + it('rates, and takes a rating back', () => { + const onRate = jest.fn(); + const { rerender } = renderWithTheme(); + fireEvent.press(screen.getByRole('button', { name: 'Bad response' })); + expect(onRate).toHaveBeenLastCalledWith('negative'); + + rerender( + + + , + ); + const good = screen.getByRole('button', { name: 'Good response' }); + expect(good.props.accessibilityState).toEqual({ selected: true }); + fireEvent.press(good); + expect(onRate).toHaveBeenLastCalledWith(null); + }); +}); + +const iconNamed = + (name: string): AIIcon => + () => {`icon:${name}`}; + +describe('icons', () => { + it('draws with the theme’s icons over the defaults', () => { + render( + + + , + ); + expect(screen.getByText('icon:Copy')).toBeTruthy(); + expect(screen.getByText('icon:Share')).toBeTruthy(); + expect(screen.queryByText('icon:ThumbsUp')).toBeNull(); + }); + + it('keeps icons as they are when themes merge', () => { + const Copy = iconNamed('Copy'); + const Share = iconNamed('Share'); + const theme = mergeThemes({ style: { icons: { Copy } } }); + expect(theme.icons?.Copy).toBe(Copy); + const merged = mergeThemes({ style: { icons: { Share } }, theme }); + expect(merged.icons).toEqual({ Copy, Share }); + expect(merged.colors).toEqual(theme.colors); + }); + + it.each([ + [{ kind: 'file', name: 'report.pdf' }, 'Pdf'], + [{ kind: 'file', mimeType: 'application/pdf', name: 'report' }, 'Pdf'], + [{ kind: 'file', name: 'song.mp3' }, 'Audio'], + [{ kind: 'file', mimeType: 'video/mp4', name: 'clip' }, 'Video'], + [{ kind: 'file', name: 'backup.zip' }, 'Archive'], + [{ kind: 'file', mimeType: 'application/vnd.rar', name: 'x' }, 'Archive'], + [{ kind: 'image', name: 'photo' }, 'Image'], + [{ kind: 'file', name: 'notes.txt' }, 'File'], + ])('gives a file of %o the %s icon', (attachment, icon) => { + const icons = Object.fromEntries( + Object.keys(defaultAIIcons).map((name) => [name, iconNamed(name)]), + ); + render( + + + , + ); + expect(screen.getByText(`icon:${icon}`)).toBeTruthy(); + }); +}); + +describe('Attachment slot', () => { + const Mine = ({ attachment, role }: AIMessageAttachmentProps) => ( + {`mine:${role}:${attachment.name}`} + ); + + it('shows the person’s attachments with a view of your own', () => { + renderWithTheme( + , + ); + expect(screen.getByText('mine:user:a.pdf')).toBeTruthy(); + }); + + it('shows the agent’s attachments with a view of your own', () => { + renderWithTheme( + , + ); + expect(screen.getByText('mine:assistant:b.csv')).toBeTruthy(); + }); +}); diff --git a/package/src/ai-components/contexts/theme-context.tsx b/package/src/ai-components/contexts/theme-context.tsx index b1f68bf1ae..3a6f995a50 100644 --- a/package/src/ai-components/contexts/theme-context.tsx +++ b/package/src/ai-components/contexts/theme-context.tsx @@ -26,15 +26,23 @@ export type ThemeContextValue = { export const mergeThemes = (params: MergedThemesParams) => { const { style, theme } = params; + // Icons are components, which the JSON copy below would drop and a deep merge would take apart, + // so they are carried over as they are. + const { icons: themeIcons, ...themeRest } = theme ?? {}; + const { icons: styleIcons, ...styleRest } = style ?? {}; // TODO: Use structured-clone here or something similar. const finalTheme = ( - !theme || Object.keys(theme).length === 0 + !theme || Object.keys(themeRest).length === 0 ? JSON.parse(JSON.stringify(defaultTheme)) - : JSON.parse(JSON.stringify(theme)) + : JSON.parse(JSON.stringify(themeRest)) ) as Theme; if (style) { - merge(finalTheme, style); + merge(finalTheme, styleRest); + } + + if (themeIcons || styleIcons) { + finalTheme.icons = { ...themeIcons, ...styleIcons } as Theme['icons']; } return finalTheme; diff --git a/package/src/ai-components/icons.ts b/package/src/ai-components/icons.ts new file mode 100644 index 0000000000..48a1a02145 --- /dev/null +++ b/package/src/ai-components/icons.ts @@ -0,0 +1,98 @@ +import { type ComponentType, useMemo } from 'react'; + +import { useTheme } from './contexts/theme-context'; +import { + ArchiveIcon, + AudioIcon, + CancelledIcon, + CheckIcon, + ChevronIcon, + CopiedIcon, + CopyIcon, + DeviceIcon, + EditIcon, + FailedIcon, + FileIcon, + HandRaisedIcon, + ImageIcon, + InfoIcon, + KeyIcon, + PdfIcon, + ReasoningIcon, + RegenerateIcon, + ShareIcon, + ThumbsDownFilledIcon, + ThumbsDownIcon, + ThumbsUpFilledIcon, + ThumbsUpIcon, + VideoIcon, +} from './internal/icons/AgentIcons'; + +/** + * What an icon of the agent replies' views is drawn with. An icon is decorative (the text or + * label beside it says what it means), so one of your own hides itself from screen readers, and + * one that points somewhere, such as `Chevron`, mirrors itself in a right-to-left layout. + */ +export type AIIconProps = { color: string; size?: number }; + +export type AIIcon = ComponentType; + +/** The icons of the agent replies' views, by what they stand for. */ +export const defaultAIIcons = { + /** A file of an archive type, such as ZIP, on its attachment. */ + Archive: ArchiveIcon, + /** An audio file, on its attachment. */ + Audio: AudioIcon, + /** A tool call that was cancelled or declined. */ + Cancelled: CancelledIcon, + /** A tool call that completed. */ + Check: CheckIcon, + /** A reasoning step's fold. */ + Chevron: ChevronIcon, + /** The copy action, for a moment after it copied. */ + Copied: CopiedIcon, + /** The copy action. */ + Copy: CopyIcon, + /** A tool call that waits for the person's device. */ + Device: DeviceIcon, + /** The edit action on the person's message. */ + Edit: EditIcon, + /** A tool call that didn't complete. */ + Failed: FailedIcon, + /** A file of a type without an icon of its own, on its attachment. */ + File: FileIcon, + /** A tool call that waits for the person's approval. */ + HandRaised: HandRaisedIcon, + /** An image shown as a file, such as one without a URL, on its attachment. */ + Image: ImageIcon, + /** A step this version can't show. */ + Info: InfoIcon, + /** A tool call that waits for the person to sign in. */ + Key: KeyIcon, + /** A PDF, on its attachment. */ + Pdf: PdfIcon, + /** A reasoning step. */ + Reasoning: ReasoningIcon, + /** The regenerate action. */ + Regenerate: RegenerateIcon, + /** The share action. */ + Share: ShareIcon, + /** The bad response rating, and the same once chosen. */ + ThumbsDown: ThumbsDownIcon, + ThumbsDownFilled: ThumbsDownFilledIcon, + /** The good response rating, and the same once chosen. */ + ThumbsUp: ThumbsUpIcon, + ThumbsUpFilled: ThumbsUpFilledIcon, + /** A video, on its attachment. */ + Video: VideoIcon, +} satisfies Record; + +export type AIIcons = Record; + +/** The icons to draw with: the theme's `icons`, over the defaults. */ +export const useAIIcons = (): AIIcons => { + const { + theme: { icons }, + } = useTheme(); + return useMemo(() => ({ ...defaultAIIcons, ...icons }), [icons]); +}; diff --git a/package/src/ai-components/index.ts b/package/src/ai-components/index.ts index 23d3042976..3d79049094 100644 --- a/package/src/ai-components/index.ts +++ b/package/src/ai-components/index.ts @@ -6,6 +6,8 @@ export * from './StreamingMessageView'; export * from './AITypingIndicatorView'; export * from './voice-visualization'; +export * from './icons'; +export * from './message'; export * from './message-parts'; export * from './reasoning'; export * from './tool-calls'; diff --git a/package/src/ai-components/internal/icons/AgentIcons.tsx b/package/src/ai-components/internal/icons/AgentIcons.tsx index 188bd0fe9b..cec2259f0e 100644 --- a/package/src/ai-components/internal/icons/AgentIcons.tsx +++ b/package/src/ai-components/internal/icons/AgentIcons.tsx @@ -2,15 +2,27 @@ import React from 'react'; import { I18nManager } from 'react-native'; import Svg, { Path } from 'react-native-svg'; -// The agent reply steps' icons, on a 20 × 20 grid with a 1.5 stroke, as in stream-chat-react's -// AI components. Decorative: the text beside each one says what it means. +// The agent replies' icons, on a 20 × 20 grid with a 1.5 stroke, as in stream-chat-react's +// AI components. Decorative: the text beside each one, or its button's label, says what it means. export type AgentIconProps = { color: string; size?: number }; const circle = 'M17.5 10C17.5 14.14 14.14 17.5 10 17.5C5.86 17.5 2.5 14.14 2.5 10C2.5 5.86 5.86 2.5 10 2.5C14.14 2.5 17.5 5.86 17.5 10Z'; -const strokeIcon = (d: string, mirrorsInRTL = false) => { +type StrokeIconOptions = { + /** Filled with its colour as well as drawn, such as a chosen rating. */ + filled?: boolean; + /** Flipped in a right-to-left layout, for one that points somewhere. */ + mirrorsInRTL?: boolean; + /** Turned around the grid's centre, in degrees. */ + rotation?: number; +}; + +const strokeIcon = ( + d: string, + { filled = false, mirrorsInRTL = false, rotation }: StrokeIconOptions = {}, +) => { const Icon = ({ color, size = 16 }: AgentIconProps) => ( { > ); @@ -37,7 +55,7 @@ export const ReasoningIcon = strokeIcon( 'M7.5 15.625H12.5M8.125 18.125H11.875M10 1.875C6.89 1.875 4.375 4.39 4.375 7.5C4.375 9.53 5.45 11.31 7.06 12.3C7.33 12.47 7.5 12.76 7.5 13.08V13.75H12.5V13.08C12.5 12.76 12.67 12.47 12.94 12.3C14.55 11.31 15.625 9.53 15.625 7.5C15.625 4.39 13.11 1.875 10 1.875Z', ); -export const ChevronIcon = strokeIcon('M7.5 4.375L13.125 10L7.5 15.625', true); +export const ChevronIcon = strokeIcon('M7.5 4.375L13.125 10L7.5 15.625', { mirrorsInRTL: true }); export const CheckIcon = strokeIcon('M4.375 10.625L8.125 14.375L15.625 5.625'); @@ -58,3 +76,61 @@ export const KeyIcon = strokeIcon( export const DeviceIcon = strokeIcon( 'M8.125 15H11.875M6.25 18.125H13.75C14.44 18.125 15 17.565 15 16.875V3.125C15 2.435 14.44 1.875 13.75 1.875H6.25C5.56 1.875 5 2.435 5 3.125V16.875C5 17.565 5.56 18.125 6.25 18.125Z', ); + +// A message's actions, from stream-chat-react's icon set. + +export const CopyIcon = strokeIcon( + 'M13.125 13.125H16.875V3.125H6.875V6.875M3.125 6.875H13.125V16.875H3.125V6.875Z', +); + +export const CopiedIcon = strokeIcon('M3.125 11.25L7.5 15.625L17.5 5.625'); + +export const ShareIcon = strokeIcon( + 'M13.125 6.25L10 3.125M10 3.125L6.875 6.25M10 3.125V12.5M6.875 8.75H5C4.31 8.75 3.75 9.31 3.75 10V15.625C3.75 16.315 4.31 16.875 5 16.875H15C15.69 16.875 16.25 16.315 16.25 15.625V10C16.25 9.31 15.69 8.75 15 8.75H13.125', +); + +export const RegenerateIcon = strokeIcon( + 'M13.125 7.5H16.875M16.875 7.5V3.75M16.875 7.5L14.67 5.29C13.39 4.01 11.65 3.29 9.84 3.28C8.03 3.27 6.29 3.97 5 5.23M6.875 12.5H3.125M3.125 12.5V16.25M3.125 12.5L5.33 14.71C6.61 15.99 8.35 16.71 10.16 16.72C11.97 16.73 13.71 16.03 15 14.77', +); + +export const EditIcon = strokeIcon( + 'M10.625 5L15 9.375M7.24 16.875H3.75C3.58 16.875 3.43 16.81 3.31 16.69C3.19 16.57 3.125 16.42 3.125 16.25V12.76C3.125 12.59 3.19 12.43 3.31 12.32L12.94 2.68C13.06 2.57 13.22 2.5 13.38 2.5C13.55 2.5 13.71 2.57 13.83 2.68L17.32 6.17C17.43 6.29 17.5 6.45 17.5 6.61C17.5 6.78 17.43 6.94 17.32 7.06L7.68 16.69C7.57 16.81 7.41 16.875 7.24 16.875Z', +); + +export const FileIcon = strokeIcon( + 'M11.875 2.5H4.375C4.21 2.5 4.05 2.57 3.93 2.68C3.82 2.8 3.75 2.96 3.75 3.125V16.875C3.75 17.04 3.82 17.2 3.93 17.32C4.05 17.43 4.21 17.5 4.375 17.5H15.625C15.79 17.5 15.95 17.43 16.07 17.32C16.18 17.2 16.25 17.04 16.25 16.875V6.875M11.875 2.5L16.25 6.875M11.875 2.5V6.875H16.25', +); + +const thumb = + 'M5.625 8.75V16.25M5.625 8.75H3.75C3.405 8.75 3.125 9.03 3.125 9.375V15.625C3.125 15.97 3.405 16.25 3.75 16.25H5.625M5.625 8.75L8.75 2.5C9.44 2.5 10.1 2.77 10.59 3.26C11.08 3.75 11.25 4.41 11.25 5.1V6.875H15.94C16.12 6.875 16.29 6.91 16.45 6.99C16.61 7.06 16.75 7.17 16.86 7.3C16.98 7.43 17.06 7.59 17.11 7.75C17.16 7.92 17.17 8.09 17.15 8.27L16.21 15.77C16.17 16.07 16.03 16.34 15.8 16.54C15.58 16.74 15.29 16.85 14.99 16.85H5.625'; + +export const ThumbsUpIcon = strokeIcon(thumb); + +export const ThumbsUpFilledIcon = strokeIcon(thumb, { filled: true }); + +export const ThumbsDownIcon = strokeIcon(thumb, { rotation: 180 }); + +export const ThumbsDownFilledIcon = strokeIcon(thumb, { filled: true, rotation: 180 }); + +// A file's type, on its attachment's tile. PDF and waveform are drawn to match StreamChatAI's +// `doc.richtext` and `waveform`, which stream-chat-react's set lacks. + +export const PdfIcon = strokeIcon( + 'M11.875 2.5H4.375C4.21 2.5 4.05 2.57 3.93 2.68C3.82 2.8 3.75 2.96 3.75 3.125V16.875C3.75 17.04 3.82 17.2 3.93 17.32C4.05 17.43 4.21 17.5 4.375 17.5H15.625C15.79 17.5 15.95 17.43 16.07 17.32C16.18 17.2 16.25 17.04 16.25 16.875V6.875M11.875 2.5L16.25 6.875M11.875 2.5V6.875H16.25M6.875 10.625H13.125M6.875 13.75H13.125', +); + +export const ImageIcon = strokeIcon( + 'M4.43 16.875L13 8.31C13.05 8.25 13.12 8.2 13.2 8.17C13.27 8.14 13.36 8.125 13.44 8.125C13.52 8.125 13.6 8.14 13.68 8.17C13.75 8.2 13.82 8.25 13.88 8.31L16.875 11.3M3.75 3.125H16.25C16.6 3.125 16.875 3.4 16.875 3.75V16.25C16.875 16.6 16.6 16.875 16.25 16.875H3.75C3.4 16.875 3.125 16.6 3.125 16.25V3.75C3.125 3.4 3.4 3.125 3.75 3.125ZM8.75 7.5C8.75 8.19 8.19 8.75 7.5 8.75C6.81 8.75 6.25 8.19 6.25 7.5C6.25 6.81 6.81 6.25 7.5 6.25C8.19 6.25 8.75 6.81 8.75 7.5Z', +); + +export const AudioIcon = strokeIcon( + 'M3.75 8.125V11.875M6.875 5.625V14.375M10 3.125V16.875M13.125 6.875V13.125M16.25 8.75V11.25', +); + +export const VideoIcon = strokeIcon( + 'M15.625 8.75L19.375 6.25V13.75L15.625 11.25M2.5 5H15C15.35 5 15.625 5.28 15.625 5.625V14.375C15.625 14.72 15.35 15 15 15H2.5C2.15 15 1.875 14.72 1.875 14.375V5.625C1.875 5.28 2.15 5 2.5 5Z', +); + +export const ArchiveIcon = strokeIcon( + 'M16.875 7.5V15C16.875 15.17 16.81 15.32 16.69 15.44C16.57 15.56 16.42 15.625 16.25 15.625H3.75C3.58 15.625 3.43 15.56 3.31 15.44C3.19 15.32 3.125 15.17 3.125 15V7.5M8.125 10.625H11.875M2.5 4.375H17.5C17.85 4.375 18.125 4.65 18.125 5V6.875C18.125 7.22 17.85 7.5 17.5 7.5H2.5C2.15 7.5 1.875 7.22 1.875 6.875V5C1.875 4.65 2.15 4.375 2.5 4.375Z', +); diff --git a/package/src/ai-components/message-parts/AIMessageParts.tsx b/package/src/ai-components/message-parts/AIMessageParts.tsx index 707fd727ee..7d2403594f 100644 --- a/package/src/ai-components/message-parts/AIMessageParts.tsx +++ b/package/src/ai-components/message-parts/AIMessageParts.tsx @@ -7,7 +7,7 @@ import { AIToolCall } from './AIToolCall'; import type { AIMessagePartData } from './types'; import { useAITranslation, useTheme } from '../contexts'; -import { InfoIcon } from '../internal/icons/AgentIcons'; +import { useAIIcons } from '../icons'; import { AIReasoning } from '../reasoning/AIReasoning'; import type { AIToolApprover } from '../tool-calls/AIToolApprover'; @@ -32,6 +32,7 @@ export const AIMessagePart = ({ approver, part, reasoningText }: AIMessagePartPr const { theme: { colors, messageParts }, } = useTheme(); + const { Info } = useAIIcons(); if (part.reasoning) { return ; @@ -47,7 +48,7 @@ export const AIMessagePart = ({ approver, part, reasoningText }: AIMessagePartPr } return ( - + diff --git a/package/src/ai-components/message-parts/AIToolCall.tsx b/package/src/ai-components/message-parts/AIToolCall.tsx index 1a1422ed21..ffe3427b49 100644 --- a/package/src/ai-components/message-parts/AIToolCall.tsx +++ b/package/src/ai-components/message-parts/AIToolCall.tsx @@ -5,15 +5,8 @@ import { isDeclined, isToolCallFinished } from './parseMessageParts'; import type { AIToolCallPart } from './types'; import { useAITranslation, useTheme } from '../contexts'; +import { useAIIcons } from '../icons'; import { formatCallDuration } from '../internal/format'; -import { - CancelledIcon, - CheckIcon, - DeviceIcon, - FailedIcon, - HandRaisedIcon, - KeyIcon, -} from '../internal/icons/AgentIcons'; import { Shimmer } from '../internal/Shimmer'; export type AIToolCallProps = { @@ -29,6 +22,7 @@ export const AIToolCall = ({ part }: AIToolCallProps) => { const { theme: { colors, messageParts }, } = useTheme(); + const { Cancelled, Check, Device, Failed, HandRaised, Key } = useAIIcons(); const title = part.displayTitle ?? part.name.replace(/_/g, ' '); const detail = (() => { @@ -53,17 +47,17 @@ export const AIToolCall = ({ part }: AIToolCallProps) => { const icon = (() => { switch (part.status) { case 'awaiting_approval': - return ; + return ; case 'awaiting_authorization': - return ; + return ; case 'awaiting_client': - return ; + return ; case 'completed': - return ; + return ; case 'failed': - return ; + return ; case 'cancelled': - return ; + return ; default: return ; } diff --git a/package/src/ai-components/message/AIAssistantMessage.tsx b/package/src/ai-components/message/AIAssistantMessage.tsx new file mode 100644 index 0000000000..176f63dd84 --- /dev/null +++ b/package/src/ai-components/message/AIAssistantMessage.tsx @@ -0,0 +1,82 @@ +import React, { type ComponentType } from 'react'; +import { StyleSheet, View } from 'react-native'; + +import { + AIMessageAttachment, + type AIMessageAttachmentData, + type AIMessageAttachmentProps, +} from './AIMessageAttachment'; + +import { useTheme } from '../contexts'; +import { AIMessageParts } from '../message-parts/AIMessageParts'; +import type { AIMessagePartData } from '../message-parts/types'; +import { StreamingMessageView } from '../StreamingMessageView'; +import type { AIToolApprover } from '../tool-calls/AIToolApprover'; + +export type AIAssistantMessageProps = { + /** The answer, in Markdown. */ + text: string; + /** + * Who answers tool calls' questions and sign-ins on this device, to ask them. Defaults to the + * one of the nearest `AIToolApproverProvider`. + */ + approver?: AIToolApprover; + /** Shows each attachment, instead of `AIMessageAttachment`, such as for a kind of your own. */ + Attachment?: ComponentType; + /** What the agent sent with it, such as images and files. */ + attachments?: AIMessageAttachmentData[]; + /** Whether the agent is still writing it. The text that arrives meanwhile is typed out. */ + isGenerating?: boolean; + /** Opens a link in the answer, instead of the system's handler. */ + onLink?: (url: string) => void; + /** The steps the agent took, in order, shown before the answer. */ + parts?: AIMessagePartData[]; + /** The whole of a reasoning step's thinking, when the app has it. */ + reasoningText?: (stepId: string) => string | undefined; +}; + +/** + * The agent's reply: its steps (reasoning, tool calls and their questions), then its answer, + * typed out as it streams in, then what it sent with it. The counterpart of + * `AIAssistantMessageView`. + */ +export const AIAssistantMessage = ({ + approver, + Attachment = AIMessageAttachment, + attachments = [], + isGenerating = false, + onLink, + parts = [], + reasoningText, + text, +}: AIAssistantMessageProps) => { + const { + theme: { message }, + } = useTheme(); + const styles = message?.assistantMessage; + return ( + + {parts.length > 0 ? ( + + ) : null} + {/* What arrives after mount is typed out; a reply loaded from history shows at once. */} + {text !== '' ? : null} + {attachments.length > 0 ? ( + + {attachments.map((attachment, index) => ( + + ))} + + ) : null} + + ); +}; + +const defaultStyles = StyleSheet.create({ + attachments: { alignItems: 'flex-start', gap: 8 }, + container: { gap: 8 }, +}); diff --git a/package/src/ai-components/message/AIMessageAttachment.tsx b/package/src/ai-components/message/AIMessageAttachment.tsx new file mode 100644 index 0000000000..81e63364d9 --- /dev/null +++ b/package/src/ai-components/message/AIMessageAttachment.tsx @@ -0,0 +1,247 @@ +import React from 'react'; +import { Image, Linking, Pressable, StyleSheet, Text, View } from 'react-native'; + +import { useAITranslation, useTheme } from '../contexts'; +import { type AIIcons, useAIIcons } from '../icons'; + +/** + * What a message carries besides its text, such as an image or a file. The counterpart of + * `AIMessageAttachment` in StreamChatAI. + */ +export type AIMessageAttachmentData = { + /** `image` shows a thumbnail; `file` shows its name, type and size. An open set. */ + kind: 'image' | 'file' | (string & {}); + /** Data of its own, such as what an attachment of a kind of your own needs. */ + extraData?: Record; + /** Its media type, such as `application/pdf`. Without it, the type comes from its name or URL. */ + mimeType?: string; + /** Its name, such as the file's. */ + name?: string; + /** Its size, in bytes. */ + size?: number; + /** Where it is: a web URL, or a local one of something the app holds, such as a photo just picked. */ + url?: string; +}; + +/** The extension of its name, else of its URL. */ +const pathExtensionOf = (attachment: AIMessageAttachmentData) => { + for (const candidate of [attachment.name, attachment.url]) { + if (!candidate) continue; + const path = candidate.split(/[?#]/)[0] ?? ''; + const file = path.split('/').pop() ?? ''; + const dot = file.lastIndexOf('.'); + if (dot > 0 && dot < file.length - 1) return file.slice(dot + 1); + } + return undefined; +}; + +const extensionOf = (attachment: AIMessageAttachmentData) => + pathExtensionOf(attachment) ?? attachment.mimeType?.split('/')[1]?.split(/[+;]/)[0]; + +const ARCHIVE_EXTENSIONS = ['7z', 'bz2', 'gz', 'rar', 'tar', 'tgz', 'xz', 'zip']; +const ARCHIVE_SUBTYPES = [ + 'gzip', + 'vnd.rar', + 'x-7z-compressed', + 'x-bzip2', + 'x-rar-compressed', + 'x-tar', + 'zip', +]; +const AUDIO_EXTENSIONS = ['aac', 'flac', 'm4a', 'mp3', 'ogg', 'opus', 'wav']; +const IMAGE_EXTENSIONS = ['bmp', 'gif', 'heic', 'heif', 'jpeg', 'jpg', 'png', 'svg', 'webp']; +const VIDEO_EXTENSIONS = ['avi', 'm4v', 'mkv', 'mov', 'mp4', 'webm']; + +/** + * The icon of a file's type, from its media type, else from the extension of its name or URL, as + * in StreamChatAI and the Android AI SDK. + */ +const fileTypeIcon = ( + attachment: AIMessageAttachmentData, +): keyof Pick => { + const [type, subtype] = attachment.mimeType?.toLowerCase().split(';')[0]?.trim().split('/') ?? []; + const extension = pathExtensionOf(attachment)?.toLowerCase(); + const is = (wanted: string, extensions: string[]) => + type ? type === wanted : !!extension && extensions.includes(extension); + if (type ? subtype === 'pdf' : extension === 'pdf') return 'Pdf'; + if (is('image', IMAGE_EXTENSIONS)) return 'Image'; + if (is('audio', AUDIO_EXTENSIONS)) return 'Audio'; + if (is('video', VIDEO_EXTENSIONS)) return 'Video'; + if ( + type + ? !!subtype && ARCHIVE_SUBTYPES.includes(subtype) + : !!extension && ARCHIVE_EXTENSIONS.includes(extension) + ) { + return 'Archive'; + } + return !type && !extension && attachment.kind === 'image' ? 'Image' : 'File'; +}; + +const FILE_SIZE_UNITS = ['B', 'kB', 'MB', 'GB']; + +/** A size in bytes to three significant digits, such as "2.34 MB", as the chat SDK writes it. */ +export const formatFileSize = (bytes: number) => { + const exponent = + bytes <= 0 + ? 0 + : Math.min(Math.floor(Math.log(bytes) / Math.log(1024)), FILE_SIZE_UNITS.length - 1); + const mantissa = bytes / 1024 ** exponent; + // `toPrecision` turns exponential past three integer digits (1000–1023 of a unit). + const formatted = mantissa >= 1000 ? Math.round(mantissa).toString() : mantissa.toPrecision(3); + return `${formatted} ${FILE_SIZE_UNITS[exponent]}`; +}; + +/** Its type and size, such as "PDF · 2.34 MB". */ +export const attachmentDetails = (attachment: AIMessageAttachmentData) => + [ + extensionOf(attachment)?.toUpperCase(), + attachment.size !== undefined ? formatFileSize(attachment.size) : undefined, + ] + .filter(Boolean) + .join(' · '); + +// Only a web URL opens; anything else, such as a local file, a data URL or `javascript:`, doesn't. +const isWebUrl = (url: string | undefined): url is string => !!url && /^https?:\/\//i.test(url); + +const open = (url: string) => { + Linking.openURL(url).catch(() => undefined); +}; + +export type AIMessageAttachmentProps = { + attachment: AIMessageAttachmentData; + /** Who sent it. */ + role?: 'user' | 'assistant' | (string & {}); +}; + +/** + * An image or a file on a message: an image as a thumbnail, a file with its type's icon, name, type + * and size. A file at a web URL opens in the browser when tapped; an image doesn't open, as in + * StreamChatAI and the Android AI SDK. The counterpart of `AIMessageAttachmentView`. + */ +export const AIMessageAttachment = ({ + attachment, + role = 'assistant', +}: AIMessageAttachmentProps) => { + const { t } = useAITranslation(); + const { + theme: { colors, message }, + } = useTheme(); + const styles = message?.attachment; + const icons = useAIIcons(); + const { url } = attachment; + const name = + attachment.name || + (url && !url.startsWith('data:') ? url.split(/[?#]/)[0]?.split('/').pop() : undefined) || + (attachment.kind === 'image' ? t('Image') : t('File')); + + if (attachment.kind === 'image' && url) { + return ( + + + + ); + } + + const details = attachmentDetails(attachment); + const TypeIcon = icons[fileTypeIcon(attachment)]; + const content = ( + <> + + + + + + {name} + + {details ? ( + + {details} + + ) : null} + + + ); + const fileStyle = [ + defaultStyles.file, + { + backgroundColor: message?.attachmentBackground ?? colors.white, + borderColor: message?.attachmentBorder ?? colors.grey_gainsboro, + }, + styles?.file, + ]; + + return isWebUrl(url) ? ( + open(url)} + style={({ pressed }) => [fileStyle, { opacity: pressed ? 0.7 : 1 }]} + testID={`ai-message-attachment-${role}`} + > + {content} + + ) : ( + + {content} + + ); +}; + +const defaultStyles = StyleSheet.create({ + // The Android AI SDK's card: a 36 tile with a 20 icon, 8 padding, 10 apart, at most 280 wide. + file: { + alignItems: 'center', + borderRadius: 16, + borderWidth: 1, + flexDirection: 'row', + gap: 10, + maxWidth: 280, + padding: 8, + }, + fileDetails: { fontSize: 12 }, + fileIcon: { + alignItems: 'center', + borderRadius: 8, + height: 36, + justifyContent: 'center', + width: 36, + }, + fileName: { fontSize: 14, fontWeight: '500' }, + fileText: { flexShrink: 1 }, + image: { borderRadius: 16, height: 180, overflow: 'hidden', width: 180 }, +}); diff --git a/package/src/ai-components/message/AIReplyActions.tsx b/package/src/ai-components/message/AIReplyActions.tsx new file mode 100644 index 0000000000..6cdc1969dc --- /dev/null +++ b/package/src/ai-components/message/AIReplyActions.tsx @@ -0,0 +1,134 @@ +import React, { useEffect, useRef, useState } from 'react'; +import { Pressable, Share, StyleSheet, View } from 'react-native'; + +import { useAITranslation, useTheme } from '../contexts'; +import { type AIIcon, useAIIcons } from '../icons'; +import { setClipboardString } from '../services/clipboard-service'; + +/** How the person rated a reply. */ +export type AIMessageRating = 'positive' | 'negative'; + +export type AIReplyActionsProps = { + /** The reply's text, which copying and sharing hand on. */ + text: string; + /** Rates the reply, or takes the rating back with `null`. Without it, no rating is offered. */ + onRate?: (rating: AIMessageRating | null) => void; + /** Asks the agent for a new reply in its place. Without it, no regenerate button. */ + onRegenerate?: () => void; + /** How the person rated the reply, if they did. */ + rating?: AIMessageRating | null; +}; + +/** How long the copy button shows its check. */ +const COPIED_MS = 1500; + +type ActionProps = { + Icon: AIIcon; + label: string; + onPress: () => void; + selected?: boolean; +}; + +const Action = ({ Icon, label, onPress, selected }: ActionProps) => { + const { + theme: { colors, message }, + } = useTheme(); + const styles = message?.replyActions; + return ( + [ + defaultStyles.button, + { opacity: pressed ? 0.5 : 1 }, + styles?.button, + ]} + > + + + ); +}; + +/** + * The actions under a finished reply: copy (when the app has a clipboard module), share through + * the system's share sheet, regenerate, and a thumbs up or down rating. The counterpart of + * `AIReplyActionsView`. + */ +export const AIReplyActions = ({ onRate, onRegenerate, rating, text }: AIReplyActionsProps) => { + const { t } = useAITranslation(); + const { + theme: { message }, + } = useTheme(); + const icons = useAIIcons(); + const [isCopied, setIsCopied] = useState(false); + const timer = useRef>(undefined); + useEffect(() => () => clearTimeout(timer.current), []); + + const setString = setClipboardString; + const copy = setString + ? () => { + setString(text); + setIsCopied(true); + clearTimeout(timer.current); + timer.current = setTimeout(() => setIsCopied(false), COPIED_MS); + } + : undefined; + + // The person closing the share sheet resolves too; there's nothing to do either way. + const share = () => { + Share.share({ message: text }).catch(() => undefined); + }; + + const positive = rating === 'positive'; + const negative = rating === 'negative'; + + return ( + + {copy ? ( + + ) : null} + + {onRegenerate ? ( + + ) : null} + {onRate ? ( + <> + onRate(positive ? null : 'positive')} + selected={positive} + /> + onRate(negative ? null : 'negative')} + selected={negative} + /> + + ) : null} + + ); +}; + +const defaultStyles = StyleSheet.create({ + // StreamChatAI's 40 buttons, 8 apart; the Android AI SDK's 18 icons. + button: { + alignItems: 'center', + height: 40, + justifyContent: 'center', + width: 40, + }, + container: { alignItems: 'center', flexDirection: 'row', gap: 8 }, +}); diff --git a/package/src/ai-components/message/AIUserMessage.tsx b/package/src/ai-components/message/AIUserMessage.tsx new file mode 100644 index 0000000000..1443c44da2 --- /dev/null +++ b/package/src/ai-components/message/AIUserMessage.tsx @@ -0,0 +1,121 @@ +import React, { type ComponentType } from 'react'; +import { I18nManager, Pressable, StyleSheet, Text, View } from 'react-native'; + +import { + AIMessageAttachment, + type AIMessageAttachmentData, + type AIMessageAttachmentProps, +} from './AIMessageAttachment'; + +import { useAITranslation, useTheme } from '../contexts'; +import { useAIIcons } from '../icons'; + +export type AIUserMessageProps = { + /** What the person wrote. */ + text: string; + /** Shows each attachment, instead of `AIMessageAttachment`, such as for a kind of your own. */ + Attachment?: ComponentType; + /** What they sent with it, such as images and files. */ + attachments?: AIMessageAttachmentData[]; + /** Edits the message, such as by putting its text back in the composer. Without it, no edit button. */ + onEdit?: () => void; +}; + +/** + * The person's message: its attachments, then its text in a bubble on the trailing side. The + * counterpart of `AIUserMessageView`. + */ +export const AIUserMessage = ({ + Attachment = AIMessageAttachment, + attachments = [], + onEdit, + text, +}: AIUserMessageProps) => { + const { t } = useAITranslation(); + const { + theme: { colors, message }, + } = useTheme(); + const { Edit } = useAIIcons(); + const styles = message?.userMessage; + + return ( + + {attachments.length > 0 ? ( + + {attachments.map((attachment, index) => ( + + ))} + + ) : null} + {text !== '' ? ( + + {onEdit ? ( + [ + defaultStyles.editButton, + { opacity: pressed ? 0.5 : 1 }, + styles?.editButton, + ]} + > + + + ) : null} + + + {text} + + + + ) : null} + + ); +}; + +// StreamChatAI's and the Android AI SDK's bubble: the design tokens' default surface and text. +const USER_MESSAGE_BACKGROUND = '#EBEEF1'; +const USER_MESSAGE_TEXT = '#1A1B25'; + +const defaultStyles = StyleSheet.create({ + // One under the other, as in StreamChatAI and the Android AI SDK. + attachments: { alignItems: 'flex-end', gap: 8 }, + bubble: { + borderRadius: 20, + flexShrink: 1, + paddingHorizontal: 16, + paddingVertical: 10, + }, + container: { + alignItems: 'flex-end', + gap: 8, + paddingStart: 48, + }, + editButton: { padding: 4 }, + row: { + alignItems: 'flex-end', + flexDirection: 'row', + gap: 4, + maxWidth: '100%', + }, + text: { + fontSize: 16, + lineHeight: 22, + writingDirection: I18nManager.isRTL ? 'rtl' : 'ltr', + }, +}); diff --git a/package/src/ai-components/message/index.ts b/package/src/ai-components/message/index.ts new file mode 100644 index 0000000000..eb9d1e94e1 --- /dev/null +++ b/package/src/ai-components/message/index.ts @@ -0,0 +1,4 @@ +export * from './AIAssistantMessage'; +export * from './AIMessageAttachment'; +export * from './AIReplyActions'; +export * from './AIUserMessage'; diff --git a/package/src/ai-components/reasoning/AIReasoning.tsx b/package/src/ai-components/reasoning/AIReasoning.tsx index 9b4bae937a..5ec972bcaf 100644 --- a/package/src/ai-components/reasoning/AIReasoning.tsx +++ b/package/src/ai-components/reasoning/AIReasoning.tsx @@ -14,8 +14,8 @@ import { FINISHED_REASONING_SHOWN_MS, useAIThinkingTurn } from './thinkingClock' import type { ScrollViewRef } from '../../types/react-native-compat'; import { type DeepPartial, StreamTheme, useAITranslation, useTheme } from '../contexts'; +import { useAIIcons } from '../icons'; import { formatMinutesAndSeconds, useNow } from '../internal/format'; -import { ChevronIcon, ReasoningIcon } from '../internal/icons/AgentIcons'; import { Shimmer } from '../internal/Shimmer'; import { Markdown } from '../markdown'; import type { AIReasoningPart } from '../message-parts/types'; @@ -183,6 +183,7 @@ export const AIReasoning = ({ }: AIReasoningProps) => { const { t, userLanguage } = useAITranslation(); const { theme } = useTheme(); + const { Chevron, Reasoning } = useAIIcons(); const { colors, messageParts } = theme; const reasoningStyle = messageParts?.reasoning; const turn = useAIThinkingTurn(); @@ -309,7 +310,7 @@ export const AIReasoning = ({ style={[styles.header, reasoningStyle?.header]} > - + {title} @@ -327,7 +328,7 @@ export const AIReasoning = ({ transform: [{ rotate: expanded ? (I18nManager.isRTL ? '-90deg' : '90deg') : '0deg' }], }} > - + {expanded ? ( diff --git a/package/src/ai-components/services/clipboard-service/expo.ts b/package/src/ai-components/services/clipboard-service/expo.ts index 755027e699..0de43ec266 100644 --- a/package/src/ai-components/services/clipboard-service/expo.ts +++ b/package/src/ai-components/services/clipboard-service/expo.ts @@ -1,4 +1,9 @@ -let Clipboard: { setString: (string: string) => void } | undefined; +let Clipboard: + | { + setString?: (string: string) => void; + setStringAsync?: (string: string) => Promise; + } + | undefined; try { Clipboard = require('expo-clipboard'); @@ -6,6 +11,15 @@ try { // `expo-clipboard` is not installed. } -export const setClipboardString = Clipboard - ? (string: string) => Clipboard?.setString(string) - : undefined; +// `expo-clipboard` removed the synchronous `setString` in favour of `setStringAsync`; older +// versions only have `setString`. +export const setClipboardString = + Clipboard && (Clipboard.setStringAsync || Clipboard.setString) + ? (string: string) => { + if (Clipboard?.setStringAsync) { + Clipboard.setStringAsync(string).catch(() => undefined); + } else { + Clipboard?.setString?.(string); + } + } + : undefined; diff --git a/package/src/ai-components/theme.ts b/package/src/ai-components/theme.ts index af50497239..6d5c63ff5c 100644 --- a/package/src/ai-components/theme.ts +++ b/package/src/ai-components/theme.ts @@ -1,5 +1,6 @@ import type { ImageStyle, TextStyle, ViewStyle } from 'react-native'; +import type { AIIcons } from './icons'; import type { MarkdownStyle } from './markdown'; export const colors = { @@ -117,6 +118,53 @@ export type Theme = { text?: TextStyle; }; }; + /** + * Icons of your own for the agent replies' views, by name, over `defaultAIIcons`. Kept as they + * are when themes merge. + */ + icons?: Partial; + /** + * The person's and the agent's messages, what they carry, and the actions under a reply. + * Optional, so a hand-built `Theme` without it keeps type-checking. + */ + message?: { + /** The person's bubble. */ + userMessageBackground?: string; + userMessageText?: string; + /** The icons of the actions under a reply. */ + messageAction?: string; + /** A file's card, and the tile with its type's icon. */ + attachmentBackground?: string; + attachmentBorder?: string; + attachmentIcon?: string; + attachmentIconBackground?: string; + attachmentName?: string; + attachmentDetails?: string; + /** What an image shows while it loads, or when it can't. */ + attachmentImagePlaceholder?: string; + userMessage?: { + container?: ViewStyle; + attachments?: ViewStyle; + bubble?: ViewStyle; + text?: TextStyle; + editButton?: ViewStyle; + }; + assistantMessage?: { + container?: ViewStyle; + attachments?: ViewStyle; + }; + attachment?: { + image?: ImageStyle; + file?: ViewStyle; + fileIcon?: ViewStyle; + fileName?: TextStyle; + fileDetails?: TextStyle; + }; + replyActions?: { + container?: ViewStyle; + button?: ViewStyle; + }; + }; }; export const defaultTheme: Theme = { @@ -227,4 +275,21 @@ export const defaultTheme: Theme = { }, unsupported: { container: {}, text: {} }, }, + message: { + // StreamChatAI's and the Android AI SDK's defaults, from the shared design tokens. + userMessageBackground: '#EBEEF1', + userMessageText: '#1A1B25', + messageAction: '#414552', + attachmentBackground: '#FFFFFF', + attachmentBorder: '#D5DBE1', + attachmentIcon: '#FFFFFF', + attachmentIconBackground: '#005FFF', + attachmentName: '#1A1B25', + attachmentDetails: '#414552', + attachmentImagePlaceholder: '#F6F8FA', + userMessage: { container: {}, attachments: {}, bubble: {}, text: {}, editButton: {} }, + assistantMessage: { container: {}, attachments: {} }, + attachment: { image: {}, file: {}, fileIcon: {}, fileName: {}, fileDetails: {} }, + replyActions: { container: {}, button: {} }, + }, }; From 7aa684599f3d0ba102281ff70ead8094698ac25e Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 23:57:08 +0100 Subject: [PATCH 06/23] feat(ai-components): render AIConversation on LegendList Ported from #3929 on v9-release. stream-chat 10 keeps mime_type, file_size and the AI fields under `custom`, so steps and messages read them through withCustomFields, as React does. --- examples/ExpoChatGPTSample/agent/agentChat.ts | 93 ++++ examples/ExpoChatGPTSample/agent/photos.ts | 14 + .../VoiceMode/useVoiceConversation.ts | 22 +- .../ExpoChatGPTSample/contexts/AppContext.tsx | 177 ++++--- .../hooks/useSendAIMessage.ts | 30 +- .../ExpoChatGPTSample/screens/ChatContent.tsx | 258 +++------- package/bin/validate-entry-isolation.js | 6 + package/jest.config.js | 3 + package/package.json | 1 + .../ai-components/AITypingIndicatorView.tsx | 3 +- .../ai-components/StreamingMessageView.tsx | 11 + .../ai-components/__tests__/Markdown.test.tsx | 13 + .../__tests__/StreamingMessageView.test.tsx | 27 + .../__tests__/conversation.test.tsx | 486 ++++++++++++++++++ .../__tests__/conversationRows.test.ts | 92 ++++ .../__tests__/messageParts.test.ts | 16 + .../__tests__/messageViews.test.tsx | 28 + .../ai-components/__tests__/signIn.test.tsx | 34 +- .../conversation/AIConversation.tsx | 331 ++++++++++++ .../src/ai-components/conversation/entries.ts | 133 +++++ .../conversation/fromStreamMessage.ts | 87 ++++ .../src/ai-components/conversation/index.ts | 4 + .../conversation/list/AIChatList.tsx | 176 +++++++ .../list/useChatListController.ts | 72 +++ .../conversation/list/useEndInView.ts | 46 ++ .../conversation/list/useLoadOlder.ts | 97 ++++ .../conversation/list/usePinnedAnchor.ts | 133 +++++ .../src/ai-components/conversation/types.ts | 45 ++ .../conversation/useConversationRows.ts | 169 ++++++ package/src/ai-components/icons.ts | 3 + package/src/ai-components/index.ts | 1 + .../ai-components/internal/customFields.ts | 11 + .../internal/icons/AgentIcons.tsx | 3 + package/src/ai-components/internal/tokens.ts | 4 + .../markdown/components/InlineCode.tsx | 1 + .../markdown/components/Text.tsx | 9 +- package/src/ai-components/markdown/styles.ts | 47 +- package/src/ai-components/markdown/types.ts | 1 + .../message-composer/ComposerView.tsx | 3 +- .../message-parts/AIMessageParts.tsx | 3 +- .../message-parts/AIToolApproval.tsx | 16 +- .../message-parts/AIToolAuthorization.tsx | 31 +- .../message-parts/AIToolCall.tsx | 17 +- .../message-parts/parseMessageParts.ts | 13 +- .../message/AIAssistantMessage.tsx | 9 +- .../message/AIMessageAttachment.tsx | 5 +- .../ai-components/message/AIUserMessage.tsx | 5 +- .../ai-components/reasoning/AIReasoning.tsx | 15 +- package/src/ai-components/theme.ts | 34 ++ .../tool-calls/AIToolApprover.tsx | 8 +- .../tool-calls/authSessionAuthorizer.ts | 17 +- yarn.lock | 33 ++ 52 files changed, 2545 insertions(+), 351 deletions(-) create mode 100644 examples/ExpoChatGPTSample/agent/agentChat.ts create mode 100644 examples/ExpoChatGPTSample/agent/photos.ts create mode 100644 package/src/ai-components/__tests__/conversation.test.tsx create mode 100644 package/src/ai-components/__tests__/conversationRows.test.ts create mode 100644 package/src/ai-components/conversation/AIConversation.tsx create mode 100644 package/src/ai-components/conversation/entries.ts create mode 100644 package/src/ai-components/conversation/fromStreamMessage.ts create mode 100644 package/src/ai-components/conversation/index.ts create mode 100644 package/src/ai-components/conversation/list/AIChatList.tsx create mode 100644 package/src/ai-components/conversation/list/useChatListController.ts create mode 100644 package/src/ai-components/conversation/list/useEndInView.ts create mode 100644 package/src/ai-components/conversation/list/useLoadOlder.ts create mode 100644 package/src/ai-components/conversation/list/usePinnedAnchor.ts create mode 100644 package/src/ai-components/conversation/types.ts create mode 100644 package/src/ai-components/conversation/useConversationRows.ts create mode 100644 package/src/ai-components/internal/customFields.ts create mode 100644 package/src/ai-components/internal/tokens.ts diff --git a/examples/ExpoChatGPTSample/agent/agentChat.ts b/examples/ExpoChatGPTSample/agent/agentChat.ts new file mode 100644 index 0000000000..0515cf4235 --- /dev/null +++ b/examples/ExpoChatGPTSample/agent/agentChat.ts @@ -0,0 +1,93 @@ +import { LiveReasoning, type Session } from '@stream-io/calypso'; +import { type Channel, type Event, type LocalMessage, StateStore } from 'stream-chat'; + +/** How many messages are read at a time. */ +const PAGE_SIZE = 30; + +export type AgentChatState = { + /** The channel's messages, oldest first. */ + messages: readonly LocalMessage[]; + /** Whether the channel holds messages older than those loaded. */ + hasOlderMessages: boolean; + /** Whether the first page of messages is in. */ + isLoaded: boolean; +}; + +/** + * A written conversation as the AI components want it: the messages of the channel the router + * keeps it in, and the whole of each reply's thinking. + * + * The router writes everything there. Asking goes through the session, which records the question + * in the channel and answers it there: the reply's text, and its steps (each round of reasoning and + * each tool call) as attachments. + */ +export class AgentChat { + /** Each reply's thinking in full, from its live updates. A stored step only keeps its opening. */ + readonly reasoning = new LiveReasoning(); + + readonly state = new StateStore({ + hasOlderMessages: false, + isLoaded: false, + messages: [], + }); + + private stops: (() => void)[] = []; + + constructor( + readonly session: Session, + readonly channel: Channel, + ) {} + + /** Watches the channel and loads its latest messages. */ + async follow() { + // Every event rather than the list kept for us: each live update carries its own window of the + // reply's thinking, and folding them into one would lose all but the last. + const subscription = this.channel.on((event: Event) => { + if (event.type === 'message.updated' && event.message) { + this.reasoning.read(event.message as { id: string; reasoning?: unknown }); + } + if ( + event.type === 'message.new' || + event.type === 'message.updated' || + event.type === 'message.deleted' + ) { + // A new array, in which only the messages that changed are new objects. + this.state.partialNext({ messages: [...this.channel.state.messages] }); + } + }); + this.stops.push(() => subscription.unsubscribe()); + + const watched = await this.channel.watch({ messages: { limit: PAGE_SIZE } }); + this.state.partialNext({ + hasOlderMessages: (watched.messages?.length ?? 0) >= PAGE_SIZE, + isLoaded: true, + messages: [...this.channel.state.messages], + }); + } + + /** Loads the messages before the oldest one loaded. Throws when it can't, to be asked again. */ + loadOlderMessages = async () => { + const oldest = this.state.getLatestValue().messages[0]?.id; + if (!oldest) return; + const page = await this.channel.query({ messages: { id_lt: oldest, limit: PAGE_SIZE } }); + this.state.partialNext({ + hasOlderMessages: page.messages.length >= PAGE_SIZE, + messages: [...this.channel.state.messages], + }); + }; + + /** Stops following the channel. */ + close() { + for (const stop of this.stops.splice(0)) stop(); + } +} + +/** A reply the agent wrote. */ +export const isAgentReply = (message: LocalMessage) => message.ai_generated === true; + +/** A reply with nothing to show yet: no text and no steps. */ +export const isEmptyReply = (message: LocalMessage) => + !message.text && + !(message.attachments ?? []).some( + (attachment) => attachment.type === 'ai_reasoning' || attachment.type === 'ai_tool_call', + ); diff --git a/examples/ExpoChatGPTSample/agent/photos.ts b/examples/ExpoChatGPTSample/agent/photos.ts new file mode 100644 index 0000000000..3dc3d8d0bf --- /dev/null +++ b/examples/ExpoChatGPTSample/agent/photos.ts @@ -0,0 +1,14 @@ +import { manipulateAsync, SaveFormat } from 'expo-image-manipulator'; + +// The router takes photos inline, so they are shrunk first. +const MAX_IMAGE_WIDTH = 1280; + +/** A photo the person picked, as the data URI the router takes. */ +export const toDataUri = async (uri: string) => { + const { base64 } = await manipulateAsync(uri, [{ resize: { width: MAX_IMAGE_WIDTH } }], { + base64: true, + compress: 0.7, + format: SaveFormat.JPEG, + }); + return `data:image/jpeg;base64,${base64}`; +}; diff --git a/examples/ExpoChatGPTSample/components/VoiceMode/useVoiceConversation.ts b/examples/ExpoChatGPTSample/components/VoiceMode/useVoiceConversation.ts index d635f1e7d0..d24052bbcb 100644 --- a/examples/ExpoChatGPTSample/components/VoiceMode/useVoiceConversation.ts +++ b/examples/ExpoChatGPTSample/components/VoiceMode/useVoiceConversation.ts @@ -2,7 +2,7 @@ import { useEffect, useRef, useState } from 'react'; import * as Speech from 'expo-speech'; -import { AIStates, useAIState, useChannelContext, useStableCallback } from 'stream-chat-expo'; +import { useStableCallback } from 'stream-chat-expo'; import { dictationStore, @@ -12,13 +12,12 @@ import { type VoiceAgentState, } from 'stream-chat-expo/ai-components'; +import { useAppContext } from '../../contexts/AppContext'; import { useSendAIMessage } from '../../hooks/useSendAIMessage'; /** A turn the agent is never seen working on ends after this long. */ const TURN_TIMEOUT_MS = 20_000; -const WORKING_STATES: string[] = [AIStates.Thinking, AIStates.Generating, AIStates.ExternalSources]; - // Marks voice mode's dictation as its own, so the composer underneath ignores it. const DICTATION_SOURCE = { source: 'voice-mode' }; @@ -45,8 +44,9 @@ export const useVoiceConversation = (active: boolean) => { const { state: dictationState, error } = useDictationState(); const { transcript } = useDictationTranscript(); const send = useSendAIMessage(); - const { channel } = useChannelContext(); - const { aiState } = useAIState(channel); + const { agentState, chat } = useAppContext(); + // The agent answering, from the session's socket. + const isWorking = agentState.kind === 'responding' || agentState.kind === 'working'; // The messages already in the channel when the question was sent, so the reply is found by // identity rather than by comparing the server's timestamps with the device's clock. @@ -65,11 +65,11 @@ export const useVoiceConversation = (active: boolean) => { const latestReply = useStableCallback(() => { const seenIds = turn.current?.seenIds ?? new Set(); - const messages = channel?.messagePaginator.items ?? []; + const messages = chat?.state.getLatestValue().messages ?? []; for (let i = messages.length - 1; i >= 0; i--) { const message = messages[i]; if (seenIds.has(message.id)) break; - if (message.custom?.ai_generated && message.text) return message.text; + if (message.ai_generated && message.text) return message.text; } return null; }); @@ -129,25 +129,25 @@ export const useVoiceConversation = (active: boolean) => { } turn.current = { sawWorking: false, - seenIds: new Set((channel?.messagePaginator.items ?? []).map((message) => message.id)), + seenIds: new Set((chat?.state.getLatestValue().messages ?? []).map((message) => message.id)), }; setPhase('thinking'); void send({ text }); timeout.current = setTimeout(() => { if (!turn.current?.sawWorking) finishTurn(); }, TURN_TIMEOUT_MS); - }, [active, channel, dictationState, error, finishTurn, listen, phase, send, transcript]); + }, [active, chat, dictationState, error, finishTurn, listen, phase, send, transcript]); // Follows the agent: its turn ends once it has been seen working and is done. useEffect(() => { const current = turn.current; if (!active || !current) return; - if (WORKING_STATES.includes(aiState)) { + if (isWorking) { current.sawWorking = true; } else if (current.sawWorking) { finishTurn(); } - }, [active, aiState, finishTurn]); + }, [active, isWorking, finishTurn]); return { // Only an error from the current attempt, not one left over while dictation restarts. diff --git a/examples/ExpoChatGPTSample/contexts/AppContext.tsx b/examples/ExpoChatGPTSample/contexts/AppContext.tsx index 78addabde3..f08cf43639 100644 --- a/examples/ExpoChatGPTSample/contexts/AppContext.tsx +++ b/examples/ExpoChatGPTSample/contexts/AppContext.tsx @@ -1,25 +1,21 @@ import type { PropsWithChildren } from 'react'; import React, { useEffect, useMemo, useRef, useState } from 'react'; -import { - type ConversationSnapshot, - type ImageSource, - LiveReasoning, - type Session, -} from '@stream-io/calypso'; -import type { Channel, Event, StreamChat } from 'stream-chat'; -import { useStableCallback } from 'stream-chat-expo'; +import { type ConversationSnapshot, type Session } from '@stream-io/calypso'; +import type { Channel, LocalMessage, StreamChat } from 'stream-chat'; +import { useStableCallback, useStateStore } from 'stream-chat-expo'; import { AIClientIdentity, type AIMessageRating, + type AIPendingMessage, type AIToolApprover, createAuthSessionAuthorizer, } from 'stream-chat-expo/ai-components'; +import { AgentChat, type AgentChatState, isAgentReply, isEmptyReply } from '../agent/agentChat'; import { agents, agentSessions, streamToken } from '../agent/agents'; import { AgentConfig } from '../agent/config'; - -type Conversation = { channel: Channel; reasoning: LiveReasoning; session: Session }; +import { toDataUri } from '../agent/photos'; const IDLE: ConversationSnapshot = { kind: 'idle', skills: [] }; @@ -38,31 +34,37 @@ const declineLogin = (session: Session, toolCallId: string) => .held() .send({ allowed: false, tool_call_id: toolCallId, type: 'tool_approval' }); +// A question on its way: what shows until Stream's copy arrives, and how many of the person's +// messages there were when it was asked, so its copy is the first one after them. +type Asked = { message: AIPendingMessage; after: number }; + export type AppContextValue = { /** What the agent is doing in the open conversation, from the session's socket. */ agentState: ConversationSnapshot; /** Answers the open conversation's tool calls that ask first. */ approver: AIToolApprover | undefined; - /** The channel the router writes the open conversation into. */ - channel: Channel | undefined; + /** The open conversation's messages and live reasoning. */ + chat: AgentChat | undefined; /** Bumped when the drawer's list of conversations may have changed. */ conversationsVersion: number; /** Why the last conversation couldn't be opened, or what asking failed with. */ failure: string | undefined; + /** Whether a question is on its way or being answered, from the first thought to the last word. */ + isBusy: boolean; + /** Whether the agent is answering and its reply shows nothing yet. */ + isThinking: boolean; + /** The question on its way, until Stream's copy of it arrives. */ + pending: AIPendingMessage | undefined; /** The open conversation's session id. */ sessionId: string | undefined; - /** Asks the agent, with photos as data URIs. Its reply arrives in the channel. */ - ask: (text: string, images?: ImageSource[]) => Promise; + /** Asks the agent, with photos by their local URIs. Its reply arrives in the channel. */ + ask: (text: string, photos?: string[]) => Promise; /** Carries on a conversation from the drawer, by its session id. */ openConversation: (sessionId: string) => void; /** Rates a reply, or takes the rating back with null. */ rate: (messageId: string, rating: AIMessageRating | null) => void; /** How the person rated replies, by message. Kept for this run only: the router takes no ratings yet. */ ratings: Readonly>; - /** The whole of a reply's reasoning step so far, from its live updates. */ - reasoningText: (messageId: string, stepId: string) => string | undefined; - /** Bumped as reasoning arrives, so the steps showing it render again. */ - reasoningVersion: number; /** Starts a new conversation. */ startConversation: () => void; /** Stops the reply being written. What it wrote so far stays. */ @@ -73,14 +75,15 @@ export const AppContext = React.createContext({ agentState: IDLE, approver: undefined, ask: async () => {}, - channel: undefined, + chat: undefined, conversationsVersion: 0, failure: undefined, + isBusy: false, + isThinking: false, openConversation: () => {}, + pending: undefined, rate: () => {}, ratings: {}, - reasoningText: () => undefined, - reasoningVersion: 0, sessionId: undefined, startConversation: () => {}, stop: () => {}, @@ -88,14 +91,31 @@ export const AppContext = React.createContext({ const messageOf = (error: unknown) => (error instanceof Error ? error.message : String(error)); +const userMessageCount = (messages: readonly LocalMessage[]) => + messages.filter((message) => !isAgentReply(message)).length; + +// What the provider follows of the messages: what decides whether the agent is thinking or busy, +// and when a question's copy has arrived. +const selector = ({ messages }: AgentChatState) => ({ + last: messages[messages.length - 1] as LocalMessage | undefined, + lastReply: [...messages].reverse().find(isAgentReply), + userMessages: userMessageCount(messages), +}); + export const AppProvider = ({ client, children }: PropsWithChildren<{ client: StreamChat }>) => { - const [conversation, setConversation] = useState(); + const [chat, setChat] = useState(); const [failure, setFailure] = useState(); const [agentState, setAgentState] = useState(IDLE); - const [reasoningVersion, setReasoningVersion] = useState(0); const [conversationsVersion, setConversationsVersion] = useState(0); const [ratings, setRatings] = useState>>({}); - const current = useRef(undefined); + const [asked, setAsked] = useState(); + // The newest message when something was last asked here, until it fails or is stopped. + const [askedAfter, setAskedAfter] = useState(); + const current = useRef(undefined); + const askedRef = useRef(undefined); + askedRef.current = asked; + + const followed = useStateStore(chat?.state, selector); // Opening one conversation after another: the newest wins, and one that was never asked // anything is deleted rather than left in the drawer. @@ -104,12 +124,16 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St try { const session = await opening(); const { channel } = await session.chat(); + const next = new AgentChat(session, channel as Channel); + await next.follow(); const previous = current.current; - const next = { channel: channel as Channel, reasoning: new LiveReasoning(), session }; current.current = next; - setConversation(next); + setAsked(undefined); + setAskedAfter(undefined); + setChat(next); if (previous && previous.session.id !== session.id) { - const unused = previous.channel.state.messages.length === 0; + previous.close(); + const unused = previous.state.getLatestValue().messages.length === 0; await (unused ? previous.session.delete() : previous.session.close()).catch(() => {}); } setConversationsVersion((version) => version + 1); @@ -134,20 +158,47 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St void open(() => agentSessions().resume(sessionId)); }); - const ask = useStableCallback(async (text: string, images: ImageSource[] = []) => { - const session = current.current?.session; - if (!session) { + const isResponding = agentState.kind === 'responding' || agentState.kind === 'working'; + const isBusy = !!asked || isResponding || followed?.lastReply?.generating === true; + + const ask = useStableCallback(async (text: string, photos: string[] = []) => { + const opened = current.current; + if (!opened || isBusy) { return; } + // Shows at once, with the photos as they were picked. + const asking: Asked = { + after: userMessageCount(opened.state.getLatestValue().messages), + message: { + attachments: photos.map((url) => ({ kind: 'image', url })), + id: crypto.randomUUID(), + text, + }, + }; + const messages = opened.state.getLatestValue().messages; setFailure(undefined); + setAsked(asking); + setAskedAfter(messages[messages.length - 1]?.id ?? ''); try { - await session.responses.create(text, images.length > 0 ? { images } : undefined); + const images = await Promise.all(photos.map(async (uri) => ({ url: await toDataUri(uri) }))); + await opened.session.responses.create(text, images.length > 0 ? { images } : undefined); } catch (error) { setFailure(messageOf(error)); + setAskedAfter(undefined); + if (askedRef.current === asking) setAsked(undefined); } }); + // Stream's copy of the question on its way is the person's first message after the ones there + // were when they asked: once it is in, the question on its way goes. + useEffect(() => { + if (asked && followed && followed.userMessages > asked.after) { + setAsked(undefined); + } + }, [asked, followed]); + const stop = useStableCallback(() => { + setAskedAfter(undefined); try { current.current?.session.interrupt(); } catch { @@ -167,45 +218,46 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St }), ); - const reasoningText = useStableCallback((messageId: string, stepId: string) => - current.current?.reasoning.get(messageId, stepId), - ); + // Whether the agent is answering and its reply shows nothing yet. The session can pass through + // idle between the question and the first of its reply, which would hide it for a moment. + const last = followed?.last; + const replyHasBegun = !!last && isAgentReply(last) && !isEmptyReply(last); + const isAwaitingReply = + askedAfter !== undefined && + !!last && + last.id !== askedAfter && + (!isAgentReply(last) || (last.generating === true && !replyHasBegun)); + const isThinking = + agentState.kind === 'responding' + ? !replyHasBegun + : agentState.kind === 'working' + ? false + : isAwaitingReply; - // The open conversation: what the agent is doing, and every live update's slice of thinking, - // read from each event rather than the message list, which can fold several into one. + // The open conversation: what the agent is doing, from the session's socket. useEffect(() => { - if (!conversation) { + if (!chat) { return; } - const { channel, reasoning, session } = conversation; + const { session } = chat; setAgentState(session.state.get()); - const stopState = session.state.subscribe(() => { + return session.state.subscribe(() => { const state = session.state.get(); setAgentState(state); // The router titles a conversation as it goes. if (state.kind === 'idle') setConversationsVersion((version) => version + 1); }); - const subscription = channel.on('message.updated', (event: Event) => { - if (event.message) { - reasoning.read(event.message as { id: string; reasoning?: unknown }); - setReasoningVersion((version) => version + 1); - } - }); - return () => { - stopState(); - subscription.unsubscribe(); - }; - }, [conversation]); + }, [chat]); const approver = useMemo( () => - conversation + chat ? { authorizer: createAuthSessionAuthorizer({ callbackScheme: SIGN_IN_SCHEME, decline: (call) => new Promise((resolve) => { - declineLogin(conversation.session, call.id); + declineLogin(chat.session, call.id); resolve(); }), }), @@ -213,13 +265,13 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St // `decide` sends over the session's socket and throws when it can't. decide: (call, allowed) => new Promise((resolve) => { - conversation.session.decide(call.id, { allowed }); + chat.session.decide(call.id, { allowed }); resolve(); }), userId: AgentConfig.user.id, } : undefined, - [conversation], + [chat], ); useEffect(() => { @@ -241,20 +293,22 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St }; }, [client, startConversation]); + const pending = asked?.message; const contextValue = useMemo( () => ({ agentState, approver, ask, - channel: conversation?.channel, + chat, conversationsVersion, failure, + isBusy, + isThinking, openConversation, + pending, rate, ratings, - reasoningText, - reasoningVersion, - sessionId: conversation?.session.id, + sessionId: chat?.session.id, startConversation, stop, }), @@ -262,14 +316,15 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St agentState, approver, ask, - conversation, + chat, conversationsVersion, failure, + isBusy, + isThinking, openConversation, + pending, rate, ratings, - reasoningText, - reasoningVersion, startConversation, stop, ], diff --git a/examples/ExpoChatGPTSample/hooks/useSendAIMessage.ts b/examples/ExpoChatGPTSample/hooks/useSendAIMessage.ts index efd6c8de9b..368441f67d 100644 --- a/examples/ExpoChatGPTSample/hooks/useSendAIMessage.ts +++ b/examples/ExpoChatGPTSample/hooks/useSendAIMessage.ts @@ -1,5 +1,3 @@ -import { manipulateAsync, SaveFormat } from 'expo-image-manipulator'; - import { useStableCallback } from 'stream-chat-expo'; import type { ComposerViewProps } from 'stream-chat-expo/ai-components'; @@ -8,32 +6,18 @@ import { useAppContext } from '../contexts/AppContext'; export type AIMessage = Parameters[0]; -// The router takes photos inline, so they are shrunk first. -const MAX_IMAGE_WIDTH = 1280; - -const toDataUri = async (uri: string) => { - const { base64 } = await manipulateAsync(uri, [{ resize: { width: MAX_IMAGE_WIDTH } }], { - base64: true, - compress: 0.7, - format: SaveFormat.JPEG, - }); - return `data:image/jpeg;base64,${base64}`; -}; - /** - * Asks the agent, with the photos picked in the composer. The router writes the question and its - * reply into the conversation's channel. Shared by the composer and voice mode. + * Asks the agent, with the photos picked in the composer. The question shows at once, and the + * router writes it and its reply into the conversation's channel. Shared by the composer and voice + * mode. */ export const useSendAIMessage = () => { const { ask } = useAppContext(); return useStableCallback(async ({ text, attachments }: AIMessage) => { - const photos = (attachments ?? []).filter( - (asset) => !asset.type || asset.type.startsWith('image'), - ); - const images = await Promise.all( - photos.map(async (asset) => ({ url: await toDataUri(asset.uri) })), - ); - await ask(text, images); + const photos = (attachments ?? []) + .filter((asset) => !asset.type || asset.type.startsWith('image')) + .map((asset) => asset.uri); + await ask(text, photos); }); }; diff --git a/examples/ExpoChatGPTSample/screens/ChatContent.tsx b/examples/ExpoChatGPTSample/screens/ChatContent.tsx index e5eb2e9118..65aec8a102 100644 --- a/examples/ExpoChatGPTSample/screens/ChatContent.tsx +++ b/examples/ExpoChatGPTSample/screens/ChatContent.tsx @@ -1,143 +1,32 @@ -import { useMemo } from 'react'; -import { - ActivityIndicator, - Platform, - StatusBar, - StyleSheet, - Text, - useWindowDimensions, - View, -} from 'react-native'; +import { useCallback, useMemo, useState } from 'react'; +import { ActivityIndicator, Platform, StatusBar, StyleSheet, Text, View } from 'react-native'; import Animated, { FadeIn, FadeOut } from 'react-native-reanimated'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { useHeaderHeight } from 'expo-router/react-navigation'; -import type { Attachment, LocalMessage } from 'stream-chat'; -import type { MessageProps } from 'stream-chat-expo'; -import { - Channel, - isLocalUrl, - MessageList, - usePaginatedMessageListContext, - WithComponents, -} from 'stream-chat-expo'; +import { KeyboardCompatibleView, useStateStore } from 'stream-chat-expo'; import { - AIAssistantMessage, - type AIMessageAttachmentData, - AIReplyActions, - AIThinking, - AIToolApproverProvider, + AIConversation, + type AIMessage, AITypingIndicatorView, - AIUserMessage, ComposerView, - parseMessageParts, + createFromStreamMessage, } from 'stream-chat-expo/ai-components'; +import type { AgentChat, AgentChatState } from '../agent/agentChat'; import { bottomSheetOptions } from '../bottomSheetOptions'; import { VoiceMode } from '../components/VoiceMode'; import { useAppContext } from '../contexts/AppContext'; import { useSendAIMessage } from '../hooks/useSendAIMessage'; -// Images and files only: a reply's steps (`ai_*`) are read as its parts, and other kinds, such as -// the router's `connector_authorization`, a second copy of a sign-in its tool call's step asks for, -// aren't shown. -const attachmentOf = (attachment: Attachment): AIMessageAttachmentData | undefined => { - const name = attachment.title ?? attachment.fallback ?? undefined; - if (attachment.type === 'image') { - const url = attachment.image_url ?? attachment.asset_url ?? attachment.thumb_url; - return url ? { kind: 'image', mimeType: attachment.mime_type, name, url } : undefined; - } - if (attachment.type === 'file') { - const size = Number(attachment.file_size); - return attachment.asset_url - ? { - kind: 'file', - mimeType: attachment.mime_type, - name, - size: Number.isFinite(size) && size > 0 ? size : undefined, - url: attachment.asset_url, - } - : undefined; - } - return undefined; -}; - -const attachmentsOf = (attachments: Attachment[] | undefined) => - (attachments ?? []) - .map(attachmentOf) - .filter((attachment): attachment is AIMessageAttachmentData => !!attachment); - -// The person's message, dimmed while its photos upload. -const UserMessage = ({ message }: { message: LocalMessage }) => { - const attachments = useMemo(() => attachmentsOf(message.attachments), [message.attachments]); - const isUploading = useMemo( - () => - (message.attachments ?? []).some( - (attachment) => - (attachment.image_url && isLocalUrl(attachment.image_url)) || - (attachment.asset_url && isLocalUrl(attachment.asset_url)), - ), - [message.attachments], - ); - return ( - - - - ); -}; - -// A reply: its steps, its answer and what it sent, then its actions once it's written. -const AssistantMessage = ({ message: listed }: { message: LocalMessage }) => { - // Read so the reply renders again as each live update arrives. - const { channel, rate, ratings, reasoningText, reasoningVersion } = useAppContext(); - // The list passes a row its message again only when it changes in ways it compares, which a - // step changing on its own, such as a tool call starting to wait for a sign-in, isn't. The - // channel's state has every update. - const message = channel?.state.findMessage(listed.id) ?? listed; - const parts = useMemo(() => parseMessageParts(message.attachments), [message.attachments]); - const attachments = useMemo(() => attachmentsOf(message.attachments), [message.attachments]); - const stepText = useMemo( - () => (stepId: string) => reasoningText(message.id, stepId), - // eslint-disable-next-line react-hooks/exhaustive-deps -- a new function per slice of reasoning - [message.id, reasoningText, reasoningVersion], - ); - const isGenerating = message.generating === true; - const text = message.text ?? ''; - // The list sizes a row to its content; a reply takes the row's whole width. - const { width } = useWindowDimensions(); - return ( - - - {!isGenerating && text ? ( - rate(message.id, rating)} - rating={ratings[message.id] ?? null} - text={text} - /> - ) : null} - - ); -}; - -const CustomMessage = ({ message }: MessageProps) => - message.ai_generated ? : ; - const CustomComposerView = () => { - const { agentState, stop } = useAppContext(); + const { isBusy, stop } = useAppContext(); // The router stops through its own session, not with an AI indicator event. const stopGenerating = useMemo(() => () => Promise.resolve(stop()), [stop]); - const isGenerating = agentState.kind === 'responding' || agentState.kind === 'working'; - const safeAreaInsets = useSafeAreaInsets(); const insets = useMemo( () => ({ @@ -156,62 +45,73 @@ const CustomComposerView = () => { bottomSheetOptions={bottomSheetOptions} bottomSheetInsets={insets} onSendMessage={serializeToMessage} - isGenerating={isGenerating} + isGenerating={isBusy} stopGenerating={stopGenerating} /> ); }; -const EmptyStateIndicator = () => ( - - What can I help with ? - -); +const EmptyState = () => What can I help with ?; -// What the agent is doing while its reply shows nothing yet: thinking, or running skills. -const CustomAITypingIndicatorView = () => { - const { agentState } = useAppContext(); - const { messages } = usePaginatedMessageListContext(); - const latest = messages[messages.length - 1]; - const replyHasBegun = - !!latest?.ai_generated && (!!latest.text || parseMessageParts(latest.attachments).length > 0); +const conversationSelector = ({ hasOlderMessages, messages }: AgentChatState) => ({ + hasOlderMessages, + messages, +}); - if (agentState.kind === 'working') { - const skills = agentState.skills.map((skill) => skill.replace(/_/g, ' ')).join(' and '); - return ( - - - - ); - } - if (agentState.kind === 'responding' && !replyHasBegun) { - return ( - - - - ); - } - return null; -}; +/** The open conversation, read from its channel by `AgentChat`. */ +const Conversation = ({ chat }: { chat: AgentChat }) => { + const { agentState, approver, isThinking, pending, rate, ratings } = useAppContext(); + const { hasOlderMessages, messages } = useStateStore(chat.state, conversationSelector); -const RenderNull = () => null; + // The same message object for each Stream message that didn't change, so only the messages an + // update touched render again. The live thinking is read through `chat.reasoning`. + const [fromStreamMessage] = useState(() => + createFromStreamMessage({ + reasoningText: (messageId, stepId) => chat.reasoning.get(messageId, stepId), + }), + ); + const conversation = useMemo( + () => messages.map(fromStreamMessage), + [fromStreamMessage, messages], + ); -const additionalFlatListProps = { - maintainVisibleContentPosition: { - minIndexForVisible: 0, - autoscrollToTopThreshold: 0, - }, - ListHeaderComponent: undefined, + const replyActions = useCallback( + (message: AIMessage) => ({ + onRate: (rating: Parameters[1]) => rate(message.id, rating), + rating: ratings[message.id] ?? null, + }), + [rate, ratings], + ); + + // What the agent is doing while it runs skills, under the latest turn. + const footer = + agentState.kind === 'working' ? ( + skill.replace(/_/g, ' ')).join(' and ')}`} + /> + ) : undefined; + + return ( + } + footer={footer} + isThinking={isThinking} + loadOlderMessages={hasOlderMessages ? chat.loadOlderMessages : undefined} + messages={conversation} + pendingMessage={pending} + replyActions={replyActions} + /> + ); }; export const ChatContent = () => { - const { approver, channel, failure } = useAppContext(); + const { chat, failure } = useAppContext(); const { bottom } = useSafeAreaInsets(); - // The channel sits below the drawer header, so the keyboard offset has to account for it - // (without it, Android keeps the composer behind the keyboard). + // The conversation sits below the drawer header, so the keyboard offset has to account for it. const headerHeight = useHeaderHeight(); - if (!channel) { + if (!chat) { return ( {failure ? {failure} : } @@ -221,38 +121,24 @@ export const ChatContent = () => { return ( - - - - - - {failure ? {failure} : null} - - - - - + + + {failure ? {failure} : null} + + + ); }; const styles = StyleSheet.create({ wrapper: { flex: 1, backgroundColor: '#fcfcfc' }, + flex: { flex: 1 }, emptyContainer: { flex: 1, width: '100%', @@ -260,12 +146,6 @@ const styles = StyleSheet.create({ justifyContent: 'center', alignItems: 'center', }, - emptyContainerText: { fontSize: 24, fontWeight: 'bold' }, + emptyText: { fontSize: 24, fontWeight: 'bold' }, failureText: { color: '#B00020', fontSize: 15, paddingHorizontal: 24, textAlign: 'center' }, - assistantMessage: { gap: 4 }, - message: { paddingHorizontal: 16, paddingVertical: 8 }, - aiTypingIndicatorWrapper: { - paddingHorizontal: 24, - paddingVertical: 12, - }, }); diff --git a/package/bin/validate-entry-isolation.js b/package/bin/validate-entry-isolation.js index 480df61224..ff19b10468 100644 --- a/package/bin/validate-entry-isolation.js +++ b/package/bin/validate-entry-isolation.js @@ -29,6 +29,7 @@ const WRAPPERS = ['native-package', 'expo-package'].map((name) => path.join(PACK // not among them: the main entry point imports both too.) const AI_ONLY_PACKAGES = [ '@khanacademy/simple-markdown', + '@legendapp/list', '@shopify/react-native-skia', 'react-syntax-highlighter', 'victory-native', @@ -44,6 +45,11 @@ const AI_ALLOWED_FILES = [ path.join(CORE_SRC, 'contexts/translationContext/types.ts'), path.join(CORE_SRC, 'contexts/utils/isTestEnvironment.ts'), path.join(CORE_SRC, 'i18n/utils.ts'), + // The design tokens (plain data, no imports), which the AI components size their text with. Metro + // picks the platform's file; this check resolves the fallback, which re-exports the web one. + ...['', '.ios', '.android', '.web'].map((platform) => + path.join(CORE_SRC, `theme/generated/light/StreamTokens${platform}.ts`), + ), ...WRAPPERS.flatMap((wrapper) => [ path.join(wrapper, 'ai-components/index.js'), path.join(wrapper, 'src/optionalDependencies/NativeShimmerView.ts'), diff --git a/package/jest.config.js b/package/jest.config.js index 1ac5fcc627..bc708eeee4 100644 --- a/package/jest.config.js +++ b/package/jest.config.js @@ -5,6 +5,9 @@ module.exports = { // cap worker count on CI ...(process.env.CI ? { maxWorkers: 2 } : {}), moduleNameMapper: { + // LegendList's exports map has only a `react-native` condition for its native build, which + // Jest's node conditions skip for the web build (react-dom). + '^@legendapp/list/(react-native|reanimated)$': '/node_modules/@legendapp/list/$1.js', 'mock-builders(.*)$': '/src/mock-builders$1', // See __mocks__/SvgTouchableMixin.js - react-native-svg reads `Touchable.Mixin` off the // react-native root, and RN 0.87's late, non-enumerable re-export of it is lost through diff --git a/package/package.json b/package/package.json index 68f6b14b53..84e8985979 100644 --- a/package/package.json +++ b/package/package.json @@ -71,6 +71,7 @@ "dependencies": { "@gorhom/bottom-sheet": "^5.2.14", "@khanacademy/simple-markdown": "^2.1.0", + "@legendapp/list": "^3.7.1", "@stream-io/i18n": "^1.1.0", "@stream-io/state-store": "^1.2.0", "@ungap/structured-clone": "^1.3.0", diff --git a/package/src/ai-components/AITypingIndicatorView.tsx b/package/src/ai-components/AITypingIndicatorView.tsx index b648e33699..926182f3c3 100644 --- a/package/src/ai-components/AITypingIndicatorView.tsx +++ b/package/src/ai-components/AITypingIndicatorView.tsx @@ -3,6 +3,7 @@ import { StyleSheet, Text } from 'react-native'; import { useTheme } from './contexts'; import { Shimmer } from './internal/Shimmer'; +import { primitives } from './internal/tokens'; export const AITypingIndicatorView = ({ text }: { text: string }) => { const styles = useStyles(); @@ -21,7 +22,7 @@ const useStyles = () => { () => StyleSheet.create({ text: { - fontSize: 16, + fontSize: primitives.typographyFontSizeMd, color: theme.colors.grey_dark, }, wrapper: { diff --git a/package/src/ai-components/StreamingMessageView.tsx b/package/src/ai-components/StreamingMessageView.tsx index eb23f1ee2b..2efe179e50 100644 --- a/package/src/ai-components/StreamingMessageView.tsx +++ b/package/src/ai-components/StreamingMessageView.tsx @@ -21,8 +21,19 @@ export const useStreamingMessage = ({ const [streamedMessageText, setStreamedMessageText] = useState(text); const textCursor = useRef(text.length); + // Only text added to the end is typed out. Another text, such as another message's when a list + // reuses this view, shows at once, from this render rather than after the old text's. + const [lastText, setLastText] = useState(text); + if (text !== lastText) { + setLastText(text); + if (!text.startsWith(lastText)) setStreamedMessageText(text); + } + const typedToward = useRef(text); + useEffect(() => { const textLength = text.length; + if (!text.startsWith(typedToward.current)) textCursor.current = textLength; + typedToward.current = text; const interval = setInterval(() => { if (!text || textCursor.current >= textLength) { clearInterval(interval); diff --git a/package/src/ai-components/__tests__/Markdown.test.tsx b/package/src/ai-components/__tests__/Markdown.test.tsx index bec3a50777..5c672621ff 100644 --- a/package/src/ai-components/__tests__/Markdown.test.tsx +++ b/package/src/ai-components/__tests__/Markdown.test.tsx @@ -1,8 +1,10 @@ import React from 'react'; +import { StyleSheet } from 'react-native'; import { fireEvent, render, screen } from '@testing-library/react-native'; import { Markdown, mergeThemes, StreamTheme } from '../index'; +import { primitives } from '../internal/tokens'; const mockSetString = jest.fn(); jest.mock( @@ -101,6 +103,17 @@ describe('ai-components Markdown', () => { expect(screen.getByText('json')).toBeTruthy(); }); + it('sizes text from the design tokens, leaving headings and inline code their own size', () => { + renderMarkdown('## Heading\n\nbody with `code`'); + + const fontSizeOf = (text: string) => + StyleSheet.flatten(screen.getByText(text).props.style).fontSize; + expect(fontSizeOf('body with ')).toBe(primitives.typographyFontSizeMd); + // The size comes from the heading or inline code around the text. + expect(fontSizeOf('Heading')).toBeUndefined(); + expect(fontSizeOf('code')).toBeUndefined(); + }); + it('applies theme overrides', () => { const theme = mergeThemes({ style: { markdown: { text: { color: 'rgb(1, 2, 3)' } } } }); render( diff --git a/package/src/ai-components/__tests__/StreamingMessageView.test.tsx b/package/src/ai-components/__tests__/StreamingMessageView.test.tsx index 1c1824f99a..a2231d8da2 100644 --- a/package/src/ai-components/__tests__/StreamingMessageView.test.tsx +++ b/package/src/ai-components/__tests__/StreamingMessageView.test.tsx @@ -41,6 +41,33 @@ describe('ai-components streaming', () => { expect(result.current.streamedMessageText).toBe('Hello world'); }); + it('shows another text at once rather than typing it out', () => { + const { result, rerender } = renderHook( + ({ text }: { text: string }) => + useStreamingMessage({ letterInterval: 10, renderingLetterCount: 2, text }), + { initialProps: { text: 'Hi' } }, + ); + + // As when a list reuses the view for another, longer message. + rerender({ text: 'Goodbye, and thanks for all the fish' }); + expect(result.current.streamedMessageText).toBe('Goodbye, and thanks for all the fish'); + act(() => { + jest.advanceTimersByTime(100); + }); + expect(result.current.streamedMessageText).toBe('Goodbye, and thanks for all the fish'); + + // And one shorter than what was shown. + rerender({ text: 'Bye' }); + expect(result.current.streamedMessageText).toBe('Bye'); + + // Text added to the end of it is still typed out. + rerender({ text: 'Bye now' }); + act(() => { + jest.advanceTimersByTime(10); + }); + expect(result.current.streamedMessageText).toBe('Bye n'); + }); + it('optimistically closes a code fence while it is being streamed', () => { const { result, rerender } = renderHook( ({ text }: { text: string }) => diff --git a/package/src/ai-components/__tests__/conversation.test.tsx b/package/src/ai-components/__tests__/conversation.test.tsx new file mode 100644 index 0000000000..28526631a9 --- /dev/null +++ b/package/src/ai-components/__tests__/conversation.test.tsx @@ -0,0 +1,486 @@ +import React, { type ReactElement, useEffect } from 'react'; +import { Text } from 'react-native'; + +import { act, fireEvent, render, screen } from '@testing-library/react-native'; +import type { Attachment } from 'stream-chat'; + +import { AIPendingMessageTracker, turnsOf } from '../conversation/entries'; +import { + AIConversation, + type AIConversationAssistantMessageProps, + type AIConversationUserMessageProps, + type AIMessage, + type AIToolApprover, + createFromStreamMessage, + fromStreamMessage, + parseMessageParts, + StreamTheme, +} from '../index'; + +jest.mock('../services/clipboard-service', () => ({ setClipboardString: jest.fn() })); + +const user = (id: string, text: string): AIMessage => ({ id, role: 'user', text }); +const reply = (id: string, text: string, extra: Partial = {}): AIMessage => ({ + id, + role: 'assistant', + text, + ...extra, +}); + +const layoutOf = (height: number) => ({ + nativeEvent: { layout: { height, width: 400, x: 0, y: 0 } }, +}); + +// Lets the list apply what it measured, which it does on a later tick, with fake timers too. +const settle = async (ms = 20) => { + if ('clock' in setTimeout) { + await act(async () => { + await jest.advanceTimersByTimeAsync(ms); + }); + } else { + await act(() => new Promise((resolve) => setTimeout(resolve, ms))); + } +}; + +// The list renders its rows once it knows its size, and lets them be touched once each one is +// measured, which a real layout does. +const layOut = async () => { + const list = screen.queryByTestId('ai-conversation-list'); + if (!list) return; + act(() => { + fireEvent(list, 'layout', layoutOf(800)); + }); + // Measuring rows can bring more into the range it renders: until every row is measured. + for (let pass = 0; pass < 10; pass++) { + const measured = screen.UNSAFE_root.findAll( + (node) => typeof node.type === 'string' && !!node.props.onLayout && node !== list, + ); + act(() => { + for (const node of measured) node.props.onLayout(layoutOf(60)); + }); + await settle(); + const waiting = screen.UNSAFE_root.findAll( + (node) => typeof node.type === 'string' && node.props.pointerEvents === 'none', + ); + if (waiting.length === 0) return; + } +}; + +// A scroll event at the start of the list. +const atStart = { + nativeEvent: { + contentInset: { bottom: 0, left: 0, right: 0, top: 0 }, + contentOffset: { x: 0, y: 0 }, + contentSize: { height: 800, width: 400 }, + layoutMeasurement: { height: 800, width: 400 }, + }, +}; + +// The person scrolling to the start of the list. +const scrollToStart = () => { + act(() => { + fireEvent.scroll(screen.getByTestId('ai-conversation-list'), atStart); + }); +}; + +const renderConversation = async (ui: ReactElement) => { + const result = render({ui}); + await layOut(); + return { + ...result, + rerender: async (next: ReactElement) => { + result.rerender({next}); + await layOut(); + }, + }; +}; + +afterEach(() => { + jest.useRealTimers(); +}); + +describe('AIConversation', () => { + it('shows the empty content while there are no messages', async () => { + await renderConversation( + Ask me anything} messages={[]} />, + ); + expect(screen.getByText('Ask me anything')).toBeTruthy(); + expect(screen.queryByTestId('ai-conversation-list')).toBeNull(); + }); + + it('groups the messages into turns, each starting with a question', async () => { + const messages = [ + reply('a0', 'Hi, I am Juno.'), + user('u1', 'What time is it?'), + reply('a1', 'It is noon.'), + user('u2', 'And the weather?'), + reply('a2', 'Sunny.'), + ]; + const turns = turnsOf(new AIPendingMessageTracker().entries(messages)); + expect(turns.map((turn) => turn.id)).toEqual(['a0', 'u1', 'u2']); + expect(turns[2].entries.map((entry) => entry.id)).toEqual(['u2', 'a2']); + expect(turns[2].entries[1].isLatestReply).toBe(true); + + await renderConversation(footer} messages={messages} />); + for (const { text } of messages) expect(screen.getByText(text)).toBeTruthy(); + expect(screen.getAllByText('footer')).toHaveLength(1); + }); + + it('shows a pending message with the agent thinking, until its reply starts', async () => { + const mounts: string[] = []; + const UserMessage = ({ text }: AIConversationUserMessageProps) => { + useEffect(() => { + mounts.push(text); + }, [text]); + return {text}; + }; + const history = [user('u1', 'Hello'), reply('a1', 'Hi!')]; + const pending = { id: 'local-1', text: 'Plan my day' }; + const { rerender } = await renderConversation( + , + ); + expect(screen.getByText('Plan my day')).toBeTruthy(); + expect(screen.getByRole('button', { name: 'Thinking...' })).toBeTruthy(); + + // Stream's copy arrives with another id: it takes over the pending row, under its key. + await rerender( + , + ); + expect(screen.getAllByText('Plan my day')).toHaveLength(1); + expect(screen.getByRole('button', { name: 'Thinking...' })).toBeTruthy(); + + // A reply being written that shows nothing yet: still thinking. + await rerender( + , + ); + expect(screen.getByRole('button', { name: 'Thinking...' })).toBeTruthy(); + + await rerender( + , + ); + expect(screen.queryByRole('button', { name: 'Thinking...' })).toBeNull(); + expect(screen.getByText('Here is your plan.')).toBeTruthy(); + // The row the pending message showed in was never mounted again. + expect(mounts.filter((text) => text === 'Plan my day')).toHaveLength(1); + }); + + it('finds the copy of a pending message by its id', async () => { + const pending = { id: 'm-2', text: 'Draft' }; + const { rerender } = await renderConversation( + , + ); + // the earlier message saying the same is not its copy + expect(screen.getAllByText('Draft')).toHaveLength(2); + await rerender( + , + ); + expect(screen.getAllByText('Draft')).toHaveLength(2); + }); + + it('shows the agent thinking when told to', async () => { + await renderConversation(); + expect(screen.getByRole('button', { name: 'Thinking...' })).toBeTruthy(); + }); + + it('shows reply actions under finished replies only', async () => { + const onRate = jest.fn(); + const replyActions = (message: AIMessage, { isLatestReply }: { isLatestReply: boolean }) => ({ + onRate: (rating: 'positive' | 'negative' | null) => onRate(message.id, rating, isLatestReply), + rating: null, + }); + const { rerender } = await renderConversation( + , + ); + expect(screen.getAllByTestId('ai-reply-actions')).toHaveLength(1); + fireEvent.press(screen.getByRole('button', { name: 'Good response' })); + expect(onRate).toHaveBeenCalledWith('a1', 'positive', false); + + await rerender( + , + ); + expect(screen.queryByTestId('ai-reply-actions')).toBeNull(); + }); + + it('shows a new rating when the messages stay the same', async () => { + const messages = [user('u1', 'One'), reply('a1', 'Done.')]; + const rated = (rating: 'positive' | null) => ( + ({ onRate: jest.fn(), rating })} /> + ); + const { rerender } = await renderConversation(rated(null)); + expect(screen.getByRole('button', { name: 'Good response', selected: false })).toBeTruthy(); + await rerender(rated('positive')); + expect(screen.getByRole('button', { name: 'Good response', selected: true })).toBeTruthy(); + }); + + it('renders messages through the slots', async () => { + const UserMessage = ({ text }: AIConversationUserMessageProps) => user:{text}; + const AssistantMessage = ({ message, text }: AIConversationAssistantMessageProps) => ( + + agent:{message.id}:{text} + + ); + await renderConversation( + actions} + UserMessage={UserMessage} + />, + ); + expect(screen.getByText('user:Hi')).toBeTruthy(); + expect(screen.getByText('agent:a1:Hello')).toBeTruthy(); + expect(screen.getByText('actions')).toBeTruthy(); + }); + + it('renders again only the messages that changed', async () => { + const renders: string[] = []; + const AssistantMessage = ({ message, text }: AIConversationAssistantMessageProps) => { + renders.push(message.id); + return {text}; + }; + const first = reply('a1', 'Done.'); + const conversation = (latest: AIMessage) => ( + + ); + const { rerender } = await renderConversation( + conversation(reply('a2', 'Wri', { isGenerating: true })), + ); + renders.length = 0; + await rerender(conversation(reply('a2', 'Writing', { isGenerating: true }))); + expect(screen.getByText('Writing')).toBeTruthy(); + expect(renders).toEqual(['a2']); + }); + + it('shows reasoning, tool calls and approvals of a reply', async () => { + let fail = true; + const decide = jest.fn(() => + fail ? Promise.reject(new Error('offline')) : new Promise(() => undefined), + ); + const approver: AIToolApprover = { clientId: 'device-1', decide, userId: 'alex' }; + const parts = parseMessageParts([ + { duration_ms: 3000, id: 'r1', preview: 'Thinking about the weather.', status: 'completed', type: 'ai_reasoning', v: 1 }, + { + approval: { title: 'Share your location?' }, + display_title: 'Checking your location', + executor: 'client', + id: 'call_1', + name: 'get_location', + status: 'awaiting_approval', + target_client_id: 'device-1', + target_user_id: 'alex', + type: 'ai_tool_call', + v: 1, + }, + ]); // prettier-ignore + await renderConversation( + , + ); + expect(screen.getByRole('button', { name: /Thought for 3s/ })).toBeTruthy(); + expect(screen.getByText('Checking your location')).toBeTruthy(); + expect(screen.getByText('Waiting for approval')).toBeTruthy(); + + fireEvent.press(screen.getByRole('button', { name: 'Allow' })); + expect(decide).toHaveBeenCalledWith(parts[1].toolCall, true); + await act(() => Promise.resolve()); + expect(screen.getByText("Your answer couldn't be sent. Try again.")).toBeTruthy(); + + fail = false; + fireEvent.press(screen.getByRole('button', { name: "Don't Allow" })); + expect(decide).toHaveBeenLastCalledWith(parts[1].toolCall, false); + expect(screen.queryByText("Your answer couldn't be sent. Try again.")).toBeNull(); + expect(screen.getByRole('button', { name: "Don't Allow" })).toBeDisabled(); + }); + + describe('loading older messages', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + it('loads once the spinner has been in view a moment', async () => { + const loadOlderMessages = jest.fn(() => Promise.resolve()); + const { rerender } = await renderConversation( + , + ); + expect(screen.getByRole('progressbar', { name: 'Loading older messages' })).toBeTruthy(); + loadOlderMessages.mockClear(); + scrollToStart(); + act(() => jest.advanceTimersByTime(299)); + expect(loadOlderMessages).not.toHaveBeenCalled(); + act(() => jest.advanceTimersByTime(1)); + expect(loadOlderMessages).toHaveBeenCalledTimes(1); + + await rerender( + , + ); + expect(screen.queryByRole('progressbar')).toBeNull(); + }); + + it('asks again after a load that failed, while the spinner stays in view', async () => { + const loadOlderMessages = jest + .fn, []>() + .mockRejectedValueOnce(new Error('offline')) + .mockResolvedValue(undefined); + await renderConversation( + , + ); + scrollToStart(); + await act(async () => { + await jest.advanceTimersByTimeAsync(300); + }); + expect(loadOlderMessages).toHaveBeenCalledTimes(1); + await act(async () => { + await jest.advanceTimersByTimeAsync(2999); + }); + expect(loadOlderMessages).toHaveBeenCalledTimes(1); + await act(async () => { + await jest.advanceTimersByTimeAsync(1); + }); + expect(loadOlderMessages).toHaveBeenCalledTimes(2); + }); + + it('waits for the list to come to rest', async () => { + const loadOlderMessages = jest.fn(() => Promise.resolve()); + await renderConversation( + , + ); + const list = screen.getByTestId('ai-conversation-list'); + loadOlderMessages.mockClear(); + + // A finger holds the list at its start. + act(() => { + fireEvent(list, 'scrollBeginDrag', atStart); + }); + scrollToStart(); + act(() => jest.advanceTimersByTime(1000)); + expect(loadOlderMessages).not.toHaveBeenCalled(); + + // It lets go, and the list glides on. + act(() => { + fireEvent(list, 'scrollEndDrag', atStart); + fireEvent(list, 'momentumScrollBegin', atStart); + }); + act(() => jest.advanceTimersByTime(1000)); + expect(loadOlderMessages).not.toHaveBeenCalled(); + + act(() => { + fireEvent(list, 'momentumScrollEnd', atStart); + }); + act(() => jest.advanceTimersByTime(300)); + expect(loadOlderMessages).toHaveBeenCalledTimes(1); + }); + }); +}); + +describe('fromStreamMessage', () => { + it('adapts a reply with its steps and attachments', () => { + const reasoningText = jest.fn(() => 'The whole thought'); + // stream-chat keeps the fields its types don't list under `custom`, such as a step's. + const step = { + custom: { id: 'r1', status: 'streaming', v: 1 }, + type: 'ai_reasoning', + } as unknown as Attachment; + const message = fromStreamMessage( + { + attachments: [ + step, + { custom: {}, image_url: 'https://cdn.example.com/a.jpg', title: 'a.jpg', type: 'image' }, + { + asset_url: 'https://cdn.example.com/plan.pdf', + custom: { file_size: 2400, mime_type: 'application/pdf' }, + title: 'plan.pdf', + type: 'file', + }, + { custom: {}, type: 'connector_authorization' }, + ], + custom: { ai_generated: true, generating: true }, + id: 'm1', + text: 'Working on it', + }, + { reasoningText }, + ); + expect(message).toMatchObject({ + attachments: [ + { kind: 'image', name: 'a.jpg', url: 'https://cdn.example.com/a.jpg' }, + { + kind: 'file', + mimeType: 'application/pdf', + name: 'plan.pdf', + size: 2400, + url: 'https://cdn.example.com/plan.pdf', + }, + ], + id: 'm1', + isGenerating: true, + role: 'assistant', + text: 'Working on it', + }); + expect(message.parts?.map((part) => part.id)).toEqual(['r1']); + expect(message.reasoningText?.('r1')).toBe('The whole thought'); + expect(reasoningText).toHaveBeenCalledWith('m1', 'r1'); + }); + + it('adapts the person’s message', () => { + expect(fromStreamMessage({ attachments: [], id: 'u1', text: 'Hi' })).toMatchObject({ + isGenerating: false, + parts: [], + role: 'user', + }); + }); + + it('gives back the same message for the same Stream message, through its cache', () => { + const adapt = createFromStreamMessage({ isAssistant: (message) => message.id === 'a1' }); + const stream = { attachments: [], id: 'a1', text: 'Hi' }; + const adapted = adapt(stream); + expect(adapted.role).toBe('assistant'); + expect(adapt(stream)).toBe(adapted); + expect(adapt({ ...stream, text: 'Hi there' })).not.toBe(adapted); + }); +}); diff --git a/package/src/ai-components/__tests__/conversationRows.test.ts b/package/src/ai-components/__tests__/conversationRows.test.ts new file mode 100644 index 0000000000..6f159fffcc --- /dev/null +++ b/package/src/ai-components/__tests__/conversationRows.test.ts @@ -0,0 +1,92 @@ +import { renderHook } from '@testing-library/react-native'; + +import type { AIMessage, AIPendingMessage } from '../conversation/types'; +import { + type ConversationRow, + useConversationRows, + type UseConversationRowsOptions, +} from '../conversation/useConversationRows'; + +const user = (id: string, text: string): AIMessage => ({ id, role: 'user', text }); +const reply = (id: string, text: string, extra: Partial = {}): AIMessage => ({ + id, + role: 'assistant', + text, + ...extra, +}); + +const render = (options: UseConversationRowsOptions) => + renderHook((props: UseConversationRowsOptions) => useConversationRows(props), { + initialProps: options, + }); + +const shapeOf = (row: ConversationRow) => + row.kind === 'tail' + ? `tail:${row.turnKey}${row.isThinking ? ':thinking' : ''}` + : `${row.key}:${row.position}:${row.turnKey}`; + +describe('useConversationRows', () => { + it('lists the messages by turn and anchors the latest turn', () => { + const { result } = render({ + messages: [user('q1', 'One?'), reply('a1', 'One.'), user('q2', 'Two?'), reply('a2', 'Two.')], + }); + + expect(result.current.rows.map(shapeOf)).toEqual([ + 'q1:turn:q1', + 'a1:message:q1', + 'q2:turn:q2', + 'a2:message:q2', + ]); + expect(result.current.anchorKey).toBe('q2'); + expect(result.current.isEmpty).toBe(false); + }); + + it('is empty without messages, with nothing to anchor', () => { + const { result } = render({ messages: [] }); + + expect(result.current.rows).toEqual([]); + expect(result.current.anchorKey).toBeUndefined(); + expect(result.current.isEmpty).toBe(true); + }); + + it('ends the latest turn with a row while the agent thinks or there is a footer', () => { + const messages = [user('q1', 'One?')]; + const { rerender, result } = render({ messages }); + expect(result.current.rows.map(shapeOf)).toEqual(['q1:turn:q1']); + + rerender({ isThinking: true, messages }); + expect(result.current.rows.map(shapeOf)).toEqual(['q1:turn:q1', 'tail:q1:thinking']); + + rerender({ footer: 'Working', messages: [...messages, reply('a1', 'One.')] }); + expect(result.current.rows.map(shapeOf)).toEqual(['q1:turn:q1', 'a1:message:q1', 'tail:q1']); + }); + + it('anchors a message on its way, and keeps its row for its copy', () => { + const pending: AIPendingMessage = { id: 'local-1', text: 'Three?' }; + const messages = [user('q1', 'One?'), reply('a1', 'One.')]; + const { rerender, result } = render({ messages, pendingMessage: pending }); + expect(result.current.anchorKey).toBe('local-1'); + const pendingRow = result.current.rows[2]; + expect(pendingRow && shapeOf(pendingRow)).toBe('local-1:turn:local-1'); + + // The copy arrives under the server's id, and takes over the row and its key. + rerender({ messages: [...messages, user('q3', 'Three?')], pendingMessage: pending }); + expect(result.current.anchorKey).toBe('local-1'); + expect(result.current.rows.map(shapeOf)).toEqual([ + 'q1:turn:q1', + 'a1:message:q1', + 'local-1:turn:local-1', + 'tail:local-1:thinking', + ]); + }); + + it('keeps the object of every row that did not change', () => { + const q1 = user('q1', 'One?'); + const { rerender, result } = render({ messages: [q1, reply('a1', 'On')] }); + const [firstRow, firstReply] = result.current.rows; + + rerender({ messages: [q1, reply('a1', 'One.')] }); + expect(result.current.rows[0]).toBe(firstRow); + expect(result.current.rows[1]).not.toBe(firstReply); + }); +}); diff --git a/package/src/ai-components/__tests__/messageParts.test.ts b/package/src/ai-components/__tests__/messageParts.test.ts index 731be7ecc8..7e8fe64518 100644 --- a/package/src/ai-components/__tests__/messageParts.test.ts +++ b/package/src/ai-components/__tests__/messageParts.test.ts @@ -148,6 +148,22 @@ describe('parseMessageParts', () => { expect(parts.every(isSupportedPart)).toBe(true); }); + it('reads the fields stream-chat keeps under custom', () => { + const [part] = parseMessageParts([ + { + custom: { duration_ms: 1200, id: 'r1', status: 'completed', summary: 'Checked', v: 1 }, + type: 'ai_reasoning', + }, + ]); + expect(part).toMatchObject({ id: 'r1', kind: 'ai_reasoning', version: 1 }); + expect(part.payload).toMatchObject({ id: 'r1', status: 'completed', type: 'ai_reasoning' }); + expect(part.reasoning).toMatchObject({ + durationMs: 1200, + status: 'completed', + summary: 'Checked', + }); + }); + it('reads a reasoning step', () => { expect(parseMessagePart(reasoning)?.reasoning).toEqual({ durationMs: 4200, diff --git a/package/src/ai-components/__tests__/messageViews.test.tsx b/package/src/ai-components/__tests__/messageViews.test.tsx index c91ea64079..6136644133 100644 --- a/package/src/ai-components/__tests__/messageViews.test.tsx +++ b/package/src/ai-components/__tests__/messageViews.test.tsx @@ -150,6 +150,34 @@ describe('AIAssistantMessage', () => { expect(screen.getByText('chart.csv')).toBeTruthy(); }); + it('starts its steps afresh for another message, whose step ids may repeat', () => { + // Every reply's first reasoning step can be `r1`: only the message tells them apart. + const parts = parseMessageParts([ + { id: 'r1', status: 'completed', summary: 'Looked it up', type: 'ai_reasoning' }, + ]); + const view = (id: string) => ( + + 'The whole thought'} + text='The answer' + /> + + ); + const { rerender } = render(view('a')); + fireEvent.press(screen.getByRole('button', { expanded: false })); + expect(screen.getByText('The whole thought')).toBeTruthy(); + + rerender(view('a')); + expect(screen.getByRole('button', { expanded: true })).toBeTruthy(); + + // As when a list reuses the view for another reply. + rerender(view('b')); + expect(screen.getByRole('button', { expanded: false })).toBeTruthy(); + expect(screen.queryByText('The whole thought')).toBeNull(); + }); + it('is busy while the agent writes it', () => { renderWithTheme(); expect(screen.getByTestId('ai-assistant-message').props.accessibilityState).toEqual({ diff --git a/package/src/ai-components/__tests__/signIn.test.tsx b/package/src/ai-components/__tests__/signIn.test.tsx index 7dca68fd39..2f3007b568 100644 --- a/package/src/ai-components/__tests__/signIn.test.tsx +++ b/package/src/ai-components/__tests__/signIn.test.tsx @@ -76,7 +76,7 @@ describe('AIToolAuthorization', () => { expect(screen.queryByText('Connect Linear')).toBeNull(); }); - it('connects, and lets the person try again when the sign-in closes', async () => { + it('connects, and lets the person try again when the sign-in ends unconnected', async () => { let finish: (connected: boolean) => void = () => undefined; const connect = jest.fn( () => @@ -103,6 +103,24 @@ describe('AIToolAuthorization', () => { expect(screen.getByRole('button', { name: 'Connect' })).toBeEnabled(); }); + it('declines a sign-in the person closed, which cannot be opened again', async () => { + const decline = jest.fn(() => Promise.reject(new Error('offline'))); + renderWithTheme( + Promise.resolve('closed'), decline })} + part={signIn} + />, + ); + fireEvent.press(screen.getByRole('button', { name: 'Connect' })); + await act(() => Promise.resolve()); + expect(decline).toHaveBeenCalledWith(signIn); + + // As with "Not now", an answer that couldn't be sent can be given again. + await act(() => Promise.resolve()); + expect(screen.getByText("Your answer couldn't be sent. Try again.")).toBeTruthy(); + expect(screen.getByRole('button', { name: 'Not now' })).toBeEnabled(); + }); + it('keeps connecting once connected, until the step changes', async () => { renderWithTheme( { ); await expect(authorizer.connect(signIn)).resolves.toBe(false); - // The person closed the sheet. + // The person closed the sheet: the sign-in can't be opened again. native.openAuthSession.mockResolvedValueOnce(''); - await expect(authorizer.connect(signIn)).resolves.toBe(false); + await expect(authorizer.connect(signIn)).resolves.toBe('closed'); await authorizer.decline(signIn); expect(decline).toHaveBeenCalledWith(signIn); }); + it('lets a closed sign-in be tried again with declineOnClose off', async () => { + const authorizer = createAuthSessionAuthorizer({ + callbackScheme: 'myapp', + decline: jest.fn(), + declineOnClose: false, + }); + native.openAuthSession.mockResolvedValueOnce(''); + await expect(authorizer.connect(signIn)).resolves.toBe(false); + }); + it('rejects when the sign-in could not be opened', async () => { const authorizer = createAuthSessionAuthorizer({ callbackScheme: 'myapp', decline: jest.fn() }); diff --git a/package/src/ai-components/conversation/AIConversation.tsx b/package/src/ai-components/conversation/AIConversation.tsx new file mode 100644 index 0000000000..12f1dc9760 --- /dev/null +++ b/package/src/ai-components/conversation/AIConversation.tsx @@ -0,0 +1,331 @@ +import React, { + type ComponentType, + createContext, + memo, + type ReactNode, + useContext, + useMemo, +} from 'react'; +import { StyleSheet, View } from 'react-native'; + +import type { LegendListRenderItemProps } from '@legendapp/list/react-native'; + +import type { AIConversationEntry } from './entries'; +import { AIChatList } from './list/AIChatList'; +import type { AIMessage, AIPendingMessage } from './types'; +import { type ConversationRow, type RowPosition, useConversationRows } from './useConversationRows'; + +import { useTheme } from '../contexts'; +import { primitives } from '../internal/tokens'; +import { AIAssistantMessage } from '../message/AIAssistantMessage'; +import type { AIMessageAttachmentData } from '../message/AIMessageAttachment'; +import { AIReplyActions, type AIReplyActionsProps } from '../message/AIReplyActions'; +import { AIUserMessage } from '../message/AIUserMessage'; +import type { AIMessagePartData } from '../message-parts/types'; +import { AIThinking } from '../reasoning/AIReasoning'; +import { type AIThinkingClock, AIThinkingTurnContext } from '../reasoning/thinkingClock'; +import { type AIToolApprover, AIToolApproverProvider } from '../tool-calls/AIToolApprover'; + +export type AIConversationUserMessageProps = { + /** What the person sent with it, such as images and files. */ + attachments: AIMessageAttachmentData[]; + /** What the person wrote. */ + text: string; + /** The message, or `undefined` for one on its way. */ + message?: AIMessage; +}; + +export type AIConversationAssistantMessageProps = { + attachments: AIMessageAttachmentData[]; + isGenerating: boolean; + message: AIMessage; + parts: AIMessagePartData[]; + text: string; + approver?: AIToolApprover; + reasoningText?: (stepId: string) => string | undefined; +}; + +export type AIConversationReplyActionsProps = { + /** Whether it is the reply that ends the conversation, which can be asked for again. */ + isLatestReply: boolean; + message: AIMessage; + text: string; +}; + +export type AIConversationProps = { + /** The conversation, oldest message first. */ + messages: AIMessage[]; + /** Who answers tool calls' questions and sign-ins on this device, to ask them. */ + approver?: AIToolApprover; + /** + * The view of the agent's replies. Defaults to `AIAssistantMessage`. + * + * The list reuses its rows: a view scrolled out of sight renders another reply next, so state a + * view keeps of its own has to start afresh for a new `message.id` (such as with a `key`). + * The same holds for `UserMessage` and `ReplyActions`. + */ + AssistantMessage?: ComponentType; + /** + * Room left under the latest turn, for a composer floating over the bottom of the conversation. + * The question of a new turn pins to the top of what is visible above it. + */ + bottomInset?: number; + /** What the conversation shows while there are no messages, centered in it. */ + emptyContent?: ReactNode; + /** A view under the latest turn's messages, such as what the agent is doing, or an error with a way to retry. */ + footer?: ReactNode; + /** + * Whether the agent is answering the latest message and its reply shows nothing yet. + * Meanwhile the latest turn ends with `AIThinking`, and the reply's first reasoning step takes + * over its seconds. The conversation also knows it is while `pendingMessage` waits, and while + * the latest reply is being written but shows nothing. + */ + isThinking?: boolean; + /** + * Loads the messages before the first one, which the conversation asks for as the person + * scrolls to its start. It shows a spinner meanwhile, and what is on screen stays in place as + * they arrive. Pass `undefined` once there are none left. + */ + loadOlderMessages?: () => Promise | void; + /** + * A message the person sent that is not in `messages` yet. Keep passing it until its reply + * starts: the conversation stops showing it once its copy arrives. + */ + pendingMessage?: AIPendingMessage | null; + /** The actions under each finished reply. Defaults to `AIReplyActions`; `null` shows none. */ + ReplyActions?: ComponentType | null; + /** + * What the default `AIReplyActions` offers for a reply besides copy and share, such as its + * rating and how to rate or regenerate it. Every message renders again when it changes, so keep + * it the same between renders (`useCallback`) unless what it returns changed. + */ + replyActions?: ( + message: AIMessage, + context: { isLatestReply: boolean }, + ) => Omit | undefined; + /** The view of the person's messages. Defaults to `AIUserMessage`. */ + UserMessage?: ComponentType; +}; + +const DefaultUserMessage = ({ attachments, text }: AIConversationUserMessageProps) => ( + +); + +const DefaultAssistantMessage = ({ + approver, + attachments, + isGenerating, + message, + parts, + reasoningText, + text, +}: AIConversationAssistantMessageProps) => ( + +); + +// What every row reads, rather than each row's item carrying it: a row renders again only when +// its item is a new object, or when this changes. +type ConversationContextValue = { + AssistantMessage: ComponentType; + clock: AIThinkingClock; + ReplyActions: ComponentType | null; + UserMessage: ComponentType; + approver?: AIToolApprover; + replyActions?: AIConversationProps['replyActions']; +}; + +const ConversationContext = createContext(undefined); + +// StreamChatAI's layout, from the design tokens: turns 8 apart, each with 16 above it, its +// messages 16 apart, and 16 around the whole. +const ROW_GAPS: Record = { + message: primitives.spacingMd, + turn: primitives.spacingXs + primitives.spacingMd, +}; + +type EntryViewProps = { + AssistantMessage: ComponentType; + entry: AIConversationEntry; + ReplyActions: ComponentType | null; + UserMessage: ComponentType; + approver?: AIToolApprover; +}; + +const EntryView = ({ + approver, + AssistantMessage, + entry, + ReplyActions, + UserMessage, +}: EntryViewProps) => { + const { message, pending } = entry; + if (!message) { + return pending ? ( + + ) : null; + } + if (message.role === 'user') { + return ( + + ); + } + return ( + + + {ReplyActions && !message.isGenerating && message.text !== '' ? ( + // Keyed: a row reused for another reply starts its actions afresh, such as not "Copied". + + ) : null} + + ); +}; + +const Row = memo(({ row }: { row: ConversationRow }) => { + const context = useContext(ConversationContext); + const { + theme: { conversation }, + } = useTheme(); + const clock = context?.clock; + const turn = useMemo( + () => (clock ? { clock, firstStepId: row.firstStepId, key: row.turnKey } : undefined), + [clock, row.firstStepId, row.turnKey], + ); + if (!context) return null; + return ( + + + {row.kind === 'entry' ? ( + + ) : ( + <> + {row.isThinking ? : null} + {row.footer} + + )} + + + ); +}); +Row.displayName = 'AIConversationRow'; + +const renderRow = ({ item }: LegendListRenderItemProps) => ; + +const keyOf = (row: ConversationRow) => row.key; + +// A row reused for another takes one of its kind, which renders the same views. +const typeOf = (row: ConversationRow) => + row.kind === 'tail' ? 'tail' : row.entry.isFromUser ? 'user' : 'assistant'; + +// `AIReplyActions` with what the conversation's `replyActions` offers for the reply. +const DefaultConversationReplyActions = ({ + isLatestReply, + message, + text, +}: AIConversationReplyActionsProps) => { + const offered = useContext(ConversationContext)?.replyActions; + return ; +}; + +/** + * The conversation with an AI agent, laid out by turn: each of the person's questions starts a + * turn, with the agent's reply under it. The counterpart of `AIConversationView`. + * + * - A new question pins to the top of what is visible, and the reply writes out below it, + * without moving what is on screen. The latest turn is at least as tall as the conversation. + * - It opens at the end; older messages load as the person scrolls to the start, and what is on + * screen stays in place as they arrive. + * - A button scrolls back to the latest message when the end is out of view. + * - A message the person sent shows at once (`pendingMessage`) and the agent thinks under it + * (`AIThinking`) until its reply shows something. + */ +export const AIConversation = ({ + approver, + AssistantMessage = DefaultAssistantMessage, + bottomInset = 0, + emptyContent, + footer, + isThinking = false, + loadOlderMessages, + messages, + pendingMessage, + ReplyActions: ReplyActionsProp, + replyActions, + UserMessage = DefaultUserMessage, +}: AIConversationProps) => { + const { anchorKey, clock, rows } = useConversationRows({ + footer, + isThinking, + messages, + pendingMessage, + }); + + const ReplyActions = + ReplyActionsProp === undefined ? DefaultConversationReplyActions : ReplyActionsProp; + + const contextValue = useMemo( + () => ({ approver, AssistantMessage, clock, ReplyActions, replyActions, UserMessage }), + [approver, AssistantMessage, clock, ReplyActions, replyActions, UserMessage], + ); + + return ( + + + + + + ); +}; + +const styles = StyleSheet.create({ + reply: { gap: 8 }, + row: { paddingHorizontal: primitives.spacingMd }, +}); diff --git a/package/src/ai-components/conversation/entries.ts b/package/src/ai-components/conversation/entries.ts new file mode 100644 index 0000000000..f1f38767a2 --- /dev/null +++ b/package/src/ai-components/conversation/entries.ts @@ -0,0 +1,133 @@ +import type { AIMessage, AIPendingMessage } from './types'; + +import type { AIMessageAttachmentData } from '../message/AIMessageAttachment'; + +/** A row of the conversation: a message, or a message the person sent that is on its way. */ +export type AIConversationEntry = { + /** The message's id, or, for the copy of a message that was on its way, that message's. */ + id: string; + isFromUser: boolean; + /** Whether it is a reply that ends the conversation. */ + isLatestReply: boolean; + /** The message, or `undefined` for one on its way. */ + message?: AIMessage; + /** The message on its way, until its copy arrives. */ + pending?: AIPendingMessage; + /** What the person sent with a message on its way, which its copy shows rather than load it again. */ + sentAttachments: AIMessageAttachmentData[]; +}; + +/** The person's message and the replies to it. */ +export type AIConversationTurn = { entries: AIConversationEntry[]; id: string }; + +// A reply the agent has started but that has nothing to show yet. +const showsNothing = (message: AIMessage) => + message.role !== 'user' && + message.text === '' && + !message.parts?.length && + !message.attachments?.length; + +/** Groups the rows into turns, each starting with one of the person's messages. */ +export const turnsOf = (entries: AIConversationEntry[]) => + entries.reduce((turns, entry) => { + const last = turns[turns.length - 1]; + if (!last || entry.isFromUser) turns.push({ entries: [entry], id: entry.id }); + else last.entries.push(entry); + return turns; + }, []); + +/** + * Follows a message the person sent until its copy arrives in the messages, so the copy takes + * over its row (and its key) without a flicker, and tells whether the agent is surely about to + * answer. The counterpart of `AIPendingMessageTracker`. + */ +export class AIPendingMessageTracker { + // The person's messages each pending message's copy cannot be: those there were when it was + // first seen, and those that arrived since without being its copy, such as older ones. + private seen = new Map>(); + // The pending message each copy took over, by the copy's id. + private copies = new Map(); + // The pending message of the last update, whose copy can arrive in the update that stops + // passing it. + private lastPending?: AIPendingMessage; + + // The rows of a conversation: its messages, apart from replies with nothing to show yet, and + // a message on its way until its copy arrives. + entries(messages: AIMessage[], pending?: AIPendingMessage | null): AIConversationEntry[] { + let showsPending = !!pending; + // A copy that arrives in the update that stops passing its pending message still takes over + // its row. Only then: a message sent later is not the copy of one that failed. + const awaited = pending ?? this.lastPending; + if (awaited) { + const sent = messages.filter((message) => message.role === 'user'); + const ids = new Set(sent.map((message) => message.id)); + const known = this.seen.get(awaited.id) ?? ids; + const matched = [...this.copies].find( + ([copyId, original]) => original.id === awaited.id && ids.has(copyId), + )?.[0]; + const copy = matched ?? this.copyOf(awaited, sent, known); + if (copy !== undefined) { + if (!this.copies.has(copy)) this.copies.set(copy, awaited); + showsPending = false; + } else { + this.seen.set(awaited.id, new Set([...known, ...ids])); + } + } + this.lastPending = pending ?? undefined; + + const entries: AIConversationEntry[] = messages + .filter((message) => !showsNothing(message)) + .map((message) => { + const taken = this.copies.get(message.id); + return { + id: taken?.id ?? message.id, + isFromUser: message.role === 'user', + isLatestReply: false, + message, + sentAttachments: taken?.attachments ?? [], + }; + }); + if (showsPending && pending) { + entries.push({ + id: pending.id, + isFromUser: true, + isLatestReply: false, + pending, + sentAttachments: [], + }); + } + const last = entries[entries.length - 1]; + if (last && !last.isFromUser) last.isLatestReply = true; + return entries; + } + + // The copy of a message that was on its way: the person's message with its id, when the app + // knew it. Otherwise their first after those known, which older messages loading above cannot + // be. With none of theirs known, it is their latest, when that is the only new one or says the + // same. + private copyOf(pending: AIPendingMessage, sent: AIMessage[], known: Set) { + const ids = sent.map((message) => message.id); + if (ids.includes(pending.id)) return pending.id; + let lastKnown = -1; + ids.forEach((id, index) => { + if (known.has(id)) lastKnown = index; + }); + if (lastKnown >= 0) return ids[lastKnown + 1]; + const fresh = sent.filter((message) => !known.has(message.id)); + const latest = fresh[fresh.length - 1]; + if (!latest || (fresh.length !== 1 && latest.text !== pending.text)) return undefined; + return latest.id; + } + + // Whether the latest message is surely waiting for a reply that shows nothing yet: it is on its + // way, it is the copy of the one on its way, or its reply is being written but shows nothing. + // Call it after `entries`, which matches the copies. + isAwaitingReply(messages: AIMessage[], pending?: AIPendingMessage | null) { + const last = messages[messages.length - 1]; + if (last && last.role !== 'user' && last.isGenerating && showsNothing(last)) return true; + if (!pending) return false; + const copy = [...this.copies].find(([, original]) => original.id === pending.id)?.[0]; + if (copy === undefined) return true; + return last?.id === copy; + } +} diff --git a/package/src/ai-components/conversation/fromStreamMessage.ts b/package/src/ai-components/conversation/fromStreamMessage.ts new file mode 100644 index 0000000000..d20f895033 --- /dev/null +++ b/package/src/ai-components/conversation/fromStreamMessage.ts @@ -0,0 +1,87 @@ +import type { Attachment, LocalMessage, MessageResponse } from 'stream-chat'; + +import type { AIMessage } from './types'; + +import { withCustomFields } from '../internal/customFields'; +import type { AIMessageAttachmentData } from '../message/AIMessageAttachment'; +import { parseMessageParts } from '../message-parts/parseMessageParts'; + +/** A Stream message, as the channel's state or an event has it. */ +export type AIStreamMessage = Pick & + Record; + +export type FromStreamMessageOptions = { + /** Whether the agent wrote it. Defaults to the message's `custom.ai_generated` field. */ + isAssistant?: (message: AIStreamMessage) => boolean; + /** The whole of a reasoning step's thinking, when the app has it, such as from live updates. */ + reasoningText?: (messageId: string, stepId: string) => string | undefined; +}; + +// Images and files only: the steps (`ai_*`) are read as parts, and other kinds are the app's. +const attachmentOf = (attachment: Attachment): AIMessageAttachmentData | undefined => { + const name = attachment.title ?? attachment.fallback ?? undefined; + // stream-chat keeps the file's size and type under `custom`. + const { file_size, mime_type } = attachment.custom ?? {}; + if (attachment.type === 'image') { + const url = attachment.image_url ?? attachment.asset_url ?? attachment.thumb_url; + return url ? { kind: 'image', mimeType: mime_type, name, url } : undefined; + } + if (attachment.type === 'file') { + const size = Number(file_size); + return attachment.asset_url + ? { + kind: 'file', + mimeType: mime_type, + name, + size: Number.isFinite(size) && size > 0 ? size : undefined, + url: attachment.asset_url, + } + : undefined; + } + return undefined; +}; + +/** + * Adapts a Stream message for `AIConversation`: a reply is one with `custom.ai_generated: true`, + * it is being written while `custom.generating: true`, its steps are its `ai_reasoning` and `ai_tool_call` + * attachments, and its images and files are its other attachments. + */ +export const fromStreamMessage = ( + message: AIStreamMessage, + { isAssistant, reasoningText }: FromStreamMessageOptions = {}, +): AIMessage => { + const attachments = message.attachments ?? []; + const fields = withCustomFields(message); + const assistant = isAssistant ? isAssistant(message) : fields.ai_generated === true; + return { + attachments: attachments + .map(attachmentOf) + .filter((attachment): attachment is AIMessageAttachmentData => !!attachment), + id: message.id, + isGenerating: fields.generating === true, + parts: parseMessageParts(attachments), + reasoningText: reasoningText + ? (stepId: string) => reasoningText(message.id, stepId) + : undefined, + role: assistant ? 'assistant' : 'user', + text: message.text ?? '', + }; +}; + +/** + * A `fromStreamMessage` that gives back the same `AIMessage` for the same Stream message object. + * The channel's state replaces only the messages that change, so mapping its messages through it + * on every update leaves the others as they were, and `AIConversation` renders only the changed + * ones again. Make one per conversation, such as with `useState(() => createFromStreamMessage())`. + */ +export const createFromStreamMessage = (options?: FromStreamMessageOptions) => { + const cache = new WeakMap(); + return (message: AIStreamMessage) => { + let adapted = cache.get(message); + if (!adapted) { + adapted = fromStreamMessage(message, options); + cache.set(message, adapted); + } + return adapted; + }; +}; diff --git a/package/src/ai-components/conversation/index.ts b/package/src/ai-components/conversation/index.ts new file mode 100644 index 0000000000..6917067c6e --- /dev/null +++ b/package/src/ai-components/conversation/index.ts @@ -0,0 +1,4 @@ +export * from './AIConversation'; +export * from './fromStreamMessage'; +export * from './types'; +export type { AIConversationEntry, AIConversationTurn } from './entries'; diff --git a/package/src/ai-components/conversation/list/AIChatList.tsx b/package/src/ai-components/conversation/list/AIChatList.tsx new file mode 100644 index 0000000000..a6e06b1961 --- /dev/null +++ b/package/src/ai-components/conversation/list/AIChatList.tsx @@ -0,0 +1,176 @@ +import React, { type ReactNode, useMemo, useRef } from 'react'; +import { ActivityIndicator, Pressable, StyleSheet, View } from 'react-native'; + +import type { LegendListRef, LegendListRenderItemProps } from '@legendapp/list/react-native'; +import { AnimatedLegendList } from '@legendapp/list/reanimated'; + +import { useChatListController } from './useChatListController'; + +import { useAITranslation, useTheme } from '../../contexts'; +import { useAIIcons } from '../../icons'; +import { primitives } from '../../internal/tokens'; + +export type AIChatListProps = { + /** The key of the row to pin to the top of what is visible, such as the latest question's. */ + anchorKey: string | undefined; + data: T[]; + getItemType: (item: T) => string; + keyExtractor: (item: T) => string; + renderItem: (props: LegendListRenderItemProps) => ReactNode; + /** The room for a composer floating over the bottom of the list. */ + bottomInset?: number; + /** What shows while there are no rows, centered. */ + emptyContent?: ReactNode; + /** Loads the rows before the first one. `undefined` once there are none left. */ + loadOlderMessages?: () => Promise | void; + testID?: string; +}; + +const LoadingOlder = () => { + const { t } = useAITranslation(); + const { + theme: { colors, conversation }, + } = useTheme(); + return ( + + + + ); +}; + +const ScrollToLatestButton = ({ onPress }: { onPress: () => void }) => { + const { t } = useAITranslation(); + const { + theme: { colors, conversation }, + } = useTheme(); + const { ArrowDown } = useAIIcons(); + return ( + [ + styles.scrollButton, + { + backgroundColor: conversation?.scrollButtonBackground ?? colors.white, + borderColor: conversation?.scrollButtonBorder ?? colors.grey_gainsboro, + opacity: pressed ? 0.7 : 1, + }, + conversation?.scrollButton, + ]} + > + + + ); +}; + +/** + * The list of a chat with an agent: virtualised on Reanimated LegendList, reusing the views of + * rows scrolled out of sight. It pins `anchorKey`'s row to the top of what is visible, opens at the + * end, loads older rows as the person scrolls to the start, and offers a button back to the latest + * row when the end is out of view. What the rows are is the caller's. + */ +export const AIChatList = ({ + anchorKey, + bottomInset = 0, + data, + emptyContent, + getItemType, + keyExtractor, + loadOlderMessages, + renderItem, + testID, +}: AIChatListProps) => { + const { + theme: { conversation }, + } = useTheme(); + const listRef = useRef(null); + const isEmpty = data.length === 0; + const { contentFloor, isEndInView, listProps, loadsOlder, onContainerLayout, scrollToEnd } = + useChatListController({ + anchorKey, + bottomInset, + data, + keyExtractor, + listRef, + loadOlderMessages: isEmpty ? undefined : loadOlderMessages, + }); + + // Flattened: the list reads the floor from it. + const contentContainerStyle = useMemo( + () => + StyleSheet.flatten([ + { minHeight: contentFloor, paddingBottom: primitives.spacingMd + bottomInset }, + conversation?.content, + ]), + [bottomInset, contentFloor, conversation?.content], + ); + + return ( + + {isEmpty ? ( + + {emptyContent} + + ) : ( + + )} + {!isEndInView && !isEmpty ? ( + + + + ) : null} + + ); +}; + +const styles = StyleSheet.create({ + container: { flex: 1 }, + empty: { alignItems: 'center', flex: 1, justifyContent: 'center' }, + loadingOlder: { alignItems: 'center', paddingTop: primitives.spacingMd }, + scrollButton: { + alignItems: 'center', + borderRadius: 20, + borderWidth: 1, + elevation: 2, + height: 40, + justifyContent: 'center', + // StreamChatAI's second elevation. + shadowColor: '#000000', + shadowOffset: { height: 2, width: 0 }, + shadowOpacity: 0.14, + shadowRadius: 3, + width: 40, + }, + scrollButtonContainer: { alignItems: 'center', left: 0, position: 'absolute', right: 0 }, +}); diff --git a/package/src/ai-components/conversation/list/useChatListController.ts b/package/src/ai-components/conversation/list/useChatListController.ts new file mode 100644 index 0000000000..d339923345 --- /dev/null +++ b/package/src/ai-components/conversation/list/useChatListController.ts @@ -0,0 +1,72 @@ +import { type RefObject, useCallback, useRef } from 'react'; + +import type { LegendListRef } from '@legendapp/list/react-native'; + +import { useEndInView } from './useEndInView'; +import { useLoadOlder } from './useLoadOlder'; +import { usePinnedAnchor } from './usePinnedAnchor'; + +import { useReduceMotion } from '../../voice-visualization/useReduceMotion'; + +export type ChatListControllerOptions = { + /** The key of the row to pin to the top, such as the latest question's. */ + anchorKey: string | undefined; + /** The room for a composer floating over the bottom of the list. */ + bottomInset: number; + data: readonly T[]; + keyExtractor: (item: T) => string; + listRef: RefObject; + /** Loads the rows before the first one. `undefined` once there are none left. */ + loadOlderMessages?: () => Promise | void; +}; + +/** + * How a chat list behaves, apart from how it looks: the row to pin, whether the end is in view, + * and loading older rows. Renders nothing: `listProps` go on a LegendList given `listRef`, and + * the rest is for the views around it. Any LegendList that keeps `anchoredEndSpace` works. + */ +export const useChatListController = ({ + anchorKey, + bottomInset, + data, + keyExtractor, + listRef, + loadOlderMessages, +}: ChatListControllerOptions) => { + const reduceMotion = useReduceMotion(); + // The list's positions leave its header out: the spinner while older messages load. + const headerSize = useRef(0); + + const { isEndInView, scheduleCheckEnd } = useEndInView({ bottomInset, headerSize, listRef }); + const { anchoredEndSpace, contentFloor, onContainerLayout, onMetricsChange } = usePinnedAnchor({ + anchorKey, + data, + headerSize, + keyExtractor, + listRef, + onAnchorLayout: scheduleCheckEnd, + }); + const loadOlder = useLoadOlder({ listRef, loadOlderMessages }); + + const scrollToEnd = useCallback(() => { + void listRef.current?.scrollToEnd({ animated: !reduceMotion }); + }, [listRef, reduceMotion]); + + return { + /** The least height of the list's content, for its `contentContainerStyle`. */ + contentFloor, + isEndInView, + /** Whether older rows load as the list nears its start, so it shows that they are loading. */ + loadsOlder: !!loadOlder, + listProps: { + anchoredEndSpace, + onItemSizeChanged: scheduleCheckEnd, + onMetricsChange, + onScroll: scheduleCheckEnd, + ...loadOlder, + }, + /** For the view the list fills, which is there before the list is. */ + onContainerLayout, + scrollToEnd, + }; +}; diff --git a/package/src/ai-components/conversation/list/useEndInView.ts b/package/src/ai-components/conversation/list/useEndInView.ts new file mode 100644 index 0000000000..c9f5144a38 --- /dev/null +++ b/package/src/ai-components/conversation/list/useEndInView.ts @@ -0,0 +1,46 @@ +import { type RefObject, useCallback, useEffect, useRef, useState } from 'react'; + +import type { LegendListRef } from '@legendapp/list/react-native'; + +// How far from the end still counts as at the end. +const END_SLACK = 32; + +export type UseEndInViewOptions = { + /** The room for a composer floating over the bottom of the list, which is not in view. */ + bottomInset: number; + /** The height of the list's header, which its positions leave out. */ + headerSize: RefObject; + listRef: RefObject; +}; + +/** + * Whether the end of the last row's content is in view: not the room the anchored latest turn + * leaves under a short reply, nor the room for the floating composer. Call `scheduleCheckEnd` + * whenever that may have changed: a scroll, or a row changing size without one, such as an answer + * typing out. It checks once a frame at most. + */ +export const useEndInView = ({ bottomInset, headerSize, listRef }: UseEndInViewOptions) => { + const [isEndInView, setIsEndInView] = useState(true); + const bottomInsetRef = useRef(bottomInset); + bottomInsetRef.current = bottomInset; + const checkEnd = useCallback(() => { + const state = listRef.current?.getState(); + const last = (state?.data.length ?? 0) - 1; + if (!state || last < 0) return; + const end = headerSize.current + state.positionAtIndex(last) + state.sizeAtIndex(last); + const visibleEnd = state.scroll + state.scrollLength - bottomInsetRef.current; + setIsEndInView(end <= visibleEnd + END_SLACK); + }, [headerSize, listRef]); + + const checkFrame = useRef(undefined); + const scheduleCheckEnd = useCallback(() => { + if (checkFrame.current !== undefined) return; + checkFrame.current = requestAnimationFrame(() => { + checkFrame.current = undefined; + checkEnd(); + }); + }, [checkEnd]); + useEffect(() => () => cancelAnimationFrame(checkFrame.current ?? 0), []); + + return { isEndInView, scheduleCheckEnd }; +}; diff --git a/package/src/ai-components/conversation/list/useLoadOlder.ts b/package/src/ai-components/conversation/list/useLoadOlder.ts new file mode 100644 index 0000000000..53fad39ef7 --- /dev/null +++ b/package/src/ai-components/conversation/list/useLoadOlder.ts @@ -0,0 +1,97 @@ +import { type RefObject, useCallback, useEffect, useRef } from 'react'; + +import type { LegendListRef } from '@legendapp/list/react-native'; + +// How long the spinner above the turns stays in view before older messages load, so a list that +// opens at its end does not load for the frame its start shows. +const LOAD_OLDER_DELAY_MS = 300; +// How long to wait before asking again after loading older messages failed. +const LOAD_OLDER_RETRY_MS = 3000; +// How close to the start, as a share of the list's height, the spinner counts as in view. +const START_THRESHOLD = 0.1; + +export type UseLoadOlderOptions = { + listRef: RefObject; + /** Loads the rows before the first one. `undefined` once there are none left. */ + loadOlderMessages?: () => Promise | void; +}; + +/** + * Loads older messages once the spinner above the turns has been in view a moment, and again + * while it stays in view after a load that failed, or brought nothing while there is more. Only + * while the list is at rest: on iOS, what arrives above while a finger holds the list, or while + * it glides or bounces, pushes what is in view down rather than staying out of sight. + * + * Returns the list's handlers for it, or none while there is nothing to load. + */ +export const useLoadOlder = ({ listRef, loadOlderMessages }: UseLoadOlderOptions) => { + const dragging = useRef(false); + const gliding = useRef(false); + const loadOlderRef = useRef(loadOlderMessages); + loadOlderRef.current = loadOlderMessages; + const loading = useRef(false); + const loadTimer = useRef | undefined>(undefined); + const mounted = useRef(true); + const scheduleLoadOlder = useCallback( + (delay: number) => { + clearTimeout(loadTimer.current); + loadTimer.current = setTimeout(async () => { + const load = loadOlderRef.current; + if (dragging.current || gliding.current) return; + if (loading.current || !load || !listRef.current?.getState().isNearStart) return; + loading.current = true; + let failed = false; + try { + await load(); + } catch { + failed = true; + } + loading.current = false; + if (mounted.current) { + scheduleLoadOlder(failed ? LOAD_OLDER_RETRY_MS : LOAD_OLDER_DELAY_MS); + } + }, delay); + }, + [listRef], + ); + const onStartReached = useCallback( + () => scheduleLoadOlder(LOAD_OLDER_DELAY_MS), + [scheduleLoadOlder], + ); + // When the list comes to rest, a load the spinner asked for while it moved goes ahead. A glide + // that follows the drag starts before the delay is up, and asks again when it ends. + // A drag that stops a glide ends it, which may not say so itself. + const onScrollBeginDrag = useCallback(() => { + dragging.current = true; + gliding.current = false; + }, []); + const onScrollEndDrag = useCallback(() => { + dragging.current = false; + scheduleLoadOlder(LOAD_OLDER_DELAY_MS); + }, [scheduleLoadOlder]); + const onMomentumScrollBegin = useCallback(() => { + gliding.current = true; + }, []); + const onMomentumScrollEnd = useCallback(() => { + gliding.current = false; + scheduleLoadOlder(LOAD_OLDER_DELAY_MS); + }, [scheduleLoadOlder]); + + useEffect( + () => () => { + mounted.current = false; + clearTimeout(loadTimer.current); + }, + [], + ); + + if (!loadOlderMessages) return undefined; + return { + onMomentumScrollBegin, + onMomentumScrollEnd, + onScrollBeginDrag, + onScrollEndDrag, + onStartReached, + onStartReachedThreshold: START_THRESHOLD, + }; +}; diff --git a/package/src/ai-components/conversation/list/usePinnedAnchor.ts b/package/src/ai-components/conversation/list/usePinnedAnchor.ts new file mode 100644 index 0000000000..68262e1d79 --- /dev/null +++ b/package/src/ai-components/conversation/list/usePinnedAnchor.ts @@ -0,0 +1,133 @@ +import { type RefObject, useCallback, useLayoutEffect, useMemo, useRef, useState } from 'react'; +import type { LayoutChangeEvent } from 'react-native'; + +import type { AnchoredEndSpaceConfig, LegendListRef } from '@legendapp/list/react-native'; + +import { useReduceMotion } from '../../voice-visualization/useReduceMotion'; + +export type UsePinnedAnchorOptions = { + /** The key of the row to pin to the top, such as the latest question's. */ + anchorKey: string | undefined; + data: readonly T[]; + /** The height of the list's header, which its positions leave out. */ + headerSize: RefObject; + keyExtractor: (item: T) => string; + listRef: RefObject; + /** Called once the anchor is laid out, and whenever the room the list leaves under it changes. */ + onAnchorLayout?: () => void; +}; + +/** + * Pins a row to the top of what is visible: the list leaves room under it (`anchoredEndSpace`), + * and a new anchor is scrolled to once it is laid out. A content floor keeps it there while what + * is below it shrinks, before the list has measured the change. + */ +export const usePinnedAnchor = ({ + anchorKey, + data, + headerSize, + keyExtractor, + listRef, + onAnchorLayout, +}: UsePinnedAnchorOptions) => { + const reduceMotion = useReduceMotion(); + const onAnchorLayoutRef = useRef(onAnchorLayout); + onAnchorLayoutRef.current = onAnchorLayout; + + // Where to scroll once the anchor is laid out: a new question pins to the top, and a + // conversation replaced as a whole opens at its end. + const pendingScroll = useRef<{ animated: boolean; key: string } | undefined>(undefined); + const shown = useRef<{ anchorKey?: string; mounted: boolean }>({ mounted: false }); + useLayoutEffect(() => { + const previous = shown.current; + shown.current = { anchorKey, mounted: true }; + if (!previous.mounted || anchorKey === undefined || anchorKey === previous.anchorKey) { + return; + } + const continues = + previous.anchorKey !== undefined && + data.some((item) => keyExtractor(item) === previous.anchorKey); + pendingScroll.current = { animated: continues && !reduceMotion, key: anchorKey }; + }, [anchorKey, data, keyExtractor, reduceMotion]); + + const scrollToPending = useCallback( + (key: string | undefined) => { + const target = pendingScroll.current; + const list = listRef.current; + if (!target || !list || key !== target.key) return; + pendingScroll.current = undefined; + void list.scrollToKey({ animated: target.animated, key: target.key, viewPosition: 0 }); + }, + [listRef], + ); + + // The least the content can be: the latest question's top plus the list's height, as the native + // SDKs' latest turn is at least as tall as the conversation. The list leaves that room under the + // latest turn itself, but only once it has measured what changed, while the content shrinks at + // once when a row below the question does (a reasoning step folding, the thinking going), and the + // list grows at once when the keyboard goes, either of which would pull the question down. So the + // floor is the tallest the list has been at this width, such as without the keyboard. + const viewport = useRef({ maxLength: 0, width: 0 }); + const [contentFloor, setContentFloor] = useState(0); + const updateContentFloor = useCallback( + (key: string | undefined) => { + const state = listRef.current?.getState(); + const position = key === undefined ? undefined : state?.positionByKey(key); + if (!state || position === undefined) { + setContentFloor(0); + return; + } + viewport.current.maxLength = Math.max(viewport.current.maxLength, state.scrollLength); + setContentFloor(Math.ceil(position + headerSize.current + viewport.current.maxLength)); + }, + [headerSize, listRef], + ); + const anchorKeyRef = useRef(anchorKey); + anchorKeyRef.current = anchorKey; + // The list fills this view, which is there before the list is, such as while the conversation + // is empty and the keyboard comes up for its first question. A new width, such as on rotation, + // has heights of its own. + const onContainerLayout = useCallback( + ({ nativeEvent: { layout } }: LayoutChangeEvent) => { + if (layout.width !== viewport.current.width) { + viewport.current = { maxLength: 0, width: layout.width }; + } + if (layout.height <= viewport.current.maxLength) return; + viewport.current.maxLength = layout.height; + updateContentFloor(anchorKeyRef.current); + }, + [updateContentFloor], + ); + // The spinner above the turns coming or going moves the latest question. + const onMetricsChange = useCallback( + ({ headerSize: size }: { headerSize: number }) => { + if (headerSize.current === size) return; + headerSize.current = size; + updateContentFloor(anchorKeyRef.current); + }, + [headerSize, updateContentFloor], + ); + + const anchorIndex = useMemo( + () => + anchorKey === undefined ? -1 : data.findIndex((item) => keyExtractor(item) === anchorKey), + [anchorKey, data, keyExtractor], + ); + const anchoredEndSpace = useMemo( + () => + anchorIndex >= 0 + ? { + anchorIndex, + onReady: ({ anchorKey: readyKey }) => { + updateContentFloor(readyKey); + scrollToPending(readyKey); + onAnchorLayoutRef.current?.(); + }, + onSizeChanged: () => onAnchorLayoutRef.current?.(), + } + : undefined, + [anchorIndex, scrollToPending, updateContentFloor], + ); + + return { anchoredEndSpace, contentFloor, onContainerLayout, onMetricsChange }; +}; diff --git a/package/src/ai-components/conversation/types.ts b/package/src/ai-components/conversation/types.ts new file mode 100644 index 0000000000..42b16e61f4 --- /dev/null +++ b/package/src/ai-components/conversation/types.ts @@ -0,0 +1,45 @@ +import type { AIMessageAttachmentData } from '../message/AIMessageAttachment'; +import type { AIMessagePartData } from '../message-parts/types'; + +/** Who wrote a message: the person using the app (`user`) or the agent (`assistant`). */ +export type AIMessageRole = 'user' | 'assistant' | (string & {}); + +/** + * A message of the conversation as the AI components show it. The counterpart of the + * `AIMessage` protocol in StreamChatAI. `fromStreamMessage` makes one from a Stream message. + */ +export type AIMessage = { + id: string; + /** Who wrote it. Each of the person's messages starts a turn. */ + role: AIMessageRole; + /** What it says. A reply's text is Markdown. */ + text: string; + /** What it carries besides its text, such as images and files. */ + attachments?: AIMessageAttachmentData[]; + /** Whether the agent is still writing it. */ + isGenerating?: boolean; + /** The steps the agent took for it, in order: reasoning, tool calls and kinds of its own. */ + parts?: AIMessagePartData[]; + /** + * The whole of a reasoning step's thinking when the app has it, such as from live updates. + * Without it, the step shows its own excerpt. It is read on every render, so look it up + * rather than work it out. + */ + reasoningText?: (stepId: string) => string | undefined; +}; + +/** + * A message the person sent that is not in the conversation's messages yet. The conversation + * shows it until its copy arrives, and shows the agent thinking meanwhile. + */ +export type AIPendingMessage = { + /** + * The message's id once it arrives, when you know it (such as the id you send it with). Its + * copy is then found by it rather than by where it lands. Keep it the same while it waits. + */ + id: string; + /** What the person wrote. */ + text: string; + /** What they sent with it, such as the photos and files they picked. Its copy shows these too. */ + attachments?: AIMessageAttachmentData[]; +}; diff --git a/package/src/ai-components/conversation/useConversationRows.ts b/package/src/ai-components/conversation/useConversationRows.ts new file mode 100644 index 0000000000..01b19f0868 --- /dev/null +++ b/package/src/ai-components/conversation/useConversationRows.ts @@ -0,0 +1,169 @@ +import { type ReactNode, useLayoutEffect, useMemo, useRef, useState } from 'react'; + +import { + type AIConversationEntry, + type AIConversationTurn, + AIPendingMessageTracker, + turnsOf, +} from './entries'; +import type { AIMessage, AIPendingMessage } from './types'; + +import type { AIMessageAttachmentData } from '../message/AIMessageAttachment'; +import { AIThinkingClock } from '../reasoning/thinkingClock'; + +/** + * Where a row is: the first of a turn, or within one. The first turn has the same room above it as + * the others, so older messages arriving above it leave it where it is. + */ +export type RowPosition = 'turn' | 'message'; + +/** A row of the list: one message, with the turn it belongs to. */ +export type EntryRow = { + entry: AIConversationEntry; + key: string; + kind: 'entry'; + position: RowPosition; + turnKey: string; + /** The turn's first reasoning step, which takes over the seconds the turn was seen thinking. */ + firstStepId?: string; +}; + +/** + * What ends the latest turn: the agent thinking, and the app's footer. A row of its own rather than + * the list's footer: the list lays its rows out itself, together with the room it leaves under the + * latest turn, while a footer that goes away shrinks the content at once, which moves the view. + */ +export type TailRow = { + isThinking: boolean; + key: string; + kind: 'tail'; + turnKey: string; + firstStepId?: string; + footer?: ReactNode; +}; + +export type ConversationRow = EntryRow | TailRow; + +const TAIL_KEY = 'ai-conversation-tail'; + +const sameAttachments = (a: AIMessageAttachmentData[], b: AIMessageAttachmentData[]) => + a === b || (a.length === 0 && b.length === 0); + +// Whether a row can be kept as it was: the list renders a row again only for a new item. +const isSameRow = (row: EntryRow, next: Omit) => + row.entry.message === next.entry.message && + row.entry.pending === next.entry.pending && + row.entry.isLatestReply === next.entry.isLatestReply && + sameAttachments(row.entry.sentAttachments, next.entry.sentAttachments) && + row.firstStepId === next.firstStepId && + row.position === next.position && + row.turnKey === next.turnKey; + +// The turn's first reasoning step, which takes over the seconds the turn was seen thinking. +const firstStepOf = (turn: AIConversationTurn | undefined) => { + for (const { message } of turn?.entries ?? []) { + if (!message || message.role === 'user') continue; + const step = message.parts?.find((part) => part.reasoning); + if (step) return step.id; + } + return undefined; +}; + +export type UseConversationRowsOptions = { + messages: AIMessage[]; + footer?: ReactNode; + isThinking?: boolean; + pendingMessage?: AIPendingMessage | null; +}; + +export type ConversationRows = { + /** The key of the latest turn's first row, which pins to the top: `undefined` with no turns. */ + anchorKey: string | undefined; + /** The turns' thinking clock, for `AIThinkingTurnContext`. */ + clock: AIThinkingClock; + /** The rows to list: each message, then what ends the latest turn while there is any. */ + rows: ConversationRow[]; + /** Whether there are no messages to show yet, not even one on its way. */ + isEmpty: boolean; +}; + +/** + * The conversation's data, as rows for a list: its messages grouped into turns, a message on its + * way until its copy arrives, and what ends the latest turn. Each unchanged row keeps its object, + * so the list renders only the rows that changed. Knows nothing about the list itself. + */ +export const useConversationRows = ({ + footer, + isThinking = false, + messages, + pendingMessage, +}: UseConversationRowsOptions): ConversationRows => { + const [tracker] = useState(() => new AIPendingMessageTracker()); + const [clock] = useState(() => new AIThinkingClock()); + + const entries = useMemo( + () => tracker.entries(messages, pendingMessage), + [messages, pendingMessage, tracker], + ); + // after `entries`, which matched the copies + const thinking = isThinking || tracker.isAwaitingReply(messages, pendingMessage); + const turns = useMemo(() => turnsOf(entries), [entries]); + const latest = turns[turns.length - 1]; + const latestHasReplies = (latest?.entries.length ?? 0) > 1; + const latestFirstStepId = firstStepOf(latest); + const latestHasFirstStep = latestFirstStepId !== undefined; + // Before the views below read it in their effects and next renders: a turn's thinking starts + // and ends once. + useLayoutEffect(() => { + clock.update(latest?.id, thinking, latestHasReplies, latestHasFirstStep); + }, [clock, latest?.id, latestHasFirstStep, latestHasReplies, thinking]); + + // The rows, keeping each unchanged one's object, so the list renders only the rows that changed. + const rowCache = useRef(new Map()); + const entryRows = useMemo(() => { + const cache = new Map(); + const next: EntryRow[] = []; + turns.forEach((turn) => { + const firstStepId = firstStepOf(turn); + turn.entries.forEach((entry, index) => { + const fields = { + entry, + firstStepId, + kind: 'entry' as const, + position: (index > 0 ? 'message' : 'turn') as RowPosition, + turnKey: turn.id, + }; + const previous = rowCache.current.get(entry.id); + const row = + previous && isSameRow(previous, fields) ? previous : { ...fields, key: entry.id }; + cache.set(entry.id, row); + next.push(row); + }); + }); + rowCache.current = cache; + return next; + }, [turns]); + + // A turn's id is its first entry's, so the latest turn's first row has its key. + const lastTurnKey = latest?.id; + const tailRow = useMemo( + () => + lastTurnKey !== undefined && (thinking || footer) + ? { + firstStepId: latestFirstStepId, + footer, + isThinking: thinking, + key: TAIL_KEY, + kind: 'tail', + turnKey: lastTurnKey, + } + : undefined, + [footer, lastTurnKey, latestFirstStepId, thinking], + ); + const rows = useMemo( + () => (tailRow ? [...entryRows, tailRow] : entryRows), + [entryRows, tailRow], + ); + + return { anchorKey: lastTurnKey, clock, isEmpty: entryRows.length === 0, rows }; +}; diff --git a/package/src/ai-components/icons.ts b/package/src/ai-components/icons.ts index 48a1a02145..18a8227d5c 100644 --- a/package/src/ai-components/icons.ts +++ b/package/src/ai-components/icons.ts @@ -3,6 +3,7 @@ import { type ComponentType, useMemo } from 'react'; import { useTheme } from './contexts/theme-context'; import { ArchiveIcon, + ArrowDownIcon, AudioIcon, CancelledIcon, CheckIcon, @@ -41,6 +42,8 @@ export type AIIcon = ComponentType; export const defaultAIIcons = { /** A file of an archive type, such as ZIP, on its attachment. */ Archive: ArchiveIcon, + /** The button back to the latest message of a conversation. */ + ArrowDown: ArrowDownIcon, /** An audio file, on its attachment. */ Audio: AudioIcon, /** A tool call that was cancelled or declined. */ diff --git a/package/src/ai-components/index.ts b/package/src/ai-components/index.ts index 3d79049094..cbdf3d1eb9 100644 --- a/package/src/ai-components/index.ts +++ b/package/src/ai-components/index.ts @@ -6,6 +6,7 @@ export * from './StreamingMessageView'; export * from './AITypingIndicatorView'; export * from './voice-visualization'; +export * from './conversation'; export * from './icons'; export * from './message'; export * from './message-parts'; diff --git a/package/src/ai-components/internal/customFields.ts b/package/src/ai-components/internal/customFields.ts new file mode 100644 index 0000000000..7180e09986 --- /dev/null +++ b/package/src/ai-components/internal/customFields.ts @@ -0,0 +1,11 @@ +const isObject = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +/** + * The object's fields, with those stream-chat keeps under `custom` moved up beside them. An object + * read straight from the API has them at the top level already, and reads the same. + */ +export const withCustomFields = (object: Record): Record => { + const { custom, ...standard } = object; + return isObject(custom) ? { ...standard, ...custom } : object; +}; diff --git a/package/src/ai-components/internal/icons/AgentIcons.tsx b/package/src/ai-components/internal/icons/AgentIcons.tsx index cec2259f0e..c452555bca 100644 --- a/package/src/ai-components/internal/icons/AgentIcons.tsx +++ b/package/src/ai-components/internal/icons/AgentIcons.tsx @@ -55,6 +55,9 @@ export const ReasoningIcon = strokeIcon( 'M7.5 15.625H12.5M8.125 18.125H11.875M10 1.875C6.89 1.875 4.375 4.39 4.375 7.5C4.375 9.53 5.45 11.31 7.06 12.3C7.33 12.47 7.5 12.76 7.5 13.08V13.75H12.5V13.08C12.5 12.76 12.67 12.47 12.94 12.3C14.55 11.31 15.625 9.53 15.625 7.5C15.625 4.39 13.11 1.875 10 1.875Z', ); +// Points down in either direction, so it isn't mirrored. +export const ArrowDownIcon = strokeIcon('M10 3.75V16.25M4.375 10.625L10 16.25L15.625 10.625'); + export const ChevronIcon = strokeIcon('M7.5 4.375L13.125 10L7.5 15.625', { mirrorsInRTL: true }); export const CheckIcon = strokeIcon('M4.375 10.625L8.125 14.375L15.625 5.625'); diff --git a/package/src/ai-components/internal/tokens.ts b/package/src/ai-components/internal/tokens.ts new file mode 100644 index 0000000000..6607e25bde --- /dev/null +++ b/package/src/ai-components/internal/tokens.ts @@ -0,0 +1,4 @@ +// The chat SDK's design tokens, as the AI components read them. Straight from the generated file +// rather than through `theme/index.ts`, which reaches more of the SDK than the AI entry point may. +// Metro picks the platform's tokens (`StreamTokens.{ios,android,web}.ts`). +export { primitives } from '../../theme/generated/light/StreamTokens'; diff --git a/package/src/ai-components/markdown/components/InlineCode.tsx b/package/src/ai-components/markdown/components/InlineCode.tsx index 8663c8c3d0..1f42af175e 100644 --- a/package/src/ai-components/markdown/components/InlineCode.tsx +++ b/package/src/ai-components/markdown/components/InlineCode.tsx @@ -8,6 +8,7 @@ export const InlineCode = ({ children, styles }: MarkdownComponentProps) => ( export const renderInlineCode: RuleRenderFunction = ({ node, output, state, styles }) => { state.withinText = true; + state.withinInlineCode = true; return ( {output(node.content, state)} diff --git a/package/src/ai-components/markdown/components/Text.tsx b/package/src/ai-components/markdown/components/Text.tsx index b6287e02fa..14bbb873d6 100644 --- a/package/src/ai-components/markdown/components/Text.tsx +++ b/package/src/ai-components/markdown/components/Text.tsx @@ -1,11 +1,15 @@ import { useMemo } from 'react'; -import { Text } from 'react-native'; +import { StyleSheet, Text } from 'react-native'; import type { MarkdownComponentProps, RuleRenderFunction } from '../types'; export const TextUI = ({ children, styles, state }: MarkdownComponentProps) => { + // Text in a heading or inline code takes that one's size rather than the body's. + const usesBodySize = !state.withinHeading && !state.withinInlineCode; const textStyle = useMemo(() => { - const style = [styles.text, state.style]; + // eslint-disable-next-line @typescript-eslint/no-unused-vars -- left out to inherit the wrapper's size + const { fontSize, lineHeight, ...unsized } = StyleSheet.flatten(styles.text) ?? {}; + const style = [usesBodySize ? styles.text : unsized, state.style]; if (state.withinQuote) { style.push(styles.blockQuoteText); @@ -17,6 +21,7 @@ export const TextUI = ({ children, styles, state }: MarkdownComponentProps) => { return style; }, [ + usesBodySize, state.style, state.withinLink, state.withinQuote, diff --git a/package/src/ai-components/markdown/styles.ts b/package/src/ai-components/markdown/styles.ts index a23706b896..0a3ebd1303 100644 --- a/package/src/ai-components/markdown/styles.ts +++ b/package/src/ai-components/markdown/styles.ts @@ -1,7 +1,18 @@ import { Dimensions, Platform, StyleSheet } from 'react-native'; +import { primitives } from '../internal/tokens'; import type { Theme } from '../theme'; +// The answer's text, from the design tokens: 17 on iOS and 16 on Android. +const BODY_SIZE = primitives.typographyFontSizeMd; + +// A heading, scaled from the body as StreamChatAI's markdown scales it, in semibold. +const heading = (scale: number) => ({ + fontSize: Math.round(BODY_SIZE * scale), + fontWeight: '600' as const, + lineHeight: Math.round(BODY_SIZE * scale * 1.25), +}); + export const getLocalStyles = ({ theme }: { theme: Theme }) => { const colors = theme.colors; @@ -78,27 +89,13 @@ export const getLocalStyles = ({ theme }: { theme: Theme }) => { em: { fontStyle: 'italic', }, - heading: { - fontWeight: '200', - }, - heading1: { - fontSize: 32, - }, - heading2: { - fontSize: 24, - }, - heading3: { - fontSize: 18, - }, - heading4: { - fontSize: 16, - }, - heading5: { - fontSize: 13, - }, - heading6: { - fontSize: 11, - }, + heading: heading(1), + heading1: heading(2), + heading2: heading(1.5), + heading3: heading(1.17), + heading4: heading(1), + heading5: heading(0.83), + heading6: heading(0.67), hr: { backgroundColor: colors.grey_gainsboro, height: 1, @@ -121,7 +118,7 @@ export const getLocalStyles = ({ theme }: { theme: Theme }) => { color: colors.accent_red, fontFamily: Platform.OS === 'ios' ? 'Courier' : 'Monospace', fontWeight: '500', - fontSize: 13, + fontSize: primitives.typographyFontSizeSm, padding: 3, paddingHorizontal: 5, }, @@ -134,10 +131,12 @@ export const getLocalStyles = ({ theme }: { theme: Theme }) => { }, listItemBullet: { fontSize: 20, - lineHeight: 20, + lineHeight: primitives.typographyLineHeightRelaxed, }, listItemNumber: { + fontSize: BODY_SIZE, fontWeight: '500', + lineHeight: primitives.typographyLineHeightRelaxed, }, listItemText: { flex: 0, @@ -218,6 +217,8 @@ export const getLocalStyles = ({ theme }: { theme: Theme }) => { }, text: { color: colors.black, + fontSize: BODY_SIZE, + lineHeight: primitives.typographyLineHeightRelaxed, }, textRow: { flexDirection: 'row', diff --git a/package/src/ai-components/markdown/types.ts b/package/src/ai-components/markdown/types.ts index 42ddec10a1..4f8ef6109b 100644 --- a/package/src/ai-components/markdown/types.ts +++ b/package/src/ai-components/markdown/types.ts @@ -48,6 +48,7 @@ export type MarkdownState = State & { withinText?: boolean; withinQuote?: boolean; withinHeading?: boolean; + withinInlineCode?: boolean; withinLink?: boolean; withinList?: boolean; withinParagraphWithImage?: boolean; diff --git a/package/src/ai-components/message-composer/ComposerView.tsx b/package/src/ai-components/message-composer/ComposerView.tsx index f214323ece..90ac4ea30b 100644 --- a/package/src/ai-components/message-composer/ComposerView.tsx +++ b/package/src/ai-components/message-composer/ComposerView.tsx @@ -15,6 +15,7 @@ import { MediaPreviewList } from './MediaPreviewList'; import { BottomSheet } from '../components/BottomSheet'; import { ComposerProvider, useMessageComposerContext, useTheme } from '../contexts'; +import { primitives } from '../internal/tokens'; import type { AbstractMediaPickerService, MediaPickerState, @@ -179,7 +180,7 @@ const useStyles = () => { }, textInput: { flex: 1, - fontSize: 16, + fontSize: primitives.typographyFontSizeMd, color: colors.black, paddingVertical: 0, paddingHorizontal: 0, diff --git a/package/src/ai-components/message-parts/AIMessageParts.tsx b/package/src/ai-components/message-parts/AIMessageParts.tsx index 7d2403594f..481631ca0e 100644 --- a/package/src/ai-components/message-parts/AIMessageParts.tsx +++ b/package/src/ai-components/message-parts/AIMessageParts.tsx @@ -8,6 +8,7 @@ import type { AIMessagePartData } from './types'; import { useAITranslation, useTheme } from '../contexts'; import { useAIIcons } from '../icons'; +import { primitives } from '../internal/tokens'; import { AIReasoning } from '../reasoning/AIReasoning'; import type { AIToolApprover } from '../tool-calls/AIToolApprover'; @@ -93,5 +94,5 @@ const styles = StyleSheet.create({ parts: { gap: 8, width: '100%' }, tool: { gap: 8 }, unsupported: { alignItems: 'center', flexDirection: 'row', gap: 8 }, - unsupportedText: { flexShrink: 1, fontSize: 14 }, + unsupportedText: { flexShrink: 1, fontSize: primitives.typographyFontSizeSm }, }); diff --git a/package/src/ai-components/message-parts/AIToolApproval.tsx b/package/src/ai-components/message-parts/AIToolApproval.tsx index 73e3d2efa6..75e0e80204 100644 --- a/package/src/ai-components/message-parts/AIToolApproval.tsx +++ b/package/src/ai-components/message-parts/AIToolApproval.tsx @@ -5,6 +5,7 @@ import { isAwaitingApproval } from './parseMessageParts'; import type { AIToolApprovalData, AIToolCallPart } from './types'; import { useAITranslation, useTheme } from '../contexts'; +import { primitives } from '../internal/tokens'; import { type AIToolApprover, useAIToolApprover } from '../tool-calls/AIToolApprover'; /** Where the person's answer to a tool call's question is. */ @@ -183,11 +184,18 @@ const styles = StyleSheet.create({ paddingHorizontal: 14, paddingVertical: 6, }, - buttonText: { fontSize: 14, fontWeight: '600' }, + buttonText: { fontSize: primitives.typographyFontSizeSm, fontWeight: '600' }, container: { borderRadius: 16, borderWidth: 1, gap: 12, padding: 12 }, declineButton: { borderWidth: 1 }, - failure: { fontSize: 13 }, - message: { fontSize: 14, lineHeight: 20 }, + failure: { fontSize: primitives.typographyFontSizeXs }, + message: { + fontSize: primitives.typographyFontSizeXs, + lineHeight: primitives.typographyLineHeightNormal, + }, text: { gap: 4 }, - title: { fontSize: 15, fontWeight: '600', lineHeight: 20 }, + title: { + fontSize: primitives.typographyFontSizeSm, + fontWeight: '600', + lineHeight: primitives.typographyLineHeightNormal, + }, }); diff --git a/package/src/ai-components/message-parts/AIToolAuthorization.tsx b/package/src/ai-components/message-parts/AIToolAuthorization.tsx index 599960f8be..288c5dd373 100644 --- a/package/src/ai-components/message-parts/AIToolAuthorization.tsx +++ b/package/src/ai-components/message-parts/AIToolAuthorization.tsx @@ -6,6 +6,7 @@ import type { AIToolAuthorizationData, AIToolCallPart } from './types'; import { useAITranslation, useTheme } from '../contexts'; import { useNow } from '../internal/format'; +import { primitives } from '../internal/tokens'; import { type AIToolApprover, type AIToolAuthorizer, @@ -205,15 +206,21 @@ export const AIToolAuthorization = ({ if (mounted.current) setState(next); }; + const sendDecline = (signIn: AIToolAuthorizer) => { + update({ ...idle, isDeclining: true }); + signIn.decline(part).catch(() => update({ ...idle, declineFailed: true })); + }; + const connect = (signIn: AIToolAuthorizer) => { if (state.isConnecting || state.isDeclining) return; setState({ ...idle, isConnecting: true }); signIn.connect(part).then( // Your backend has the provider's answer before the sign-in comes back, and its change to - // the step hides this. Until then it says it is connecting, unless the person closed the - // sign-in or the provider did not connect them. + // the step hides this. Until then it says it is connecting, unless the provider did not + // connect them. A sign-in the person closed can't be opened again, so it declines. (connected) => { - if (!connected) update(idle); + if (connected === 'closed') sendDecline(signIn); + else if (!connected) update(idle); }, () => update({ ...idle, connectFailed: true }), ); @@ -221,8 +228,7 @@ export const AIToolAuthorization = ({ const decline = (signIn: AIToolAuthorizer) => { if (state.isDeclining) return; - setState({ ...idle, isDeclining: true }); - signIn.decline(part).catch(() => update({ ...idle, declineFailed: true })); + sendDecline(signIn); }; const actions = { @@ -256,11 +262,18 @@ const styles = StyleSheet.create({ paddingHorizontal: 14, paddingVertical: 6, }, - buttonText: { fontSize: 14, fontWeight: '600' }, + buttonText: { fontSize: primitives.typographyFontSizeSm, fontWeight: '600' }, container: { borderRadius: 16, borderWidth: 1, gap: 12, padding: 12 }, declineButton: { borderWidth: 1 }, - failure: { fontSize: 13 }, - message: { fontSize: 14, lineHeight: 20 }, + failure: { fontSize: primitives.typographyFontSizeXs }, + message: { + fontSize: primitives.typographyFontSizeXs, + lineHeight: primitives.typographyLineHeightNormal, + }, text: { gap: 4 }, - title: { fontSize: 15, fontWeight: '600', lineHeight: 20 }, + title: { + fontSize: primitives.typographyFontSizeSm, + fontWeight: '600', + lineHeight: primitives.typographyLineHeightNormal, + }, }); diff --git a/package/src/ai-components/message-parts/AIToolCall.tsx b/package/src/ai-components/message-parts/AIToolCall.tsx index ffe3427b49..569dc020b1 100644 --- a/package/src/ai-components/message-parts/AIToolCall.tsx +++ b/package/src/ai-components/message-parts/AIToolCall.tsx @@ -8,6 +8,7 @@ import { useAITranslation, useTheme } from '../contexts'; import { useAIIcons } from '../icons'; import { formatCallDuration } from '../internal/format'; import { Shimmer } from '../internal/Shimmer'; +import { primitives } from '../internal/tokens'; export type AIToolCallProps = { part: AIToolCallPart; @@ -95,12 +96,22 @@ export const AIToolCall = ({ part }: AIToolCallProps) => { const styles = StyleSheet.create({ container: { alignItems: 'flex-start', flexDirection: 'row', gap: 8 }, - detail: { fontSize: 13, lineHeight: 18 }, - duration: { fontSize: 13, fontVariant: ['tabular-nums'], lineHeight: 20 }, + detail: { + fontSize: primitives.typographyFontSizeXs, + lineHeight: primitives.typographyLineHeightTight, + }, + duration: { + fontSize: primitives.typographyFontSizeXs, + fontVariant: ['tabular-nums'], + lineHeight: primitives.typographyLineHeightNormal, + }, icon: { marginTop: 2 }, // The platform spinner, sized to the 16 the icons are drawn at. spinner: { height: 16, transform: [{ scale: 0.8 }], width: 16 }, text: { flex: 1, gap: 2, minWidth: 0 }, - title: { fontSize: 15, lineHeight: 20 }, + title: { + fontSize: primitives.typographyFontSizeSm, + lineHeight: primitives.typographyLineHeightNormal, + }, titleShimmer: { alignSelf: 'flex-start' }, }); diff --git a/package/src/ai-components/message-parts/parseMessageParts.ts b/package/src/ai-components/message-parts/parseMessageParts.ts index 999412e4c1..ce4326b524 100644 --- a/package/src/ai-components/message-parts/parseMessageParts.ts +++ b/package/src/ai-components/message-parts/parseMessageParts.ts @@ -7,6 +7,8 @@ import type { AIToolCallStatus, } from './types'; +import { withCustomFields } from '../internal/customFields'; + /** The attachment type prefix every AI step uses. */ export const AI_MESSAGE_PART_TYPE_PREFIX = 'ai_'; /** The newest format version of the built-in kinds this SDK understands. */ @@ -116,15 +118,16 @@ export const parseMessagePart = ( if (typeof type !== 'string' || !type.startsWith(AI_MESSAGE_PART_TYPE_PREFIX)) { return undefined; } - const id = string(attachment, 'id') ?? `${type}-${position}`; - const version = int(attachment, 'v') ?? 1; + const fields = withCustomFields(attachment); + const id = string(fields, 'id') ?? `${type}-${position}`; + const version = int(fields, 'v') ?? 1; const known = version <= AI_MESSAGE_PART_SUPPORTED_VERSION; return { id, kind: type, - payload: attachment, - reasoning: known && type === 'ai_reasoning' ? parseReasoning(id, attachment) : undefined, - toolCall: known && type === 'ai_tool_call' ? parseToolCall(id, attachment) : undefined, + payload: fields, + reasoning: known && type === 'ai_reasoning' ? parseReasoning(id, fields) : undefined, + toolCall: known && type === 'ai_tool_call' ? parseToolCall(id, fields) : undefined, version, }; }; diff --git a/package/src/ai-components/message/AIAssistantMessage.tsx b/package/src/ai-components/message/AIAssistantMessage.tsx index 176f63dd84..b72914f4e0 100644 --- a/package/src/ai-components/message/AIAssistantMessage.tsx +++ b/package/src/ai-components/message/AIAssistantMessage.tsx @@ -25,6 +25,12 @@ export type AIAssistantMessageProps = { Attachment?: ComponentType; /** What the agent sent with it, such as images and files. */ attachments?: AIMessageAttachmentData[]; + /** + * The message's id. The steps keep their state for it, such as a reasoning step opened by + * hand, and start afresh for another, as when a list reuses this view for another reply. Step + * ids are only unique within a message. + */ + id?: string; /** Whether the agent is still writing it. The text that arrives meanwhile is typed out. */ isGenerating?: boolean; /** Opens a link in the answer, instead of the system's handler. */ @@ -44,6 +50,7 @@ export const AIAssistantMessage = ({ approver, Attachment = AIMessageAttachment, attachments = [], + id, isGenerating = false, onLink, parts = [], @@ -61,7 +68,7 @@ export const AIAssistantMessage = ({ testID='ai-assistant-message' > {parts.length > 0 ? ( - + ) : null} {/* What arrives after mount is typed out; a reply loaded from history shows at once. */} {text !== '' ? : null} diff --git a/package/src/ai-components/message/AIMessageAttachment.tsx b/package/src/ai-components/message/AIMessageAttachment.tsx index 81e63364d9..c443f9b216 100644 --- a/package/src/ai-components/message/AIMessageAttachment.tsx +++ b/package/src/ai-components/message/AIMessageAttachment.tsx @@ -3,6 +3,7 @@ import { Image, Linking, Pressable, StyleSheet, Text, View } from 'react-native' import { useAITranslation, useTheme } from '../contexts'; import { type AIIcons, useAIIcons } from '../icons'; +import { primitives } from '../internal/tokens'; /** * What a message carries besides its text, such as an image or a file. The counterpart of @@ -233,7 +234,7 @@ const defaultStyles = StyleSheet.create({ maxWidth: 280, padding: 8, }, - fileDetails: { fontSize: 12 }, + fileDetails: { fontSize: primitives.typographyFontSizeXs }, fileIcon: { alignItems: 'center', borderRadius: 8, @@ -241,7 +242,7 @@ const defaultStyles = StyleSheet.create({ justifyContent: 'center', width: 36, }, - fileName: { fontSize: 14, fontWeight: '500' }, + fileName: { fontSize: primitives.typographyFontSizeSm, fontWeight: '600' }, fileText: { flexShrink: 1 }, image: { borderRadius: 16, height: 180, overflow: 'hidden', width: 180 }, }); diff --git a/package/src/ai-components/message/AIUserMessage.tsx b/package/src/ai-components/message/AIUserMessage.tsx index 1443c44da2..c53dfc5fa0 100644 --- a/package/src/ai-components/message/AIUserMessage.tsx +++ b/package/src/ai-components/message/AIUserMessage.tsx @@ -9,6 +9,7 @@ import { import { useAITranslation, useTheme } from '../contexts'; import { useAIIcons } from '../icons'; +import { primitives } from '../internal/tokens'; export type AIUserMessageProps = { /** What the person wrote. */ @@ -114,8 +115,8 @@ const defaultStyles = StyleSheet.create({ maxWidth: '100%', }, text: { - fontSize: 16, - lineHeight: 22, + fontSize: primitives.typographyFontSizeMd, + lineHeight: primitives.typographyLineHeightRelaxed, writingDirection: I18nManager.isRTL ? 'rtl' : 'ltr', }, }); diff --git a/package/src/ai-components/reasoning/AIReasoning.tsx b/package/src/ai-components/reasoning/AIReasoning.tsx index 5ec972bcaf..19ebbc4fa9 100644 --- a/package/src/ai-components/reasoning/AIReasoning.tsx +++ b/package/src/ai-components/reasoning/AIReasoning.tsx @@ -17,6 +17,7 @@ import { type DeepPartial, StreamTheme, useAITranslation, useTheme } from '../co import { useAIIcons } from '../icons'; import { formatMinutesAndSeconds, useNow } from '../internal/format'; import { Shimmer } from '../internal/Shimmer'; +import { primitives } from '../internal/tokens'; import { Markdown } from '../markdown'; import type { AIReasoningPart } from '../message-parts/types'; import type { Theme } from '../theme'; @@ -285,7 +286,11 @@ export const AIReasoning = ({ () => ({ markdown: { paragraph: { marginBottom: 4, marginTop: 4 }, - text: { color: colors.grey, fontSize: 14, lineHeight: 20 }, + text: { + color: colors.grey, + fontSize: primitives.typographyFontSizeSm, + lineHeight: primitives.typographyLineHeightNormal, + }, }, }), [colors.grey], @@ -372,6 +377,10 @@ const styles = StyleSheet.create({ maxWidth: '100%', }, lead: { alignItems: 'center', flexDirection: 'row', gap: 6 }, - summary: { flexShrink: 1, fontSize: 14 }, - title: { fontSize: 14, fontVariant: ['tabular-nums'], fontWeight: '500' }, + summary: { flexShrink: 1, fontSize: primitives.typographyFontSizeSm }, + title: { + fontSize: primitives.typographyFontSizeSm, + fontVariant: ['tabular-nums'], + fontWeight: '500', + }, }); diff --git a/package/src/ai-components/theme.ts b/package/src/ai-components/theme.ts index 6d5c63ff5c..4f90929835 100644 --- a/package/src/ai-components/theme.ts +++ b/package/src/ai-components/theme.ts @@ -118,6 +118,27 @@ export type Theme = { text?: TextStyle; }; }; + /** + * `AIConversation`: the list of turns, the spinner while older messages load, and the button back + * to the latest message. Optional, so a hand-built `Theme` without it keeps type-checking. + */ + conversation?: { + /** The spinner while older messages load. */ + loadingIndicator?: string; + /** The button back to the latest message. */ + scrollButtonBackground?: string; + scrollButtonBorder?: string; + scrollButtonIcon?: string; + container?: ViewStyle; + /** The list's content: the turns, with the room around them. */ + content?: ViewStyle; + /** What shows while there are no messages. */ + empty?: ViewStyle; + loadingOlder?: ViewStyle; + scrollButton?: ViewStyle; + /** The view around each message, and around what ends the latest turn. */ + row?: ViewStyle; + }; /** * Icons of your own for the agent replies' views, by name, over `defaultAIIcons`. Kept as they * are when themes merge. @@ -275,6 +296,19 @@ export const defaultTheme: Theme = { }, unsupported: { container: {}, text: {} }, }, + conversation: { + // StreamChatAI's defaults, from the shared design tokens. + loadingIndicator: '#414552', + scrollButtonBackground: '#FFFFFF', + scrollButtonBorder: '#D5DBE1', + scrollButtonIcon: '#1A1B25', + container: {}, + content: {}, + empty: {}, + loadingOlder: {}, + scrollButton: {}, + row: {}, + }, message: { // StreamChatAI's and the Android AI SDK's defaults, from the shared design tokens. userMessageBackground: '#EBEEF1', diff --git a/package/src/ai-components/tool-calls/AIToolApprover.tsx b/package/src/ai-components/tool-calls/AIToolApprover.tsx index 73709e81e2..d8a28f9110 100644 --- a/package/src/ai-components/tool-calls/AIToolApprover.tsx +++ b/package/src/ai-components/tool-calls/AIToolApprover.tsx @@ -11,10 +11,12 @@ export type AIToolAuthorizer = { /** * Opens the call's sign-in. Resolves `true` once the account is connected: your backend then * updates the call's step, and until it does the sign-in reads as connecting. Resolves `false` - * when the person closed the sign-in or it ended without connecting, so they can try again. - * Rejects when the sign-in could not be opened. + * when it ended without connecting, so the person can try again. Resolves `'closed'` when the + * person closed it and it can't be opened again, as the router's sign-in can't once its page + * has opened: the call is then declined, as with "Not now". Rejects when the sign-in could + * not be opened. */ - connect: (call: AIToolCallPart) => Promise; + connect: (call: AIToolCallPart) => Promise; /** * Tells your backend the person won't sign in now. Your backend ends the call, which never * runs. A rejection lets the person answer again. diff --git a/package/src/ai-components/tool-calls/authSessionAuthorizer.ts b/package/src/ai-components/tool-calls/authSessionAuthorizer.ts index 8fe64f716e..bf162ac354 100644 --- a/package/src/ai-components/tool-calls/authSessionAuthorizer.ts +++ b/package/src/ai-components/tool-calls/authSessionAuthorizer.ts @@ -86,6 +86,12 @@ export type AuthSessionAuthorizerOptions = { * the person answer again. */ decline: (call: AIToolCallPart) => Promise; + /** + * Whether closing the sign-in declines the call, as "Not now" does. Defaults to `true`: the + * router's sign-in can be opened once, so after a close its page would only say it expired, and + * asking again starts a new one. Pass `false` if your app can give the call a new sign-in. + */ + declineOnClose?: boolean; }; export type AuthSessionAuthorizer = AIToolAuthorizer & { callbackScheme: string }; @@ -95,13 +101,14 @@ export type AuthSessionAuthorizer = AIToolAuthorizer & { callbackScheme: string * `ASWebAuthenticationSession` on iOS, and on Android the browser's Auth Tab, or a Custom Tab when * the browser has none. * - * `connect` resolves `true` when the return URL says the account is connected, and `false` when - * the person closed the sign-in or it ended otherwise. It rejects with an `AIConnectorSignInError` - * when the sign-in couldn't be opened. + * `connect` resolves `true` when the return URL says the account is connected, `'closed'` when + * the person closed the sign-in (`false` with `declineOnClose: false`), and `false` when it ended + * otherwise. It rejects with an `AIConnectorSignInError` when the sign-in couldn't be opened. */ export const createAuthSessionAuthorizer = ({ callbackScheme, decline, + declineOnClose = true, }: AuthSessionAuthorizerOptions): AuthSessionAuthorizer => { const connect = async (call: AIToolCallPart) => { const authorization = call.authorization; @@ -118,7 +125,9 @@ export const createAuthSessionAuthorizer = ({ error instanceof Error ? error.message : undefined, ); } - return returned !== '' && readConnectorSignInResult(returned)?.status === 'connected'; + // Nothing comes back when the person closed the sheet or the tab. + if (returned === '') return declineOnClose ? ('closed' as const) : false; + return readConnectorSignInResult(returned)?.status === 'connected'; }; return { callbackScheme, connect, decline }; diff --git a/yarn.lock b/yarn.lock index 3f76961f56..ea2fe376b4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3877,6 +3877,22 @@ __metadata: languageName: node linkType: hard +"@legendapp/list@npm:^3.7.1": + version: 3.7.1 + resolution: "@legendapp/list@npm:3.7.1" + dependencies: + use-sync-external-store: "npm:^1.5.0" + peerDependencies: + react: "*" + peerDependenciesMeta: + react-dom: + optional: true + react-native: + optional: true + checksum: 10c0/a445838e5b99a637fe88b0badc408dc2b9763e57e9e107c0122135359555521c126c44e50767754ec297ec2788874d9b9fec76ce3c90fad5444586e40addd303 + languageName: node + linkType: hard + "@napi-rs/wasm-runtime@npm:^0.2.11": version: 0.2.12 resolution: "@napi-rs/wasm-runtime@npm:0.2.12" @@ -6128,6 +6144,21 @@ __metadata: languageName: node linkType: hard +"@stream-io/calypso@npm:^0.2.3": + version: 0.2.4 + resolution: "@stream-io/calypso@npm:0.2.4" + peerDependencies: + "@stream-io/video-client": ">=1.0.0" + stream-chat: ">=9.0.0" + peerDependenciesMeta: + "@stream-io/video-client": + optional: true + stream-chat: + optional: true + checksum: 10c0/91cd9f8300306d055d757a14871b583474a51aa3c0ef807f23018cf0a5450c8c08771435727e0b8e259afe640b0d77608373f1c516e17456cc8fe13df01f7406 + languageName: node + linkType: hard + "@stream-io/i18n@npm:^1.1.0": version: 1.1.0 resolution: "@stream-io/i18n@npm:1.1.0" @@ -10950,6 +10981,7 @@ __metadata: "@react-native-community/netinfo": "npm:12.0.1" "@rnx-kit/metro-config": "npm:^2.1.2" "@shopify/react-native-skia": "npm:^2.14.0" + "@stream-io/calypso": "npm:^0.2.3" "@stream-io/typescript-config": "workspace:^" "@types/react": "npm:~19.3.0" expo: "npm:^58.0.8" @@ -19478,6 +19510,7 @@ __metadata: "@babel/runtime": "npm:^7.29.2" "@gorhom/bottom-sheet": "npm:^5.2.14" "@khanacademy/simple-markdown": "npm:^2.1.0" + "@legendapp/list": "npm:^3.7.1" "@op-engineering/op-sqlite": "npm:^18.2.5" "@react-native-community/netinfo": "npm:^12.0.1" "@react-native/babel-preset": "npm:0.88.0" From 00fd5d632c5ccfb6195ba1fc69abc242ba13e8cf Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 23:57:08 +0100 Subject: [PATCH 07/23] feat(ai-components): chat options, suggestions and attachment preview fixes Ported from #3931 on v9-release. --- .../ExpoChatGPTSample/bottomSheetOptions.tsx | 45 ------ examples/ExpoChatGPTSample/chatOptions.ts | 19 +++ .../ExpoChatGPTSample/contexts/AppContext.tsx | 68 ++++---- .../hooks/useSendAIMessage.ts | 8 +- .../ExpoChatGPTSample/screens/ChatContent.tsx | 41 ++++- .../__tests__/ComposerView.test.tsx | 103 ++++++++++++ .../contexts/composer-context.tsx | 13 +- package/src/ai-components/index.ts | 1 + .../message-composer/ActionSheet.tsx | 27 +++- .../message-composer/ChatOptions.tsx | 146 ++++++++++++++++++ .../message-composer/ComposerView.tsx | 36 ++++- .../message-composer/MediaPreviewList.tsx | 42 ++++- .../ai-components/message-composer/index.ts | 1 + .../src/ai-components/store/composer/store.ts | 4 + .../store/composer/useComposerChatOption.ts | 15 ++ package/src/ai-components/store/index.ts | 1 + .../suggestions/AISuggestions.tsx | 73 +++++++++ .../src/ai-components/suggestions/index.ts | 1 + package/src/ai-components/theme.ts | 56 +++++++ 19 files changed, 603 insertions(+), 97 deletions(-) delete mode 100644 examples/ExpoChatGPTSample/bottomSheetOptions.tsx create mode 100644 examples/ExpoChatGPTSample/chatOptions.ts create mode 100644 package/src/ai-components/message-composer/ChatOptions.tsx create mode 100644 package/src/ai-components/store/composer/useComposerChatOption.ts create mode 100644 package/src/ai-components/suggestions/AISuggestions.tsx create mode 100644 package/src/ai-components/suggestions/index.ts diff --git a/examples/ExpoChatGPTSample/bottomSheetOptions.tsx b/examples/ExpoChatGPTSample/bottomSheetOptions.tsx deleted file mode 100644 index b83226ad5d..0000000000 --- a/examples/ExpoChatGPTSample/bottomSheetOptions.tsx +++ /dev/null @@ -1,45 +0,0 @@ -import React from 'react'; -import { Alert } from 'react-native'; - -import { Copy, Edit, FilePickerIcon, Flag, IconProps, Picture } from 'stream-chat-expo'; - -// The sheet renders each icon without props, and some SDK icons (e.g. `Flag`, `Copy`) only draw -// with an explicit stroke, so every icon gets the same colour here. -const withStroke = (Icon: React.ComponentType) => { - const StrokedIcon = () => ; - StrokedIcon.displayName = `Stroked(${Icon.displayName ?? Icon.name})`; - return StrokedIcon; -}; - -export const bottomSheetOptions = [ - { - title: 'Create Image', - subtitle: 'Visualize anything', - action: () => Alert.alert('Pressed on Create Image !'), - Icon: withStroke(Picture), - }, - { - title: 'Thinking', - subtitle: 'Think longer for better answers', - action: () => Alert.alert('Pressed on Thinking !'), - Icon: withStroke(Flag), - }, - { - title: 'Deep research', - subtitle: 'Get a detailed report', - action: () => Alert.alert('Pressed on Deep research !'), - Icon: withStroke(FilePickerIcon), - }, - { - title: 'Web search', - subtitle: 'Find real-time news and info', - action: () => Alert.alert('Pressed on Web search !'), - Icon: withStroke(Copy), - }, - { - title: 'Study and learn', - subtitle: 'Learn a new concept', - action: () => Alert.alert('Pressed on Study and learn !'), - Icon: withStroke(Edit), - }, -]; diff --git a/examples/ExpoChatGPTSample/chatOptions.ts b/examples/ExpoChatGPTSample/chatOptions.ts new file mode 100644 index 0000000000..955430edfe --- /dev/null +++ b/examples/ExpoChatGPTSample/chatOptions.ts @@ -0,0 +1,19 @@ +import type { AIChatOption } from 'stream-chat-expo/ai-components'; + +/** Asks with more time to reason, as Juno does. */ +export const THINK_HARDER: AIChatOption = { + description: 'More time to reason before answering', + id: 'think_harder', + shortTitle: 'Think', + title: 'Think harder', +}; + +export const chatOptions = [THINK_HARDER]; + +/** Starters for an empty conversation. */ +export const STARTERS = [ + "What's on my calendar today?", + 'Remind me to call mum at 6pm', + 'Plan a weekend in Lisbon', + 'What should I cook tonight?', +]; diff --git a/examples/ExpoChatGPTSample/contexts/AppContext.tsx b/examples/ExpoChatGPTSample/contexts/AppContext.tsx index f08cf43639..920815accf 100644 --- a/examples/ExpoChatGPTSample/contexts/AppContext.tsx +++ b/examples/ExpoChatGPTSample/contexts/AppContext.tsx @@ -57,8 +57,11 @@ export type AppContextValue = { pending: AIPendingMessage | undefined; /** The open conversation's session id. */ sessionId: string | undefined; - /** Asks the agent, with photos by their local URIs. Its reply arrives in the channel. */ - ask: (text: string, photos?: string[]) => Promise; + /** + * Asks the agent, with photos by their local URIs, and with more time to reason when + * `thinkHarder`. Its reply arrives in the channel. + */ + ask: (text: string, photos?: string[], thinkHarder?: boolean) => Promise; /** Carries on a conversation from the drawer, by its session id. */ openConversation: (sessionId: string) => void; /** Rates a reply, or takes the rating back with null. */ @@ -161,33 +164,40 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St const isResponding = agentState.kind === 'responding' || agentState.kind === 'working'; const isBusy = !!asked || isResponding || followed?.lastReply?.generating === true; - const ask = useStableCallback(async (text: string, photos: string[] = []) => { - const opened = current.current; - if (!opened || isBusy) { - return; - } - // Shows at once, with the photos as they were picked. - const asking: Asked = { - after: userMessageCount(opened.state.getLatestValue().messages), - message: { - attachments: photos.map((url) => ({ kind: 'image', url })), - id: crypto.randomUUID(), - text, - }, - }; - const messages = opened.state.getLatestValue().messages; - setFailure(undefined); - setAsked(asking); - setAskedAfter(messages[messages.length - 1]?.id ?? ''); - try { - const images = await Promise.all(photos.map(async (uri) => ({ url: await toDataUri(uri) }))); - await opened.session.responses.create(text, images.length > 0 ? { images } : undefined); - } catch (error) { - setFailure(messageOf(error)); - setAskedAfter(undefined); - if (askedRef.current === asking) setAsked(undefined); - } - }); + const ask = useStableCallback( + async (text: string, photos: string[] = [], thinkHarder = false) => { + const opened = current.current; + if (!opened || isBusy) { + return; + } + // Shows at once, with the photos as they were picked. + const asking: Asked = { + after: userMessageCount(opened.state.getLatestValue().messages), + message: { + attachments: photos.map((url) => ({ kind: 'image', url })), + id: crypto.randomUUID(), + text, + }, + }; + const messages = opened.state.getLatestValue().messages; + setFailure(undefined); + setAsked(asking); + setAskedAfter(messages[messages.length - 1]?.id ?? ''); + try { + const images = await Promise.all( + photos.map(async (uri) => ({ url: await toDataUri(uri) })), + ); + await opened.session.responses.create(text, { + ...(images.length > 0 ? { images } : {}), + ...(thinkHarder ? { reasoningEffort: 'high' } : {}), + }); + } catch (error) { + setFailure(messageOf(error)); + setAskedAfter(undefined); + if (askedRef.current === asking) setAsked(undefined); + } + }, + ); // Stream's copy of the question on its way is the person's first message after the ones there // were when they asked: once it is in, the question on its way goes. diff --git a/examples/ExpoChatGPTSample/hooks/useSendAIMessage.ts b/examples/ExpoChatGPTSample/hooks/useSendAIMessage.ts index 368441f67d..f597430149 100644 --- a/examples/ExpoChatGPTSample/hooks/useSendAIMessage.ts +++ b/examples/ExpoChatGPTSample/hooks/useSendAIMessage.ts @@ -2,22 +2,24 @@ import { useStableCallback } from 'stream-chat-expo'; import type { ComposerViewProps } from 'stream-chat-expo/ai-components'; +import { THINK_HARDER } from '../chatOptions'; import { useAppContext } from '../contexts/AppContext'; export type AIMessage = Parameters[0]; /** - * Asks the agent, with the photos picked in the composer. The question shows at once, and the + * Asks the agent, with the photos picked in the composer, and more time to reason when "Think + * harder" is picked. The question shows at once, and the * router writes it and its reply into the conversation's channel. Shared by the composer and voice * mode. */ export const useSendAIMessage = () => { const { ask } = useAppContext(); - return useStableCallback(async ({ text, attachments }: AIMessage) => { + return useStableCallback(async ({ attachments, chatOption, text }: AIMessage) => { const photos = (attachments ?? []) .filter((asset) => !asset.type || asset.type.startsWith('image')) .map((asset) => asset.uri); - await ask(text, photos); + await ask(text, photos, chatOption?.id === THINK_HARDER.id); }); }; diff --git a/examples/ExpoChatGPTSample/screens/ChatContent.tsx b/examples/ExpoChatGPTSample/screens/ChatContent.tsx index 65aec8a102..fa051c4843 100644 --- a/examples/ExpoChatGPTSample/screens/ChatContent.tsx +++ b/examples/ExpoChatGPTSample/screens/ChatContent.tsx @@ -10,18 +10,22 @@ import { KeyboardCompatibleView, useStateStore } from 'stream-chat-expo'; import { AIConversation, type AIMessage, + AISuggestions, AITypingIndicatorView, ComposerView, createFromStreamMessage, + createNewComposerStore, } from 'stream-chat-expo/ai-components'; import type { AgentChat, AgentChatState } from '../agent/agentChat'; -import { bottomSheetOptions } from '../bottomSheetOptions'; +import { chatOptions, STARTERS, THINK_HARDER } from '../chatOptions'; import { VoiceMode } from '../components/VoiceMode'; import { useAppContext } from '../contexts/AppContext'; import { useSendAIMessage } from '../hooks/useSendAIMessage'; -const CustomComposerView = () => { +type Composer = ReturnType; + +const CustomComposerView = ({ composer }: { composer: Composer }) => { const { isBusy, stop } = useAppContext(); // The router stops through its own session, not with an AI indicator event. @@ -42,7 +46,8 @@ const CustomComposerView = () => { return ( { const EmptyState = () => What can I help with ?; +const isEmptySelector = ({ messages }: AgentChatState) => ({ isEmpty: messages.length === 0 }); + +/** + * Starters, above the composer, until the conversation has a first question. One is asked with the + * option picked in the composer, as the composer would ask it. + */ +const Starters = ({ chat, composer }: { chat: AgentChat; composer: Composer }) => { + const { ask, pending } = useAppContext(); + const { isEmpty } = useStateStore(chat.state, isEmptySelector); + if (!isEmpty || pending) return null; + const onSelect = (text: string) => { + const { chatOption } = composer.getLatestValue(); + void ask(text, [], chatOption?.id === THINK_HARDER.id); + }; + return ; +}; + +/** The starters and the composer, sharing the composer's draft and picked option. */ +const ComposerArea = ({ chat }: { chat: AgentChat }) => { + const [composer] = useState(createNewComposerStore); + return ( + <> + + + + ); +}; + const conversationSelector = ({ hasOlderMessages, messages }: AgentChatState) => ({ hasOlderMessages, messages, @@ -130,7 +163,7 @@ export const ChatContent = () => { {failure ? {failure} : null} - + ); diff --git a/package/src/ai-components/__tests__/ComposerView.test.tsx b/package/src/ai-components/__tests__/ComposerView.test.tsx index ba6b8c5318..096fa12b2a 100644 --- a/package/src/ai-components/__tests__/ComposerView.test.tsx +++ b/package/src/ai-components/__tests__/ComposerView.test.tsx @@ -4,6 +4,7 @@ import { Image, Text } from 'react-native'; import { act, fireEvent, render, screen, waitFor } from '@testing-library/react-native'; import { + AISuggestions, ComposerView, type ComposerViewProps, sheetStore, @@ -110,4 +111,106 @@ describe('ai-components ComposerView', () => { screen.UNSAFE_queryAllByType(Text).some((t) => t.props.children === 'With an image'), ).toBe(false); }); + + it('sends the picked chat option, which stays picked until removed', async () => { + const think = { + description: 'More time to reason before answering', + id: 'think_harder', + shortTitle: 'Think', + title: 'Think harder', + }; + const { onSendMessage } = renderComposer({ chatOptions: [think] }); + + fireEvent.press(screen.getByText('+')); + expect(screen.getByText('More time to reason before answering')).toBeTruthy(); + expect(screen.getByRole('radio', { checked: false })).toBeTruthy(); + await act(async () => { + fireEvent.press(screen.getByText('Think harder')); + await flushPromises(); + }); + expect(sheetStore.getLatestValue().open).toBe(false); + expect(screen.getByTestId('ai-selected-chat-option')).toBeTruthy(); + expect(screen.getByText('Think')).toBeTruthy(); + + fireEvent.changeText(screen.getByPlaceholderText('Ask anything'), 'Why is the sky blue?'); + await act(async () => { + fireEvent.press(screen.getByTestId('send-up')); + await flushPromises(); + }); + expect(onSendMessage).toHaveBeenCalledWith( + expect.objectContaining({ chatOption: think, text: 'Why is the sky blue?' }), + ); + expect(screen.getByText('Think')).toBeTruthy(); + + fireEvent.press(screen.getByLabelText('Remove option: Think')); + expect(screen.queryByTestId('ai-selected-chat-option')).toBeNull(); + + fireEvent.changeText(screen.getByPlaceholderText('Ask anything'), 'And now?'); + await act(async () => { + fireEvent.press(screen.getByTestId('send-up')); + await flushPromises(); + }); + expect(onSendMessage).toHaveBeenLastCalledWith( + expect.objectContaining({ chatOption: undefined, text: 'And now?' }), + ); + }); + + it('removes the picked chat option when it is picked again', async () => { + renderComposer({ chatOptions: [{ id: 'search', title: 'Web search' }] }); + + act(() => { + sheetStoreApi.openSheet(); + }); + await act(async () => { + fireEvent.press(screen.getByText('Web search')); + await flushPromises(); + }); + // Without a short title, the chip shows the title. + expect(screen.getByLabelText('Remove option: Web search')).toBeTruthy(); + + act(() => { + sheetStoreApi.openSheet(); + }); + expect(screen.getByRole('radio', { checked: true })).toBeTruthy(); + await act(async () => { + fireEvent.press(screen.getByRole('radio')); + await flushPromises(); + }); + expect(screen.queryByTestId('ai-selected-chat-option')).toBeNull(); + }); + + it('shows a picked file that is not an image as a file, with a way to remove it', () => { + const mediaPickerService = new FakeMediaPickerService(); + mediaPickerService.appendAssets([ + { name: 'clip.mp4', size: 2048, type: 'video/mp4', uri: 'file:///clip.mp4' }, + ]); + renderComposer({ mediaPickerService }); + + expect(screen.UNSAFE_queryAllByType(Image)).toHaveLength(0); + expect(screen.getByText('clip.mp4')).toBeTruthy(); + expect(screen.getByText('MP4 · 2.00 kB')).toBeTruthy(); + + fireEvent.press(screen.getByLabelText('Remove attachment')); + expect(mediaPickerService.state.getLatestValue().assets).toEqual([]); + }); +}); + +describe('AISuggestions', () => { + it('hands the picked suggestion over, and shows nothing without any', () => { + const onSelect = jest.fn(); + const { rerender } = render( + + + , + ); + fireEvent.press(screen.getByRole('button', { name: 'Plan a weekend in Lisbon' })); + expect(onSelect).toHaveBeenCalledWith('Plan a weekend in Lisbon'); + + rerender( + + + , + ); + expect(screen.queryByTestId('ai-suggestions')).toBeNull(); + }); }); diff --git a/package/src/ai-components/contexts/composer-context.tsx b/package/src/ai-components/contexts/composer-context.tsx index b825409744..982f029217 100644 --- a/package/src/ai-components/contexts/composer-context.tsx +++ b/package/src/ai-components/contexts/composer-context.tsx @@ -4,6 +4,7 @@ import type { StateStore } from '@stream-io/state-store'; import { useStableCallback } from '../internal/hooks/useStableCallback'; import type { ComposerViewProps } from '../message-composer'; +import type { AIChatOption } from '../message-composer/ChatOptions'; import { MediaPickerService } from '../services'; import type { AbstractMediaPickerService } from '../services/media-picker-service/AbstractMediaPickerService'; import { type ComposerState, createNewComposerStore } from '../store'; @@ -11,6 +12,8 @@ import { type ComposerState, createNewComposerStore } from '../store'; export type ComposerContext = { state: StateStore; setText: (text: string) => void | Promise; + /** Picks the chat option for the next message, or removes it with `null`. */ + setChatOption: (option: AIChatOption | null) => void; sendMessage: () => void | Promise; mediaPickerService?: AbstractMediaPickerService; }; @@ -33,6 +36,9 @@ export const ComposerProvider = ({ const [state] = useState(() => stateOverride ?? createNewComposerStore()); const setText = useStableCallback((text: string) => state.partialNext({ text })); + const setChatOption = useStableCallback((chatOption: AIChatOption | null) => + state.partialNext({ chatOption }), + ); const clearState = useStableCallback(() => { setText(''); @@ -40,11 +46,12 @@ export const ComposerProvider = ({ }); const sendMessage = useStableCallback(async () => { - const { text, custom } = state.getLatestValue(); + const { chatOption, custom, text } = state.getLatestValue(); const data = { text, attachments: mediaPickerService?.state.getLatestValue().assets, + chatOption: chatOption ?? undefined, custom, }; @@ -54,8 +61,8 @@ export const ComposerProvider = ({ }); const contextValue = useMemo( - () => ({ mediaPickerService, state, setText, sendMessage, clearState }), - [state, mediaPickerService, setText, sendMessage, clearState], + () => ({ mediaPickerService, state, setChatOption, setText, sendMessage, clearState }), + [state, mediaPickerService, setChatOption, setText, sendMessage, clearState], ); return {children}; diff --git a/package/src/ai-components/index.ts b/package/src/ai-components/index.ts index cbdf3d1eb9..c2558185ce 100644 --- a/package/src/ai-components/index.ts +++ b/package/src/ai-components/index.ts @@ -11,6 +11,7 @@ export * from './icons'; export * from './message'; export * from './message-parts'; export * from './reasoning'; +export * from './suggestions'; export * from './tool-calls'; export * from './components'; diff --git a/package/src/ai-components/message-composer/ActionSheet.tsx b/package/src/ai-components/message-composer/ActionSheet.tsx index 5b4f999342..2a3e38d4da 100644 --- a/package/src/ai-components/message-composer/ActionSheet.tsx +++ b/package/src/ai-components/message-composer/ActionSheet.tsx @@ -1,6 +1,7 @@ import React, { useCallback, useMemo } from 'react'; import { Pressable, ScrollView, StyleSheet, Text, View } from 'react-native'; +import { ChatOptionsList } from './ChatOptions'; import type { BottomSheetOption, ComposerViewProps } from './ComposerView'; import { withCloseSheet } from './utils/withCloseSheet'; @@ -8,13 +9,17 @@ import { withCloseSheet } from './utils/withCloseSheet'; import { useMessageComposerContext, useTheme } from '../contexts'; import { Camera } from '../internal/icons/Camera'; import { Picture } from '../internal/icons/Picture'; -import { sheetStoreApi } from '../store'; +import { sheetStoreApi, useComposerChatOption } from '../store'; -type BottomSheetContentProps = Pick; +type BottomSheetContentProps = Pick< + ComposerViewProps, + 'bottomSheetOptions' | 'bottomSheetInsets' | 'chatOptions' +>; export const BottomSheetContent = ({ bottomSheetOptions, bottomSheetInsets, + chatOptions, }: BottomSheetContentProps) => { const { theme: { @@ -23,7 +28,9 @@ export const BottomSheetContent = ({ } = useTheme(); const styles = useStyles(); - const { mediaPickerService } = useMessageComposerContext(); + const { mediaPickerService, setChatOption } = useMessageComposerContext(); + const { chatOption } = useComposerChatOption(); + const selectedId = chatOption?.id ?? null; const onContentSizeChange = useCallback( (_: unknown, height: number) => sheetStoreApi.setHeight(height), @@ -54,6 +61,20 @@ export const BottomSheetContent = ({ /> + {chatOptions && chatOptions.length > 0 ? ( + <> + + + setChatOption(option.id === selectedId ? null : option), + )} + options={chatOptions} + selectedId={selectedId} + /> + + ) : null} + {bottomSheetOptions && bottomSheetOptions.length > 0 ? ( <> diff --git a/package/src/ai-components/message-composer/ChatOptions.tsx b/package/src/ai-components/message-composer/ChatOptions.tsx new file mode 100644 index 0000000000..bf39e22dd5 --- /dev/null +++ b/package/src/ai-components/message-composer/ChatOptions.tsx @@ -0,0 +1,146 @@ +import React, { type ReactNode } from 'react'; +import { Pressable, StyleSheet, Text, View } from 'react-native'; + +import { useAITranslation, useTheme } from '../contexts'; +import { Close } from '../internal/icons/Close'; +import { primitives } from '../internal/tokens'; + +/** + * A way to ask, which the person picks for their next message, such as "Think harder". The + * counterpart of `ChatOption` in StreamChatAI. + */ +export type AIChatOption = { + id: string; + /** The option's name in the menu, such as "Think harder". */ + title: string; + /** Data of your own, such as what the option sends with the message. */ + customData?: Record; + /** A line under the title in the menu, such as "Takes longer, for harder questions". */ + description?: string; + /** An icon shown in the menu and on the chip. */ + icon?: ReactNode; + /** The option's name on the chip once picked, such as "Think". Defaults to `title`. */ + shortTitle?: string; +}; + +export type ChatOptionsListProps = { + onSelect: (option: AIChatOption) => void; + options: AIChatOption[]; + selectedId?: string | null; +}; + +/** + * The items of the chat options menu, one per option, each with its icon, title and description. + * `ComposerView` shows them in its sheet, under the camera and photos. The counterpart of + * `ChatOptionsList`. + */ +export const ChatOptionsList = ({ onSelect, options, selectedId }: ChatOptionsListProps) => { + const { + theme: { chatOptions: theme }, + } = useTheme(); + + return ( + + {options.map((option) => ( + onSelect(option)} + style={({ pressed }) => [styles.item, { opacity: pressed ? 0.7 : 1 }, theme?.item]} + > + {option.icon ? ( + {option.icon} + ) : null} + + + {option.title} + + {option.description ? ( + + {option.description} + + ) : null} + + + ))} + + ); +}; + +export type SelectedChatOptionChipProps = { + onRemove: () => void; + option: AIChatOption; + disabled?: boolean; +}; + +/** The picked option, with a button that removes it. The counterpart of `SelectedChatOptionChip`. */ +export const SelectedChatOptionChip = ({ + disabled, + onRemove, + option, +}: SelectedChatOptionChipProps) => { + const { t } = useAITranslation(); + const { + theme: { chatOptions: theme }, + } = useTheme(); + const title = option.shortTitle ?? option.title; + + return ( + + {option.icon ? {option.icon} : null} + {title} + + + + + ); +}; + +const styles = StyleSheet.create({ + chip: { + alignItems: 'center', + alignSelf: 'flex-start', + borderRadius: primitives.radiusXl, + flexDirection: 'row', + gap: primitives.spacingXs, + padding: primitives.spacingXs, + }, + chipIcon: { alignItems: 'center', justifyContent: 'center' }, + chipRemoveButton: { alignItems: 'center', justifyContent: 'center' }, + chipTitle: { + fontSize: primitives.typographyFontSizeMd, + fontWeight: '600', + lineHeight: primitives.typographyLineHeightNormal, + }, + item: { alignItems: 'center', flexDirection: 'row', gap: primitives.spacingMd }, + itemDescription: { + fontSize: primitives.typographyFontSizeSm, + lineHeight: primitives.typographyLineHeightNormal, + }, + itemIcon: { alignItems: 'center', justifyContent: 'center' }, + itemText: { flex: 1 }, + itemTitle: { + fontSize: primitives.typographyFontSizeMd, + fontWeight: '600', + lineHeight: primitives.typographyLineHeightNormal, + }, + list: { gap: primitives.spacingMd, paddingVertical: primitives.spacingMd }, +}); diff --git a/package/src/ai-components/message-composer/ComposerView.tsx b/package/src/ai-components/message-composer/ComposerView.tsx index 90ac4ea30b..060f1a7c69 100644 --- a/package/src/ai-components/message-composer/ComposerView.tsx +++ b/package/src/ai-components/message-composer/ComposerView.tsx @@ -1,7 +1,7 @@ import React, { useEffect, useMemo, useRef } from 'react'; import { Platform, StyleSheet, TextInput, View } from 'react-native'; -import Animated, { LinearTransition } from 'react-native-reanimated'; +import Animated, { FadeIn, FadeOut, LinearTransition } from 'react-native-reanimated'; import type { StateStore } from '@stream-io/state-store'; @@ -9,6 +9,7 @@ import { ActionButton } from './ActionButton'; import { BottomSheetContent } from './ActionSheet'; import { AttachmentButton } from './AttachmentButton'; +import { type AIChatOption, SelectedChatOptionChip } from './ChatOptions'; import { MediaPreviewList } from './MediaPreviewList'; @@ -20,7 +21,7 @@ import type { AbstractMediaPickerService, MediaPickerState, } from '../services/media-picker-service/AbstractMediaPickerService'; -import { COMPOSER_DICTATION_SOURCE, useComposerText } from '../store'; +import { COMPOSER_DICTATION_SOURCE, useComposerChatOption, useComposerText } from '../store'; import type { ComposerState } from '../store'; import { useDictationState } from '../store/dictation/useDictationState'; import { useDictationTranscript } from '../store/dictation/useDictationTranscript'; @@ -43,9 +44,17 @@ export type ComposerViewProps = { onSendMessage: (opts: { text: string; attachments?: MediaPickerState['assets']; + /** The chat option picked for the message. */ + chatOption?: AIChatOption; custom?: Record; }) => Promise; bottomSheetOptions?: BottomSheetOption[]; + /** + * Ways to ask, such as "Think harder", listed in the sheet under the camera and photos. The one + * picked shows as a chip in the composer, is sent with each message as `chatOption`, and stays + * picked until the person removes it. It is kept in `state`, as `chatOption`. + */ + chatOptions?: AIChatOption[]; isGenerating?: boolean; stopGenerating?: () => Promise; bottomSheetInsets?: BottomSheetInsets; @@ -56,6 +65,7 @@ export type ComposerViewProps = { export const ComposerViewUI = ({ bottomSheetOptions = [], bottomSheetInsets, + chatOptions, isGenerating, stopGenerating, }: ComposerViewProps) => { @@ -76,6 +86,7 @@ export const ComposerViewUI = ({ layout={LinearTransition.duration(150)} > + @@ -86,12 +97,29 @@ export const ComposerViewUI = ({ ); }; +/** The chat option picked for the next message, if any, with a button that removes it. */ +const ChatOptionChip = () => { + const { chatOption } = useComposerChatOption(); + const { setChatOption } = useMessageComposerContext(); + if (!chatOption) return null; + return ( + + setChatOption(null)} option={chatOption} /> + + ); +}; + export const AIMessageInput = () => { const { theme: { @@ -141,6 +169,10 @@ export const ComposerView = (props: ComposerViewProps) => ( ); +const chipStyles = StyleSheet.create({ + chip: { alignSelf: 'stretch', paddingBottom: primitives.spacingXs }, +}); + const useStyles = () => { const { theme } = useTheme(); diff --git a/package/src/ai-components/message-composer/MediaPreviewList.tsx b/package/src/ai-components/message-composer/MediaPreviewList.tsx index c69e859606..e3daef0eb2 100644 --- a/package/src/ai-components/message-composer/MediaPreviewList.tsx +++ b/package/src/ai-components/message-composer/MediaPreviewList.tsx @@ -1,12 +1,18 @@ import React, { useMemo } from 'react'; -import { Image, Pressable, ScrollView, StyleSheet } from 'react-native'; +import { Image, Pressable, ScrollView, StyleSheet, View } from 'react-native'; import Animated, { FadeIn, FadeOut, LinearTransition } from 'react-native-reanimated'; -import { useMessageComposerContext, useTheme } from '../contexts'; +import { useAITranslation, useMessageComposerContext, useTheme } from '../contexts'; import { Close } from '../internal/icons/Close'; +import { AIMessageAttachment } from '../message/AIMessageAttachment'; +import type { MediaAsset } from '../services/media-picker-service/AbstractMediaPickerService'; import { useMediaPickerState } from '../services/media-picker-service/hooks/useMediaPickerState'; +// An asset without a type is a photo, as the pickers give them. +const isImage = (asset: MediaAsset) => !asset.type || asset.type.startsWith('image'); + export const MediaPreviewList = () => { + const { t } = useAITranslation(); const { mediaPickerService } = useMessageComposerContext(); const { attachments } = useMediaPickerState({ service: mediaPickerService }) ?? {}; const { @@ -42,13 +48,32 @@ export const MediaPreviewList = () => { exiting={FadeOut.duration(150)} layout={LinearTransition.duration(150)} > - + {isImage(attachment) ? ( + + ) : ( + // Anything else, such as a video, can't show in an `Image`, so it shows as a file. + + + + )} mediaPickerService?.removeAsset(index)} > @@ -71,6 +96,7 @@ const useStyles = () => { }, mediaPreviewContentContainerStyle: { flexGrow: 1, paddingBottom: 8 }, mediaPreviewImage: { borderRadius: 12, marginRight: 8 }, + mediaPreviewFile: { height: 100, justifyContent: 'center', marginRight: 8, width: 220 }, mediaPreviewRemoveButton: { position: 'absolute', top: 8, diff --git a/package/src/ai-components/message-composer/index.ts b/package/src/ai-components/message-composer/index.ts index 415e82ed71..fec4681e25 100644 --- a/package/src/ai-components/message-composer/index.ts +++ b/package/src/ai-components/message-composer/index.ts @@ -3,3 +3,4 @@ export * from './ActionButton'; export * from './AttachmentButton'; export * from './MediaPreviewList'; export * from './SpeechToTextButton'; +export * from './ChatOptions'; diff --git a/package/src/ai-components/store/composer/store.ts b/package/src/ai-components/store/composer/store.ts index fa6e1a6fc1..580a7bb36f 100644 --- a/package/src/ai-components/store/composer/store.ts +++ b/package/src/ai-components/store/composer/store.ts @@ -1,7 +1,11 @@ import { StateStore } from '@stream-io/state-store'; +import type { AIChatOption } from '../../message-composer/ChatOptions'; + export type ComposerState = { text: string; + /** The chat option picked for the next message. It stays picked after a message is sent. */ + chatOption?: AIChatOption | null; custom?: Record; }; diff --git a/package/src/ai-components/store/composer/useComposerChatOption.ts b/package/src/ai-components/store/composer/useComposerChatOption.ts new file mode 100644 index 0000000000..64bad9fad2 --- /dev/null +++ b/package/src/ai-components/store/composer/useComposerChatOption.ts @@ -0,0 +1,15 @@ +import { useStateStore } from '@stream-io/state-store/react-bindings'; + +import { type ComposerState } from './store'; + +import { useMessageComposerContext } from '../../contexts'; + +const selector = ({ chatOption }: ComposerState) => ({ + chatOption, +}); + +/** The chat option picked for the next message, if any. */ +export const useComposerChatOption = () => { + const { state } = useMessageComposerContext(); + return useStateStore(state, selector); +}; diff --git a/package/src/ai-components/store/index.ts b/package/src/ai-components/store/index.ts index 338237289c..43ad719de2 100644 --- a/package/src/ai-components/store/index.ts +++ b/package/src/ai-components/store/index.ts @@ -2,6 +2,7 @@ export * from './bottom-sheet/store'; export * from './bottom-sheet/useBottomSheetState'; export * from './composer/store'; +export * from './composer/useComposerChatOption'; export * from './composer/useComposerHasText'; export * from './composer/useComposerText'; diff --git a/package/src/ai-components/suggestions/AISuggestions.tsx b/package/src/ai-components/suggestions/AISuggestions.tsx new file mode 100644 index 0000000000..aebec3c97b --- /dev/null +++ b/package/src/ai-components/suggestions/AISuggestions.tsx @@ -0,0 +1,73 @@ +import React from 'react'; +import { Pressable, ScrollView, StyleSheet, Text } from 'react-native'; + +import { useTheme } from '../contexts'; +import { primitives } from '../internal/tokens'; + +export type AISuggestionsProps = { + /** Called with the suggestion the person picked, such as to send it. */ + onSelect: (suggestion: string) => void; + /** Starters for a conversation, such as "What's on my calendar today?". */ + suggestions: string[]; +}; + +/** + * A row of starter chips for an empty conversation, scrolling sideways. Picking one hands its text + * to `onSelect`. The counterpart of `SuggestionsView`. + */ +export const AISuggestions = ({ onSelect, suggestions }: AISuggestionsProps) => { + const { + theme: { suggestions: theme }, + } = useTheme(); + if (suggestions.length === 0) return null; + + return ( + + {suggestions.map((suggestion) => ( + onSelect(suggestion)} + style={({ pressed }) => [ + styles.item, + { backgroundColor: theme?.background, opacity: pressed ? 0.7 : 1 }, + theme?.item, + ]} + > + + {suggestion} + + + ))} + + ); +}; + +const styles = StyleSheet.create({ + container: { flexGrow: 0 }, + content: { + gap: primitives.spacingXs, + paddingHorizontal: primitives.spacingMd, + paddingVertical: primitives.spacingXs, + }, + item: { + borderRadius: primitives.radiusXl, + justifyContent: 'center', + maxWidth: 160, + padding: primitives.spacingMd, + }, + itemText: { + fontSize: primitives.typographyFontSizeSm, + lineHeight: primitives.typographyLineHeightNormal, + }, +}); diff --git a/package/src/ai-components/suggestions/index.ts b/package/src/ai-components/suggestions/index.ts new file mode 100644 index 0000000000..f077b729cd --- /dev/null +++ b/package/src/ai-components/suggestions/index.ts @@ -0,0 +1 @@ +export * from './AISuggestions'; diff --git a/package/src/ai-components/theme.ts b/package/src/ai-components/theme.ts index 4f90929835..4d7cb68c43 100644 --- a/package/src/ai-components/theme.ts +++ b/package/src/ai-components/theme.ts @@ -186,6 +186,37 @@ export type Theme = { button?: ViewStyle; }; }; + /** + * The composer's chat options: the menu in its sheet, and the chip of the one picked. Optional, + * so a hand-built `Theme` without it keeps type-checking. + */ + chatOptions?: { + /** The picked option's chip. */ + chipBackground?: string; + chipText?: string; + /** An option's description in the menu. */ + descriptionText?: string; + /** An option's title in the menu. */ + titleText?: string; + list?: ViewStyle; + item?: ViewStyle; + itemIcon?: ViewStyle; + itemTitle?: TextStyle; + itemDescription?: TextStyle; + chip?: ViewStyle; + chipIcon?: ViewStyle; + chipTitle?: TextStyle; + chipRemoveButton?: ViewStyle; + }; + /** `AISuggestions`: the starters of an empty conversation. Optional, as above. */ + suggestions?: { + background?: string; + text?: string; + container?: ViewStyle; + content?: ViewStyle; + item?: ViewStyle; + itemText?: TextStyle; + }; }; export const defaultTheme: Theme = { @@ -326,4 +357,29 @@ export const defaultTheme: Theme = { attachment: { image: {}, file: {}, fileIcon: {}, fileName: {}, fileDetails: {} }, replyActions: { container: {}, button: {} }, }, + chatOptions: { + // StreamChatAI's defaults, from the shared design tokens. + chipBackground: '#FFFFFF', + chipText: '#005FFF', + descriptionText: '#414552', + titleText: '#1A1B25', + list: {}, + item: {}, + itemIcon: {}, + itemTitle: {}, + itemDescription: {}, + chip: {}, + chipIcon: {}, + chipTitle: {}, + chipRemoveButton: {}, + }, + suggestions: { + // StreamChatAI's defaults, from the shared design tokens. + background: '#EBEEF1', + text: '#1A1B25', + container: {}, + content: {}, + item: {}, + itemText: {}, + }, }; From 7ff36af2010223d05b53e614a57ee3e6a678f338 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 23:57:08 +0100 Subject: [PATCH 08/23] feat(ai-components): client tool runner Ported from #3933 on v9-release. --- .../ExpoChatGPTSample/agent/deviceTools.ts | 128 +++++++++++ .../ExpoChatGPTSample/contexts/AppContext.tsx | 47 +++- .../__tests__/clientToolRunner.test.ts | 215 ++++++++++++++++++ .../tool-calls/AIClientToolRunner.ts | 188 +++++++++++++++ package/src/ai-components/tool-calls/index.ts | 1 + 5 files changed, 575 insertions(+), 4 deletions(-) create mode 100644 examples/ExpoChatGPTSample/agent/deviceTools.ts create mode 100644 package/src/ai-components/__tests__/clientToolRunner.test.ts create mode 100644 package/src/ai-components/tool-calls/AIClientToolRunner.ts diff --git a/examples/ExpoChatGPTSample/agent/deviceTools.ts b/examples/ExpoChatGPTSample/agent/deviceTools.ts new file mode 100644 index 0000000000..c1c830d732 --- /dev/null +++ b/examples/ExpoChatGPTSample/agent/deviceTools.ts @@ -0,0 +1,128 @@ +import { Tools } from '@stream-io/calypso'; + +import { + type AIClientTool, + type AIClientToolResult, + AIClientToolRunner, + type AIToolCallPart, + completedToolResult, +} from 'stream-chat-expo/ai-components'; + +/** What only this phone knows: its clock and time zone. */ +const localTime: AIClientTool = { + definition: { + annotations: { readOnlyHint: true }, + description: + "Gets the date, time and time zone on the person's phone. The only way to know their local time.", + inputSchema: { properties: {}, type: 'object' }, + name: 'get_local_time', + }, + run: () => { + const now = new Date(); + return completedToolResult( + { + local: now.toString(), + timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone, + utc: now.toISOString(), + }, + 'Shared the local time', + ); + }, +}; + +const DEVICE_TOOLS = [localTime]; + +// What a call says it is doing on the reply's step, by tool. +const DISPLAY_TITLES: Record = { get_local_time: "Checking your phone's time" }; + +/** + * The results the runner reported, by call, for the session's socket to answer with. A call can + * reach the socket before its step shows it waiting for this phone, or after. + */ +class Reports { + private readonly reported = new Map(); + private readonly waiting = new Map void>(); + + report(callId: string, result: AIClientToolResult) { + const waiting = this.waiting.get(callId); + this.waiting.delete(callId); + if (waiting) waiting(result); + else this.reported.set(callId, result); + } + + wait(callId: string, signal: AbortSignal) { + return new Promise((resolve, reject) => { + const reported = this.reported.get(callId); + this.reported.delete(callId); + if (reported) return resolve(reported); + this.waiting.set(callId, resolve); + signal.addEventListener('abort', () => { + this.waiting.delete(callId); + reject(signal.reason); + }); + }); + } +} + +/** + * The phone's tools, run by the AI components' `AIClientToolRunner` from each reply's steps, as + * an app whose backend holds the session would run them. + * + * Here the phone holds the session too. The router sends it each call over the session's socket + * and, for a question that named this install, shows the call on the reply as waiting for it. The + * runner runs it from the reply, and the socket answers the router with what the runner reported. + * A call no step addresses to this phone, from a question that couldn't name the install, is run + * by the socket itself. + */ +export const createDeviceTools = ({ clientId, userId }: { clientId: string; userId: string }) => { + const reports = new Reports(); + const runner = new AIClientToolRunner({ clientId, tools: DEVICE_TOOLS, userId }); + + const runUnaddressed = async (name: string, call: AIToolCallPart) => { + const tool = DEVICE_TOOLS.find(({ definition }) => definition.name === name); + try { + return (await tool?.run(call)) ?? { failure: `This phone has no ${name}` }; + } catch { + return { failure: "The tool couldn't run" }; + } + }; + + const deviceTools = { + /** Whether the latest question named this install, so its calls wait for this phone. */ + addressed: false, + runner, + /** Hands the runner's result to the session's socket. */ + send: (callId: string, result: AIClientToolResult) => { + reports.report(callId, result); + return Promise.resolve(); + }, + /** How the router is told of the tools: declared on the session, with the socket's answer. */ + sessionTools: () => { + const tools = new Tools(); + for (const { description, name, parameters } of runner.registrations) { + tools.register({ + description, + displayTitle: DISPLAY_TITLES[name], + executor: 'client', + name, + parameters, + run: async (input, signal, call) => { + const { failure, output } = deviceTools.addressed + ? await reports.wait(call.id, signal) + : await runUnaddressed(name, { + arguments: input, + executor: 'client', + id: call.id, + name, + status: 'running', + }); + if (failure) throw new Error(failure); + return output; + }, + }); + } + return tools; + }, + }; + return deviceTools; +}; diff --git a/examples/ExpoChatGPTSample/contexts/AppContext.tsx b/examples/ExpoChatGPTSample/contexts/AppContext.tsx index 920815accf..e6dc394d04 100644 --- a/examples/ExpoChatGPTSample/contexts/AppContext.tsx +++ b/examples/ExpoChatGPTSample/contexts/AppContext.tsx @@ -10,11 +10,13 @@ import { type AIPendingMessage, type AIToolApprover, createAuthSessionAuthorizer, + parseMessageParts, } from 'stream-chat-expo/ai-components'; import { AgentChat, type AgentChatState, isAgentReply, isEmptyReply } from '../agent/agentChat'; import { agents, agentSessions, streamToken } from '../agent/agents'; import { AgentConfig } from '../agent/config'; +import { createDeviceTools } from '../agent/deviceTools'; import { toDataUri } from '../agent/photos'; const IDLE: ConversationSnapshot = { kind: 'idle', skills: [] }; @@ -22,6 +24,9 @@ const IDLE: ConversationSnapshot = { kind: 'idle', skills: [] }; // This install, for the tool calls addressed to it. const clientId = AIClientIdentity.installId(); +// The tools this phone runs for the agent, such as telling it the local time. +const deviceTools = createDeviceTools({ clientId, userId: AgentConfig.user.id }); + // Where a sign-in comes back to: the app's own scheme, which the web auth session hands back. // The router takes only https or a native app's own scheme in reverse domain form (RFC 8252). const SIGN_IN_SCHEME = 'io.getstream.expochatgptsample'; @@ -34,6 +39,30 @@ const declineLogin = (session: Session, toolCallId: string) => .held() .send({ allowed: false, tool_call_id: toolCallId, type: 'tool_approval' }); +// Asks. The router addresses a client tool's call to the install a question came from, which +// only its `respond` endpoint takes. `responses` (calypso's `responses.create`) takes photos and a +// reasoning effort but no install, so a question with either reaches the device's tools without +// one, and the phone runs them itself. +const respond = async ( + session: Session, + text: string, + options: { images: { url: string }[]; reasoningEffort?: string }, +) => { + if (options.images.length > 0 || options.reasoningEffort) { + deviceTools.addressed = false; + await session.responses.create(text, { + ...(options.images.length > 0 ? { images: options.images } : {}), + ...(options.reasoningEffort ? { reasoningEffort: options.reasoningEffort } : {}), + }); + return; + } + deviceTools.addressed = true; + await agents.post('/v1/agents/sessions/{id}/respond', { + body: { client_id: clientId, request_id: crypto.randomUUID(), text }, + path: { id: session.id }, + }); +}; + // A question on its way: what shows until Stream's copy arrives, and how many of the person's // messages there were when it was asked, so its copy is the first one after them. type Asked = { message: AIPendingMessage; after: number }; @@ -120,6 +149,15 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St const followed = useStateStore(chat?.state, selector); + // Runs the calls of the latest reply that wait for this phone, each once. + const lastReply = followed?.lastReply; + useEffect(() => { + if (!lastReply) return; + deviceTools.runner.run(parseMessageParts(lastReply.attachments), (call, result) => + deviceTools.send(call.id, result), + ); + }, [lastReply]); + // Opening one conversation after another: the newest wins, and one that was never asked // anything is deleted rather than left in the drawer. const open = useStableCallback(async (opening: () => Promise) => { @@ -150,6 +188,7 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St agentSessions().create({ modelOverwrites: { llm: AgentConfig.chatModel }, return_url: SIGN_IN_RETURN_URL, + tools: deviceTools.sessionTools(), }), ); }); @@ -158,7 +197,7 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St if (sessionId === current.current?.session.id) { return; } - void open(() => agentSessions().resume(sessionId)); + void open(() => agentSessions().resume(sessionId, { tools: deviceTools.sessionTools() })); }); const isResponding = agentState.kind === 'responding' || agentState.kind === 'working'; @@ -187,9 +226,9 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St const images = await Promise.all( photos.map(async (uri) => ({ url: await toDataUri(uri) })), ); - await opened.session.responses.create(text, { - ...(images.length > 0 ? { images } : {}), - ...(thinkHarder ? { reasoningEffort: 'high' } : {}), + await respond(opened.session, text, { + images, + reasoningEffort: thinkHarder ? 'high' : undefined, }); } catch (error) { setFailure(messageOf(error)); diff --git a/package/src/ai-components/__tests__/clientToolRunner.test.ts b/package/src/ai-components/__tests__/clientToolRunner.test.ts new file mode 100644 index 0000000000..80265885ed --- /dev/null +++ b/package/src/ai-components/__tests__/clientToolRunner.test.ts @@ -0,0 +1,215 @@ +import { + type AIClientTool, + AIClientToolRunner, + type AIClientToolSend, + type AIMessagePartData, + completedToolResult, + failedToolResult, + parseMessagePart, +} from '../index'; + +const step = (fields: Record) => + parseMessagePart({ + executor: 'client', + id: 'toolu_01A', + name: 'athena_device_location', + status: 'awaiting_client', + target_client_id: 'ios-1', + target_user_id: 'u_1', + type: 'ai_tool_call', + ...fields, + }) as AIMessagePartData; + +const awaiting = ({ + client = 'ios-1', + id = 'toolu_01A', + name = 'athena_device_location', + user = 'u_1', +} = {}) => [step({ id, name, target_client_id: client, target_user_id: user })]; + +const countingTool = () => { + const tool = { + definition: { name: 'athena_device_location' }, + run: jest.fn(() => completedToolResult({ city: 'Skopje' }, 'Shared approximate location')), + } satisfies AIClientTool; + return tool; +}; + +const settle = async () => { + for (let i = 0; i < 20; i += 1) await Promise.resolve(); +}; + +const runnerWith = (tools: AIClientTool[], maxAttempts?: number) => + new AIClientToolRunner({ clientId: 'ios-1', maxAttempts, tools, userId: 'u_1' }); + +describe('AIClientToolRunner', () => { + it('runs a call once, and only on the device it is addressed to', async () => { + const tool = countingTool(); + const runner = runnerWith([tool]); + const send = jest.fn, Parameters>(() => + Promise.resolve(), + ); + + runner.run(awaiting(), send); + runner.run(awaiting(), send); + await settle(); + runner.run(awaiting(), send); + runner.run(awaiting({ client: 'ios-2', id: 'toolu_02' }), send); + runner.run(awaiting({ id: 'toolu_03', user: 'u_2' }), send); + runner.run(awaiting({ id: 'toolu_04', name: 'unknown_tool' }), send); + await settle(); + + expect(tool.run).toHaveBeenCalledTimes(1); + expect(send).toHaveBeenCalledTimes(1); + expect(send.mock.calls[0][0].id).toBe('toolu_01A'); + expect(send.mock.calls[0][1]).toEqual({ + output: { city: 'Skopje' }, + summary: 'Shared approximate location', + }); + }); + + it('sends a result that could not be sent again, without running the tool again', async () => { + const tool = countingTool(); + const runner = runnerWith([tool]); + let attempts = 0; + const send = () => { + attempts += 1; + return attempts === 1 ? Promise.reject(new Error('offline')) : Promise.resolve(); + }; + + for (let i = 0; i < 3; i += 1) { + runner.run(awaiting(), send); + await settle(); + } + + expect(tool.run).toHaveBeenCalledTimes(1); + expect(attempts).toBe(2); + }); + + it('gives up on a result after its attempts', async () => { + const tool = countingTool(); + const runner = runnerWith([tool], 2); + const send = jest.fn(() => Promise.reject(new Error('offline'))); + + for (let i = 0; i < 4; i += 1) { + runner.run(awaiting(), send); + await settle(); + } + + expect(tool.run).toHaveBeenCalledTimes(1); + expect(send).toHaveBeenCalledTimes(2); + }); + + it('does not run a call again while it is still running', async () => { + let finish: () => void = () => undefined; + const tool: AIClientTool = { + definition: { name: 'athena_device_location' }, + run: jest.fn( + () => + new Promise((resolve) => { + finish = () => resolve(completedToolResult({ city: 'Skopje' })); + }), + ), + }; + const runner = runnerWith([tool]); + const send = jest.fn(() => Promise.resolve()); + + runner.run(awaiting(), send); + await settle(); + runner.run(awaiting(), send); + finish(); + await settle(); + + expect(tool.run).toHaveBeenCalledTimes(1); + expect(send).toHaveBeenCalledTimes(1); + }); + + it('does not run a call that is finished', async () => { + const tool = countingTool(); + const runner = runnerWith([tool]); + + runner.run([step({ status: 'completed' })], jest.fn()); + await settle(); + + expect(tool.run).not.toHaveBeenCalled(); + }); + + it('reports a tool that throws as failed', async () => { + const tool: AIClientTool = { + definition: { name: 'athena_device_location' }, + run: () => Promise.reject(new Error('no GPS')), + }; + const send = jest.fn(() => Promise.resolve()); + + runnerWith([tool]).run(awaiting(), send); + await settle(); + + expect(send).toHaveBeenCalledWith(expect.objectContaining({ id: 'toolu_01A' }), { + failure: "The tool couldn't run", + }); + }); + + it('sorts its tool names and keeps the first tool of a name', async () => { + const named = (name: string) => + ({ + definition: { name }, + run: jest.fn(() => failedToolResult('Not now')), + }) satisfies AIClientTool; + const first = named('athena_device_location'); + const duplicate = named('athena_device_location'); + const runner = runnerWith([named('open_camera'), first, duplicate]); + + runner.run(awaiting(), () => Promise.resolve()); + await settle(); + + expect(runner.toolNames).toEqual(['athena_device_location', 'open_camera']); + expect(first.run).toHaveBeenCalledTimes(1); + expect(duplicate.run).not.toHaveBeenCalled(); + }); + + it('describes every tool to the agent', () => { + const weather: AIClientTool = { + definition: { + description: 'Gets the weather for a city.', + inputSchema: { + properties: { city: { type: 'string' } }, + required: ['city'], + type: 'object', + }, + name: 'get_weather', + }, + instructions: 'Use it when the person asks about the weather.', + run: () => failedToolResult('Not now'), + showExternalSourcesIndicator: true, + }; + const runner = runnerWith([weather, countingTool()]); + + expect(runner.registrations).toEqual([ + // A tool without a description or instructions. + { description: '', name: 'athena_device_location', showExternalSourcesIndicator: false }, + { + description: 'Gets the weather for a city.', + instructions: 'Use it when the person asks about the weather.', + name: 'get_weather', + parameters: { + properties: { city: { type: 'string' } }, + required: ['city'], + type: 'object', + }, + showExternalSourcesIndicator: true, + }, + ]); + }); + + it('describes a tool without a description by its instructions', () => { + const location: AIClientTool = { + definition: { name: 'get_location' }, + instructions: 'Ask before sharing the location.', + run: () => failedToolResult('Not now'), + }; + + expect(runnerWith([location]).registrations[0].description).toBe( + 'Ask before sharing the location.', + ); + }); +}); diff --git a/package/src/ai-components/tool-calls/AIClientToolRunner.ts b/package/src/ai-components/tool-calls/AIClientToolRunner.ts new file mode 100644 index 0000000000..237f3f4923 --- /dev/null +++ b/package/src/ai-components/tool-calls/AIClientToolRunner.ts @@ -0,0 +1,188 @@ +import { isAwaitingClient } from '../message-parts/parseMessageParts'; +import type { AIMessagePartData, AIToolCallPart } from '../message-parts/types'; + +/** + * The name, description and argument schema of a client tool. It reads the same JSON as a Model + * Context Protocol tool (`name`, `description`, `inputSchema` and the `readOnlyHint` of its + * `annotations`), so a tool defined with the MCP SDK can be used as it is. The counterpart of + * `AIClientToolDefinition` in StreamChatAI. + */ +export type AIClientToolDefinition = { + /** The unique name of the tool. */ + name: string; + /** Hints about how the tool behaves, as a Model Context Protocol tool's `annotations`. */ + annotations?: { + /** Whether the tool only reads, and changes nothing. */ + readOnlyHint?: boolean; + }; + /** A description of what the tool does, for the agent. */ + description?: string; + /** The JSON schema of the tool's arguments. */ + inputSchema?: Record; +}; + +/** What a device reports for a call. The counterpart of `AIClientToolResult`. */ +export type AIClientToolResult = { + /** Why the device could not run the call, such as "Location not shared". Shown on the step and told to the model. */ + failure?: string; + /** The result for the model, a value that serializes to JSON. */ + output?: unknown; + /** + * A short outcome every channel member sees on the step, such as "Shared approximate location". + * Keep the data itself out of it. + */ + summary?: string; +}; + +/** A completed call, with its result for the model. */ +export const completedToolResult = (output: unknown, summary?: string): AIClientToolResult => ({ + output, + summary, +}); + +/** A call the device could not run. */ +export const failedToolResult = (reason: string): AIClientToolResult => ({ failure: reason }); + +/** + * A tool this device runs when an AI agent asks for it. Describe it to the agent with + * `AIClientToolRunner`'s `registrations`. The agent writes each call on its reply as an + * `ai_tool_call` step with `executor: client` and `status: awaiting_client`, addressed to one + * person and one install, and an `AIClientToolRunner` on that install runs it. The counterpart + * of `AIClientTool`. + */ +export type AIClientTool = { + /** The tool's name, description and argument schema, as the agent sees them. */ + definition: AIClientToolDefinition; + /** + * Runs one call. A call whose tool asks the person first reaches the device only once they + * allowed it. A tool that throws reports that it failed. + */ + run: (call: AIToolCallPart) => AIClientToolResult | Promise; + /** Guidance for the agent on when and how to use the tool, sent with its definition. */ + instructions?: string; + /** Whether the app shows that the tool reaches outside the conversation while it runs. */ + showExternalSourcesIndicator?: boolean; +}; + +/** How a client tool is described to the agent, for your backend to register it. */ +export type AIClientToolRegistration = { + /** What the tool does: its description, or its instructions when it has none. */ + description: string; + /** The unique name of the tool. This is how the agent identifies it. */ + name: string; + /** Whether the app shows that the tool reaches outside the conversation while it runs. */ + showExternalSourcesIndicator: boolean; + /** Guidance for the agent on when and how to use the tool. */ + instructions?: string; + /** The JSON schema of the tool's arguments. */ + parameters?: Record; +}; + +/** Hands a call's result to your backend. A rejection offers it again on a later `run`. */ +export type AIClientToolSend = (call: AIToolCallPart, result: AIClientToolResult) => Promise; + +export type AIClientToolRunnerOptions = { + /** This install, matched against a call's `target_client_id`, such as `AIClientIdentity.installId()`. */ + clientId: string; + tools: AIClientTool[]; + /** The person signed in, matched against a call's `target_user_id`. */ + userId: string; + /** How many times a result is offered to `send` before the runner gives up on it. Defaults to 3. */ + maxAttempts?: number; +}; + +type CallState = { + attempts: number; + result?: AIClientToolResult; + sending: boolean; + sent: boolean; +}; + +/** + * Runs the client tool calls addressed to this device, each once. Give it each AI reply's steps + * as they update. It runs a call when the call awaits this person and this install, the runner + * has a tool of that name and the call has not run here before, then hands the result to `send`. + * A result that could not be sent is sent again on a later update, without running the tool + * again. The counterpart of `AIClientToolRunner` in StreamChatAI. + * + * ```ts + * const runner = new AIClientToolRunner({ clientId: AIClientIdentity.installId(), tools: [localTime], userId }); + * runner.run(message.parts, (call, result) => backend.send(call.id, result)); + * ``` + */ +export class AIClientToolRunner { + readonly clientId: string; + readonly maxAttempts: number; + readonly userId: string; + private readonly calls = new Map(); + private readonly tools = new Map(); + + constructor({ clientId, maxAttempts = 3, tools, userId }: AIClientToolRunnerOptions) { + this.clientId = clientId; + this.maxAttempts = maxAttempts; + this.userId = userId; + for (const tool of tools) { + // The first tool of a name wins. + if (!this.tools.has(tool.definition.name)) this.tools.set(tool.definition.name, tool); + } + } + + /** The names of the tools this device can run, sorted. */ + get toolNames() { + return [...this.tools.keys()].sort(); + } + + /** + * How this device's tools are described to the agent, sorted by name. Send them to your + * backend so the agent knows it can call them. + */ + get registrations(): AIClientToolRegistration[] { + return this.toolNames.map((name) => { + const { definition, instructions, showExternalSourcesIndicator } = this.tools.get(name)!; + return { + description: definition.description ?? instructions ?? '', + name: definition.name, + showExternalSourcesIndicator: showExternalSourcesIndicator ?? false, + ...(instructions !== undefined ? { instructions } : {}), + ...(definition.inputSchema !== undefined ? { parameters: definition.inputSchema } : {}), + }; + }); + } + + /** Runs the calls among `parts` that await this device, and sends their results. */ + run(parts: readonly AIMessagePartData[], send: AIClientToolSend) { + for (const { toolCall: call } of parts) { + if (!call || !isAwaitingClient(call, this.userId, this.clientId)) continue; + const tool = this.tools.get(call.name); + if (!tool) continue; + const state = this.calls.get(call.id) ?? { attempts: 0, sending: false, sent: false }; + if (state.sending || state.sent || state.attempts >= this.maxAttempts) continue; + state.sending = true; + this.calls.set(call.id, state); + void this.deliver(tool, call, state, send); + } + } + + private async deliver( + tool: AIClientTool, + call: AIToolCallPart, + state: CallState, + send: AIClientToolSend, + ) { + if (!state.result) { + try { + state.result = await tool.run(call); + } catch { + state.result = failedToolResult("The tool couldn't run"); + } + } + state.attempts += 1; + try { + await send(call, state.result); + state.sent = true; + } catch { + // Offered again on the next update that still shows the call waiting. + } + state.sending = false; + } +} diff --git a/package/src/ai-components/tool-calls/index.ts b/package/src/ai-components/tool-calls/index.ts index 98cd79cca7..32eb3665be 100644 --- a/package/src/ai-components/tool-calls/index.ts +++ b/package/src/ai-components/tool-calls/index.ts @@ -1,3 +1,4 @@ export * from './AIClientIdentity'; export * from './AIToolApprover'; export * from './authSessionAuthorizer'; +export * from './AIClientToolRunner'; From b89a7270aac81c2886c869d748c2c48ff468b059 Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 23:57:08 +0100 Subject: [PATCH 09/23] feat(ai-components): polish agent conversations Ported from #3934 on v9-release. --- examples/ExpoChatGPTSample/README.md | 22 ++- examples/ExpoChatGPTSample/agent/agentChat.ts | 10 ++ examples/ExpoChatGPTSample/app/_layout.tsx | 32 ++-- .../components/VoiceMode/VoiceMode.tsx | 43 ++--- examples/ExpoChatGPTSample/package.json | 1 + .../ExpoChatGPTSample/screens/ChatContent.tsx | 51 +++--- package/bin/validate-entry-isolation.js | 1 + package/expo-package/package.json | 4 + package/jest-setup.tsx | 12 ++ package/jest.config.js | 3 +- package/native-package/package.json | 4 + package/package.json | 5 + .../ai-components/StreamingMessageView.tsx | 21 ++- .../__tests__/ComposerView.test.tsx | 34 +++- .../__tests__/StreamingMessageView.test.tsx | 42 +++++ .../__tests__/conversation.test.tsx | 31 +++- .../ai-components/__tests__/keyboard.test.ts | 14 ++ .../charts/components/PieLegend.tsx | 3 +- .../charts/components/VictoryChart.tsx | 4 +- .../conversation/AIConversation.tsx | 114 ++++++++++++-- .../conversation/list/AIChatList.tsx | 115 ++++++++++---- .../list/useChatListController.ts | 35 +++- .../conversation/list/useEndInView.ts | 23 ++- .../conversation/list/useKeyboardOverlap.ts | 51 ++++++ .../conversation/list/usePinnedAnchor.ts | 82 ++++------ .../internal/hooks/useBottomSafeArea.ts | 6 + .../markdown/components/CodeBlock.tsx | 3 +- package/src/ai-components/markdown/styles.ts | 9 +- .../message-composer/ActionSheet.tsx | 7 +- .../message-composer/ComposerView.tsx | 70 ++++++-- .../message/AIAssistantMessage.tsx | 7 +- .../ai-components/message/UserMessageMenu.tsx | 149 ++++++++++++++++++ .../ai-components/reasoning/AIReasoning.tsx | 65 ++++++-- .../utils/flatten-nodes.ts | 3 +- .../syntax-highlighting/utils/renderer.tsx | 6 +- package/src/ai-components/theme.ts | 18 ++- yarn.lock | 30 +++- 37 files changed, 906 insertions(+), 224 deletions(-) create mode 100644 package/src/ai-components/__tests__/keyboard.test.ts create mode 100644 package/src/ai-components/conversation/list/useKeyboardOverlap.ts create mode 100644 package/src/ai-components/internal/hooks/useBottomSafeArea.ts create mode 100644 package/src/ai-components/message/UserMessageMenu.tsx diff --git a/examples/ExpoChatGPTSample/README.md b/examples/ExpoChatGPTSample/README.md index 4da20e2836..eb804dbc29 100644 --- a/examples/ExpoChatGPTSample/README.md +++ b/examples/ExpoChatGPTSample/README.md @@ -12,13 +12,18 @@ Ported from the `ExpoChatGPTSample` of the deprecated `@stream-io/chat-react-nat The agent runs on a [Vision-Agents](https://github.com/GetStream/Vision-Agents) router, which the app talks to through [`@stream-io/calypso`](https://www.npmjs.com/package/@stream-io/calypso) (`agent/`). -- Each conversation is a router session, kept in an `agent` channel. `` follows that channel as usual. -- Asking goes through `session.responses.create`, not the chat composer. The router writes the question and the agent's - streamed reply into the channel, and titles the conversation itself. -- The drawer lists the user's `agent` channels. `+` starts a new session, and one that is never asked anything is - deleted when you leave it. -- The reply's steps (`ai_reasoning`, `ai_tool_call` attachments) are hidden for now. They get their own views in a - later change. +- Each conversation is a router session, kept in an `agent` channel. The drawer lists the user's `agent` channels. `+` + starts a new session, and one that is never asked anything is deleted when you leave it. +- The conversation is `AIConversation`, fed from `agent/agentChat.ts`: a small store that watches the channel, copies + its messages on every `message.*` event, loads older pages, and keeps each reply's full reasoning from its live + updates. There's no ``, so the store also watches the channel again after `connection.recovered`, to pick up + what was written while the connection was down. +- Asking goes through the session, not the chat composer. The router writes the question and the streamed reply into + the channel, the reply's steps as `ai_reasoning` and `ai_tool_call` attachments, and titles the conversation itself. + A plain question goes to `respond` with this install's id, so the agent can call the phone's tools + (`agent/deviceTools.ts`). A question with photos or "Think harder" goes to `responses.create`, which takes neither. +- The keyboard is handled by the AI components through `react-native-keyboard-controller`; the app only wraps itself + in `KeyboardProvider`. The starters and the voice button sit in the composer's `header`, so they ride the keyboard. ## Running @@ -37,3 +42,6 @@ yarn workspace expochatgptsample ios # or: android `ios` and `android` sync the SDK's shared native code, run `expo prebuild` and build the app. The native `ios` and `android` folders are generated and not committed. + +On iOS, if the app crashes at launch in `dyld` with a missing symbol, reinstall the pods without Expo's +precompiled modules: `cd ios && EXPO_USE_PRECOMPILED_MODULES=0 pod install`, then build again. diff --git a/examples/ExpoChatGPTSample/agent/agentChat.ts b/examples/ExpoChatGPTSample/agent/agentChat.ts index 0515cf4235..040d7f2a69 100644 --- a/examples/ExpoChatGPTSample/agent/agentChat.ts +++ b/examples/ExpoChatGPTSample/agent/agentChat.ts @@ -57,6 +57,16 @@ export class AgentChat { }); this.stops.push(() => subscription.unsubscribe()); + const recovered = this.channel + .getClient() + .on('connection.recovered', () => void this.watchLatest().catch(() => {})); + this.stops.push(() => recovered.unsubscribe()); + + await this.watchLatest(); + } + + /** Watches the channel again and takes its latest page, in place of what was loaded. */ + private async watchLatest() { const watched = await this.channel.watch({ messages: { limit: PAGE_SIZE } }); this.state.partialNext({ hasOlderMessages: (watched.messages?.length ?? 0) >= PAGE_SIZE, diff --git a/examples/ExpoChatGPTSample/app/_layout.tsx b/examples/ExpoChatGPTSample/app/_layout.tsx index a83444ff2a..90d2893a7d 100644 --- a/examples/ExpoChatGPTSample/app/_layout.tsx +++ b/examples/ExpoChatGPTSample/app/_layout.tsx @@ -5,6 +5,7 @@ import React from 'react'; import { StyleSheet } from 'react-native'; import { GestureHandlerRootView } from 'react-native-gesture-handler'; +import { KeyboardProvider } from 'react-native-keyboard-controller'; import { SafeAreaProvider } from 'react-native-safe-area-context'; import { Drawer } from 'expo-router/drawer'; @@ -34,19 +35,24 @@ export default function Layout() { } return ( - - - - - - - - - - - - - + + + + + + + + + + + + + + + ); } diff --git a/examples/ExpoChatGPTSample/components/VoiceMode/VoiceMode.tsx b/examples/ExpoChatGPTSample/components/VoiceMode/VoiceMode.tsx index d1a0984bff..935778df95 100644 --- a/examples/ExpoChatGPTSample/components/VoiceMode/VoiceMode.tsx +++ b/examples/ExpoChatGPTSample/components/VoiceMode/VoiceMode.tsx @@ -6,6 +6,7 @@ import Svg, { Line } from 'react-native-svg'; import { StreamTheme, useDictationAudioLevel, + useTheme, type VoiceAgentState, VoiceGlow, VoiceWave, @@ -63,8 +64,8 @@ const VoiceModeOverlay = ({ onClose }: { onClose: () => void }) => { * A static waveform icon. The button stays still: a live `VoiceGlow` here would redraw every frame * for as long as the chat is open. */ -const VoiceIcon = () => ( - +const VoiceIcon = ({ color, size }: { color: string; size: number }) => ( + {[ [3, 8, 14], [7, 4, 18], @@ -74,7 +75,7 @@ const VoiceIcon = () => ( ].map(([x, y1, y2]) => ( ( ); -/** Opens a full-screen voice conversation with the AI agent. */ +/** + * Opens a full-screen voice conversation with the AI agent: a button after the composer, the size + * of the attach button before it. + */ export const VoiceMode = () => { const [open, setOpen] = useState(false); + const { + theme: { composer }, + } = useTheme(); + const size = composer.inputPillHeight; return ( <> - - setOpen(true)} - style={styles.button} - > - - - + setOpen(true)} + style={({ pressed }) => [ + styles.button, + { borderRadius: size / 2, height: size, width: size }, + pressed && styles.buttonPressed, + ]} + > + + setOpen(false)} visible={open}> {open ? setOpen(false)} /> : null} @@ -114,15 +124,12 @@ const styles = StyleSheet.create({ alignItems: 'center', backgroundColor: '#ffffff', borderColor: '#ececec', - borderRadius: 24, borderWidth: 1, - height: 48, justifyContent: 'center', overflow: 'hidden', - width: 48, }, + buttonPressed: { opacity: 0.8 }, close: { color: '#ffffff', fontSize: 22 }, - entry: { alignItems: 'flex-end', paddingHorizontal: 16, paddingBottom: 8 }, glow: { aspectRatio: 1, width: '100%' }, glowContainer: { alignSelf: 'stretch', flex: 1, justifyContent: 'center' }, header: { alignItems: 'flex-end', paddingHorizontal: 20 }, diff --git a/examples/ExpoChatGPTSample/package.json b/examples/ExpoChatGPTSample/package.json index 6b13018c84..75142db434 100644 --- a/examples/ExpoChatGPTSample/package.json +++ b/examples/ExpoChatGPTSample/package.json @@ -24,6 +24,7 @@ "react": "19.3.0", "react-native": "0.88.0", "react-native-gesture-handler": "~3.4.0", + "react-native-keyboard-controller": "1.22.7", "react-native-reanimated": "4.7.1", "react-native-safe-area-context": "~5.10.1", "react-native-screens": "~4.29.0", diff --git a/examples/ExpoChatGPTSample/screens/ChatContent.tsx b/examples/ExpoChatGPTSample/screens/ChatContent.tsx index fa051c4843..16e8d5c040 100644 --- a/examples/ExpoChatGPTSample/screens/ChatContent.tsx +++ b/examples/ExpoChatGPTSample/screens/ChatContent.tsx @@ -1,11 +1,9 @@ -import { useCallback, useMemo, useState } from 'react'; +import { type ReactNode, useCallback, useMemo, useState } from 'react'; import { ActivityIndicator, Platform, StatusBar, StyleSheet, Text, View } from 'react-native'; import Animated, { FadeIn, FadeOut } from 'react-native-reanimated'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; -import { useHeaderHeight } from 'expo-router/react-navigation'; - -import { KeyboardCompatibleView, useStateStore } from 'stream-chat-expo'; +import { useStateStore } from 'stream-chat-expo'; import { AIConversation, @@ -25,7 +23,15 @@ import { useSendAIMessage } from '../hooks/useSendAIMessage'; type Composer = ReturnType; -const CustomComposerView = ({ composer }: { composer: Composer }) => { +const CustomComposerView = ({ + composer, + header, + trailing, +}: { + composer: Composer; + header: ReactNode; + trailing: ReactNode; +}) => { const { isBusy, stop } = useAppContext(); // The router stops through its own session, not with an AI indicator event. @@ -47,11 +53,13 @@ const CustomComposerView = ({ composer }: { composer: Composer }) => { return ( ); }; @@ -75,14 +83,23 @@ const Starters = ({ chat, composer }: { chat: AgentChat; composer: Composer }) = return ; }; -/** The starters and the composer, sharing the composer's draft and picked option. */ -const ComposerArea = ({ chat }: { chat: AgentChat }) => { +/** + * The composer, with what rides the keyboard above it, a failure and the starters (which share the + * composer's draft and picked option), and the way into voice mode beside it. + */ +const ComposerArea = ({ chat, failure }: { chat: AgentChat; failure: string | undefined }) => { const [composer] = useState(createNewComposerStore); return ( - <> - - - + + {failure ? {failure} : null} + + + } + trailing={} + /> ); }; @@ -141,8 +158,6 @@ const Conversation = ({ chat }: { chat: AgentChat }) => { export const ChatContent = () => { const { chat, failure } = useAppContext(); const { bottom } = useSafeAreaInsets(); - // The conversation sits below the drawer header, so the keyboard offset has to account for it. - const headerHeight = useHeaderHeight(); if (!chat) { return ( @@ -159,19 +174,15 @@ export const ChatContent = () => { entering={FadeIn.duration(200)} exiting={FadeOut.duration(200)} > - - - {failure ? {failure} : null} - - - + {/* The keyboard is the SDK's: the conversation makes room for it, the composer rides it. */} + + ); }; const styles = StyleSheet.create({ wrapper: { flex: 1, backgroundColor: '#fcfcfc' }, - flex: { flex: 1 }, emptyContainer: { flex: 1, width: '100%', diff --git a/package/bin/validate-entry-isolation.js b/package/bin/validate-entry-isolation.js index ff19b10468..39b2865615 100644 --- a/package/bin/validate-entry-isolation.js +++ b/package/bin/validate-entry-isolation.js @@ -31,6 +31,7 @@ const AI_ONLY_PACKAGES = [ '@khanacademy/simple-markdown', '@legendapp/list', '@shopify/react-native-skia', + 'react-native-keyboard-controller', 'react-syntax-highlighter', 'victory-native', 'zod', diff --git a/package/expo-package/package.json b/package/expo-package/package.json index 0f9ad7db9e..77e1113870 100644 --- a/package/expo-package/package.json +++ b/package/expo-package/package.json @@ -47,6 +47,7 @@ "expo-sharing": "*", "expo-video": "*", "react-native": ">=0.79.0", + "react-native-keyboard-controller": ">=1.21.7", "victory-native": ">=41.0.0" }, "peerDependenciesMeta": { @@ -86,6 +87,9 @@ "expo-video": { "optional": true }, + "react-native-keyboard-controller": { + "optional": true + }, "victory-native": { "optional": true } diff --git a/package/jest-setup.tsx b/package/jest-setup.tsx index 806594ca18..fda7443b95 100644 --- a/package/jest-setup.tsx +++ b/package/jest-setup.tsx @@ -53,6 +53,18 @@ jest.mock('react-native-gesture-handler/lib/module/GestureHandlerRootViewContext const React = require('react'); return { __esModule: true, default: React.createContext(true) }; }); +// The AI conversation rides the keyboard with react-native-keyboard-controller, which ships a mock. +jest.mock('react-native-keyboard-controller', () => + require('react-native-keyboard-controller/jest'), +); +// The keyboard-aware list waits for its scroll view to report its insets before it shows its rows, +// which the mocked scroll view never does. Under Jest it is the list it wraps; how it rides the +// keyboard is checked on devices. +jest.mock('@legendapp/list/keyboard', () => ({ + ...jest.requireActual('@legendapp/list/keyboard'), + KeyboardAwareLegendList: jest.requireActual('@legendapp/list/reanimated').AnimatedLegendList, +})); + jest.mock('react-native-reanimated', () => { const RNReanimatedmock = require('react-native-reanimated/mock'); // RNGH (>= 3.3) subscribes to SharedValue-backed gesture config on the UI runtime via diff --git a/package/jest.config.js b/package/jest.config.js index bc708eeee4..8cf2405ff6 100644 --- a/package/jest.config.js +++ b/package/jest.config.js @@ -7,7 +7,8 @@ module.exports = { moduleNameMapper: { // LegendList's exports map has only a `react-native` condition for its native build, which // Jest's node conditions skip for the web build (react-dom). - '^@legendapp/list/(react-native|reanimated)$': '/node_modules/@legendapp/list/$1.js', + '^@legendapp/list/(keyboard|react-native|reanimated)$': + '/node_modules/@legendapp/list/$1.js', 'mock-builders(.*)$': '/src/mock-builders$1', // See __mocks__/SvgTouchableMixin.js - react-native-svg reads `Touchable.Mixin` off the // react-native root, and RN 0.87's late, non-enumerable re-export of it is lost through diff --git a/package/native-package/package.json b/package/native-package/package.json index b9b3f9e7dd..4ee283eb07 100644 --- a/package/native-package/package.json +++ b/package/native-package/package.json @@ -43,6 +43,7 @@ "react-native-blob-util": ">=0.22.0", "react-native-haptic-feedback": ">=2.3.0", "react-native-image-picker": ">=7.1.2", + "react-native-keyboard-controller": ">=1.21.7", "react-native-nitro-sound": ">=0.2.9", "react-native-share": ">=11.0.0", "react-native-video": ">=6.18.0", @@ -73,6 +74,9 @@ "react-native-image-picker": { "optional": true }, + "react-native-keyboard-controller": { + "optional": true + }, "react-native-nitro-sound": { "optional": true }, diff --git a/package/package.json b/package/package.json index 84e8985979..84ac82977e 100644 --- a/package/package.json +++ b/package/package.json @@ -99,6 +99,7 @@ "emoji-mart": ">=5.4.0", "react-native": ">=0.79.0", "react-native-gesture-handler": ">=2.18.0", + "react-native-keyboard-controller": ">=1.21.7", "react-native-reanimated": ">=3.16.0", "react-native-safe-area-context": ">=5.4.1", "react-native-svg": ">=15.8.0", @@ -121,6 +122,9 @@ "emoji-mart": { "optional": true }, + "react-native-keyboard-controller": { + "optional": true + }, "victory-native": { "optional": true } @@ -160,6 +164,7 @@ "react-native": "0.88.0", "react-native-builder-bob": "0.40.11", "react-native-gesture-handler": "^3.4.0", + "react-native-keyboard-controller": "^1.22.7", "react-native-reanimated": "^4.7.1", "react-native-safe-area-context": "^5.10.1", "react-native-svg": "15.15.5", diff --git a/package/src/ai-components/StreamingMessageView.tsx b/package/src/ai-components/StreamingMessageView.tsx index 2efe179e50..24c5b1e37e 100644 --- a/package/src/ai-components/StreamingMessageView.tsx +++ b/package/src/ai-components/StreamingMessageView.tsx @@ -17,16 +17,21 @@ export const useStreamingMessage = ({ letterInterval = 0, renderingLetterCount = 2, text, -}: UseStreamingMessageProps): { streamedMessageText: string } => { +}: UseStreamingMessageProps): { isWriting: boolean; streamedMessageText: string } => { const [streamedMessageText, setStreamedMessageText] = useState(text); const textCursor = useRef(text.length); + // How much of the text is typed out, to say whether it still is. + const [typedLength, setTypedLength] = useState(text.length); // Only text added to the end is typed out. Another text, such as another message's when a list // reuses this view, shows at once, from this render rather than after the old text's. const [lastText, setLastText] = useState(text); if (text !== lastText) { setLastText(text); - if (!text.startsWith(lastText)) setStreamedMessageText(text); + if (!text.startsWith(lastText)) { + setStreamedMessageText(text); + setTypedLength(text.length); + } } const typedToward = useRef(text); @@ -44,6 +49,7 @@ export const useStreamingMessage = ({ const codeBlockCounts = (newText.match(/```/g) || []).length; const shouldOptimisticallyCloseCodeBlock = codeBlockCounts > 0 && codeBlockCounts % 2 > 0; setStreamedMessageText(shouldOptimisticallyCloseCodeBlock ? newText + '```' : newText); + setTypedLength(textCursor.current); }, letterInterval); return () => { @@ -51,7 +57,7 @@ export const useStreamingMessage = ({ }; }, [letterInterval, renderingLetterCount, text]); - return { streamedMessageText }; + return { isWriting: typedLength < text.length, streamedMessageText }; }; export const StreamingMessageView = ({ @@ -60,6 +66,7 @@ export const StreamingMessageView = ({ rules, onLink: onLinkParam, letterInterval, + onWritingChange, renderingLetterCount, }: { text: string; @@ -67,15 +74,21 @@ export const StreamingMessageView = ({ rules?: MarkdownRules; onLink?: (url: string) => void; letterInterval?: number; + /** Called when the text starts or stops being typed out, such as to show what follows it once it is. */ + onWritingChange?: (isWriting: boolean) => void; renderingLetterCount?: number; }) => { const markdownText = useMemo(() => generateMarkdownText(text), [text]); - const { streamedMessageText } = useStreamingMessage({ + const { isWriting, streamedMessageText } = useStreamingMessage({ text: markdownText ?? '', letterInterval, renderingLetterCount, }); + const reportWriting = useStableCallback((writing: boolean) => onWritingChange?.(writing)); + useEffect(() => { + reportWriting(isWriting); + }, [isWriting, reportWriting]); const onLink = useStableCallback((url: string): void | Promise => onLinkParam diff --git a/package/src/ai-components/__tests__/ComposerView.test.tsx b/package/src/ai-components/__tests__/ComposerView.test.tsx index 096fa12b2a..d7b7094d38 100644 --- a/package/src/ai-components/__tests__/ComposerView.test.tsx +++ b/package/src/ai-components/__tests__/ComposerView.test.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { Image, Text } from 'react-native'; +import { Image, Pressable, Text } from 'react-native'; import { act, fireEvent, render, screen, waitFor } from '@testing-library/react-native'; @@ -195,6 +195,38 @@ describe('ai-components ComposerView', () => { }); }); +describe('ComposerView header', () => { + it('shows what it is given above the composer', () => { + render( + + Starters} + onSendMessage={jest.fn(() => Promise.resolve())} + /> + , + ); + expect(screen.getByText('Starters')).toBeTruthy(); + }); +}); + +describe('ComposerView trailing', () => { + it('shows what it is given after the message box, and its presses reach it', () => { + const onTalk = jest.fn(); + render( + + Promise.resolve())} + trailing={ + + } + /> + , + ); + fireEvent.press(screen.getByRole('button', { name: 'Talk' })); + expect(onTalk).toHaveBeenCalledTimes(1); + }); +}); + describe('AISuggestions', () => { it('hands the picked suggestion over, and shows nothing without any', () => { const onSelect = jest.fn(); diff --git a/package/src/ai-components/__tests__/StreamingMessageView.test.tsx b/package/src/ai-components/__tests__/StreamingMessageView.test.tsx index a2231d8da2..eb8be0fd01 100644 --- a/package/src/ai-components/__tests__/StreamingMessageView.test.tsx +++ b/package/src/ai-components/__tests__/StreamingMessageView.test.tsx @@ -68,6 +68,26 @@ describe('ai-components streaming', () => { expect(result.current.streamedMessageText).toBe('Bye n'); }); + it('says whether text is still being typed out', () => { + const { result, rerender } = renderHook( + ({ text }: { text: string }) => + useStreamingMessage({ letterInterval: 10, renderingLetterCount: 2, text }), + { initialProps: { text: 'Hello' } }, + ); + expect(result.current.isWriting).toBe(false); + + rerender({ text: 'Hello world' }); + expect(result.current.isWriting).toBe(true); + act(() => { + jest.advanceTimersByTime(100); + }); + expect(result.current.isWriting).toBe(false); + + // Another text shows at once, so nothing is being typed. + rerender({ text: 'Bye' }); + expect(result.current.isWriting).toBe(false); + }); + it('optimistically closes a code fence while it is being streamed', () => { const { result, rerender } = renderHook( ({ text }: { text: string }) => @@ -84,6 +104,28 @@ describe('ai-components streaming', () => { }); describe('StreamingMessageView', () => { + it('reports when it starts and stops typing', () => { + const onWritingChange = jest.fn(); + const { rerender } = render( + + + , + ); + rerender( + + + , + ); + act(() => { + jest.advanceTimersByTime(200); + }); + expect(onWritingChange.mock.calls.map(([writing]) => writing)).toEqual([false, true, false]); + }); + it('renders markdown and handles links with the provided callback', () => { const onLink = jest.fn(); render( diff --git a/package/src/ai-components/__tests__/conversation.test.tsx b/package/src/ai-components/__tests__/conversation.test.tsx index 28526631a9..f0af578261 100644 --- a/package/src/ai-components/__tests__/conversation.test.tsx +++ b/package/src/ai-components/__tests__/conversation.test.tsx @@ -105,7 +105,22 @@ describe('AIConversation', () => { Ask me anything} messages={[]} />, ); expect(screen.getByText('Ask me anything')).toBeTruthy(); - expect(screen.queryByTestId('ai-conversation-list')).toBeNull(); + // The list stays mounted under it, so it follows a keyboard opened before the first question. + expect(screen.getByTestId('ai-conversation-list')).toBeTruthy(); + }); + + it('hides the empty content once there are messages', async () => { + const { rerender } = await renderConversation( + Ask me anything} messages={[]} />, + ); + await rerender( + Ask me anything} + messages={[user('u1', 'Is 7 prime?')]} + />, + ); + expect(screen.queryByText('Ask me anything')).toBeNull(); + expect(screen.getByText('Is 7 prime?')).toBeTruthy(); }); it('groups the messages into turns, each starting with a question', async () => { @@ -231,6 +246,20 @@ describe('AIConversation', () => { expect(screen.queryByTestId('ai-reply-actions')).toBeNull(); }); + it('shows the actions under a reply once its answer is typed out', async () => { + const answer = 'word '.repeat(200).trim(); + const { rerender } = await renderConversation( + , + ); + await rerender(); + // Finished, and still typing out what arrived last. + expect(screen.queryByTestId('ai-reply-actions')).toBeNull(); + await settle(2000); + expect(screen.getByTestId('ai-reply-actions')).toBeTruthy(); + }); + it('shows a new rating when the messages stay the same', async () => { const messages = [user('u1', 'One'), reply('a1', 'Done.')]; const rated = (rating: 'positive' | null) => ( diff --git a/package/src/ai-components/__tests__/keyboard.test.ts b/package/src/ai-components/__tests__/keyboard.test.ts new file mode 100644 index 0000000000..233d585baf --- /dev/null +++ b/package/src/ai-components/__tests__/keyboard.test.ts @@ -0,0 +1,14 @@ +import { keyboardOverlapOf } from '../conversation/list/useKeyboardOverlap'; + +describe('keyboardOverlapOf', () => { + it('is how far the keyboard rises past the offset, as far as it is open', () => { + expect(keyboardOverlapOf({ height: 300, progress: 1 }, 34)).toBe(266); + // Halfway through opening, the offset counts by half. + expect(keyboardOverlapOf({ height: 150, progress: 0.5 }, 34)).toBe(133); + }); + + it('is nothing while the keyboard is down or below the offset', () => { + expect(keyboardOverlapOf({ height: 0, progress: 0 }, 34)).toBe(0); + expect(keyboardOverlapOf({ height: 20, progress: 1 }, 34)).toBe(0); + }); +}); diff --git a/package/src/ai-components/charts/components/PieLegend.tsx b/package/src/ai-components/charts/components/PieLegend.tsx index 9191674642..70ff7849db 100644 --- a/package/src/ai-components/charts/components/PieLegend.tsx +++ b/package/src/ai-components/charts/components/PieLegend.tsx @@ -1,6 +1,7 @@ import React from 'react'; import { StyleSheet, Text, View } from 'react-native'; +import { primitives } from '../../internal/tokens'; import type { Datum } from '../types'; export const PieLegend = ({ items, maxRows = 3 }: { items: Datum[]; maxRows?: number }) => { @@ -45,6 +46,6 @@ const styles = StyleSheet.create({ height: 12, }, label: { - fontSize: 13, + fontSize: primitives.typographyFontSizeXs, }, }); diff --git a/package/src/ai-components/charts/components/VictoryChart.tsx b/package/src/ai-components/charts/components/VictoryChart.tsx index a4ba35674a..615d1ab0c3 100644 --- a/package/src/ai-components/charts/components/VictoryChart.tsx +++ b/package/src/ai-components/charts/components/VictoryChart.tsx @@ -6,6 +6,7 @@ import { Area, Bar, CartesianChart, Line, Pie, PolarChart, Scatter } from 'victo import { PieLegend } from './PieLegend'; +import { primitives } from '../../internal/tokens'; import type { ChartSpec } from '../types'; const font = matchFont({ @@ -14,7 +15,8 @@ const font = matchFont({ android: 'sans-serif', default: 'serif', }), - fontSize: 12, + // A chart's labels read at the caption size, as in StreamChatAI. + fontSize: primitives.typographyFontSizeXxs, fontWeight: 'normal', fontStyle: 'normal', }); diff --git a/package/src/ai-components/conversation/AIConversation.tsx b/package/src/ai-components/conversation/AIConversation.tsx index 12f1dc9760..f3e9f58eb5 100644 --- a/package/src/ai-components/conversation/AIConversation.tsx +++ b/package/src/ai-components/conversation/AIConversation.tsx @@ -4,9 +4,17 @@ import React, { memo, type ReactNode, useContext, + useLayoutEffect, useMemo, + useState, } from 'react'; -import { StyleSheet, View } from 'react-native'; +import { type LayoutChangeEvent, StyleSheet, View } from 'react-native'; +import Animated, { + Easing, + FadeIn, + useAnimatedStyle, + useSharedValue, +} from 'react-native-reanimated'; import type { LegendListRenderItemProps } from '@legendapp/list/react-native'; @@ -42,6 +50,11 @@ export type AIConversationAssistantMessageProps = { parts: AIMessagePartData[]; text: string; approver?: AIToolApprover; + /** + * Called when the answer starts or stops being typed out. The reply's actions wait until it is + * done. A view that never calls it has its actions shown once the reply is finished. + */ + onWritingChange?: (isWriting: boolean) => void; reasoningText?: (stepId: string) => string | undefined; }; @@ -74,6 +87,14 @@ export type AIConversationProps = { emptyContent?: ReactNode; /** A view under the latest turn's messages, such as what the agent is doing, or an error with a way to retry. */ footer?: ReactNode; + /** + * How far above the bottom of the screen the composer sits, such as above a tab bar. The + * keyboard comes up over the conversation, which makes room for it below the latest turn (the + * room the pinned question leaves takes it first), and covers it only as far as it rises past + * this. Defaults to the bottom safe area. Needs a `KeyboardProvider` from + * react-native-keyboard-controller above it. + */ + keyboardOffset?: number; /** * Whether the agent is answering the latest message and its reply shows nothing yet. * Meanwhile the latest turn ends with `AIThinking`, and the reply's first reasoning step takes @@ -116,6 +137,7 @@ const DefaultAssistantMessage = ({ attachments, isGenerating, message, + onWritingChange, parts, reasoningText, text, @@ -125,6 +147,7 @@ const DefaultAssistantMessage = ({ attachments={attachments} id={message.id} isGenerating={isGenerating} + onWritingChange={onWritingChange} parts={parts} reasoningText={reasoningText} text={text} @@ -133,9 +156,17 @@ const DefaultAssistantMessage = ({ // What every row reads, rather than each row's item carrying it: a row renders again only when // its item is a new object, or when this changes. +// The tallest each row of the latest turn has been, by row key. The turn never shrinks while it is +// the latest: the room the list leaves under it to keep its question at the top follows a row that +// shrinks (a sign-in going, reasoning folding, the thinking going) only a frame later, and in that +// frame the question would be pulled down. Below a pinned turn there is room anyway, so it looks +// the same, and the room stays the list's, which the keyboard takes before it lifts anything. +type TurnFloor = { heights: Map; turnKey: string | undefined }; + type ConversationContextValue = { AssistantMessage: ComponentType; clock: AIThinkingClock; + turnFloor: TurnFloor; ReplyActions: ComponentType | null; UserMessage: ComponentType; approver?: AIToolApprover; @@ -159,6 +190,21 @@ type EntryViewProps = { approver?: AIToolApprover; }; +const REPLY_ACTIONS_IN = FadeIn.duration(200).easing(Easing.out(Easing.ease)); + +const useReplyWriting = (messageId: string | undefined, isGenerating: boolean) => { + const [writing, setWriting] = useState(); + const [watched, setWatched] = useState(); + const isWriting = !!messageId && writing === messageId; + if (messageId && (isGenerating || isWriting) && watched !== messageId) setWatched(messageId); + const onWritingChange = useMemo( + () => (value: boolean) => + setWriting((current) => (value ? messageId : current === messageId ? undefined : current)), + [messageId], + ); + return { isWriting, onWritingChange, wasWatched: !!messageId && watched === messageId }; +}; + const EntryView = ({ approver, AssistantMessage, @@ -167,6 +213,12 @@ const EntryView = ({ UserMessage, }: EntryViewProps) => { const { message, pending } = entry; + const isReply = message?.role !== 'user'; + const isGenerating = !!message?.isGenerating; + const { isWriting, onWritingChange, wasWatched } = useReplyWriting( + isReply ? message?.id : undefined, + isGenerating, + ); if (!message) { return pending ? ( @@ -188,20 +240,17 @@ const EntryView = ({ - {ReplyActions && !message.isGenerating && message.text !== '' ? ( - // Keyed: a row reused for another reply starts its actions afresh, such as not "Copied". - + {ReplyActions && !isGenerating && !isWriting && message.text !== '' ? ( + + + ) : null} ); @@ -217,12 +266,30 @@ const Row = memo(({ row }: { row: ConversationRow }) => { () => (clock ? { clock, firstStepId: row.firstStepId, key: row.turnKey } : undefined), [clock, row.firstStepId, row.turnKey], ); + // A row reused for another starts from that row's floor. + const turnFloor = context?.turnFloor; + const isHeld = !!turnFloor && row.turnKey === turnFloor.turnKey; + const floor = useSharedValue(0); + useLayoutEffect(() => { + floor.value = isHeld ? (turnFloor?.heights.get(row.key) ?? 0) : 0; + }, [floor, isHeld, row.key, turnFloor]); + const onLayout = isHeld + ? ({ nativeEvent }: LayoutChangeEvent) => { + const height = nativeEvent.layout.height; + if (height <= floor.value) return; + floor.value = height; + turnFloor?.heights.set(row.key, height); + } + : undefined; + const floorStyle = useAnimatedStyle(() => ({ minHeight: floor.value })); if (!context) return null; return ( - @@ -242,7 +309,7 @@ const Row = memo(({ row }: { row: ConversationRow }) => { )} - + ); }); Row.displayName = 'AIConversationRow'; @@ -284,6 +351,7 @@ export const AIConversation = ({ emptyContent, footer, isThinking = false, + keyboardOffset, loadOlderMessages, messages, pendingMessage, @@ -301,9 +369,26 @@ export const AIConversation = ({ const ReplyActions = ReplyActionsProp === undefined ? DefaultConversationReplyActions : ReplyActionsProp; + const latestTurnKey = useMemo( + () => rows.find((row) => row.key === anchorKey)?.turnKey, + [anchorKey, rows], + ); + const turnFloor = useMemo( + () => ({ heights: new Map(), turnKey: latestTurnKey }), + [latestTurnKey], + ); + const contextValue = useMemo( - () => ({ approver, AssistantMessage, clock, ReplyActions, replyActions, UserMessage }), - [approver, AssistantMessage, clock, ReplyActions, replyActions, UserMessage], + () => ({ + approver, + AssistantMessage, + clock, + ReplyActions, + replyActions, + turnFloor, + UserMessage, + }), + [approver, AssistantMessage, clock, ReplyActions, replyActions, turnFloor, UserMessage], ); return ( @@ -315,6 +400,7 @@ export const AIConversation = ({ data={rows} emptyContent={emptyContent} getItemType={typeOf} + keyboardOffset={keyboardOffset} keyExtractor={keyOf} loadOlderMessages={loadOlderMessages} renderItem={renderRow} diff --git a/package/src/ai-components/conversation/list/AIChatList.tsx b/package/src/ai-components/conversation/list/AIChatList.tsx index a6e06b1961..9dcd01a18a 100644 --- a/package/src/ai-components/conversation/list/AIChatList.tsx +++ b/package/src/ai-components/conversation/list/AIChatList.tsx @@ -1,13 +1,16 @@ import React, { type ReactNode, useMemo, useRef } from 'react'; import { ActivityIndicator, Pressable, StyleSheet, View } from 'react-native'; +import Animated, { Easing, Keyframe, useAnimatedStyle } from 'react-native-reanimated'; +import { KeyboardAwareLegendList } from '@legendapp/list/keyboard'; import type { LegendListRef, LegendListRenderItemProps } from '@legendapp/list/react-native'; -import { AnimatedLegendList } from '@legendapp/list/reanimated'; import { useChatListController } from './useChatListController'; +import { keyboardOverlapOf } from './useKeyboardOverlap'; import { useAITranslation, useTheme } from '../../contexts'; import { useAIIcons } from '../../icons'; +import { useBottomSafeArea } from '../../internal/hooks/useBottomSafeArea'; import { primitives } from '../../internal/tokens'; export type AIChatListProps = { @@ -21,6 +24,11 @@ export type AIChatListProps = { bottomInset?: number; /** What shows while there are no rows, centered. */ emptyContent?: ReactNode; + /** + * How far above the bottom of the screen the composer sits, such as above a tab bar: the + * keyboard covers the list only as far as it rises past that. Defaults to the bottom safe area. + */ + keyboardOffset?: number; /** Loads the rows before the first one. `undefined` once there are none left. */ loadOlderMessages?: () => Promise | void; testID?: string; @@ -43,6 +51,15 @@ const LoadingOlder = () => { ); }; +const SCROLL_BUTTON_IN = new Keyframe({ + 0: { opacity: 0, transform: [{ scale: 0.8 }] }, + 100: { easing: Easing.inOut(Easing.ease), opacity: 1, transform: [{ scale: 1 }] }, +}).duration(200); +const SCROLL_BUTTON_OUT = new Keyframe({ + 0: { opacity: 1, transform: [{ scale: 1 }] }, + 100: { easing: Easing.inOut(Easing.ease), opacity: 0, transform: [{ scale: 0.8 }] }, +}).duration(200); + const ScrollToLatestButton = ({ onPress }: { onPress: () => void }) => { const { t } = useAITranslation(); const { @@ -82,6 +99,7 @@ export const AIChatList = ({ data, emptyContent, getItemType, + keyboardOffset: keyboardOffsetProp, keyExtractor, loadOlderMessages, renderItem, @@ -92,7 +110,7 @@ export const AIChatList = ({ } = useTheme(); const listRef = useRef(null); const isEmpty = data.length === 0; - const { contentFloor, isEndInView, listProps, loadsOlder, onContainerLayout, scrollToEnd } = + const { isEndInView, keyboard, keyboardOffset, listProps, loadsOlder, scrollToEnd } = useChatListController({ anchorKey, bottomInset, @@ -102,53 +120,72 @@ export const AIChatList = ({ loadOlderMessages: isEmpty ? undefined : loadOlderMessages, }); - // Flattened: the list reads the floor from it. const contentContainerStyle = useMemo( () => StyleSheet.flatten([ - { minHeight: contentFloor, paddingBottom: primitives.spacingMd + bottomInset }, + { paddingBottom: primitives.spacingMd + bottomInset }, conversation?.content, ]), - [bottomInset, contentFloor, conversation?.content], + [bottomInset, conversation?.content], ); + const bottomSafeArea = useBottomSafeArea(); + const offset = keyboardOffsetProp ?? bottomSafeArea; + keyboardOffset.current = offset; + + // The button rides the keyboard, so it stays in sight above it. + const buttonLift = useAnimatedStyle(() => ({ + transform: [{ translateY: -keyboardOverlapOf(keyboard.value, offset) }], + })); + // What shows while there are no rows stays centered in what the keyboard leaves of the list. + const emptyLift = useAnimatedStyle(() => ({ + paddingBottom: bottomInset + keyboardOverlapOf(keyboard.value, offset), + })); + return ( - + + {/* Mounted even while there are no rows: the list follows the keyboard only from its next + move, so one mounted while the keyboard is up would leave the first turn under it. The + keyboard comes up over the list, which makes room for it below the latest turn: the room + the pinned question leaves takes it first. */} + {isEmpty ? ( - {emptyContent} - - ) : ( - - )} + + ) : null} {!isEndInView && !isEmpty ? ( - - - + + + + ) : null} ); @@ -156,7 +193,15 @@ export const AIChatList = ({ const styles = StyleSheet.create({ container: { flex: 1 }, - empty: { alignItems: 'center', flex: 1, justifyContent: 'center' }, + empty: { + alignItems: 'center', + bottom: 0, + justifyContent: 'center', + left: 0, + position: 'absolute', + right: 0, + top: 0, + }, loadingOlder: { alignItems: 'center', paddingTop: primitives.spacingMd }, scrollButton: { alignItems: 'center', diff --git a/package/src/ai-components/conversation/list/useChatListController.ts b/package/src/ai-components/conversation/list/useChatListController.ts index d339923345..845b0d6a5b 100644 --- a/package/src/ai-components/conversation/list/useChatListController.ts +++ b/package/src/ai-components/conversation/list/useChatListController.ts @@ -1,8 +1,10 @@ import { type RefObject, useCallback, useRef } from 'react'; +import { useKeyboardScrollToEnd } from '@legendapp/list/keyboard'; import type { LegendListRef } from '@legendapp/list/react-native'; import { useEndInView } from './useEndInView'; +import { useKeyboardFrame } from './useKeyboardOverlap'; import { useLoadOlder } from './useLoadOlder'; import { usePinnedAnchor } from './usePinnedAnchor'; @@ -37,8 +39,19 @@ export const useChatListController = ({ // The list's positions leave its header out: the spinner while older messages load. const headerSize = useRef(0); - const { isEndInView, scheduleCheckEnd } = useEndInView({ bottomInset, headerSize, listRef }); - const { anchoredEndSpace, contentFloor, onContainerLayout, onMetricsChange } = usePinnedAnchor({ + // How far above the bottom of the screen the list ends, which the keyboard reaches past. + const keyboardOffset = useRef(0); + const checkEnd = useRef<() => void>(undefined); + const keyboard = useKeyboardFrame(() => checkEnd.current?.()); + const { isEndInView, scheduleCheckEnd } = useEndInView({ + bottomInset, + headerSize, + keyboard, + keyboardOffset, + listRef, + }); + checkEnd.current = scheduleCheckEnd; + const { anchoredEndSpace, onMetricsChange } = usePinnedAnchor({ anchorKey, data, headerSize, @@ -48,25 +61,31 @@ export const useChatListController = ({ }); const loadOlder = useLoadOlder({ listRef, loadOlderMessages }); + // Holds the keyboard's own adjustments of the list still while it scrolls, so they don't stop it + // short of the end above the keyboard. + const { freeze, scrollMessageToEnd } = useKeyboardScrollToEnd({ listRef }); const scrollToEnd = useCallback(() => { - void listRef.current?.scrollToEnd({ animated: !reduceMotion }); - }, [listRef, reduceMotion]); + void scrollMessageToEnd({ animated: !reduceMotion, closeKeyboard: false }); + }, [reduceMotion, scrollMessageToEnd]); return { - /** The least height of the list's content, for its `contentContainerStyle`. */ - contentFloor, isEndInView, + /** The keyboard, for views that ride it, such as the button back to the latest row. */ + keyboard, + /** How far above the bottom of the screen the list ends; the list's view sets it. */ + keyboardOffset, /** Whether older rows load as the list nears its start, so it shows that they are loading. */ loadsOlder: !!loadOlder, listProps: { anchoredEndSpace, + freeze, + // The list growing or shrinking, such as under a taller composer, can bring its end in or out of view. + onLayout: scheduleCheckEnd, onItemSizeChanged: scheduleCheckEnd, onMetricsChange, onScroll: scheduleCheckEnd, ...loadOlder, }, - /** For the view the list fills, which is there before the list is. */ - onContainerLayout, scrollToEnd, }; }; diff --git a/package/src/ai-components/conversation/list/useEndInView.ts b/package/src/ai-components/conversation/list/useEndInView.ts index c9f5144a38..ab78f88765 100644 --- a/package/src/ai-components/conversation/list/useEndInView.ts +++ b/package/src/ai-components/conversation/list/useEndInView.ts @@ -1,7 +1,11 @@ import { type RefObject, useCallback, useEffect, useRef, useState } from 'react'; +import type { SharedValue } from 'react-native-reanimated'; + import type { LegendListRef } from '@legendapp/list/react-native'; +import { type KeyboardFrame, keyboardOverlapOf } from './useKeyboardOverlap'; + // How far from the end still counts as at the end. const END_SLACK = 32; @@ -10,6 +14,10 @@ export type UseEndInViewOptions = { bottomInset: number; /** The height of the list's header, which its positions leave out. */ headerSize: RefObject; + /** The keyboard, which covers the bottom of the list by as much as it reaches over it. */ + keyboard: SharedValue; + /** How far above the bottom of the screen the list ends. */ + keyboardOffset: RefObject; listRef: RefObject; }; @@ -17,9 +25,15 @@ export type UseEndInViewOptions = { * Whether the end of the last row's content is in view: not the room the anchored latest turn * leaves under a short reply, nor the room for the floating composer. Call `scheduleCheckEnd` * whenever that may have changed: a scroll, or a row changing size without one, such as an answer - * typing out. It checks once a frame at most. + * typing out, or the keyboard moving. It checks once a frame at most. */ -export const useEndInView = ({ bottomInset, headerSize, listRef }: UseEndInViewOptions) => { +export const useEndInView = ({ + bottomInset, + headerSize, + keyboard, + keyboardOffset, + listRef, +}: UseEndInViewOptions) => { const [isEndInView, setIsEndInView] = useState(true); const bottomInsetRef = useRef(bottomInset); bottomInsetRef.current = bottomInset; @@ -28,9 +42,10 @@ export const useEndInView = ({ bottomInset, headerSize, listRef }: UseEndInViewO const last = (state?.data.length ?? 0) - 1; if (!state || last < 0) return; const end = headerSize.current + state.positionAtIndex(last) + state.sizeAtIndex(last); - const visibleEnd = state.scroll + state.scrollLength - bottomInsetRef.current; + const covered = keyboardOverlapOf(keyboard.value, keyboardOffset.current); + const visibleEnd = state.scroll + state.scrollLength - bottomInsetRef.current - covered; setIsEndInView(end <= visibleEnd + END_SLACK); - }, [headerSize, listRef]); + }, [headerSize, keyboard, keyboardOffset, listRef]); const checkFrame = useRef(undefined); const scheduleCheckEnd = useCallback(() => { diff --git a/package/src/ai-components/conversation/list/useKeyboardOverlap.ts b/package/src/ai-components/conversation/list/useKeyboardOverlap.ts new file mode 100644 index 0000000000..77259684f1 --- /dev/null +++ b/package/src/ai-components/conversation/list/useKeyboardOverlap.ts @@ -0,0 +1,51 @@ +import { useRef } from 'react'; +import { KeyboardController, useKeyboardHandler } from 'react-native-keyboard-controller'; +import { runOnJS, type SharedValue, useSharedValue } from 'react-native-reanimated'; + +export type KeyboardFrame = { height: number; progress: number }; + +/** + * How far the keyboard reaches over a view that ends `offset` above the bottom of the screen, as + * `KeyboardAwareLegendList` reckons it: the keyboard's height less the offset, scaled by how far + * the keyboard is open. In a worklet or on the JS thread. + */ +export const keyboardOverlapOf = (keyboard: KeyboardFrame, offset: number) => { + 'worklet'; + return Math.max(0, keyboard.height - offset * keyboard.progress); +}; + +/** + * The keyboard's height and how far it is open, as it moves. `onSettled` runs on the JS thread + * once it has finished moving. + */ +export const useKeyboardFrame = (onSettled?: () => void): SharedValue => { + const initial = KeyboardController.state()?.height ?? 0; + const keyboard = useSharedValue({ + height: initial, + progress: initial > 0 ? 1 : 0, + }); + const settled = useRef(onSettled); + settled.current = onSettled; + const reportSettled = () => settled.current?.(); + + useKeyboardHandler( + { + onEnd: (event) => { + 'worklet'; + keyboard.value = { height: event.height, progress: event.progress }; + runOnJS(reportSettled)(); + }, + onInteractive: (event) => { + 'worklet'; + keyboard.value = { height: event.height, progress: event.progress }; + }, + onMove: (event) => { + 'worklet'; + keyboard.value = { height: event.height, progress: event.progress }; + }, + }, + [], + ); + + return keyboard; +}; diff --git a/package/src/ai-components/conversation/list/usePinnedAnchor.ts b/package/src/ai-components/conversation/list/usePinnedAnchor.ts index 68262e1d79..4831bb9c4d 100644 --- a/package/src/ai-components/conversation/list/usePinnedAnchor.ts +++ b/package/src/ai-components/conversation/list/usePinnedAnchor.ts @@ -1,5 +1,4 @@ -import { type RefObject, useCallback, useLayoutEffect, useMemo, useRef, useState } from 'react'; -import type { LayoutChangeEvent } from 'react-native'; +import { type RefObject, useCallback, useLayoutEffect, useMemo, useRef } from 'react'; import type { AnchoredEndSpaceConfig, LegendListRef } from '@legendapp/list/react-native'; @@ -19,8 +18,9 @@ export type UsePinnedAnchorOptions = { /** * Pins a row to the top of what is visible: the list leaves room under it (`anchoredEndSpace`), - * and a new anchor is scrolled to once it is laid out. A content floor keeps it there while what - * is below it shrinks, before the list has measured the change. + * and a new anchor is scrolled to once it is laid out. The keyboard-aware list keeps that room as + * the scroll view's inset, which follows what is below the row shrinking, and which the keyboard + * takes before it lifts anything. */ export const usePinnedAnchor = ({ anchorKey, @@ -36,7 +36,9 @@ export const usePinnedAnchor = ({ // Where to scroll once the anchor is laid out: a new question pins to the top, and a // conversation replaced as a whole opens at its end. - const pendingScroll = useRef<{ animated: boolean; key: string } | undefined>(undefined); + const pendingScroll = useRef<{ animated: boolean; key: string; toEnd: boolean } | undefined>( + undefined, + ); const shown = useRef<{ anchorKey?: string; mounted: boolean }>({ mounted: false }); useLayoutEffect(() => { const previous = shown.current; @@ -47,7 +49,9 @@ export const usePinnedAnchor = ({ const continues = previous.anchorKey !== undefined && data.some((item) => keyExtractor(item) === previous.anchorKey); - pendingScroll.current = { animated: continues && !reduceMotion, key: anchorKey }; + pendingScroll.current = continues + ? { animated: !reduceMotion, key: anchorKey, toEnd: false } + : { animated: false, key: anchorKey, toEnd: true }; }, [anchorKey, data, keyExtractor, reduceMotion]); const scrollToPending = useCallback( @@ -56,56 +60,33 @@ export const usePinnedAnchor = ({ const list = listRef.current; if (!target || !list || key !== target.key) return; pendingScroll.current = undefined; - void list.scrollToKey({ animated: target.animated, key: target.key, viewPosition: 0 }); + // The keyboard-aware list keeps the room under the anchor as the scroll view's inset, which + // it sets on the UI thread a frame or two after the room changes. iOS stops a scroll at the + // inset it has when the scroll starts, so a scroll started now stops short of the top. + requestAnimationFrame(() => + requestAnimationFrame(() => { + const current = listRef.current; + if (target.toEnd) { + void current?.scrollToEnd({ animated: false }); + return; + } + void current?.scrollToKey({ + animated: target.animated, + key: target.key, + viewPosition: 0, + }); + }), + ); }, [listRef], ); - // The least the content can be: the latest question's top plus the list's height, as the native - // SDKs' latest turn is at least as tall as the conversation. The list leaves that room under the - // latest turn itself, but only once it has measured what changed, while the content shrinks at - // once when a row below the question does (a reasoning step folding, the thinking going), and the - // list grows at once when the keyboard goes, either of which would pull the question down. So the - // floor is the tallest the list has been at this width, such as without the keyboard. - const viewport = useRef({ maxLength: 0, width: 0 }); - const [contentFloor, setContentFloor] = useState(0); - const updateContentFloor = useCallback( - (key: string | undefined) => { - const state = listRef.current?.getState(); - const position = key === undefined ? undefined : state?.positionByKey(key); - if (!state || position === undefined) { - setContentFloor(0); - return; - } - viewport.current.maxLength = Math.max(viewport.current.maxLength, state.scrollLength); - setContentFloor(Math.ceil(position + headerSize.current + viewport.current.maxLength)); - }, - [headerSize, listRef], - ); - const anchorKeyRef = useRef(anchorKey); - anchorKeyRef.current = anchorKey; - // The list fills this view, which is there before the list is, such as while the conversation - // is empty and the keyboard comes up for its first question. A new width, such as on rotation, - // has heights of its own. - const onContainerLayout = useCallback( - ({ nativeEvent: { layout } }: LayoutChangeEvent) => { - if (layout.width !== viewport.current.width) { - viewport.current = { maxLength: 0, width: layout.width }; - } - if (layout.height <= viewport.current.maxLength) return; - viewport.current.maxLength = layout.height; - updateContentFloor(anchorKeyRef.current); - }, - [updateContentFloor], - ); - // The spinner above the turns coming or going moves the latest question. + // The spinner above the turns: the list's positions leave it out. const onMetricsChange = useCallback( ({ headerSize: size }: { headerSize: number }) => { - if (headerSize.current === size) return; headerSize.current = size; - updateContentFloor(anchorKeyRef.current); }, - [headerSize, updateContentFloor], + [headerSize], ); const anchorIndex = useMemo( @@ -119,15 +100,14 @@ export const usePinnedAnchor = ({ ? { anchorIndex, onReady: ({ anchorKey: readyKey }) => { - updateContentFloor(readyKey); scrollToPending(readyKey); onAnchorLayoutRef.current?.(); }, onSizeChanged: () => onAnchorLayoutRef.current?.(), } : undefined, - [anchorIndex, scrollToPending, updateContentFloor], + [anchorIndex, scrollToPending], ); - return { anchoredEndSpace, contentFloor, onContainerLayout, onMetricsChange }; + return { anchoredEndSpace, onMetricsChange }; }; diff --git a/package/src/ai-components/internal/hooks/useBottomSafeArea.ts b/package/src/ai-components/internal/hooks/useBottomSafeArea.ts new file mode 100644 index 0000000000..ca5c1a2f6b --- /dev/null +++ b/package/src/ai-components/internal/hooks/useBottomSafeArea.ts @@ -0,0 +1,6 @@ +import { useContext } from 'react'; + +import { SafeAreaInsetsContext } from 'react-native-safe-area-context'; + +/** The bottom safe area, or 0 without a `SafeAreaProvider` (where `useSafeAreaInsets` throws). */ +export const useBottomSafeArea = () => useContext(SafeAreaInsetsContext)?.bottom ?? 0; diff --git a/package/src/ai-components/markdown/components/CodeBlock.tsx b/package/src/ai-components/markdown/components/CodeBlock.tsx index cbdc6833c6..6372eb8fd5 100644 --- a/package/src/ai-components/markdown/components/CodeBlock.tsx +++ b/package/src/ai-components/markdown/components/CodeBlock.tsx @@ -6,13 +6,14 @@ import { parseMermaid } from '../../charts'; import Chart from '../../charts/Chart'; import { MarkdownReactiveScrollView } from '../../components'; import { useStableCallback } from '../../internal/hooks/useStableCallback'; +import { primitives } from '../../internal/tokens'; import { setClipboardString } from '../../services'; import { SyntaxHighlighter } from '../../syntax-highlighting'; import type { MarkdownComponentProps, RuleRenderFunction } from '../types'; export const CodeBlockCopyButton = ({ onPress }: { onPress?: PressableProps['onPress'] }) => ( ({ opacity: pressed ? 0.5 : 1 })} onPress={onPress}> - {'\u29C9'} + {'\u29C9'} ); diff --git a/package/src/ai-components/markdown/styles.ts b/package/src/ai-components/markdown/styles.ts index 0a3ebd1303..8c67bc7bf3 100644 --- a/package/src/ai-components/markdown/styles.ts +++ b/package/src/ai-components/markdown/styles.ts @@ -51,7 +51,8 @@ export const getLocalStyles = ({ theme }: { theme: Theme }) => { fontWeight: '500', color: colors.black, paddingLeft: 8, - lineHeight: 14, + fontSize: primitives.typographyFontSizeXs, + lineHeight: primitives.typographyLineHeightTight, }, codeBlockWrapper: { backgroundColor: colors.code_block, @@ -69,8 +70,8 @@ export const getLocalStyles = ({ theme }: { theme: Theme }) => { codeBlockLineNumberCell: { fontFamily: Platform.OS === 'ios' ? 'Courier' : 'Monospace', color: colors.grey, - paddingVertical: 1, // (codeBlock.lineHeight - fontSize) / 2 - fontSize: 12, + fontSize: primitives.typographyFontSizeXs, + lineHeight: primitives.typographyLineHeightTight, }, codeBlockHeaderContainer: { flexDirection: 'row', @@ -78,7 +79,7 @@ export const getLocalStyles = ({ theme }: { theme: Theme }) => { marginBottom: 8, }, codeBlockHeaderTitle: { - fontSize: 12, + fontSize: primitives.typographyFontSizeXs, color: colors.grey, fontFamily: Platform.OS === 'ios' ? 'Courier' : 'Monospace', }, diff --git a/package/src/ai-components/message-composer/ActionSheet.tsx b/package/src/ai-components/message-composer/ActionSheet.tsx index 2a3e38d4da..b93373b190 100644 --- a/package/src/ai-components/message-composer/ActionSheet.tsx +++ b/package/src/ai-components/message-composer/ActionSheet.tsx @@ -9,6 +9,7 @@ import { withCloseSheet } from './utils/withCloseSheet'; import { useMessageComposerContext, useTheme } from '../contexts'; import { Camera } from '../internal/icons/Camera'; import { Picture } from '../internal/icons/Picture'; +import { primitives } from '../internal/tokens'; import { sheetStoreApi, useComposerChatOption } from '../store'; type BottomSheetContentProps = Pick< @@ -180,7 +181,7 @@ const useStyles = () => { opacity: 0.7, }, quickActionLabel: { - fontSize: 13, + fontSize: primitives.typographyFontSizeXs, fontWeight: '500', color: colors.black, marginTop: 4, @@ -207,13 +208,13 @@ const useStyles = () => { flex: 1, }, listTitle: { - fontSize: 15, + fontSize: primitives.typographyFontSizeSm, fontWeight: '500', color: colors.black, }, listSubtitle: { marginTop: 2, - fontSize: 13, + fontSize: primitives.typographyFontSizeXs, color: colors.grey_dark, }, divider: { diff --git a/package/src/ai-components/message-composer/ComposerView.tsx b/package/src/ai-components/message-composer/ComposerView.tsx index 060f1a7c69..a71f9ae2c4 100644 --- a/package/src/ai-components/message-composer/ComposerView.tsx +++ b/package/src/ai-components/message-composer/ComposerView.tsx @@ -1,6 +1,7 @@ -import React, { useEffect, useMemo, useRef } from 'react'; +import React, { type ReactNode, useEffect, useMemo, useRef } from 'react'; import { Platform, StyleSheet, TextInput, View } from 'react-native'; +import { KeyboardStickyView } from 'react-native-keyboard-controller'; import Animated, { FadeIn, FadeOut, LinearTransition } from 'react-native-reanimated'; import type { StateStore } from '@stream-io/state-store'; @@ -16,6 +17,7 @@ import { MediaPreviewList } from './MediaPreviewList'; import { BottomSheet } from '../components/BottomSheet'; import { ComposerProvider, useMessageComposerContext, useTheme } from '../contexts'; +import { useBottomSafeArea } from '../internal/hooks/useBottomSafeArea'; import { primitives } from '../internal/tokens'; import type { AbstractMediaPickerService, @@ -55,6 +57,20 @@ export type ComposerViewProps = { * picked until the person removes it. It is kept in `state`, as `chatOption`. */ chatOptions?: AIChatOption[]; + /** Shown above the composer, riding the keyboard with it, such as suggestions to start with. */ + header?: ReactNode; + /** + * How far above the bottom of the screen the composer sits, such as above a tab bar: it rides + * the keyboard up by as much as the keyboard rises past that. Defaults to the bottom safe area, + * the same as `AIConversation`'s. Needs a `KeyboardProvider` from react-native-keyboard-controller + * above it. + */ + keyboardOffset?: number; + /** + * Shown after the message box, on the same line, such as a button into voice mode. The + * counterpart of StreamChatAI's trailing composer view. + */ + trailing?: ReactNode; isGenerating?: boolean; stopGenerating?: () => Promise; bottomSheetInsets?: BottomSheetInsets; @@ -66,33 +82,48 @@ export const ComposerViewUI = ({ bottomSheetOptions = [], bottomSheetInsets, chatOptions, + header, isGenerating, + keyboardOffset, stopGenerating, + trailing, }: ComposerViewProps) => { const { theme: { - composer: { container, containerRow, inputPillContainer, inputPill }, + composer: { container, containerRow, inputPillContainer, inputPill, trailing: trailingStyle }, }, } = useTheme(); const styles = useStyles(); + const bottomSafeArea = useBottomSafeArea(); + const offset = useMemo( + () => ({ closed: 0, opened: keyboardOffset ?? bottomSafeArea }), + [bottomSafeArea, keyboardOffset], + ); return ( - - - - - - - - - - + + + {/* Edge to edge, as the room around the composer is its own. */} + {header} + + + + + + + + + + + + + {trailing ? {trailing} : null} - - + + { shadowRadius: 4, elevation: 2, }, + trailing: { + justifyContent: 'center', + marginStart: 8, + minHeight: pillHeight, + }, textInput: { flex: 1, fontSize: primitives.typographyFontSizeMd, diff --git a/package/src/ai-components/message/AIAssistantMessage.tsx b/package/src/ai-components/message/AIAssistantMessage.tsx index b72914f4e0..db12608aec 100644 --- a/package/src/ai-components/message/AIAssistantMessage.tsx +++ b/package/src/ai-components/message/AIAssistantMessage.tsx @@ -35,6 +35,8 @@ export type AIAssistantMessageProps = { isGenerating?: boolean; /** Opens a link in the answer, instead of the system's handler. */ onLink?: (url: string) => void; + /** Called when the answer starts or stops being typed out. */ + onWritingChange?: (isWriting: boolean) => void; /** The steps the agent took, in order, shown before the answer. */ parts?: AIMessagePartData[]; /** The whole of a reasoning step's thinking, when the app has it. */ @@ -53,6 +55,7 @@ export const AIAssistantMessage = ({ id, isGenerating = false, onLink, + onWritingChange, parts = [], reasoningText, text, @@ -71,7 +74,9 @@ export const AIAssistantMessage = ({ ) : null} {/* What arrives after mount is typed out; a reply loaded from history shows at once. */} - {text !== '' ? : null} + {text !== '' ? ( + + ) : null} {attachments.length > 0 ? ( {attachments.map((attachment, index) => ( diff --git a/package/src/ai-components/message/UserMessageMenu.tsx b/package/src/ai-components/message/UserMessageMenu.tsx new file mode 100644 index 0000000000..b06670bde7 --- /dev/null +++ b/package/src/ai-components/message/UserMessageMenu.tsx @@ -0,0 +1,149 @@ +import React, { type ComponentType, type ReactNode } from 'react'; +import { + I18nManager, + Modal, + Platform, + Pressable, + StyleSheet, + Text, + useWindowDimensions, + View, +} from 'react-native'; +import Animated, { Easing, Keyframe } from 'react-native-reanimated'; + +import { useTheme } from '../contexts'; +import { primitives } from '../internal/tokens'; + +/** Where the pressed bubble is on the screen, as `measureInWindow` says. */ +export type UserMessageMenuAnchor = { height: number; width: number; x: number; y: number }; + +export type UserMessageMenuItem = { + Icon: ComponentType<{ color?: string; size?: number }>; + label: string; + onPress: () => void; +}; + +type UserMessageMenuProps = { + anchor: UserMessageMenuAnchor | undefined; + /** The bubble again, drawn lifted over the dimmed screen on iOS. */ + bubble: ReactNode; + items: UserMessageMenuItem[]; + onClose: () => void; +}; + +const MENU_WIDTH = 200; +const ITEM_HEIGHT = 44; +const GAP = primitives.spacingXs; + +const MENU_IN = new Keyframe({ + 0: { opacity: 0, transform: [{ scale: 0.9 }] }, + 100: { easing: Easing.out(Easing.ease), opacity: 1, transform: [{ scale: 1 }] }, +}).duration(150); + +/** + * The menu a long press on the person's message opens, under the bubble or above it when there is + * no room, at its trailing edge. On iOS the screen dims behind the lifted bubble, as a context menu + * does; on Android it drops down from the bubble, as the Android AI SDK's does. + */ +export const UserMessageMenu = ({ anchor, bubble, items, onClose }: UserMessageMenuProps) => { + const { + theme: { colors, message }, + } = useTheme(); + const { height: screenHeight, width: screenWidth } = useWindowDimensions(); + const styles = message?.userMessage; + if (!anchor) return null; + + const menuHeight = items.length * ITEM_HEIGHT; + const fitsBelow = anchor.y + anchor.height + GAP + menuHeight <= screenHeight - GAP; + const top = fitsBelow + ? anchor.y + anchor.height + GAP + : Math.max(GAP, anchor.y - GAP - menuHeight); + // The bubble's trailing edge: its right in a left-to-right layout, its left otherwise. + const left = I18nManager.isRTL + ? Math.min(anchor.x, screenWidth - MENU_WIDTH - GAP) + : Math.max(GAP, anchor.x + anchor.width - MENU_WIDTH); + const lifts = Platform.OS === 'ios'; + + return ( + + + {lifts ? ( + + {bubble} + + ) : null} + + {items.map(({ Icon, label, onPress }) => ( + { + onClose(); + onPress(); + }} + style={({ pressed }) => [ + defaultStyles.item, + pressed && { backgroundColor: colors.white_smoke }, + styles?.menuItem, + ]} + > + + {label} + + ))} + + + ); +}; + +// The design tokens' primary text. +const MENU_TEXT = '#1A1B25'; + +const defaultStyles = StyleSheet.create({ + dim: { backgroundColor: 'rgba(0, 0, 0, 0.2)' }, + item: { + alignItems: 'center', + flexDirection: 'row', + gap: primitives.spacingXs + 4, + height: ITEM_HEIGHT, + paddingHorizontal: primitives.spacingMd, + }, + itemText: { + color: MENU_TEXT, + fontSize: primitives.typographyFontSizeMd, + }, + lifted: { + alignItems: 'flex-end', + position: 'absolute', + shadowColor: '#000000', + shadowOffset: { height: 4, width: 0 }, + shadowOpacity: 0.15, + shadowRadius: 12, + transform: [{ scale: 1.02 }], + }, + menu: { + borderRadius: 12, + elevation: 8, + overflow: 'hidden', + position: 'absolute', + shadowColor: '#000000', + shadowOffset: { height: 4, width: 0 }, + shadowOpacity: 0.15, + shadowRadius: 12, + }, +}); diff --git a/package/src/ai-components/reasoning/AIReasoning.tsx b/package/src/ai-components/reasoning/AIReasoning.tsx index 19ebbc4fa9..205b3f1066 100644 --- a/package/src/ai-components/reasoning/AIReasoning.tsx +++ b/package/src/ai-components/reasoning/AIReasoning.tsx @@ -9,6 +9,12 @@ import { Text, View, } from 'react-native'; +import Animated, { + runOnJS, + useAnimatedStyle, + useSharedValue, + withSpring, +} from 'react-native-reanimated'; import { FINISHED_REASONING_SHOWN_MS, useAIThinkingTurn } from './thinkingClock'; @@ -154,6 +160,10 @@ export type AIReasoningProps = { text?: string; }; +// StreamChatAI's fold, SwiftUI's `spring(response: 0.45, dampingFraction: 0.95)`: a stiffness of +// (2π / 0.45)² and a damping of 2 × 0.95 × √stiffness. With Reduce Motion it folds at once. +const FOLD_SPRING = { damping: 26.5, mass: 1, stiffness: 195 }; + const opensByDefault = ( isThinking: boolean, showsLiveReasoning: boolean, @@ -281,6 +291,26 @@ export const AIReasoning = ({ // Live reasoning is open before any of it arrives, but there is nothing to show until then. const expanded = open && text !== ''; + // The panel shrinks in place as it folds, so what is below it glides up rather than jumps, and + // goes once folded. Live reasoning joins at no height and unfolds; a finished trace that opens + // with the step shows at once. + const [mounted, setMounted] = useState(expanded); + if (expanded && !mounted) setMounted(true); + const progress = useSharedValue(expanded && !isThinking ? 1 : 0); + const panelHeight = useSharedValue(0); + useEffect(() => { + progress.value = withSpring(expanded ? 1 : 0, FOLD_SPRING, (finished) => { + if (finished && !expanded) runOnJS(setMounted)(false); + }); + }, [expanded, progress]); + const foldStyle = useAnimatedStyle(() => ({ + height: panelHeight.value * progress.value, + opacity: progress.value, + })); + const chevronStyle = useAnimatedStyle(() => ({ + transform: [{ rotate: `${progress.value * (I18nManager.isRTL ? -90 : 90)}deg` }], + })); + // The reasoning reads in the secondary colour, a little smaller than an answer. const panelStyle = useMemo>( () => ({ @@ -328,19 +358,28 @@ export const AIReasoning = ({ {summary} ) : null} - + - + - {expanded ? ( - - - {footnote && !isThinking && (choice === true || initiallyExpanded) ? footnote : null} - + {mounted && text !== '' ? ( + + { + panelHeight.value = nativeEvent.layout.height; + }} + style={styles.panel} + > + + + {footnote && !isThinking && (choice === true || initiallyExpanded) ? footnote : null} + + + ) : null} ); @@ -368,7 +407,8 @@ export const AIThinking = ({ since }: AIThinkingProps) => { }; const styles = StyleSheet.create({ - container: { borderStartWidth: 2, gap: 4, paddingStart: 12 }, + container: { borderStartWidth: 2, paddingStart: 12 }, + fold: { overflow: 'hidden' }, header: { alignItems: 'center', alignSelf: 'flex-start', @@ -377,6 +417,7 @@ const styles = StyleSheet.create({ maxWidth: '100%', }, lead: { alignItems: 'center', flexDirection: 'row', gap: 6 }, + panel: { left: 0, paddingTop: 4, position: 'absolute', right: 0, top: 0 }, summary: { flexShrink: 1, fontSize: primitives.typographyFontSizeSm }, title: { fontSize: primitives.typographyFontSizeSm, diff --git a/package/src/ai-components/syntax-highlighting/utils/flatten-nodes.ts b/package/src/ai-components/syntax-highlighting/utils/flatten-nodes.ts index c49e9dd9f2..f9ffbf9b75 100644 --- a/package/src/ai-components/syntax-highlighting/utils/flatten-nodes.ts +++ b/package/src/ai-components/syntax-highlighting/utils/flatten-nodes.ts @@ -3,6 +3,7 @@ import { processColor } from 'react-native'; // @ts-expect-error createStyleObject is not available in the type exports, it is still exported though import { createStyleObject } from 'react-syntax-highlighter/dist/esm/create-element'; +import { primitives } from '../../internal/tokens'; import type { ColorRange } from '../SyntaxHighlighter'; import type { SyntaxHighlighterStylesheet } from '../types'; @@ -19,7 +20,7 @@ export const flattenNodesToTextAndColorRanges = ({ stylesheet, defaultColor, fontFamily, - fontSize = 13, + fontSize = primitives.typographyFontSizeXs, }: FlattenOptions): { text: string; ranges: ColorRange[] } => { let text = ''; const rawRanges: { start: number; end: number; color: string | number }[] = []; diff --git a/package/src/ai-components/syntax-highlighting/utils/renderer.tsx b/package/src/ai-components/syntax-highlighting/utils/renderer.tsx index bce647e88e..8a0859348c 100644 --- a/package/src/ai-components/syntax-highlighting/utils/renderer.tsx +++ b/package/src/ai-components/syntax-highlighting/utils/renderer.tsx @@ -8,6 +8,7 @@ import { createStyleObject } from 'react-syntax-highlighter/dist/esm/create-elem import { cssToRNTextStyle } from './converter'; +import { primitives } from '../../internal/tokens'; import { AndroidNativeRenderer } from '../components'; import type { NativeSyntaxHighlighterProps, @@ -16,7 +17,8 @@ import type { SyntaxHighlighterStylesheet, } from '../types'; -export const DEFAULT_FONT_SIZE = 13; +// Code reads at the footnote size, as in StreamChatAI. +export const DEFAULT_FONT_SIZE = primitives.typographyFontSizeXs; export const generateNewStylesheet = ({ stylesheet, @@ -120,7 +122,7 @@ export const createNativeElement = ({ export const nativeRenderer = ({ defaultColor, fontFamily, - fontSize = 13, + fontSize = DEFAULT_FONT_SIZE, }: { defaultColor: string; fontFamily?: string; diff --git a/package/src/ai-components/theme.ts b/package/src/ai-components/theme.ts index 4d7cb68c43..356482cadf 100644 --- a/package/src/ai-components/theme.ts +++ b/package/src/ai-components/theme.ts @@ -30,6 +30,8 @@ export type Theme = { inputPillContainer: ViewStyle; inputPill: ViewStyle; textInput: TextStyle; + /** What `ComposerView`'s `trailing` sits in, after the message box. */ + trailing: ViewStyle; roundButton: ViewStyle; attachIcon: TextStyle; mediaPreviewStyle: ViewStyle; @@ -169,6 +171,10 @@ export type Theme = { bubble?: ViewStyle; text?: TextStyle; editButton?: ViewStyle; + /** The menu a long press on the message opens, and its items. */ + menu?: ViewStyle; + menuItem?: ViewStyle; + menuItemText?: TextStyle; }; assistantMessage?: { container?: ViewStyle; @@ -269,6 +275,7 @@ export const defaultTheme: Theme = { inputPillContainer: {}, inputPill: {}, textInput: {}, + trailing: {}, roundButton: {}, attachIcon: {}, mediaPreviewStyle: {}, @@ -352,7 +359,16 @@ export const defaultTheme: Theme = { attachmentName: '#1A1B25', attachmentDetails: '#414552', attachmentImagePlaceholder: '#F6F8FA', - userMessage: { container: {}, attachments: {}, bubble: {}, text: {}, editButton: {} }, + userMessage: { + container: {}, + attachments: {}, + bubble: {}, + text: {}, + editButton: {}, + menu: {}, + menuItem: {}, + menuItemText: {}, + }, assistantMessage: { container: {}, attachments: {} }, attachment: { image: {}, file: {}, fileIcon: {}, fileName: {}, fileDetails: {} }, replyActions: { container: {}, button: {} }, diff --git a/yarn.lock b/yarn.lock index ea2fe376b4..0b0b078302 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3878,8 +3878,8 @@ __metadata: linkType: hard "@legendapp/list@npm:^3.7.1": - version: 3.7.1 - resolution: "@legendapp/list@npm:3.7.1" + version: 3.7.2 + resolution: "@legendapp/list@npm:3.7.2" dependencies: use-sync-external-store: "npm:^1.5.0" peerDependencies: @@ -3889,7 +3889,7 @@ __metadata: optional: true react-native: optional: true - checksum: 10c0/a445838e5b99a637fe88b0badc408dc2b9763e57e9e107c0122135359555521c126c44e50767754ec297ec2788874d9b9fec76ce3c90fad5444586e40addd303 + checksum: 10c0/a449c27cdcdac4fbe7741410638307bbc4506a532c3a732d52e50860a2884f7f66d6b0e6108e0c6963faab0121bd7264381c180cdd0f7bdb269b8476f7101b80 languageName: node linkType: hard @@ -10995,6 +10995,7 @@ __metadata: react: "npm:19.3.0" react-native: "npm:0.88.0" react-native-gesture-handler: "npm:~3.4.0" + react-native-keyboard-controller: "npm:1.22.7" react-native-reanimated: "npm:4.7.1" react-native-safe-area-context: "npm:~5.10.1" react-native-screens: "npm:~4.29.0" @@ -17772,6 +17773,19 @@ __metadata: languageName: node linkType: hard +"react-native-keyboard-controller@npm:^1.22.7": + version: 1.23.0 + resolution: "react-native-keyboard-controller@npm:1.23.0" + dependencies: + react-native-is-edge-to-edge: "npm:^1.2.1" + peerDependencies: + react: "*" + react-native: "*" + react-native-reanimated: ">=3.0.0" + checksum: 10c0/8a38af2a51470297094810e2908a407bbf515ea049c312de7c755eecd7b2391f12f178120492989162c85b75b564d320c41065c4bed58fd783c817c4cd3d230b + languageName: node + linkType: hard + "react-native-launch-arguments@npm:^4.1.1": version: 4.1.1 resolution: "react-native-launch-arguments@npm:4.1.1" @@ -19471,6 +19485,7 @@ __metadata: expo-sharing: "*" expo-video: "*" react-native: ">=0.79.0" + react-native-keyboard-controller: ">=1.21.7" victory-native: ">=41.0.0" peerDependenciesMeta: "@shopify/react-native-skia": @@ -19497,6 +19512,8 @@ __metadata: optional: true expo-video: optional: true + react-native-keyboard-controller: + optional: true victory-native: optional: true languageName: unknown @@ -19554,6 +19571,7 @@ __metadata: react-native: "npm:0.88.0" react-native-builder-bob: "npm:0.40.11" react-native-gesture-handler: "npm:^3.4.0" + react-native-keyboard-controller: "npm:^1.22.7" react-native-markdown-package: "npm:1.8.2" react-native-reanimated: "npm:^4.7.1" react-native-safe-area-context: "npm:^5.10.1" @@ -19579,6 +19597,7 @@ __metadata: emoji-mart: ">=5.4.0" react-native: ">=0.79.0" react-native-gesture-handler: ">=2.18.0" + react-native-keyboard-controller: ">=1.21.7" react-native-reanimated: ">=3.16.0" react-native-safe-area-context: ">=5.4.1" react-native-svg: ">=15.8.0" @@ -19595,6 +19614,8 @@ __metadata: optional: true emoji-mart: optional: true + react-native-keyboard-controller: + optional: true victory-native: optional: true languageName: unknown @@ -19620,6 +19641,7 @@ __metadata: react-native-blob-util: ">=0.22.0" react-native-haptic-feedback: ">=2.3.0" react-native-image-picker: ">=7.1.2" + react-native-keyboard-controller: ">=1.21.7" react-native-nitro-sound: ">=0.2.9" react-native-share: ">=11.0.0" react-native-video: ">=6.18.0" @@ -19641,6 +19663,8 @@ __metadata: optional: true react-native-image-picker: optional: true + react-native-keyboard-controller: + optional: true react-native-nitro-sound: optional: true react-native-share: From 12417843443e0e6cb685c90ef9802db270f443dc Mon Sep 17 00:00:00 2001 From: Ivan Sekovanikj Date: Wed, 11 Nov 2026 23:57:08 +0100 Subject: [PATCH 10/23] feat(ai-components): generated UI with an A2UI renderer Ported from #3936 on v9-release. --- .prettierignore | 3 + .yarnrc.yml | 2 + .../ExpoChatGPTSample/agent/UIComponents.json | 76 + examples/ExpoChatGPTSample/agent/agentChat.ts | 6 +- .../ExpoChatGPTSample/agent/deviceTools.ts | 23 +- .../components/TripComponents.tsx | 165 + .../ExpoChatGPTSample/contexts/AppContext.tsx | 140 +- examples/ExpoChatGPTSample/custom-types.d.ts | 5 + examples/ExpoChatGPTSample/package.json | 2 +- .../ExpoChatGPTSample/screens/ChatContent.tsx | 23 +- examples/SampleApp/fastlane/Fastfile | 4 +- package.json | 5 +- .../__fixtures__/a2ui-conformance/NOTICE.md | 6 + .../agent_functions_catalog_v1_0.json | 47 + .../allowed_callers_catalog_v1_0.json | 64 + .../component_metadata_catalog_v1_0.json | 215 + ...ent_metadata_invalid_key_catalog_v1_0.json | 220 + .../catalogs/composition_catalog_v1_0.json | 40 + .../catalogs/custom_catalog_v1_0.json | 54 + .../catalogs/derive_source_catalog_v1_0.json | 79 + .../catalogs/forms_catalog_v1_0.json | 73 + .../function_metadata_catalog_v1_0.json | 73 + .../list_item_action_catalog_v1_0.json | 77 + .../catalogs/literal_map_catalog_v1_0.json | 40 + .../catalogs/simplified_catalog_v1_0.json | 123 + .../catalogs/weighted_catalog_v1_0.json | 145 + .../a2ui-conformance/core/accessibility.json | 722 +++ .../a2ui-conformance/core/actions.json | 458 ++ .../core/composition_constraints.json | 734 +++ .../a2ui-conformance/core/data_context.json | 144 + .../a2ui-conformance/core/data_deletion.json | 358 ++ .../a2ui-conformance/core/data_model.json | 2012 +++++++ .../core/data_model_pointers.json | 358 ++ .../core/error_responses.json | 1093 ++++ .../a2ui-conformance/core/expressions.json | 1270 ++++ .../a2ui-conformance/core/functions.json | 1286 ++++ .../a2ui-conformance/core/index_function.json | 270 + .../core/message_processor_v1_0.json | 5222 +++++++++++++++++ .../a2ui-conformance/core/multi_catalog.json | 237 + .../a2ui-conformance/core/reserved_keys.json | 303 + .../a2ui-conformance/core/rpc_functions.json | 908 +++ .../core/validation_result.json | 599 ++ .../a2ui-conformance/core/validator_v1_0.json | 2792 +++++++++ package/a2ui/package.json | 6 + package/bin/validate-entry-isolation.js | 45 +- package/expo-package/a2ui/index.d.ts | 1 + package/expo-package/a2ui/index.js | 6 + package/expo-package/package.json | 1 + package/native-package/a2ui/index.d.ts | 1 + package/native-package/a2ui/index.js | 6 + package/native-package/package.json | 3 +- package/package.json | 4 +- .../android/agent/AIAgentController.kt | 55 +- package/shared-native/ios/AIAgentModule.swift | 54 +- package/src/a2ui/A2UIContexts.ts | 15 + package/src/a2ui/A2UIInstructions.ts | 103 + package/src/a2ui/A2UIPlaceholder.tsx | 24 + package/src/a2ui/A2UIRenderer.tsx | 505 ++ package/src/a2ui/A2UIStream.ts | 82 + package/src/a2ui/A2UISurface.tsx | 240 + .../a2ui/__tests__/A2UIInstructions.test.ts | 233 + .../src/a2ui/__tests__/A2UIRenderer.test.tsx | 1174 ++++ .../src/a2ui/__tests__/conformance.test.ts | 579 ++ package/src/a2ui/basicCatalog.json | 1206 ++++ package/src/a2ui/basicFunctions.ts | 429 ++ package/src/a2ui/catalogDocument.ts | 121 + package/src/a2ui/catalogs.ts | 185 + package/src/a2ui/components/DateTimeInput.tsx | 380 ++ package/src/a2ui/components/Media.tsx | 264 + .../src/a2ui/components/basicComponents.tsx | 831 +++ package/src/a2ui/components/iconPaths.ts | 116 + package/src/a2ui/components/types.ts | 24 + package/src/a2ui/index.ts | 18 + package/src/a2ui/instructions.ts | 343 ++ package/src/a2ui/secrets.ts | 110 + package/src/ai-components/README.md | 52 + .../__tests__/clientToolRunner.test.ts | 22 + .../__tests__/conversation.test.tsx | 4 + .../__tests__/conversationUI.test.tsx | 481 ++ .../__tests__/generativeUI.test.ts | 634 ++ .../__tests__/messagePartViews.test.tsx | 22 + .../__tests__/messageViews.test.tsx | 4 + .../ai-components/__tests__/signIn.test.tsx | 17 + .../ai-components/__tests__/theme.test.tsx | 18 + .../ai-components/contexts/theme-context.tsx | 9 +- .../conversation/AIConversation.tsx | 58 +- .../src/ai-components/conversation/entries.ts | 33 +- .../conversation/fromStreamMessage.ts | 9 +- .../src/ai-components/conversation/index.ts | 2 +- .../src/ai-components/conversation/types.ts | 6 + .../conversation/useConversationRows.ts | 10 +- .../ai-components/generative-ui/AIUIAction.ts | 73 + .../generative-ui/AIUIArguments.ts | 117 + .../generative-ui/AIUIComponent.tsx | 104 + .../generative-ui/AIUIComponentDefinition.ts | 164 + .../generative-ui/AIUIDocument.ts | 78 + .../generative-ui/AIUIEntrance.tsx | 165 + .../generative-ui/AIUIErrorBoundary.tsx | 39 + .../generative-ui/AIUIHelpers.ts | 86 + .../ai-components/generative-ui/AIUIHost.ts | 360 ++ .../ai-components/generative-ui/AIUIMedia.ts | 45 + .../generative-ui/AIUIRenderer.ts | 61 + .../ai-components/generative-ui/AIUIView.tsx | 79 + .../src/ai-components/generative-ui/index.ts | 48 + package/src/ai-components/index.ts | 1 + .../message-parts/AIToolApproval.tsx | 37 +- .../message-parts/AIToolAuthorization.tsx | 33 +- .../ai-components/message-parts/callStates.ts | 44 + package/src/ai-components/theme.ts | 87 + .../tool-calls/AIClientToolRunner.ts | 16 +- .../tool-calls/authSessionAuthorizer.ts | 8 +- package/src/mock-builders/a2ui/ohrid.ts | 106 + yarn.lock | 138 +- 113 files changed, 30753 insertions(+), 157 deletions(-) create mode 100644 examples/ExpoChatGPTSample/agent/UIComponents.json create mode 100644 examples/ExpoChatGPTSample/components/TripComponents.tsx create mode 100644 package/__fixtures__/a2ui-conformance/NOTICE.md create mode 100644 package/__fixtures__/a2ui-conformance/catalogs/agent_functions_catalog_v1_0.json create mode 100644 package/__fixtures__/a2ui-conformance/catalogs/allowed_callers_catalog_v1_0.json create mode 100644 package/__fixtures__/a2ui-conformance/catalogs/component_metadata_catalog_v1_0.json create mode 100644 package/__fixtures__/a2ui-conformance/catalogs/component_metadata_invalid_key_catalog_v1_0.json create mode 100644 package/__fixtures__/a2ui-conformance/catalogs/composition_catalog_v1_0.json create mode 100644 package/__fixtures__/a2ui-conformance/catalogs/custom_catalog_v1_0.json create mode 100644 package/__fixtures__/a2ui-conformance/catalogs/derive_source_catalog_v1_0.json create mode 100644 package/__fixtures__/a2ui-conformance/catalogs/forms_catalog_v1_0.json create mode 100644 package/__fixtures__/a2ui-conformance/catalogs/function_metadata_catalog_v1_0.json create mode 100644 package/__fixtures__/a2ui-conformance/catalogs/list_item_action_catalog_v1_0.json create mode 100644 package/__fixtures__/a2ui-conformance/catalogs/literal_map_catalog_v1_0.json create mode 100644 package/__fixtures__/a2ui-conformance/catalogs/simplified_catalog_v1_0.json create mode 100644 package/__fixtures__/a2ui-conformance/catalogs/weighted_catalog_v1_0.json create mode 100644 package/__fixtures__/a2ui-conformance/core/accessibility.json create mode 100644 package/__fixtures__/a2ui-conformance/core/actions.json create mode 100644 package/__fixtures__/a2ui-conformance/core/composition_constraints.json create mode 100644 package/__fixtures__/a2ui-conformance/core/data_context.json create mode 100644 package/__fixtures__/a2ui-conformance/core/data_deletion.json create mode 100644 package/__fixtures__/a2ui-conformance/core/data_model.json create mode 100644 package/__fixtures__/a2ui-conformance/core/data_model_pointers.json create mode 100644 package/__fixtures__/a2ui-conformance/core/error_responses.json create mode 100644 package/__fixtures__/a2ui-conformance/core/expressions.json create mode 100644 package/__fixtures__/a2ui-conformance/core/functions.json create mode 100644 package/__fixtures__/a2ui-conformance/core/index_function.json create mode 100644 package/__fixtures__/a2ui-conformance/core/message_processor_v1_0.json create mode 100644 package/__fixtures__/a2ui-conformance/core/multi_catalog.json create mode 100644 package/__fixtures__/a2ui-conformance/core/reserved_keys.json create mode 100644 package/__fixtures__/a2ui-conformance/core/rpc_functions.json create mode 100644 package/__fixtures__/a2ui-conformance/core/validation_result.json create mode 100644 package/__fixtures__/a2ui-conformance/core/validator_v1_0.json create mode 100644 package/a2ui/package.json create mode 100644 package/expo-package/a2ui/index.d.ts create mode 100644 package/expo-package/a2ui/index.js create mode 100644 package/native-package/a2ui/index.d.ts create mode 100644 package/native-package/a2ui/index.js create mode 100644 package/src/a2ui/A2UIContexts.ts create mode 100644 package/src/a2ui/A2UIInstructions.ts create mode 100644 package/src/a2ui/A2UIPlaceholder.tsx create mode 100644 package/src/a2ui/A2UIRenderer.tsx create mode 100644 package/src/a2ui/A2UIStream.ts create mode 100644 package/src/a2ui/A2UISurface.tsx create mode 100644 package/src/a2ui/__tests__/A2UIInstructions.test.ts create mode 100644 package/src/a2ui/__tests__/A2UIRenderer.test.tsx create mode 100644 package/src/a2ui/__tests__/conformance.test.ts create mode 100644 package/src/a2ui/basicCatalog.json create mode 100644 package/src/a2ui/basicFunctions.ts create mode 100644 package/src/a2ui/catalogDocument.ts create mode 100644 package/src/a2ui/catalogs.ts create mode 100644 package/src/a2ui/components/DateTimeInput.tsx create mode 100644 package/src/a2ui/components/Media.tsx create mode 100644 package/src/a2ui/components/basicComponents.tsx create mode 100644 package/src/a2ui/components/iconPaths.ts create mode 100644 package/src/a2ui/components/types.ts create mode 100644 package/src/a2ui/index.ts create mode 100644 package/src/a2ui/instructions.ts create mode 100644 package/src/a2ui/secrets.ts create mode 100644 package/src/ai-components/__tests__/conversationUI.test.tsx create mode 100644 package/src/ai-components/__tests__/generativeUI.test.ts create mode 100644 package/src/ai-components/generative-ui/AIUIAction.ts create mode 100644 package/src/ai-components/generative-ui/AIUIArguments.ts create mode 100644 package/src/ai-components/generative-ui/AIUIComponent.tsx create mode 100644 package/src/ai-components/generative-ui/AIUIComponentDefinition.ts create mode 100644 package/src/ai-components/generative-ui/AIUIDocument.ts create mode 100644 package/src/ai-components/generative-ui/AIUIEntrance.tsx create mode 100644 package/src/ai-components/generative-ui/AIUIErrorBoundary.tsx create mode 100644 package/src/ai-components/generative-ui/AIUIHelpers.ts create mode 100644 package/src/ai-components/generative-ui/AIUIHost.ts create mode 100644 package/src/ai-components/generative-ui/AIUIMedia.ts create mode 100644 package/src/ai-components/generative-ui/AIUIRenderer.ts create mode 100644 package/src/ai-components/generative-ui/AIUIView.tsx create mode 100644 package/src/ai-components/generative-ui/index.ts create mode 100644 package/src/ai-components/message-parts/callStates.ts create mode 100644 package/src/mock-builders/a2ui/ohrid.ts diff --git a/.prettierignore b/.prettierignore index 796a04e4c9..860d0a1b29 100644 --- a/.prettierignore +++ b/.prettierignore @@ -41,6 +41,9 @@ package/src/theme/generated/ perf/gauntlet/out/ perf/gauntlet/runtime/generated/ +# A2UI's conformance cases, kept as upstream wrote them +package/__fixtures__/a2ui-conformance/ + # Workflow / tooling configs not owned by this project's Prettier .github/ .husky/ diff --git a/.yarnrc.yml b/.yarnrc.yml index e37b3de9cf..c1af242b7d 100644 --- a/.yarnrc.yml +++ b/.yarnrc.yml @@ -22,6 +22,8 @@ npmPreapprovedPackages: - '@stream-io/state-store' - '@stream-io/worker-timer' - '@stream-io/ws' + - '@stream-io/calypso' + - '@a2ui/web_core' npmPublishProvenance: true diff --git a/examples/ExpoChatGPTSample/agent/UIComponents.json b/examples/ExpoChatGPTSample/agent/UIComponents.json new file mode 100644 index 0000000000..869bd82218 --- /dev/null +++ b/examples/ExpoChatGPTSample/agent/UIComponents.json @@ -0,0 +1,76 @@ +[ + { + "name": "place_card", + "description": "One place of a plan, with its photo and a short summary, which the phone loads from Wikipedia, so it needs no image URL. Use one per sight, beach, monastery, village or day trip.", + "parameters": { + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "The place's exact English Wikipedia article title, such as \"Church of St. John at Kaneo\"" + }, + "note": { + "type": "string", + "description": "One short line about it for this plan, such as \"Best at sunset\" or \"2 hours\"" + } + }, + "required": ["title"], + "x-order": ["title", "note"] + }, + "events": [ + { + "name": "ask_about", + "description": "Tapping the card asks you about the place." + } + ] + }, + { + "name": "trip_map", + "description": "A map of a plan with a pin for each place, drawn on the phone. Use one near the top of every trip plan.", + "parameters": { + "type": "object", + "properties": { + "places": { + "type": "array", + "description": "Every place of the plan, with its coordinates", + "items": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "latitude": { "type": "number" }, + "longitude": { "type": "number" } + }, + "required": ["name", "latitude", "longitude"], + "x-order": ["name", "latitude", "longitude"] + } + } + }, + "required": ["places"], + "x-order": ["places"] + }, + "events": [ + { + "name": "select_place", + "description": "Tapping a pin asks you about that place." + } + ] + }, + { + "name": "calendar_check", + "description": "The person's own calendar for the days of a plan, read on the phone. You never see it: use it whenever a plan has dates, so they can see what the trip would clash with.", + "parameters": { + "type": "object", + "properties": { + "start": { + "type": "string", + "format": "date", + "description": "The first day, as YYYY-MM-DD" + }, + "end": { "type": "string", "format": "date", "description": "The last day, as YYYY-MM-DD" } + }, + "required": ["start", "end"], + "x-order": ["start", "end"] + }, + "events": [] + } +] diff --git a/examples/ExpoChatGPTSample/agent/agentChat.ts b/examples/ExpoChatGPTSample/agent/agentChat.ts index 040d7f2a69..0f4210849a 100644 --- a/examples/ExpoChatGPTSample/agent/agentChat.ts +++ b/examples/ExpoChatGPTSample/agent/agentChat.ts @@ -25,6 +25,9 @@ export class AgentChat { /** Each reply's thinking in full, from its live updates. A stored step only keeps its opening. */ readonly reasoning = new LiveReasoning(); + /** Whether the person asked something here, which may not have arrived in `messages` yet. */ + wasAsked = false; + readonly state = new StateStore({ hasOlderMessages: false, isLoaded: false, @@ -95,9 +98,10 @@ export class AgentChat { /** A reply the agent wrote. */ export const isAgentReply = (message: LocalMessage) => message.ai_generated === true; -/** A reply with nothing to show yet: no text and no steps. */ +/** A reply with nothing to show yet: no text, no steps and no UI. */ export const isEmptyReply = (message: LocalMessage) => !message.text && + !message.ai_ui && !(message.attachments ?? []).some( (attachment) => attachment.type === 'ai_reasoning' || attachment.type === 'ai_tool_call', ); diff --git a/examples/ExpoChatGPTSample/agent/deviceTools.ts b/examples/ExpoChatGPTSample/agent/deviceTools.ts index c1c830d732..205bbe8890 100644 --- a/examples/ExpoChatGPTSample/agent/deviceTools.ts +++ b/examples/ExpoChatGPTSample/agent/deviceTools.ts @@ -8,32 +8,33 @@ import { completedToolResult, } from 'stream-chat-expo/ai-components'; -/** What only this phone knows: its clock and time zone. */ -const localTime: AIClientTool = { +/** + * What only this phone knows: its date, clock, time zone and language. The same tool, by the same + * name, as the iOS, Android and web assistants, whose instructions call it before a plan with + * dates. + */ +const localContext: AIClientTool = { definition: { annotations: { readOnlyHint: true }, description: - "Gets the date, time and time zone on the person's phone. The only way to know their local time.", + "The person's current date, local time, timezone and language. Call it before answering anything that depends on today or the time.", inputSchema: { properties: {}, type: 'object' }, - name: 'get_local_time', + name: 'get_local_context', }, run: () => { const now = new Date(); + const { locale, timeZone } = Intl.DateTimeFormat().resolvedOptions(); return completedToolResult( - { - local: now.toString(), - timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone, - utc: now.toISOString(), - }, + { language: locale, local: now.toString(), timeZone, utc: now.toISOString() }, 'Shared the local time', ); }, }; -const DEVICE_TOOLS = [localTime]; +const DEVICE_TOOLS = [localContext]; // What a call says it is doing on the reply's step, by tool. -const DISPLAY_TITLES: Record = { get_local_time: "Checking your phone's time" }; +const DISPLAY_TITLES: Record = { get_local_context: "Checking your phone's time" }; /** * The results the runner reported, by call, for the session's socket to answer with. A call can diff --git a/examples/ExpoChatGPTSample/components/TripComponents.tsx b/examples/ExpoChatGPTSample/components/TripComponents.tsx new file mode 100644 index 0000000000..1d62b0807a --- /dev/null +++ b/examples/ExpoChatGPTSample/components/TripComponents.tsx @@ -0,0 +1,165 @@ +import { useEffect, useState } from 'react'; +import { Image, Pressable, StyleSheet, Text, View } from 'react-native'; + +import { + type AIUIComponent, + type AIUIComponentDefinition, + defineAIUIComponent, + parseAIUIComponentDefinition, +} from 'stream-chat-expo/ai-components'; + +import uiComponents from '../agent/UIComponents.json'; + +// The trip planner's components, as the Swift, Android and web assistants describe them, from the +// same `UIComponents.json` the agent's instructions are written from. +const definitionOf = (name: string): AIUIComponentDefinition => { + const definition = parseAIUIComponentDefinition( + uiComponents.find((component) => component.name === name), + ); + if (!definition) throw new Error(`UIComponents.json has no ${name}`); + return definition; +}; + +type Summary = { extract?: string; thumbnail?: { source: string } }; + +// A place's photo and first lines from Wikipedia, loaded on the phone, so the agent writes no +// image links. +const useWikipediaSummary = (title: string) => { + const [summary, setSummary] = useState(); + useEffect(() => { + const controller = new AbortController(); + fetch( + `https://en.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent(title.replace(/ /g, '_'))}`, + { signal: controller.signal }, + ) + .then((response) => (response.ok ? response.json() : undefined)) + .then((json: Summary | undefined) => setSummary(json)) + .catch(() => undefined); + return () => controller.abort(); + }, [title]); + return summary; +}; + +const PlaceCard = defineAIUIComponent<{ title: string; note?: string }>( + definitionOf('place_card'), + ({ args, context }) => { + const summary = useWikipediaSummary(args.title); + return ( + + context.emit('ask_about', `Tell me more about ${args.title}`, { title: args.title }) + } + style={({ pressed }) => [styles.card, pressed && styles.pressed]} + > + {summary?.thumbnail ? ( + + ) : ( + + )} + + + {args.title} + + {args.note ? {args.note} : null} + {summary?.extract ? ( + + {summary.extract} + + ) : null} + + + ); + }, +); + +type Place = { latitude: number; longitude: number; name: string }; + +const TripMap = defineAIUIComponent<{ places: Place[] }>( + definitionOf('trip_map'), + ({ args, context }) => ( + + {args.places.map((place, index) => ( + + context.emit('select_place', `Tell me more about ${place.name}`, { name: place.name }) + } + style={({ pressed }) => [styles.pin, pressed && styles.pressed]} + > + {index + 1} + + {place.name} + + + {place.latitude.toFixed(3)}, {place.longitude.toFixed(3)} + + + ))} + + ), +); + +const dayOf = (date: Date) => + date.toLocaleDateString(undefined, { day: 'numeric', month: 'short', weekday: 'short' }); + +const CalendarCheck = defineAIUIComponent<{ end: Date; start: Date }>( + definitionOf('calendar_check'), + ({ args }) => ( + + + {dayOf(args.start)} – {dayOf(args.end)} + + Your calendar stays on this phone. + + ), +); + +/** The app's components the agent can place in a reply's UI. */ +export const tripComponents: AIUIComponent[] = [PlaceCard, TripMap, CalendarCheck]; + +const styles = StyleSheet.create({ + calendar: { + backgroundColor: '#ffffff', + borderColor: '#ececec', + borderRadius: 12, + borderWidth: 1, + gap: 2, + padding: 12, + }, + card: { + alignItems: 'center', + backgroundColor: '#ffffff', + borderColor: '#ececec', + borderRadius: 12, + borderWidth: 1, + flexDirection: 'row', + gap: 12, + padding: 8, + }, + cardText: { flex: 1, gap: 2 }, + coordinates: { color: '#7a7a7a', fontSize: 12 }, + extract: { color: '#4b4b4b', fontSize: 13 }, + map: { backgroundColor: '#eef4ff', borderRadius: 12, gap: 4, padding: 8 }, + note: { color: '#7a7a7a', fontSize: 13 }, + photo: { borderRadius: 8, height: 64, width: 64 }, + photoPlaceholder: { backgroundColor: '#f2f2f2' }, + pin: { alignItems: 'center', flexDirection: 'row', gap: 8, paddingVertical: 6 }, + pinIndex: { + backgroundColor: '#005fff', + borderRadius: 10, + color: '#ffffff', + fontSize: 12, + fontWeight: '700', + height: 20, + lineHeight: 20, + overflow: 'hidden', + textAlign: 'center', + width: 20, + }, + pinName: { flex: 1, fontSize: 15 }, + pressed: { opacity: 0.7 }, + title: { fontSize: 15, fontWeight: '600' }, +}); diff --git a/examples/ExpoChatGPTSample/contexts/AppContext.tsx b/examples/ExpoChatGPTSample/contexts/AppContext.tsx index e6dc394d04..92e13c48fd 100644 --- a/examples/ExpoChatGPTSample/contexts/AppContext.tsx +++ b/examples/ExpoChatGPTSample/contexts/AppContext.tsx @@ -1,7 +1,7 @@ import type { PropsWithChildren } from 'react'; import React, { useEffect, useMemo, useRef, useState } from 'react'; -import { type ConversationSnapshot, type Session } from '@stream-io/calypso'; +import { type ConversationSnapshot, type Session, type UIAction } from '@stream-io/calypso'; import type { Channel, LocalMessage, StreamChat } from 'stream-chat'; import { useStableCallback, useStateStore } from 'stream-chat-expo'; import { @@ -32,34 +32,43 @@ const deviceTools = createDeviceTools({ clientId, userId: AgentConfig.user.id }) const SIGN_IN_SCHEME = 'io.getstream.expochatgptsample'; const SIGN_IN_RETURN_URL = `${SIGN_IN_SCHEME}://sign-in`; -// The person's "not now" on a sign-in. calypso 0.2.3 has no `declineLogin` yet, so this sends -// the frame Vision-Agents' JS SDK sends for it over the session's socket. -const declineLogin = (session: Session, toolCallId: string) => - (session as unknown as { held(): { send(frame: Record): void } }) - .held() - .send({ allowed: false, tool_call_id: toolCallId, type: 'tool_approval' }); - -// Asks. The router addresses a client tool's call to the install a question came from, which -// only its `respond` endpoint takes. `responses` (calypso's `responses.create`) takes photos and a -// reasoning effort but no install, so a question with either reaches the device's tools without -// one, and the phone runs them itself. +// The person's answer to a call that asks first. calypso's `decide` answers only a call this +// session was asked on its own socket, and leaves any other alone, so a question shown from the +// channel, as after reopening the conversation or reloading the app, would wait forever. Such a +// call is answered with the frames `decide` sends, which the router matches by the call's id: an +// allowed client tool's call then waits for this phone, whose runner runs it. +const answerApproval = (session: Session, toolCallId: string, allowed: boolean) => { + if (session.approvals.some((request) => request.id === toolCallId)) { + session.decide(toolCallId, { allowed }); + return; + } + const socket = ( + session as unknown as { held(): { send(frame: Record): void } } + ).held(); + socket.send({ allowed, tool_call_id: toolCallId, type: 'tool_approval' }); + if (!allowed) { + socket.send({ + error: 'The person declined this call.', + tool_call_id: toolCallId, + type: 'tool_result', + }); + } +}; + +// Asks, with calypso's `responses.create`, which takes photos, a reasoning effort and what the +// person did in a reply's UI. The router's `respond`, which addresses a client tool's call to the +// install a question came from, is only for an app's backend, and this is a phone: the calls reach +// the device's tools without an install, and the phone runs them itself. const respond = async ( session: Session, text: string, - options: { images: { url: string }[]; reasoningEffort?: string }, + options: { images: { url: string }[]; reasoningEffort?: string; uiAction?: UIAction }, ) => { - if (options.images.length > 0 || options.reasoningEffort) { - deviceTools.addressed = false; - await session.responses.create(text, { - ...(options.images.length > 0 ? { images: options.images } : {}), - ...(options.reasoningEffort ? { reasoningEffort: options.reasoningEffort } : {}), - }); - return; - } - deviceTools.addressed = true; - await agents.post('/v1/agents/sessions/{id}/respond', { - body: { client_id: clientId, request_id: crypto.randomUUID(), text }, - path: { id: session.id }, + deviceTools.addressed = false; + await session.responses.create(text, { + ...(options.images.length > 0 ? { images: options.images } : {}), + ...(options.reasoningEffort ? { reasoningEffort: options.reasoningEffort } : {}), + ...(options.uiAction ? { uiAction: options.uiAction } : {}), }); }; @@ -87,10 +96,16 @@ export type AppContextValue = { /** The open conversation's session id. */ sessionId: string | undefined; /** - * Asks the agent, with photos by their local URIs, and with more time to reason when - * `thinkHarder`. Its reply arrives in the channel. + * Asks the agent, with photos by their local URIs, with more time to reason when + * `thinkHarder`, and with what the person did in a reply's UI as `uiAction`. Its reply arrives + * in the channel. Resolves to whether it was asked: not while the agent is busy. */ - ask: (text: string, photos?: string[], thinkHarder?: boolean) => Promise; + ask: ( + text: string, + photos?: string[], + thinkHarder?: boolean, + uiAction?: UIAction, + ) => Promise; /** Carries on a conversation from the drawer, by its session id. */ openConversation: (sessionId: string) => void; /** Rates a reply, or takes the rating back with null. */ @@ -106,7 +121,7 @@ export type AppContextValue = { export const AppContext = React.createContext({ agentState: IDLE, approver: undefined, - ask: async () => {}, + ask: () => Promise.resolve(false), chat: undefined, conversationsVersion: 0, failure: undefined, @@ -144,6 +159,8 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St // The newest message when something was last asked here, until it fails or is stopped. const [askedAfter, setAskedAfter] = useState(); const current = useRef(undefined); + // The latest conversation asked to open: one that opens after it was asked for another is let go. + const openings = useRef(0); const askedRef = useRef(undefined); askedRef.current = asked; @@ -159,37 +176,51 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St }, [lastReply]); // Opening one conversation after another: the newest wins, and one that was never asked - // anything is deleted rather than left in the drawer. - const open = useStableCallback(async (opening: () => Promise) => { + // anything is deleted rather than left in the drawer. A conversation counts as asked from the + // moment the question goes, before Stream's copy of it arrives. + const letGo = (chat: AgentChat) => { + chat.close(); + const unused = !chat.wasAsked && chat.state.getLatestValue().messages.length === 0; + return (unused ? chat.session.delete() : chat.session.close()).catch(() => {}); + }; + + const open = useStableCallback(async (opening: () => Promise, isNew: boolean) => { + const attempt = ++openings.current; setFailure(undefined); + let session: Session | undefined; try { - const session = await opening(); + session = await opening(); const { channel } = await session.chat(); const next = new AgentChat(session, channel as Channel); await next.follow(); + if (attempt !== openings.current) { + // Another conversation was asked for while this one opened. + await letGo(next); + return; + } const previous = current.current; current.current = next; setAsked(undefined); setAskedAfter(undefined); setChat(next); - if (previous && previous.session.id !== session.id) { - previous.close(); - const unused = previous.state.getLatestValue().messages.length === 0; - await (unused ? previous.session.delete() : previous.session.close()).catch(() => {}); - } + if (previous && previous.session.id !== session.id) await letGo(previous); setConversationsVersion((version) => version + 1); } catch (error) { - setFailure(messageOf(error)); + // A new conversation that couldn't open was never asked anything: it isn't left in the drawer. + if (isNew) await session?.delete().catch(() => {}); + if (attempt === openings.current) setFailure(messageOf(error)); } }); const startConversation = useStableCallback(() => { - void open(() => - agentSessions().create({ - modelOverwrites: { llm: AgentConfig.chatModel }, - return_url: SIGN_IN_RETURN_URL, - tools: deviceTools.sessionTools(), - }), + void open( + () => + agentSessions().create({ + modelOverwrites: { llm: AgentConfig.chatModel }, + return_url: SIGN_IN_RETURN_URL, + tools: deviceTools.sessionTools(), + }), + true, ); }); @@ -197,17 +228,20 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St if (sessionId === current.current?.session.id) { return; } - void open(() => agentSessions().resume(sessionId, { tools: deviceTools.sessionTools() })); + void open( + () => agentSessions().resume(sessionId, { tools: deviceTools.sessionTools() }), + false, + ); }); const isResponding = agentState.kind === 'responding' || agentState.kind === 'working'; const isBusy = !!asked || isResponding || followed?.lastReply?.generating === true; const ask = useStableCallback( - async (text: string, photos: string[] = [], thinkHarder = false) => { + async (text: string, photos: string[] = [], thinkHarder = false, uiAction?: UIAction) => { const opened = current.current; if (!opened || isBusy) { - return; + return false; } // Shows at once, with the photos as they were picked. const asking: Asked = { @@ -219,6 +253,8 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St }, }; const messages = opened.state.getLatestValue().messages; + // From now on the conversation is used, even before Stream's copy of the question arrives. + opened.wasAsked = true; setFailure(undefined); setAsked(asking); setAskedAfter(messages[messages.length - 1]?.id ?? ''); @@ -229,11 +265,14 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St await respond(opened.session, text, { images, reasoningEffort: thinkHarder ? 'high' : undefined, + uiAction, }); + return true; } catch (error) { setFailure(messageOf(error)); setAskedAfter(undefined); if (askedRef.current === asking) setAsked(undefined); + return false; } }, ); @@ -306,15 +345,16 @@ export const AppProvider = ({ client, children }: PropsWithChildren<{ client: St callbackScheme: SIGN_IN_SCHEME, decline: (call) => new Promise((resolve) => { - declineLogin(chat.session, call.id); + chat.session.declineLogin(call.id); resolve(); }), }), clientId, - // `decide` sends over the session's socket and throws when it can't. + // Sends over the session's socket, and throws when it can't, so the card lets the + // person answer again. decide: (call, allowed) => new Promise((resolve) => { - chat.session.decide(call.id, { allowed }); + answerApproval(chat.session, call.id, allowed); resolve(); }), userId: AgentConfig.user.id, diff --git a/examples/ExpoChatGPTSample/custom-types.d.ts b/examples/ExpoChatGPTSample/custom-types.d.ts index 4af867514c..e8aceb7b9b 100644 --- a/examples/ExpoChatGPTSample/custom-types.d.ts +++ b/examples/ExpoChatGPTSample/custom-types.d.ts @@ -10,6 +10,7 @@ import { DefaultThreadData, DefaultUserData, } from 'stream-chat-expo'; +import type { AIUIActionJSON, AIUIDocumentJSON } from 'stream-chat-expo/ai-components'; declare module 'stream-chat' { // Module augmentation: the empty interfaces are how the default data types are applied. @@ -29,6 +30,10 @@ declare module 'stream-chat' { ai_generated?: boolean; /** Set by the router while the agent is still writing a reply. */ generating?: boolean; + /** The UI the agent wrote for a reply, which the router moves out of its text. */ + ai_ui?: AIUIDocumentJSON; + /** What the person did in a reply's UI, on their message that says so. */ + ai_ui_action?: AIUIActionJSON; } interface CustomPollOptionData extends DefaultPollOptionData {} diff --git a/examples/ExpoChatGPTSample/package.json b/examples/ExpoChatGPTSample/package.json index 75142db434..6597c83f3e 100644 --- a/examples/ExpoChatGPTSample/package.json +++ b/examples/ExpoChatGPTSample/package.json @@ -12,7 +12,7 @@ "dependencies": { "@react-native-community/netinfo": "12.0.1", "@shopify/react-native-skia": "^2.14.0", - "@stream-io/calypso": "^0.2.3", + "@stream-io/calypso": "^0.3.0", "expo": "^58.0.8", "expo-clipboard": "~58.0.3", "expo-constants": "~58.0.11", diff --git a/examples/ExpoChatGPTSample/screens/ChatContent.tsx b/examples/ExpoChatGPTSample/screens/ChatContent.tsx index 16e8d5c040..a63360aa79 100644 --- a/examples/ExpoChatGPTSample/screens/ChatContent.tsx +++ b/examples/ExpoChatGPTSample/screens/ChatContent.tsx @@ -5,11 +5,14 @@ import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { useStateStore } from 'stream-chat-expo'; +import { createA2UIRenderer } from 'stream-chat-expo/a2ui'; import { AIConversation, type AIMessage, AISuggestions, AITypingIndicatorView, + aiUIActionJSON, + AIUIHost, ComposerView, createFromStreamMessage, createNewComposerStore, @@ -17,6 +20,7 @@ import { import type { AgentChat, AgentChatState } from '../agent/agentChat'; import { chatOptions, STARTERS, THINK_HARDER } from '../chatOptions'; +import { tripComponents } from '../components/TripComponents'; import { VoiceMode } from '../components/VoiceMode'; import { useAppContext } from '../contexts/AppContext'; import { useSendAIMessage } from '../hooks/useSendAIMessage'; @@ -110,9 +114,25 @@ const conversationSelector = ({ hasOlderMessages, messages }: AgentChatState) => /** The open conversation, read from its channel by `AgentChat`. */ const Conversation = ({ chat }: { chat: AgentChat }) => { - const { agentState, approver, isThinking, pending, rate, ratings } = useAppContext(); + const { agentState, approver, ask, isThinking, pending, rate, ratings } = useAppContext(); const { hasOlderMessages, messages } = useStateStore(chat.state, conversationSelector); + // The UI the agent writes for its replies, in A2UI, with the trip planner's own components. One + // host per conversation, for as long as it shows, so what a reply's UI holds survives scrolling. + // What the person does in it goes to the agent as their next message. + const [ui] = useState( + () => + new AIUIHost({ + components: tripComponents, + renderers: [createA2UIRenderer()], + send: async (action) => { + if (!(await ask(action.label, [], false, aiUIActionJSON(action)))) { + throw new Error('The agent is still answering'); + } + }, + }), + ); + // The same message object for each Stream message that didn't change, so only the messages an // update touched render again. The live thinking is read through `chat.reasoning`. const [fromStreamMessage] = useState(() => @@ -151,6 +171,7 @@ const Conversation = ({ chat }: { chat: AgentChat }) => { messages={conversation} pendingMessage={pending} replyActions={replyActions} + ui={ui} /> ); }; diff --git a/examples/SampleApp/fastlane/Fastfile b/examples/SampleApp/fastlane/Fastfile index 42baf1e14c..7c1ddf7000 100644 --- a/examples/SampleApp/fastlane/Fastfile +++ b/examples/SampleApp/fastlane/Fastfile @@ -542,10 +542,12 @@ lane :frameworks_sizes do sh('yarn build') sh('yarn minify-bundle') sh('yarn minify-ai-components-bundle') + sh('yarn minify-a2ui-bundle') { js_bundle_size: file_size(path: 'package/lib/module/bundle.min.js'), - ai_components_size: file_size(path: 'package/lib/ai-components.min.js') + ai_components_size: file_size(path: 'package/lib/ai-components.min.js'), + a2ui_size: file_size(path: 'package/lib/a2ui.min.js') } end end diff --git a/package.json b/package.json index ee702cd07b..7bb99336e5 100644 --- a/package.json +++ b/package.json @@ -76,8 +76,9 @@ "build": "yarn workspace stream-chat-react-native-core build", "test:coverage": "yarn workspace stream-chat-react-native-core test:coverage", "test:unit": "yarn workspace stream-chat-react-native-core test:unit", - "minify-bundle": "uglifyjs package/lib/module/*.js package/lib/module/!(ai-components)/**/*.js -o package/lib/module/bundle.min.js -c -m", - "minify-ai-components-bundle": "uglifyjs package/lib/module/ai-components/**/*.js -o package/lib/ai-components.min.js -c -m" + "minify-bundle": "uglifyjs package/lib/module/*.js package/lib/module/!(ai-components|a2ui)/**/*.js -o package/lib/module/bundle.min.js -c -m", + "minify-ai-components-bundle": "uglifyjs package/lib/module/ai-components/**/*.js -o package/lib/ai-components.min.js -c -m", + "minify-a2ui-bundle": "uglifyjs package/lib/module/a2ui/**/*.js -o package/lib/a2ui.min.js -c -m" }, "packageManager": "yarn@4.15.0" } diff --git a/package/__fixtures__/a2ui-conformance/NOTICE.md b/package/__fixtures__/a2ui-conformance/NOTICE.md new file mode 100644 index 0000000000..a2472d4e53 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/NOTICE.md @@ -0,0 +1,6 @@ +# A2UI conformance suite + +The cases in `core/` and the catalogs in `catalogs/` are the conformance suite of the A2UI +specification, from [a2ui-project/a2ui](https://github.com/a2ui-project/a2ui) at commit `d378d20`, +licensed under the Apache License, Version 2.0. `src/a2ui/__tests__/conformance.test.ts` runs them. +Copied as they are: update them by copying a newer commit's `conformance/` over these. diff --git a/package/__fixtures__/a2ui-conformance/catalogs/agent_functions_catalog_v1_0.json b/package/__fixtures__/a2ui-conformance/catalogs/agent_functions_catalog_v1_0.json new file mode 100644 index 0000000000..48af1e761b --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/catalogs/agent_functions_catalog_v1_0.json @@ -0,0 +1,47 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/conformance/test_data/catalogs/agent_functions_catalog_v1_0.json", + "protocolVersion": "1.0", + "title": "Conformance Agent Functions Catalog (v1.0)", + "description": "A catalog whose function the agent may call, for the cases that compile a callRendererFunction. The functions of the other fixtures declare no allowedCallers, so only the renderer may call them.", + "catalogId": "conformance/agent_functions", + "components": {}, + "functions": { + "notify": { + "type": "object", + "description": "Shows the user a short message.", + "returnType": "void", + "allowedCallers": "rendererOrAgent", + "properties": { + "@call": { + "const": "notify" + }, + "args": { + "type": "object", + "properties": { + "message": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "message" + ], + "unevaluatedProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + }, + "$defs": { + "anyFunction": { + "oneOf": [ + { + "$ref": "#/functions/notify" + } + ] + } + } +} diff --git a/package/__fixtures__/a2ui-conformance/catalogs/allowed_callers_catalog_v1_0.json b/package/__fixtures__/a2ui-conformance/catalogs/allowed_callers_catalog_v1_0.json new file mode 100644 index 0000000000..cabfeef8cc --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/catalogs/allowed_callers_catalog_v1_0.json @@ -0,0 +1,64 @@ +{ + "protocolVersion": "1.0", + "catalogId": "https://a2ui.org/catalogs/v10/allowed_callers", + "components": { + "Spacer": { + "type": "object", + "allowedParents": [ + "Surface", + "Spacer" + ], + "allowedChildren": [ + "Spacer" + ], + "properties": { + "component": { + "const": "Spacer" + } + }, + "required": [ + "component" + ] + } + }, + "functions": { + "readSessionToken": { + "type": "object", + "description": "Returns the renderer's session token to the agent.", + "returnType": "string", + "allowedCallers": "agentOnly", + "properties": { + "@call": { + "const": "readSessionToken" + }, + "args": { + "type": "object", + "properties": {}, + "unevaluatedProperties": false + } + }, + "required": [ + "@call" + ] + } + }, + "$defs": { + "anyComponent": { + "oneOf": [ + { + "$ref": "#/components/Spacer" + } + ], + "discriminator": { + "propertyName": "component" + } + }, + "anyFunction": { + "oneOf": [ + { + "$ref": "#/functions/readSessionToken" + } + ] + } + } +} diff --git a/package/__fixtures__/a2ui-conformance/catalogs/component_metadata_catalog_v1_0.json b/package/__fixtures__/a2ui-conformance/catalogs/component_metadata_catalog_v1_0.json new file mode 100644 index 0000000000..c472e201bb --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/catalogs/component_metadata_catalog_v1_0.json @@ -0,0 +1,215 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/conformance/test_data/catalogs/component_metadata_catalog_v1_0.json", + "protocolVersion": "1.0", + "title": "Conformance Component Metadata Catalog (v1.0)", + "description": "A copy of the simplified catalog whose component definitions carry ComponentDefinition.metadata extension envelopes, for the catalog metadata round-trip cases.", + "catalogId": "conformance/component-metadata", + "components": { + "Text": { + "type": "object", + "description": "Displays a run of text.", + "properties": { + "component": { + "const": "Text" + }, + "text": { + "$ref": "common_types.json#/$defs/DynamicString" + }, + "variant": { + "type": "string", + "enum": [ + "body", + "caption" + ], + "default": "body" + } + }, + "required": [ + "component", + "text" + ], + "metadata": { + "extensions": { + "com_example_telemetry": { + "track": true + }, + "a2ui_experimental": "preview" + } + } + }, + "Button": { + "type": "object", + "description": "A control that emits an action when pressed.", + "properties": { + "component": { + "const": "Button" + }, + "child": { + "$ref": "common_types.json#/$defs/Child" + }, + "action": { + "$ref": "common_types.json#/$defs/Action" + } + }, + "required": [ + "component", + "child", + "action" + ], + "metadata": { + "extensions": { + "com_example_policy": "interactive" + } + } + }, + "Card": { + "type": "object", + "description": "A container holding a single child.", + "properties": { + "component": { + "const": "Card" + }, + "child": { + "$ref": "common_types.json#/$defs/Child" + } + }, + "required": [ + "component", + "child" + ] + }, + "Column": { + "type": "object", + "description": "A layout that stacks its children vertically.", + "properties": { + "component": { + "const": "Column" + }, + "children": { + "$ref": "common_types.json#/$defs/ChildList" + } + }, + "required": [ + "component", + "children" + ] + } + }, + "functions": { + "required": { + "type": "object", + "description": "Checks that the value is not null, undefined or empty.", + "returnType": "validationResult", + "properties": { + "@call": { + "const": "required" + }, + "args": { + "type": "object", + "properties": { + "value": {} + }, + "required": [ + "value" + ], + "unevaluatedProperties": false + } + }, + "required": [ + "@call", + "args" + ] + }, + "formatString": { + "type": "object", + "description": "Interpolates data model values into a template string.", + "returnType": "string", + "properties": { + "@call": { + "const": "formatString" + }, + "args": { + "type": "object", + "properties": { + "value": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "value" + ], + "unevaluatedProperties": false + } + }, + "required": [ + "@call", + "args" + ] + }, + "openUrl": { + "type": "object", + "description": "Opens a URL in the renderer's browser or handler.", + "returnType": "void", + "requiresUserActivation": true, + "properties": { + "@call": { + "const": "openUrl" + }, + "args": { + "type": "object", + "properties": { + "url": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "url" + ], + "unevaluatedProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + }, + "$defs": { + "anyComponent": { + "oneOf": [ + { + "$ref": "#/components/Text" + }, + { + "$ref": "#/components/Button" + }, + { + "$ref": "#/components/Card" + }, + { + "$ref": "#/components/Column" + } + ], + "discriminator": { + "propertyName": "component" + } + }, + "anyFunction": { + "oneOf": [ + { + "$ref": "#/functions/required" + }, + { + "$ref": "#/functions/formatString" + }, + { + "$ref": "#/functions/openUrl" + } + ], + "discriminator": { + "propertyName": "@call" + } + } + } +} \ No newline at end of file diff --git a/package/__fixtures__/a2ui-conformance/catalogs/component_metadata_invalid_key_catalog_v1_0.json b/package/__fixtures__/a2ui-conformance/catalogs/component_metadata_invalid_key_catalog_v1_0.json new file mode 100644 index 0000000000..3b9ce6c06a --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/catalogs/component_metadata_invalid_key_catalog_v1_0.json @@ -0,0 +1,220 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/conformance/test_data/catalogs/component_metadata_invalid_key_catalog_v1_0.json", + "protocolVersion": "1.0", + "title": "Conformance Component Metadata Catalog With Invalid Key (v1.0)", + "description": "Like component_metadata_catalog_v1_0.json, but Card.metadata.extensions has a key that is not a UAX 31 identifier.", + "catalogId": "conformance/component-metadata-invalid", + "components": { + "Text": { + "type": "object", + "description": "Displays a run of text.", + "properties": { + "component": { + "const": "Text" + }, + "text": { + "$ref": "common_types.json#/$defs/DynamicString" + }, + "variant": { + "type": "string", + "enum": [ + "body", + "caption" + ], + "default": "body" + } + }, + "required": [ + "component", + "text" + ], + "metadata": { + "extensions": { + "com_example_telemetry": { + "track": true + }, + "a2ui_experimental": "preview" + } + } + }, + "Button": { + "type": "object", + "description": "A control that emits an action when pressed.", + "properties": { + "component": { + "const": "Button" + }, + "child": { + "$ref": "common_types.json#/$defs/Child" + }, + "action": { + "$ref": "common_types.json#/$defs/Action" + } + }, + "required": [ + "component", + "child", + "action" + ], + "metadata": { + "extensions": { + "com_example_policy": "interactive" + } + } + }, + "Card": { + "type": "object", + "description": "A container holding a single child.", + "properties": { + "component": { + "const": "Card" + }, + "child": { + "$ref": "common_types.json#/$defs/Child" + } + }, + "required": [ + "component", + "child" + ], + "metadata": { + "extensions": { + "com.example.policy": "dotted-key" + } + } + }, + "Column": { + "type": "object", + "description": "A layout that stacks its children vertically.", + "properties": { + "component": { + "const": "Column" + }, + "children": { + "$ref": "common_types.json#/$defs/ChildList" + } + }, + "required": [ + "component", + "children" + ] + } + }, + "functions": { + "required": { + "type": "object", + "description": "Checks that the value is not null, undefined or empty.", + "returnType": "validationResult", + "properties": { + "@call": { + "const": "required" + }, + "args": { + "type": "object", + "properties": { + "value": {} + }, + "required": [ + "value" + ], + "unevaluatedProperties": false + } + }, + "required": [ + "@call", + "args" + ] + }, + "formatString": { + "type": "object", + "description": "Interpolates data model values into a template string.", + "returnType": "string", + "properties": { + "@call": { + "const": "formatString" + }, + "args": { + "type": "object", + "properties": { + "value": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "value" + ], + "unevaluatedProperties": false + } + }, + "required": [ + "@call", + "args" + ] + }, + "openUrl": { + "type": "object", + "description": "Opens a URL in the renderer's browser or handler.", + "returnType": "void", + "requiresUserActivation": true, + "properties": { + "@call": { + "const": "openUrl" + }, + "args": { + "type": "object", + "properties": { + "url": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "url" + ], + "unevaluatedProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + }, + "$defs": { + "anyComponent": { + "oneOf": [ + { + "$ref": "#/components/Text" + }, + { + "$ref": "#/components/Button" + }, + { + "$ref": "#/components/Card" + }, + { + "$ref": "#/components/Column" + } + ], + "discriminator": { + "propertyName": "component" + } + }, + "anyFunction": { + "oneOf": [ + { + "$ref": "#/functions/required" + }, + { + "$ref": "#/functions/formatString" + }, + { + "$ref": "#/functions/openUrl" + } + ], + "discriminator": { + "propertyName": "@call" + } + } + } +} \ No newline at end of file diff --git a/package/__fixtures__/a2ui-conformance/catalogs/composition_catalog_v1_0.json b/package/__fixtures__/a2ui-conformance/catalogs/composition_catalog_v1_0.json new file mode 100644 index 0000000000..e65fcab540 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/catalogs/composition_catalog_v1_0.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/conformance/test_data/catalogs/composition_catalog_v1_0.json", + "protocolVersion": "1.0", + "title": "Conformance Composition Catalog (v1.0)", + "description": "A two component catalog whose components declare allowedParents and allowedChildren, for the timing cases of core/message_processor_scaling.yaml.", + "catalogId": "conformance/composition", + "components": { + "Text": { + "type": "object", + "description": "Displays a run of text.", + "allowedParents": ["Column"], + "properties": { + "component": { "const": "Text" }, + "text": { "$ref": "common_types.json#/$defs/DynamicString" } + }, + "required": ["component", "text"] + }, + "Column": { + "type": "object", + "description": "A layout that stacks its children vertically.", + "allowedParents": ["Surface", "Column"], + "allowedChildren": ["Text", "Column"], + "properties": { + "component": { "const": "Column" }, + "children": { "$ref": "common_types.json#/$defs/ChildList" } + }, + "required": ["component", "children"] + } + }, + "$defs": { + "anyComponent": { + "oneOf": [ + { "$ref": "#/components/Text" }, + { "$ref": "#/components/Column" } + ], + "discriminator": { "propertyName": "component" } + } + } +} diff --git a/package/__fixtures__/a2ui-conformance/catalogs/custom_catalog_v1_0.json b/package/__fixtures__/a2ui-conformance/catalogs/custom_catalog_v1_0.json new file mode 100644 index 0000000000..a86a0ee17b --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/catalogs/custom_catalog_v1_0.json @@ -0,0 +1,54 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/conformance/test_data/catalogs/custom_catalog_v1_0.json", + "protocolVersion": "1.0", + "title": "Conformance Custom Catalog (v1.0)", + "description": "A second catalog used by the multi catalog and capability negotiation cases. Its component and function names do not overlap with the basic catalog, so a case can tell the two apart by name alone.", + "catalogId": "conformance/custom", + "components": { + "Chart": { + "type": "object", + "description": "Plots a series of numbers.", + "properties": { + "component": {"const": "Chart"}, + "values": {"type": "array", "items": {"type": "number"}}, + "caption": {"$ref": "common_types.json#/$defs/DynamicString"} + }, + "required": ["component", "values"] + }, + "Gauge": { + "type": "object", + "description": "Shows a single value on a scale.", + "properties": { + "component": {"const": "Gauge"}, + "value": {"type": "number"} + }, + "required": ["component", "value"] + } + }, + "functions": { + "percent": { + "type": "object", + "description": "Renders a ratio as a percentage string.", + "returnType": "string", + "properties": { + "@call": {"const": "percent"}, + "args": { + "type": "object", + "properties": {"value": {"type": "number"}}, + "required": ["value"], + "unevaluatedProperties": false + } + }, + "required": ["@call", "args"] + } + }, + "$defs": { + "anyComponent": { + "oneOf": [{"$ref": "#/components/Chart"}, {"$ref": "#/components/Gauge"}] + }, + "anyFunction": { + "oneOf": [{"$ref": "#/functions/percent"}] + } + } +} diff --git a/package/__fixtures__/a2ui-conformance/catalogs/derive_source_catalog_v1_0.json b/package/__fixtures__/a2ui-conformance/catalogs/derive_source_catalog_v1_0.json new file mode 100644 index 0000000000..7728c5115e --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/catalogs/derive_source_catalog_v1_0.json @@ -0,0 +1,79 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/catalogs/v10/derive_source.json", + "title": "Derive source", + "description": "The catalog the derive cases start from.", + "catalogId": "https://a2ui.org/catalogs/v10/derive_source", + "protocolVersion": "1.0", + "instructions": "Write components of https://a2ui.org/catalogs/v10/derive_source.", + "components": { + "Text": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + }, + "required": [ + "text" + ] + }, + "Image": { + "type": "object", + "properties": { + "url": { + "type": "string" + } + }, + "required": [ + "url" + ] + }, + "Button": { + "type": "object", + "properties": { + "label": { + "type": "string" + } + } + } + }, + "functions": { + "upper": { + "type": "object", + "properties": { + "call": { + "const": "upper" + }, + "args": { + "type": "object" + }, + "returnType": { + "const": "string" + } + } + }, + "lower": { + "type": "object", + "properties": { + "call": { + "const": "lower" + }, + "args": { + "type": "object" + }, + "returnType": { + "const": "string" + } + } + } + }, + "theme": { + "type": "object", + "properties": { + "primaryColor": { + "type": "string" + } + } + } +} diff --git a/package/__fixtures__/a2ui-conformance/catalogs/forms_catalog_v1_0.json b/package/__fixtures__/a2ui-conformance/catalogs/forms_catalog_v1_0.json new file mode 100644 index 0000000000..c604346cc8 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/catalogs/forms_catalog_v1_0.json @@ -0,0 +1,73 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/conformance/test_data/catalogs/forms_catalog_v1_0.json", + "protocolVersion": "1.0", + "title": "Conformance Forms Catalog (v1.0)", + "description": "An input catalog for the cases the simplified catalog cannot express: a component with enough optional properties to skip one in the middle, and a component that takes validation checks. Common types come from the protocol's common_types.json.", + "catalogId": "conformance/forms", + "components": { + "TextField": { + "type": "object", + "description": "A single line text input.", + "properties": { + "component": { "const": "TextField" }, + "label": { "$ref": "common_types.json#/$defs/DynamicString" }, + "value": { "$ref": "common_types.json#/$defs/DataBinding" }, + "placeholder": { "$ref": "common_types.json#/$defs/DynamicString" }, + "checks": { + "type": "array", + "items": { "$ref": "common_types.json#/$defs/CheckRule" } + } + }, + "required": ["component", "label"] + } + }, + "functions": { + "required": { + "type": "object", + "description": "Checks that the value is not null, undefined or empty.", + "returnType": "validationResult", + "properties": { + "@call": { "const": "required" }, + "args": { + "type": "object", + "properties": { "value": {} }, + "required": ["value"], + "unevaluatedProperties": false + } + }, + "required": ["@call", "args"] + }, + "regex": { + "type": "object", + "description": "Checks the value against a regular expression.", + "returnType": "validationResult", + "properties": { + "@call": { "const": "regex" }, + "args": { + "type": "object", + "properties": { + "value": {}, + "pattern": { "type": "string" } + }, + "required": ["value", "pattern"], + "unevaluatedProperties": false + } + }, + "required": ["@call", "args"] + } + }, + "$defs": { + "anyComponent": { + "oneOf": [{ "$ref": "#/components/TextField" }], + "discriminator": { "propertyName": "component" } + }, + "anyFunction": { + "oneOf": [ + { "$ref": "#/functions/required" }, + { "$ref": "#/functions/regex" } + ], + "discriminator": { "propertyName": "@call" } + } + } +} diff --git a/package/__fixtures__/a2ui-conformance/catalogs/function_metadata_catalog_v1_0.json b/package/__fixtures__/a2ui-conformance/catalogs/function_metadata_catalog_v1_0.json new file mode 100644 index 0000000000..d9210ad835 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/catalogs/function_metadata_catalog_v1_0.json @@ -0,0 +1,73 @@ +{ + "protocolVersion": "1.0", + "catalogId": "https://a2ui.org/catalogs/v10/function_metadata", + "components": { + "Spacer": { + "type": "object", + "allowedParents": [ + "Surface", + "Spacer" + ], + "allowedChildren": [ + "Spacer" + ], + "properties": { + "component": { + "const": "Spacer" + } + }, + "required": [ + "component" + ] + } + }, + "functions": { + "launch": { + "type": "object", + "description": "Launches a URL.", + "returnType": "void", + "allowedCallers": "rendererOnly", + "requiresUserActivation": true, + "properties": { + "@call": { + "const": "launch" + }, + "args": { + "type": "object", + "properties": { + "url": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "url" + ], + "unevaluatedProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + }, + "$defs": { + "anyComponent": { + "oneOf": [ + { + "$ref": "#/components/Spacer" + } + ], + "discriminator": { + "propertyName": "component" + } + }, + "anyFunction": { + "oneOf": [ + { + "$ref": "#/functions/launch" + } + ] + } + } +} diff --git a/package/__fixtures__/a2ui-conformance/catalogs/list_item_action_catalog_v1_0.json b/package/__fixtures__/a2ui-conformance/catalogs/list_item_action_catalog_v1_0.json new file mode 100644 index 0000000000..9db00f3100 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/catalogs/list_item_action_catalog_v1_0.json @@ -0,0 +1,77 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/conformance/test_data/catalogs/list_item_action_catalog_v1_0.json", + "protocolVersion": "1.0", + "title": "Conformance List Item Action Catalog (v1.0)", + "description": "A catalog whose component carries actions inside list items, for testing that an Express function call in a nested action slot compiles to a functionCall action.", + "catalogId": "conformance/list_item_action", + "components": { + "Text": { + "type": "object", + "description": "Displays a run of text.", + "properties": { + "component": { "const": "Text" }, + "text": { "$ref": "common_types.json#/$defs/DynamicString" } + }, + "required": ["component", "text"] + }, + "Links": { + "type": "object", + "description": "A list of labelled links, each firing an action when chosen.", + "properties": { + "component": { "const": "Links" }, + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { "type": "string" }, + "action": { "$ref": "common_types.json#/$defs/Action" } + }, + "required": ["label", "action"] + } + }, + "primary": { "$ref": "common_types.json#/$defs/Action" } + }, + "required": ["component", "items"] + }, + "Tabs": { + "type": "object", + "description": "A tabbed view whose tabItems carry a title and a Child componentId.", + "properties": { + "component": { "const": "Tabs" }, + "tabItems": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { "$ref": "common_types.json#/$defs/DynamicString" }, + "child": { "$ref": "common_types.json#/$defs/Child" } + }, + "required": ["title", "child"] + } + } + }, + "required": ["component", "tabItems"] + } + }, + "functions": { + "openUrl": { + "type": "object", + "description": "Opens a URL.", + "returnType": "void", + "properties": { + "@call": { "const": "openUrl" }, + "args": { + "type": "object", + "properties": { + "url": { "type": "string" } + }, + "required": ["url"], + "unevaluatedProperties": false + } + }, + "required": ["@call", "args"] + } + } +} diff --git a/package/__fixtures__/a2ui-conformance/catalogs/literal_map_catalog_v1_0.json b/package/__fixtures__/a2ui-conformance/catalogs/literal_map_catalog_v1_0.json new file mode 100644 index 0000000000..dfc2c4ab16 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/catalogs/literal_map_catalog_v1_0.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/conformance/test_data/catalogs/literal_map_catalog_v1_0.json", + "protocolVersion": "1.0", + "title": "Conformance Literal Map Catalog (v1.0)", + "description": "A catalog for testing that Express map literals keep their keys, including keys that share a name with an Express construct such as variable, path, call or check.", + "catalogId": "conformance/literal_map", + "components": { + "Probe": { + "type": "object", + "description": "Carries an arbitrary object payload.", + "properties": { + "component": { "const": "Probe" }, + "payload": { "type": "object" } + }, + "required": ["component", "payload"] + } + }, + "functions": { + "series": { + "type": "object", + "description": "Repeats an item a number of times, naming the loop variable.", + "returnType": "array", + "properties": { + "@call": { "const": "series" }, + "args": { + "type": "object", + "properties": { + "count": { "type": "integer" }, + "variable": { "type": "string" }, + "item": { "type": "object" } + }, + "required": ["count", "item"], + "unevaluatedProperties": false + } + }, + "required": ["@call", "args"] + } + } +} diff --git a/package/__fixtures__/a2ui-conformance/catalogs/simplified_catalog_v1_0.json b/package/__fixtures__/a2ui-conformance/catalogs/simplified_catalog_v1_0.json new file mode 100644 index 0000000000..bd577e6741 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/catalogs/simplified_catalog_v1_0.json @@ -0,0 +1,123 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/conformance/test_data/catalogs/simplified_catalog_v1_0.json", + "protocolVersion": "1.0", + "title": "Conformance Simplified Catalog (v1.0)", + "description": "A four component catalog for the agent SDK conformance suites. Like a published catalog it takes its common types from the protocol's common_types.json and defines only anyComponent and anyFunction itself.", + "catalogId": "conformance/simplified", + "components": { + "Text": { + "type": "object", + "description": "Displays a run of text.", + "properties": { + "component": { "const": "Text" }, + "text": { "$ref": "common_types.json#/$defs/DynamicString" }, + "variant": { + "type": "string", + "enum": ["body", "caption"], + "default": "body" + } + }, + "required": ["component", "text"] + }, + "Button": { + "type": "object", + "description": "A control that emits an action when pressed.", + "properties": { + "component": { "const": "Button" }, + "child": { "$ref": "common_types.json#/$defs/Child" }, + "action": { "$ref": "common_types.json#/$defs/Action" } + }, + "required": ["component", "child", "action"] + }, + "Card": { + "type": "object", + "description": "A container holding a single child.", + "properties": { + "component": { "const": "Card" }, + "child": { "$ref": "common_types.json#/$defs/Child" } + }, + "required": ["component", "child"] + }, + "Column": { + "type": "object", + "description": "A layout that stacks its children vertically.", + "properties": { + "component": { "const": "Column" }, + "children": { "$ref": "common_types.json#/$defs/ChildList" } + }, + "required": ["component", "children"] + } + }, + "functions": { + "required": { + "type": "object", + "description": "Checks that the value is not null, undefined or empty.", + "returnType": "validationResult", + "properties": { + "@call": { "const": "required" }, + "args": { + "type": "object", + "properties": { "value": {} }, + "required": ["value"], + "unevaluatedProperties": false + } + }, + "required": ["@call", "args"] + }, + "formatString": { + "type": "object", + "description": "Interpolates data model values into a template string.", + "returnType": "string", + "properties": { + "@call": { "const": "formatString" }, + "args": { + "type": "object", + "properties": { + "value": { "$ref": "common_types.json#/$defs/DynamicString" } + }, + "required": ["value"], + "unevaluatedProperties": false + } + }, + "required": ["@call", "args"] + }, + "openUrl": { + "type": "object", + "description": "Opens a URL in the renderer's browser or handler.", + "returnType": "void", + "requiresUserActivation": true, + "properties": { + "@call": { "const": "openUrl" }, + "args": { + "type": "object", + "properties": { + "url": { "$ref": "common_types.json#/$defs/DynamicString" } + }, + "required": ["url"], + "unevaluatedProperties": false + } + }, + "required": ["@call", "args"] + } + }, + "$defs": { + "anyComponent": { + "oneOf": [ + { "$ref": "#/components/Text" }, + { "$ref": "#/components/Button" }, + { "$ref": "#/components/Card" }, + { "$ref": "#/components/Column" } + ], + "discriminator": { "propertyName": "component" } + }, + "anyFunction": { + "oneOf": [ + { "$ref": "#/functions/required" }, + { "$ref": "#/functions/formatString" }, + { "$ref": "#/functions/openUrl" } + ], + "discriminator": { "propertyName": "@call" } + } + } +} diff --git a/package/__fixtures__/a2ui-conformance/catalogs/weighted_catalog_v1_0.json b/package/__fixtures__/a2ui-conformance/catalogs/weighted_catalog_v1_0.json new file mode 100644 index 0000000000..af7c5974b1 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/catalogs/weighted_catalog_v1_0.json @@ -0,0 +1,145 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/conformance/test_data/catalogs/weighted_catalog_v1_0.json", + "protocolVersion": "1.0", + "title": "Conformance Weighted Catalog (v1.0)", + "description": "A catalog for testing positional numeric argument placement when optional enum properties precede a numeric property such as weight.", + "catalogId": "conformance/weighted", + "components": { + "Text": { + "type": "object", + "description": "Displays a run of text.", + "properties": { + "component": { "const": "Text" }, + "text": { "$ref": "common_types.json#/$defs/DynamicString" }, + "variant": { + "type": "string", + "enum": ["body", "caption"], + "default": "body" + }, + "weight": { "type": "number" } + }, + "required": ["component", "text"] + }, + "Column": { + "type": "object", + "description": "A layout that stacks its children vertically.", + "properties": { + "component": { "const": "Column" }, + "children": { "$ref": "common_types.json#/$defs/ChildList" }, + "justify": { + "type": "string", + "enum": ["start", "center", "end"], + "default": "start" + }, + "align": { + "type": "string", + "enum": ["start", "center", "end"], + "default": "start" + }, + "weight": { "type": "number" } + }, + "required": ["component", "children"] + }, + "Image": { + "type": "object", + "description": "Displays an image.", + "properties": { + "component": { "const": "Image" }, + "url": { "$ref": "common_types.json#/$defs/DynamicString" }, + "description": { "$ref": "common_types.json#/$defs/DynamicString" }, + "weight": { "type": "number" } + }, + "required": ["component", "url"] + }, + "TextField": { + "type": "object", + "description": "A single-line text input.", + "properties": { + "component": { "const": "TextField" }, + "label": { "$ref": "common_types.json#/$defs/DynamicString" }, + "value": { "$ref": "common_types.json#/$defs/DataBinding" }, + "placeholder": { "$ref": "common_types.json#/$defs/DynamicString" }, + "weight": { "type": "number" } + }, + "required": ["component", "label"] + }, + "Slider": { + "type": "object", + "description": "A slider input whose value is a DynamicNumber ref.", + "properties": { + "component": { "const": "Slider" }, + "orientation": { + "type": "string", + "enum": ["horizontal", "vertical"], + "default": "horizontal" + }, + "value": { "$ref": "common_types.json#/$defs/DynamicNumber" } + }, + "required": ["component"] + }, + "DualMetric": { + "type": "object", + "description": "A component with two later numeric properties.", + "properties": { + "component": { "const": "DualMetric" }, + "variant": { + "type": "string", + "enum": ["a", "b"], + "default": "a" + }, + "min": { "type": "number" }, + "max": { "type": "number" } + }, + "required": ["component"] + }, + "Status": { + "type": "object", + "description": "A status component with a required enum property.", + "properties": { + "component": { "const": "Status" }, + "state": { + "type": "string", + "enum": ["active", "inactive"] + }, + "priority": { "type": "number" } + }, + "required": ["component", "state"] + }, + "Meter": { + "type": "object", + "description": "A meter component with title, optional unit enum, numeric level, and optional description.", + "properties": { + "component": { "const": "Meter" }, + "label": { "$ref": "common_types.json#/$defs/DynamicString" }, + "unit": { + "type": "string", + "enum": ["percent", "ratio"], + "default": "percent" + }, + "level": { "type": "number" }, + "description": { "type": "string" } + }, + "required": ["component", "label"] + } + }, + "functions": {}, + "$defs": { + "anyComponent": { + "oneOf": [ + { "$ref": "#/components/Text" }, + { "$ref": "#/components/Column" }, + { "$ref": "#/components/Image" }, + { "$ref": "#/components/TextField" }, + { "$ref": "#/components/Slider" }, + { "$ref": "#/components/DualMetric" }, + { "$ref": "#/components/Status" }, + { "$ref": "#/components/Meter" } + ], + "discriminator": { "propertyName": "component" } + }, + "anyFunction": { + "not": {} + } + } +} diff --git a/package/__fixtures__/a2ui-conformance/core/accessibility.json b/package/__fixtures__/a2ui-conformance/core/accessibility.json new file mode 100644 index 0000000000..54ccd426f4 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/core/accessibility.json @@ -0,0 +1,722 @@ +[ + { + "name": "test_accessibility_attributes_plumbing", + "description": "Verifies plumbing of AccessibilityAttributes (label, description, live, hidden) to accessibility node trees and axe-core rules.", + "protocolVersion": "1.0", + "catalogPath": "catalogs/basic/v1/catalog.json", + "action": "accessibility_check", + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "a11y_surface", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "btn1", + "status_text" + ] + }, + { + "id": "btn1", + "component": "Button", + "child": "btn1_label", + "action": { + "event": { + "name": "tap" + } + }, + "accessibility": { + "label": "Mute Notifications", + "description": "Silences notifications about this conversation" + } + }, + { + "id": "btn1_label", + "component": "Text", + "text": "Mute" + }, + { + "id": "status_text", + "component": "Text", + "text": "Status: Active", + "accessibility": { + "live": "polite" + } + } + ] + } + } + ], + "surface": { + "id": "root", + "component": "Column", + "children": [ + "btn1", + "status_text" + ], + "components": { + "btn1": { + "component": "Button", + "child": "btn1_label", + "title": "Mute", + "action": { + "event": { + "name": "tap" + } + }, + "accessibility": { + "label": "Mute Notifications", + "description": "Silences notifications about this conversation" + } + }, + "btn1_label": { + "component": "Text", + "text": "Mute" + }, + "status_text": { + "component": "Text", + "text": "Status: Active", + "accessibility": { + "live": "polite" + } + } + } + }, + "assertions": { + "axeCore": [ + "button-name", + "aria-valid-attr-value", + "aria-allowed-attr" + ], + "accessibilityTree": { + "btn1": { + "label": "Mute Notifications", + "description": "Silences notifications about this conversation" + }, + "status_text": { + "live": "polite" + } + } + } + }, + { + "name": "test_accessibility_live_region_assertive", + "description": "Verifies assertive live region announcement for critical alert banner updates.", + "protocolVersion": "1.0", + "catalogPath": "catalogs/basic/v1/catalog.json", + "action": "accessibility_check", + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "a11y_surface", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "components": [ + { + "id": "root", + "component": "Card", + "child": "alert_box" + }, + { + "id": "alert_box", + "component": "Text", + "text": "Connection Lost!", + "accessibility": { + "label": "Critical Alert", + "live": "assertive" + } + } + ] + } + } + ], + "surface": { + "id": "root", + "component": "Card", + "child": "alert_box", + "components": { + "alert_box": { + "component": "Text", + "text": "Connection Lost!", + "accessibility": { + "label": "Critical Alert", + "live": "assertive" + } + } + } + }, + "assertions": { + "axeCore": [ + "aria-valid-attr-value" + ], + "accessibilityTree": { + "alert_box": { + "label": "Critical Alert", + "live": "assertive" + } + } + } + }, + { + "name": "test_accessibility_hidden_subtree", + "description": "Verifies that setting hidden=true hides the component and its children from assistive technologies.", + "protocolVersion": "1.0", + "catalogPath": "catalogs/basic/v1/catalog.json", + "action": "accessibility_check", + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "a11y_surface", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "components": [ + { + "id": "root", + "component": "Column", + "accessibility": { + "hidden": true + }, + "children": [ + "visible_text", + "decorative_icon" + ] + }, + { + "id": "visible_text", + "component": "Text", + "text": "Visible Content" + }, + { + "id": "decorative_icon", + "component": "Icon", + "name": "star" + } + ] + } + } + ], + "surface": { + "id": "root", + "component": "Column", + "accessibility": { + "hidden": true + }, + "children": [ + "visible_text", + "decorative_icon" + ], + "components": { + "visible_text": { + "component": "Text", + "text": "Visible Content" + }, + "decorative_icon": { + "component": "Icon", + "name": "star" + } + } + }, + "assertions": { + "axeCore": [ + "aria-hidden-focus" + ], + "accessibilityTree": { + "root": { + "hidden": true + } + } + } + }, + { + "name": "test_accessibility_implicit_label_inference", + "description": "Verifies default screen reader semantic inference from visible text properties when explicit accessibility attributes are omitted.", + "protocolVersion": "1.0", + "catalogPath": "catalogs/basic/v1/catalog.json", + "action": "accessibility_check", + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "a11y_surface", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "submit_btn" + ] + }, + { + "id": "submit_btn", + "component": "Button", + "child": "submit_btn_label", + "action": { + "event": { + "name": "tap" + } + } + }, + { + "id": "submit_btn_label", + "component": "Text", + "text": "Submit Form" + } + ] + } + } + ], + "surface": { + "id": "root", + "component": "Column", + "children": [ + "submit_btn" + ], + "components": { + "submit_btn": { + "component": "Button", + "child": "submit_btn_label", + "title": "Submit Form", + "action": { + "event": { + "name": "tap" + } + } + }, + "submit_btn_label": { + "component": "Text", + "text": "Submit Form" + } + } + }, + "assertions": { + "axeCore": [ + "button-name" + ], + "accessibilityTree": { + "submit_btn": { + "label": "Submit Form" + } + } + } + }, + { + "name": "test_accessibility_interactive_roles_and_states", + "description": "Verifies accessibility role mapping and ARIA states for Button, TextField, CheckBox, ChoicePicker.", + "protocolVersion": "1.0", + "catalogPath": "catalogs/basic/v1/catalog.json", + "action": "accessibility_check", + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "a11y_surface", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "btn", + "text_input", + "check_box", + "choice_picker" + ] + }, + { + "id": "btn", + "component": "Button", + "child": "btn_label", + "action": { + "event": { + "name": "tap" + } + }, + "accessibility": { + "label": "Submit Feedback" + } + }, + { + "id": "btn_label", + "component": "Text", + "text": "Submit" + }, + { + "id": "text_input", + "component": "TextField", + "label": "Your Email", + "value": "", + "accessibility": { + "description": "Enter a valid email address" + } + }, + { + "id": "check_box", + "component": "CheckBox", + "label": "I agree to terms", + "value": true, + "accessibility": { + "label": "Accept Terms and Conditions" + } + }, + { + "id": "choice_picker", + "component": "ChoicePicker", + "variant": "mutuallyExclusive", + "value": [ + "1" + ], + "options": [ + { + "label": "Option 1", + "value": "1" + }, + { + "label": "Option 2", + "value": "2" + } + ], + "accessibility": { + "label": "Select Preference" + } + } + ] + } + } + ], + "surface": { + "id": "root", + "component": "Column", + "children": [ + "btn", + "text_input", + "check_box", + "choice_picker" + ], + "components": { + "btn": { + "component": "Button", + "child": "btn_label", + "title": "Submit", + "action": { + "event": { + "name": "tap" + } + }, + "accessibility": { + "label": "Submit Feedback" + } + }, + "btn_label": { + "component": "Text", + "text": "Submit" + }, + "text_input": { + "component": "TextField", + "label": "Your Email", + "value": "", + "accessibility": { + "description": "Enter a valid email address" + } + }, + "check_box": { + "component": "CheckBox", + "label": "I agree to terms", + "value": true, + "checked": true, + "accessibility": { + "label": "Accept Terms and Conditions" + } + }, + "choice_picker": { + "component": "ChoicePicker", + "variant": "mutuallyExclusive", + "value": [ + "1" + ], + "options": [ + { + "label": "Option 1", + "value": "1" + }, + { + "label": "Option 2", + "value": "2" + } + ], + "selectedIndex": 0, + "accessibility": { + "label": "Select Preference" + } + } + } + }, + "assertions": { + "axeCore": [ + "button-name", + "aria-valid-attr-value", + "aria-allowed-attr" + ], + "accessibilityTree": { + "btn": { + "label": "Submit Feedback", + "role": "button" + }, + "text_input": { + "description": "Enter a valid email address", + "role": "textbox" + }, + "check_box": { + "label": "Accept Terms and Conditions", + "role": "checkbox", + "checked": true + }, + "choice_picker": { + "label": "Select Preference", + "role": "radiogroup" + } + } + } + }, + { + "name": "test_accessibility_dynamic_label_and_hidden_binding", + "description": "Verifies dynamic data bindings in accessibility attributes (label, hidden) reactively update accessibility trees.", + "protocolVersion": "1.0", + "catalogPath": "catalogs/basic/v1/catalog.json", + "action": "accessibility_check", + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "a11y_surface", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "dataModel": { + "buttonLabel": "Mute Audio", + "isButtonHidden": false, + "isStatusHidden": true + }, + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "dynamic_btn", + "dynamic_status" + ] + }, + { + "id": "dynamic_btn", + "component": "Button", + "child": "dynamic_btn_label", + "action": { + "event": { + "name": "tap" + } + }, + "accessibility": { + "label": { + "@path": "/buttonLabel" + }, + "hidden": { + "@path": "/isButtonHidden" + } + } + }, + { + "id": "dynamic_btn_label", + "component": "Text", + "text": "Dynamic Action" + }, + { + "id": "dynamic_status", + "component": "Text", + "text": "Status", + "accessibility": { + "hidden": { + "@path": "/isStatusHidden" + } + } + } + ] + } + } + ], + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "a11y_surface", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "dataModel": { + "buttonLabel": "Mute Audio", + "isButtonHidden": false, + "isStatusHidden": true + }, + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "dynamic_btn", + "dynamic_status" + ] + }, + { + "id": "dynamic_btn", + "component": "Button", + "child": "dynamic_btn_label", + "action": { + "event": { + "name": "tap" + } + }, + "accessibility": { + "label": { + "@path": "/buttonLabel" + }, + "hidden": { + "@path": "/isButtonHidden" + } + } + }, + { + "id": "dynamic_btn_label", + "component": "Text", + "text": "Dynamic Action" + }, + { + "id": "dynamic_status", + "component": "Text", + "text": "Status", + "accessibility": { + "hidden": { + "@path": "/isStatusHidden" + } + } + } + ] + } + } + ], + "assertions": { + "axeCore": [ + "button-name", + "aria-hidden-focus" + ], + "accessibilityTree": { + "dynamic_btn": { + "label": "Mute Audio", + "hidden": false + }, + "dynamic_status": { + "hidden": true + } + } + } + }, + { + "messages": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "a11y_surface", + "path": "/buttonLabel", + "value": "Unmute Audio" + } + }, + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "a11y_surface", + "path": "/isButtonHidden", + "value": true + } + } + ], + "assertions": { + "axeCore": [ + "button-name", + "aria-hidden-focus" + ], + "accessibilityTree": { + "dynamic_btn": { + "label": "Unmute Audio", + "hidden": true + }, + "dynamic_status": { + "hidden": true + } + } + } + } + ], + "surface": { + "id": "root", + "component": "Column", + "children": [ + "dynamic_btn", + "dynamic_status" + ], + "components": { + "dynamic_btn": { + "component": "Button", + "child": "dynamic_btn_label", + "title": "Dynamic Action", + "action": { + "event": { + "name": "tap" + } + }, + "accessibility": { + "label": { + "@path": "/buttonLabel" + }, + "hidden": { + "@path": "/isButtonHidden" + } + } + }, + "dynamic_btn_label": { + "component": "Text", + "text": "Dynamic Action" + }, + "dynamic_status": { + "component": "Text", + "text": "Status", + "accessibility": { + "hidden": { + "@path": "/isStatusHidden" + } + } + } + } + }, + "assertions": { + "axeCore": [ + "button-name", + "aria-hidden-focus" + ], + "accessibilityTree": { + "dynamic_btn": { + "label": { + "path": "/buttonLabel" + }, + "hidden": { + "path": "/isButtonHidden" + } + }, + "dynamic_status": { + "hidden": { + "path": "/isStatusHidden" + } + } + } + } + } +] \ No newline at end of file diff --git a/package/__fixtures__/a2ui-conformance/core/actions.json b/package/__fixtures__/a2ui-conformance/core/actions.json new file mode 100644 index 0000000000..2570480951 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/core/actions.json @@ -0,0 +1,458 @@ +[ + { + "name": "test_action_event_basic", + "description": "Verifies dispatching a basic server event with name only.", + "action": "dispatch_action", + "actionPayload": { + "event": { + "name": "submit" + } + }, + "expectDispatched": { + "name": "submit" + } + }, + { + "name": "test_action_event_with_literal_context", + "description": "Verifies event action preserves literal context properties.", + "action": "dispatch_action", + "actionPayload": { + "event": { + "name": "filter_change", + "context": { + "query": "books", + "limit": 10, + "active": true + } + } + }, + "expectDispatched": { + "name": "filter_change", + "context": { + "query": "books", + "limit": 10, + "active": true + } + } + }, + { + "name": "test_action_event_context_literal_keys_path_and_call", + "description": "Verifies context keys literally named 'path' and 'call' are preserved as literal keys (D1).", + "action": "dispatch_action", + "actionPayload": { + "event": { + "name": "save", + "context": { + "path": "/custom/file.txt", + "call": "external", + "regular": "data" + } + } + }, + "expectDispatched": { + "name": "save", + "context": { + "path": "/custom/file.txt", + "call": "external", + "regular": "data" + } + } + }, + { + "name": "test_action_event_with_data_binding", + "description": "Verifies event action dynamically resolves bound context values from DataModel.", + "action": "dispatch_action", + "dataModel": { + "user": { + "id": "u_123", + "role": "admin" + } + }, + "actionPayload": { + "event": { + "name": "user_action", + "context": { + "userId": { + "path": "/user/id" + }, + "role": { + "path": "/user/role" + }, + "staticKey": "constant" + } + } + }, + "expectDispatched": { + "name": "user_action", + "context": { + "userId": "u_123", + "role": "admin", + "staticKey": "constant" + } + } + }, + { + "name": "test_action_event_with_relative_scope", + "description": "Verifies event action resolves relative data bindings against the provided scope.", + "action": "dispatch_action", + "scope": "/items/1", + "dataModel": { + "items": [ + { + "title": "Item 0", + "id": "id_0" + }, + { + "title": "Item 1", + "id": "id_1" + } + ] + }, + "actionPayload": { + "event": { + "name": "select_item", + "context": { + "selectedId": { + "path": "id" + }, + "selectedTitle": { + "path": "title" + } + } + } + }, + "expectDispatched": { + "name": "select_item", + "context": { + "selectedId": "id_1", + "selectedTitle": "Item 1" + } + } + }, + { + "name": "test_action_event_with_user_message", + "description": "Verifies event action preserves and dispatches userMessage.", + "action": "dispatch_action", + "actionPayload": { + "event": { + "name": "feedback", + "userMessage": "User requested additional details", + "context": { + "score": 5 + } + } + }, + "expectDispatched": { + "name": "feedback", + "userMessage": "User requested additional details", + "context": { + "score": 5 + } + } + }, + { + "name": "test_action_direct_name_shorthand", + "description": "Verifies v0.9 direct action name shorthand is dispatched as an event.", + "action": "dispatch_action", + "actionPayload": { + "name": "click", + "context": { + "target": "btn1" + } + }, + "expectDispatched": { + "name": "click", + "context": { + "target": "btn1" + } + } + }, + { + "name": "test_action_event_with_dispatch_metadata", + "description": "Verifies that metadata passed to dispatchAction is attached to the dispatched action as action.metadata (v1.0 extension envelope).", + "action": "dispatch_action", + "protocolVersion": "1.0", + "actionPayload": { + "event": { + "name": "submit", + "context": { + "form": "signup" + } + } + }, + "metadata": { + "extensions": { + "com_example_attestation": "signed", + "a2ui_experimental": { + "trace": 7 + } + } + }, + "expectDispatched": { + "name": "submit", + "context": { + "form": "signup" + }, + "metadata": { + "extensions": { + "com_example_attestation": "signed", + "a2ui_experimental": { + "trace": 7 + } + } + } + } + }, + { + "name": "test_action_event_without_dispatch_metadata_has_none", + "description": "Verifies that an action dispatched without metadata carries no action.metadata envelope.", + "action": "dispatch_action", + "protocolVersion": "1.0", + "actionPayload": { + "event": { + "name": "submit" + } + }, + "expectDispatched": { + "name": "submit", + "metadata": null + } + }, + { + "name": "test_action_event_dispatch_metadata_invalid_uax31_key_error", + "description": "Verifies that dispatchAction rejects metadata whose extensions key is not a UAX 31 identifier and dispatches nothing.", + "action": "dispatch_action", + "protocolVersion": "1.0", + "actionPayload": { + "event": { + "name": "submit" + } + }, + "metadata": { + "extensions": { + "com-example-attestation": "signed" + } + }, + "expectError": { + "category": "ValidationError" + } + }, + { + "name": "test_action_v10_event_with_at_path_and_at_call_context", + "description": "Verifies v1.0 @path and @call context bindings, @@ literal prefix unescaping, and @index inside template scope.", + "action": "dispatch_action", + "protocolVersion": "1.0", + "surfaceId": "main_surface", + "sourceComponentId": "row_btn", + "scope": "/items/1", + "dataModel": { + "items": [ + { + "title": "First" + }, + { + "title": "Second" + } + ] + }, + "actionPayload": { + "event": { + "name": "item_selected", + "context": { + "itemTitle": { + "@path": "title" + }, + "upperTitle": { + "@call": "formatString", + "args": { + "value": "Selected: ${title}" + } + }, + "itemIndex": { + "@call": "@index" + }, + "oneBasedIndex": { + "@call": "@index", + "args": { + "offset": 1 + } + }, + "escapedKey": { + "@@path": "/literal/not/bound" + } + } + } + }, + "expectDispatched": { + "name": "item_selected", + "surfaceId": "main_surface", + "sourceComponentId": "row_btn", + "timestampIso8601": true, + "context": { + "itemTitle": "Second", + "upperTitle": "Selected: Second", + "itemIndex": 1, + "oneBasedIndex": 2, + "escapedKey": { + "@path": "/literal/not/bound" + } + } + } + }, + { + "name": "test_action_v10_event_context_literal_path_and_call_keys", + "description": "Verifies v1.0 literal 'path' and 'call' keys inside event.context are preserved as literal keys through GenericBinder / NodeResolver without being misread as v0.9 bindings.", + "action": "dispatch_action", + "protocolVersion": "1.0", + "surfaceId": "main_surface", + "sourceComponentId": "save_btn", + "dataModel": { + "user": "Ada" + }, + "actionPayload": { + "event": { + "name": "save_document", + "context": { + "path": "/documents/report.md", + "call": "sync", + "author": { + "@path": "/user" + } + } + } + }, + "expectDispatched": { + "name": "save_document", + "surfaceId": "main_surface", + "sourceComponentId": "save_btn", + "timestampIso8601": true, + "context": { + "path": "/documents/report.md", + "call": "sync", + "author": "Ada" + } + } + }, + { + "name": "test_action_function_call_local_mutation_no_event_dispatched", + "description": "Verifies a local functionCall action executes locally, mutates dataModel, and dispatches no outbound event (expectDispatched is null).", + "action": "dispatch_action", + "protocolVersion": "1.0", + "surfaceId": "main_surface", + "sourceComponentId": "inc_btn", + "dataModel": { + "count": 1 + }, + "actionPayload": { + "functionCall": { + "@call": "incrementCounter", + "args": { + "path": "/count", + "step": 2 + } + } + }, + "expectDataModel": { + "count": 3 + }, + "expectDispatched": null + }, + { + "name": "test_action_function_call_open_url_with_user_activation_allowed", + "description": "Verifies openUrl functionCall action succeeds when userActivation is true and dispatches no outbound event.", + "action": "dispatch_action", + "protocolVersion": "1.0", + "surfaceId": "main_surface", + "sourceComponentId": "link_btn", + "userActivation": true, + "actionPayload": { + "functionCall": { + "@call": "openUrl", + "args": { + "url": "https://example.com" + } + } + }, + "expectDispatched": null + }, + { + "name": "test_action_function_call_open_url_without_user_activation_denied", + "description": "Verifies openUrl functionCall action fails when userActivation is false and emits EXECUTION_ERROR / INVALID_ACTION on surface.onError.", + "action": "dispatch_action", + "protocolVersion": "1.0", + "surfaceId": "main_surface", + "sourceComponentId": "link_btn", + "userActivation": false, + "actionPayload": { + "functionCall": { + "@call": "openUrl", + "args": { + "url": "https://example.com" + } + } + }, + "expectDispatched": null, + "expectError": { + "surfaceId": "main_surface", + "message": "openUrl" + } + }, + { + "name": "test_action_event_missing_name_error", + "description": "Verifies dispatch_action with a missing event name emits INVALID_ACTION on surface.onError.", + "action": "dispatch_action", + "protocolVersion": "1.0", + "surfaceId": "main_surface", + "sourceComponentId": "bad_btn", + "actionPayload": { + "event": { + "context": { + "foo": "bar" + } + } + }, + "expectDispatched": null, + "expectError": { + "code": "INVALID_ACTION", + "surfaceId": "main_surface" + } + }, + { + "name": "test_action_event_non_string_user_message_error", + "description": "Verifies dispatch_action with a non-string userMessage emits INVALID_ACTION on surface.onError.", + "action": "dispatch_action", + "protocolVersion": "1.0", + "surfaceId": "main_surface", + "sourceComponentId": "bad_msg_btn", + "actionPayload": { + "event": { + "name": "submit", + "userMessage": 12345 + } + }, + "expectDispatched": null, + "expectError": { + "code": "INVALID_ACTION", + "surfaceId": "main_surface" + } + }, + { + "name": "test_action_function_call_execution_error", + "description": "Verifies a failing local functionCall action emits EXECUTION_ERROR on surface.onError and dispatches no outbound event.", + "action": "dispatch_action", + "protocolVersion": "1.0", + "surfaceId": "main_surface", + "sourceComponentId": "fail_btn", + "actionPayload": { + "functionCall": { + "@call": "failingAction", + "args": {} + } + }, + "expectDispatched": null, + "expectError": { + "code": "EXECUTION_ERROR", + "surfaceId": "main_surface" + } + } +] \ No newline at end of file diff --git a/package/__fixtures__/a2ui-conformance/core/composition_constraints.json b/package/__fixtures__/a2ui-conformance/core/composition_constraints.json new file mode 100644 index 0000000000..201b787401 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/core/composition_constraints.json @@ -0,0 +1,734 @@ +[ + { + "name": "test_composition_surface_implicit_parent_container", + "description": "Verifies root components treating Surface as canonical parent container when allowedParents includes Surface.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "process_messages", + "validationConfig": {}, + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "t1" + ] + }, + { + "id": "t1", + "component": "Text", + "text": "Hello World" + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "s1": { + "components": { + "root": { + "component": "Column", + "children": [ + "t1" + ] + }, + "t1": { + "component": "Text", + "text": "Hello World" + } + } + } + } + } + }, + { + "name": "test_composition_unallowed_parent_error", + "description": "Verifies validation error thrown when component is placed under an unallowed parent component type.", + "catalog": { + "catalogId": "custom", + "protocolVersion": "1.0", + "components": { + "ColumnHeader": { + "type": "object", + "allowedParents": [ + "Column" + ] + }, + "Row": { + "type": "object", + "properties": { + "children": { + "$ref": "#/$defs/ChildList" + } + } + } + } + }, + "action": "process_messages", + "validationConfig": {}, + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom", + "components": [ + { + "id": "root", + "component": "Row", + "children": [ + "invalid_header" + ] + }, + { + "id": "invalid_header", + "component": "ColumnHeader" + } + ] + } + } + ] + } + ], + "expectError": { + "category": "ValidationError", + "code": "UNALLOWED_PARENT", + "surfaceId": "s1", + "path": "/components/1", + "message": "Component 'invalid_header' (ColumnHeader) cannot be placed under parent 'root' (Row). Allowed parents: ['Column'].", + "details": [ + { + "path": "/components/1", + "code": "UNALLOWED_PARENT", + "surfaceId": "s1" + } + ], + "rendererError": { + "version": "1.0", + "error": { + "code": "UNALLOWED_PARENT", + "surfaceId": "s1", + "path": "/components/1", + "message": "Component 'invalid_header' (ColumnHeader) cannot be placed under parent 'root' (Row). Allowed parents: ['Column']." + } + } + } + }, + { + "name": "test_composition_unallowed_parent_root_error", + "description": "Verifies validation error thrown when root component does not allow Surface as its implicit parent.", + "catalog": { + "protocolVersion": "1.0", + "catalogSchema": { + "catalogId": "custom", + "components": { + "ColumnHeader": { + "allowedParents": [ + "Column" + ] + } + } + } + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom", + "components": [ + { + "id": "root", + "component": "ColumnHeader" + } + ] + } + } + ] + } + ], + "expectError": { + "category": "ValidationError", + "code": "UNALLOWED_PARENT", + "surfaceId": "s1", + "path": "/components/0", + "message": "Component 'root' (ColumnHeader) cannot be placed under parent 'Surface' (Surface). Allowed parents: ['Column'].", + "details": [ + { + "path": "/components/0", + "code": "UNALLOWED_PARENT", + "surfaceId": "s1" + } + ], + "rendererError": { + "version": "1.0", + "error": { + "code": "UNALLOWED_PARENT", + "surfaceId": "s1", + "path": "/components/0", + "message": "Component 'root' (ColumnHeader) cannot be placed under parent 'Surface' (Surface). Allowed parents: ['Column']." + } + } + } + }, + { + "name": "test_composition_unallowed_child_error", + "description": "Verifies validation error thrown when container contains an unallowed child component type.", + "catalog": { + "catalogId": "custom", + "protocolVersion": "1.0", + "components": { + "RestrictedBox": { + "type": "object", + "allowedChildren": [ + "Text", + "Button" + ], + "properties": { + "children": { + "$ref": "#/$defs/ChildList" + } + } + }, + "Video": { + "type": "object" + } + } + }, + "action": "process_messages", + "validationConfig": {}, + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom", + "components": [ + { + "id": "root", + "component": "RestrictedBox", + "children": [ + "v1" + ] + }, + { + "id": "v1", + "component": "Video" + } + ] + } + } + ] + } + ], + "expectError": { + "category": "ValidationError", + "code": "UNALLOWED_CHILD", + "surfaceId": "s1", + "path": "/components/0/children/0", + "message": "Container 'root' (RestrictedBox) cannot contain child 'v1' (Video). Allowed children: ['Text', 'Button'].", + "details": [ + { + "path": "/components/0/children/0", + "code": "UNALLOWED_CHILD", + "surfaceId": "s1" + } + ], + "rendererError": { + "version": "1.0", + "error": { + "code": "UNALLOWED_CHILD", + "surfaceId": "s1", + "path": "/components/0/children/0", + "message": "Container 'root' (RestrictedBox) cannot contain child 'v1' (Video). Allowed children: ['Text', 'Button']." + } + } + } + }, + { + "name": "test_composition_dropped_child_is_not_checked_against_surface", + "description": "A container rewritten to drop a child leaves a component that nothing references. Only the surface's root is checked against Surface in allowedParents, so when orphans are allowed the update is accepted, even though the child's allowedParents does not include Surface. The other checks are on, so the processor checks the whole surface.", + "catalog": { + "catalogId": "custom", + "protocolVersion": "1.0", + "components": { + "Box": { + "type": "object", + "properties": { + "children": { + "$ref": "#/$defs/ChildList" + } + } + }, + "Header": { + "type": "object", + "allowedParents": [ + "Box" + ] + } + } + }, + "action": "process_messages", + "validationConfig": { + "allowOrphanComponents": true + }, + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom", + "components": [ + { + "id": "root", + "component": "Box", + "children": [ + "h" + ] + }, + { + "id": "h", + "component": "Header" + } + ] + } + } + ] + }, + { + "payload": [ + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Box", + "children": [] + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "s1": { + "components": { + "root": { + "component": "Box", + "children": [] + }, + "h": { + "component": "Header" + } + } + } + } + } + }, + { + "name": "test_composition_dropped_child_is_not_checked_against_surface_when_references_may_dangle", + "description": "The same payloads as test_composition_dropped_child_is_not_checked_against_surface, when references to components that have not arrived are also allowed. An SDK may then check only the components each update changes; the dropped child is still not checked against Surface.", + "catalog": { + "catalogId": "custom", + "protocolVersion": "1.0", + "components": { + "Box": { + "type": "object", + "properties": { + "children": { + "$ref": "#/$defs/ChildList" + } + } + }, + "Header": { + "type": "object", + "allowedParents": [ + "Box" + ] + } + } + }, + "action": "process_messages", + "validationConfig": { + "allowOrphanComponents": true, + "allowDanglingReferences": true + }, + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom", + "components": [ + { + "id": "root", + "component": "Box", + "children": [ + "h" + ] + }, + { + "id": "h", + "component": "Header" + } + ] + } + } + ] + }, + { + "payload": [ + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Box", + "children": [] + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "s1": { + "components": { + "root": { + "component": "Box", + "children": [] + }, + "h": { + "component": "Header" + } + } + } + } + } + }, + { + "name": "test_composition_dropped_child_is_reported_as_orphan", + "description": "The same payloads as test_composition_dropped_child_is_not_checked_against_surface, when orphans are not allowed. The topology check reports the dropped child as unreachable from the root; it is not reported as UNALLOWED_PARENT against Surface.", + "catalog": { + "catalogId": "custom", + "protocolVersion": "1.0", + "components": { + "Box": { + "type": "object", + "properties": { + "children": { + "$ref": "#/$defs/ChildList" + } + } + }, + "Header": { + "type": "object", + "allowedParents": [ + "Box" + ] + } + } + }, + "action": "process_messages", + "validationConfig": { + "allowOrphanComponents": false + }, + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom", + "components": [ + { + "id": "root", + "component": "Box", + "children": [ + "h" + ] + }, + { + "id": "h", + "component": "Header" + } + ] + } + } + ] + }, + { + "payload": [ + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Box", + "children": [] + } + ] + } + } + ], + "expectError": { + "category": "IntegrityError", + "message": "Component 'h' is not reachable from 'root'" + } + } + ] + }, + { + "name": "test_composition_root_referenced_by_another_component_checked_against_surface", + "description": "The surface's root is checked against Surface in allowedParents even when another component also lists it as a child. The other component is an orphan, which the config allows.", + "catalog": { + "catalogId": "custom", + "protocolVersion": "1.0", + "components": { + "Box": { + "type": "object", + "properties": { + "children": { + "$ref": "#/$defs/ChildList" + } + } + }, + "Card": { + "type": "object", + "allowedParents": [ + "Box" + ] + } + } + }, + "action": "process_messages", + "validationConfig": { + "allowOrphanComponents": true + }, + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom", + "components": [ + { + "id": "root", + "component": "Card" + }, + { + "id": "box", + "component": "Box", + "children": [ + "root" + ] + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "code": "UNALLOWED_PARENT", + "message": "Component 'root' (Card) cannot be placed under parent 'Surface' (Surface). Allowed parents: ['Box']." + } + } + ] + }, + { + "name": "test_composition_multiple_parents_disallowed_first", + "description": "Every parent of a component is checked against its allowedParents. Here the disallowed parent comes first in surface order.", + "catalog": { + "catalogId": "custom", + "protocolVersion": "1.0", + "components": { + "Box": { + "type": "object", + "properties": { + "children": { + "$ref": "#/$defs/ChildList" + } + } + }, + "Card": { + "type": "object", + "properties": { + "children": { + "$ref": "#/$defs/ChildList" + } + } + }, + "Header": { + "type": "object", + "allowedParents": [ + "Box" + ] + } + } + }, + "action": "process_messages", + "validationConfig": {}, + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom", + "components": [ + { + "id": "root", + "component": "Box", + "children": [ + "card", + "b" + ] + }, + { + "id": "card", + "component": "Card", + "children": [ + "h" + ] + }, + { + "id": "b", + "component": "Box", + "children": [ + "h" + ] + }, + { + "id": "h", + "component": "Header" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "code": "UNALLOWED_PARENT", + "message": "Component 'h' (Header) cannot be placed under parent 'card' (Card). Allowed parents: ['Box']." + } + } + ] + }, + { + "name": "test_composition_multiple_parents_disallowed_second", + "description": "The same as test_composition_multiple_parents_disallowed_first, with the disallowed parent second in surface order.", + "catalog": { + "catalogId": "custom", + "protocolVersion": "1.0", + "components": { + "Box": { + "type": "object", + "properties": { + "children": { + "$ref": "#/$defs/ChildList" + } + } + }, + "Card": { + "type": "object", + "properties": { + "children": { + "$ref": "#/$defs/ChildList" + } + } + }, + "Header": { + "type": "object", + "allowedParents": [ + "Box" + ] + } + } + }, + "action": "process_messages", + "validationConfig": {}, + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom", + "components": [ + { + "id": "root", + "component": "Box", + "children": [ + "b", + "card" + ] + }, + { + "id": "b", + "component": "Box", + "children": [ + "h" + ] + }, + { + "id": "card", + "component": "Card", + "children": [ + "h" + ] + }, + { + "id": "h", + "component": "Header" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "code": "UNALLOWED_PARENT", + "message": "Component 'h' (Header) cannot be placed under parent 'card' (Card). Allowed parents: ['Box']." + } + } + ] + } +] \ No newline at end of file diff --git a/package/__fixtures__/a2ui-conformance/core/data_context.json b/package/__fixtures__/a2ui-conformance/core/data_context.json new file mode 100644 index 0000000000..10c6456d07 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/core/data_context.json @@ -0,0 +1,144 @@ +[ + { + "name": "test_resolve_path_relative_against_context", + "description": "Resolves a relative path against a scoped context path.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "resolve_path", + "args": { + "path": "name", + "contextPath": "/user" + }, + "expect": "/user/name" + }, + { + "name": "test_resolve_path_nested_relative_against_context", + "description": "Resolves a multi-segment relative path against a scoped context path.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "resolve_path", + "args": { + "path": "address/city", + "contextPath": "/user" + }, + "expect": "/user/address/city" + }, + { + "name": "test_resolve_path_absolute_ignores_context", + "description": "An absolute path starting with `/` ignores the context path.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "resolve_path", + "args": { + "path": "/list/0", + "contextPath": "/user" + }, + "expect": "/list/0" + }, + { + "name": "test_resolve_path_relative_against_root", + "description": "Resolves a relative path against the root context `/`.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "resolve_path", + "args": { + "path": "user/name", + "contextPath": "/" + }, + "expect": "/user/name" + }, + { + "name": "test_resolve_path_relative_without_context", + "description": "Resolves a relative path when no context path is provided.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "resolve_path", + "args": { + "path": "foo" + }, + "expect": "/foo" + }, + { + "name": "test_resolve_path_context_with_trailing_slash", + "description": "Strips a trailing slash on the context path before joining a relative path.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "resolve_path", + "args": { + "path": "foo", + "contextPath": "/bar/" + }, + "expect": "/bar/foo" + }, + { + "name": "test_resolve_path_empty_in_scoped_context", + "description": "An empty relative path resolves to the current context path.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "resolve_path", + "args": { + "path": "", + "contextPath": "/user" + }, + "expect": "/user" + }, + { + "name": "test_resolve_path_empty_in_trailing_slash_context", + "description": "An empty relative path in a context with a trailing slash resolves to the trimmed context path.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "resolve_path", + "args": { + "path": "", + "contextPath": "/user/" + }, + "expect": "/user" + }, + { + "name": "test_resolve_path_dot_in_scoped_context", + "description": "A `.` path resolves to the current context path.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "resolve_path", + "args": { + "path": ".", + "contextPath": "/user" + }, + "expect": "/user" + }, + { + "name": "test_resolve_path_empty_in_root_context", + "description": "An empty relative path in the root context resolves to `/`.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "resolve_path", + "args": { + "path": "", + "contextPath": "/" + }, + "expect": "/" + }, + { + "name": "test_resolve_path_dot_in_root_context", + "description": "A `.` path in the root context resolves to `/`.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "resolve_path", + "args": { + "path": ".", + "contextPath": "/" + }, + "expect": "/" + } +] \ No newline at end of file diff --git a/package/__fixtures__/a2ui-conformance/core/data_deletion.json b/package/__fixtures__/a2ui-conformance/core/data_deletion.json new file mode 100644 index 0000000000..64471a4081 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/core/data_deletion.json @@ -0,0 +1,358 @@ +[ + { + "name": "test_data_deletion_value_null", + "description": "Verifies that updateDataModel with value=null deletes the specified JSON Pointer path.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic", + "dataModel": { + "user": { + "profile": { + "name": "Alice", + "bio": "Software Engineer" + } + } + } + } + } + ] + }, + { + "payload": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/user/profile/bio", + "value": null + } + } + ] + } + ], + "expect": { + "surfaces": { + "s1": { + "dataModel": { + "user": { + "profile": { + "name": "Alice" + } + } + } + } + } + } + }, + { + "name": "test_data_deletion_nested_pointer_sibling_preservation", + "description": "Verifies deleting a child property via value=null preserves sibling keys in parent object.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic", + "dataModel": { + "settings": { + "notifications": { + "email": true, + "push": false, + "sms": true + } + } + } + } + } + ] + }, + { + "payload": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/settings/notifications/email", + "value": null + } + } + ] + } + ], + "expect": { + "surfaces": { + "s1": { + "dataModel": { + "settings": { + "notifications": { + "push": false, + "sms": true + } + } + } + } + } + } + }, + { + "name": "test_data_deletion_top_level_key", + "description": "Verifies deleting a top-level JSON Pointer path via value=null.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic", + "dataModel": { + "persistentData": "keep", + "transientData": "remove" + } + } + } + ] + }, + { + "payload": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/transientData", + "value": null + } + } + ] + } + ], + "expect": { + "surfaces": { + "s1": { + "dataModel": { + "persistentData": "keep" + } + } + } + } + }, + { + "name": "test_data_deletion_parent_object_recursive", + "description": "Verifies deleting an intermediate parent object key via value=null recursively removes all nested child properties.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic", + "dataModel": { + "settings": { + "notifications": { + "email": true, + "push": false, + "sms": true + }, + "theme": "dark" + } + } + } + } + ] + }, + { + "payload": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/settings/notifications", + "value": null + } + } + ] + } + ], + "expect": { + "surfaces": { + "s1": { + "dataModel": { + "settings": { + "theme": "dark" + } + } + } + } + } + }, + { + "name": "test_data_deletion_all_keys_empty_data_model", + "description": "Verifies deleting the only top-level key via value=null results in an empty dataModel.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic", + "dataModel": { + "user": { + "profile": { + "name": "Alice" + } + } + } + } + } + ] + }, + { + "payload": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/user", + "value": null + } + } + ] + } + ], + "expect": { + "surfaces": { + "s1": { + "dataModel": {} + } + } + } + }, + { + "name": "test_data_deletion_array_index_preservation", + "description": "Verifies deleting array element via value=null preserves outer array length and structure with null.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic", + "dataModel": { + "items": [ + "a", + "b", + "c" + ] + } + } + } + ] + }, + { + "payload": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/items/1", + "value": null + } + } + ] + } + ], + "expect": { + "surfaces": { + "s1": { + "dataModel": { + "items": [ + "a", + null, + "c" + ] + } + } + } + } + }, + { + "name": "test_data_deletion_root_replacement", + "description": "Verifies updating root path / with value=null resets data model to empty object.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic", + "dataModel": { + "user": { + "name": "Alice" + }, + "count": 42 + } + } + } + ] + }, + { + "payload": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/", + "value": null + } + } + ] + } + ], + "expect": { + "surfaces": { + "s1": { + "dataModel": {} + } + } + } + } +] \ No newline at end of file diff --git a/package/__fixtures__/a2ui-conformance/core/data_model.json b/package/__fixtures__/a2ui-conformance/core/data_model.json new file mode 100644 index 0000000000..c4cf040e49 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/core/data_model.json @@ -0,0 +1,2012 @@ +[ + { + "name": "test_data_model_initializes_empty", + "description": "A model created without data starts as an empty object.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "steps": [ + { + "op": "get", + "path": "/", + "expect": {} + } + ] + }, + { + "name": "test_data_model_retrieves_root", + "description": "The root path returns the whole data model.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "get", + "path": "/", + "expect": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + } + } + ] + }, + { + "name": "test_data_model_retrieves_nested_path", + "description": "Nested object paths resolve segment by segment.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "get", + "path": "/user/name", + "expect": "Alice" + }, + { + "op": "get", + "path": "/user/settings/theme", + "expect": "dark" + } + ] + }, + { + "name": "test_data_model_retrieves_list_items", + "description": "List elements resolve by numeric segment.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "get", + "path": "/items/0", + "expect": "a" + }, + { + "op": "get", + "path": "/items/1", + "expect": "b" + } + ] + }, + { + "name": "test_data_model_absent_for_missing_paths", + "description": "Paths that do not exist resolve to no value.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "get", + "path": "/user/age", + "expect_absent": true + }, + { + "op": "get", + "path": "/unknown/path", + "expect_absent": true + } + ] + }, + { + "name": "test_data_model_absent_through_empty_segment", + "description": "Traversing through a segment that holds no value yields no value.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "set", + "path": "/nullable/inner", + "value": 1 + }, + { + "op": "delete", + "path": "/nullable/inner" + }, + { + "op": "get", + "path": "/nullable/deep/path", + "expect_absent": true + } + ] + }, + { + "name": "test_data_model_sets_existing_path", + "description": "Setting an existing path replaces its value.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "set", + "path": "/user/name", + "value": "Bob" + }, + { + "op": "get", + "path": "/user/name", + "expect": "Bob" + } + ] + }, + { + "name": "test_data_model_sets_new_path", + "description": "Setting a new key on an existing object adds it.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "set", + "path": "/user/age", + "value": 30 + }, + { + "op": "get", + "path": "/user/age", + "expect": 30 + } + ] + }, + { + "name": "test_data_model_creates_intermediate_objects", + "description": "Missing intermediate objects are created on write.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "set", + "path": "/a/b/c", + "value": "foo" + }, + { + "op": "get", + "path": "/a/b/c", + "expect": "foo" + }, + { + "op": "get", + "path": "/a/b", + "expect_type": "object" + } + ] + }, + { + "name": "test_data_model_delete_removes_key", + "description": "Deleting a path removes the key from its parent object.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "delete", + "path": "/user/name" + }, + { + "op": "get", + "path": "/user/name", + "expect_absent": true + }, + { + "op": "get", + "path": "/user", + "expect": { + "settings": { + "theme": "dark" + } + } + } + ] + }, + { + "name": "test_data_model_list_set_and_get", + "description": "Writing a numeric segment into a missing path creates a list.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "set", + "path": "/list/0", + "value": "hello" + }, + { + "op": "get", + "path": "/list/0", + "expect": "hello" + }, + { + "op": "get", + "path": "/list", + "expect_type": "list" + } + ] + }, + { + "name": "test_data_model_list_append", + "description": "Writing successive indices appends to a list.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "set", + "path": "/list/0", + "value": "hello" + }, + { + "op": "set", + "path": "/list/1", + "value": "world" + }, + { + "op": "get", + "path": "/list", + "expect": [ + "hello", + "world" + ] + } + ] + }, + { + "name": "test_data_model_list_update_index", + "description": "Writing an existing index replaces that element.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "set", + "path": "/items/1", + "value": "updated" + }, + { + "op": "get", + "path": "/items", + "expect": [ + "a", + "updated", + "c" + ] + } + ] + }, + { + "name": "test_data_model_creates_nested_structures", + "description": "Auto-vivification picks a list or an object based on the next segment.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "set", + "path": "/a/b/0/c", + "value": 123 + }, + { + "op": "get", + "path": "/a/b/0/c", + "expect": 123 + }, + { + "op": "get", + "path": "/a/b", + "expect_type": "list" + }, + { + "op": "get", + "path": "/a/b/0", + "expect_type": "object" + }, + { + "op": "set", + "path": "/x/y/z", + "value": "hello" + }, + { + "op": "get", + "path": "/x/y/z", + "expect": "hello" + }, + { + "op": "set", + "path": "/nestedList/0/0", + "value": "inner" + }, + { + "op": "get", + "path": "/nestedList/0/0", + "expect": "inner" + }, + { + "op": "get", + "path": "/nestedList", + "expect_type": "list" + }, + { + "op": "get", + "path": "/nestedList/0", + "expect_type": "list" + } + ] + }, + { + "name": "test_data_model_get_out_of_range_index", + "description": "Out of range and non numeric list indices resolve to no value on read.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "get", + "path": "/items/99", + "expect_absent": true + }, + { + "op": "get", + "path": "/items/-1", + "expect_absent": true + }, + { + "op": "get", + "path": "/items/invalid", + "expect_absent": true + } + ] + }, + { + "name": "test_data_model_normalizes_trailing_slash", + "description": "A trailing slash is not a distinct path.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "watch": [ + "/foo" + ], + "steps": [ + { + "op": "set", + "path": "/foo/", + "value": "bar", + "expect_notified": [ + "/foo" + ] + }, + { + "op": "get", + "path": "/foo/", + "expect": "bar" + }, + { + "op": "get", + "path": "/foo", + "expect": "bar" + } + ] + }, + { + "name": "test_data_model_replaces_root", + "description": "Setting the root path replaces the whole model.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "watch": [ + "/", + "/unrelated" + ], + "steps": [ + { + "op": "set", + "path": "/", + "value": { + "newRoot": "foo" + }, + "expect_notified": [ + "/" + ], + "expect_values": { + "/": { + "newRoot": "foo" + } + } + }, + { + "op": "get", + "path": "", + "expect": { + "newRoot": "foo" + } + } + ] + }, + { + "name": "test_data_model_escaped_slash", + "description": "A ~1 escape addresses a key containing a slash (RFC 6901).", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "set", + "path": "/user/detailed~1info", + "value": "some info" + }, + { + "op": "get", + "path": "/user/detailed~1info", + "expect": "some info" + }, + { + "op": "get", + "path": "/user", + "expect": { + "name": "Alice", + "settings": { + "theme": "dark" + }, + "detailed/info": "some info" + } + } + ] + }, + { + "name": "test_data_model_escaped_tilde", + "description": "A ~0 escape addresses a key containing a tilde (RFC 6901).", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "set", + "path": "/user/profile~0name", + "value": "profile~name" + }, + { + "op": "get", + "path": "/user/profile~0name", + "expect": "profile~name" + }, + { + "op": "get", + "path": "/user", + "expect": { + "name": "Alice", + "settings": { + "theme": "dark" + }, + "profile~name": "profile~name" + } + } + ] + }, + { + "name": "test_data_model_escaped_mixed", + "description": "Both escapes may appear in one segment.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "set", + "path": "/user/a~0b~1c", + "value": "value" + }, + { + "op": "get", + "path": "/user/a~0b~1c", + "expect": "value" + }, + { + "op": "get", + "path": "/user", + "expect": { + "name": "Alice", + "settings": { + "theme": "dark" + }, + "a~b/c": "value" + } + } + ] + }, + { + "name": "test_data_model_escape_order", + "description": "A ~01 sequence decodes to ~1, not to a slash.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "set", + "path": "/user/a~01b", + "value": "value" + }, + { + "op": "get", + "path": "/user/a~01b", + "expect": "value" + }, + { + "op": "get", + "path": "/user", + "expect": { + "name": "Alice", + "settings": { + "theme": "dark" + }, + "a~1b": "value" + } + } + ] + }, + { + "name": "test_data_model_rejects_write_through_primitive", + "description": "Writing through a primitive intermediate segment is an error.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "set", + "path": "/user/name", + "value": "not an object" + }, + { + "op": "set", + "path": "/user/name/first", + "value": "Alice", + "expect_error": { + "category": "DataError", + "message": "Cannot set path" + } + } + ] + }, + { + "name": "test_data_model_rejects_write_through_list_primitive", + "description": "Writing through a primitive list element is an error.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "set", + "path": "/items/0/foo", + "value": "bar", + "expect_error": { + "category": "DataError", + "message": "Cannot set path" + } + } + ] + }, + { + "name": "test_data_model_rejects_non_numeric_list_segment", + "description": "A non numeric segment cannot address a list element on write.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "set", + "path": "/items/foo", + "value": "bar", + "expect_error": { + "category": "DataError", + "message": "non-numeric segment" + } + } + ] + }, + { + "name": "test_data_model_rejects_non_numeric_list_segment_intermediate", + "description": "A non numeric intermediate segment cannot address a list element.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "set", + "path": "/items/foo/bar", + "value": "value", + "expect_error": { + "category": "DataError", + "message": "non-numeric segment" + } + } + ] + }, + { + "name": "test_data_model_notifies_exact_path", + "description": "An observer fires when its own path is written.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "watch": [ + "/user/name" + ], + "steps": [ + { + "op": "set", + "path": "/user/name", + "value": "Charlie", + "expect_values": { + "/user/name": "Charlie" + }, + "expect_notified": [ + "/user/name" + ] + } + ] + }, + { + "name": "test_data_model_notifies_ancestor", + "description": "An observer on a container fires when a descendant is written.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "watch": [ + "/user" + ], + "steps": [ + { + "op": "set", + "path": "/user/name", + "value": "Dave", + "expect_values": { + "/user": { + "name": "Dave", + "settings": { + "theme": "dark" + } + } + }, + "expect_notified": [ + "/user" + ] + } + ] + }, + { + "name": "test_data_model_notifies_descendant", + "description": "An observer on a leaf fires when an ancestor is replaced.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "watch": [ + "/user/settings/theme" + ], + "steps": [ + { + "op": "set", + "path": "/user/settings", + "value": { + "theme": "light" + }, + "expect_values": { + "/user/settings/theme": "light" + }, + "expect_notified": [ + "/user/settings/theme" + ] + } + ] + }, + { + "name": "test_data_model_notifies_root_observer", + "description": "A root observer fires on any write.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "watch": [ + "/" + ], + "steps": [ + { + "op": "set", + "path": "/newProp", + "value": "test", + "expect_notified": [ + "/" + ] + } + ] + }, + { + "name": "test_data_model_notifies_parent_on_child_write", + "description": "A container observer sees the updated container value.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "watch": [ + "/parent" + ], + "steps": [ + { + "op": "set", + "path": "/parent", + "value": { + "child": "initial" + } + }, + { + "op": "set", + "path": "/parent/child", + "value": "updated", + "expect_values": { + "/parent": { + "child": "updated" + } + } + } + ] + }, + { + "name": "test_data_model_does_not_notify_unrelated", + "description": "An observer does not fire for an unrelated path.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "a": 1, + "b": 2 + }, + "watch": [ + "/b" + ], + "steps": [ + { + "op": "set", + "path": "/a", + "value": 99, + "expect_notified": [] + } + ] + }, + { + "name": "test_data_model_does_not_notify_prefix_sibling", + "description": "A sibling whose name merely shares a prefix is not a descendant.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "foo": 1, + "foobar": 2 + }, + "watch": [ + "/foo", + "/foobar" + ], + "steps": [ + { + "op": "set", + "path": "/foo", + "value": 99, + "expect_notified": [ + "/foo" + ] + } + ] + }, + { + "name": "test_data_model_multiple_observers_same_path", + "description": "Every observer on a path is notified.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "watch": [ + "/user/name", + "/user/name" + ], + "steps": [ + { + "op": "set", + "path": "/user/name", + "value": "Eve", + "expect_values": { + "/user/name": "Eve" + }, + "expect_notified": [ + "/user/name", + "/user/name" + ] + } + ] + }, + { + "name": "test_data_model_observe_absent_path", + "description": "Observing a path that does not exist yet starts empty and fires on creation.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "watch": [ + "/non/existent" + ], + "steps": [ + { + "op": "get", + "path": "/non/existent", + "expect_absent": true + }, + { + "op": "set", + "path": "/non/existent", + "value": "exists now", + "expect_values": { + "/non/existent": "exists now" + }, + "expect_notified": [ + "/non/existent" + ] + } + ] + }, + { + "name": "test_data_model_stops_notifying_after_dispose", + "description": "Disposing the model detaches every observer.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + }, + "items": [ + "a", + "b", + "c" + ] + }, + "watch": [ + "/" + ], + "steps": [ + { + "op": "dispose" + }, + { + "op": "set", + "path": "/foo", + "value": "bar", + "expect_notified": [] + } + ] + }, + { + "name": "test_data_model_does_not_notify_unchanged_descendant", + "description": "An observer on a descendant path does not fire when an ancestor is rewritten but the observer's own value is still absent.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "watch": [ + "/a/b" + ], + "steps": [ + { + "op": "set", + "path": "/a", + "value": { + "c": 1 + }, + "expect_notified": [] + } + ] + }, + { + "name": "test_data_model_does_not_notify_same_value_rewrite", + "description": "Writing a path the value it already holds does not fire its observers.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "k": "v" + }, + "watch": [ + "/k" + ], + "steps": [ + { + "op": "set", + "path": "/k", + "value": "v", + "expect_notified": [] + }, + { + "op": "set", + "path": "/k", + "value": "w", + "expect_notified": [ + "/k" + ] + } + ] + }, + { + "name": "test_data_model_reactivity_child_deletion_notifies_parent_and_child", + "description": "Deleting a child property notifies both the child path and ancestor containers.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice", + "settings": { + "theme": "dark" + } + } + }, + "watch": [ + "/user", + "/user/name" + ], + "steps": [ + { + "op": "delete", + "path": "/user/name", + "expect_notified": [ + "/user", + "/user/name" + ], + "expect_values": { + "/user": { + "settings": { + "theme": "dark" + } + } + } + }, + { + "op": "get", + "path": "/user/name", + "expect_absent": true + } + ] + }, + { + "name": "test_data_model_reactivity_array_element_update_notifies_array", + "description": "Updating an array element notifies both the element observer and the array observer.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "items": [ + "a", + "b", + "c" + ] + }, + "watch": [ + "/items", + "/items/1" + ], + "steps": [ + { + "op": "set", + "path": "/items/1", + "value": "updated", + "expect_notified": [ + "/items", + "/items/1" + ], + "expect_values": { + "/items": [ + "a", + "updated", + "c" + ], + "/items/1": "updated" + } + } + ] + }, + { + "name": "test_data_model_reactivity_ancestor_deletion_notifies_descendants", + "description": "Deleting a parent container notifies observers on nested descendant paths whose values become absent.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "settings": { + "theme": "dark" + } + } + }, + "watch": [ + "/user", + "/user/settings", + "/user/settings/theme" + ], + "steps": [ + { + "op": "delete", + "path": "/user", + "expect_notified": [ + "/user", + "/user/settings", + "/user/settings/theme" + ] + }, + { + "op": "get", + "path": "/user/settings/theme", + "expect_absent": true + } + ] + }, + { + "name": "test_data_model_delete_out_of_range_list_index_is_noop", + "description": "Deleting a list index past the end of the list leaves the list unchanged rather than padding it out to that index.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "delete", + "path": "/items/10" + }, + { + "op": "get", + "path": "/items", + "expect": [ + "a", + "b", + "c" + ] + } + ] + }, + { + "name": "test_data_model_notifies_descendant_on_root_replacement", + "description": "Replacing the root notifies observers on descendant paths whose values change or become absent.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice" + }, + "stale": "present" + }, + "watch": [ + "/user/name", + "/stale" + ], + "steps": [ + { + "op": "set", + "path": "/", + "value": { + "user": { + "name": "Bob" + } + }, + "expect_notified": [ + "/user/name", + "/stale" + ], + "expect_values": { + "/user/name": "Bob" + } + }, + { + "op": "get", + "path": "/stale", + "expect_absent": true + } + ] + }, + { + "name": "test_data_model_rejects_forbidden_path_segments", + "description": "Paths containing __proto__, constructor, or prototype are rejected on both read and write with DataError.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": { + "name": "Alice" + } + }, + "steps": [ + { + "op": "set", + "path": "/__proto__/polluted", + "value": "hacked", + "expect_error": { + "category": "DataError", + "message": "Forbidden path segment" + } + }, + { + "op": "get", + "path": "/__proto__/polluted", + "expect_error": { + "category": "DataError", + "message": "Forbidden path segment" + } + }, + { + "op": "set", + "path": "/constructor/prototype/polluted", + "value": "hacked", + "expect_error": { + "category": "DataError", + "message": "Forbidden path segment" + } + }, + { + "op": "get", + "path": "/constructor/prototype/polluted", + "expect_error": { + "category": "DataError", + "message": "Forbidden path segment" + } + }, + { + "op": "set", + "path": "/user/prototype/polluted", + "value": "hacked", + "expect_error": { + "category": "DataError", + "message": "Forbidden path segment" + } + }, + { + "op": "get", + "path": "/user/prototype/polluted", + "expect_error": { + "category": "DataError", + "message": "Forbidden path segment" + } + } + ] + }, + { + "name": "test_data_model_allows_own_properties_named_after_object_methods", + "description": "Unset keys like toString, valueOf, and hasOwnProperty are absent on read, and can be written and read back as ordinary keys.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": {}, + "steps": [ + { + "op": "get", + "path": "/toString", + "expect_absent": true + }, + { + "op": "get", + "path": "/valueOf", + "expect_absent": true + }, + { + "op": "get", + "path": "/hasOwnProperty", + "expect_absent": true + }, + { + "op": "set", + "path": "/toString", + "value": "custom toString" + }, + { + "op": "get", + "path": "/toString", + "expect": "custom toString" + }, + { + "op": "set", + "path": "/valueOf/nested", + "value": "custom valueOf" + }, + { + "op": "get", + "path": "/valueOf/nested", + "expect": "custom valueOf" + } + ] + }, + { + "name": "test_data_model_delete_absent_deep_path_is_noop", + "description": "Deleting a deep path that does not exist is a no-op that neither vivifies intermediate containers nor notifies observers.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": {}, + "watch": [ + "/", + "/nonexistent" + ], + "steps": [ + { + "op": "delete", + "path": "/nonexistent/child/path", + "expect_notified": [] + }, + { + "op": "get", + "path": "/", + "expect": {} + }, + { + "op": "get", + "path": "/nonexistent", + "expect_absent": true + } + ] + }, + { + "name": "test_data_model_sparse_list_write_pads_with_null", + "description": "Writing an index past the current end of a list pads intermediate positions with null/absent values.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": {}, + "steps": [ + { + "op": "set", + "path": "/data/lists/1/value", + "value": 42 + }, + { + "op": "get", + "path": "/data/lists/1/value", + "expect": 42 + }, + { + "op": "get", + "path": "/data/lists/0", + "expect_absent": true + }, + { + "op": "get", + "path": "/data/lists", + "expect_type": "list" + } + ] + }, + { + "name": "test_data_model_rejects_excessive_list_index", + "description": "List indices exceeding the maximum supported bound (10000) are rejected with DataError.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "set", + "path": "/items/10000", + "value": "max_element" + }, + { + "op": "get", + "path": "/items/10000", + "expect": "max_element" + }, + { + "op": "set", + "path": "/items/10001", + "value": "x", + "expect_error": { + "category": "DataError" + } + }, + { + "op": "set", + "path": "/items/999999999", + "value": "x", + "expect_error": { + "category": "DataError" + } + }, + { + "op": "set", + "path": "/items/10001/name", + "value": "x", + "expect_error": { + "category": "DataError" + } + }, + { + "op": "set", + "path": "/newArray/10000000", + "value": "x", + "expect_error": { + "category": "DataError" + } + }, + { + "op": "set", + "path": "/newArray/10000000/field", + "value": "x", + "expect_error": { + "category": "DataError" + } + } + ] + }, + { + "name": "test_data_model_rejects_leading_zero_list_index", + "description": "Leading-zero list indices (RFC 6901 §4) resolve to absent on read and raise DataError on write.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "items": [ + "a", + "b", + "c" + ] + }, + "steps": [ + { + "op": "get", + "path": "/items/01", + "expect_absent": true + }, + { + "op": "set", + "path": "/items/01", + "value": "value", + "expect_error": { + "category": "DataError", + "message": "non-numeric segment" + } + }, + { + "op": "set", + "path": "/items/01/nested", + "value": "value", + "expect_error": { + "category": "DataError", + "message": "non-numeric segment" + } + } + ] + }, + { + "name": "test_data_model_rejects_write_through_primitive_root", + "description": "Writing a child path when the root itself holds a primitive (including falsy primitives) raises DataError and leaves the root intact.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": {}, + "steps": [ + { + "op": "set", + "path": "/", + "value": 1 + }, + { + "op": "set", + "path": "/a", + "value": 2, + "expect_error": { + "category": "DataError", + "message": "Cannot set path" + } + }, + { + "op": "get", + "path": "/", + "expect": 1 + }, + { + "op": "set", + "path": "/", + "value": false + }, + { + "op": "set", + "path": "/a", + "value": 2, + "expect_error": { + "category": "DataError", + "message": "Cannot set path" + } + }, + { + "op": "get", + "path": "/", + "expect": false + }, + { + "op": "set", + "path": "/", + "value": 0 + }, + { + "op": "set", + "path": "/a", + "value": 2, + "expect_error": { + "category": "DataError", + "message": "Cannot set path" + } + }, + { + "op": "get", + "path": "/", + "expect": 0 + }, + { + "op": "set", + "path": "/", + "value": "" + }, + { + "op": "set", + "path": "/a", + "value": 2, + "expect_error": { + "category": "DataError", + "message": "Cannot set path" + } + }, + { + "op": "get", + "path": "/", + "expect": "" + } + ] + }, + { + "name": "test_data_model_set_through_null_placeholder_vivifies", + "description": "Writing through an explicit null value in an object or list replaces the null slot with a container.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": { + "user": null, + "counts": [ + null + ] + }, + "steps": [ + { + "op": "set", + "path": "/user/name", + "value": "Bob" + }, + { + "op": "set", + "path": "/counts/0/total", + "value": 1 + }, + { + "op": "get", + "path": "/user/name", + "expect": "Bob" + }, + { + "op": "get", + "path": "/counts/0/total", + "expect": 1 + } + ] + }, + { + "name": "test_data_model_normalizes_empty_path_segments", + "description": "Multiple or trailing slashes in a path are collapsed into canonical segments on read, write, and watch.", + "catalog": { + "protocolVersion": "v0.9" + }, + "action": "data_model", + "initial": {}, + "watch": [ + "/a/b" + ], + "steps": [ + { + "op": "set", + "path": "//a///b//", + "value": "val", + "expect_notified": [ + "/a/b" + ], + "expect_values": { + "/a/b": "val" + } + }, + { + "op": "get", + "path": "/a/b", + "expect": "val" + }, + { + "op": "get", + "path": "//a///b//", + "expect": "val" + }, + { + "op": "get", + "path": "///", + "expect": { + "a": { + "b": "val" + } + } + } + ] + } +] \ No newline at end of file diff --git a/package/__fixtures__/a2ui-conformance/core/data_model_pointers.json b/package/__fixtures__/a2ui-conformance/core/data_model_pointers.json new file mode 100644 index 0000000000..94df6744c4 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/core/data_model_pointers.json @@ -0,0 +1,358 @@ +[ + { + "name": "test_pointer_escape_characters", + "description": "Verifies RFC 6901 JSON Pointer escape sequence resolution (~0 for ~ and ~1 for /).", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic", + "dataModel": { + "a/b": { + "c~d": "original_value" + }, + "slash/key": 100 + } + } + } + ] + }, + { + "payload": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/a~1b/c~0d", + "value": "escaped_value" + } + } + ] + } + ], + "expect": { + "surfaces": { + "s1": { + "dataModel": { + "a/b": { + "c~d": "escaped_value" + }, + "slash/key": 100 + } + } + } + } + }, + { + "name": "test_pointer_array_indexing", + "description": "Verifies numeric array indexing in JSON Pointers (/items/0, /items/1).", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic", + "dataModel": { + "items": [ + "first", + "second", + "third" + ] + } + } + } + ] + }, + { + "payload": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/items/1", + "value": "updated_second" + } + } + ] + } + ], + "expect": { + "surfaces": { + "s1": { + "dataModel": { + "items": [ + "first", + "updated_second", + "third" + ] + } + } + } + } + }, + { + "name": "test_pointer_auto_vivification", + "description": "Verifies automatic intermediate object creation when setting deep uncreated JSON Pointer paths.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic", + "dataModel": {} + } + } + ] + }, + { + "payload": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/deeply/nested/structure/key", + "value": "vivified_value" + } + } + ] + } + ], + "expect": { + "surfaces": { + "s1": { + "dataModel": { + "deeply": { + "nested": { + "structure": { + "key": "vivified_value" + } + } + } + } + } + } + } + }, + { + "name": "test_pointer_extended_escape_sequences", + "description": "Verifies advanced RFC 6901 escape sequence combinations (~10 for /0, ~00 for ~0, ~11 for /1, and ~01 for ~1).", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic", + "dataModel": { + "m~0": "val0", + "m/1": "val1", + "m~1": "val2", + "m/0": "val3" + } + } + } + ] + }, + { + "payload": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/m~00", + "value": "updated0" + } + }, + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/m~11", + "value": "updated1" + } + }, + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/m~01", + "value": "updated2" + } + }, + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/m~10", + "value": "updated3" + } + } + ] + } + ], + "expect": { + "surfaces": { + "s1": { + "dataModel": { + "m~0": "updated0", + "m/1": "updated1", + "m~1": "updated2", + "m/0": "updated3" + } + } + } + } + }, + { + "name": "test_pointer_reject_leading_zero_array_index", + "description": "Verifies rejection of leading zero array indices (e.g. /items/01) per RFC 6901 §4.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic", + "dataModel": { + "items": [ + "a", + "b", + "c" + ] + } + } + } + ] + }, + { + "payload": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/items/01", + "value": "invalid" + } + } + ], + "expectError": { + "category": "DataError", + "message": "non-numeric segment" + } + } + ] + }, + { + "name": "test_pointer_reject_write_through_primitive", + "description": "Verifies DataError when writing nested path through primitive intermediate node.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic", + "dataModel": { + "user": "Alice" + } + } + } + ] + }, + { + "payload": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/user/name/first", + "value": "Alice" + } + } + ], + "expectError": { + "category": "DataError", + "message": "Cannot set path" + } + } + ] + }, + { + "name": "test_pointer_reject_non_numeric_array_segment", + "description": "Verifies DataError when setting non-numeric key on array target.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic", + "dataModel": { + "items": [ + "a", + "b", + "c" + ] + } + } + } + ] + }, + { + "payload": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/items/invalidKey", + "value": "value" + } + } + ], + "expectError": { + "category": "DataError", + "message": "non-numeric segment" + } + } + ] + } +] \ No newline at end of file diff --git a/package/__fixtures__/a2ui-conformance/core/error_responses.json b/package/__fixtures__/a2ui-conformance/core/error_responses.json new file mode 100644 index 0000000000..d8a05e004c --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/core/error_responses.json @@ -0,0 +1,1093 @@ +[ + { + "name": "test_error_response_create_surface_invalid_theme_property", + "description": "Emits VALIDATION_FAILED with /theme/primaryColor when createSurface theme property has wrong type.", + "action": "validate", + "protocolVersion": "v0.9", + "catalogPaths": [ + "specification/v0_9/catalogs/basic/catalog.json" + ], + "payload": [ + { + "version": "v0.9", + "createSurface": { + "surfaceId": "main", + "catalogId": "https://a2ui.org/specification/v0_9/catalogs/basic/catalog.json", + "theme": { + "primaryColor": 12345 + } + } + } + ], + "expectError": { + "category": "ValidationError", + "surfaceId": "main", + "path": "/theme/primaryColor", + "rendererError": { + "version": "v0.9", + "error": { + "code": "VALIDATION_FAILED", + "surfaceId": "main", + "path": "/theme/primaryColor", + "message": "primaryColor" + } + } + } + }, + { + "name": "test_error_response_create_surface_unknown_catalog_id", + "description": "Emits CATALOG_NOT_FOUND when createSurface references an unregistered catalogId.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "https://example.com/unknown_catalog.json" + } + } + ], + "expectError": { + "category": "CatalogError", + "surfaceId": "main", + "rendererError": { + "version": "1.0", + "error": { + "code": "CATALOG_NOT_FOUND", + "surfaceId": "main", + "message": "unknown_catalog" + } + } + } + }, + { + "name": "test_error_response_create_surface_incompatible_catalog_version", + "description": "Emits CATALOG_NOT_FOUND when createSurface binds a v0.9 catalog to a v1.0 surface.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "specification/v0_9/catalogs/basic/catalog.json" + ], + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "https://a2ui.org/specification/v0_9/catalogs/basic/catalog.json" + } + } + ], + "expectError": { + "category": "CatalogError", + "surfaceId": "main", + "rendererError": { + "version": "1.0", + "error": { + "code": "CATALOG_NOT_FOUND", + "surfaceId": "main", + "message": "v0_9" + } + } + } + }, + { + "name": "test_error_response_create_surface_duplicate_surface_id", + "description": "Emits INTEGRITY_ERROR when createSurface is sent for an already existing surfaceId.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + } + ], + "expectError": { + "category": "IntegrityError", + "surfaceId": "main", + "rendererError": { + "version": "1.0", + "error": { + "code": "INTEGRITY_ERROR", + "surfaceId": "main", + "message": "already exists" + } + } + } + }, + { + "name": "test_error_response_update_components_unknown_surface", + "description": "Emits INTEGRITY_ERROR when updateComponents targets a non-existent surfaceId.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "payload": [ + { + "version": "1.0", + "updateComponents": { + "surfaceId": "missing_surface", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Hello" + } + ] + } + } + ], + "expectError": { + "category": "IntegrityError", + "surfaceId": "missing_surface", + "rendererError": { + "version": "1.0", + "error": { + "code": "INTEGRITY_ERROR", + "surfaceId": "missing_surface", + "message": "missing_surface" + } + } + } + }, + { + "name": "test_error_response_update_components_duplicate_id_in_batch", + "description": "Emits INTEGRITY_ERROR when updateComponents contains duplicate component IDs in the same batch.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "main", + "components": [ + { + "id": "root", + "component": "Text", + "text": "First" + }, + { + "id": "root", + "component": "Text", + "text": "Duplicate" + } + ] + } + } + ], + "expectError": { + "category": "IntegrityError", + "surfaceId": "main", + "rendererError": { + "version": "1.0", + "error": { + "code": "INTEGRITY_ERROR", + "surfaceId": "main", + "message": "root" + } + } + } + }, + { + "name": "test_error_response_update_components_unknown_type_strict", + "description": "Emits VALIDATION_FAILED with /components/0/component when component type is unknown in strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "main", + "components": [ + { + "id": "root", + "component": "NonExistentWidget" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "surfaceId": "main", + "path": "/components/0/component", + "rendererError": { + "version": "1.0", + "error": { + "code": "VALIDATION_FAILED", + "surfaceId": "main", + "path": "/components/0/component", + "message": "NonExistentWidget" + } + } + } + }, + { + "name": "test_error_response_update_components_invalid_property_type", + "description": "Emits VALIDATION_FAILED with /components/0/text when a component property has the wrong type.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "main", + "components": [ + { + "id": "root", + "component": "Text", + "text": 12345 + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "surfaceId": "main", + "path": "/components/0/text", + "rendererError": { + "version": "1.0", + "error": { + "code": "VALIDATION_FAILED", + "surfaceId": "main", + "path": "/components/0/text", + "message": "text" + } + } + } + }, + { + "name": "test_error_response_update_components_extra_forbidden_property", + "description": "Emits VALIDATION_FAILED when a component includes an undeclared extra property in strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "main", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Hello", + "unknownExtraProp": true + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "surfaceId": "main", + "rendererError": { + "version": "1.0", + "error": { + "code": "VALIDATION_FAILED", + "surfaceId": "main", + "path": "/components/0/unknownExtraProp", + "message": "unknownExtraProp" + } + } + } + }, + { + "name": "test_error_response_update_components_invalid_uax31_id", + "description": "Emits VALIDATION_FAILED with /components/0/id when a v1.0 component ID violates UAX #31.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "main", + "components": [ + { + "id": "123-invalid", + "component": "Text", + "text": "Hello" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "surfaceId": "main", + "path": "/components/0/id", + "rendererError": { + "version": "1.0", + "error": { + "code": "VALIDATION_FAILED", + "surfaceId": "main", + "path": "/components/0/id", + "message": "123-invalid" + } + } + } + }, + { + "name": "test_error_response_update_components_unallowed_parent", + "description": "Emits UNALLOWED_PARENT with /components/1 when a child component is placed inside a disallowed parent.", + "action": "validate", + "protocolVersion": "1.0", + "catalog": { + "catalogId": "test-comp-catalog", + "protocolVersion": "1.0", + "components": { + "Container": { + "type": "object", + "properties": { + "component": { + "const": "Container" + }, + "children": { + "$ref": "common_types.json#/$defs/ChildList" + } + }, + "required": [ + "component" + ] + }, + "RestrictedItem": { + "type": "object", + "allowedParents": [ + "SpecialList" + ], + "properties": { + "component": { + "const": "RestrictedItem" + } + }, + "required": [ + "component" + ] + } + } + }, + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "test-comp-catalog" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "main", + "components": [ + { + "id": "root", + "component": "Container", + "children": [ + "item1" + ] + }, + { + "id": "item1", + "component": "RestrictedItem" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "code": "UNALLOWED_PARENT", + "surfaceId": "main", + "path": "/components/1", + "details": [ + { + "path": "/components/1", + "code": "UNALLOWED_PARENT", + "surfaceId": "main" + } + ], + "rendererError": { + "version": "1.0", + "error": { + "code": "UNALLOWED_PARENT", + "surfaceId": "main", + "path": "/components/1", + "message": "RestrictedItem" + } + } + } + }, + { + "name": "test_error_response_update_components_unallowed_parent_root", + "description": "Emits UNALLOWED_PARENT with /components/0 when the root component does not allow Surface as parent.", + "action": "validate", + "protocolVersion": "1.0", + "catalog": { + "catalogId": "test-comp-catalog", + "protocolVersion": "1.0", + "components": { + "RestrictedItem": { + "type": "object", + "allowedParents": [ + "SpecialList" + ], + "properties": { + "component": { + "const": "RestrictedItem" + } + }, + "required": [ + "component" + ] + } + } + }, + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "test-comp-catalog" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "main", + "components": [ + { + "id": "root", + "component": "RestrictedItem" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "code": "UNALLOWED_PARENT", + "surfaceId": "main", + "path": "/components/0", + "details": [ + { + "path": "/components/0", + "code": "UNALLOWED_PARENT", + "surfaceId": "main" + } + ], + "rendererError": { + "version": "1.0", + "error": { + "code": "UNALLOWED_PARENT", + "surfaceId": "main", + "path": "/components/0", + "message": "RestrictedItem" + } + } + } + }, + { + "name": "test_error_response_update_components_unallowed_child", + "description": "Emits UNALLOWED_CHILD with /components/0/children/0 when a parent references a disallowed child type.", + "action": "validate", + "protocolVersion": "1.0", + "catalog": { + "catalogId": "test-comp-catalog", + "protocolVersion": "1.0", + "components": { + "StrictList": { + "type": "object", + "allowedChildren": [ + "ListItem" + ], + "properties": { + "component": { + "const": "StrictList" + }, + "children": { + "$ref": "common_types.json#/$defs/ChildList" + } + }, + "required": [ + "component" + ] + }, + "OtherItem": { + "type": "object", + "properties": { + "component": { + "const": "OtherItem" + } + }, + "required": [ + "component" + ] + } + } + }, + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "test-comp-catalog" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "main", + "components": [ + { + "id": "root", + "component": "StrictList", + "children": [ + "item1" + ] + }, + { + "id": "item1", + "component": "OtherItem" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "code": "UNALLOWED_CHILD", + "surfaceId": "main", + "path": "/components/0/children/0", + "details": [ + { + "path": "/components/0/children/0", + "code": "UNALLOWED_CHILD", + "surfaceId": "main" + } + ], + "rendererError": { + "version": "1.0", + "error": { + "code": "UNALLOWED_CHILD", + "surfaceId": "main", + "path": "/components/0/children/0", + "message": "OtherItem" + } + } + } + }, + { + "name": "test_error_response_update_components_circular_reference", + "description": "Emits RECURSION_LIMIT_EXCEEDED when components form a cycle in strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "main", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "child1" + ] + }, + { + "id": "child1", + "component": "Column", + "children": [ + "root" + ] + } + ] + } + } + ], + "expectError": { + "category": "RecursionError", + "surfaceId": "main", + "rendererError": { + "version": "1.0", + "error": { + "code": "RECURSION_LIMIT_EXCEEDED", + "surfaceId": "main", + "message": "root" + } + } + } + }, + { + "name": "test_error_response_update_components_dangling_child_strict", + "description": "Emits INTEGRITY_ERROR when a component references a non-existent child ID in strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "main", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "missing_child" + ] + } + ] + } + } + ], + "expectError": { + "category": "IntegrityError", + "surfaceId": "main", + "rendererError": { + "version": "1.0", + "error": { + "code": "INTEGRITY_ERROR", + "surfaceId": "main", + "message": "missing_child" + } + } + } + }, + { + "name": "test_error_response_update_components_orphaned_component_strict", + "description": "Emits INTEGRITY_ERROR when a component is unreachable from root in strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "main", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Root" + }, + { + "id": "orphan1", + "component": "Text", + "text": "Orphan" + } + ] + } + } + ], + "expectError": { + "category": "IntegrityError", + "surfaceId": "main", + "rendererError": { + "version": "1.0", + "error": { + "code": "INTEGRITY_ERROR", + "surfaceId": "main", + "message": "orphan1" + } + } + } + }, + { + "name": "test_error_response_update_components_buffered_streaming_non_strict", + "description": "Accepts dangling child references and orphaned components without error when strictMode is false.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": false, + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "main", + "components": [ + { + "id": "orphan1", + "component": "Text", + "text": "Buffered child" + }, + { + "id": "root", + "component": "Column", + "children": [ + "orphan1", + "dangling_child" + ] + } + ] + } + } + ] + }, + { + "payload": [ + { + "version": "1.0", + "updateComponents": { + "surfaceId": "main", + "components": [ + { + "id": "dangling_child", + "component": "Text", + "text": "Arrived later" + } + ] + } + } + ] + } + ] + }, + { + "name": "test_error_response_update_data_model_unknown_surface", + "description": "Emits INTEGRITY_ERROR when updateDataModel targets a non-existent surfaceId.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "payload": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "missing_surface", + "path": "/user/name", + "value": "Alice" + } + } + ], + "expectError": { + "category": "IntegrityError", + "surfaceId": "missing_surface", + "rendererError": { + "version": "1.0", + "error": { + "code": "INTEGRITY_ERROR", + "surfaceId": "missing_surface", + "message": "missing_surface" + } + } + } + }, + { + "name": "test_error_response_update_data_model_prototype_pollution", + "description": "Emits DATA_ERROR when updateDataModel targets a forbidden prototype-pollution path segment.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "main", + "path": "/__proto__/polluted", + "value": true + } + } + ], + "expectError": { + "category": "DataError", + "surfaceId": "main", + "rendererError": { + "version": "1.0", + "error": { + "code": "DATA_ERROR", + "surfaceId": "main", + "message": "__proto__" + } + } + } + }, + { + "name": "test_error_response_update_data_model_invalid_json_pointer", + "description": "Emits VALIDATION_FAILED with /path when updateDataModel path is not a valid JSON Pointer.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "main", + "path": "not_a_pointer", + "value": "Alice" + } + } + ], + "expectError": { + "category": "ValidationError", + "surfaceId": "main", + "path": "/path", + "rendererError": { + "version": "1.0", + "error": { + "code": "VALIDATION_FAILED", + "surfaceId": "main", + "path": "/path", + "message": "path" + } + } + } + }, + { + "name": "test_error_response_delete_surface_missing_surface_id", + "description": "Emits VALIDATION_FAILED with surfaceId '' and /surfaceId when deleteSurface omits surfaceId.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "payload": [ + { + "version": "1.0", + "deleteSurface": {} + } + ], + "expectError": { + "category": "ValidationError", + "surfaceId": "", + "path": "/surfaceId", + "rendererError": { + "version": "1.0", + "error": { + "code": "VALIDATION_FAILED", + "surfaceId": "", + "path": "/surfaceId", + "message": "surfaceId" + } + } + } + }, + { + "name": "test_error_response_call_renderer_function_missing_call_function", + "description": "Emits INVALID_FUNCTION_CALL correlated by functionCallId without surfaceId when callFunction is missing.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "payload": [ + { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-err-1" + } + } + ], + "expectError": { + "category": "ValidationError", + "functionCallId": "call-err-1", + "rendererError": { + "version": "1.0", + "error": { + "code": "INVALID_FUNCTION_CALL", + "functionCallId": "call-err-1", + "message": "callFunction" + } + } + } + }, + { + "name": "test_error_response_call_renderer_function_non_object_args", + "description": "Emits INVALID_FUNCTION_CALL correlated by functionCallId when callFunction.args is not an object.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "payload": [ + { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-err-2", + "callFunction": { + "@call": "formatString", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "args": "not_an_object" + } + } + } + ], + "expectError": { + "category": "ValidationError", + "functionCallId": "call-err-2", + "rendererError": { + "version": "1.0", + "error": { + "code": "INVALID_FUNCTION_CALL", + "functionCallId": "call-err-2", + "message": "args" + } + } + } + }, + { + "name": "test_error_response_call_renderer_function_missing_catalog_id", + "description": "Emits INVALID_FUNCTION_CALL correlated by functionCallId when callFunction omits required catalogId.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "payload": [ + { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-err-3", + "callFunction": { + "@call": "formatString", + "args": { + "value": "hello" + } + } + } + } + ], + "expectError": { + "category": "ValidationError", + "functionCallId": "call-err-3", + "rendererError": { + "version": "1.0", + "error": { + "code": "INVALID_FUNCTION_CALL", + "functionCallId": "call-err-3", + "message": "catalogId" + } + } + } + } +] \ No newline at end of file diff --git a/package/__fixtures__/a2ui-conformance/core/expressions.json b/package/__fixtures__/a2ui-conformance/core/expressions.json new file mode 100644 index 0000000000..30733740bb --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/core/expressions.json @@ -0,0 +1,1270 @@ +[ + { + "name": "test_expr_plain_text_is_one_literal", + "description": "A template with no interpolation yields the input unchanged.", + "action": "parse_expression_template", + "input": "hello world", + "expect": [ + "hello world" + ] + }, + { + "name": "test_expr_empty_template", + "description": "An empty template yields no content.", + "action": "parse_expression_template", + "input": "", + "expect": [] + }, + { + "name": "test_expr_relative_path", + "description": "A bare identifier is a relative data binding.", + "action": "parse_expression_template", + "input": "${user}", + "expect": [ + { + "path": "user" + } + ] + }, + { + "name": "test_expr_absolute_path", + "description": "An identifier starting with a slash is an absolute data binding.", + "action": "parse_expression_template", + "input": "value is ${/user/name}", + "expect": [ + "value is ", + { + "path": "/user/name" + } + ] + }, + { + "name": "test_expr_path_with_punctuation", + "description": "Dots, dashes and underscores are part of a path.", + "action": "parse_expression_template", + "input": "${my-path.with_underscores}", + "expect": [ + { + "path": "my-path.with_underscores" + } + ] + }, + { + "name": "test_expr_non_ascii_path", + "description": "Path segments may use any UAX 31 XID_Continue character.", + "action": "parse_expression_template", + "input": "${/użytkownik/größe}", + "expect": [ + { + "path": "/użytkownik/größe" + } + ] + }, + { + "name": "test_expr_error_non_identifier_character_in_path", + "description": "A character outside XID_Continue, such as ½, ends the path.", + "action": "parse_expression_template", + "input": "${a½}", + "expect_error": { + "category": "ParseError", + "message": "Unexpected characters" + } + }, + { + "name": "test_expr_combining_marks_and_supplementary_plane_path", + "description": "Path segments accept combining marks in XID_Continue, including right after a slash or a dot, and supplementary-plane XID_Continue code points such as 𠮷.", + "action": "parse_expression_template", + "input": "${/a/́b/grüße/𠮷.́c}", + "expect": [ + { + "path": "/a/́b/grüße/𠮷.́c" + } + ] + }, + { + "name": "test_expr_error_supplementary_plane_non_identifier_in_path", + "description": "A supplementary-plane character outside XID_Continue, such as 🚀, ends the path.", + "action": "parse_expression_template", + "input": "${a🚀}", + "expect_error": { + "category": "ParseError", + "message": "Unexpected characters" + } + }, + { + "name": "test_expr_literal_between_bindings", + "description": "Literal text surrounding interpolations is preserved.", + "action": "parse_expression_template", + "input": "a ${x} b ${y} c", + "expect": [ + "a ", + { + "path": "x" + }, + " b ", + { + "path": "y" + }, + " c" + ] + }, + { + "name": "test_expr_adjacent_bindings", + "description": "Interpolations with nothing between them stay separate parts.", + "action": "parse_expression_template", + "input": "${x}${y}", + "expect": [ + { + "path": "x" + }, + { + "path": "y" + } + ] + }, + { + "name": "test_expr_single_quoted_string", + "description": "Single-quoted literals parse to their contents.", + "action": "parse_expression_template", + "input": "${'hello world'}", + "expect": [ + "hello world" + ] + }, + { + "name": "test_expr_double_quoted_string", + "description": "Double-quoted literals parse to their contents.", + "action": "parse_expression_template", + "input": "${\"hello world\"}", + "expect": [ + "hello world" + ] + }, + { + "name": "test_expr_string_escapes", + "description": "Backslash escapes inside string literals are unescaped.", + "action": "parse_expression_template", + "input": "${'line\\nbreak'}", + "expect": [ + "line\nbreak" + ] + }, + { + "name": "test_expr_string_escaped_quote", + "description": "An escaped quote does not terminate the literal.", + "action": "parse_expression_template", + "input": "${'it\\'s'}", + "expect": [ + "it's" + ] + }, + { + "name": "test_expr_integer_literal", + "description": "Integer literals parse to numbers.", + "action": "parse_expression_template", + "input": "${42}", + "expect": [ + 42 + ] + }, + { + "name": "test_expr_decimal_literal", + "description": "Decimal literals parse to numbers.", + "action": "parse_expression_template", + "input": "${1.5}", + "expect": [ + 1.5 + ] + }, + { + "name": "test_expr_negative_integer_literal", + "description": "Negative integer literals parse to negative numbers.", + "action": "parse_expression_template", + "input": "${-42}", + "expect": [ + -42 + ] + }, + { + "name": "test_expr_negative_decimal_literal", + "description": "Negative decimal literals parse to negative numbers.", + "action": "parse_expression_template", + "input": "${-3.14}", + "expect": [ + -3.14 + ] + }, + { + "name": "test_expr_exponential_notation_positive", + "description": "Scientific/exponential notation parses to numbers.", + "action": "parse_expression_template", + "input": "${1e5}", + "expect": [ + 100000 + ] + }, + { + "name": "test_expr_exponential_notation_negative_exponent", + "description": "Scientific notation with negative exponent parses to floating point numbers.", + "action": "parse_expression_template", + "input": "${1.5e-3}", + "expect": [ + 0.0015 + ] + }, + { + "name": "test_expr_exponential_notation_capital_e", + "description": "Scientific notation with capital E and plus sign parses correctly.", + "action": "parse_expression_template", + "input": "${2.5E+4}", + "expect": [ + 25000 + ] + }, + { + "name": "test_expr_leading_zeros", + "description": "Leading zeros do not change the value.", + "action": "parse_expression_template", + "input": "${007}", + "expect": [ + 7 + ] + }, + { + "name": "test_expr_boolean_literals", + "description": "The true and false keywords parse to booleans.", + "action": "parse_expression_template", + "input": "${true} ${false}", + "expect": [ + true, + " ", + false + ] + }, + { + "name": "test_expr_keyword_prefix_is_a_path", + "description": "An identifier that merely starts with a keyword is a path.", + "action": "parse_expression_template", + "input": "${trueish}", + "expect": [ + { + "path": "trueish" + } + ] + }, + { + "name": "test_expr_keyword_followed_by_non_ascii_letter_is_a_path", + "description": "A non-ASCII XID_Continue character after a keyword makes it part of a path.", + "action": "parse_expression_template", + "input": "${nullß}", + "expect": [ + { + "path": "nullß" + } + ] + }, + { + "name": "test_expr_error_keyword_followed_by_non_identifier_character", + "description": "A character outside XID_Continue, such as ½, does not extend a keyword.", + "action": "parse_expression_template", + "input": "${true½}", + "expect_error": { + "category": "ParseError", + "message": "Unexpected characters" + } + }, + { + "name": "test_expr_function_call_no_args", + "description": "A function call with no arguments parses to an empty args map.", + "action": "parse_expression_template", + "input": "${now()}", + "expect": [ + { + "call": "now", + "args": {}, + "returnType": "any" + } + ] + }, + { + "name": "test_expr_function_call_named_args", + "description": "Function arguments are named and comma separated.", + "action": "parse_expression_template", + "input": "sum is ${add(a: 10, b: 20)}", + "expect": [ + "sum is ", + { + "call": "add", + "args": { + "a": 10, + "b": 20 + }, + "returnType": "any" + } + ] + }, + { + "name": "test_expr_non_ascii_function_and_argument_names", + "description": "Function and argument names may use any UAX 31 XID_Continue character.", + "action": "parse_expression_template", + "input": "${größe(wartość: 1, 値: 2)}", + "expect": [ + { + "call": "größe", + "args": { + "wartość": 1, + "値": 2 + }, + "returnType": "any" + } + ] + }, + { + "name": "test_expr_error_non_identifier_character_in_argument_name", + "description": "A character outside XID_Continue, such as ½, is not part of an argument name.", + "action": "parse_expression_template", + "input": "${add(½: 1)}", + "expect_error": { + "category": "ParseError", + "message": "after argument name" + } + }, + { + "name": "test_expr_function_call_negative_args", + "description": "Function arguments may be negative numeric literals.", + "action": "parse_expression_template", + "input": "${add(a: -5, b: 10)}", + "expect": [ + { + "call": "add", + "args": { + "a": -5, + "b": 10 + }, + "returnType": "any" + } + ] + }, + { + "name": "test_expr_function_call_mixed_arg_types", + "description": "Arguments may be literals, paths or nested calls.", + "action": "parse_expression_template", + "input": "${format(value: /amount, suffix: ' USD', fallback: zero())}", + "expect": [ + { + "call": "format", + "args": { + "value": { + "path": "/amount" + }, + "suffix": " USD", + "fallback": { + "call": "zero", + "args": {}, + "returnType": "any" + } + }, + "returnType": "any" + } + ] + }, + { + "name": "test_expr_nested_function_calls", + "description": "Function arguments may themselves be function calls.", + "action": "parse_expression_template", + "input": "${outer(a: inner(b: 1))}", + "expect": [ + { + "call": "outer", + "args": { + "a": { + "call": "inner", + "args": { + "b": 1 + }, + "returnType": "any" + } + }, + "returnType": "any" + } + ] + }, + { + "name": "test_expr_nested_interpolation", + "description": "An interpolation inside an interpolation resolves to the inner expression.", + "action": "parse_expression_template", + "input": "${${'hello'}}", + "expect": [ + "hello" + ] + }, + { + "name": "test_expr_whitespace_around_expression", + "description": "Whitespace inside the interpolation braces is ignored.", + "action": "parse_expression_template", + "input": "${ /user/name }", + "expect": [ + { + "path": "/user/name" + } + ] + }, + { + "name": "test_expr_escaped_marker_is_literal", + "description": "An escaped marker resolves to a literal \"${\" and is not interpolated.", + "action": "parse_expression_template", + "input": "\\${not_interpolated}", + "expect": [ + "${not_interpolated}" + ] + }, + { + "name": "test_expr_escaped_marker_between_text", + "description": "An escaped marker inside surrounding text stays literal.", + "action": "parse_expression_template", + "input": "before \\${x} after", + "expect": [ + "before ${x} after" + ] + }, + { + "name": "test_expr_escaped_and_real_marker", + "description": "An escaped marker and a real interpolation can appear in one template.", + "action": "parse_expression_template", + "input": "\\${literal} ${real}", + "expect": [ + "${literal} ", + { + "path": "real" + } + ] + }, + { + "name": "test_expr_error_unclosed_interpolation", + "description": "A template whose interpolation is never closed is rejected.", + "action": "parse_expression_template", + "input": "hello ${world", + "expect_error": { + "category": "ParseError", + "message": "Unclosed interpolation" + } + }, + { + "name": "test_expr_error_missing_closing_paren", + "description": "A function call whose parentheses are unbalanced is rejected.", + "action": "parse_expression_template", + "input": "${add(a: 1, b: 2}", + "expect_error": { + "category": "ParseError", + "message": "after function arguments" + } + }, + { + "name": "test_expr_error_missing_arg_colon", + "description": "A function argument without a colon is rejected.", + "action": "parse_expression_template", + "input": "${add(a 1)}", + "expect_error": { + "category": "ParseError", + "message": "after argument name" + } + }, + { + "name": "test_expr_error_trailing_characters", + "description": "Characters after a complete expression are rejected.", + "action": "parse_expression_template", + "input": "${true false}", + "expect_error": { + "category": "ParseError", + "message": "Unexpected characters" + } + }, + { + "name": "test_expr_number_with_trailing_point", + "description": "A trailing decimal point is accepted, as every client implementation does today. The grammar is stated in each parser rather than delegated to the platform's number parser, so this stays true.\n", + "action": "parse_expression_template", + "input": "${1.}", + "expect": [ + 1 + ] + }, + { + "name": "test_expr_error_invalid_number_two_points", + "description": "A number literal with two decimal points is rejected as a protocol error.", + "action": "parse_expression_template", + "input": "${1.2.3}", + "expect_error": { + "category": "ParseError", + "message": "Invalid number literal" + } + }, + { + "name": "test_expr_leading_dot_decimal_literal", + "description": "A decimal literal may omit the digits before the decimal point.", + "action": "parse_expression_template", + "input": "${.5}", + "expect": [ + 0.5 + ] + }, + { + "name": "test_expr_negative_leading_dot_decimal_literal", + "description": "A leading-dot decimal literal may carry a minus sign.", + "action": "parse_expression_template", + "input": "${-.5}", + "expect": [ + -0.5 + ] + }, + { + "name": "test_expr_positive_leading_dot_decimal_literal", + "description": "A leading-dot decimal literal may carry a plus sign.", + "action": "parse_expression_template", + "input": "${+.5}", + "expect": [ + 0.5 + ] + }, + { + "name": "test_expr_leading_dot_exponential_notation", + "description": "A leading-dot decimal literal may carry an exponent.", + "action": "parse_expression_template", + "input": "${.5e2}", + "expect": [ + 50 + ] + }, + { + "name": "test_expr_function_call_leading_dot_args", + "description": "Function arguments may be signed leading-dot decimal literals.", + "action": "parse_expression_template", + "input": "${scale(by: -.5, offset: .25)}", + "expect": [ + { + "call": "scale", + "args": { + "by": -0.5, + "offset": 0.25 + }, + "returnType": "any" + } + ] + }, + { + "name": "test_expr_leading_dot_path_is_not_a_number", + "description": "A leading dot starts a number only when a digit follows it. Otherwise the token stays a path, as does a sign followed by a dot and no digit.\n", + "action": "parse_expression_template", + "input": "${.foo} ${-.}", + "expect": [ + { + "path": ".foo" + }, + " ", + { + "path": "-." + } + ] + }, + { + "name": "test_expr_error_leading_dot_number_two_points", + "description": "A leading-dot number literal with a second decimal point is rejected.", + "action": "parse_expression_template", + "input": "${.5.5}", + "expect_error": { + "category": "ParseError", + "message": "Invalid number literal" + } + }, + { + "name": "test_expr_error_leading_dot_number_empty_exponent", + "description": "A signed leading-dot number literal with an empty exponent is rejected.", + "action": "parse_expression_template", + "input": "${-.5e}", + "expect_error": { + "category": "ParseError", + "message": "Invalid number literal" + } + }, + { + "name": "test_expr_positive_integer_literal", + "description": "An integer literal with a leading plus sign is parsed as a number.", + "action": "parse_expression_template", + "input": "${+1}", + "expect": [ + 1 + ] + }, + { + "name": "test_expr_hyphen_inside_path", + "description": "A hyphen after an identifier character stays part of the path token.", + "action": "parse_expression_template", + "input": "${a-1}", + "expect": [ + { + "path": "a-1" + } + ] + }, + { + "name": "test_expr_hyphen_before_non_digit_is_a_path", + "description": "A leading hyphen not followed by a digit or a dot-and-digit stays a path.", + "action": "parse_expression_template", + "input": "${-a}", + "expect": [ + { + "path": "-a" + } + ] + }, + { + "name": "test_expr_error_exponent_without_digits", + "description": "A number literal whose exponent has no digits is rejected.", + "action": "parse_expression_template", + "input": "${1e}", + "expect_error": { + "category": "ParseError", + "message": "Invalid number literal" + } + }, + { + "name": "test_expr_max_safe_integer_literals", + "description": "Integer literals at ±(2^53 - 1) are returned with exact value in every engine.", + "action": "parse_expression_template", + "input": "${9007199254740991} ${-9007199254740991}", + "expect": [ + 9007199254740991, + " ", + -9007199254740991 + ] + }, + { + "name": "test_expr_largest_exponent_literal", + "description": "A number literal near the top of the double range is accepted.", + "action": "parse_expression_template", + "input": "${1e308}", + "expect": [ + 1e+308 + ] + }, + { + "name": "test_expr_exponent_underflow_is_zero", + "description": "A number literal that underflows the double range evaluates to zero.", + "action": "parse_expression_template", + "input": "${1e-999}", + "expect": [ + 0 + ] + }, + { + "name": "test_expr_error_exponent_out_of_range", + "description": "A positive exponent literal that overflows the double range is rejected.", + "action": "parse_expression_template", + "input": "${1e999}", + "expect_error": { + "category": "ParseError", + "message": "out of range" + } + }, + { + "name": "test_expr_error_negative_exponent_out_of_range", + "description": "A negative number literal that overflows the double range is rejected.", + "action": "parse_expression_template", + "input": "${-1e999}", + "expect_error": { + "category": "ParseError", + "message": "out of range" + } + }, + { + "name": "test_expr_error_integer_out_of_range", + "description": "An integer literal of 10^309 is outside the double range and is rejected.", + "action": "parse_expression_template", + "input": "${1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000}", + "expect_error": { + "category": "ParseError", + "message": "out of range" + } + }, + { + "name": "test_expression_parser_string_interpolation_multiple_bindings", + "description": "Verifies ${/path} interpolation across multiple variables and static text.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "basic", + "dataModel": { + "user": { + "name": "Alice", + "unreadCount": 5 + } + }, + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "formatString", + "args": { + "value": "Hello, ${/user/name}! You have ${/user/unreadCount} unread messages." + } + } + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "main": { + "components": { + "root": { + "component": "Text", + "text": "Hello, Alice! You have 5 unread messages." + } + } + } + } + } + }, + { + "name": "test_expression_parser_escaped_interpolation_sequence", + "description": "Verifies \\${...} renders literal ${...} without evaluation.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "basic", + "dataModel": { + "amount": 100 + }, + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "formatString", + "args": { + "value": "Price is \\${/amount} USD" + } + } + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "main": { + "components": { + "root": { + "component": "Text", + "text": "Price is ${/amount} USD" + } + } + } + } + } + }, + { + "name": "test_expression_parser_nested_function_call", + "description": "Verifies evaluating nested function calls with named arguments inside ${...}.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "basic", + "dataModel": { + "price": 49.99 + }, + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "formatString", + "args": { + "value": "Total: ${formatCurrency(value: /price, currency: 'USD')}" + } + } + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "main": { + "components": { + "root": { + "component": "Text", + "text": "Total: $49.99" + } + } + } + } + } + }, + { + "name": "test_expression_parser_data_type_coercion_matrix", + "description": "Verifies expression string coercion for booleans, numbers, and missing values.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "basic", + "dataModel": { + "isActive": true, + "score": 98.5 + }, + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "formatString", + "args": { + "value": "Active: ${/isActive}, Score: ${/score}, Missing: ${/nonexistent}" + } + } + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "main": { + "components": { + "root": { + "component": "Text", + "text": "Active: true, Score: 98.5, Missing: " + } + } + } + } + } + }, + { + "name": "test_expression_parser_syntax_error_unclosed_brace", + "description": "Verifies error when expression contains unclosed interpolation brace.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "basic", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "formatString", + "args": { + "value": "Hello ${/unclosed" + } + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Unclosed interpolation" + } + } + ] + }, + { + "name": "test_expression_parser_max_depth_exceeded_error", + "description": "Verifies error when expression nesting exceeds the maximum recursion depth. The value nests 101 interpolations, one past the limit of 100 that every engine applies.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "basic", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "formatString", + "args": { + "value": "${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${1}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}" + } + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Max recursion depth" + } + } + ] + }, + { + "name": "test_expr_error_max_depth_nested_interpolations", + "description": "An interpolation nested past the recursion limit of 100 is rejected. This template nests 101 deep.", + "action": "parse_expression_template", + "input": "${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${1}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}", + "expect_error": { + "category": "ParseError", + "message": "Max recursion depth" + } + }, + { + "name": "test_expr_error_max_depth_function_arguments", + "description": "A function-call argument counts toward the recursion limit, not just a nested interpolation. This template nests 101 calls, one past the limit.", + "action": "parse_expression_template", + "input": "${f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: f(a: 1)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))}", + "expect_error": { + "category": "ParseError", + "message": "Max recursion depth" + } + }, + { + "name": "test_expr_null_keyword_produces_no_part", + "description": "The null keyword inside an interpolation resolves to empty content.", + "action": "parse_expression_template", + "input": "a${null}b", + "expect": [ + "ab" + ] + }, + { + "name": "test_expr_null_keyword_argument_is_empty_string", + "description": "The null keyword passed as a function argument is an empty string.", + "action": "parse_expression_template", + "input": "${f(x: null)}", + "expect": [ + { + "call": "f", + "args": { + "x": "" + }, + "returnType": "any" + } + ] + }, + { + "name": "test_expr_nested_path_interpolation", + "description": "An interpolation nested inside another interpolation resolves to the inner path binding.", + "action": "parse_expression_template", + "input": "${${nested}}", + "expect": [ + { + "path": "nested" + } + ] + }, + { + "name": "test_expr_function_call_double_quoted_arg", + "description": "Function arguments may be double-quoted string literals.", + "action": "parse_expression_template", + "input": "${upper(text: \"hello\")}", + "expect": [ + { + "call": "upper", + "args": { + "text": "hello" + }, + "returnType": "any" + } + ] + }, + { + "name": "test_expr_string_escapes_tab_cr_backslash", + "description": "Tab, carriage-return, and backslash escapes inside string literals are unescaped.", + "action": "parse_expression_template", + "input": "${'a\\tb\\rc\\\\d'}", + "expect": [ + "a\tb\rc\\d" + ] + }, + { + "name": "test_expr_string_unknown_escape_keeps_character", + "description": "An unrecognized backslash escape inside a string literal yields the escaped character.", + "action": "parse_expression_template", + "input": "${'a\\qb'}", + "expect": [ + "aqb" + ] + }, + { + "name": "test_expr_negative_number_with_trailing_point", + "description": "A negative number literal may end with a trailing decimal point.", + "action": "parse_expression_template", + "input": "${-42.}", + "expect": [ + -42 + ] + }, + { + "name": "test_expr_positive_zero_with_trailing_point", + "description": "A positive zero literal with a trailing decimal point evaluates to zero.", + "action": "parse_expression_template", + "input": "${+0.}", + "expect": [ + 0 + ] + }, + { + "name": "test_expr_negative_zero_literal", + "description": "A negative zero integer literal evaluates to zero.", + "action": "parse_expression_template", + "input": "${-0}", + "expect": [ + 0 + ] + }, + { + "name": "test_expr_negative_exponential_notation", + "description": "A negative number literal in exponential notation parses to a negative number.", + "action": "parse_expression_template", + "input": "${-2e3}", + "expect": [ + -2000 + ] + }, + { + "name": "test_expr_negative_leading_dot_exponential_notation", + "description": "A negative leading-dot decimal literal with a negative exponent parses to a number.", + "action": "parse_expression_template", + "input": "${-.5E-1}", + "expect": [ + -0.05 + ] + }, + { + "name": "test_expr_function_call_signed_and_exponential_args", + "description": "Function arguments may be negative exponential or positive signed number literals.", + "action": "parse_expression_template", + "input": "${clamp(value: -1.5e2, max: +10)}", + "expect": [ + { + "call": "clamp", + "args": { + "value": -150, + "max": 10 + }, + "returnType": "any" + } + ] + }, + { + "name": "test_expr_dot_slash_relative_path", + "description": "A token starting with './' is parsed as a path.", + "action": "parse_expression_template", + "input": "${./x}", + "expect": [ + { + "path": "./x" + } + ] + }, + { + "name": "test_expr_leading_dot_before_exponent_letter_is_a_path", + "description": "A leading dot followed by 'e' and digits is a path, not a number literal.", + "action": "parse_expression_template", + "input": "${.e5}", + "expect": [ + { + "path": ".e5" + } + ] + }, + { + "name": "test_expr_dot_digit_inside_path", + "description": "A dot followed by a digit after an identifier character stays part of the path.", + "action": "parse_expression_template", + "input": "${a.5}", + "expect": [ + { + "path": "a.5" + } + ] + }, + { + "name": "test_expr_slash_dot_digit_inside_path", + "description": "A segment starting with a dot and digit inside a slash-delimited path stays a path.", + "action": "parse_expression_template", + "input": "${/items/.5}", + "expect": [ + { + "path": "/items/.5" + } + ] + }, + { + "name": "test_expr_slash_hyphen_digit_inside_path", + "description": "A segment starting with a hyphen and digit inside a slash-delimited path stays a path.", + "action": "parse_expression_template", + "input": "${/items/-1}", + "expect": [ + { + "path": "/items/-1" + } + ] + }, + { + "name": "test_expr_parent_directory_relative_path", + "description": "A token starting with '../' is parsed as a path.", + "action": "parse_expression_template", + "input": "${../x}", + "expect": [ + { + "path": "../x" + } + ] + }, + { + "name": "test_expr_function_call_trailing_comma", + "description": "A trailing comma after the last function argument is accepted.", + "action": "parse_expression_template", + "input": "${add(a: 1, b: 2,)}", + "expect": [ + { + "call": "add", + "args": { + "a": 1, + "b": 2 + }, + "returnType": "any" + } + ] + }, + { + "name": "test_expr_error_leading_dot_exponent_sign_without_digits", + "description": "A leading-dot number literal whose exponent has a sign but no digits is rejected.", + "action": "parse_expression_template", + "input": "${.5e+}", + "expect_error": { + "category": "ParseError", + "message": "Invalid number literal" + } + }, + { + "name": "test_expr_error_exponent_plus_without_digits", + "description": "A number literal whose exponent has a plus sign but no digits is rejected.", + "action": "parse_expression_template", + "input": "${1e+}", + "expect_error": { + "category": "ParseError", + "message": "Invalid number literal" + } + }, + { + "name": "test_expr_error_exponent_minus_without_digits", + "description": "A number literal whose exponent has a minus sign but no digits is rejected.", + "action": "parse_expression_template", + "input": "${1E-}", + "expect_error": { + "category": "ParseError", + "message": "Invalid number literal" + } + }, + { + "name": "test_expr_error_trailing_characters_after_number", + "description": "Identifier characters immediately following a number literal are rejected.", + "action": "parse_expression_template", + "input": "${-1x}", + "expect_error": { + "category": "ParseError", + "message": "Unexpected characters" + } + }, + { + "name": "test_expr_error_mantissa_overflow_out_of_range", + "description": "A number literal whose mantissa overflows the double range at 10^308 is rejected.", + "action": "parse_expression_template", + "input": "${2e308}", + "expect_error": { + "category": "ParseError", + "message": "out of range" + } + }, + { + "name": "test_expr_error_space_separated_identifiers", + "description": "Space-separated identifiers inside an interpolation are rejected.", + "action": "parse_expression_template", + "input": "${a b}", + "expect_error": { + "category": "ParseError", + "message": "Unexpected characters" + } + }, + { + "name": "test_expr_error_positional_function_arg", + "description": "A positional function argument without a name and colon is rejected.", + "action": "parse_expression_template", + "input": "${f(1)}", + "expect_error": { + "category": "ParseError", + "message": "after argument name" + } + }, + { + "name": "test_expr_max_depth_nested_interpolations", + "description": "An interpolation nested right at the recursion limit of 100 is accepted.", + "action": "parse_expression_template", + "input": "${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${${x}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}", + "expect": [ + { + "path": "x" + } + ] + } +] \ No newline at end of file diff --git a/package/__fixtures__/a2ui-conformance/core/functions.json b/package/__fixtures__/a2ui-conformance/core/functions.json new file mode 100644 index 0000000000..9a6d3a17bb --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/core/functions.json @@ -0,0 +1,1286 @@ +[ + { + "name": "test_function_format_date_tr35_tokens", + "description": "Verifies formatDate TR35 token expansion (yyyy-MM-dd).", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "basic", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "formatDate", + "args": { + "value": "2026-09-04T12:00:00Z", + "format": "yyyy-MM-dd" + } + } + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "main": { + "components": { + "root": { + "component": "Text", + "text": "2026-09-04" + } + } + } + } + } + }, + { + "name": "test_function_format_currency_locale_and_symbol", + "description": "Verifies formatCurrency with currency symbol, decimals, and grouping separators.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "basic", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "formatCurrency", + "args": { + "value": 1234.5, + "currency": "USD", + "decimals": 2 + } + } + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "main": { + "components": { + "root": { + "component": "Text", + "text": "$1,234.50" + } + } + } + } + } + }, + { + "name": "test_function_pluralize_categories", + "description": "Verifies pluralize resolving CLDR categories (one, other).", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "basic", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "t1", + "t2" + ] + }, + { + "id": "t1", + "component": "Text", + "text": { + "@call": "pluralize", + "args": { + "value": 1, + "one": "1 item", + "other": "items" + } + } + }, + { + "id": "t2", + "component": "Text", + "text": { + "@call": "pluralize", + "args": { + "value": 5, + "one": "1 item", + "other": "5 items" + } + } + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "main": { + "components": { + "t1": { + "component": "Text", + "text": "1 item" + }, + "t2": { + "component": "Text", + "text": "5 items" + } + } + } + } + } + }, + { + "name": "test_function_logical_and_or_not", + "description": "Verifies and, or, and not logical function evaluations.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "basic", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "t_and", + "t_or", + "t_not" + ] + }, + { + "id": "t_and", + "component": "Text", + "text": { + "@call": "and", + "args": { + "values": [ + true, + false + ] + } + } + }, + { + "id": "t_or", + "component": "Text", + "text": { + "@call": "or", + "args": { + "values": [ + true, + false + ] + } + } + }, + { + "id": "t_not", + "component": "Text", + "text": { + "@call": "not", + "args": { + "value": false + } + } + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "main": { + "components": { + "t_and": { + "component": "Text", + "text": false + }, + "t_or": { + "component": "Text", + "text": true + }, + "t_not": { + "component": "Text", + "text": true + } + } + } + } + } + }, + { + "name": "test_function_invalid_argument_schema_error", + "description": "Verifies ValidationError when a function is invoked with an argument of the wrong type.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "basic", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "formatNumber", + "args": { + "value": "not_a_number" + } + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError" + } + } + ] + }, + { + "name": "test_evaluate_function_required_string_valid", + "description": "Verifies required returns true for non-empty string.", + "action": "evaluate_function", + "function": "required", + "args": { + "value": "hello" + }, + "expect": true + }, + { + "name": "test_evaluate_function_required_string_empty", + "description": "Verifies required returns false for empty string.", + "action": "evaluate_function", + "function": "required", + "args": { + "value": "" + }, + "expect": false + }, + { + "name": "test_evaluate_function_required_null", + "description": "Verifies required returns false for null.", + "action": "evaluate_function", + "function": "required", + "args": { + "value": null + }, + "expect": false + }, + { + "name": "test_evaluate_function_required_array_valid", + "description": "Verifies required returns true for non-empty array.", + "action": "evaluate_function", + "function": "required", + "args": { + "value": [ + 1, + 2 + ] + }, + "expect": true + }, + { + "name": "test_evaluate_function_required_array_empty", + "description": "Verifies required returns false for empty array.", + "action": "evaluate_function", + "function": "required", + "args": { + "value": [] + }, + "expect": false + }, + { + "name": "test_evaluate_function_required_object_empty", + "description": "Verifies required returns true for empty object (D20).", + "action": "evaluate_function", + "function": "required", + "args": { + "value": {} + }, + "expect": true + }, + { + "name": "test_evaluate_function_regex_match", + "description": "Verifies regex returns true on matching string.", + "action": "evaluate_function", + "function": "regex", + "args": { + "pattern": "^[0-9]+$", + "value": "12345" + }, + "expect": true + }, + { + "name": "test_evaluate_function_regex_mismatch", + "description": "Verifies regex returns false on non-matching string.", + "action": "evaluate_function", + "function": "regex", + "args": { + "pattern": "^[0-9]+$", + "value": "abc" + }, + "expect": false + }, + { + "name": "test_evaluate_function_regex_invalid_pattern", + "description": "Verifies regex raises ExpressionError on invalid regex pattern (D21).", + "action": "evaluate_function", + "function": "regex", + "args": { + "pattern": "[invalid(", + "value": "abc" + }, + "expectError": { + "category": "ExpressionError" + } + }, + { + "name": "test_evaluate_function_email_valid", + "description": "Verifies email returns true on standard email address.", + "action": "evaluate_function", + "function": "email", + "args": { + "value": "user@example.com" + }, + "expect": true + }, + { + "name": "test_evaluate_function_email_invalid_no_domain", + "description": "Verifies email returns false on missing domain.", + "action": "evaluate_function", + "function": "email", + "args": { + "value": "invalid-email" + }, + "expect": false + }, + { + "name": "test_evaluate_function_email_invalid_single_char_tld", + "description": "Verifies email returns false on single-character TLD (D22).", + "action": "evaluate_function", + "function": "email", + "args": { + "value": "test@test.c" + }, + "expect": false + }, + { + "name": "test_evaluate_function_length_string_valid", + "description": "Verifies length returns true when string is within bounds.", + "action": "evaluate_function", + "function": "length", + "args": { + "value": "hello", + "min": 3, + "max": 10 + }, + "expect": true + }, + { + "name": "test_evaluate_function_length_string_too_short", + "description": "Verifies length returns false when string is shorter than min.", + "action": "evaluate_function", + "function": "length", + "args": { + "value": "hi", + "min": 3 + }, + "expect": false + }, + { + "name": "test_evaluate_function_length_string_too_long", + "description": "Verifies length returns false when string is longer than max.", + "action": "evaluate_function", + "function": "length", + "args": { + "value": "hello world", + "max": 5 + }, + "expect": false + }, + { + "name": "test_evaluate_function_length_array_count_valid", + "description": "Verifies length counts array elements, not character length (D24).", + "action": "evaluate_function", + "function": "length", + "args": { + "value": [ + "a", + "b", + "c" + ], + "min": 2, + "max": 4 + }, + "expect": true + }, + { + "name": "test_evaluate_function_length_array_count_too_short", + "description": "Verifies length returns false when array has fewer elements than min (D24).", + "action": "evaluate_function", + "function": "length", + "args": { + "value": [ + "a" + ], + "min": 2 + }, + "expect": false + }, + { + "name": "test_evaluate_function_length_null_value", + "description": "Verifies length counts a null value, such as an unset field, as length 0.", + "protocolVersion": "v0.9", + "action": "evaluate_function", + "function": "length", + "args": { + "value": null, + "max": 10 + }, + "expect": true + }, + { + "name": "test_evaluate_function_length_null_value_too_short", + "description": "Verifies length rejects a null value when min is greater than 0.", + "protocolVersion": "v0.9", + "action": "evaluate_function", + "function": "length", + "args": { + "value": null, + "min": 1 + }, + "expect": false + }, + { + "name": "test_evaluate_function_numeric_valid_float", + "description": "Verifies numeric returns true for float within bounds.", + "action": "evaluate_function", + "function": "numeric", + "args": { + "value": 42.5, + "min": 0, + "max": 100 + }, + "expect": true + }, + { + "name": "test_evaluate_function_numeric_valid_string", + "description": "Verifies numeric returns true for valid numeric string.", + "action": "evaluate_function", + "function": "numeric", + "args": { + "value": "42.5", + "min": 0, + "max": 100 + }, + "expect": true + }, + { + "name": "test_evaluate_function_numeric_too_small", + "description": "Verifies numeric returns false when value is below min.", + "action": "evaluate_function", + "function": "numeric", + "args": { + "value": -5, + "min": 0 + }, + "expect": false + }, + { + "name": "test_evaluate_function_numeric_too_large", + "description": "Verifies numeric returns false when value is above max.", + "action": "evaluate_function", + "function": "numeric", + "args": { + "value": 150, + "max": 100 + }, + "expect": false + }, + { + "name": "test_evaluate_function_numeric_unparseable_string", + "description": "Verifies numeric returns false for unparseable string without raising (D23).", + "action": "evaluate_function", + "function": "numeric", + "args": { + "value": "not_a_number" + }, + "expect": false + }, + { + "name": "test_evaluate_function_and_all_true", + "description": "Verifies and returns true when all values are true.", + "action": "evaluate_function", + "function": "and", + "args": { + "values": [ + true, + true + ] + }, + "expect": true + }, + { + "name": "test_evaluate_function_and_one_false", + "description": "Verifies and returns false when any value is false.", + "action": "evaluate_function", + "function": "and", + "args": { + "values": [ + true, + false + ] + }, + "expect": false + }, + { + "name": "test_evaluate_function_and_truthiness_coercion_true", + "description": "Verifies and truthiness coercion on non-boolean values (D26).", + "action": "evaluate_function", + "function": "and", + "args": { + "values": [ + 1, + "non-empty" + ] + }, + "expect": true + }, + { + "name": "test_evaluate_function_and_truthiness_coercion_false", + "description": "Verifies and truthiness coercion with a falsy value (D26).", + "action": "evaluate_function", + "function": "and", + "args": { + "values": [ + 1, + "" + ] + }, + "expect": false + }, + { + "name": "test_evaluate_function_and_fewer_than_two_values_error", + "description": "Verifies and requires at least 2 values per schema (D25).", + "action": "evaluate_function", + "function": "and", + "args": { + "values": [ + true + ] + }, + "expectError": { + "category": "ExpressionError" + } + }, + { + "name": "test_evaluate_function_or_all_false", + "description": "Verifies or returns false when all values are false.", + "action": "evaluate_function", + "function": "or", + "args": { + "values": [ + false, + false + ] + }, + "expect": false + }, + { + "name": "test_evaluate_function_or_one_true", + "description": "Verifies or returns true when at least one value is true.", + "action": "evaluate_function", + "function": "or", + "args": { + "values": [ + false, + true + ] + }, + "expect": true + }, + { + "name": "test_evaluate_function_or_truthiness_coercion_true", + "description": "Verifies or truthiness coercion on non-boolean values (D26).", + "action": "evaluate_function", + "function": "or", + "args": { + "values": [ + 0, + "yes" + ] + }, + "expect": true + }, + { + "name": "test_evaluate_function_or_fewer_than_two_values_error", + "description": "Verifies or requires at least 2 values per schema (D25).", + "action": "evaluate_function", + "function": "or", + "args": { + "values": [ + false + ] + }, + "expectError": { + "category": "ExpressionError" + } + }, + { + "name": "test_evaluate_function_not_true", + "description": "Verifies not returns false for true.", + "action": "evaluate_function", + "function": "not", + "args": { + "value": true + }, + "expect": false + }, + { + "name": "test_evaluate_function_not_false", + "description": "Verifies not returns true for false.", + "action": "evaluate_function", + "function": "not", + "args": { + "value": false + }, + "expect": true + }, + { + "name": "test_evaluate_function_not_truthiness_falsy", + "description": "Verifies not truthiness coercion for falsy value (D26).", + "action": "evaluate_function", + "function": "not", + "args": { + "value": "" + }, + "expect": true + }, + { + "name": "test_evaluate_function_not_truthiness_truthy", + "description": "Verifies not truthiness coercion for truthy value (D26).", + "action": "evaluate_function", + "function": "not", + "args": { + "value": "hello" + }, + "expect": false + }, + { + "name": "test_evaluate_function_and_validation_result_operands", + "description": "Verifies and reads a failing ValidationResult operand as false in v1.0.", + "protocolVersion": "1.0", + "action": "evaluate_function", + "function": "and", + "args": { + "values": [ + { + "valid": true + }, + { + "valid": false, + "message": "Required", + "code": "required" + } + ] + }, + "expect": false + }, + { + "name": "test_evaluate_function_or_validation_result_operands", + "description": "Verifies or over two failing ValidationResult operands is false in v1.0.", + "protocolVersion": "1.0", + "action": "evaluate_function", + "function": "or", + "args": { + "values": [ + { + "valid": false + }, + { + "valid": false + } + ] + }, + "expect": false + }, + { + "name": "test_evaluate_function_not_validation_result_operand", + "description": "Verifies not inverts the validity of a ValidationResult operand in v1.0.", + "protocolVersion": "1.0", + "action": "evaluate_function", + "function": "not", + "args": { + "value": { + "valid": false + } + }, + "expect": true + }, + { + "name": "test_evaluate_function_not_validation_result_operand_v0_9", + "description": "Verifies v0.9, whose validators return booleans, treats a {valid} object as a plain truthy object.", + "protocolVersion": "v0.9", + "action": "evaluate_function", + "function": "not", + "args": { + "value": { + "valid": false + } + }, + "expect": false + }, + { + "name": "test_evaluate_function_pluralize_explicit_zero", + "description": "Verifies pluralize selects explicit zero category over CLDR (D27).", + "action": "evaluate_function", + "function": "pluralize", + "args": { + "value": 0, + "zero": "No items", + "one": "1 item", + "other": "{n} items" + }, + "expect": "No items" + }, + { + "name": "test_evaluate_function_pluralize_explicit_one", + "description": "Verifies pluralize selects explicit one category.", + "action": "evaluate_function", + "function": "pluralize", + "args": { + "value": 1, + "zero": "No items", + "one": "1 item", + "other": "{n} items" + }, + "expect": "1 item" + }, + { + "name": "test_evaluate_function_pluralize_explicit_two", + "description": "Verifies pluralize selects explicit two category (D27).", + "action": "evaluate_function", + "function": "pluralize", + "args": { + "value": 2, + "zero": "No items", + "one": "1 item", + "two": "A couple items", + "other": "{n} items" + }, + "expect": "A couple items" + }, + { + "name": "test_evaluate_function_pluralize_explicit_empty_string", + "description": "Verifies pluralize preserves explicit empty string without falling back to other (D27).", + "action": "evaluate_function", + "function": "pluralize", + "args": { + "value": 0, + "zero": "", + "other": "default" + }, + "expect": "" + }, + { + "name": "test_evaluate_function_pluralize_cldr_other", + "description": "Verifies pluralize falls back to other for unlisted category.", + "action": "evaluate_function", + "function": "pluralize", + "args": { + "value": 5, + "one": "1 item", + "other": "Many items" + }, + "expect": "Many items" + }, + { + "name": "test_evaluate_function_format_number_decimals", + "description": "Verifies formatNumber formats with decimals and grouping.", + "action": "evaluate_function", + "function": "formatNumber", + "args": { + "value": 1234.567, + "decimals": 2, + "grouping": true + }, + "expect": "1,234.57" + }, + { + "name": "test_evaluate_function_format_number_zero_decimals", + "description": "Verifies formatNumber formats with zero decimals and no grouping.", + "action": "evaluate_function", + "function": "formatNumber", + "args": { + "value": 1234.567, + "decimals": 0, + "grouping": false + }, + "expect": "1235" + }, + { + "name": "test_evaluate_function_format_currency_usd", + "description": "Verifies formatCurrency formats USD currency.", + "action": "evaluate_function", + "function": "formatCurrency", + "args": { + "value": 99.99, + "currency": "USD", + "decimals": 2 + }, + "expect": "$99.99" + }, + { + "name": "test_evaluate_function_format_date_custom_pattern", + "description": "Verifies formatDate formats date with custom pattern.", + "action": "evaluate_function", + "function": "formatDate", + "args": { + "value": "2026-08-26T12:00:00Z", + "format": "yyyy-MM-dd" + }, + "expect": "2026-08-26" + }, + { + "name": "test_evaluate_function_format_date_iso_normalized", + "description": "Verifies formatDate normalizes ISO output to UTC YYYY-MM-DDTHH:mm:ss.sssZ (D29).", + "action": "evaluate_function", + "function": "formatDate", + "args": { + "value": "2026-08-26T12:00:00.000Z", + "format": "ISO" + }, + "expect": "2026-08-26T12:00:00.000Z" + }, + { + "name": "test_evaluate_function_format_date_impossible_day", + "description": "Verifies formatDate returns an empty string for a day that does not exist, rather than rolling it into the next month.", + "action": "evaluate_function", + "function": "formatDate", + "args": { + "value": "2026-02-30", + "format": "yyyy-MM-dd" + }, + "expect": "" + }, + { + "name": "test_evaluate_function_format_date_impossible_month", + "description": "Verifies formatDate returns an empty string for a month that does not exist, rather than rolling it into the next year.", + "action": "evaluate_function", + "function": "formatDate", + "args": { + "value": "2026-13-01", + "format": "yyyy-MM-dd" + }, + "expect": "" + }, + { + "name": "test_evaluate_function_format_date_utc_time", + "description": "Verifies formatDate renders the time fields of a UTC timestamp as written, whatever the host time zone.", + "action": "evaluate_function", + "function": "formatDate", + "args": { + "value": "2026-09-04T23:30:00Z", + "format": "yyyy-MM-dd HH:mm" + }, + "expect": "2026-09-04 23:30" + }, + { + "name": "test_evaluate_function_format_date_without_offset", + "description": "Verifies formatDate reads a timestamp without an offset as written (as UTC), never as host-local time.", + "action": "evaluate_function", + "function": "formatDate", + "args": { + "value": "2026-09-04T23:30:00", + "format": "yyyy-MM-dd HH:mm" + }, + "expect": "2026-09-04 23:30" + }, + { + "name": "test_evaluate_function_format_date_keeps_written_offset", + "description": "Verifies formatDate renders a timestamp with an offset in that offset, so the written wall-clock date and time are kept.", + "protocolVersion": "1.0", + "action": "evaluate_function", + "function": "formatDate", + "args": { + "value": "2026-09-04T23:30:00-05:00", + "format": "yyyy-MM-dd HH:mm" + }, + "expect": "2026-09-04 23:30" + }, + { + "name": "test_evaluate_function_format_string_basic", + "description": "Verifies formatString evaluates static template string.", + "action": "evaluate_function", + "function": "formatString", + "args": { + "value": "Hello World" + }, + "expect": "Hello World" + }, + { + "name": "test_evaluate_function_format_string_null_coercion", + "description": "Verifies formatString coerces null expression to empty string (D18).", + "action": "evaluate_function", + "function": "formatString", + "args": { + "value": "Greeting: ${null}" + }, + "expect": "Greeting: " + }, + { + "name": "test_evaluate_function_open_url_valid_https", + "description": "Verifies openUrl accepts valid https scheme.", + "action": "evaluate_function", + "function": "openUrl", + "args": { + "url": "https://example.com" + }, + "expect": null + }, + { + "name": "test_evaluate_function_open_url_disallowed_scheme", + "description": "Verifies openUrl rejects disallowed URL scheme (D30).", + "action": "evaluate_function", + "function": "openUrl", + "args": { + "url": "javascript:alert(1)" + }, + "expectError": { + "category": "ExpressionError" + } + }, + { + "name": "test_evaluate_function_v10_runs_in_named_catalog", + "description": "Verifies a v1.0 call that names a catalogId runs in that catalog rather than the surface default. The default is the conformance custom catalog, which declares no formatString; the call names the basic catalog, which does.", + "action": "evaluate_function", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json", + "conformance/test_data/catalogs/custom_catalog_v1_0.json" + ], + "surface": { + "catalogId": "conformance/custom" + }, + "function": "formatString", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "args": { + "value": "Hello" + }, + "expect": "Hello" + }, + { + "name": "test_evaluate_function_v10_runs_in_named_catalog_on_surface_without_default", + "description": "Verifies a v1.0 call that names a catalogId runs on a surface that has no default catalog.", + "action": "evaluate_function", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "surface": {}, + "function": "formatString", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "args": { + "value": "Hello" + }, + "expect": "Hello" + }, + { + "name": "test_evaluate_function_v10_runs_in_surface_default", + "description": "Verifies a v1.0 call that names no catalogId runs in the surface default catalog (the basic catalog), while the conformance custom catalog is also available on the surface.", + "action": "evaluate_function", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json", + "conformance/test_data/catalogs/custom_catalog_v1_0.json" + ], + "surface": { + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + }, + "function": "formatString", + "args": { + "value": "Hello" + }, + "expect": "Hello" + }, + { + "name": "test_evaluate_function_v10_without_catalog_id_on_surface_without_default_errors", + "description": "Verifies a v1.0 call that names no catalogId fails with a CatalogError on a surface that has no default catalog, even though the only available catalog declares the function. There is no fallback to the available catalogs.", + "action": "evaluate_function", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "surface": {}, + "function": "formatString", + "args": { + "value": "Hello" + }, + "expectError": { + "category": "CatalogError", + "message": "has no default catalogId" + } + }, + { + "name": "test_evaluate_function_v10_unknown_catalog_id_errors", + "description": "Verifies a v1.0 call that names a catalogId the surface does not have fails with a CatalogError, rather than falling back to the surface default, which declares the function.", + "action": "evaluate_function", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "surface": { + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + }, + "function": "formatString", + "catalogId": "missing-catalog", + "args": { + "value": "Hello" + }, + "expectError": { + "category": "CatalogError", + "message": "Catalog not found: missing-catalog" + } + }, + { + "name": "test_evaluate_function_v10_catalog_of_other_protocol_version_errors", + "description": "Verifies a v1.0 call that names the v0.9 basic catalog fails with a CatalogError. The client has that catalog, but a v1.0 surface only makes v1.0 catalogs available.", + "action": "evaluate_function", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json", + "specification/v0_9/catalogs/basic/catalog.json" + ], + "surface": { + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + }, + "function": "formatString", + "catalogId": "https://a2ui.org/specification/v0_9/catalogs/basic/catalog.json", + "args": { + "value": "Hello" + }, + "expectError": { + "category": "CatalogError", + "message": "Catalog not found" + } + }, + { + "name": "test_evaluate_function_v10_nested_call_does_not_inherit_catalog", + "description": "Verifies a call nested in the args of a v1.0 call resolves its catalog on its own. The outer call names the basic catalog; the inner call names no catalogId, and the surface has no default catalog, so the inner call fails with a CatalogError.", + "action": "evaluate_function", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "surface": {}, + "function": "formatString", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "args": { + "value": { + "@call": "formatString", + "args": { + "value": "hi" + } + } + }, + "expectError": { + "category": "CatalogError", + "message": "has no default catalogId" + } + }, + { + "name": "test_evaluate_function_v09_ignores_call_catalog_id", + "description": "Verifies a catalogId on a call is ignored before v1.0. On a v0.9 surface the call runs in the surface catalog, so naming a catalog the client does not have is not an error.", + "action": "evaluate_function", + "protocolVersion": "v0.9", + "catalogPaths": [ + "specification/v0_9/catalogs/basic/catalog.json" + ], + "surface": { + "catalogId": "https://a2ui.org/specification/v0_9/catalogs/basic/catalog.json" + }, + "function": "formatString", + "catalogId": "missing-catalog", + "args": { + "value": "Hello" + }, + "expect": "Hello" + }, + { + "name": "test_evaluate_function_v10_validation_result_coerced_in_format_number_grouping", + "description": "Verifies a failing ValidationResult bound to formatNumber's DynamicBoolean grouping argument reads as false, turning grouping off. An object is truthy in most languages, so an engine that does not coerce it groups the digits.", + "action": "evaluate_function", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "surface": { + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + }, + "function": "formatNumber", + "args": { + "value": 1234, + "decimals": 0, + "grouping": { + "@call": "required", + "args": { + "value": "" + } + } + }, + "expect": "1234" + }, + { + "name": "test_evaluate_function_v10_validation_result_coerced_in_format_number_grouping_valid", + "description": "Verifies a passing ValidationResult bound to formatNumber's DynamicBoolean grouping argument reads as true, keeping grouping on.", + "action": "evaluate_function", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "surface": { + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + }, + "function": "formatNumber", + "args": { + "value": 1234, + "decimals": 0, + "grouping": { + "@call": "required", + "args": { + "value": "x" + } + } + }, + "expect": "1,234" + }, + { + "name": "test_evaluate_function_v10_validation_result_coerced_in_not_operand_on_surface", + "description": "Verifies a ValidationResult resolved from a nested validation call into not's DynamicBoolean operand is read by its validity when the call runs on a surface.", + "action": "evaluate_function", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "surface": { + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + }, + "function": "not", + "args": { + "value": { + "@call": "required", + "args": { + "value": "" + } + } + }, + "expect": true + }, + { + "name": "test_evaluate_function_v10_validation_result_coerced_in_and_operands_on_surface", + "description": "Verifies the ValidationResults resolved from nested validation calls into and's DynamicBoolean operands are read by their validity when the call runs on a surface, as in the specification's enable-when-valid example.", + "action": "evaluate_function", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "surface": { + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + }, + "dataModel": { + "email": "a@b.c", + "phone": "" + }, + "function": "and", + "args": { + "values": [ + { + "@call": "required", + "args": { + "value": { + "@path": "/email" + } + } + }, + { + "@call": "required", + "args": { + "value": { + "@path": "/phone" + } + } + } + ] + }, + "expect": false + }, + { + "name": "test_evaluate_function_v10_agent_only_function_errors", + "description": "Verifies a v1.0 call on a surface to a function whose allowedCallers is agentOnly fails with an ExpressionError and does not run.", + "action": "evaluate_function", + "protocolVersion": "1.0", + "catalogPaths": [ + "conformance/test_data/catalogs/allowed_callers_catalog_v1_0.json" + ], + "surface": { + "catalogId": "https://a2ui.org/catalogs/v10/allowed_callers" + }, + "function": "readSessionToken", + "args": {}, + "expectError": { + "category": "ExpressionError", + "message": "cannot be called by renderer" + } + } +] \ No newline at end of file diff --git a/package/__fixtures__/a2ui-conformance/core/index_function.json b/package/__fixtures__/a2ui-conformance/core/index_function.json new file mode 100644 index 0000000000..283f21106e --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/core/index_function.json @@ -0,0 +1,270 @@ +[ + { + "name": "test_index_function_in_collection_loop", + "description": "Verifies built-in @index function evaluation during collection template iteration.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "basic", + "dataModel": { + "tasks": [ + "Buy Milk", + "Clean Kitchen", + "Pay Bills" + ] + }, + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "list" + ] + }, + { + "id": "list", + "component": "List", + "children": { + "path": "/tasks", + "componentId": "row" + } + }, + { + "id": "row", + "component": "Row", + "children": [ + "item_index", + "item_text" + ] + }, + { + "id": "item_index", + "component": "Text", + "text": { + "@call": "@index", + "args": {} + } + }, + { + "id": "item_text", + "component": "Text", + "text": { + "@path": "" + } + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "main": { + "components": { + "item_index_0": { + "text": "0" + }, + "item_text_0": { + "text": "Buy Milk" + }, + "item_index_1": { + "text": "1" + }, + "item_text_1": { + "text": "Clean Kitchen" + }, + "item_index_2": { + "text": "2" + }, + "item_text_2": { + "text": "Pay Bills" + } + } + } + } + } + }, + { + "name": "test_index_function_with_offset", + "description": "Verifies built-in @index function with offset argument (@index(1) for 1-based indexing).", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "basic", + "dataModel": { + "players": [ + "Alice", + "Bob" + ] + }, + "components": [ + { + "id": "root", + "component": "List", + "children": { + "path": "/players", + "componentId": "text" + } + }, + { + "id": "text", + "component": "Text", + "text": { + "@call": "@index", + "args": { + "offset": 1 + } + } + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "main": { + "components": { + "text_0": { + "text": "1" + }, + "text_1": { + "text": "2" + } + } + } + } + } + }, + { + "name": "test_index_function_nested_path", + "description": "Verifies built-in @index function evaluation inside a deeply nested child property path of a template list.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "basic", + "dataModel": { + "users": [ + { + "name": "Alice", + "address": { + "city": "Seattle" + } + }, + { + "name": "Bob", + "address": { + "city": "Portland" + } + } + ] + }, + "components": [ + { + "id": "root", + "component": "List", + "children": { + "path": "/users", + "componentId": "col" + } + }, + { + "id": "col", + "component": "Column", + "children": [ + "nested_text" + ] + }, + { + "id": "nested_text", + "component": "Text", + "text": { + "@call": "@index", + "args": { + "offset": 1 + } + } + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "main": { + "components": { + "nested_text_0": { + "text": "1" + }, + "nested_text_1": { + "text": "2" + } + } + } + } + } + }, + { + "name": "test_index_function_outside_loop_error", + "description": "Verifies error thrown when @index function is called outside a collection iteration scope.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "basic", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "@index", + "args": {} + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "@index function can only be evaluated inside a collection template iteration scope." + } + } + ] + } +] \ No newline at end of file diff --git a/package/__fixtures__/a2ui-conformance/core/message_processor_v1_0.json b/package/__fixtures__/a2ui-conformance/core/message_processor_v1_0.json new file mode 100644 index 0000000000..05b297adb9 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/core/message_processor_v1_0.json @@ -0,0 +1,5222 @@ +[ + { + "name": "test_v10_create_surface_inline_initialization", + "description": "Tests v1.0 createSurface message with inline components and initial dataModel initialization.", + "action": "process_messages", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_v10", + "catalogId": "test-catalog-v10", + "sendDataModel": true, + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "text1" + ] + }, + { + "id": "text1", + "component": "Text", + "text": "Inline Text" + } + ], + "dataModel": { + "counter": 42 + } + } + } + ], + "expect": { + "surfaces": { + "s_v10": { + "exists": true, + "sendDataModel": true, + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "text1" + ] + }, + { + "id": "text1", + "component": "Text", + "text": "Inline Text" + } + ], + "dataModel": { + "counter": 42 + } + } + } + } + }, + { + "name": "test_v10_create_surface_optional_catalog_id", + "description": "Tests that v1.0 createSurface permits omitting catalogId.", + "action": "process_messages", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_v10_nocat" + } + } + ], + "expect": { + "surfaces": { + "s_v10_nocat": { + "exists": true + } + } + } + }, + { + "name": "test_v10_create_surface_with_metadata_extensions", + "description": "Tests v1.0 createSurface with metadata and extension descriptors.", + "action": "process_messages", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_v10_meta", + "catalogId": "test-catalog-v10", + "metadata": { + "extensions": { + "a2ui_experimental": { + "streaming": true + } + } + } + } + } + ], + "expect": { + "surfaces": { + "s_v10_meta": { + "exists": true + } + } + } + }, + { + "name": "test_v10_create_surface_metadata_extension_key_must_be_identifier", + "description": "Tests that v1.0 createSurface rejects metadata extensions whose key is not a Unicode identifier (UAX 31).", + "action": "process_messages", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_v10_meta_bad_key", + "catalogId": "test-catalog-v10", + "metadata": { + "extensions": { + "a2ui:experimental": { + "streaming": true + } + } + } + } + } + ], + "expectError": { + "category": "ValidationError" + } + }, + { + "name": "test_v10_create_surface_metadata_only_holds_extensions", + "description": "Tests that v1.0 createSurface rejects metadata keys other than extensions.", + "action": "process_messages", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_v10_meta_bad_field", + "catalogId": "test-catalog-v10", + "metadata": { + "owner": "me" + } + } + } + ], + "expectError": { + "category": "ValidationError" + } + }, + { + "name": "test_v10_component_catalog_override", + "description": "Tests that individual components in v1.0 can specify a catalogId override different from the surface default.", + "action": "process_messages", + "catalogs": [ + { + "catalogId": "main-catalog", + "protocolVersion": "1.0" + }, + { + "catalogId": "charts-catalog", + "protocolVersion": "1.0" + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_v10_multi", + "catalogId": "main-catalog" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_v10_multi", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "c1" + ] + }, + { + "id": "c1", + "component": "PieChart", + "catalogId": "charts-catalog", + "data": [ + 10, + 20, + 30 + ] + } + ] + } + } + ], + "expect": { + "surfaces": { + "s_v10_multi": { + "exists": true, + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "c1" + ] + }, + { + "id": "c1", + "component": "PieChart", + "data": [ + 10, + 20, + 30 + ] + } + ] + } + } + } + }, + { + "name": "test_v10_update_data_model_deletion", + "description": "Tests that updating a path with value null deletes the key from the data model in v1.0.", + "action": "process_messages", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_v10_del", + "catalogId": "test-catalog-v10", + "dataModel": { + "user": { + "name": "Alice", + "temporaryToken": "abc123xyz" + } + } + } + }, + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s_v10_del", + "path": "/user/temporaryToken", + "value": null + } + } + ], + "expect": { + "surfaces": { + "s_v10_del": { + "dataModel": { + "user": { + "name": "Alice" + } + } + } + } + } + }, + { + "name": "test_v10_get_renderer_capabilities", + "description": "Tests generating v1.0 renderer capabilities structure.", + "action": "get_renderer_capabilities", + "catalogs": [ + { + "catalogId": "cat-v10", + "protocolVersion": "1.0" + } + ], + "args": { + "includeInlineCatalogs": false, + "version": "1.0" + }, + "expect": { + "1.0": { + "supportedCatalogIds": [ + "cat-v10" + ] + } + } + }, + { + "name": "test_v10_get_renderer_capabilities_inline_catalogs", + "description": "Tests that v1.0 inline catalogs list each component as a plain schema, without the v0.9 ComponentCommon allOf envelope, because v1.0 applies ComponentCommon at the message envelope.", + "action": "get_renderer_capabilities", + "catalogs": [ + { + "catalogId": "cat-v10", + "protocolVersion": "1.0", + "components": { + "Button": { + "type": "object", + "properties": { + "component": { + "const": "Button" + }, + "label": { + "type": "string", + "description": "The button label" + } + }, + "required": [ + "component", + "label" + ] + } + } + } + ], + "args": { + "includeInlineCatalogs": true, + "version": "1.0" + }, + "expect": { + "1.0": { + "supportedCatalogIds": [ + "cat-v10" + ], + "inlineCatalogs": [ + { + "catalogId": "cat-v10", + "protocolVersion": "1.0", + "components": { + "Button": { + "type": "object", + "properties": { + "component": { + "const": "Button" + }, + "label": { + "type": "string", + "description": "The button label" + } + }, + "required": [ + "component", + "label" + ] + } + } + } + ] + } + } + }, + { + "name": "test_multiple_update_types_error", + "description": "Tests that sending a single message envelope containing multiple update actions raises a ValidationError.", + "action": "process_messages", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "test-catalog-v10" + }, + "deleteSurface": { + "surfaceId": "s1" + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Message contains multiple conflicting update actions" + } + }, + { + "name": "test_batch_message_array_and_wrapper", + "description": "Tests that message processor correctly processes message list arrays and wrapped container payloads.", + "action": "process_messages", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "test-catalog-v10" + } + }, + { + "version": "1.0", + "createSurface": { + "surfaceId": "s2", + "catalogId": "test-catalog-v10" + } + } + ] + }, + "expect": { + "surfaces": { + "s1": { + "exists": true + }, + "s2": { + "exists": true + } + } + } + }, + { + "name": "test_v10_rejects_v08_begin_rendering_action", + "description": "Tests that v1.0 MessageProcessor rejects v0.8 beginRendering action when protocolVersion is v1.0.", + "action": "process_messages", + "protocolVersion": "1.0", + "messages": [ + { + "version": "1.0", + "beginRendering": { + "surfaceId": "s_v10_invalid", + "catalogId": "test-catalog-v10" + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "action 'beginRendering' is not supported in protocol version 1.0" + } + }, + { + "name": "test_v10_non_string_surface_id_error", + "description": "Tests that a v1.0 action with a non-string surfaceId raises a ValidationError.", + "action": "process_messages", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": 12345, + "catalogId": "test-catalog-v10" + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "surfaceId must be a string" + } + }, + { + "name": "test_v10_update_components_mismatched_catalog_protocol_version_error", + "description": "Tests that updating a component with a catalog of a different protocol version raises CatalogError.", + "action": "process_messages", + "catalogs": [ + { + "catalogId": "cat-v10", + "protocolVersion": "1.0" + }, + { + "catalogId": "cat-v09", + "protocolVersion": "v0.9" + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_mismatch", + "catalogId": "cat-v10" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_mismatch", + "components": [ + { + "id": "c1", + "component": "Button", + "catalogId": "cat-v09" + } + ] + } + } + ], + "expectError": { + "category": "CatalogError", + "surfaceId": "s_mismatch", + "rendererError": { + "version": "1.0", + "error": { + "code": "CATALOG_NOT_FOUND", + "surfaceId": "s_mismatch", + "message": "cat-v09" + } + } + } + }, + { + "name": "test_unsupported_protocol_version_error", + "description": "Verifies rejecting unsupported protocol version strings (v99.0).", + "action": "process_messages", + "messages": [ + { + "version": "v99.0", + "createSurface": { + "surfaceId": "s1" + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Unsupported protocol version 'v99.0'" + } + }, + { + "name": "test_v091_version_string_compatibility", + "description": "Verifies processing messages with version v0.9.1.", + "action": "process_messages", + "protocolVersion": "v0.9.1", + "messages": [ + { + "version": "v0.9.1", + "createSurface": { + "surfaceId": "s_v091", + "catalogId": "basic" + } + } + ], + "expect": { + "surfaces": { + "s_v091": { + "exists": true + } + } + } + }, + { + "name": "test_missing_version_field_in_v10_process_messages_error", + "description": "Verifies ValidationError on missing version field during process_messages.", + "action": "process_messages", + "messages": [ + { + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic" + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Missing required version field" + } + }, + { + "name": "test_batch_multi_stage_lifecycle_pipeline", + "description": "Verifies create, component update, data update, and component mutation within a single batch array.", + "action": "process_messages", + "protocolVersion": "1.0", + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_pipeline", + "catalogId": "test-catalog-v10" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_pipeline", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "text1" + ] + }, + { + "id": "text1", + "component": "Text", + "text": "Initial" + } + ] + } + }, + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s_pipeline", + "path": "/status", + "value": "ready" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_pipeline", + "components": [ + { + "id": "text1", + "component": "Text", + "text": "Updated" + } + ] + } + } + ], + "expect": { + "surfaces": { + "s_pipeline": { + "exists": true, + "dataModel": { + "status": "ready" + }, + "components": [ + { + "id": "root", + "component": "Column" + }, + { + "id": "text1", + "component": "Text" + } + ] + } + } + } + }, + { + "name": "test_batch_duplicate_component_ids_in_same_message_error", + "description": "Verifies IntegrityError on duplicate component IDs within a single updateComponents message.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_dup", + "catalogId": "basic" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_dup", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "c1" + ] + }, + { + "id": "c1", + "component": "Text", + "text": "First" + }, + { + "id": "c1", + "component": "Text", + "text": "Duplicate" + } + ] + } + } + ], + "expectError": { + "category": "IntegrityError", + "surfaceId": "s_dup", + "rendererError": { + "version": "1.0", + "error": { + "code": "INTEGRITY_ERROR", + "surfaceId": "s_dup", + "message": "c1" + } + } + } + }, + { + "name": "test_batch_atomic_rollback_on_candidate_topology_cycle", + "description": "Verifies candidate topology pre-flight cycle check aborts without mutating existing valid component tree.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_cycle", + "catalogId": "basic", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "c1" + ] + }, + { + "id": "c1", + "component": "Text", + "text": "Valid Text" + } + ] + } + } + ], + "expect": { + "surfaces": { + "s_cycle": { + "exists": true, + "components": [ + { + "id": "root", + "component": "Column" + }, + { + "id": "c1", + "component": "Text" + } + ] + } + } + } + }, + { + "messages": [ + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_cycle", + "components": [ + { + "id": "c1", + "component": "Column", + "children": [ + "c2" + ] + }, + { + "id": "c2", + "component": "Column", + "children": [ + "c1" + ] + } + ] + } + } + ], + "expectError": { + "category": "IntegrityError", + "surfaceId": "s_cycle", + "rendererError": { + "version": "1.0", + "error": { + "code": "RECURSION_LIMIT_EXCEEDED", + "surfaceId": "s_cycle", + "message": "Circular" + } + } + } + } + ] + }, + { + "name": "test_update_components_on_deleted_surface_error", + "description": "Verifies updating components on a deleted surface raises IntegrityError.", + "action": "process_messages", + "protocolVersion": "1.0", + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_del", + "catalogId": "basic" + } + }, + { + "version": "1.0", + "deleteSurface": { + "surfaceId": "s_del" + } + } + ], + "expect": { + "surfaces": { + "s_del": { + "exists": false + } + } + } + }, + { + "messages": [ + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_del", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Fail" + } + ] + } + } + ], + "expectError": { + "category": "IntegrityError" + } + } + ] + }, + { + "name": "test_update_data_model_on_deleted_surface_error", + "description": "Verifies updating data model on a deleted surface raises IntegrityError.", + "action": "process_messages", + "protocolVersion": "1.0", + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_del_data", + "catalogId": "basic" + } + }, + { + "version": "1.0", + "deleteSurface": { + "surfaceId": "s_del_data" + } + } + ], + "expect": { + "surfaces": { + "s_del_data": { + "exists": false + } + } + } + }, + { + "messages": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s_del_data", + "path": "/key", + "value": "value" + } + } + ], + "expectError": { + "category": "IntegrityError" + } + } + ] + }, + { + "name": "test_delete_non_existent_surface_is_safe_noop", + "description": "Verifies deleting a non-existent surface ID executes cleanly as a no-op.", + "action": "process_messages", + "protocolVersion": "1.0", + "messages": [ + { + "version": "1.0", + "deleteSurface": { + "surfaceId": "non_existent_surface_id" + } + } + ], + "expect": { + "surfaces": { + "non_existent_surface_id": { + "exists": false + } + } + } + }, + { + "name": "test_permissive_mode_allows_orphan_components", + "description": "Verifies strictMode false accepts components unreachable from root.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": false, + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_permissive_orphan", + "catalogId": "test-catalog-v10", + "components": [ + { + "id": "root", + "component": "Column", + "children": [] + }, + { + "id": "orphan_comp", + "component": "Text", + "text": "Unreachable Text" + } + ] + } + } + ], + "expect": { + "surfaces": { + "s_permissive_orphan": { + "exists": true, + "components": [ + { + "id": "root", + "component": "Column" + }, + { + "id": "orphan_comp", + "component": "Text" + } + ] + } + } + } + }, + { + "name": "test_permissive_mode_allows_dangling_references", + "description": "Verifies strictMode false accepts containers referencing uncreated child IDs.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": false, + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_permissive_dangling", + "catalogId": "test-catalog-v10", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "uncreated_child_node" + ] + } + ] + } + } + ], + "expect": { + "surfaces": { + "s_permissive_dangling": { + "exists": true, + "components": [ + { + "id": "root", + "component": "Column" + } + ] + } + } + } + }, + { + "name": "test_v10_call_validation_config_relaxes_strict_processor", + "description": "A strict processor accepts an orphan component in a call whose validationConfig allows orphans, and rejects one in the next call, which has no validationConfig.", + "action": "process_messages", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "validationConfig": { + "allowOrphanComponents": true + }, + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_relaxed_call", + "catalogId": "https://a2ui.org/catalogs/basic/v1/catalog.json", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Root" + }, + { + "id": "orphan", + "component": "Text", + "text": "Orphan" + } + ] + } + } + ], + "expect": { + "surfaces": { + "s_relaxed_call": { + "exists": true, + "components": [ + { + "id": "root", + "component": "Text" + }, + { + "id": "orphan", + "component": "Text" + } + ] + } + } + } + }, + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_strict_call", + "catalogId": "https://a2ui.org/catalogs/basic/v1/catalog.json", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Root" + }, + { + "id": "orphan", + "component": "Text", + "text": "Orphan" + } + ] + } + } + ], + "expectError": { + "message": "is not reachable from" + } + } + ] + }, + { + "name": "test_v10_call_validation_config_tightens_permissive_processor", + "description": "A processor created without validation rules rejects an orphan component in a call whose validationConfig is strict, and accepts one in the next call, which has no validationConfig.", + "action": "process_messages", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": false, + "steps": [ + { + "validationConfig": {}, + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_strict_call", + "catalogId": "https://a2ui.org/catalogs/basic/v1/catalog.json", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Root" + }, + { + "id": "orphan", + "component": "Text", + "text": "Orphan" + } + ] + } + } + ], + "expectError": { + "message": "is not reachable from" + } + }, + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_permissive_call", + "catalogId": "https://a2ui.org/catalogs/basic/v1/catalog.json", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Root" + }, + { + "id": "orphan", + "component": "Text", + "text": "Orphan" + } + ] + } + } + ], + "expect": { + "surfaces": { + "s_permissive_call": { + "exists": true, + "components": [ + { + "id": "root", + "component": "Text" + }, + { + "id": "orphan", + "component": "Text" + } + ] + } + } + } + } + ] + }, + { + "name": "test_v10_call_validation_config_leaves_out_rules_as_strict", + "description": "A call whose validationConfig allows only orphans still rejects a dangling child reference, since the rules it leaves out are strict, even on a processor created without validation rules.", + "action": "process_messages", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": false, + "steps": [ + { + "validationConfig": { + "allowOrphanComponents": true + }, + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_dangling_call", + "catalogId": "https://a2ui.org/catalogs/basic/v1/catalog.json", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "missing_child" + ] + } + ] + } + } + ], + "expectError": { + "message": "Dangling reference" + } + } + ] + }, + { + "name": "test_composition_constraints_preserved_on_partial_parent_update", + "description": "Verifies updating container property without redefining children preserves existing child relationships.", + "action": "process_messages", + "protocolVersion": "1.0", + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_partial_parent", + "catalogId": "test-catalog-v10", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "text_child" + ] + }, + { + "id": "text_child", + "component": "Text", + "text": "Child Text" + } + ] + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_partial_parent", + "components": [ + { + "id": "text_child", + "component": "Text", + "text": "Updated Child Text" + } + ] + } + } + ], + "expect": { + "surfaces": { + "s_partial_parent": { + "exists": true, + "components": [ + { + "id": "root", + "component": "Column" + }, + { + "id": "text_child", + "component": "Text" + } + ] + } + } + } + }, + { + "name": "test_global_recursion_depth_limit_exceeded_error", + "description": "Verifies object nesting exceeding limit (>50) raises RecursionError.", + "action": "validate", + "protocolVersion": "1.0", + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_rec", + "catalogId": "basic" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_rec", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Deep", + "meta": { + "l1": { + "l2": { + "l3": { + "l4": { + "l5": { + "l6": { + "l7": { + "l8": { + "l9": { + "l10": { + "l11": { + "l12": { + "l13": { + "l14": { + "l15": { + "l16": { + "l17": { + "l18": { + "l19": { + "l20": { + "l21": { + "l22": { + "l23": { + "l24": { + "l25": { + "l26": { + "l27": { + "l28": { + "l29": { + "l30": { + "l31": { + "l32": { + "l33": { + "l34": { + "l35": { + "l36": { + "l37": { + "l38": { + "l39": { + "l40": { + "l41": { + "l42": { + "l43": { + "l44": { + "l45": { + "l46": { + "l47": { + "l48": { + "l49": { + "l50": { + "l51": { + "l52": "too deep" + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + ] + } + } + ], + "expectError": { + "category": "RecursionError", + "message": "Global recursion limit exceeded" + } + } + ] + }, + { + "name": "test_v10_component_without_catalog_id_on_surface_without_default_errors", + "description": "A component that names no catalogId on a surface that names no default catalog cannot be resolved, so the update fails instead of falling back to the first registered catalog.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Hello" + } + ] + } + } + ], + "expectError": { + "category": "CatalogError", + "surfaceId": "s", + "message": "has no default catalogId", + "rendererError": { + "version": "1.0", + "error": { + "code": "CATALOG_NOT_FOUND", + "surfaceId": "s", + "message": "has no default catalogId" + } + } + } + }, + { + "name": "test_v10_components_naming_catalogs_on_surface_without_default", + "description": "A surface without a default catalog accepts components that each name their catalog.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Column", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "children": [ + "greeting", + "badge" + ] + }, + { + "id": "greeting", + "component": "Text", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "text": "Hello" + }, + { + "id": "badge", + "component": "Badge", + "catalogId": "custom-v10", + "label": "New" + } + ] + } + } + ], + "expect": { + "surfaces": { + "s": { + "exists": true, + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "greeting", + "badge" + ] + }, + { + "id": "greeting", + "component": "Text", + "text": "Hello" + }, + { + "id": "badge", + "component": "Badge", + "label": "New" + } + ] + } + } + } + }, + { + "name": "test_v10_component_from_named_catalog_with_closed_schema_accepted", + "description": "A component's catalogId belongs to the component envelope (ComponentCommon), not to its properties. A catalog whose component schema is closed and declares no catalogId accepts a component that names that catalog.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogs": [ + { + "catalogId": "closed-v10", + "protocolVersion": "1.0", + "components": { + "Tag": { + "type": "object", + "properties": { + "label": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "functions": {} + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Tag", + "catalogId": "closed-v10", + "label": "Hello" + } + ] + } + } + ], + "expect": { + "surfaces": { + "s": { + "exists": true, + "components": [ + { + "id": "root", + "component": "Tag", + "label": "Hello" + } + ] + } + } + } + }, + { + "name": "test_v10_function_call_naming_catalog_without_that_function_errors", + "description": "Under strictMode: true (allow_unknown_elements=False), a function call that names a catalogId is validated against that catalog's function schema, which does not declare formatString. With strictMode: false, the call is accepted at processing time so a renderer can route it via callAgentFunction.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "formatString", + "catalogId": "custom-v10", + "args": { + "value": "Hello" + } + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "surfaceId": "s", + "message": "Unrecognized function 'formatString'", + "rendererError": { + "version": "1.0", + "error": { + "code": "VALIDATION_FAILED", + "surfaceId": "s", + "path": "/components/0/text", + "message": "Unrecognized function 'formatString'" + } + } + } + }, + { + "name": "test_v10_function_call_naming_catalog_with_invalid_args_errors", + "description": "A function call that names a catalogId has its args validated against that catalog's function definition.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "phrase": { + "type": "string" + } + }, + "required": [ + "phrase" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "shout", + "catalogId": "custom-v10", + "args": { + "phrase": 3 + } + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "surfaceId": "s", + "message": "phrase", + "rendererError": { + "version": "1.0", + "error": { + "code": "VALIDATION_FAILED", + "surfaceId": "s", + "path": "/components/0/text/args/phrase", + "message": "phrase" + } + } + } + }, + { + "name": "test_v10_function_call_naming_catalog_with_valid_args", + "description": "A component from the surface default catalog may call a function from another catalog by naming its catalogId.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "shout", + "catalogId": "custom-v10", + "args": { + "value": "Hello" + } + } + } + ] + } + } + ], + "expect": { + "surfaces": { + "s": { + "exists": true + } + } + } + }, + { + "name": "test_v10_function_call_naming_unknown_catalog_errors", + "description": "A function call that names a catalogId the client does not have is rejected.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "shout", + "catalogId": "missing-catalog", + "args": { + "value": "Hello" + } + } + } + ] + } + } + ], + "expectError": { + "category": "CatalogError", + "message": "Catalog not found: missing-catalog" + } + }, + { + "name": "test_v10_function_call_with_empty_catalog_id_errors", + "description": "A function call whose catalogId is an empty string names a catalog, so it does not fall back to the surface default. No catalog has an empty ID, so the call is rejected.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "formatString", + "catalogId": "", + "args": { + "value": "Hello" + } + } + } + ] + } + } + ], + "expectError": { + "category": "CatalogError", + "message": "Catalog not found" + } + }, + { + "name": "test_v10_component_with_non_string_catalog_id_errors", + "description": "A component's catalogId must be a string. A number is rejected as invalid instead of being ignored, which would resolve the component to the surface default.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "catalogId": 123, + "text": "Hello" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "catalogId" + } + }, + { + "name": "test_v10_function_call_without_catalog_id_uses_surface_default", + "description": "A function call that names no catalogId resolves against the surface default catalog, not the catalog of the component that contains it. formatString is declared by the basic catalog only.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Badge", + "catalogId": "custom-v10", + "label": { + "@call": "formatString", + "args": { + "value": "Hello" + } + } + } + ] + } + } + ], + "expect": { + "surfaces": { + "s": { + "exists": true + } + } + } + }, + { + "name": "test_v10_function_call_without_catalog_id_to_unknown_function_errors", + "description": "Under strictMode: true (allow_unknown_elements=False), a function call that names no catalogId is validated against the surface default catalog's function schema, which does not declare notAFunction. With strictMode: false, the call is accepted at processing time so a renderer can route it via callAgentFunction.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "notAFunction", + "args": {} + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Unrecognized function 'notAFunction'" + } + }, + { + "name": "test_v10_function_call_without_catalog_id_on_surface_without_default_errors", + "description": "A function call that names no catalogId on a surface without a default catalog cannot be resolved, even when the component's own catalog declares the function.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Badge", + "catalogId": "custom-v10", + "label": { + "@call": "shout", + "args": { + "value": "Hello" + } + } + } + ] + } + } + ], + "expectError": { + "category": "CatalogError", + "surfaceId": "s", + "message": "has no default catalogId", + "rendererError": { + "version": "1.0", + "error": { + "code": "CATALOG_NOT_FOUND", + "surfaceId": "s", + "message": "has no default catalogId" + } + } + } + }, + { + "name": "test_v10_index_function_on_surface_without_default", + "description": "@index is a universal system function that belongs to no catalog, so a template may call it without a catalogId on a surface that has no default catalog.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "List", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "children": { + "path": "/items", + "componentId": "row" + } + }, + { + "id": "row", + "component": "Text", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "text": { + "@call": "@index", + "args": { + "offset": 1 + } + } + } + ] + } + } + ], + "expect": { + "surfaces": { + "s": { + "exists": true + } + } + } + }, + { + "name": "test_v10_index_function_on_surface_with_default", + "description": "@index is accepted on a surface whose default catalog is the basic catalog; it is not looked up as a catalog function.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "List", + "children": { + "path": "/items", + "componentId": "row" + } + }, + { + "id": "row", + "component": "Text", + "text": { + "@call": "@index", + "args": { + "offset": 1 + } + } + } + ] + } + } + ], + "expect": { + "surfaces": { + "s": { + "exists": true + } + } + } + }, + { + "name": "test_v10_component_update_without_catalog_id_on_surface_without_default_errors", + "description": "An update that names no catalogId resolves against the surface default, not the catalog of the component's earlier version, so it fails on a surface without a default catalog.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "text": "Hello" + } + ] + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Goodbye" + } + ] + } + } + ], + "expectError": { + "category": "CatalogError", + "surfaceId": "s", + "message": "has no default catalogId", + "rendererError": { + "version": "1.0", + "error": { + "code": "CATALOG_NOT_FOUND", + "surfaceId": "s", + "message": "has no default catalogId" + } + } + } + }, + { + "name": "test_v10_component_update_without_catalog_id_uses_surface_default", + "description": "An update that names no catalogId resolves against the surface default catalog even when the component's earlier version named another catalog. The basic catalog declares no Badge, so the update is rejected.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + } + } + } + ], + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Badge", + "catalogId": "custom-v10", + "label": "First" + } + ] + } + } + ], + "expect": { + "surfaces": { + "s": { + "exists": true, + "components": [ + { + "id": "root", + "component": "Badge" + } + ] + } + } + } + }, + { + "messages": [ + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Badge", + "label": "Second" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "'Badge'" + } + } + ] + }, + { + "name": "test_v10_nested_call_in_args_resolves_its_own_catalog", + "description": "A call nested in the args of another call resolves its catalog on its own and does not inherit the catalog of the call around it. The outer wrap names custom-v10; the inner shout names no catalogId, so it resolves to the surface default (the basic catalog), which under strictMode: true rejects shout because the basic catalog does not declare it.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + }, + "Tags": { + "type": "object", + "properties": { + "component": { + "const": "Tags" + }, + "items": { + "type": "array" + } + }, + "required": [ + "component", + "items" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + }, + "wrap": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "wrap" + }, + "args": { + "type": "object", + "properties": { + "value": {} + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "wrap", + "catalogId": "custom-v10", + "args": { + "value": { + "@call": "shout", + "args": { + "value": "hi" + } + } + } + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Unrecognized function 'shout'" + } + }, + { + "name": "test_v10_nested_call_in_args_naming_its_catalog", + "description": "A call nested in the args of another call is accepted when it names the catalog that declares it, even though the surface default (the basic catalog) does not declare it.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + }, + "Tags": { + "type": "object", + "properties": { + "component": { + "const": "Tags" + }, + "items": { + "type": "array" + } + }, + "required": [ + "component", + "items" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + }, + "wrap": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "wrap" + }, + "args": { + "type": "object", + "properties": { + "value": {} + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "wrap", + "catalogId": "custom-v10", + "args": { + "value": { + "@call": "shout", + "catalogId": "custom-v10", + "args": { + "value": "hi" + } + } + } + } + } + ] + } + } + ], + "expect": { + "surfaces": { + "s": { + "exists": true + } + } + } + }, + { + "name": "test_v10_nested_call_in_args_on_surface_without_default_errors", + "description": "On a surface without a default catalog, a call nested in the args of a call that names custom-v10 cannot be resolved when it names no catalogId itself, even though custom-v10 declares it.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + }, + "Tags": { + "type": "object", + "properties": { + "component": { + "const": "Tags" + }, + "items": { + "type": "array" + } + }, + "required": [ + "component", + "items" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + }, + "wrap": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "wrap" + }, + "args": { + "type": "object", + "properties": { + "value": {} + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Badge", + "catalogId": "custom-v10", + "label": { + "@call": "wrap", + "catalogId": "custom-v10", + "args": { + "value": { + "@call": "shout", + "args": { + "value": "hi" + } + } + } + } + } + ] + } + } + ], + "expectError": { + "category": "CatalogError", + "surfaceId": "s", + "message": "has no default catalogId", + "rendererError": { + "version": "1.0", + "error": { + "code": "CATALOG_NOT_FOUND", + "surfaceId": "s", + "message": "has no default catalogId" + } + } + } + }, + { + "name": "test_v10_call_in_index_args_is_checked", + "description": "@index needs no catalog, but a call in its args is still resolved and checked. Here the offset calls notAFunction, which under strictMode: true is rejected because the surface default (the basic catalog) does not declare it.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + }, + "Tags": { + "type": "object", + "properties": { + "component": { + "const": "Tags" + }, + "items": { + "type": "array" + } + }, + "required": [ + "component", + "items" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + }, + "wrap": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "wrap" + }, + "args": { + "type": "object", + "properties": { + "value": {} + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "List", + "children": { + "path": "/items", + "componentId": "row" + } + }, + { + "id": "row", + "component": "Text", + "text": { + "@call": "@index", + "args": { + "offset": { + "@call": "notAFunction", + "args": {} + } + } + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Unrecognized function 'notAFunction'" + } + }, + { + "name": "test_v10_call_in_index_args_on_surface_without_default_errors", + "description": "On a surface without a default catalog, @index itself is accepted, but a call in its args that names no catalogId cannot be resolved.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + }, + "Tags": { + "type": "object", + "properties": { + "component": { + "const": "Tags" + }, + "items": { + "type": "array" + } + }, + "required": [ + "component", + "items" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + }, + "wrap": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "wrap" + }, + "args": { + "type": "object", + "properties": { + "value": {} + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "List", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "children": { + "path": "/items", + "componentId": "row" + } + }, + { + "id": "row", + "component": "Text", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "text": { + "@call": "@index", + "args": { + "offset": { + "@call": "formatNumber", + "args": { + "value": 1 + } + } + } + } + } + ] + } + } + ], + "expectError": { + "category": "CatalogError", + "surfaceId": "s", + "message": "has no default catalogId", + "rendererError": { + "version": "1.0", + "error": { + "code": "CATALOG_NOT_FOUND", + "surfaceId": "s", + "message": "has no default catalogId" + } + } + } + }, + { + "name": "test_v10_calls_in_lists_naming_their_catalog", + "description": "Calls inside a list property, and inside objects nested in that list, are resolved one by one; each names custom-v10, which declares shout.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + }, + "Tags": { + "type": "object", + "properties": { + "component": { + "const": "Tags" + }, + "items": { + "type": "array" + } + }, + "required": [ + "component", + "items" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + }, + "wrap": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "wrap" + }, + "args": { + "type": "object", + "properties": { + "value": {} + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Tags", + "catalogId": "custom-v10", + "items": [ + { + "@call": "shout", + "catalogId": "custom-v10", + "args": { + "value": "a" + } + }, + { + "nested": [ + { + "@call": "shout", + "catalogId": "custom-v10", + "args": { + "value": "b" + } + } + ] + } + ] + } + ] + } + } + ], + "expect": { + "surfaces": { + "s": { + "exists": true + } + } + } + }, + { + "name": "test_v10_call_in_list_without_catalog_id_uses_surface_default", + "description": "A call inside an object nested in a list property is resolved like any other call: it names no catalogId, so it resolves to the surface default (the basic catalog), which under strictMode: true rejects shout because the basic catalog does not declare it, even though the component and the list's first call use custom-v10.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + }, + "Tags": { + "type": "object", + "properties": { + "component": { + "const": "Tags" + }, + "items": { + "type": "array" + } + }, + "required": [ + "component", + "items" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + }, + "wrap": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "wrap" + }, + "args": { + "type": "object", + "properties": { + "value": {} + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Tags", + "catalogId": "custom-v10", + "items": [ + { + "@call": "shout", + "catalogId": "custom-v10", + "args": { + "value": "a" + } + }, + { + "nested": [ + { + "@call": "shout", + "args": { + "value": "b" + } + } + ] + } + ] + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Unrecognized function 'shout'" + } + }, + { + "name": "test_v10_call_in_list_with_invalid_args_errors", + "description": "A call inside an object nested in a list property has its args checked against the catalog it names; custom-v10's shout requires a string phrase.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + }, + "Tags": { + "type": "object", + "properties": { + "component": { + "const": "Tags" + }, + "items": { + "type": "array" + } + }, + "required": [ + "component", + "items" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "phrase": { + "type": "string" + } + }, + "required": [ + "phrase" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + }, + "wrap": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "wrap" + }, + "args": { + "type": "object", + "properties": { + "value": {} + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Tags", + "catalogId": "custom-v10", + "items": [ + { + "nested": [ + { + "@call": "shout", + "catalogId": "custom-v10", + "args": { + "phrase": 2 + } + } + ] + } + ] + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "phrase" + } + }, + { + "name": "test_v10_undefined_system_function_errors", + "description": "The @ namespace is reserved for system functions, and v1.0 defines only @index. A call to @foo is rejected rather than looked up in a catalog.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + }, + "Tags": { + "type": "object", + "properties": { + "component": { + "const": "Tags" + }, + "items": { + "type": "array" + } + }, + "required": [ + "component", + "items" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + }, + "wrap": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "wrap" + }, + "args": { + "type": "object", + "properties": { + "value": {} + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "@foo" + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "@foo" + } + }, + { + "name": "test_v10_function_call_with_empty_name_errors", + "description": "A function call's name must be a UAX #31 identifier. An empty name is not one, so the call is rejected rather than skipped.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "" + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "valid UAX #31 identifier" + } + }, + { + "name": "test_v10_call_without_catalog_id_on_surface_without_default_errors_without_strict_mode", + "description": "Catalog resolution does not depend on strict validation. With strictMode off, a call that names no catalogId on a surface without a default catalog is still rejected.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": false, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + }, + "Tags": { + "type": "object", + "properties": { + "component": { + "const": "Tags" + }, + "items": { + "type": "array" + } + }, + "required": [ + "component", + "items" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + }, + "wrap": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "wrap" + }, + "args": { + "type": "object", + "properties": { + "value": {} + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "text": { + "@call": "formatString", + "args": { + "value": "hi" + } + } + } + ] + } + } + ], + "expectError": { + "category": "CatalogError", + "surfaceId": "s", + "message": "has no default catalogId", + "rendererError": { + "version": "1.0", + "error": { + "code": "CATALOG_NOT_FOUND", + "surfaceId": "s", + "message": "has no default catalogId" + } + } + } + }, + { + "name": "test_v10_call_naming_unknown_catalog_errors_without_strict_mode", + "description": "Catalog resolution does not depend on strict validation. With strictMode off, a call that names a catalogId the client does not have is still rejected.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": false, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + }, + "Tags": { + "type": "object", + "properties": { + "component": { + "const": "Tags" + }, + "items": { + "type": "array" + } + }, + "required": [ + "component", + "items" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + }, + "wrap": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "wrap" + }, + "args": { + "type": "object", + "properties": { + "value": {} + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "formatString", + "catalogId": "missing-catalog", + "args": { + "value": "hi" + } + } + } + ] + } + } + ], + "expectError": { + "category": "CatalogError", + "message": "Catalog not found: missing-catalog" + } + }, + { + "name": "test_v10_call_to_undeclared_function_without_strict_mode_accepted", + "description": "With strictMode off (default renderer processing), a function call whose catalog resolves is accepted even when that local catalog does not declare the function, so the renderer can forward it to the agent via callAgentFunction.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": false, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "notAFunction", + "args": {} + } + } + ] + } + } + ], + "expect": { + "surfaces": { + "s": { + "exists": true + } + } + } + }, + { + "name": "test_v10_call_naming_catalog_of_other_protocol_version_errors", + "description": "Every catalog a v1.0 surface uses must be built for v1.0. A call that names legacy-v09, a v0.9 catalog the client has, cannot be resolved on a v1.0 surface. The message check only requires the error to name the catalog, since SDKs word the reason differently.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + }, + "Tags": { + "type": "object", + "properties": { + "component": { + "const": "Tags" + }, + "items": { + "type": "array" + } + }, + "required": [ + "component", + "items" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + }, + "wrap": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "wrap" + }, + "args": { + "type": "object", + "properties": { + "value": {} + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + }, + { + "catalogId": "legacy-v09", + "protocolVersion": "v0.9", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "formatString", + "catalogId": "legacy-v09", + "args": { + "value": "hi" + } + } + } + ] + } + } + ], + "expectError": { + "category": "CatalogError", + "message": "legacy-v09" + } + }, + { + "name": "test_v10_call_with_non_string_catalog_id_errors", + "description": "A call's catalogId must be a string. A number is rejected as invalid instead of being ignored, which would resolve the call to the surface default.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + }, + "Tags": { + "type": "object", + "properties": { + "component": { + "const": "Tags" + }, + "items": { + "type": "array" + } + }, + "required": [ + "component", + "items" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + }, + "wrap": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "wrap" + }, + "args": { + "type": "object", + "properties": { + "value": {} + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "formatString", + "catalogId": 3, + "args": { + "value": "hi" + } + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError" + } + }, + { + "name": "test_v10_component_naming_catalog_is_checked_against_that_catalog", + "description": "A component that names a catalogId is validated against that catalog, not the surface default. custom-v10 declares no Text, although the default (the basic catalog) does.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "catalogs": [ + { + "catalogId": "custom-v10", + "protocolVersion": "1.0", + "components": { + "Badge": { + "type": "object", + "properties": { + "component": { + "const": "Badge" + }, + "label": { + "$ref": "common_types.json#/$defs/DynamicString" + } + }, + "required": [ + "component", + "label" + ] + }, + "Tags": { + "type": "object", + "properties": { + "component": { + "const": "Tags" + }, + "items": { + "type": "array" + } + }, + "required": [ + "component", + "items" + ] + } + }, + "functions": { + "shout": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "shout" + }, + "args": { + "type": "object", + "properties": { + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + }, + "wrap": { + "type": "object", + "returnType": "string", + "properties": { + "@call": { + "const": "wrap" + }, + "args": { + "type": "object", + "properties": { + "value": {} + }, + "required": [ + "value" + ], + "additionalProperties": false + } + }, + "required": [ + "@call", + "args" + ] + } + } + } + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "catalogId": "custom-v10", + "text": "Hello" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Text" + } + }, + { + "name": "test_v10_component_without_catalog_id_errors_with_a_single_catalog", + "description": "There is no fallback to the catalogs the client supports, even when the case supplies only one. A component that names no catalogId on a surface without a default catalog is rejected.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": true, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Hello" + } + ] + } + } + ], + "expectError": { + "category": "CatalogError", + "surfaceId": "s", + "message": "has no default catalogId", + "rendererError": { + "version": "1.0", + "error": { + "code": "CATALOG_NOT_FOUND", + "surfaceId": "s", + "message": "has no default catalogId" + } + } + } + }, + { + "name": "test_v10_get_renderer_data_model_includes_surface_without_default", + "description": "A v1.0 surface created without a default catalog keeps the protocol version of the message that created it, so it is part of a v1.0 data model request. Its data model is empty.", + "action": "get_renderer_data_model", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "sendDataModel": true + } + } + ], + "args": { + "version": "1.0" + }, + "expect": { + "version": "1.0", + "surfaces": { + "s": {} + } + } + }, + { + "name": "test_v10_get_renderer_data_model_excludes_surface_without_default_from_v09", + "description": "A v1.0 surface created without a default catalog is not part of a v0.9 data model request, so the request finds no surface to report.", + "action": "get_renderer_data_model", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "sendDataModel": true + } + } + ], + "args": { + "version": "v0.9" + }, + "expect": null + }, + { + "name": "test_v10_get_renderer_data_model_filters_by_surface_protocol_version", + "description": "With a v1.0 surface that has no default catalog and a v0.9 surface side by side, a v0.9 data model request reports only the v0.9 surface. The surfaces are created in separate steps, one batch per protocol version.", + "action": "get_renderer_data_model", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json", + "specification/v0_9/catalogs/basic/catalog.json" + ], + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s", + "sendDataModel": true + } + } + ] + }, + { + "messages": [ + { + "version": "v0.9", + "createSurface": { + "surfaceId": "t", + "catalogId": "https://a2ui.org/specification/v0_9/catalogs/basic/catalog.json", + "sendDataModel": true + } + } + ] + } + ], + "args": { + "version": "v0.9" + }, + "expect": { + "version": "v0.9", + "surfaces": { + "t": {} + } + } + }, + { + "name": "test_v10_update_components_stores_component_metadata", + "description": "Tests that v1.0 updateComponents keeps ComponentCommon.metadata apart from component properties and exposes it as component metadata.", + "action": "process_messages", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_v10_comp_meta", + "catalogId": "test-catalog-v10", + "metadata": { + "extensions": { + "com_example_session": "abc" + } + } + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_v10_comp_meta", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Hello", + "metadata": { + "extensions": { + "com_example_trace": "t1" + } + } + } + ] + } + } + ], + "expect": { + "surfaces": { + "s_v10_comp_meta": { + "exists": true, + "metadata": { + "extensions": { + "com_example_session": "abc" + } + }, + "components": [ + { + "id": "root", + "component": "Text", + "text": "Hello", + "metadata": { + "extensions": { + "com_example_trace": "t1" + } + } + } + ] + } + } + } + }, + { + "name": "test_v10_update_components_replaces_component_metadata", + "description": "Tests that a later v1.0 updateComponents entry replaces a component's metadata in full rather than merging it.", + "action": "process_messages", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_v10_comp_meta_replace", + "catalogId": "test-catalog-v10" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_v10_comp_meta_replace", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Hello", + "metadata": { + "extensions": { + "com_example_trace": "t1", + "com_example_state": "draft" + } + } + } + ] + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_v10_comp_meta_replace", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Hello", + "metadata": { + "extensions": { + "com_example_state": "final" + } + } + } + ] + } + } + ], + "expect": { + "surfaces": { + "s_v10_comp_meta_replace": { + "exists": true, + "components": [ + { + "id": "root", + "component": "Text", + "text": "Hello", + "metadata": { + "extensions": { + "com_example_state": "final" + } + } + } + ] + } + } + } + }, + { + "name": "test_v10_update_components_omitted_metadata_clears_component_metadata", + "description": "Tests that a v1.0 updateComponents entry without metadata clears the component's previously stored metadata, because an update replaces the definition in full.", + "action": "process_messages", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_v10_comp_meta_clear", + "catalogId": "test-catalog-v10" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_v10_comp_meta_clear", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Hello", + "metadata": { + "extensions": { + "com_example_trace": "t1" + } + } + } + ] + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_v10_comp_meta_clear", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Hello" + } + ] + } + } + ], + "expect": { + "surfaces": { + "s_v10_comp_meta_clear": { + "exists": true, + "components": [ + { + "id": "root", + "component": "Text", + "text": "Hello", + "metadata": null + } + ] + } + } + } + }, + { + "name": "test_v10_update_components_metadata_extension_key_must_be_identifier", + "description": "Tests that v1.0 updateComponents rejects component metadata extensions whose key is not a Unicode identifier (UAX 31) when the component is first created.", + "action": "process_messages", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_v10_comp_meta_bad_key", + "catalogId": "test-catalog-v10" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_v10_comp_meta_bad_key", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Hello", + "metadata": { + "extensions": { + "com.example.trace": "t1" + } + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError" + } + }, + { + "name": "test_v10_update_components_metadata_extension_key_must_be_identifier_on_update", + "description": "Tests that v1.0 updateComponents rejects an invalid UAX 31 extension key on an existing component and leaves the previously stored metadata untouched.", + "action": "process_messages", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_v10_comp_meta_bad_key_update", + "catalogId": "test-catalog-v10" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_v10_comp_meta_bad_key_update", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Hello", + "metadata": { + "extensions": { + "com_example_trace": "t1" + } + } + } + ] + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_v10_comp_meta_bad_key_update", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Hello", + "metadata": { + "extensions": { + "com-example-trace": "t2" + } + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError" + } + }, + { + "name": "test_v10_permissive_incremental_streaming_out_of_order_before_root", + "description": "Multi-step strictMode: false incremental streaming case verifying that out-of-order child components arriving before root emit zero errors and assemble cleanly once root arrives.", + "action": "process_messages", + "protocolVersion": "1.0", + "strictMode": false, + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_stream", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_stream", + "components": [ + { + "id": "leaf_text", + "component": "Text", + "text": "Streamed Leaf" + } + ] + } + } + ], + "expect": { + "surfaces": { + "s_stream": { + "exists": true, + "components": [ + { + "id": "leaf_text", + "component": "Text", + "text": "Streamed Leaf" + } + ] + } + } + } + }, + { + "messages": [ + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_stream", + "components": [ + { + "id": "mid_card", + "component": "Card", + "child": "leaf_text" + } + ] + } + } + ], + "expect": { + "surfaces": { + "s_stream": { + "exists": true, + "components": [ + { + "id": "leaf_text", + "component": "Text", + "text": "Streamed Leaf" + }, + { + "id": "mid_card", + "component": "Card", + "child": "leaf_text" + } + ] + } + } + } + }, + { + "messages": [ + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s_stream", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "mid_card" + ] + } + ] + } + } + ], + "expect": { + "surfaces": { + "s_stream": { + "exists": true, + "components": [ + { + "id": "leaf_text", + "component": "Text", + "text": "Streamed Leaf" + }, + { + "id": "mid_card", + "component": "Card", + "child": "leaf_text" + }, + { + "id": "root", + "component": "Column", + "children": [ + "mid_card" + ] + } + ] + } + } + } + } + ] + } +] \ No newline at end of file diff --git a/package/__fixtures__/a2ui-conformance/core/multi_catalog.json b/package/__fixtures__/a2ui-conformance/core/multi_catalog.json new file mode 100644 index 0000000000..d12fcec8f1 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/core/multi_catalog.json @@ -0,0 +1,237 @@ +[ + { + "name": "test_multi_catalog_explicit_catalog_id_resolution", + "description": "Verifies resolving component definition using explicit catalogId specified on component instance.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "select_catalog", + "args": { + "surface": { + "id": "main_surface", + "defaultCatalogId": "basic", + "supportedCatalogIds": [ + "basic", + "chart_catalog" + ] + }, + "components": { + "c1": { + "component": "Text", + "catalogId": "basic", + "text": "Default Catalog Component" + }, + "c2": { + "component": "LineChart", + "catalogId": "chart_catalog", + "data": "/series/line1" + } + } + }, + "expectSelected": "chart_catalog" + }, + { + "name": "test_multi_catalog_surface_default_fallback", + "description": "Verifies component falling back to surface defaultCatalogId when catalogId is omitted.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "select_catalog", + "args": { + "surface": { + "id": "main_surface", + "defaultCatalogId": "basic", + "supportedCatalogIds": [ + "basic", + "chart_catalog" + ] + }, + "components": { + "c1": { + "component": "Button", + "title": "Click Me" + } + } + }, + "expectSelected": "basic" + }, + { + "name": "test_multi_catalog_unregistered_catalog_error", + "description": "Verifies error thrown when component specifies catalogId not listed in surface supportedCatalogIds.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "select_catalog", + "args": { + "surface": { + "id": "main_surface", + "defaultCatalogId": "basic", + "supportedCatalogIds": [ + "basic" + ] + }, + "components": { + "c1": { + "component": "CustomWidget", + "catalogId": "unregistered_vendor_catalog" + } + } + }, + "expectError": { + "category": "CatalogError", + "message": "Catalog 'unregistered_vendor_catalog' is not supported by surface 'main_surface'." + } + }, + { + "name": "test_multi_catalog_version_mismatch_error", + "description": "Verifies error thrown when mixed catalogs specify conflicting protocol versions.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "select_catalog", + "args": { + "surface": { + "id": "main_surface", + "defaultCatalogId": "basic_v1_0", + "supportedCatalogIds": [ + "basic_v1_0", + "legacy_v0_9_catalog" + ] + }, + "catalogs": { + "basic_v1_0": { + "protocolVersion": "1.0" + }, + "legacy_v0_9_catalog": { + "protocolVersion": "v0.9" + } + } + }, + "expectError": { + "category": "CatalogError", + "message": "Protocol version mismatch: cannot mix catalog 'legacy_v0_9_catalog' (v0.9) with surface version 1.0." + } + }, + { + "name": "test_multi_catalog_function_resolution_precedence", + "description": "Verifies function calls with explicit catalogId resolve to the specified catalog.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "select_catalog", + "args": { + "surface": { + "id": "main_surface", + "defaultCatalogId": "basic", + "supportedCatalogIds": [ + "basic", + "custom_math_catalog" + ] + }, + "functionCall": { + "call": "add", + "catalogId": "custom_math_catalog", + "args": { + "a": 5, + "b": 10 + } + } + }, + "expectSelected": "custom_math_catalog" + }, + { + "name": "test_multi_catalog_cross_catalog_component_composition", + "description": "Verifies creating a surface with multiple supportedCatalogIds and rendering components from distinct catalogs in a single hierarchy.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "select_catalog", + "args": { + "surface": { + "id": "main_surface", + "defaultCatalogId": "basic", + "supportedCatalogIds": [ + "basic", + "chart_catalog", + "map_catalog" + ] + }, + "components": { + "c1": { + "component": "Text", + "catalogId": "basic", + "text": "Dashboard" + }, + "c2": { + "component": "BarChart", + "catalogId": "chart_catalog" + }, + "c3": { + "component": "GeoMap", + "catalogId": "map_catalog" + } + } + }, + "expectSelected": "map_catalog" + }, + { + "name": "test_multi_catalog_function_call_catalog_id_override", + "description": "Verifies a function call naming a non-default catalogId is evaluated by that catalog.", + "action": "validate", + "catalogs": [ + { + "catalogId": "app-cat", + "protocolVersion": "1.0", + "components": { + "Text": { + "type": "object", + "properties": { + "text": { + "$ref": "#/$defs/DynamicString" + } + } + } + } + } + ], + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "main", + "catalogId": "app-cat", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "formatDate", + "catalogId": "basic", + "args": { + "value": "2026-09-04T12:00:00Z", + "format": "yyyy-MM-dd" + } + } + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "main": { + "components": { + "root": { + "component": "Text", + "text": "2026-09-04" + } + } + } + } + } + } +] \ No newline at end of file diff --git a/package/__fixtures__/a2ui-conformance/core/reserved_keys.json b/package/__fixtures__/a2ui-conformance/core/reserved_keys.json new file mode 100644 index 0000000000..44b13ef899 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/core/reserved_keys.json @@ -0,0 +1,303 @@ +[ + { + "name": "test_reserved_at_path_valid", + "description": "Valid v1.0 DataBinding using the reserved @path prefix.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@path": "/user/displayName" + } + } + ] + } + } + ] + }, + { + "name": "test_reserved_at_call_valid", + "description": "Valid v1.0 FunctionCall using the reserved @call prefix.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "formatString", + "args": { + "value": "Hello world" + } + } + } + ] + } + } + ] + }, + { + "name": "test_reserved_at_call_index", + "description": "Valid v1.0 FunctionCall invoking the @index system context function within a List template.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "basic", + "components": [ + { + "id": "root", + "component": "List", + "children": { + "componentId": "itemTemplate", + "path": "/items" + } + }, + { + "id": "itemTemplate", + "component": "Text", + "text": { + "@call": "@index", + "args": { + "offset": 1 + } + } + } + ] + } + } + ] + }, + { + "name": "test_plain_object_with_literal_path_and_call", + "description": "A plain object containing 'path' and 'call' keys is valid in v1.0 and is not intercepted as a dynamic DataBinding or FunctionCall directive.", + "catalog": { + "catalogId": "custom", + "protocolVersion": "1.0", + "components": { + "CustomCard": { + "type": "object", + "properties": { + "details": { + "$ref": "common_types.json#/$defs/DynamicValue" + } + } + } + } + }, + "action": "validate", + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom", + "components": [ + { + "id": "root", + "component": "CustomCard", + "details": { + "path": "/var/log/system.log", + "call": "audit" + } + } + ] + } + } + ] + }, + { + "name": "test_plain_object_escaped_doubled_at_key", + "description": "Literal @ keys escaped with prefix doubling (@@path, @@type) validate successfully as plain object keys.", + "catalog": { + "catalogId": "custom", + "protocolVersion": "1.0", + "components": { + "CustomCard": { + "type": "object", + "properties": { + "details": { + "$ref": "common_types.json#/$defs/DynamicValue" + } + } + } + } + }, + "action": "validate", + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom", + "components": [ + { + "id": "root", + "component": "CustomCard", + "details": { + "@@path": "/static/file", + "@@type": "Resource" + } + } + ] + } + } + ] + }, + { + "name": "test_disallowed_unknown_single_at_key", + "description": "Dynamic objects containing unknown single-@ keys (^@([^@]|$)) are disallowed to preserve backward compatibility for future directives.", + "catalog": { + "catalogId": "custom", + "protocolVersion": "1.0", + "components": { + "CustomCard": { + "type": "object", + "properties": { + "details": { + "$ref": "common_types.json#/$defs/DynamicValue" + } + } + } + } + }, + "action": "validate", + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom", + "components": [ + { + "id": "root", + "component": "CustomCard", + "details": { + "@if": true, + "then": "active" + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError" + } + }, + { + "name": "test_disallowed_bare_at_key", + "description": "A key consisting of a single @ with no following character is also reserved and disallowed. It must be escaped as \"@@\" to appear as literal data.", + "catalog": { + "catalogId": "custom", + "protocolVersion": "1.0", + "components": { + "CustomCard": { + "type": "object", + "properties": { + "details": { + "$ref": "common_types.json#/$defs/DynamicValue" + } + } + } + } + }, + "action": "validate", + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom", + "components": [ + { + "id": "root", + "component": "CustomCard", + "details": { + "@": "reserved" + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError" + } + }, + { + "name": "test_escaped_doubled_at_unescaping", + "description": "Verifies that doubled-@ keys ('@@path') are unescaped to '@path' during dynamic evaluation in v1.0.", + "catalog": { + "catalogId": "custom", + "protocolVersion": "1.0", + "components": { + "CustomCard": { + "type": "object", + "properties": { + "details": { + "$ref": "common_types.json#/$defs/DynamicValue" + } + } + } + } + }, + "action": "process_messages", + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom", + "components": [ + { + "id": "root", + "component": "CustomCard", + "details": { + "@@path": "/static/file" + } + } + ] + } + } + ], + "expect": { + "surfaces": { + "s1": { + "components": [ + { + "id": "root", + "component": "CustomCard", + "details": { + "@path": "/static/file" + } + } + ] + } + } + } + } +] \ No newline at end of file diff --git a/package/__fixtures__/a2ui-conformance/core/rpc_functions.json b/package/__fixtures__/a2ui-conformance/core/rpc_functions.json new file mode 100644 index 0000000000..873b662f45 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/core/rpc_functions.json @@ -0,0 +1,908 @@ +[ + { + "name": "test_rpc_call_renderer_function_success", + "description": "Verifies agent-initiated callRendererFunction on a function with allowedCallers=rendererOrAgent.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "message": { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-101", + "callFunction": { + "@call": "playMedia", + "catalogId": "media_catalog", + "args": { + "mediaId": "video-789", + "autoplay": true + } + } + } + }, + "functionMetadata": { + "playMedia": { + "allowedCallers": "rendererOrAgent", + "requiresUserActivation": false + } + } + }, + "expect": { + "response": { + "version": "1.0", + "rendererFunctionResponse": { + "functionCallId": "call-101", + "value": { + "playing": true, + "timestamp": 0 + } + } + } + } + }, + { + "name": "test_rpc_call_renderer_function_unauthorized_caller", + "description": "Verifies callRendererFunction fails with INVALID_FUNCTION_CALL when allowedCallers is rendererOnly.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "message": { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-102", + "callFunction": { + "@call": "internalStateReset", + "catalogId": "system_catalog", + "args": {} + } + } + }, + "functionMetadata": { + "internalStateReset": { + "allowedCallers": "rendererOnly", + "requiresUserActivation": false + } + } + }, + "expect": { + "response": { + "version": "1.0", + "rendererFunctionResponse": { + "functionCallId": "call-102", + "error": { + "code": "INVALID_FUNCTION_CALL", + "message": "Function 'internalStateReset' cannot be called by agent (allowedCallers is rendererOnly)." + } + } + } + } + }, + { + "name": "test_rpc_call_renderer_function_requires_user_activation", + "description": "Verifies callRendererFunction execution when requiresUserActivation is true and user gesture is present.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "userActivationPresent": true, + "message": { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-103", + "callFunction": { + "@call": "openExternalUrl", + "catalogId": "basic", + "args": { + "url": "https://example.com/details" + } + } + } + }, + "functionMetadata": { + "openExternalUrl": { + "allowedCallers": "rendererOrAgent", + "requiresUserActivation": true + } + } + }, + "expect": { + "response": { + "version": "1.0", + "rendererFunctionResponse": { + "functionCallId": "call-103", + "value": { + "opened": true + } + } + } + } + }, + { + "name": "test_rpc_call_agent_function_dispatch", + "description": "Verifies renderer-initiated callAgentFunction message generation and response correlation via functionCallId.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "outboundCall": { + "surfaceId": "main_surface", + "functionCallId": "agent-call-55", + "callFunction": { + "@call": "queryInventory", + "catalogId": "store_catalog", + "args": { + "sku": "ITEM-404" + } + } + }, + "inboundResponse": { + "version": "1.0", + "agentFunctionResponse": { + "functionCallId": "agent-call-55", + "value": { + "inStock": 12, + "price": "$29.99" + } + } + } + }, + "expect": { + "correlatedCallId": "agent-call-55", + "result": { + "inStock": 12, + "price": "$29.99" + } + } + }, + { + "name": "test_rpc_call_renderer_function_user_activation_denied", + "description": "Verifies callRendererFunction fails with INVALID_FUNCTION_CALL when requiresUserActivation is true but user gesture context is absent.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "userActivationPresent": false, + "message": { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-104", + "callFunction": { + "@call": "openExternalUrl", + "catalogId": "basic", + "args": { + "url": "https://example.com/details" + } + } + } + }, + "functionMetadata": { + "openExternalUrl": { + "allowedCallers": "rendererOrAgent", + "requiresUserActivation": true + } + } + }, + "expect": { + "response": { + "version": "1.0", + "rendererFunctionResponse": { + "functionCallId": "call-104", + "error": { + "code": "INVALID_FUNCTION_CALL", + "message": "Function 'openExternalUrl' requires user activation context to execute." + } + } + } + } + }, + { + "name": "test_rpc_call_renderer_function_execution_exception", + "description": "Verifies callRendererFunction handles runtime function execution exceptions and returns EXECUTION_ERROR.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "message": { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-105", + "callFunction": { + "@call": "failingFunction", + "catalogId": "basic", + "args": {} + } + } + }, + "functionMetadata": { + "failingFunction": { + "allowedCallers": "rendererOrAgent", + "requiresUserActivation": false + } + } + }, + "expect": { + "response": { + "version": "1.0", + "rendererFunctionResponse": { + "functionCallId": "call-105", + "error": { + "code": "EXECUTION_ERROR", + "message": "An error occurred during function execution." + } + } + } + } + }, + { + "name": "test_rpc_call_renderer_function_agent_only_boundary", + "description": "Verifies agent-initiated callRendererFunction on a function with allowedCallers=agentOnly.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "message": { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-106", + "callFunction": { + "@call": "syncState", + "catalogId": "basic", + "args": {} + } + } + }, + "functionMetadata": { + "syncState": { + "allowedCallers": "agentOnly", + "requiresUserActivation": false + } + } + }, + "expect": { + "response": { + "version": "1.0", + "rendererFunctionResponse": { + "functionCallId": "call-106", + "value": null + } + } + } + }, + { + "name": "test_rpc_call_renderer_function_unregistered_function", + "description": "Verifies callRendererFunction fails with INVALID_FUNCTION_CALL when calling an unregistered function.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "message": { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-107", + "callFunction": { + "@call": "unknownFunc", + "catalogId": "basic", + "args": {} + } + } + }, + "functionMetadata": {} + }, + "expect": { + "response": { + "version": "1.0", + "rendererFunctionResponse": { + "functionCallId": "call-107", + "error": { + "code": "INVALID_FUNCTION_CALL", + "message": "Function not found: unknownFunc" + } + } + } + } + }, + { + "name": "test_rpc_call_renderer_function_unregistered_catalog", + "description": "Verifies callRendererFunction fails with INVALID_FUNCTION_CALL when specifying an unregistered catalogId.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "message": { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-108", + "callFunction": { + "@call": "playMedia", + "catalogId": "non_existent_catalog", + "args": {} + } + } + }, + "functionMetadata": { + "playMedia": { + "allowedCallers": "rendererOrAgent" + } + } + }, + "expect": { + "response": { + "version": "1.0", + "rendererFunctionResponse": { + "functionCallId": "call-108", + "error": { + "code": "INVALID_FUNCTION_CALL", + "message": "Catalog not found: non_existent_catalog" + } + } + } + } + }, + { + "name": "test_rpc_call_renderer_function_invalid_payload_missing_call_function", + "description": "Verifies envelope/schema validation error when callRendererFunction is missing required callFunction property.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "message": { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-109" + } + } + }, + "expect": { + "error": { + "code": "INVALID_FUNCTION_CALL", + "functionCallId": "call-109", + "message": "callFunction" + } + }, + "expectError": { + "category": "ValidationError", + "functionCallId": "call-109", + "rendererError": { + "version": "1.0", + "error": { + "code": "INVALID_FUNCTION_CALL", + "functionCallId": "call-109", + "message": "callFunction" + } + } + } + }, + { + "name": "test_rpc_call_renderer_function_invalid_payload_non_object_args", + "description": "Verifies envelope/schema validation error when callRendererFunction has non-object args.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "message": { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-110", + "callFunction": { + "@call": "playMedia", + "catalogId": "media_catalog", + "args": "not-an-object" + } + } + } + }, + "expect": { + "error": { + "code": "INVALID_FUNCTION_CALL", + "functionCallId": "call-110", + "message": "args" + } + }, + "expectError": { + "category": "ValidationError", + "functionCallId": "call-110", + "rendererError": { + "version": "1.0", + "error": { + "code": "INVALID_FUNCTION_CALL", + "functionCallId": "call-110", + "message": "args" + } + } + } + }, + { + "name": "test_rpc_call_renderer_function_unknown_surface_id", + "description": "Verifies callRendererFunction targeting an unknown surfaceId returns rendererFunctionResponse with INVALID_FUNCTION_CALL.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "message": { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-111", + "surfaceId": "unknown_surface", + "callFunction": { + "@call": "playMedia", + "catalogId": "media_catalog", + "args": { + "mediaId": "video-789" + } + } + } + }, + "functionMetadata": { + "playMedia": { + "allowedCallers": "rendererOrAgent" + } + } + }, + "expect": { + "response": { + "version": "1.0", + "rendererFunctionResponse": { + "functionCallId": "call-111", + "error": { + "code": "INVALID_FUNCTION_CALL", + "message": "unknown_surface" + } + } + } + } + }, + { + "name": "test_rpc_call_renderer_function_invalid_payload_missing_catalog_id", + "description": "Verifies a callRendererFunction whose callFunction has no catalogId fails schema validation, even when a held catalog defines the function.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "message": { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-110", + "callFunction": { + "@call": "playMedia", + "args": { + "mediaId": "video-789" + } + } + } + }, + "functionMetadata": { + "playMedia": { + "allowedCallers": "rendererOrAgent", + "requiresUserActivation": false + } + } + }, + "expect": { + "error": { + "message": "catalogId" + } + } + }, + { + "name": "test_rpc_agent_function_response_inbound", + "description": "Verifies inbound agentFunctionResponse message processing by MessageProcessor.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "message": { + "version": "1.0", + "agentFunctionResponse": { + "functionCallId": "call-201", + "value": { + "status": "verified" + } + } + } + }, + "expect": { + "response": null + } + }, + { + "name": "test_rpc_call_renderer_function_invalid_argument_schema", + "description": "Verifies callRendererFunction returns INVALID_FUNCTION_CALL when argument schema validation fails.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "message": { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-301", + "callFunction": { + "@call": "calculateTax", + "catalogId": "basic", + "args": { + "amount": "invalid_string_amount" + } + } + } + }, + "functionMetadata": { + "calculateTax": { + "allowedCallers": "rendererOrAgent", + "parameters": { + "type": "object", + "properties": { + "amount": { + "type": "number" + } + }, + "required": [ + "amount" + ] + } + } + } + }, + "expect": { + "response": { + "version": "1.0", + "rendererFunctionResponse": { + "functionCallId": "call-301", + "error": { + "code": "INVALID_FUNCTION_CALL", + "message": "Invalid" + } + } + } + } + }, + { + "name": "test_rpc_call_renderer_function_protocol_version_mismatch", + "description": "Verifies callRendererFunction returns INVALID_FUNCTION_CALL when catalog protocol version mismatches message protocol version.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "catalogId": "legacy_cat", + "catalogVersion": "v0.9", + "message": { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-302", + "callFunction": { + "@call": "playMedia", + "catalogId": "legacy_cat", + "args": { + "mediaId": "vid1" + } + } + } + }, + "functionMetadata": { + "playMedia": { + "allowedCallers": "rendererOrAgent" + } + } + }, + "expect": { + "response": { + "version": "1.0", + "rendererFunctionResponse": { + "functionCallId": "call-302", + "error": { + "code": "INVALID_FUNCTION_CALL", + "message": "does not match message protocol version" + } + } + } + } + }, + { + "name": "test_rpc_call_agent_function_timeout_handling", + "description": "Verifies callAgentFunction timeout emitting TIMEOUT error code.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "outboundCall": { + "surfaceId": "main_surface", + "functionCallId": "call-timeout-1", + "timeoutMs": 10, + "callFunction": { + "@call": "longRunningTask", + "catalogId": "basic", + "args": {} + } + } + }, + "expectError": { + "code": "TIMEOUT" + } + }, + { + "name": "test_rpc_call_agent_function_duplicate_call_id_error", + "description": "Verifies callAgentFunction rejecting duplicate pending functionCallId with DUPLICATE error code.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "outboundCall": { + "surfaceId": "main_surface", + "functionCallId": "call-dup-1", + "callFunction": { + "@call": "fetchData", + "catalogId": "basic", + "args": {} + } + }, + "secondOutboundCall": { + "surfaceId": "main_surface", + "functionCallId": "call-dup-1", + "callFunction": { + "@call": "fetchData", + "catalogId": "basic", + "args": {} + } + } + }, + "expectError": { + "code": "DUPLICATE" + } + }, + { + "name": "test_rpc_call_renderer_function_with_surface_id_and_dynamic_args", + "description": "Verifies callRendererFunction with surfaceId resolves @path and @call in callFunction.args against the target surface's root DataContext (/).", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "surfaceId": "media_surface", + "dataModel": { + "media": { + "id": "video-456" + }, + "prefix": "now-" + }, + "message": { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-401", + "surfaceId": "media_surface", + "callFunction": { + "@call": "echoArgs", + "catalogId": "media_catalog", + "args": { + "mediaId": { + "@path": "/media/id" + }, + "formatted": { + "@call": "formatString", + "catalogId": "basic", + "args": { + "value": "${/prefix}${/media/id}" + } + } + } + } + } + }, + "functionMetadata": { + "echoArgs": { + "allowedCallers": "rendererOrAgent", + "requiresUserActivation": false + } + } + }, + "expect": { + "response": { + "version": "1.0", + "rendererFunctionResponse": { + "functionCallId": "call-401", + "value": { + "mediaId": "video-456", + "formatted": "now-video-456" + } + } + } + } + }, + { + "name": "test_rpc_call_renderer_function_without_surface_id_isolated_data_model", + "description": "Verifies that when surfaceId is omitted on callRendererFunction, @path in callFunction.args resolves against an isolated empty {} data model even when a surface with data exists.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "surfaceId": "existing_surface", + "dataModel": { + "secret": "should-not-leak" + }, + "message": { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-402", + "callFunction": { + "@call": "echoArgs", + "catalogId": "media_catalog", + "args": { + "leaked": { + "@path": "/secret" + }, + "constant": "safe" + } + } + } + }, + "functionMetadata": { + "echoArgs": { + "allowedCallers": "rendererOrAgent", + "requiresUserActivation": false + } + } + }, + "expect": { + "response": { + "version": "1.0", + "rendererFunctionResponse": { + "functionCallId": "call-402", + "value": { + "leaked": null, + "constant": "safe" + } + } + } + } + }, + { + "name": "test_rpc_call_renderer_function_index_at_root_execution_error", + "description": "Verifies that evaluating @index at the root DataContext in callRendererFunction fails with EXECUTION_ERROR.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "surfaceId": "main_surface", + "dataModel": {}, + "message": { + "version": "1.0", + "callRendererFunction": { + "functionCallId": "call-403", + "surfaceId": "main_surface", + "callFunction": { + "@call": "echoArgs", + "catalogId": "basic", + "args": { + "idx": { + "@call": "@index" + } + } + } + } + }, + "functionMetadata": { + "echoArgs": { + "allowedCallers": "rendererOrAgent", + "requiresUserActivation": false + } + } + }, + "expect": { + "response": { + "version": "1.0", + "rendererFunctionResponse": { + "functionCallId": "call-403", + "error": { + "code": "EXECUTION_ERROR", + "message": "index" + } + } + } + } + }, + { + "name": "test_rpc_ui_binding_triggers_call_agent_function_async_resolution", + "description": "Verifies a component property bound to an agent function initially resolves to null/undefined while emitting outbound callAgentFunction, and updates reactively when agentFunctionResponse arrives.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "uiBindingScenario": { + "surfaceId": "main_surface", + "catalogId": "store_catalog", + "component": { + "id": "root", + "component": "Text", + "text": { + "@call": "fetchRemoteGreeting", + "catalogId": "store_catalog", + "args": { + "user": "Ada" + } + } + }, + "inboundResponseValue": "Hello from agent, Ada!" + } + }, + "expect": { + "initialValue": null, + "outboundCall": { + "version": "1.0", + "callAgentFunction": { + "surfaceId": "main_surface", + "callFunction": { + "@call": "fetchRemoteGreeting", + "catalogId": "store_catalog", + "args": { + "user": "Ada" + } + } + } + }, + "resolvedValue": "Hello from agent, Ada!" + } + }, + { + "name": "test_rpc_ui_action_missing_local_function_fallback_to_agent", + "description": "Verifies an undeclared function invoked from a UI action in relaxed mode falls back to dispatching callAgentFunction to the agent.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "handle_rpc", + "args": { + "uiActionFallbackScenario": { + "surfaceId": "main_surface", + "catalogId": "store_catalog", + "actionPayload": { + "functionCall": { + "@call": "undeclaredAgentAction", + "catalogId": "store_catalog", + "args": { + "orderId": "ord-99" + } + } + }, + "inboundResponseValue": { + "accepted": true + } + } + }, + "expect": { + "outboundCall": { + "version": "1.0", + "callAgentFunction": { + "surfaceId": "main_surface", + "callFunction": { + "@call": "undeclaredAgentAction", + "catalogId": "store_catalog", + "args": { + "orderId": "ord-99" + } + } + } + }, + "result": { + "accepted": true + } + } + } +] \ No newline at end of file diff --git a/package/__fixtures__/a2ui-conformance/core/validation_result.json b/package/__fixtures__/a2ui-conformance/core/validation_result.json new file mode 100644 index 0000000000..14ebd7c363 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/core/validation_result.json @@ -0,0 +1,599 @@ +[ + { + "name": "test_validation_result_dynamic_object_return", + "description": "Verifies CheckRule evaluation capturing dynamic ValidationResult object returned from custom validation function.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "form_surface", + "catalogId": "basic", + "dataModel": { + "email": "invalid-email-address" + }, + "components": [ + { + "id": "root", + "component": "TextField", + "label": "Email", + "value": { + "@path": "/email" + }, + "checks": [ + { + "condition": { + "@call": "email", + "args": { + "value": { + "@path": "/email" + } + } + } + } + ] + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "form_surface": { + "isValid": { + "root": false + }, + "validationErrors": { + "root": [ + "Please enter a valid email address (e.g. user@example.com)." + ] + }, + "validationResults": { + "root": [ + { + "valid": false, + "code": "INVALID_EMAIL", + "message": "Please enter a valid email address (e.g. user@example.com).", + "severity": "error" + } + ] + }, + "validationResult": { + "root": { + "valid": false, + "code": "INVALID_EMAIL", + "message": "Please enter a valid email address (e.g. user@example.com).", + "severity": "error" + } + } + } + } + } + }, + { + "name": "test_validation_result_boolean_fallback", + "description": "Verifies CheckRule evaluation when validation function returns primitive boolean, falling back to CheckRule message.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "form_surface", + "catalogId": "basic", + "dataModel": { + "age": 15 + }, + "components": [ + { + "id": "root", + "component": "TextField", + "label": "Age", + "value": { + "@path": "/age" + }, + "checks": [ + { + "condition": { + "@call": "numeric", + "args": { + "value": { + "@path": "/age" + }, + "min": 18 + } + }, + "message": "Must be 18 or older." + } + ] + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "form_surface": { + "isValid": { + "root": false + }, + "validationErrors": { + "root": [ + "Must be 18 or older." + ] + }, + "validationResults": { + "root": [ + { + "valid": false, + "message": "Must be 18 or older.", + "severity": "error" + } + ] + }, + "validationResult": { + "root": { + "valid": false, + "message": "Must be 18 or older." + } + } + } + } + } + }, + { + "name": "test_validation_result_warning_and_info_non_blocking", + "description": "Verifies that CheckRules returning severity warning or info populate validationResults while keeping isValid true and validationErrors empty.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "form_surface", + "catalogId": "basic", + "dataModel": { + "password": "secret", + "warningCheck": { + "valid": false, + "code": "WEAK_PASSWORD", + "message": "Password could be stronger.", + "severity": "warning" + }, + "infoCheck": { + "valid": false, + "code": "PASSWORD_HINT", + "message": "Consider adding symbols.", + "severity": "info" + } + }, + "components": [ + { + "id": "root", + "component": "TextField", + "label": "Password", + "value": { + "@path": "/password" + }, + "checks": [ + { + "condition": { + "@path": "/warningCheck" + } + }, + { + "condition": { + "@path": "/infoCheck" + } + } + ] + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "form_surface": { + "isValid": { + "root": true + }, + "validationErrors": { + "root": [] + }, + "validationResults": { + "root": [ + { + "valid": false, + "code": "WEAK_PASSWORD", + "message": "Password could be stronger.", + "severity": "warning" + }, + { + "valid": false, + "code": "PASSWORD_HINT", + "message": "Consider adding symbols.", + "severity": "info" + } + ] + } + } + } + } + }, + { + "name": "test_validation_result_multiple_rules_preserve_order", + "description": "Verifies that a component with two failing error rules and one failing warning rule preserves declaration order in validationErrors and validationResults.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "form_surface", + "catalogId": "basic", + "dataModel": { + "email": "bad", + "warningCheck": { + "valid": false, + "code": "DOMAIN_WARNING", + "message": "External domain requires approval.", + "severity": "warning" + } + }, + "components": [ + { + "id": "root", + "component": "TextField", + "label": "Corporate Email", + "value": { + "@path": "/email" + }, + "checks": [ + { + "condition": { + "@call": "email", + "args": { + "value": { + "@path": "/email" + } + } + }, + "message": "Must be a valid email address." + }, + { + "condition": { + "@path": "/warningCheck" + } + }, + { + "condition": { + "@call": "length", + "args": { + "value": { + "@path": "/email" + }, + "min": 10 + } + }, + "message": "Email must be at least 10 characters." + } + ] + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "form_surface": { + "isValid": { + "root": false + }, + "validationErrors": { + "root": [ + "Must be a valid email address.", + "Email must be at least 10 characters." + ] + }, + "validationResults": { + "root": [ + { + "valid": false, + "code": "INVALID_EMAIL", + "message": "Must be a valid email address.", + "severity": "error" + }, + { + "valid": false, + "code": "DOMAIN_WARNING", + "message": "External domain requires approval.", + "severity": "warning" + }, + { + "valid": false, + "message": "Email must be at least 10 characters.", + "severity": "error" + } + ] + } + } + } + } + }, + { + "name": "test_validation_result_reactive_recovery_on_data_update", + "description": "Verifies that failing validation recovers reactively when updateDataModel provides a valid value in a subsequent step.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "form_surface", + "catalogId": "basic", + "dataModel": { + "email": "not-an-email" + }, + "components": [ + { + "id": "root", + "component": "TextField", + "label": "Email", + "value": { + "@path": "/email" + }, + "checks": [ + { + "condition": { + "@call": "email", + "args": { + "value": { + "@path": "/email" + } + } + } + } + ] + } + ] + } + } + ], + "expect": { + "surfaces": { + "form_surface": { + "isValid": { + "root": false + }, + "validationErrors": { + "root": [ + "Please enter a valid email address (e.g. user@example.com)." + ] + }, + "validationResults": { + "root": [ + { + "valid": false, + "code": "INVALID_EMAIL", + "message": "Please enter a valid email address (e.g. user@example.com).", + "severity": "error" + } + ] + } + } + } + } + }, + { + "payload": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "form_surface", + "path": "/email", + "value": "user@example.com" + } + } + ], + "expect": { + "surfaces": { + "form_surface": { + "isValid": { + "root": true + }, + "validationErrors": { + "root": [] + }, + "validationResults": { + "root": [] + } + } + } + } + } + ] + }, + { + "name": "test_validation_result_coercion_in_logical_and_or_not", + "description": "Verifies that ValidationResult objects returned by functions inside logical and, or, and not expressions are coerced via their .valid boolean property.", + "catalog": { + "protocolVersion": "1.0" + }, + "action": "validate", + "steps": [ + { + "payload": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "form_surface", + "catalogId": "basic", + "dataModel": { + "email": "invalid-email", + "allowOverride": true + }, + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "and_field", + "or_field", + "not_field" + ] + }, + { + "id": "and_field", + "component": "TextField", + "label": "And Field", + "value": { + "@path": "/email" + }, + "checks": [ + { + "condition": { + "@call": "and", + "args": { + "values": [ + { + "@call": "email", + "args": { + "value": { + "@path": "/email" + } + } + }, + { + "@path": "/allowOverride" + } + ] + } + }, + "message": "And check failed." + } + ] + }, + { + "id": "or_field", + "component": "TextField", + "label": "Or Field", + "value": { + "@path": "/email" + }, + "checks": [ + { + "condition": { + "@call": "or", + "args": { + "values": [ + { + "@call": "email", + "args": { + "value": { + "@path": "/email" + } + } + }, + { + "@path": "/allowOverride" + } + ] + } + }, + "message": "Or check failed." + } + ] + }, + { + "id": "not_field", + "component": "TextField", + "label": "Not Field", + "value": { + "@path": "/email" + }, + "checks": [ + { + "condition": { + "@call": "not", + "args": { + "value": { + "@call": "email", + "args": { + "value": { + "@path": "/email" + } + } + } + } + }, + "message": "Not check failed." + } + ] + } + ] + } + } + ] + } + ], + "expect": { + "surfaces": { + "form_surface": { + "isValid": { + "and_field": false, + "or_field": true, + "not_field": true + }, + "validationErrors": { + "and_field": [ + "And check failed." + ], + "or_field": [], + "not_field": [] + }, + "validationResults": { + "and_field": [ + { + "valid": false, + "message": "And check failed.", + "severity": "error" + } + ], + "or_field": [], + "not_field": [] + } + } + } + } + } +] \ No newline at end of file diff --git a/package/__fixtures__/a2ui-conformance/core/validator_v1_0.json b/package/__fixtures__/a2ui-conformance/core/validator_v1_0.json new file mode 100644 index 0000000000..611c15a926 --- /dev/null +++ b/package/__fixtures__/a2ui-conformance/core/validator_v1_0.json @@ -0,0 +1,2792 @@ +[ + { + "name": "test_validator_1_0", + "description": "Tests schema validation of v1.0 message envelopes (inline components, optional catalogId, data deletion) against the basic v1.0 catalog schema.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/catalogs/basic/v1/catalog.json", + "sendDataModel": true, + "components": [ + { + "id": "root", + "component": "Text", + "text": "Inline Text" + } + ], + "dataModel": { + "counter": 100 + } + } + } + ] + }, + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s_nocat" + } + } + ] + }, + { + "messages": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/user/token", + "value": null + } + } + ] + }, + { + "messages": [ + { + "createSurface": { + "surfaceId": "s1" + } + } + ], + "expectError": { + "category": "ValidationError", + "details": [ + { + "path": "messages.0.version", + "code": "missing_field" + } + ] + } + }, + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": 123 + } + } + ], + "expectError": { + "category": "ValidationError", + "details": [ + { + "path": "messages.0.createSurface.surfaceId", + "code": "type_mismatch" + } + ] + } + } + ] + }, + { + "name": "test_custom_catalog_1_0", + "description": "Tests component property schema validation in v1.0 against custom catalog schema.", + "action": "validate", + "protocolVersion": "1.0", + "catalog": { + "catalogId": "custom", + "components": { + "Canvas": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "component": { + "const": "Canvas" + }, + "children": { + "type": "array", + "items": { + "$ref": "common_types.json#/$defs/ComponentId" + } + } + }, + "required": [ + "component", + "children" + ] + }, + "Chart": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "component": { + "const": "Chart" + }, + "chartType": { + "enum": [ + "doughnut", + "pie" + ] + }, + "title": { + "type": "string" + } + }, + "required": [ + "component" + ] + } + } + }, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Canvas", + "children": [ + "c1", + "c2" + ] + }, + { + "id": "c1", + "component": "Chart", + "chartType": "doughnut", + "title": "Chart 1" + }, + { + "id": "c2", + "component": "Chart", + "chartType": "pie", + "title": "Chart 2" + } + ] + } + } + ] + }, + { + "messages": [ + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "c1", + "component": "Chart", + "chartType": "bar" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "surfaceId": "s1", + "path": "/components/0/chartType", + "details": [ + { + "path": "/chartType", + "code": "invalid_value", + "surfaceId": "s1" + } + ], + "rendererError": { + "version": "1.0", + "error": { + "code": "VALIDATION_FAILED", + "surfaceId": "s1", + "path": "/components/0/chartType", + "message": "chartType" + } + } + } + }, + { + "messages": [ + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "c1", + "component": "Canvas" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "surfaceId": "s1", + "path": "/components/0/children", + "details": [ + { + "path": "/children", + "code": "missing_field", + "surfaceId": "s1" + } + ], + "rendererError": { + "version": "1.0", + "error": { + "code": "VALIDATION_FAILED", + "surfaceId": "s1", + "path": "/components/0/children", + "message": "children" + } + } + } + }, + { + "messages": [ + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "c1", + "component": "UnknownComponent" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "surfaceId": "s1", + "path": "/components/0/component", + "details": [ + { + "path": "/component", + "code": "invalid_value", + "surfaceId": "s1" + } + ], + "rendererError": { + "version": "1.0", + "error": { + "code": "VALIDATION_FAILED", + "surfaceId": "s1", + "path": "/components/0/component", + "message": "UnknownComponent" + } + } + } + } + ] + }, + { + "name": "test_validator_theme_schema", + "description": "Tests schema validation of surface theme properties against catalog theme schema.", + "action": "validate", + "catalog": { + "catalogId": "theme_cat", + "theme": { + "type": "object", + "properties": { + "primaryColor": { + "type": "string", + "pattern": "^#[0-9a-fA-F]{6}$" + } + }, + "required": [ + "primaryColor" + ], + "additionalProperties": false + } + }, + "steps": [ + { + "messages": [ + { + "version": "v0.9", + "createSurface": { + "surfaceId": "s1", + "catalogId": "theme_cat", + "theme": { + "primaryColor": "#ff0000" + } + } + } + ] + }, + { + "messages": [ + { + "version": "v0.9", + "createSurface": { + "surfaceId": "s1", + "catalogId": "theme_cat", + "theme": { + "primaryColor": "not-a-hex" + } + } + } + ], + "expectError": { + "category": "ValidationError" + } + } + ] + }, + { + "name": "test_v10_topology_circular_reference_error", + "description": "Tests that circular component references fail validation in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/catalogs/basic/v1/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "c1" + ] + }, + { + "id": "c1", + "component": "Column", + "children": [ + "root" + ] + } + ] + } + } + ], + "expectError": "Circular component reference" + } + ] + }, + { + "name": "test_v10_topology_dangling_child_reference_error", + "description": "Tests that dangling child references fail validation in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/catalogs/basic/v1/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "missing_child" + ] + } + ] + } + } + ], + "expectError": "Dangling reference" + } + ] + }, + { + "name": "test_v10_topology_self_reference_error", + "description": "Tests that self-referencing components fail validation in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/catalogs/basic/v1/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "root" + ] + } + ] + } + } + ], + "expectError": "Self-reference detected" + } + ] + }, + { + "name": "test_v10_topology_orphaned_component_error", + "description": "Tests that unreachable orphaned components fail validation in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/catalogs/basic/v1/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Root" + }, + { + "id": "orphan", + "component": "Text", + "text": "Orphan" + } + ] + } + } + ], + "expectError": "is not reachable from" + } + ] + }, + { + "name": "test_v10_topology_missing_root_component_error", + "description": "Tests that component updates missing root component fail validation in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/catalogs/basic/v1/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "c1", + "component": "Text", + "text": "No Root" + } + ] + } + } + ], + "expectError": "Missing root component" + } + ] + }, + { + "name": "test_v10_unknown_component_type", + "description": "Tests that unknown component types fail schema validation in v1.0.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/catalogs/basic/v1/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "UnknownWidget", + "foo": "bar" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "surfaceId": "s1", + "path": "/components/0/component", + "rendererError": { + "version": "1.0", + "error": { + "code": "VALIDATION_FAILED", + "surfaceId": "s1", + "path": "/components/0/component", + "message": "UnknownWidget" + } + } + } + } + ] + }, + { + "name": "test_v10_unknown_component_property", + "description": "Tests that extra unknown component properties fail schema validation in v1.0.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/catalogs/basic/v1/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Button", + "label": "Click Me", + "unknownAttribute": "someValue" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "surfaceId": "s1", + "rendererError": { + "version": "1.0", + "error": { + "code": "VALIDATION_FAILED", + "surfaceId": "s1", + "path": "/components/0", + "message": "Button" + } + } + } + } + ] + }, + { + "name": "test_v10_unknown_nested_function", + "description": "Tests that unknown nested functions fail schema validation in v1.0.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/catalogs/basic/v1/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Button", + "child": "t1", + "action": { + "functionCall": { + "@call": "unknownFunc", + "args": { + "arg": "val" + } + } + } + }, + { + "id": "t1", + "component": "Text", + "text": "Click Me" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "surfaceId": "s1", + "rendererError": { + "version": "1.0", + "error": { + "code": "VALIDATION_FAILED", + "surfaceId": "s1", + "path": "/components/0/action/functionCall", + "message": "unknownFunc" + } + } + } + } + ] + }, + { + "name": "test_v10_multiple_action_envelope_error", + "description": "Tests that having multiple action keys in a single message envelope fails validation in v1.0.", + "action": "validate", + "protocolVersion": "1.0", + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1" + }, + "deleteSurface": { + "surfaceId": "s1" + } + } + ], + "expectError": { + "category": "ValidationError" + } + } + ] + }, + { + "name": "test_v10_invalid_json_pointer_path_error", + "description": "Tests that invalid JSON Pointer path syntax fails validation in v1.0.", + "action": "validate", + "protocolVersion": "1.0", + "steps": [ + { + "messages": [ + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "invalid path [0]", + "value": "data" + } + } + ], + "expectError": { + "category": "ValidationError" + } + } + ] + }, + { + "name": "test_v10_uax31_invalid_identifier_error", + "description": "Tests that component IDs violating UAX#31 identifier syntax fail validation in v1.0.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/catalogs/basic/v1/catalog.json", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "123invalid_id" + ] + }, + { + "id": "123invalid_id", + "component": "Text", + "text": "Invalid ID" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "surfaceId": "s1", + "path": "/components/1/id", + "message": "Component id '123invalid_id' must be a valid", + "rendererError": { + "version": "1.0", + "error": { + "code": "VALIDATION_FAILED", + "surfaceId": "s1", + "path": "/components/1/id", + "message": "123invalid_id" + } + } + } + } + ] + }, + { + "name": "test_v10_topology_template_child_reachable", + "description": "Tests that a component referenced only as a list template is reachable from the root in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "List", + "children": { + "componentId": "item", + "path": "/items" + } + }, + { + "id": "item", + "component": "Text", + "text": "Reachable" + } + ] + } + } + ] + } + ] + }, + { + "name": "test_v10_topology_template_dangling_reference_error", + "description": "Tests that a list template naming a component that does not exist fails validation in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "List", + "children": { + "componentId": "missing_id", + "path": "/items" + } + }, + { + "id": "item", + "component": "Text", + "text": "Unused" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "references non-existent component 'missing_id'" + } + } + ] + }, + { + "name": "test_v10_invalid_binding_path_error", + "description": "Tests that a data binding path with an invalid JSON Pointer escape ('~2') fails validation in v1.0.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@path": "/invalid/escape/~2" + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Invalid path syntax" + } + } + ] + }, + { + "name": "test_v10_component_nesting_depth_limit_exceeded_error", + "description": "Tests that a chain of nested components deeper than the global limit (50) fails validation in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Card", + "child": "c0" + }, + { + "id": "c0", + "component": "Card", + "child": "c1" + }, + { + "id": "c1", + "component": "Card", + "child": "c2" + }, + { + "id": "c2", + "component": "Card", + "child": "c3" + }, + { + "id": "c3", + "component": "Card", + "child": "c4" + }, + { + "id": "c4", + "component": "Card", + "child": "c5" + }, + { + "id": "c5", + "component": "Card", + "child": "c6" + }, + { + "id": "c6", + "component": "Card", + "child": "c7" + }, + { + "id": "c7", + "component": "Card", + "child": "c8" + }, + { + "id": "c8", + "component": "Card", + "child": "c9" + }, + { + "id": "c9", + "component": "Card", + "child": "c10" + }, + { + "id": "c10", + "component": "Card", + "child": "c11" + }, + { + "id": "c11", + "component": "Card", + "child": "c12" + }, + { + "id": "c12", + "component": "Card", + "child": "c13" + }, + { + "id": "c13", + "component": "Card", + "child": "c14" + }, + { + "id": "c14", + "component": "Card", + "child": "c15" + }, + { + "id": "c15", + "component": "Card", + "child": "c16" + }, + { + "id": "c16", + "component": "Card", + "child": "c17" + }, + { + "id": "c17", + "component": "Card", + "child": "c18" + }, + { + "id": "c18", + "component": "Card", + "child": "c19" + }, + { + "id": "c19", + "component": "Card", + "child": "c20" + }, + { + "id": "c20", + "component": "Card", + "child": "c21" + }, + { + "id": "c21", + "component": "Card", + "child": "c22" + }, + { + "id": "c22", + "component": "Card", + "child": "c23" + }, + { + "id": "c23", + "component": "Card", + "child": "c24" + }, + { + "id": "c24", + "component": "Card", + "child": "c25" + }, + { + "id": "c25", + "component": "Card", + "child": "c26" + }, + { + "id": "c26", + "component": "Card", + "child": "c27" + }, + { + "id": "c27", + "component": "Card", + "child": "c28" + }, + { + "id": "c28", + "component": "Card", + "child": "c29" + }, + { + "id": "c29", + "component": "Card", + "child": "c30" + }, + { + "id": "c30", + "component": "Card", + "child": "c31" + }, + { + "id": "c31", + "component": "Card", + "child": "c32" + }, + { + "id": "c32", + "component": "Card", + "child": "c33" + }, + { + "id": "c33", + "component": "Card", + "child": "c34" + }, + { + "id": "c34", + "component": "Card", + "child": "c35" + }, + { + "id": "c35", + "component": "Card", + "child": "c36" + }, + { + "id": "c36", + "component": "Card", + "child": "c37" + }, + { + "id": "c37", + "component": "Card", + "child": "c38" + }, + { + "id": "c38", + "component": "Card", + "child": "c39" + }, + { + "id": "c39", + "component": "Card", + "child": "c40" + }, + { + "id": "c40", + "component": "Card", + "child": "c41" + }, + { + "id": "c41", + "component": "Card", + "child": "c42" + }, + { + "id": "c42", + "component": "Card", + "child": "c43" + }, + { + "id": "c43", + "component": "Card", + "child": "c44" + }, + { + "id": "c44", + "component": "Card", + "child": "c45" + }, + { + "id": "c45", + "component": "Card", + "child": "c46" + }, + { + "id": "c46", + "component": "Card", + "child": "c47" + }, + { + "id": "c47", + "component": "Card", + "child": "c48" + }, + { + "id": "c48", + "component": "Card", + "child": "c49" + }, + { + "id": "c49", + "component": "Card", + "child": "c50" + }, + { + "id": "c50", + "component": "Card", + "child": "c51" + }, + { + "id": "c51", + "component": "Card", + "child": "c52" + }, + { + "id": "c52", + "component": "Card", + "child": "c53" + }, + { + "id": "c53", + "component": "Card", + "child": "c54" + }, + { + "id": "c54", + "component": "Card", + "child": "c55" + }, + { + "id": "c55", + "component": "Text", + "text": "Leaf" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Global recursion limit exceeded" + } + } + ] + }, + { + "name": "test_v10_component_nesting_depth_within_limit", + "description": "Tests that a chain of 41 nested components, within the global limit (50), passes validation in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Card", + "child": "c0" + }, + { + "id": "c0", + "component": "Card", + "child": "c1" + }, + { + "id": "c1", + "component": "Card", + "child": "c2" + }, + { + "id": "c2", + "component": "Card", + "child": "c3" + }, + { + "id": "c3", + "component": "Card", + "child": "c4" + }, + { + "id": "c4", + "component": "Card", + "child": "c5" + }, + { + "id": "c5", + "component": "Card", + "child": "c6" + }, + { + "id": "c6", + "component": "Card", + "child": "c7" + }, + { + "id": "c7", + "component": "Card", + "child": "c8" + }, + { + "id": "c8", + "component": "Card", + "child": "c9" + }, + { + "id": "c9", + "component": "Card", + "child": "c10" + }, + { + "id": "c10", + "component": "Card", + "child": "c11" + }, + { + "id": "c11", + "component": "Card", + "child": "c12" + }, + { + "id": "c12", + "component": "Card", + "child": "c13" + }, + { + "id": "c13", + "component": "Card", + "child": "c14" + }, + { + "id": "c14", + "component": "Card", + "child": "c15" + }, + { + "id": "c15", + "component": "Card", + "child": "c16" + }, + { + "id": "c16", + "component": "Card", + "child": "c17" + }, + { + "id": "c17", + "component": "Card", + "child": "c18" + }, + { + "id": "c18", + "component": "Card", + "child": "c19" + }, + { + "id": "c19", + "component": "Card", + "child": "c20" + }, + { + "id": "c20", + "component": "Card", + "child": "c21" + }, + { + "id": "c21", + "component": "Card", + "child": "c22" + }, + { + "id": "c22", + "component": "Card", + "child": "c23" + }, + { + "id": "c23", + "component": "Card", + "child": "c24" + }, + { + "id": "c24", + "component": "Card", + "child": "c25" + }, + { + "id": "c25", + "component": "Card", + "child": "c26" + }, + { + "id": "c26", + "component": "Card", + "child": "c27" + }, + { + "id": "c27", + "component": "Card", + "child": "c28" + }, + { + "id": "c28", + "component": "Card", + "child": "c29" + }, + { + "id": "c29", + "component": "Card", + "child": "c30" + }, + { + "id": "c30", + "component": "Card", + "child": "c31" + }, + { + "id": "c31", + "component": "Card", + "child": "c32" + }, + { + "id": "c32", + "component": "Card", + "child": "c33" + }, + { + "id": "c33", + "component": "Card", + "child": "c34" + }, + { + "id": "c34", + "component": "Card", + "child": "c35" + }, + { + "id": "c35", + "component": "Card", + "child": "c36" + }, + { + "id": "c36", + "component": "Card", + "child": "c37" + }, + { + "id": "c37", + "component": "Card", + "child": "c38" + }, + { + "id": "c38", + "component": "Card", + "child": "c39" + }, + { + "id": "c39", + "component": "Card", + "child": "c40" + }, + { + "id": "c40", + "component": "Text", + "text": "Leaf" + } + ] + } + } + ] + } + ] + }, + { + "name": "test_v10_multi_surface_independent_roots", + "description": "Tests that two surfaces are each validated against their own component tree in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "surface_a", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "createSurface": { + "surfaceId": "surface_b", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "surface_a", + "components": [ + { + "id": "root", + "component": "Card", + "child": "child_a" + }, + { + "id": "child_a", + "component": "Text", + "text": "Hello A" + } + ] + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "surface_b", + "components": [ + { + "id": "root", + "component": "Card", + "child": "child_b" + }, + { + "id": "child_b", + "component": "Text", + "text": "Hello B" + } + ] + } + } + ] + } + ] + }, + { + "name": "test_v10_multi_surface_missing_root_error", + "description": "Tests that a missing root on one of two surfaces fails validation in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "surface_a", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "createSurface": { + "surfaceId": "surface_b", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "surface_a", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Hello A" + } + ] + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "surface_b", + "components": [ + { + "id": "not_root", + "component": "Text", + "text": "Hello B" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Missing root component" + } + } + ] + }, + { + "name": "test_v10_incremental_update_without_root", + "description": "Tests that an updateComponents message that omits the root component passes once the root has been sent, in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "card" + ] + }, + { + "id": "card", + "component": "Card", + "child": "text" + }, + { + "id": "text", + "component": "Text", + "text": "Initial" + } + ] + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "card", + "component": "Card", + "child": "text" + }, + { + "id": "text", + "component": "Text", + "text": "Updated" + } + ] + } + } + ] + } + ] + }, + { + "name": "test_v10_incremental_update_self_reference_error", + "description": "Tests that an incremental updateComponents message introducing a self-reference fails validation in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "card" + ] + }, + { + "id": "card", + "component": "Card", + "child": "text" + }, + { + "id": "text", + "component": "Text", + "text": "Initial" + } + ] + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "card", + "component": "Card", + "child": "card" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Self-reference detected" + } + } + ] + }, + { + "name": "test_v10_incremental_update_circular_reference_error", + "description": "Tests that an incremental updateComponents message introducing a cycle fails validation in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "card" + ] + }, + { + "id": "card", + "component": "Card", + "child": "text" + }, + { + "id": "text", + "component": "Text", + "text": "Initial" + } + ] + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "card", + "component": "Card", + "child": "loop" + }, + { + "id": "loop", + "component": "Card", + "child": "card" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Circular reference detected" + } + } + ] + }, + { + "name": "test_v10_incremental_update_duplicate_component_id_error", + "description": "Tests that an incremental updateComponents message listing the same component ID twice fails validation in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Column", + "children": [ + "card" + ] + }, + { + "id": "card", + "component": "Card", + "child": "text" + }, + { + "id": "text", + "component": "Text", + "text": "Initial" + } + ] + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "text", + "component": "Text", + "text": "A" + }, + { + "id": "text", + "component": "Text", + "text": "B" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Duplicate component ID" + } + } + ] + }, + { + "name": "test_v10_recursion_limit_function_call_depth_exceeded_error", + "description": "Tests that nested function calls exceeding the maximum depth (5) fail validation in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "f1", + "args": { + "v": { + "@call": "f2", + "args": { + "v": { + "@call": "f3", + "args": { + "v": { + "@call": "f4", + "args": { + "v": { + "@call": "f5", + "args": { + "v": { + "@call": "f6", + "args": {} + } + } + } + } + } + } + } + } + } + } + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "functionCall depth" + } + } + ] + }, + { + "name": "test_v10_data_model_nesting_depth_limit_exceeded_error", + "description": "Tests that an updateDataModel value nested deeper than the global limit (50) fails validation in v1.0.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Hello" + } + ] + } + }, + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/", + "value": { + "level": 0, + "next": { + "level": 1, + "next": { + "level": 2, + "next": { + "level": 3, + "next": { + "level": 4, + "next": { + "level": 5, + "next": { + "level": 6, + "next": { + "level": 7, + "next": { + "level": 8, + "next": { + "level": 9, + "next": { + "level": 10, + "next": { + "level": 11, + "next": { + "level": 12, + "next": { + "level": 13, + "next": { + "level": 14, + "next": { + "level": 15, + "next": { + "level": 16, + "next": { + "level": 17, + "next": { + "level": 18, + "next": { + "level": 19, + "next": { + "level": 20, + "next": { + "level": 21, + "next": { + "level": 22, + "next": { + "level": 23, + "next": { + "level": 24, + "next": { + "level": 25, + "next": { + "level": 26, + "next": { + "level": 27, + "next": { + "level": 28, + "next": { + "level": 29, + "next": { + "level": 30, + "next": { + "level": 31, + "next": { + "level": 32, + "next": { + "level": 33, + "next": { + "level": 34, + "next": { + "level": 35, + "next": { + "level": 36, + "next": { + "level": 37, + "next": { + "level": 38, + "next": { + "level": 39, + "next": { + "level": 40, + "next": { + "level": 41, + "next": { + "level": 42, + "next": { + "level": 43, + "next": { + "level": 44, + "next": { + "level": 45, + "next": { + "level": 46, + "next": { + "level": 47, + "next": { + "level": 48, + "next": { + "level": 49, + "next": { + "level": 50, + "next": { + "level": 51, + "next": { + "level": 52, + "next": { + "level": 53, + "next": { + "level": 54, + "next": { + "level": 55 + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Global recursion limit exceeded" + } + } + ] + }, + { + "name": "test_v10_topology_structured_array_item_dangling_child_error", + "description": "Tests that a dangling child reference inside a structured array item (Tabs.tabs[].child) fails validation in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Tabs", + "tabs": [ + { + "title": "Overview", + "child": "tab_one" + }, + { + "title": "Details", + "child": "missing_tab" + } + ] + }, + { + "id": "tab_one", + "component": "Text", + "text": "First" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "missing_tab" + } + } + ] + }, + { + "name": "test_v10_topology_plain_string_property_not_treated_as_child_ref", + "description": "Tests that a plain string property whose value happens to match another component's ID does not make that component reachable in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Text", + "text": "orphan_id" + }, + { + "id": "orphan_id", + "component": "Text", + "text": "Still orphaned" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "orphan_id" + } + } + ] + }, + { + "name": "test_v10_topology_component_id_array_dangling_reference_error", + "description": "Tests that every item of an array property whose items are ComponentId is a child reference, whatever the property is called, so an item naming no component fails validation in v1.0 strict mode even when no other item names one.", + "action": "validate", + "protocolVersion": "1.0", + "catalog": { + "catalogId": "custom", + "components": { + "Stack": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "component": { + "const": "Stack" + }, + "slots": { + "type": "array", + "items": { + "$ref": "common_types.json#/$defs/ComponentId" + } + } + }, + "required": [ + "component", + "slots" + ] + } + } + }, + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Stack", + "slots": [ + "missing_slot" + ] + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "missing_slot" + } + } + ] + }, + { + "name": "test_v10_topology_component_id_array_child_reachable", + "description": "Tests that a component named in an array property whose items are ComponentId is reachable from the root in v1.0 strict mode, whatever the property is called.", + "action": "validate", + "protocolVersion": "1.0", + "catalog": { + "catalogId": "custom", + "components": { + "Stack": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "component": { + "const": "Stack" + }, + "slots": { + "type": "array", + "items": { + "$ref": "common_types.json#/$defs/ComponentId" + } + } + }, + "required": [ + "component", + "slots" + ] + } + } + }, + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Stack", + "slots": [ + "first" + ] + }, + { + "id": "first", + "component": "Stack", + "slots": [] + } + ] + } + } + ] + } + ] + }, + { + "name": "test_v10_topology_component_id_array_partial_dangling_reference_error", + "description": "A property declared as an array of ComponentId directly (an array whose items are a ComponentId $ref, not a ChildList) is read as a list of child references. With one held and one missing item, the missing item is reported as a dangling reference at its index in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalog": { + "catalogId": "custom", + "components": { + "Stack": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "component": { + "const": "Stack" + }, + "slots": { + "type": "array", + "items": { + "$ref": "common_types.json#/$defs/ComponentId" + } + } + }, + "required": [ + "component", + "slots" + ] + }, + "Label": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "component": { + "const": "Label" + }, + "text": { + "type": "string" + } + }, + "required": [ + "component", + "text" + ] + } + } + }, + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Stack", + "slots": [ + "held", + "missing" + ] + }, + { + "id": "held", + "component": "Label", + "text": "held" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "references non-existent component 'missing' in field 'slots[1]'" + } + } + ] + }, + { + "name": "test_v10_topology_component_id_array_circular_reference_error", + "description": "A property declared as an array of ComponentId directly (an array whose items are a ComponentId $ref, not a ChildList) is read as a list of child references, so a cycle through it is detected in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalog": { + "catalogId": "custom", + "components": { + "Stack": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "component": { + "const": "Stack" + }, + "slots": { + "type": "array", + "items": { + "$ref": "common_types.json#/$defs/ComponentId" + } + } + }, + "required": [ + "component", + "slots" + ] + }, + "Label": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "component": { + "const": "Label" + }, + "text": { + "type": "string" + } + }, + "required": [ + "component", + "text" + ] + } + } + }, + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "custom" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Stack", + "slots": [ + "loop_a" + ] + }, + { + "id": "loop_a", + "component": "Stack", + "slots": [ + "loop_b" + ] + }, + { + "id": "loop_b", + "component": "Stack", + "slots": [ + "loop_a" + ] + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Circular reference detected" + } + } + ] + }, + { + "name": "test_v10_child_list_template_missing_path_error", + "description": "Tests that a ChildList template object missing its required 'path' property fails schema validation in v1.0.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "List", + "children": { + "componentId": "item" + } + }, + { + "id": "item", + "component": "Text", + "text": "Item" + } + ] + } + } + ], + "expectError": { + "category": "ValidationError" + } + } + ] + }, + { + "name": "test_v10_update_data_model_invalid_path_error", + "description": "Tests that an updateDataModel message with an invalid JSON Pointer escape ('~2') fails validation in v1.0.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateDataModel": { + "surfaceId": "s1", + "path": "/user/~2invalid", + "value": "Alice" + } + } + ], + "expectError": { + "category": "ValidationError", + "message": "Invalid path syntax" + } + } + ] + }, + { + "name": "test_v10_function_argument_schema_mismatch_error", + "description": "Tests that a function call passing an argument of the wrong type fails schema validation in v1.0.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Text", + "text": { + "@call": "formatNumber", + "args": { + "value": "not_a_number" + } + } + } + ] + } + } + ], + "expectError": { + "category": "ValidationError", + "surfaceId": "s1", + "path": "/components/0/text/args/value", + "rendererError": { + "version": "1.0", + "error": { + "code": "VALIDATION_FAILED", + "surfaceId": "s1", + "path": "/components/0/text/args/value", + "message": "value" + } + } + } + } + ] + }, + { + "name": "test_v10_incremental_update_same_component_id_across_messages", + "description": "Tests that updating the same component ID across separate updateComponents messages succeeds in v1.0 strict mode.", + "action": "validate", + "protocolVersion": "1.0", + "catalogPaths": [ + "catalogs/basic/v1/catalog.json" + ], + "strictMode": true, + "steps": [ + { + "messages": [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "s1", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json" + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Initial" + } + ] + } + }, + { + "version": "1.0", + "updateComponents": { + "surfaceId": "s1", + "components": [ + { + "id": "root", + "component": "Text", + "text": "Updated" + } + ] + } + } + ] + } + ] + } +] \ No newline at end of file diff --git a/package/a2ui/package.json b/package/a2ui/package.json new file mode 100644 index 0000000000..7f6827791c --- /dev/null +++ b/package/a2ui/package.json @@ -0,0 +1,6 @@ +{ + "main": "../lib/commonjs/a2ui/index.js", + "module": "../lib/module/a2ui/index.js", + "react-native": "../src/a2ui/index.ts", + "types": "../lib/typescript/a2ui/index.d.ts" +} diff --git a/package/bin/validate-entry-isolation.js b/package/bin/validate-entry-isolation.js index 39b2865615..d9d7c43940 100644 --- a/package/bin/validate-entry-isolation.js +++ b/package/bin/validate-entry-isolation.js @@ -9,7 +9,9 @@ * the AI components or their dependencies. Apps that don't use AI must not bundle any of it. * 2. The AI entry points (core `src/ai-components/index.ts` and the wrappers' * `ai-components/index.js`) may only reach the AI sources plus an explicit allowlist of other - * files. Apps that only use the AI components must not bundle the chat SDK. + * files. Apps that only use the AI components must not bundle the chat SDK. Nor the A2UI + * renderer: the `a2ui` entry points (core `src/a2ui/index.ts` and the wrappers' + * `a2ui/index.js`) reach it and `@a2ui/web_core`, besides what the AI entry points reach. * 3. The codegen spec copies the wrappers need (codegen only reads a package's own `jsSrcsDir`) stay * byte-identical to the core source of truth. * @@ -23,6 +25,7 @@ const ts = require('typescript'); const PACKAGE_DIR = path.resolve(__dirname, '..'); const CORE_SRC = path.join(PACKAGE_DIR, 'src'); const AI_SRC = path.join(CORE_SRC, 'ai-components'); +const A2UI_SRC = path.join(CORE_SRC, 'a2ui'); const WRAPPERS = ['native-package', 'expo-package'].map((name) => path.join(PACKAGE_DIR, name)); // Third-party packages only the AI components may use. (`lodash` and `@stream-io/state-store` are @@ -37,6 +40,10 @@ const AI_ONLY_PACKAGES = [ 'zod', ]; +// Third-party packages only the A2UI renderer may use: its own entry point keeps them out of apps +// that use the AI components without it. +const A2UI_ONLY_PACKAGES = ['@a2ui/web_core']; + // Files outside `src/ai-components` that the AI entry points are allowed to reach. const AI_ALLOWED_FILES = [ path.join(CORE_SRC, 'native.ts'), @@ -91,6 +98,9 @@ const resolveSpecifier = (specifier, fromFile) => { if (specifier === 'stream-chat-react-native-core/ai-components') { return { file: path.join(AI_SRC, 'index.ts') }; } + if (specifier === 'stream-chat-react-native-core/a2ui') { + return { file: path.join(A2UI_SRC, 'index.ts') }; + } return { pkg: packageNameOf(specifier), specifier }; }; @@ -176,13 +186,15 @@ for (const entry of [ for (const [file, chain] of files) { if ( file.startsWith(AI_SRC + path.sep) || - file.includes(`${path.sep}ai-components${path.sep}`) + file.startsWith(A2UI_SRC + path.sep) || + file.includes(`${path.sep}ai-components${path.sep}`) || + file.includes(`${path.sep}a2ui${path.sep}`) ) { report(`${relative(entry)} reaches the AI components (${relative(file)})`, chain); } } for (const [specifier, chain] of packages) { - if (AI_ONLY_PACKAGES.includes(packageNameOf(specifier))) { + if ([...AI_ONLY_PACKAGES, ...A2UI_ONLY_PACKAGES].includes(packageNameOf(specifier))) { report(`${relative(entry)} reaches the AI-only dependency "${specifier}"`, chain); } } @@ -202,6 +214,33 @@ for (const entry of [ ); } } + for (const [specifier, chain] of packages) { + if (AI_FORBIDDEN_PACKAGES.includes(specifier)) { + report(`${relative(entry)} reaches "${specifier}"`, chain); + } + if (A2UI_ONLY_PACKAGES.includes(packageNameOf(specifier))) { + report(`${relative(entry)} reaches the A2UI-only dependency "${specifier}"`, chain); + } + } +} + +// 2b. The A2UI entry points reach the AI components and the A2UI renderer only. +const A2UI_WRAPPER_ENTRIES = WRAPPERS.map((w) => path.join(w, 'a2ui/index.js')); +for (const entry of [path.join(A2UI_SRC, 'index.ts'), ...A2UI_WRAPPER_ENTRIES]) { + const { files, packages } = walk(entry); + for (const [file, chain] of files) { + if ( + !file.startsWith(AI_SRC + path.sep) && + !file.startsWith(A2UI_SRC + path.sep) && + !AI_ALLOWED_FILES.includes(file) && + !A2UI_WRAPPER_ENTRIES.includes(file) + ) { + report( + `${relative(entry)} reaches ${relative(file)}, which is not in the AI allowlist`, + chain, + ); + } + } for (const [specifier, chain] of packages) { if (AI_FORBIDDEN_PACKAGES.includes(specifier)) { report(`${relative(entry)} reaches "${specifier}"`, chain); diff --git a/package/expo-package/a2ui/index.d.ts b/package/expo-package/a2ui/index.d.ts new file mode 100644 index 0000000000..7c6e147c82 --- /dev/null +++ b/package/expo-package/a2ui/index.d.ts @@ -0,0 +1 @@ +export * from 'stream-chat-react-native-core/a2ui'; diff --git a/package/expo-package/a2ui/index.js b/package/expo-package/a2ui/index.js new file mode 100644 index 0000000000..ca0f6bd382 --- /dev/null +++ b/package/expo-package/a2ui/index.js @@ -0,0 +1,6 @@ +// The A2UI renderer for the AI components' generated UI, an entry point of its own so an app whose +// agent never writes A2UI doesn't bundle the A2UI core. It draws with the AI components, whose +// entry point registers the native pieces they need. +import '../ai-components'; + +export * from 'stream-chat-react-native-core/a2ui'; diff --git a/package/expo-package/package.json b/package/expo-package/package.json index 77e1113870..7822a6c49c 100644 --- a/package/expo-package/package.json +++ b/package/expo-package/package.json @@ -22,6 +22,7 @@ "react-native.config.js", "package.json", "ai-components", + "a2ui", "app.plugin.js", "plugin" ], diff --git a/package/native-package/a2ui/index.d.ts b/package/native-package/a2ui/index.d.ts new file mode 100644 index 0000000000..7c6e147c82 --- /dev/null +++ b/package/native-package/a2ui/index.d.ts @@ -0,0 +1 @@ +export * from 'stream-chat-react-native-core/a2ui'; diff --git a/package/native-package/a2ui/index.js b/package/native-package/a2ui/index.js new file mode 100644 index 0000000000..ca0f6bd382 --- /dev/null +++ b/package/native-package/a2ui/index.js @@ -0,0 +1,6 @@ +// The A2UI renderer for the AI components' generated UI, an entry point of its own so an app whose +// agent never writes A2UI doesn't bundle the A2UI core. It draws with the AI components, whose +// entry point registers the native pieces they need. +import '../ai-components'; + +export * from 'stream-chat-react-native-core/a2ui'; diff --git a/package/native-package/package.json b/package/native-package/package.json index 4ee283eb07..c46bfe2165 100644 --- a/package/native-package/package.json +++ b/package/native-package/package.json @@ -23,7 +23,8 @@ "*.podspec", "react-native.config.js", "package.json", - "ai-components" + "ai-components", + "a2ui" ], "license": "SEE LICENSE IN LICENSE", "main": "src/index.js", diff --git a/package/package.json b/package/package.json index 84ac82977e..081761a6e0 100644 --- a/package/package.json +++ b/package/package.json @@ -21,7 +21,8 @@ "lib", "package.json", "README.md", - "ai-components" + "ai-components", + "a2ui" ], "scripts": { "shared-native:sync": "bash ./scripts/sync-shared-native.sh all", @@ -69,6 +70,7 @@ ] }, "dependencies": { + "@a2ui/web_core": "~0.15.0", "@gorhom/bottom-sheet": "^5.2.14", "@khanacademy/simple-markdown": "^2.1.0", "@legendapp/list": "^3.7.1", diff --git a/package/shared-native/android/agent/AIAgentController.kt b/package/shared-native/android/agent/AIAgentController.kt index af4dbd41e5..8914e1b5c2 100644 --- a/package/shared-native/android/agent/AIAgentController.kt +++ b/package/shared-native/android/agent/AIAgentController.kt @@ -13,6 +13,7 @@ import com.facebook.react.bridge.LifecycleEventListener import com.facebook.react.bridge.Promise import com.facebook.react.bridge.ReactApplicationContext import java.io.File +import java.lang.ref.WeakReference import java.util.Locale import java.util.UUID @@ -39,8 +40,13 @@ class AIAgentController(private val reactContext: ReactApplicationContext) : private class SignIn( val callbackScheme: String, val promise: Promise, - val launcher: ActivityResultLauncher?, + /** The key the Auth Tab's result comes back under, or null for a Custom Tab. */ + val resultKey: String?, ) { + /** Where the Auth Tab's result is received, and the activity it is registered on. */ + var launcher: ActivityResultLauncher? = null + var host: WeakReference? = null + /** Whether the person left the app for the Custom Tab, so coming back means it was closed. */ var left = false } @@ -91,7 +97,7 @@ class AIAgentController(private val reactContext: ReactApplicationContext) : ) { openAuthTab(activity, browser, uri, callbackScheme, promise) } else { - signIn = SignIn(callbackScheme, promise, launcher = null) + signIn = SignIn(callbackScheme, promise, resultKey = null) CustomTabsIntent.Builder().build().launchUrl(activity, uri) } } catch (error: Exception) { @@ -108,16 +114,27 @@ class AIAgentController(private val reactContext: ReactApplicationContext) : callbackScheme: String, promise: Promise, ) { - // Registered for this sign-in alone and dropped once it answers, since a TurboModule can't - // register before the activity starts, as `AuthTabIntent.registerActivityResultLauncher` - // requires. Auth Tab's own contract only passes the intent through and reads the result's - // data, which is what `StartActivityForResult` does. - lateinit var launcher: ActivityResultLauncher - launcher = activity.activityResultRegistry.register( - "stream-chat-ai-sign-in-${launches++}", + val current = SignIn(callbackScheme, promise, resultKey = "stream-chat-ai-sign-in-${launches++}") + signIn = current + val launcher = receiveResult(current, activity) + AgentAuthTab.launch(launcher, browser, uri, callbackScheme) + } + + // Registers where `current`'s Auth Tab result is received, on `activity`. It is registered for + // this sign-in alone and dropped once it answers, since a TurboModule can't register before the + // activity starts, as `AuthTabIntent.registerActivityResultLauncher` requires. Auth Tab's own + // contract only passes the intent through and reads the result's data, which is what + // `StartActivityForResult` does. An activity recreated while the tab is open, such as one + // rotated or reclaimed in the background, keeps the result under the same key until it is + // registered on it again, which `onHostResume` does. + private fun receiveResult(current: SignIn, activity: ComponentActivity): ActivityResultLauncher { + current.launcher?.unregister() + current.host = WeakReference(activity) + val launcher = activity.activityResultRegistry.register( + current.resultKey!!, ActivityResultContracts.StartActivityForResult(), ) { result -> - if (signIn?.launcher !== launcher) return@register + if (signIn !== current) return@register when (result.resultCode) { // Auth Tab's RESULT_OK and RESULT_CANCELED are the activity's own. Activity.RESULT_OK -> finish(result.data?.data?.toString() ?: "") @@ -125,8 +142,8 @@ class AIAgentController(private val reactContext: ReactApplicationContext) : else -> fail("The sign-in didn't complete.") } } - signIn = SignIn(callbackScheme, promise, launcher) - AgentAuthTab.launch(launcher, browser, uri, callbackScheme) + current.launcher = launcher + return launcher } private fun finish(callback: String) { @@ -148,7 +165,7 @@ class AIAgentController(private val reactContext: ReactApplicationContext) : override fun onNewIntent(intent: Intent) { val current = signIn ?: return val data = intent.data ?: return - if (current.launcher == null && data.scheme.equals(current.callbackScheme, ignoreCase = true)) { + if (current.resultKey == null && data.scheme.equals(current.callbackScheme, ignoreCase = true)) { finish(data.toString()) } } @@ -161,12 +178,20 @@ class AIAgentController(private val reactContext: ReactApplicationContext) : ) = Unit override fun onHostPause() { - signIn?.takeIf { it.launcher == null }?.left = true + signIn?.takeIf { it.resultKey == null }?.left = true } override fun onHostResume() { val current = signIn ?: return - if (current.launcher == null && current.left) finish("") + if (current.resultKey != null) { + // The activity was recreated while the Auth Tab was open: its result waits on this one. + val activity = reactContext.currentActivity + if (activity is ComponentActivity && current.host?.get() !== activity) { + receiveResult(current, activity) + } + return + } + if (current.left) finish("") } override fun onHostDestroy() = Unit diff --git a/package/shared-native/ios/AIAgentModule.swift b/package/shared-native/ios/AIAgentModule.swift index e7e4eac086..15ebbc8c3c 100644 --- a/package/shared-native/ios/AIAgentModule.swift +++ b/package/shared-native/ios/AIAgentModule.swift @@ -13,13 +13,55 @@ public class AIAgentModule: NSObject { private var completion: ((String?, String?) -> Void)? private let anchor = PresentationAnchor() - /// This install's identifier, created on first use. The same key and form as StreamChatAI's, - /// so a native app moving to React Native keeps its id. + /// This install's identifier, created on first use. It is kept in a file left out of backups, + /// as on Android, so a phone restored from another's backup never answers tool calls meant + /// for that one. An id kept the way StreamChatAI keeps it, in UserDefaults, carries over, so a + /// native app moving to React Native keeps its id. @objc public static func installId() -> String { - if let saved = UserDefaults.standard.string(forKey: installIdKey) { return saved } - let created = "ios-" + UUID().uuidString - UserDefaults.standard.set(created, forKey: installIdKey) - return created + let file = installIdFile() + if let file, let saved = try? String(contentsOf: file, encoding: .utf8), !saved.isEmpty { + return saved + } + let id = UserDefaults.standard.string(forKey: installIdKey) ?? "ios-" + UUID().uuidString + guard let file, keep(id, in: file) else { + // Without the file, UserDefaults keeps it, backed up as it is. + UserDefaults.standard.set(id, forKey: installIdKey) + return id + } + // The backups from now on don't carry it. + UserDefaults.standard.removeObject(forKey: installIdKey) + return id + } + + // Application Support/io.getstream.ai/install-id. + private static func installIdFile() -> URL? { + guard let support = try? FileManager.default.url( + for: .applicationSupportDirectory, + in: .userDomainMask, + appropriateFor: nil, + create: true + ) else { return nil } + return support.appendingPathComponent("io.getstream.ai", isDirectory: true) + .appendingPathComponent("install-id") + } + + // Writes `id` to `file`, left out of backups. Whether it did. + private static func keep(_ id: String, in file: URL) -> Bool { + do { + try FileManager.default.createDirectory( + at: file.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + try id.write(to: file, atomically: true, encoding: .utf8) + var values = URLResourceValues() + values.isExcludedFromBackup = true + var excluded = file + try excluded.setResourceValues(values) + return true + } catch { + try? FileManager.default.removeItem(at: file) + return false + } } /// Opens `url` in a web authentication session. `completion` gets the URL the page handed diff --git a/package/src/a2ui/A2UIContexts.ts b/package/src/a2ui/A2UIContexts.ts new file mode 100644 index 0000000000..767f22ac66 --- /dev/null +++ b/package/src/a2ui/A2UIContexts.ts @@ -0,0 +1,15 @@ +import { createContext } from 'react'; + +import type { A2UIEngine } from './A2UIRenderer'; + +/** The engine of the surfaces below, for what in them reaches the host, such as links. */ +export const A2UIEngineContext = createContext(undefined); + +/** The reply the surfaces below belong to, and whether the agent is still writing it. */ +export const A2UIReplyContext = createContext< + | { + isStreaming: boolean; + messageId: string; + } + | undefined +>(undefined); diff --git a/package/src/a2ui/A2UIInstructions.ts b/package/src/a2ui/A2UIInstructions.ts new file mode 100644 index 0000000000..2be41e4a0c --- /dev/null +++ b/package/src/a2ui/A2UIInstructions.ts @@ -0,0 +1,103 @@ +// What an agent is told to write A2UI with, and what checks it. +import { A2UI_VERSION } from './A2UIStream'; + +import type { AIUIComponent } from '../ai-components/generative-ui/AIUIComponent'; +import type { AIUIComponentDefinition } from '../ai-components/generative-ui/AIUIComponentDefinition'; +import { isRecord } from '../ai-components/generative-ui/AIUIHelpers'; +import type { AIClientToolDefinition } from '../ai-components/tool-calls/AIClientToolRunner'; + +/** + * What an agent is told to write replies' UI in A2UI with: the basic catalog, or the part of it + * `keep` names, with the app's components, and what the app adds. The same as the + * `prompt.json` StreamChatA2UI's instructions are written from. + */ +export type A2UIInstructionsConfig = { + /** The id of the catalog with the app's components, as the renderer names it. */ + catalogId?: string; + /** The app's components, or their portable definitions, such as a `UIComponents.json`. */ + components?: (AIUIComponent | AIUIComponentDefinition)[]; + /** + * Replies with their UI, as the agent should write them. Each block must render with the + * catalog, or no instructions are written: a broken example teaches the model broken UI. + */ + examples?: string[]; + /** The info string of a reply's UI block, as the agents router reads it. */ + fence?: string; + /** The basic catalog's components and functions to describe; all of them when absent. */ + keep?: string[]; + /** + * The basic catalog's components and functions to leave out, such as those the app's renderer + * leaves out with its own `removing`, after `keep`. + */ + removing?: string[]; + /** Guidance on using the app's components. */ + notes?: string[]; + /** What the model is told first, such as which answers to write as UI. */ + preamble?: string; + /** Rules added after the protocol's. */ + rules?: string[]; + /** The app's tools the agent can call from the UI it writes, as functions of the catalog. */ + tools?: AIClientToolDefinition[]; +}; + +/** The A2UI an example holds that would not render, and why. */ +export class A2UIInstructionsError extends Error { + readonly problems: string[]; + + constructor(problems: string[]) { + super(`An example's A2UI would not render:\n${problems.join('\n')}`); + this.name = 'A2UIInstructionsError'; + this.problems = problems; + } +} + +/** A description of the A2UI v1.0 messages and the rules that keep what they make valid. */ +export const A2UI_PROTOCOL_OVERVIEW = `# Generating user interfaces with A2UI v1.0 + +You render user interfaces by emitting A2UI messages. Every message is a JSON object with a \`"version": "1.0"\` field plus exactly one of these keys: + +- \`createSurface\`: creates a surface. Requires \`surfaceId\`, and accepts \`catalogId\`, \`sendDataModel\`, \`components\` and \`dataModel\` so a complete UI can be sent in one message. +- \`updateComponents\`: adds or replaces components of an existing surface. Requires \`surfaceId\` and \`components\`. +- \`updateDataModel\`: writes to the surface data model. Requires \`surfaceId\` and \`value\`, and accepts a JSON Pointer \`path\`. Setting \`value\` to \`null\` deletes the entry at \`path\`. +- \`deleteSurface\`: removes a surface. Requires \`surfaceId\`. +- \`callRendererFunction\`: asks the renderer to run one of its functions. +- \`agentFunctionResponse\`: answers a \`callAgentFunction\` request from the renderer. + +Rules: + +1. Components are a flat adjacency list. Each component has a unique \`id\`, a \`component\` type, and its properties inline. Exactly one component must have the id \`root\`. An id is an identifier: letters, digits and underscores, not starting with a digit, such as \`saturday_plan\` (never \`saturday-plan\`). +2. Children are referenced by id, never nested inline. Use an array of ids, or \`{"componentId": "...", "path": "/items"}\` to repeat a template for every item of a data model list. +3. Any property may be a literal, a data binding \`{"@path": "/a/b"}\` or a function call \`{"@call": "name", "args": {...}}\`. Inside a list template, paths without a leading slash resolve relative to the current item, and \`{"@call": "@index"}\` returns the item's position. A literal object key that starts with \`@\` is written with \`@@\`, so \`"@@type"\` means \`"@type"\`. +4. Put content that changes in the data model and bind to it, so you can update values with \`updateDataModel\` instead of resending components. +5. Interactive components declare an \`action\`, which either dispatches an event to you (\`{"event": {"name": "...", "context": {...}}}\`) or runs a renderer function (\`{"functionCall": {"@call": "openUrl", "args": {"url": "..."}}}\`). +6. Surface ids must be unique for the lifetime of the renderer. Creating a surface that already exists is an error: delete it first, or send \`updateComponents\` and \`updateDataModel\` to change it.`; + +export const writing = (fence: string, catalogId: string) => `## Writing UI in A2UI + +Write a reply's UI in one fenced block whose info string is \`${fence}\`, after one short sentence saying what it shows, which is what people read wherever the UI can't show. In the block, write A2UI messages, one JSON object per line, and nothing else: + +1. First a \`createSurface\` with a \`surfaceId\` new to this reply, the \`catalogId\` \`${catalogId}\`, and the \`dataModel\` the UI's inputs start from. +2. Then \`updateComponents\` messages for that surface with a few components each, \`root\` first, so the UI shows as you write it. + +A reply's UI is shown as you wrote it: never write \`callRendererFunction\` or \`agentFunctionResponse\`, and never change an earlier reply's UI, but write a new one. What the person does in it comes back as their next message. Give every event action a \`userMessage\`, the words of that message, such as "Update the plan", and a \`context\` with what you need to know, bound to the data model.`; + +// A JSON value with its object keys sorted, so the same catalog always reads the same. +export const sorted = (value: unknown): unknown => + Array.isArray(value) + ? value.map(sorted) + : isRecord(value) + ? Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, sorted(value[key])]), + ) + : value; + +export const fenceOf = (fence?: string) => fence ?? `ui:a2ui@${A2UI_VERSION}`; + +// The A2UI blocks of an example reply. +export const blocksIn = (example: string, fence: string) => + example + .split(`\`\`\`${fence}\n`) + .slice(1) + .map((block) => block.split('\n```')[0]); diff --git a/package/src/a2ui/A2UIPlaceholder.tsx b/package/src/a2ui/A2UIPlaceholder.tsx new file mode 100644 index 0000000000..2f10223cb3 --- /dev/null +++ b/package/src/a2ui/A2UIPlaceholder.tsx @@ -0,0 +1,24 @@ +import React from 'react'; +import { StyleSheet, View } from 'react-native'; + +import { useTheme } from '../ai-components/contexts'; +import { Shimmer } from '../ai-components/internal/Shimmer'; +import { primitives } from '../ai-components/internal/tokens'; + +/** What shows while a reply's first surface is being written. */ +export const A2UIPlaceholder = () => { + const { + theme: { colors }, + } = useTheme(); + return ( + + + + + + ); +}; + +const styles = StyleSheet.create({ + placeholder: { borderRadius: primitives.radiusXl, height: 120 }, +}); diff --git a/package/src/a2ui/A2UIRenderer.tsx b/package/src/a2ui/A2UIRenderer.tsx new file mode 100644 index 0000000000..63be21dab4 --- /dev/null +++ b/package/src/a2ui/A2UIRenderer.tsx @@ -0,0 +1,505 @@ +import React, { useLayoutEffect, useMemo, useSyncExternalStore } from 'react'; +import { StyleSheet, View } from 'react-native'; + +import { + type ActionPayload, + type Catalog, + type ComponentApi, + MessageProcessor, + type SurfaceModel, +} from '@a2ui/web_core'; + +import { A2UIEngineContext, A2UIReplyContext } from './A2UIContexts'; +import { A2UIPlaceholder } from './A2UIPlaceholder'; +import { + A2UI_VERSION, + a2uiMessagesIn, + a2uiSourceOf, + normalizeA2UIMessage, + supportsA2UIVersion, +} from './A2UIStream'; +import { A2UISurfaceView } from './A2UISurface'; +import { A2UI_DEFAULT_CATALOG_ID, a2uiTypeName } from './catalogDocument'; +import { a2uiCatalogsOf } from './catalogs'; +import { A2UI_BASIC_COMPONENTS } from './components/basicComponents'; +import type { A2UIComponents, A2UIIcons } from './components/types'; +import { a2uiInstructions } from './instructions'; +import { + secretContextKeysOf, + secretPathsOf, + secretValuesOf, + withoutPaths, + withoutSecrets, +} from './secrets'; + +import { useTheme } from '../ai-components/contexts'; +import type { AIUIDocument } from '../ai-components/generative-ui/AIUIDocument'; +import { isRecord } from '../ai-components/generative-ui/AIUIHelpers'; +import type { AIUIHost } from '../ai-components/generative-ui/AIUIHost'; +import { + aiUIMediaSource, + type AIUIMediaURL, + aiUIPublicMediaURL, +} from '../ai-components/generative-ui/AIUIMedia'; +import type { AIUIEngine, AIUIRenderer } from '../ai-components/generative-ui/AIUIRenderer'; +import { primitives } from '../ai-components/internal/tokens'; + +export { A2UI_DEFAULT_CATALOG_ID }; + +/** What went wrong in a reply's A2UI, as the A2UI specification reports it to an agent. */ +export type A2UIError = { + /** A code such as `VALIDATION_FAILED` or `EXPRESSION_ERROR`. */ + code: string; + message: string; + /** Where in the message it went wrong, as a JSON pointer. */ + path?: string; + surfaceId?: string; +}; + +export type A2UIRendererOptions = { + /** + * The id the agent names the catalog with the app's components by. A surface naming the + * basic catalog shows them too, and a surface naming none is made with this one. + */ + catalogId?: string; + /** + * The views of the basic catalog's components to use instead of the default ones, by their + * A2UI type, such as `{ Card: MyCard }`. Each gets what the agent wrote for it, resolved. + */ + components?: A2UIComponents; + /** + * The app's icons for the names A2UI's Icon takes, such as `{ shoppingCart: MyCartIcon }`, over + * the Material icons it draws by default. An app's own names work too, when its agent is told + * them. + */ + icons?: A2UIIcons; + /** + * What the app tells the model before A2UI's own instructions in `capabilities`, such as which + * answers to write as UI. + */ + instructions?: string; + /** The locale numbers and dates are formatted in. `en-US` by default. */ + locale?: string; + /** + * Decides what an Image, Video or AudioPlayer loads for the link the agent wrote: the address + * to load, or `null` not to load it. By default only public web links load, never addresses on + * the person's network, such as `192.168.1.1` or `localhost`. + */ + mediaUrl?: AIUIMediaURL; + /** + * The basic catalog's components and functions this renderer leaves out, such as `['Video']`. + * The agent's instructions leave them out too, and a reply that places one shows the rest. + */ + removing?: string[]; + /** + * Tells the app what went wrong in a reply's A2UI, such as a message that isn't JSON or a + * component the catalog doesn't have, with the reply's id. The rest of the reply's UI still + * shows. What is wrong with the renderer rather than a reply, such as an app's component that + * can't join the catalog, comes without an id. Logged by default. + */ + onError?: (error: A2UIError, messageId?: string) => void; +}; + +const reportA2UIError = (options: A2UIRendererOptions, error: A2UIError, messageId?: string) => { + if (options.onError) options.onError(error, messageId); + else console.error(`A reply's A2UI can't show: ${error.code} ${error.message}`, error); +}; + +type RendererError = { + code?: unknown; + message?: unknown; + path?: unknown; + surfaceId?: unknown; +}; + +const errorOf = (error: RendererError): A2UIError => ({ + code: typeof error.code === 'string' ? error.code : 'EXECUTION_ERROR', + message: typeof error.message === 'string' ? error.message : String(error.message), + ...(typeof error.path === 'string' ? { path: error.path } : {}), + ...(typeof error.surfaceId === 'string' ? { surfaceId: error.surfaceId } : {}), +}); + +// An action's name in words, such as `Submit form` for `submit_form`, for an action whose button +// says nothing. +const wordsOf = (name: string) => { + const words = name + .replace(/([a-z0-9])([A-Z])/g, '$1 $2') + .split(/[_\-\s]+/) + .filter(Boolean) + .join(' ') + .toLowerCase(); + return words ? words.charAt(0).toUpperCase() + words.slice(1) : name; +}; + +/** + * The surfaces of one reply: its own processor, fed the messages of its document as they + * complete. Messages that changed, rather than grew, start its surfaces again. + */ +export class A2UIReplyState { + isStreaming = false; + + private applied: string[] = []; + private readonly engine: A2UIEngine; + private listeners = new Set<() => void>(); + private readonly messageId: string; + private processor: MessageProcessor; + private reported = new Set(); + private shown: SurfaceModel[] = []; + private showedNothing = false; + private stopWatching: () => void; + + constructor(engine: A2UIEngine, messageId: string) { + this.engine = engine; + this.messageId = messageId; + this.processor = this.makeProcessor(); + this.stopWatching = this.watch(); + } + + /** The reply's surfaces, in the order the agent made them. */ + surfaces = (): SurfaceModel[] => this.shown; + + subscribe = (listener: () => void) => { + this.listeners.add(listener); + return () => { + this.listeners.delete(listener); + }; + }; + + /** Applies the messages of `source` not applied yet. */ + update(source: string, isStreaming: boolean) { + this.isStreaming = isStreaming; + const messages = a2uiMessagesIn(source); + const grew = + messages.length >= this.applied.length && + this.applied.every((message, index) => messages[index] === message); + if (!grew) { + this.stopWatching(); + this.processor.dispose(); + this.processor = this.makeProcessor(); + this.stopWatching = this.watch(); + this.applied = []; + } + for (const raw of messages.slice(this.applied.length)) { + let json: unknown; + try { + json = JSON.parse(raw); + } catch { + json = undefined; + } + if (!isRecord(json)) { + this.report({ code: 'VALIDATION_FAILED', message: 'A message isn’t a JSON object' }); + continue; + } + this.feed(normalizeA2UIMessage(json, this.engine.catalogId)); + } + this.applied = messages; + this.refresh(); + // A finished reply that made no surface shows its text, which the host shows when asked + // again: it asked before these messages were applied. + const showsNothing = !isStreaming && this.shown.length === 0; + if (showsNothing !== this.showedNothing) { + this.showedNothing = showsNothing; + this.engine.host.refresh(); + } + } + + dispose() { + this.stopWatching(); + this.processor.dispose(); + this.listeners.clear(); + } + + // Feeds a message to the processor. A batch of components that one bad component spoils is + // fed again a component at a time, so the rest show, as they would while being written. + private feed(message: Record) { + const errors = this.processor.processMessagesWithErrors(message as never); + const reported = errors.length ? (this.oneAtATime(message) ?? errors) : errors; + for (const { error } of reported) this.report(errorOf(error as RendererError)); + } + + // The components of a message one at a time, with what goes wrong with each, or `undefined` + // when the message has no batch of them to feed again. A surface made with components is made + // before they are added, and stays when they fail, so they are added to it. + private oneAtATime(message: Record) { + const { createSurface, updateComponents } = message; + const batch = isRecord(updateComponents) + ? updateComponents + : isRecord(createSurface) + ? createSurface + : undefined; + if (!batch || !Array.isArray(batch.components) || batch.components.length < 2) { + return undefined; + } + const { surfaceId } = batch; + if ( + batch === createSurface && + (typeof surfaceId !== 'string' || !this.processor.model.getSurface(surfaceId)) + ) { + return undefined; + } + return batch.components.flatMap((component: unknown) => + this.processor.processMessagesWithErrors({ + updateComponents: { components: [component], surfaceId }, + version: message.version, + } as never), + ); + } + + private makeProcessor() { + return new MessageProcessor(this.engine.catalogs(), (action) => this.act(action)); + } + + // Tells the agent what the person did, in the words of the button they tapped, else its name. + // Never in the words the agent gave the action, which could say something the button doesn't; + // those go along as `userMessage`. What the person typed into an obscured field goes nowhere: + // not in the context, the message or the data model, which goes when the surface asks for it. + private act(action: ActionPayload) { + const surface = this.processor.model.getSurface(action.surfaceId); + const paths = secretPathsOf(surface); + const secrets = secretValuesOf(surface); + const context = withoutSecrets(action.context ?? {}, secrets) as Record; + for (const key of secretContextKeysOf(surface, action.sourceComponentId)) delete context[key]; + const userMessage = + typeof action.userMessage === 'string' + ? withoutSecrets(action.userMessage, secrets).trim() + : ''; + const dataModel = surface?.sendDataModel + ? withoutSecrets(withoutPaths(surface.dataModel.get('/'), paths), secrets) + : undefined; + void this.engine.host.send({ + format: 'a2ui', + label: this.buttonText(action) ?? wordsOf(action.name), + messageId: this.messageId, + name: action.name, + payload: { + context, + sourceComponentId: action.sourceComponentId, + surfaceId: action.surfaceId, + ...(userMessage ? { userMessage } : {}), + ...(dataModel !== undefined ? { dataModel } : {}), + }, + }); + } + + // What the tapped button says: its child Text, written out or bound to the data model. + private buttonText(action: ActionPayload) { + const surface = this.processor.model.getSurface(action.surfaceId); + const components = surface?.componentsModel; + const child: unknown = components?.get(action.sourceComponentId)?.properties.child; + if (typeof child !== 'string') return undefined; + const text: unknown = components?.get(child)?.properties.text; + if (typeof text === 'string') return text.trim() === '' ? undefined : text; + if (isRecord(text) && typeof text['@path'] === 'string' && text['@path'].startsWith('/')) { + const bound: unknown = surface?.dataModel.get(text['@path']); + return typeof bound === 'string' && bound.trim() !== '' ? bound : undefined; + } + return undefined; + } + + private watch() { + const subscriptions = [ + this.processor.onSurfaceCreated(() => this.refresh()), + this.processor.onSurfaceDeleted(() => this.refresh()), + this.processor.model.onError.subscribe((error) => + this.report(errorOf(error as RendererError)), + ), + ]; + return () => subscriptions.forEach((subscription) => subscription.unsubscribe()); + } + + private refresh() { + const surfaces = [...this.processor.getSurfaces().values()]; + if ( + surfaces.length === this.shown.length && + surfaces.every((surface, index) => surface === this.shown[index]) + ) { + return; + } + this.shown = surfaces; + this.listeners.forEach((listener) => listener()); + } + + // Each error once: a reply's messages are fed again as it streams, and the processor tells + // the same error both as it fails a message, with the place in it, and to the surface. + private report(error: A2UIError) { + const key = [error.code, error.message, error.surfaceId].join('\u0000'); + if (this.reported.has(key)) return; + this.reported.add(key); + this.engine.report(error, this.messageId); + } +} + +/** A reply's surfaces, following its document as the agent writes it. */ +const A2UIReplyContent = ({ + document, + engine, + messageId, +}: { + document: AIUIDocument; + engine: A2UIEngine; + messageId: string; +}) => { + const reply = engine.replyOf(document, messageId); + const source = a2uiSourceOf(document) ?? ''; + const { isStreaming } = document; + useLayoutEffect(() => { + reply.update(source, isStreaming); + }, [isStreaming, reply, source]); + const surfaces = useSyncExternalStore(reply.subscribe, reply.surfaces); + const { theme } = useTheme(); + const replyContext = useMemo(() => ({ isStreaming, messageId }), [isStreaming, messageId]); + if (surfaces.length === 0) return isStreaming ? : null; + return ( + + + + {surfaces.map((surface) => ( + + ))} + + + + ); +}; + +/** + * Shows the A2UI of one conversation. Each reply's surfaces are its own, kept for as long as + * the host keeps the engine, so what the person entered survives scrolling. + */ +export class A2UIEngine implements AIUIEngine { + readonly catalogId: string; + readonly components: A2UIComponents; + readonly host: AIUIHost; + readonly icons: A2UIIcons; + readonly mediaUrl: AIUIMediaURL; + /** Each reply's surfaces, by the reply's id. */ + readonly replies = new Map(); + + private made?: Catalog[]; + private readonly options: A2UIRendererOptions; + + constructor(host: AIUIHost, options: A2UIRendererOptions) { + this.host = host; + this.options = options; + this.catalogId = options.catalogId ?? A2UI_DEFAULT_CATALOG_ID; + // The basic catalog's views, the app's own over them. + this.components = { ...A2UI_BASIC_COMPONENTS, ...options.components }; + this.icons = options.icons ?? {}; + this.mediaUrl = options.mediaUrl ?? aiUIPublicMediaURL; + } + + /** The address an Image, Video or AudioPlayer loads for a link the agent wrote, if any. */ + mediaSource(value: unknown) { + return aiUIMediaSource(value, this.mediaUrl); + } + + /** The catalogs a reply's surfaces are made with, made once. */ + catalogs(): Catalog[] { + this.made ??= a2uiCatalogsOf({ + catalogId: this.catalogId, + components: this.host.components, + locale: this.options.locale, + removing: this.options.removing, + onComponentError: (component, error) => + this.report({ + code: 'CATALOG_ERROR', + message: `The component ${component.definition.name} can't join the catalog: ${String(error)}`, + }), + onToolError: (tool, error) => + this.report({ + code: 'CATALOG_ERROR', + message: `The tool ${tool.name} can't join the catalog: ${String(error)}`, + }), + openUrl: (url) => this.host.open(url), + runTool: (name, args, options) => this.host.runTool(name, args, options), + tools: this.host.toolDefinitions, + }); + return this.made; + } + + /** The app's component the agent places by its A2UI type, such as `PlaceCard`. */ + appComponent(type: string) { + return this.host.components.find( + (component) => a2uiTypeName(component.definition.name) === type, + ); + } + + /** + * Tells the app what went wrong, after the current render: a reply's messages are applied + * as it first shows, and an app may set state when told. + */ + report(error: A2UIError, messageId?: string) { + queueMicrotask(() => reportA2UIError(this.options, error, messageId)); + } + + /** The surfaces of a reply, made with its messages so far the first time they are asked for. */ + replyOf(document: AIUIDocument, messageId: string) { + let reply = this.replies.get(messageId); + if (!reply) { + reply = new A2UIReplyState(this, messageId); + this.replies.set(messageId, reply); + reply.update(a2uiSourceOf(document) ?? '', document.isStreaming); + } + return reply; + } + + remove(messageId: string) { + this.replies.get(messageId)?.dispose(); + this.replies.delete(messageId); + } + + view(document: AIUIDocument, messageId: string) { + if (a2uiSourceOf(document) === undefined) return null; + // A finished reply whose A2UI made no surface shows its text instead, once that is known. + if (!document.isStreaming && this.replyOf(document, messageId).surfaces().length === 0) { + return null; + } + return ; + } +} + +/** + * The renderer of [A2UI](https://a2ui.org) 1.0 for `AIUIHost`, on the A2UI core + * (`@a2ui/web_core`), with the basic catalog's components drawn natively. The app's components + * join its catalog, so the agent places them like the basic catalog's own. The counterpart of + * `A2UIRenderer` in StreamChatA2UI. + */ +export const createA2UIRenderer = (options: A2UIRendererOptions = {}): AIUIRenderer => { + const catalogId = options.catalogId ?? A2UI_DEFAULT_CATALOG_ID; + return { + // The fence, the catalog and the instructions for both, so the agent's backend has its model + // write UI this renderer shows. Without the instructions, when they can't be written, the rest + // still goes. + capabilities: async (components, tools) => { + const fence = `ui:a2ui@${A2UI_VERSION}`; + let instructions: string | undefined; + try { + instructions = await a2uiInstructions({ + catalogId, + components, + fence, + ...(options.instructions?.trim() ? { preamble: options.instructions.trim() } : {}), + removing: options.removing, + tools, + }); + } catch (error) { + reportA2UIError(options, { + code: 'EXECUTION_ERROR', + message: `The A2UI instructions couldn't be written: ${String(error)}`, + }); + } + return { + catalog_id: catalogId, + fence, + ...(instructions === undefined ? {} : { instructions }), + }; + }, + createEngine: (host) => new A2UIEngine(host, options), + format: 'a2ui', + supports: supportsA2UIVersion, + }; +}; + +const styles = StyleSheet.create({ + surfaces: { gap: primitives.spacingSm }, +}); diff --git a/package/src/a2ui/A2UIStream.ts b/package/src/a2ui/A2UIStream.ts new file mode 100644 index 0000000000..849bd64be4 --- /dev/null +++ b/package/src/a2ui/A2UIStream.ts @@ -0,0 +1,82 @@ +import type { AIUIDocument } from '../ai-components/generative-ui/AIUIDocument'; +import { isRecord } from '../ai-components/generative-ui/AIUIHelpers'; + +/** The version of A2UI this renderer reads. */ +export const A2UI_VERSION = '1.0'; + +// The spellings of 1.0 an agent may write, such as from instructions written before its +// release, or by copying the fence's `ui:a2ui@1.0` in another form. +const VERSION_SPELLINGS = new Set(['1', '1.0', '1.0.0', 'v1', 'v1.0']); + +/** Whether a reply's A2UI of `version`, or of no version, is one this renderer reads. */ +export const supportsA2UIVersion = (version: string | undefined) => + version === undefined || VERSION_SPELLINGS.has(version); + +/** + * The complete A2UI messages in `text`, as written: every top-level JSON object, in order. + * One object per line, a pretty-printed one or a JSON array of them all read the same, and + * words around them are skipped. An object still being written is left out until it closes, so + * a reply's UI shows each message as it completes. + */ +export const a2uiMessagesIn = (text: string): string[] => { + const messages: string[] = []; + let depth = 0; + let start = 0; + let inString = false; + let escaped = false; + for (let index = 0; index < text.length; index++) { + const char = text[index]; + if (inString) { + if (escaped) escaped = false; + else if (char === '\\') escaped = true; + else if (char === '"') inString = false; + continue; + } + if (char === '"' && depth > 0) { + inString = true; + } else if (char === '{') { + if (depth === 0) start = index; + depth++; + } else if (char === '}' && depth > 0) { + depth--; + if (depth === 0) messages.push(text.slice(start, index + 1)); + } + } + return messages; +}; + +/** + * The messages of a document as text: as the agent wrote them, or written out when a backend + * sent them as JSON. `undefined` for content that holds no messages. + */ +export const a2uiSourceOf = (document: AIUIDocument): string | undefined => { + const { content } = document; + if (typeof content === 'string') return content; + if (typeof content === 'object' && content !== null) { + try { + return JSON.stringify(content); + } catch { + return undefined; + } + } + return undefined; +}; + +/** + * A message as the processor reads it: 1.0 in its own spelling, and a surface the agent made + * without naming a catalog made with `catalogId`, the catalog with the app's components. + */ +export const normalizeA2UIMessage = ( + message: Record, + catalogId: string, +): Record => { + let normalized = message; + if (typeof message.version === 'string' && VERSION_SPELLINGS.has(message.version)) { + normalized = { ...normalized, version: A2UI_VERSION }; + } + const { createSurface } = normalized; + if (isRecord(createSurface) && createSurface.catalogId === undefined) { + normalized = { ...normalized, createSurface: { ...createSurface, catalogId } }; + } + return normalized; +}; diff --git a/package/src/a2ui/A2UISurface.tsx b/package/src/a2ui/A2UISurface.tsx new file mode 100644 index 0000000000..88c0e9347a --- /dev/null +++ b/package/src/a2ui/A2UISurface.tsx @@ -0,0 +1,240 @@ +import React, { + createContext, + memo, + useCallback, + useContext, + useLayoutEffect, + useMemo, + useSyncExternalStore, +} from 'react'; +import type { AccessibilityProps } from 'react-native'; +import Animated from 'react-native-reanimated'; + +import { + type ComponentApi, + type ComponentNode, + effect, + getValue, + isComponentNode, + isWritable, + NodeResolver, + peekValue, + ResolvedBinding, + type Signal, + type SurfaceModel, +} from '@a2ui/web_core'; + +import { A2UIEngineContext, A2UIReplyContext } from './A2UIContexts'; +import { keepSecretPath } from './secrets'; + +import { AIUIComponentView } from '../ai-components/generative-ui/AIUIComponent'; +import { useAIUIEntrance } from '../ai-components/generative-ui/AIUIEntrance'; +import { isRecord } from '../ai-components/generative-ui/AIUIHelpers'; + +/** The value of an A2UI signal, following it. */ +export const useSignalValue = (signal: Signal) => { + const subscribe = useCallback( + (onChange: () => void) => + effect(() => { + getValue(signal); + onChange(); + }), + [signal], + ); + const getSnapshot = useCallback(() => peekValue(signal), [signal]); + return useSyncExternalStore(subscribe, getSnapshot); +}; + +const isPlainObject = (value: unknown): value is Record => { + if (!isRecord(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +}; + +const viewValueOf = (value: unknown): unknown => { + if (isComponentNode(value)) return value; + if (value instanceof ResolvedBinding) return viewValueOf(value.value); + if (Array.isArray(value)) return value.map(viewValueOf); + if (isPlainObject(value)) return viewPropsOf(value); + return value; +}; + +const noop = () => undefined; + +// What a component's view reads: bindings as their values, with a setter for each one the person +// can change, named after it, such as `setValue` for `value`. Children stay nodes. +const viewPropsOf = (props: Record) => { + const view: Record = {}; + for (const [key, value] of Object.entries(props)) { + if (value instanceof ResolvedBinding) { + view[key] = viewValueOf(value.value); + view[`set${key.charAt(0).toUpperCase()}${key.slice(1)}`] = isWritable(value) + ? value.set + : noop; + } else view[key] = viewValueOf(value); + } + return view; +}; + +/** The axis the parent of a part lays its children out on, which a part's weight grows along. */ +export const A2UIAxisContext = createContext<'column' | 'row'>('column'); + +const said = (value: unknown) => + value === undefined || value === null || value === '' ? undefined : String(value); + +// What the agent tells assistive technology about a part, with its `accessibility`: what to call +// it, what more to say about it, whether to announce its changes, and whether to leave it out. +const accessibilityOf = (accessibility: unknown): AccessibilityProps => { + if (!isRecord(accessibility)) return {}; + const { description, hidden, label, live } = accessibility; + const named = said(label); + const described = said(description); + return { + ...(named ? { accessibilityLabel: named } : {}), + ...(described ? { accessibilityHint: described } : {}), + ...(live === 'polite' || live === 'assertive' ? { accessibilityLiveRegion: live } : {}), + ...(hidden === true + ? { accessibilityElementsHidden: true, importantForAccessibility: 'no-hide-descendants' } + : {}), + }; +}; + +// One of the app's components where the agent placed it, with the arguments it declares. +const PlacedComponent = ({ + node, + props, +}: { + node: ComponentNode; + props: Record; +}) => { + const engine = useContext(A2UIEngineContext); + const reply = useContext(A2UIReplyContext); + const component = engine?.appComponent(node.type); + const names = component?.definition.parameters.map(({ name }) => name); + const args = useMemo( + () => + Object.fromEntries( + (names ?? []).filter((name) => name in props).map((name) => [name, props[name]]), + ), + // eslint-disable-next-line react-hooks/exhaustive-deps -- the names change with the component + [component, props], + ); + if (!engine || !reply || !component) return null; + const { messageId } = reply; + const { name } = component.definition; + return ( + + void engine.host.send({ + format: 'a2ui', + label, + messageId, + name: event, + payload: { ...payload, component: name }, + }), + host: engine.host, + isStreaming: reply.isStreaming, + messageId, + }} + /> + ); +}; + +/** + * One part of a surface: its view from the basic catalog, or the app's component of its type, + * coming in when it shows, with its weight and what the agent tells assistive technology. A part + * whose type no catalog has, or that is still on its way, shows nothing. + */ +// The path of the data model a TextField writes what the person types to, when it is obscured, +// such as a password, which must never reach the agent. +const secretPathOf = (node: ComponentNode, resolved: Record) => { + if (node.type !== 'TextField') return undefined; + const variant = + resolved.variant instanceof ResolvedBinding ? resolved.variant.value : resolved.variant; + const { value } = resolved; + return variant === 'obscured' && value instanceof ResolvedBinding && isWritable(value) + ? value.path + : undefined; +}; + +const A2UINode = memo(({ node }: { node: ComponentNode }) => { + const engine = useContext(A2UIEngineContext); + const axis = useContext(A2UIAxisContext); + const resolved = useSignalValue(node.props) as Record; + const props = useMemo(() => viewPropsOf(resolved), [resolved]); + // Before the person can type: whichever view draws the field, what they type stays theirs. + const secretPath = secretPathOf(node, resolved); + const surface = node.context?.dataContext.surface; + useLayoutEffect(() => { + if (secretPath && surface) keepSecretPath(surface, secretPath); + }, [secretPath, surface]); + const { scope, style } = useAIUIEntrance(); + if (!engine || node.isPlaceholder || node.state !== 'resolved') return null; + const View = engine.components[node.type]; + const isApp = !View && !!engine.appComponent(node.type); + if (!View && !isApp) return null; + // A weighted part shares the room its row leaves, as flex-grow from nothing does. + const weight = typeof props.weight === 'number' && axis === 'row' ? props.weight : undefined; + return ( + + {scope( + View ? : , + )} + + ); +}); +A2UINode.displayName = 'A2UINode'; + +/** A child of a part, as its view gets it: a node, or nothing while it is on its way. */ +export const A2UIChild = ({ node }: { node: unknown }) => + isComponentNode(node) ? : null; + +/** The children of a part, as its view gets them. */ +export const A2UIChildren = ({ nodes }: { nodes: unknown }) => ( + <> + {Array.isArray(nodes) + ? nodes.map((node) => + isComponentNode(node) ? : null, + ) + : null} + +); + +/** A surface of a reply: its tree of parts, from its root, as the agent writes them. */ +export const A2UISurfaceView = ({ surface }: { surface: SurfaceModel }) => { + const box = useMemo<{ resolver?: NodeResolver }>( + () => ({}), + // A new box for each surface, which its resolver is made for. + // eslint-disable-next-line react-hooks/exhaustive-deps + [surface], + ); + const subscribe = useCallback( + (onChange: () => void) => { + const resolver = new NodeResolver(surface, surface.defaultCatalog); + box.resolver = resolver; + const stop = effect(() => { + getValue(resolver.rootNode); + onChange(); + }); + return () => { + stop(); + resolver.dispose(); + if (box.resolver === resolver) box.resolver = undefined; + }; + }, + [box, surface], + ); + const getSnapshot = useCallback( + () => (box.resolver ? peekValue(box.resolver.rootNode) : undefined), + [box], + ); + const root = useSyncExternalStore(subscribe, getSnapshot); + return root ? : null; +}; diff --git a/package/src/a2ui/__tests__/A2UIInstructions.test.ts b/package/src/a2ui/__tests__/A2UIInstructions.test.ts new file mode 100644 index 0000000000..145994b08a --- /dev/null +++ b/package/src/a2ui/__tests__/A2UIInstructions.test.ts @@ -0,0 +1,233 @@ +// What an agent is told, written in a script or a build too. +import { parseAIUIComponentDefinition } from '../../ai-components'; +import { ohridLines, uiComponents } from '../../mock-builders/a2ui/ohrid'; +import { + A2UI_DEFAULT_CATALOG_ID, + a2uiInstructions, + A2UIInstructionsError, + a2uiProblems, +} from '../index'; + +const components = uiComponents.flatMap((json) => { + const definition = parseAIUIComponentDefinition(json); + return definition ? [definition] : []; +}); + +// The basic catalog's part the Swift assistant's `a2ui/prompt.json` keeps. +const keep = [ + 'Text', + 'Icon', + 'Row', + 'Column', + 'List', + 'Card', + 'Tabs', + 'Divider', + 'Button', + 'TextField', + 'CheckBox', + 'ChoicePicker', + 'Slider', + 'DateTimeInput', + 'formatString', + 'formatDate', + 'formatNumber', + 'openUrl', + 'required', +]; + +const example = (lines: string[]) => + [ + 'Example — a weekend trip:', + 'A weekend in Ohrid by the lake: a map, your calendar and a plan for each day.', + `\`\`\`ui:a2ui@1.0\n${lines.join('\n')}\n\`\`\``, + ].join('\n\n'); + +const line = (message: Record) => JSON.stringify({ version: '1.0', ...message }); + +const surface = (components: Record[]) => [ + line({ createSurface: { catalogId: A2UI_DEFAULT_CATALOG_ID, surfaceId: 's' } }), + line({ updateComponents: { components, surfaceId: 's' } }), +]; + +describe('a2uiInstructions', () => { + it('writes what StreamChatA2UI writes from the same prompt.json, in its order', async () => { + const text = await a2uiInstructions({ + components, + examples: [example(ohridLines)], + keep, + notes: ['- PlaceCard, TripMap and CalendarCheck are the phone’s own views.'], + preamble: 'Some answers work better as interactive UI than as text.', + rules: ['Never add images for places.', 'Write the whole UI again.'], + }); + + const headings = text.split('\n').filter((row) => /^#{1,2} /.test(row)); + expect(headings).toEqual([ + '## Writing UI in A2UI', + '# Generating user interfaces with A2UI v1.0', + '## Catalog Guidelines', + "## The app's components", + '## The catalog `getstream.io:app-catalog`', + '## Rules', + '## Examples', + ]); + expect(text.startsWith('Some answers work better')).toBe(true); + expect(text).toContain('whose info string is `ui:a2ui@1.0`'); + expect(text).toContain('the `catalogId` `getstream.io:app-catalog`'); + expect(text).toContain('1. Never add images for places.\n2. Write the whole UI again.'); + + const catalog = JSON.parse((text.match(/```json\n(\{.*\})\n```/) ?? [])[1] ?? '{}') as { + catalogId: string; + components: Record; + }; + expect(catalog.catalogId).toBe(A2UI_DEFAULT_CATALOG_ID); + expect(Object.keys(catalog.components)).toEqual( + expect.arrayContaining(['PlaceCard', 'TripMap', 'CalendarCheck', 'Text', 'Tabs']), + ); + expect(catalog.components).not.toHaveProperty('Video'); + expect(catalog.components).not.toHaveProperty('Modal'); + expect(catalog.components.CalendarCheck).toEqual({ + description: + "The person's own calendar for the days of a plan, read on the phone. You never see it: use it whenever a plan has dates, so they can see what the trip would clash with.", + properties: { + component: { const: 'CalendarCheck' }, + end: { + description: 'The last day, as YYYY-MM-DD', + format: 'date', + type: 'string', + }, + start: { + description: 'The first day, as YYYY-MM-DD', + format: 'date', + type: 'string', + }, + }, + required: ['component', 'end', 'start'], + type: 'object', + }); + expect(JSON.stringify(catalog.components.PlaceCard)).toContain( + 'Tapping the card asks you about the place.', + ); + }); + + it('writes nothing when an example would not render, and says why', async () => { + const broken = [ + ...ohridLines.slice(0, 2), + line({ + updateComponents: { + components: [{ component: 'Globe', id: 'map' }], + surfaceId: 'ohrid-weekend', + }, + }), + ]; + const asked = a2uiInstructions({ components, examples: [example(broken)], keep }); + await expect(asked).rejects.toBeInstanceOf(A2UIInstructionsError); + await expect(asked).rejects.toMatchObject({ + problems: expect.arrayContaining([expect.stringMatching(/^example: map: .*Globe/)]), + }); + }); +}); + +describe('a2uiProblems', () => { + it('finds nothing in the Ohrid plan', async () => { + expect(await a2uiProblems(ohridLines.join('\n'), { components })).toEqual([]); + }); + + it('says what doesn’t fit the catalog: components, properties and their values', async () => { + const problems = await a2uiProblems( + surface([ + { children: ['title', 'map', 'go'], component: 'Column', id: 'root' }, + { component: 'Text', id: 'title', size: 3, text: 'A weekend' }, + { component: 'Globe', id: 'map' }, + { child: 'title', component: 'Button', id: 'go' }, + ]).join('\n'), + { components }, + ); + expect(problems).toEqual([ + expect.stringMatching(/^go: .*action/), + expect.stringMatching(/^map: .*Globe/), + expect.stringMatching(/^title: .*size/), + ]); + }); + + it('says what the components make together: references, root, orphans, cycles and ids', async () => { + const dangling = await a2uiProblems( + surface([ + { children: ['a', 'gone'], component: 'Column', id: 'root' }, + { component: 'Text', id: 'a', text: 'Here' }, + { component: 'Text', id: 'lost', text: 'Nobody holds this' }, + ]).join('\n'), + ); + expect(dangling).toEqual([ + 'root holds gone, which is not defined', + 'lost is defined but nothing holds it', + ]); + + expect( + await a2uiProblems(surface([{ component: 'Text', id: 'top', text: 'Hi' }]).join('\n')), + ).toEqual(['surface s has no component with the id root']); + + const cycle = await a2uiProblems( + surface([ + { children: ['loop'], component: 'Column', id: 'root' }, + { children: ['root'], component: 'Column', id: 'loop' }, + ]).join('\n'), + ); + expect(cycle).toEqual([expect.stringMatching(/holds itself through its children/)]); + + const named = await a2uiProblems( + surface([ + { children: ['saturday-plan'], component: 'Column', id: 'root' }, + { component: 'Text', id: 'saturday-plan', text: 'Morning' }, + ]).join('\n'), + ); + expect(named).toEqual([expect.stringMatching(/^saturday-plan: .*identifier/)]); + }); + + it('says what is wrong with the messages themselves', async () => { + const problems = await a2uiProblems( + [ + '{not json}', + line({ updateComponents: { components: [], surfaceId: 'later' } }), + line({ callRendererFunction: { name: 'openUrl' } }), + line({ createSurface: { catalogId: 'someone-else', surfaceId: 's' } }), + line({ + updateComponents: { + components: [ + { component: 'Text', id: 'root', text: 'One' }, + { component: 'Text', id: 'root', text: 'Two' }, + ], + surfaceId: 's', + }, + }), + ].join('\n'), + ); + expect(problems).toEqual([ + "this isn't JSON: {not json}", + 'surface later gets components before it is created', + "a reply's UI calls no functions", + 'surface s names the catalog someone-else, not getstream.io:app-catalog', + 'surface s defines root more than once in one message', + ]); + expect( + await a2uiProblems( + [ + ...surface([{ component: 'Text', id: 'root', text: 'One' }]), + line({ createSurface: { catalogId: A2UI_DEFAULT_CATALOG_ID, surfaceId: 's' } }), + ].join('\n'), + ), + ).toEqual([expect.stringMatching(/^surface s is created again/)]); + expect( + await a2uiProblems( + [ + ...surface([{ component: 'Text', id: 'root', text: 'One' }]), + line({ deleteSurface: { surfaceId: 's' } }), + ...surface([{ component: 'Text', id: 'root', text: 'Two' }]), + ].join('\n'), + ), + ).toEqual([]); + expect(await a2uiProblems('Just words.')).toEqual([ + 'nothing renders: there is no createSurface', + ]); + }); +}); diff --git a/package/src/a2ui/__tests__/A2UIRenderer.test.tsx b/package/src/a2ui/__tests__/A2UIRenderer.test.tsx new file mode 100644 index 0000000000..3f95138422 --- /dev/null +++ b/package/src/a2ui/__tests__/A2UIRenderer.test.tsx @@ -0,0 +1,1174 @@ +import React from 'react'; +import { Pressable, ScrollView, StyleSheet, Text, TextInput } from 'react-native'; + +import { act, fireEvent, render, screen, waitFor } from '@testing-library/react-native'; + +import { + type AIUIAction, + AIUIHost, + AIUIView, + defineAIUIComponent, + parseAIUIComponentDefinition, + parseAIUIDocument, + StreamTheme, +} from '../../ai-components'; +import { aiUIMediaSource, isPublicWebURL } from '../../ai-components/generative-ui/AIUIMedia'; +import type { + AIClientTool, + AIClientToolDefinition, +} from '../../ai-components/tool-calls/AIClientToolRunner'; +import { ohridLines, uiComponents } from '../../mock-builders/a2ui/ohrid'; +import { NativeHandlers } from '../../native'; +import { a2uiMessagesIn, normalizeA2UIMessage, supportsA2UIVersion } from '../A2UIStream'; +import basicCatalogJSON from '../basicCatalog.json'; +import { a2uiAppCatalogDocument, a2uiTypeName } from '../catalogDocument'; +import { formatMoment, parseMoment } from '../components/DateTimeInput'; +import { A2UI_ICON_PATHS } from '../components/iconPaths'; +import { A2UI_DEFAULT_CATALOG_ID, type A2UIRendererOptions, createA2UIRenderer } from '../index'; +import { withoutPaths, withoutSecrets } from '../secrets'; + +const definitionOf = (name: string) => { + const definition = parseAIUIComponentDefinition( + uiComponents.find((component) => component.name === name), + ); + if (!definition) throw new Error(`no ${name}`); + return definition; +}; + +const placeCard = defineAIUIComponent<{ note?: string; title: string }>( + definitionOf('place_card'), + ({ args, context }) => ( + + context.emit('ask_about', `Tell me more about ${args.title}`, { title: args.title }) + } + testID='place-card' + > + + {args.title} + {args.note ? ` (${args.note})` : ''} + + + ), +); + +const tripMap = defineAIUIComponent<{ places: { name: string }[] }>( + definitionOf('trip_map'), + ({ args }) => {args.places.map(({ name }) => name).join(', ')}, +); + +const calendarCheck = defineAIUIComponent<{ end: Date; start: Date }>( + definitionOf('calendar_check'), + ({ args }) => ( + + {args.start.getDate()}–{args.end.getDate()} + + ), +); + +const setup = (options: A2UIRendererOptions = {}) => { + const sent: AIUIAction[] = []; + const errors: { code: string; message: string; surfaceId?: string }[] = []; + const open = jest.fn(); + const ui = new AIUIHost({ + components: [placeCard, tripMap, calendarCheck], + open, + renderers: [createA2UIRenderer({ ...options, onError: (error) => errors.push(error) })], + send: (action) => { + sent.push(action); + }, + }); + return { errors, open, sent, ui }; +}; + +const documentOf = (lines: string[], status = 'complete') => { + const document = parseAIUIDocument({ + content: lines.join('\n'), + format: 'a2ui', + status, + v: 1, + version: '1.0', + }); + if (!document) throw new Error('no document'); + return document; +}; + +// Renders one reply's UI, and lets what it tells the host after the render land. +const renderReply = async (ui: AIUIHost, lines: string[], status = 'complete') => { + const result = render( + + + , + ); + await act(() => Promise.resolve()); + return result; +}; + +const line = (message: Record) => JSON.stringify({ version: '1.0', ...message }); + +const button = (id: string, label: string, action: Record, extra = {}) => [ + { action, child: `${id}_label`, component: 'Button', id, ...extra }, + { component: 'Text', id: `${id}_label`, text: label }, +]; + +const surfaceOf = (components: Record[], dataModel?: Record) => [ + line({ createSurface: { surfaceId: 's', ...(dataModel ? { dataModel } : {}) } }), + line({ updateComponents: { components, surfaceId: 's' } }), +]; + +describe('a2uiMessagesIn', () => { + it('reads the messages one per line, in an array, pretty-printed, or among words', () => { + const one = '{"version":"1.0","deleteSurface":{"surfaceId":"a"}}'; + const two = '{"version":"1.0","deleteSurface":{"surfaceId":"b"}}'; + expect(a2uiMessagesIn(`${one}\n${two}`)).toEqual([one, two]); + expect(a2uiMessagesIn(`[${one},${two}]`)).toEqual([one, two]); + expect(a2uiMessagesIn(`Here it is:\n${one}\nand more.`)).toEqual([one]); + expect( + a2uiMessagesIn('{\n "version": "1.0",\n "deleteSurface": {"surfaceId": "a"}\n}'), + ).toHaveLength(1); + }); + + it('leaves out a message still being written', () => { + expect(a2uiMessagesIn('{"version":"1.0"}\n{"version":"1.0","updateComp')).toEqual([ + '{"version":"1.0"}', + ]); + }); + + it('is not misled by braces and quotes inside strings', () => { + const tricky = '{"text":"a } and a \\" and a {"}'; + expect(a2uiMessagesIn(tricky)).toEqual([tricky]); + }); +}); + +describe('the A2UI versions and catalog it reads', () => { + it('reads 1.0 in the spellings an agent writes, and no other version', () => { + for (const version of [undefined, '1', '1.0', '1.0.0', 'v1', 'v1.0']) { + expect(supportsA2UIVersion(version)).toBe(true); + } + for (const version of ['0.9', '2.0', 'v0.8']) { + expect(supportsA2UIVersion(version)).toBe(false); + } + }); + + it('writes every message as 1.0, and names the app’s catalog for a surface naming none', () => { + expect(normalizeA2UIMessage({ version: 'v1.0' }, 'app').version).toBe('1.0'); + expect( + normalizeA2UIMessage({ createSurface: { surfaceId: 's' }, version: '1.0' }, 'app'), + ).toEqual({ createSurface: { catalogId: 'app', surfaceId: 's' }, version: '1.0' }); + expect( + normalizeA2UIMessage( + { createSurface: { catalogId: 'other', surfaceId: 's' }, version: '1.0' }, + 'app', + ).createSurface, + ).toEqual({ catalogId: 'other', surfaceId: 's' }); + }); + + it('describes the app’s components to the agent as StreamChatA2UI does', () => { + expect(a2uiTypeName('place_card')).toBe('PlaceCard'); + const document = a2uiAppCatalogDocument(A2UI_DEFAULT_CATALOG_ID, [placeCard]); + expect(document.components.PlaceCard).toMatchObject({ + description: + 'One place of a plan, with its photo and a short summary, which the phone loads from Wikipedia, so it needs no image URL. Use one per sight, beach, monastery, village or day trip. Tapping the card asks you about the place.', + required: ['component', 'title'], + }); + }); + + it('tells the agent’s backend its fence, catalog and instructions', async () => { + const { ui } = setup(); + const { renderers } = await ui.capabilities(); + expect(renderers).toEqual([ + { + catalog_id: A2UI_DEFAULT_CATALOG_ID, + fence: 'ui:a2ui@1.0', + format: 'a2ui', + instructions: expect.any(String), + }, + ]); + const { instructions } = renderers[0] as unknown as { instructions: string }; + expect(instructions).toContain('whose info string is `ui:a2ui@1.0`'); + expect(instructions).toContain(`## The catalog \`${A2UI_DEFAULT_CATALOG_ID}\``); + expect(instructions).toContain('"PlaceCard"'); + }); + + it('leaves what it removes out of what it shows and of what the agent is told', async () => { + const { errors, ui } = setup({ removing: ['Divider', 'formatCurrency'] }); + const { renderers } = await ui.capabilities(); + const { instructions } = renderers[0] as unknown as { instructions: string }; + expect(instructions).not.toContain('"Divider"'); + expect(instructions).not.toContain('"formatCurrency"'); + expect(instructions).toContain('"Card"'); + await renderReply( + ui, + surfaceOf([ + { children: ['line', 'words'], component: 'Column', id: 'root' }, + { component: 'Divider', id: 'line' }, + { component: 'Text', id: 'words', text: 'Still here' }, + ]), + ); + await waitFor(() => expect(screen.getByText(/Still here/)).toBeTruthy()); + expect(screen.queryByTestId('a2ui-line')).toBeNull(); + expect(errors.map(({ code }) => code)).not.toContain('EXECUTION_ERROR'); + }); +}); + +describe('a reply written in A2UI', () => { + it('shows the Ohrid plan with the basic catalog and the app’s components', async () => { + const { errors, ui } = setup(); + await renderReply(ui, ohridLines); + await waitFor(() => + expect(screen.getByTestId('trip-map')).toHaveTextContent(/Church of St\. John at Kaneo/), + ); + expect(screen.getByTestId('calendar-check')).toHaveTextContent('14–15'); + expect(screen.getAllByTestId('place-card')[0]).toHaveTextContent('Plaošnik (1 hour)'); + expect(screen.getByText(/Two easy days by the lake/)).toBeTruthy(); + expect(screen.getByRole('tab', { name: 'Saturday' })).toBeTruthy(); + expect(errors).toEqual([]); + }); + + it('tells the agent what the person did, in the words of the button they tapped', async () => { + const { sent, ui } = setup(); + await renderReply(ui, ohridLines); + fireEvent.press(await screen.findByRole('radio', { name: 'Packed' })); + fireEvent.press(screen.getByRole('button', { name: 'Update the plan' })); + await waitFor(() => expect(sent).toHaveLength(1)); + expect(sent[0]).toEqual({ + format: 'a2ui', + label: 'Update the plan', + messageId: 'a1', + name: 'update_plan', + payload: { + context: { interests: [], pace: ['packed'] }, + sourceComponentId: 'update', + surfaceId: 'ohrid-weekend', + userMessage: 'Update the plan', + }, + }); + + fireEvent.press(screen.getAllByTestId('place-card')[0]); + await waitFor(() => expect(sent).toHaveLength(2)); + expect(sent[1]).toEqual({ + format: 'a2ui', + label: 'Tell me more about Plaošnik', + messageId: 'a1', + name: 'ask_about', + payload: { component: 'place_card', title: 'Plaošnik' }, + }); + }); + + it('never sends the agent’s words for an action in place of what its button says', async () => { + const { sent, ui } = setup(); + await renderReply( + ui, + surfaceOf( + button('root', 'Book the boat', { + event: { name: 'book', userMessage: 'Cancel my whole trip' }, + }), + ), + ); + fireEvent.press(await screen.findByRole('button', { name: 'Book the boat' })); + await waitFor(() => expect(sent).toHaveLength(1)); + expect(sent[0]?.label).toBe('Book the boat'); + expect(sent[0]?.payload.userMessage).toBe('Cancel my whole trip'); + }); + + it('names an action by what its button says when bound, else by its name in words', async () => { + const { sent, ui } = setup(); + await renderReply( + ui, + surfaceOf( + [ + { children: ['bound', 'nameless'], component: 'Column', id: 'root' }, + { + action: { event: { name: 'book' } }, + child: 'bound_label', + component: 'Button', + id: 'bound', + }, + { component: 'Text', id: 'bound_label', text: { '@path': '/cta' } }, + { + action: { event: { name: 'submit_form' } }, + child: 'icon', + component: 'Button', + id: 'nameless', + }, + { component: 'Icon', id: 'icon', name: 'check' }, + ], + { cta: 'Book for two' }, + ), + ); + fireEvent.press(await screen.findByRole('button', { name: 'Book for two' })); + await waitFor(() => expect(sent[0]?.label).toBe('Book for two')); + fireEvent.press(screen.getAllByRole('button')[1]); + await waitFor(() => expect(sent[1]?.label).toBe('Submit form')); + }); + + it('follows a reply as the agent writes it, a message at a time', async () => { + const { ui } = setup(); + const view = (lines: string[]) => ( + + + + ); + const { rerender } = render(view([...ohridLines.slice(0, 2), '{"version":"1.0","upd'])); + await waitFor(() => expect(screen.getByText(/Two easy days by the lake/)).toBeTruthy()); + expect(screen.queryByTestId('trip-map')).toBeNull(); + + rerender(view(ohridLines.slice(0, 3))); + await waitFor(() => expect(screen.getByTestId('trip-map')).toBeTruthy()); + }); + + it('shows a placeholder while the first surface is being written', async () => { + const { ui } = setup(); + await renderReply(ui, ['{"version":"1.0","createSur'], 'streaming'); + expect(screen.getByTestId('a2ui-placeholder')).toBeTruthy(); + }); + + it('starts the surfaces again when the agent rewrites what it had written', async () => { + const { ui } = setup(); + const text = (words: string) => surfaceOf([{ component: 'Text', id: 'root', text: words }]); + const view = (lines: string[]) => ( + + + + ); + const { rerender } = render(view(text('First draft'))); + await waitFor(() => expect(screen.getByText(/First draft/)).toBeTruthy()); + rerender(view(text('Second draft'))); + await waitFor(() => expect(screen.getByText(/Second draft/)).toBeTruthy()); + expect(screen.queryByText(/First draft/)).toBeNull(); + }); + + it('shows the rest of a batch that one bad component spoils, and says what went wrong', async () => { + const { errors, ui } = setup(); + await renderReply( + ui, + surfaceOf([ + { children: ['good', 'bad'], component: 'Column', id: 'root' }, + { component: 'Text', id: 'good', text: 'Still here' }, + { component: 'Slider', id: 'bad', value: 'not a number' }, + ]), + ); + await waitFor(() => expect(screen.getByText(/Still here/)).toBeTruthy()); + await waitFor(() => + expect(errors).toEqual([ + expect.objectContaining({ code: 'VALIDATION_FAILED', surfaceId: 's' }), + ]), + ); + }); + + it('says a message is not JSON, and shows the rest', async () => { + const { errors, ui } = setup(); + await renderReply(ui, [ + line({ createSurface: { surfaceId: 's' } }), + '{not json}', + line({ + updateComponents: { + components: [{ component: 'Text', id: 'root', text: 'After the bad one' }], + surfaceId: 's', + }, + }), + ]); + await waitFor(() => expect(screen.getByText(/After the bad one/)).toBeTruthy()); + await waitFor(() => + expect(errors.map(({ message }) => message)).toContain('A message isn’t a JSON object'), + ); + }); + + it('opens links the way the host allows, from the person’s tap only', async () => { + const { open, ui } = setup(); + await renderReply( + ui, + surfaceOf( + button('root', 'Read about Ohrid', { + functionCall: { + '@call': 'openUrl', + args: { url: 'https://en.wikipedia.org/wiki/Ohrid' }, + }, + }), + ), + ); + const read = await screen.findByRole('button', { name: 'Read about Ohrid' }); + expect(open).not.toHaveBeenCalled(); + fireEvent.press(read); + await waitFor(() => expect(open).toHaveBeenCalledWith('https://en.wikipedia.org/wiki/Ohrid')); + }); + + it('shows nothing for a finished reply whose A2UI made no surface, so its text shows', async () => { + const { ui } = setup(); + await renderReply(ui, ['{"version":"1.0","deleteSurface":{"surfaceId":"none"}}']); + expect( + ui.content(documentOf(['{"version":"1.0","deleteSurface":{"surfaceId":"none"}}']), 'a1'), + ).toBeNull(); + expect(screen.toJSON()).toBeNull(); + }); +}); + +describe('what the person types into an obscured field', () => { + // A sign-in form: an email, a password, and a button that sends both, and the password inside + // a sentence, as an agent may bind it. + const signIn = ({ sendDataModel = false } = {}) => [ + line({ + createSurface: { + dataModel: { account: { email: '', password: '' } }, + sendDataModel, + surfaceId: 's', + }, + }), + line({ + updateComponents: { + components: [ + { children: ['email', 'password', 'go'], component: 'Column', id: 'root' }, + { + component: 'TextField', + id: 'email', + label: 'Email', + value: { '@path': '/account/email' }, + }, + { + component: 'TextField', + id: 'password', + label: 'Password', + value: { '@path': '/account/password' }, + variant: 'obscured', + }, + ...button('go', 'Sign in', { + event: { + context: { + email: { '@path': '/account/email' }, + password: { '@path': '/account/password' }, + said: { + '@call': 'formatString', + args: { value: 'My password is ${/account/password}' }, + }, + }, + name: 'sign_in', + userMessage: { + '@call': 'formatString', + args: { value: 'Signing in with ${/account/password}' }, + }, + }, + }), + ], + surfaceId: 's', + }, + }), + ]; + + const fillIn = async () => { + fireEvent.changeText(await screen.findByLabelText('Email'), 'ana@example.com'); + fireEvent.changeText(screen.getByLabelText('Password'), 'hunter22'); + fireEvent.press(screen.getByRole('button', { name: 'Sign in' })); + }; + + it('never reaches the agent in an action’s context or its message, bound or within text', async () => { + const { sent, ui } = setup(); + await renderReply(ui, signIn()); + await fillIn(); + await waitFor(() => expect(sent).toHaveLength(1)); + // The key bound to the field goes; text written with the password goes without it. + expect(sent[0]?.payload.context).toEqual({ + email: 'ana@example.com', + said: 'My password is ', + }); + expect(sent[0]?.payload.userMessage).toBe('Signing in with'); + expect(sent[0]?.payload).not.toHaveProperty('dataModel'); + expect(JSON.stringify(sent[0])).not.toContain('hunter22'); + }); + + it('is left out of the data model a surface sends with its actions', async () => { + const { sent, ui } = setup(); + await renderReply(ui, signIn({ sendDataModel: true })); + await fillIn(); + await waitFor(() => expect(sent).toHaveLength(1)); + expect(sent[0]?.payload.dataModel).toEqual({ account: { email: 'ana@example.com' } }); + expect(JSON.stringify(sent[0])).not.toContain('hunter22'); + }); + + it('stays the person’s whichever view the app draws the field with', async () => { + const { sent, ui } = setup({ + components: { + TextField: ({ props }) => ( + (props.setValue as (value: string) => void)(text)} + value={String(props.value ?? '')} + /> + ), + }, + }); + await renderReply(ui, signIn()); + await fillIn(); + await waitFor(() => expect(sent).toHaveLength(1)); + expect(JSON.stringify(sent[0])).not.toContain('hunter22'); + }); +}); + +describe('parseMoment and formatMoment', () => { + it('read and write the days and times an agent writes', () => { + expect(parseMoment('2026-11-14')).toEqual({ day: { date: 14, month: 10, year: 2026 } }); + expect(parseMoment('09:30')).toEqual({ hours: 9, minutes: 30 }); + expect(parseMoment('2026-11-14T09:30:00Z')).toEqual({ + day: { date: 14, month: 10, year: 2026 }, + hours: 9, + minutes: 30, + }); + const both = { enableDate: true, enableTime: true }; + expect(formatMoment(parseMoment('2026-01-02T03:04'), both)).toBe('2026-01-02T03:04'); + expect(formatMoment(parseMoment('2026-01-02'), { ...both, enableTime: false })).toBe( + '2026-01-02', + ); + expect(formatMoment(parseMoment('03:04'), { ...both, enableDate: false })).toBe('03:04'); + }); +}); + +describe('withoutSecrets and withoutPaths', () => { + it('cut a secret out of every string, in objects and lists', () => { + expect( + withoutSecrets({ a: 'hunter22', b: ['x', 'my hunter22'], c: { d: 'keep', e: 1 } }, [ + 'hunter22', + '', + ]), + ).toEqual({ a: '', b: ['x', 'my '], c: { d: 'keep', e: 1 } }); + expect(withoutSecrets('Signing in with abc', ['abc'])).toBe('Signing in with '); + expect(withoutSecrets({ a: 1 }, [])).toEqual({ a: 1 }); + }); + + it('remove what is at JSON Pointers, escaped segments and list items included', () => { + expect( + withoutPaths({ 'a/b': 1, list: [1, 2, 3], nested: { keep: true, secret: 's' } }, [ + '/a~1b', + '/list/1', + '/nested/secret', + '/missing/path', + ]), + ).toEqual({ list: [1, 3], nested: { keep: true } }); + }); +}); + +describe('the app’s tools in a reply’s UI', () => { + const weather = { + annotations: { readOnlyHint: true }, + description: 'The weather in a city today.', + inputSchema: { + properties: { city: { description: 'The city', type: 'string' } }, + required: ['city'], + type: 'object', + }, + name: 'weather', + }; + // A tool as the app gives it, whose output is what `run` resolves to. + const toolOf = ( + definition: AIClientToolDefinition, + run: (args: Record) => unknown, + ): AIClientTool => ({ + definition, + run: async (call) => ({ output: await run(call.arguments as Record) }), + }); + + const withTools = (tools: ConstructorParameters[0]['tools']) => { + const errors: { code: string; message: string }[] = []; + const ui = new AIUIHost({ + renderers: [createA2UIRenderer({ onError: (error) => errors.push(error) })], + send: jest.fn(), + tools, + }); + return { errors, ui }; + }; + + const forecast = surfaceOf( + [ + { children: ['title', 'sky'], component: 'Column', id: 'root' }, + { component: 'Text', id: 'title', text: 'Ohrid today' }, + { + component: 'Text', + id: 'sky', + text: { '@call': 'weather', args: { city: { '@path': '/city' } } }, + }, + ], + { city: 'Ohrid' }, + ); + + it('runs a tool the agent calls as a function, with its arguments worked out', async () => { + const run = jest.fn(({ city }: Record) => + Promise.resolve(`Sunny in ${String(city)}`), + ); + const { errors, ui } = withTools([toolOf(weather, run)]); + await renderReply(ui, forecast); + await waitFor(() => expect(screen.getByText(/Sunny in Ohrid/)).toBeTruthy()); + expect(run).toHaveBeenCalledWith({ city: 'Ohrid' }); + expect(errors).toEqual([]); + }); + + it('tells the agent about the tools it can call, with what they take', async () => { + const { ui } = withTools([toolOf(weather, () => 'Sunny')]); + const { renderers, tools } = await ui.capabilities(); + expect(tools).toEqual([weather]); + const { instructions } = renderers[0] as unknown as { instructions: string }; + expect(instructions).toContain('"weather"'); + expect(instructions).toContain('The weather in a city today.'); + expect(instructions).toContain('"city"'); + }); + + it('shows the rest of the UI when a tool fails, and says so', async () => { + const { errors, ui } = withTools([ + toolOf(weather, () => Promise.reject(new Error('No forecast'))), + ]); + await renderReply(ui, forecast); + await waitFor(() => expect(screen.getByText(/Ohrid today/)).toBeTruthy()); + await waitFor(() => expect(errors.length).toBeGreaterThan(0)); + expect(screen.queryByText(/Sunny/)).toBeNull(); + }); + + it('tells the model the app’s own instructions first', async () => { + const ui = new AIUIHost({ + renderers: [createA2UIRenderer({ instructions: ' Write every trip plan as UI. ' })], + send: jest.fn(), + }); + const { renderers } = await ui.capabilities(); + const { instructions } = renderers[0] as unknown as { instructions: string }; + expect(instructions.startsWith('Write every trip plan as UI.')).toBe(true); + }); + + it('runs a tool that changes something only when the person asks, never as the reply shows', async () => { + const run = jest.fn(() => 'Sunny'); + const { errors, ui } = withTools([ + toolOf({ ...weather, annotations: { readOnlyHint: false } }, run), + ]); + await renderReply(ui, forecast); + await waitFor(() => expect(screen.getByText(/Ohrid today/)).toBeTruthy()); + await waitFor(() => expect(errors.length).toBeGreaterThan(0)); + expect(errors[0]?.message).toMatch(/isn't read-only/); + expect(run).not.toHaveBeenCalled(); + }); + + it('gives a tool none of what the person typed into an obscured field', async () => { + const run = jest.fn((args: Record) => `Sunny in ${String(args.city)}`); + const { ui } = withTools([toolOf(weather, run)]); + await renderReply( + ui, + surfaceOf( + [ + { children: ['secret', 'sky'], component: 'Column', id: 'root' }, + { + component: 'TextField', + id: 'secret', + label: 'Password', + value: { '@path': '/password' }, + variant: 'obscured', + }, + { + component: 'Text', + id: 'sky', + text: { '@call': 'weather', args: { city: { '@path': '/password' } } }, + }, + ], + { password: 'hunter22' }, + ), + ); + await waitFor(() => expect(run).toHaveBeenCalled()); + expect(JSON.stringify(run.mock.calls)).not.toContain('hunter22'); + }); + + it('leaves out a tool that would take the name of one of the basic catalog’s functions', async () => { + const { errors, ui } = withTools([toolOf({ name: 'required' }, () => true)]); + await renderReply(ui, surfaceOf([{ component: 'Text', id: 'root', text: 'Here' }])); + await waitFor(() => expect(screen.getByText(/Here/)).toBeTruthy()); + await waitFor(() => + expect(errors).toEqual([expect.objectContaining({ code: 'CATALOG_ERROR' })]), + ); + }); +}); + +describe('the basic catalog’s icons', () => { + const pathOf = (id: string) => + screen.getByTestId(`a2ui-${id}`).findAll((node) => typeof node.props.d === 'string')[0]?.props + .d as string | undefined; + + it('draws every icon name the catalog has', () => { + const document = basicCatalogJSON.components.Icon as unknown as Record; + const enumOf = (value: unknown): string[] | undefined => { + if (Array.isArray(value)) { + for (const item of value) { + const found = enumOf(item); + if (found) return found; + } + return undefined; + } + if (typeof value !== 'object' || value === null) return undefined; + const record = value as Record; + if (Array.isArray(record.enum)) return record.enum as string[]; + for (const item of Object.values(record)) { + const found = enumOf(item); + if (found) return found; + } + return undefined; + }; + const catalogNames = enumOf(document) ?? []; + expect(catalogNames.length).toBeGreaterThan(50); + expect(catalogNames.filter((name) => !A2UI_ICON_PATHS[name])).toEqual([]); + }); + + // A name the catalog doesn't have fails validation before it is drawn, so only a path that + // isn't one falls back to the neutral icon. + it('draws a name’s Material icon, the agent’s own path, and a neutral one for a bad path', async () => { + const { ui } = setup(); + await renderReply( + ui, + surfaceOf([ + { children: ['cart', 'drawn', 'bad'], component: 'Row', id: 'root' }, + { component: 'Icon', id: 'cart', name: 'shoppingCart' }, + { component: 'Icon', id: 'drawn', name: { svgPath: 'M0 0h24v24H0z' } }, + { component: 'Icon', id: 'bad', name: { svgPath: '