diff --git a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts index f16d1f5eee02..e4280a8cef54 100644 --- a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts @@ -52,7 +52,7 @@ import { import type { TracingChannel } from 'node:diagnostics_channel'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../ai/core/gen-ai-attributes'; import { isEveGenAiRecordingDefault } from './gen-ai-recording-mode'; -import type { GenAiOptions } from '../../ai/core/utils'; +import { getEvaluationOutputMessages, type GenAiOptions } from '../../ai/core/utils'; import { getProviderMetadataAttributes, LAST_STEP_ONLY_USAGE_KEYS } from '../../ai/vercel-ai'; import { WORKERS_AI_INTEGRATION_NAME } from '../../ai/workers-ai/constants'; import { bindTracingChannelToSpan } from '../../tracing-channel'; @@ -689,7 +689,7 @@ function getOutputMessages( finishReason: string | undefined, ): string | undefined { if (type === 'experimental_evaluate' || type === 'experimental_decide') { - return stringify([{ type: 'evaluation', answers: withProviderConfidence(result) }]); + return getEvaluationOutputMessages(withProviderAnswerFields(result)); } // `languageModelCall` exposes the response as a `content` parts array; top-level results expose // `text` + `toolCalls`. Both normalize into the OTel `gen_ai.output.messages` assistant message. @@ -701,26 +701,36 @@ function getOutputMessages( } /** - * The AI SDK TypeSafe and OpenAI providers move each answer's `confidence` out of the answers into - * `providerMetadata..confidence` (keyed by question id). Put it back so decide answers keep it. + * Providers move answer fields into `providerMetadata`: TypeSafe and OpenAI as `.confidence[id]`, + * OpenRouter as `openrouter.answers[id].{confidence,legend}`. Put them back on the answers. */ -function withProviderConfidence(result: Record): unknown { +function withProviderAnswerFields(result: Record): unknown { const { answers, providerMetadata } = result; - const metadata = isObjectLike(providerMetadata) - ? [providerMetadata.typesafe, providerMetadata.openai].find(m => isObjectLike(m) && isObjectLike(m.confidence)) - : undefined; - const confidence = isObjectLike(metadata) && isObjectLike(metadata.confidence) ? metadata.confidence : undefined; - if (!confidence || !isObjectLike(answers)) { + if (!isObjectLike(answers) || !isObjectLike(providerMetadata)) { + return answers; + } + const confidenceById = [providerMetadata.typesafe, providerMetadata.openai] + .map(metadata => (isObjectLike(metadata) ? metadata.confidence : undefined)) + .find(isObjectLike); + const { openrouter } = providerMetadata; + const openrouterById = isObjectLike(openrouter) && isObjectLike(openrouter.answers) ? openrouter.answers : undefined; + if (!confidenceById && !openrouterById) { return answers; } return Object.fromEntries( - Object.entries(answers).map(([id, answer]) => [ - id, - isObjectLike(answer) && typeof confidence[id] === 'number' && answer.confidence === undefined - ? { ...answer, confidence: confidence[id] } - : answer, - ]), + Object.entries(answers).map(([id, answer]) => { + if (!isObjectLike(answer)) { + return [id, answer]; + } + const openrouterAnswer = openrouterById?.[id]; + const fields = isObjectLike(openrouterAnswer) ? openrouterAnswer : {}; + const { + confidence = asNumber(confidenceById?.[id]) ?? asNumber(fields.confidence), + legend = isObjectLike(fields.legend) ? fields.legend : undefined, + } = answer; + return [id, { ...answer, confidence, legend }]; + }), ); } diff --git a/packages/server-utils/test/integrations/vercel-ai/decide.test.ts b/packages/server-utils/test/integrations/vercel-ai/decide.test.ts index ae668d64e338..8a225cab123b 100644 --- a/packages/server-utils/test/integrations/vercel-ai/decide.test.ts +++ b/packages/server-utils/test/integrations/vercel-ai/decide.test.ts @@ -29,6 +29,27 @@ describe('Vercel AI decide spans', () => { getMainCarrier().__SENTRY__ = undefined; }); + function recordedAnswers(provider: string, result: Message['result']): unknown { + const message = { + type: 'experimental_decide', + event: { + provider, + modelId: 'typesafe/jev-1.13', + state: 'I was charged twice and the app keeps crashing', + questions: {}, + recordInputs: true, + recordOutputs: true, + }, + result, + } satisfies Message; + + const span = createSpanFromMessage(message, {} as Parameters[1])!; + enrichSpanOnEnd(span, message, {} as Parameters[2]); + span.end(); + + return JSON.parse(spanToStaticSpanJSON(endedSpans[0]!).data?.[GEN_AI_OUTPUT_MESSAGES] as string)[0].answers; + } + // `ai` 7.0.128 renamed the channel event from `experimental_evaluate` to `experimental_decide`. it.each(['experimental_evaluate', 'experimental_decide'] as const)('creates a decide span for `%s`', type => { const message = { @@ -58,29 +79,80 @@ describe('Vercel AI decide spans', () => { }); it.each(['typesafe', 'openai'])('restores answer confidence from `providerMetadata.%s`', provider => { - const message = { - type: 'experimental_decide', - event: { - provider, - modelId: 'gpt-6-luna', - state: 'I was charged twice', - questions: { department: { type: 'choice', criteria: { billing: 'Charges', support: 'Other' } } }, - recordInputs: true, - recordOutputs: true, - }, - result: { - answers: { department: { type: 'choice', choice: 'billing' } }, - providerMetadata: { [provider]: { confidence: { department: 0.91 } } }, - }, - } satisfies Message; + const answers = recordedAnswers(provider, { + answers: { department: { type: 'choice', choice: 'billing' } }, + providerMetadata: { [provider]: { confidence: { department: 0.91 } } }, + }); - const span = createSpanFromMessage(message, {} as Parameters[1])!; - enrichSpanOnEnd(span, message, {} as Parameters[2]); - span.end(); + expect(answers).toEqual({ department: { type: 'choice', choice: 'billing', confidence: 0.91 } }); + }); - const json = spanToStaticSpanJSON(endedSpans[0]!); - expect(JSON.parse(json.data?.[GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ - { type: 'evaluation', answers: { department: { type: 'choice', choice: 'billing', confidence: 0.91 } } }, - ]); + describe('OpenRouter evaluation models', () => { + it('restores choice confidence from `providerMetadata.openrouter.answers`', () => { + const answers = recordedAnswers('openrouter', { + answers: { + department: { type: 'choice', choice: 'billing', probabilities: { billing: 0.82, technical: 0.18 } }, + wantsRefund: { type: 'boolean', probability: 0.97 }, + }, + providerMetadata: { + openrouter: { provider: 'TypeSafe', answers: { department: { confidence: 0.64 }, wantsRefund: {} } }, + }, + }); + + expect(answers).toEqual({ + department: { + type: 'choice', + choice: 'billing', + probabilities: { billing: 0.82, technical: 0.18 }, + confidence: 0.64, + }, + wantsRefund: { type: 'boolean', probability: 0.97 }, + }); + }); + + it('restores score confidence and legend from `providerMetadata.openrouter.answers`', () => { + const legend = { '0': 'Not urgent', '1': 'Handle today', '2': 'Page on-call' }; + + const answers = recordedAnswers('openrouter', { + answers: { urgency: { type: 'score', score: 1.8, probabilities: { '0': 0, '1': 0.2, '2': 0.8 } } }, + providerMetadata: { openrouter: { answers: { urgency: { confidence: 0.31, legend } } } }, + }); + + expect(answers).toEqual({ + urgency: { + type: 'score', + score: 1.8, + probabilities: { '0': 0, '1': 0.2, '2': 0.8 }, + confidence: 0.31, + legend, + }, + }); + }); + + it.each([ + ['no provider metadata', undefined], + ['no `answers` entry', { openrouter: { provider: 'TypeSafe', usage: { cost: 0.0004 } } }], + ['no entry for the question', { openrouter: { answers: { department: { confidence: 0.64 } } } }], + ])('leaves answers unchanged with %s', (_, providerMetadata) => { + const answers = recordedAnswers('openrouter', { + answers: { urgency: { type: 'score', score: 1.8 } }, + providerMetadata, + }); + + expect(answers).toEqual({ urgency: { type: 'score', score: 1.8 } }); + }); + + it('keeps confidence and legend already on the answer', () => { + const answers = recordedAnswers('openrouter', { + answers: { urgency: { type: 'score', score: 1.8, confidence: 0.9, legend: { '2': 'Page on-call' } } }, + providerMetadata: { + openrouter: { answers: { urgency: { confidence: 0.31, legend: { '2': 'Escalate' } } } }, + }, + }); + + expect(answers).toEqual({ + urgency: { type: 'score', score: 1.8, confidence: 0.9, legend: { '2': 'Page on-call' } }, + }); + }); }); });