From 0f4e9a99cb6307385e8ac948b1391e53b641a181 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:35:39 +0000 Subject: [PATCH 1/5] ref!: Don't make generated types part of public api --- js/eslint.config.ts | 10 + js/src/eval-parameters.ts | 2 +- js/src/framework-types.ts | 2 +- js/src/framework.ts | 12 +- js/src/framework2.ts | 43 +- js/src/functions/invoke.ts | 12 +- js/src/functions/stream.ts | 14 +- js/src/gitutil.ts | 5 +- js/src/isomorph.ts | 5 +- js/src/logger.ts | 188 +++-- js/src/prompt-cache/parameters-cache.ts | 10 +- js/src/prompt-cache/prompt-cache.ts | 10 +- js/src/prompt-schemas.ts | 16 +- js/src/prompt-types.test.ts | 135 ++++ js/src/public-types.test.ts | 2 + js/src/sandbox.ts | 4 +- js/src/sdk-boundary.test.ts | 70 ++ js/src/sdk-contracts.test.ts | 269 +++++++ js/src/sdk-schemas.ts | 717 ++++++++++++++++++ js/src/sdk-types.ts | 573 ++++++++++++++ .../api-compatibility.test.ts | 16 + js/util/git_fields.ts | 2 +- js/util/object.ts | 9 +- js/util/span_identifier_v4.ts | 2 +- 24 files changed, 1987 insertions(+), 141 deletions(-) create mode 100644 js/src/prompt-types.test.ts create mode 100644 js/src/sdk-boundary.test.ts create mode 100644 js/src/sdk-contracts.test.ts create mode 100644 js/src/sdk-schemas.ts create mode 100644 js/src/sdk-types.ts diff --git a/js/eslint.config.ts b/js/eslint.config.ts index 80829409c..c718d52c8 100644 --- a/js/eslint.config.ts +++ b/js/eslint.config.ts @@ -169,6 +169,16 @@ export default [ "error", { patterns: [ + { + group: [ + "**/generated_types", + "**/generated_plain_types", + "**/generated_types.ts", + "**/generated_plain_types.ts", + ], + message: + "Generated backend definitions are compatibility-test fixtures. Use SDK-owned types and validators instead.", + }, { group: [ "./exports", diff --git a/js/src/eval-parameters.ts b/js/src/eval-parameters.ts index bc15dc105..9a21208ad 100644 --- a/js/src/eval-parameters.ts +++ b/js/src/eval-parameters.ts @@ -5,7 +5,7 @@ import { promptDefinitionToPromptData, type PromptDefinitionWithTools, } from "./prompt-schemas"; -import { PromptData as promptDataSchema } from "./generated_types"; +import { PromptData as promptDataSchema } from "./sdk-schemas"; export type EvalParameters = Record< string, diff --git a/js/src/framework-types.ts b/js/src/framework-types.ts index 0f9d45c26..27279c8b0 100644 --- a/js/src/framework-types.ts +++ b/js/src/framework-types.ts @@ -1,4 +1,4 @@ -import type { IfExistsType as IfExists } from "./generated_plain_types"; +import type { IfExists } from "./sdk-types"; export type GenericFunction = | ((input: Input) => Output) diff --git a/js/src/framework.ts b/js/src/framework.ts index 9d2bef9d3..770e810df 100644 --- a/js/src/framework.ts +++ b/js/src/framework.ts @@ -7,13 +7,13 @@ import { SpanTypeAttribute, spanObjectTypeV3ToTypedString, } from "../util/index"; -import { ObjectReference as ObjectReferenceSchema } from "./generated_types"; +import { ObjectReference as ObjectReferenceSchema } from "./sdk-schemas"; import type { - GitMetadataSettingsType as GitMetadataSettings, - ObjectReferenceType as ObjectReference, - RepoInfoType as RepoInfo, - SSEProgressEventDataType as SSEProgressEventData, -} from "./generated_plain_types"; + GitMetadataSettings, + ObjectReference, + RepoInfo, + SSEProgressEventData, +} from "./sdk-types"; import { queue } from "async"; import iso from "./isomorph"; diff --git a/js/src/framework2.ts b/js/src/framework2.ts index ed1eef15f..fb9ee26ea 100644 --- a/js/src/framework2.ts +++ b/js/src/framework2.ts @@ -3,17 +3,16 @@ import type { Trace } from "./trace"; import iso from "./isomorph"; import { slugify } from "../util/string_util"; import { z } from "zod/v3"; -import { Project as projectSchema } from "./generated_types"; +import { Project as projectSchema } from "./sdk-schemas"; import type { - FunctionTypeEnumType as FunctionType, - IfExistsType as IfExists, - SavedFunctionIdType as SavedFunctionId, - PromptBlockDataType as PromptBlockData, - PromptDataType as PromptData, - ToolFunctionDefinitionType as ToolFunctionDefinition, - ExtendedSavedFunctionIdType as ExtendedSavedFunctionId, - FunctionDataType, -} from "./generated_plain_types"; + FunctionTypeEnum as FunctionType, + IfExists, + SavedFunctionId, + PromptBlockData, + PromptData, + ToolFunctionDefinition, + ExtendedSavedFunctionId, +} from "./sdk-types"; import { loadPrettyXact, TransactionId } from "../util/index"; import { _internalGetGlobalState, @@ -121,7 +120,7 @@ class Project { } await login(); const projectMap = new ProjectNameIdMap(); - const functionDefinitions: FunctionEvent[] = []; + const functionDefinitions: PromptFunctionEvent[] = []; if (this._publishableCodeFunctions.length > 0) { // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. console.warn( @@ -499,7 +498,7 @@ export class CodePrompt { async toFunctionDefinition( projectNameToId: ProjectNameIdMap, - ): Promise { + ): Promise { const prompt_data = { ...this.prompt, }; @@ -666,7 +665,7 @@ export class CodeParameters { async toFunctionDefinition( projectNameToId: ProjectNameIdMap, - ): Promise { + ): Promise { const schema = serializeEvalParameterstoParametersSchema(this.schema); return { project_id: await projectNameToId.resolve(this.project), @@ -776,13 +775,11 @@ function getDefaultDataFromParametersSchema( ); } -interface FunctionEvent { +interface FunctionEventBase { project_id: string; slug: string; name: string; description: string; - prompt_data?: PromptData; - function_data: FunctionDataType; function_type?: FunctionType; if_exists?: IfExists; tags?: string[]; @@ -790,6 +787,20 @@ interface FunctionEvent { environments?: { slug: string }[]; } +interface PromptFunctionEvent extends FunctionEventBase { + prompt_data: PromptData; + function_data: { type: "prompt" }; +} + +interface ParametersFunctionEvent extends FunctionEventBase { + function_type: "parameters"; + function_data: { + type: "parameters"; + data: Record; + __schema: ParametersSchema; + }; +} + class ProjectNameIdMap { private nameToId: Record = {}; private idToName: Record = {}; diff --git a/js/src/functions/invoke.ts b/js/src/functions/invoke.ts index 1cfe9a5b3..ac6e3f1a4 100644 --- a/js/src/functions/invoke.ts +++ b/js/src/functions/invoke.ts @@ -1,10 +1,10 @@ -import { FunctionId as functionIdSchema } from "../generated_types"; +import { FunctionId as functionIdSchema } from "../sdk-schemas"; import type { - InvokeFunctionType as InvokeFunctionRequest, - ChatCompletionMessageParamType as Message, - StreamingModeType as StreamingMode, - FunctionTypeEnumType as FunctionType, -} from "../generated_plain_types"; + InvokeFunction as InvokeFunctionRequest, + ChatCompletionMessageParam as Message, + StreamingMode, + FunctionTypeEnum as FunctionType, +} from "../sdk-types"; import { _internalGetGlobalState, BraintrustState, diff --git a/js/src/functions/stream.ts b/js/src/functions/stream.ts index 901891119..ae0ba64e1 100644 --- a/js/src/functions/stream.ts +++ b/js/src/functions/stream.ts @@ -2,12 +2,12 @@ import { CallEvent as callEventSchema, SSEConsoleEventData as sseConsoleEventDataSchema, SSEProgressEventData as sseProgressEventDataSchema, -} from "../generated_types"; +} from "../sdk-schemas"; import type { - CallEventType as CallEvent, - SSEConsoleEventDataType, - SSEProgressEventDataType, -} from "../generated_plain_types"; + CallEvent, + SSEConsoleEventData, + SSEProgressEventData, +} from "../sdk-types"; import { createParser, EventSourceParser, @@ -25,8 +25,8 @@ export type BraintrustStreamChunk = | { type: "reasoning_delta"; data: string } | { type: "json_delta"; data: string } | { type: "error"; data: string } - | { type: "console"; data: SSEConsoleEventDataType } - | { type: "progress"; data: SSEProgressEventDataType } + | { type: "console"; data: SSEConsoleEventData } + | { type: "progress"; data: SSEProgressEventData } | { type: "start"; data: string } | { type: "done"; data: string }; diff --git a/js/src/gitutil.ts b/js/src/gitutil.ts index 9a687ebb3..091404ce6 100644 --- a/js/src/gitutil.ts +++ b/js/src/gitutil.ts @@ -1,7 +1,4 @@ -import type { - GitMetadataSettingsType as GitMetadataSettings, - RepoInfoType as RepoInfo, -} from "./generated_plain_types"; +import type { GitMetadataSettings, RepoInfo } from "./sdk-types"; import { debugLogger } from "./debug-logger"; import { runGitCommand } from "./git-command"; diff --git a/js/src/isomorph.ts b/js/src/isomorph.ts index c02657d31..c06635d0d 100644 --- a/js/src/isomorph.ts +++ b/js/src/isomorph.ts @@ -1,7 +1,4 @@ -import type { - GitMetadataSettingsType as GitMetadataSettings, - RepoInfoType as RepoInfo, -} from "./generated_plain_types"; +import type { GitMetadataSettings, RepoInfo } from "./sdk-types"; import { newGlobalTracingChannel, type GlobalHookAsyncLocalStorage, diff --git a/js/src/logger.ts b/js/src/logger.ts index 13d85fbf6..b7bd554c0 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -76,27 +76,25 @@ import { DatasetSnapshot as datasetSnapshotSchema, PromptData as promptDataSchema, Prompt as promptSchema, -} from "./generated_types"; +} from "./sdk-schemas"; import type { - AnyModelParamsType as AnyModelParam, - AttachmentReferenceType as AttachmentReference, - BraintrustAttachmentReferenceType as BraintrustAttachmentReference, - ChatCompletionToolType as ChatCompletionTool, - ExternalAttachmentReferenceType as ExternalAttachmentReference, - ModelParamsType as ModelParams, - AttachmentStatusType as AttachmentStatus, - GitMetadataSettingsType as GitMetadataSettings, - ChatCompletionMessageParamType as Message, - ChatCompletionOpenAIMessageParamType as OpenAIMessage, - DatasetSnapshotType as DatasetSnapshot, - PromptDataType as PromptData, - PromptType as PromptRow, - PromptSessionEventType as PromptSessionEvent, - RepoInfoType as RepoInfo, - ObjectReferenceType as ObjectReference, - PromptBlockDataType as PromptBlockData, - ResponseFormatJsonSchemaType as ResponseFormatJsonSchema, -} from "./generated_plain_types"; + AttachmentReference, + BraintrustAttachmentReference, + ChatCompletionTool, + ExternalAttachmentReference, + ModelParams, + AttachmentStatus, + GitMetadataSettings, + ChatCompletionMessageParam as Message, + ChatCompletionOpenAIMessageParam as OpenAIMessage, + DatasetSnapshot, + PromptData, + FunctionTypeEnum, + PromptSessionEvent, + RepoInfo, + ObjectReference, + PromptBlockData, +} from "./sdk-types"; const BRAINTRUST_ATTACHMENT = BraintrustAttachmentReferenceSchema.shape.type.value; @@ -8797,37 +8795,52 @@ export class Dataset extends ObjectFetcher { } type CompiledPromptResponseFormat = - Exclude extends infer ResponseFormat - ? ResponseFormat extends { - type: "json_schema"; - json_schema: infer JsonSchema; - } - ? Omit & { - json_schema: Omit< - Extract, - "schema" - > & { - schema?: Record; - }; - } - : ResponseFormat - : never; - -// "none" is not assignable to older openai clients so we gotta exclude it -type CompiledPromptReasoningEffort = Exclude< - AnyModelParam["reasoning_effort"], - "none" ->; - -export type CompiledPromptParams = Omit< - NonNullable["params"], - "use_cache" | "response_format" | "reasoning_effort" -> & - Omit & { - reasoning_effort?: CompiledPromptReasoningEffort; - response_format?: CompiledPromptResponseFormat; - model: NonNullable["model"]>; - }; + | { type: "json_object" } + | { type: "text" } + | { + type: "json_schema"; + json_schema: { + name: string; + description?: string; + schema?: Record; + strict?: boolean | null; + }; + }; + +// Keep the compiled API independent of the backend's provider parameter union. +// "none" is excluded from reasoning_effort for compatibility with older OpenAI clients. +export interface CompiledPromptParams { + model: string; + max_tokens: number; + temperature?: number; + top_p?: number; + max_completion_tokens?: number; + frequency_penalty?: number; + presence_penalty?: number; + response_format?: CompiledPromptResponseFormat; + tool_choice?: + | "auto" + | "none" + | "required" + | { + type: "function"; + function: { name: string }; + }; + function_call?: "auto" | "none" | { name: string }; + n?: number; + stop?: string[]; + reasoning_effort?: "minimal" | "low" | "medium" | "high"; + verbosity?: "low" | "medium" | "high"; + top_k?: number; + stop_sequences?: string[]; + reasoning_enabled?: boolean; + reasoning_budget?: number; + /** Legacy parameter; prefer max_tokens. */ + max_tokens_to_sample?: number; + maxOutputTokens?: number; + topP?: number; + topK?: number; +} export type ChatPrompt = { messages: OpenAIMessage[]; @@ -8858,9 +8871,13 @@ export type CompiledPrompt = : // eslint-disable-next-line @typescript-eslint/no-empty-object-type {}); -export type DefaultPromptArgs = Partial< - CompiledPromptParams & AnyModelParam & ChatPrompt & CompletionPrompt ->; +export interface DefaultPromptArgs + extends + Partial, + Partial, + Partial { + use_cache?: boolean; +} function isAttachmentObject(value: unknown): boolean { return ( @@ -9024,17 +9041,26 @@ export function renderMessageImpl( }; } +interface PromptMetadata { + name: string; + slug: string; + project_id?: string; + description?: string | null; + created?: string | null; + prompt_data?: PromptData | null; + tags?: string[] | null; + // Preserve the generated prompt row's permissive metadata contract. + // eslint-disable-next-line @typescript-eslint/no-empty-object-type + metadata?: {} | null; + function_type?: FunctionTypeEnum | null; +} + export type PromptRowWithId< HasId extends boolean = true, HasVersion extends boolean = true, -> = Omit & - Partial> & - (HasId extends true - ? Pick - : Partial>) & - (HasVersion extends true - ? Pick - : Partial>); +> = PromptMetadata & + (HasId extends true ? { id: string } : { id?: string }) & + (HasVersion extends true ? { _xact_id: string } : { _xact_id?: string }); export function deserializePlainStringAsJSON(s: string) { if (s.trim() === "") { @@ -9121,6 +9147,16 @@ export function renderPromptParams( return params; } +export interface SerializedPrompt { + metadata: PromptRowWithId | PromptSessionEvent; + defaults: DefaultPromptArgs; + noTrace: boolean; +} + +// Initialized by the class so cache serialization can access private fields +// without exposing a serialization method on the public Prompt API. +export let serializePromptForCache: (prompt: Prompt) => SerializedPrompt; + export class Prompt< HasId extends boolean = true, HasVersion extends boolean = true, @@ -9457,13 +9493,12 @@ export class Prompt< ); } - /** @internal */ - public _internalSerializeForCache() { - return { - metadata: this.metadata, - defaults: this.defaults, - noTrace: this.noTrace, - }; + static { + serializePromptForCache = (prompt) => ({ + metadata: prompt.metadata, + defaults: prompt.defaults, + noTrace: prompt.noTrace, + }); } public static fromPromptData( @@ -9482,6 +9517,14 @@ export class Prompt< } } +export interface SerializedParameters { + metadata: ParametersRow; +} + +export let serializeParametersForCache: ( + parameters: RemoteEvalParameters, +) => SerializedParameters; + export class RemoteEvalParameters< HasId extends boolean = true, HasVersion extends boolean = true, @@ -9528,9 +9571,10 @@ export class RemoteEvalParameters< return (this.metadata.function_data.data ?? {}) as T; } - /** @internal */ - public _internalSerializeForCache() { - return { metadata: this.metadata }; + static { + serializeParametersForCache = (parameters) => ({ + metadata: parameters.metadata, + }); } public validate(data: unknown): boolean { diff --git a/js/src/prompt-cache/parameters-cache.ts b/js/src/prompt-cache/parameters-cache.ts index bb81a7a9c..530cfb28a 100644 --- a/js/src/prompt-cache/parameters-cache.ts +++ b/js/src/prompt-cache/parameters-cache.ts @@ -1,4 +1,8 @@ -import { RemoteEvalParameters } from "../logger"; +import { + RemoteEvalParameters, + serializeParametersForCache, + type SerializedParameters, +} from "../logger"; import { LRUCache } from "../lru-cache"; import { DiskCache } from "./disk-cache"; @@ -35,7 +39,7 @@ export type ParametersMemoryCacheEntry = { }; export type ParametersDiskCacheEntry = { - value: ReturnType; + value: SerializedParameters; resolvedOrgIdentity?: string; }; @@ -111,7 +115,7 @@ export class ParametersCache { this.memoryCache?.set(cacheKey, memoryEntry); if (this.diskCache) { await this.diskCache.set(cacheKey, { - value: value._internalSerializeForCache(), + value: serializeParametersForCache(value), resolvedOrgIdentity: this.expectedResolvedOrgIdentity, }); } diff --git a/js/src/prompt-cache/prompt-cache.ts b/js/src/prompt-cache/prompt-cache.ts index eb58ca0c2..8fa7b291a 100644 --- a/js/src/prompt-cache/prompt-cache.ts +++ b/js/src/prompt-cache/prompt-cache.ts @@ -1,4 +1,8 @@ -import { Prompt } from "../logger"; +import { + Prompt, + serializePromptForCache, + type SerializedPrompt, +} from "../logger"; import { LRUCache } from "../lru-cache"; import { DiskCache } from "./disk-cache"; @@ -40,7 +44,7 @@ export type PromptMemoryCacheEntry = { }; export type PromptDiskCacheEntry = { - value: ReturnType; + value: SerializedPrompt; resolvedOrgIdentity?: string; }; @@ -168,7 +172,7 @@ export class PromptCache { this.memoryCache?.set(cacheKey, memoryEntry); if (this.diskCache) { await this.diskCache.set(cacheKey, { - value: value._internalSerializeForCache(), + value: serializePromptForCache(value), resolvedOrgIdentity: this.expectedResolvedOrgIdentity, }); } diff --git a/js/src/prompt-schemas.ts b/js/src/prompt-schemas.ts index c83c420f0..94a433212 100644 --- a/js/src/prompt-schemas.ts +++ b/js/src/prompt-schemas.ts @@ -1,19 +1,19 @@ import type { - ToolFunctionDefinitionType as ToolFunctionDefinition, - ChatCompletionMessageParamType, - ModelParamsType, - PromptBlockDataType as PromptBlockData, - PromptDataType as PromptData, -} from "./generated_plain_types"; + ToolFunctionDefinition, + ChatCompletionMessageParam, + ModelParams, + PromptBlockData, + PromptData, +} from "./sdk-types"; // This roughly maps to promptBlockDataSchema, but is more ergonomic for the user. export type PromptContents = | { prompt: string } - | { messages: ChatCompletionMessageParamType[] }; + | { messages: ChatCompletionMessageParam[] }; export type PromptDefinition = PromptContents & { model: string; - params?: ModelParamsType; + params?: ModelParams; templateFormat?: "mustache" | "nunjucks" | "none"; environments?: string[]; }; diff --git a/js/src/prompt-types.test.ts b/js/src/prompt-types.test.ts new file mode 100644 index 000000000..51df00f1b --- /dev/null +++ b/js/src/prompt-types.test.ts @@ -0,0 +1,135 @@ +import { expectTypeOf, test } from "vitest"; +import type { ChatCompletionCreateParamsNonStreaming } from "openai/resources/chat/completions"; +import type { CompletionCreateParamsNonStreaming } from "openai/resources/completions"; +import { + Prompt, + type CompiledPrompt, + type CompiledPromptParams, + type DefaultPromptArgs, + type PromptRowWithId, +} from "./logger"; +import type { CodeParameters, CodePrompt } from "./framework2"; +import type { + AnyModelParamsType, + FunctionDataType, + PromptDataType, + PromptType, +} from "./generated_plain_types"; + +test("compiled prompts preserve provider parameter types and compatibility", () => { + // The generated backend types are a compatibility reference, not the public API. + type BackendParams = Omit< + AnyModelParamsType, + "use_cache" | "response_format" | "reasoning_effort" + > & { model: string }; + type Params = Omit< + CompiledPromptParams, + "response_format" | "reasoning_effort" + >; + expectTypeOf().toMatchTypeOf(); + expectTypeOf().toMatchTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf< + CompiledPrompt<"chat"> + >().toMatchTypeOf(); + expectTypeOf< + CompiledPrompt<"completion"> + >().toMatchTypeOf(); + expectTypeOf().toEqualTypeOf< + "minimal" | "low" | "medium" | "high" | undefined + >(); + expectTypeOf().toEqualTypeOf< + boolean | undefined + >(); + + if (false) { + const prompt = null as unknown as Prompt; + // @ts-expect-error Cache serialization is internal, not a Prompt method. + prompt._internalSerializeForCache(); + expectTypeOf(prompt.build({})).toEqualTypeOf>(); + expectTypeOf(prompt.build({}, { flavor: "completion" })).toEqualTypeOf< + CompiledPrompt<"completion"> + >(); + expectTypeOf(prompt.buildWithAttachments({})).toEqualTypeOf< + Promise> + >(); + const defaults: DefaultPromptArgs = { + response_format: { + type: "json_schema", + json_schema: { + name: "result", + schema: { type: "object" }, + strict: null, + }, + }, + use_cache: true, + }; + new Prompt( + { id: "id", _xact_id: "version", name: "name", slug: "slug" }, + defaults, + false, + ); + defaults.response_format = { + type: "json_schema", + json_schema: { + name: "result", + // @ts-expect-error Compiled schemas are objects, not templates. + schema: "{{schema}}", + }, + }; + // @ts-expect-error Preserve compatibility with older OpenAI clients. + defaults.reasoning_effort = "none"; + } +}); + +test("prompt metadata preserves required and optional identifiers", () => { + type Metadata = Omit< + PromptType, + "log_id" | "org_id" | "project_id" | "id" | "_xact_id" + > & { + project_id?: string; + }; + type RequiredRow = Metadata & { id: string; _xact_id: string }; + type OptionalRow = Metadata & { id?: string; _xact_id?: string }; + expectTypeOf>().toMatchTypeOf(); + expectTypeOf().toMatchTypeOf>(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf>().toMatchTypeOf(); + expectTypeOf().toMatchTypeOf>(); + expectTypeOf["id"]>().toEqualTypeOf(); + expectTypeOf["_xact_id"]>().toEqualTypeOf< + string | undefined + >(); + expectTypeOf["id"]>().toEqualTypeOf< + string | undefined + >(); + expectTypeOf< + PromptRowWithId["_xact_id"] + >().toEqualTypeOf(); + expectTypeOf["id"]>().toEqualTypeOf(); + expectTypeOf["id"]>().toEqualTypeOf< + string | undefined + >(); +}); + +test("function definitions expose only the variant each builder produces", () => { + type PromptEvent = Awaited>; + type ParametersEvent = Awaited< + ReturnType + >; + expectTypeOf().toEqualTypeOf<{ + type: "prompt"; + }>(); + expectTypeOf< + PromptEvent["prompt_data"] + >().branded.toEqualTypeOf(); + expectTypeOf< + ParametersEvent["function_data"]["type"] + >().toEqualTypeOf<"parameters">(); + expectTypeOf< + ParametersEvent["function_type"] + >().toEqualTypeOf<"parameters">(); + expectTypeOf().toMatchTypeOf< + Extract + >(); +}); diff --git a/js/src/public-types.test.ts b/js/src/public-types.test.ts index 3a37a9cf7..2870520fd 100644 --- a/js/src/public-types.test.ts +++ b/js/src/public-types.test.ts @@ -53,6 +53,8 @@ test("does not expose runtime schemas or implementation helpers", () => { "SpanImpl", "IDGenerator", "_exportsForTestingOnly", + "serializePromptForCache", + "serializeParametersForCache", "default", ]) { expect(publicExports).not.toHaveProperty(name); diff --git a/js/src/sandbox.ts b/js/src/sandbox.ts index 58c9f45be..4f91fc753 100644 --- a/js/src/sandbox.ts +++ b/js/src/sandbox.ts @@ -1,6 +1,6 @@ import { z } from "zod/v3"; import { slugify } from "../util/string_util"; -import type { IfExistsType } from "./generated_plain_types"; +import type { IfExists } from "./sdk-types"; import { type BraintrustState, _internalGetGlobalState } from "./logger"; /** @@ -32,7 +32,7 @@ interface RegisterSandboxOptions { /** Optional metadata. */ metadata?: Record; /** What to do if function already exists. Defaults to "replace". */ - ifExists?: IfExistsType; + ifExists?: IfExists; /** Braintrust API key. Uses BRAINTRUST_API_KEY env var if not provided. */ apiKey?: string; /** Braintrust app URL. Uses default if not provided. */ diff --git a/js/src/sdk-boundary.test.ts b/js/src/sdk-boundary.test.ts new file mode 100644 index 000000000..f283bce28 --- /dev/null +++ b/js/src/sdk-boundary.test.ts @@ -0,0 +1,70 @@ +import { readFileSync, readdirSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { build } from "esbuild"; +import ts from "typescript"; +import { expect, test } from "vitest"; + +test("production sources cannot depend on generated backend definitions", () => { + for (const directory of [__dirname, resolve(__dirname, "../util")]) { + for (const file of readdirSync(directory, { recursive: true })) { + const name = String(file); + if ( + !/\.[cm]?tsx?$/.test(name) || + /\.test\.[cm]?tsx?$/.test(name) || + /(?:^|\/)generated_(?:plain_)?types\.ts$/.test(name) + ) { + continue; + } + const path = join(directory, name); + const source = ts.createSourceFile( + path, + readFileSync(path, "utf8"), + ts.ScriptTarget.Latest, + true, + ); + const dependencies: string[] = []; + function visit(node: ts.Node) { + if ( + ts.isStringLiteral(node) && + /(?:^|\/)generated_(?:plain_)?types(?:\.[cm]?[jt]s)?$/.test(node.text) + ) { + dependencies.push(node.text); + } + node.forEachChild(visit); + } + visit(source); + expect(dependencies, path).toEqual([]); + } + } +}); + +test("public bundles exclude the backend schema modules", async () => { + const result = await build({ + entryPoints: [ + "node", + "browser", + "edge-light", + "workerd", + "instrumentation", + ].map((entry) => resolve(__dirname, entry, "index.ts")), + bundle: true, + packages: "external", + platform: "node", + format: "esm", + splitting: true, + outdir: "sdk-boundary-check", + write: false, + metafile: true, + logLevel: "silent", + }); + expect( + Object.keys(result.metafile.inputs).filter((file) => + /generated_(?:plain_)?types/.test(file), + ), + ).toEqual([]); + expect( + Object.keys(result.metafile.inputs).some((file) => + file.endsWith("sdk-schemas.ts"), + ), + ).toBe(true); +}); diff --git a/js/src/sdk-contracts.test.ts b/js/src/sdk-contracts.test.ts new file mode 100644 index 000000000..1d7b5db92 --- /dev/null +++ b/js/src/sdk-contracts.test.ts @@ -0,0 +1,269 @@ +import { resolve } from "node:path"; +import ts from "typescript"; +import { expect, expectTypeOf, test } from "vitest"; +import { zodToJsonSchema } from "zod-to-json-schema"; +import type * as Backend from "./generated_plain_types"; +import * as backendSchemas from "./generated_types"; +import type * as SDK from "./sdk-types"; +import * as schemas from "./sdk-schemas"; + +test("model parameters preserve provider field autocomplete", () => { + const suggestions = []; + for (const [module, name] of [ + ["./sdk-types", "ModelParams"], + ["./generated_plain_types", "ModelParamsType"], + ]) { + const file = resolve(__dirname, "__model_params_completion.ts"); + const source = `import type { ${name} as Params } from "${module}";\nconst params: Params = {\n\n};`; + const service = ts.createLanguageService({ + getScriptFileNames: () => [file], + getScriptVersion: () => "0", + getScriptSnapshot: (path) => { + const text = path === file ? source : ts.sys.readFile(path); + return text === undefined + ? undefined + : ts.ScriptSnapshot.fromString(text); + }, + getCurrentDirectory: () => __dirname, + getCompilationSettings: () => ({ + strict: true, + target: ts.ScriptTarget.ES2022, + }), + getDefaultLibFileName: ts.getDefaultLibFilePath, + fileExists: ts.sys.fileExists, + readFile: ts.sys.readFile, + readDirectory: ts.sys.readDirectory, + }); + try { + suggestions.push( + service + .getCompletionsAtPosition(file, source.lastIndexOf("\n};"), {}) + ?.entries.filter( + (entry) => + entry.kind === ts.ScriptElementKind.memberVariableElement, + ) + .map((entry) => entry.name) + .sort(), + ); + } finally { + service.dispose(); + } + } + expect(suggestions[0]).toEqual(suggestions[1]); + expect(suggestions[0]).toEqual( + expect.arrayContaining([ + "temperature", + "top_p", + "topP", + "topK", + "max_tokens", + "maxOutputTokens", + "response_format", + ]), + ); +}); + +// Backend definitions are compatibility fixtures, never production dependencies. +test("SDK-owned contracts remain compatible with the backend", () => { + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf< + Exclude + >().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().branded.toEqualTypeOf(); + expectTypeOf< + Exclude + >().toEqualTypeOf(); + expectTypeOf().branded.toEqualTypeOf(); + // Structural equality accommodates explicit fields replacing mapped intersections. + expectTypeOf().branded.toEqualTypeOf(); + expectTypeOf().toExtend(); + expectTypeOf().toExtend(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().branded.toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().branded.toEqualTypeOf(); + expectTypeOf().branded.toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); +}); + +test.each([ + "PromptData", + "ObjectReference", + "Project", + "AttachmentReference", + "BraintrustAttachmentReference", + "BraintrustModelParams", + "ChatCompletionTool", + "ExternalAttachmentReference", + "ResponseFormatJsonSchema", + "AttachmentStatus", + "GitMetadataSettings", + "DatasetSnapshot", + "Prompt", + "FunctionId", + "CallEvent", + "SSEConsoleEventData", + "SSEProgressEventData", +] as const)("%s preserves backend validation constraints", (name) => { + expect(zodToJsonSchema(schemas[name])).toEqual( + zodToJsonSchema(backendSchemas[name]), + ); +}); + +const id = "12345678-1234-4234-8234-123456789012"; +const fixtures: Partial> = { + AttachmentReference: [ + { + type: "braintrust_attachment", + filename: "file", + content_type: "text/plain", + key: "key", + }, + { + type: "external_attachment", + filename: "file", + content_type: "text/plain", + url: "https://example.com/file", + }, + { + type: "braintrust_attachment", + filename: "", + content_type: "text/plain", + key: "key", + }, + ], + AttachmentStatus: [{ upload_status: "done" }, { upload_status: "invalid" }], + FunctionId: [ + { function_id: id }, + { project_name: "project", slug: "prompt" }, + { global_function: "scorer" }, + { prompt_session_id: id, prompt_session_function_id: id }, + { + inline_prompt: { prompt: { type: "completion", content: "Hello" } }, + function_type: "llm", + }, + ], + PromptData: [ + { + prompt: { type: "chat", messages: [{ role: "user" }, { role: "tool" }] }, + options: { model: "model", params: { temperature: 0, custom: true } }, + extra: "strip", + }, + { + prompt: { type: "completion", content: "Hello" }, + options: null, + parser: null, + }, + { prompt: { type: "chat", messages: [{ role: "invalid" }] } }, + ], + Prompt: [ + { + id, + project_id: id, + org_id: id, + _xact_id: "version", + log_id: "p", + name: "prompt", + slug: "prompt", + extra: "strip", + }, + ], + DatasetSnapshot: [ + { + id, + dataset_id: id, + name: "snapshot", + description: null, + xact_id: "version", + created: null, + }, + ], + ObjectReference: [ + { object_type: "dataset", object_id: id, id }, + { object_type: "invalid", object_id: id, id }, + ], + Project: [ + { + id, + org_id: id, + name: "project", + settings: { blind_reviews: true }, + extra: "strip", + }, + ], + CallEvent: [ + { event: "start" }, + { event: "done" }, + { event: "text_delta", data: "hello" }, + { event: "invalid", data: "hello" }, + ], + GitMetadataSettings: [ + { collect: "all" }, + { collect: "some", fields: ["commit"] }, + { collect: "some", fields: ["invalid"] }, + ], +}; + +test.each(Object.keys(schemas) as (keyof typeof schemas)[])( + "%s preserves parsing, defaults, and errors", + (name) => { + for (const input of [ + undefined, + null, + {}, + "invalid", + ...(fixtures[name] ?? []), + ]) { + const actual = schemas[name].safeParse(input); + const expected = backendSchemas[name].safeParse(input); + expect(actual.success, JSON.stringify(input)).toBe(expected.success); + if (actual.success && expected.success) { + expect(actual.data).toEqual(expected.data); + } else if (!actual.success && !expected.success) { + // Nested union errors have instance-specific methods; compare their + // serialized diagnostic payload, including codes, paths, and messages. + expect(JSON.parse(JSON.stringify(actual.error.issues))).toEqual( + JSON.parse(JSON.stringify(expected.error.issues)), + ); + } + } + }, +); diff --git a/js/src/sdk-schemas.ts b/js/src/sdk-schemas.ts new file mode 100644 index 000000000..c7d67e5a9 --- /dev/null +++ b/js/src/sdk-schemas.ts @@ -0,0 +1,717 @@ +/** + * Runtime validators for the SDK-owned subset of the Braintrust API. + * Compatibility with backend validators is checked in sdk-contracts.test.ts. + */ +import { z } from "zod/v3"; + +export const ResponseFormatJsonSchema = z.object({ + name: z.string(), + description: z.string().optional(), + schema: z + .union([z.object({}).partial().passthrough(), z.string()]) + .optional(), + strict: z.union([z.boolean(), z.null()]).optional(), +}); + +const ResponseFormatNullish = z.union([ + z.object({ type: z.literal("json_object") }), + z.object({ + type: z.literal("json_schema"), + json_schema: ResponseFormatJsonSchema, + }), + z.object({ type: z.literal("text") }), + z.null(), +]); + +export const BraintrustAttachmentReference = z.object({ + type: z.literal("braintrust_attachment"), + filename: z.string().min(1), + content_type: z.string().min(1), + key: z.string().min(1), +}); + +export const ExternalAttachmentReference = z.object({ + type: z.literal("external_attachment"), + filename: z.string().min(1), + content_type: z.string().min(1), + url: z.string().min(1), +}); + +export const AttachmentReference = z.discriminatedUnion("type", [ + BraintrustAttachmentReference, + ExternalAttachmentReference, +]); + +const UploadStatus = z.enum(["uploading", "done", "error"]); + +export const AttachmentStatus = z.object({ + upload_status: UploadStatus, + error_message: z.string().optional(), +}); + +const FunctionTypeEnum = z.enum([ + "llm", + "scorer", + "task", + "tool", + "custom_view", + "preprocessor", + "facet", + "classifier", + "tag", + "parameters", + "sandbox", +]); + +const SavedFunctionId = z.union([ + z.object({ + type: z.literal("function"), + id: z.string(), + version: z.string().optional(), + }), + z.object({ + type: z.literal("global"), + name: z.string(), + function_type: FunctionTypeEnum.optional().default("scorer"), + }), +]); + +export const BraintrustModelParams = z + .object({ + use_cache: z.boolean(), + reasoning_enabled: z.boolean(), + reasoning_budget: z.number(), + }) + .partial(); + +export const CallEvent = z.union([ + z.object({ + id: z.string().optional(), + data: z.string(), + event: z.literal("text_delta"), + }), + z.object({ + id: z.string().optional(), + data: z.string(), + event: z.literal("reasoning_delta"), + }), + z.object({ + id: z.string().optional(), + data: z.string(), + event: z.literal("json_delta"), + }), + z.object({ + id: z.string().optional(), + data: z.string(), + event: z.literal("progress"), + }), + z.object({ + id: z.string().optional(), + data: z.string(), + event: z.literal("error"), + }), + z.object({ + id: z.string().optional(), + data: z.string(), + event: z.literal("console"), + }), + z.object({ + id: z.string().optional(), + event: z.literal("start"), + data: z.literal(""), + }), + z.object({ + id: z.string().optional(), + event: z.literal("done"), + data: z.literal(""), + }), +]); + +const ChatCompletionContentPartTextWithTitle = z.object({ + text: z.string().default(""), + type: z.literal("text"), + cache_control: z + .object({ + type: z.literal("ephemeral"), + ttl: z.enum(["5m", "1h"]).optional(), + }) + .optional(), +}); + +const ChatCompletionContentPartImageWithTitle = z.object({ + image_url: z.object({ + url: z.string(), + detail: z + .union([z.literal("auto"), z.literal("low"), z.literal("high")]) + .optional(), + }), + type: z.literal("image_url"), + cache_control: z + .object({ + type: z.literal("ephemeral"), + ttl: z.enum(["5m", "1h"]).optional(), + }) + .optional(), +}); + +const ChatCompletionContentPartFileFile = z + .object({ file_data: z.string(), filename: z.string(), file_id: z.string() }) + .partial(); + +const ChatCompletionContentPartFileWithTitle = z.object({ + file: ChatCompletionContentPartFileFile, + type: z.literal("file"), + cache_control: z + .object({ + type: z.literal("ephemeral"), + ttl: z.enum(["5m", "1h"]).optional(), + }) + .optional(), +}); + +const ChatCompletionContentPart = z.union([ + ChatCompletionContentPartTextWithTitle, + ChatCompletionContentPartImageWithTitle, + ChatCompletionContentPartFileWithTitle, +]); + +const ChatCompletionContentPartText = z.object({ + text: z.string().default(""), + type: z.literal("text"), + cache_control: z + .object({ + type: z.literal("ephemeral"), + ttl: z.enum(["5m", "1h"]).optional(), + }) + .optional(), +}); + +const ChatCompletionMessageToolCall = z.object({ + id: z.string(), + function: z.object({ arguments: z.string(), name: z.string() }), + type: z.literal("function"), +}); + +const ChatCompletionMessageReasoning = z + .object({ id: z.string(), content: z.string() }) + .partial(); + +const ChatCompletionMessageParam = z.union([ + z.object({ + content: z.union([z.string(), z.array(ChatCompletionContentPartText)]), + role: z.literal("system"), + name: z.string().optional(), + }), + z.object({ + content: z.union([z.string(), z.array(ChatCompletionContentPart)]), + role: z.literal("user"), + name: z.string().optional(), + }), + z.object({ + role: z.literal("assistant"), + content: z + .union([z.string(), z.array(ChatCompletionContentPartText), z.null()]) + .optional(), + function_call: z + .object({ arguments: z.string(), name: z.string() }) + .optional(), + name: z.string().optional(), + tool_calls: z.array(ChatCompletionMessageToolCall).optional(), + reasoning: z.array(ChatCompletionMessageReasoning).optional(), + reasoning_signature: z.string().optional(), + }), + z.object({ + content: z.union([z.string(), z.array(ChatCompletionContentPartText)]), + role: z.literal("tool"), + tool_call_id: z.string().default(""), + }), + z.object({ + content: z.union([z.string(), z.null()]), + name: z.string(), + role: z.literal("function"), + }), + z.object({ + content: z.union([z.string(), z.array(ChatCompletionContentPartText)]), + role: z.literal("developer"), + name: z.string().optional(), + }), + z.object({ + role: z.literal("model"), + content: z.union([z.string(), z.null()]).optional(), + }), +]); + +export const ChatCompletionTool = z.object({ + function: z.object({ + name: z.string(), + description: z.string().optional(), + parameters: z.object({}).partial().passthrough().optional(), + }), + type: z.literal("function"), +}); + +const ObjectReferenceNullish = z.union([ + z.object({ + object_type: z.enum([ + "project_logs", + "experiment", + "dataset", + "prompt", + "function", + "prompt_session", + ]), + object_id: z.string().uuid(), + id: z.string(), + _xact_id: z.union([z.string(), z.null()]).optional(), + created: z.union([z.string(), z.null()]).optional(), + }), + z.null(), +]); + +export const DatasetSnapshot = z.object({ + id: z.string().uuid(), + dataset_id: z.string().uuid(), + name: z.string(), + description: z.union([z.string(), z.null()]), + xact_id: z.string(), + created: z.union([z.string(), z.null()]), +}); + +const PromptBlockDataNullish = z.union([ + z.object({ + type: z.literal("chat"), + messages: z.array(ChatCompletionMessageParam), + tools: z.string().optional(), + }), + z.object({ type: z.literal("completion"), content: z.string() }), + z.null(), +]); + +const ModelParams = z.union([ + z + .object({ + use_cache: z.boolean(), + reasoning_enabled: z.boolean(), + reasoning_budget: z.number(), + temperature: z.number(), + top_p: z.number(), + max_tokens: z.number(), + max_completion_tokens: z.number(), + frequency_penalty: z.number(), + presence_penalty: z.number(), + response_format: ResponseFormatNullish, + tool_choice: z.union([ + z.literal("auto"), + z.literal("none"), + z.literal("required"), + z.object({ + type: z.literal("function"), + function: z.object({ name: z.string() }), + }), + ]), + function_call: z.union([ + z.literal("auto"), + z.literal("none"), + z.object({ name: z.string() }), + ]), + n: z.number(), + stop: z.array(z.string()), + reasoning_effort: z.enum(["none", "minimal", "low", "medium", "high"]), + verbosity: z.enum(["low", "medium", "high"]), + }) + .partial() + .passthrough(), + z + .object({ + use_cache: z.boolean().optional(), + reasoning_enabled: z.boolean().optional(), + reasoning_budget: z.number().optional(), + max_tokens: z.number(), + temperature: z.number(), + top_p: z.number().optional(), + top_k: z.number().optional(), + stop_sequences: z.array(z.string()).optional(), + max_tokens_to_sample: z.number().optional(), + }) + .passthrough(), + z + .object({ + use_cache: z.boolean(), + reasoning_enabled: z.boolean(), + reasoning_budget: z.number(), + temperature: z.number(), + maxOutputTokens: z.number(), + topP: z.number(), + topK: z.number(), + }) + .partial() + .passthrough(), + z + .object({ + use_cache: z.boolean(), + reasoning_enabled: z.boolean(), + reasoning_budget: z.number(), + temperature: z.number(), + topK: z.number(), + }) + .partial() + .passthrough(), + z + .object({ + use_cache: z.boolean(), + reasoning_enabled: z.boolean(), + reasoning_budget: z.number(), + }) + .partial() + .passthrough(), +]); + +const PromptOptionsNullish = z.union([ + z + .object({ + model: z.string(), + params: ModelParams, + position: z.string(), + endpoint_name: z.union([z.string(), z.null()]), + }) + .partial(), + z.null(), +]); + +const PromptParserNullish = z.union([ + z.object({ + type: z.literal("llm_classifier"), + use_cot: z.boolean(), + choice_scores: z.record(z.number().gte(0).lte(1)).optional(), + choice: z.array(z.string()).optional(), + allow_no_match: z.boolean().optional(), + allow_skip: z.boolean().optional(), + }), + z.null(), +]); + +const PreprocessorId = z.union([ + z.object({ + type: z.literal("function"), + id: z.string(), + version: z.string().optional(), + }), + z.object({ + type: z.literal("global"), + name: z.string(), + function_type: z.literal("preprocessor").optional().default("preprocessor"), + }), + z.object({ type: z.literal("inline"), code: z.string().min(1) }), + z.null(), +]); + +const PromptDataNullish = z.union([ + z + .object({ + prompt: PromptBlockDataNullish, + options: PromptOptionsNullish, + parser: PromptParserNullish, + preprocessor: PreprocessorId, + tool_functions: z.union([z.array(SavedFunctionId), z.null()]), + template_format: z.union([ + z.enum(["mustache", "nunjucks", "none"]), + z.null(), + ]), + mcp: z.union([ + z.record( + z.union([ + z.object({ + type: z.literal("id"), + id: z.string().uuid(), + is_disabled: z.boolean().optional(), + enabled_tools: z + .union([z.array(z.string()), z.null()]) + .optional(), + }), + z.object({ + type: z.literal("url"), + url: z.string(), + is_disabled: z.boolean().optional(), + enabled_tools: z + .union([z.array(z.string()), z.null()]) + .optional(), + }), + ]), + ), + z.null(), + ]), + origin: z.union([ + z + .object({ + prompt_id: z.string(), + project_id: z.string(), + prompt_version: z.string(), + }) + .partial(), + z.null(), + ]), + }) + .partial(), + z.null(), +]); + +const FunctionTypeEnumNullish = z.union([ + z.enum([ + "llm", + "scorer", + "task", + "tool", + "custom_view", + "preprocessor", + "facet", + "classifier", + "tag", + "parameters", + "sandbox", + ]), + z.null(), +]); + +const FunctionFormat = z.enum(["llm", "code", "global", "graph", "topic_map"]); + +export const PromptData = z + .object({ + prompt: PromptBlockDataNullish, + options: PromptOptionsNullish, + parser: PromptParserNullish, + preprocessor: PreprocessorId, + tool_functions: z.union([z.array(SavedFunctionId), z.null()]), + template_format: z.union([ + z.enum(["mustache", "nunjucks", "none"]), + z.null(), + ]), + mcp: z.union([ + z.record( + z.union([ + z.object({ + type: z.literal("id"), + id: z.string().uuid(), + is_disabled: z.boolean().optional(), + enabled_tools: z.union([z.array(z.string()), z.null()]).optional(), + }), + z.object({ + type: z.literal("url"), + url: z.string(), + is_disabled: z.boolean().optional(), + enabled_tools: z.union([z.array(z.string()), z.null()]).optional(), + }), + ]), + ), + z.null(), + ]), + origin: z.union([ + z + .object({ + prompt_id: z.string(), + project_id: z.string(), + prompt_version: z.string(), + }) + .partial(), + z.null(), + ]), + }) + .partial(); + +export const FunctionId = z.union([ + z.object({ function_id: z.string(), version: z.string().optional() }), + z.object({ + project_name: z.string(), + slug: z.string(), + version: z.string().optional(), + }), + z.object({ + global_function: z.string(), + function_type: FunctionTypeEnum.optional().default("scorer"), + }), + z.object({ + prompt_session_id: z.string(), + prompt_session_function_id: z.string(), + version: z.string().optional(), + }), + z.object({ + inline_context: z.object({ + runtime: z.enum(["node", "python", "browser", "quickjs"]), + version: z.string(), + }), + code: z.string(), + function_type: FunctionTypeEnum.and(z.unknown()).optional(), + name: z.union([z.string(), z.null()]).optional(), + }), + z.object({ + inline_prompt: PromptData.optional(), + inline_function: z.object({}).partial().passthrough(), + function_type: FunctionTypeEnum.optional().default("scorer"), + name: z.union([z.string(), z.null()]).optional(), + }), + z.object({ + inline_prompt: PromptData, + function_type: FunctionTypeEnum.optional().default("scorer"), + name: z.union([z.string(), z.null()]).optional(), + }), +]); + +const FunctionObjectType = z.enum([ + "prompt", + "tool", + "scorer", + "task", + "workflow", + "custom_view", + "preprocessor", + "facet", + "classifier", + "parameters", + "sandbox", +]); + +const FunctionOutputType = z.enum([ + "completion", + "score", + "facet", + "classification", + "any", +]); + +export const GitMetadataSettings = z.object({ + collect: z.enum(["all", "none", "some"]), + fields: z + .array( + z.enum([ + "commit", + "branch", + "tag", + "dirty", + "author_name", + "author_email", + "commit_message", + "commit_time", + "git_diff", + ]), + ) + .optional(), +}); + +const NullableSavedFunctionId = z.union([ + z.object({ + type: z.literal("function"), + id: z.string(), + version: z.string().optional(), + }), + z.object({ + type: z.literal("global"), + name: z.string(), + function_type: FunctionTypeEnum.optional().default("scorer"), + }), + z.null(), +]); + +export const ObjectReference = z.object({ + object_type: z.enum([ + "project_logs", + "experiment", + "dataset", + "prompt", + "function", + "prompt_session", + ]), + object_id: z.string().uuid(), + id: z.string(), + _xact_id: z.union([z.string(), z.null()]).optional(), + created: z.union([z.string(), z.null()]).optional(), +}); + +const ProjectSettings = z.union([ + z + .object({ + comparison_key: z.union([z.string(), z.null()]), + baseline_experiment_id: z.union([z.string(), z.null()]), + spanFieldOrder: z.union([ + z.array( + z.object({ + object_type: z.string(), + column_id: z.string(), + position: z.string(), + layout: z + .union([z.literal("full"), z.literal("two_column"), z.null()]) + .optional(), + }), + ), + z.null(), + ]), + remote_eval_sources: z.union([ + z.array( + z.object({ + url: z.string(), + name: z.union([z.string(), z.null()]).optional(), + description: z.union([z.string(), z.null()]).optional(), + }), + ), + z.null(), + ]), + disable_realtime_queries: z.union([z.boolean(), z.null()]), + monitor_charts_use_metrics_start: z.union([z.boolean(), z.null()]), + blind_reviews: z.union([z.boolean(), z.null()]), + default_preprocessor: NullableSavedFunctionId, + }) + .partial(), + z.null(), +]); + +export const Project = z.object({ + id: z.string().uuid(), + org_id: z.string().uuid(), + name: z.string(), + description: z.union([z.string(), z.null()]).optional(), + created: z.union([z.string(), z.null()]).optional(), + deleted_at: z.union([z.string(), z.null()]).optional(), + user_id: z.union([z.string(), z.null()]).optional(), + settings: ProjectSettings.optional(), +}); + +export const Prompt = z.object({ + id: z.string().uuid(), + _xact_id: z.string(), + project_id: z.string().uuid(), + log_id: z.literal("p"), + org_id: z.string().uuid(), + name: z.string(), + slug: z.string(), + description: z.union([z.string(), z.null()]).optional(), + created: z.union([z.string(), z.null()]).optional(), + prompt_data: PromptDataNullish.optional(), + tags: z.union([z.array(z.string()), z.null()]).optional(), + metadata: z + .union([z.object({}).partial().passthrough(), z.null()]) + .optional(), + function_type: FunctionTypeEnumNullish.optional(), +}); + +export const SSEConsoleEventData = z.object({ + stream: z.enum(["stderr", "stdout"]), + message: z.string(), +}); + +export const SSEProgressEventData = z.object({ + id: z.string(), + object_type: FunctionObjectType, + origin: ObjectReferenceNullish.and(z.unknown()).optional(), + format: FunctionFormat, + output_type: FunctionOutputType, + name: z.string(), + event: z.enum([ + "reasoning_delta", + "text_delta", + "json_delta", + "error", + "console", + "start", + "done", + "progress", + ]), + data: z.string(), +}); diff --git a/js/src/sdk-types.ts b/js/src/sdk-types.ts new file mode 100644 index 000000000..1fdc3f6a9 --- /dev/null +++ b/js/src/sdk-types.ts @@ -0,0 +1,573 @@ +/* eslint-disable @typescript-eslint/no-empty-object-type -- Preserve the existing wire contracts, including permissive JSON fields. */ +/* eslint-disable @typescript-eslint/no-explicit-any -- Provider parameter maps intentionally accept arbitrary values. */ +/** + * SDK-owned wire contracts. Keep backend-generated definitions in compatibility + * tests only; changes here should describe the fields the SDK actually uses. + */ +export interface ResponseFormatJsonSchema { + name: string; + description?: string | undefined; + schema?: ({} | string) | undefined; + strict?: (boolean | null) | undefined; +} +export type ResponseFormatNullish = + | { + type: "json_object"; + } + | { + type: "json_schema"; + json_schema: ResponseFormatJsonSchema; + } + | { + type: "text"; + } + | null; +export type AsyncScoringState = + | { + status: "enabled"; + token: string; + function_ids: Array; + skip_logging?: (boolean | null) | undefined; + triggered_functions?: ({} | null) | undefined; + last_triggered_xact_id?: (string | number | null) | undefined; + } + | { + status: "disabled"; + } + | null; +export type AsyncScoringControl = + | { + kind: "score_update"; + token?: string | undefined; + } + | { + kind: "state_override"; + state: AsyncScoringState; + } + | { + kind: "state_force_reselect"; + } + | { + kind: "state_enabled_force_rescore"; + } + | { + kind: "trigger_functions"; + triggered_functions: Array<{ + function_id?: unknown | undefined; + scope: + | { + type: "span"; + } + | { + type: "trace"; + }; + idempotency_key?: string | undefined; + }>; + } + | { + kind: "complete_triggered_functions"; + function_ids: Array; + triggered_xact_id: string; + } + | { + kind: "mark_attempt_failed"; + function_ids: Array; + }; +export interface BraintrustAttachmentReference { + type: "braintrust_attachment"; + filename: string; + content_type: string; + key: string; +} +export interface ExternalAttachmentReference { + type: "external_attachment"; + filename: string; + content_type: string; + url: string; +} +export type AttachmentReference = + | BraintrustAttachmentReference + | ExternalAttachmentReference; +export type UploadStatus = "uploading" | "done" | "error"; +export interface AttachmentStatus { + upload_status: UploadStatus; + error_message?: string | undefined; +} +export type FunctionTypeEnum = + | "llm" + | "scorer" + | "task" + | "tool" + | "custom_view" + | "preprocessor" + | "facet" + | "classifier" + | "tag" + | "parameters" + | "sandbox"; +export type SavedFunctionId = + | { + type: "function"; + id: string; + version?: string | undefined; + } + | { + type: "global"; + name: string; + function_type: FunctionTypeEnum; + }; +export type CallEvent = + | { + id?: string | undefined; + data: string; + event: "text_delta"; + } + | { + id?: string | undefined; + data: string; + event: "reasoning_delta"; + } + | { + id?: string | undefined; + data: string; + event: "json_delta"; + } + | { + id?: string | undefined; + data: string; + event: "progress"; + } + | { + id?: string | undefined; + data: string; + event: "error"; + } + | { + id?: string | undefined; + data: string; + event: "console"; + } + | { + id?: string | undefined; + event: "start"; + data: ""; + } + | { + id?: string | undefined; + event: "done"; + data: ""; + }; +interface CacheControl { + type: "ephemeral"; + ttl?: "5m" | "1h" | undefined; +} + +export interface ChatCompletionContentPartImageWithTitle { + image_url: { + url: string; + detail?: ("auto" | "low" | "high") | undefined; + }; + type: "image_url"; + cache_control?: CacheControl | undefined; +} +export interface ChatCompletionContentPartFileFile { + file_data?: string; + filename?: string; + file_id?: string; +} +export interface ChatCompletionContentPartFileWithTitle { + file: ChatCompletionContentPartFileFile; + type: "file"; + cache_control?: CacheControl | undefined; +} +export type ChatCompletionContentPart = + | ChatCompletionContentPartText + | ChatCompletionContentPartImageWithTitle + | ChatCompletionContentPartFileWithTitle; +export interface ChatCompletionContentPartText { + text: string; + type: "text"; + cache_control?: CacheControl | undefined; +} +export interface ChatCompletionMessageToolCall { + id: string; + function: { + arguments: string; + name: string; + }; + type: "function"; +} +export interface ChatCompletionMessageReasoning { + id?: string; + content?: string; +} +export type ChatCompletionMessageParam = + | ChatCompletionOpenAIMessageParam + | { + role: "model"; + content?: string | null | undefined; + }; +export type ChatCompletionOpenAIMessageParam = + | { + content: string | Array; + role: "system"; + name?: string | undefined; + } + | { + content: string | Array; + role: "user"; + name?: string | undefined; + } + | { + role: "assistant"; + content?: + | (string | Array | null) + | undefined; + function_call?: + | { + arguments: string; + name: string; + } + | undefined; + name?: string | undefined; + tool_calls?: Array | undefined; + reasoning?: Array | undefined; + reasoning_signature?: string | undefined; + } + | { + content: string | Array; + role: "tool"; + tool_call_id: string; + } + | { + content: string | null; + name: string; + role: "function"; + } + | { + content: string | Array; + role: "developer"; + name?: string | undefined; + }; +export interface ChatCompletionTool { + function: { + name: string; + description?: string | undefined; + parameters?: {} | undefined; + }; + type: "function"; +} +export interface DatasetSnapshot { + id: string; + dataset_id: string; + name: string; + description: string | null; + xact_id: string; + created: string | null; +} +export type RepoInfo = { + commit?: string | null; + branch?: string | null; + tag?: string | null; + dirty?: boolean | null; + author_name?: string | null; + author_email?: string | null; + commit_message?: string | null; + commit_time?: string | null; + git_diff?: string | null; +} | null; +export type ExtendedSavedFunctionId = + | SavedFunctionId + | { + type: "slug"; + project_id: string; + slug: string; + }; +// Keep provider fields visible to editor completion while accepting custom options. +export type ModelParams = + | { + use_cache?: boolean; + reasoning_enabled?: boolean; + reasoning_budget?: number; + temperature?: number; + top_p?: number; + max_tokens?: number; + max_completion_tokens?: number; + frequency_penalty?: number; + presence_penalty?: number; + response_format?: ResponseFormatNullish; + tool_choice?: + | "auto" + | "none" + | "required" + | { + type: "function"; + function: { + name: string; + }; + }; + function_call?: + | "auto" + | "none" + | { + name: string; + }; + n?: number; + stop?: Array; + reasoning_effort?: "none" | "minimal" | "low" | "medium" | "high"; + verbosity?: "low" | "medium" | "high"; + [key: string]: any; + } + | { + use_cache?: boolean | undefined; + reasoning_enabled?: boolean | undefined; + reasoning_budget?: number | undefined; + max_tokens: number; + temperature: number; + top_p?: number | undefined; + top_k?: number | undefined; + stop_sequences?: Array | undefined; + max_tokens_to_sample?: number | undefined; + [key: string]: any; + } + | { + use_cache?: boolean; + reasoning_enabled?: boolean; + reasoning_budget?: number; + temperature?: number; + maxOutputTokens?: number; + topP?: number; + topK?: number; + [key: string]: any; + } + | { + use_cache?: boolean; + reasoning_enabled?: boolean; + reasoning_budget?: number; + temperature?: number; + topK?: number; + [key: string]: any; + } + | { + use_cache?: boolean; + reasoning_enabled?: boolean; + reasoning_budget?: number; + [key: string]: any; + }; +export type PromptOptionsNullish = { + model?: string; + params?: ModelParams; + position?: string; + endpoint_name?: string | null; +} | null; +export type PromptParserNullish = { + type: "llm_classifier"; + use_cot: boolean; + choice_scores?: {} | undefined; + choice?: Array | undefined; + allow_no_match?: boolean | undefined; + allow_skip?: boolean | undefined; +} | null; +export type PreprocessorId = + | { + type: "function"; + id: string; + version?: string | undefined; + } + | { + type: "global"; + name: string; + function_type: "preprocessor"; + } + | { + type: "inline"; + code: string; + } + | null; +export type PromptBlockData = + | { + type: "chat"; + messages: Array; + tools?: string | undefined; + } + | { + type: "completion"; + content: string; + }; +export type FunctionFormat = "llm" | "code" | "global" | "graph" | "topic_map"; +export interface PromptData { + prompt?: PromptBlockData | null; + options?: PromptOptionsNullish; + parser?: PromptParserNullish; + preprocessor?: PreprocessorId; + tool_functions?: Array | null; + template_format?: ("mustache" | "nunjucks" | "none") | null; + mcp?: {} | null; + origin?: { + prompt_id?: string; + project_id?: string; + prompt_version?: string; + } | null; +} +export type FunctionId = + | { + function_id: string; + version?: string | undefined; + } + | { + project_name: string; + slug: string; + version?: string | undefined; + } + | { + global_function: string; + function_type: FunctionTypeEnum; + } + | { + prompt_session_id: string; + prompt_session_function_id: string; + version?: string | undefined; + } + | { + inline_context: { + runtime: "node" | "python" | "browser" | "quickjs"; + version: string; + }; + code: string; + function_type?: FunctionTypeEnum | undefined; + name?: (string | null) | undefined; + } + | { + inline_prompt?: PromptData | undefined; + inline_function: {}; + function_type: FunctionTypeEnum; + name?: (string | null) | undefined; + } + | { + inline_prompt: PromptData; + function_type: FunctionTypeEnum; + name?: (string | null) | undefined; + }; +export type FunctionObjectType = + | "prompt" + | "tool" + | "scorer" + | "task" + | "workflow" + | "custom_view" + | "preprocessor" + | "facet" + | "classifier" + | "parameters" + | "sandbox"; +export type FunctionOutputType = + | "completion" + | "score" + | "facet" + | "classification" + | "any"; +export interface GitMetadataSettings { + collect: "all" | "none" | "some"; + fields?: + | Array< + | "commit" + | "branch" + | "tag" + | "dirty" + | "author_name" + | "author_email" + | "commit_message" + | "commit_time" + | "git_diff" + > + | undefined; +} +export type IfExists = "error" | "ignore" | "replace"; +export type InvokeParent = + | { + object_type: "project_logs" | "experiment" | "playground_logs"; + object_id: string; + row_ids?: + | ({ + id: string; + span_id: string; + root_span_id: string; + } | null) + | undefined; + propagated_event?: ({} | null) | undefined; + } + | string; +export type StreamingMode = ("auto" | "parallel" | "json" | "text") | null; +export type InvokeFunction = FunctionId & { + input?: unknown; + expected?: unknown; + metadata?: {} | null; + tags?: Array | null; + messages?: Array; + parent?: InvokeParent; + stream?: boolean | null; + mode?: StreamingMode; + strict?: boolean | null; + mcp_auth?: {}; + overrides?: {} | null; + endpoint_name?: string | null; +}; +export interface ObjectReference { + object_type: + | "project_logs" + | "experiment" + | "dataset" + | "prompt" + | "function" + | "prompt_session"; + object_id: string; + id: string; + _xact_id?: (string | null) | undefined; + created?: (string | null) | undefined; +} +export interface PromptSessionEvent { + id: string; + _xact_id: string; + created: string; + _pagination_key?: (string | null) | undefined; + project_id: string; + prompt_session_id: string; + prompt_session_data?: unknown | undefined; + prompt_data?: unknown | undefined; + function_data?: unknown | undefined; + function_type?: FunctionTypeEnum | null | undefined; + object_data?: unknown | undefined; + completion?: unknown | undefined; + tags?: (Array | null) | undefined; +} +export interface SSEConsoleEventData { + stream: "stderr" | "stdout"; + message: string; +} +export interface SSEProgressEventData { + id: string; + object_type: FunctionObjectType; + origin?: (ObjectReference | null) | undefined; + format: FunctionFormat; + output_type: FunctionOutputType; + name: string; + event: + | "reasoning_delta" + | "text_delta" + | "json_delta" + | "error" + | "console" + | "start" + | "done" + | "progress"; + data: string; +} +export interface ToolFunctionDefinition { + type: "function"; + function: { + name: string; + description?: string | undefined; + parameters?: {} | undefined; + strict?: (boolean | null) | undefined; + }; +} diff --git a/js/tests/api-compatibility/api-compatibility.test.ts b/js/tests/api-compatibility/api-compatibility.test.ts index 6241d23d6..0b32fb552 100644 --- a/js/tests/api-compatibility/api-compatibility.test.ts +++ b/js/tests/api-compatibility/api-compatibility.test.ts @@ -2983,6 +2983,22 @@ describe("API Compatibility", () => { } }); + test("keeps prompt implementation types out of declaration chunks", () => { + const dist = path.join(__dirname, "..", "..", "dist"); + const declarations = fs + .readdirSync(dist, { recursive: true }) + .filter((file) => /\.d\.(?:ts|mts)$/.test(String(file))) + .map((file) => fs.readFileSync(path.join(dist, String(file)), "utf8")) + .join("\n"); + + expect(declarations).toContain("interface CompiledPromptParams"); + expect(declarations).not.toContain("_internalSerializeForCache"); + expect(declarations).not.toMatch(/generated_(?:plain_)?types/); + expect(declarations).not.toMatch( + /\b(?:FunctionDataType|AnyModelParamsType|PromptType)\b/, + ); + }); + test("should not regress public API surface for all entrypoints", async () => { if (!publishedVersion) { console.log("Skipping test: No published version available"); diff --git a/js/util/git_fields.ts b/js/util/git_fields.ts index 482a43526..a33b07ffe 100644 --- a/js/util/git_fields.ts +++ b/js/util/git_fields.ts @@ -1,4 +1,4 @@ -import type { GitMetadataSettingsType as GitMetadataSettings } from "./generated_plain_types"; +import type { GitMetadataSettings } from "../src/sdk-types"; export function mergeGitMetadataSettings( s1: GitMetadataSettings, diff --git a/js/util/object.ts b/js/util/object.ts index 074aa6229..e34e014da 100644 --- a/js/util/object.ts +++ b/js/util/object.ts @@ -1,7 +1,4 @@ -import type { - AsyncScoringControlType as AsyncScoringControl, - ObjectReferenceType, -} from "./generated_plain_types"; +import type { AsyncScoringControl, ObjectReference } from "../src/sdk-types"; import { Source, ASYNC_SCORING_CONTROL_FIELD, @@ -23,7 +20,7 @@ export type OtherExperimentLogFields = { classifications?: Record; metadata: Record; metrics: Record; - origin: ObjectReferenceType; + origin: ObjectReference; span_attributes: Record; [ASYNC_SCORING_CONTROL_FIELD]: AsyncScoringControl; [MERGE_PATHS_FIELD]: string[][]; @@ -101,7 +98,7 @@ export type DatasetEvent = { tags?: string[]; metadata?: unknown; created?: string; - origin?: ObjectReferenceType; + origin?: ObjectReference; id: string; dataset_id: string; } & ({ expected?: unknown } | { output?: unknown }); diff --git a/js/util/span_identifier_v4.ts b/js/util/span_identifier_v4.ts index 771d72aec..844730365 100644 --- a/js/util/span_identifier_v4.ts +++ b/js/util/span_identifier_v4.ts @@ -20,7 +20,7 @@ import { uint8ArrayToString, } from "./bytes"; import { z } from "zod/v3"; -import type { InvokeFunctionType as InvokeFunctionRequest } from "./generated_plain_types"; +import type { InvokeFunction as InvokeFunctionRequest } from "../src/sdk-types"; import { mergeDicts } from "./object_util"; const ENCODING_VERSION_NUMBER_V4 = 4; From bb8df849e708ba92574a0c6ffbbd4b3fb1b8d6da Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:54:04 +0000 Subject: [PATCH 2/5] Update PR #2517 --- .changeset/simplify-public-sdk-types.md | 5 + js/src/sdk-contracts.test.ts | 119 +++++++++++++----------- 2 files changed, 69 insertions(+), 55 deletions(-) create mode 100644 .changeset/simplify-public-sdk-types.md diff --git a/.changeset/simplify-public-sdk-types.md b/.changeset/simplify-public-sdk-types.md new file mode 100644 index 000000000..0cff8f061 --- /dev/null +++ b/.changeset/simplify-public-sdk-types.md @@ -0,0 +1,5 @@ +--- +"braintrust": major +--- + +ref!: Don't make generated types part of public api diff --git a/js/src/sdk-contracts.test.ts b/js/src/sdk-contracts.test.ts index 1d7b5db92..a3010c049 100644 --- a/js/src/sdk-contracts.test.ts +++ b/js/src/sdk-contracts.test.ts @@ -1,4 +1,4 @@ -import { resolve } from "node:path"; +import { posix, win32 } from "node:path"; import ts from "typescript"; import { expect, expectTypeOf, test } from "vitest"; import { zodToJsonSchema } from "zod-to-json-schema"; @@ -7,61 +7,70 @@ import * as backendSchemas from "./generated_types"; import type * as SDK from "./sdk-types"; import * as schemas from "./sdk-schemas"; -test("model parameters preserve provider field autocomplete", () => { - const suggestions = []; - for (const [module, name] of [ - ["./sdk-types", "ModelParams"], - ["./generated_plain_types", "ModelParamsType"], - ]) { - const file = resolve(__dirname, "__model_params_completion.ts"); - const source = `import type { ${name} as Params } from "${module}";\nconst params: Params = {\n\n};`; - const service = ts.createLanguageService({ - getScriptFileNames: () => [file], - getScriptVersion: () => "0", - getScriptSnapshot: (path) => { - const text = path === file ? source : ts.sys.readFile(path); - return text === undefined - ? undefined - : ts.ScriptSnapshot.fromString(text); - }, - getCurrentDirectory: () => __dirname, - getCompilationSettings: () => ({ - strict: true, - target: ts.ScriptTarget.ES2022, - }), - getDefaultLibFileName: ts.getDefaultLibFilePath, - fileExists: ts.sys.fileExists, - readFile: ts.sys.readFile, - readDirectory: ts.sys.readDirectory, - }); - try { - suggestions.push( - service - .getCompletionsAtPosition(file, source.lastIndexOf("\n};"), {}) - ?.entries.filter( - (entry) => - entry.kind === ts.ScriptElementKind.memberVariableElement, - ) - .map((entry) => entry.name) - .sort(), - ); - } finally { - service.dispose(); +test.each([ + ["POSIX", posix], + ["Windows", win32], +] as const)( + "model parameters preserve provider field autocomplete (%s paths)", + (_name, path) => { + const suggestions = []; + for (const [module, name] of [ + ["./sdk-types", "ModelParams"], + ["./generated_plain_types", "ModelParamsType"], + ]) { + // TypeScript normalizes script names before requesting their snapshots. + const file = path + .join(__dirname, "__model_params_completion.ts") + .replaceAll("\\", "/"); + const source = `import type { ${name} as Params } from "${module}";\nconst params: Params = {\n\n};`; + const service = ts.createLanguageService({ + getScriptFileNames: () => [file], + getScriptVersion: () => "0", + getScriptSnapshot: (path) => { + const text = path === file ? source : ts.sys.readFile(path); + return text === undefined + ? undefined + : ts.ScriptSnapshot.fromString(text); + }, + getCurrentDirectory: () => __dirname, + getCompilationSettings: () => ({ + strict: true, + target: ts.ScriptTarget.ES2022, + }), + getDefaultLibFileName: ts.getDefaultLibFilePath, + fileExists: ts.sys.fileExists, + readFile: ts.sys.readFile, + readDirectory: ts.sys.readDirectory, + }); + try { + suggestions.push( + service + .getCompletionsAtPosition(file, source.lastIndexOf("\n};"), {}) + ?.entries.filter( + (entry) => + entry.kind === ts.ScriptElementKind.memberVariableElement, + ) + .map((entry) => entry.name) + .sort(), + ); + } finally { + service.dispose(); + } } - } - expect(suggestions[0]).toEqual(suggestions[1]); - expect(suggestions[0]).toEqual( - expect.arrayContaining([ - "temperature", - "top_p", - "topP", - "topK", - "max_tokens", - "maxOutputTokens", - "response_format", - ]), - ); -}); + expect(suggestions[0]).toEqual(suggestions[1]); + expect(suggestions[0]).toEqual( + expect.arrayContaining([ + "temperature", + "top_p", + "topP", + "topK", + "max_tokens", + "maxOutputTokens", + "response_format", + ]), + ); + }, +); // Backend definitions are compatibility fixtures, never production dependencies. test("SDK-owned contracts remain compatible with the backend", () => { From 7abf19ecc3223850fdc88cba3966931d187643d3 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 24 Sep 2026 14:12:37 +0000 Subject: [PATCH 3/5] chill down on tests --- js/eslint.config.ts | 34 ++-- js/src/sdk-boundary.test.ts | 70 -------- js/src/sdk-contracts.test.ts | 328 ++++++++++++----------------------- js/src/sdk-schemas.ts | 2 +- 4 files changed, 140 insertions(+), 294 deletions(-) delete mode 100644 js/src/sdk-boundary.test.ts diff --git a/js/eslint.config.ts b/js/eslint.config.ts index c718d52c8..00dad5c0a 100644 --- a/js/eslint.config.ts +++ b/js/eslint.config.ts @@ -20,6 +20,17 @@ const entryFiles = tsupConfig.flatMap((config) => { return entries; }); +const generatedTypeImports = { + group: [ + "**/generated_types", + "**/generated_plain_types", + "**/generated_types.ts", + "**/generated_plain_types.ts", + ], + message: + "Generated backend definitions are compatibility-test fixtures. Use SDK-owned types and validators instead.", +}; + export default [ { ignores: [ @@ -161,6 +172,18 @@ export default [ "no-restricted-properties": "off", }, }, + { + files: ["src/**/*.ts", "src/**/*.tsx", "util/**/*.ts"], + languageOptions: { + parser: tsparser, + }, + plugins: { + "@typescript-eslint": tseslint, + }, + rules: { + "no-restricted-imports": ["error", { patterns: [generatedTypeImports] }], + }, + }, { files: ["src/**/*.ts", "src/**/*.tsx"], ignores: [...entryFiles, "**/*.test.ts", "**/*.test.tsx"], @@ -169,16 +192,7 @@ export default [ "error", { patterns: [ - { - group: [ - "**/generated_types", - "**/generated_plain_types", - "**/generated_types.ts", - "**/generated_plain_types.ts", - ], - message: - "Generated backend definitions are compatibility-test fixtures. Use SDK-owned types and validators instead.", - }, + generatedTypeImports, { group: [ "./exports", diff --git a/js/src/sdk-boundary.test.ts b/js/src/sdk-boundary.test.ts deleted file mode 100644 index f283bce28..000000000 --- a/js/src/sdk-boundary.test.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { readFileSync, readdirSync } from "node:fs"; -import { join, resolve } from "node:path"; -import { build } from "esbuild"; -import ts from "typescript"; -import { expect, test } from "vitest"; - -test("production sources cannot depend on generated backend definitions", () => { - for (const directory of [__dirname, resolve(__dirname, "../util")]) { - for (const file of readdirSync(directory, { recursive: true })) { - const name = String(file); - if ( - !/\.[cm]?tsx?$/.test(name) || - /\.test\.[cm]?tsx?$/.test(name) || - /(?:^|\/)generated_(?:plain_)?types\.ts$/.test(name) - ) { - continue; - } - const path = join(directory, name); - const source = ts.createSourceFile( - path, - readFileSync(path, "utf8"), - ts.ScriptTarget.Latest, - true, - ); - const dependencies: string[] = []; - function visit(node: ts.Node) { - if ( - ts.isStringLiteral(node) && - /(?:^|\/)generated_(?:plain_)?types(?:\.[cm]?[jt]s)?$/.test(node.text) - ) { - dependencies.push(node.text); - } - node.forEachChild(visit); - } - visit(source); - expect(dependencies, path).toEqual([]); - } - } -}); - -test("public bundles exclude the backend schema modules", async () => { - const result = await build({ - entryPoints: [ - "node", - "browser", - "edge-light", - "workerd", - "instrumentation", - ].map((entry) => resolve(__dirname, entry, "index.ts")), - bundle: true, - packages: "external", - platform: "node", - format: "esm", - splitting: true, - outdir: "sdk-boundary-check", - write: false, - metafile: true, - logLevel: "silent", - }); - expect( - Object.keys(result.metafile.inputs).filter((file) => - /generated_(?:plain_)?types/.test(file), - ), - ).toEqual([]); - expect( - Object.keys(result.metafile.inputs).some((file) => - file.endsWith("sdk-schemas.ts"), - ), - ).toBe(true); -}); diff --git a/js/src/sdk-contracts.test.ts b/js/src/sdk-contracts.test.ts index a3010c049..00ec7cd07 100644 --- a/js/src/sdk-contracts.test.ts +++ b/js/src/sdk-contracts.test.ts @@ -1,76 +1,9 @@ -import { posix, win32 } from "node:path"; -import ts from "typescript"; -import { expect, expectTypeOf, test } from "vitest"; -import { zodToJsonSchema } from "zod-to-json-schema"; +import { expectTypeOf, test } from "vitest"; +import type { z } from "zod/v3"; import type * as Backend from "./generated_plain_types"; -import * as backendSchemas from "./generated_types"; +import type * as backendSchemas from "./generated_types"; import type * as SDK from "./sdk-types"; -import * as schemas from "./sdk-schemas"; - -test.each([ - ["POSIX", posix], - ["Windows", win32], -] as const)( - "model parameters preserve provider field autocomplete (%s paths)", - (_name, path) => { - const suggestions = []; - for (const [module, name] of [ - ["./sdk-types", "ModelParams"], - ["./generated_plain_types", "ModelParamsType"], - ]) { - // TypeScript normalizes script names before requesting their snapshots. - const file = path - .join(__dirname, "__model_params_completion.ts") - .replaceAll("\\", "/"); - const source = `import type { ${name} as Params } from "${module}";\nconst params: Params = {\n\n};`; - const service = ts.createLanguageService({ - getScriptFileNames: () => [file], - getScriptVersion: () => "0", - getScriptSnapshot: (path) => { - const text = path === file ? source : ts.sys.readFile(path); - return text === undefined - ? undefined - : ts.ScriptSnapshot.fromString(text); - }, - getCurrentDirectory: () => __dirname, - getCompilationSettings: () => ({ - strict: true, - target: ts.ScriptTarget.ES2022, - }), - getDefaultLibFileName: ts.getDefaultLibFilePath, - fileExists: ts.sys.fileExists, - readFile: ts.sys.readFile, - readDirectory: ts.sys.readDirectory, - }); - try { - suggestions.push( - service - .getCompletionsAtPosition(file, source.lastIndexOf("\n};"), {}) - ?.entries.filter( - (entry) => - entry.kind === ts.ScriptElementKind.memberVariableElement, - ) - .map((entry) => entry.name) - .sort(), - ); - } finally { - service.dispose(); - } - } - expect(suggestions[0]).toEqual(suggestions[1]); - expect(suggestions[0]).toEqual( - expect.arrayContaining([ - "temperature", - "top_p", - "topP", - "topK", - "max_tokens", - "maxOutputTokens", - "response_format", - ]), - ); - }, -); +import type * as schemas from "./sdk-schemas"; // Backend definitions are compatibility fixtures, never production dependencies. test("SDK-owned contracts remain compatible with the backend", () => { @@ -133,146 +66,115 @@ test("SDK-owned contracts remain compatible with the backend", () => { expectTypeOf().toEqualTypeOf(); }); -test.each([ - "PromptData", - "ObjectReference", - "Project", - "AttachmentReference", - "BraintrustAttachmentReference", - "BraintrustModelParams", - "ChatCompletionTool", - "ExternalAttachmentReference", - "ResponseFormatJsonSchema", - "AttachmentStatus", - "GitMetadataSettings", - "DatasetSnapshot", - "Prompt", - "FunctionId", - "CallEvent", - "SSEConsoleEventData", - "SSEProgressEventData", -] as const)("%s preserves backend validation constraints", (name) => { - expect(zodToJsonSchema(schemas[name])).toEqual( - zodToJsonSchema(backendSchemas[name]), - ); +test("runtime validators accept and return compatible types", () => { + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf< + z.input + >().toEqualTypeOf< + z.input + >(); + expectTypeOf< + z.output + >().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf< + z.input + >().toEqualTypeOf< + z.input + >(); + expectTypeOf< + z.output + >().toEqualTypeOf< + z.output + >(); + expectTypeOf< + z.input + >().toEqualTypeOf>(); + expectTypeOf< + z.output + >().toEqualTypeOf>(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); }); - -const id = "12345678-1234-4234-8234-123456789012"; -const fixtures: Partial> = { - AttachmentReference: [ - { - type: "braintrust_attachment", - filename: "file", - content_type: "text/plain", - key: "key", - }, - { - type: "external_attachment", - filename: "file", - content_type: "text/plain", - url: "https://example.com/file", - }, - { - type: "braintrust_attachment", - filename: "", - content_type: "text/plain", - key: "key", - }, - ], - AttachmentStatus: [{ upload_status: "done" }, { upload_status: "invalid" }], - FunctionId: [ - { function_id: id }, - { project_name: "project", slug: "prompt" }, - { global_function: "scorer" }, - { prompt_session_id: id, prompt_session_function_id: id }, - { - inline_prompt: { prompt: { type: "completion", content: "Hello" } }, - function_type: "llm", - }, - ], - PromptData: [ - { - prompt: { type: "chat", messages: [{ role: "user" }, { role: "tool" }] }, - options: { model: "model", params: { temperature: 0, custom: true } }, - extra: "strip", - }, - { - prompt: { type: "completion", content: "Hello" }, - options: null, - parser: null, - }, - { prompt: { type: "chat", messages: [{ role: "invalid" }] } }, - ], - Prompt: [ - { - id, - project_id: id, - org_id: id, - _xact_id: "version", - log_id: "p", - name: "prompt", - slug: "prompt", - extra: "strip", - }, - ], - DatasetSnapshot: [ - { - id, - dataset_id: id, - name: "snapshot", - description: null, - xact_id: "version", - created: null, - }, - ], - ObjectReference: [ - { object_type: "dataset", object_id: id, id }, - { object_type: "invalid", object_id: id, id }, - ], - Project: [ - { - id, - org_id: id, - name: "project", - settings: { blind_reviews: true }, - extra: "strip", - }, - ], - CallEvent: [ - { event: "start" }, - { event: "done" }, - { event: "text_delta", data: "hello" }, - { event: "invalid", data: "hello" }, - ], - GitMetadataSettings: [ - { collect: "all" }, - { collect: "some", fields: ["commit"] }, - { collect: "some", fields: ["invalid"] }, - ], -}; - -test.each(Object.keys(schemas) as (keyof typeof schemas)[])( - "%s preserves parsing, defaults, and errors", - (name) => { - for (const input of [ - undefined, - null, - {}, - "invalid", - ...(fixtures[name] ?? []), - ]) { - const actual = schemas[name].safeParse(input); - const expected = backendSchemas[name].safeParse(input); - expect(actual.success, JSON.stringify(input)).toBe(expected.success); - if (actual.success && expected.success) { - expect(actual.data).toEqual(expected.data); - } else if (!actual.success && !expected.success) { - // Nested union errors have instance-specific methods; compare their - // serialized diagnostic payload, including codes, paths, and messages. - expect(JSON.parse(JSON.stringify(actual.error.issues))).toEqual( - JSON.parse(JSON.stringify(expected.error.issues)), - ); - } - } - }, -); diff --git a/js/src/sdk-schemas.ts b/js/src/sdk-schemas.ts index c7d67e5a9..8ee7eba35 100644 --- a/js/src/sdk-schemas.ts +++ b/js/src/sdk-schemas.ts @@ -1,6 +1,6 @@ /** * Runtime validators for the SDK-owned subset of the Braintrust API. - * Compatibility with backend validators is checked in sdk-contracts.test.ts. + * Input/output type compatibility is checked in sdk-contracts.test.ts. */ import { z } from "zod/v3"; From 33d1aead920471f39c0c74869a9b8af755e0d2b3 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:19:18 +0000 Subject: [PATCH 4/5] Update PR #2517 --- js/src/sdk-schemas.ts | 292 +++++++++++------------------------------- 1 file changed, 74 insertions(+), 218 deletions(-) diff --git a/js/src/sdk-schemas.ts b/js/src/sdk-schemas.ts index 8ee7eba35..70ae00ae1 100644 --- a/js/src/sdk-schemas.ts +++ b/js/src/sdk-schemas.ts @@ -127,15 +127,15 @@ export const CallEvent = z.union([ }), ]); -const ChatCompletionContentPartTextWithTitle = z.object({ +const CacheControl = z.object({ + type: z.literal("ephemeral"), + ttl: z.enum(["5m", "1h"]).optional(), +}); + +const ChatCompletionContentPartText = z.object({ text: z.string().default(""), type: z.literal("text"), - cache_control: z - .object({ - type: z.literal("ephemeral"), - ttl: z.enum(["5m", "1h"]).optional(), - }) - .optional(), + cache_control: CacheControl.optional(), }); const ChatCompletionContentPartImageWithTitle = z.object({ @@ -146,12 +146,7 @@ const ChatCompletionContentPartImageWithTitle = z.object({ .optional(), }), type: z.literal("image_url"), - cache_control: z - .object({ - type: z.literal("ephemeral"), - ttl: z.enum(["5m", "1h"]).optional(), - }) - .optional(), + cache_control: CacheControl.optional(), }); const ChatCompletionContentPartFileFile = z @@ -161,31 +156,15 @@ const ChatCompletionContentPartFileFile = z const ChatCompletionContentPartFileWithTitle = z.object({ file: ChatCompletionContentPartFileFile, type: z.literal("file"), - cache_control: z - .object({ - type: z.literal("ephemeral"), - ttl: z.enum(["5m", "1h"]).optional(), - }) - .optional(), + cache_control: CacheControl.optional(), }); const ChatCompletionContentPart = z.union([ - ChatCompletionContentPartTextWithTitle, + ChatCompletionContentPartText, ChatCompletionContentPartImageWithTitle, ChatCompletionContentPartFileWithTitle, ]); -const ChatCompletionContentPartText = z.object({ - text: z.string().default(""), - type: z.literal("text"), - cache_control: z - .object({ - type: z.literal("ephemeral"), - ttl: z.enum(["5m", "1h"]).optional(), - }) - .optional(), -}); - const ChatCompletionMessageToolCall = z.object({ id: z.string(), function: z.object({ arguments: z.string(), name: z.string() }), @@ -250,23 +229,20 @@ export const ChatCompletionTool = z.object({ type: z.literal("function"), }); -const ObjectReferenceNullish = z.union([ - z.object({ - object_type: z.enum([ - "project_logs", - "experiment", - "dataset", - "prompt", - "function", - "prompt_session", - ]), - object_id: z.string().uuid(), - id: z.string(), - _xact_id: z.union([z.string(), z.null()]).optional(), - created: z.union([z.string(), z.null()]).optional(), - }), - z.null(), -]); +export const ObjectReference = z.object({ + object_type: z.enum([ + "project_logs", + "experiment", + "dataset", + "prompt", + "function", + "prompt_session", + ]), + object_id: z.string().uuid(), + id: z.string(), + _xact_id: z.union([z.string(), z.null()]).optional(), + created: z.union([z.string(), z.null()]).optional(), +}); export const DatasetSnapshot = z.object({ id: z.string().uuid(), @@ -288,82 +264,58 @@ const PromptBlockDataNullish = z.union([ ]); const ModelParams = z.union([ - z - .object({ - use_cache: z.boolean(), - reasoning_enabled: z.boolean(), - reasoning_budget: z.number(), - temperature: z.number(), - top_p: z.number(), - max_tokens: z.number(), - max_completion_tokens: z.number(), - frequency_penalty: z.number(), - presence_penalty: z.number(), - response_format: ResponseFormatNullish, - tool_choice: z.union([ - z.literal("auto"), - z.literal("none"), - z.literal("required"), - z.object({ - type: z.literal("function"), - function: z.object({ name: z.string() }), - }), - ]), - function_call: z.union([ - z.literal("auto"), - z.literal("none"), - z.object({ name: z.string() }), - ]), - n: z.number(), - stop: z.array(z.string()), - reasoning_effort: z.enum(["none", "minimal", "low", "medium", "high"]), - verbosity: z.enum(["low", "medium", "high"]), - }) - .partial() - .passthrough(), - z - .object({ - use_cache: z.boolean().optional(), - reasoning_enabled: z.boolean().optional(), - reasoning_budget: z.number().optional(), - max_tokens: z.number(), - temperature: z.number(), - top_p: z.number().optional(), - top_k: z.number().optional(), - stop_sequences: z.array(z.string()).optional(), - max_tokens_to_sample: z.number().optional(), - }) - .passthrough(), - z - .object({ - use_cache: z.boolean(), - reasoning_enabled: z.boolean(), - reasoning_budget: z.number(), - temperature: z.number(), - maxOutputTokens: z.number(), - topP: z.number(), - topK: z.number(), - }) + BraintrustModelParams.extend({ + temperature: z.number(), + top_p: z.number(), + max_tokens: z.number(), + max_completion_tokens: z.number(), + frequency_penalty: z.number(), + presence_penalty: z.number(), + response_format: ResponseFormatNullish, + tool_choice: z.union([ + z.literal("auto"), + z.literal("none"), + z.literal("required"), + z.object({ + type: z.literal("function"), + function: z.object({ name: z.string() }), + }), + ]), + function_call: z.union([ + z.literal("auto"), + z.literal("none"), + z.object({ name: z.string() }), + ]), + n: z.number(), + stop: z.array(z.string()), + reasoning_effort: z.enum(["none", "minimal", "low", "medium", "high"]), + verbosity: z.enum(["low", "medium", "high"]), + }) .partial() .passthrough(), - z - .object({ - use_cache: z.boolean(), - reasoning_enabled: z.boolean(), - reasoning_budget: z.number(), - temperature: z.number(), - topK: z.number(), - }) + BraintrustModelParams.extend({ + max_tokens: z.number(), + temperature: z.number(), + top_p: z.number().optional(), + top_k: z.number().optional(), + stop_sequences: z.array(z.string()).optional(), + max_tokens_to_sample: z.number().optional(), + }).passthrough(), + BraintrustModelParams.extend({ + temperature: z.number(), + maxOutputTokens: z.number(), + topP: z.number(), + topK: z.number(), + }) .partial() .passthrough(), - z - .object({ - use_cache: z.boolean(), - reasoning_enabled: z.boolean(), - reasoning_budget: z.number(), - }) + BraintrustModelParams.extend({ + temperature: z.number(), + topK: z.number(), + }) .partial() .passthrough(), + BraintrustModelParams.passthrough(), ]); const PromptOptionsNullish = z.union([ @@ -405,73 +357,6 @@ const PreprocessorId = z.union([ z.null(), ]); -const PromptDataNullish = z.union([ - z - .object({ - prompt: PromptBlockDataNullish, - options: PromptOptionsNullish, - parser: PromptParserNullish, - preprocessor: PreprocessorId, - tool_functions: z.union([z.array(SavedFunctionId), z.null()]), - template_format: z.union([ - z.enum(["mustache", "nunjucks", "none"]), - z.null(), - ]), - mcp: z.union([ - z.record( - z.union([ - z.object({ - type: z.literal("id"), - id: z.string().uuid(), - is_disabled: z.boolean().optional(), - enabled_tools: z - .union([z.array(z.string()), z.null()]) - .optional(), - }), - z.object({ - type: z.literal("url"), - url: z.string(), - is_disabled: z.boolean().optional(), - enabled_tools: z - .union([z.array(z.string()), z.null()]) - .optional(), - }), - ]), - ), - z.null(), - ]), - origin: z.union([ - z - .object({ - prompt_id: z.string(), - project_id: z.string(), - prompt_version: z.string(), - }) - .partial(), - z.null(), - ]), - }) - .partial(), - z.null(), -]); - -const FunctionTypeEnumNullish = z.union([ - z.enum([ - "llm", - "scorer", - "task", - "tool", - "custom_view", - "preprocessor", - "facet", - "classifier", - "tag", - "parameters", - "sandbox", - ]), - z.null(), -]); - const FunctionFormat = z.enum(["llm", "code", "global", "graph", "topic_map"]); export const PromptData = z @@ -596,35 +481,6 @@ export const GitMetadataSettings = z.object({ .optional(), }); -const NullableSavedFunctionId = z.union([ - z.object({ - type: z.literal("function"), - id: z.string(), - version: z.string().optional(), - }), - z.object({ - type: z.literal("global"), - name: z.string(), - function_type: FunctionTypeEnum.optional().default("scorer"), - }), - z.null(), -]); - -export const ObjectReference = z.object({ - object_type: z.enum([ - "project_logs", - "experiment", - "dataset", - "prompt", - "function", - "prompt_session", - ]), - object_id: z.string().uuid(), - id: z.string(), - _xact_id: z.union([z.string(), z.null()]).optional(), - created: z.union([z.string(), z.null()]).optional(), -}); - const ProjectSettings = z.union([ z .object({ @@ -656,7 +512,7 @@ const ProjectSettings = z.union([ disable_realtime_queries: z.union([z.boolean(), z.null()]), monitor_charts_use_metrics_start: z.union([z.boolean(), z.null()]), blind_reviews: z.union([z.boolean(), z.null()]), - default_preprocessor: NullableSavedFunctionId, + default_preprocessor: z.union([...SavedFunctionId.options, z.null()]), }) .partial(), z.null(), @@ -683,12 +539,12 @@ export const Prompt = z.object({ slug: z.string(), description: z.union([z.string(), z.null()]).optional(), created: z.union([z.string(), z.null()]).optional(), - prompt_data: PromptDataNullish.optional(), + prompt_data: z.union([PromptData, z.null()]).optional(), tags: z.union([z.array(z.string()), z.null()]).optional(), metadata: z .union([z.object({}).partial().passthrough(), z.null()]) .optional(), - function_type: FunctionTypeEnumNullish.optional(), + function_type: z.union([FunctionTypeEnum, z.null()]).optional(), }); export const SSEConsoleEventData = z.object({ @@ -699,7 +555,7 @@ export const SSEConsoleEventData = z.object({ export const SSEProgressEventData = z.object({ id: z.string(), object_type: FunctionObjectType, - origin: ObjectReferenceNullish.and(z.unknown()).optional(), + origin: z.union([ObjectReference, z.null()]).and(z.unknown()).optional(), format: FunctionFormat, output_type: FunctionOutputType, name: z.string(), From b00f1d4d082a13c8f1f24325fc7f92ab73f20c5b Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 24 Sep 2026 16:01:30 +0000 Subject: [PATCH 5/5] Update PR #2517 --- e2e/helpers/mock-braintrust-server.test.ts | 61 ++++++++++++++++++++++ e2e/helpers/mock-braintrust-server.ts | 53 ++++++++++++------- e2e/scripts/run-e2e-tests.mjs | 2 +- 3 files changed, 95 insertions(+), 21 deletions(-) create mode 100644 e2e/helpers/mock-braintrust-server.test.ts diff --git a/e2e/helpers/mock-braintrust-server.test.ts b/e2e/helpers/mock-braintrust-server.test.ts new file mode 100644 index 000000000..4ba4af9d1 --- /dev/null +++ b/e2e/helpers/mock-braintrust-server.test.ts @@ -0,0 +1,61 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { startMockBraintrustServer } from "./mock-braintrust-server"; + +afterEach(() => vi.restoreAllMocks()); + +it.each([ + { path: "/logs3", statuses: [502, 503, 504, 200], succeeds: true }, + { path: "/logs3", statuses: [502, 502, 502, 502], succeeds: false }, + { path: "/logs3", statuses: [400], succeeds: false }, + { path: "/api/project/register", statuses: [502], succeeds: false }, +])( + "forwards $path with responses $statuses", + async ({ path, statuses, succeeds }) => { + const realFetch = globalThis.fetch; + const forward = vi.spyOn(globalThis, "fetch"); + for (const status of statuses) { + forward.mockResolvedValueOnce(new Response("{}", { status })); + } + const server = await startMockBraintrustServer({ + prodForwarding: { + apiKey: "test-forwarding-key", + apiUrl: "https://forwarding.invalid", + appUrl: "https://forwarding.invalid", + orgId: "test-org", + orgName: "test-org", + projectId: "test-project", + projectName: "tmp-luca-forwarding-test", + }, + }); + const body = JSON.stringify({ + api_version: 2, + rows: [{ id: "test-row", project_id: "test-project" }], + }); + try { + const response = await realFetch(`${server.url}${path}`, { + method: "POST", + body, + }); + await response.text(); + } finally { + if (succeeds) { + await server.close(); + } else { + await expect(server.close()).rejects.toThrow( + `prodForwarding failed for POST ${path}: ${statuses.at(-1)}`, + ); + } + } + + expect(forward).toHaveBeenCalledTimes(statuses.length); + for (const [url, init] of forward.mock.calls) { + expect(String(url)).toBe(`https://forwarding.invalid${path}`); + expect(init?.body).toBe(body); + expect(new Headers(init?.headers).get("authorization")).toBe( + "Bearer test-forwarding-key", + ); + } + expect(server.requests).toHaveLength(1); + expect(server.payloads).toHaveLength(path === "/logs3" ? 1 : 0); + }, +); diff --git a/e2e/helpers/mock-braintrust-server.ts b/e2e/helpers/mock-braintrust-server.ts index 7f59a57ed..78e072e15 100644 --- a/e2e/helpers/mock-braintrust-server.ts +++ b/e2e/helpers/mock-braintrust-server.ts @@ -6,6 +6,7 @@ import type { ServerResponse, } from "node:http"; import type { AddressInfo } from "node:net"; +import { setTimeout as sleep } from "node:timers/promises"; import type { ProdForwarding } from "./prod-forwarding"; export type JsonValue = @@ -472,29 +473,41 @@ export async function startMockBraintrustServer( } headers.set("authorization", `Bearer ${prodForwarding.apiKey}`); - const response = await fetch(url, { - body: - prodRequest.method === "GET" || prodRequest.method === "HEAD" - ? undefined - : prodRequest.rawBody, - headers, - method: prodRequest.method, - }); + // Log rows have stable IDs, so retrying an upload does not duplicate spans. + // Keep registration and other requests single-attempt. + const maxAttempts = prodRequest.path === "/logs3" ? 4 : 1; + for (let attempt = 1; ; attempt++) { + const response = await fetch(url, { + body: + prodRequest.method === "GET" || prodRequest.method === "HEAD" + ? undefined + : prodRequest.rawBody, + headers, + method: prodRequest.method, + }); - if (!response.ok) { - const responseText = await response.text().catch(() => ""); - throw new Error( - `prodForwarding failed for ${capturedRequest.method} ${capturedRequest.path}: ${response.status} ${response.statusText}${ - responseText ? `: ${responseText.slice(0, 500)}` : "" - }`, - ); - } + if (!response.ok) { + const responseText = await response.text().catch(() => ""); + if ( + attempt < maxAttempts && + [502, 503, 504].includes(response.status) + ) { + await sleep(500 * 2 ** (attempt - 1)); + continue; + } + throw new Error( + `prodForwarding failed for ${capturedRequest.method} ${capturedRequest.path}: ${response.status} ${response.statusText}${ + responseText ? `: ${responseText.slice(0, 500)}` : "" + }`, + ); + } - if (options.drainResponseBody) { - await response.arrayBuffer(); - } + if (options.drainResponseBody) { + await response.arrayBuffer(); + } - return response; + return response; + } } const server = createServer((req, res) => { diff --git a/e2e/scripts/run-e2e-tests.mjs b/e2e/scripts/run-e2e-tests.mjs index b4254e6b7..5b53a6f3d 100644 --- a/e2e/scripts/run-e2e-tests.mjs +++ b/e2e/scripts/run-e2e-tests.mjs @@ -22,7 +22,7 @@ const scenarioArgs = rawArgs.filter((arg) => arg !== "--update"); const testTargets = scenarioArgs.length > 0 ? scenarioArgs.map((arg) => scenarioPathArg(arg)) - : await defaultScenarioTestPaths(); + : [...(await defaultScenarioTestPaths()), "helpers"]; const vitestArgs = [ "run", "--run",