diff --git a/.changeset/simplify-public-sdk-types.md b/.changeset/simplify-public-sdk-types.md new file mode 100644 index 000000000..0cff8f061 --- /dev/null +++ b/.changeset/simplify-public-sdk-types.md @@ -0,0 +1,5 @@ +--- +"braintrust": major +--- + +ref!: Don't make generated types part of public api diff --git a/e2e/helpers/mock-braintrust-server.test.ts b/e2e/helpers/mock-braintrust-server.test.ts new file mode 100644 index 000000000..4ba4af9d1 --- /dev/null +++ b/e2e/helpers/mock-braintrust-server.test.ts @@ -0,0 +1,61 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { startMockBraintrustServer } from "./mock-braintrust-server"; + +afterEach(() => vi.restoreAllMocks()); + +it.each([ + { path: "/logs3", statuses: [502, 503, 504, 200], succeeds: true }, + { path: "/logs3", statuses: [502, 502, 502, 502], succeeds: false }, + { path: "/logs3", statuses: [400], succeeds: false }, + { path: "/api/project/register", statuses: [502], succeeds: false }, +])( + "forwards $path with responses $statuses", + async ({ path, statuses, succeeds }) => { + const realFetch = globalThis.fetch; + const forward = vi.spyOn(globalThis, "fetch"); + for (const status of statuses) { + forward.mockResolvedValueOnce(new Response("{}", { status })); + } + const server = await startMockBraintrustServer({ + prodForwarding: { + apiKey: "test-forwarding-key", + apiUrl: "https://forwarding.invalid", + appUrl: "https://forwarding.invalid", + orgId: "test-org", + orgName: "test-org", + projectId: "test-project", + projectName: "tmp-luca-forwarding-test", + }, + }); + const body = JSON.stringify({ + api_version: 2, + rows: [{ id: "test-row", project_id: "test-project" }], + }); + try { + const response = await realFetch(`${server.url}${path}`, { + method: "POST", + body, + }); + await response.text(); + } finally { + if (succeeds) { + await server.close(); + } else { + await expect(server.close()).rejects.toThrow( + `prodForwarding failed for POST ${path}: ${statuses.at(-1)}`, + ); + } + } + + expect(forward).toHaveBeenCalledTimes(statuses.length); + for (const [url, init] of forward.mock.calls) { + expect(String(url)).toBe(`https://forwarding.invalid${path}`); + expect(init?.body).toBe(body); + expect(new Headers(init?.headers).get("authorization")).toBe( + "Bearer test-forwarding-key", + ); + } + expect(server.requests).toHaveLength(1); + expect(server.payloads).toHaveLength(path === "/logs3" ? 1 : 0); + }, +); diff --git a/e2e/helpers/mock-braintrust-server.ts b/e2e/helpers/mock-braintrust-server.ts index 7f59a57ed..78e072e15 100644 --- a/e2e/helpers/mock-braintrust-server.ts +++ b/e2e/helpers/mock-braintrust-server.ts @@ -6,6 +6,7 @@ import type { ServerResponse, } from "node:http"; import type { AddressInfo } from "node:net"; +import { setTimeout as sleep } from "node:timers/promises"; import type { ProdForwarding } from "./prod-forwarding"; export type JsonValue = @@ -472,29 +473,41 @@ export async function startMockBraintrustServer( } headers.set("authorization", `Bearer ${prodForwarding.apiKey}`); - const response = await fetch(url, { - body: - prodRequest.method === "GET" || prodRequest.method === "HEAD" - ? undefined - : prodRequest.rawBody, - headers, - method: prodRequest.method, - }); + // Log rows have stable IDs, so retrying an upload does not duplicate spans. + // Keep registration and other requests single-attempt. + const maxAttempts = prodRequest.path === "/logs3" ? 4 : 1; + for (let attempt = 1; ; attempt++) { + const response = await fetch(url, { + body: + prodRequest.method === "GET" || prodRequest.method === "HEAD" + ? undefined + : prodRequest.rawBody, + headers, + method: prodRequest.method, + }); - if (!response.ok) { - const responseText = await response.text().catch(() => ""); - throw new Error( - `prodForwarding failed for ${capturedRequest.method} ${capturedRequest.path}: ${response.status} ${response.statusText}${ - responseText ? `: ${responseText.slice(0, 500)}` : "" - }`, - ); - } + if (!response.ok) { + const responseText = await response.text().catch(() => ""); + if ( + attempt < maxAttempts && + [502, 503, 504].includes(response.status) + ) { + await sleep(500 * 2 ** (attempt - 1)); + continue; + } + throw new Error( + `prodForwarding failed for ${capturedRequest.method} ${capturedRequest.path}: ${response.status} ${response.statusText}${ + responseText ? `: ${responseText.slice(0, 500)}` : "" + }`, + ); + } - if (options.drainResponseBody) { - await response.arrayBuffer(); - } + if (options.drainResponseBody) { + await response.arrayBuffer(); + } - return response; + return response; + } } const server = createServer((req, res) => { diff --git a/e2e/scripts/run-e2e-tests.mjs b/e2e/scripts/run-e2e-tests.mjs index b4254e6b7..5b53a6f3d 100644 --- a/e2e/scripts/run-e2e-tests.mjs +++ b/e2e/scripts/run-e2e-tests.mjs @@ -22,7 +22,7 @@ const scenarioArgs = rawArgs.filter((arg) => arg !== "--update"); const testTargets = scenarioArgs.length > 0 ? scenarioArgs.map((arg) => scenarioPathArg(arg)) - : await defaultScenarioTestPaths(); + : [...(await defaultScenarioTestPaths()), "helpers"]; const vitestArgs = [ "run", "--run", diff --git a/js/eslint.config.ts b/js/eslint.config.ts index 80829409c..00dad5c0a 100644 --- a/js/eslint.config.ts +++ b/js/eslint.config.ts @@ -20,6 +20,17 @@ const entryFiles = tsupConfig.flatMap((config) => { return entries; }); +const generatedTypeImports = { + group: [ + "**/generated_types", + "**/generated_plain_types", + "**/generated_types.ts", + "**/generated_plain_types.ts", + ], + message: + "Generated backend definitions are compatibility-test fixtures. Use SDK-owned types and validators instead.", +}; + export default [ { ignores: [ @@ -161,6 +172,18 @@ export default [ "no-restricted-properties": "off", }, }, + { + files: ["src/**/*.ts", "src/**/*.tsx", "util/**/*.ts"], + languageOptions: { + parser: tsparser, + }, + plugins: { + "@typescript-eslint": tseslint, + }, + rules: { + "no-restricted-imports": ["error", { patterns: [generatedTypeImports] }], + }, + }, { files: ["src/**/*.ts", "src/**/*.tsx"], ignores: [...entryFiles, "**/*.test.ts", "**/*.test.tsx"], @@ -169,6 +192,7 @@ export default [ "error", { patterns: [ + generatedTypeImports, { group: [ "./exports", diff --git a/js/src/eval-parameters.ts b/js/src/eval-parameters.ts index bc15dc105..9a21208ad 100644 --- a/js/src/eval-parameters.ts +++ b/js/src/eval-parameters.ts @@ -5,7 +5,7 @@ import { promptDefinitionToPromptData, type PromptDefinitionWithTools, } from "./prompt-schemas"; -import { PromptData as promptDataSchema } from "./generated_types"; +import { PromptData as promptDataSchema } from "./sdk-schemas"; export type EvalParameters = Record< string, diff --git a/js/src/framework-types.ts b/js/src/framework-types.ts index 0f9d45c26..27279c8b0 100644 --- a/js/src/framework-types.ts +++ b/js/src/framework-types.ts @@ -1,4 +1,4 @@ -import type { IfExistsType as IfExists } from "./generated_plain_types"; +import type { IfExists } from "./sdk-types"; export type GenericFunction = | ((input: Input) => Output) diff --git a/js/src/framework.ts b/js/src/framework.ts index bce815f8f..2e7fea418 100644 --- a/js/src/framework.ts +++ b/js/src/framework.ts @@ -7,13 +7,13 @@ import { SpanTypeAttribute, spanObjectTypeV3ToTypedString, } from "../util/index"; -import { ObjectReference as ObjectReferenceSchema } from "./generated_types"; +import { ObjectReference as ObjectReferenceSchema } from "./sdk-schemas"; import type { - GitMetadataSettingsType as GitMetadataSettings, - ObjectReferenceType as ObjectReference, - RepoInfoType as RepoInfo, - SSEProgressEventDataType as SSEProgressEventData, -} from "./generated_plain_types"; + GitMetadataSettings, + ObjectReference, + RepoInfo, + SSEProgressEventData, +} from "./sdk-types"; import { queue } from "async"; import iso from "./isomorph"; diff --git a/js/src/framework2.ts b/js/src/framework2.ts index ed1eef15f..fb9ee26ea 100644 --- a/js/src/framework2.ts +++ b/js/src/framework2.ts @@ -3,17 +3,16 @@ import type { Trace } from "./trace"; import iso from "./isomorph"; import { slugify } from "../util/string_util"; import { z } from "zod/v3"; -import { Project as projectSchema } from "./generated_types"; +import { Project as projectSchema } from "./sdk-schemas"; import type { - FunctionTypeEnumType as FunctionType, - IfExistsType as IfExists, - SavedFunctionIdType as SavedFunctionId, - PromptBlockDataType as PromptBlockData, - PromptDataType as PromptData, - ToolFunctionDefinitionType as ToolFunctionDefinition, - ExtendedSavedFunctionIdType as ExtendedSavedFunctionId, - FunctionDataType, -} from "./generated_plain_types"; + FunctionTypeEnum as FunctionType, + IfExists, + SavedFunctionId, + PromptBlockData, + PromptData, + ToolFunctionDefinition, + ExtendedSavedFunctionId, +} from "./sdk-types"; import { loadPrettyXact, TransactionId } from "../util/index"; import { _internalGetGlobalState, @@ -121,7 +120,7 @@ class Project { } await login(); const projectMap = new ProjectNameIdMap(); - const functionDefinitions: FunctionEvent[] = []; + const functionDefinitions: PromptFunctionEvent[] = []; if (this._publishableCodeFunctions.length > 0) { // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. console.warn( @@ -499,7 +498,7 @@ export class CodePrompt { async toFunctionDefinition( projectNameToId: ProjectNameIdMap, - ): Promise { + ): Promise { const prompt_data = { ...this.prompt, }; @@ -666,7 +665,7 @@ export class CodeParameters { async toFunctionDefinition( projectNameToId: ProjectNameIdMap, - ): Promise { + ): Promise { const schema = serializeEvalParameterstoParametersSchema(this.schema); return { project_id: await projectNameToId.resolve(this.project), @@ -776,13 +775,11 @@ function getDefaultDataFromParametersSchema( ); } -interface FunctionEvent { +interface FunctionEventBase { project_id: string; slug: string; name: string; description: string; - prompt_data?: PromptData; - function_data: FunctionDataType; function_type?: FunctionType; if_exists?: IfExists; tags?: string[]; @@ -790,6 +787,20 @@ interface FunctionEvent { environments?: { slug: string }[]; } +interface PromptFunctionEvent extends FunctionEventBase { + prompt_data: PromptData; + function_data: { type: "prompt" }; +} + +interface ParametersFunctionEvent extends FunctionEventBase { + function_type: "parameters"; + function_data: { + type: "parameters"; + data: Record; + __schema: ParametersSchema; + }; +} + class ProjectNameIdMap { private nameToId: Record = {}; private idToName: Record = {}; diff --git a/js/src/functions/invoke.ts b/js/src/functions/invoke.ts index 1cfe9a5b3..ac6e3f1a4 100644 --- a/js/src/functions/invoke.ts +++ b/js/src/functions/invoke.ts @@ -1,10 +1,10 @@ -import { FunctionId as functionIdSchema } from "../generated_types"; +import { FunctionId as functionIdSchema } from "../sdk-schemas"; import type { - InvokeFunctionType as InvokeFunctionRequest, - ChatCompletionMessageParamType as Message, - StreamingModeType as StreamingMode, - FunctionTypeEnumType as FunctionType, -} from "../generated_plain_types"; + InvokeFunction as InvokeFunctionRequest, + ChatCompletionMessageParam as Message, + StreamingMode, + FunctionTypeEnum as FunctionType, +} from "../sdk-types"; import { _internalGetGlobalState, BraintrustState, diff --git a/js/src/functions/stream.ts b/js/src/functions/stream.ts index 901891119..ae0ba64e1 100644 --- a/js/src/functions/stream.ts +++ b/js/src/functions/stream.ts @@ -2,12 +2,12 @@ import { CallEvent as callEventSchema, SSEConsoleEventData as sseConsoleEventDataSchema, SSEProgressEventData as sseProgressEventDataSchema, -} from "../generated_types"; +} from "../sdk-schemas"; import type { - CallEventType as CallEvent, - SSEConsoleEventDataType, - SSEProgressEventDataType, -} from "../generated_plain_types"; + CallEvent, + SSEConsoleEventData, + SSEProgressEventData, +} from "../sdk-types"; import { createParser, EventSourceParser, @@ -25,8 +25,8 @@ export type BraintrustStreamChunk = | { type: "reasoning_delta"; data: string } | { type: "json_delta"; data: string } | { type: "error"; data: string } - | { type: "console"; data: SSEConsoleEventDataType } - | { type: "progress"; data: SSEProgressEventDataType } + | { type: "console"; data: SSEConsoleEventData } + | { type: "progress"; data: SSEProgressEventData } | { type: "start"; data: string } | { type: "done"; data: string }; diff --git a/js/src/gitutil.ts b/js/src/gitutil.ts index 9a687ebb3..091404ce6 100644 --- a/js/src/gitutil.ts +++ b/js/src/gitutil.ts @@ -1,7 +1,4 @@ -import type { - GitMetadataSettingsType as GitMetadataSettings, - RepoInfoType as RepoInfo, -} from "./generated_plain_types"; +import type { GitMetadataSettings, RepoInfo } from "./sdk-types"; import { debugLogger } from "./debug-logger"; import { runGitCommand } from "./git-command"; diff --git a/js/src/isomorph.ts b/js/src/isomorph.ts index c02657d31..c06635d0d 100644 --- a/js/src/isomorph.ts +++ b/js/src/isomorph.ts @@ -1,7 +1,4 @@ -import type { - GitMetadataSettingsType as GitMetadataSettings, - RepoInfoType as RepoInfo, -} from "./generated_plain_types"; +import type { GitMetadataSettings, RepoInfo } from "./sdk-types"; import { newGlobalTracingChannel, type GlobalHookAsyncLocalStorage, diff --git a/js/src/logger.ts b/js/src/logger.ts index 13d85fbf6..b7bd554c0 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -76,27 +76,25 @@ import { DatasetSnapshot as datasetSnapshotSchema, PromptData as promptDataSchema, Prompt as promptSchema, -} from "./generated_types"; +} from "./sdk-schemas"; import type { - AnyModelParamsType as AnyModelParam, - AttachmentReferenceType as AttachmentReference, - BraintrustAttachmentReferenceType as BraintrustAttachmentReference, - ChatCompletionToolType as ChatCompletionTool, - ExternalAttachmentReferenceType as ExternalAttachmentReference, - ModelParamsType as ModelParams, - AttachmentStatusType as AttachmentStatus, - GitMetadataSettingsType as GitMetadataSettings, - ChatCompletionMessageParamType as Message, - ChatCompletionOpenAIMessageParamType as OpenAIMessage, - DatasetSnapshotType as DatasetSnapshot, - PromptDataType as PromptData, - PromptType as PromptRow, - PromptSessionEventType as PromptSessionEvent, - RepoInfoType as RepoInfo, - ObjectReferenceType as ObjectReference, - PromptBlockDataType as PromptBlockData, - ResponseFormatJsonSchemaType as ResponseFormatJsonSchema, -} from "./generated_plain_types"; + AttachmentReference, + BraintrustAttachmentReference, + ChatCompletionTool, + ExternalAttachmentReference, + ModelParams, + AttachmentStatus, + GitMetadataSettings, + ChatCompletionMessageParam as Message, + ChatCompletionOpenAIMessageParam as OpenAIMessage, + DatasetSnapshot, + PromptData, + FunctionTypeEnum, + PromptSessionEvent, + RepoInfo, + ObjectReference, + PromptBlockData, +} from "./sdk-types"; const BRAINTRUST_ATTACHMENT = BraintrustAttachmentReferenceSchema.shape.type.value; @@ -8797,37 +8795,52 @@ export class Dataset extends ObjectFetcher { } type CompiledPromptResponseFormat = - Exclude extends infer ResponseFormat - ? ResponseFormat extends { - type: "json_schema"; - json_schema: infer JsonSchema; - } - ? Omit & { - json_schema: Omit< - Extract, - "schema" - > & { - schema?: Record; - }; - } - : ResponseFormat - : never; - -// "none" is not assignable to older openai clients so we gotta exclude it -type CompiledPromptReasoningEffort = Exclude< - AnyModelParam["reasoning_effort"], - "none" ->; - -export type CompiledPromptParams = Omit< - NonNullable["params"], - "use_cache" | "response_format" | "reasoning_effort" -> & - Omit & { - reasoning_effort?: CompiledPromptReasoningEffort; - response_format?: CompiledPromptResponseFormat; - model: NonNullable["model"]>; - }; + | { type: "json_object" } + | { type: "text" } + | { + type: "json_schema"; + json_schema: { + name: string; + description?: string; + schema?: Record; + strict?: boolean | null; + }; + }; + +// Keep the compiled API independent of the backend's provider parameter union. +// "none" is excluded from reasoning_effort for compatibility with older OpenAI clients. +export interface CompiledPromptParams { + model: string; + max_tokens: number; + temperature?: number; + top_p?: number; + max_completion_tokens?: number; + frequency_penalty?: number; + presence_penalty?: number; + response_format?: CompiledPromptResponseFormat; + tool_choice?: + | "auto" + | "none" + | "required" + | { + type: "function"; + function: { name: string }; + }; + function_call?: "auto" | "none" | { name: string }; + n?: number; + stop?: string[]; + reasoning_effort?: "minimal" | "low" | "medium" | "high"; + verbosity?: "low" | "medium" | "high"; + top_k?: number; + stop_sequences?: string[]; + reasoning_enabled?: boolean; + reasoning_budget?: number; + /** Legacy parameter; prefer max_tokens. */ + max_tokens_to_sample?: number; + maxOutputTokens?: number; + topP?: number; + topK?: number; +} export type ChatPrompt = { messages: OpenAIMessage[]; @@ -8858,9 +8871,13 @@ export type CompiledPrompt = : // eslint-disable-next-line @typescript-eslint/no-empty-object-type {}); -export type DefaultPromptArgs = Partial< - CompiledPromptParams & AnyModelParam & ChatPrompt & CompletionPrompt ->; +export interface DefaultPromptArgs + extends + Partial, + Partial, + Partial { + use_cache?: boolean; +} function isAttachmentObject(value: unknown): boolean { return ( @@ -9024,17 +9041,26 @@ export function renderMessageImpl( }; } +interface PromptMetadata { + name: string; + slug: string; + project_id?: string; + description?: string | null; + created?: string | null; + prompt_data?: PromptData | null; + tags?: string[] | null; + // Preserve the generated prompt row's permissive metadata contract. + // eslint-disable-next-line @typescript-eslint/no-empty-object-type + metadata?: {} | null; + function_type?: FunctionTypeEnum | null; +} + export type PromptRowWithId< HasId extends boolean = true, HasVersion extends boolean = true, -> = Omit & - Partial> & - (HasId extends true - ? Pick - : Partial>) & - (HasVersion extends true - ? Pick - : Partial>); +> = PromptMetadata & + (HasId extends true ? { id: string } : { id?: string }) & + (HasVersion extends true ? { _xact_id: string } : { _xact_id?: string }); export function deserializePlainStringAsJSON(s: string) { if (s.trim() === "") { @@ -9121,6 +9147,16 @@ export function renderPromptParams( return params; } +export interface SerializedPrompt { + metadata: PromptRowWithId | PromptSessionEvent; + defaults: DefaultPromptArgs; + noTrace: boolean; +} + +// Initialized by the class so cache serialization can access private fields +// without exposing a serialization method on the public Prompt API. +export let serializePromptForCache: (prompt: Prompt) => SerializedPrompt; + export class Prompt< HasId extends boolean = true, HasVersion extends boolean = true, @@ -9457,13 +9493,12 @@ export class Prompt< ); } - /** @internal */ - public _internalSerializeForCache() { - return { - metadata: this.metadata, - defaults: this.defaults, - noTrace: this.noTrace, - }; + static { + serializePromptForCache = (prompt) => ({ + metadata: prompt.metadata, + defaults: prompt.defaults, + noTrace: prompt.noTrace, + }); } public static fromPromptData( @@ -9482,6 +9517,14 @@ export class Prompt< } } +export interface SerializedParameters { + metadata: ParametersRow; +} + +export let serializeParametersForCache: ( + parameters: RemoteEvalParameters, +) => SerializedParameters; + export class RemoteEvalParameters< HasId extends boolean = true, HasVersion extends boolean = true, @@ -9528,9 +9571,10 @@ export class RemoteEvalParameters< return (this.metadata.function_data.data ?? {}) as T; } - /** @internal */ - public _internalSerializeForCache() { - return { metadata: this.metadata }; + static { + serializeParametersForCache = (parameters) => ({ + metadata: parameters.metadata, + }); } public validate(data: unknown): boolean { diff --git a/js/src/prompt-cache/parameters-cache.ts b/js/src/prompt-cache/parameters-cache.ts index bb81a7a9c..530cfb28a 100644 --- a/js/src/prompt-cache/parameters-cache.ts +++ b/js/src/prompt-cache/parameters-cache.ts @@ -1,4 +1,8 @@ -import { RemoteEvalParameters } from "../logger"; +import { + RemoteEvalParameters, + serializeParametersForCache, + type SerializedParameters, +} from "../logger"; import { LRUCache } from "../lru-cache"; import { DiskCache } from "./disk-cache"; @@ -35,7 +39,7 @@ export type ParametersMemoryCacheEntry = { }; export type ParametersDiskCacheEntry = { - value: ReturnType; + value: SerializedParameters; resolvedOrgIdentity?: string; }; @@ -111,7 +115,7 @@ export class ParametersCache { this.memoryCache?.set(cacheKey, memoryEntry); if (this.diskCache) { await this.diskCache.set(cacheKey, { - value: value._internalSerializeForCache(), + value: serializeParametersForCache(value), resolvedOrgIdentity: this.expectedResolvedOrgIdentity, }); } diff --git a/js/src/prompt-cache/prompt-cache.ts b/js/src/prompt-cache/prompt-cache.ts index eb58ca0c2..8fa7b291a 100644 --- a/js/src/prompt-cache/prompt-cache.ts +++ b/js/src/prompt-cache/prompt-cache.ts @@ -1,4 +1,8 @@ -import { Prompt } from "../logger"; +import { + Prompt, + serializePromptForCache, + type SerializedPrompt, +} from "../logger"; import { LRUCache } from "../lru-cache"; import { DiskCache } from "./disk-cache"; @@ -40,7 +44,7 @@ export type PromptMemoryCacheEntry = { }; export type PromptDiskCacheEntry = { - value: ReturnType; + value: SerializedPrompt; resolvedOrgIdentity?: string; }; @@ -168,7 +172,7 @@ export class PromptCache { this.memoryCache?.set(cacheKey, memoryEntry); if (this.diskCache) { await this.diskCache.set(cacheKey, { - value: value._internalSerializeForCache(), + value: serializePromptForCache(value), resolvedOrgIdentity: this.expectedResolvedOrgIdentity, }); } diff --git a/js/src/prompt-schemas.ts b/js/src/prompt-schemas.ts index c83c420f0..94a433212 100644 --- a/js/src/prompt-schemas.ts +++ b/js/src/prompt-schemas.ts @@ -1,19 +1,19 @@ import type { - ToolFunctionDefinitionType as ToolFunctionDefinition, - ChatCompletionMessageParamType, - ModelParamsType, - PromptBlockDataType as PromptBlockData, - PromptDataType as PromptData, -} from "./generated_plain_types"; + ToolFunctionDefinition, + ChatCompletionMessageParam, + ModelParams, + PromptBlockData, + PromptData, +} from "./sdk-types"; // This roughly maps to promptBlockDataSchema, but is more ergonomic for the user. export type PromptContents = | { prompt: string } - | { messages: ChatCompletionMessageParamType[] }; + | { messages: ChatCompletionMessageParam[] }; export type PromptDefinition = PromptContents & { model: string; - params?: ModelParamsType; + params?: ModelParams; templateFormat?: "mustache" | "nunjucks" | "none"; environments?: string[]; }; diff --git a/js/src/prompt-types.test.ts b/js/src/prompt-types.test.ts new file mode 100644 index 000000000..51df00f1b --- /dev/null +++ b/js/src/prompt-types.test.ts @@ -0,0 +1,135 @@ +import { expectTypeOf, test } from "vitest"; +import type { ChatCompletionCreateParamsNonStreaming } from "openai/resources/chat/completions"; +import type { CompletionCreateParamsNonStreaming } from "openai/resources/completions"; +import { + Prompt, + type CompiledPrompt, + type CompiledPromptParams, + type DefaultPromptArgs, + type PromptRowWithId, +} from "./logger"; +import type { CodeParameters, CodePrompt } from "./framework2"; +import type { + AnyModelParamsType, + FunctionDataType, + PromptDataType, + PromptType, +} from "./generated_plain_types"; + +test("compiled prompts preserve provider parameter types and compatibility", () => { + // The generated backend types are a compatibility reference, not the public API. + type BackendParams = Omit< + AnyModelParamsType, + "use_cache" | "response_format" | "reasoning_effort" + > & { model: string }; + type Params = Omit< + CompiledPromptParams, + "response_format" | "reasoning_effort" + >; + expectTypeOf().toMatchTypeOf(); + expectTypeOf().toMatchTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf< + CompiledPrompt<"chat"> + >().toMatchTypeOf(); + expectTypeOf< + CompiledPrompt<"completion"> + >().toMatchTypeOf(); + expectTypeOf().toEqualTypeOf< + "minimal" | "low" | "medium" | "high" | undefined + >(); + expectTypeOf().toEqualTypeOf< + boolean | undefined + >(); + + if (false) { + const prompt = null as unknown as Prompt; + // @ts-expect-error Cache serialization is internal, not a Prompt method. + prompt._internalSerializeForCache(); + expectTypeOf(prompt.build({})).toEqualTypeOf>(); + expectTypeOf(prompt.build({}, { flavor: "completion" })).toEqualTypeOf< + CompiledPrompt<"completion"> + >(); + expectTypeOf(prompt.buildWithAttachments({})).toEqualTypeOf< + Promise> + >(); + const defaults: DefaultPromptArgs = { + response_format: { + type: "json_schema", + json_schema: { + name: "result", + schema: { type: "object" }, + strict: null, + }, + }, + use_cache: true, + }; + new Prompt( + { id: "id", _xact_id: "version", name: "name", slug: "slug" }, + defaults, + false, + ); + defaults.response_format = { + type: "json_schema", + json_schema: { + name: "result", + // @ts-expect-error Compiled schemas are objects, not templates. + schema: "{{schema}}", + }, + }; + // @ts-expect-error Preserve compatibility with older OpenAI clients. + defaults.reasoning_effort = "none"; + } +}); + +test("prompt metadata preserves required and optional identifiers", () => { + type Metadata = Omit< + PromptType, + "log_id" | "org_id" | "project_id" | "id" | "_xact_id" + > & { + project_id?: string; + }; + type RequiredRow = Metadata & { id: string; _xact_id: string }; + type OptionalRow = Metadata & { id?: string; _xact_id?: string }; + expectTypeOf>().toMatchTypeOf(); + expectTypeOf().toMatchTypeOf>(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf>().toMatchTypeOf(); + expectTypeOf().toMatchTypeOf>(); + expectTypeOf["id"]>().toEqualTypeOf(); + expectTypeOf["_xact_id"]>().toEqualTypeOf< + string | undefined + >(); + expectTypeOf["id"]>().toEqualTypeOf< + string | undefined + >(); + expectTypeOf< + PromptRowWithId["_xact_id"] + >().toEqualTypeOf(); + expectTypeOf["id"]>().toEqualTypeOf(); + expectTypeOf["id"]>().toEqualTypeOf< + string | undefined + >(); +}); + +test("function definitions expose only the variant each builder produces", () => { + type PromptEvent = Awaited>; + type ParametersEvent = Awaited< + ReturnType + >; + expectTypeOf().toEqualTypeOf<{ + type: "prompt"; + }>(); + expectTypeOf< + PromptEvent["prompt_data"] + >().branded.toEqualTypeOf(); + expectTypeOf< + ParametersEvent["function_data"]["type"] + >().toEqualTypeOf<"parameters">(); + expectTypeOf< + ParametersEvent["function_type"] + >().toEqualTypeOf<"parameters">(); + expectTypeOf().toMatchTypeOf< + Extract + >(); +}); diff --git a/js/src/public-types.test.ts b/js/src/public-types.test.ts index 3a37a9cf7..2870520fd 100644 --- a/js/src/public-types.test.ts +++ b/js/src/public-types.test.ts @@ -53,6 +53,8 @@ test("does not expose runtime schemas or implementation helpers", () => { "SpanImpl", "IDGenerator", "_exportsForTestingOnly", + "serializePromptForCache", + "serializeParametersForCache", "default", ]) { expect(publicExports).not.toHaveProperty(name); diff --git a/js/src/sandbox.ts b/js/src/sandbox.ts index 58c9f45be..4f91fc753 100644 --- a/js/src/sandbox.ts +++ b/js/src/sandbox.ts @@ -1,6 +1,6 @@ import { z } from "zod/v3"; import { slugify } from "../util/string_util"; -import type { IfExistsType } from "./generated_plain_types"; +import type { IfExists } from "./sdk-types"; import { type BraintrustState, _internalGetGlobalState } from "./logger"; /** @@ -32,7 +32,7 @@ interface RegisterSandboxOptions { /** Optional metadata. */ metadata?: Record; /** What to do if function already exists. Defaults to "replace". */ - ifExists?: IfExistsType; + ifExists?: IfExists; /** Braintrust API key. Uses BRAINTRUST_API_KEY env var if not provided. */ apiKey?: string; /** Braintrust app URL. Uses default if not provided. */ diff --git a/js/src/sdk-contracts.test.ts b/js/src/sdk-contracts.test.ts new file mode 100644 index 000000000..00ec7cd07 --- /dev/null +++ b/js/src/sdk-contracts.test.ts @@ -0,0 +1,180 @@ +import { expectTypeOf, test } from "vitest"; +import type { z } from "zod/v3"; +import type * as Backend from "./generated_plain_types"; +import type * as backendSchemas from "./generated_types"; +import type * as SDK from "./sdk-types"; +import type * as schemas from "./sdk-schemas"; + +// Backend definitions are compatibility fixtures, never production dependencies. +test("SDK-owned contracts remain compatible with the backend", () => { + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf< + Exclude + >().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().branded.toEqualTypeOf(); + expectTypeOf< + Exclude + >().toEqualTypeOf(); + expectTypeOf().branded.toEqualTypeOf(); + // Structural equality accommodates explicit fields replacing mapped intersections. + expectTypeOf().branded.toEqualTypeOf(); + expectTypeOf().toExtend(); + expectTypeOf().toExtend(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().branded.toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().branded.toEqualTypeOf(); + expectTypeOf().branded.toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); +}); + +test("runtime validators accept and return compatible types", () => { + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf< + z.input + >().toEqualTypeOf< + z.input + >(); + expectTypeOf< + z.output + >().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf< + z.input + >().toEqualTypeOf< + z.input + >(); + expectTypeOf< + z.output + >().toEqualTypeOf< + z.output + >(); + expectTypeOf< + z.input + >().toEqualTypeOf>(); + expectTypeOf< + z.output + >().toEqualTypeOf>(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); + expectTypeOf>().toEqualTypeOf< + z.input + >(); + expectTypeOf>().toEqualTypeOf< + z.output + >(); +}); diff --git a/js/src/sdk-schemas.ts b/js/src/sdk-schemas.ts new file mode 100644 index 000000000..70ae00ae1 --- /dev/null +++ b/js/src/sdk-schemas.ts @@ -0,0 +1,573 @@ +/** + * Runtime validators for the SDK-owned subset of the Braintrust API. + * Input/output type compatibility is checked in sdk-contracts.test.ts. + */ +import { z } from "zod/v3"; + +export const ResponseFormatJsonSchema = z.object({ + name: z.string(), + description: z.string().optional(), + schema: z + .union([z.object({}).partial().passthrough(), z.string()]) + .optional(), + strict: z.union([z.boolean(), z.null()]).optional(), +}); + +const ResponseFormatNullish = z.union([ + z.object({ type: z.literal("json_object") }), + z.object({ + type: z.literal("json_schema"), + json_schema: ResponseFormatJsonSchema, + }), + z.object({ type: z.literal("text") }), + z.null(), +]); + +export const BraintrustAttachmentReference = z.object({ + type: z.literal("braintrust_attachment"), + filename: z.string().min(1), + content_type: z.string().min(1), + key: z.string().min(1), +}); + +export const ExternalAttachmentReference = z.object({ + type: z.literal("external_attachment"), + filename: z.string().min(1), + content_type: z.string().min(1), + url: z.string().min(1), +}); + +export const AttachmentReference = z.discriminatedUnion("type", [ + BraintrustAttachmentReference, + ExternalAttachmentReference, +]); + +const UploadStatus = z.enum(["uploading", "done", "error"]); + +export const AttachmentStatus = z.object({ + upload_status: UploadStatus, + error_message: z.string().optional(), +}); + +const FunctionTypeEnum = z.enum([ + "llm", + "scorer", + "task", + "tool", + "custom_view", + "preprocessor", + "facet", + "classifier", + "tag", + "parameters", + "sandbox", +]); + +const SavedFunctionId = z.union([ + z.object({ + type: z.literal("function"), + id: z.string(), + version: z.string().optional(), + }), + z.object({ + type: z.literal("global"), + name: z.string(), + function_type: FunctionTypeEnum.optional().default("scorer"), + }), +]); + +export const BraintrustModelParams = z + .object({ + use_cache: z.boolean(), + reasoning_enabled: z.boolean(), + reasoning_budget: z.number(), + }) + .partial(); + +export const CallEvent = z.union([ + z.object({ + id: z.string().optional(), + data: z.string(), + event: z.literal("text_delta"), + }), + z.object({ + id: z.string().optional(), + data: z.string(), + event: z.literal("reasoning_delta"), + }), + z.object({ + id: z.string().optional(), + data: z.string(), + event: z.literal("json_delta"), + }), + z.object({ + id: z.string().optional(), + data: z.string(), + event: z.literal("progress"), + }), + z.object({ + id: z.string().optional(), + data: z.string(), + event: z.literal("error"), + }), + z.object({ + id: z.string().optional(), + data: z.string(), + event: z.literal("console"), + }), + z.object({ + id: z.string().optional(), + event: z.literal("start"), + data: z.literal(""), + }), + z.object({ + id: z.string().optional(), + event: z.literal("done"), + data: z.literal(""), + }), +]); + +const CacheControl = z.object({ + type: z.literal("ephemeral"), + ttl: z.enum(["5m", "1h"]).optional(), +}); + +const ChatCompletionContentPartText = z.object({ + text: z.string().default(""), + type: z.literal("text"), + cache_control: CacheControl.optional(), +}); + +const ChatCompletionContentPartImageWithTitle = z.object({ + image_url: z.object({ + url: z.string(), + detail: z + .union([z.literal("auto"), z.literal("low"), z.literal("high")]) + .optional(), + }), + type: z.literal("image_url"), + cache_control: CacheControl.optional(), +}); + +const ChatCompletionContentPartFileFile = z + .object({ file_data: z.string(), filename: z.string(), file_id: z.string() }) + .partial(); + +const ChatCompletionContentPartFileWithTitle = z.object({ + file: ChatCompletionContentPartFileFile, + type: z.literal("file"), + cache_control: CacheControl.optional(), +}); + +const ChatCompletionContentPart = z.union([ + ChatCompletionContentPartText, + ChatCompletionContentPartImageWithTitle, + ChatCompletionContentPartFileWithTitle, +]); + +const ChatCompletionMessageToolCall = z.object({ + id: z.string(), + function: z.object({ arguments: z.string(), name: z.string() }), + type: z.literal("function"), +}); + +const ChatCompletionMessageReasoning = z + .object({ id: z.string(), content: z.string() }) + .partial(); + +const ChatCompletionMessageParam = z.union([ + z.object({ + content: z.union([z.string(), z.array(ChatCompletionContentPartText)]), + role: z.literal("system"), + name: z.string().optional(), + }), + z.object({ + content: z.union([z.string(), z.array(ChatCompletionContentPart)]), + role: z.literal("user"), + name: z.string().optional(), + }), + z.object({ + role: z.literal("assistant"), + content: z + .union([z.string(), z.array(ChatCompletionContentPartText), z.null()]) + .optional(), + function_call: z + .object({ arguments: z.string(), name: z.string() }) + .optional(), + name: z.string().optional(), + tool_calls: z.array(ChatCompletionMessageToolCall).optional(), + reasoning: z.array(ChatCompletionMessageReasoning).optional(), + reasoning_signature: z.string().optional(), + }), + z.object({ + content: z.union([z.string(), z.array(ChatCompletionContentPartText)]), + role: z.literal("tool"), + tool_call_id: z.string().default(""), + }), + z.object({ + content: z.union([z.string(), z.null()]), + name: z.string(), + role: z.literal("function"), + }), + z.object({ + content: z.union([z.string(), z.array(ChatCompletionContentPartText)]), + role: z.literal("developer"), + name: z.string().optional(), + }), + z.object({ + role: z.literal("model"), + content: z.union([z.string(), z.null()]).optional(), + }), +]); + +export const ChatCompletionTool = z.object({ + function: z.object({ + name: z.string(), + description: z.string().optional(), + parameters: z.object({}).partial().passthrough().optional(), + }), + type: z.literal("function"), +}); + +export const ObjectReference = z.object({ + object_type: z.enum([ + "project_logs", + "experiment", + "dataset", + "prompt", + "function", + "prompt_session", + ]), + object_id: z.string().uuid(), + id: z.string(), + _xact_id: z.union([z.string(), z.null()]).optional(), + created: z.union([z.string(), z.null()]).optional(), +}); + +export const DatasetSnapshot = z.object({ + id: z.string().uuid(), + dataset_id: z.string().uuid(), + name: z.string(), + description: z.union([z.string(), z.null()]), + xact_id: z.string(), + created: z.union([z.string(), z.null()]), +}); + +const PromptBlockDataNullish = z.union([ + z.object({ + type: z.literal("chat"), + messages: z.array(ChatCompletionMessageParam), + tools: z.string().optional(), + }), + z.object({ type: z.literal("completion"), content: z.string() }), + z.null(), +]); + +const ModelParams = z.union([ + BraintrustModelParams.extend({ + temperature: z.number(), + top_p: z.number(), + max_tokens: z.number(), + max_completion_tokens: z.number(), + frequency_penalty: z.number(), + presence_penalty: z.number(), + response_format: ResponseFormatNullish, + tool_choice: z.union([ + z.literal("auto"), + z.literal("none"), + z.literal("required"), + z.object({ + type: z.literal("function"), + function: z.object({ name: z.string() }), + }), + ]), + function_call: z.union([ + z.literal("auto"), + z.literal("none"), + z.object({ name: z.string() }), + ]), + n: z.number(), + stop: z.array(z.string()), + reasoning_effort: z.enum(["none", "minimal", "low", "medium", "high"]), + verbosity: z.enum(["low", "medium", "high"]), + }) + .partial() + .passthrough(), + BraintrustModelParams.extend({ + max_tokens: z.number(), + temperature: z.number(), + top_p: z.number().optional(), + top_k: z.number().optional(), + stop_sequences: z.array(z.string()).optional(), + max_tokens_to_sample: z.number().optional(), + }).passthrough(), + BraintrustModelParams.extend({ + temperature: z.number(), + maxOutputTokens: z.number(), + topP: z.number(), + topK: z.number(), + }) + .partial() + .passthrough(), + BraintrustModelParams.extend({ + temperature: z.number(), + topK: z.number(), + }) + .partial() + .passthrough(), + BraintrustModelParams.passthrough(), +]); + +const PromptOptionsNullish = z.union([ + z + .object({ + model: z.string(), + params: ModelParams, + position: z.string(), + endpoint_name: z.union([z.string(), z.null()]), + }) + .partial(), + z.null(), +]); + +const PromptParserNullish = z.union([ + z.object({ + type: z.literal("llm_classifier"), + use_cot: z.boolean(), + choice_scores: z.record(z.number().gte(0).lte(1)).optional(), + choice: z.array(z.string()).optional(), + allow_no_match: z.boolean().optional(), + allow_skip: z.boolean().optional(), + }), + z.null(), +]); + +const PreprocessorId = z.union([ + z.object({ + type: z.literal("function"), + id: z.string(), + version: z.string().optional(), + }), + z.object({ + type: z.literal("global"), + name: z.string(), + function_type: z.literal("preprocessor").optional().default("preprocessor"), + }), + z.object({ type: z.literal("inline"), code: z.string().min(1) }), + z.null(), +]); + +const FunctionFormat = z.enum(["llm", "code", "global", "graph", "topic_map"]); + +export const PromptData = z + .object({ + prompt: PromptBlockDataNullish, + options: PromptOptionsNullish, + parser: PromptParserNullish, + preprocessor: PreprocessorId, + tool_functions: z.union([z.array(SavedFunctionId), z.null()]), + template_format: z.union([ + z.enum(["mustache", "nunjucks", "none"]), + z.null(), + ]), + mcp: z.union([ + z.record( + z.union([ + z.object({ + type: z.literal("id"), + id: z.string().uuid(), + is_disabled: z.boolean().optional(), + enabled_tools: z.union([z.array(z.string()), z.null()]).optional(), + }), + z.object({ + type: z.literal("url"), + url: z.string(), + is_disabled: z.boolean().optional(), + enabled_tools: z.union([z.array(z.string()), z.null()]).optional(), + }), + ]), + ), + z.null(), + ]), + origin: z.union([ + z + .object({ + prompt_id: z.string(), + project_id: z.string(), + prompt_version: z.string(), + }) + .partial(), + z.null(), + ]), + }) + .partial(); + +export const FunctionId = z.union([ + z.object({ function_id: z.string(), version: z.string().optional() }), + z.object({ + project_name: z.string(), + slug: z.string(), + version: z.string().optional(), + }), + z.object({ + global_function: z.string(), + function_type: FunctionTypeEnum.optional().default("scorer"), + }), + z.object({ + prompt_session_id: z.string(), + prompt_session_function_id: z.string(), + version: z.string().optional(), + }), + z.object({ + inline_context: z.object({ + runtime: z.enum(["node", "python", "browser", "quickjs"]), + version: z.string(), + }), + code: z.string(), + function_type: FunctionTypeEnum.and(z.unknown()).optional(), + name: z.union([z.string(), z.null()]).optional(), + }), + z.object({ + inline_prompt: PromptData.optional(), + inline_function: z.object({}).partial().passthrough(), + function_type: FunctionTypeEnum.optional().default("scorer"), + name: z.union([z.string(), z.null()]).optional(), + }), + z.object({ + inline_prompt: PromptData, + function_type: FunctionTypeEnum.optional().default("scorer"), + name: z.union([z.string(), z.null()]).optional(), + }), +]); + +const FunctionObjectType = z.enum([ + "prompt", + "tool", + "scorer", + "task", + "workflow", + "custom_view", + "preprocessor", + "facet", + "classifier", + "parameters", + "sandbox", +]); + +const FunctionOutputType = z.enum([ + "completion", + "score", + "facet", + "classification", + "any", +]); + +export const GitMetadataSettings = z.object({ + collect: z.enum(["all", "none", "some"]), + fields: z + .array( + z.enum([ + "commit", + "branch", + "tag", + "dirty", + "author_name", + "author_email", + "commit_message", + "commit_time", + "git_diff", + ]), + ) + .optional(), +}); + +const ProjectSettings = z.union([ + z + .object({ + comparison_key: z.union([z.string(), z.null()]), + baseline_experiment_id: z.union([z.string(), z.null()]), + spanFieldOrder: z.union([ + z.array( + z.object({ + object_type: z.string(), + column_id: z.string(), + position: z.string(), + layout: z + .union([z.literal("full"), z.literal("two_column"), z.null()]) + .optional(), + }), + ), + z.null(), + ]), + remote_eval_sources: z.union([ + z.array( + z.object({ + url: z.string(), + name: z.union([z.string(), z.null()]).optional(), + description: z.union([z.string(), z.null()]).optional(), + }), + ), + z.null(), + ]), + disable_realtime_queries: z.union([z.boolean(), z.null()]), + monitor_charts_use_metrics_start: z.union([z.boolean(), z.null()]), + blind_reviews: z.union([z.boolean(), z.null()]), + default_preprocessor: z.union([...SavedFunctionId.options, z.null()]), + }) + .partial(), + z.null(), +]); + +export const Project = z.object({ + id: z.string().uuid(), + org_id: z.string().uuid(), + name: z.string(), + description: z.union([z.string(), z.null()]).optional(), + created: z.union([z.string(), z.null()]).optional(), + deleted_at: z.union([z.string(), z.null()]).optional(), + user_id: z.union([z.string(), z.null()]).optional(), + settings: ProjectSettings.optional(), +}); + +export const Prompt = z.object({ + id: z.string().uuid(), + _xact_id: z.string(), + project_id: z.string().uuid(), + log_id: z.literal("p"), + org_id: z.string().uuid(), + name: z.string(), + slug: z.string(), + description: z.union([z.string(), z.null()]).optional(), + created: z.union([z.string(), z.null()]).optional(), + prompt_data: z.union([PromptData, z.null()]).optional(), + tags: z.union([z.array(z.string()), z.null()]).optional(), + metadata: z + .union([z.object({}).partial().passthrough(), z.null()]) + .optional(), + function_type: z.union([FunctionTypeEnum, z.null()]).optional(), +}); + +export const SSEConsoleEventData = z.object({ + stream: z.enum(["stderr", "stdout"]), + message: z.string(), +}); + +export const SSEProgressEventData = z.object({ + id: z.string(), + object_type: FunctionObjectType, + origin: z.union([ObjectReference, z.null()]).and(z.unknown()).optional(), + format: FunctionFormat, + output_type: FunctionOutputType, + name: z.string(), + event: z.enum([ + "reasoning_delta", + "text_delta", + "json_delta", + "error", + "console", + "start", + "done", + "progress", + ]), + data: z.string(), +}); diff --git a/js/src/sdk-types.ts b/js/src/sdk-types.ts new file mode 100644 index 000000000..1fdc3f6a9 --- /dev/null +++ b/js/src/sdk-types.ts @@ -0,0 +1,573 @@ +/* eslint-disable @typescript-eslint/no-empty-object-type -- Preserve the existing wire contracts, including permissive JSON fields. */ +/* eslint-disable @typescript-eslint/no-explicit-any -- Provider parameter maps intentionally accept arbitrary values. */ +/** + * SDK-owned wire contracts. Keep backend-generated definitions in compatibility + * tests only; changes here should describe the fields the SDK actually uses. + */ +export interface ResponseFormatJsonSchema { + name: string; + description?: string | undefined; + schema?: ({} | string) | undefined; + strict?: (boolean | null) | undefined; +} +export type ResponseFormatNullish = + | { + type: "json_object"; + } + | { + type: "json_schema"; + json_schema: ResponseFormatJsonSchema; + } + | { + type: "text"; + } + | null; +export type AsyncScoringState = + | { + status: "enabled"; + token: string; + function_ids: Array; + skip_logging?: (boolean | null) | undefined; + triggered_functions?: ({} | null) | undefined; + last_triggered_xact_id?: (string | number | null) | undefined; + } + | { + status: "disabled"; + } + | null; +export type AsyncScoringControl = + | { + kind: "score_update"; + token?: string | undefined; + } + | { + kind: "state_override"; + state: AsyncScoringState; + } + | { + kind: "state_force_reselect"; + } + | { + kind: "state_enabled_force_rescore"; + } + | { + kind: "trigger_functions"; + triggered_functions: Array<{ + function_id?: unknown | undefined; + scope: + | { + type: "span"; + } + | { + type: "trace"; + }; + idempotency_key?: string | undefined; + }>; + } + | { + kind: "complete_triggered_functions"; + function_ids: Array; + triggered_xact_id: string; + } + | { + kind: "mark_attempt_failed"; + function_ids: Array; + }; +export interface BraintrustAttachmentReference { + type: "braintrust_attachment"; + filename: string; + content_type: string; + key: string; +} +export interface ExternalAttachmentReference { + type: "external_attachment"; + filename: string; + content_type: string; + url: string; +} +export type AttachmentReference = + | BraintrustAttachmentReference + | ExternalAttachmentReference; +export type UploadStatus = "uploading" | "done" | "error"; +export interface AttachmentStatus { + upload_status: UploadStatus; + error_message?: string | undefined; +} +export type FunctionTypeEnum = + | "llm" + | "scorer" + | "task" + | "tool" + | "custom_view" + | "preprocessor" + | "facet" + | "classifier" + | "tag" + | "parameters" + | "sandbox"; +export type SavedFunctionId = + | { + type: "function"; + id: string; + version?: string | undefined; + } + | { + type: "global"; + name: string; + function_type: FunctionTypeEnum; + }; +export type CallEvent = + | { + id?: string | undefined; + data: string; + event: "text_delta"; + } + | { + id?: string | undefined; + data: string; + event: "reasoning_delta"; + } + | { + id?: string | undefined; + data: string; + event: "json_delta"; + } + | { + id?: string | undefined; + data: string; + event: "progress"; + } + | { + id?: string | undefined; + data: string; + event: "error"; + } + | { + id?: string | undefined; + data: string; + event: "console"; + } + | { + id?: string | undefined; + event: "start"; + data: ""; + } + | { + id?: string | undefined; + event: "done"; + data: ""; + }; +interface CacheControl { + type: "ephemeral"; + ttl?: "5m" | "1h" | undefined; +} + +export interface ChatCompletionContentPartImageWithTitle { + image_url: { + url: string; + detail?: ("auto" | "low" | "high") | undefined; + }; + type: "image_url"; + cache_control?: CacheControl | undefined; +} +export interface ChatCompletionContentPartFileFile { + file_data?: string; + filename?: string; + file_id?: string; +} +export interface ChatCompletionContentPartFileWithTitle { + file: ChatCompletionContentPartFileFile; + type: "file"; + cache_control?: CacheControl | undefined; +} +export type ChatCompletionContentPart = + | ChatCompletionContentPartText + | ChatCompletionContentPartImageWithTitle + | ChatCompletionContentPartFileWithTitle; +export interface ChatCompletionContentPartText { + text: string; + type: "text"; + cache_control?: CacheControl | undefined; +} +export interface ChatCompletionMessageToolCall { + id: string; + function: { + arguments: string; + name: string; + }; + type: "function"; +} +export interface ChatCompletionMessageReasoning { + id?: string; + content?: string; +} +export type ChatCompletionMessageParam = + | ChatCompletionOpenAIMessageParam + | { + role: "model"; + content?: string | null | undefined; + }; +export type ChatCompletionOpenAIMessageParam = + | { + content: string | Array; + role: "system"; + name?: string | undefined; + } + | { + content: string | Array; + role: "user"; + name?: string | undefined; + } + | { + role: "assistant"; + content?: + | (string | Array | null) + | undefined; + function_call?: + | { + arguments: string; + name: string; + } + | undefined; + name?: string | undefined; + tool_calls?: Array | undefined; + reasoning?: Array | undefined; + reasoning_signature?: string | undefined; + } + | { + content: string | Array; + role: "tool"; + tool_call_id: string; + } + | { + content: string | null; + name: string; + role: "function"; + } + | { + content: string | Array; + role: "developer"; + name?: string | undefined; + }; +export interface ChatCompletionTool { + function: { + name: string; + description?: string | undefined; + parameters?: {} | undefined; + }; + type: "function"; +} +export interface DatasetSnapshot { + id: string; + dataset_id: string; + name: string; + description: string | null; + xact_id: string; + created: string | null; +} +export type RepoInfo = { + commit?: string | null; + branch?: string | null; + tag?: string | null; + dirty?: boolean | null; + author_name?: string | null; + author_email?: string | null; + commit_message?: string | null; + commit_time?: string | null; + git_diff?: string | null; +} | null; +export type ExtendedSavedFunctionId = + | SavedFunctionId + | { + type: "slug"; + project_id: string; + slug: string; + }; +// Keep provider fields visible to editor completion while accepting custom options. +export type ModelParams = + | { + use_cache?: boolean; + reasoning_enabled?: boolean; + reasoning_budget?: number; + temperature?: number; + top_p?: number; + max_tokens?: number; + max_completion_tokens?: number; + frequency_penalty?: number; + presence_penalty?: number; + response_format?: ResponseFormatNullish; + tool_choice?: + | "auto" + | "none" + | "required" + | { + type: "function"; + function: { + name: string; + }; + }; + function_call?: + | "auto" + | "none" + | { + name: string; + }; + n?: number; + stop?: Array; + reasoning_effort?: "none" | "minimal" | "low" | "medium" | "high"; + verbosity?: "low" | "medium" | "high"; + [key: string]: any; + } + | { + use_cache?: boolean | undefined; + reasoning_enabled?: boolean | undefined; + reasoning_budget?: number | undefined; + max_tokens: number; + temperature: number; + top_p?: number | undefined; + top_k?: number | undefined; + stop_sequences?: Array | undefined; + max_tokens_to_sample?: number | undefined; + [key: string]: any; + } + | { + use_cache?: boolean; + reasoning_enabled?: boolean; + reasoning_budget?: number; + temperature?: number; + maxOutputTokens?: number; + topP?: number; + topK?: number; + [key: string]: any; + } + | { + use_cache?: boolean; + reasoning_enabled?: boolean; + reasoning_budget?: number; + temperature?: number; + topK?: number; + [key: string]: any; + } + | { + use_cache?: boolean; + reasoning_enabled?: boolean; + reasoning_budget?: number; + [key: string]: any; + }; +export type PromptOptionsNullish = { + model?: string; + params?: ModelParams; + position?: string; + endpoint_name?: string | null; +} | null; +export type PromptParserNullish = { + type: "llm_classifier"; + use_cot: boolean; + choice_scores?: {} | undefined; + choice?: Array | undefined; + allow_no_match?: boolean | undefined; + allow_skip?: boolean | undefined; +} | null; +export type PreprocessorId = + | { + type: "function"; + id: string; + version?: string | undefined; + } + | { + type: "global"; + name: string; + function_type: "preprocessor"; + } + | { + type: "inline"; + code: string; + } + | null; +export type PromptBlockData = + | { + type: "chat"; + messages: Array; + tools?: string | undefined; + } + | { + type: "completion"; + content: string; + }; +export type FunctionFormat = "llm" | "code" | "global" | "graph" | "topic_map"; +export interface PromptData { + prompt?: PromptBlockData | null; + options?: PromptOptionsNullish; + parser?: PromptParserNullish; + preprocessor?: PreprocessorId; + tool_functions?: Array | null; + template_format?: ("mustache" | "nunjucks" | "none") | null; + mcp?: {} | null; + origin?: { + prompt_id?: string; + project_id?: string; + prompt_version?: string; + } | null; +} +export type FunctionId = + | { + function_id: string; + version?: string | undefined; + } + | { + project_name: string; + slug: string; + version?: string | undefined; + } + | { + global_function: string; + function_type: FunctionTypeEnum; + } + | { + prompt_session_id: string; + prompt_session_function_id: string; + version?: string | undefined; + } + | { + inline_context: { + runtime: "node" | "python" | "browser" | "quickjs"; + version: string; + }; + code: string; + function_type?: FunctionTypeEnum | undefined; + name?: (string | null) | undefined; + } + | { + inline_prompt?: PromptData | undefined; + inline_function: {}; + function_type: FunctionTypeEnum; + name?: (string | null) | undefined; + } + | { + inline_prompt: PromptData; + function_type: FunctionTypeEnum; + name?: (string | null) | undefined; + }; +export type FunctionObjectType = + | "prompt" + | "tool" + | "scorer" + | "task" + | "workflow" + | "custom_view" + | "preprocessor" + | "facet" + | "classifier" + | "parameters" + | "sandbox"; +export type FunctionOutputType = + | "completion" + | "score" + | "facet" + | "classification" + | "any"; +export interface GitMetadataSettings { + collect: "all" | "none" | "some"; + fields?: + | Array< + | "commit" + | "branch" + | "tag" + | "dirty" + | "author_name" + | "author_email" + | "commit_message" + | "commit_time" + | "git_diff" + > + | undefined; +} +export type IfExists = "error" | "ignore" | "replace"; +export type InvokeParent = + | { + object_type: "project_logs" | "experiment" | "playground_logs"; + object_id: string; + row_ids?: + | ({ + id: string; + span_id: string; + root_span_id: string; + } | null) + | undefined; + propagated_event?: ({} | null) | undefined; + } + | string; +export type StreamingMode = ("auto" | "parallel" | "json" | "text") | null; +export type InvokeFunction = FunctionId & { + input?: unknown; + expected?: unknown; + metadata?: {} | null; + tags?: Array | null; + messages?: Array; + parent?: InvokeParent; + stream?: boolean | null; + mode?: StreamingMode; + strict?: boolean | null; + mcp_auth?: {}; + overrides?: {} | null; + endpoint_name?: string | null; +}; +export interface ObjectReference { + object_type: + | "project_logs" + | "experiment" + | "dataset" + | "prompt" + | "function" + | "prompt_session"; + object_id: string; + id: string; + _xact_id?: (string | null) | undefined; + created?: (string | null) | undefined; +} +export interface PromptSessionEvent { + id: string; + _xact_id: string; + created: string; + _pagination_key?: (string | null) | undefined; + project_id: string; + prompt_session_id: string; + prompt_session_data?: unknown | undefined; + prompt_data?: unknown | undefined; + function_data?: unknown | undefined; + function_type?: FunctionTypeEnum | null | undefined; + object_data?: unknown | undefined; + completion?: unknown | undefined; + tags?: (Array | null) | undefined; +} +export interface SSEConsoleEventData { + stream: "stderr" | "stdout"; + message: string; +} +export interface SSEProgressEventData { + id: string; + object_type: FunctionObjectType; + origin?: (ObjectReference | null) | undefined; + format: FunctionFormat; + output_type: FunctionOutputType; + name: string; + event: + | "reasoning_delta" + | "text_delta" + | "json_delta" + | "error" + | "console" + | "start" + | "done" + | "progress"; + data: string; +} +export interface ToolFunctionDefinition { + type: "function"; + function: { + name: string; + description?: string | undefined; + parameters?: {} | undefined; + strict?: (boolean | null) | undefined; + }; +} diff --git a/js/tests/api-compatibility/api-compatibility.test.ts b/js/tests/api-compatibility/api-compatibility.test.ts index 6241d23d6..0b32fb552 100644 --- a/js/tests/api-compatibility/api-compatibility.test.ts +++ b/js/tests/api-compatibility/api-compatibility.test.ts @@ -2983,6 +2983,22 @@ describe("API Compatibility", () => { } }); + test("keeps prompt implementation types out of declaration chunks", () => { + const dist = path.join(__dirname, "..", "..", "dist"); + const declarations = fs + .readdirSync(dist, { recursive: true }) + .filter((file) => /\.d\.(?:ts|mts)$/.test(String(file))) + .map((file) => fs.readFileSync(path.join(dist, String(file)), "utf8")) + .join("\n"); + + expect(declarations).toContain("interface CompiledPromptParams"); + expect(declarations).not.toContain("_internalSerializeForCache"); + expect(declarations).not.toMatch(/generated_(?:plain_)?types/); + expect(declarations).not.toMatch( + /\b(?:FunctionDataType|AnyModelParamsType|PromptType)\b/, + ); + }); + test("should not regress public API surface for all entrypoints", async () => { if (!publishedVersion) { console.log("Skipping test: No published version available"); diff --git a/js/util/git_fields.ts b/js/util/git_fields.ts index 482a43526..a33b07ffe 100644 --- a/js/util/git_fields.ts +++ b/js/util/git_fields.ts @@ -1,4 +1,4 @@ -import type { GitMetadataSettingsType as GitMetadataSettings } from "./generated_plain_types"; +import type { GitMetadataSettings } from "../src/sdk-types"; export function mergeGitMetadataSettings( s1: GitMetadataSettings, diff --git a/js/util/object.ts b/js/util/object.ts index 074aa6229..e34e014da 100644 --- a/js/util/object.ts +++ b/js/util/object.ts @@ -1,7 +1,4 @@ -import type { - AsyncScoringControlType as AsyncScoringControl, - ObjectReferenceType, -} from "./generated_plain_types"; +import type { AsyncScoringControl, ObjectReference } from "../src/sdk-types"; import { Source, ASYNC_SCORING_CONTROL_FIELD, @@ -23,7 +20,7 @@ export type OtherExperimentLogFields = { classifications?: Record; metadata: Record; metrics: Record; - origin: ObjectReferenceType; + origin: ObjectReference; span_attributes: Record; [ASYNC_SCORING_CONTROL_FIELD]: AsyncScoringControl; [MERGE_PATHS_FIELD]: string[][]; @@ -101,7 +98,7 @@ export type DatasetEvent = { tags?: string[]; metadata?: unknown; created?: string; - origin?: ObjectReferenceType; + origin?: ObjectReference; id: string; dataset_id: string; } & ({ expected?: unknown } | { output?: unknown }); diff --git a/js/util/span_identifier_v4.ts b/js/util/span_identifier_v4.ts index 771d72aec..844730365 100644 --- a/js/util/span_identifier_v4.ts +++ b/js/util/span_identifier_v4.ts @@ -20,7 +20,7 @@ import { uint8ArrayToString, } from "./bytes"; import { z } from "zod/v3"; -import type { InvokeFunctionType as InvokeFunctionRequest } from "./generated_plain_types"; +import type { InvokeFunction as InvokeFunctionRequest } from "../src/sdk-types"; import { mergeDicts } from "./object_util"; const ENCODING_VERSION_NUMBER_V4 = 4;