ref!: Don't make generated types part of public api

.changeset/simplify-public-sdk-types.md

@@ -0,0 +1,5 @@
+---
+"braintrust": major
+---
+
+ref!: Don't make generated types part of public api

e2e/helpers/mock-braintrust-server.test.ts

@@ -0,0 +1,61 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { startMockBraintrustServer } from "./mock-braintrust-server";
+
+afterEach(() => vi.restoreAllMocks());
+
+it.each([
+  { path: "/logs3", statuses: [502, 503, 504, 200], succeeds: true },
+  { path: "/logs3", statuses: [502, 502, 502, 502], succeeds: false },
+  { path: "/logs3", statuses: [400], succeeds: false },
+  { path: "/api/project/register", statuses: [502], succeeds: false },
+])(
+  "forwards $path with responses $statuses",
+  async ({ path, statuses, succeeds }) => {
+    const realFetch = globalThis.fetch;
+    const forward = vi.spyOn(globalThis, "fetch");
+    for (const status of statuses) {
+      forward.mockResolvedValueOnce(new Response("{}", { status }));
+    }
+    const server = await startMockBraintrustServer({
+      prodForwarding: {
+        apiKey: "test-forwarding-key",
+        apiUrl: "https://forwarding.invalid",
+        appUrl: "https://forwarding.invalid",
+        orgId: "test-org",
+        orgName: "test-org",
+        projectId: "test-project",
+        projectName: "tmp-luca-forwarding-test",
+      },
+    });
+    const body = JSON.stringify({
+      api_version: 2,
+      rows: [{ id: "test-row", project_id: "test-project" }],
+    });
+    try {
+      const response = await realFetch(`${server.url}${path}`, {
+        method: "POST",
+        body,
+      });
+      await response.text();
+    } finally {
+      if (succeeds) {
+        await server.close();
+      } else {
+        await expect(server.close()).rejects.toThrow(
+          `prodForwarding failed for POST ${path}: ${statuses.at(-1)}`,
+        );
+      }
+    }
+
+    expect(forward).toHaveBeenCalledTimes(statuses.length);
+    for (const [url, init] of forward.mock.calls) {
+      expect(String(url)).toBe(`https://forwarding.invalid${path}`);
+      expect(init?.body).toBe(body);
+      expect(new Headers(init?.headers).get("authorization")).toBe(
+        "Bearer test-forwarding-key",
+      );
+    }
+    expect(server.requests).toHaveLength(1);
+    expect(server.payloads).toHaveLength(path === "/logs3" ? 1 : 0);
+  },
+);

e2e/helpers/mock-braintrust-server.ts

@@ -6,6 +6,7 @@ import type {
   ServerResponse,
 } from "node:http";
 import type { AddressInfo } from "node:net";
+import { setTimeout as sleep } from "node:timers/promises";
 import type { ProdForwarding } from "./prod-forwarding";
 
 export type JsonValue =
@@ -472,29 +473,41 @@ export async function startMockBraintrustServer(
     }
     headers.set("authorization", `Bearer ${prodForwarding.apiKey}`);
 
-    const response = await fetch(url, {
-      body:
-        prodRequest.method === "GET" || prodRequest.method === "HEAD"
-          ? undefined
-          : prodRequest.rawBody,
-      headers,
-      method: prodRequest.method,
-    });
+    // Log rows have stable IDs, so retrying an upload does not duplicate spans.
+    // Keep registration and other requests single-attempt.
+    const maxAttempts = prodRequest.path === "/logs3" ? 4 : 1;
+    for (let attempt = 1; ; attempt++) {
+      const response = await fetch(url, {
+        body:
+          prodRequest.method === "GET" || prodRequest.method === "HEAD"
+            ? undefined
+            : prodRequest.rawBody,
+        headers,
+        method: prodRequest.method,
+      });
 
-    if (!response.ok) {
-      const responseText = await response.text().catch(() => "");
-      throw new Error(
-        `prodForwarding failed for ${capturedRequest.method} ${capturedRequest.path}: ${response.status} ${response.statusText}${
-          responseText ? `: ${responseText.slice(0, 500)}` : ""
-        }`,
-      );
-    }
+      if (!response.ok) {
+        const responseText = await response.text().catch(() => "");
+        if (
+          attempt < maxAttempts &&
+          [502, 503, 504].includes(response.status)
+        ) {
+          await sleep(500 * 2 ** (attempt - 1));
+          continue;
+        }
+        throw new Error(
+          `prodForwarding failed for ${capturedRequest.method} ${capturedRequest.path}: ${response.status} ${response.statusText}${
+            responseText ? `: ${responseText.slice(0, 500)}` : ""
+          }`,
+        );
+      }
 
-    if (options.drainResponseBody) {
-      await response.arrayBuffer();
-    }
+      if (options.drainResponseBody) {
+        await response.arrayBuffer();
+      }
 
-    return response;
+      return response;
+    }
   }
 
   const server = createServer((req, res) => {

e2e/scripts/run-e2e-tests.mjs

@@ -22,7 +22,7 @@ const scenarioArgs = rawArgs.filter((arg) => arg !== "--update");
 const testTargets =
   scenarioArgs.length > 0
     ? scenarioArgs.map((arg) => scenarioPathArg(arg))
-    : await defaultScenarioTestPaths();
+    : [...(await defaultScenarioTestPaths()), "helpers"];
 const vitestArgs = [
   "run",
   "--run",

js/eslint.config.ts

@@ -20,6 +20,17 @@ const entryFiles = tsupConfig.flatMap((config) => {
   return entries;
 });
 
+const generatedTypeImports = {
+  group: [
+    "**/generated_types",
+    "**/generated_plain_types",
+    "**/generated_types.ts",
+    "**/generated_plain_types.ts",
+  ],
+  message:
+    "Generated backend definitions are compatibility-test fixtures. Use SDK-owned types and validators instead.",
+};
+
 export default [
   {
     ignores: [
@@ -161,6 +172,18 @@ export default [
       "no-restricted-properties": "off",
     },
   },
+  {
+    files: ["src/**/*.ts", "src/**/*.tsx", "util/**/*.ts"],
+    languageOptions: {
+      parser: tsparser,
+    },
+    plugins: {
+      "@typescript-eslint": tseslint,
+    },
+    rules: {
+      "no-restricted-imports": ["error", { patterns: [generatedTypeImports] }],
+    },
+  },
   {
     files: ["src/**/*.ts", "src/**/*.tsx"],
     ignores: [...entryFiles, "**/*.test.ts", "**/*.test.tsx"],
@@ -169,6 +192,7 @@ export default [
         "error",
         {
           patterns: [
+            generatedTypeImports,
             {
               group: [
                 "./exports",

js/src/eval-parameters.ts

@@ -5,7 +5,7 @@ import {
   promptDefinitionToPromptData,
   type PromptDefinitionWithTools,
 } from "./prompt-schemas";
-import { PromptData as promptDataSchema } from "./generated_types";
+import { PromptData as promptDataSchema } from "./sdk-schemas";
 
 export type EvalParameters = Record<
   string,

js/src/framework-types.ts

@@ -1,4 +1,4 @@
-import type { IfExistsType as IfExists } from "./generated_plain_types";
+import type { IfExists } from "./sdk-types";
 
 export type GenericFunction<Input, Output> =
   | ((input: Input) => Output)

js/src/framework.ts

@@ -7,13 +7,13 @@ import {
   SpanTypeAttribute,
   spanObjectTypeV3ToTypedString,
 } from "../util/index";
-import { ObjectReference as ObjectReferenceSchema } from "./generated_types";
+import { ObjectReference as ObjectReferenceSchema } from "./sdk-schemas";
 import type {
-  GitMetadataSettingsType as GitMetadataSettings,
-  ObjectReferenceType as ObjectReference,
-  RepoInfoType as RepoInfo,
-  SSEProgressEventDataType as SSEProgressEventData,
-} from "./generated_plain_types";
+  GitMetadataSettings,
+  ObjectReference,
+  RepoInfo,
+  SSEProgressEventData,
+} from "./sdk-types";
 import { queue } from "async";
 
 import iso from "./isomorph";

js/src/framework2.ts

@@ -3,17 +3,16 @@ import type { Trace } from "./trace";
 import iso from "./isomorph";
 import { slugify } from "../util/string_util";
 import { z } from "zod/v3";
-import { Project as projectSchema } from "./generated_types";
+import { Project as projectSchema } from "./sdk-schemas";
 import type {
-  FunctionTypeEnumType as FunctionType,
-  IfExistsType as IfExists,
-  SavedFunctionIdType as SavedFunctionId,
-  PromptBlockDataType as PromptBlockData,
-  PromptDataType as PromptData,
-  ToolFunctionDefinitionType as ToolFunctionDefinition,
-  ExtendedSavedFunctionIdType as ExtendedSavedFunctionId,
-  FunctionDataType,
-} from "./generated_plain_types";
+  FunctionTypeEnum as FunctionType,
+  IfExists,
+  SavedFunctionId,
+  PromptBlockData,
+  PromptData,
+  ToolFunctionDefinition,
+  ExtendedSavedFunctionId,
+} from "./sdk-types";
 import { loadPrettyXact, TransactionId } from "../util/index";
 import {
   _internalGetGlobalState,
@@ -121,7 +120,7 @@ class Project {
     }
     await login();
     const projectMap = new ProjectNameIdMap();
-    const functionDefinitions: FunctionEvent[] = [];
+    const functionDefinitions: PromptFunctionEvent[] = [];
     if (this._publishableCodeFunctions.length > 0) {
       // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage.
       console.warn(
@@ -499,7 +498,7 @@ export class CodePrompt {
 
   async toFunctionDefinition(
     projectNameToId: ProjectNameIdMap,
-  ): Promise<FunctionEvent> {
+  ): Promise<PromptFunctionEvent> {
     const prompt_data = {
       ...this.prompt,
     };
@@ -666,7 +665,7 @@ export class CodeParameters {
 
   async toFunctionDefinition(
     projectNameToId: ProjectNameIdMap,
-  ): Promise<FunctionEvent> {
+  ): Promise<ParametersFunctionEvent> {
     const schema = serializeEvalParameterstoParametersSchema(this.schema);
     return {
       project_id: await projectNameToId.resolve(this.project),
@@ -776,20 +775,32 @@ function getDefaultDataFromParametersSchema(
   );
 }
 
-interface FunctionEvent {
+interface FunctionEventBase {
   project_id: string;
   slug: string;
   name: string;
   description: string;
-  prompt_data?: PromptData;
-  function_data: FunctionDataType;
   function_type?: FunctionType;
   if_exists?: IfExists;
   tags?: string[];
   metadata?: Record<string, unknown>;
   environments?: { slug: string }[];
 }
 
+interface PromptFunctionEvent extends FunctionEventBase {
+  prompt_data: PromptData;
+  function_data: { type: "prompt" };
+}
+
+interface ParametersFunctionEvent extends FunctionEventBase {
+  function_type: "parameters";
+  function_data: {
+    type: "parameters";
+    data: Record<string, unknown>;
+    __schema: ParametersSchema;
+  };
+}
+
 class ProjectNameIdMap {
   private nameToId: Record<string, string> = {};
   private idToName: Record<string, string> = {};

js/src/functions/invoke.ts

@@ -1,10 +1,10 @@
-import { FunctionId as functionIdSchema } from "../generated_types";
+import { FunctionId as functionIdSchema } from "../sdk-schemas";
 import type {
-  InvokeFunctionType as InvokeFunctionRequest,
-  ChatCompletionMessageParamType as Message,
-  StreamingModeType as StreamingMode,
-  FunctionTypeEnumType as FunctionType,
-} from "../generated_plain_types";
+  InvokeFunction as InvokeFunctionRequest,
+  ChatCompletionMessageParam as Message,
+  StreamingMode,
+  FunctionTypeEnum as FunctionType,
+} from "../sdk-types";
 import {
   _internalGetGlobalState,
   BraintrustState,

js/src/functions/stream.ts

@@ -2,12 +2,12 @@ import {
   CallEvent as callEventSchema,
   SSEConsoleEventData as sseConsoleEventDataSchema,
   SSEProgressEventData as sseProgressEventDataSchema,
-} from "../generated_types";
+} from "../sdk-schemas";
 import type {
-  CallEventType as CallEvent,
-  SSEConsoleEventDataType,
-  SSEProgressEventDataType,
-} from "../generated_plain_types";
+  CallEvent,
+  SSEConsoleEventData,
+  SSEProgressEventData,
+} from "../sdk-types";
 import {
   createParser,
   EventSourceParser,
@@ -25,8 +25,8 @@ export type BraintrustStreamChunk =
   | { type: "reasoning_delta"; data: string }
   | { type: "json_delta"; data: string }
   | { type: "error"; data: string }
-  | { type: "console"; data: SSEConsoleEventDataType }
-  | { type: "progress"; data: SSEProgressEventDataType }
+  | { type: "console"; data: SSEConsoleEventData }
+  | { type: "progress"; data: SSEProgressEventData }
   | { type: "start"; data: string }
   | { type: "done"; data: string };
 

js/src/gitutil.ts

@@ -1,7 +1,4 @@
-import type {
-  GitMetadataSettingsType as GitMetadataSettings,
-  RepoInfoType as RepoInfo,
-} from "./generated_plain_types";
+import type { GitMetadataSettings, RepoInfo } from "./sdk-types";
 import { debugLogger } from "./debug-logger";
 import { runGitCommand } from "./git-command";
 

js/src/isomorph.ts

@@ -1,7 +1,4 @@
-import type {
-  GitMetadataSettingsType as GitMetadataSettings,
-  RepoInfoType as RepoInfo,
-} from "./generated_plain_types";
+import type { GitMetadataSettings, RepoInfo } from "./sdk-types";
 import {
   newGlobalTracingChannel,
   type GlobalHookAsyncLocalStorage,