diff --git a/.github/instructions/dotnet-e2e.instructions.md b/.github/instructions/dotnet-e2e.instructions.md new file mode 100644 index 0000000000..8dcf7d5330 --- /dev/null +++ b/.github/instructions/dotnet-e2e.instructions.md @@ -0,0 +1,9 @@ +--- +applyTo: "dotnet/test/E2E/**/*.cs" +--- + +# .NET E2E test instructions + +- Create and resume sessions through `E2ETestContext` using `Ctx.CreateSessionAsync` and `Ctx.ResumeSessionAsync`. Do not call these methods directly on `CopilotClient`; the context applies the backend selected by the E2E matrix while preserving providers explicitly configured by the test. +- Create clients with `Ctx.CreateClient` so they receive the harness environment, CLI path, authentication defaults, transport handling, and lifecycle tracking. +- Instantiate `CopilotClient` directly only when client construction, startup, shutdown, or disposal is the behavior under test. Keep cleanup explicit in those tests, and still create any sessions through `Ctx` so they participate in backend coverage. diff --git a/.github/workflows/dotnet-sdk-tests.yml b/.github/workflows/dotnet-sdk-tests.yml index dcf559228c..ecd5dcd09c 100644 --- a/.github/workflows/dotnet-sdk-tests.yml +++ b/.github/workflows/dotnet-sdk-tests.yml @@ -28,19 +28,35 @@ permissions: jobs: test: - name: ".NET SDK Tests (${{ matrix.os }}, ${{ matrix.transport }})" + name: ".NET SDK Tests (${{ matrix.os }}, ${{ matrix.transport }}, ${{ matrix.backend }})" if: github.event.repository.fork == false env: POWERSHELL_UPDATECHECK: Off + COPILOT_SDK_E2E_BACKEND: ${{ matrix.backend }} + DOTNET_TEST_FILTER: ${{ matrix.test-filter }} strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] transport: ["default", "inprocess"] - # TODO: Re-enable after fixing in-process sqlite file locking on shutdown on Windows + backend: [capi] + # TODO: Re-enable after fixing in-process sqlite file locking on shutdown on Windows. exclude: - os: windows-latest transport: "inprocess" + include: + - os: ubuntu-latest + transport: inprocess + backend: anthropic-messages + test-filter: "FullyQualifiedName~GitHub.Copilot.Test.E2E&E2EBackend!=SelfConfiguredBackend&E2EBackend!=CapiOnly" + - os: ubuntu-latest + transport: inprocess + backend: openai-responses + test-filter: "FullyQualifiedName~GitHub.Copilot.Test.E2E&E2EBackend!=SelfConfiguredBackend&E2EBackend!=CapiOnly" + - os: ubuntu-latest + transport: inprocess + backend: openai-completions + test-filter: "FullyQualifiedName~GitHub.Copilot.Test.E2E&E2EBackend!=SelfConfiguredBackend&E2EBackend!=CapiOnly" runs-on: ${{ matrix.os }} defaults: run: @@ -92,4 +108,9 @@ jobs: - name: Run .NET SDK tests env: COPILOT_HMAC_KEY: ${{ secrets.COPILOT_DEVELOPER_CLI_INTEGRATION_HMAC_KEY }} - run: dotnet test --no-build -v n + run: | + args=(--no-build -v n) + if [[ -n "$DOTNET_TEST_FILTER" ]]; then + args+=(--filter "$DOTNET_TEST_FILTER") + fi + dotnet test "${args[@]}" diff --git a/dotnet/test/AssemblyInfo.cs b/dotnet/test/AssemblyInfo.cs index 9380ca48cd..6f5f258a91 100644 --- a/dotnet/test/AssemblyInfo.cs +++ b/dotnet/test/AssemblyInfo.cs @@ -5,7 +5,7 @@ using Xunit; using GitHub.Copilot.Test.Harness; -// Each E2E test class fixture spins up its own Copilot CLI subprocess plus a CapiProxy +// Each E2E test class fixture spins up its own Copilot CLI subprocess plus a ReplayProxy // (replaying HTTP proxy) Node.js subprocess. With ~25 test classes, running them in parallel // would launch ~50 long-lived Node.js processes simultaneously and exhaust both file // descriptors and memory on developer machines and CI runners (especially Windows). Tests diff --git a/dotnet/test/E2E/ByokBearerTokenProviderE2ETests.cs b/dotnet/test/E2E/ByokBearerTokenProviderE2ETests.cs index 5973dc61c6..4d2cb5e34d 100644 --- a/dotnet/test/E2E/ByokBearerTokenProviderE2ETests.cs +++ b/dotnet/test/E2E/ByokBearerTokenProviderE2ETests.cs @@ -36,6 +36,7 @@ namespace GitHub.Copilot.Test.E2E; /// and the resulting token reaches that provider's endpoint. /// /// +[Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public class ByokBearerTokenProviderE2ETests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "byok_bearer_token_provider", output) { diff --git a/dotnet/test/E2E/ClientE2ETests.cs b/dotnet/test/E2E/ClientE2ETests.cs index 5de6ce159a..d166223d65 100644 --- a/dotnet/test/E2E/ClientE2ETests.cs +++ b/dotnet/test/E2E/ClientE2ETests.cs @@ -9,8 +9,10 @@ namespace GitHub.Copilot.Test.E2E; // These tests bypass E2ETestBase because they are about how the CLI subprocess is started // Other test classes should instead inherit from E2ETestBase -public class ClientE2ETests +public class ClientE2ETests(E2ETestFixture fixture) : IClassFixture { + private E2ETestContext Ctx => fixture.Ctx; + [Theory] [InlineData(true)] // stdio transport [InlineData(false)] // TCP transport @@ -66,7 +68,7 @@ public async Task Should_Force_Stop_Without_Cleanup(bool useStdio) { using var client = new CopilotClient(new CopilotClientOptions { Connection = useStdio ? RuntimeConnection.ForStdio() : RuntimeConnection.ForTcp() }); - await client.CreateSessionAsync(new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll }); + await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll }); await client.ForceStopAsync(); } @@ -165,7 +167,7 @@ public async Task Should_List_Models_When_Authenticated(bool useStdio) public async Task Should_Not_Throw_When_Disposing_Session_After_Stopping_Client(bool useStdio) { await using var client = new CopilotClient(new CopilotClientOptions { Connection = useStdio ? RuntimeConnection.ForStdio() : RuntimeConnection.ForTcp() }); - await using var session = await client.CreateSessionAsync(new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll }); + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll }); await client.StopAsync(); } @@ -201,7 +203,7 @@ public async Task Should_Report_Error_With_Stderr_When_CLI_Fails_To_Start(bool u // Verify subsequent calls also fail (don't hang) var ex2 = await Assert.ThrowsAnyAsync(async () => { - var session = await client.CreateSessionAsync(new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll }); + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll }); await session.SendAsync(new MessageOptions { Prompt = "test" }); }); Assert.True( @@ -219,7 +221,7 @@ public async Task Should_Report_Error_With_Stderr_When_CLI_Fails_To_Start(bool u public async Task Should_Allow_CreateSession_Called_Without_PermissionHandler(bool useStdio) { await using var client = new CopilotClient(new CopilotClientOptions { Connection = useStdio ? RuntimeConnection.ForStdio() : RuntimeConnection.ForTcp() }); - await using var session = await client.CreateSessionAsync(new SessionConfig()); + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig()); Assert.NotNull(session.SessionId); } @@ -234,7 +236,7 @@ public async Task Should_Allow_ResumeSession_Called_Without_PermissionHandler() { Connection = RuntimeConnection.ForTcp(connectionToken: connectionToken), }); - await using var originalSession = await client.CreateSessionAsync(new SessionConfig()); + await using var originalSession = await ctx.CreateSessionAsync(client, new SessionConfig()); var port = client.RuntimePort ?? throw new InvalidOperationException("Client must be using TCP transport to support multi-client resume."); @@ -243,7 +245,7 @@ public async Task Should_Allow_ResumeSession_Called_Without_PermissionHandler() { Connection = RuntimeConnection.ForUri($"localhost:{port}", connectionToken: connectionToken), }); - await using var resumedSession = await resumeClient.ResumeSessionAsync(originalSession.SessionId, new()); + await using var resumedSession = await ctx.ResumeSessionAsync(resumeClient, originalSession.SessionId, new()); Assert.Equal(originalSession.SessionId, resumedSession.SessionId); } diff --git a/dotnet/test/E2E/ClientOptionsE2ETests.cs b/dotnet/test/E2E/ClientOptionsE2ETests.cs index a8ceda5b08..bf995563c4 100644 --- a/dotnet/test/E2E/ClientOptionsE2ETests.cs +++ b/dotnet/test/E2E/ClientOptionsE2ETests.cs @@ -7,6 +7,7 @@ using System.Net; using System.Net.Sockets; using System.Text.Json; +using GitHub.Copilot.Test.Harness; using Xunit; using Xunit.Abstractions; @@ -41,7 +42,7 @@ public async Task Should_Use_Client_Cwd_For_Default_WorkingDirectory() WorkingDirectory = clientCwd, }); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -110,7 +111,7 @@ public async Task Should_Propagate_Process_Options_To_Spawned_Cli() Assert.Equal("dotnet-sdk-e2e", capturedEnv.GetProperty("COPILOT_OTEL_SOURCE_NAME").GetString()); Assert.Equal("true", capturedEnv.GetProperty("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT").GetString()); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { EnableConfigDiscovery = true, IncludeSubAgentStreamingEvents = false, @@ -139,7 +140,7 @@ public async Task Should_Forward_EnableSessionTelemetry_In_Wire_Request() await client.StartAsync(); // When explicitly set to false, it should appear in the wire request - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { EnableSessionTelemetry = false, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -166,7 +167,7 @@ public async Task Should_Omit_EnableSessionTelemetry_When_Not_Set() await client.StartAsync(); // When omitted (null/default), the field should not be present in the wire request - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -191,7 +192,7 @@ public async Task Should_Forward_Granular_Multitenancy_Fields_In_Create_Wire_Req await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { SkipEmbeddingRetrieval = false, OrganizationCustomInstructions = "Follow org policy.", @@ -219,6 +220,7 @@ public async Task Should_Forward_Granular_Multitenancy_Fields_In_Create_Wire_Req } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Forward_Advanced_Session_Options_In_Create_Wire_Request() { var (cliPath, capturePath) = await CreateFakeCliCaptureAsync(); @@ -232,7 +234,7 @@ public async Task Should_Forward_Advanced_Session_Options_In_Create_Wire_Request await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { ClientName = "advanced-create-client", Model = "claude-sonnet-4.5", @@ -366,6 +368,7 @@ public async Task Should_Forward_Advanced_Session_Options_In_Create_Wire_Request } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Forward_Singular_Provider_Options_In_Create_Wire_Request() { var (cliPath, capturePath) = await CreateFakeCliCaptureAsync(); @@ -378,7 +381,7 @@ public async Task Should_Forward_Singular_Provider_Options_In_Create_Wire_Reques await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { Model = "claude-sonnet-4.5", Provider = new ProviderConfig @@ -432,7 +435,7 @@ public async Task Should_Apply_Empty_Mode_Defaults_To_CreateSession_Wire_Request await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, AvailableTools = new ToolSet().AddBuiltIn(BuiltInTools.Isolated), @@ -471,7 +474,7 @@ public async Task Should_Propagate_Activity_TraceContext_To_Session_Create_And_S activity.TraceStateString = "vendor=create-send"; activity.Start(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -555,7 +558,7 @@ public async Task Should_Propagate_Activity_TraceContext_To_Session_Resume() activity.TraceStateString = "vendor=resume"; activity.Start(); - var session = await client.ResumeSessionAsync("trace-resume-session", new ResumeSessionConfig + var session = await Ctx.ResumeSessionAsync(client, "trace-resume-session", new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -582,7 +585,7 @@ public async Task Should_Forward_Granular_Multitenancy_Fields_In_Resume_Wire_Req await client.StartAsync(); - var session = await client.ResumeSessionAsync("resume-session", new ResumeSessionConfig + var session = await Ctx.ResumeSessionAsync(client, "resume-session", new ResumeSessionConfig { SkipEmbeddingRetrieval = false, OrganizationCustomInstructions = "Resume org policy.", @@ -624,7 +627,7 @@ public async Task Should_Forward_Advanced_Session_Options_In_Resume_Wire_Request await client.StartAsync(); - var session = await client.ResumeSessionAsync("advanced-resume-session", new ResumeSessionConfig + var session = await Ctx.ResumeSessionAsync(client, "advanced-resume-session", new ResumeSessionConfig { ClientName = "advanced-resume-client", Model = "claude-haiku-4.5", @@ -706,7 +709,7 @@ public async Task Should_Apply_Empty_Mode_Defaults_To_ResumeSession_Wire_Request await client.StartAsync(); - var session = await client.ResumeSessionAsync("resume-empty-session", new ResumeSessionConfig + var session = await Ctx.ResumeSessionAsync(client, "resume-empty-session", new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, AvailableTools = new ToolSet().AddBuiltIn(BuiltInTools.Isolated), diff --git a/dotnet/test/E2E/CopilotRequestCancelErrorE2ETests.cs b/dotnet/test/E2E/CopilotRequestCancelErrorE2ETests.cs index 99e9e2546e..a9b645d2ac 100644 --- a/dotnet/test/E2E/CopilotRequestCancelErrorE2ETests.cs +++ b/dotnet/test/E2E/CopilotRequestCancelErrorE2ETests.cs @@ -54,7 +54,7 @@ public async Task Reports_A_Thrown_Callback_Error_Instead_Of_Hanging() await using var client = CreateClientWith(handler); await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -81,7 +81,7 @@ public async Task Observes_Runtime_Cancellation_Of_An_In_Flight_Inference_Reques await using var client = CreateClientWith(handler); await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); diff --git a/dotnet/test/E2E/CopilotRequestSessionIdE2ETests.cs b/dotnet/test/E2E/CopilotRequestSessionIdE2ETests.cs index 08dd075643..fd00cc9b99 100644 --- a/dotnet/test/E2E/CopilotRequestSessionIdE2ETests.cs +++ b/dotnet/test/E2E/CopilotRequestSessionIdE2ETests.cs @@ -28,13 +28,14 @@ private CopilotClient CreateClientWith(RecordingRequestHandler provider) => }); [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Threads_The_Session_Id_Into_A_Capi_Session_Inference_Request() { var provider = new RecordingRequestHandler(); await using var client = CreateClientWith(provider); await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -64,13 +65,14 @@ public async Task Threads_The_Session_Id_Into_A_Capi_Session_Inference_Request() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Threads_The_Session_Id_Into_A_Byok_Session_Inference_Request() { var provider = new RecordingRequestHandler(); await using var client = CreateClientWith(provider); await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, // BYOK providers require an explicit model id. diff --git a/dotnet/test/E2E/CopilotRequestWebSocketE2ETests.cs b/dotnet/test/E2E/CopilotRequestWebSocketE2ETests.cs index 9bb19df7d5..80ccdb8c90 100644 --- a/dotnet/test/E2E/CopilotRequestWebSocketE2ETests.cs +++ b/dotnet/test/E2E/CopilotRequestWebSocketE2ETests.cs @@ -31,6 +31,7 @@ namespace GitHub.Copilot.Test.E2E; /// message. Without the eager start the turn never completes and this test /// times out. /// +[Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public class CopilotRequestWebSocketE2ETests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "copilot_request_websocket", output) { @@ -54,7 +55,7 @@ public async Task Services_A_WebSocket_Turn_End_To_End_Via_The_Request_Handler() }, environment: env); await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); diff --git a/dotnet/test/E2E/GitHubTelemetryForwardingE2ETests.cs b/dotnet/test/E2E/GitHubTelemetryForwardingE2ETests.cs index 343c4815b7..80d0ccb0b6 100644 --- a/dotnet/test/E2E/GitHubTelemetryForwardingE2ETests.cs +++ b/dotnet/test/E2E/GitHubTelemetryForwardingE2ETests.cs @@ -12,6 +12,9 @@ namespace GitHub.Copilot.Test.E2E; #pragma warning disable GHCP001 // GitHub telemetry forwarding is experimental. +// TODO(BYOK): Anthropic Messages produced no GitHub telemetry notification. Determine whether +// provider-backed sessions should forward the same telemetry before keeping this CAPI-only. +[Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public class GitHubTelemetryForwardingE2ETests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "github_telemetry", output) { @@ -32,7 +35,7 @@ public async Task Should_Forward_GitHub_Telemetry_For_A_Live_Session() CopilotSession? session = null; try { - session = await client.CreateSessionAsync(new SessionConfig + session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); diff --git a/dotnet/test/E2E/ModeEmptyE2ETests.cs b/dotnet/test/E2E/ModeEmptyE2ETests.cs index df1bbc857d..433e6a745c 100644 --- a/dotnet/test/E2E/ModeEmptyE2ETests.cs +++ b/dotnet/test/E2E/ModeEmptyE2ETests.cs @@ -21,7 +21,7 @@ public class ModeEmptyE2ETests(E2ETestFixture fixture, ITestOutputHelper output) public async Task Empty_Mode_Isolated_Set_Shell_Tool_Is_Not_Exposed() { await using var client = Ctx.CreateClient(options: EmptyModeOptions(Ctx)); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, AvailableTools = new ToolSet().AddBuiltIn(BuiltInTools.Isolated), @@ -45,7 +45,7 @@ public async Task Empty_Mode_Isolated_Set_Shell_Tool_Is_Not_Exposed() public async Task Empty_Mode_Builtin_Star_Exposes_All_Built_In_Tools() { await using var client = Ctx.CreateClient(options: EmptyModeOptions(Ctx)); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, AvailableTools = new ToolSet().AddBuiltIn("*"), @@ -65,7 +65,7 @@ public async Task Empty_Mode_Excluded_Tools_Subtracts_From_Available_Tools() { var shellToolName = OperatingSystem.IsWindows() ? "powershell" : "bash"; await using var client = Ctx.CreateClient(options: EmptyModeOptions(Ctx)); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, AvailableTools = new ToolSet().AddBuiltIn("*"), @@ -85,7 +85,7 @@ public async Task Empty_Mode_Excluded_Tools_Subtracts_From_Available_Tools() public async Task Empty_Mode_Strips_Environment_Context_From_The_System_Message_By_Default() { await using var client = Ctx.CreateClient(options: EmptyModeOptions(Ctx)); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, AvailableTools = new ToolSet().AddBuiltIn(BuiltInTools.Isolated), @@ -109,7 +109,7 @@ public async Task Empty_Mode_Strips_Environment_Context_From_The_System_Message_ public async Task Empty_Mode_System_Message_Replace_Llm_Follows_Caller_Content_Verbatim() { await using var client = Ctx.CreateClient(options: EmptyModeOptions(Ctx)); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, AvailableTools = new ToolSet().AddBuiltIn(BuiltInTools.Isolated), @@ -128,7 +128,7 @@ public async Task Empty_Mode_System_Message_Replace_Llm_Follows_Caller_Content_V public async Task Empty_Mode_Append_Caller_Instruction_Takes_Effect_And_Env_Context_Stripped() { await using var client = Ctx.CreateClient(options: EmptyModeOptions(Ctx)); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, AvailableTools = new ToolSet().AddBuiltIn(BuiltInTools.Isolated), diff --git a/dotnet/test/E2E/ModeHandlersE2ETests.cs b/dotnet/test/E2E/ModeHandlersE2ETests.cs index a397999f73..47e80192e9 100644 --- a/dotnet/test/E2E/ModeHandlersE2ETests.cs +++ b/dotnet/test/E2E/ModeHandlersE2ETests.cs @@ -8,6 +8,7 @@ namespace GitHub.Copilot.Test.E2E; +[Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public class ModeHandlersE2ETests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "mode_handlers", output) { @@ -24,7 +25,7 @@ public async Task Should_Invoke_Exit_Plan_Mode_Handler_When_Model_Uses_Tool() TaskCreationOptions.RunContinuationsAsynchronously); await using var client = CreateAuthenticatedClient(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { GitHubToken = Token, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -92,7 +93,7 @@ public async Task Should_Invoke_Auto_Mode_Switch_Handler_When_Rate_Limited() TaskCreationOptions.RunContinuationsAsynchronously); await using var client = CreateAuthenticatedClient(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { GitHubToken = Token, OnPermissionRequest = PermissionHandler.ApproveAll, diff --git a/dotnet/test/E2E/MultiClientCommandsElicitationE2ETests.cs b/dotnet/test/E2E/MultiClientCommandsElicitationE2ETests.cs index d60c21709c..d869dc8164 100644 --- a/dotnet/test/E2E/MultiClientCommandsElicitationE2ETests.cs +++ b/dotnet/test/E2E/MultiClientCommandsElicitationE2ETests.cs @@ -59,7 +59,7 @@ public async Task InitializeAsync() await Ctx.ConfigureForTestAsync("multi_client", _testName); // Trigger connection so we can read the port - var initSession = await Client1.CreateSessionAsync(new SessionConfig + var initSession = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -102,7 +102,7 @@ public async Task DisposeAsync() [Fact] public async Task Client_Receives_Commands_Changed_When_Another_Client_Joins_With_Commands() { - var session1 = await Client1.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -120,7 +120,7 @@ public async Task Client_Receives_Commands_Changed_When_Another_Client_Joins_Wit }); // Client2 joins with commands - var session2 = await Client2.ResumeSessionAsync(session1.SessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(Client2, session1.SessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Commands = @@ -148,7 +148,7 @@ public async Task Client_Receives_Commands_Changed_When_Another_Client_Joins_Wit public async Task Capabilities_Changed_Fires_When_Second_Client_Joins_With_Elicitation_Handler() { // Client1 creates session without elicitation - var session1 = await Client1.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -168,7 +168,7 @@ public async Task Capabilities_Changed_Fires_When_Second_Client_Joins_With_Elici }); // Client2 joins WITH elicitation handler — triggers capabilities.changed - var session2 = await Client2.ResumeSessionAsync(session1.SessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(Client2, session1.SessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, OnElicitationRequest = _ => Task.FromResult(new ElicitationResult @@ -194,7 +194,7 @@ public async Task Capabilities_Changed_Fires_When_Second_Client_Joins_With_Elici public async Task Capabilities_Changed_Fires_When_Elicitation_Provider_Disconnects() { // Client1 creates session without elicitation - var session1 = await Client1.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -222,7 +222,7 @@ public async Task Capabilities_Changed_Fires_When_Elicitation_Provider_Disconnec }); // Client3 joins WITH elicitation handler - await _client3.ResumeSessionAsync(session1.SessionId, new ResumeSessionConfig + await Ctx.ResumeSessionAsync(_client3, session1.SessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, OnElicitationRequest = _ => Task.FromResult(new ElicitationResult diff --git a/dotnet/test/E2E/MultiClientE2ETests.cs b/dotnet/test/E2E/MultiClientE2ETests.cs index faaf383935..4dbe7190ad 100644 --- a/dotnet/test/E2E/MultiClientE2ETests.cs +++ b/dotnet/test/E2E/MultiClientE2ETests.cs @@ -58,7 +58,7 @@ public async Task InitializeAsync() await Ctx.ConfigureForTestAsync("multi_client", _testName); // Trigger connection so we can read the port - var initSession = await Client1.CreateSessionAsync(new SessionConfig + var initSession = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -96,13 +96,13 @@ public async Task Both_Clients_See_Tool_Request_And_Completion_Events() { var tool = AIFunctionFactory.Create(MagicNumber, "magic_number"); - var session1 = await Client1.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Tools = [tool], }); - var session2 = await Client2.ResumeSessionAsync(session1.SessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(Client2, session1.SessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -148,7 +148,7 @@ public async Task One_Client_Approves_Permission_And_Both_See_The_Result() { var client1PermissionRequests = new List(); - var session1 = await Client1.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = (request, _) => { @@ -158,7 +158,7 @@ public async Task One_Client_Approves_Permission_And_Both_See_The_Result() }); // Client 2 resumes — its handler never completes, so only client 1's approval takes effect - var session2 = await Client2.ResumeSessionAsync(session1.SessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(Client2, session1.SessionId, new ResumeSessionConfig { OnPermissionRequest = (_, _) => new TaskCompletionSource().Task, }); @@ -200,13 +200,13 @@ await session1.SendAsync(new MessageOptions [Fact] public async Task One_Client_Rejects_Permission_And_Both_See_The_Result() { - var session1 = await Client1.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = (_, _) => Task.FromResult(PermissionDecision.Reject()), }); // Client 2 resumes — its handler never completes - var session2 = await Client2.ResumeSessionAsync(session1.SessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(Client2, session1.SessionId, new ResumeSessionConfig { OnPermissionRequest = (_, _) => new TaskCompletionSource().Task, }); @@ -252,13 +252,13 @@ public async Task Two_Clients_Register_Different_Tools_And_Agent_Uses_Both() var toolA = AIFunctionFactory.Create(CityLookup, "city_lookup"); var toolB = AIFunctionFactory.Create(CurrencyLookup, "currency_lookup"); - var session1 = await Client1.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Tools = [toolA], }); - var session2 = await Client2.ResumeSessionAsync(session1.SessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(Client2, session1.SessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Tools = [toolB], @@ -294,13 +294,13 @@ public async Task Disconnecting_Client_Removes_Its_Tools() var toolA = AIFunctionFactory.Create(StableTool, "stable_tool"); var toolB = AIFunctionFactory.Create(EphemeralTool, "ephemeral_tool"); - var session1 = await Client1.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Tools = [toolA], }); - await Client2.ResumeSessionAsync(session1.SessionId, new ResumeSessionConfig + await Ctx.ResumeSessionAsync(Client2, session1.SessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Tools = [toolB], diff --git a/dotnet/test/E2E/MultiProviderRegistryE2ETests.cs b/dotnet/test/E2E/MultiProviderRegistryE2ETests.cs index 8a75cc3c1a..80827cc867 100644 --- a/dotnet/test/E2E/MultiProviderRegistryE2ETests.cs +++ b/dotnet/test/E2E/MultiProviderRegistryE2ETests.cs @@ -18,6 +18,7 @@ namespace GitHub.Copilot.Test.E2E; /// session, be launched, and route inference to the configured provider with /// the configured wire model and headers. /// +[Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public class MultiProviderRegistryE2ETests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "multi_provider_registry", output) { diff --git a/dotnet/test/E2E/PendingWorkResumeE2ETests.cs b/dotnet/test/E2E/PendingWorkResumeE2ETests.cs index b330795905..b3ca218190 100644 --- a/dotnet/test/E2E/PendingWorkResumeE2ETests.cs +++ b/dotnet/test/E2E/PendingWorkResumeE2ETests.cs @@ -30,7 +30,7 @@ public async Task Should_Continue_Pending_Permission_Request_After_Resume() var cliUrl = GetCliUrl(server); using var suspendedClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) }); - var session1 = await suspendedClient.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(suspendedClient, new SessionConfig { Tools = [AIFunctionFactory.Create(ResumePermissionTool, "resume_permission_tool")], OnPermissionRequest = (request, _) => @@ -57,7 +57,7 @@ await session1.SendAsync(new MessageOptions await suspendedClient.ForceStopAsync(); await using var resumedTcpClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) }); - var session2 = await resumedTcpClient.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(resumedTcpClient, sessionId, new ResumeSessionConfig { ContinuePendingWork = true, OnPermissionRequest = (_, _) => Task.FromResult(PermissionDecision.NoResult()), @@ -99,7 +99,7 @@ public async Task Should_Continue_Pending_External_Tool_Request_After_Resume() var cliUrl = GetCliUrl(server); using var suspendedClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) }); - var session1 = await suspendedClient.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(suspendedClient, new SessionConfig { Tools = [AIFunctionFactory.Create(BlockingExternalTool, "resume_external_tool")], OnPermissionRequest = PermissionHandler.ApproveAll, @@ -120,7 +120,7 @@ await session1.SendAsync(new MessageOptions await suspendedClient.ForceStopAsync(); await using var resumedClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) }); - var session2 = await resumedClient.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(resumedClient, sessionId, new ResumeSessionConfig { ContinuePendingWork = true, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -175,7 +175,7 @@ private async Task AssertPendingExternalToolHandleableOnResumeAsync( var cliUrl = GetCliUrl(server); using var suspendedClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) }); - var session1 = await suspendedClient.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(suspendedClient, new SessionConfig { Tools = [AIFunctionFactory.Create(BlockingExternalTool, "resume_external_tool")], OnPermissionRequest = PermissionHandler.ApproveAll, @@ -216,7 +216,7 @@ await session1.SendAsync(new MessageOptions resumeConfig.Tools = [AIFunctionFactory.Create(ResumedExternalTool, "resume_external_tool")]; } - var session2 = await resumedClient.ResumeSessionAsync(sessionId, resumeConfig); + var session2 = await Ctx.ResumeSessionAsync(resumedClient, sessionId, resumeConfig); var resumeEvent = await GetSingleResumeEventAsync(session2); Assert.Equal(false, resumeEvent.Data.ContinuePendingWork); @@ -276,7 +276,7 @@ public async Task Should_Continue_Parallel_Pending_External_Tool_Requests_After_ var cliUrl = GetCliUrl(server); using var suspendedClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) }); - var session1 = await suspendedClient.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(suspendedClient, new SessionConfig { Tools = [ @@ -306,7 +306,7 @@ await Task.WhenAll( await suspendedClient.ForceStopAsync(); await using var resumedClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) }); - var session2 = await resumedClient.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(resumedClient, sessionId, new ResumeSessionConfig { ContinuePendingWork = true, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -357,7 +357,7 @@ public async Task Should_Resume_Successfully_When_No_Pending_Work_Exists() string sessionId; await using (var firstClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) })) { - var firstSession = await firstClient.CreateSessionAsync(new SessionConfig + var firstSession = await Ctx.CreateSessionAsync(firstClient, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -370,7 +370,7 @@ public async Task Should_Resume_Successfully_When_No_Pending_Work_Exists() } await using var resumedClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) }); - var resumedSession = await resumedClient.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var resumedSession = await Ctx.ResumeSessionAsync(resumedClient, sessionId, new ResumeSessionConfig { ContinuePendingWork = true, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -395,7 +395,7 @@ public async Task Should_Report_ContinuePendingWork_True_In_Resume_Event() string sessionId; await using (var firstClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) })) { - var firstSession = await firstClient.CreateSessionAsync(new SessionConfig + var firstSession = await Ctx.CreateSessionAsync(firstClient, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -411,7 +411,7 @@ public async Task Should_Report_ContinuePendingWork_True_In_Resume_Event() } await using var resumedClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) }); - var resumedSession = await resumedClient.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var resumedSession = await Ctx.ResumeSessionAsync(resumedClient, sessionId, new ResumeSessionConfig { ContinuePendingWork = true, OnPermissionRequest = PermissionHandler.ApproveAll, diff --git a/dotnet/test/E2E/PerSessionAuthE2ETests.cs b/dotnet/test/E2E/PerSessionAuthE2ETests.cs index d1226f3737..4b370768a1 100644 --- a/dotnet/test/E2E/PerSessionAuthE2ETests.cs +++ b/dotnet/test/E2E/PerSessionAuthE2ETests.cs @@ -8,6 +8,7 @@ namespace GitHub.Copilot.Test.E2E; +[Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public class PerSessionAuthE2ETests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "per-session-auth", output) { /// @@ -74,7 +75,7 @@ public async Task ShouldAuthenticateWithGitHubToken() { await SetupCopilotUsersAsync(); - await using var session = await AuthClient.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(AuthClient, new SessionConfig { GitHubToken = "token-alice", OnPermissionRequest = PermissionHandler.ApproveAll, @@ -90,13 +91,13 @@ public async Task ShouldIsolateAuthBetweenSessions() { await SetupCopilotUsersAsync(); - await using var sessionA = await AuthClient.CreateSessionAsync(new SessionConfig + await using var sessionA = await Ctx.CreateSessionAsync(AuthClient, new SessionConfig { GitHubToken = "token-alice", OnPermissionRequest = PermissionHandler.ApproveAll, }); - await using var sessionB = await AuthClient.CreateSessionAsync(new SessionConfig + await using var sessionB = await Ctx.CreateSessionAsync(AuthClient, new SessionConfig { GitHubToken = "token-bob", OnPermissionRequest = PermissionHandler.ApproveAll, @@ -116,7 +117,7 @@ public async Task ShouldBeUnauthenticatedWithoutToken() { var noAuthClient = CreateNoAuthTestClient(); - await using var session = await noAuthClient.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(noAuthClient, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -131,7 +132,7 @@ public async Task ShouldFailWithInvalidToken() { await SetupCopilotUsersAsync(); - var ex = await Assert.ThrowsAnyAsync(() => AuthClient.CreateSessionAsync(new SessionConfig + var ex = await Assert.ThrowsAnyAsync(() => Ctx.CreateSessionAsync(AuthClient, new SessionConfig { GitHubToken = "invalid-token", OnPermissionRequest = PermissionHandler.ApproveAll, diff --git a/dotnet/test/E2E/PermissionE2ETests.cs b/dotnet/test/E2E/PermissionE2ETests.cs index 8c1904701a..2225dcba89 100644 --- a/dotnet/test/E2E/PermissionE2ETests.cs +++ b/dotnet/test/E2E/PermissionE2ETests.cs @@ -205,7 +205,7 @@ public async Task Should_Resume_Session_With_Permission_Handler() await session1.DisposeAsync(); // Resume with permission handler - var session2 = await Client.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(Client, sessionId, new ResumeSessionConfig { OnPermissionRequest = (request, invocation) => { @@ -279,7 +279,7 @@ public async Task Should_Deny_Tool_Operations_When_Handler_Explicitly_Denies_Aft await session1.SendAndWaitAsync(new MessageOptions { Prompt = "What is 1+1?" }); await session1.DisposeAsync(); - var session2 = await Client.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(Client, sessionId, new ResumeSessionConfig { OnPermissionRequest = (_, _) => Task.FromResult(PermissionDecision.UserNotAvailable()) diff --git a/dotnet/test/E2E/ProviderEndpointE2ETests.cs b/dotnet/test/E2E/ProviderEndpointE2ETests.cs index 7ea4eecf39..d2bf06982f 100644 --- a/dotnet/test/E2E/ProviderEndpointE2ETests.cs +++ b/dotnet/test/E2E/ProviderEndpointE2ETests.cs @@ -27,11 +27,12 @@ private CopilotClient CreateProviderEndpointClient() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task ShouldReturnByokProviderEndpointWhenCustomProviderIsConfigured() { var client = CreateProviderEndpointClient(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Provider = new ProviderConfig @@ -64,11 +65,12 @@ public async Task ShouldReturnByokProviderEndpointWhenCustomProviderIsConfigured } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task ShouldReturnCapiProviderEndpointForOAuthAuthenticatedSession() { var client = CreateProviderEndpointClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); diff --git a/dotnet/test/E2E/RpcExtensionsLoadedE2ETests.cs b/dotnet/test/E2E/RpcExtensionsLoadedE2ETests.cs index af959bd7eb..0a3513e4b5 100644 --- a/dotnet/test/E2E/RpcExtensionsLoadedE2ETests.cs +++ b/dotnet/test/E2E/RpcExtensionsLoadedE2ETests.cs @@ -189,7 +189,7 @@ public async Task Discovers_Loads_And_Reports_Running_Extension(string sourceVal await using var client = CreateExtensionsClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { EnableConfigDiscovery = true, WorkingDirectory = workingDirectory, @@ -214,7 +214,7 @@ public async Task Disable_Then_Enable_Cycles_Extension_Status() await using var client = CreateExtensionsClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { EnableConfigDiscovery = true, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -240,7 +240,7 @@ public async Task Reload_Picks_Up_Extension_Added_After_Session_Create() // Start the session BEFORE writing the extension so the initial discovery sees nothing. await using var client = CreateExtensionsClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { EnableConfigDiscovery = true, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -285,7 +285,7 @@ public async Task Failed_Extension_Reports_Failed_Status() await using var client = CreateExtensionsClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { EnableConfigDiscovery = true, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -306,7 +306,7 @@ public async Task Multiple_Extensions_Are_Discovered_Independently() await using var client = CreateExtensionsClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { EnableConfigDiscovery = true, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -328,7 +328,7 @@ public async Task Reload_Preserves_Disabled_State_Across_Calls() await using var client = CreateExtensionsClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { EnableConfigDiscovery = true, OnPermissionRequest = PermissionHandler.ApproveAll, diff --git a/dotnet/test/E2E/RpcMcpAndSkillsE2ETests.cs b/dotnet/test/E2E/RpcMcpAndSkillsE2ETests.cs index 350aac4274..0d2942d4bf 100644 --- a/dotnet/test/E2E/RpcMcpAndSkillsE2ETests.cs +++ b/dotnet/test/E2E/RpcMcpAndSkillsE2ETests.cs @@ -189,7 +189,7 @@ public async Task Should_List_Extensions() { Connection = RuntimeConnection.ForStdio(args: ["--yolo"]), }); - await using var session = await yoloClient.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(yoloClient, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -208,7 +208,7 @@ public async Task Should_List_Extensions() public async Task Should_Round_Trip_Mcp_App_Host_Context() { await using var client = CreateMcpAppsClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -253,7 +253,7 @@ public async Task Should_Diagnose_And_Report_Mcp_App_Capability_Errors() new Dictionary { ["MCP_APP_RPC_VALUE"] = "from-app-rpc" }; await using var client = CreateMcpAppsClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { McpServers = mcpServers, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -292,7 +292,7 @@ public async Task Should_Report_Error_When_Mcp_App_Resource_Is_Not_Available() { const string serverName = "rpc-apps-resource-server"; await using var client = CreateMcpAppsClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { McpServers = CreateTestMcpServers(serverName), OnPermissionRequest = PermissionHandler.ApproveAll, @@ -368,7 +368,7 @@ public async Task Should_Report_Error_When_Extensions_Are_Not_Available() { Connection = RuntimeConnection.ForStdio(args: ["--yolo"]), }); - await using var session = await yoloClient.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(yoloClient, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); diff --git a/dotnet/test/E2E/RpcServerE2ETests.cs b/dotnet/test/E2E/RpcServerE2ETests.cs index 47df256157..2df8593cc4 100644 --- a/dotnet/test/E2E/RpcServerE2ETests.cs +++ b/dotnet/test/E2E/RpcServerE2ETests.cs @@ -160,6 +160,7 @@ public async Task Should_Reject_Llm_Inference_Response_Frames_For_Missing_Reques } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Call_Rpc_Models_List_With_Typed_Result() { const string token = "rpc-models-token"; @@ -175,6 +176,7 @@ public async Task Should_Call_Rpc_Models_List_With_Typed_Result() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Call_Rpc_Account_GetQuota_When_Authenticated() { const string token = "rpc-quota-token"; @@ -256,7 +258,7 @@ public async Task Should_List_Find_And_Inspect_Persisted_Session_State() var missingTaskId = $"missing-task-{Guid.NewGuid():N}"; var missingSessionId = Guid.NewGuid().ToString(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { SessionId = sessionId, WorkingDirectory = workingDirectory, @@ -308,7 +310,7 @@ public async Task Should_Enrich_Basic_Session_Metadata() var sessionId = Guid.NewGuid().ToString(); var workingDirectory = CreateUniqueWorkDirectory("server-rpc-enrich"); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { SessionId = sessionId, WorkingDirectory = workingDirectory, @@ -353,7 +355,7 @@ public async Task Should_Close_Active_Session_And_Release_Lock() await using var client = CreateAuthenticatedClient(token); var sessionId = Guid.NewGuid().ToString(); var workingDirectory = CreateUniqueWorkDirectory("server-rpc-close"); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { SessionId = sessionId, WorkingDirectory = workingDirectory, @@ -378,7 +380,7 @@ public async Task Should_Check_In_Use_Session_From_Another_Runtime_And_Release_L var sessionId = Guid.NewGuid().ToString(); var workingDirectory = CreateUniqueWorkDirectory("server-rpc-in-use"); await using var otherClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForStdio() }); - await using var otherSession = await otherClient.CreateSessionAsync(new SessionConfig + await using var otherSession = await Ctx.CreateSessionAsync(otherClient, new SessionConfig { SessionId = sessionId, WorkingDirectory = workingDirectory, @@ -419,7 +421,7 @@ public async Task Should_Prune_DryRun_And_BulkDelete_Persisted_Session() var missingSessionId = Guid.NewGuid().ToString(); var workingDirectory = CreateUniqueWorkDirectory("server-rpc-delete"); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { SessionId = sessionId, WorkingDirectory = workingDirectory, diff --git a/dotnet/test/E2E/RpcSessionStateE2ETests.cs b/dotnet/test/E2E/RpcSessionStateE2ETests.cs index 93af399203..dfd76fb34f 100644 --- a/dotnet/test/E2E/RpcSessionStateE2ETests.cs +++ b/dotnet/test/E2E/RpcSessionStateE2ETests.cs @@ -46,7 +46,7 @@ public async Task Should_Call_Session_Rpc_Model_SwitchTo() await isolatedCtx.ConfigureForTestAsync("rpc_session_state", nameof(Should_Call_Session_Rpc_Model_SwitchTo)); var isolatedClient = isolatedCtx.CreateClient(); - await using var session = await isolatedClient.CreateSessionAsync(new SessionConfig + await using var session = await isolatedCtx.CreateSessionAsync(isolatedClient, new SessionConfig { Model = "claude-sonnet-4.5", OnPermissionRequest = PermissionHandler.ApproveAll, @@ -443,7 +443,7 @@ public async Task Should_Set_ReasoningEffort_And_Auto_Name() public async Task Should_Set_Auth_Credentials() { await using var client = Ctx.CreateClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); diff --git a/dotnet/test/E2E/RpcSessionStateExtrasE2ETests.cs b/dotnet/test/E2E/RpcSessionStateExtrasE2ETests.cs index f3f3d5d5de..28ec9b7cfe 100644 --- a/dotnet/test/E2E/RpcSessionStateExtrasE2ETests.cs +++ b/dotnet/test/E2E/RpcSessionStateExtrasE2ETests.cs @@ -20,6 +20,7 @@ public class RpcSessionStateExtrasE2ETests(E2ETestFixture fixture, ITestOutputHe : E2ETestBase(fixture, "rpc_session_state_extras", output) { [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_List_Models_For_Session() { // model.list resolves models through the session's own auth context, which requires the @@ -29,7 +30,7 @@ public async Task Should_List_Models_For_Session() const string token = "rpc-session-model-list-token"; await ConfigureAuthenticatedUserAsync(token); await using var client = CreateAuthenticatedClient(token); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { Model = "claude-sonnet-4.5", OnPermissionRequest = PermissionHandler.ApproveAll, @@ -44,6 +45,7 @@ public async Task Should_List_Models_For_Session() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Add_Byok_Provider_And_Model_At_Runtime() { await using var session = await CreateSessionAsync(); diff --git a/dotnet/test/E2E/RpcTasksAndHandlersE2ETests.cs b/dotnet/test/E2E/RpcTasksAndHandlersE2ETests.cs index 9b9738a2ee..989fef7c55 100644 --- a/dotnet/test/E2E/RpcTasksAndHandlersE2ETests.cs +++ b/dotnet/test/E2E/RpcTasksAndHandlersE2ETests.cs @@ -72,6 +72,9 @@ await AssertImplementedFailureAsync( } [Fact] + // TODO(BYOK): Provider-backed task agents handled an invalid model differently. Verify that + // BYOK model validation should reject it consistently before keeping this CAPI-only. + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Report_Implemented_Error_For_Invalid_Task_Agent_Model() { var session = await CreateSessionAsync(); diff --git a/dotnet/test/E2E/SessionConfigE2ETests.cs b/dotnet/test/E2E/SessionConfigE2ETests.cs index 45cdef2016..ad313b116d 100644 --- a/dotnet/test/E2E/SessionConfigE2ETests.cs +++ b/dotnet/test/E2E/SessionConfigE2ETests.cs @@ -22,6 +22,9 @@ public class SessionConfigE2ETests(E2ETestFixture fixture, ITestOutputHelper out "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="); [Fact] + // TODO(BYOK): Anthropic Messages history diverged after enabling vision via SetModel. Verify + // that model capability overrides work for provider-backed sessions before keeping this CAPI-only. + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Vision_Disabled_Then_Enabled_Via_SetModel() { await File.WriteAllBytesAsync(Path.Join(Ctx.WorkDir, "test.png"), Png1X1); @@ -62,6 +65,9 @@ await session.SetModelAsync( } [Fact] + // TODO(BYOK): Anthropic Messages history diverged after disabling vision via SetModel. Verify + // that model capability overrides work for provider-backed sessions before keeping this CAPI-only. + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Vision_Enabled_Then_Disabled_Via_SetModel() { await File.WriteAllBytesAsync(Path.Join(Ctx.WorkDir, "test.png"), Png1X1); @@ -121,6 +127,7 @@ public async Task Should_Use_Custom_SessionId() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Apply_ReasoningEffort_On_Session_Create() { const string reasoningModelId = "custom-reasoning-model"; @@ -140,6 +147,7 @@ public async Task Should_Apply_ReasoningEffort_On_Session_Create() } [Theory] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] [InlineData("low")] [InlineData("medium")] [InlineData("high")] @@ -162,6 +170,7 @@ public async Task Should_Apply_All_ReasoningEffort_Values_On_Session_Create(stri } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Apply_ReasoningEffort_On_Session_Resume() { var originalSession = await CreateSessionAsync(); @@ -182,6 +191,9 @@ public async Task Should_Apply_ReasoningEffort_On_Session_Resume() } [Fact] + // TODO(BYOK): The Anthropic user-agent omitted ClientName and contained only its provider SDK + // identifier. Determine the expected propagation for custom providers before keeping this CAPI-only. + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Forward_ClientName_In_UserAgent() { var session = await CreateSessionAsync(new SessionConfig @@ -198,6 +210,7 @@ public async Task Should_Forward_ClientName_In_UserAgent() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Forward_Custom_Provider_Headers_On_Create() { var session = await CreateSessionAsync(new SessionConfig @@ -217,6 +230,7 @@ public async Task Should_Forward_Custom_Provider_Headers_On_Create() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Forward_Custom_Provider_Headers_On_Resume() { var session1 = await CreateSessionAsync(); @@ -239,6 +253,7 @@ public async Task Should_Forward_Custom_Provider_Headers_On_Resume() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Forward_Provider_Wire_Model() { // Verifies that ProviderConfig.WireModel overrides the model name sent to @@ -270,6 +285,7 @@ public async Task Should_Forward_Provider_Wire_Model() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Use_Provider_Model_Id_As_Wire_Model() { // ProviderConfig.ModelId drives both the runtime resolved model AND the wire model @@ -564,13 +580,14 @@ public async Task Should_Apply_Excluded_Built_In_Agents_On_Resume() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Enable_Citations_For_Anthropic_File_Attachments_On_Create() { var handler = new RecordingRequestHandler(); await using var client = CreateClientWithRequestHandler(handler); await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Model = "claude-sonnet-4.5", @@ -595,6 +612,7 @@ await session.SendAndWaitAsync(new MessageOptions } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Enable_Citations_For_Anthropic_File_Attachments_On_Resume() { const string connectionToken = "citation-resume-token"; @@ -604,7 +622,7 @@ public async Task Should_Enable_Citations_For_Anthropic_File_Attachments_On_Resu RuntimeConnection.ForTcp(connectionToken: connectionToken)); await client.StartAsync(); - var session1 = await client.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -616,7 +634,7 @@ public async Task Should_Enable_Citations_For_Anthropic_File_Attachments_On_Resu Connection = RuntimeConnection.ForUri($"localhost:{port}", connectionToken: connectionToken), }); - var session2 = await resumeClient.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(resumeClient, sessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Model = "claude-sonnet-4.5", @@ -642,6 +660,7 @@ await session2.SendAndWaitAsync(new MessageOptions } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Create_Session_With_Custom_Provider_Config() { // Per the TS test (session_config.e2e.test.ts), this only verifies that a @@ -669,6 +688,9 @@ public async Task Should_Create_Session_With_Custom_Provider_Config() } [Fact] + // TODO(BYOK): Anthropic Messages request history diverged while replaying this blob attachment. + // Confirm native clients preserve blob/image turns before keeping this CAPI-only. + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Accept_Blob_Attachments() { // Write the image to disk so the model can view it if it tries diff --git a/dotnet/test/E2E/SessionE2ETests.cs b/dotnet/test/E2E/SessionE2ETests.cs index bcfb46295a..bcc8fc268d 100644 --- a/dotnet/test/E2E/SessionE2ETests.cs +++ b/dotnet/test/E2E/SessionE2ETests.cs @@ -232,7 +232,7 @@ public async Task Should_Reject_Resuming_Active_Session_Using_The_Same_Client() var sessionId = session1.SessionId; var exception = await Assert.ThrowsAsync(() => - Client.ResumeSessionAsync(sessionId, new ResumeSessionConfig + Ctx.ResumeSessionAsync(Client, sessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, })); @@ -251,7 +251,7 @@ public async Task Should_Resume_A_Session_Using_A_New_Client() Assert.Contains("2", answer!.Data.Content ?? string.Empty); using var newClient = Ctx.CreateClient(); - var session2 = await newClient.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(newClient, sessionId, new ResumeSessionConfig { ContinuePendingWork = true, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -287,7 +287,7 @@ public async Task Resumes_A_Persisted_Session_From_A_New_Client_When_An_Mcp_OAut Assert.Contains("2", answer!.Data.Content ?? string.Empty); using var newClient = Ctx.CreateClient(); - await using var session2 = await newClient.ResumeSessionAsync(sessionId, new ResumeSessionConfig + await using var session2 = await Ctx.ResumeSessionAsync(newClient, sessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, OnMcpAuthRequest = CancelMcpAuthAsync, @@ -732,6 +732,9 @@ public async Task DisposeAsync_From_Handler_Does_Not_Deadlock() } [Fact] + // TODO(BYOK): Anthropic Messages request history diverged while replaying this blob attachment. + // Confirm native clients preserve blob/image turns before keeping this CAPI-only. + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Accept_Blob_Attachments() { var pngBase64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="; @@ -922,6 +925,7 @@ await session.SendAndWaitAsync(new MessageOptions } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Create_Session_With_Custom_Provider() { var session = await CreateSessionAsync(new SessionConfig @@ -947,6 +951,7 @@ public async Task Should_Create_Session_With_Custom_Provider() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Create_Session_With_Azure_Provider() { var session = await CreateSessionAsync(new SessionConfig @@ -976,6 +981,7 @@ public async Task Should_Create_Session_With_Azure_Provider() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Resume_Session_With_Custom_Provider() { var session = await CreateSessionAsync(); diff --git a/dotnet/test/E2E/SessionFsE2ETests.cs b/dotnet/test/E2E/SessionFsE2ETests.cs index b1b4848adc..cf91e3ddc8 100644 --- a/dotnet/test/E2E/SessionFsE2ETests.cs +++ b/dotnet/test/E2E/SessionFsE2ETests.cs @@ -28,7 +28,7 @@ public async Task Should_Route_File_Operations_Through_The_Session_Fs_Provider() { await using var client = CreateSessionFsClient(providerRoot); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = s => new TestSessionFsHandler(s.SessionId, providerRoot), @@ -58,7 +58,7 @@ public async Task Should_Load_Session_Data_From_Fs_Provider_On_Resume() await using var client = CreateSessionFsClient(providerRoot); Func createSessionFsHandler = s => new TestSessionFsHandler(s.SessionId, providerRoot); - var session1 = await client.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = createSessionFsHandler, @@ -72,7 +72,7 @@ public async Task Should_Load_Session_Data_From_Fs_Provider_On_Resume() var eventsPath = GetStoredPath(providerRoot, sessionId, $"{SessionFsConfig.SessionStatePath}/events.jsonl"); await WaitForConditionAsync(() => File.Exists(eventsPath)); - var session2 = await client.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(client, sessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = createSessionFsHandler, @@ -97,7 +97,7 @@ public async Task Should_Reject_SetProvider_When_Sessions_Already_Exist() await using var client1 = CreateSessionFsClient(providerRoot, useStdio: false, tcpConnectionToken: "session-fs-shared-token"); var createSessionFsHandler = (Func)(s => new TestSessionFsHandler(s.SessionId, providerRoot)); - _ = await client1.CreateSessionAsync(new SessionConfig + _ = await Ctx.CreateSessionAsync(client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = createSessionFsHandler, @@ -319,7 +319,7 @@ public async Task Should_Map_Large_Output_Handling_Into_SessionFs() var suppliedFileContent = new string('x', largeContentSize); await using var client = CreateSessionFsClient(providerRoot); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = s => new TestSessionFsHandler(s.SessionId, providerRoot), @@ -361,7 +361,7 @@ public async Task Should_Succeed_With_Compaction_While_Using_SessionFs() try { await using var client = CreateSessionFsClient(providerRoot); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = s => new TestSessionFsHandler(s.SessionId, providerRoot), @@ -400,7 +400,7 @@ public async Task Should_Write_Workspace_Metadata_Via_SessionFs() try { await using var client = CreateSessionFsClient(providerRoot); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = s => new TestSessionFsHandler(s.SessionId, providerRoot), @@ -433,7 +433,7 @@ public async Task Should_Persist_Plan_Md_Via_SessionFs() try { await using var client = CreateSessionFsClient(providerRoot); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = s => new TestSessionFsHandler(s.SessionId, providerRoot), diff --git a/dotnet/test/E2E/SessionFsSqliteE2ETests.cs b/dotnet/test/E2E/SessionFsSqliteE2ETests.cs index bf7bdf3b5e..8ed86c72e3 100644 --- a/dotnet/test/E2E/SessionFsSqliteE2ETests.cs +++ b/dotnet/test/E2E/SessionFsSqliteE2ETests.cs @@ -31,7 +31,7 @@ public async Task Should_Route_Sql_Queries_Through_The_Sessionfs_Sqlite_Handler( { await using var client = CreateSessionFsClient(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = s => new InMemorySessionFsSqliteHandler(s.SessionId, _sqliteCalls), @@ -65,7 +65,7 @@ public async Task Should_Allow_Subagents_To_Use_Sql_Tool_Via_Inherited_Sessionfs await using var client = CreateSessionFsClient(); var handler = (InMemorySessionFsSqliteHandler?)null; - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = s => diff --git a/dotnet/test/E2E/StreamingFidelityE2ETests.cs b/dotnet/test/E2E/StreamingFidelityE2ETests.cs index fec00f1cb1..4df9ca4420 100644 --- a/dotnet/test/E2E/StreamingFidelityE2ETests.cs +++ b/dotnet/test/E2E/StreamingFidelityE2ETests.cs @@ -2,6 +2,7 @@ * Copyright (c) Microsoft Corporation. All rights reserved. *--------------------------------------------------------------------------------------------*/ +using GitHub.Copilot.Test.Harness; using Xunit; using Xunit.Abstractions; @@ -46,6 +47,9 @@ public async Task Should_Produce_Delta_Events_When_Streaming_Is_Enabled() } [Fact] + // TODO(BYOK): Anthropic Messages emitted delta events with Streaming=false. Investigate the + // native-client streaming contract before keeping this disabled for every BYOK backend. + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Not_Produce_Deltas_When_Streaming_Is_Disabled() { var session = await CreateSessionAsync(new SessionConfig { Streaming = false }); @@ -79,7 +83,7 @@ public async Task Should_Produce_Deltas_After_Session_Resume() // Resume using a new client using var newClient = Ctx.CreateClient(); - var session2 = await newClient.ResumeSessionAsync(session.SessionId, + var session2 = await Ctx.ResumeSessionAsync(newClient, session.SessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Streaming = true }); var events = new List(); @@ -106,6 +110,9 @@ public async Task Should_Produce_Deltas_After_Session_Resume() } [Fact] + // TODO(BYOK): Anthropic Messages emitted delta events after resuming with Streaming=false. + // Investigate the native-client streaming contract before keeping this disabled for every BYOK backend. + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Not_Produce_Deltas_After_Session_Resume_With_Streaming_Disabled() { var session = await CreateSessionAsync(new SessionConfig { Streaming = true }); @@ -114,7 +121,7 @@ public async Task Should_Not_Produce_Deltas_After_Session_Resume_With_Streaming_ // Resume using a new client with streaming DISABLED using var newClient = Ctx.CreateClient(); - var session2 = await newClient.ResumeSessionAsync(session.SessionId, + var session2 = await Ctx.ResumeSessionAsync(newClient, session.SessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Streaming = false }); var events = new List(); diff --git a/dotnet/test/E2E/SubagentHooksE2ETests.cs b/dotnet/test/E2E/SubagentHooksE2ETests.cs index a718c56c1a..8314b8c098 100644 --- a/dotnet/test/E2E/SubagentHooksE2ETests.cs +++ b/dotnet/test/E2E/SubagentHooksE2ETests.cs @@ -30,7 +30,7 @@ public async Task Should_Invoke_PreToolUse_And_PostToolUse_Hooks_For_Sub_Agent_T RequestHandler = requestHandler }, environment: env); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Hooks = new SessionHooks diff --git a/dotnet/test/E2E/SuspendE2ETests.cs b/dotnet/test/E2E/SuspendE2ETests.cs index f531f0e59c..b88a9897be 100644 --- a/dotnet/test/E2E/SuspendE2ETests.cs +++ b/dotnet/test/E2E/SuspendE2ETests.cs @@ -58,7 +58,7 @@ public async Task Should_Allow_Resume_And_Continue_Conversation_After_Suspend() string sessionId; await using (var client1 = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: sharedToken) })) { - var session1 = await client1.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -78,7 +78,7 @@ await session1.SendAndWaitAsync(new MessageOptions // A different client should be able to pick the session back up. The previous // turn was completed before suspend, so there is no pending work to continue. await using var client2 = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: sharedToken) }); - var session2 = await client2.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(client2, sessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); diff --git a/dotnet/test/E2E/TelemetryExportE2ETests.cs b/dotnet/test/E2E/TelemetryExportE2ETests.cs index 2dd9d591a4..48e3b53ad6 100644 --- a/dotnet/test/E2E/TelemetryExportE2ETests.cs +++ b/dotnet/test/E2E/TelemetryExportE2ETests.cs @@ -34,7 +34,7 @@ public async Task Should_Export_File_Telemetry_For_Sdk_Interactions() }, }); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { Tools = [AIFunctionFactory.Create(EchoTelemetryMarker, toolName, "Echoes a marker string for telemetry validation.")], OnPermissionRequest = PermissionHandler.ApproveAll, diff --git a/dotnet/test/E2E/ToolsE2ETests.cs b/dotnet/test/E2E/ToolsE2ETests.cs index 7424cfa3a3..ea615fbc4a 100644 --- a/dotnet/test/E2E/ToolsE2ETests.cs +++ b/dotnet/test/E2E/ToolsE2ETests.cs @@ -306,7 +306,7 @@ public async Task Invokes_Custom_Tool_With_Permission_Handler() { var permissionRequests = new List(); - var session = await Client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(Client, new SessionConfig { Tools = [AIFunctionFactory.Create(EncryptStringForPermission, "encrypt_string")], OnPermissionRequest = (request, invocation) => @@ -340,7 +340,7 @@ public async Task Denies_Custom_Tool_When_Permission_Denied() { var toolHandlerCalled = false; - var session = await Client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(Client, new SessionConfig { Tools = [AIFunctionFactory.Create(EncryptStringDenied, "encrypt_string")], OnPermissionRequest = async (request, invocation) => PermissionDecision.Reject(), diff --git a/dotnet/test/Harness/E2ETestBackend.cs b/dotnet/test/Harness/E2ETestBackend.cs new file mode 100644 index 0000000000..04808ed7a9 --- /dev/null +++ b/dotnet/test/Harness/E2ETestBackend.cs @@ -0,0 +1,122 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +namespace GitHub.Copilot.Test.Harness; + +internal enum E2ETestBackend +{ + Capi, + AnthropicMessages, + OpenAIResponses, + OpenAICompletions, +} + +internal static class E2ETestBackendConfiguration +{ + internal const string EnvironmentVariable = "COPILOT_SDK_E2E_BACKEND"; + private const string AnthropicDefaultModel = "claude-sonnet-4.5"; + private const string OpenAIDefaultModel = "gpt-4.1"; + private const string FakeCredential = "fake-byok-credential-for-e2e-tests"; + + internal static E2ETestBackend Current + => Parse(Environment.GetEnvironmentVariable(EnvironmentVariable)); + + internal static E2ETestBackend Parse(string? value) + => value?.Trim().ToLowerInvariant() switch + { + null or "" or "capi" => E2ETestBackend.Capi, + "anthropic-messages" => E2ETestBackend.AnthropicMessages, + "openai-responses" => E2ETestBackend.OpenAIResponses, + "openai-completions" => E2ETestBackend.OpenAICompletions, + _ => throw new ArgumentOutOfRangeException( + nameof(value), + value, + $"Unsupported {EnvironmentVariable} value. Expected capi, anthropic-messages, openai-responses, or openai-completions."), + }; + + internal static string ToWireName(this E2ETestBackend backend) + => backend switch + { + E2ETestBackend.Capi => "capi", + E2ETestBackend.AnthropicMessages => "anthropic-messages", + E2ETestBackend.OpenAIResponses => "openai-responses", + E2ETestBackend.OpenAICompletions => "openai-completions", + _ => throw new ArgumentOutOfRangeException(nameof(backend), backend, null), + }; + + internal static void ApplyProvider( + this E2ETestBackend backend, + SessionConfig config, + string proxyUrl) + { + if (backend == E2ETestBackend.Capi + || config.Provider is not null + || config.Providers is not null) + { + return; + } + + var model = config.Model ??= backend.GetDefaultModel(); + config.Provider = CreateProvider(backend, proxyUrl, model); + } + + internal static void ApplyProvider( + this E2ETestBackend backend, + ResumeSessionConfig config, + string proxyUrl) + { + if (backend == E2ETestBackend.Capi || config.Provider is not null) + { + return; + } + + var model = config.Model ??= backend.GetDefaultModel(); + config.Provider = CreateProvider(backend, proxyUrl, model); + } + + private static string GetDefaultModel(this E2ETestBackend backend) + => backend switch + { + E2ETestBackend.AnthropicMessages => AnthropicDefaultModel, + E2ETestBackend.OpenAIResponses or E2ETestBackend.OpenAICompletions => OpenAIDefaultModel, + _ => throw new ArgumentOutOfRangeException(nameof(backend), backend, null), + }; + + private static ProviderConfig CreateProvider( + E2ETestBackend backend, + string proxyUrl, + string model) + => new() + { + BaseUrl = proxyUrl, + Type = backend switch + { + E2ETestBackend.AnthropicMessages => "anthropic", + E2ETestBackend.OpenAIResponses or E2ETestBackend.OpenAICompletions => "openai", + _ => throw new ArgumentOutOfRangeException(nameof(backend), backend, null), + }, + WireApi = backend switch + { + E2ETestBackend.AnthropicMessages => null, + E2ETestBackend.OpenAIResponses => "responses", + E2ETestBackend.OpenAICompletions => "completions", + _ => throw new ArgumentOutOfRangeException(nameof(backend), backend, null), + }, + BearerToken = FakeCredential, + ModelId = model, + WireModel = model, + }; +} + +internal static class E2ETestTraits +{ + // Trait key used by workflow filters to classify backend compatibility. + internal const string Backend = "E2EBackend"; + + // Requires the default CAPI backend and is excluded from BYOK legs. + internal const string CapiOnly = "CapiOnly"; + + // Owns its backend setup and must not inherit the backend selected by the test matrix. + internal const string SelfConfiguredBackend = "SelfConfiguredBackend"; +} diff --git a/dotnet/test/Harness/E2ETestBase.cs b/dotnet/test/Harness/E2ETestBase.cs index ddd1b894bb..a3006389bb 100644 --- a/dotnet/test/Harness/E2ETestBase.cs +++ b/dotnet/test/Harness/E2ETestBase.cs @@ -76,7 +76,7 @@ protected Task CreateSessionAsync(SessionConfig? config = null) { config ??= new SessionConfig(); config.OnPermissionRequest ??= PermissionHandler.ApproveAll; - return Client.CreateSessionAsync(config); + return Ctx.CreateSessionAsync(Client, config); } /// @@ -96,7 +96,7 @@ protected async Task ResumeSessionAsync(string sessionId, Resume { Connection = RuntimeConnection.ForUri($"localhost:{port}", connectionToken: E2ETestFixture.SharedTcpConnectionToken), }); - return await client.ResumeSessionAsync(sessionId, config); + return await Ctx.ResumeSessionAsync(client, sessionId, config); } protected static string GetSystemMessage(ParsedHttpExchange exchange) diff --git a/dotnet/test/Harness/E2ETestContext.cs b/dotnet/test/Harness/E2ETestContext.cs index 4c9b892ff2..c01f4efcff 100644 --- a/dotnet/test/Harness/E2ETestContext.cs +++ b/dotnet/test/Harness/E2ETestContext.cs @@ -20,13 +20,13 @@ public sealed class E2ETestContext : IAsyncDisposable /// Optional logger injected by tests; applied to all clients created via . public ILogger? Logger { get; set; } - private readonly CapiProxy _proxy; + private readonly ReplayProxy _proxy; private readonly string _repoRoot; private readonly object _clientsLock = new(); private readonly List _persistentClients = []; private readonly List _transientClients = []; - private E2ETestContext(string homeDir, string workDir, string proxyUrl, CapiProxy proxy, string repoRoot) + private E2ETestContext(string homeDir, string workDir, string proxyUrl, ReplayProxy proxy, string repoRoot) { HomeDir = homeDir; WorkDir = workDir; @@ -50,7 +50,7 @@ public static async Task CreateAsync() homeDir = ResolveSymlinks(homeDir); workDir = ResolveSymlinks(workDir); - var proxy = new CapiProxy(); + var proxy = new ReplayProxy(); var proxyUrl = await proxy.StartAsync(); await proxy.SetCopilotUserByTokenAsync(DefaultGitHubToken, new CopilotUserConfig( Login: "e2e-test-user", @@ -163,7 +163,10 @@ public async Task ConfigureForTestAsync(string testFile, [CallerMemberName] stri // to avoid case collisions on case-insensitive filesystems (macOS/Windows) var sanitizedName = Regex.Replace(testName!, @"[^a-zA-Z0-9]", "_").ToLowerInvariant(); var snapshotPath = Path.Combine(_repoRoot, "test", "snapshots", testFile, $"{sanitizedName}.yaml"); - await _proxy.ConfigureAsync(snapshotPath, WorkDir); + await _proxy.ConfigureAsync( + snapshotPath, + WorkDir, + E2ETestBackendConfiguration.Current.ToWireName()); } public Task> GetExchangesAsync() @@ -352,6 +355,25 @@ public CopilotClient CreateClient( return client; } + public Task CreateSessionAsync( + CopilotClient client, + SessionConfig? config = null) + { + config ??= new SessionConfig(); + E2ETestBackendConfiguration.Current.ApplyProvider(config, ProxyUrl); + return client.CreateSessionAsync(config); + } + + public Task ResumeSessionAsync( + CopilotClient client, + string sessionId, + ResumeSessionConfig? config = null) + { + config ??= new ResumeSessionConfig(); + E2ETestBackendConfiguration.Current.ApplyProvider(config, ProxyUrl); + return client.ResumeSessionAsync(sessionId, config); + } + public void UntrackClient(CopilotClient client) { lock (_clientsLock) diff --git a/dotnet/test/Harness/CapiProxy.cs b/dotnet/test/Harness/ReplayProxy.cs similarity index 94% rename from dotnet/test/Harness/CapiProxy.cs rename to dotnet/test/Harness/ReplayProxy.cs index 39c95fa683..895ebccb87 100644 --- a/dotnet/test/Harness/CapiProxy.cs +++ b/dotnet/test/Harness/ReplayProxy.cs @@ -13,7 +13,7 @@ namespace GitHub.Copilot.Test.Harness; -public sealed partial class CapiProxy : IAsyncDisposable +public sealed partial class ReplayProxy : IAsyncDisposable { private Process? _process; private Task? _startupTask; @@ -76,7 +76,7 @@ async Task StartCoreAsync() { var metadata = JsonSerializer.Deserialize( match.Groups["metadata"].Value, - CapiProxyJsonContext.Default.ProxyStartupMetadata); + ReplayProxyJsonContext.Default.ProxyStartupMetadata); ConnectProxyUrl = metadata?.ConnectProxyUrl; CaFilePath = metadata?.CaFilePath; } @@ -150,16 +150,19 @@ public async Task StopAsync(bool skipWritingCache = false) _startupTask = null; } - public async Task ConfigureAsync(string filePath, string workDir) + public async Task ConfigureAsync(string filePath, string workDir, string backend) { var url = await (_startupTask ?? throw new InvalidOperationException("Proxy not started")); using var client = new HttpClient(); - var response = await client.PostAsJsonAsync($"{url}/config", new ConfigureRequest(filePath, workDir), CapiProxyJsonContext.Default.ConfigureRequest); + var response = await client.PostAsJsonAsync( + $"{url}/config", + new ConfigureRequest(filePath, workDir, backend), + ReplayProxyJsonContext.Default.ConfigureRequest); response.EnsureSuccessStatusCode(); } - private record ConfigureRequest(string FilePath, string WorkDir); + private record ConfigureRequest(string FilePath, string WorkDir, string Backend); private record ProxyStartupMetadata(string? ConnectProxyUrl, string? CaFilePath); @@ -168,7 +171,7 @@ public async Task> GetExchangesAsync() var url = await (_startupTask ?? throw new InvalidOperationException("Proxy not started")); using var client = new HttpClient(); - return await client.GetFromJsonAsync($"{url}/exchanges", CapiProxyJsonContext.Default.ListParsedHttpExchange) + return await client.GetFromJsonAsync($"{url}/exchanges", ReplayProxyJsonContext.Default.ListParsedHttpExchange) ?? []; } @@ -178,7 +181,7 @@ public async Task SetCopilotUserByTokenAsync(string token, CopilotUserConfig res using var client = new HttpClient(); var payload = new CopilotUserByTokenRequest(token, response); - var resp = await client.PostAsJsonAsync($"{url}/copilot-user-config", payload, CapiProxyJsonContext.Default.CopilotUserByTokenRequest); + var resp = await client.PostAsJsonAsync($"{url}/copilot-user-config", payload, ReplayProxyJsonContext.Default.CopilotUserByTokenRequest); resp.EnsureSuccessStatusCode(); } @@ -205,7 +208,7 @@ private static string FindRepoRoot() [JsonSerializable(typeof(CopilotUserByTokenRequest))] [JsonSerializable(typeof(Dictionary))] [JsonSerializable(typeof(ProxyStartupMetadata))] - private partial class CapiProxyJsonContext : JsonSerializerContext; + private partial class ReplayProxyJsonContext : JsonSerializerContext; } public record CopilotUserByTokenRequest(string Token, CopilotUserConfig Response); diff --git a/dotnet/test/Unit/E2ETestBackendTests.cs b/dotnet/test/Unit/E2ETestBackendTests.cs new file mode 100644 index 0000000000..f7c39a5083 --- /dev/null +++ b/dotnet/test/Unit/E2ETestBackendTests.cs @@ -0,0 +1,80 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +using GitHub.Copilot.Test.Harness; +using Xunit; + +namespace GitHub.Copilot.Test.Unit; + +public class E2ETestBackendTests +{ + [Theory] + [InlineData(null, "capi")] + [InlineData("", "capi")] + [InlineData("capi", "capi")] + [InlineData("ANTHROPIC-MESSAGES", "anthropic-messages")] + [InlineData("openai-responses", "openai-responses")] + [InlineData("openai-completions", "openai-completions")] + public void ParsesBackend(string? value, string expected) + => Assert.Equal(expected, E2ETestBackendConfiguration.Parse(value).ToWireName()); + + [Fact] + public void RejectsUnknownBackend() + => Assert.Throws( + () => E2ETestBackendConfiguration.Parse("unknown")); + + [Theory] + [InlineData("anthropic-messages", "anthropic", null, "claude-sonnet-4.5")] + [InlineData("openai-responses", "openai", "responses", "gpt-4.1")] + [InlineData("openai-completions", "openai", "completions", "gpt-4.1")] + public void AppliesProvider( + string backendValue, + string expectedType, + string? expectedWireApi, + string expectedModel) + { + var backend = E2ETestBackendConfiguration.Parse(backendValue); + var config = new SessionConfig(); + backend.ApplyProvider(config, "http://localhost:1234"); + + Assert.Equal(expectedModel, config.Model); + Assert.Equal("http://localhost:1234", config.Provider!.BaseUrl); + Assert.Equal(expectedType, config.Provider.Type); + Assert.Equal(expectedWireApi, config.Provider.WireApi); + Assert.Equal(expectedModel, config.Provider.ModelId); + Assert.Equal(expectedModel, config.Provider.WireModel); + Assert.False(string.IsNullOrEmpty(config.Provider.BearerToken)); + } + + [Fact] + public void PreservesExplicitModel() + { + var config = new SessionConfig { Model = "test-model" }; + E2ETestBackend.OpenAIResponses.ApplyProvider(config, "http://localhost:1234"); + + Assert.Equal("test-model", config.Model); + Assert.Equal("test-model", config.Provider!.ModelId); + Assert.Equal("test-model", config.Provider.WireModel); + } + + [Fact] + public void PreservesExplicitProvider() + { + var provider = new ProviderConfig + { + Type = "custom", + ModelId = "provider-model", + }; + var config = new SessionConfig + { + Model = "session-model", + Provider = provider, + }; + + E2ETestBackend.OpenAIResponses.ApplyProvider(config, "http://localhost:1234"); + + Assert.Equal("session-model", config.Model); + Assert.Same(provider, config.Provider); + } +} diff --git a/test/harness/anthropicMessagesAdapter.ts b/test/harness/anthropicMessagesAdapter.ts new file mode 100644 index 0000000000..acc74a2bf9 --- /dev/null +++ b/test/harness/anthropicMessagesAdapter.ts @@ -0,0 +1,396 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import type { ChatCompletion } from "openai/resources/chat/completions"; +import { + CanonicalMessage, + CanonicalToolCall, + formatSseEvent, + functionToolCalls, + isObject, + JsonObject, +} from "./modelProtocolAdapterShared"; + +export const anthropicMessagesEndpoint = "/v1/messages"; + +type CanonicalContentPart = + | { type: "text"; text: string } + | { type: "image_url"; image_url: { url: string } } + | { + type: "file"; + file: { file_data: string; filename?: string }; + }; + +type AnthropicContentBlock = + | { type: "text"; text: string; citations?: null } + | { + type: "image" | "document"; + source?: { type?: string; media_type?: string; data?: string }; + } + | { type: "tool_use"; id: string; name: string; input: unknown } + | { + type: "tool_result"; + tool_use_id?: string; + content?: string | Array<{ type?: string; text?: string }>; + }; + +type AnthropicMessageParam = { + role: "user" | "assistant"; + content: string | AnthropicContentBlock[]; +}; + +type AnthropicRequest = { + model: string; + messages: AnthropicMessageParam[]; + system?: string | Array<{ type?: string; text?: string }>; + max_tokens?: number; + temperature?: number; + top_p?: number; + stream?: boolean; + tools?: Array<{ + name: string; + description?: string; + input_schema?: JsonObject; + }>; + tool_choice?: + | { type: "auto" | "any" | "none" } + | { type: "tool"; name: string }; +}; + +type AnthropicStopReason = + | "end_turn" + | "max_tokens" + | "stop_sequence" + | "tool_use" + | "refusal"; + +export type AnthropicMessage = { + id: string; + type: "message"; + role: "assistant"; + content: Array< + | { type: "text"; text: string; citations: null } + | { type: "tool_use"; id: string; name: string; input: unknown } + >; + model: string; + stop_reason: AnthropicStopReason | null; + stop_sequence: string | null; + usage: { + input_tokens: number; + output_tokens: number; + cache_creation_input_tokens: number | null; + cache_read_input_tokens: number | null; + }; +}; + +const finishReasonToStopReason: Record = { + stop: "end_turn", + length: "max_tokens", + tool_calls: "tool_use", + function_call: "tool_use", + content_filter: "refusal", +}; + +export function anthropicMessagesRequestToChatCompletion( + requestBody: string, +): string { + const request = JSON.parse(requestBody) as AnthropicRequest; + const messages: CanonicalMessage[] = []; + + const system = anthropicSystemToString(request.system); + if (system) messages.push({ role: "system", content: system }); + + for (const message of request.messages) { + messages.push(...convertAnthropicMessage(message)); + } + + return JSON.stringify({ + model: request.model, + messages, + ...(request.max_tokens !== undefined + ? { max_tokens: request.max_tokens } + : {}), + ...(request.temperature !== undefined + ? { temperature: request.temperature } + : {}), + ...(request.top_p !== undefined ? { top_p: request.top_p } : {}), + ...(request.stream !== undefined ? { stream: request.stream } : {}), + ...(request.tools + ? { + tools: request.tools.map((tool) => ({ + type: "function", + function: { + name: tool.name, + ...(tool.description ? { description: tool.description } : {}), + parameters: tool.input_schema ?? { + type: "object", + properties: {}, + }, + }, + })), + } + : {}), + ...(request.tool_choice + ? { tool_choice: convertToolChoice(request.tool_choice) } + : {}), + }); +} + +function anthropicSystemToString( + system: AnthropicRequest["system"], +): string | undefined { + if (typeof system === "string") return system; + if (!Array.isArray(system)) return undefined; + return system + .map((block) => (typeof block.text === "string" ? block.text : "")) + .filter(Boolean) + .join("\n"); +} + +function convertAnthropicMessage( + message: AnthropicMessageParam, +): CanonicalMessage[] { + return message.role === "user" + ? convertAnthropicUserMessage(message) + : convertAnthropicAssistantMessage(message); +} + +function normalizeContent( + content: AnthropicMessageParam["content"], +): AnthropicContentBlock[] { + return typeof content === "string" + ? [{ type: "text", text: content }] + : content; +} + +function convertAnthropicUserMessage( + message: AnthropicMessageParam, +): CanonicalMessage[] { + const result: CanonicalMessage[] = []; + const contentParts: CanonicalContentPart[] = []; + + const flushUserContent = () => { + if (contentParts.length === 0) return; + const onlyText = contentParts.every((part) => part.type === "text"); + result.push({ + role: "user", + content: onlyText + ? contentParts + .map((part) => (part.type === "text" ? part.text : "")) + .join("\n") + : [...contentParts], + }); + contentParts.length = 0; + }; + + for (const block of normalizeContent(message.content)) { + if (block.type === "text") { + contentParts.push({ type: "text", text: block.text }); + } else if ( + (block.type === "image" || block.type === "document") && + block.source?.type === "base64" && + block.source.data + ) { + const dataUrl = `data:${ + block.source.media_type ?? + (block.type === "image" ? "image/png" : "application/pdf") + };base64,${block.source.data}`; + contentParts.push( + block.type === "image" + ? { type: "image_url", image_url: { url: dataUrl } } + : { type: "file", file: { file_data: dataUrl } }, + ); + } else if (block.type === "tool_result") { + flushUserContent(); + result.push({ + role: "tool", + tool_call_id: block.tool_use_id ?? "", + content: anthropicToolResultContent(block.content), + }); + } + } + + flushUserContent(); + return result; +} + +function convertAnthropicAssistantMessage( + message: AnthropicMessageParam, +): CanonicalMessage[] { + const text: string[] = []; + const toolCalls: CanonicalToolCall[] = []; + for (const block of normalizeContent(message.content)) { + if (block.type === "text") { + text.push(block.text); + } else if (block.type === "tool_use") { + toolCalls.push({ + id: block.id, + type: "function", + function: { + name: block.name, + arguments: JSON.stringify(block.input ?? {}), + }, + }); + } + } + + return [ + { + role: "assistant", + content: text.length ? text.join("") : null, + ...(toolCalls.length ? { tool_calls: toolCalls } : {}), + }, + ]; +} + +function anthropicToolResultContent(content: unknown): string { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; + return content + .map((part) => + isObject(part) && typeof part.text === "string" ? part.text : "", + ) + .filter(Boolean) + .join("\n"); +} + +function convertToolChoice( + choice: NonNullable, +): unknown { + switch (choice.type) { + case "auto": + return "auto"; + case "any": + return "required"; + case "none": + return "none"; + case "tool": + return { type: "function", function: { name: choice.name } }; + } +} + +export function chatCompletionResponseToAnthropicMessage( + response: ChatCompletion, +): AnthropicMessage { + const content: AnthropicMessage["content"] = []; + for (const choice of response.choices) { + if (choice.message.content) { + content.push({ + type: "text", + text: choice.message.content, + citations: null, + }); + } + for (const toolCall of functionToolCalls(choice.message)) { + content.push({ + type: "tool_use", + id: toolCall.id, + name: toolCall.function.name, + input: safeParseJson(toolCall.function.arguments), + }); + } + } + + const finishReason = response.choices.at(-1)?.finish_reason; + return { + id: response.id, + type: "message", + role: "assistant", + content, + model: response.model, + stop_reason: finishReason + ? (finishReasonToStopReason[finishReason] ?? null) + : null, + stop_sequence: null, + usage: { + input_tokens: response.usage?.prompt_tokens ?? 0, + output_tokens: response.usage?.completion_tokens ?? 0, + cache_creation_input_tokens: null, + cache_read_input_tokens: null, + }, + }; +} + +export function chatCompletionResponseToAnthropicSseChunks( + response: ChatCompletion, +): string[] { + const message = chatCompletionResponseToAnthropicMessage(response); + const chunks = [ + formatSseEvent("message_start", { + type: "message_start", + message: { + ...message, + content: [], + stop_reason: null, + usage: { ...message.usage, output_tokens: 1 }, + }, + }), + ]; + + for (let index = 0; index < message.content.length; index++) { + const block = message.content[index]; + if (block.type === "text") { + chunks.push( + formatSseEvent("content_block_start", { + type: "content_block_start", + index, + content_block: { type: "text", text: "", citations: null }, + }), + formatSseEvent("content_block_delta", { + type: "content_block_delta", + index, + delta: { type: "text_delta", text: block.text }, + }), + ); + } else { + chunks.push( + formatSseEvent("content_block_start", { + type: "content_block_start", + index, + content_block: { + type: "tool_use", + id: block.id, + name: block.name, + input: {}, + }, + }), + formatSseEvent("content_block_delta", { + type: "content_block_delta", + index, + delta: { + type: "input_json_delta", + partial_json: JSON.stringify(block.input ?? {}), + }, + }), + ); + } + chunks.push( + formatSseEvent("content_block_stop", { + type: "content_block_stop", + index, + }), + ); + } + + chunks.push( + formatSseEvent("message_delta", { + type: "message_delta", + delta: { + stop_reason: message.stop_reason, + stop_sequence: message.stop_sequence, + }, + usage: { output_tokens: message.usage.output_tokens }, + }), + formatSseEvent("message_stop", { type: "message_stop" }), + ); + return chunks; +} + +function safeParseJson(value: string): unknown { + try { + return JSON.parse(value); + } catch { + return {}; + } +} diff --git a/test/harness/modelProtocolAdapterShared.ts b/test/harness/modelProtocolAdapterShared.ts new file mode 100644 index 0000000000..1f879da5d0 --- /dev/null +++ b/test/harness/modelProtocolAdapterShared.ts @@ -0,0 +1,39 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +export type JsonObject = Record; + +export type CanonicalToolCall = { + id: string; + type: "function"; + function: { name: string; arguments: string }; +}; + +export type CanonicalMessage = { + role: "system" | "user" | "assistant" | "tool"; + content?: string | unknown[] | null; + tool_call_id?: string; + tool_calls?: CanonicalToolCall[]; +}; + +export function functionToolCalls(message: unknown): CanonicalToolCall[] { + if (!isObject(message) || !Array.isArray(message.tool_calls)) return []; + return message.tool_calls.filter( + (toolCall): toolCall is CanonicalToolCall => + isObject(toolCall) && + typeof toolCall.id === "string" && + toolCall.type === "function" && + isObject(toolCall.function) && + typeof toolCall.function.name === "string" && + typeof toolCall.function.arguments === "string", + ); +} + +export function formatSseEvent(type: string, data: unknown): string { + return `event: ${type}\ndata: ${JSON.stringify(data)}\n\n`; +} + +export function isObject(value: unknown): value is JsonObject { + return value !== null && typeof value === "object" && !Array.isArray(value); +} diff --git a/test/harness/modelProtocolAdapters.test.ts b/test/harness/modelProtocolAdapters.test.ts new file mode 100644 index 0000000000..ddb6fe40bf --- /dev/null +++ b/test/harness/modelProtocolAdapters.test.ts @@ -0,0 +1,641 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { ChatCompletion } from "openai/resources/chat/completions"; +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import yaml from "yaml"; +import { + anthropicMessagesRequestToChatCompletion, + chatCompletionResponseToAnthropicMessage, + chatCompletionResponseToAnthropicSseChunks, +} from "./anthropicMessagesAdapter"; +import { + chatCompletionResponseToResponsesApiMessage, + chatCompletionResponseToResponsesApiSseChunks, + responsesApiRequestToChatCompletion, +} from "./responsesApiAdapter"; +import { + NormalizedData, + ReplayBackend, + ReplayingCapiProxy, +} from "./replayingCapiProxy"; + +type ByokBackend = Exclude; + +const backends: ReplayBackend[] = [ + "capi", + "anthropic-messages", + "openai-responses", + "openai-completions", +]; + +const endpoints: Record = { + capi: "/chat/completions", + "anthropic-messages": "/v1/messages", + "openai-responses": "/responses", + "openai-completions": "/chat/completions", +}; + +const models: Record = { + capi: "gpt-4.1", + "anthropic-messages": "claude-sonnet-4.5", + "openai-responses": "gpt-4.1", + "openai-completions": "gpt-4.1", +}; + +const completionWithTool: ChatCompletion = { + id: "completion-1", + object: "chat.completion", + created: 123, + model: "test-model", + choices: [ + { + index: 0, + message: { + role: "assistant", + content: "Calling a tool", + refusal: null, + tool_calls: [ + { + id: "call-1", + type: "function", + function: { name: "lookup", arguments: '{"value":42}' }, + }, + ], + }, + logprobs: null, + finish_reason: "tool_calls", + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 5, + total_tokens: 15, + }, +}; + +function requestFor( + backend: ReplayBackend, + prompt: string, +): Record { + const model = models[backend]; + switch (backend) { + case "anthropic-messages": + return { + model, + system: "Be helpful", + messages: [{ role: "user", content: prompt }], + max_tokens: 128, + }; + case "openai-responses": + return { + model, + instructions: "Be helpful", + input: [ + { + type: "message", + role: "user", + content: [{ type: "input_text", text: prompt }], + }, + ], + }; + case "capi": + case "openai-completions": + return { + model, + messages: [ + { role: "system", content: "Be helpful" }, + { role: "user", content: prompt }, + ], + }; + } +} + +async function postJson( + proxyUrl: string, + endpoint: string, + body: unknown, +): Promise { + return fetch(`${proxyUrl}${endpoint}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); +} + +describe("Anthropic Messages adapter", () => { + test("normalizes messages, binary content, and tools", () => { + const result = JSON.parse( + anthropicMessagesRequestToChatCompletion( + JSON.stringify({ + model: "test-model", + system: [{ type: "text", text: "Be helpful" }], + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Inspect this" }, + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "AQID", + }, + }, + ], + }, + { + role: "assistant", + content: [ + { + type: "tool_use", + id: "call-1", + name: "lookup", + input: { value: 42 }, + }, + ], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "call-1", + content: "found", + }, + ], + }, + ], + tools: [ + { + name: "lookup", + description: "Find a value", + input_schema: { type: "object" }, + }, + ], + stream: true, + }), + ), + ) as { + messages: Array>; + tools: Array>; + stream: boolean; + }; + + expect(result.messages.map((message) => message.role)).toEqual([ + "system", + "user", + "assistant", + "tool", + ]); + expect(result.messages[1].content).toEqual([ + { type: "text", text: "Inspect this" }, + { + type: "image_url", + image_url: { url: "data:image/png;base64,AQID" }, + }, + ]); + expect(result.messages[2].tool_calls).toEqual([ + { + id: "call-1", + type: "function", + function: { name: "lookup", arguments: '{"value":42}' }, + }, + ]); + expect(result.messages[3]).toMatchObject({ + tool_call_id: "call-1", + content: "found", + }); + expect(result.tools).toHaveLength(1); + expect(result.stream).toBe(true); + }); + + test("renders JSON and streaming tool responses", () => { + const message = + chatCompletionResponseToAnthropicMessage(completionWithTool); + expect(message.stop_reason).toBe("tool_use"); + expect(message.usage).toMatchObject({ + cache_creation_input_tokens: null, + cache_read_input_tokens: null, + }); + expect(message.content.map((block) => block.type)).toEqual([ + "text", + "tool_use", + ]); + + const stream = + chatCompletionResponseToAnthropicSseChunks(completionWithTool).join(""); + expect(stream).toContain("event: message_start"); + expect(stream).toContain("event: content_block_delta"); + expect(stream).toContain("event: message_stop"); + }); + + test("combines tools from multiple canonical choices", () => { + const secondChoice = structuredClone(completionWithTool.choices[0]); + secondChoice.message.content = null; + secondChoice.message.tool_calls![0] = { + id: "call-2", + type: "function", + function: { name: "inspect", arguments: '{"path":"file.txt"}' }, + }; + + const message = chatCompletionResponseToAnthropicMessage({ + ...completionWithTool, + choices: [completionWithTool.choices[0], secondChoice], + }); + expect( + message.content + .filter((block) => block.type === "tool_use") + .map((block) => block.name), + ).toEqual(["lookup", "inspect"]); + }); +}); + +describe("OpenAI Responses adapter", () => { + test("normalizes messages, binary content, and tools", () => { + const result = JSON.parse( + responsesApiRequestToChatCompletion( + JSON.stringify({ + model: "test-model", + instructions: "Be helpful", + input: [ + { + type: "message", + role: "user", + content: [ + { type: "input_text", text: "Inspect this" }, + { + type: "input_image", + image_url: "data:image/png;base64,AQID", + }, + ], + }, + { + type: "function_call", + call_id: "call-1", + name: "lookup", + arguments: '{"value":42}', + }, + { + type: "function_call_output", + call_id: "call-1", + output: "found", + }, + ], + tools: [ + { + type: "function", + name: "lookup", + parameters: { type: "object" }, + }, + ], + }), + ), + ) as { + messages: Array>; + tools: Array>; + }; + + expect(result.messages.map((message) => message.role)).toEqual([ + "system", + "user", + "assistant", + "tool", + ]); + expect(result.messages[1].content).toEqual([ + { type: "text", text: "Inspect this" }, + { + type: "image_url", + image_url: { url: "data:image/png;base64,AQID" }, + }, + ]); + expect(result.messages[2].tool_calls).toEqual([ + { + id: "call-1", + type: "function", + function: { name: "lookup", arguments: '{"value":42}' }, + }, + ]); + expect(result.messages[3]).toMatchObject({ + tool_call_id: "call-1", + content: "found", + }); + expect(result.tools).toHaveLength(1); + }); + + test("renders JSON and streaming tool responses", () => { + const response = + chatCompletionResponseToResponsesApiMessage(completionWithTool); + const nextResponse = + chatCompletionResponseToResponsesApiMessage(completionWithTool); + expect(response).toMatchObject({ + object: "response", + created_at: completionWithTool.created, + status: "completed", + incomplete_details: null, + error: null, + }); + expect(response.output[0].id).not.toBe(nextResponse.output[0].id); + expect(response.output.map((item) => item.type)).toEqual([ + "message", + "function_call", + ]); + + const chunks = + chatCompletionResponseToResponsesApiSseChunks(completionWithTool); + const events = chunks.map( + (chunk) => + JSON.parse(chunk.split("\ndata: ")[1]) as Record, + ); + const stream = chunks.join(""); + expect(stream).toContain("event: response.created"); + expect(stream).toContain("event: response.in_progress"); + expect(stream).toContain("event: response.output_text.delta"); + expect(stream).toContain('"sequence_number":0'); + expect(stream).toContain("event: response.completed"); + + expect(events[0]).toMatchObject({ + type: "response.created", + response: { status: "in_progress", output: [] }, + }); + expect(events[1]).toMatchObject({ + type: "response.in_progress", + response: { status: "in_progress", output: [] }, + }); + + const addedItems = events.filter( + (event) => event.type === "response.output_item.added", + ); + expect(addedItems).toMatchObject([ + { + item: { + type: "message", + status: "in_progress", + content: [], + }, + }, + { + item: { + type: "function_call", + status: "in_progress", + arguments: "", + }, + }, + ]); + expect( + events.find((event) => event.type === "response.content_part.added"), + ).toMatchObject({ + part: { type: "output_text", text: "" }, + }); + + const completedItems = events.filter( + (event) => event.type === "response.output_item.done", + ); + expect(completedItems).toMatchObject([ + { + item: { + type: "message", + status: "completed", + content: [{ type: "output_text", text: "Calling a tool" }], + }, + }, + { + item: { + type: "function_call", + status: "completed", + arguments: '{"value":42}', + }, + }, + ]); + }); +}); + +describe("protocol-aware replay", () => { + let tempDir: string; + let workDir: string; + let cachePath: string; + + async function writeSnapshot( + messages: NormalizedData["conversations"][number]["messages"], + ): Promise { + await writeFile( + cachePath, + yaml.stringify({ + models: ["captured-capi-model"], + conversations: [{ messages }], + } satisfies NormalizedData), + ); + } + + async function withProxy( + backend: ReplayBackend, + action: (proxyUrl: string) => Promise, + ): Promise { + const proxy = new ReplayingCapiProxy( + "http://localhost:9999", + cachePath, + workDir, + ); + const proxyUrl = await proxy.start(); + await proxy.updateConfig({ filePath: cachePath, workDir, backend }); + try { + await action(proxyUrl); + } finally { + await proxy.stop(true); + } + } + + beforeEach(async () => { + tempDir = await mkdtemp(path.join(os.tmpdir(), "protocol-replay-")); + workDir = path.join(tempDir, "work"); + cachePath = path.join(tempDir, "cache.yaml"); + await writeSnapshot([ + { role: "system", content: "${system}" }, + { role: "user", content: "Hello" }, + { role: "assistant", content: "Hi there!" }, + ]); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + test.each(backends)( + "replays one model-independent snapshot through %s", + async (backend) => { + await withProxy(backend, async (proxyUrl) => { + const response = await postJson( + proxyUrl, + endpoints[backend], + requestFor(backend, "Hello"), + ); + expect(response.status).toBe(200); + const body = (await response.json()) as Record; + expect(body.model).toBe(models[backend]); + + const exchanges = (await ( + await fetch(`${proxyUrl}/exchanges`) + ).json()) as Array<{ + request: { + model: string; + messages: Array<{ role: string; content: unknown }>; + }; + response?: unknown; + }>; + expect(exchanges).toHaveLength(1); + expect(exchanges[0].request.model).toBe(models[backend]); + expect(exchanges[0].request.messages.at(-1)).toEqual({ + role: "user", + content: "Hello", + }); + if ( + backend === "anthropic-messages" || + backend === "openai-responses" + ) { + expect(exchanges[0].response).toBeUndefined(); + } + }); + }, + ); + + test("does not rewrite canonical snapshots after BYOK replay", async () => { + const original = await readFile(cachePath, "utf8"); + const proxy = new ReplayingCapiProxy( + "http://localhost:9999", + cachePath, + workDir, + ); + const proxyUrl = await proxy.start(); + await proxy.updateConfig({ + filePath: cachePath, + workDir, + backend: "openai-responses", + }); + + let stopped = false; + try { + const response = await postJson( + proxyUrl, + endpoints["openai-responses"], + requestFor("openai-responses", "Hello"), + ); + expect(response.status).toBe(200); + await proxy.stop(); + stopped = true; + expect(await readFile(cachePath, "utf8")).toBe(original); + } finally { + if (!stopped) await proxy.stop(true); + } + }); + + test.each(["openai-responses", "openai-completions"] as const)( + "coalesces adjacent user messages from %s", + async (backend) => { + await writeSnapshot([ + { role: "system", content: "${system}" }, + { role: "user", content: "Hook context" }, + { role: "user", content: "Hello" }, + { role: "assistant", content: "Hi there!" }, + ]); + const request = requestFor(backend, "Hello"); + const hook = + "Hook context\n\n\n2026-01-01T00:00:00Z\n\n"; + if (backend === "openai-responses") { + (request.input as unknown[]).unshift({ + type: "message", + role: "user", + content: [{ type: "input_text", text: hook }], + }); + } else { + (request.messages as unknown[]).splice(1, 0, { + role: "user", + content: hook, + }); + } + + await withProxy(backend, async (proxyUrl) => { + const response = await postJson( + proxyUrl, + endpoints[backend], + request, + ); + expect(response.status).toBe(200); + }); + }, + ); + + test("normalizes Anthropic spacing between adjacent user turns", async () => { + await writeSnapshot([ + { role: "system", content: "${system}" }, + { role: "user", content: "First prompt" }, + { role: "user", content: "Recovery prompt" }, + { role: "assistant", content: "Recovered" }, + ]); + await withProxy("anthropic-messages", async (proxyUrl) => { + const response = await postJson( + proxyUrl, + endpoints["anthropic-messages"], + requestFor( + "anthropic-messages", + "First prompt\n\n\n\n\nRecovery prompt", + ), + ); + expect(response.status).toBe(200); + }); + }); + + test.each(backends)( + "replays compaction responses through %s", + async (backend) => { + await writeSnapshot([ + { role: "system", content: "${system}" }, + { role: "user", content: "${compaction_prompt}" }, + { + role: "assistant", + content: + "CompactedHistoryCheckpoint", + }, + ]); + await withProxy(backend, async (proxyUrl) => { + const response = await postJson( + proxyUrl, + endpoints[backend], + requestFor(backend, "${compaction_prompt}"), + ); + expect(response.status).toBe(200); + const body = JSON.stringify(await response.json()); + expect(body).toContain(""); + expect(body).toContain(""); + expect(body).toContain(""); + }); + }, + ); + + test("rejects an inference request over the wrong protocol", async () => { + await withProxy("anthropic-messages", async (proxyUrl) => { + const response = await postJson( + proxyUrl, + endpoints["openai-completions"], + requestFor("openai-completions", "Hello"), + ); + expect(response.status).toBe(400); + await expect(response.text()).resolves.toContain("protocol_mismatch"); + }); + }); + + test("keeps foreign model endpoints unavailable in CAPI mode", async () => { + await withProxy("capi", async (proxyUrl) => { + const response = await postJson( + proxyUrl, + endpoints["openai-responses"], + requestFor("openai-responses", "Hello"), + ); + expect(response.status).toBe(404); + }); + }); +}); diff --git a/test/harness/replayingCapiProxy.ts b/test/harness/replayingCapiProxy.ts index 6a9271de29..1c29fb7559 100644 --- a/test/harness/replayingCapiProxy.ts +++ b/test/harness/replayingCapiProxy.ts @@ -2,7 +2,7 @@ * Copyright (c) Microsoft Corporation. All rights reserved. *--------------------------------------------------------------------------------------------*/ -import { existsSync, appendFileSync } from "fs"; +import { appendFileSync, existsSync } from "fs"; import { mkdir, readFile, writeFile } from "fs/promises"; import type { ChatCompletion, @@ -19,10 +19,80 @@ import { CapturingHttpProxy, PerformRequestOptions, } from "./capturingHttpProxy"; +import { + anthropicMessagesEndpoint, + anthropicMessagesRequestToChatCompletion, + chatCompletionResponseToAnthropicMessage, + chatCompletionResponseToAnthropicSseChunks, +} from "./anthropicMessagesAdapter"; +import { + chatCompletionResponseToResponsesApiMessage, + chatCompletionResponseToResponsesApiSseChunks, + responsesApiRequestToChatCompletion, + responsesEndpoint, +} from "./responsesApiAdapter"; import { iife, ShellConfig, sleep } from "./util"; export const workingDirPlaceholder = "${workdir}"; const chatCompletionEndpoint = "/chat/completions"; +export type ReplayBackend = + | "capi" + | "anthropic-messages" + | "openai-responses" + | "openai-completions"; + +type ReplayProtocol = { + endpoint: string; + normalizeRequest?: (body: string) => string; + responseBody?: (response: ChatCompletion) => unknown; + responseChunks: (response: ChatCompletion) => string[]; + responseEndChunk?: string; + errorBody?: (code: string | undefined, message: string) => unknown; + canonicalResponse?: boolean; +}; + +const chatCompletionsProtocol = { + endpoint: chatCompletionEndpoint, + responseChunks: (response) => + convertToStreamingResponseChunks(response).map( + (chunk) => `data: ${JSON.stringify(chunk)}\n\n`, + ), + responseEndChunk: "data: [DONE]\n\n", + canonicalResponse: true, +} satisfies ReplayProtocol; + +const replayProtocols: Record = { + capi: chatCompletionsProtocol, + "openai-completions": { + ...chatCompletionsProtocol, + normalizeRequest: coalesceAdjacentUserMessages, + }, + "anthropic-messages": { + endpoint: anthropicMessagesEndpoint, + normalizeRequest: (body) => + coalesceAdjacentUserMessages( + anthropicMessagesRequestToChatCompletion(body), + ), + responseBody: chatCompletionResponseToAnthropicMessage, + responseChunks: chatCompletionResponseToAnthropicSseChunks, + errorBody: (code, message) => { + const type = code ?? "rate_limited"; + return { type: "error", error: { type, message } }; + }, + }, + "openai-responses": { + endpoint: responsesEndpoint, + normalizeRequest: (body) => + coalesceAdjacentUserMessages(responsesApiRequestToChatCompletion(body)), + responseBody: chatCompletionResponseToResponsesApiMessage, + responseChunks: chatCompletionResponseToResponsesApiSseChunks, + }, +}; + +const modelEndpoints = new Set( + Object.values(replayProtocols).map((protocol) => protocol.endpoint), +); + const shellConfig = process.platform === "win32" ? ShellConfig.powerShell : ShellConfig.bash; const normalizedToolNames: Record = { @@ -90,6 +160,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { filePath, workDir, testInfo, + backend: "capi", toolResultNormalizers: [...this.defaultToolResultNormalizers], }; } @@ -112,7 +183,10 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { // In CI mode (GITHUB_ACTIONS=true) we never write — the snapshots are read-only. // Otherwise tests that exercise only a subset of a multi-conversation snapshot // would silently overwrite the file with that subset, breaking subsequent runs. - if (this.state && process.env.GITHUB_ACTIONS !== "true") { + if ( + this.state?.backend === "capi" && + process.env.GITHUB_ACTIONS !== "true" + ) { await writeCapturesToDisk(this.exchanges, this.state); } @@ -120,6 +194,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { filePath: config.filePath, workDir: config.workDir, testInfo: config.testInfo, + backend: parseReplayBackend(config.backend), toolResultNormalizers: [...this.defaultToolResultNormalizers], }; @@ -134,15 +209,20 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { normalizeToolResultOrder(this.state.storedData.conversations); normalizeStoredUserMessages(this.state.storedData.conversations); normalizeStoredToolMessages(this.state.storedData.conversations); + normalizeStoredMessagesForBackend( + this.state.storedData.conversations, + this.state.backend, + ); } } async stop(skipWritingCache?: boolean): Promise { await super.stop(); - // In CI mode we never write — the snapshots are read-only. + // CAPI is the authoritative capture path. BYOK modes only verify that the + // same canonical snapshots replay through each provider protocol. if ( - this.state && + this.state?.backend === "capi" && !skipWritingCache && process.env.GITHUB_ACTIONS !== "true" ) { @@ -224,17 +304,21 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { options.requestOptions.path === "/exchanges" && options.requestOptions.method === "GET" ) { - const chatCompletionExchanges = this.exchanges.filter( - (e) => e.request.url === chatCompletionEndpoint, - ); + const protocol = + replayProtocols[this.state?.backend ?? "capi"]; const parsedExchanges = await Promise.all( - chatCompletionExchanges.map((e) => - parseHttpExchange( - e.request.body, - e.response?.body, - e.request.headers, + this.exchanges + .filter((exchange) => exchange.request.url === protocol.endpoint) + .map((exchange) => + parseHttpExchange( + protocol.normalizeRequest?.(exchange.request.body) ?? + exchange.request.body, + protocol.canonicalResponse + ? exchange.response?.body + : undefined, + exchange.request.headers, + ), ), - ), ); options.onResponseStart(200, {}); options.onData(Buffer.from(JSON.stringify(parsedExchanges))); @@ -336,16 +420,42 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { options.onResponseEnd(); return; } - - // Handle /chat/completions endpoint + const requestPath = options.requestOptions.path ?? ""; + const protocol = replayProtocols[state.backend]; if ( - state.storedData && - options.requestOptions.path === chatCompletionEndpoint && - options.body + modelEndpoints.has(requestPath) && + state.backend !== "capi" && + requestPath !== protocol.endpoint ) { + const message = `Expected ${protocol.endpoint} for backend ${state.backend}, received ${requestPath}`; + options.onResponseStart(400, { + "content-type": "application/json", + ...commonResponseHeaders, + }); + options.onData( + Buffer.from( + JSON.stringify({ + error: { type: "protocol_mismatch", message }, + }), + ), + ); + options.onResponseEnd(); + return; + } + + const isModelRequest = requestPath === protocol.endpoint; + // Every protocol enters the existing Chat Completions snapshot matcher. + const normalizedBody = + isModelRequest && options.body + ? (protocol.normalizeRequest?.(options.body) ?? options.body) + : options.body; + if (state.storedData && isModelRequest && normalizedBody) { + const streamingIsRequested = + (JSON.parse(normalizedBody) as { stream?: boolean }).stream === true; + const savedError = await findSavedChatCompletionError( state.storedData, - options.body, + normalizedBody, state.workDir, state.toolResultNormalizers, ); @@ -361,14 +471,12 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { options.onResponseStart(savedError.status, headers); options.onData( Buffer.from( - JSON.stringify({ - error: { - message: - savedError.message ?? "Rate limited by test snapshot", - type: savedError.code ?? "rate_limited", - code: savedError.code ?? "rate_limited", - }, - }), + JSON.stringify( + (protocol.errorBody ?? openAIErrorBody)( + savedError.code, + savedError.message ?? "Rate limited by test snapshot", + ), + ), ), ); options.onResponseEnd(); @@ -377,45 +485,19 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { const savedResponse = await findSavedChatCompletionResponse( state.storedData, - options.body, + normalizedBody, state.workDir, state.toolResultNormalizers, ); if (savedResponse) { - const streamingIsRequested = - options.body && - (JSON.parse(options.body) as { stream?: boolean }).stream === - true; - - if (streamingIsRequested) { - const headers = { - "content-type": "text/event-stream", - ...commonResponseHeaders, - }; - options.onResponseStart(200, headers); - for (const chunk of convertToStreamingResponseChunks( - savedResponse, - )) { - options.onData( - Buffer.from(`data: ${JSON.stringify(chunk)}\n\n`), - ); - if (this.slowStreaming) { - await sleep(100); - } - } - options.onData(Buffer.from("data: [DONE]\n\n")); - options.onResponseEnd(); - } else { - const body = JSON.stringify(savedResponse); - const headers = { - "content-type": "application/json", - ...commonResponseHeaders, - }; - options.onResponseStart(200, headers); - options.onData(Buffer.from(body)); - options.onResponseEnd(); - } + await this.respondWithProtocol( + options, + protocol, + savedResponse, + streamingIsRequested, + commonResponseHeaders, + ); return; } @@ -425,15 +507,11 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { if ( await isRequestOnlySnapshot( state.storedData, - options.body, + normalizedBody, state.workDir, state.toolResultNormalizers, ) ) { - const streamingIsRequested = - options.body && - (JSON.parse(options.body) as { stream?: boolean }).stream === - true; const headers = { "content-type": streamingIsRequested ? "text/event-stream" @@ -450,7 +528,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { // Beyond this point, we're only going to be able to supply responses in CI if we have a snapshot, // and we only store snapshots for chat completion. For anything else (e.g., custom-agents fetches), // return 404 so the CLI treats them as unavailable instead of erroring. - if (options.requestOptions.path !== chatCompletionEndpoint) { + if (!isModelRequest) { const headers = { "content-type": "application/json", "x-github-request-id": "proxy-not-found", @@ -466,13 +544,14 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { // Fallback to normal proxying if no cached response found // This implicitly captures the new exchange too const isCI = process.env.GITHUB_ACTIONS === "true"; - if (isCI) { + if (isCI || state.backend !== "capi") { await exitWithNoMatchingRequestError( options, state.testInfo, state.workDir, state.toolResultNormalizers, state.storedData, + normalizedBody, ); return; } @@ -482,6 +561,43 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { } }); } + + private async respondWithProtocol( + options: PerformRequestOptions, + protocol: ReplayProtocol, + response: ChatCompletion, + streaming: boolean, + commonHeaders: Record, + ): Promise { + if (!streaming) { + options.onResponseStart(200, { + "content-type": "application/json", + ...commonHeaders, + }); + options.onData( + Buffer.from( + JSON.stringify(protocol.responseBody?.(response) ?? response), + ), + ); + options.onResponseEnd(); + return; + } + + options.onResponseStart(200, { + "content-type": "text/event-stream", + ...commonHeaders, + }); + for (const chunk of protocol.responseChunks(response)) { + options.onData(Buffer.from(chunk)); + if (this.slowStreaming) { + await sleep(100); + } + } + if (protocol.responseEndChunk) { + options.onData(Buffer.from(protocol.responseEndChunk)); + } + options.onResponseEnd(); + } } async function writeCapturesToDisk( @@ -592,11 +708,12 @@ async function exitWithNoMatchingRequestError( workDir: string, toolResultNormalizers: ToolResultNormalizer[], storedData?: NormalizedData, + requestBody?: string, ) { let diagnostics: string; try { const normalized = await parseAndNormalizeRequest( - options.body, + requestBody ?? options.body, workDir, toolResultNormalizers, ); @@ -605,8 +722,11 @@ async function exitWithNoMatchingRequestError( let rawMessages: unknown[] = []; try { rawMessages = - (JSON.parse(options.body ?? "{}") as { messages?: unknown[] }) - .messages ?? []; + ( + JSON.parse(requestBody ?? options.body ?? "{}") as { + messages?: unknown[]; + } + ).messages ?? []; } catch { /* non-JSON body */ } @@ -775,6 +895,60 @@ async function transformHttpExchanges( return { models: Array.from(dedupedModels), conversations: dedupedExchanges }; } +function parseReplayBackend(value: unknown): ReplayBackend { + if (value === undefined || value === null || value === "") return "capi"; + if (typeof value === "string" && Object.hasOwn(replayProtocols, value)) { + return value as ReplayBackend; + } + throw new Error(`Unsupported replay backend: ${String(value)}`); +} + +function coalesceAdjacentUserMessages(requestBody: string): string { + const request = JSON.parse(requestBody) as { + messages?: Array<{ + role?: string; + content?: unknown; + [key: string]: unknown; + }>; + }; + if (!request.messages) return requestBody; + + const messages: NonNullable = []; + for (const message of request.messages) { + const previous = messages.at(-1); + if ( + previous?.role === "user" && + message.role === "user" && + typeof previous.content === "string" && + typeof message.content === "string" + ) { + previous.content = `${previous.content.trimEnd()}\n\n\n${message.content.trimStart()}`; + } else { + messages.push(message); + } + } + + for (const message of messages) { + if (message.role === "user" && typeof message.content === "string") { + message.content = normalizeUserMessage(message.content).replace( + /\n{5,}/g, + "\n\n\n", + ); + } + } + + request.messages = messages; + return JSON.stringify(request); +} + +function openAIErrorBody( + code: string | undefined, + message: string, +): unknown { + const type = code ?? "rate_limited"; + return { error: { message, type, code: type } }; +} + function normalizeFilenames( conversations: NormalizedConversation[], workDir: string, @@ -1080,6 +1254,51 @@ function normalizeStoredUserMessages(conversations: NormalizedConversation[]) { } } +function normalizeStoredMessagesForBackend( + conversations: NormalizedConversation[], + backend: ReplayBackend, +) { + if (backend === "capi") return; + + for (const conversation of conversations) { + conversation.messages = coalesceMessages( + conversation.messages, + backend !== "openai-completions", + ); + } +} + +function coalesceMessages( + messages: NormalizedMessage[], + coalesceAssistantMessages: boolean, +): NormalizedMessage[] { + const result: NormalizedMessage[] = []; + for (const message of messages) { + const previous = result.at(-1); + const shouldCoalesce = + previous?.role === message.role && + ((coalesceAssistantMessages && message.role === "assistant") || + message.role === "user"); + if (!shouldCoalesce) { + result.push(message); + continue; + } + + const separator = message.role === "user" ? "\n\n\n" : ""; + const previousContent = previous.content ?? ""; + const currentContent = message.content ?? ""; + const content = `${previousContent}${previousContent && currentContent ? separator : ""}${currentContent}`; + if (content) previous.content = content; + + const toolCalls = [ + ...(previous.tool_calls ?? []), + ...(message.tool_calls ?? []), + ]; + if (toolCalls.length) previous.tool_calls = toolCalls; + } + return result; +} + // Re-normalizes the built-in tool enumeration in stored tool results at load // time. Snapshots recorded before normalizeAvailableToolNames collapsed the // whole list (or recorded against an older tool set) still contain the literal @@ -1543,6 +1762,7 @@ type ReplayingCapiProxyState = { filePath: string; workDir: string; testInfo?: { file: string; line?: number }; + backend: ReplayBackend; storedData?: NormalizedData | undefined; toolResultNormalizers: ToolResultNormalizer[]; }; diff --git a/test/harness/responsesApiAdapter.ts b/test/harness/responsesApiAdapter.ts new file mode 100644 index 0000000000..16568f6e03 --- /dev/null +++ b/test/harness/responsesApiAdapter.ts @@ -0,0 +1,437 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import { randomUUID } from "node:crypto"; +import type { ChatCompletion } from "openai/resources/chat/completions"; +import type { Response as OpenAIResponse } from "openai/resources/responses/responses"; +import { + CanonicalMessage, + formatSseEvent, + functionToolCalls, + isObject, + JsonObject, +} from "./modelProtocolAdapterShared"; + +export const responsesEndpoint = "/responses"; + +type ResponsesRequest = { + model: string; + instructions?: string; + input?: string | JsonObject[]; + stream?: boolean; + tools?: JsonObject[]; + tool_choice?: unknown; + temperature?: number | null; + top_p?: number | null; + parallel_tool_calls?: boolean | null; +}; + +export type ResponsesApiResponse = OpenAIResponse; + +export function responsesApiRequestToChatCompletion( + requestBody: string, +): string { + const request = JSON.parse(requestBody) as ResponsesRequest; + const messages: CanonicalMessage[] = []; + if (request.instructions) { + messages.push({ role: "system", content: request.instructions }); + } + + if (typeof request.input === "string") { + messages.push({ role: "user", content: request.input }); + } else { + for (const item of request.input ?? []) { + const converted = responseInputItemToCanonicalMessages(item); + messages.push(...converted); + } + } + + return JSON.stringify({ + model: request.model, + messages: coalesceAssistantMessages(messages), + ...(request.tools ? { tools: convertResponsesTools(request.tools) } : {}), + ...(request.tool_choice !== undefined + ? { tool_choice: convertResponsesToolChoice(request.tool_choice) } + : {}), + ...(request.stream !== undefined ? { stream: request.stream } : {}), + ...(request.temperature !== undefined && request.temperature !== null + ? { temperature: request.temperature } + : {}), + ...(request.top_p !== undefined && request.top_p !== null + ? { top_p: request.top_p } + : {}), + ...(request.parallel_tool_calls !== undefined && + request.parallel_tool_calls !== null + ? { parallel_tool_calls: request.parallel_tool_calls } + : {}), + }); +} + +function responseInputItemToCanonicalMessages( + item: JsonObject, +): CanonicalMessage[] { + if (item.type === "function_call") { + const callId = + typeof item.call_id === "string" + ? item.call_id + : typeof item.id === "string" + ? item.id + : ""; + return [ + { + role: "assistant", + content: null, + tool_calls: [ + { + id: callId, + type: "function", + function: { + name: typeof item.name === "string" ? item.name : "", + arguments: + typeof item.arguments === "string" ? item.arguments : "{}", + }, + }, + ], + }, + ]; + } + + if (item.type === "function_call_output") { + return [ + { + role: "tool", + tool_call_id: typeof item.call_id === "string" ? item.call_id : "", + content: + typeof item.output === "string" + ? item.output + : JSON.stringify(item.output ?? ""), + }, + ]; + } + + if (item.type === "reasoning") return []; + + if ( + item.type !== "message" && + item.role !== "user" && + item.role !== "assistant" && + item.role !== "system" + ) { + return []; + } + + const role = + item.role === "assistant" || item.role === "system" ? item.role : "user"; + if (typeof item.content === "string") { + return [{ role, content: item.content }]; + } + if (!Array.isArray(item.content)) return [{ role, content: "" }]; + + const parts: unknown[] = []; + for (const part of item.content) { + if (!isObject(part)) continue; + if ( + (part.type === "input_text" || part.type === "output_text") && + typeof part.text === "string" + ) { + parts.push({ type: "text", text: part.text }); + } else if ( + part.type === "input_image" && + typeof part.image_url === "string" + ) { + parts.push({ + type: "image_url", + image_url: { + url: part.image_url, + ...(typeof part.detail === "string" ? { detail: part.detail } : {}), + }, + }); + } else if ( + part.type === "input_file" && + typeof part.file_data === "string" + ) { + parts.push({ + type: "file", + file: { + file_data: part.file_data, + ...(typeof part.filename === "string" + ? { filename: part.filename } + : {}), + }, + }); + } + } + + const onlyText = parts.every( + (part) => isObject(part) && part.type === "text", + ); + return [ + { + role, + content: onlyText + ? parts + .map((part) => + isObject(part) && typeof part.text === "string" ? part.text : "", + ) + .join("") + : parts, + }, + ]; +} + +function coalesceAssistantMessages( + messages: CanonicalMessage[], +): CanonicalMessage[] { + const result: CanonicalMessage[] = []; + for (const message of messages) { + const previous = result[result.length - 1]; + if (message.role === "assistant" && previous?.role === "assistant") { + const previousText = + typeof previous.content === "string" ? previous.content : ""; + const currentText = + typeof message.content === "string" ? message.content : ""; + previous.content = `${previousText}${currentText}` || null; + const toolCalls = [ + ...(previous.tool_calls ?? []), + ...(message.tool_calls ?? []), + ]; + if (toolCalls.length) previous.tool_calls = toolCalls; + } else { + result.push(message); + } + } + return result; +} + +function convertResponsesTools(tools: JsonObject[]): JsonObject[] { + return tools + .filter((tool) => tool.type === "function" && typeof tool.name === "string") + .map((tool) => ({ + type: "function", + function: { + name: tool.name, + ...(typeof tool.description === "string" + ? { description: tool.description } + : {}), + ...(isObject(tool.parameters) ? { parameters: tool.parameters } : {}), + ...(typeof tool.strict === "boolean" ? { strict: tool.strict } : {}), + }, + })); +} + +function convertResponsesToolChoice(toolChoice: unknown): unknown { + if ( + toolChoice === "auto" || + toolChoice === "none" || + toolChoice === "required" + ) { + return toolChoice; + } + if ( + isObject(toolChoice) && + toolChoice.type === "function" && + typeof toolChoice.name === "string" + ) { + return { + type: "function", + function: { name: toolChoice.name }, + }; + } + return undefined; +} + +export function chatCompletionResponseToResponsesApiMessage( + response: ChatCompletion, +): ResponsesApiResponse { + const output: ResponsesApiResponse["output"] = []; + const outputText: string[] = []; + + for (const choice of response.choices) { + if (choice.message.content) { + const text = choice.message.content; + outputText.push(text); + output.push({ + type: "message", + id: `msg_${randomUUID()}`, + role: "assistant", + status: "completed", + content: [ + { + type: "output_text", + text, + annotations: [], + }, + ], + }); + } + for (const toolCall of functionToolCalls(choice.message)) { + output.push({ + type: "function_call", + id: `fc_${toolCall.id}`, + call_id: toolCall.id, + name: toolCall.function.name, + arguments: toolCall.function.arguments, + status: "completed", + }); + } + } + + const finishReason = response.choices[0]?.finish_reason; + return { + id: response.id, + object: "response", + created_at: response.created, + model: response.model, + status: "completed", + output, + output_text: outputText.join(""), + incomplete_details: + finishReason === "length" + ? { reason: "max_output_tokens" } + : finishReason === "content_filter" + ? { reason: "content_filter" } + : null, + error: null, + instructions: null, + metadata: null, + parallel_tool_calls: false, + temperature: null, + tool_choice: "auto", + tools: [], + top_p: null, + usage: { + input_tokens: response.usage?.prompt_tokens ?? 0, + output_tokens: response.usage?.completion_tokens ?? 0, + total_tokens: response.usage?.total_tokens ?? 0, + input_tokens_details: { + cached_tokens: + response.usage?.prompt_tokens_details?.cached_tokens ?? 0, + }, + output_tokens_details: { + reasoning_tokens: + response.usage?.completion_tokens_details?.reasoning_tokens ?? 0, + }, + }, + }; +} + +export function chatCompletionResponseToResponsesApiSseChunks( + response: ChatCompletion, +): string[] { + const fullResponse = chatCompletionResponseToResponsesApiMessage(response); + const skeleton = { + ...fullResponse, + status: "in_progress" as const, + output: [], + output_text: "", + usage: undefined, + }; + const chunks: string[] = []; + let sequenceNumber = 0; + const event = (type: string, data: JsonObject) => + formatSseEvent(type, { + type, + sequence_number: sequenceNumber++, + ...data, + }); + + chunks.push( + event("response.created", { response: skeleton }), + event("response.in_progress", { response: skeleton }), + ); + + for ( + let outputIndex = 0; + outputIndex < fullResponse.output.length; + outputIndex++ + ) { + const item = fullResponse.output[outputIndex]; + const addedItem = + item.type === "message" + ? { ...item, status: "in_progress" as const, content: [] } + : item.type === "function_call" + ? { ...item, status: "in_progress" as const, arguments: "" } + : item; + chunks.push( + event("response.output_item.added", { + output_index: outputIndex, + item: addedItem, + }), + ); + + if (item.type === "message" && Array.isArray(item.content)) { + for ( + let contentIndex = 0; + contentIndex < item.content.length; + contentIndex++ + ) { + const part = item.content[contentIndex]; + chunks.push( + event("response.content_part.added", { + item_id: item.id, + output_index: outputIndex, + content_index: contentIndex, + part: + isObject(part) && part.type === "output_text" + ? { ...part, text: "" } + : part, + }), + ); + if ( + isObject(part) && + part.type === "output_text" && + typeof part.text === "string" + ) { + chunks.push( + event("response.output_text.delta", { + item_id: item.id, + output_index: outputIndex, + content_index: contentIndex, + delta: part.text, + logprobs: [], + }), + event("response.output_text.done", { + item_id: item.id, + output_index: outputIndex, + content_index: contentIndex, + text: part.text, + logprobs: [], + }), + ); + } + chunks.push( + event("response.content_part.done", { + item_id: item.id, + output_index: outputIndex, + content_index: contentIndex, + part, + }), + ); + } + } else if (item.type === "function_call") { + chunks.push( + event("response.function_call_arguments.delta", { + item_id: item.id, + output_index: outputIndex, + delta: item.arguments, + }), + event("response.function_call_arguments.done", { + item_id: item.id, + output_index: outputIndex, + arguments: item.arguments, + }), + ); + } + + chunks.push( + event("response.output_item.done", { + output_index: outputIndex, + item, + }), + ); + } + + chunks.push(event("response.completed", { response: fullResponse })); + return chunks; +}