From 09d489622780ada4290b515364fb3c50b2f8d581 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Tue, 14 Jul 2026 20:18:21 +0200 Subject: [PATCH 01/10] Add .NET BYOK E2E coverage Reuse conceptual replay snapshots across Anthropic Messages, OpenAI Responses, and OpenAI Chat Completions, and add three Ubuntu in-process CI legs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 85a40918-bd95-440b-b6e9-97c757a71f8c --- .github/workflows/dotnet-sdk-tests.yml | 80 ++- .../E2E/ByokBearerTokenProviderE2ETests.cs | 1 + dotnet/test/E2E/ClientE2ETests.cs | 16 +- dotnet/test/E2E/ClientOptionsE2ETests.cs | 29 +- .../E2E/CopilotRequestCancelErrorE2ETests.cs | 4 +- .../E2E/CopilotRequestSessionIdE2ETests.cs | 6 +- .../E2E/CopilotRequestWebSocketE2ETests.cs | 3 +- .../E2E/GitHubTelemetryForwardingE2ETests.cs | 3 +- dotnet/test/E2E/ModeEmptyE2ETests.cs | 12 +- dotnet/test/E2E/ModeHandlersE2ETests.cs | 5 +- .../MultiClientCommandsElicitationE2ETests.cs | 14 +- dotnet/test/E2E/MultiClientE2ETests.cs | 22 +- .../test/E2E/MultiProviderRegistryE2ETests.cs | 1 + dotnet/test/E2E/PendingWorkResumeE2ETests.cs | 24 +- dotnet/test/E2E/PerSessionAuthE2ETests.cs | 11 +- dotnet/test/E2E/PermissionE2ETests.cs | 4 +- dotnet/test/E2E/ProviderEndpointE2ETests.cs | 6 +- .../test/E2E/RpcExtensionsLoadedE2ETests.cs | 12 +- dotnet/test/E2E/RpcMcpAndSkillsE2ETests.cs | 10 +- dotnet/test/E2E/RpcServerE2ETests.cs | 12 +- dotnet/test/E2E/RpcSessionStateE2ETests.cs | 4 +- .../test/E2E/RpcSessionStateExtrasE2ETests.cs | 4 +- .../test/E2E/RpcTasksAndHandlersE2ETests.cs | 1 + dotnet/test/E2E/SessionConfigE2ETests.cs | 20 +- dotnet/test/E2E/SessionE2ETests.cs | 10 +- dotnet/test/E2E/SessionFsE2ETests.cs | 16 +- dotnet/test/E2E/SessionFsSqliteE2ETests.cs | 4 +- dotnet/test/E2E/StreamingFidelityE2ETests.cs | 7 +- dotnet/test/E2E/SubagentHooksE2ETests.cs | 2 +- dotnet/test/E2E/SuspendE2ETests.cs | 4 +- dotnet/test/E2E/TelemetryExportE2ETests.cs | 2 +- dotnet/test/E2E/ToolsE2ETests.cs | 4 +- dotnet/test/Harness/CapiProxy.cs | 9 +- dotnet/test/Harness/E2ETestBackend.cs | 97 +++ dotnet/test/Harness/E2ETestBase.cs | 4 +- dotnet/test/Harness/E2ETestContext.cs | 24 +- dotnet/test/Unit/E2ETestBackendTests.cs | 46 ++ test/harness/anthropicMessagesAdapter.ts | 579 ++++++++++++++++ test/harness/modelProtocolAdapters.test.ts | 619 +++++++++++++++++ test/harness/replayingCapiProxy.ts | 634 ++++++++++++++++-- test/harness/responsesApiAdapter.ts | 568 ++++++++++++++++ test/harness/sseParser.ts | 33 + 42 files changed, 2752 insertions(+), 214 deletions(-) create mode 100644 dotnet/test/Harness/E2ETestBackend.cs create mode 100644 dotnet/test/Unit/E2ETestBackendTests.cs create mode 100644 test/harness/anthropicMessagesAdapter.ts create mode 100644 test/harness/modelProtocolAdapters.test.ts create mode 100644 test/harness/responsesApiAdapter.ts create mode 100644 test/harness/sseParser.ts diff --git a/.github/workflows/dotnet-sdk-tests.yml b/.github/workflows/dotnet-sdk-tests.yml index dcf559228c..ce8e5890fa 100644 --- a/.github/workflows/dotnet-sdk-tests.yml +++ b/.github/workflows/dotnet-sdk-tests.yml @@ -7,19 +7,19 @@ on: pull_request: types: [opened, synchronize, reopened, ready_for_review] paths: - - 'dotnet/**' - - 'test/**' - - 'nodejs/package.json' - - '.github/workflows/dotnet-sdk-tests.yml' - - '!**/*.md' - - '!**/LICENSE*' - - '!**/.gitignore' - - '!**/.editorconfig' - - '!**/*.png' - - '!**/*.jpg' - - '!**/*.jpeg' - - '!**/*.gif' - - '!**/*.svg' + - "dotnet/**" + - "test/**" + - "nodejs/package.json" + - ".github/workflows/dotnet-sdk-tests.yml" + - "!**/*.md" + - "!**/LICENSE*" + - "!**/.gitignore" + - "!**/.editorconfig" + - "!**/*.png" + - "!**/*.jpg" + - "!**/*.jpeg" + - "!**/*.gif" + - "!**/*.svg" workflow_dispatch: merge_group: @@ -28,19 +28,49 @@ permissions: jobs: test: - name: ".NET SDK Tests (${{ matrix.os }}, ${{ matrix.transport }})" + name: ".NET SDK Tests (${{ matrix.os }}, ${{ matrix.transport }}, ${{ matrix.backend }})" if: github.event.repository.fork == false env: POWERSHELL_UPDATECHECK: Off + COPILOT_SDK_E2E_BACKEND: ${{ matrix.backend }} + DOTNET_TEST_FILTER: ${{ matrix.test-filter }} strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - transport: ["default", "inprocess"] - # TODO: Re-enable after fixing in-process sqlite file locking on shutdown on Windows - exclude: + include: + - os: ubuntu-latest + transport: default + backend: capi + test-filter: "" + - os: macos-latest + transport: default + backend: capi + test-filter: "" - os: windows-latest - transport: "inprocess" + transport: default + backend: capi + test-filter: "" + - os: ubuntu-latest + transport: inprocess + backend: capi + test-filter: "" + - os: macos-latest + transport: inprocess + backend: capi + test-filter: "" + # TODO: Add Windows after fixing in-process sqlite file locking on shutdown. + - os: ubuntu-latest + transport: inprocess + backend: anthropic-messages + test-filter: "FullyQualifiedName~GitHub.Copilot.Test.E2E&E2EBackend!=SelfConfiguredProvider&E2EBackend!=CapiOnly" + - os: ubuntu-latest + transport: inprocess + backend: openai-responses + test-filter: "FullyQualifiedName~GitHub.Copilot.Test.E2E&E2EBackend!=SelfConfiguredProvider&E2EBackend!=CapiOnly" + - os: ubuntu-latest + transport: inprocess + backend: openai-completions + test-filter: "FullyQualifiedName~GitHub.Copilot.Test.E2E&E2EBackend!=SelfConfiguredProvider&E2EBackend!=CapiOnly" runs-on: ${{ matrix.os }} defaults: run: @@ -81,6 +111,11 @@ jobs: working-directory: ./test/harness run: npm ci --ignore-scripts + - name: Run replay harness tests + if: matrix.os == 'ubuntu-latest' && matrix.transport == 'default' + working-directory: ./test/harness + run: npm test + - name: Warm up PowerShell if: runner.os == 'Windows' run: pwsh.exe -Command "Write-Host 'PowerShell ready'" @@ -92,4 +127,9 @@ jobs: - name: Run .NET SDK tests env: COPILOT_HMAC_KEY: ${{ secrets.COPILOT_DEVELOPER_CLI_INTEGRATION_HMAC_KEY }} - run: dotnet test --no-build -v n + run: | + args=(--no-build -v n) + if [[ -n "$DOTNET_TEST_FILTER" ]]; then + args+=(--filter "$DOTNET_TEST_FILTER") + fi + dotnet test "${args[@]}" diff --git a/dotnet/test/E2E/ByokBearerTokenProviderE2ETests.cs b/dotnet/test/E2E/ByokBearerTokenProviderE2ETests.cs index 5973dc61c6..26e19c009a 100644 --- a/dotnet/test/E2E/ByokBearerTokenProviderE2ETests.cs +++ b/dotnet/test/E2E/ByokBearerTokenProviderE2ETests.cs @@ -36,6 +36,7 @@ namespace GitHub.Copilot.Test.E2E; /// and the resulting token reaches that provider's endpoint. /// /// +[Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public class ByokBearerTokenProviderE2ETests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "byok_bearer_token_provider", output) { diff --git a/dotnet/test/E2E/ClientE2ETests.cs b/dotnet/test/E2E/ClientE2ETests.cs index 5de6ce159a..d166223d65 100644 --- a/dotnet/test/E2E/ClientE2ETests.cs +++ b/dotnet/test/E2E/ClientE2ETests.cs @@ -9,8 +9,10 @@ namespace GitHub.Copilot.Test.E2E; // These tests bypass E2ETestBase because they are about how the CLI subprocess is started // Other test classes should instead inherit from E2ETestBase -public class ClientE2ETests +public class ClientE2ETests(E2ETestFixture fixture) : IClassFixture { + private E2ETestContext Ctx => fixture.Ctx; + [Theory] [InlineData(true)] // stdio transport [InlineData(false)] // TCP transport @@ -66,7 +68,7 @@ public async Task Should_Force_Stop_Without_Cleanup(bool useStdio) { using var client = new CopilotClient(new CopilotClientOptions { Connection = useStdio ? RuntimeConnection.ForStdio() : RuntimeConnection.ForTcp() }); - await client.CreateSessionAsync(new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll }); + await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll }); await client.ForceStopAsync(); } @@ -165,7 +167,7 @@ public async Task Should_List_Models_When_Authenticated(bool useStdio) public async Task Should_Not_Throw_When_Disposing_Session_After_Stopping_Client(bool useStdio) { await using var client = new CopilotClient(new CopilotClientOptions { Connection = useStdio ? RuntimeConnection.ForStdio() : RuntimeConnection.ForTcp() }); - await using var session = await client.CreateSessionAsync(new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll }); + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll }); await client.StopAsync(); } @@ -201,7 +203,7 @@ public async Task Should_Report_Error_With_Stderr_When_CLI_Fails_To_Start(bool u // Verify subsequent calls also fail (don't hang) var ex2 = await Assert.ThrowsAnyAsync(async () => { - var session = await client.CreateSessionAsync(new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll }); + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll }); await session.SendAsync(new MessageOptions { Prompt = "test" }); }); Assert.True( @@ -219,7 +221,7 @@ public async Task Should_Report_Error_With_Stderr_When_CLI_Fails_To_Start(bool u public async Task Should_Allow_CreateSession_Called_Without_PermissionHandler(bool useStdio) { await using var client = new CopilotClient(new CopilotClientOptions { Connection = useStdio ? RuntimeConnection.ForStdio() : RuntimeConnection.ForTcp() }); - await using var session = await client.CreateSessionAsync(new SessionConfig()); + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig()); Assert.NotNull(session.SessionId); } @@ -234,7 +236,7 @@ public async Task Should_Allow_ResumeSession_Called_Without_PermissionHandler() { Connection = RuntimeConnection.ForTcp(connectionToken: connectionToken), }); - await using var originalSession = await client.CreateSessionAsync(new SessionConfig()); + await using var originalSession = await ctx.CreateSessionAsync(client, new SessionConfig()); var port = client.RuntimePort ?? throw new InvalidOperationException("Client must be using TCP transport to support multi-client resume."); @@ -243,7 +245,7 @@ public async Task Should_Allow_ResumeSession_Called_Without_PermissionHandler() { Connection = RuntimeConnection.ForUri($"localhost:{port}", connectionToken: connectionToken), }); - await using var resumedSession = await resumeClient.ResumeSessionAsync(originalSession.SessionId, new()); + await using var resumedSession = await ctx.ResumeSessionAsync(resumeClient, originalSession.SessionId, new()); Assert.Equal(originalSession.SessionId, resumedSession.SessionId); } diff --git a/dotnet/test/E2E/ClientOptionsE2ETests.cs b/dotnet/test/E2E/ClientOptionsE2ETests.cs index a8ceda5b08..4489ef442f 100644 --- a/dotnet/test/E2E/ClientOptionsE2ETests.cs +++ b/dotnet/test/E2E/ClientOptionsE2ETests.cs @@ -7,6 +7,7 @@ using System.Net; using System.Net.Sockets; using System.Text.Json; +using GitHub.Copilot.Test.Harness; using Xunit; using Xunit.Abstractions; @@ -41,7 +42,7 @@ public async Task Should_Use_Client_Cwd_For_Default_WorkingDirectory() WorkingDirectory = clientCwd, }); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -110,7 +111,7 @@ public async Task Should_Propagate_Process_Options_To_Spawned_Cli() Assert.Equal("dotnet-sdk-e2e", capturedEnv.GetProperty("COPILOT_OTEL_SOURCE_NAME").GetString()); Assert.Equal("true", capturedEnv.GetProperty("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT").GetString()); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { EnableConfigDiscovery = true, IncludeSubAgentStreamingEvents = false, @@ -139,7 +140,7 @@ public async Task Should_Forward_EnableSessionTelemetry_In_Wire_Request() await client.StartAsync(); // When explicitly set to false, it should appear in the wire request - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { EnableSessionTelemetry = false, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -166,7 +167,7 @@ public async Task Should_Omit_EnableSessionTelemetry_When_Not_Set() await client.StartAsync(); // When omitted (null/default), the field should not be present in the wire request - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -191,7 +192,7 @@ public async Task Should_Forward_Granular_Multitenancy_Fields_In_Create_Wire_Req await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { SkipEmbeddingRetrieval = false, OrganizationCustomInstructions = "Follow org policy.", @@ -219,6 +220,7 @@ public async Task Should_Forward_Granular_Multitenancy_Fields_In_Create_Wire_Req } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public async Task Should_Forward_Advanced_Session_Options_In_Create_Wire_Request() { var (cliPath, capturePath) = await CreateFakeCliCaptureAsync(); @@ -232,7 +234,7 @@ public async Task Should_Forward_Advanced_Session_Options_In_Create_Wire_Request await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { ClientName = "advanced-create-client", Model = "claude-sonnet-4.5", @@ -366,6 +368,7 @@ public async Task Should_Forward_Advanced_Session_Options_In_Create_Wire_Request } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public async Task Should_Forward_Singular_Provider_Options_In_Create_Wire_Request() { var (cliPath, capturePath) = await CreateFakeCliCaptureAsync(); @@ -378,7 +381,7 @@ public async Task Should_Forward_Singular_Provider_Options_In_Create_Wire_Reques await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { Model = "claude-sonnet-4.5", Provider = new ProviderConfig @@ -432,7 +435,7 @@ public async Task Should_Apply_Empty_Mode_Defaults_To_CreateSession_Wire_Request await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, AvailableTools = new ToolSet().AddBuiltIn(BuiltInTools.Isolated), @@ -471,7 +474,7 @@ public async Task Should_Propagate_Activity_TraceContext_To_Session_Create_And_S activity.TraceStateString = "vendor=create-send"; activity.Start(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -555,7 +558,7 @@ public async Task Should_Propagate_Activity_TraceContext_To_Session_Resume() activity.TraceStateString = "vendor=resume"; activity.Start(); - var session = await client.ResumeSessionAsync("trace-resume-session", new ResumeSessionConfig + var session = await Ctx.ResumeSessionAsync(client, "trace-resume-session", new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -582,7 +585,7 @@ public async Task Should_Forward_Granular_Multitenancy_Fields_In_Resume_Wire_Req await client.StartAsync(); - var session = await client.ResumeSessionAsync("resume-session", new ResumeSessionConfig + var session = await Ctx.ResumeSessionAsync(client, "resume-session", new ResumeSessionConfig { SkipEmbeddingRetrieval = false, OrganizationCustomInstructions = "Resume org policy.", @@ -624,7 +627,7 @@ public async Task Should_Forward_Advanced_Session_Options_In_Resume_Wire_Request await client.StartAsync(); - var session = await client.ResumeSessionAsync("advanced-resume-session", new ResumeSessionConfig + var session = await Ctx.ResumeSessionAsync(client, "advanced-resume-session", new ResumeSessionConfig { ClientName = "advanced-resume-client", Model = "claude-haiku-4.5", @@ -706,7 +709,7 @@ public async Task Should_Apply_Empty_Mode_Defaults_To_ResumeSession_Wire_Request await client.StartAsync(); - var session = await client.ResumeSessionAsync("resume-empty-session", new ResumeSessionConfig + var session = await Ctx.ResumeSessionAsync(client, "resume-empty-session", new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, AvailableTools = new ToolSet().AddBuiltIn(BuiltInTools.Isolated), diff --git a/dotnet/test/E2E/CopilotRequestCancelErrorE2ETests.cs b/dotnet/test/E2E/CopilotRequestCancelErrorE2ETests.cs index 99e9e2546e..a9b645d2ac 100644 --- a/dotnet/test/E2E/CopilotRequestCancelErrorE2ETests.cs +++ b/dotnet/test/E2E/CopilotRequestCancelErrorE2ETests.cs @@ -54,7 +54,7 @@ public async Task Reports_A_Thrown_Callback_Error_Instead_Of_Hanging() await using var client = CreateClientWith(handler); await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -81,7 +81,7 @@ public async Task Observes_Runtime_Cancellation_Of_An_In_Flight_Inference_Reques await using var client = CreateClientWith(handler); await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); diff --git a/dotnet/test/E2E/CopilotRequestSessionIdE2ETests.cs b/dotnet/test/E2E/CopilotRequestSessionIdE2ETests.cs index 08dd075643..1aa1442a7d 100644 --- a/dotnet/test/E2E/CopilotRequestSessionIdE2ETests.cs +++ b/dotnet/test/E2E/CopilotRequestSessionIdE2ETests.cs @@ -28,13 +28,14 @@ private CopilotClient CreateClientWith(RecordingRequestHandler provider) => }); [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Threads_The_Session_Id_Into_A_Capi_Session_Inference_Request() { var provider = new RecordingRequestHandler(); await using var client = CreateClientWith(provider); await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -64,13 +65,14 @@ public async Task Threads_The_Session_Id_Into_A_Capi_Session_Inference_Request() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public async Task Threads_The_Session_Id_Into_A_Byok_Session_Inference_Request() { var provider = new RecordingRequestHandler(); await using var client = CreateClientWith(provider); await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, // BYOK providers require an explicit model id. diff --git a/dotnet/test/E2E/CopilotRequestWebSocketE2ETests.cs b/dotnet/test/E2E/CopilotRequestWebSocketE2ETests.cs index 9bb19df7d5..c1cc92fc52 100644 --- a/dotnet/test/E2E/CopilotRequestWebSocketE2ETests.cs +++ b/dotnet/test/E2E/CopilotRequestWebSocketE2ETests.cs @@ -31,6 +31,7 @@ namespace GitHub.Copilot.Test.E2E; /// message. Without the eager start the turn never completes and this test /// times out. /// +[Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public class CopilotRequestWebSocketE2ETests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "copilot_request_websocket", output) { @@ -54,7 +55,7 @@ public async Task Services_A_WebSocket_Turn_End_To_End_Via_The_Request_Handler() }, environment: env); await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); diff --git a/dotnet/test/E2E/GitHubTelemetryForwardingE2ETests.cs b/dotnet/test/E2E/GitHubTelemetryForwardingE2ETests.cs index 343c4815b7..16312a90b2 100644 --- a/dotnet/test/E2E/GitHubTelemetryForwardingE2ETests.cs +++ b/dotnet/test/E2E/GitHubTelemetryForwardingE2ETests.cs @@ -12,6 +12,7 @@ namespace GitHub.Copilot.Test.E2E; #pragma warning disable GHCP001 // GitHub telemetry forwarding is experimental. +[Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public class GitHubTelemetryForwardingE2ETests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "github_telemetry", output) { @@ -32,7 +33,7 @@ public async Task Should_Forward_GitHub_Telemetry_For_A_Live_Session() CopilotSession? session = null; try { - session = await client.CreateSessionAsync(new SessionConfig + session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); diff --git a/dotnet/test/E2E/ModeEmptyE2ETests.cs b/dotnet/test/E2E/ModeEmptyE2ETests.cs index df1bbc857d..433e6a745c 100644 --- a/dotnet/test/E2E/ModeEmptyE2ETests.cs +++ b/dotnet/test/E2E/ModeEmptyE2ETests.cs @@ -21,7 +21,7 @@ public class ModeEmptyE2ETests(E2ETestFixture fixture, ITestOutputHelper output) public async Task Empty_Mode_Isolated_Set_Shell_Tool_Is_Not_Exposed() { await using var client = Ctx.CreateClient(options: EmptyModeOptions(Ctx)); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, AvailableTools = new ToolSet().AddBuiltIn(BuiltInTools.Isolated), @@ -45,7 +45,7 @@ public async Task Empty_Mode_Isolated_Set_Shell_Tool_Is_Not_Exposed() public async Task Empty_Mode_Builtin_Star_Exposes_All_Built_In_Tools() { await using var client = Ctx.CreateClient(options: EmptyModeOptions(Ctx)); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, AvailableTools = new ToolSet().AddBuiltIn("*"), @@ -65,7 +65,7 @@ public async Task Empty_Mode_Excluded_Tools_Subtracts_From_Available_Tools() { var shellToolName = OperatingSystem.IsWindows() ? "powershell" : "bash"; await using var client = Ctx.CreateClient(options: EmptyModeOptions(Ctx)); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, AvailableTools = new ToolSet().AddBuiltIn("*"), @@ -85,7 +85,7 @@ public async Task Empty_Mode_Excluded_Tools_Subtracts_From_Available_Tools() public async Task Empty_Mode_Strips_Environment_Context_From_The_System_Message_By_Default() { await using var client = Ctx.CreateClient(options: EmptyModeOptions(Ctx)); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, AvailableTools = new ToolSet().AddBuiltIn(BuiltInTools.Isolated), @@ -109,7 +109,7 @@ public async Task Empty_Mode_Strips_Environment_Context_From_The_System_Message_ public async Task Empty_Mode_System_Message_Replace_Llm_Follows_Caller_Content_Verbatim() { await using var client = Ctx.CreateClient(options: EmptyModeOptions(Ctx)); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, AvailableTools = new ToolSet().AddBuiltIn(BuiltInTools.Isolated), @@ -128,7 +128,7 @@ public async Task Empty_Mode_System_Message_Replace_Llm_Follows_Caller_Content_V public async Task Empty_Mode_Append_Caller_Instruction_Takes_Effect_And_Env_Context_Stripped() { await using var client = Ctx.CreateClient(options: EmptyModeOptions(Ctx)); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, AvailableTools = new ToolSet().AddBuiltIn(BuiltInTools.Isolated), diff --git a/dotnet/test/E2E/ModeHandlersE2ETests.cs b/dotnet/test/E2E/ModeHandlersE2ETests.cs index a397999f73..47e80192e9 100644 --- a/dotnet/test/E2E/ModeHandlersE2ETests.cs +++ b/dotnet/test/E2E/ModeHandlersE2ETests.cs @@ -8,6 +8,7 @@ namespace GitHub.Copilot.Test.E2E; +[Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public class ModeHandlersE2ETests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "mode_handlers", output) { @@ -24,7 +25,7 @@ public async Task Should_Invoke_Exit_Plan_Mode_Handler_When_Model_Uses_Tool() TaskCreationOptions.RunContinuationsAsynchronously); await using var client = CreateAuthenticatedClient(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { GitHubToken = Token, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -92,7 +93,7 @@ public async Task Should_Invoke_Auto_Mode_Switch_Handler_When_Rate_Limited() TaskCreationOptions.RunContinuationsAsynchronously); await using var client = CreateAuthenticatedClient(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { GitHubToken = Token, OnPermissionRequest = PermissionHandler.ApproveAll, diff --git a/dotnet/test/E2E/MultiClientCommandsElicitationE2ETests.cs b/dotnet/test/E2E/MultiClientCommandsElicitationE2ETests.cs index d60c21709c..d869dc8164 100644 --- a/dotnet/test/E2E/MultiClientCommandsElicitationE2ETests.cs +++ b/dotnet/test/E2E/MultiClientCommandsElicitationE2ETests.cs @@ -59,7 +59,7 @@ public async Task InitializeAsync() await Ctx.ConfigureForTestAsync("multi_client", _testName); // Trigger connection so we can read the port - var initSession = await Client1.CreateSessionAsync(new SessionConfig + var initSession = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -102,7 +102,7 @@ public async Task DisposeAsync() [Fact] public async Task Client_Receives_Commands_Changed_When_Another_Client_Joins_With_Commands() { - var session1 = await Client1.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -120,7 +120,7 @@ public async Task Client_Receives_Commands_Changed_When_Another_Client_Joins_Wit }); // Client2 joins with commands - var session2 = await Client2.ResumeSessionAsync(session1.SessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(Client2, session1.SessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Commands = @@ -148,7 +148,7 @@ public async Task Client_Receives_Commands_Changed_When_Another_Client_Joins_Wit public async Task Capabilities_Changed_Fires_When_Second_Client_Joins_With_Elicitation_Handler() { // Client1 creates session without elicitation - var session1 = await Client1.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -168,7 +168,7 @@ public async Task Capabilities_Changed_Fires_When_Second_Client_Joins_With_Elici }); // Client2 joins WITH elicitation handler — triggers capabilities.changed - var session2 = await Client2.ResumeSessionAsync(session1.SessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(Client2, session1.SessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, OnElicitationRequest = _ => Task.FromResult(new ElicitationResult @@ -194,7 +194,7 @@ public async Task Capabilities_Changed_Fires_When_Second_Client_Joins_With_Elici public async Task Capabilities_Changed_Fires_When_Elicitation_Provider_Disconnects() { // Client1 creates session without elicitation - var session1 = await Client1.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -222,7 +222,7 @@ public async Task Capabilities_Changed_Fires_When_Elicitation_Provider_Disconnec }); // Client3 joins WITH elicitation handler - await _client3.ResumeSessionAsync(session1.SessionId, new ResumeSessionConfig + await Ctx.ResumeSessionAsync(_client3, session1.SessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, OnElicitationRequest = _ => Task.FromResult(new ElicitationResult diff --git a/dotnet/test/E2E/MultiClientE2ETests.cs b/dotnet/test/E2E/MultiClientE2ETests.cs index faaf383935..4dbe7190ad 100644 --- a/dotnet/test/E2E/MultiClientE2ETests.cs +++ b/dotnet/test/E2E/MultiClientE2ETests.cs @@ -58,7 +58,7 @@ public async Task InitializeAsync() await Ctx.ConfigureForTestAsync("multi_client", _testName); // Trigger connection so we can read the port - var initSession = await Client1.CreateSessionAsync(new SessionConfig + var initSession = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -96,13 +96,13 @@ public async Task Both_Clients_See_Tool_Request_And_Completion_Events() { var tool = AIFunctionFactory.Create(MagicNumber, "magic_number"); - var session1 = await Client1.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Tools = [tool], }); - var session2 = await Client2.ResumeSessionAsync(session1.SessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(Client2, session1.SessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -148,7 +148,7 @@ public async Task One_Client_Approves_Permission_And_Both_See_The_Result() { var client1PermissionRequests = new List(); - var session1 = await Client1.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = (request, _) => { @@ -158,7 +158,7 @@ public async Task One_Client_Approves_Permission_And_Both_See_The_Result() }); // Client 2 resumes — its handler never completes, so only client 1's approval takes effect - var session2 = await Client2.ResumeSessionAsync(session1.SessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(Client2, session1.SessionId, new ResumeSessionConfig { OnPermissionRequest = (_, _) => new TaskCompletionSource().Task, }); @@ -200,13 +200,13 @@ await session1.SendAsync(new MessageOptions [Fact] public async Task One_Client_Rejects_Permission_And_Both_See_The_Result() { - var session1 = await Client1.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = (_, _) => Task.FromResult(PermissionDecision.Reject()), }); // Client 2 resumes — its handler never completes - var session2 = await Client2.ResumeSessionAsync(session1.SessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(Client2, session1.SessionId, new ResumeSessionConfig { OnPermissionRequest = (_, _) => new TaskCompletionSource().Task, }); @@ -252,13 +252,13 @@ public async Task Two_Clients_Register_Different_Tools_And_Agent_Uses_Both() var toolA = AIFunctionFactory.Create(CityLookup, "city_lookup"); var toolB = AIFunctionFactory.Create(CurrencyLookup, "currency_lookup"); - var session1 = await Client1.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Tools = [toolA], }); - var session2 = await Client2.ResumeSessionAsync(session1.SessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(Client2, session1.SessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Tools = [toolB], @@ -294,13 +294,13 @@ public async Task Disconnecting_Client_Removes_Its_Tools() var toolA = AIFunctionFactory.Create(StableTool, "stable_tool"); var toolB = AIFunctionFactory.Create(EphemeralTool, "ephemeral_tool"); - var session1 = await Client1.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(Client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Tools = [toolA], }); - await Client2.ResumeSessionAsync(session1.SessionId, new ResumeSessionConfig + await Ctx.ResumeSessionAsync(Client2, session1.SessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Tools = [toolB], diff --git a/dotnet/test/E2E/MultiProviderRegistryE2ETests.cs b/dotnet/test/E2E/MultiProviderRegistryE2ETests.cs index 8a75cc3c1a..f04398a7ff 100644 --- a/dotnet/test/E2E/MultiProviderRegistryE2ETests.cs +++ b/dotnet/test/E2E/MultiProviderRegistryE2ETests.cs @@ -18,6 +18,7 @@ namespace GitHub.Copilot.Test.E2E; /// session, be launched, and route inference to the configured provider with /// the configured wire model and headers. /// +[Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public class MultiProviderRegistryE2ETests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "multi_provider_registry", output) { diff --git a/dotnet/test/E2E/PendingWorkResumeE2ETests.cs b/dotnet/test/E2E/PendingWorkResumeE2ETests.cs index b330795905..b3ca218190 100644 --- a/dotnet/test/E2E/PendingWorkResumeE2ETests.cs +++ b/dotnet/test/E2E/PendingWorkResumeE2ETests.cs @@ -30,7 +30,7 @@ public async Task Should_Continue_Pending_Permission_Request_After_Resume() var cliUrl = GetCliUrl(server); using var suspendedClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) }); - var session1 = await suspendedClient.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(suspendedClient, new SessionConfig { Tools = [AIFunctionFactory.Create(ResumePermissionTool, "resume_permission_tool")], OnPermissionRequest = (request, _) => @@ -57,7 +57,7 @@ await session1.SendAsync(new MessageOptions await suspendedClient.ForceStopAsync(); await using var resumedTcpClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) }); - var session2 = await resumedTcpClient.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(resumedTcpClient, sessionId, new ResumeSessionConfig { ContinuePendingWork = true, OnPermissionRequest = (_, _) => Task.FromResult(PermissionDecision.NoResult()), @@ -99,7 +99,7 @@ public async Task Should_Continue_Pending_External_Tool_Request_After_Resume() var cliUrl = GetCliUrl(server); using var suspendedClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) }); - var session1 = await suspendedClient.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(suspendedClient, new SessionConfig { Tools = [AIFunctionFactory.Create(BlockingExternalTool, "resume_external_tool")], OnPermissionRequest = PermissionHandler.ApproveAll, @@ -120,7 +120,7 @@ await session1.SendAsync(new MessageOptions await suspendedClient.ForceStopAsync(); await using var resumedClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) }); - var session2 = await resumedClient.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(resumedClient, sessionId, new ResumeSessionConfig { ContinuePendingWork = true, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -175,7 +175,7 @@ private async Task AssertPendingExternalToolHandleableOnResumeAsync( var cliUrl = GetCliUrl(server); using var suspendedClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) }); - var session1 = await suspendedClient.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(suspendedClient, new SessionConfig { Tools = [AIFunctionFactory.Create(BlockingExternalTool, "resume_external_tool")], OnPermissionRequest = PermissionHandler.ApproveAll, @@ -216,7 +216,7 @@ await session1.SendAsync(new MessageOptions resumeConfig.Tools = [AIFunctionFactory.Create(ResumedExternalTool, "resume_external_tool")]; } - var session2 = await resumedClient.ResumeSessionAsync(sessionId, resumeConfig); + var session2 = await Ctx.ResumeSessionAsync(resumedClient, sessionId, resumeConfig); var resumeEvent = await GetSingleResumeEventAsync(session2); Assert.Equal(false, resumeEvent.Data.ContinuePendingWork); @@ -276,7 +276,7 @@ public async Task Should_Continue_Parallel_Pending_External_Tool_Requests_After_ var cliUrl = GetCliUrl(server); using var suspendedClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) }); - var session1 = await suspendedClient.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(suspendedClient, new SessionConfig { Tools = [ @@ -306,7 +306,7 @@ await Task.WhenAll( await suspendedClient.ForceStopAsync(); await using var resumedClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) }); - var session2 = await resumedClient.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(resumedClient, sessionId, new ResumeSessionConfig { ContinuePendingWork = true, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -357,7 +357,7 @@ public async Task Should_Resume_Successfully_When_No_Pending_Work_Exists() string sessionId; await using (var firstClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) })) { - var firstSession = await firstClient.CreateSessionAsync(new SessionConfig + var firstSession = await Ctx.CreateSessionAsync(firstClient, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -370,7 +370,7 @@ public async Task Should_Resume_Successfully_When_No_Pending_Work_Exists() } await using var resumedClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) }); - var resumedSession = await resumedClient.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var resumedSession = await Ctx.ResumeSessionAsync(resumedClient, sessionId, new ResumeSessionConfig { ContinuePendingWork = true, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -395,7 +395,7 @@ public async Task Should_Report_ContinuePendingWork_True_In_Resume_Event() string sessionId; await using (var firstClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) })) { - var firstSession = await firstClient.CreateSessionAsync(new SessionConfig + var firstSession = await Ctx.CreateSessionAsync(firstClient, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -411,7 +411,7 @@ public async Task Should_Report_ContinuePendingWork_True_In_Resume_Event() } await using var resumedClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: SharedToken) }); - var resumedSession = await resumedClient.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var resumedSession = await Ctx.ResumeSessionAsync(resumedClient, sessionId, new ResumeSessionConfig { ContinuePendingWork = true, OnPermissionRequest = PermissionHandler.ApproveAll, diff --git a/dotnet/test/E2E/PerSessionAuthE2ETests.cs b/dotnet/test/E2E/PerSessionAuthE2ETests.cs index d1226f3737..4b370768a1 100644 --- a/dotnet/test/E2E/PerSessionAuthE2ETests.cs +++ b/dotnet/test/E2E/PerSessionAuthE2ETests.cs @@ -8,6 +8,7 @@ namespace GitHub.Copilot.Test.E2E; +[Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public class PerSessionAuthE2ETests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "per-session-auth", output) { /// @@ -74,7 +75,7 @@ public async Task ShouldAuthenticateWithGitHubToken() { await SetupCopilotUsersAsync(); - await using var session = await AuthClient.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(AuthClient, new SessionConfig { GitHubToken = "token-alice", OnPermissionRequest = PermissionHandler.ApproveAll, @@ -90,13 +91,13 @@ public async Task ShouldIsolateAuthBetweenSessions() { await SetupCopilotUsersAsync(); - await using var sessionA = await AuthClient.CreateSessionAsync(new SessionConfig + await using var sessionA = await Ctx.CreateSessionAsync(AuthClient, new SessionConfig { GitHubToken = "token-alice", OnPermissionRequest = PermissionHandler.ApproveAll, }); - await using var sessionB = await AuthClient.CreateSessionAsync(new SessionConfig + await using var sessionB = await Ctx.CreateSessionAsync(AuthClient, new SessionConfig { GitHubToken = "token-bob", OnPermissionRequest = PermissionHandler.ApproveAll, @@ -116,7 +117,7 @@ public async Task ShouldBeUnauthenticatedWithoutToken() { var noAuthClient = CreateNoAuthTestClient(); - await using var session = await noAuthClient.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(noAuthClient, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -131,7 +132,7 @@ public async Task ShouldFailWithInvalidToken() { await SetupCopilotUsersAsync(); - var ex = await Assert.ThrowsAnyAsync(() => AuthClient.CreateSessionAsync(new SessionConfig + var ex = await Assert.ThrowsAnyAsync(() => Ctx.CreateSessionAsync(AuthClient, new SessionConfig { GitHubToken = "invalid-token", OnPermissionRequest = PermissionHandler.ApproveAll, diff --git a/dotnet/test/E2E/PermissionE2ETests.cs b/dotnet/test/E2E/PermissionE2ETests.cs index 8c1904701a..2225dcba89 100644 --- a/dotnet/test/E2E/PermissionE2ETests.cs +++ b/dotnet/test/E2E/PermissionE2ETests.cs @@ -205,7 +205,7 @@ public async Task Should_Resume_Session_With_Permission_Handler() await session1.DisposeAsync(); // Resume with permission handler - var session2 = await Client.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(Client, sessionId, new ResumeSessionConfig { OnPermissionRequest = (request, invocation) => { @@ -279,7 +279,7 @@ public async Task Should_Deny_Tool_Operations_When_Handler_Explicitly_Denies_Aft await session1.SendAndWaitAsync(new MessageOptions { Prompt = "What is 1+1?" }); await session1.DisposeAsync(); - var session2 = await Client.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(Client, sessionId, new ResumeSessionConfig { OnPermissionRequest = (_, _) => Task.FromResult(PermissionDecision.UserNotAvailable()) diff --git a/dotnet/test/E2E/ProviderEndpointE2ETests.cs b/dotnet/test/E2E/ProviderEndpointE2ETests.cs index 7ea4eecf39..2cc4a47425 100644 --- a/dotnet/test/E2E/ProviderEndpointE2ETests.cs +++ b/dotnet/test/E2E/ProviderEndpointE2ETests.cs @@ -27,11 +27,12 @@ private CopilotClient CreateProviderEndpointClient() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public async Task ShouldReturnByokProviderEndpointWhenCustomProviderIsConfigured() { var client = CreateProviderEndpointClient(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Provider = new ProviderConfig @@ -64,11 +65,12 @@ public async Task ShouldReturnByokProviderEndpointWhenCustomProviderIsConfigured } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task ShouldReturnCapiProviderEndpointForOAuthAuthenticatedSession() { var client = CreateProviderEndpointClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); diff --git a/dotnet/test/E2E/RpcExtensionsLoadedE2ETests.cs b/dotnet/test/E2E/RpcExtensionsLoadedE2ETests.cs index af959bd7eb..0a3513e4b5 100644 --- a/dotnet/test/E2E/RpcExtensionsLoadedE2ETests.cs +++ b/dotnet/test/E2E/RpcExtensionsLoadedE2ETests.cs @@ -189,7 +189,7 @@ public async Task Discovers_Loads_And_Reports_Running_Extension(string sourceVal await using var client = CreateExtensionsClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { EnableConfigDiscovery = true, WorkingDirectory = workingDirectory, @@ -214,7 +214,7 @@ public async Task Disable_Then_Enable_Cycles_Extension_Status() await using var client = CreateExtensionsClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { EnableConfigDiscovery = true, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -240,7 +240,7 @@ public async Task Reload_Picks_Up_Extension_Added_After_Session_Create() // Start the session BEFORE writing the extension so the initial discovery sees nothing. await using var client = CreateExtensionsClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { EnableConfigDiscovery = true, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -285,7 +285,7 @@ public async Task Failed_Extension_Reports_Failed_Status() await using var client = CreateExtensionsClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { EnableConfigDiscovery = true, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -306,7 +306,7 @@ public async Task Multiple_Extensions_Are_Discovered_Independently() await using var client = CreateExtensionsClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { EnableConfigDiscovery = true, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -328,7 +328,7 @@ public async Task Reload_Preserves_Disabled_State_Across_Calls() await using var client = CreateExtensionsClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { EnableConfigDiscovery = true, OnPermissionRequest = PermissionHandler.ApproveAll, diff --git a/dotnet/test/E2E/RpcMcpAndSkillsE2ETests.cs b/dotnet/test/E2E/RpcMcpAndSkillsE2ETests.cs index 350aac4274..0d2942d4bf 100644 --- a/dotnet/test/E2E/RpcMcpAndSkillsE2ETests.cs +++ b/dotnet/test/E2E/RpcMcpAndSkillsE2ETests.cs @@ -189,7 +189,7 @@ public async Task Should_List_Extensions() { Connection = RuntimeConnection.ForStdio(args: ["--yolo"]), }); - await using var session = await yoloClient.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(yoloClient, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -208,7 +208,7 @@ public async Task Should_List_Extensions() public async Task Should_Round_Trip_Mcp_App_Host_Context() { await using var client = CreateMcpAppsClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -253,7 +253,7 @@ public async Task Should_Diagnose_And_Report_Mcp_App_Capability_Errors() new Dictionary { ["MCP_APP_RPC_VALUE"] = "from-app-rpc" }; await using var client = CreateMcpAppsClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { McpServers = mcpServers, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -292,7 +292,7 @@ public async Task Should_Report_Error_When_Mcp_App_Resource_Is_Not_Available() { const string serverName = "rpc-apps-resource-server"; await using var client = CreateMcpAppsClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { McpServers = CreateTestMcpServers(serverName), OnPermissionRequest = PermissionHandler.ApproveAll, @@ -368,7 +368,7 @@ public async Task Should_Report_Error_When_Extensions_Are_Not_Available() { Connection = RuntimeConnection.ForStdio(args: ["--yolo"]), }); - await using var session = await yoloClient.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(yoloClient, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); diff --git a/dotnet/test/E2E/RpcServerE2ETests.cs b/dotnet/test/E2E/RpcServerE2ETests.cs index 47df256157..2df8593cc4 100644 --- a/dotnet/test/E2E/RpcServerE2ETests.cs +++ b/dotnet/test/E2E/RpcServerE2ETests.cs @@ -160,6 +160,7 @@ public async Task Should_Reject_Llm_Inference_Response_Frames_For_Missing_Reques } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Call_Rpc_Models_List_With_Typed_Result() { const string token = "rpc-models-token"; @@ -175,6 +176,7 @@ public async Task Should_Call_Rpc_Models_List_With_Typed_Result() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Call_Rpc_Account_GetQuota_When_Authenticated() { const string token = "rpc-quota-token"; @@ -256,7 +258,7 @@ public async Task Should_List_Find_And_Inspect_Persisted_Session_State() var missingTaskId = $"missing-task-{Guid.NewGuid():N}"; var missingSessionId = Guid.NewGuid().ToString(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { SessionId = sessionId, WorkingDirectory = workingDirectory, @@ -308,7 +310,7 @@ public async Task Should_Enrich_Basic_Session_Metadata() var sessionId = Guid.NewGuid().ToString(); var workingDirectory = CreateUniqueWorkDirectory("server-rpc-enrich"); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { SessionId = sessionId, WorkingDirectory = workingDirectory, @@ -353,7 +355,7 @@ public async Task Should_Close_Active_Session_And_Release_Lock() await using var client = CreateAuthenticatedClient(token); var sessionId = Guid.NewGuid().ToString(); var workingDirectory = CreateUniqueWorkDirectory("server-rpc-close"); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { SessionId = sessionId, WorkingDirectory = workingDirectory, @@ -378,7 +380,7 @@ public async Task Should_Check_In_Use_Session_From_Another_Runtime_And_Release_L var sessionId = Guid.NewGuid().ToString(); var workingDirectory = CreateUniqueWorkDirectory("server-rpc-in-use"); await using var otherClient = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForStdio() }); - await using var otherSession = await otherClient.CreateSessionAsync(new SessionConfig + await using var otherSession = await Ctx.CreateSessionAsync(otherClient, new SessionConfig { SessionId = sessionId, WorkingDirectory = workingDirectory, @@ -419,7 +421,7 @@ public async Task Should_Prune_DryRun_And_BulkDelete_Persisted_Session() var missingSessionId = Guid.NewGuid().ToString(); var workingDirectory = CreateUniqueWorkDirectory("server-rpc-delete"); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { SessionId = sessionId, WorkingDirectory = workingDirectory, diff --git a/dotnet/test/E2E/RpcSessionStateE2ETests.cs b/dotnet/test/E2E/RpcSessionStateE2ETests.cs index 93af399203..ab5808302d 100644 --- a/dotnet/test/E2E/RpcSessionStateE2ETests.cs +++ b/dotnet/test/E2E/RpcSessionStateE2ETests.cs @@ -46,7 +46,7 @@ public async Task Should_Call_Session_Rpc_Model_SwitchTo() await isolatedCtx.ConfigureForTestAsync("rpc_session_state", nameof(Should_Call_Session_Rpc_Model_SwitchTo)); var isolatedClient = isolatedCtx.CreateClient(); - await using var session = await isolatedClient.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(isolatedClient, new SessionConfig { Model = "claude-sonnet-4.5", OnPermissionRequest = PermissionHandler.ApproveAll, @@ -443,7 +443,7 @@ public async Task Should_Set_ReasoningEffort_And_Auto_Name() public async Task Should_Set_Auth_Credentials() { await using var client = Ctx.CreateClient(); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); diff --git a/dotnet/test/E2E/RpcSessionStateExtrasE2ETests.cs b/dotnet/test/E2E/RpcSessionStateExtrasE2ETests.cs index f3f3d5d5de..ca339668e9 100644 --- a/dotnet/test/E2E/RpcSessionStateExtrasE2ETests.cs +++ b/dotnet/test/E2E/RpcSessionStateExtrasE2ETests.cs @@ -20,6 +20,7 @@ public class RpcSessionStateExtrasE2ETests(E2ETestFixture fixture, ITestOutputHe : E2ETestBase(fixture, "rpc_session_state_extras", output) { [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_List_Models_For_Session() { // model.list resolves models through the session's own auth context, which requires the @@ -29,7 +30,7 @@ public async Task Should_List_Models_For_Session() const string token = "rpc-session-model-list-token"; await ConfigureAuthenticatedUserAsync(token); await using var client = CreateAuthenticatedClient(token); - await using var session = await client.CreateSessionAsync(new SessionConfig + await using var session = await Ctx.CreateSessionAsync(client, new SessionConfig { Model = "claude-sonnet-4.5", OnPermissionRequest = PermissionHandler.ApproveAll, @@ -44,6 +45,7 @@ public async Task Should_List_Models_For_Session() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public async Task Should_Add_Byok_Provider_And_Model_At_Runtime() { await using var session = await CreateSessionAsync(); diff --git a/dotnet/test/E2E/RpcTasksAndHandlersE2ETests.cs b/dotnet/test/E2E/RpcTasksAndHandlersE2ETests.cs index 9b9738a2ee..fe6a20b2d0 100644 --- a/dotnet/test/E2E/RpcTasksAndHandlersE2ETests.cs +++ b/dotnet/test/E2E/RpcTasksAndHandlersE2ETests.cs @@ -72,6 +72,7 @@ await AssertImplementedFailureAsync( } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Report_Implemented_Error_For_Invalid_Task_Agent_Model() { var session = await CreateSessionAsync(); diff --git a/dotnet/test/E2E/SessionConfigE2ETests.cs b/dotnet/test/E2E/SessionConfigE2ETests.cs index 45cdef2016..e665632847 100644 --- a/dotnet/test/E2E/SessionConfigE2ETests.cs +++ b/dotnet/test/E2E/SessionConfigE2ETests.cs @@ -22,6 +22,7 @@ public class SessionConfigE2ETests(E2ETestFixture fixture, ITestOutputHelper out "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="); [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Vision_Disabled_Then_Enabled_Via_SetModel() { await File.WriteAllBytesAsync(Path.Join(Ctx.WorkDir, "test.png"), Png1X1); @@ -62,6 +63,7 @@ await session.SetModelAsync( } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Vision_Enabled_Then_Disabled_Via_SetModel() { await File.WriteAllBytesAsync(Path.Join(Ctx.WorkDir, "test.png"), Png1X1); @@ -121,6 +123,7 @@ public async Task Should_Use_Custom_SessionId() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public async Task Should_Apply_ReasoningEffort_On_Session_Create() { const string reasoningModelId = "custom-reasoning-model"; @@ -140,6 +143,7 @@ public async Task Should_Apply_ReasoningEffort_On_Session_Create() } [Theory] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] [InlineData("low")] [InlineData("medium")] [InlineData("high")] @@ -162,6 +166,7 @@ public async Task Should_Apply_All_ReasoningEffort_Values_On_Session_Create(stri } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public async Task Should_Apply_ReasoningEffort_On_Session_Resume() { var originalSession = await CreateSessionAsync(); @@ -182,6 +187,7 @@ public async Task Should_Apply_ReasoningEffort_On_Session_Resume() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Forward_ClientName_In_UserAgent() { var session = await CreateSessionAsync(new SessionConfig @@ -198,6 +204,7 @@ public async Task Should_Forward_ClientName_In_UserAgent() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public async Task Should_Forward_Custom_Provider_Headers_On_Create() { var session = await CreateSessionAsync(new SessionConfig @@ -217,6 +224,7 @@ public async Task Should_Forward_Custom_Provider_Headers_On_Create() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public async Task Should_Forward_Custom_Provider_Headers_On_Resume() { var session1 = await CreateSessionAsync(); @@ -239,6 +247,7 @@ public async Task Should_Forward_Custom_Provider_Headers_On_Resume() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public async Task Should_Forward_Provider_Wire_Model() { // Verifies that ProviderConfig.WireModel overrides the model name sent to @@ -270,6 +279,7 @@ public async Task Should_Forward_Provider_Wire_Model() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public async Task Should_Use_Provider_Model_Id_As_Wire_Model() { // ProviderConfig.ModelId drives both the runtime resolved model AND the wire model @@ -564,13 +574,14 @@ public async Task Should_Apply_Excluded_Built_In_Agents_On_Resume() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public async Task Should_Enable_Citations_For_Anthropic_File_Attachments_On_Create() { var handler = new RecordingRequestHandler(); await using var client = CreateClientWithRequestHandler(handler); await client.StartAsync(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Model = "claude-sonnet-4.5", @@ -595,6 +606,7 @@ await session.SendAndWaitAsync(new MessageOptions } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public async Task Should_Enable_Citations_For_Anthropic_File_Attachments_On_Resume() { const string connectionToken = "citation-resume-token"; @@ -604,7 +616,7 @@ public async Task Should_Enable_Citations_For_Anthropic_File_Attachments_On_Resu RuntimeConnection.ForTcp(connectionToken: connectionToken)); await client.StartAsync(); - var session1 = await client.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -616,7 +628,7 @@ public async Task Should_Enable_Citations_For_Anthropic_File_Attachments_On_Resu Connection = RuntimeConnection.ForUri($"localhost:{port}", connectionToken: connectionToken), }); - var session2 = await resumeClient.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(resumeClient, sessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Model = "claude-sonnet-4.5", @@ -642,6 +654,7 @@ await session2.SendAndWaitAsync(new MessageOptions } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public async Task Should_Create_Session_With_Custom_Provider_Config() { // Per the TS test (session_config.e2e.test.ts), this only verifies that a @@ -669,6 +682,7 @@ public async Task Should_Create_Session_With_Custom_Provider_Config() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Accept_Blob_Attachments() { // Write the image to disk so the model can view it if it tries diff --git a/dotnet/test/E2E/SessionE2ETests.cs b/dotnet/test/E2E/SessionE2ETests.cs index bcfb46295a..bf6da85474 100644 --- a/dotnet/test/E2E/SessionE2ETests.cs +++ b/dotnet/test/E2E/SessionE2ETests.cs @@ -232,7 +232,7 @@ public async Task Should_Reject_Resuming_Active_Session_Using_The_Same_Client() var sessionId = session1.SessionId; var exception = await Assert.ThrowsAsync(() => - Client.ResumeSessionAsync(sessionId, new ResumeSessionConfig + Ctx.ResumeSessionAsync(Client, sessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, })); @@ -251,7 +251,7 @@ public async Task Should_Resume_A_Session_Using_A_New_Client() Assert.Contains("2", answer!.Data.Content ?? string.Empty); using var newClient = Ctx.CreateClient(); - var session2 = await newClient.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(newClient, sessionId, new ResumeSessionConfig { ContinuePendingWork = true, OnPermissionRequest = PermissionHandler.ApproveAll, @@ -287,7 +287,7 @@ public async Task Resumes_A_Persisted_Session_From_A_New_Client_When_An_Mcp_OAut Assert.Contains("2", answer!.Data.Content ?? string.Empty); using var newClient = Ctx.CreateClient(); - await using var session2 = await newClient.ResumeSessionAsync(sessionId, new ResumeSessionConfig + await using var session2 = await Ctx.ResumeSessionAsync(newClient, sessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, OnMcpAuthRequest = CancelMcpAuthAsync, @@ -732,6 +732,7 @@ public async Task DisposeAsync_From_Handler_Does_Not_Deadlock() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Accept_Blob_Attachments() { var pngBase64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="; @@ -922,6 +923,7 @@ await session.SendAndWaitAsync(new MessageOptions } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public async Task Should_Create_Session_With_Custom_Provider() { var session = await CreateSessionAsync(new SessionConfig @@ -947,6 +949,7 @@ public async Task Should_Create_Session_With_Custom_Provider() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public async Task Should_Create_Session_With_Azure_Provider() { var session = await CreateSessionAsync(new SessionConfig @@ -976,6 +979,7 @@ public async Task Should_Create_Session_With_Azure_Provider() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] public async Task Should_Resume_Session_With_Custom_Provider() { var session = await CreateSessionAsync(); diff --git a/dotnet/test/E2E/SessionFsE2ETests.cs b/dotnet/test/E2E/SessionFsE2ETests.cs index b1b4848adc..cf91e3ddc8 100644 --- a/dotnet/test/E2E/SessionFsE2ETests.cs +++ b/dotnet/test/E2E/SessionFsE2ETests.cs @@ -28,7 +28,7 @@ public async Task Should_Route_File_Operations_Through_The_Session_Fs_Provider() { await using var client = CreateSessionFsClient(providerRoot); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = s => new TestSessionFsHandler(s.SessionId, providerRoot), @@ -58,7 +58,7 @@ public async Task Should_Load_Session_Data_From_Fs_Provider_On_Resume() await using var client = CreateSessionFsClient(providerRoot); Func createSessionFsHandler = s => new TestSessionFsHandler(s.SessionId, providerRoot); - var session1 = await client.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = createSessionFsHandler, @@ -72,7 +72,7 @@ public async Task Should_Load_Session_Data_From_Fs_Provider_On_Resume() var eventsPath = GetStoredPath(providerRoot, sessionId, $"{SessionFsConfig.SessionStatePath}/events.jsonl"); await WaitForConditionAsync(() => File.Exists(eventsPath)); - var session2 = await client.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(client, sessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = createSessionFsHandler, @@ -97,7 +97,7 @@ public async Task Should_Reject_SetProvider_When_Sessions_Already_Exist() await using var client1 = CreateSessionFsClient(providerRoot, useStdio: false, tcpConnectionToken: "session-fs-shared-token"); var createSessionFsHandler = (Func)(s => new TestSessionFsHandler(s.SessionId, providerRoot)); - _ = await client1.CreateSessionAsync(new SessionConfig + _ = await Ctx.CreateSessionAsync(client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = createSessionFsHandler, @@ -319,7 +319,7 @@ public async Task Should_Map_Large_Output_Handling_Into_SessionFs() var suppliedFileContent = new string('x', largeContentSize); await using var client = CreateSessionFsClient(providerRoot); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = s => new TestSessionFsHandler(s.SessionId, providerRoot), @@ -361,7 +361,7 @@ public async Task Should_Succeed_With_Compaction_While_Using_SessionFs() try { await using var client = CreateSessionFsClient(providerRoot); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = s => new TestSessionFsHandler(s.SessionId, providerRoot), @@ -400,7 +400,7 @@ public async Task Should_Write_Workspace_Metadata_Via_SessionFs() try { await using var client = CreateSessionFsClient(providerRoot); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = s => new TestSessionFsHandler(s.SessionId, providerRoot), @@ -433,7 +433,7 @@ public async Task Should_Persist_Plan_Md_Via_SessionFs() try { await using var client = CreateSessionFsClient(providerRoot); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = s => new TestSessionFsHandler(s.SessionId, providerRoot), diff --git a/dotnet/test/E2E/SessionFsSqliteE2ETests.cs b/dotnet/test/E2E/SessionFsSqliteE2ETests.cs index bf7bdf3b5e..8ed86c72e3 100644 --- a/dotnet/test/E2E/SessionFsSqliteE2ETests.cs +++ b/dotnet/test/E2E/SessionFsSqliteE2ETests.cs @@ -31,7 +31,7 @@ public async Task Should_Route_Sql_Queries_Through_The_Sessionfs_Sqlite_Handler( { await using var client = CreateSessionFsClient(); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = s => new InMemorySessionFsSqliteHandler(s.SessionId, _sqliteCalls), @@ -65,7 +65,7 @@ public async Task Should_Allow_Subagents_To_Use_Sql_Tool_Via_Inherited_Sessionfs await using var client = CreateSessionFsClient(); var handler = (InMemorySessionFsSqliteHandler?)null; - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, CreateSessionFsProvider = s => diff --git a/dotnet/test/E2E/StreamingFidelityE2ETests.cs b/dotnet/test/E2E/StreamingFidelityE2ETests.cs index fec00f1cb1..b10b7bc846 100644 --- a/dotnet/test/E2E/StreamingFidelityE2ETests.cs +++ b/dotnet/test/E2E/StreamingFidelityE2ETests.cs @@ -2,6 +2,7 @@ * Copyright (c) Microsoft Corporation. All rights reserved. *--------------------------------------------------------------------------------------------*/ +using GitHub.Copilot.Test.Harness; using Xunit; using Xunit.Abstractions; @@ -46,6 +47,7 @@ public async Task Should_Produce_Delta_Events_When_Streaming_Is_Enabled() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Not_Produce_Deltas_When_Streaming_Is_Disabled() { var session = await CreateSessionAsync(new SessionConfig { Streaming = false }); @@ -79,7 +81,7 @@ public async Task Should_Produce_Deltas_After_Session_Resume() // Resume using a new client using var newClient = Ctx.CreateClient(); - var session2 = await newClient.ResumeSessionAsync(session.SessionId, + var session2 = await Ctx.ResumeSessionAsync(newClient, session.SessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Streaming = true }); var events = new List(); @@ -106,6 +108,7 @@ public async Task Should_Produce_Deltas_After_Session_Resume() } [Fact] + [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Not_Produce_Deltas_After_Session_Resume_With_Streaming_Disabled() { var session = await CreateSessionAsync(new SessionConfig { Streaming = true }); @@ -114,7 +117,7 @@ public async Task Should_Not_Produce_Deltas_After_Session_Resume_With_Streaming_ // Resume using a new client with streaming DISABLED using var newClient = Ctx.CreateClient(); - var session2 = await newClient.ResumeSessionAsync(session.SessionId, + var session2 = await Ctx.ResumeSessionAsync(newClient, session.SessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Streaming = false }); var events = new List(); diff --git a/dotnet/test/E2E/SubagentHooksE2ETests.cs b/dotnet/test/E2E/SubagentHooksE2ETests.cs index a718c56c1a..8314b8c098 100644 --- a/dotnet/test/E2E/SubagentHooksE2ETests.cs +++ b/dotnet/test/E2E/SubagentHooksE2ETests.cs @@ -30,7 +30,7 @@ public async Task Should_Invoke_PreToolUse_And_PostToolUse_Hooks_For_Sub_Agent_T RequestHandler = requestHandler }, environment: env); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, Hooks = new SessionHooks diff --git a/dotnet/test/E2E/SuspendE2ETests.cs b/dotnet/test/E2E/SuspendE2ETests.cs index f531f0e59c..b88a9897be 100644 --- a/dotnet/test/E2E/SuspendE2ETests.cs +++ b/dotnet/test/E2E/SuspendE2ETests.cs @@ -58,7 +58,7 @@ public async Task Should_Allow_Resume_And_Continue_Conversation_After_Suspend() string sessionId; await using (var client1 = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: sharedToken) })) { - var session1 = await client1.CreateSessionAsync(new SessionConfig + var session1 = await Ctx.CreateSessionAsync(client1, new SessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); @@ -78,7 +78,7 @@ await session1.SendAndWaitAsync(new MessageOptions // A different client should be able to pick the session back up. The previous // turn was completed before suspend, so there is no pending work to continue. await using var client2 = Ctx.CreateClient(options: new CopilotClientOptions { Connection = RuntimeConnection.ForUri(cliUrl, connectionToken: sharedToken) }); - var session2 = await client2.ResumeSessionAsync(sessionId, new ResumeSessionConfig + var session2 = await Ctx.ResumeSessionAsync(client2, sessionId, new ResumeSessionConfig { OnPermissionRequest = PermissionHandler.ApproveAll, }); diff --git a/dotnet/test/E2E/TelemetryExportE2ETests.cs b/dotnet/test/E2E/TelemetryExportE2ETests.cs index 2dd9d591a4..48e3b53ad6 100644 --- a/dotnet/test/E2E/TelemetryExportE2ETests.cs +++ b/dotnet/test/E2E/TelemetryExportE2ETests.cs @@ -34,7 +34,7 @@ public async Task Should_Export_File_Telemetry_For_Sdk_Interactions() }, }); - var session = await client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(client, new SessionConfig { Tools = [AIFunctionFactory.Create(EchoTelemetryMarker, toolName, "Echoes a marker string for telemetry validation.")], OnPermissionRequest = PermissionHandler.ApproveAll, diff --git a/dotnet/test/E2E/ToolsE2ETests.cs b/dotnet/test/E2E/ToolsE2ETests.cs index 7424cfa3a3..ea615fbc4a 100644 --- a/dotnet/test/E2E/ToolsE2ETests.cs +++ b/dotnet/test/E2E/ToolsE2ETests.cs @@ -306,7 +306,7 @@ public async Task Invokes_Custom_Tool_With_Permission_Handler() { var permissionRequests = new List(); - var session = await Client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(Client, new SessionConfig { Tools = [AIFunctionFactory.Create(EncryptStringForPermission, "encrypt_string")], OnPermissionRequest = (request, invocation) => @@ -340,7 +340,7 @@ public async Task Denies_Custom_Tool_When_Permission_Denied() { var toolHandlerCalled = false; - var session = await Client.CreateSessionAsync(new SessionConfig + var session = await Ctx.CreateSessionAsync(Client, new SessionConfig { Tools = [AIFunctionFactory.Create(EncryptStringDenied, "encrypt_string")], OnPermissionRequest = async (request, invocation) => PermissionDecision.Reject(), diff --git a/dotnet/test/Harness/CapiProxy.cs b/dotnet/test/Harness/CapiProxy.cs index 39c95fa683..0e8de207ab 100644 --- a/dotnet/test/Harness/CapiProxy.cs +++ b/dotnet/test/Harness/CapiProxy.cs @@ -150,16 +150,19 @@ public async Task StopAsync(bool skipWritingCache = false) _startupTask = null; } - public async Task ConfigureAsync(string filePath, string workDir) + public async Task ConfigureAsync(string filePath, string workDir, string backend) { var url = await (_startupTask ?? throw new InvalidOperationException("Proxy not started")); using var client = new HttpClient(); - var response = await client.PostAsJsonAsync($"{url}/config", new ConfigureRequest(filePath, workDir), CapiProxyJsonContext.Default.ConfigureRequest); + var response = await client.PostAsJsonAsync( + $"{url}/config", + new ConfigureRequest(filePath, workDir, backend), + CapiProxyJsonContext.Default.ConfigureRequest); response.EnsureSuccessStatusCode(); } - private record ConfigureRequest(string FilePath, string WorkDir); + private record ConfigureRequest(string FilePath, string WorkDir, string Backend); private record ProxyStartupMetadata(string? ConnectProxyUrl, string? CaFilePath); diff --git a/dotnet/test/Harness/E2ETestBackend.cs b/dotnet/test/Harness/E2ETestBackend.cs new file mode 100644 index 0000000000..1cfa3e67e1 --- /dev/null +++ b/dotnet/test/Harness/E2ETestBackend.cs @@ -0,0 +1,97 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +namespace GitHub.Copilot.Test.Harness; + +internal enum E2ETestBackend +{ + Capi, + AnthropicMessages, + OpenAIResponses, + OpenAICompletions, +} + +internal static class E2ETestBackendConfiguration +{ + internal const string EnvironmentVariable = "COPILOT_SDK_E2E_BACKEND"; + private const string DefaultModel = "claude-sonnet-4.5"; + private const string FakeCredential = "fake-byok-credential-for-e2e-tests"; + + internal static E2ETestBackend Current + => Parse(Environment.GetEnvironmentVariable(EnvironmentVariable)); + + internal static E2ETestBackend Parse(string? value) + => value?.Trim().ToLowerInvariant() switch + { + null or "" or "capi" => E2ETestBackend.Capi, + "anthropic-messages" => E2ETestBackend.AnthropicMessages, + "openai-responses" => E2ETestBackend.OpenAIResponses, + "openai-completions" => E2ETestBackend.OpenAICompletions, + _ => throw new InvalidOperationException( + $"Unsupported {EnvironmentVariable} value '{value}'. Expected capi, anthropic-messages, openai-responses, or openai-completions."), + }; + + internal static string ToWireName(this E2ETestBackend backend) + => backend switch + { + E2ETestBackend.Capi => "capi", + E2ETestBackend.AnthropicMessages => "anthropic-messages", + E2ETestBackend.OpenAIResponses => "openai-responses", + E2ETestBackend.OpenAICompletions => "openai-completions", + _ => throw new ArgumentOutOfRangeException(nameof(backend), backend, null), + }; + + internal static void ApplyProvider( + this E2ETestBackend backend, + SessionConfig config, + string proxyUrl) + { + if (backend == E2ETestBackend.Capi + || config.Provider is not null + || config.Providers is not null) + { + return; + } + + config.Model ??= DefaultModel; + config.Provider = CreateProvider(backend, proxyUrl); + } + + internal static void ApplyProvider( + this E2ETestBackend backend, + ResumeSessionConfig config, + string proxyUrl) + { + if (backend == E2ETestBackend.Capi || config.Provider is not null) + { + return; + } + + config.Model ??= DefaultModel; + config.Provider = CreateProvider(backend, proxyUrl); + } + + private static ProviderConfig CreateProvider(E2ETestBackend backend, string proxyUrl) + => new() + { + BaseUrl = proxyUrl, + Type = backend == E2ETestBackend.AnthropicMessages ? "anthropic" : "openai", + WireApi = backend switch + { + E2ETestBackend.OpenAIResponses => "responses", + E2ETestBackend.OpenAICompletions => "completions", + _ => null, + }, + BearerToken = FakeCredential, + ModelId = DefaultModel, + WireModel = DefaultModel, + }; +} + +internal static class E2ETestTraits +{ + internal const string Backend = "E2EBackend"; + internal const string CapiOnly = "CapiOnly"; + internal const string SelfConfiguredProvider = "SelfConfiguredProvider"; +} diff --git a/dotnet/test/Harness/E2ETestBase.cs b/dotnet/test/Harness/E2ETestBase.cs index ddd1b894bb..a3006389bb 100644 --- a/dotnet/test/Harness/E2ETestBase.cs +++ b/dotnet/test/Harness/E2ETestBase.cs @@ -76,7 +76,7 @@ protected Task CreateSessionAsync(SessionConfig? config = null) { config ??= new SessionConfig(); config.OnPermissionRequest ??= PermissionHandler.ApproveAll; - return Client.CreateSessionAsync(config); + return Ctx.CreateSessionAsync(Client, config); } /// @@ -96,7 +96,7 @@ protected async Task ResumeSessionAsync(string sessionId, Resume { Connection = RuntimeConnection.ForUri($"localhost:{port}", connectionToken: E2ETestFixture.SharedTcpConnectionToken), }); - return await client.ResumeSessionAsync(sessionId, config); + return await Ctx.ResumeSessionAsync(client, sessionId, config); } protected static string GetSystemMessage(ParsedHttpExchange exchange) diff --git a/dotnet/test/Harness/E2ETestContext.cs b/dotnet/test/Harness/E2ETestContext.cs index 4c9b892ff2..b39dfcee45 100644 --- a/dotnet/test/Harness/E2ETestContext.cs +++ b/dotnet/test/Harness/E2ETestContext.cs @@ -163,7 +163,10 @@ public async Task ConfigureForTestAsync(string testFile, [CallerMemberName] stri // to avoid case collisions on case-insensitive filesystems (macOS/Windows) var sanitizedName = Regex.Replace(testName!, @"[^a-zA-Z0-9]", "_").ToLowerInvariant(); var snapshotPath = Path.Combine(_repoRoot, "test", "snapshots", testFile, $"{sanitizedName}.yaml"); - await _proxy.ConfigureAsync(snapshotPath, WorkDir); + await _proxy.ConfigureAsync( + snapshotPath, + WorkDir, + E2ETestBackendConfiguration.Current.ToWireName()); } public Task> GetExchangesAsync() @@ -352,6 +355,25 @@ public CopilotClient CreateClient( return client; } + public Task CreateSessionAsync( + CopilotClient client, + SessionConfig? config = null) + { + config ??= new SessionConfig(); + E2ETestBackendConfiguration.Current.ApplyProvider(config, ProxyUrl); + return client.CreateSessionAsync(config); + } + + public Task ResumeSessionAsync( + CopilotClient client, + string sessionId, + ResumeSessionConfig? config = null) + { + config ??= new ResumeSessionConfig(); + E2ETestBackendConfiguration.Current.ApplyProvider(config, ProxyUrl); + return client.ResumeSessionAsync(sessionId, config); + } + public void UntrackClient(CopilotClient client) { lock (_clientsLock) diff --git a/dotnet/test/Unit/E2ETestBackendTests.cs b/dotnet/test/Unit/E2ETestBackendTests.cs new file mode 100644 index 0000000000..1b90c7a2ff --- /dev/null +++ b/dotnet/test/Unit/E2ETestBackendTests.cs @@ -0,0 +1,46 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +using GitHub.Copilot.Test.Harness; +using Xunit; + +namespace GitHub.Copilot.Test.Unit; + +public class E2ETestBackendTests +{ + [Theory] + [InlineData(null, "capi")] + [InlineData("", "capi")] + [InlineData("capi", "capi")] + [InlineData("ANTHROPIC-MESSAGES", "anthropic-messages")] + [InlineData("openai-responses", "openai-responses")] + [InlineData("openai-completions", "openai-completions")] + public void ParsesBackend(string? value, string expected) + => Assert.Equal(expected, E2ETestBackendConfiguration.Parse(value).ToWireName()); + + [Fact] + public void RejectsUnknownBackend() + => Assert.Throws( + () => E2ETestBackendConfiguration.Parse("unknown")); + + [Theory] + [InlineData("anthropic-messages", "anthropic", null)] + [InlineData("openai-responses", "openai", "responses")] + [InlineData("openai-completions", "openai", "completions")] + public void AppliesProvider( + string backendValue, + string expectedType, + string? expectedWireApi) + { + var backend = E2ETestBackendConfiguration.Parse(backendValue); + var config = new SessionConfig(); + backend.ApplyProvider(config, "http://localhost:1234"); + + Assert.Equal("claude-sonnet-4.5", config.Model); + Assert.Equal("http://localhost:1234", config.Provider!.BaseUrl); + Assert.Equal(expectedType, config.Provider.Type); + Assert.Equal(expectedWireApi, config.Provider.WireApi); + Assert.False(string.IsNullOrEmpty(config.Provider.BearerToken)); + } +} diff --git a/test/harness/anthropicMessagesAdapter.ts b/test/harness/anthropicMessagesAdapter.ts new file mode 100644 index 0000000000..6ead783e30 --- /dev/null +++ b/test/harness/anthropicMessagesAdapter.ts @@ -0,0 +1,579 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import type { ChatCompletion } from "openai/resources/chat/completions"; +import { parseSseEvents } from "./sseParser"; + +export const anthropicMessagesEndpoint = "/v1/messages"; + +type JsonObject = Record; + +type CanonicalContentPart = + | { type: "text"; text: string } + | { type: "image_url"; image_url: { url: string } } + | { + type: "file"; + file: { file_data: string; filename?: string }; + }; + +type CanonicalMessage = { + role: "system" | "user" | "assistant" | "tool"; + content?: string | CanonicalContentPart[] | null; + tool_call_id?: string; + tool_calls?: CanonicalToolCall[]; +}; + +type CanonicalToolCall = { + id: string; + type: "function"; + function: { name: string; arguments: string }; +}; + +type AnthropicContentBlock = + | { type: "text"; text: string; citations?: null } + | { + type: "image" | "document"; + source?: { type?: string; media_type?: string; data?: string }; + } + | { type: "tool_use"; id: string; name: string; input: unknown } + | { + type: "tool_result"; + tool_use_id?: string; + content?: string | Array<{ type?: string; text?: string }>; + }; + +type AnthropicMessageParam = { + role: "user" | "assistant"; + content: string | AnthropicContentBlock[]; +}; + +type AnthropicRequest = { + model: string; + messages: AnthropicMessageParam[]; + system?: string | Array<{ type?: string; text?: string }>; + max_tokens?: number; + temperature?: number; + top_p?: number; + stream?: boolean; + tools?: Array<{ + name: string; + description?: string; + input_schema?: JsonObject; + }>; + tool_choice?: + | { type: "auto" | "any" | "none" } + | { type: "tool"; name: string }; +}; + +type AnthropicStopReason = + | "end_turn" + | "max_tokens" + | "stop_sequence" + | "tool_use" + | "refusal"; + +export type AnthropicMessage = { + id: string; + type: "message"; + role: "assistant"; + content: Array< + | { type: "text"; text: string; citations: null } + | { type: "tool_use"; id: string; name: string; input: unknown } + >; + model: string; + stop_reason: AnthropicStopReason | null; + stop_sequence: string | null; + usage: { + input_tokens: number; + output_tokens: number; + cache_creation_input_tokens: number | null; + cache_read_input_tokens: number | null; + }; +}; + +const finishReasonToStopReason: Record = { + stop: "end_turn", + length: "max_tokens", + tool_calls: "tool_use", + function_call: "tool_use", + content_filter: "refusal", +}; + +const stopReasonToFinishReason: Record = { + end_turn: "stop", + stop_sequence: "stop", + max_tokens: "length", + tool_use: "tool_calls", + refusal: "content_filter", +}; + +export function anthropicMessagesRequestToChatCompletion( + requestBody: string, +): string { + const request = JSON.parse(requestBody) as AnthropicRequest; + const messages: CanonicalMessage[] = []; + + const system = anthropicSystemToString(request.system); + if (system) messages.push({ role: "system", content: system }); + + for (const message of request.messages) { + messages.push(...convertAnthropicMessage(message)); + } + + return JSON.stringify({ + model: request.model, + messages, + ...(request.max_tokens !== undefined + ? { max_tokens: request.max_tokens } + : {}), + ...(request.temperature !== undefined + ? { temperature: request.temperature } + : {}), + ...(request.top_p !== undefined ? { top_p: request.top_p } : {}), + ...(request.stream !== undefined ? { stream: request.stream } : {}), + ...(request.tools + ? { + tools: request.tools.map((tool) => ({ + type: "function", + function: { + name: tool.name, + ...(tool.description ? { description: tool.description } : {}), + parameters: tool.input_schema ?? { + type: "object", + properties: {}, + }, + }, + })), + } + : {}), + ...(request.tool_choice + ? { tool_choice: convertToolChoice(request.tool_choice) } + : {}), + }); +} + +function anthropicSystemToString( + system: AnthropicRequest["system"], +): string | undefined { + if (typeof system === "string") return system; + if (!Array.isArray(system)) return undefined; + return system + .map((block) => (typeof block.text === "string" ? block.text : "")) + .filter(Boolean) + .join("\n"); +} + +function convertAnthropicMessage( + message: AnthropicMessageParam, +): CanonicalMessage[] { + return message.role === "user" + ? convertAnthropicUserMessage(message) + : convertAnthropicAssistantMessage(message); +} + +function normalizeContent( + content: AnthropicMessageParam["content"], +): AnthropicContentBlock[] { + return typeof content === "string" + ? [{ type: "text", text: content }] + : content; +} + +function convertAnthropicUserMessage( + message: AnthropicMessageParam, +): CanonicalMessage[] { + const result: CanonicalMessage[] = []; + const contentParts: CanonicalContentPart[] = []; + + const flushUserContent = () => { + if (contentParts.length === 0) return; + const onlyText = contentParts.every((part) => part.type === "text"); + result.push({ + role: "user", + content: onlyText + ? contentParts + .map((part) => (part.type === "text" ? part.text : "")) + .join("\n") + : [...contentParts], + }); + contentParts.length = 0; + }; + + for (const block of normalizeContent(message.content)) { + if (block.type === "text") { + contentParts.push({ type: "text", text: block.text }); + } else if ( + (block.type === "image" || block.type === "document") && + block.source?.type === "base64" && + block.source.data + ) { + const dataUrl = `data:${ + block.source.media_type ?? + (block.type === "image" ? "image/png" : "application/pdf") + };base64,${block.source.data}`; + contentParts.push( + block.type === "image" + ? { type: "image_url", image_url: { url: dataUrl } } + : { type: "file", file: { file_data: dataUrl } }, + ); + } else if (block.type === "tool_result") { + flushUserContent(); + result.push({ + role: "tool", + tool_call_id: block.tool_use_id ?? "", + content: anthropicToolResultContent(block.content), + }); + } + } + + flushUserContent(); + return result; +} + +function convertAnthropicAssistantMessage( + message: AnthropicMessageParam, +): CanonicalMessage[] { + const text: string[] = []; + const toolCalls: CanonicalToolCall[] = []; + for (const block of normalizeContent(message.content)) { + if (block.type === "text") { + text.push(block.text); + } else if (block.type === "tool_use") { + toolCalls.push({ + id: block.id, + type: "function", + function: { + name: block.name, + arguments: JSON.stringify(block.input ?? {}), + }, + }); + } + } + + return [ + { + role: "assistant", + content: text.length ? text.join("") : null, + ...(toolCalls.length ? { tool_calls: toolCalls } : {}), + }, + ]; +} + +function anthropicToolResultContent(content: unknown): string { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; + return content + .map((part) => + isObject(part) && typeof part.text === "string" ? part.text : "", + ) + .filter(Boolean) + .join("\n"); +} + +function convertToolChoice( + choice: NonNullable, +): unknown { + switch (choice.type) { + case "auto": + return "auto"; + case "any": + return "required"; + case "none": + return "none"; + case "tool": + return { type: "function", function: { name: choice.name } }; + } +} + +export function chatCompletionResponseToAnthropicMessage( + response: ChatCompletion, +): AnthropicMessage { + const content: AnthropicMessage["content"] = []; + for (const choice of response.choices) { + if (choice.message.content) { + content.push({ + type: "text", + text: choice.message.content, + citations: null, + }); + } + for (const toolCall of functionToolCalls(choice.message)) { + content.push({ + type: "tool_use", + id: toolCall.id, + name: toolCall.function.name, + input: safeParseJson(toolCall.function.arguments), + }); + } + } + + const finishReason = response.choices.at(-1)?.finish_reason; + return { + id: response.id, + type: "message", + role: "assistant", + content, + model: response.model, + stop_reason: finishReason + ? (finishReasonToStopReason[finishReason] ?? null) + : null, + stop_sequence: null, + usage: { + input_tokens: response.usage?.prompt_tokens ?? 0, + output_tokens: response.usage?.completion_tokens ?? 0, + cache_creation_input_tokens: null, + cache_read_input_tokens: null, + }, + }; +} + +export function chatCompletionResponseToAnthropicSseChunks( + response: ChatCompletion, +): string[] { + const message = chatCompletionResponseToAnthropicMessage(response); + const chunks = [ + formatSseEvent("message_start", { + type: "message_start", + message: { + ...message, + content: [], + stop_reason: null, + usage: { ...message.usage, output_tokens: 1 }, + }, + }), + ]; + + for (let index = 0; index < message.content.length; index++) { + const block = message.content[index]; + if (block.type === "text") { + chunks.push( + formatSseEvent("content_block_start", { + type: "content_block_start", + index, + content_block: { type: "text", text: "", citations: null }, + }), + formatSseEvent("content_block_delta", { + type: "content_block_delta", + index, + delta: { type: "text_delta", text: block.text }, + }), + ); + } else { + chunks.push( + formatSseEvent("content_block_start", { + type: "content_block_start", + index, + content_block: { + type: "tool_use", + id: block.id, + name: block.name, + input: {}, + }, + }), + formatSseEvent("content_block_delta", { + type: "content_block_delta", + index, + delta: { + type: "input_json_delta", + partial_json: JSON.stringify(block.input ?? {}), + }, + }), + ); + } + chunks.push( + formatSseEvent("content_block_stop", { + type: "content_block_stop", + index, + }), + ); + } + + chunks.push( + formatSseEvent("message_delta", { + type: "message_delta", + delta: { + stop_reason: message.stop_reason, + stop_sequence: message.stop_sequence, + }, + usage: { output_tokens: message.usage.output_tokens }, + }), + formatSseEvent("message_stop", { type: "message_stop" }), + ); + return chunks; +} + +export function anthropicMessageResponseToChatCompletion( + requestBody: string, + responseBody: string, +): ChatCompletion { + const request = JSON.parse(requestBody) as AnthropicRequest; + const message = JSON.parse(responseBody) as AnthropicMessage; + const text: string[] = []; + const toolCalls: CanonicalToolCall[] = []; + for (const block of message.content) { + if (block.type === "text") { + text.push(block.text); + } else { + toolCalls.push({ + id: block.id, + type: "function", + function: { + name: block.name, + arguments: JSON.stringify(block.input ?? {}), + }, + }); + } + } + + return { + id: message.id, + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model: request.model, + choices: [ + { + index: 0, + message: { + role: "assistant", + content: text.length ? text.join("") : null, + refusal: null, + ...(toolCalls.length ? { tool_calls: toolCalls } : {}), + }, + logprobs: null, + finish_reason: + (message.stop_reason && + stopReasonToFinishReason[message.stop_reason]) || + "stop", + }, + ], + usage: { + prompt_tokens: + message.usage.input_tokens + + (message.usage.cache_read_input_tokens ?? 0), + completion_tokens: message.usage.output_tokens, + total_tokens: + message.usage.input_tokens + + (message.usage.cache_read_input_tokens ?? 0) + + message.usage.output_tokens, + }, + } as ChatCompletion; +} + +export function aggregateAnthropicSseToMessage( + body: string, +): AnthropicMessage | null { + let message: AnthropicMessage | null = null; + const blocks: AnthropicMessage["content"] = []; + const toolInputs: string[] = []; + + for (const event of parseSseEvents(body)) { + if (event.type === "message_start" && isObject(event.message)) { + message = { + ...(event.message as AnthropicMessage), + content: [], + }; + } else if ( + event.type === "content_block_start" && + typeof event.index === "number" && + isObject(event.content_block) + ) { + const block = event.content_block; + if (block.type === "text") { + blocks[event.index] = { + type: "text", + text: typeof block.text === "string" ? block.text : "", + citations: null, + }; + } else if ( + block.type === "tool_use" && + typeof block.id === "string" && + typeof block.name === "string" + ) { + blocks[event.index] = { + type: "tool_use", + id: block.id, + name: block.name, + input: {}, + }; + toolInputs[event.index] = ""; + } + } else if ( + event.type === "content_block_delta" && + typeof event.index === "number" && + isObject(event.delta) + ) { + const block = blocks[event.index]; + if ( + block?.type === "text" && + event.delta.type === "text_delta" && + typeof event.delta.text === "string" + ) { + block.text += event.delta.text; + } else if ( + block?.type === "tool_use" && + event.delta.type === "input_json_delta" && + typeof event.delta.partial_json === "string" + ) { + toolInputs[event.index] += event.delta.partial_json; + } + } else if ( + event.type === "content_block_stop" && + typeof event.index === "number" + ) { + const block = blocks[event.index]; + if (block?.type === "tool_use") { + block.input = safeParseJson(toolInputs[event.index] || "{}"); + } + } else if ( + event.type === "message_delta" && + message && + isObject(event.delta) + ) { + if (typeof event.delta.stop_reason === "string") { + message.stop_reason = event.delta.stop_reason as AnthropicStopReason; + } + if ( + isObject(event.usage) && + typeof event.usage.output_tokens === "number" + ) { + message.usage.output_tokens = event.usage.output_tokens; + } + } + } + + if (!message) return null; + message.content = blocks.filter( + (block): block is AnthropicMessage["content"][number] => Boolean(block), + ); + return message; +} + +function functionToolCalls(message: unknown): CanonicalToolCall[] { + if (!isObject(message) || !Array.isArray(message.tool_calls)) return []; + return message.tool_calls.filter( + (toolCall): toolCall is CanonicalToolCall => + isObject(toolCall) && + typeof toolCall.id === "string" && + toolCall.type === "function" && + isObject(toolCall.function) && + typeof toolCall.function.name === "string" && + typeof toolCall.function.arguments === "string", + ); +} + +function formatSseEvent(event: string, data: unknown): string { + return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`; +} + +function safeParseJson(value: string): unknown { + try { + return JSON.parse(value); + } catch { + return {}; + } +} + +function isObject(value: unknown): value is JsonObject { + return value !== null && typeof value === "object" && !Array.isArray(value); +} diff --git a/test/harness/modelProtocolAdapters.test.ts b/test/harness/modelProtocolAdapters.test.ts new file mode 100644 index 0000000000..088e5d898a --- /dev/null +++ b/test/harness/modelProtocolAdapters.test.ts @@ -0,0 +1,619 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { ChatCompletion } from "openai/resources/chat/completions"; +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import yaml from "yaml"; +import { + aggregateAnthropicSseToMessage, + anthropicMessageResponseToChatCompletion, + anthropicMessagesRequestToChatCompletion, + chatCompletionResponseToAnthropicMessage, + chatCompletionResponseToAnthropicSseChunks, +} from "./anthropicMessagesAdapter"; +import { + aggregateResponsesApiSseToResponse, + chatCompletionResponseToResponsesApiMessage, + chatCompletionResponseToResponsesApiSseChunks, + responsesApiRequestToChatCompletion, + responsesApiResponseToChatCompletion, +} from "./responsesApiAdapter"; +import { + NormalizedData, + ReplayBackend, + ReplayingCapiProxy, +} from "./replayingCapiProxy"; + +const completionWithTool: ChatCompletion = { + id: "completion-1", + object: "chat.completion", + created: 123, + model: "test-model", + choices: [ + { + index: 0, + message: { + role: "assistant", + content: "Calling a tool", + refusal: null, + tool_calls: [ + { + id: "call-1", + type: "function", + function: { + name: "lookup", + arguments: '{"value":42}', + }, + }, + ], + }, + logprobs: null, + finish_reason: "tool_calls", + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 5, + total_tokens: 15, + }, +}; + +describe("Anthropic Messages adapter", () => { + test("normalizes system, images, tools, and tool results", () => { + const result = JSON.parse( + anthropicMessagesRequestToChatCompletion( + JSON.stringify({ + model: "test-model", + system: [{ type: "text", text: "Be helpful" }], + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Inspect this" }, + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "AQID", + }, + }, + ], + }, + { + role: "assistant", + content: [ + { + type: "tool_use", + id: "call-1", + name: "lookup", + input: { value: 42 }, + }, + ], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "call-1", + content: "found", + }, + ], + }, + ], + tools: [ + { + name: "lookup", + description: "Find a value", + input_schema: { type: "object" }, + }, + ], + stream: true, + }), + ), + ) as { + messages: Array>; + tools: Array>; + stream: boolean; + }; + + expect(result.messages).toEqual([ + { role: "system", content: "Be helpful" }, + { + role: "user", + content: [ + { type: "text", text: "Inspect this" }, + { + type: "image_url", + image_url: { url: "data:image/png;base64,AQID" }, + }, + ], + }, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call-1", + type: "function", + function: { + name: "lookup", + arguments: '{"value":42}', + }, + }, + ], + }, + { role: "tool", tool_call_id: "call-1", content: "found" }, + ]); + expect(result.tools).toHaveLength(1); + expect(result.stream).toBe(true); + }); + + test("round-trips JSON and streaming responses", () => { + const message = + chatCompletionResponseToAnthropicMessage(completionWithTool); + expect(message.stop_reason).toBe("tool_use"); + expect(message.content.map((block) => block.type)).toEqual([ + "text", + "tool_use", + ]); + + const request = JSON.stringify({ + model: "test-model", + messages: [{ role: "user", content: "Hello" }], + }); + const fromJson = anthropicMessageResponseToChatCompletion( + request, + JSON.stringify(message), + ); + expect(fromJson.choices[0].message.tool_calls).toHaveLength(1); + + const stream = + chatCompletionResponseToAnthropicSseChunks(completionWithTool).join(""); + expect(stream).toContain("event: message_start"); + expect(stream).toContain("event: message_stop"); + const aggregated = aggregateAnthropicSseToMessage(stream); + expect(aggregated).not.toBeNull(); + expect(aggregated!.content).toEqual(message.content); + }); + + test("combines CAPI multi-choice assistant messages", () => { + const message = chatCompletionResponseToAnthropicMessage({ + ...completionWithTool, + choices: [ + completionWithTool.choices[0], + { + index: 1, + message: { + role: "assistant", + content: null, + refusal: null, + tool_calls: [ + { + id: "call-2", + type: "function", + function: { name: "inspect", arguments: '{"path":"file.txt"}' }, + }, + ], + }, + logprobs: null, + finish_reason: "tool_calls", + }, + ], + }); + + expect( + message.content + .filter((block) => block.type === "tool_use") + .map((block) => block.name), + ).toEqual(["lookup", "inspect"]); + }); +}); + +describe("OpenAI Responses adapter", () => { + test("normalizes instructions, binary content, and function history", () => { + const result = JSON.parse( + responsesApiRequestToChatCompletion( + JSON.stringify({ + model: "test-model", + instructions: "Be helpful", + input: [ + { + type: "message", + role: "user", + content: [ + { type: "input_text", text: "Inspect this" }, + { + type: "input_image", + image_url: "data:image/png;base64,AQID", + }, + ], + }, + { + type: "function_call", + call_id: "call-1", + name: "lookup", + arguments: '{"value":42}', + }, + { + type: "function_call_output", + call_id: "call-1", + output: "found", + }, + ], + tools: [ + { + type: "function", + name: "lookup", + parameters: { type: "object" }, + }, + ], + }), + ), + ) as { + messages: Array>; + tools: Array>; + }; + + expect(result.messages).toEqual([ + { role: "system", content: "Be helpful" }, + { + role: "user", + content: [ + { type: "text", text: "Inspect this" }, + { + type: "image_url", + image_url: { + url: "data:image/png;base64,AQID", + }, + }, + ], + }, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call-1", + type: "function", + function: { + name: "lookup", + arguments: '{"value":42}', + }, + }, + ], + }, + { role: "tool", tool_call_id: "call-1", content: "found" }, + ]); + expect(result.tools).toHaveLength(1); + }); + + test("round-trips JSON and streaming responses", () => { + const response = + chatCompletionResponseToResponsesApiMessage(completionWithTool); + expect(response.output.map((item) => item.type)).toEqual([ + "message", + "function_call", + ]); + + const request = JSON.stringify({ model: "test-model", input: "Hello" }); + const fromJson = responsesApiResponseToChatCompletion( + request, + JSON.stringify(response), + ); + expect(fromJson.choices[0].message.tool_calls).toHaveLength(1); + + const stream = + chatCompletionResponseToResponsesApiSseChunks(completionWithTool).join( + "", + ); + expect(stream).toContain("event: response.created"); + expect(stream).toContain("event: response.completed"); + const aggregated = aggregateResponsesApiSseToResponse(stream); + expect(aggregated?.output_text).toBe(response.output_text); + expect(aggregated?.output.map((item) => item.type)).toEqual([ + "message", + "function_call", + ]); + }); +}); + +describe("protocol-aware replay", () => { + let tempDir: string; + let workDir: string; + let cachePath: string; + + beforeEach(async () => { + tempDir = await mkdtemp(path.join(os.tmpdir(), "protocol-replay-")); + workDir = path.join(tempDir, "work"); + cachePath = path.join(tempDir, "cache.yaml"); + await writeFile( + cachePath, + yaml.stringify({ + models: ["test-model"], + conversations: [ + { + messages: [ + { role: "system", content: "${system}" }, + { role: "user", content: "Hello" }, + { role: "assistant", content: "Hi there!" }, + ], + }, + ], + } satisfies NormalizedData), + ); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + test.each([ + { + backend: "anthropic-messages" as ReplayBackend, + endpoint: "/v1/messages", + request: { + model: "test-model", + system: "Be helpful", + messages: [{ role: "user", content: "Hello" }], + max_tokens: 128, + }, + assertBody: (body: Record) => { + expect(body.type).toBe("message"); + expect(body.content).toEqual([ + { type: "text", text: "Hi there!", citations: null }, + ]); + }, + }, + { + backend: "openai-responses" as ReplayBackend, + endpoint: "/responses", + request: { + model: "test-model", + instructions: "Be helpful", + input: [ + { + type: "message", + role: "user", + content: [{ type: "input_text", text: "Hello" }], + }, + ], + }, + assertBody: (body: Record) => { + expect(body.object).toBe("response"); + expect(body.output_text).toBe("Hi there!"); + }, + }, + { + backend: "openai-completions" as ReplayBackend, + endpoint: "/chat/completions", + request: { + model: "test-model", + messages: [ + { role: "system", content: "Be helpful" }, + { role: "user", content: "Hello" }, + ], + }, + assertBody: (body: Record) => { + expect(body.object).toBe("chat.completion"); + }, + }, + ])("replays $backend and exposes canonical exchanges", async (testCase) => { + const proxy = new ReplayingCapiProxy( + "http://localhost:9999", + cachePath, + workDir, + ); + const proxyUrl = await proxy.start(); + await proxy.updateConfig({ + filePath: cachePath, + workDir, + backend: testCase.backend, + }); + + try { + const response = await fetch(`${proxyUrl}${testCase.endpoint}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(testCase.request), + }); + expect(response.status).toBe(200); + testCase.assertBody((await response.json()) as Record); + + const exchangesResponse = await fetch(`${proxyUrl}/exchanges`); + const exchanges = (await exchangesResponse.json()) as Array<{ + request: { messages: Array<{ role: string; content: unknown }> }; + }>; + expect(exchanges).toHaveLength(1); + expect(exchanges[0].request.messages.at(-1)).toEqual({ + role: "user", + content: "Hello", + }); + } finally { + await proxy.stop(true); + } + }); + + test.each([ + { + backend: "openai-responses" as ReplayBackend, + endpoint: "/responses", + request: { + model: "test-model", + instructions: "Be helpful", + input: [ + { + type: "message", + role: "user", + content: [ + { + type: "input_text", + text: "Hook context\n\n\n2026-01-01T00:00:00Z\n\n", + }, + ], + }, + { + type: "message", + role: "user", + content: [{ type: "input_text", text: "Hello" }], + }, + ], + }, + }, + { + backend: "openai-completions" as ReplayBackend, + endpoint: "/chat/completions", + request: { + model: "test-model", + messages: [ + { role: "system", content: "Be helpful" }, + { + role: "user", + content: + "Hook context\n\n\n2026-01-01T00:00:00Z\n\n", + }, + { role: "user", content: "Hello" }, + ], + }, + }, + ])("coalesces adjacent user messages for $backend", async (testCase) => { + await writeFile( + cachePath, + yaml.stringify({ + models: ["test-model"], + conversations: [ + { + messages: [ + { role: "system", content: "${system}" }, + { role: "user", content: "Hook context" }, + { role: "user", content: "Hello" }, + { role: "assistant", content: "Hi there!" }, + ], + }, + ], + } satisfies NormalizedData), + ); + const proxy = new ReplayingCapiProxy( + "http://localhost:9999", + cachePath, + workDir, + ); + const proxyUrl = await proxy.start(); + await proxy.updateConfig({ + filePath: cachePath, + workDir, + backend: testCase.backend, + }); + + try { + const response = await fetch(`${proxyUrl}${testCase.endpoint}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(testCase.request), + }); + expect(response.status).toBe(200); + } finally { + await proxy.stop(true); + } + }); + + test.each([ + { + backend: "anthropic-messages" as ReplayBackend, + endpoint: "/v1/messages", + request: { + model: "test-model", + system: "Be helpful", + messages: [{ role: "user", content: "${compaction_prompt}" }], + max_tokens: 128, + }, + }, + { + backend: "openai-responses" as ReplayBackend, + endpoint: "/responses", + request: { + model: "test-model", + instructions: "Be helpful", + input: [ + { + type: "message", + role: "user", + content: [{ type: "input_text", text: "${compaction_prompt}" }], + }, + ], + }, + }, + { + backend: "openai-completions" as ReplayBackend, + endpoint: "/chat/completions", + request: { + model: "test-model", + messages: [ + { role: "system", content: "Be helpful" }, + { role: "user", content: "${compaction_prompt}" }, + ], + }, + }, + ])("synthesizes a compaction response for $backend", async (testCase) => { + const proxy = new ReplayingCapiProxy( + "http://localhost:9999", + cachePath, + workDir, + ); + const proxyUrl = await proxy.start(); + await proxy.updateConfig({ + filePath: cachePath, + workDir, + backend: testCase.backend, + }); + + try { + const response = await fetch(`${proxyUrl}${testCase.endpoint}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(testCase.request), + }); + expect(response.status).toBe(200); + const body = JSON.stringify(await response.json()); + expect(body).toContain(""); + expect(body).toContain(""); + expect(body).toContain(""); + } finally { + await proxy.stop(true); + } + }); + + test("rejects an inference request sent over the wrong protocol", async () => { + const proxy = new ReplayingCapiProxy( + "http://localhost:9999", + cachePath, + workDir, + ); + const proxyUrl = await proxy.start(); + await proxy.updateConfig({ + filePath: cachePath, + workDir, + backend: "anthropic-messages", + }); + + try { + const response = await fetch(`${proxyUrl}/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "test-model", + messages: [{ role: "user", content: "Hello" }], + }), + }); + expect(response.status).toBe(400); + await expect(response.text()).resolves.toContain("protocol_mismatch"); + } finally { + await proxy.stop(true); + } + }); +}); diff --git a/test/harness/replayingCapiProxy.ts b/test/harness/replayingCapiProxy.ts index 740f8ab746..379224763c 100644 --- a/test/harness/replayingCapiProxy.ts +++ b/test/harness/replayingCapiProxy.ts @@ -19,10 +19,43 @@ import { CapturingHttpProxy, PerformRequestOptions, } from "./capturingHttpProxy"; +import { + aggregateAnthropicSseToMessage, + anthropicMessageResponseToChatCompletion, + anthropicMessagesEndpoint, + anthropicMessagesRequestToChatCompletion, + chatCompletionResponseToAnthropicMessage, + chatCompletionResponseToAnthropicSseChunks, +} from "./anthropicMessagesAdapter"; +import { + aggregateResponsesApiSseToResponse, + chatCompletionResponseToResponsesApiMessage, + chatCompletionResponseToResponsesApiSseChunks, + responsesApiRequestToChatCompletion, + responsesApiResponseToChatCompletion, + responsesEndpoint, +} from "./responsesApiAdapter"; import { iife, ShellConfig, sleep } from "./util"; export const workingDirPlaceholder = "${workdir}"; const chatCompletionEndpoint = "/chat/completions"; +const replayedModelRequestEndpoints = new Set([ + chatCompletionEndpoint, + anthropicMessagesEndpoint, + responsesEndpoint, +]); + +export type ReplayBackend = + | "capi" + | "anthropic-messages" + | "openai-responses" + | "openai-completions"; + +type ModelProtocol = + | "anthropic-messages" + | "openai-responses" + | "openai-completions"; + const shellConfig = process.platform === "win32" ? ShellConfig.powerShell : ShellConfig.bash; const normalizedToolNames: Record = { @@ -90,6 +123,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { filePath, workDir, testInfo, + backend: "capi", toolResultNormalizers: [...this.defaultToolResultNormalizers], }; } @@ -120,6 +154,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { filePath: config.filePath, workDir: config.workDir, testInfo: config.testInfo, + backend: parseReplayBackend(config.backend), toolResultNormalizers: [...this.defaultToolResultNormalizers], }; @@ -133,6 +168,10 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { this.state.storedData = yaml.parse(content) as NormalizedData; normalizeToolResultOrder(this.state.storedData.conversations); normalizeStoredUserMessages(this.state.storedData.conversations); + normalizeStoredMessagesForBackend( + this.state.storedData.conversations, + this.state.backend, + ); } } @@ -223,9 +262,12 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { options.requestOptions.path === "/exchanges" && options.requestOptions.method === "GET" ) { - const chatCompletionExchanges = this.exchanges.filter( - (e) => e.request.url === chatCompletionEndpoint, - ); + const chatCompletionExchanges = this.exchanges + .filter((e) => replayedModelRequestEndpoints.has(e.request.url)) + .map((exchange) => + convertModelExchangeToOpenAI(exchange, this.state?.backend), + ) + .filter((e) => e.request.url === chatCompletionEndpoint); const parsedExchanges = await Promise.all( chatCompletionExchanges.map((e) => parseHttpExchange( @@ -336,15 +378,81 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { return; } - // Handle /chat/completions endpoint + const requestPath = options.requestOptions.path ?? ""; + const protocol = protocolForEndpoint(requestPath); + const expectedEndpoint = endpointForBackend(state.backend); if ( - state.storedData && - options.requestOptions.path === chatCompletionEndpoint && - options.body + protocol && + state.backend !== "capi" && + requestPath !== expectedEndpoint ) { + const message = `Expected ${expectedEndpoint} for backend ${state.backend}, received ${requestPath}`; + options.onResponseStart(400, { + "content-type": "application/json", + ...commonResponseHeaders, + }); + options.onData( + Buffer.from( + JSON.stringify({ + error: { type: "protocol_mismatch", message }, + }), + ), + ); + options.onResponseEnd(); + return; + } + + // Handle model inference endpoints. Foreign wire formats are converted + // to the canonical Chat Completions shape used by the snapshot matcher. + if (state.storedData && protocol && options.body) { + const normalizedBody = normalizeModelRequest( + protocol, + options.body, + state.backend, + ); + const compactionResponse = + state.backend === "capi" + ? undefined + : createCompactionResponse(normalizedBody); + if (compactionResponse) { + const streamingIsRequested = + (JSON.parse(options.body) as { stream?: boolean }).stream === + true; + await this.respondWithProtocol( + options, + protocol, + compactionResponse, + streamingIsRequested, + commonResponseHeaders, + ); + return; + } + + const postCompactionResponse = + state.backend === "capi" + ? undefined + : createPostCompactionResponse( + state.storedData, + normalizedBody, + state.workDir, + ); + if (postCompactionResponse) { + const streamingIsRequested = + (JSON.parse(options.body) as { stream?: boolean }).stream === + true; + await this.respondWithProtocol( + options, + protocol, + postCompactionResponse, + streamingIsRequested, + commonResponseHeaders, + ); + return; + } + const savedError = await findSavedChatCompletionError( state.storedData, - options.body, + normalizedBody, state.workDir, state.toolResultNormalizers, ); @@ -360,14 +468,13 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { options.onResponseStart(savedError.status, headers); options.onData( Buffer.from( - JSON.stringify({ - error: { - message: - savedError.message ?? "Rate limited by test snapshot", - type: savedError.code ?? "rate_limited", - code: savedError.code ?? "rate_limited", - }, - }), + JSON.stringify( + createProtocolError( + protocol, + savedError.code, + savedError.message ?? "Rate limited by test snapshot", + ), + ), ), ); options.onResponseEnd(); @@ -376,45 +483,23 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { const savedResponse = await findSavedChatCompletionResponse( state.storedData, - options.body, + normalizedBody, state.workDir, state.toolResultNormalizers, ); if (savedResponse) { const streamingIsRequested = - options.body && (JSON.parse(options.body) as { stream?: boolean }).stream === - true; - - if (streamingIsRequested) { - const headers = { - "content-type": "text/event-stream", - ...commonResponseHeaders, - }; - options.onResponseStart(200, headers); - for (const chunk of convertToStreamingResponseChunks( - savedResponse, - )) { - options.onData( - Buffer.from(`data: ${JSON.stringify(chunk)}\n\n`), - ); - if (this.slowStreaming) { - await sleep(100); - } - } - options.onData(Buffer.from("data: [DONE]\n\n")); - options.onResponseEnd(); - } else { - const body = JSON.stringify(savedResponse); - const headers = { - "content-type": "application/json", - ...commonResponseHeaders, - }; - options.onResponseStart(200, headers); - options.onData(Buffer.from(body)); - options.onResponseEnd(); - } + true; + + await this.respondWithProtocol( + options, + protocol, + savedResponse, + streamingIsRequested, + commonResponseHeaders, + ); return; } @@ -424,15 +509,14 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { if ( await isRequestOnlySnapshot( state.storedData, - options.body, + normalizedBody, state.workDir, state.toolResultNormalizers, ) ) { const streamingIsRequested = - options.body && (JSON.parse(options.body) as { stream?: boolean }).stream === - true; + true; const headers = { "content-type": streamingIsRequested ? "text/event-stream" @@ -449,7 +533,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { // Beyond this point, we're only going to be able to supply responses in CI if we have a snapshot, // and we only store snapshots for chat completion. For anything else (e.g., custom-agents fetches), // return 404 so the CLI treats them as unavailable instead of erroring. - if (options.requestOptions.path !== chatCompletionEndpoint) { + if (!protocol) { const headers = { "content-type": "application/json", "x-github-request-id": "proxy-not-found", @@ -465,13 +549,16 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { // Fallback to normal proxying if no cached response found // This implicitly captures the new exchange too const isCI = process.env.GITHUB_ACTIONS === "true"; - if (isCI) { + if (isCI || state.backend !== "capi") { await exitWithNoMatchingRequestError( options, state.testInfo, state.workDir, state.toolResultNormalizers, state.storedData, + options.body + ? normalizeModelRequest(protocol, options.body, state.backend) + : options.body, ); return; } @@ -481,6 +568,53 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { } }); } + + private async respondWithProtocol( + options: PerformRequestOptions, + protocol: ModelProtocol, + response: ChatCompletion, + streaming: boolean, + commonHeaders: Record, + ): Promise { + if (!streaming) { + const body = + protocol === "anthropic-messages" + ? chatCompletionResponseToAnthropicMessage(response) + : protocol === "openai-responses" + ? chatCompletionResponseToResponsesApiMessage(response) + : response; + options.onResponseStart(200, { + "content-type": "application/json", + ...commonHeaders, + }); + options.onData(Buffer.from(JSON.stringify(body))); + options.onResponseEnd(); + return; + } + + const chunks = + protocol === "anthropic-messages" + ? chatCompletionResponseToAnthropicSseChunks(response) + : protocol === "openai-responses" + ? chatCompletionResponseToResponsesApiSseChunks(response) + : convertToStreamingResponseChunks(response).map( + (chunk) => `data: ${JSON.stringify(chunk)}\n\n`, + ); + options.onResponseStart(200, { + "content-type": "text/event-stream", + ...commonHeaders, + }); + for (const chunk of chunks) { + options.onData(Buffer.from(chunk)); + if (this.slowStreaming) { + await sleep(100); + } + } + if (protocol === "openai-completions") { + options.onData(Buffer.from("data: [DONE]\n\n")); + } + options.onResponseEnd(); + } } async function writeCapturesToDisk( @@ -542,18 +676,9 @@ function diagnoseMatchFailure( for (let c = 0; c < storedData.conversations.length; c++) { const saved = storedData.conversations[c].messages; - // Same check as findAssistantIndexAfterPrefix: request must be a strict prefix - if (requestMessages.length >= saved.length) { - lines.push( - `Conversation ${c} (${saved.length} messages): ` + - `skipped — request has ${requestMessages.length} messages, need fewer than ${saved.length}.`, - ); - continue; - } - // Find the first message that doesn't match let mismatchIndex = -1; - for (let i = 0; i < requestMessages.length; i++) { + for (let i = 0; i < Math.min(requestMessages.length, saved.length); i++) { if (JSON.stringify(requestMessages[i]) !== JSON.stringify(saved[i])) { mismatchIndex = i; break; @@ -567,10 +692,20 @@ function diagnoseMatchFailure( : "(no raw message)"; lines.push( `Conversation ${c} (${saved.length} messages): mismatch at message ${mismatchIndex}:`, - ` request: ${JSON.stringify(requestMessages[mismatchIndex]).slice(0, 200)}`, - ` saved: ${JSON.stringify(saved[mismatchIndex]).slice(0, 200)}`, + ` request: ${JSON.stringify(requestMessages[mismatchIndex]).slice(0, 1000)}`, + ` saved: ${JSON.stringify(saved[mismatchIndex]).slice(0, 1000)}`, ` raw (pre-normalization): ${raw}`, ); + } else if (requestMessages.length >= saved.length) { + const extraMessage = requestMessages[saved.length]; + lines.push( + `Conversation ${c} (${saved.length} messages): matched the stored conversation, but the request needs a later response.`, + ...(extraMessage + ? [ + ` first extra request message: ${JSON.stringify(extraMessage).slice(0, 200)}`, + ] + : []), + ); } else { // Prefix matched, but the next saved message isn't an assistant turn const nextRole = @@ -591,11 +726,12 @@ async function exitWithNoMatchingRequestError( workDir: string, toolResultNormalizers: ToolResultNormalizer[], storedData?: NormalizedData, + requestBody?: string, ) { let diagnostics: string; try { const normalized = await parseAndNormalizeRequest( - options.body, + requestBody ?? options.body, workDir, toolResultNormalizers, ); @@ -604,8 +740,11 @@ async function exitWithNoMatchingRequestError( let rawMessages: unknown[] = []; try { rawMessages = - (JSON.parse(options.body ?? "{}") as { messages?: unknown[] }) - .messages ?? []; + ( + JSON.parse(requestBody ?? options.body ?? "{}") as { + messages?: unknown[]; + } + ).messages ?? []; } catch { /* non-JSON body */ } @@ -754,6 +893,7 @@ async function transformHttpExchanges( toolResultNormalizers: ToolResultNormalizer[], ): Promise { const chatCompletionExchanges = httpExchanges + .map((exchange) => convertModelExchangeToOpenAI(exchange)) .filter((e) => e.request.url === chatCompletionEndpoint) .filter(excludeFailedResponses); const allTurns = await Promise.all( @@ -774,6 +914,296 @@ async function transformHttpExchanges( return { models: Array.from(dedupedModels), conversations: dedupedExchanges }; } +function parseReplayBackend(value: unknown): ReplayBackend { + if (value === undefined || value === null || value === "") return "capi"; + if ( + value === "capi" || + value === "anthropic-messages" || + value === "openai-responses" || + value === "openai-completions" + ) { + return value; + } + throw new Error(`Unsupported replay backend: ${String(value)}`); +} + +function endpointForBackend(backend: ReplayBackend): string { + switch (backend) { + case "anthropic-messages": + return anthropicMessagesEndpoint; + case "openai-responses": + return responsesEndpoint; + case "capi": + case "openai-completions": + return chatCompletionEndpoint; + } +} + +function protocolForEndpoint(path: string): ModelProtocol | undefined { + switch (path) { + case anthropicMessagesEndpoint: + return "anthropic-messages"; + case responsesEndpoint: + return "openai-responses"; + case chatCompletionEndpoint: + return "openai-completions"; + default: + return undefined; + } +} + +function normalizeModelRequest( + protocol: ModelProtocol, + requestBody: string, + backend?: ReplayBackend, +): string { + switch (protocol) { + case "anthropic-messages": + return anthropicMessagesRequestToChatCompletion(requestBody); + case "openai-responses": + return coalesceAdjacentUserMessages( + responsesApiRequestToChatCompletion(requestBody), + true, + ); + case "openai-completions": + return backend === "openai-completions" + ? coalesceAdjacentUserMessages(requestBody, true) + : requestBody; + } +} + +function coalesceAdjacentUserMessages( + requestBody: string, + normalizeByokSpacing: boolean, +): string { + const request = JSON.parse(requestBody) as { + messages?: Array<{ + role?: string; + content?: unknown; + [key: string]: unknown; + }>; + }; + if (!request.messages) return requestBody; + + const messages: NonNullable = []; + for (const message of request.messages) { + const previous = messages.at(-1); + if ( + previous?.role === "user" && + message.role === "user" && + typeof previous.content === "string" && + typeof message.content === "string" + ) { + previous.content = `${previous.content.trimEnd()}\n\n\n${message.content.trimStart()}`; + } else { + messages.push(message); + } + } + + if (normalizeByokSpacing) { + for (const message of messages) { + if (message.role === "user" && typeof message.content === "string") { + message.content = normalizeUserMessage(message.content).replace( + /\n{5,}/g, + "\n\n\n", + ); + } + } + } + + request.messages = messages; + return JSON.stringify(request); +} + +function createCompactionResponse( + requestBody: string, +): ChatCompletion | undefined { + const request = JSON.parse(requestBody) as { + model?: string; + messages?: Array<{ role?: string; content?: unknown }>; + }; + const messages = (request.messages ?? []).map((message) => ({ + ...message, + content: + message.role === "user" && typeof message.content === "string" + ? normalizeUserMessage(message.content) + : message.content, + })); + if ( + !messages.some( + (message) => + message.role === "user" && + typeof message.content === "string" && + message.content.includes("${compaction_prompt}"), + ) + ) { + return undefined; + } + + const history = messages + .filter((message) => message.role !== "system") + .map((message) => { + const content = + typeof message.content === "string" + ? message.content.replace("${compaction_prompt}", "").trim() + : ""; + return `${message.role ?? "unknown"}: ${content}`; + }) + .filter((line) => !line.endsWith(": ")) + .join("\n\n"); + const content = [ + "Conversation compacted by the E2E replay proxy.", + `${history}`, + "Compacted E2E conversation", + ].join("\n"); + + return { + id: "compaction-e2e-response", + object: "chat.completion", + created: 0, + model: request.model ?? "test-model", + choices: [ + { + index: 0, + message: { + role: "assistant", + content, + refusal: null, + }, + finish_reason: "stop", + logprobs: null, + }, + ], + usage: { + prompt_tokens: 100, + completion_tokens: 50, + total_tokens: 150, + }, + }; +} + +function createPostCompactionResponse( + storedData: NormalizedData, + requestBody: string, + workDir: string, +): ChatCompletion | undefined { + const request = JSON.parse(requestBody) as { + model?: string; + messages?: Array<{ role?: string; content?: unknown }>; + }; + const hasCompactedHistory = (request.messages ?? []).some( + (message) => + message.role === "user" && + typeof message.content === "string" && + message.content.startsWith( + "Some of the conversation history has been summarized to free up context.", + ), + ); + if (!hasCompactedHistory) return undefined; + + for (const conversation of storedData.conversations) { + let replyIndex = conversation.messages.length - 1; + while ( + replyIndex >= 0 && + conversation.messages[replyIndex].role !== "assistant" + ) { + replyIndex--; + } + if (replyIndex >= 0) { + return createOpenAIResponse( + request.model ?? storedData.models[0] ?? "test-model", + conversation.messages, + replyIndex, + workDir, + ); + } + } + return undefined; +} + +function createProtocolError( + protocol: ModelProtocol, + code: string | undefined, + message: string, +): unknown { + const type = code ?? "rate_limited"; + return protocol === "anthropic-messages" + ? { type: "error", error: { type, message } } + : { error: { message, type, code: type } }; +} + +function convertModelExchangeToOpenAI( + exchange: CapturedExchange, + backend?: ReplayBackend, +): CapturedExchange { + const protocol = protocolForEndpoint(exchange.request.url); + if (!protocol || protocol === "openai-completions") return exchange; + + try { + const requestBody = normalizeModelRequest( + protocol, + exchange.request.body, + backend, + ); + let response = exchange.response; + if ( + response?.body && + response.statusCode >= 200 && + response.statusCode < 300 + ) { + const contentType = String(response.headers["content-type"] ?? ""); + const streaming = contentType.includes("text/event-stream"); + let completion: ChatCompletion | undefined; + if (protocol === "anthropic-messages") { + if (streaming) { + const message = aggregateAnthropicSseToMessage(response.body); + if (message) { + completion = anthropicMessageResponseToChatCompletion( + exchange.request.body, + JSON.stringify(message), + ); + } + } else { + completion = anthropicMessageResponseToChatCompletion( + exchange.request.body, + response.body, + ); + } + } else if (streaming) { + const responsesResponse = aggregateResponsesApiSseToResponse( + response.body, + ); + if (responsesResponse) { + completion = responsesApiResponseToChatCompletion( + exchange.request.body, + JSON.stringify(responsesResponse), + ); + } + } else { + completion = responsesApiResponseToChatCompletion( + exchange.request.body, + response.body, + ); + } + + if (!completion) return exchange; + response = { ...response, body: JSON.stringify(completion) }; + } + + return { + ...exchange, + request: { + ...exchange.request, + url: chatCompletionEndpoint, + body: requestBody, + }, + response, + }; + } catch { + return exchange; + } +} + function normalizeFilenames( conversations: NormalizedConversation[], workDir: string, @@ -1048,7 +1478,10 @@ function transformOpenAIRequestMessage( function normalizeUserMessage(content: string): string { return normalizeSkillContextFrontmatter(content) - .replace(taskCompletionNotificationPattern, taskCompletionNotificationReplacement) + .replace( + taskCompletionNotificationPattern, + taskCompletionNotificationReplacement, + ) .replace(/.*?<\/current_datetime>/g, "") .replace(/[\s\S]*?<\/reminder>/g, "") .replace(/[\s\S]*?<\/system_reminder>/g, "") @@ -1079,6 +1512,53 @@ function normalizeStoredUserMessages(conversations: NormalizedConversation[]) { } } +function normalizeStoredMessagesForBackend( + conversations: NormalizedConversation[], + backend: ReplayBackend, +) { + if (backend === "capi") return; + + for (const conversation of conversations) { + conversation.messages = coalesceMessages( + conversation.messages, + backend !== "openai-completions", + true, + ); + } +} + +function coalesceMessages( + messages: NormalizedMessage[], + coalesceAssistantMessages: boolean, + coalesceUserMessages: boolean, +): NormalizedMessage[] { + const result: NormalizedMessage[] = []; + for (const message of messages) { + const previous = result.at(-1); + const shouldCoalesce = + previous?.role === message.role && + ((coalesceAssistantMessages && message.role === "assistant") || + (coalesceUserMessages && message.role === "user")); + if (!shouldCoalesce) { + result.push(message); + continue; + } + + const separator = message.role === "user" ? "\n\n\n" : ""; + const previousContent = previous.content ?? ""; + const currentContent = message.content ?? ""; + const content = `${previousContent}${previousContent && currentContent ? separator : ""}${currentContent}`; + if (content) previous.content = content; + + const toolCalls = [ + ...(previous.tool_calls ?? []), + ...(message.tool_calls ?? []), + ]; + if (toolCalls.length) previous.tool_calls = toolCalls; + } + return result; +} + function normalizeSkillContextFrontmatter(content: string): string { // Runtime versions may include or omit SKILL.md metadata in the prompt context. return content.replace( @@ -1295,10 +1775,17 @@ function findAssistantIndexAfterPrefix( savedMessages: NormalizedMessage[], ): number | undefined { const logFile = process.env.PROXY_DEBUG_LOG; - const log = (msg: string) => { if (logFile) try { appendFileSync(logFile, msg + "\n"); } catch {} }; + const log = (msg: string) => { + if (logFile) + try { + appendFileSync(logFile, msg + "\n"); + } catch {} + }; if (requestMessages.length >= savedMessages.length) { - log(`prefix check failed: request.length=${requestMessages.length} >= saved.length=${savedMessages.length}`); + log( + `prefix check failed: request.length=${requestMessages.length} >= saved.length=${savedMessages.length}`, + ); return undefined; } @@ -1323,7 +1810,9 @@ function findAssistantIndexAfterPrefix( return nextIndex; } - log(`no assistant at nextIndex=${nextIndex}, saved.length=${savedMessages.length}`); + log( + `no assistant at nextIndex=${nextIndex}, saved.length=${savedMessages.length}`, + ); return undefined; } @@ -1546,6 +2035,7 @@ type ReplayingCapiProxyState = { filePath: string; workDir: string; testInfo?: { file: string; line?: number }; + backend: ReplayBackend; storedData?: NormalizedData | undefined; toolResultNormalizers: ToolResultNormalizer[]; }; diff --git a/test/harness/responsesApiAdapter.ts b/test/harness/responsesApiAdapter.ts new file mode 100644 index 0000000000..753d61a5ee --- /dev/null +++ b/test/harness/responsesApiAdapter.ts @@ -0,0 +1,568 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import { randomUUID } from "node:crypto"; +import type { ChatCompletion } from "openai/resources/chat/completions"; +import { parseSseEvents } from "./sseParser"; + +export const responsesEndpoint = "/responses"; + +type JsonObject = Record; + +type CanonicalToolCall = { + id: string; + type: "function"; + function: { name: string; arguments: string }; +}; + +type CanonicalMessage = { + role: "system" | "user" | "assistant" | "tool"; + content?: string | unknown[] | null; + tool_call_id?: string; + tool_calls?: CanonicalToolCall[]; +}; + +type ResponsesRequest = { + model: string; + instructions?: string; + input?: string | JsonObject[]; + stream?: boolean; + tools?: JsonObject[]; + tool_choice?: unknown; + temperature?: number | null; + top_p?: number | null; + parallel_tool_calls?: boolean | null; +}; + +export type ResponsesApiResponse = { + id: string; + object: "response"; + created_at: number; + model: string; + status: "completed"; + output: JsonObject[]; + output_text: string; + incomplete_details: JsonObject | null; + error: null; + instructions: null; + metadata: null; + parallel_tool_calls: boolean; + temperature: null; + tool_choice: "auto"; + tools: []; + top_p: null; + usage: { + input_tokens: number; + output_tokens: number; + total_tokens: number; + input_tokens_details: { cached_tokens: number }; + output_tokens_details: { reasoning_tokens: number }; + }; +}; + +export function responsesApiRequestToChatCompletion( + requestBody: string, +): string { + const request = JSON.parse(requestBody) as ResponsesRequest; + const messages: CanonicalMessage[] = []; + if (request.instructions) { + messages.push({ role: "system", content: request.instructions }); + } + + if (typeof request.input === "string") { + messages.push({ role: "user", content: request.input }); + } else { + for (const item of request.input ?? []) { + const converted = responseInputItemToCanonicalMessages(item); + messages.push(...converted); + } + } + + return JSON.stringify({ + model: request.model, + messages: coalesceAssistantMessages(messages), + ...(request.tools ? { tools: convertResponsesTools(request.tools) } : {}), + ...(request.tool_choice !== undefined + ? { tool_choice: convertResponsesToolChoice(request.tool_choice) } + : {}), + ...(request.stream !== undefined ? { stream: request.stream } : {}), + ...(request.temperature !== undefined && request.temperature !== null + ? { temperature: request.temperature } + : {}), + ...(request.top_p !== undefined && request.top_p !== null + ? { top_p: request.top_p } + : {}), + ...(request.parallel_tool_calls !== undefined && + request.parallel_tool_calls !== null + ? { parallel_tool_calls: request.parallel_tool_calls } + : {}), + }); +} + +function responseInputItemToCanonicalMessages( + item: JsonObject, +): CanonicalMessage[] { + if (item.type === "function_call") { + const callId = + typeof item.call_id === "string" + ? item.call_id + : typeof item.id === "string" + ? item.id + : ""; + return [ + { + role: "assistant", + content: null, + tool_calls: [ + { + id: callId, + type: "function", + function: { + name: typeof item.name === "string" ? item.name : "", + arguments: + typeof item.arguments === "string" ? item.arguments : "{}", + }, + }, + ], + }, + ]; + } + + if (item.type === "function_call_output") { + return [ + { + role: "tool", + tool_call_id: typeof item.call_id === "string" ? item.call_id : "", + content: + typeof item.output === "string" + ? item.output + : JSON.stringify(item.output ?? ""), + }, + ]; + } + + if (item.type === "reasoning") return []; + + if ( + item.type !== "message" && + item.role !== "user" && + item.role !== "assistant" && + item.role !== "system" + ) { + return []; + } + + const role = + item.role === "assistant" || item.role === "system" ? item.role : "user"; + if (typeof item.content === "string") { + return [{ role, content: item.content }]; + } + if (!Array.isArray(item.content)) return [{ role, content: "" }]; + + const parts: unknown[] = []; + for (const part of item.content) { + if (!isObject(part)) continue; + if ( + (part.type === "input_text" || part.type === "output_text") && + typeof part.text === "string" + ) { + parts.push({ type: "text", text: part.text }); + } else if ( + part.type === "input_image" && + typeof part.image_url === "string" + ) { + parts.push({ + type: "image_url", + image_url: { + url: part.image_url, + ...(typeof part.detail === "string" ? { detail: part.detail } : {}), + }, + }); + } else if ( + part.type === "input_file" && + typeof part.file_data === "string" + ) { + parts.push({ + type: "file", + file: { + file_data: part.file_data, + ...(typeof part.filename === "string" + ? { filename: part.filename } + : {}), + }, + }); + } + } + + const onlyText = parts.every( + (part) => isObject(part) && part.type === "text", + ); + return [ + { + role, + content: onlyText + ? parts + .map((part) => + isObject(part) && typeof part.text === "string" ? part.text : "", + ) + .join("") + : parts, + }, + ]; +} + +function coalesceAssistantMessages( + messages: CanonicalMessage[], +): CanonicalMessage[] { + const result: CanonicalMessage[] = []; + for (const message of messages) { + const previous = result[result.length - 1]; + if (message.role === "assistant" && previous?.role === "assistant") { + const previousText = + typeof previous.content === "string" ? previous.content : ""; + const currentText = + typeof message.content === "string" ? message.content : ""; + previous.content = `${previousText}${currentText}` || null; + const toolCalls = [ + ...(previous.tool_calls ?? []), + ...(message.tool_calls ?? []), + ]; + if (toolCalls.length) previous.tool_calls = toolCalls; + } else { + result.push(message); + } + } + return result; +} + +function convertResponsesTools(tools: JsonObject[]): JsonObject[] { + return tools + .filter((tool) => tool.type === "function" && typeof tool.name === "string") + .map((tool) => ({ + type: "function", + function: { + name: tool.name, + ...(typeof tool.description === "string" + ? { description: tool.description } + : {}), + ...(isObject(tool.parameters) ? { parameters: tool.parameters } : {}), + ...(typeof tool.strict === "boolean" ? { strict: tool.strict } : {}), + }, + })); +} + +function convertResponsesToolChoice(toolChoice: unknown): unknown { + if ( + toolChoice === "auto" || + toolChoice === "none" || + toolChoice === "required" + ) { + return toolChoice; + } + if ( + isObject(toolChoice) && + toolChoice.type === "function" && + typeof toolChoice.name === "string" + ) { + return { + type: "function", + function: { name: toolChoice.name }, + }; + } + return undefined; +} + +export function chatCompletionResponseToResponsesApiMessage( + response: ChatCompletion, +): ResponsesApiResponse { + const output: JsonObject[] = []; + const outputText: string[] = []; + + for (const choice of response.choices) { + if (choice.message.content) { + const text = choice.message.content; + outputText.push(text); + output.push({ + type: "message", + id: `msg_${randomUUID()}`, + role: "assistant", + status: "completed", + content: [ + { + type: "output_text", + text, + annotations: [], + }, + ], + }); + } + for (const toolCall of functionToolCalls(choice.message)) { + output.push({ + type: "function_call", + id: `fc_${toolCall.id}`, + call_id: toolCall.id, + name: toolCall.function.name, + arguments: toolCall.function.arguments, + status: "completed", + }); + } + } + + const finishReason = response.choices[0]?.finish_reason; + return { + id: response.id, + object: "response", + created_at: response.created, + model: response.model, + status: "completed", + output, + output_text: outputText.join(""), + incomplete_details: + finishReason === "length" + ? { reason: "max_output_tokens" } + : finishReason === "content_filter" + ? { reason: "content_filter" } + : null, + error: null, + instructions: null, + metadata: null, + parallel_tool_calls: false, + temperature: null, + tool_choice: "auto", + tools: [], + top_p: null, + usage: { + input_tokens: response.usage?.prompt_tokens ?? 0, + output_tokens: response.usage?.completion_tokens ?? 0, + total_tokens: response.usage?.total_tokens ?? 0, + input_tokens_details: { + cached_tokens: + response.usage?.prompt_tokens_details?.cached_tokens ?? 0, + }, + output_tokens_details: { + reasoning_tokens: + response.usage?.completion_tokens_details?.reasoning_tokens ?? 0, + }, + }, + }; +} + +export function chatCompletionResponseToResponsesApiSseChunks( + response: ChatCompletion, +): string[] { + const fullResponse = chatCompletionResponseToResponsesApiMessage(response); + const skeleton = { + ...fullResponse, + output: [], + output_text: "", + usage: undefined, + }; + const chunks: string[] = []; + let sequenceNumber = 0; + const event = (type: string, data: JsonObject) => + formatSseEvent(type, { + type, + sequence_number: sequenceNumber++, + ...data, + }); + + chunks.push( + event("response.created", { response: skeleton }), + event("response.in_progress", { response: skeleton }), + ); + + for ( + let outputIndex = 0; + outputIndex < fullResponse.output.length; + outputIndex++ + ) { + const item = fullResponse.output[outputIndex]; + chunks.push( + event("response.output_item.added", { + output_index: outputIndex, + item, + }), + ); + + if (item.type === "message" && Array.isArray(item.content)) { + for ( + let contentIndex = 0; + contentIndex < item.content.length; + contentIndex++ + ) { + const part = item.content[contentIndex]; + chunks.push( + event("response.content_part.added", { + item_id: item.id, + output_index: outputIndex, + content_index: contentIndex, + part, + }), + ); + if ( + isObject(part) && + part.type === "output_text" && + typeof part.text === "string" + ) { + chunks.push( + event("response.output_text.delta", { + item_id: item.id, + output_index: outputIndex, + content_index: contentIndex, + delta: part.text, + logprobs: [], + }), + event("response.output_text.done", { + item_id: item.id, + output_index: outputIndex, + content_index: contentIndex, + text: part.text, + logprobs: [], + }), + ); + } + chunks.push( + event("response.content_part.done", { + item_id: item.id, + output_index: outputIndex, + content_index: contentIndex, + part, + }), + ); + } + } else if (item.type === "function_call") { + chunks.push( + event("response.function_call_arguments.delta", { + item_id: item.id, + output_index: outputIndex, + delta: item.arguments, + }), + event("response.function_call_arguments.done", { + item_id: item.id, + output_index: outputIndex, + arguments: item.arguments, + }), + ); + } + + chunks.push( + event("response.output_item.done", { + output_index: outputIndex, + item, + }), + ); + } + + chunks.push(event("response.completed", { response: fullResponse })); + return chunks; +} + +export function responsesApiResponseToChatCompletion( + requestBody: string, + responseBody: string, +): ChatCompletion { + const request = JSON.parse(requestBody) as ResponsesRequest; + const response = JSON.parse(responseBody) as ResponsesApiResponse; + const text: string[] = []; + const toolCalls: CanonicalToolCall[] = []; + + for (const item of response.output) { + if (item.type === "message" && Array.isArray(item.content)) { + for (const part of item.content) { + if ( + isObject(part) && + part.type === "output_text" && + typeof part.text === "string" + ) { + text.push(part.text); + } + } + } else if ( + item.type === "function_call" && + typeof item.call_id === "string" + ) { + toolCalls.push({ + id: item.call_id, + type: "function", + function: { + name: typeof item.name === "string" ? item.name : "", + arguments: typeof item.arguments === "string" ? item.arguments : "{}", + }, + }); + } + } + + const incompleteReason = response.incomplete_details?.reason; + return { + id: response.id, + object: "chat.completion", + created: response.created_at, + model: request.model, + choices: [ + { + index: 0, + message: { + role: "assistant", + content: text.length ? text.join("") : null, + refusal: null, + ...(toolCalls.length ? { tool_calls: toolCalls } : {}), + }, + logprobs: null, + finish_reason: + incompleteReason === "max_output_tokens" + ? "length" + : incompleteReason === "content_filter" + ? "content_filter" + : toolCalls.length + ? "tool_calls" + : "stop", + }, + ], + usage: { + prompt_tokens: response.usage?.input_tokens ?? 0, + completion_tokens: response.usage?.output_tokens ?? 0, + total_tokens: response.usage?.total_tokens ?? 0, + }, + } as ChatCompletion; +} + +export function aggregateResponsesApiSseToResponse( + body: string, +): ResponsesApiResponse | null { + let snapshot: ResponsesApiResponse | null = null; + const output: JsonObject[] = []; + for (const event of parseSseEvents(body)) { + if (event.type === "response.completed" && isObject(event.response)) { + return event.response as ResponsesApiResponse; + } + if (event.type === "response.created" && isObject(event.response)) { + snapshot = event.response as ResponsesApiResponse; + } + if (event.type === "response.output_item.done" && isObject(event.item)) { + output.push(event.item); + } + } + return snapshot ? { ...snapshot, output } : null; +} + +function functionToolCalls(message: unknown): CanonicalToolCall[] { + if (!isObject(message) || !Array.isArray(message.tool_calls)) return []; + return message.tool_calls.filter( + (toolCall): toolCall is CanonicalToolCall => + isObject(toolCall) && + typeof toolCall.id === "string" && + toolCall.type === "function" && + isObject(toolCall.function) && + typeof toolCall.function.name === "string" && + typeof toolCall.function.arguments === "string", + ); +} + +function formatSseEvent(type: string, data: unknown): string { + return `event: ${type}\ndata: ${JSON.stringify(data)}\n\n`; +} + +function isObject(value: unknown): value is JsonObject { + return value !== null && typeof value === "object" && !Array.isArray(value); +} diff --git a/test/harness/sseParser.ts b/test/harness/sseParser.ts new file mode 100644 index 0000000000..4f3a8f3455 --- /dev/null +++ b/test/harness/sseParser.ts @@ -0,0 +1,33 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +/** + * Parses JSON Server-Sent Events. Malformed events and `[DONE]` sentinels are + * ignored so recorded provider streams can be normalized best-effort. + */ +export function parseSseEvents( + body: string, +): Array<{ type: string } & Record> { + const events: Array<{ type: string } & Record> = []; + for (const block of body.split(/\r?\n\r?\n/)) { + if (!block.trim()) continue; + + const dataLines = block + .split(/\r?\n/) + .filter((line) => line.startsWith("data:")) + .map((line) => line.slice(5).replace(/^ /, "")); + const payload = dataLines.join("\n"); + if (!payload || payload === "[DONE]") continue; + + try { + const parsed = JSON.parse(payload) as Record; + if (typeof parsed.type === "string") { + events.push(parsed as { type: string } & Record); + } + } catch { + // Ignore malformed events in a partially captured stream. + } + } + return events; +} From 551fa02dd544652a71ebeeb52f38efccb6b5f3a2 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Tue, 14 Jul 2026 21:43:09 +0200 Subject: [PATCH 02/10] Normalize Anthropic adjacent user turns Collapse runtime-specific blank-line expansion so conceptual replay snapshots match recovery turns consistently. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 85a40918-bd95-440b-b6e9-97c757a71f8c --- test/harness/modelProtocolAdapters.test.ts | 51 ++++++++++++++++++++++ test/harness/replayingCapiProxy.ts | 5 ++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/test/harness/modelProtocolAdapters.test.ts b/test/harness/modelProtocolAdapters.test.ts index 088e5d898a..3cde6f9f55 100644 --- a/test/harness/modelProtocolAdapters.test.ts +++ b/test/harness/modelProtocolAdapters.test.ts @@ -522,6 +522,57 @@ describe("protocol-aware replay", () => { } }); + test("normalizes Anthropic spacing for adjacent user turns", async () => { + await writeFile( + cachePath, + yaml.stringify({ + models: ["test-model"], + conversations: [ + { + messages: [ + { role: "system", content: "${system}" }, + { role: "user", content: "First prompt" }, + { role: "user", content: "Recovery prompt" }, + { role: "assistant", content: "Recovered" }, + ], + }, + ], + } satisfies NormalizedData), + ); + const proxy = new ReplayingCapiProxy( + "http://localhost:9999", + cachePath, + workDir, + ); + const proxyUrl = await proxy.start(); + await proxy.updateConfig({ + filePath: cachePath, + workDir, + backend: "anthropic-messages", + }); + + try { + const response = await fetch(`${proxyUrl}/v1/messages`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "test-model", + system: "Be helpful", + messages: [ + { + role: "user", + content: "First prompt\n\n\n\n\nRecovery prompt", + }, + ], + max_tokens: 128, + }), + }); + expect(response.status).toBe(200); + } finally { + await proxy.stop(true); + } + }); + test.each([ { backend: "anthropic-messages" as ReplayBackend, diff --git a/test/harness/replayingCapiProxy.ts b/test/harness/replayingCapiProxy.ts index 379224763c..6bde66919d 100644 --- a/test/harness/replayingCapiProxy.ts +++ b/test/harness/replayingCapiProxy.ts @@ -959,7 +959,10 @@ function normalizeModelRequest( ): string { switch (protocol) { case "anthropic-messages": - return anthropicMessagesRequestToChatCompletion(requestBody); + return coalesceAdjacentUserMessages( + anthropicMessagesRequestToChatCompletion(requestBody), + true, + ); case "openai-responses": return coalesceAdjacentUserMessages( responsesApiRequestToChatCompletion(requestBody), From 97a7c05b916854bb8aa0c09dfb62c43300ea7ea8 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Thu, 16 Jul 2026 19:24:21 +0200 Subject: [PATCH 03/10] Document potential BYOK E2E gaps Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 85a40918-bd95-440b-b6e9-97c757a71f8c --- dotnet/test/E2E/GitHubTelemetryForwardingE2ETests.cs | 2 ++ dotnet/test/E2E/RpcTasksAndHandlersE2ETests.cs | 2 ++ dotnet/test/E2E/SessionConfigE2ETests.cs | 8 ++++++++ dotnet/test/E2E/SessionE2ETests.cs | 2 ++ dotnet/test/E2E/StreamingFidelityE2ETests.cs | 4 ++++ 5 files changed, 18 insertions(+) diff --git a/dotnet/test/E2E/GitHubTelemetryForwardingE2ETests.cs b/dotnet/test/E2E/GitHubTelemetryForwardingE2ETests.cs index 16312a90b2..80d0ccb0b6 100644 --- a/dotnet/test/E2E/GitHubTelemetryForwardingE2ETests.cs +++ b/dotnet/test/E2E/GitHubTelemetryForwardingE2ETests.cs @@ -12,6 +12,8 @@ namespace GitHub.Copilot.Test.E2E; #pragma warning disable GHCP001 // GitHub telemetry forwarding is experimental. +// TODO(BYOK): Anthropic Messages produced no GitHub telemetry notification. Determine whether +// provider-backed sessions should forward the same telemetry before keeping this CAPI-only. [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public class GitHubTelemetryForwardingE2ETests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "github_telemetry", output) diff --git a/dotnet/test/E2E/RpcTasksAndHandlersE2ETests.cs b/dotnet/test/E2E/RpcTasksAndHandlersE2ETests.cs index fe6a20b2d0..989fef7c55 100644 --- a/dotnet/test/E2E/RpcTasksAndHandlersE2ETests.cs +++ b/dotnet/test/E2E/RpcTasksAndHandlersE2ETests.cs @@ -72,6 +72,8 @@ await AssertImplementedFailureAsync( } [Fact] + // TODO(BYOK): Provider-backed task agents handled an invalid model differently. Verify that + // BYOK model validation should reject it consistently before keeping this CAPI-only. [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Report_Implemented_Error_For_Invalid_Task_Agent_Model() { diff --git a/dotnet/test/E2E/SessionConfigE2ETests.cs b/dotnet/test/E2E/SessionConfigE2ETests.cs index e665632847..01095df109 100644 --- a/dotnet/test/E2E/SessionConfigE2ETests.cs +++ b/dotnet/test/E2E/SessionConfigE2ETests.cs @@ -22,6 +22,8 @@ public class SessionConfigE2ETests(E2ETestFixture fixture, ITestOutputHelper out "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="); [Fact] + // TODO(BYOK): Anthropic Messages history diverged after enabling vision via SetModel. Verify + // that model capability overrides work for provider-backed sessions before keeping this CAPI-only. [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Vision_Disabled_Then_Enabled_Via_SetModel() { @@ -63,6 +65,8 @@ await session.SetModelAsync( } [Fact] + // TODO(BYOK): Anthropic Messages history diverged after disabling vision via SetModel. Verify + // that model capability overrides work for provider-backed sessions before keeping this CAPI-only. [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Vision_Enabled_Then_Disabled_Via_SetModel() { @@ -187,6 +191,8 @@ public async Task Should_Apply_ReasoningEffort_On_Session_Resume() } [Fact] + // TODO(BYOK): The Anthropic user-agent omitted ClientName and contained only its provider SDK + // identifier. Determine the expected propagation for custom providers before keeping this CAPI-only. [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Forward_ClientName_In_UserAgent() { @@ -682,6 +688,8 @@ public async Task Should_Create_Session_With_Custom_Provider_Config() } [Fact] + // TODO(BYOK): Anthropic Messages request history diverged while replaying this blob attachment. + // Confirm native clients preserve blob/image turns before keeping this CAPI-only. [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Accept_Blob_Attachments() { diff --git a/dotnet/test/E2E/SessionE2ETests.cs b/dotnet/test/E2E/SessionE2ETests.cs index bf6da85474..8d9a0346db 100644 --- a/dotnet/test/E2E/SessionE2ETests.cs +++ b/dotnet/test/E2E/SessionE2ETests.cs @@ -732,6 +732,8 @@ public async Task DisposeAsync_From_Handler_Does_Not_Deadlock() } [Fact] + // TODO(BYOK): Anthropic Messages request history diverged while replaying this blob attachment. + // Confirm native clients preserve blob/image turns before keeping this CAPI-only. [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Accept_Blob_Attachments() { diff --git a/dotnet/test/E2E/StreamingFidelityE2ETests.cs b/dotnet/test/E2E/StreamingFidelityE2ETests.cs index b10b7bc846..4df9ca4420 100644 --- a/dotnet/test/E2E/StreamingFidelityE2ETests.cs +++ b/dotnet/test/E2E/StreamingFidelityE2ETests.cs @@ -47,6 +47,8 @@ public async Task Should_Produce_Delta_Events_When_Streaming_Is_Enabled() } [Fact] + // TODO(BYOK): Anthropic Messages emitted delta events with Streaming=false. Investigate the + // native-client streaming contract before keeping this disabled for every BYOK backend. [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Not_Produce_Deltas_When_Streaming_Is_Disabled() { @@ -108,6 +110,8 @@ public async Task Should_Produce_Deltas_After_Session_Resume() } [Fact] + // TODO(BYOK): Anthropic Messages emitted delta events after resuming with Streaming=false. + // Investigate the native-client streaming contract before keeping this disabled for every BYOK backend. [Trait(E2ETestTraits.Backend, E2ETestTraits.CapiOnly)] public async Task Should_Not_Produce_Deltas_After_Session_Resume_With_Streaming_Disabled() { From 6b6c00e5d87c90f3ebbabdb27381fa913eab1feb Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Thu, 16 Jul 2026 19:43:45 +0200 Subject: [PATCH 04/10] Restore workflow path quoting Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 85a40918-bd95-440b-b6e9-97c757a71f8c --- .github/workflows/dotnet-sdk-tests.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/dotnet-sdk-tests.yml b/.github/workflows/dotnet-sdk-tests.yml index ce8e5890fa..892e8dc943 100644 --- a/.github/workflows/dotnet-sdk-tests.yml +++ b/.github/workflows/dotnet-sdk-tests.yml @@ -7,19 +7,19 @@ on: pull_request: types: [opened, synchronize, reopened, ready_for_review] paths: - - "dotnet/**" - - "test/**" - - "nodejs/package.json" - - ".github/workflows/dotnet-sdk-tests.yml" - - "!**/*.md" - - "!**/LICENSE*" - - "!**/.gitignore" - - "!**/.editorconfig" - - "!**/*.png" - - "!**/*.jpg" - - "!**/*.jpeg" - - "!**/*.gif" - - "!**/*.svg" + - 'dotnet/**' + - 'test/**' + - 'nodejs/package.json' + - '.github/workflows/dotnet-sdk-tests.yml' + - '!**/*.md' + - '!**/LICENSE*' + - '!**/.gitignore' + - '!**/.editorconfig' + - '!**/*.png' + - '!**/*.jpg' + - '!**/*.jpeg' + - '!**/*.gif' + - '!**/*.svg' workflow_dispatch: merge_group: From 25645be18f0040f7fc9d575288aa7e93e9d02095 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Thu, 16 Jul 2026 19:45:34 +0200 Subject: [PATCH 05/10] Preserve the .NET test matrix Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 85a40918-bd95-440b-b6e9-97c757a71f8c --- .github/workflows/dotnet-sdk-tests.yml | 28 +++++++------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/.github/workflows/dotnet-sdk-tests.yml b/.github/workflows/dotnet-sdk-tests.yml index 892e8dc943..44050392ab 100644 --- a/.github/workflows/dotnet-sdk-tests.yml +++ b/.github/workflows/dotnet-sdk-tests.yml @@ -37,28 +37,14 @@ jobs: strategy: fail-fast: false matrix: - include: - - os: ubuntu-latest - transport: default - backend: capi - test-filter: "" - - os: macos-latest - transport: default - backend: capi - test-filter: "" + os: [ubuntu-latest, macos-latest, windows-latest] + transport: ["default", "inprocess"] + backend: [capi] + # TODO: Re-enable after fixing in-process sqlite file locking on shutdown on Windows. + exclude: - os: windows-latest - transport: default - backend: capi - test-filter: "" - - os: ubuntu-latest - transport: inprocess - backend: capi - test-filter: "" - - os: macos-latest - transport: inprocess - backend: capi - test-filter: "" - # TODO: Add Windows after fixing in-process sqlite file locking on shutdown. + transport: "inprocess" + include: - os: ubuntu-latest transport: inprocess backend: anthropic-messages From 581fc1d9440064eefd6638e15ec0b0b11dfc51f2 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Thu, 16 Jul 2026 19:49:03 +0200 Subject: [PATCH 06/10] Clarify replay harness test leg Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 85a40918-bd95-440b-b6e9-97c757a71f8c --- .github/workflows/dotnet-sdk-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dotnet-sdk-tests.yml b/.github/workflows/dotnet-sdk-tests.yml index 44050392ab..31d791b01e 100644 --- a/.github/workflows/dotnet-sdk-tests.yml +++ b/.github/workflows/dotnet-sdk-tests.yml @@ -98,7 +98,7 @@ jobs: run: npm ci --ignore-scripts - name: Run replay harness tests - if: matrix.os == 'ubuntu-latest' && matrix.transport == 'default' + if: matrix.os == 'ubuntu-latest' && matrix.transport == 'default' && matrix.backend == 'capi' working-directory: ./test/harness run: npm test From 34b6a6072ca8867557b14d637c1f27144d9e2a3b Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Thu, 16 Jul 2026 19:53:06 +0200 Subject: [PATCH 07/10] Use existing replay harness test coverage Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 85a40918-bd95-440b-b6e9-97c757a71f8c --- .github/workflows/dotnet-sdk-tests.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/dotnet-sdk-tests.yml b/.github/workflows/dotnet-sdk-tests.yml index 31d791b01e..91e92c0454 100644 --- a/.github/workflows/dotnet-sdk-tests.yml +++ b/.github/workflows/dotnet-sdk-tests.yml @@ -97,11 +97,6 @@ jobs: working-directory: ./test/harness run: npm ci --ignore-scripts - - name: Run replay harness tests - if: matrix.os == 'ubuntu-latest' && matrix.transport == 'default' && matrix.backend == 'capi' - working-directory: ./test/harness - run: npm test - - name: Warm up PowerShell if: runner.os == 'Windows' run: pwsh.exe -Command "Write-Host 'PowerShell ready'" From 818f289858b20fbd83b2209ad448a7cce55e4225 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Fri, 17 Jul 2026 00:18:58 +0200 Subject: [PATCH 08/10] Simplify BYOK replay harness Make non-CAPI replay explicitly read-only so provider response parsing and SSE aggregation are unnecessary. Keep protocol-complete forward rendering, clarify backend trait naming, and rename the .NET proxy wrapper to reflect its protocol-neutral role. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 85a40918-bd95-440b-b6e9-97c757a71f8c --- .../instructions/dotnet-e2e.instructions.md | 9 + .github/workflows/dotnet-sdk-tests.yml | 6 +- dotnet/test/AssemblyInfo.cs | 2 +- .../E2E/ByokBearerTokenProviderE2ETests.cs | 2 +- dotnet/test/E2E/ClientOptionsE2ETests.cs | 4 +- .../E2E/CopilotRequestSessionIdE2ETests.cs | 2 +- .../E2E/CopilotRequestWebSocketE2ETests.cs | 2 +- .../test/E2E/MultiProviderRegistryE2ETests.cs | 2 +- dotnet/test/E2E/ProviderEndpointE2ETests.cs | 2 +- .../test/E2E/RpcSessionStateExtrasE2ETests.cs | 2 +- dotnet/test/E2E/SessionConfigE2ETests.cs | 20 +- dotnet/test/E2E/SessionE2ETests.cs | 6 +- dotnet/test/Harness/E2ETestBackend.cs | 51 +- dotnet/test/Harness/E2ETestContext.cs | 6 +- .../Harness/{CapiProxy.cs => ReplayProxy.cs} | 12 +- dotnet/test/Unit/E2ETestBackendTests.cs | 47 +- test/harness/anthropicMessagesAdapter.ts | 199 +---- test/harness/modelProtocolAdapterShared.ts | 39 + test/harness/modelProtocolAdapters.test.ts | 691 ++++++++---------- test/harness/replayingCapiProxy.ts | 166 ++--- test/harness/responsesApiAdapter.ts | 162 +--- test/harness/sseParser.ts | 33 - 22 files changed, 514 insertions(+), 951 deletions(-) create mode 100644 .github/instructions/dotnet-e2e.instructions.md rename dotnet/test/Harness/{CapiProxy.cs => ReplayProxy.cs} (96%) create mode 100644 test/harness/modelProtocolAdapterShared.ts delete mode 100644 test/harness/sseParser.ts diff --git a/.github/instructions/dotnet-e2e.instructions.md b/.github/instructions/dotnet-e2e.instructions.md new file mode 100644 index 0000000000..8dcf7d5330 --- /dev/null +++ b/.github/instructions/dotnet-e2e.instructions.md @@ -0,0 +1,9 @@ +--- +applyTo: "dotnet/test/E2E/**/*.cs" +--- + +# .NET E2E test instructions + +- Create and resume sessions through `E2ETestContext` using `Ctx.CreateSessionAsync` and `Ctx.ResumeSessionAsync`. Do not call these methods directly on `CopilotClient`; the context applies the backend selected by the E2E matrix while preserving providers explicitly configured by the test. +- Create clients with `Ctx.CreateClient` so they receive the harness environment, CLI path, authentication defaults, transport handling, and lifecycle tracking. +- Instantiate `CopilotClient` directly only when client construction, startup, shutdown, or disposal is the behavior under test. Keep cleanup explicit in those tests, and still create any sessions through `Ctx` so they participate in backend coverage. diff --git a/.github/workflows/dotnet-sdk-tests.yml b/.github/workflows/dotnet-sdk-tests.yml index 91e92c0454..ecd5dcd09c 100644 --- a/.github/workflows/dotnet-sdk-tests.yml +++ b/.github/workflows/dotnet-sdk-tests.yml @@ -48,15 +48,15 @@ jobs: - os: ubuntu-latest transport: inprocess backend: anthropic-messages - test-filter: "FullyQualifiedName~GitHub.Copilot.Test.E2E&E2EBackend!=SelfConfiguredProvider&E2EBackend!=CapiOnly" + test-filter: "FullyQualifiedName~GitHub.Copilot.Test.E2E&E2EBackend!=SelfConfiguredBackend&E2EBackend!=CapiOnly" - os: ubuntu-latest transport: inprocess backend: openai-responses - test-filter: "FullyQualifiedName~GitHub.Copilot.Test.E2E&E2EBackend!=SelfConfiguredProvider&E2EBackend!=CapiOnly" + test-filter: "FullyQualifiedName~GitHub.Copilot.Test.E2E&E2EBackend!=SelfConfiguredBackend&E2EBackend!=CapiOnly" - os: ubuntu-latest transport: inprocess backend: openai-completions - test-filter: "FullyQualifiedName~GitHub.Copilot.Test.E2E&E2EBackend!=SelfConfiguredProvider&E2EBackend!=CapiOnly" + test-filter: "FullyQualifiedName~GitHub.Copilot.Test.E2E&E2EBackend!=SelfConfiguredBackend&E2EBackend!=CapiOnly" runs-on: ${{ matrix.os }} defaults: run: diff --git a/dotnet/test/AssemblyInfo.cs b/dotnet/test/AssemblyInfo.cs index 9380ca48cd..6f5f258a91 100644 --- a/dotnet/test/AssemblyInfo.cs +++ b/dotnet/test/AssemblyInfo.cs @@ -5,7 +5,7 @@ using Xunit; using GitHub.Copilot.Test.Harness; -// Each E2E test class fixture spins up its own Copilot CLI subprocess plus a CapiProxy +// Each E2E test class fixture spins up its own Copilot CLI subprocess plus a ReplayProxy // (replaying HTTP proxy) Node.js subprocess. With ~25 test classes, running them in parallel // would launch ~50 long-lived Node.js processes simultaneously and exhaust both file // descriptors and memory on developer machines and CI runners (especially Windows). Tests diff --git a/dotnet/test/E2E/ByokBearerTokenProviderE2ETests.cs b/dotnet/test/E2E/ByokBearerTokenProviderE2ETests.cs index 26e19c009a..4d2cb5e34d 100644 --- a/dotnet/test/E2E/ByokBearerTokenProviderE2ETests.cs +++ b/dotnet/test/E2E/ByokBearerTokenProviderE2ETests.cs @@ -36,7 +36,7 @@ namespace GitHub.Copilot.Test.E2E; /// and the resulting token reaches that provider's endpoint. /// /// -[Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] +[Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public class ByokBearerTokenProviderE2ETests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "byok_bearer_token_provider", output) { diff --git a/dotnet/test/E2E/ClientOptionsE2ETests.cs b/dotnet/test/E2E/ClientOptionsE2ETests.cs index 4489ef442f..bf995563c4 100644 --- a/dotnet/test/E2E/ClientOptionsE2ETests.cs +++ b/dotnet/test/E2E/ClientOptionsE2ETests.cs @@ -220,7 +220,7 @@ public async Task Should_Forward_Granular_Multitenancy_Fields_In_Create_Wire_Req } [Fact] - [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Forward_Advanced_Session_Options_In_Create_Wire_Request() { var (cliPath, capturePath) = await CreateFakeCliCaptureAsync(); @@ -368,7 +368,7 @@ public async Task Should_Forward_Advanced_Session_Options_In_Create_Wire_Request } [Fact] - [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Forward_Singular_Provider_Options_In_Create_Wire_Request() { var (cliPath, capturePath) = await CreateFakeCliCaptureAsync(); diff --git a/dotnet/test/E2E/CopilotRequestSessionIdE2ETests.cs b/dotnet/test/E2E/CopilotRequestSessionIdE2ETests.cs index 1aa1442a7d..fd00cc9b99 100644 --- a/dotnet/test/E2E/CopilotRequestSessionIdE2ETests.cs +++ b/dotnet/test/E2E/CopilotRequestSessionIdE2ETests.cs @@ -65,7 +65,7 @@ public async Task Threads_The_Session_Id_Into_A_Capi_Session_Inference_Request() } [Fact] - [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Threads_The_Session_Id_Into_A_Byok_Session_Inference_Request() { var provider = new RecordingRequestHandler(); diff --git a/dotnet/test/E2E/CopilotRequestWebSocketE2ETests.cs b/dotnet/test/E2E/CopilotRequestWebSocketE2ETests.cs index c1cc92fc52..80ccdb8c90 100644 --- a/dotnet/test/E2E/CopilotRequestWebSocketE2ETests.cs +++ b/dotnet/test/E2E/CopilotRequestWebSocketE2ETests.cs @@ -31,7 +31,7 @@ namespace GitHub.Copilot.Test.E2E; /// message. Without the eager start the turn never completes and this test /// times out. /// -[Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] +[Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public class CopilotRequestWebSocketE2ETests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "copilot_request_websocket", output) { diff --git a/dotnet/test/E2E/MultiProviderRegistryE2ETests.cs b/dotnet/test/E2E/MultiProviderRegistryE2ETests.cs index f04398a7ff..80827cc867 100644 --- a/dotnet/test/E2E/MultiProviderRegistryE2ETests.cs +++ b/dotnet/test/E2E/MultiProviderRegistryE2ETests.cs @@ -18,7 +18,7 @@ namespace GitHub.Copilot.Test.E2E; /// session, be launched, and route inference to the configured provider with /// the configured wire model and headers. /// -[Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] +[Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public class MultiProviderRegistryE2ETests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "multi_provider_registry", output) { diff --git a/dotnet/test/E2E/ProviderEndpointE2ETests.cs b/dotnet/test/E2E/ProviderEndpointE2ETests.cs index 2cc4a47425..d2bf06982f 100644 --- a/dotnet/test/E2E/ProviderEndpointE2ETests.cs +++ b/dotnet/test/E2E/ProviderEndpointE2ETests.cs @@ -27,7 +27,7 @@ private CopilotClient CreateProviderEndpointClient() } [Fact] - [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task ShouldReturnByokProviderEndpointWhenCustomProviderIsConfigured() { var client = CreateProviderEndpointClient(); diff --git a/dotnet/test/E2E/RpcSessionStateExtrasE2ETests.cs b/dotnet/test/E2E/RpcSessionStateExtrasE2ETests.cs index ca339668e9..28ec9b7cfe 100644 --- a/dotnet/test/E2E/RpcSessionStateExtrasE2ETests.cs +++ b/dotnet/test/E2E/RpcSessionStateExtrasE2ETests.cs @@ -45,7 +45,7 @@ public async Task Should_List_Models_For_Session() } [Fact] - [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Add_Byok_Provider_And_Model_At_Runtime() { await using var session = await CreateSessionAsync(); diff --git a/dotnet/test/E2E/SessionConfigE2ETests.cs b/dotnet/test/E2E/SessionConfigE2ETests.cs index 01095df109..ad313b116d 100644 --- a/dotnet/test/E2E/SessionConfigE2ETests.cs +++ b/dotnet/test/E2E/SessionConfigE2ETests.cs @@ -127,7 +127,7 @@ public async Task Should_Use_Custom_SessionId() } [Fact] - [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Apply_ReasoningEffort_On_Session_Create() { const string reasoningModelId = "custom-reasoning-model"; @@ -147,7 +147,7 @@ public async Task Should_Apply_ReasoningEffort_On_Session_Create() } [Theory] - [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] [InlineData("low")] [InlineData("medium")] [InlineData("high")] @@ -170,7 +170,7 @@ public async Task Should_Apply_All_ReasoningEffort_Values_On_Session_Create(stri } [Fact] - [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Apply_ReasoningEffort_On_Session_Resume() { var originalSession = await CreateSessionAsync(); @@ -210,7 +210,7 @@ public async Task Should_Forward_ClientName_In_UserAgent() } [Fact] - [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Forward_Custom_Provider_Headers_On_Create() { var session = await CreateSessionAsync(new SessionConfig @@ -230,7 +230,7 @@ public async Task Should_Forward_Custom_Provider_Headers_On_Create() } [Fact] - [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Forward_Custom_Provider_Headers_On_Resume() { var session1 = await CreateSessionAsync(); @@ -253,7 +253,7 @@ public async Task Should_Forward_Custom_Provider_Headers_On_Resume() } [Fact] - [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Forward_Provider_Wire_Model() { // Verifies that ProviderConfig.WireModel overrides the model name sent to @@ -285,7 +285,7 @@ public async Task Should_Forward_Provider_Wire_Model() } [Fact] - [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Use_Provider_Model_Id_As_Wire_Model() { // ProviderConfig.ModelId drives both the runtime resolved model AND the wire model @@ -580,7 +580,7 @@ public async Task Should_Apply_Excluded_Built_In_Agents_On_Resume() } [Fact] - [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Enable_Citations_For_Anthropic_File_Attachments_On_Create() { var handler = new RecordingRequestHandler(); @@ -612,7 +612,7 @@ await session.SendAndWaitAsync(new MessageOptions } [Fact] - [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Enable_Citations_For_Anthropic_File_Attachments_On_Resume() { const string connectionToken = "citation-resume-token"; @@ -660,7 +660,7 @@ await session2.SendAndWaitAsync(new MessageOptions } [Fact] - [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Create_Session_With_Custom_Provider_Config() { // Per the TS test (session_config.e2e.test.ts), this only verifies that a diff --git a/dotnet/test/E2E/SessionE2ETests.cs b/dotnet/test/E2E/SessionE2ETests.cs index 8d9a0346db..bcc8fc268d 100644 --- a/dotnet/test/E2E/SessionE2ETests.cs +++ b/dotnet/test/E2E/SessionE2ETests.cs @@ -925,7 +925,7 @@ await session.SendAndWaitAsync(new MessageOptions } [Fact] - [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Create_Session_With_Custom_Provider() { var session = await CreateSessionAsync(new SessionConfig @@ -951,7 +951,7 @@ public async Task Should_Create_Session_With_Custom_Provider() } [Fact] - [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Create_Session_With_Azure_Provider() { var session = await CreateSessionAsync(new SessionConfig @@ -981,7 +981,7 @@ public async Task Should_Create_Session_With_Azure_Provider() } [Fact] - [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredProvider)] + [Trait(E2ETestTraits.Backend, E2ETestTraits.SelfConfiguredBackend)] public async Task Should_Resume_Session_With_Custom_Provider() { var session = await CreateSessionAsync(); diff --git a/dotnet/test/Harness/E2ETestBackend.cs b/dotnet/test/Harness/E2ETestBackend.cs index 1cfa3e67e1..e0d1c4c53d 100644 --- a/dotnet/test/Harness/E2ETestBackend.cs +++ b/dotnet/test/Harness/E2ETestBackend.cs @@ -2,6 +2,8 @@ * Copyright (c) Microsoft Corporation. All rights reserved. *--------------------------------------------------------------------------------------------*/ +using System.Diagnostics; + namespace GitHub.Copilot.Test.Harness; internal enum E2ETestBackend @@ -15,7 +17,8 @@ internal enum E2ETestBackend internal static class E2ETestBackendConfiguration { internal const string EnvironmentVariable = "COPILOT_SDK_E2E_BACKEND"; - private const string DefaultModel = "claude-sonnet-4.5"; + private const string AnthropicDefaultModel = "claude-sonnet-4.5"; + private const string OpenAIDefaultModel = "gpt-4.1"; private const string FakeCredential = "fake-byok-credential-for-e2e-tests"; internal static E2ETestBackend Current @@ -28,7 +31,7 @@ internal static E2ETestBackend Parse(string? value) "anthropic-messages" => E2ETestBackend.AnthropicMessages, "openai-responses" => E2ETestBackend.OpenAIResponses, "openai-completions" => E2ETestBackend.OpenAICompletions, - _ => throw new InvalidOperationException( + _ => throw new UnreachableException( $"Unsupported {EnvironmentVariable} value '{value}'. Expected capi, anthropic-messages, openai-responses, or openai-completions."), }; @@ -39,7 +42,7 @@ internal static string ToWireName(this E2ETestBackend backend) E2ETestBackend.AnthropicMessages => "anthropic-messages", E2ETestBackend.OpenAIResponses => "openai-responses", E2ETestBackend.OpenAICompletions => "openai-completions", - _ => throw new ArgumentOutOfRangeException(nameof(backend), backend, null), + _ => throw new UnreachableException(), }; internal static void ApplyProvider( @@ -54,8 +57,8 @@ internal static void ApplyProvider( return; } - config.Model ??= DefaultModel; - config.Provider = CreateProvider(backend, proxyUrl); + var model = config.Model ??= backend.GetDefaultModel(); + config.Provider = CreateProvider(backend, proxyUrl, model); } internal static void ApplyProvider( @@ -68,30 +71,52 @@ internal static void ApplyProvider( return; } - config.Model ??= DefaultModel; - config.Provider = CreateProvider(backend, proxyUrl); + var model = config.Model ??= backend.GetDefaultModel(); + config.Provider = CreateProvider(backend, proxyUrl, model); } - private static ProviderConfig CreateProvider(E2ETestBackend backend, string proxyUrl) + private static string GetDefaultModel(this E2ETestBackend backend) + => backend switch + { + E2ETestBackend.AnthropicMessages => AnthropicDefaultModel, + E2ETestBackend.OpenAIResponses or E2ETestBackend.OpenAICompletions => OpenAIDefaultModel, + _ => throw new UnreachableException(), + }; + + private static ProviderConfig CreateProvider( + E2ETestBackend backend, + string proxyUrl, + string model) => new() { BaseUrl = proxyUrl, - Type = backend == E2ETestBackend.AnthropicMessages ? "anthropic" : "openai", + Type = backend switch + { + E2ETestBackend.AnthropicMessages => "anthropic", + E2ETestBackend.OpenAIResponses or E2ETestBackend.OpenAICompletions => "openai", + _ => throw new UnreachableException(), + }, WireApi = backend switch { + E2ETestBackend.AnthropicMessages => null, E2ETestBackend.OpenAIResponses => "responses", E2ETestBackend.OpenAICompletions => "completions", - _ => null, + _ => throw new UnreachableException(), }, BearerToken = FakeCredential, - ModelId = DefaultModel, - WireModel = DefaultModel, + ModelId = model, + WireModel = model, }; } internal static class E2ETestTraits { + // Trait key used by workflow filters to classify backend compatibility. internal const string Backend = "E2EBackend"; + + // Requires the default CAPI backend and is excluded from BYOK legs. internal const string CapiOnly = "CapiOnly"; - internal const string SelfConfiguredProvider = "SelfConfiguredProvider"; + + // Owns its backend setup and must not inherit the backend selected by the test matrix. + internal const string SelfConfiguredBackend = "SelfConfiguredBackend"; } diff --git a/dotnet/test/Harness/E2ETestContext.cs b/dotnet/test/Harness/E2ETestContext.cs index b39dfcee45..c01f4efcff 100644 --- a/dotnet/test/Harness/E2ETestContext.cs +++ b/dotnet/test/Harness/E2ETestContext.cs @@ -20,13 +20,13 @@ public sealed class E2ETestContext : IAsyncDisposable /// Optional logger injected by tests; applied to all clients created via . public ILogger? Logger { get; set; } - private readonly CapiProxy _proxy; + private readonly ReplayProxy _proxy; private readonly string _repoRoot; private readonly object _clientsLock = new(); private readonly List _persistentClients = []; private readonly List _transientClients = []; - private E2ETestContext(string homeDir, string workDir, string proxyUrl, CapiProxy proxy, string repoRoot) + private E2ETestContext(string homeDir, string workDir, string proxyUrl, ReplayProxy proxy, string repoRoot) { HomeDir = homeDir; WorkDir = workDir; @@ -50,7 +50,7 @@ public static async Task CreateAsync() homeDir = ResolveSymlinks(homeDir); workDir = ResolveSymlinks(workDir); - var proxy = new CapiProxy(); + var proxy = new ReplayProxy(); var proxyUrl = await proxy.StartAsync(); await proxy.SetCopilotUserByTokenAsync(DefaultGitHubToken, new CopilotUserConfig( Login: "e2e-test-user", diff --git a/dotnet/test/Harness/CapiProxy.cs b/dotnet/test/Harness/ReplayProxy.cs similarity index 96% rename from dotnet/test/Harness/CapiProxy.cs rename to dotnet/test/Harness/ReplayProxy.cs index 0e8de207ab..895ebccb87 100644 --- a/dotnet/test/Harness/CapiProxy.cs +++ b/dotnet/test/Harness/ReplayProxy.cs @@ -13,7 +13,7 @@ namespace GitHub.Copilot.Test.Harness; -public sealed partial class CapiProxy : IAsyncDisposable +public sealed partial class ReplayProxy : IAsyncDisposable { private Process? _process; private Task? _startupTask; @@ -76,7 +76,7 @@ async Task StartCoreAsync() { var metadata = JsonSerializer.Deserialize( match.Groups["metadata"].Value, - CapiProxyJsonContext.Default.ProxyStartupMetadata); + ReplayProxyJsonContext.Default.ProxyStartupMetadata); ConnectProxyUrl = metadata?.ConnectProxyUrl; CaFilePath = metadata?.CaFilePath; } @@ -158,7 +158,7 @@ public async Task ConfigureAsync(string filePath, string workDir, string backend var response = await client.PostAsJsonAsync( $"{url}/config", new ConfigureRequest(filePath, workDir, backend), - CapiProxyJsonContext.Default.ConfigureRequest); + ReplayProxyJsonContext.Default.ConfigureRequest); response.EnsureSuccessStatusCode(); } @@ -171,7 +171,7 @@ public async Task> GetExchangesAsync() var url = await (_startupTask ?? throw new InvalidOperationException("Proxy not started")); using var client = new HttpClient(); - return await client.GetFromJsonAsync($"{url}/exchanges", CapiProxyJsonContext.Default.ListParsedHttpExchange) + return await client.GetFromJsonAsync($"{url}/exchanges", ReplayProxyJsonContext.Default.ListParsedHttpExchange) ?? []; } @@ -181,7 +181,7 @@ public async Task SetCopilotUserByTokenAsync(string token, CopilotUserConfig res using var client = new HttpClient(); var payload = new CopilotUserByTokenRequest(token, response); - var resp = await client.PostAsJsonAsync($"{url}/copilot-user-config", payload, CapiProxyJsonContext.Default.CopilotUserByTokenRequest); + var resp = await client.PostAsJsonAsync($"{url}/copilot-user-config", payload, ReplayProxyJsonContext.Default.CopilotUserByTokenRequest); resp.EnsureSuccessStatusCode(); } @@ -208,7 +208,7 @@ private static string FindRepoRoot() [JsonSerializable(typeof(CopilotUserByTokenRequest))] [JsonSerializable(typeof(Dictionary))] [JsonSerializable(typeof(ProxyStartupMetadata))] - private partial class CapiProxyJsonContext : JsonSerializerContext; + private partial class ReplayProxyJsonContext : JsonSerializerContext; } public record CopilotUserByTokenRequest(string Token, CopilotUserConfig Response); diff --git a/dotnet/test/Unit/E2ETestBackendTests.cs b/dotnet/test/Unit/E2ETestBackendTests.cs index 1b90c7a2ff..eda24e1941 100644 --- a/dotnet/test/Unit/E2ETestBackendTests.cs +++ b/dotnet/test/Unit/E2ETestBackendTests.cs @@ -2,6 +2,7 @@ * Copyright (c) Microsoft Corporation. All rights reserved. *--------------------------------------------------------------------------------------------*/ +using System.Diagnostics; using GitHub.Copilot.Test.Harness; using Xunit; @@ -21,26 +22,60 @@ public void ParsesBackend(string? value, string expected) [Fact] public void RejectsUnknownBackend() - => Assert.Throws( + => Assert.Throws( () => E2ETestBackendConfiguration.Parse("unknown")); [Theory] - [InlineData("anthropic-messages", "anthropic", null)] - [InlineData("openai-responses", "openai", "responses")] - [InlineData("openai-completions", "openai", "completions")] + [InlineData("anthropic-messages", "anthropic", null, "claude-sonnet-4.5")] + [InlineData("openai-responses", "openai", "responses", "gpt-4.1")] + [InlineData("openai-completions", "openai", "completions", "gpt-4.1")] public void AppliesProvider( string backendValue, string expectedType, - string? expectedWireApi) + string? expectedWireApi, + string expectedModel) { var backend = E2ETestBackendConfiguration.Parse(backendValue); var config = new SessionConfig(); backend.ApplyProvider(config, "http://localhost:1234"); - Assert.Equal("claude-sonnet-4.5", config.Model); + Assert.Equal(expectedModel, config.Model); Assert.Equal("http://localhost:1234", config.Provider!.BaseUrl); Assert.Equal(expectedType, config.Provider.Type); Assert.Equal(expectedWireApi, config.Provider.WireApi); + Assert.Equal(expectedModel, config.Provider.ModelId); + Assert.Equal(expectedModel, config.Provider.WireModel); Assert.False(string.IsNullOrEmpty(config.Provider.BearerToken)); } + + [Fact] + public void PreservesExplicitModel() + { + var config = new SessionConfig { Model = "test-model" }; + E2ETestBackend.OpenAIResponses.ApplyProvider(config, "http://localhost:1234"); + + Assert.Equal("test-model", config.Model); + Assert.Equal("test-model", config.Provider!.ModelId); + Assert.Equal("test-model", config.Provider.WireModel); + } + + [Fact] + public void PreservesExplicitProvider() + { + var provider = new ProviderConfig + { + Type = "custom", + ModelId = "provider-model", + }; + var config = new SessionConfig + { + Model = "session-model", + Provider = provider, + }; + + E2ETestBackend.OpenAIResponses.ApplyProvider(config, "http://localhost:1234"); + + Assert.Equal("session-model", config.Model); + Assert.Same(provider, config.Provider); + } } diff --git a/test/harness/anthropicMessagesAdapter.ts b/test/harness/anthropicMessagesAdapter.ts index 6ead783e30..acc74a2bf9 100644 --- a/test/harness/anthropicMessagesAdapter.ts +++ b/test/harness/anthropicMessagesAdapter.ts @@ -3,12 +3,17 @@ *--------------------------------------------------------------------------------------------*/ import type { ChatCompletion } from "openai/resources/chat/completions"; -import { parseSseEvents } from "./sseParser"; +import { + CanonicalMessage, + CanonicalToolCall, + formatSseEvent, + functionToolCalls, + isObject, + JsonObject, +} from "./modelProtocolAdapterShared"; export const anthropicMessagesEndpoint = "/v1/messages"; -type JsonObject = Record; - type CanonicalContentPart = | { type: "text"; text: string } | { type: "image_url"; image_url: { url: string } } @@ -17,19 +22,6 @@ type CanonicalContentPart = file: { file_data: string; filename?: string }; }; -type CanonicalMessage = { - role: "system" | "user" | "assistant" | "tool"; - content?: string | CanonicalContentPart[] | null; - tool_call_id?: string; - tool_calls?: CanonicalToolCall[]; -}; - -type CanonicalToolCall = { - id: string; - type: "function"; - function: { name: string; arguments: string }; -}; - type AnthropicContentBlock = | { type: "text"; text: string; citations?: null } | { @@ -100,14 +92,6 @@ const finishReasonToStopReason: Record = { content_filter: "refusal", }; -const stopReasonToFinishReason: Record = { - end_turn: "stop", - stop_sequence: "stop", - max_tokens: "length", - tool_use: "tool_calls", - refusal: "content_filter", -}; - export function anthropicMessagesRequestToChatCompletion( requestBody: string, ): string { @@ -403,169 +387,6 @@ export function chatCompletionResponseToAnthropicSseChunks( return chunks; } -export function anthropicMessageResponseToChatCompletion( - requestBody: string, - responseBody: string, -): ChatCompletion { - const request = JSON.parse(requestBody) as AnthropicRequest; - const message = JSON.parse(responseBody) as AnthropicMessage; - const text: string[] = []; - const toolCalls: CanonicalToolCall[] = []; - for (const block of message.content) { - if (block.type === "text") { - text.push(block.text); - } else { - toolCalls.push({ - id: block.id, - type: "function", - function: { - name: block.name, - arguments: JSON.stringify(block.input ?? {}), - }, - }); - } - } - - return { - id: message.id, - object: "chat.completion", - created: Math.floor(Date.now() / 1000), - model: request.model, - choices: [ - { - index: 0, - message: { - role: "assistant", - content: text.length ? text.join("") : null, - refusal: null, - ...(toolCalls.length ? { tool_calls: toolCalls } : {}), - }, - logprobs: null, - finish_reason: - (message.stop_reason && - stopReasonToFinishReason[message.stop_reason]) || - "stop", - }, - ], - usage: { - prompt_tokens: - message.usage.input_tokens + - (message.usage.cache_read_input_tokens ?? 0), - completion_tokens: message.usage.output_tokens, - total_tokens: - message.usage.input_tokens + - (message.usage.cache_read_input_tokens ?? 0) + - message.usage.output_tokens, - }, - } as ChatCompletion; -} - -export function aggregateAnthropicSseToMessage( - body: string, -): AnthropicMessage | null { - let message: AnthropicMessage | null = null; - const blocks: AnthropicMessage["content"] = []; - const toolInputs: string[] = []; - - for (const event of parseSseEvents(body)) { - if (event.type === "message_start" && isObject(event.message)) { - message = { - ...(event.message as AnthropicMessage), - content: [], - }; - } else if ( - event.type === "content_block_start" && - typeof event.index === "number" && - isObject(event.content_block) - ) { - const block = event.content_block; - if (block.type === "text") { - blocks[event.index] = { - type: "text", - text: typeof block.text === "string" ? block.text : "", - citations: null, - }; - } else if ( - block.type === "tool_use" && - typeof block.id === "string" && - typeof block.name === "string" - ) { - blocks[event.index] = { - type: "tool_use", - id: block.id, - name: block.name, - input: {}, - }; - toolInputs[event.index] = ""; - } - } else if ( - event.type === "content_block_delta" && - typeof event.index === "number" && - isObject(event.delta) - ) { - const block = blocks[event.index]; - if ( - block?.type === "text" && - event.delta.type === "text_delta" && - typeof event.delta.text === "string" - ) { - block.text += event.delta.text; - } else if ( - block?.type === "tool_use" && - event.delta.type === "input_json_delta" && - typeof event.delta.partial_json === "string" - ) { - toolInputs[event.index] += event.delta.partial_json; - } - } else if ( - event.type === "content_block_stop" && - typeof event.index === "number" - ) { - const block = blocks[event.index]; - if (block?.type === "tool_use") { - block.input = safeParseJson(toolInputs[event.index] || "{}"); - } - } else if ( - event.type === "message_delta" && - message && - isObject(event.delta) - ) { - if (typeof event.delta.stop_reason === "string") { - message.stop_reason = event.delta.stop_reason as AnthropicStopReason; - } - if ( - isObject(event.usage) && - typeof event.usage.output_tokens === "number" - ) { - message.usage.output_tokens = event.usage.output_tokens; - } - } - } - - if (!message) return null; - message.content = blocks.filter( - (block): block is AnthropicMessage["content"][number] => Boolean(block), - ); - return message; -} - -function functionToolCalls(message: unknown): CanonicalToolCall[] { - if (!isObject(message) || !Array.isArray(message.tool_calls)) return []; - return message.tool_calls.filter( - (toolCall): toolCall is CanonicalToolCall => - isObject(toolCall) && - typeof toolCall.id === "string" && - toolCall.type === "function" && - isObject(toolCall.function) && - typeof toolCall.function.name === "string" && - typeof toolCall.function.arguments === "string", - ); -} - -function formatSseEvent(event: string, data: unknown): string { - return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`; -} - function safeParseJson(value: string): unknown { try { return JSON.parse(value); @@ -573,7 +394,3 @@ function safeParseJson(value: string): unknown { return {}; } } - -function isObject(value: unknown): value is JsonObject { - return value !== null && typeof value === "object" && !Array.isArray(value); -} diff --git a/test/harness/modelProtocolAdapterShared.ts b/test/harness/modelProtocolAdapterShared.ts new file mode 100644 index 0000000000..1f879da5d0 --- /dev/null +++ b/test/harness/modelProtocolAdapterShared.ts @@ -0,0 +1,39 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +export type JsonObject = Record; + +export type CanonicalToolCall = { + id: string; + type: "function"; + function: { name: string; arguments: string }; +}; + +export type CanonicalMessage = { + role: "system" | "user" | "assistant" | "tool"; + content?: string | unknown[] | null; + tool_call_id?: string; + tool_calls?: CanonicalToolCall[]; +}; + +export function functionToolCalls(message: unknown): CanonicalToolCall[] { + if (!isObject(message) || !Array.isArray(message.tool_calls)) return []; + return message.tool_calls.filter( + (toolCall): toolCall is CanonicalToolCall => + isObject(toolCall) && + typeof toolCall.id === "string" && + toolCall.type === "function" && + isObject(toolCall.function) && + typeof toolCall.function.name === "string" && + typeof toolCall.function.arguments === "string", + ); +} + +export function formatSseEvent(type: string, data: unknown): string { + return `event: ${type}\ndata: ${JSON.stringify(data)}\n\n`; +} + +export function isObject(value: unknown): value is JsonObject { + return value !== null && typeof value === "object" && !Array.isArray(value); +} diff --git a/test/harness/modelProtocolAdapters.test.ts b/test/harness/modelProtocolAdapters.test.ts index 3cde6f9f55..1b7466db9e 100644 --- a/test/harness/modelProtocolAdapters.test.ts +++ b/test/harness/modelProtocolAdapters.test.ts @@ -2,25 +2,21 @@ * Copyright (c) Microsoft Corporation. All rights reserved. *--------------------------------------------------------------------------------------------*/ -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import type { ChatCompletion } from "openai/resources/chat/completions"; import { afterEach, beforeEach, describe, expect, test } from "vitest"; import yaml from "yaml"; import { - aggregateAnthropicSseToMessage, - anthropicMessageResponseToChatCompletion, anthropicMessagesRequestToChatCompletion, chatCompletionResponseToAnthropicMessage, chatCompletionResponseToAnthropicSseChunks, } from "./anthropicMessagesAdapter"; import { - aggregateResponsesApiSseToResponse, chatCompletionResponseToResponsesApiMessage, chatCompletionResponseToResponsesApiSseChunks, responsesApiRequestToChatCompletion, - responsesApiResponseToChatCompletion, } from "./responsesApiAdapter"; import { NormalizedData, @@ -28,6 +24,26 @@ import { ReplayingCapiProxy, } from "./replayingCapiProxy"; +type ByokBackend = Exclude; + +const backends: ByokBackend[] = [ + "anthropic-messages", + "openai-responses", + "openai-completions", +]; + +const endpoints: Record = { + "anthropic-messages": "/v1/messages", + "openai-responses": "/responses", + "openai-completions": "/chat/completions", +}; + +const models: Record = { + "anthropic-messages": "claude-sonnet-4.5", + "openai-responses": "gpt-4.1", + "openai-completions": "gpt-4.1", +}; + const completionWithTool: ChatCompletion = { id: "completion-1", object: "chat.completion", @@ -44,10 +60,7 @@ const completionWithTool: ChatCompletion = { { id: "call-1", type: "function", - function: { - name: "lookup", - arguments: '{"value":42}', - }, + function: { name: "lookup", arguments: '{"value":42}' }, }, ], }, @@ -62,8 +75,56 @@ const completionWithTool: ChatCompletion = { }, }; +function requestFor( + backend: ByokBackend, + prompt: string, +): Record { + const model = models[backend]; + switch (backend) { + case "anthropic-messages": + return { + model, + system: "Be helpful", + messages: [{ role: "user", content: prompt }], + max_tokens: 128, + }; + case "openai-responses": + return { + model, + instructions: "Be helpful", + input: [ + { + type: "message", + role: "user", + content: [{ type: "input_text", text: prompt }], + }, + ], + }; + case "openai-completions": + return { + model, + messages: [ + { role: "system", content: "Be helpful" }, + { role: "user", content: prompt }, + ], + }; + } +} + +async function postJson( + proxyUrl: string, + endpoint: string, + body: unknown, +): Promise { + return fetch(`${proxyUrl}${endpoint}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); +} + describe("Anthropic Messages adapter", () => { - test("normalizes system, images, tools, and tool results", () => { + test("normalizes messages, binary content, and tools", () => { const result = JSON.parse( anthropicMessagesRequestToChatCompletion( JSON.stringify({ @@ -122,91 +183,67 @@ describe("Anthropic Messages adapter", () => { stream: boolean; }; - expect(result.messages).toEqual([ - { role: "system", content: "Be helpful" }, + expect(result.messages.map((message) => message.role)).toEqual([ + "system", + "user", + "assistant", + "tool", + ]); + expect(result.messages[1].content).toEqual([ + { type: "text", text: "Inspect this" }, { - role: "user", - content: [ - { type: "text", text: "Inspect this" }, - { - type: "image_url", - image_url: { url: "data:image/png;base64,AQID" }, - }, - ], + type: "image_url", + image_url: { url: "data:image/png;base64,AQID" }, }, + ]); + expect(result.messages[2].tool_calls).toEqual([ { - role: "assistant", - content: null, - tool_calls: [ - { - id: "call-1", - type: "function", - function: { - name: "lookup", - arguments: '{"value":42}', - }, - }, - ], + id: "call-1", + type: "function", + function: { name: "lookup", arguments: '{"value":42}' }, }, - { role: "tool", tool_call_id: "call-1", content: "found" }, ]); + expect(result.messages[3]).toMatchObject({ + tool_call_id: "call-1", + content: "found", + }); expect(result.tools).toHaveLength(1); expect(result.stream).toBe(true); }); - test("round-trips JSON and streaming responses", () => { + test("renders JSON and streaming tool responses", () => { const message = chatCompletionResponseToAnthropicMessage(completionWithTool); expect(message.stop_reason).toBe("tool_use"); + expect(message.usage).toMatchObject({ + cache_creation_input_tokens: null, + cache_read_input_tokens: null, + }); expect(message.content.map((block) => block.type)).toEqual([ "text", "tool_use", ]); - const request = JSON.stringify({ - model: "test-model", - messages: [{ role: "user", content: "Hello" }], - }); - const fromJson = anthropicMessageResponseToChatCompletion( - request, - JSON.stringify(message), - ); - expect(fromJson.choices[0].message.tool_calls).toHaveLength(1); - const stream = chatCompletionResponseToAnthropicSseChunks(completionWithTool).join(""); expect(stream).toContain("event: message_start"); + expect(stream).toContain("event: content_block_delta"); expect(stream).toContain("event: message_stop"); - const aggregated = aggregateAnthropicSseToMessage(stream); - expect(aggregated).not.toBeNull(); - expect(aggregated!.content).toEqual(message.content); }); - test("combines CAPI multi-choice assistant messages", () => { + test("combines tools from multiple canonical choices", () => { + const secondChoice = structuredClone(completionWithTool.choices[0]); + secondChoice.message.content = null; + secondChoice.message.tool_calls![0] = { + id: "call-2", + type: "function", + function: { name: "inspect", arguments: '{"path":"file.txt"}' }, + }; + const message = chatCompletionResponseToAnthropicMessage({ ...completionWithTool, - choices: [ - completionWithTool.choices[0], - { - index: 1, - message: { - role: "assistant", - content: null, - refusal: null, - tool_calls: [ - { - id: "call-2", - type: "function", - function: { name: "inspect", arguments: '{"path":"file.txt"}' }, - }, - ], - }, - logprobs: null, - finish_reason: "tool_calls", - }, - ], + choices: [completionWithTool.choices[0], secondChoice], }); - expect( message.content .filter((block) => block.type === "tool_use") @@ -216,7 +253,7 @@ describe("Anthropic Messages adapter", () => { }); describe("OpenAI Responses adapter", () => { - test("normalizes instructions, binary content, and function history", () => { + test("normalizes messages, binary content, and tools", () => { const result = JSON.parse( responsesApiRequestToChatCompletion( JSON.stringify({ @@ -260,66 +297,60 @@ describe("OpenAI Responses adapter", () => { tools: Array>; }; - expect(result.messages).toEqual([ - { role: "system", content: "Be helpful" }, + expect(result.messages.map((message) => message.role)).toEqual([ + "system", + "user", + "assistant", + "tool", + ]); + expect(result.messages[1].content).toEqual([ + { type: "text", text: "Inspect this" }, { - role: "user", - content: [ - { type: "text", text: "Inspect this" }, - { - type: "image_url", - image_url: { - url: "data:image/png;base64,AQID", - }, - }, - ], + type: "image_url", + image_url: { url: "data:image/png;base64,AQID" }, }, + ]); + expect(result.messages[2].tool_calls).toEqual([ { - role: "assistant", - content: null, - tool_calls: [ - { - id: "call-1", - type: "function", - function: { - name: "lookup", - arguments: '{"value":42}', - }, - }, - ], + id: "call-1", + type: "function", + function: { name: "lookup", arguments: '{"value":42}' }, }, - { role: "tool", tool_call_id: "call-1", content: "found" }, ]); + expect(result.messages[3]).toMatchObject({ + tool_call_id: "call-1", + content: "found", + }); expect(result.tools).toHaveLength(1); }); - test("round-trips JSON and streaming responses", () => { + test("renders JSON and streaming tool responses", () => { const response = chatCompletionResponseToResponsesApiMessage(completionWithTool); + const nextResponse = + chatCompletionResponseToResponsesApiMessage(completionWithTool); + expect(response).toMatchObject({ + object: "response", + created_at: completionWithTool.created, + status: "completed", + incomplete_details: null, + error: null, + }); + expect(response.output[0].id).not.toBe(nextResponse.output[0].id); expect(response.output.map((item) => item.type)).toEqual([ "message", "function_call", ]); - const request = JSON.stringify({ model: "test-model", input: "Hello" }); - const fromJson = responsesApiResponseToChatCompletion( - request, - JSON.stringify(response), - ); - expect(fromJson.choices[0].message.tool_calls).toHaveLength(1); - const stream = chatCompletionResponseToResponsesApiSseChunks(completionWithTool).join( "", ); expect(stream).toContain("event: response.created"); - expect(stream).toContain("event: response.completed"); - const aggregated = aggregateResponsesApiSseToResponse(stream); - expect(aggregated?.output_text).toBe(response.output_text); - expect(aggregated?.output.map((item) => item.type)).toEqual([ - "message", - "function_call", - ]); + expect(stream).toContain("event: response.in_progress"); + expect(stream).toContain("event: response.output_text.delta"); + expect(stream).toContain('"sequence_number":0'); + expect(stream).toContain("event: response.completed"); }); }); @@ -328,217 +359,88 @@ describe("protocol-aware replay", () => { let workDir: string; let cachePath: string; - beforeEach(async () => { - tempDir = await mkdtemp(path.join(os.tmpdir(), "protocol-replay-")); - workDir = path.join(tempDir, "work"); - cachePath = path.join(tempDir, "cache.yaml"); + async function writeSnapshot( + messages: NormalizedData["conversations"][number]["messages"], + ): Promise { await writeFile( cachePath, yaml.stringify({ - models: ["test-model"], - conversations: [ - { - messages: [ - { role: "system", content: "${system}" }, - { role: "user", content: "Hello" }, - { role: "assistant", content: "Hi there!" }, - ], - }, - ], + models: ["captured-capi-model"], + conversations: [{ messages }], } satisfies NormalizedData), ); - }); + } - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - test.each([ - { - backend: "anthropic-messages" as ReplayBackend, - endpoint: "/v1/messages", - request: { - model: "test-model", - system: "Be helpful", - messages: [{ role: "user", content: "Hello" }], - max_tokens: 128, - }, - assertBody: (body: Record) => { - expect(body.type).toBe("message"); - expect(body.content).toEqual([ - { type: "text", text: "Hi there!", citations: null }, - ]); - }, - }, - { - backend: "openai-responses" as ReplayBackend, - endpoint: "/responses", - request: { - model: "test-model", - instructions: "Be helpful", - input: [ - { - type: "message", - role: "user", - content: [{ type: "input_text", text: "Hello" }], - }, - ], - }, - assertBody: (body: Record) => { - expect(body.object).toBe("response"); - expect(body.output_text).toBe("Hi there!"); - }, - }, - { - backend: "openai-completions" as ReplayBackend, - endpoint: "/chat/completions", - request: { - model: "test-model", - messages: [ - { role: "system", content: "Be helpful" }, - { role: "user", content: "Hello" }, - ], - }, - assertBody: (body: Record) => { - expect(body.object).toBe("chat.completion"); - }, - }, - ])("replays $backend and exposes canonical exchanges", async (testCase) => { + async function withProxy( + backend: ByokBackend, + action: (proxyUrl: string) => Promise, + ): Promise { const proxy = new ReplayingCapiProxy( "http://localhost:9999", cachePath, workDir, ); const proxyUrl = await proxy.start(); - await proxy.updateConfig({ - filePath: cachePath, - workDir, - backend: testCase.backend, - }); - + await proxy.updateConfig({ filePath: cachePath, workDir, backend }); try { - const response = await fetch(`${proxyUrl}${testCase.endpoint}`, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(testCase.request), - }); - expect(response.status).toBe(200); - testCase.assertBody((await response.json()) as Record); - - const exchangesResponse = await fetch(`${proxyUrl}/exchanges`); - const exchanges = (await exchangesResponse.json()) as Array<{ - request: { messages: Array<{ role: string; content: unknown }> }; - }>; - expect(exchanges).toHaveLength(1); - expect(exchanges[0].request.messages.at(-1)).toEqual({ - role: "user", - content: "Hello", - }); + await action(proxyUrl); } finally { await proxy.stop(true); } + } + + beforeEach(async () => { + tempDir = await mkdtemp(path.join(os.tmpdir(), "protocol-replay-")); + workDir = path.join(tempDir, "work"); + cachePath = path.join(tempDir, "cache.yaml"); + await writeSnapshot([ + { role: "system", content: "${system}" }, + { role: "user", content: "Hello" }, + { role: "assistant", content: "Hi there!" }, + ]); }); - test.each([ - { - backend: "openai-responses" as ReplayBackend, - endpoint: "/responses", - request: { - model: "test-model", - instructions: "Be helpful", - input: [ - { - type: "message", - role: "user", - content: [ - { - type: "input_text", - text: "Hook context\n\n\n2026-01-01T00:00:00Z\n\n", - }, - ], - }, - { - type: "message", - role: "user", - content: [{ type: "input_text", text: "Hello" }], - }, - ], - }, - }, - { - backend: "openai-completions" as ReplayBackend, - endpoint: "/chat/completions", - request: { - model: "test-model", - messages: [ - { role: "system", content: "Be helpful" }, - { - role: "user", - content: - "Hook context\n\n\n2026-01-01T00:00:00Z\n\n", - }, - { role: "user", content: "Hello" }, - ], - }, - }, - ])("coalesces adjacent user messages for $backend", async (testCase) => { - await writeFile( - cachePath, - yaml.stringify({ - models: ["test-model"], - conversations: [ - { - messages: [ - { role: "system", content: "${system}" }, - { role: "user", content: "Hook context" }, - { role: "user", content: "Hello" }, - { role: "assistant", content: "Hi there!" }, - ], - }, - ], - } satisfies NormalizedData), - ); - const proxy = new ReplayingCapiProxy( - "http://localhost:9999", - cachePath, - workDir, - ); - const proxyUrl = await proxy.start(); - await proxy.updateConfig({ - filePath: cachePath, - workDir, - backend: testCase.backend, - }); + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); - try { - const response = await fetch(`${proxyUrl}${testCase.endpoint}`, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(testCase.request), + test.each(backends)( + "replays one model-independent snapshot through %s", + async (backend) => { + await withProxy(backend, async (proxyUrl) => { + const response = await postJson( + proxyUrl, + endpoints[backend], + requestFor(backend, "Hello"), + ); + expect(response.status).toBe(200); + const body = (await response.json()) as Record; + expect(body.model).toBe(models[backend]); + + const exchanges = (await ( + await fetch(`${proxyUrl}/exchanges`) + ).json()) as Array<{ + request: { + model: string; + messages: Array<{ role: string; content: unknown }>; + }; + response?: unknown; + }>; + expect(exchanges).toHaveLength(1); + expect(exchanges[0].request.model).toBe(models[backend]); + expect(exchanges[0].request.messages.at(-1)).toEqual({ + role: "user", + content: "Hello", + }); + if (backend !== "openai-completions") { + expect(exchanges[0].response).toBeUndefined(); + } }); - expect(response.status).toBe(200); - } finally { - await proxy.stop(true); - } - }); + }, + ); - test("normalizes Anthropic spacing for adjacent user turns", async () => { - await writeFile( - cachePath, - yaml.stringify({ - models: ["test-model"], - conversations: [ - { - messages: [ - { role: "system", content: "${system}" }, - { role: "user", content: "First prompt" }, - { role: "user", content: "Recovery prompt" }, - { role: "assistant", content: "Recovered" }, - ], - }, - ], - } satisfies NormalizedData), - ); + test("does not rewrite canonical snapshots after BYOK replay", async () => { + const original = await readFile(cachePath, "utf8"); const proxy = new ReplayingCapiProxy( "http://localhost:9999", cachePath, @@ -548,123 +450,108 @@ describe("protocol-aware replay", () => { await proxy.updateConfig({ filePath: cachePath, workDir, - backend: "anthropic-messages", + backend: "openai-responses", }); + let stopped = false; try { - const response = await fetch(`${proxyUrl}/v1/messages`, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - model: "test-model", - system: "Be helpful", - messages: [ - { - role: "user", - content: "First prompt\n\n\n\n\nRecovery prompt", - }, - ], - max_tokens: 128, - }), - }); + const response = await postJson( + proxyUrl, + endpoints["openai-responses"], + requestFor("openai-responses", "Hello"), + ); expect(response.status).toBe(200); + await proxy.stop(); + stopped = true; + expect(await readFile(cachePath, "utf8")).toBe(original); } finally { - await proxy.stop(true); + if (!stopped) await proxy.stop(true); } }); - test.each([ - { - backend: "anthropic-messages" as ReplayBackend, - endpoint: "/v1/messages", - request: { - model: "test-model", - system: "Be helpful", - messages: [{ role: "user", content: "${compaction_prompt}" }], - max_tokens: 128, - }, - }, - { - backend: "openai-responses" as ReplayBackend, - endpoint: "/responses", - request: { - model: "test-model", - instructions: "Be helpful", - input: [ - { - type: "message", - role: "user", - content: [{ type: "input_text", text: "${compaction_prompt}" }], - }, - ], - }, - }, - { - backend: "openai-completions" as ReplayBackend, - endpoint: "/chat/completions", - request: { - model: "test-model", - messages: [ - { role: "system", content: "Be helpful" }, - { role: "user", content: "${compaction_prompt}" }, - ], - }, - }, - ])("synthesizes a compaction response for $backend", async (testCase) => { - const proxy = new ReplayingCapiProxy( - "http://localhost:9999", - cachePath, - workDir, - ); - const proxyUrl = await proxy.start(); - await proxy.updateConfig({ - filePath: cachePath, - workDir, - backend: testCase.backend, - }); + test.each(["openai-responses", "openai-completions"] as const)( + "coalesces adjacent user messages from %s", + async (backend) => { + await writeSnapshot([ + { role: "system", content: "${system}" }, + { role: "user", content: "Hook context" }, + { role: "user", content: "Hello" }, + { role: "assistant", content: "Hi there!" }, + ]); + const request = requestFor(backend, "Hello"); + const hook = + "Hook context\n\n\n2026-01-01T00:00:00Z\n\n"; + if (backend === "openai-responses") { + (request.input as unknown[]).unshift({ + type: "message", + role: "user", + content: [{ type: "input_text", text: hook }], + }); + } else { + (request.messages as unknown[]).splice(1, 0, { + role: "user", + content: hook, + }); + } - try { - const response = await fetch(`${proxyUrl}${testCase.endpoint}`, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(testCase.request), + await withProxy(backend, async (proxyUrl) => { + const response = await postJson( + proxyUrl, + endpoints[backend], + request, + ); + expect(response.status).toBe(200); }); - expect(response.status).toBe(200); - const body = JSON.stringify(await response.json()); - expect(body).toContain(""); - expect(body).toContain(""); - expect(body).toContain(""); - } finally { - await proxy.stop(true); - } - }); + }, + ); - test("rejects an inference request sent over the wrong protocol", async () => { - const proxy = new ReplayingCapiProxy( - "http://localhost:9999", - cachePath, - workDir, - ); - const proxyUrl = await proxy.start(); - await proxy.updateConfig({ - filePath: cachePath, - workDir, - backend: "anthropic-messages", + test("normalizes Anthropic spacing between adjacent user turns", async () => { + await writeSnapshot([ + { role: "system", content: "${system}" }, + { role: "user", content: "First prompt" }, + { role: "user", content: "Recovery prompt" }, + { role: "assistant", content: "Recovered" }, + ]); + await withProxy("anthropic-messages", async (proxyUrl) => { + const response = await postJson( + proxyUrl, + endpoints["anthropic-messages"], + requestFor( + "anthropic-messages", + "First prompt\n\n\n\n\nRecovery prompt", + ), + ); + expect(response.status).toBe(200); }); + }); - try { - const response = await fetch(`${proxyUrl}/chat/completions`, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - model: "test-model", - messages: [{ role: "user", content: "Hello" }], - }), + test.each(backends)( + "synthesizes compaction responses through %s", + async (backend) => { + await withProxy(backend, async (proxyUrl) => { + const response = await postJson( + proxyUrl, + endpoints[backend], + requestFor(backend, "${compaction_prompt}"), + ); + expect(response.status).toBe(200); + const body = JSON.stringify(await response.json()); + expect(body).toContain(""); + expect(body).toContain(""); + expect(body).toContain(""); }); + }, + ); + + test("rejects an inference request over the wrong protocol", async () => { + await withProxy("anthropic-messages", async (proxyUrl) => { + const response = await postJson( + proxyUrl, + endpoints["openai-completions"], + requestFor("openai-completions", "Hello"), + ); expect(response.status).toBe(400); await expect(response.text()).resolves.toContain("protocol_mismatch"); - } finally { - await proxy.stop(true); - } + }); }); }); diff --git a/test/harness/replayingCapiProxy.ts b/test/harness/replayingCapiProxy.ts index 6bde66919d..cee6d65c6a 100644 --- a/test/harness/replayingCapiProxy.ts +++ b/test/harness/replayingCapiProxy.ts @@ -2,7 +2,7 @@ * Copyright (c) Microsoft Corporation. All rights reserved. *--------------------------------------------------------------------------------------------*/ -import { existsSync, appendFileSync } from "fs"; +import { existsSync } from "fs"; import { mkdir, readFile, writeFile } from "fs/promises"; import type { ChatCompletion, @@ -20,31 +20,21 @@ import { PerformRequestOptions, } from "./capturingHttpProxy"; import { - aggregateAnthropicSseToMessage, - anthropicMessageResponseToChatCompletion, anthropicMessagesEndpoint, anthropicMessagesRequestToChatCompletion, chatCompletionResponseToAnthropicMessage, chatCompletionResponseToAnthropicSseChunks, } from "./anthropicMessagesAdapter"; import { - aggregateResponsesApiSseToResponse, chatCompletionResponseToResponsesApiMessage, chatCompletionResponseToResponsesApiSseChunks, responsesApiRequestToChatCompletion, - responsesApiResponseToChatCompletion, responsesEndpoint, } from "./responsesApiAdapter"; import { iife, ShellConfig, sleep } from "./util"; export const workingDirPlaceholder = "${workdir}"; const chatCompletionEndpoint = "/chat/completions"; -const replayedModelRequestEndpoints = new Set([ - chatCompletionEndpoint, - anthropicMessagesEndpoint, - responsesEndpoint, -]); - export type ReplayBackend = | "capi" | "anthropic-messages" @@ -146,7 +136,10 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { // In CI mode (GITHUB_ACTIONS=true) we never write — the snapshots are read-only. // Otherwise tests that exercise only a subset of a multi-conversation snapshot // would silently overwrite the file with that subset, breaking subsequent runs. - if (this.state && process.env.GITHUB_ACTIONS !== "true") { + if ( + this.state?.backend === "capi" && + process.env.GITHUB_ACTIONS !== "true" + ) { await writeCapturesToDisk(this.exchanges, this.state); } @@ -178,9 +171,10 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { async stop(skipWritingCache?: boolean): Promise { await super.stop(); - // In CI mode we never write — the snapshots are read-only. + // CAPI is the authoritative capture path. BYOK modes only verify that the + // same canonical snapshots replay through each provider protocol. if ( - this.state && + this.state?.backend === "capi" && !skipWritingCache && process.env.GITHUB_ACTIONS !== "true" ) { @@ -262,18 +256,35 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { options.requestOptions.path === "/exchanges" && options.requestOptions.method === "GET" ) { - const chatCompletionExchanges = this.exchanges - .filter((e) => replayedModelRequestEndpoints.has(e.request.url)) - .map((exchange) => - convertModelExchangeToOpenAI(exchange, this.state?.backend), - ) - .filter((e) => e.request.url === chatCompletionEndpoint); + const modelExchanges = this.exchanges.flatMap((exchange) => { + const protocol = protocolForEndpoint(exchange.request.url); + if (!protocol) return []; + + try { + return [ + { + requestBody: normalizeModelRequest( + protocol, + exchange.request.body, + this.state?.backend, + ), + responseBody: + protocol === "openai-completions" + ? exchange.response?.body + : undefined, + requestHeaders: exchange.request.headers, + }, + ]; + } catch { + return []; + } + }); const parsedExchanges = await Promise.all( - chatCompletionExchanges.map((e) => + modelExchanges.map((exchange) => parseHttpExchange( - e.request.body, - e.response?.body, - e.request.headers, + exchange.requestBody, + exchange.responseBody, + exchange.requestHeaders, ), ), ); @@ -787,11 +798,14 @@ async function findSavedChatCompletionResponse( toolResultNormalizers, ); const requestMessages = normalized.conversations[0]?.messages ?? []; - const requestModel = normalized.models[0]; - if (!requestModel) { + const responseModel = normalized.models[0]; + if (!responseModel) { throw new Error("Unable to determine model from request"); } + // Snapshot model IDs are capture metadata, not match criteria. Render the + // canonical response using the active request's model so one transcript can + // replay against different provider/model profiles. // Now find a matching cached conversation (i.e., one for which this request is a prefix) for (const conversation of storedData.conversations) { const replyIndex = findAssistantIndexAfterPrefix( @@ -800,7 +814,7 @@ async function findSavedChatCompletionResponse( ); if (replyIndex !== undefined) { return createOpenAIResponse( - requestModel, + responseModel, conversation.messages, replyIndex, workDir, @@ -893,7 +907,6 @@ async function transformHttpExchanges( toolResultNormalizers: ToolResultNormalizer[], ): Promise { const chatCompletionExchanges = httpExchanges - .map((exchange) => convertModelExchangeToOpenAI(exchange)) .filter((e) => e.request.url === chatCompletionEndpoint) .filter(excludeFailedResponses); const allTurns = await Promise.all( @@ -1135,78 +1148,6 @@ function createProtocolError( : { error: { message, type, code: type } }; } -function convertModelExchangeToOpenAI( - exchange: CapturedExchange, - backend?: ReplayBackend, -): CapturedExchange { - const protocol = protocolForEndpoint(exchange.request.url); - if (!protocol || protocol === "openai-completions") return exchange; - - try { - const requestBody = normalizeModelRequest( - protocol, - exchange.request.body, - backend, - ); - let response = exchange.response; - if ( - response?.body && - response.statusCode >= 200 && - response.statusCode < 300 - ) { - const contentType = String(response.headers["content-type"] ?? ""); - const streaming = contentType.includes("text/event-stream"); - let completion: ChatCompletion | undefined; - if (protocol === "anthropic-messages") { - if (streaming) { - const message = aggregateAnthropicSseToMessage(response.body); - if (message) { - completion = anthropicMessageResponseToChatCompletion( - exchange.request.body, - JSON.stringify(message), - ); - } - } else { - completion = anthropicMessageResponseToChatCompletion( - exchange.request.body, - response.body, - ); - } - } else if (streaming) { - const responsesResponse = aggregateResponsesApiSseToResponse( - response.body, - ); - if (responsesResponse) { - completion = responsesApiResponseToChatCompletion( - exchange.request.body, - JSON.stringify(responsesResponse), - ); - } - } else { - completion = responsesApiResponseToChatCompletion( - exchange.request.body, - response.body, - ); - } - - if (!completion) return exchange; - response = { ...response, body: JSON.stringify(completion) }; - } - - return { - ...exchange, - request: { - ...exchange.request, - url: chatCompletionEndpoint, - body: requestBody, - }, - response, - }; - } catch { - return exchange; - } -} - function normalizeFilenames( conversations: NormalizedConversation[], workDir: string, @@ -1777,28 +1718,14 @@ function findAssistantIndexAfterPrefix( requestMessages: NormalizedMessage[], savedMessages: NormalizedMessage[], ): number | undefined { - const logFile = process.env.PROXY_DEBUG_LOG; - const log = (msg: string) => { - if (logFile) - try { - appendFileSync(logFile, msg + "\n"); - } catch {} - }; - if (requestMessages.length >= savedMessages.length) { - log( - `prefix check failed: request.length=${requestMessages.length} >= saved.length=${savedMessages.length}`, - ); return undefined; } for (let i = 0; i < requestMessages.length; i++) { - const reqMsg = JSON.stringify(requestMessages[i]); - const savedMsg = JSON.stringify(savedMessages[i]); - if (reqMsg !== savedMsg) { - log(`mismatch at index ${i}:`); - log(` REQ: ${reqMsg.substring(0, 1000)}`); - log(` SAVED: ${savedMsg.substring(0, 1000)}`); + if ( + JSON.stringify(requestMessages[i]) !== JSON.stringify(savedMessages[i]) + ) { return undefined; } } @@ -1809,13 +1736,9 @@ function findAssistantIndexAfterPrefix( nextIndex < savedMessages.length && savedMessages[nextIndex].role === "assistant" ) { - log(`MATCH found at index ${nextIndex}`); return nextIndex; } - log( - `no assistant at nextIndex=${nextIndex}, saved.length=${savedMessages.length}`, - ); return undefined; } @@ -2074,6 +1997,7 @@ interface NormalizedErrorResponse { } export interface NormalizedData { + /** Captured model IDs used to replay /models; conversations are model-agnostic. */ models: string[]; errors?: NormalizedErrorResponse[]; conversations: NormalizedConversation[]; diff --git a/test/harness/responsesApiAdapter.ts b/test/harness/responsesApiAdapter.ts index 753d61a5ee..27650b7897 100644 --- a/test/harness/responsesApiAdapter.ts +++ b/test/harness/responsesApiAdapter.ts @@ -4,25 +4,18 @@ import { randomUUID } from "node:crypto"; import type { ChatCompletion } from "openai/resources/chat/completions"; -import { parseSseEvents } from "./sseParser"; +import type { Response as OpenAIResponse } from "openai/resources/responses/responses"; +import { + CanonicalMessage, + CanonicalToolCall, + formatSseEvent, + functionToolCalls, + isObject, + JsonObject, +} from "./modelProtocolAdapterShared"; export const responsesEndpoint = "/responses"; -type JsonObject = Record; - -type CanonicalToolCall = { - id: string; - type: "function"; - function: { name: string; arguments: string }; -}; - -type CanonicalMessage = { - role: "system" | "user" | "assistant" | "tool"; - content?: string | unknown[] | null; - tool_call_id?: string; - tool_calls?: CanonicalToolCall[]; -}; - type ResponsesRequest = { model: string; instructions?: string; @@ -35,31 +28,7 @@ type ResponsesRequest = { parallel_tool_calls?: boolean | null; }; -export type ResponsesApiResponse = { - id: string; - object: "response"; - created_at: number; - model: string; - status: "completed"; - output: JsonObject[]; - output_text: string; - incomplete_details: JsonObject | null; - error: null; - instructions: null; - metadata: null; - parallel_tool_calls: boolean; - temperature: null; - tool_choice: "auto"; - tools: []; - top_p: null; - usage: { - input_tokens: number; - output_tokens: number; - total_tokens: number; - input_tokens_details: { cached_tokens: number }; - output_tokens_details: { reasoning_tokens: number }; - }; -}; +export type ResponsesApiResponse = OpenAIResponse; export function responsesApiRequestToChatCompletion( requestBody: string, @@ -276,7 +245,7 @@ function convertResponsesToolChoice(toolChoice: unknown): unknown { export function chatCompletionResponseToResponsesApiMessage( response: ChatCompletion, ): ResponsesApiResponse { - const output: JsonObject[] = []; + const output: ResponsesApiResponse["output"] = []; const outputText: string[] = []; for (const choice of response.choices) { @@ -457,112 +426,3 @@ export function chatCompletionResponseToResponsesApiSseChunks( chunks.push(event("response.completed", { response: fullResponse })); return chunks; } - -export function responsesApiResponseToChatCompletion( - requestBody: string, - responseBody: string, -): ChatCompletion { - const request = JSON.parse(requestBody) as ResponsesRequest; - const response = JSON.parse(responseBody) as ResponsesApiResponse; - const text: string[] = []; - const toolCalls: CanonicalToolCall[] = []; - - for (const item of response.output) { - if (item.type === "message" && Array.isArray(item.content)) { - for (const part of item.content) { - if ( - isObject(part) && - part.type === "output_text" && - typeof part.text === "string" - ) { - text.push(part.text); - } - } - } else if ( - item.type === "function_call" && - typeof item.call_id === "string" - ) { - toolCalls.push({ - id: item.call_id, - type: "function", - function: { - name: typeof item.name === "string" ? item.name : "", - arguments: typeof item.arguments === "string" ? item.arguments : "{}", - }, - }); - } - } - - const incompleteReason = response.incomplete_details?.reason; - return { - id: response.id, - object: "chat.completion", - created: response.created_at, - model: request.model, - choices: [ - { - index: 0, - message: { - role: "assistant", - content: text.length ? text.join("") : null, - refusal: null, - ...(toolCalls.length ? { tool_calls: toolCalls } : {}), - }, - logprobs: null, - finish_reason: - incompleteReason === "max_output_tokens" - ? "length" - : incompleteReason === "content_filter" - ? "content_filter" - : toolCalls.length - ? "tool_calls" - : "stop", - }, - ], - usage: { - prompt_tokens: response.usage?.input_tokens ?? 0, - completion_tokens: response.usage?.output_tokens ?? 0, - total_tokens: response.usage?.total_tokens ?? 0, - }, - } as ChatCompletion; -} - -export function aggregateResponsesApiSseToResponse( - body: string, -): ResponsesApiResponse | null { - let snapshot: ResponsesApiResponse | null = null; - const output: JsonObject[] = []; - for (const event of parseSseEvents(body)) { - if (event.type === "response.completed" && isObject(event.response)) { - return event.response as ResponsesApiResponse; - } - if (event.type === "response.created" && isObject(event.response)) { - snapshot = event.response as ResponsesApiResponse; - } - if (event.type === "response.output_item.done" && isObject(event.item)) { - output.push(event.item); - } - } - return snapshot ? { ...snapshot, output } : null; -} - -function functionToolCalls(message: unknown): CanonicalToolCall[] { - if (!isObject(message) || !Array.isArray(message.tool_calls)) return []; - return message.tool_calls.filter( - (toolCall): toolCall is CanonicalToolCall => - isObject(toolCall) && - typeof toolCall.id === "string" && - toolCall.type === "function" && - isObject(toolCall.function) && - typeof toolCall.function.name === "string" && - typeof toolCall.function.arguments === "string", - ); -} - -function formatSseEvent(type: string, data: unknown): string { - return `event: ${type}\ndata: ${JSON.stringify(data)}\n\n`; -} - -function isObject(value: unknown): value is JsonObject { - return value !== null && typeof value === "object" && !Array.isArray(value); -} diff --git a/test/harness/sseParser.ts b/test/harness/sseParser.ts deleted file mode 100644 index 4f3a8f3455..0000000000 --- a/test/harness/sseParser.ts +++ /dev/null @@ -1,33 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - *--------------------------------------------------------------------------------------------*/ - -/** - * Parses JSON Server-Sent Events. Malformed events and `[DONE]` sentinels are - * ignored so recorded provider streams can be normalized best-effort. - */ -export function parseSseEvents( - body: string, -): Array<{ type: string } & Record> { - const events: Array<{ type: string } & Record> = []; - for (const block of body.split(/\r?\n\r?\n/)) { - if (!block.trim()) continue; - - const dataLines = block - .split(/\r?\n/) - .filter((line) => line.startsWith("data:")) - .map((line) => line.slice(5).replace(/^ /, "")); - const payload = dataLines.join("\n"); - if (!payload || payload === "[DONE]") continue; - - try { - const parsed = JSON.parse(payload) as Record; - if (typeof parsed.type === "string") { - events.push(parsed as { type: string } & Record); - } - } catch { - // Ignore malformed events in a partially captured stream. - } - } - return events; -} From 0846618982f2b54d4e3ae3c127f5b5c7180ff275 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Fri, 17 Jul 2026 00:41:10 +0200 Subject: [PATCH 09/10] Unify replay protocol paths Select a protocol descriptor once per backend so CAPI and BYOK share routing, canonical matching, errors, JSON/SSE responses, and exchange inspection. Replay compaction directly from canonical snapshots instead of synthesizing provider-specific responses. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 85a40918-bd95-440b-b6e9-97c757a71f8c --- test/harness/modelProtocolAdapters.test.ts | 41 +- test/harness/replayingCapiProxy.ts | 457 ++++++--------------- 2 files changed, 164 insertions(+), 334 deletions(-) diff --git a/test/harness/modelProtocolAdapters.test.ts b/test/harness/modelProtocolAdapters.test.ts index 1b7466db9e..fc64b5cdc2 100644 --- a/test/harness/modelProtocolAdapters.test.ts +++ b/test/harness/modelProtocolAdapters.test.ts @@ -26,19 +26,22 @@ import { type ByokBackend = Exclude; -const backends: ByokBackend[] = [ +const backends: ReplayBackend[] = [ + "capi", "anthropic-messages", "openai-responses", "openai-completions", ]; -const endpoints: Record = { +const endpoints: Record = { + capi: "/chat/completions", "anthropic-messages": "/v1/messages", "openai-responses": "/responses", "openai-completions": "/chat/completions", }; -const models: Record = { +const models: Record = { + capi: "gpt-4.1", "anthropic-messages": "claude-sonnet-4.5", "openai-responses": "gpt-4.1", "openai-completions": "gpt-4.1", @@ -76,7 +79,7 @@ const completionWithTool: ChatCompletion = { }; function requestFor( - backend: ByokBackend, + backend: ReplayBackend, prompt: string, ): Record { const model = models[backend]; @@ -100,6 +103,7 @@ function requestFor( }, ], }; + case "capi": case "openai-completions": return { model, @@ -372,7 +376,7 @@ describe("protocol-aware replay", () => { } async function withProxy( - backend: ByokBackend, + backend: ReplayBackend, action: (proxyUrl: string) => Promise, ): Promise { const proxy = new ReplayingCapiProxy( @@ -432,7 +436,10 @@ describe("protocol-aware replay", () => { role: "user", content: "Hello", }); - if (backend !== "openai-completions") { + if ( + backend === "anthropic-messages" || + backend === "openai-responses" + ) { expect(exchanges[0].response).toBeUndefined(); } }); @@ -526,8 +533,17 @@ describe("protocol-aware replay", () => { }); test.each(backends)( - "synthesizes compaction responses through %s", + "replays compaction responses through %s", async (backend) => { + await writeSnapshot([ + { role: "system", content: "${system}" }, + { role: "user", content: "${compaction_prompt}" }, + { + role: "assistant", + content: + "CompactedHistoryCheckpoint", + }, + ]); await withProxy(backend, async (proxyUrl) => { const response = await postJson( proxyUrl, @@ -554,4 +570,15 @@ describe("protocol-aware replay", () => { await expect(response.text()).resolves.toContain("protocol_mismatch"); }); }); + + test("keeps foreign model endpoints unavailable in CAPI mode", async () => { + await withProxy("capi", async (proxyUrl) => { + const response = await postJson( + proxyUrl, + endpoints["openai-responses"], + requestFor("openai-responses", "Hello"), + ); + expect(response.status).toBe(404); + }); + }); }); diff --git a/test/harness/replayingCapiProxy.ts b/test/harness/replayingCapiProxy.ts index cee6d65c6a..266cfa503f 100644 --- a/test/harness/replayingCapiProxy.ts +++ b/test/harness/replayingCapiProxy.ts @@ -2,7 +2,7 @@ * Copyright (c) Microsoft Corporation. All rights reserved. *--------------------------------------------------------------------------------------------*/ -import { existsSync } from "fs"; +import { appendFileSync, existsSync } from "fs"; import { mkdir, readFile, writeFile } from "fs/promises"; import type { ChatCompletion, @@ -41,10 +41,57 @@ export type ReplayBackend = | "openai-responses" | "openai-completions"; -type ModelProtocol = - | "anthropic-messages" - | "openai-responses" - | "openai-completions"; +type ReplayProtocol = { + endpoint: string; + normalizeRequest?: (body: string) => string; + responseBody?: (response: ChatCompletion) => unknown; + responseChunks: (response: ChatCompletion) => string[]; + responseEndChunk?: string; + errorBody?: (code: string | undefined, message: string) => unknown; + canonicalResponse?: boolean; +}; + +const chatCompletionsProtocol = { + endpoint: chatCompletionEndpoint, + responseChunks: (response) => + convertToStreamingResponseChunks(response).map( + (chunk) => `data: ${JSON.stringify(chunk)}\n\n`, + ), + responseEndChunk: "data: [DONE]\n\n", + canonicalResponse: true, +} satisfies ReplayProtocol; + +const replayProtocols: Record = { + capi: chatCompletionsProtocol, + "openai-completions": { + ...chatCompletionsProtocol, + normalizeRequest: coalesceAdjacentUserMessages, + }, + "anthropic-messages": { + endpoint: anthropicMessagesEndpoint, + normalizeRequest: (body) => + coalesceAdjacentUserMessages( + anthropicMessagesRequestToChatCompletion(body), + ), + responseBody: chatCompletionResponseToAnthropicMessage, + responseChunks: chatCompletionResponseToAnthropicSseChunks, + errorBody: (code, message) => { + const type = code ?? "rate_limited"; + return { type: "error", error: { type, message } }; + }, + }, + "openai-responses": { + endpoint: responsesEndpoint, + normalizeRequest: (body) => + coalesceAdjacentUserMessages(responsesApiRequestToChatCompletion(body)), + responseBody: chatCompletionResponseToResponsesApiMessage, + responseChunks: chatCompletionResponseToResponsesApiSseChunks, + }, +}; + +const modelEndpoints = new Set( + Object.values(replayProtocols).map((protocol) => protocol.endpoint), +); const shellConfig = process.platform === "win32" ? ShellConfig.powerShell : ShellConfig.bash; @@ -256,37 +303,21 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { options.requestOptions.path === "/exchanges" && options.requestOptions.method === "GET" ) { - const modelExchanges = this.exchanges.flatMap((exchange) => { - const protocol = protocolForEndpoint(exchange.request.url); - if (!protocol) return []; - - try { - return [ - { - requestBody: normalizeModelRequest( - protocol, - exchange.request.body, - this.state?.backend, - ), - responseBody: - protocol === "openai-completions" - ? exchange.response?.body - : undefined, - requestHeaders: exchange.request.headers, - }, - ]; - } catch { - return []; - } - }); + const protocol = + replayProtocols[this.state?.backend ?? "capi"]; const parsedExchanges = await Promise.all( - modelExchanges.map((exchange) => - parseHttpExchange( - exchange.requestBody, - exchange.responseBody, - exchange.requestHeaders, + this.exchanges + .filter((exchange) => exchange.request.url === protocol.endpoint) + .map((exchange) => + parseHttpExchange( + protocol.normalizeRequest?.(exchange.request.body) ?? + exchange.request.body, + protocol.canonicalResponse + ? exchange.response?.body + : undefined, + exchange.request.headers, + ), ), - ), ); options.onResponseStart(200, {}); options.onData(Buffer.from(JSON.stringify(parsedExchanges))); @@ -388,16 +419,14 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { options.onResponseEnd(); return; } - const requestPath = options.requestOptions.path ?? ""; - const protocol = protocolForEndpoint(requestPath); - const expectedEndpoint = endpointForBackend(state.backend); + const protocol = replayProtocols[state.backend]; if ( - protocol && + modelEndpoints.has(requestPath) && state.backend !== "capi" && - requestPath !== expectedEndpoint + requestPath !== protocol.endpoint ) { - const message = `Expected ${expectedEndpoint} for backend ${state.backend}, received ${requestPath}`; + const message = `Expected ${protocol.endpoint} for backend ${state.backend}, received ${requestPath}`; options.onResponseStart(400, { "content-type": "application/json", ...commonResponseHeaders, @@ -413,53 +442,15 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { return; } - // Handle model inference endpoints. Foreign wire formats are converted - // to the canonical Chat Completions shape used by the snapshot matcher. - if (state.storedData && protocol && options.body) { - const normalizedBody = normalizeModelRequest( - protocol, - options.body, - state.backend, - ); - const compactionResponse = - state.backend === "capi" - ? undefined - : createCompactionResponse(normalizedBody); - if (compactionResponse) { - const streamingIsRequested = - (JSON.parse(options.body) as { stream?: boolean }).stream === - true; - await this.respondWithProtocol( - options, - protocol, - compactionResponse, - streamingIsRequested, - commonResponseHeaders, - ); - return; - } - - const postCompactionResponse = - state.backend === "capi" - ? undefined - : createPostCompactionResponse( - state.storedData, - normalizedBody, - state.workDir, - ); - if (postCompactionResponse) { - const streamingIsRequested = - (JSON.parse(options.body) as { stream?: boolean }).stream === - true; - await this.respondWithProtocol( - options, - protocol, - postCompactionResponse, - streamingIsRequested, - commonResponseHeaders, - ); - return; - } + const isModelRequest = requestPath === protocol.endpoint; + // Every protocol enters the existing Chat Completions snapshot matcher. + const normalizedBody = + isModelRequest && options.body + ? (protocol.normalizeRequest?.(options.body) ?? options.body) + : options.body; + if (state.storedData && isModelRequest && normalizedBody) { + const streamingIsRequested = + (JSON.parse(normalizedBody) as { stream?: boolean }).stream === true; const savedError = await findSavedChatCompletionError( state.storedData, @@ -480,8 +471,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { options.onData( Buffer.from( JSON.stringify( - createProtocolError( - protocol, + (protocol.errorBody ?? openAIErrorBody)( savedError.code, savedError.message ?? "Rate limited by test snapshot", ), @@ -500,10 +490,6 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { ); if (savedResponse) { - const streamingIsRequested = - (JSON.parse(options.body) as { stream?: boolean }).stream === - true; - await this.respondWithProtocol( options, protocol, @@ -525,9 +511,6 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { state.toolResultNormalizers, ) ) { - const streamingIsRequested = - (JSON.parse(options.body) as { stream?: boolean }).stream === - true; const headers = { "content-type": streamingIsRequested ? "text/event-stream" @@ -544,7 +527,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { // Beyond this point, we're only going to be able to supply responses in CI if we have a snapshot, // and we only store snapshots for chat completion. For anything else (e.g., custom-agents fetches), // return 404 so the CLI treats them as unavailable instead of erroring. - if (!protocol) { + if (!isModelRequest) { const headers = { "content-type": "application/json", "x-github-request-id": "proxy-not-found", @@ -567,9 +550,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { state.workDir, state.toolResultNormalizers, state.storedData, - options.body - ? normalizeModelRequest(protocol, options.body, state.backend) - : options.body, + normalizedBody, ); return; } @@ -582,47 +563,37 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { private async respondWithProtocol( options: PerformRequestOptions, - protocol: ModelProtocol, + protocol: ReplayProtocol, response: ChatCompletion, streaming: boolean, commonHeaders: Record, ): Promise { if (!streaming) { - const body = - protocol === "anthropic-messages" - ? chatCompletionResponseToAnthropicMessage(response) - : protocol === "openai-responses" - ? chatCompletionResponseToResponsesApiMessage(response) - : response; options.onResponseStart(200, { "content-type": "application/json", ...commonHeaders, }); - options.onData(Buffer.from(JSON.stringify(body))); + options.onData( + Buffer.from( + JSON.stringify(protocol.responseBody?.(response) ?? response), + ), + ); options.onResponseEnd(); return; } - const chunks = - protocol === "anthropic-messages" - ? chatCompletionResponseToAnthropicSseChunks(response) - : protocol === "openai-responses" - ? chatCompletionResponseToResponsesApiSseChunks(response) - : convertToStreamingResponseChunks(response).map( - (chunk) => `data: ${JSON.stringify(chunk)}\n\n`, - ); options.onResponseStart(200, { "content-type": "text/event-stream", ...commonHeaders, }); - for (const chunk of chunks) { + for (const chunk of protocol.responseChunks(response)) { options.onData(Buffer.from(chunk)); if (this.slowStreaming) { await sleep(100); } } - if (protocol === "openai-completions") { - options.onData(Buffer.from("data: [DONE]\n\n")); + if (protocol.responseEndChunk) { + options.onData(Buffer.from(protocol.responseEndChunk)); } options.onResponseEnd(); } @@ -687,9 +658,18 @@ function diagnoseMatchFailure( for (let c = 0; c < storedData.conversations.length; c++) { const saved = storedData.conversations[c].messages; + // Same check as findAssistantIndexAfterPrefix: request must be a strict prefix + if (requestMessages.length >= saved.length) { + lines.push( + `Conversation ${c} (${saved.length} messages): ` + + `skipped — request has ${requestMessages.length} messages, need fewer than ${saved.length}.`, + ); + continue; + } + // Find the first message that doesn't match let mismatchIndex = -1; - for (let i = 0; i < Math.min(requestMessages.length, saved.length); i++) { + for (let i = 0; i < requestMessages.length; i++) { if (JSON.stringify(requestMessages[i]) !== JSON.stringify(saved[i])) { mismatchIndex = i; break; @@ -703,20 +683,10 @@ function diagnoseMatchFailure( : "(no raw message)"; lines.push( `Conversation ${c} (${saved.length} messages): mismatch at message ${mismatchIndex}:`, - ` request: ${JSON.stringify(requestMessages[mismatchIndex]).slice(0, 1000)}`, - ` saved: ${JSON.stringify(saved[mismatchIndex]).slice(0, 1000)}`, + ` request: ${JSON.stringify(requestMessages[mismatchIndex]).slice(0, 200)}`, + ` saved: ${JSON.stringify(saved[mismatchIndex]).slice(0, 200)}`, ` raw (pre-normalization): ${raw}`, ); - } else if (requestMessages.length >= saved.length) { - const extraMessage = requestMessages[saved.length]; - lines.push( - `Conversation ${c} (${saved.length} messages): matched the stored conversation, but the request needs a later response.`, - ...(extraMessage - ? [ - ` first extra request message: ${JSON.stringify(extraMessage).slice(0, 200)}`, - ] - : []), - ); } else { // Prefix matched, but the next saved message isn't an assistant turn const nextRole = @@ -798,14 +768,11 @@ async function findSavedChatCompletionResponse( toolResultNormalizers, ); const requestMessages = normalized.conversations[0]?.messages ?? []; - const responseModel = normalized.models[0]; - if (!responseModel) { + const requestModel = normalized.models[0]; + if (!requestModel) { throw new Error("Unable to determine model from request"); } - // Snapshot model IDs are capture metadata, not match criteria. Render the - // canonical response using the active request's model so one transcript can - // replay against different provider/model profiles. // Now find a matching cached conversation (i.e., one for which this request is a prefix) for (const conversation of storedData.conversations) { const replyIndex = findAssistantIndexAfterPrefix( @@ -814,7 +781,7 @@ async function findSavedChatCompletionResponse( ); if (replyIndex !== undefined) { return createOpenAIResponse( - responseModel, + requestModel, conversation.messages, replyIndex, workDir, @@ -929,69 +896,13 @@ async function transformHttpExchanges( function parseReplayBackend(value: unknown): ReplayBackend { if (value === undefined || value === null || value === "") return "capi"; - if ( - value === "capi" || - value === "anthropic-messages" || - value === "openai-responses" || - value === "openai-completions" - ) { - return value; + if (typeof value === "string" && Object.hasOwn(replayProtocols, value)) { + return value as ReplayBackend; } throw new Error(`Unsupported replay backend: ${String(value)}`); } -function endpointForBackend(backend: ReplayBackend): string { - switch (backend) { - case "anthropic-messages": - return anthropicMessagesEndpoint; - case "openai-responses": - return responsesEndpoint; - case "capi": - case "openai-completions": - return chatCompletionEndpoint; - } -} - -function protocolForEndpoint(path: string): ModelProtocol | undefined { - switch (path) { - case anthropicMessagesEndpoint: - return "anthropic-messages"; - case responsesEndpoint: - return "openai-responses"; - case chatCompletionEndpoint: - return "openai-completions"; - default: - return undefined; - } -} - -function normalizeModelRequest( - protocol: ModelProtocol, - requestBody: string, - backend?: ReplayBackend, -): string { - switch (protocol) { - case "anthropic-messages": - return coalesceAdjacentUserMessages( - anthropicMessagesRequestToChatCompletion(requestBody), - true, - ); - case "openai-responses": - return coalesceAdjacentUserMessages( - responsesApiRequestToChatCompletion(requestBody), - true, - ); - case "openai-completions": - return backend === "openai-completions" - ? coalesceAdjacentUserMessages(requestBody, true) - : requestBody; - } -} - -function coalesceAdjacentUserMessages( - requestBody: string, - normalizeByokSpacing: boolean, -): string { +function coalesceAdjacentUserMessages(requestBody: string): string { const request = JSON.parse(requestBody) as { messages?: Array<{ role?: string; @@ -1016,14 +927,12 @@ function coalesceAdjacentUserMessages( } } - if (normalizeByokSpacing) { - for (const message of messages) { - if (message.role === "user" && typeof message.content === "string") { - message.content = normalizeUserMessage(message.content).replace( - /\n{5,}/g, - "\n\n\n", - ); - } + for (const message of messages) { + if (message.role === "user" && typeof message.content === "string") { + message.content = normalizeUserMessage(message.content).replace( + /\n{5,}/g, + "\n\n\n", + ); } } @@ -1031,121 +940,12 @@ function coalesceAdjacentUserMessages( return JSON.stringify(request); } -function createCompactionResponse( - requestBody: string, -): ChatCompletion | undefined { - const request = JSON.parse(requestBody) as { - model?: string; - messages?: Array<{ role?: string; content?: unknown }>; - }; - const messages = (request.messages ?? []).map((message) => ({ - ...message, - content: - message.role === "user" && typeof message.content === "string" - ? normalizeUserMessage(message.content) - : message.content, - })); - if ( - !messages.some( - (message) => - message.role === "user" && - typeof message.content === "string" && - message.content.includes("${compaction_prompt}"), - ) - ) { - return undefined; - } - - const history = messages - .filter((message) => message.role !== "system") - .map((message) => { - const content = - typeof message.content === "string" - ? message.content.replace("${compaction_prompt}", "").trim() - : ""; - return `${message.role ?? "unknown"}: ${content}`; - }) - .filter((line) => !line.endsWith(": ")) - .join("\n\n"); - const content = [ - "Conversation compacted by the E2E replay proxy.", - `${history}`, - "Compacted E2E conversation", - ].join("\n"); - - return { - id: "compaction-e2e-response", - object: "chat.completion", - created: 0, - model: request.model ?? "test-model", - choices: [ - { - index: 0, - message: { - role: "assistant", - content, - refusal: null, - }, - finish_reason: "stop", - logprobs: null, - }, - ], - usage: { - prompt_tokens: 100, - completion_tokens: 50, - total_tokens: 150, - }, - }; -} - -function createPostCompactionResponse( - storedData: NormalizedData, - requestBody: string, - workDir: string, -): ChatCompletion | undefined { - const request = JSON.parse(requestBody) as { - model?: string; - messages?: Array<{ role?: string; content?: unknown }>; - }; - const hasCompactedHistory = (request.messages ?? []).some( - (message) => - message.role === "user" && - typeof message.content === "string" && - message.content.startsWith( - "Some of the conversation history has been summarized to free up context.", - ), - ); - if (!hasCompactedHistory) return undefined; - - for (const conversation of storedData.conversations) { - let replyIndex = conversation.messages.length - 1; - while ( - replyIndex >= 0 && - conversation.messages[replyIndex].role !== "assistant" - ) { - replyIndex--; - } - if (replyIndex >= 0) { - return createOpenAIResponse( - request.model ?? storedData.models[0] ?? "test-model", - conversation.messages, - replyIndex, - workDir, - ); - } - } - return undefined; -} - -function createProtocolError( - protocol: ModelProtocol, +function openAIErrorBody( code: string | undefined, message: string, ): unknown { const type = code ?? "rate_limited"; - return protocol === "anthropic-messages" - ? { type: "error", error: { type, message } } - : { error: { message, type, code: type } }; + return { error: { message, type, code: type } }; } function normalizeFilenames( @@ -1422,10 +1222,7 @@ function transformOpenAIRequestMessage( function normalizeUserMessage(content: string): string { return normalizeSkillContextFrontmatter(content) - .replace( - taskCompletionNotificationPattern, - taskCompletionNotificationReplacement, - ) + .replace(taskCompletionNotificationPattern, taskCompletionNotificationReplacement) .replace(/.*?<\/current_datetime>/g, "") .replace(/[\s\S]*?<\/reminder>/g, "") .replace(/[\s\S]*?<\/system_reminder>/g, "") @@ -1466,7 +1263,6 @@ function normalizeStoredMessagesForBackend( conversation.messages = coalesceMessages( conversation.messages, backend !== "openai-completions", - true, ); } } @@ -1474,7 +1270,6 @@ function normalizeStoredMessagesForBackend( function coalesceMessages( messages: NormalizedMessage[], coalesceAssistantMessages: boolean, - coalesceUserMessages: boolean, ): NormalizedMessage[] { const result: NormalizedMessage[] = []; for (const message of messages) { @@ -1482,7 +1277,7 @@ function coalesceMessages( const shouldCoalesce = previous?.role === message.role && ((coalesceAssistantMessages && message.role === "assistant") || - (coalesceUserMessages && message.role === "user")); + message.role === "user"); if (!shouldCoalesce) { result.push(message); continue; @@ -1718,14 +1513,21 @@ function findAssistantIndexAfterPrefix( requestMessages: NormalizedMessage[], savedMessages: NormalizedMessage[], ): number | undefined { + const logFile = process.env.PROXY_DEBUG_LOG; + const log = (msg: string) => { if (logFile) try { appendFileSync(logFile, msg + "\n"); } catch {} }; + if (requestMessages.length >= savedMessages.length) { + log(`prefix check failed: request.length=${requestMessages.length} >= saved.length=${savedMessages.length}`); return undefined; } for (let i = 0; i < requestMessages.length; i++) { - if ( - JSON.stringify(requestMessages[i]) !== JSON.stringify(savedMessages[i]) - ) { + const reqMsg = JSON.stringify(requestMessages[i]); + const savedMsg = JSON.stringify(savedMessages[i]); + if (reqMsg !== savedMsg) { + log(`mismatch at index ${i}:`); + log(` REQ: ${reqMsg.substring(0, 1000)}`); + log(` SAVED: ${savedMsg.substring(0, 1000)}`); return undefined; } } @@ -1736,9 +1538,11 @@ function findAssistantIndexAfterPrefix( nextIndex < savedMessages.length && savedMessages[nextIndex].role === "assistant" ) { + log(`MATCH found at index ${nextIndex}`); return nextIndex; } + log(`no assistant at nextIndex=${nextIndex}, saved.length=${savedMessages.length}`); return undefined; } @@ -1997,7 +1801,6 @@ interface NormalizedErrorResponse { } export interface NormalizedData { - /** Captured model IDs used to replay /models; conversations are model-agnostic. */ models: string[]; errors?: NormalizedErrorResponse[]; conversations: NormalizedConversation[]; From 707a59d082a85144c76643314b50b08d0c8a40d2 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Fri, 17 Jul 2026 09:59:42 +0200 Subject: [PATCH 10/10] Fix Responses replay stream lifecycle Keep initial Responses events in progress and empty until their matching delta and done events, and use the isolated E2E context when injecting a BYOK provider. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 85a40918-bd95-440b-b6e9-97c757a71f8c --- dotnet/test/E2E/RpcSessionStateE2ETests.cs | 2 +- test/harness/modelProtocolAdapters.test.ts | 73 +++++++++++++++++++--- test/harness/responsesApiAdapter.ts | 14 ++++- 3 files changed, 78 insertions(+), 11 deletions(-) diff --git a/dotnet/test/E2E/RpcSessionStateE2ETests.cs b/dotnet/test/E2E/RpcSessionStateE2ETests.cs index ab5808302d..dfd76fb34f 100644 --- a/dotnet/test/E2E/RpcSessionStateE2ETests.cs +++ b/dotnet/test/E2E/RpcSessionStateE2ETests.cs @@ -46,7 +46,7 @@ public async Task Should_Call_Session_Rpc_Model_SwitchTo() await isolatedCtx.ConfigureForTestAsync("rpc_session_state", nameof(Should_Call_Session_Rpc_Model_SwitchTo)); var isolatedClient = isolatedCtx.CreateClient(); - await using var session = await Ctx.CreateSessionAsync(isolatedClient, new SessionConfig + await using var session = await isolatedCtx.CreateSessionAsync(isolatedClient, new SessionConfig { Model = "claude-sonnet-4.5", OnPermissionRequest = PermissionHandler.ApproveAll, diff --git a/test/harness/modelProtocolAdapters.test.ts b/test/harness/modelProtocolAdapters.test.ts index fc64b5cdc2..ddb6fe40bf 100644 --- a/test/harness/modelProtocolAdapters.test.ts +++ b/test/harness/modelProtocolAdapters.test.ts @@ -346,15 +346,72 @@ describe("OpenAI Responses adapter", () => { "function_call", ]); - const stream = - chatCompletionResponseToResponsesApiSseChunks(completionWithTool).join( - "", - ); + const chunks = + chatCompletionResponseToResponsesApiSseChunks(completionWithTool); + const events = chunks.map( + (chunk) => + JSON.parse(chunk.split("\ndata: ")[1]) as Record, + ); + const stream = chunks.join(""); expect(stream).toContain("event: response.created"); - expect(stream).toContain("event: response.in_progress"); - expect(stream).toContain("event: response.output_text.delta"); - expect(stream).toContain('"sequence_number":0'); - expect(stream).toContain("event: response.completed"); + expect(stream).toContain("event: response.in_progress"); + expect(stream).toContain("event: response.output_text.delta"); + expect(stream).toContain('"sequence_number":0'); + expect(stream).toContain("event: response.completed"); + + expect(events[0]).toMatchObject({ + type: "response.created", + response: { status: "in_progress", output: [] }, + }); + expect(events[1]).toMatchObject({ + type: "response.in_progress", + response: { status: "in_progress", output: [] }, + }); + + const addedItems = events.filter( + (event) => event.type === "response.output_item.added", + ); + expect(addedItems).toMatchObject([ + { + item: { + type: "message", + status: "in_progress", + content: [], + }, + }, + { + item: { + type: "function_call", + status: "in_progress", + arguments: "", + }, + }, + ]); + expect( + events.find((event) => event.type === "response.content_part.added"), + ).toMatchObject({ + part: { type: "output_text", text: "" }, + }); + + const completedItems = events.filter( + (event) => event.type === "response.output_item.done", + ); + expect(completedItems).toMatchObject([ + { + item: { + type: "message", + status: "completed", + content: [{ type: "output_text", text: "Calling a tool" }], + }, + }, + { + item: { + type: "function_call", + status: "completed", + arguments: '{"value":42}', + }, + }, + ]); }); }); diff --git a/test/harness/responsesApiAdapter.ts b/test/harness/responsesApiAdapter.ts index 70dc2b9af3..16568f6e03 100644 --- a/test/harness/responsesApiAdapter.ts +++ b/test/harness/responsesApiAdapter.ts @@ -322,6 +322,7 @@ export function chatCompletionResponseToResponsesApiSseChunks( const fullResponse = chatCompletionResponseToResponsesApiMessage(response); const skeleton = { ...fullResponse, + status: "in_progress" as const, output: [], output_text: "", usage: undefined, @@ -346,10 +347,16 @@ export function chatCompletionResponseToResponsesApiSseChunks( outputIndex++ ) { const item = fullResponse.output[outputIndex]; + const addedItem = + item.type === "message" + ? { ...item, status: "in_progress" as const, content: [] } + : item.type === "function_call" + ? { ...item, status: "in_progress" as const, arguments: "" } + : item; chunks.push( event("response.output_item.added", { output_index: outputIndex, - item, + item: addedItem, }), ); @@ -365,7 +372,10 @@ export function chatCompletionResponseToResponsesApiSseChunks( item_id: item.id, output_index: outputIndex, content_index: contentIndex, - part, + part: + isObject(part) && part.type === "output_text" + ? { ...part, text: "" } + : part, }), ); if (