From 7e2d64db5b9676c8335019c7c0f2923fb7f04e3d Mon Sep 17 00:00:00 2001 From: Hoang Snowy Date: Thu, 22 Oct 2026 23:13:18 +0700 Subject: [PATCH 1/2] =?UTF-8?q?feat(insights):=20every=20Board=20agent=20r?= =?UTF-8?q?un=20counts=20=E2=80=94=20Coding,=20Testing,=20Discovery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Insights showed nothing while Autopilot's agents worked all day: it reads pipeline runs + run_metrics, and only the 5-agent pipeline wrote them. Now every LLM call made inside an LlmCallScope is recorded (ObservedLlmClient, wrapped by the factory only while a scope is open — tokens incl. cache read/write and reasoning, cost, latency, provider, outcome), and each Board agent step is stored as a run with those calls (IPipelineRunRepository.SaveAgentRunAsync; no pipeline artifacts, so its run detail reads as not found). Coding and Testing runs on the Board and New idea (Discovery) record themselves; the Thorough engine keeps its own pipeline run. Calls made on a member's computer cost $0 and show their ≈ API figure, as Insights already labels them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018WAdQ7Rm24xss5QDSvXtJB --- src/AgentOs.Domain/Llm/LlmCallScope.cs | 70 +++++++++++ src/AgentOs.Modules.Llm/LlmClientFactory.cs | 8 +- src/AgentOs.Modules.Llm/ObservedLlmClient.cs | 36 ++++++ .../Persistence/IPipelineRunRepository.cs | 16 +++ .../Repositories/PipelineRunRepository.cs | 46 ++++++++ .../Components/Pages/BoardApp.razor | 13 ++- .../Services/BoardTasks/BoardTaskService.cs | 51 +++++++- .../Metrics/AgentRunMetricsTests.cs | 110 ++++++++++++++++++ 8 files changed, 345 insertions(+), 5 deletions(-) create mode 100644 src/AgentOs.Domain/Llm/LlmCallScope.cs create mode 100644 src/AgentOs.Modules.Llm/ObservedLlmClient.cs create mode 100644 tests/AgentOs.Tests/Metrics/AgentRunMetricsTests.cs diff --git a/src/AgentOs.Domain/Llm/LlmCallScope.cs b/src/AgentOs.Domain/Llm/LlmCallScope.cs new file mode 100644 index 00000000..a38e631e --- /dev/null +++ b/src/AgentOs.Domain/Llm/LlmCallScope.cs @@ -0,0 +1,70 @@ +// Metrics first: every LLM call an agent makes inside a scope is recorded — tokens (fresh / cache read / cache write / +// output / reasoning), cost, latency, provider, outcome. The Board's agents (Coding, Reviewer, Testing, Discovery, the +// Autopilot Orchestrator) run outside the 5-agent pipeline, so their calls never reached Insights; the caller opens a +// scope around one agent run and stores what it collected as one run with its calls. + +using System; +using System.Collections.Generic; +using System.Threading; + +namespace AgentOs.Domain.Llm; + +/// One LLM call as recorded for metrics. +public sealed record LlmCallRecord( + string? AgentRole, + string Model, + string Provider, + int InputTokens, + int OutputTokens, + int CacheReadTokens, + int CacheWriteTokens, + int ReasoningTokens, + decimal CostUsd, + TimeSpan Latency, + bool Success, + string? Error, + DateTimeOffset At); + +/// Collects the LLM calls made in the current async flow while it is open (innermost scope wins). +public sealed class LlmCallScope : IDisposable +{ + private static readonly AsyncLocal CurrentScope = new(); + + private readonly LlmCallScope? _previous; + private readonly List _calls = []; + private bool _disposed; + + private LlmCallScope(LlmCallScope? previous) => _previous = previous; + + /// The open scope of this flow, or null. + public static LlmCallScope? Current => CurrentScope.Value; + + /// Open a scope for this async flow; dispose to close it. + public static LlmCallScope Begin() + { + var scope = new LlmCallScope(CurrentScope.Value); + CurrentScope.Value = scope; + return scope; + } + + /// The calls recorded so far. + public IReadOnlyList Calls + { + get { lock (_calls) { return [.. _calls]; } } + } + + /// Record one call. + public void Add(LlmCallRecord call) + { + ArgumentNullException.ThrowIfNull(call); + lock (_calls) { _calls.Add(call); } + } + + /// + public void Dispose() + { + if (_disposed) { return; } + _disposed = true; + if (ReferenceEquals(CurrentScope.Value, this)) { CurrentScope.Value = _previous; } + } +} diff --git a/src/AgentOs.Modules.Llm/LlmClientFactory.cs b/src/AgentOs.Modules.Llm/LlmClientFactory.cs index cc2a258b..f4a6b5ac 100644 --- a/src/AgentOs.Modules.Llm/LlmClientFactory.cs +++ b/src/AgentOs.Modules.Llm/LlmClientFactory.cs @@ -77,14 +77,18 @@ public ILlmClient Create(string providerName) // No fallbacks declared → return the bare client, identical to the pre-failover behavior. if (fallbacks.Count == 0) { - return primary; + return Observed(primary); } var chain = new List(1 + fallbacks.Count) { primary }; chain.AddRange(fallbacks); - return new FailoverLlmClient(chain, _services.GetRequiredService>()); + return Observed(new FailoverLlmClient(chain, _services.GetRequiredService>())); } + // Metrics first: inside an open LlmCallScope (a Board agent run) every call is recorded for Insights. + private static ILlmClient Observed(ILlmClient client) => + AgentOs.Domain.Llm.LlmCallScope.Current is { } scope ? new ObservedLlmClient(client, scope) : client; + // The primary must resolve — a missing primary is a hard misconfiguration. private ILlmClient ResolveRequired(string effective, string requested) { diff --git a/src/AgentOs.Modules.Llm/ObservedLlmClient.cs b/src/AgentOs.Modules.Llm/ObservedLlmClient.cs new file mode 100644 index 00000000..fe2e514e --- /dev/null +++ b/src/AgentOs.Modules.Llm/ObservedLlmClient.cs @@ -0,0 +1,36 @@ +// Reports each call to the open LlmCallScope (see AgentOs.Domain.Llm.LlmCallScope). The factory only wraps a client +// when a scope is open, so the pipeline's own metrics path and every unscoped call are untouched. + +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using AgentOs.Domain.Llm; + +namespace AgentOs.Modules.Llm; + +/// An that records its calls into the scope that was open when it was created. +internal sealed class ObservedLlmClient(ILlmClient inner, LlmCallScope scope) : ILlmClient +{ + public string Provider => inner.Provider; + + public async Task SendAsync(LlmRequest request, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + var watch = Stopwatch.StartNew(); + try + { + var r = await inner.SendAsync(request, cancellationToken).ConfigureAwait(false); + scope.Add(new LlmCallRecord(request.AgentRole, r.Model, r.Provider, r.InputTokens, r.OutputTokens, + r.CacheReadTokens, r.CacheWriteTokens, r.ReasoningTokens, r.CostUsd, + r.Latency > TimeSpan.Zero ? r.Latency : watch.Elapsed, true, null, DateTimeOffset.UtcNow)); + return r; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + scope.Add(new LlmCallRecord(request.AgentRole, request.Model, inner.Provider, 0, 0, 0, 0, 0, 0m, watch.Elapsed, + false, ex.Message.Length > 500 ? ex.Message[..500] : ex.Message, DateTimeOffset.UtcNow)); + throw; + } + } +} diff --git a/src/AgentOs.Modules.Pipeline/Persistence/IPipelineRunRepository.cs b/src/AgentOs.Modules.Pipeline/Persistence/IPipelineRunRepository.cs index 80d366e4..e81b85c5 100644 --- a/src/AgentOs.Modules.Pipeline/Persistence/IPipelineRunRepository.cs +++ b/src/AgentOs.Modules.Pipeline/Persistence/IPipelineRunRepository.cs @@ -19,6 +19,10 @@ public interface IPipelineRunRepository /// tenant's workflow spend. Task SaveAsync(PipelineRunRecord record, string tenantId, CancellationToken ct = default); + /// Stores one Board agent run (Coding, Reviewer, Testing, Discovery, Autopilot Orchestrator) with the + /// LLM calls it made, so Insights counts it. It has no pipeline artifacts: its run detail reads as not found. + Task SaveAgentRunAsync(AgentRunRecord record, string tenantId, CancellationToken ct = default) => Task.CompletedTask; + /// Gets a single full run by Id (null if not found). Task GetAsync(Guid id, CancellationToken ct = default); @@ -64,6 +68,18 @@ public sealed record PipelineRunRecord( DateTimeOffset CreatedAtUtc, DateTimeOffset CompletedAtUtc); +/// One Board agent run and its LLM calls (see ). +/// What ran, shown in the run history (e.g. "Coding · #15 Build SVG game viewer"). +/// The agent name its calls are filed under when a call carries no role. +public sealed record AgentRunRecord( + Guid Id, + string Label, + string Agent, + bool Success, + DateTimeOffset StartedAtUtc, + DateTimeOffset CompletedAtUtc, + IReadOnlyList Calls); + /// Summary of a single run for the history list. public sealed record PipelineRunSummary( Guid Id, diff --git a/src/AgentOs.Modules.Pipeline/Persistence/Repositories/PipelineRunRepository.cs b/src/AgentOs.Modules.Pipeline/Persistence/Repositories/PipelineRunRepository.cs index aef624a0..7b843e61 100644 --- a/src/AgentOs.Modules.Pipeline/Persistence/Repositories/PipelineRunRepository.cs +++ b/src/AgentOs.Modules.Pipeline/Persistence/Repositories/PipelineRunRepository.cs @@ -77,6 +77,52 @@ public async Task SaveAsync(PipelineRunRecord record, string tenantId, Cancellat await db.SaveChangesAsync(ct); } + public async Task SaveAgentRunAsync(AgentRunRecord record, string tenantId, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(record); + ArgumentException.ThrowIfNullOrWhiteSpace(tenantId); + var entity = new PipelineRunEntity + { + Id = record.Id, + TenantId = tenantId, + UserStoryText = record.Label, + Status = record.Success ? "Completed" : "Failed", + TotalCostUsd = record.Calls.Sum(c => c.CostUsd), + TotalTokensIn = record.Calls.Sum(c => c.InputTokens), + TotalTokensOut = record.Calls.Sum(c => c.OutputTokens), + IterationCount = 0, + CreatedAtUtc = record.StartedAtUtc, + CompletedAtUtc = record.CompletedAtUtc, + // No pipeline artifacts: the run detail deserializes this to null and reads as not found. + ResultJson = "null", + }; + foreach (var c in record.Calls) + { + entity.Metrics.Add(new RunMetricEntity + { + RunId = record.Id, + TenantId = tenantId, + KcId = "board", + Iteration = 0, + AgentName = string.IsNullOrWhiteSpace(c.AgentRole) ? record.Agent : c.AgentRole, + Model = c.Model, + Provider = c.Provider, + TokensIn = c.InputTokens, + TokensOut = c.OutputTokens, + CacheReadTokens = c.CacheReadTokens, + CacheWriteTokens = c.CacheWriteTokens, + ReasoningTokens = c.ReasoningTokens, + LatencyMs = c.Latency.TotalMilliseconds, + CostUsd = c.CostUsd, + Success = c.Success, + ErrorMessage = c.Error, + TimestampUtc = c.At, + }); + } + db.PipelineRuns.Add(entity); + await db.SaveChangesAsync(ct); + } + public async Task GetAsync(Guid id, CancellationToken ct = default) { var entity = await db.PipelineRuns diff --git a/src/AgentOs.Web/Components/Pages/BoardApp.razor b/src/AgentOs.Web/Components/Pages/BoardApp.razor index f91f7e58..5383a16d 100644 --- a/src/AgentOs.Web/Components/Pages/BoardApp.razor +++ b/src/AgentOs.Web/Components/Pages/BoardApp.razor @@ -1569,7 +1569,18 @@ else if (_tab == Tab.Sessions) // on its members' computers sends Discovery there too — it used to fall back to the keyless offline demo). using var _identity = AgentOs.SharedKernel.Identity.AmbientIdentity.Push(_tenantId, _userId); var svc = scope.ServiceProvider.GetRequiredService(); - _preview = await svc.GenerateAsync(_ideaText.Trim()); + // Metrics first: the Requirement + Decomposition calls are stored as one Discovery run for Insights. + var started = DateTimeOffset.UtcNow; + using (var calls = AgentOs.Domain.Llm.LlmCallScope.Begin()) + { + try { _preview = await svc.GenerateAsync(_ideaText.Trim()); } + finally + { + var idea = _ideaText.Trim().ReplaceLineEndings(" "); + await TaskService.SaveAgentRunAsync(_tenantId, _userId, "Discovery", + "Discovery · " + (idea.Length > 120 ? idea[..120] + "…" : idea), started, calls.Calls); + } + } _draftRows.Clear(); for (var k = 0; k < _preview.Tickets.Count; k++) diff --git a/src/AgentOs.Web/Services/BoardTasks/BoardTaskService.cs b/src/AgentOs.Web/Services/BoardTasks/BoardTaskService.cs index c30b1e14..293a182b 100644 --- a/src/AgentOs.Web/Services/BoardTasks/BoardTaskService.cs +++ b/src/AgentOs.Web/Services/BoardTasks/BoardTaskService.cs @@ -138,7 +138,11 @@ public async Task StartAsync(string tenantId, Guid sessionId, string? } var stopLease = stops.Begin(tenantId, session.Id); // Fire-and-forget in a fresh scope, independent of whoever asked (a circuit, or the orchestrator). - _ = Task.Run(() => RunAsync(tenantId, session, board, pat, children, cliProfile, change, stopLease), CancellationToken.None); + // The Thorough engine stores its own pipeline run + metrics; every other run is recorded here for Insights. + _ = Task.Run(() => EngineSelection.IsQuality(session.Brain) + ? RunAsync(tenantId, session, board, pat, children, cliProfile, change, stopLease) + : RecordedAsync(tenantId, session, "Coding", () => RunAsync(tenantId, session, board, pat, children, cliProfile, change, stopLease)), + CancellationToken.None); return TaskStart.Started; } @@ -170,7 +174,8 @@ public async Task TestAsync(string tenantId, Guid sessionId, string? await sessions.UpdateRepoRunResultAsync(tenantId, c.Id, "Testing", c.BranchName, c.PrUrl, null, ct).ConfigureAwait(false); } var stopLease = stops.Begin(tenantId, session.Id); - _ = Task.Run(() => TestChangesAsync(tenantId, session, board, pat, changes, cliProfile, stopLease), CancellationToken.None); + _ = Task.Run(() => RecordedAsync(tenantId, session, "Testing", () => TestChangesAsync(tenantId, session, board, pat, changes, cliProfile, stopLease)), + CancellationToken.None); return TaskStart.Started; } @@ -371,6 +376,48 @@ private static string ShortTestName(string test) return $"{name}-{DateTimeOffset.UtcNow:yyyyMMddHHmm}"; } + /// + /// Metrics first: run one agent step of a task inside an LLM call scope, then store it as a run with its calls + /// (tokens, latency, provider, outcome) — Insights reads those. Best-effort: metrics never fail the task. + /// + internal async Task RecordedAsync(string tenantId, RemoteSessionEntity session, string agent, Func run) + { + ArgumentNullException.ThrowIfNull(session); + ArgumentNullException.ThrowIfNull(run); + var started = DateTimeOffset.UtcNow; + using var calls = AgentOs.Domain.Llm.LlmCallScope.Begin(); + try + { + await run().ConfigureAwait(false); + } + finally + { + await SaveAgentRunAsync(tenantId, session.MemberUserId, agent, $"{agent} · #{session.IssueNumber} {session.Title}", started, calls.Calls) + .ConfigureAwait(false); + } + } + + /// Store an agent run and its LLM calls (nothing when it made none). Never throws. + public async Task SaveAgentRunAsync(string tenantId, string memberId, string agent, string label, DateTimeOffset started, + IReadOnlyList calls) + { + ArgumentNullException.ThrowIfNull(calls); + if (calls.Count == 0) { return; } + try + { + using var _identity = AgentOs.SharedKernel.Identity.AmbientIdentity.Push(tenantId, memberId); + using var scope = scopes.CreateScope(); + await scope.ServiceProvider.GetRequiredService() + .SaveAgentRunAsync(new AgentOs.Modules.Pipeline.Persistence.AgentRunRecord(Guid.NewGuid(), + label.Length > 200 ? label[..200] : label, agent, calls.All(c => c.Success), started, DateTimeOffset.UtcNow, calls), tenantId) + .ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _ = ex; // metrics are best-effort: a lost row never fails the task + } + } + /// The number of a pull request from its URL (…/pull/42). public static int? PrNumber(string? url) { diff --git a/tests/AgentOs.Tests/Metrics/AgentRunMetricsTests.cs b/tests/AgentOs.Tests/Metrics/AgentRunMetricsTests.cs new file mode 100644 index 00000000..0c2b006f --- /dev/null +++ b/tests/AgentOs.Tests/Metrics/AgentRunMetricsTests.cs @@ -0,0 +1,110 @@ +// Metrics first for the Board's agents: inside an LlmCallScope every call is recorded, and an agent run is stored so +// Insights counts it (Insights showed nothing while Autopilot's agents worked all day). + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AgentOs.Domain.Llm; +using AgentOs.Modules.Llm; +using AgentOs.Modules.Pipeline.Persistence; +using AgentOs.Modules.Pipeline.Persistence.Repositories; +using AgentOs.SharedKernel.Identity; +using Microsoft.EntityFrameworkCore; +using NSubstitute; +using NSubstitute.ExceptionExtensions; +using Shouldly; +using Xunit; + +namespace AgentOs.Tests.Metrics; + +public sealed class AgentRunMetricsTests +{ + private static LlmRequest Req(string? role = "Reviewer") => new("sys", "user", "claude-sonnet-4-6") { AgentRole = role }; + + [Fact] + public void Scope_InnermostWins_AndRestoresOnDispose() + { + LlmCallScope.Current.ShouldBeNull(); + using (var outer = LlmCallScope.Begin()) + { + using (var inner = LlmCallScope.Begin()) + { + LlmCallScope.Current.ShouldBeSameAs(inner); + } + LlmCallScope.Current.ShouldBeSameAs(outer); + } + LlmCallScope.Current.ShouldBeNull(); + } + + [Fact] + public async Task Observed_RecordsASuccessfulCall() + { + var inner = Substitute.For(); + inner.Provider.Returns("RemoteAgent"); + inner.SendAsync(Arg.Any(), Arg.Any()) + .Returns(new LlmResponse("ok", 1200, 300, 0m, TimeSpan.FromSeconds(42), "claude-sonnet-4-6", "RemoteAgent") { CacheReadTokens = 100 }); + using var scope = LlmCallScope.Begin(); + + await new ObservedLlmClient(inner, scope).SendAsync(Req()); + + var call = scope.Calls.ShouldHaveSingleItem(); + call.AgentRole.ShouldBe("Reviewer"); + call.InputTokens.ShouldBe(1200); + call.CacheReadTokens.ShouldBe(100); + call.Provider.ShouldBe("RemoteAgent"); + call.Latency.ShouldBe(TimeSpan.FromSeconds(42)); + call.Success.ShouldBeTrue(); + } + + [Fact] + public async Task Observed_RecordsAFailedCall_AndRethrows() + { + var inner = Substitute.For(); + inner.Provider.Returns("RemoteAgent"); + inner.SendAsync(Arg.Any(), Arg.Any()).ThrowsAsync(new TimeoutException("runner timed out")); + using var scope = LlmCallScope.Begin(); + + await Should.ThrowAsync(() => new ObservedLlmClient(inner, scope).SendAsync(Req())); + + var call = scope.Calls.ShouldHaveSingleItem(); + call.Success.ShouldBeFalse(); + call.Error.ShouldBe("runner timed out"); + } + + private sealed class FixedTenant(string id) : ITenantContext + { + public string TenantId { get; } = id; + public string? UserId => "u"; + public string? UserName => "u"; + public IReadOnlyList Roles { get; } = []; + public bool IsAuthenticated => true; + public bool IsAdmin => false; + } + + [Fact] + public async Task SaveAgentRun_IsCountedByTheCostSummary() + { + var options = new DbContextOptionsBuilder().UseInMemoryDatabase($"agentrun-{Guid.NewGuid()}").Options; + var now = DateTimeOffset.UtcNow; + LlmCallRecord Call(string? role, int input, bool ok = true) => + new(role, "claude-sonnet-4-6", "RemoteAgent", input, 100, 0, 0, 0, 0m, TimeSpan.FromSeconds(3), ok, ok ? null : "x", now); + + await using (var db = new PipelineDbContext(options, new FixedTenant("t1"))) + { + await new PipelineRunRepository(db, new FixedTenant("t1")).SaveAgentRunAsync( + new AgentRunRecord(Guid.NewGuid(), "Coding · #15 Build SVG game viewer", "Coding", true, now, now, [Call(null, 1000), Call("Reviewer", 500)]), "t1"); + } + + await using var read = new PipelineDbContext(options, new FixedTenant("x")); + var repo = new PipelineRunRepository(read, new FixedTenant("x")); + var sum = await repo.GetCostSummaryForTenantAsync("t1"); + + sum.CallCount.ShouldBe(2); + sum.RunCount.ShouldBe(1); + sum.TotalTokensIn.ShouldBe(1500); + sum.ByAgent.Select(b => b.Key).ShouldBe(["Coding", "Reviewer"], ignoreOrder: true); + (await repo.ListForTenantAsync("t1")).ShouldHaveSingleItem().UserStoryPreview.ShouldContain("#15"); + } +} From 83c0b81d9a863117b1eafcf01731bf3f70841dd1 Mon Sep 17 00:00:00 2001 From: Hoang Snowy Date: Thu, 22 Oct 2026 23:13:53 +0700 Subject: [PATCH 2/2] feat(insights): Reviewer runs are recorded too Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018WAdQ7Rm24xss5QDSvXtJB --- src/AgentOs.Web/Services/BoardTasks/BoardTaskService.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/AgentOs.Web/Services/BoardTasks/BoardTaskService.cs b/src/AgentOs.Web/Services/BoardTasks/BoardTaskService.cs index 293a182b..3b4776cc 100644 --- a/src/AgentOs.Web/Services/BoardTasks/BoardTaskService.cs +++ b/src/AgentOs.Web/Services/BoardTasks/BoardTaskService.cs @@ -208,7 +208,8 @@ public async Task ReviewAsync(string tenantId, Guid sessionId, string await sessions.UpdateRepoRunResultAsync(tenantId, c.Id, "Reviewing", c.BranchName, c.PrUrl, null, ct).ConfigureAwait(false); } var stopLease = stops.Begin(tenantId, session.Id); - _ = Task.Run(() => ReviewChangesAsync(tenantId, session, board, pat, changes, cliProfile, stopLease), CancellationToken.None); + _ = Task.Run(() => RecordedAsync(tenantId, session, "Reviewer", () => ReviewChangesAsync(tenantId, session, board, pat, changes, cliProfile, stopLease)), + CancellationToken.None); return TaskStart.Started; }