diff --git a/src/AgentOs.Domain/Llm/LlmCallScope.cs b/src/AgentOs.Domain/Llm/LlmCallScope.cs
new file mode 100644
index 00000000..a38e631e
--- /dev/null
+++ b/src/AgentOs.Domain/Llm/LlmCallScope.cs
@@ -0,0 +1,70 @@
+// Metrics first: every LLM call an agent makes inside a scope is recorded — tokens (fresh / cache read / cache write /
+// output / reasoning), cost, latency, provider, outcome. The Board's agents (Coding, Reviewer, Testing, Discovery, the
+// Autopilot Orchestrator) run outside the 5-agent pipeline, so their calls never reached Insights; the caller opens a
+// scope around one agent run and stores what it collected as one run with its calls.
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+
+namespace AgentOs.Domain.Llm;
+
+/// One LLM call as recorded for metrics.
+public sealed record LlmCallRecord(
+ string? AgentRole,
+ string Model,
+ string Provider,
+ int InputTokens,
+ int OutputTokens,
+ int CacheReadTokens,
+ int CacheWriteTokens,
+ int ReasoningTokens,
+ decimal CostUsd,
+ TimeSpan Latency,
+ bool Success,
+ string? Error,
+ DateTimeOffset At);
+
+/// Collects the LLM calls made in the current async flow while it is open (innermost scope wins).
+public sealed class LlmCallScope : IDisposable
+{
+ private static readonly AsyncLocal CurrentScope = new();
+
+ private readonly LlmCallScope? _previous;
+ private readonly List _calls = [];
+ private bool _disposed;
+
+ private LlmCallScope(LlmCallScope? previous) => _previous = previous;
+
+ /// The open scope of this flow, or null.
+ public static LlmCallScope? Current => CurrentScope.Value;
+
+ /// Open a scope for this async flow; dispose to close it.
+ public static LlmCallScope Begin()
+ {
+ var scope = new LlmCallScope(CurrentScope.Value);
+ CurrentScope.Value = scope;
+ return scope;
+ }
+
+ /// The calls recorded so far.
+ public IReadOnlyList Calls
+ {
+ get { lock (_calls) { return [.. _calls]; } }
+ }
+
+ /// Record one call.
+ public void Add(LlmCallRecord call)
+ {
+ ArgumentNullException.ThrowIfNull(call);
+ lock (_calls) { _calls.Add(call); }
+ }
+
+ ///
+ public void Dispose()
+ {
+ if (_disposed) { return; }
+ _disposed = true;
+ if (ReferenceEquals(CurrentScope.Value, this)) { CurrentScope.Value = _previous; }
+ }
+}
diff --git a/src/AgentOs.Modules.Llm/LlmClientFactory.cs b/src/AgentOs.Modules.Llm/LlmClientFactory.cs
index cc2a258b..f4a6b5ac 100644
--- a/src/AgentOs.Modules.Llm/LlmClientFactory.cs
+++ b/src/AgentOs.Modules.Llm/LlmClientFactory.cs
@@ -77,14 +77,18 @@ public ILlmClient Create(string providerName)
// No fallbacks declared → return the bare client, identical to the pre-failover behavior.
if (fallbacks.Count == 0)
{
- return primary;
+ return Observed(primary);
}
var chain = new List(1 + fallbacks.Count) { primary };
chain.AddRange(fallbacks);
- return new FailoverLlmClient(chain, _services.GetRequiredService>());
+ return Observed(new FailoverLlmClient(chain, _services.GetRequiredService>()));
}
+ // Metrics first: inside an open LlmCallScope (a Board agent run) every call is recorded for Insights.
+ private static ILlmClient Observed(ILlmClient client) =>
+ AgentOs.Domain.Llm.LlmCallScope.Current is { } scope ? new ObservedLlmClient(client, scope) : client;
+
// The primary must resolve — a missing primary is a hard misconfiguration.
private ILlmClient ResolveRequired(string effective, string requested)
{
diff --git a/src/AgentOs.Modules.Llm/ObservedLlmClient.cs b/src/AgentOs.Modules.Llm/ObservedLlmClient.cs
new file mode 100644
index 00000000..fe2e514e
--- /dev/null
+++ b/src/AgentOs.Modules.Llm/ObservedLlmClient.cs
@@ -0,0 +1,36 @@
+// Reports each call to the open LlmCallScope (see AgentOs.Domain.Llm.LlmCallScope). The factory only wraps a client
+// when a scope is open, so the pipeline's own metrics path and every unscoped call are untouched.
+
+using System;
+using System.Diagnostics;
+using System.Threading;
+using System.Threading.Tasks;
+using AgentOs.Domain.Llm;
+
+namespace AgentOs.Modules.Llm;
+
+/// An that records its calls into the scope that was open when it was created.
+internal sealed class ObservedLlmClient(ILlmClient inner, LlmCallScope scope) : ILlmClient
+{
+ public string Provider => inner.Provider;
+
+ public async Task SendAsync(LlmRequest request, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(request);
+ var watch = Stopwatch.StartNew();
+ try
+ {
+ var r = await inner.SendAsync(request, cancellationToken).ConfigureAwait(false);
+ scope.Add(new LlmCallRecord(request.AgentRole, r.Model, r.Provider, r.InputTokens, r.OutputTokens,
+ r.CacheReadTokens, r.CacheWriteTokens, r.ReasoningTokens, r.CostUsd,
+ r.Latency > TimeSpan.Zero ? r.Latency : watch.Elapsed, true, null, DateTimeOffset.UtcNow));
+ return r;
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ scope.Add(new LlmCallRecord(request.AgentRole, request.Model, inner.Provider, 0, 0, 0, 0, 0, 0m, watch.Elapsed,
+ false, ex.Message.Length > 500 ? ex.Message[..500] : ex.Message, DateTimeOffset.UtcNow));
+ throw;
+ }
+ }
+}
diff --git a/src/AgentOs.Modules.Pipeline/Persistence/IPipelineRunRepository.cs b/src/AgentOs.Modules.Pipeline/Persistence/IPipelineRunRepository.cs
index 80d366e4..e81b85c5 100644
--- a/src/AgentOs.Modules.Pipeline/Persistence/IPipelineRunRepository.cs
+++ b/src/AgentOs.Modules.Pipeline/Persistence/IPipelineRunRepository.cs
@@ -19,6 +19,10 @@ public interface IPipelineRunRepository
/// tenant's workflow spend.
Task SaveAsync(PipelineRunRecord record, string tenantId, CancellationToken ct = default);
+ /// Stores one Board agent run (Coding, Reviewer, Testing, Discovery, Autopilot Orchestrator) with the
+ /// LLM calls it made, so Insights counts it. It has no pipeline artifacts: its run detail reads as not found.
+ Task SaveAgentRunAsync(AgentRunRecord record, string tenantId, CancellationToken ct = default) => Task.CompletedTask;
+
/// Gets a single full run by Id (null if not found).
Task GetAsync(Guid id, CancellationToken ct = default);
@@ -64,6 +68,18 @@ public sealed record PipelineRunRecord(
DateTimeOffset CreatedAtUtc,
DateTimeOffset CompletedAtUtc);
+/// One Board agent run and its LLM calls (see ).
+/// What ran, shown in the run history (e.g. "Coding · #15 Build SVG game viewer").
+/// The agent name its calls are filed under when a call carries no role.
+public sealed record AgentRunRecord(
+ Guid Id,
+ string Label,
+ string Agent,
+ bool Success,
+ DateTimeOffset StartedAtUtc,
+ DateTimeOffset CompletedAtUtc,
+ IReadOnlyList Calls);
+
/// Summary of a single run for the history list.
public sealed record PipelineRunSummary(
Guid Id,
diff --git a/src/AgentOs.Modules.Pipeline/Persistence/Repositories/PipelineRunRepository.cs b/src/AgentOs.Modules.Pipeline/Persistence/Repositories/PipelineRunRepository.cs
index aef624a0..7b843e61 100644
--- a/src/AgentOs.Modules.Pipeline/Persistence/Repositories/PipelineRunRepository.cs
+++ b/src/AgentOs.Modules.Pipeline/Persistence/Repositories/PipelineRunRepository.cs
@@ -77,6 +77,52 @@ public async Task SaveAsync(PipelineRunRecord record, string tenantId, Cancellat
await db.SaveChangesAsync(ct);
}
+ public async Task SaveAgentRunAsync(AgentRunRecord record, string tenantId, CancellationToken ct = default)
+ {
+ ArgumentNullException.ThrowIfNull(record);
+ ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
+ var entity = new PipelineRunEntity
+ {
+ Id = record.Id,
+ TenantId = tenantId,
+ UserStoryText = record.Label,
+ Status = record.Success ? "Completed" : "Failed",
+ TotalCostUsd = record.Calls.Sum(c => c.CostUsd),
+ TotalTokensIn = record.Calls.Sum(c => c.InputTokens),
+ TotalTokensOut = record.Calls.Sum(c => c.OutputTokens),
+ IterationCount = 0,
+ CreatedAtUtc = record.StartedAtUtc,
+ CompletedAtUtc = record.CompletedAtUtc,
+ // No pipeline artifacts: the run detail deserializes this to null and reads as not found.
+ ResultJson = "null",
+ };
+ foreach (var c in record.Calls)
+ {
+ entity.Metrics.Add(new RunMetricEntity
+ {
+ RunId = record.Id,
+ TenantId = tenantId,
+ KcId = "board",
+ Iteration = 0,
+ AgentName = string.IsNullOrWhiteSpace(c.AgentRole) ? record.Agent : c.AgentRole,
+ Model = c.Model,
+ Provider = c.Provider,
+ TokensIn = c.InputTokens,
+ TokensOut = c.OutputTokens,
+ CacheReadTokens = c.CacheReadTokens,
+ CacheWriteTokens = c.CacheWriteTokens,
+ ReasoningTokens = c.ReasoningTokens,
+ LatencyMs = c.Latency.TotalMilliseconds,
+ CostUsd = c.CostUsd,
+ Success = c.Success,
+ ErrorMessage = c.Error,
+ TimestampUtc = c.At,
+ });
+ }
+ db.PipelineRuns.Add(entity);
+ await db.SaveChangesAsync(ct);
+ }
+
public async Task GetAsync(Guid id, CancellationToken ct = default)
{
var entity = await db.PipelineRuns
diff --git a/src/AgentOs.Web/Components/Pages/BoardApp.razor b/src/AgentOs.Web/Components/Pages/BoardApp.razor
index f91f7e58..5383a16d 100644
--- a/src/AgentOs.Web/Components/Pages/BoardApp.razor
+++ b/src/AgentOs.Web/Components/Pages/BoardApp.razor
@@ -1569,7 +1569,18 @@ else if (_tab == Tab.Sessions)
// on its members' computers sends Discovery there too — it used to fall back to the keyless offline demo).
using var _identity = AgentOs.SharedKernel.Identity.AmbientIdentity.Push(_tenantId, _userId);
var svc = scope.ServiceProvider.GetRequiredService();
- _preview = await svc.GenerateAsync(_ideaText.Trim());
+ // Metrics first: the Requirement + Decomposition calls are stored as one Discovery run for Insights.
+ var started = DateTimeOffset.UtcNow;
+ using (var calls = AgentOs.Domain.Llm.LlmCallScope.Begin())
+ {
+ try { _preview = await svc.GenerateAsync(_ideaText.Trim()); }
+ finally
+ {
+ var idea = _ideaText.Trim().ReplaceLineEndings(" ");
+ await TaskService.SaveAgentRunAsync(_tenantId, _userId, "Discovery",
+ "Discovery · " + (idea.Length > 120 ? idea[..120] + "…" : idea), started, calls.Calls);
+ }
+ }
_draftRows.Clear();
for (var k = 0; k < _preview.Tickets.Count; k++)
diff --git a/src/AgentOs.Web/Services/BoardTasks/BoardTaskService.cs b/src/AgentOs.Web/Services/BoardTasks/BoardTaskService.cs
index c30b1e14..3b4776cc 100644
--- a/src/AgentOs.Web/Services/BoardTasks/BoardTaskService.cs
+++ b/src/AgentOs.Web/Services/BoardTasks/BoardTaskService.cs
@@ -138,7 +138,11 @@ public async Task StartAsync(string tenantId, Guid sessionId, string?
}
var stopLease = stops.Begin(tenantId, session.Id);
// Fire-and-forget in a fresh scope, independent of whoever asked (a circuit, or the orchestrator).
- _ = Task.Run(() => RunAsync(tenantId, session, board, pat, children, cliProfile, change, stopLease), CancellationToken.None);
+ // The Thorough engine stores its own pipeline run + metrics; every other run is recorded here for Insights.
+ _ = Task.Run(() => EngineSelection.IsQuality(session.Brain)
+ ? RunAsync(tenantId, session, board, pat, children, cliProfile, change, stopLease)
+ : RecordedAsync(tenantId, session, "Coding", () => RunAsync(tenantId, session, board, pat, children, cliProfile, change, stopLease)),
+ CancellationToken.None);
return TaskStart.Started;
}
@@ -170,7 +174,8 @@ public async Task TestAsync(string tenantId, Guid sessionId, string?
await sessions.UpdateRepoRunResultAsync(tenantId, c.Id, "Testing", c.BranchName, c.PrUrl, null, ct).ConfigureAwait(false);
}
var stopLease = stops.Begin(tenantId, session.Id);
- _ = Task.Run(() => TestChangesAsync(tenantId, session, board, pat, changes, cliProfile, stopLease), CancellationToken.None);
+ _ = Task.Run(() => RecordedAsync(tenantId, session, "Testing", () => TestChangesAsync(tenantId, session, board, pat, changes, cliProfile, stopLease)),
+ CancellationToken.None);
return TaskStart.Started;
}
@@ -203,7 +208,8 @@ public async Task ReviewAsync(string tenantId, Guid sessionId, string
await sessions.UpdateRepoRunResultAsync(tenantId, c.Id, "Reviewing", c.BranchName, c.PrUrl, null, ct).ConfigureAwait(false);
}
var stopLease = stops.Begin(tenantId, session.Id);
- _ = Task.Run(() => ReviewChangesAsync(tenantId, session, board, pat, changes, cliProfile, stopLease), CancellationToken.None);
+ _ = Task.Run(() => RecordedAsync(tenantId, session, "Reviewer", () => ReviewChangesAsync(tenantId, session, board, pat, changes, cliProfile, stopLease)),
+ CancellationToken.None);
return TaskStart.Started;
}
@@ -371,6 +377,48 @@ private static string ShortTestName(string test)
return $"{name}-{DateTimeOffset.UtcNow:yyyyMMddHHmm}";
}
+ ///
+ /// Metrics first: run one agent step of a task inside an LLM call scope, then store it as a run with its calls
+ /// (tokens, latency, provider, outcome) — Insights reads those. Best-effort: metrics never fail the task.
+ ///
+ internal async Task RecordedAsync(string tenantId, RemoteSessionEntity session, string agent, Func run)
+ {
+ ArgumentNullException.ThrowIfNull(session);
+ ArgumentNullException.ThrowIfNull(run);
+ var started = DateTimeOffset.UtcNow;
+ using var calls = AgentOs.Domain.Llm.LlmCallScope.Begin();
+ try
+ {
+ await run().ConfigureAwait(false);
+ }
+ finally
+ {
+ await SaveAgentRunAsync(tenantId, session.MemberUserId, agent, $"{agent} · #{session.IssueNumber} {session.Title}", started, calls.Calls)
+ .ConfigureAwait(false);
+ }
+ }
+
+ /// Store an agent run and its LLM calls (nothing when it made none). Never throws.
+ public async Task SaveAgentRunAsync(string tenantId, string memberId, string agent, string label, DateTimeOffset started,
+ IReadOnlyList calls)
+ {
+ ArgumentNullException.ThrowIfNull(calls);
+ if (calls.Count == 0) { return; }
+ try
+ {
+ using var _identity = AgentOs.SharedKernel.Identity.AmbientIdentity.Push(tenantId, memberId);
+ using var scope = scopes.CreateScope();
+ await scope.ServiceProvider.GetRequiredService()
+ .SaveAgentRunAsync(new AgentOs.Modules.Pipeline.Persistence.AgentRunRecord(Guid.NewGuid(),
+ label.Length > 200 ? label[..200] : label, agent, calls.All(c => c.Success), started, DateTimeOffset.UtcNow, calls), tenantId)
+ .ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ _ = ex; // metrics are best-effort: a lost row never fails the task
+ }
+ }
+
/// The number of a pull request from its URL (…/pull/42).
public static int? PrNumber(string? url)
{
diff --git a/tests/AgentOs.Tests/Metrics/AgentRunMetricsTests.cs b/tests/AgentOs.Tests/Metrics/AgentRunMetricsTests.cs
new file mode 100644
index 00000000..0c2b006f
--- /dev/null
+++ b/tests/AgentOs.Tests/Metrics/AgentRunMetricsTests.cs
@@ -0,0 +1,110 @@
+// Metrics first for the Board's agents: inside an LlmCallScope every call is recorded, and an agent run is stored so
+// Insights counts it (Insights showed nothing while Autopilot's agents worked all day).
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using AgentOs.Domain.Llm;
+using AgentOs.Modules.Llm;
+using AgentOs.Modules.Pipeline.Persistence;
+using AgentOs.Modules.Pipeline.Persistence.Repositories;
+using AgentOs.SharedKernel.Identity;
+using Microsoft.EntityFrameworkCore;
+using NSubstitute;
+using NSubstitute.ExceptionExtensions;
+using Shouldly;
+using Xunit;
+
+namespace AgentOs.Tests.Metrics;
+
+public sealed class AgentRunMetricsTests
+{
+ private static LlmRequest Req(string? role = "Reviewer") => new("sys", "user", "claude-sonnet-4-6") { AgentRole = role };
+
+ [Fact]
+ public void Scope_InnermostWins_AndRestoresOnDispose()
+ {
+ LlmCallScope.Current.ShouldBeNull();
+ using (var outer = LlmCallScope.Begin())
+ {
+ using (var inner = LlmCallScope.Begin())
+ {
+ LlmCallScope.Current.ShouldBeSameAs(inner);
+ }
+ LlmCallScope.Current.ShouldBeSameAs(outer);
+ }
+ LlmCallScope.Current.ShouldBeNull();
+ }
+
+ [Fact]
+ public async Task Observed_RecordsASuccessfulCall()
+ {
+ var inner = Substitute.For();
+ inner.Provider.Returns("RemoteAgent");
+ inner.SendAsync(Arg.Any(), Arg.Any())
+ .Returns(new LlmResponse("ok", 1200, 300, 0m, TimeSpan.FromSeconds(42), "claude-sonnet-4-6", "RemoteAgent") { CacheReadTokens = 100 });
+ using var scope = LlmCallScope.Begin();
+
+ await new ObservedLlmClient(inner, scope).SendAsync(Req());
+
+ var call = scope.Calls.ShouldHaveSingleItem();
+ call.AgentRole.ShouldBe("Reviewer");
+ call.InputTokens.ShouldBe(1200);
+ call.CacheReadTokens.ShouldBe(100);
+ call.Provider.ShouldBe("RemoteAgent");
+ call.Latency.ShouldBe(TimeSpan.FromSeconds(42));
+ call.Success.ShouldBeTrue();
+ }
+
+ [Fact]
+ public async Task Observed_RecordsAFailedCall_AndRethrows()
+ {
+ var inner = Substitute.For();
+ inner.Provider.Returns("RemoteAgent");
+ inner.SendAsync(Arg.Any(), Arg.Any()).ThrowsAsync(new TimeoutException("runner timed out"));
+ using var scope = LlmCallScope.Begin();
+
+ await Should.ThrowAsync(() => new ObservedLlmClient(inner, scope).SendAsync(Req()));
+
+ var call = scope.Calls.ShouldHaveSingleItem();
+ call.Success.ShouldBeFalse();
+ call.Error.ShouldBe("runner timed out");
+ }
+
+ private sealed class FixedTenant(string id) : ITenantContext
+ {
+ public string TenantId { get; } = id;
+ public string? UserId => "u";
+ public string? UserName => "u";
+ public IReadOnlyList Roles { get; } = [];
+ public bool IsAuthenticated => true;
+ public bool IsAdmin => false;
+ }
+
+ [Fact]
+ public async Task SaveAgentRun_IsCountedByTheCostSummary()
+ {
+ var options = new DbContextOptionsBuilder().UseInMemoryDatabase($"agentrun-{Guid.NewGuid()}").Options;
+ var now = DateTimeOffset.UtcNow;
+ LlmCallRecord Call(string? role, int input, bool ok = true) =>
+ new(role, "claude-sonnet-4-6", "RemoteAgent", input, 100, 0, 0, 0, 0m, TimeSpan.FromSeconds(3), ok, ok ? null : "x", now);
+
+ await using (var db = new PipelineDbContext(options, new FixedTenant("t1")))
+ {
+ await new PipelineRunRepository(db, new FixedTenant("t1")).SaveAgentRunAsync(
+ new AgentRunRecord(Guid.NewGuid(), "Coding · #15 Build SVG game viewer", "Coding", true, now, now, [Call(null, 1000), Call("Reviewer", 500)]), "t1");
+ }
+
+ await using var read = new PipelineDbContext(options, new FixedTenant("x"));
+ var repo = new PipelineRunRepository(read, new FixedTenant("x"));
+ var sum = await repo.GetCostSummaryForTenantAsync("t1");
+
+ sum.CallCount.ShouldBe(2);
+ sum.RunCount.ShouldBe(1);
+ sum.TotalTokensIn.ShouldBe(1500);
+ sum.ByAgent.Select(b => b.Key).ShouldBe(["Coding", "Reviewer"], ignoreOrder: true);
+ (await repo.ListForTenantAsync("t1")).ShouldHaveSingleItem().UserStoryPreview.ShouldContain("#15");
+ }
+}