feat(insights): real token usage from the member's computer; one row per agent role

src/AgentOs.Modules.RemoteAgent/IRemoteAgentBroker.cs

@@ -30,7 +30,21 @@ namespace AgentOs.Modules.RemoteAgent;
 public sealed record RemoteExecRequest(string Id, string SystemPrompt, string UserPrompt, string Model, string? Cli = null, string? Autonomy = null);
 
 /// <summary>A remote runner's reply for a <see cref="RemoteExecRequest"/>.</summary>
-public sealed record RemoteExecResult(string Id, bool Ok, string Content, string? Error);
+public sealed record RemoteExecResult(string Id, bool Ok, string Content, string? Error)
+{
+    /// <summary>Real input tokens the runner's CLI reported (cache reads/writes included); null = not reported
+    /// (an older runner, or a CLI that reports no usage) and the server estimates.</summary>
+    public int? InputTokens { get; init; }
+
+    /// <summary>Real output tokens the CLI reported; null = not reported.</summary>
+    public int? OutputTokens { get; init; }
+
+    /// <summary>Input tokens served from the prompt cache (a subset of <see cref="InputTokens"/>).</summary>
+    public int? CacheReadTokens { get; init; }
+
+    /// <summary>Input tokens written to the prompt cache (a subset of <see cref="InputTokens"/>).</summary>
+    public int? CacheWriteTokens { get; init; }
+}
 
 /// <summary>Identity of a connected runner, established by the hub's pairing handshake.</summary>
 public sealed record RunnerConnection(Guid RunnerId, string TenantId, string OwnerUserId);

src/AgentOs.Modules.RemoteAgent/RemoteAgentLlmClient.cs

@@ -125,11 +125,14 @@ void OnProgress(RemoteProgress p)
         // the standard rough heuristic). These counts are honest estimates — they let the UI show a real
         // token figure and derive a "what this would have cost on a metered API" number, without ever
         // claiming a billed spend. The estimate is deliberately NOT folded into CostUsd.
-        var inTok = EstimateTokens(request.SystemPrompt) + EstimateTokens(request.UserPrompt);
-        var outTok = EstimateTokens(result.Content);
+        // A current runner sends the CLI's real usage (the whole agentic run: every tool turn, cache reads included);
+        // only an older runner, or a CLI that reports none, falls back to the estimate.
+        var real = result.InputTokens is not null && result.OutputTokens is not null;
+        var inTok = real ? result.InputTokens!.Value : EstimateTokens(request.SystemPrompt) + EstimateTokens(request.UserPrompt);
+        var outTok = real ? result.OutputTokens!.Value : EstimateTokens(result.Content);
         LlmTelemetry.RecordSuccess(activity, genAiSystem, request.Model, request.Model, inTok, outTok, 0m, stopwatch.Elapsed.TotalSeconds);
-        _logger.LogInformation("[RemoteAgent] request {Id} handled by a remote agent ({Count} connected); ~{In}->{Out} est. tokens, 0 API cost.",
-            id, _broker.AgentCount, inTok, outTok);
+        _logger.LogInformation("[RemoteAgent] request {Id} handled by a remote agent ({Count} connected); {In}->{Out} tokens ({Kind}), 0 API cost.",
+            id, _broker.AgentCount, inTok, outTok, real ? "reported by the CLI" : "estimated");
 
         return new LlmResponse(
             Content: result.Content,
@@ -138,7 +141,11 @@ void OnProgress(RemoteProgress p)
             CostUsd: 0m,
             Latency: stopwatch.Elapsed,
             Model: request.Model,
-            Provider: Provider);
+            Provider: Provider)
+        {
+            CacheReadTokens = real ? result.CacheReadTokens ?? 0 : 0,
+            CacheWriteTokens = real ? result.CacheWriteTokens ?? 0 : 0,
+        };
     }
 
     // Rough token estimate for text the CLI produced no usage numbers for: ~4 characters per token, the

src/AgentOs.RemoteAgent/CliProgress.cs

@@ -28,7 +28,21 @@ internal static readonly (string Key, string Label)[] Steps =
     ];
 
     /// <summary>The final event of a stream-json run.</summary>
-    internal sealed record Result(bool IsError, string Text, JsonElement? Usage, decimal? CostUsd);
+    internal sealed record Result(bool IsError, string Text, JsonElement? Usage, decimal? CostUsd)
+    {
+        /// <summary>The run's real token usage, normalised so cache reads/writes are a subset of the input (Claude's
+        /// usage reports input_tokens WITHOUT them). Null when the CLI reported none.</summary>
+        public (int Input, int Output, int CacheRead, int CacheWrite)? Tokens()
+        {
+            if (Usage is not { } u) { return null; }
+            int Get(string name) => u.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.Number && v.TryGetInt32(out var n) ? n : 0;
+            var fresh = Get("input_tokens");
+            var read = Get("cache_read_input_tokens");
+            var write = Get("cache_creation_input_tokens");
+            var output = Get("output_tokens");
+            return fresh + read + write + output == 0 ? null : (fresh + read + write, output, read, write);
+        }
+    }
 
     /// <summary>The step a tool call belongs to, or null when it doesn't move the task along (e.g. a `pwd`).</summary>
     internal static string? StepFor(string tool, JsonElement input)

src/AgentOs.RemoteAgent/Program.cs

@@ -321,9 +321,17 @@ static async Task<RemoteExecResult> RunLlmAsync(RemoteExecRequest request, Func<
             await process.WaitForExitAsync().ConfigureAwait(false);
             if (final is not null)
             {
-                return final.IsError
+                // Metrics first: the CLI's own usage travels back, so the server shows real tokens, not an estimate.
+                var used = final.Tokens();
+                return (final.IsError
                     ? new RemoteExecResult(request.Id, false, string.Empty, final.Text.Length > 0 ? final.Text : streamErr.Trim())
-                    : new RemoteExecResult(request.Id, true, final.Text.Trim(), null);
+                    : new RemoteExecResult(request.Id, true, final.Text.Trim(), null)) with
+                {
+                    InputTokens = used?.Input,
+                    OutputTokens = used?.Output,
+                    CacheReadTokens = used?.CacheRead,
+                    CacheWriteTokens = used?.CacheWrite,
+                };
             }
             // No result event (killed by Stop, crashed, not logged in): fall through to the exit-code report.
             stdout = raw.ToString();
@@ -560,7 +568,14 @@ internal static partial class Program
 // ── DTOs — must match the server's shapes ───────────────────────────────────────────────────────
 
 internal sealed record RemoteExecRequest(string Id, string SystemPrompt, string UserPrompt, string Model, string? Cli = null, string? Autonomy = null);
-internal sealed record RemoteExecResult(string Id, bool Ok, string Content, string? Error);
+internal sealed record RemoteExecResult(string Id, bool Ok, string Content, string? Error)
+{
+    // Real usage reported by the CLI (cache reads/writes are a subset of the input); null = not reported.
+    public int? InputTokens { get; init; }
+    public int? OutputTokens { get; init; }
+    public int? CacheReadTokens { get; init; }
+    public int? CacheWriteTokens { get; init; }
+}
 internal sealed record RunnerToolCall(string RequestId, string ToolCallId, string ToolName, string JsonInput);
 internal sealed record RunnerToolResult(string RequestId, string ToolCallId, bool Ok, string JsonOutput, string? Error);
 

src/AgentOs.Web/Components/Pages/InsightsApp.razor

@@ -147,7 +147,7 @@ else
 
         @* ── Chart: per-agent (calls + cost) ─────────────────────────────────────────────── *@
         <Panel Title="By agent">
-            @{ var agents = _summary!.ByAgent.OrderByDescending(a => a.TokensIn + a.TokensOut).ToList(); var maxAgent = agents.Count == 0 ? 1 : Math.Max(1, agents.Max(a => a.TokensIn + a.TokensOut)); }
+            @{ var agents = ByRole(_summary!.ByAgent).OrderByDescending(a => a.TokensIn + a.TokensOut).ToList(); var maxAgent = agents.Count == 0 ? 1 : Math.Max(1, agents.Max(a => a.TokensIn + a.TokensOut)); }
             @if (agents.Count == 0)
             {
                 <p class="muted ins-note">No agent calls in this range.</p>
@@ -204,7 +204,7 @@ else
 
         @* ── Chart: latency + QA rounds ──────────────────────────────────────────────────── *@
         <Panel Title="Latency & QA rounds">
-            @{ var lat = _extra?.ByAgentLatency ?? []; var maxLat = lat.Count == 0 ? 0.0 : lat.Max(l => l.AvgLatencyMs); }
+            @{ var lat = LatencyByRole(_extra?.ByAgentLatency ?? []).ToList(); var maxLat = lat.Count == 0 ? 0.0 : lat.Max(l => l.AvgLatencyMs); }
             @if (lat.Count == 0)
             {
                 <p class="muted ins-note">No latency data in this range.</p>
@@ -409,6 +409,19 @@ else
     private static string AgentShort(string agent) => agent
         .Replace("Agent", "", StringComparison.Ordinal).Trim() is { Length: > 0 } s ? s : agent;
 
+    // Weighted average across the role's names (calls × average latency).
+    internal static IEnumerable<AgentOs.Modules.Pipeline.Persistence.AgentLatencyBucket> LatencyByRole(IEnumerable<AgentOs.Modules.Pipeline.Persistence.AgentLatencyBucket> buckets) =>
+        buckets.GroupBy(b => AgentShort(b.Agent), StringComparer.OrdinalIgnoreCase)
+            .Select(g => new AgentOs.Modules.Pipeline.Persistence.AgentLatencyBucket(g.Key,
+                g.Sum(b => b.Calls) == 0 ? 0 : g.Sum(b => b.AvgLatencyMs * b.Calls) / g.Sum(b => b.Calls), g.Sum(b => b.Calls)))
+            .OrderByDescending(b => b.AvgLatencyMs);
+
+    // One row per role: the 5-agent pipeline files calls as "CodingAgent", the Board's agents as "Coding" — the same role.
+    internal static IEnumerable<AgentOs.Modules.Pipeline.Persistence.CostBucket> ByRole(IEnumerable<AgentOs.Modules.Pipeline.Persistence.CostBucket> buckets) =>
+        buckets.GroupBy(b => AgentShort(b.Key), StringComparer.OrdinalIgnoreCase)
+            .Select(g => new AgentOs.Modules.Pipeline.Persistence.CostBucket(g.Key, g.Sum(b => b.CostUsd), g.Sum(b => b.TokensIn),
+                g.Sum(b => b.TokensOut), g.Sum(b => b.Calls), g.Sum(b => b.CacheReadTokens), g.Sum(b => b.CacheWriteTokens)));
+
     private Badge.BadgeVariant BudgetBadge() => _budget?.State switch
     {
         BudgetState.Exceeded => Badge.BadgeVariant.Err,

tests/AgentOs.Tests/Llm/RemoteAgentTests.cs

@@ -127,6 +127,41 @@ public async Task Client_RunnerReportsProgress_ReachesThatRequestOnly()
         steps.ShouldBe(["testing"]);
     }
 
+    // Metrics first: a current runner sends the CLI's real usage; the ~4-chars-per-token estimate is only a fallback.
+    [Fact]
+    public async Task Client_RunnerReportsUsage_UsesTheRealTokens()
+    {
+        var broker = new InProcessRemoteAgentBroker();
+        using var reg = broker.RegisterRunner("conn-1", Conn());
+        broker.Dispatched += d => broker.Complete(new RemoteExecResult(d.Request.Id, true, "ok", null)
+        {
+            InputTokens = 250_000, OutputTokens = 9_000, CacheReadTokens = 200_000, CacheWriteTokens = 30_000,
+        });
+        var client = new RemoteAgentLlmClient(broker, Tenant(), NullLogger<RemoteAgentLlmClient>.Instance);
+
+        var r = await client.SendAsync(new LlmRequest("s", "u", "m"));
+
+        r.InputTokens.ShouldBe(250_000);
+        r.OutputTokens.ShouldBe(9_000);
+        r.CacheReadTokens.ShouldBe(200_000);
+        r.CacheWriteTokens.ShouldBe(30_000);
+        r.CostUsd.ShouldBe(0m); // the member's subscription — never billed on the server
+    }
+
+    [Fact]
+    public async Task Client_OlderRunnerWithoutUsage_Estimates()
+    {
+        var broker = new InProcessRemoteAgentBroker();
+        using var reg = broker.RegisterRunner("conn-1", Conn());
+        broker.Dispatched += d => broker.Complete(new RemoteExecResult(d.Request.Id, true, new string('x', 400), null));
+        var client = new RemoteAgentLlmClient(broker, Tenant(), NullLogger<RemoteAgentLlmClient>.Instance);
+
+        var r = await client.SendAsync(new LlmRequest("s", "u", "m"));
+
+        r.OutputTokens.ShouldBe(100);
+        r.CacheReadTokens.ShouldBe(0);
+    }
+
     [Fact]
     public async Task Broker_NoRunner_DispatchThrows()
     {

tests/AgentOs.Tests/Sessions/CliProgressTests.cs

@@ -57,6 +57,15 @@ public void Parse_ResultEvent_ReturnsTheAnswerUsageAndCost()
         final.Text.ShouldBe("{\"branch\":\"issue-7-ai-fix\",\"summary\":\"Done.\"}");
         final.CostUsd.ShouldBe(0.4213m);
         final.Usage!.Value.GetProperty("cache_read_input_tokens").GetInt32().ShouldBe(3000);
+        // Claude reports input WITHOUT cache reads/writes; normalised, the cache is a subset of the input.
+        final.Tokens().ShouldBe((3012, 900, 3000, 0));
+    }
+
+    [Fact]
+    public void Tokens_NoUsageReported_IsNull()
+    {
+        var (_, final) = CliProgress.Parse("""{"type":"result","is_error":false,"result":"ok"}""");
+        final!.Tokens().ShouldBeNull();
     }
 
     [Theory]

tests/AgentOs.Tests/Web/InsightsByRoleTests.cs

@@ -0,0 +1,39 @@
+// Insights shows one row per agent role: the 5-agent pipeline files calls as "CodingAgent", the Board's agents as
+// "Coding" — on the lab both showed as two separate "Coding" rows.
+
+using AgentOs.Modules.Pipeline.Persistence;
+using AgentOs.Web.Components.Pages;
+using Shouldly;
+using Xunit;
+
+namespace AgentOs.Tests.Web;
+
+public sealed class InsightsByRoleTests
+{
+    [Fact]
+    public void ByRole_MergesTheSameRole_AndSumsEverything()
+    {
+        CostBucket[] buckets =
+        [
+            new("CodingAgent", 0.01m, 1000, 200, 3, 100, 10),
+            new("Coding", 0m, 5000, 900, 5, 4000, 300),
+            new("Reviewer", 0m, 700, 50, 1),
+        ];
+
+        var rows = InsightsApp.ByRole(buckets).ToDictionary(b => b.Key);
+
+        rows.Count.ShouldBe(2);
+        rows["Coding"].TokensIn.ShouldBe(6000);
+        rows["Coding"].Calls.ShouldBe(8);
+        rows["Coding"].CacheReadTokens.ShouldBe(4100);
+        rows["Coding"].CostUsd.ShouldBe(0.01m);
+    }
+
+    [Fact]
+    public void LatencyByRole_IsCallWeighted()
+    {
+        AgentLatencyBucket[] buckets = [new("CodingAgent", 1000, 1), new("Coding", 4000, 3)];
+
+        InsightsApp.LatencyByRole(buckets).ShouldHaveSingleItem().AvgLatencyMs.ShouldBe(3250);
+    }
+}