From 4cb02710aab568750876e14f3e19f0ba0bef5878 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Thu, 12 Nov 2026 12:26:28 +0000 Subject: [PATCH] fix(router): learn from real token counts, show hedges, price in thinking tokens From the first live routed calls (2026-11-12): - Infernet reports usage 0: treat 0 as unknown and estimate, so cost and per-token speed are learned (a 14B call took 108 s, router guessed 11 s) - a hedge records its own step when it fires (the aborted loser settled after the receipt was built and vanished from it) - thinking models (qwen3) count 3x output tokens outside math/reasoning: qwen3:4b spent 200 s reasoning about three primary colors - one slow sample (cold model load) is capped at 400 ms/token Co-Authored-By: Claude Opus 5.5 --- apps/web/src/lib/router/engine.ts | 10 ++++++---- apps/web/src/lib/router/planner.ts | 9 +++++++-- apps/web/src/lib/router/stats.ts | 6 +++++- apps/web/src/lib/router/types.ts | 2 +- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/apps/web/src/lib/router/engine.ts b/apps/web/src/lib/router/engine.ts index b6049527..2753ca20 100644 --- a/apps/web/src/lib/router/engine.ts +++ b/apps/web/src/lib/router/engine.ts @@ -64,8 +64,8 @@ export async function execute(args: { const signals = [args.signal, Number.isFinite(deadline) ? AbortSignal.timeout(Math.max(1000, remainingMs())) : undefined].filter(Boolean) as AbortSignal[]; try { const r = await call(model, msgs, { maxTokens, temperature: opts.temperature, signal: signals.length ? AbortSignal.any(signals) : undefined }); - const inTok = r.inTokens ?? inEst; - const outTok = r.outTokens ?? estimateTokens(r.text); + const inTok = r.inTokens || inEst; // 0 = not reported + const outTok = r.outTokens || estimateTokens(r.text); const m = price.get(model); const costUsd = m ? (inTok * m.priceIn + outTok * m.priceOut) / 1e6 : 0; spent += costUsd; @@ -111,6 +111,8 @@ export async function execute(args: { } if (!canAfford(backup, f.promptTokens, 300)) return a; state.fired = true; + // Recorded now: the losing call may only settle after the receipt is built. + steps.push({ role: "hedge", model: backup, ok: true, ms: 0, inTokens: 0, outTokens: 0, costUsd: 0, note: `${model} had not answered after ${Math.round(hedgeMs)} ms; same request sent to ${backup}, first answer wins` }); const b = run(backup, second, "fallback"); const winner = await firstNonNull([a, b]); if (winner?.model === model) second.abort(); @@ -124,8 +126,8 @@ export async function execute(args: { const inEst = estimateTokens(messages.map((m) => m.content).join("\n")); try { const r = await call(model, messages, { maxTokens: opts.maxTokens, temperature: opts.temperature, signal }); - const inTok = r.inTokens ?? inEst; - const outTok = r.outTokens ?? estimateTokens(r.text); + const inTok = r.inTokens || inEst; // 0 = not reported + const outTok = r.outTokens || estimateTokens(r.text); const m = price.get(model); const costUsd = m ? (inTok * m.priceIn + outTok * m.priceOut) / 1e6 : 0; spent += costUsd; diff --git a/apps/web/src/lib/router/planner.ts b/apps/web/src/lib/router/planner.ts index c483731d..0afbe70c 100644 --- a/apps/web/src/lib/router/planner.ts +++ b/apps/web/src/lib/router/planner.ts @@ -92,8 +92,13 @@ export function viewModels(models: ModelProfile[], f: Features, stats: Stats, op p *= 1 - Math.min(0.5, stats.errorRate(m.id)); if (m.providers !== undefined && m.busy !== undefined && m.providers > 0 && m.busy >= m.providers) p *= 0.92; p = Math.max(0.01, Math.min(0.99, p)); - const costUsd = (f.promptTokens * m.priceIn + out * m.priceOut) / 1e6; - const latencyMs = 400 + out * speedMsPerToken(m, stats) + f.promptTokens * 0.15; + // "Thinking" models write their reasoning out first: on anything but + // math/reasoning that is ~3x the tokens for the same answer (qwen3:4b + // took 200 s on "name three primary colors", 2026-11-12). + const thinks = m.tags.includes("reasoning") && f.category !== "math" && f.category !== "reasoning"; + const mOut = thinks ? out * 3 : out; + const costUsd = (f.promptTokens * m.priceIn + mOut * m.priceOut) / 1e6; + const latencyMs = 400 + mOut * speedMsPerToken(m, stats) + f.promptTokens * 0.15; return { m, p, costUsd, latencyMs }; }) .sort((a, b) => b.p - a.p || a.costUsd - b.costUsd || a.m.id.localeCompare(b.m.id)); diff --git a/apps/web/src/lib/router/stats.ts b/apps/web/src/lib/router/stats.ts index 5b7797f6..ae29213c 100644 --- a/apps/web/src/lib/router/stats.ts +++ b/apps/web/src/lib/router/stats.ts @@ -14,6 +14,8 @@ export const PRIOR_WEIGHT = 6; export const ERROR_STREAK_COOLDOWN = 3; export const COOLDOWN_MS = 2 * 60_000; const EWMA = 0.2; +/** One observation counts as at most this slow (ms per output token). */ +export const MAX_SAMPLE_MS_PER_TOKEN = 400; export type QualityRow = { ok: number; fail: number }; export type HealthRow = { @@ -91,7 +93,9 @@ export class Stats { h.coolUntil = undefined; if (r.outTokens > 8) { // Split wall time into overhead + per-token cost with a fixed 400 ms overhead guess. - const per = Math.max(1, (r.ms - 400) / r.outTokens); + // Capped: a cold model load on a node (60-100 s once) must not + // read as the model's speed forever; EWMA does the rest. + const per = Math.min(MAX_SAMPLE_MS_PER_TOKEN, Math.max(1, (r.ms - 400) / r.outTokens)); h.msPerToken = h.msPerToken === undefined ? per : h.msPerToken * (1 - EWMA) + per * EWMA; h.overheadMs = h.overheadMs === undefined ? 400 : h.overheadMs * (1 - EWMA) + 400 * EWMA; } diff --git a/apps/web/src/lib/router/types.ts b/apps/web/src/lib/router/types.ts index f5d44000..4792fa1b 100644 --- a/apps/web/src/lib/router/types.ts +++ b/apps/web/src/lib/router/types.ts @@ -100,7 +100,7 @@ export type RouteOptions = { export type Verification = { pass: boolean; score: number; issues: string[] }; -export type StepRole = "draft" | "verify" | "judge" | "escalate" | "critic" | "revise" | "fallback" | "synthesize"; +export type StepRole = "draft" | "verify" | "judge" | "escalate" | "critic" | "revise" | "fallback" | "synthesize" | "hedge"; export type StepRecord = { role: StepRole;