diff --git a/cmd/agentop/cmd_pricing.go b/cmd/agentop/cmd_pricing.go index 159c0c3fc..a19457502 100644 --- a/cmd/agentop/cmd_pricing.go +++ b/cmd/agentop/cmd_pricing.go @@ -20,7 +20,8 @@ import ( // // This exists because no config file can answer the question. `pricing:` shows what the // operator wrote; the figures a request is charged come from that PLUS a rate table -// compiled into the binary PLUS any shipped gateway discount. Storing the table in the +// compiled into the binary, and on a local install downloaded from LiteLLM, PLUS any +// shipped gateway discount. Storing the table in the // config instead would freeze every install at the rates current on its install date, // silently, because that file is written once and never refreshed — which is the exact // staleness the pricing work was done to remove. @@ -42,9 +43,10 @@ Usage: agentop pricing --host what that endpoint is charged, discount applied agentop pricing --json raw JSON -The rates come from a table built into the binary (vendor list, refreshed per release), -your `+"`pricing:`"+` config, and any gateway discount Cortex ships. --host is the useful -form: it resolves all three the way a request would. +The rates come from a table built into the binary (vendor list), your `+"`pricing:`"+` +config, and any gateway discount Cortex ships. A local install updates the built-in table +from LiteLLM's price list, downloaded hourly, so a new model is priced without a new +release. --host is the useful form: it resolves all of it the way a request would. Flags: `) @@ -221,7 +223,8 @@ type thresholdRow struct { } type describeBody struct { - UpstreamCommit string `json:"upstreamCommit"` + UpstreamCommit string `json:"upstreamCommit"` + ListFetchedAt time.Time `json:"listFetchedAt"` Rows []struct { Host string `json:"host"` Model string `json:"model"` @@ -250,6 +253,9 @@ func renderTable(body []byte, stdout, stderr io.Writer) int { if d.UpstreamCommit != "" { fmt.Fprintf(stdout, " bundled rates generated from litellm %s\n", short(d.UpstreamCommit)) } + if !d.ListFetchedAt.IsZero() { + fmt.Fprintf(stdout, " rates from litellm's price list, downloaded %s\n", d.ListFetchedAt.UTC().Format("2006-01-02 15:04 MST")) + } fmt.Fprintf(stdout, "\n %-22s %-30s %9s %9s %9s %9s %s\n", "endpoint", "model", "input", "cache-wr", "cache-rd", "output", "from") for _, r := range d.Rows { diff --git a/cmd/agentop/cmd_pricing_test.go b/cmd/agentop/cmd_pricing_test.go index df674f4c0..05ceedb36 100644 --- a/cmd/agentop/cmd_pricing_test.go +++ b/cmd/agentop/cmd_pricing_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" "github.com/rossoctl/cortex/core/cost/pricing" ) @@ -127,6 +128,37 @@ func TestRunPricing_TableViewListsRowsAndDiscounts(t *testing.T) { } } +// A local install prices from a downloaded list, and the header must say so. +func TestRunPricing_TableViewSaysWhenTheListWasDownloaded(t *testing.T) { + all := [pricing.NumTiers]bool{} + for i := range all { + all[i] = true + } + tab, err := pricing.BuildWithList(nil, &pricing.List{ + Entries: []pricing.Entry{{Host: "*", Model: "claude-opus-5-5", Prov: pricing.ProvBundled, + Rates: pricing.Rates{Base: [pricing.NumTiers]float64{pricing.TierInput: 4e-06, pricing.TierCacheWrite: 5e-06, + pricing.TierCacheRead: 2e-07, pricing.TierOutput: 2e-05}, Set: all}}}, + FetchedAt: time.Date(2026, 10, 8, 21, 0, 0, 0, time.UTC), + }) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(pricing.NewRegistry(tab).Handler()) + t.Cleanup(srv.Close) + + var out, errb bytes.Buffer + if code := runPricing([]string{"--stats-url", srv.URL}, &out, &errb); code != 0 { + t.Fatalf("exit %d: %s", code, errb.String()) + } + got := out.String() + if !strings.Contains(got, "rates from litellm's price list, downloaded 2026-10-08 21:00 UTC") { + t.Errorf("output does not say when the list was downloaded:\n%s", got) + } + if !strings.Contains(got, "bundled rates generated from litellm "+short(pricing.BundledUpstreamCommit)) { + t.Errorf("output does not name the shipped commit, whose rows the list does not name are still in the table:\n%s", got) + } +} + // A tier with no rate must render "-", never 0.00: pricing.Cost refuses to price a // request that used such a tier, so a zero would misrepresent a coverage gap as free. func TestRunPricing_UnsetTierRendersAsAbsent(t *testing.T) { diff --git a/cmd/cortex/main.go b/cmd/cortex/main.go index 0579d821d..43133ee1d 100644 --- a/cmd/cortex/main.go +++ b/cmd/cortex/main.go @@ -466,6 +466,9 @@ func main() { // disagree with the plugins about what a request cost. The table is swapped in // place instead; the pointer never changes. See pricing.Registry. pricingRegistry := pricing.NewRegistry(nil) + // Every table goes into the registry through live, which also applies a downloaded price + // list on a local install (runPriceList) without undoing a config reload, or the reverse. + live := &livePricing{reg: pricingRegistry} // This binary is hardcoded to proxy-sidecar. Rejecting other modes // early gives operators a clear boot-time error instead of silently @@ -494,7 +497,7 @@ func main() { // built so a plugin's Configure sees the new table, and swapped in place so // the usage aggregator — which holds this same registry from before the // reload — sees it too. - tab, err := pricing.Build(c.Pricing) + tab, err := live.build(c.Pricing) if err != nil { return nil, nil, nil, fmt.Errorf("pricing: %w", err) } @@ -533,10 +536,10 @@ func main() { // applyPricing puts a prepared table into effect. Called on the reload goroutine, which is // serialised, so the single pending slot needs no lock. applyPricing := func(c *config.Config) { - if pendingPricing != nil { - pricingRegistry.Swap(pendingPricing) - pendingPricing = nil + if pendingPricing != nil && c != nil { + live.Swap(c.Pricing, pendingPricing) } + pendingPricing = nil if c != nil { c.Pricing.WarnIfUnpinned(slog.Default()) } @@ -582,6 +585,10 @@ func main() { if err := rld.Start(ctx); err != nil { log.Fatalf("reloader: %v", err) } + // After the first config is applied, so the list is combined with it from the start. + if localInstall && cfg.Pricing.BundledEnabled() { + go runPriceList(ctx, live) + } var sessions *session.Store var usageAgg *usage.Aggregator diff --git a/cmd/cortex/pricing_list.go b/cmd/cortex/pricing_list.go new file mode 100644 index 000000000..f198da6f1 --- /dev/null +++ b/cmd/cortex/pricing_list.go @@ -0,0 +1,88 @@ +package main + +import ( + "context" + "log/slog" + "path/filepath" + "sync" + "time" + + "github.com/rossoctl/cortex/core/cost/pricing" + "github.com/rossoctl/cortex/core/cost/pricing/pricelist" +) + +// priceListInterval is how often a local install asks whether LiteLLM's price list changed. +// An unchanged list answers 304 with no body, so checking hourly costs nothing; a new model's +// price then reaches the proxy within the hour LiteLLM lists it. +const priceListInterval = time.Hour + +// priceListFile is where a local install keeps the last downloaded list, under ~/.cortex. +const priceListFile = "price-list.json" + +// livePricing keeps the rate table in step with two inputs that change independently: the +// config, which the reloader commits, and LiteLLM's price list, which the downloader applies. +// Either one rebuilds the table from both, under one lock, so neither can undo the other. +type livePricing struct { + reg *pricing.Registry + + mu sync.Mutex + committed bool // a config has been accepted; until then a list only waits + cfg *pricing.Config + list *pricing.List // nil until one is downloaded; the shipped table is used meanwhile +} + +// build prepares a table for a config the reloader has not accepted yet. It applies nothing, +// so a config that is then refused never prices traffic. +func (l *livePricing) build(cfg *pricing.Config) (*pricing.Table, error) { + l.mu.Lock() + list := l.list + l.mu.Unlock() + return pricing.BuildWithList(cfg, list) +} + +// Swap applies an accepted config. prepared is the table build made for it, and is what +// goes live unless a list arrived since: then it is rebuilt with that list, or the reload +// would put an older one back. +// +// Named Swap because that is what the reload closure must do, and what +// TestEveryBinaryInjectsPricing looks for inside it: a binary whose reload never swaps the +// table keeps its boot-time rates forever. +func (l *livePricing) Swap(cfg *pricing.Config, prepared *pricing.Table) { + l.mu.Lock() + defer l.mu.Unlock() + l.committed, l.cfg = true, cfg + tab := prepared + if fresh, err := pricing.BuildWithList(cfg, l.list); err == nil { + tab = fresh + } + l.reg.Swap(tab) +} + +// setList applies a downloaded list to the accepted config. +func (l *livePricing) setList(list *pricing.List) { + l.mu.Lock() + defer l.mu.Unlock() + if !l.committed { + l.list = list + return + } + tab, err := pricing.BuildWithList(l.cfg, list) + if err != nil { + slog.Warn("pricelist: the downloaded price list does not combine with the config; keeping the current prices", + "error", err) + return + } + l.list = list + l.reg.Swap(tab) +} + +// runPriceList keeps a local install's list prices current until ctx ends. Not run outside a +// local install: a sidecar makes no outbound call it was not configured to make, and keeps the +// shipped table. +func runPriceList(ctx context.Context, l *livePricing) { + f := &pricelist.Fetcher{} + if dir, err := defaultCortexDir(); err == nil { + f.CacheFile = filepath.Join(dir, priceListFile) + } + f.Run(ctx, priceListInterval, l.setList) +} diff --git a/cmd/cortex/pricing_list_test.go b/cmd/cortex/pricing_list_test.go new file mode 100644 index 000000000..fd9edc276 --- /dev/null +++ b/cmd/cortex/pricing_list_test.go @@ -0,0 +1,99 @@ +package main + +import ( + "testing" + "time" + + "github.com/rossoctl/cortex/core/cost/pricing" +) + +// opus55 is a downloaded list pricing a model newer than any build's shipped table. +func opus55(input float64) *pricing.List { + all := [pricing.NumTiers]bool{} + for i := range all { + all[i] = true + } + var base [pricing.NumTiers]float64 + base[pricing.TierInput] = input / 1e6 + base[pricing.TierOutput] = 5 * input / 1e6 + base[pricing.TierCacheRead] = input / 20 / 1e6 + base[pricing.TierCacheWrite] = 1.25 * input / 1e6 + return &pricing.List{ + Entries: []pricing.Entry{{Host: "*", Model: "claude-opus-5-5", Prov: pricing.ProvBundled, Rates: pricing.Rates{Base: base, Set: all}}}, + FetchedAt: time.Date(2026, 10, 8, 21, 0, 0, 0, time.UTC), + } +} + +// override prices claude-sonnet-5 on api.anthropic.com, so a test can see the config survive. +func override() *pricing.Config { + return &pricing.Config{Endpoints: []pricing.EndpointConfig{{ + Hosts: []string{"api.anthropic.com"}, + Models: map[string]pricing.ModelConfig{"claude-sonnet-5": {TierRates: pricing.TierRates{InputCostPerMillion: 1}}}, + }}} +} + +func inputRate(t *testing.T, reg *pricing.Registry, model string) (float64, pricing.Provenance) { + t.Helper() + r, p := reg.Resolve("api.anthropic.com", model, 0) + return r.Base[pricing.TierInput] * 1e6, p +} + +func near(a, b float64) bool { return a-b < 1e-9 && b-a < 1e-9 } + +// commitConfig does what a reload does: prepare a table, then swap it in. +func commitConfig(t *testing.T, l *livePricing, cfg *pricing.Config) { + t.Helper() + tab, err := l.build(cfg) + if err != nil { + t.Fatalf("build: %v", err) + } + l.Swap(cfg, tab) +} + +func TestLivePricing_ADownloadedListIsAppliedWithTheConfig(t *testing.T) { + l := &livePricing{reg: pricing.NewRegistry(nil)} + commitConfig(t, l, override()) + l.setList(opus55(4)) + + if got, p := inputRate(t, l.reg, "claude-opus-5-5"); p != pricing.ProvBundled || !near(got, 4) { + t.Errorf("claude-opus-5-5 = %v (%s), want 4 from the downloaded list", got, p) + } + if got, p := inputRate(t, l.reg, "claude-sonnet-5"); p != pricing.ProvConfigured || !near(got, 1) { + t.Errorf("claude-sonnet-5 = %v (%s), want the configured 1: a download must not drop the config", got, p) + } +} + +func TestLivePricing_AReloadDoesNotPutBackAnOlderList(t *testing.T) { + l := &livePricing{reg: pricing.NewRegistry(nil)} + commitConfig(t, l, nil) + l.setList(opus55(4)) + + // A reload prepares its table, a download lands, then the reload commits. + prepared, err := l.build(override()) + if err != nil { + t.Fatalf("build: %v", err) + } + l.setList(opus55(3)) + l.Swap(override(), prepared) + + if got, _ := inputRate(t, l.reg, "claude-opus-5-5"); !near(got, 3) { + t.Errorf("claude-opus-5-5 = %v after the reload, want 3: the list that arrived during it", got) + } + if _, p := inputRate(t, l.reg, "claude-sonnet-5"); p != pricing.ProvConfigured { + t.Errorf("claude-sonnet-5 provenance = %s, want configured from the reloaded config", p) + } +} + +func TestLivePricing_AListBeforeTheFirstConfigWaitsForIt(t *testing.T) { + // Applied on its own, the list would be priced with no config at all — every + // configured rate missing until the first reload. + l := &livePricing{reg: pricing.NewRegistry(nil)} + l.setList(opus55(4)) + if _, p := inputRate(t, l.reg, "claude-opus-5-5"); p != pricing.ProvNone { + t.Errorf("provenance before any config = %s, want none: nothing applied yet", p) + } + commitConfig(t, l, override()) + if got, _ := inputRate(t, l.reg, "claude-opus-5-5"); !near(got, 4) { + t.Errorf("claude-opus-5-5 = %v after the first config, want 4 from the waiting list", got) + } +} diff --git a/core/cost/pricing/config.go b/core/cost/pricing/config.go index 5da8de26d..ee76b1ec3 100644 --- a/core/cost/pricing/config.go +++ b/core/cost/pricing/config.go @@ -211,28 +211,7 @@ func (c *Config) BundledEnabled() bool { // Configured rows are emitted at ProvConfigured so they outrank anything bundled // at equal specificity: an override is an override. func Build(cfg *Config) (*Table, error) { - var entries []Entry - var mults []MultiplierRule - if cfg.BundledEnabled() { - entries = append(entries, Bundled()...) - // So do the free models' zero rates, kept out of Bundled() because that is the - // generated table its golden test pins to the LiteLLM snapshot. - entries = append(entries, bundledFreeRates()...) - // Shipped gateway discounts travel with the shipped rates: the rates are - // vendor list, and for the gateways named here list is a third too high. - // Disabling the bundled table disables both, which is the right pairing — - // a multiplier on rates you did not ship scales somebody else's numbers. - mults = append(mults, bundledMultipliers()...) - } - if cfg != nil { - configured, err := cfg.entries() - if err != nil { - return nil, err - } - entries = append(entries, configured...) - mults = append(mults, cfg.multipliers()...) - } - return NewTable(entries, mults...) + return BuildWithList(cfg, nil) } // multipliers converts the config's endpoint blocks into multiplier rules. diff --git a/core/cost/pricing/describe.go b/core/cost/pricing/describe.go index e6e41808a..fcf5755dd 100644 --- a/core/cost/pricing/describe.go +++ b/core/cost/pricing/describe.go @@ -3,6 +3,7 @@ package pricing import ( "sort" "strings" + "time" ) // This file exists because a config file cannot answer the question operators @@ -69,6 +70,7 @@ type Description struct { // UpstreamCommit is the LiteLLM commit the bundled rates were generated from, so a // figure can be traced to its source without reading the binary. UpstreamCommit string `json:"upstreamCommit,omitempty"` + ListFetchedAt time.Time `json:"listFetchedAt,omitzero"` Rows []RowView `json:"rows"` Multipliers []MultiplierView `json:"multipliers,omitempty"` } @@ -136,6 +138,7 @@ func (t *Table) Describe() Description { if t == nil { return out } + out.ListFetchedAt = t.listFetchedAt for i := range t.rows { r := &t.rows[i] row := RowView{ diff --git a/core/cost/pricing/internal/pricegen/pricegen.go b/core/cost/pricing/internal/pricegen/pricegen.go index db1d397c1..f5837ccf7 100644 --- a/core/cost/pricing/internal/pricegen/pricegen.go +++ b/core/cost/pricing/internal/pricegen/pricegen.go @@ -1,10 +1,11 @@ // Package pricegen turns LiteLLM's public model_prices_and_context_window.json // into pricing.Entry rows and renders them as Go source. // -// It lives in internal/ because it is build tooling, not runtime code, and it is -// a library rather than living inside the generator command so the golden test can -// run the exact same transform against a committed snapshot. A generator whose -// transform only exists inside a main package cannot be tested without a network. +// It is a library rather than living inside the generator command so the golden test +// can run the exact same transform against a committed snapshot — a generator whose +// transform only exists inside a main package cannot be tested without a network — and +// so pricelist can run it on a list downloaded at runtime. It stays in internal/ because +// only those two callers should depend on its output shape. package pricegen import ( diff --git a/core/cost/pricing/list.go b/core/cost/pricing/list.go new file mode 100644 index 000000000..94bd2aa17 --- /dev/null +++ b/core/cost/pricing/list.go @@ -0,0 +1,76 @@ +package pricing + +import ( + "strings" + "time" +) + +// List is a set of vendor list prices obtained at runtime: LiteLLM's public price map, +// downloaded rather than compiled in. See core/cost/pricing/pricelist. +// +// It exists because the shipped table is frozen at build time, and a model released after +// the build is priced by its family row — the newest member the build knew of. When that +// member costs more, every request to the new model is overstated with nothing to say so: +// claude-opus-5-5 was charged at claude-opus-5's rates for nine days, 1.68x what the gateway +// billed, although LiteLLM had listed its price a week before the first request. +type List struct { + // Produced by the same transform as bundled.go, so the two are interchangeable row for row. + Entries []Entry + // FetchedAt is when the list was downloaded, reported by Describe. + FetchedAt time.Time +} + +// BuildWithList is Build with list's rows. +// +// Everything else Build ships still applies — the gateway discounts and the free rates are +// hand-maintained and the list carries neither — and so does `bundled: false`, which turns the +// list off with the rest. +// +// A nil list is the shipped table: Build(cfg) is BuildWithList(cfg, nil). +func BuildWithList(cfg *Config, list *List) (*Table, error) { + var entries []Entry + var mults []MultiplierRule + bundled := cfg.BundledEnabled() + if bundled { + listed := map[string]bool{} + if list != nil { + entries = append(entries, list.Entries...) + for _, e := range list.Entries { + listed[rowKey(e)] = true + } + } + for _, e := range Bundled() { + if !listed[rowKey(e)] { + entries = append(entries, e) + } + } + // So do the free models' zero rates, kept out of Bundled() because that is the + // generated table its golden test pins to the LiteLLM snapshot. + entries = append(entries, bundledFreeRates()...) + // Shipped gateway discounts travel with the shipped rates: the rates are + // vendor list, and for the gateways named here list is a third too high. + // Disabling the bundled table disables both, which is the right pairing — + // a multiplier on rates you did not ship scales somebody else's numbers. + mults = append(mults, bundledMultipliers()...) + } + if cfg != nil { + configured, err := cfg.entries() + if err != nil { + return nil, err + } + entries = append(entries, configured...) + mults = append(mults, cfg.multipliers()...) + } + tab, err := NewTable(entries, mults...) + if err != nil { + return nil, err + } + if bundled && list != nil { + tab.listFetchedAt = list.FetchedAt + } + return tab, nil +} + +func rowKey(e Entry) string { + return strings.ToLower(e.Host) + "\x00" + strings.ToLower(e.Model) +} diff --git a/core/cost/pricing/list_test.go b/core/cost/pricing/list_test.go new file mode 100644 index 000000000..ca86d56fb --- /dev/null +++ b/core/cost/pricing/list_test.go @@ -0,0 +1,157 @@ +package pricing + +import ( + "math" + "testing" + "time" +) + +// opus55List is a downloaded list naming one model the shipped table does not: Opus 5.5, which +// costs less than the Opus 5 the shipped table's family row would have charged for it. +func opus55List() *List { + return &List{ + Entries: []Entry{{Host: "*", Model: "claude-opus-5-5", Prov: ProvBundled, Rates: Rates{ + Base: [numTiers]float64{TierInput: 4e-06, TierCacheWrite: 5e-06, TierCacheRead: 2e-07, TierOutput: 2e-05}, + Set: [numTiers]bool{TierInput: true, TierCacheWrite: true, TierCacheRead: true, TierOutput: true}, + }}}, + FetchedAt: time.Date(2026, 10, 8, 21, 0, 0, 0, time.UTC), + } +} + +func near(a, b float64) bool { return math.Abs(a-b) < 1e-9 } + +func TestBuildWithList_PricesFromTheList(t *testing.T) { + tab, err := BuildWithList(nil, opus55List()) + if err != nil { + t.Fatalf("BuildWithList: %v", err) + } + r, p := tab.Resolve("api.anthropic.com", "claude-opus-5-5", 0) + if p != ProvBundled { + t.Fatalf("provenance = %s, want bundled", p) + } + if got := inputPerMillion(r); !near(got, 4.00) { + t.Errorf("input rate = %v, want 4.00 from the list, not the shipped table's Opus family row", got) + } +} + +// listWithOpusFamily is opus55List plus the family glob pricegen emits from its newest Opus. +func listWithOpusFamily() *List { + l := opus55List() + glob := l.Entries[0] + glob.Model = "*claude-*opus-*" + l.Entries = append(l.Entries, glob) + return l +} + +func TestBuildWithList_AModelTheListDoesNotNameKeepsItsShippedRate(t *testing.T) { + shipped, err := Build(nil) + if err != nil { + t.Fatalf("Build: %v", err) + } + tab, err := BuildWithList(nil, listWithOpusFamily()) + if err != nil { + t.Fatalf("BuildWithList: %v", err) + } + for _, model := range []string{ + "claude-opus-4-1", // a shipped exact row, which the list's family glob also matches + "claude-opus-5", + "claude-sonnet-5", // a shipped exact row nothing in the list matches + "claude-haiku-9", // a shipped family glob the list has no counterpart for + } { + want, wp := shipped.Resolve("api.anthropic.com", model, 0) + got, gp := tab.Resolve("api.anthropic.com", model, 0) + if gp != wp || !near(inputPerMillion(got), inputPerMillion(want)) { + t.Errorf("%s = %v (%s), want the shipped %v (%s)", model, inputPerMillion(got), gp, inputPerMillion(want), wp) + } + } +} + +func TestBuildWithList_TheListsRowWinsForAModelBothName(t *testing.T) { + l := listWithOpusFamily() + opus5 := l.Entries[0] + opus5.Model = "Claude-Opus-5" // the table matches model names case-insensitively + opus5.Rates.Base[TierInput] = 4.5e-06 + l.Entries = append(l.Entries, opus5) + tab, err := BuildWithList(nil, l) + if err != nil { + t.Fatalf("BuildWithList: %v", err) + } + for model, want := range map[string]float64{ + "claude-opus-5": 4.50, // the list's exact row + "claude-opus-9": 4.00, // the list's family glob + } { + if r, p := tab.Resolve("api.anthropic.com", model, 0); p != ProvBundled || !near(inputPerMillion(r), want) { + t.Errorf("%s = %v (%s), want %v from the list", model, inputPerMillion(r), p, want) + } + } +} + +func TestBuildWithList_KeepsTheShippedDiscountAndFreeRates(t *testing.T) { + tab, err := BuildWithList(nil, opus55List()) + if err != nil { + t.Fatalf("BuildWithList: %v", err) + } + r, _ := tab.Resolve("ete-litellm.ai-models.vpc-int.res.ibm.com", "claude-opus-5-5", 0) + if got := inputPerMillion(r); !near(got, 3.04) { + t.Errorf("gateway input rate = %v, want 3.04 (0.76 x the list's 4.00)", got) + } + if _, p := tab.Resolve("opencode.ai", "grok-code-free", 0); p != ProvBundled { + t.Errorf("Zen free model provenance = %s, want bundled: the list carries no free rates", p) + } +} + +func TestBuildWithList_ConfiguredOutranksTheList(t *testing.T) { + tab, err := BuildWithList(mustYAML(t, ` +endpoints: + - hosts: [api.anthropic.com] + models: + claude-opus-5-5: + input_cost_per_million: 1.00 +`), opus55List()) + if err != nil { + t.Fatalf("BuildWithList: %v", err) + } + r, p := tab.Resolve("api.anthropic.com", "claude-opus-5-5", 0) + if p != ProvConfigured || !near(inputPerMillion(r), 1.00) { + t.Errorf("got %s at %v, want configured at 1.00", p, inputPerMillion(r)) + } +} + +func TestBuildWithList_BundledFalseIgnoresTheList(t *testing.T) { + tab, err := BuildWithList(mustYAML(t, "bundled: false\n"), opus55List()) + if err != nil { + t.Fatalf("BuildWithList: %v", err) + } + if _, p := tab.Resolve("api.anthropic.com", "claude-opus-5-5", 0); p != ProvNone { + t.Errorf("provenance = %s, want none: bundled: false turns off every list price", p) + } +} + +func TestBuildWithList_NilListIsTheShippedTable(t *testing.T) { + tab, err := BuildWithList(nil, nil) + if err != nil { + t.Fatalf("BuildWithList: %v", err) + } + if _, p := tab.Resolve("api.anthropic.com", "claude-opus-5", 0); p != ProvBundled { + t.Errorf("provenance = %s, want bundled from the shipped table", p) + } + if d := tab.Describe(); d.UpstreamCommit != BundledUpstreamCommit || !d.ListFetchedAt.IsZero() { + t.Errorf("describe = commit %q fetched %v, want the shipped commit and no fetch time", d.UpstreamCommit, d.ListFetchedAt) + } +} + +func TestBuildWithList_DescribeNamesTheDownload(t *testing.T) { + // agentop pricing prints where the rates came from. + l := opus55List() + tab, err := BuildWithList(nil, l) + if err != nil { + t.Fatalf("BuildWithList: %v", err) + } + d := tab.Describe() + if d.UpstreamCommit != BundledUpstreamCommit { + t.Errorf("upstream commit = %q, want %q: the shipped rows the list does not name are still in the table", d.UpstreamCommit, BundledUpstreamCommit) + } + if !d.ListFetchedAt.Equal(l.FetchedAt) { + t.Errorf("fetched at = %v, want %v", d.ListFetchedAt, l.FetchedAt) + } +} diff --git a/core/cost/pricing/pricelist/pricelist.go b/core/cost/pricing/pricelist/pricelist.go new file mode 100644 index 000000000..0ec0465e8 --- /dev/null +++ b/core/cost/pricing/pricelist/pricelist.go @@ -0,0 +1,230 @@ +// Package pricelist keeps vendor list prices current by downloading LiteLLM's public price +// map while the proxy runs, rather than only when the binary is built. +// +// The shipped table (pricing.Bundled) is generated from the same map, so a downloaded list is +// the shipped table brought up to date: same transform, same rows, newer data. That is the +// whole fix for a model released after the build — LiteLLM listed claude-opus-5-5 on +// 2026-09-22, a week before its first request here, and the build in use priced it as +// claude-opus-5 for nine days. +package pricelist + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "log/slog" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/rossoctl/cortex/core/cost/pricing" + "github.com/rossoctl/cortex/core/cost/pricing/internal/pricegen" +) + +// DefaultURL is LiteLLM's price map, the file bundled.go is generated from. +const DefaultURL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" + +// maxBytes bounds one download. The map is 3.1 MB today; this leaves room for it to grow +// without letting a misbehaving server fill memory. +const maxBytes = 16 << 20 + +// Fetcher downloads the price map and keeps the last good copy on disk. +// +// Not safe for concurrent use. The proxy has one caller, Run. +type Fetcher struct { + // URL is where the price map is read from; DefaultURL when empty. + URL string + // CacheFile holds the last list that downloaded and parsed, so a restart prices from it + // before the network answers, or without the network at all. Empty keeps nothing on disk. + CacheFile string + // Client makes the request; one with a 30s timeout when nil. + Client *http.Client + // Now stamps FetchedAt; time.Now when nil. + Now func() time.Time + + // etag is the version of the list last applied. Sent as If-None-Match, so an unchanged + // list costs a 304 with no body — which is what makes checking every hour free. + etag string +} + +// cacheFile is CacheFile's contents. Prices is the map already reduced to the rows the +// table reads (pricegen.Filter): kilobytes, where the download is megabytes. +type cacheFile struct { + ETag string `json:"etag,omitempty"` + FetchedAt time.Time `json:"fetchedAt"` + Prices json.RawMessage `json:"prices"` +} + +// Cached returns the list saved by an earlier download, or nil when there is none. +// +// A file that cannot be read back is an error rather than nil, so the caller can say the +// saved copy was ignored; the next download replaces it either way. +func (f *Fetcher) Cached() (*pricing.List, error) { + if f.CacheFile == "" { + return nil, nil + } + b, err := os.ReadFile(f.CacheFile) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var c cacheFile + if err := json.Unmarshal(b, &c); err != nil { + return nil, fmt.Errorf("pricelist: %s: %w", f.CacheFile, err) + } + entries, err := pricegen.Entries(c.Prices) + if err != nil { + return nil, fmt.Errorf("pricelist: %s: %w", f.CacheFile, err) + } + if len(entries) == 0 { + return nil, fmt.Errorf("pricelist: %s has no %s row with a rate", f.CacheFile, pricegen.AnthropicProvider) + } + f.etag = c.ETag + return &pricing.List{Entries: entries, FetchedAt: c.FetchedAt}, nil +} + +// Fetch downloads the list if it changed since the last one applied. It returns nil and no +// error when it has not. +// +// A list that cannot be used — an error status, a body that is not the price map, one with +// no rows the table reads — is an error and leaves the saved copy alone, so a bad upstream +// day never replaces a working table. +func (f *Fetcher) Fetch(ctx context.Context) (*pricing.List, error) { + url := f.URL + if url == "" { + url = DefaultURL + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + if f.etag != "" { + req.Header.Set("If-None-Match", f.etag) + } + client := f.Client + if client == nil { + client = &http.Client{Timeout: 30 * time.Second} + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusNotModified: + return nil, nil + case http.StatusOK: + default: + return nil, fmt.Errorf("pricelist: %s answered %s", url, resp.Status) + } + raw, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1)) + if err != nil { + return nil, err + } + if len(raw) > maxBytes { + return nil, fmt.Errorf("pricelist: %s is larger than %d bytes", url, maxBytes) + } + prices, err := pricegen.Filter(raw, "") + if err != nil { + return nil, err + } + entries, err := pricegen.Entries(prices) + if err != nil { + return nil, err + } + if len(entries) == 0 { + return nil, fmt.Errorf("pricelist: %s has no %s row with a rate", url, pricegen.AnthropicProvider) + } + now := time.Now + if f.Now != nil { + now = f.Now + } + l := &pricing.List{Entries: entries, FetchedAt: now()} + etag := resp.Header.Get("ETag") + if err := f.save(cacheFile{ETag: etag, FetchedAt: l.FetchedAt, Prices: prices}); err != nil { + // The list is good and is applied; only the copy a restart would start from is + // stale. The ETag is not kept, so the next check downloads and tries to save again. + slog.Warn("pricelist: could not save the price list; a restart will start from the previous copy", + "file", f.CacheFile, "error", err) + return l, nil + } + f.etag = etag + return l, nil +} + +// save writes c to CacheFile through a rename, so a crash mid-write leaves the previous copy. +func (f *Fetcher) save(c cacheFile) error { + if f.CacheFile == "" { + return nil + } + b, err := json.Marshal(c) + if err != nil { + return err + } + dir := filepath.Dir(f.CacheFile) + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + tmp, err := os.CreateTemp(dir, ".price-list-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) // a no-op once renamed + if _, err := tmp.Write(b); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), f.CacheFile) +} + +// Run applies the saved list, then checks for a newer one now and every interval until ctx +// ends, applying each change. +// +// A failed check keeps whatever is applied and is logged once per run of failures: a laptop +// offline for an afternoon should not log a warning an hour. +func (f *Fetcher) Run(ctx context.Context, every time.Duration, apply func(*pricing.List)) { + if l, err := f.Cached(); err != nil { + slog.Warn("pricelist: ignoring the saved price list", "file", f.CacheFile, "error", err) + } else if l != nil { + apply(l) + } + tick := time.NewTicker(every) + defer tick.Stop() + failing := false + for { + l, err := f.Fetch(ctx) + switch { + case err != nil && ctx.Err() != nil: + return + case err != nil: + if !failing { + slog.Warn("pricelist: could not download the price list; prices stay as they are until it can", + "error", err) + } + failing = true + default: + if failing { + slog.Info("pricelist: the price list can be downloaded again") + } + failing = false + if l != nil { + apply(l) + slog.Info("pricelist: prices updated from the downloaded list", "models", len(l.Entries)) + } + } + select { + case <-ctx.Done(): + return + case <-tick.C: + } + } +} diff --git a/core/cost/pricing/pricelist/pricelist_test.go b/core/cost/pricing/pricelist/pricelist_test.go new file mode 100644 index 000000000..89d9b67d6 --- /dev/null +++ b/core/cost/pricing/pricelist/pricelist_test.go @@ -0,0 +1,268 @@ +package pricelist + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/rossoctl/cortex/core/cost/pricing" +) + +// priceMap is LiteLLM's shape in miniature: a non-model row, a first-party Anthropic model +// and the same model through Bedrock, which carries its own rates and must not be read. +const priceMap = `{ + "sample_spec": {"input_cost_per_token": 0}, + "claude-opus-5-5": {"litellm_provider": "anthropic", "input_cost_per_token": 4e-06, + "cache_creation_input_token_cost": 5e-06, "cache_read_input_token_cost": 2e-07, + "output_cost_per_token": 2e-05}, + "us.anthropic.claude-opus-5-5": {"litellm_provider": "bedrock_converse", + "input_cost_per_token": 4.4e-06, "output_cost_per_token": 2.2e-05} +}` + +var fetchTime = time.Date(2026, 10, 8, 21, 0, 0, 0, time.UTC) + +// server serves body under etag and answers 304 to a request carrying it. +type server struct { + mu sync.Mutex + body string + etag string + code int + calls int + sent []string // If-None-Match of each request +} + +func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + s.calls++ + s.sent = append(s.sent, r.Header.Get("If-None-Match")) + if s.code != 0 { + w.WriteHeader(s.code) + return + } + if s.etag != "" && r.Header.Get("If-None-Match") == s.etag { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("ETag", s.etag) + _, _ = w.Write([]byte(s.body)) +} + +func newFetcher(t *testing.T, s *server) (*Fetcher, string) { + t.Helper() + ts := httptest.NewServer(s) + t.Cleanup(ts.Close) + cache := filepath.Join(t.TempDir(), "price-list.json") + return &Fetcher{URL: ts.URL, CacheFile: cache, Now: func() time.Time { return fetchTime }}, cache +} + +func inputPerMillion(t *testing.T, l *pricing.List, model string) float64 { + t.Helper() + tab, err := pricing.BuildWithList(nil, l) + if err != nil { + t.Fatalf("BuildWithList: %v", err) + } + r, p := tab.Resolve("api.anthropic.com", model, 0) + if p == pricing.ProvNone { + t.Fatalf("%s is not priced by the downloaded list", model) + } + return r.Base[pricing.TierInput] * 1e6 +} + +func TestFetch_ReturnsTheAnthropicRates(t *testing.T) { + f, _ := newFetcher(t, &server{body: priceMap, etag: `"v1"`}) + l, err := f.Fetch(context.Background()) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if l == nil { + t.Fatal("Fetch returned no list for a 200") + } + if got := inputPerMillion(t, l, "claude-opus-5-5"); got < 3.999 || got > 4.001 { + t.Errorf("input rate = %v, want 4.00 from the first-party row, not Bedrock's 4.40", got) + } + if !l.FetchedAt.Equal(fetchTime) { + t.Errorf("FetchedAt = %v, want %v", l.FetchedAt, fetchTime) + } +} + +func TestFetch_AnUnchangedListIsNotDownloadedAgain(t *testing.T) { + s := &server{body: priceMap, etag: `"v1"`} + f, _ := newFetcher(t, s) + if _, err := f.Fetch(context.Background()); err != nil { + t.Fatalf("first Fetch: %v", err) + } + l, err := f.Fetch(context.Background()) + if err != nil { + t.Fatalf("second Fetch: %v", err) + } + if l != nil { + t.Error("second Fetch returned a list, want nil: the server said 304") + } + if s.sent[1] != `"v1"` { + t.Errorf("second request sent If-None-Match %q, want the ETag from the first", s.sent[1]) + } +} + +func TestFetch_RefusesAListItCannotUse(t *testing.T) { + for name, s := range map[string]*server{ + "server error": {code: http.StatusInternalServerError}, + "not json": {body: "rate limited"}, + "no anthropic row": {body: `{"gpt-9": {"litellm_provider": "openai", "input_cost_per_token": 1e-06}}`}, + "no rate": {body: rateless}, + "renamed rate": {body: `{"claude-opus-9": {"litellm_provider": "anthropic", "input_cost_per_token_v2": 4e-06}}`}, + "too large": {body: `{"x": "` + strings.Repeat("a", maxBytes) + `"}`}, + } { + t.Run(name, func(t *testing.T) { + f, cache := newFetcher(t, s) + l, err := f.Fetch(context.Background()) + if err == nil { + t.Fatalf("Fetch returned %v and no error", l) + } + if _, statErr := os.Stat(cache); !os.IsNotExist(statErr) { + t.Errorf("a refused list was written to the cache (stat: %v)", statErr) + } + }) + } +} + +// rateless is an Anthropic row that decodes but carries no rate the table reads. +const rateless = `{"claude-opus-9": {"litellm_provider": "anthropic", "input_cost_per_token": 0}}` + +func TestFetch_AListWithNoRateLeavesTheSavedCopyAlone(t *testing.T) { + s := &server{body: priceMap, etag: `"v1"`} + f, cache := newFetcher(t, s) + if _, err := f.Fetch(context.Background()); err != nil { + t.Fatalf("first Fetch: %v", err) + } + s.mu.Lock() + s.body, s.etag = rateless, `"v2"` + s.mu.Unlock() + if l, err := f.Fetch(context.Background()); err == nil { + t.Fatalf("Fetch returned %v and no error for a list with no rate", l) + } + if _, err := f.Fetch(context.Background()); err == nil { + t.Fatal("third Fetch returned no error") + } + if s.sent[2] != `"v1"` { + t.Errorf("third request sent If-None-Match %q, want %q: the refused list's ETag was kept", s.sent[2], `"v1"`) + } + l, err := (&Fetcher{CacheFile: cache}).Cached() + if err != nil || l == nil { + t.Fatalf("Cached = %v, %v; want the first download", l, err) + } + if got := inputPerMillion(t, l, "claude-opus-5-5"); got < 3.999 || got > 4.001 { + t.Errorf("cached input rate = %v, want 4.00 from the first download", got) + } +} + +func TestCached_ASavedListWithNoRateIsAnError(t *testing.T) { + cache := filepath.Join(t.TempDir(), "price-list.json") + saved := `{"etag": "\"v1\"", "fetchedAt": "2026-10-08T21:00:00Z", "prices": ` + rateless + `}` + if err := os.WriteFile(cache, []byte(saved), 0o600); err != nil { + t.Fatal(err) + } + s := &server{body: priceMap, etag: `"v1"`} + f, _ := newFetcher(t, s) + f.CacheFile = cache + if l, err := f.Cached(); err == nil { + t.Fatalf("Cached = %v and no error for a saved list with no rate", l) + } + l, err := f.Fetch(context.Background()) + if err != nil || l == nil { + t.Fatalf("Fetch = %v, %v; want a download: the refused copy's ETag must not answer 304", l, err) + } +} + +func TestCached_ARestartStartsFromTheLastDownload(t *testing.T) { + s := &server{body: priceMap, etag: `"v1"`} + f, cache := newFetcher(t, s) + if _, err := f.Fetch(context.Background()); err != nil { + t.Fatalf("Fetch: %v", err) + } + + restarted := &Fetcher{URL: f.URL, CacheFile: cache} + l, err := restarted.Cached() + if err != nil { + t.Fatalf("Cached: %v", err) + } + if l == nil { + t.Fatal("Cached returned no list after a successful download") + } + if got := inputPerMillion(t, l, "claude-opus-5-5"); got < 3.999 || got > 4.001 { + t.Errorf("cached input rate = %v, want 4.00", got) + } + if !l.FetchedAt.Equal(fetchTime) { + t.Errorf("cached FetchedAt = %v, want the original download's %v", l.FetchedAt, fetchTime) + } + // The saved ETag goes out with the first request, so a restart costs a 304, not a download. + if l, err := restarted.Fetch(context.Background()); err != nil || l != nil { + t.Errorf("Fetch after restart = %v, %v; want nil, nil (304)", l, err) + } +} + +func TestCached_NoFileIsNoList(t *testing.T) { + f := &Fetcher{CacheFile: filepath.Join(t.TempDir(), "absent.json")} + l, err := f.Cached() + if l != nil || err != nil { + t.Errorf("Cached = %v, %v; want nil, nil", l, err) + } +} + +func TestCached_ACorruptFileIsAnError(t *testing.T) { + cache := filepath.Join(t.TempDir(), "price-list.json") + if err := os.WriteFile(cache, []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + f := &Fetcher{CacheFile: cache} + if l, err := f.Cached(); err == nil { + t.Errorf("Cached = %v and no error for a corrupt file", l) + } +} + +func TestRun_AppliesTheCacheThenEachChange(t *testing.T) { + s := &server{body: priceMap, etag: `"v1"`} + f, cache := newFetcher(t, s) + if _, err := f.Fetch(context.Background()); err != nil { + t.Fatalf("seeding the cache: %v", err) + } + + // Upstream changes while the proxy is down. + s.mu.Lock() + s.body = strings.Replace(priceMap, `"input_cost_per_token": 4e-06`, `"input_cost_per_token": 3e-06`, 1) + s.etag = `"v2"` + s.mu.Unlock() + + var mu sync.Mutex + var applied []float64 + done := make(chan struct{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + restarted := &Fetcher{URL: f.URL, CacheFile: cache, Now: f.Now} + go func() { + restarted.Run(ctx, time.Hour, func(l *pricing.List) { + mu.Lock() + defer mu.Unlock() + applied = append(applied, inputPerMillion(t, l, "claude-opus-5-5")) + if len(applied) == 2 { + close(done) + } + }) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Run did not apply the cached list and then the changed one") + } + mu.Lock() + defer mu.Unlock() + if applied[0] < 3.999 || applied[0] > 4.001 || applied[1] < 2.999 || applied[1] > 3.001 { + t.Errorf("applied input rates = %v, want [4 3]: the cached list first, then the change", applied) + } +} diff --git a/core/cost/pricing/table.go b/core/cost/pricing/table.go index 2fed452a9..5af7448fe 100644 --- a/core/cost/pricing/table.go +++ b/core/cost/pricing/table.go @@ -3,6 +3,7 @@ package pricing import ( "fmt" "strings" + "time" "github.com/gobwas/glob" ) @@ -38,6 +39,8 @@ type Table struct { // pair: one rule scales every model that gateway serves, including ones no row // names explicitly. mults []multRule + + listFetchedAt time.Time } type multRule struct { diff --git a/docs/pricing.md b/docs/pricing.md index 8db6e7786..86b824f8f 100644 --- a/docs/pricing.md +++ b/docs/pricing.md @@ -56,6 +56,15 @@ Generated, not hand-written: `core/cost/pricing/bundled.go` carries `model_prices_and_context_window.json`, pinned to one commit that `agentop pricing --json` reports as `upstreamCommit`. +**A local install does not stay on that commit.** It downloads the same file at startup and +then hourly. An unchanged file answers `304` with no body, so checking costs nothing. The +last good download is kept in `~/.cortex/price-list.json`, so a restart, or a laptop with no +network, prices from it. A download that fails, or is not a usable price map, changes +nothing. `agentop pricing` then reports `rates from litellm's price list, downloaded