From b8d0f8f30745d36f59cd08f90baed5c4383a7391 Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Thu, 8 Oct 2026 22:26:48 -0400 Subject: [PATCH 1/2] feat: Download LiteLLM's price list at runtime on a local install The shipped price table is generated from LiteLLM's price map at build time, so a model released after the build is priced by its family row. claude-opus-5-5 was charged at claude-opus-5's rates for nine days, 1.68x what the gateway billed, although LiteLLM had listed it a week before the first request. A local install now downloads the same file at startup and hourly, runs it through the same pricegen transform, and swaps it in with the config. An unchanged file answers 304 with no body. The last good copy is kept in ~/.cortex/price-list.json for restarts and offline starts, and a failed or unusable download changes nothing. Sidecars keep the compiled-in table and make no new outbound call. Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- cmd/agentop/cmd_pricing.go | 18 +- cmd/agentop/cmd_pricing_test.go | 33 +++ cmd/cortex/main.go | 15 +- cmd/cortex/pricing_list.go | 88 +++++++ cmd/cortex/pricing_list_test.go | 99 ++++++++ core/cost/pricing/config.go | 23 +- core/cost/pricing/describe.go | 8 +- .../pricing/internal/pricegen/pricegen.go | 10 +- core/cost/pricing/list.go | 66 ++++++ core/cost/pricing/list_test.go | 119 ++++++++++ core/cost/pricing/pricelist/pricelist.go | 224 ++++++++++++++++++ core/cost/pricing/pricelist/pricelist_test.go | 218 +++++++++++++++++ core/cost/pricing/table.go | 4 + docs/pricing.md | 25 +- 14 files changed, 909 insertions(+), 41 deletions(-) create mode 100644 cmd/cortex/pricing_list.go create mode 100644 cmd/cortex/pricing_list_test.go create mode 100644 core/cost/pricing/list.go create mode 100644 core/cost/pricing/list_test.go create mode 100644 core/cost/pricing/pricelist/pricelist.go create mode 100644 core/cost/pricing/pricelist/pricelist_test.go diff --git a/cmd/agentop/cmd_pricing.go b/cmd/agentop/cmd_pricing.go index 159c0c3fc..c8325d9d0 100644 --- a/cmd/agentop/cmd_pricing.go +++ b/cmd/agentop/cmd_pricing.go @@ -20,7 +20,8 @@ import ( // // This exists because no config file can answer the question. `pricing:` shows what the // operator wrote; the figures a request is charged come from that PLUS a rate table -// compiled into the binary PLUS any shipped gateway discount. Storing the table in the +// compiled into the binary, or on a local install downloaded from LiteLLM, PLUS any +// shipped gateway discount. Storing the table in the // config instead would freeze every install at the rates current on its install date, // silently, because that file is written once and never refreshed — which is the exact // staleness the pricing work was done to remove. @@ -42,9 +43,10 @@ Usage: agentop pricing --host what that endpoint is charged, discount applied agentop pricing --json raw JSON -The rates come from a table built into the binary (vendor list, refreshed per release), -your `+"`pricing:`"+` config, and any gateway discount Cortex ships. --host is the useful -form: it resolves all three the way a request would. +The rates come from a table built into the binary (vendor list), your `+"`pricing:`"+` +config, and any gateway discount Cortex ships. A local install replaces the built-in table +with LiteLLM's price list, downloaded hourly, so a new model is priced without a new +release. --host is the useful form: it resolves all of it the way a request would. Flags: `) @@ -221,7 +223,8 @@ type thresholdRow struct { } type describeBody struct { - UpstreamCommit string `json:"upstreamCommit"` + UpstreamCommit string `json:"upstreamCommit"` + ListFetchedAt time.Time `json:"listFetchedAt"` Rows []struct { Host string `json:"host"` Model string `json:"model"` @@ -247,7 +250,10 @@ func renderTable(body []byte, stdout, stderr io.Writer) int { return 1 } fmt.Fprintf(stdout, "Pricing table (%d rows)\n", len(d.Rows)) - if d.UpstreamCommit != "" { + switch { + case !d.ListFetchedAt.IsZero(): + fmt.Fprintf(stdout, " rates from litellm's price list, downloaded %s\n", d.ListFetchedAt.UTC().Format("2006-01-02 15:04 MST")) + case d.UpstreamCommit != "": fmt.Fprintf(stdout, " bundled rates generated from litellm %s\n", short(d.UpstreamCommit)) } fmt.Fprintf(stdout, "\n %-22s %-30s %9s %9s %9s %9s %s\n", diff --git a/cmd/agentop/cmd_pricing_test.go b/cmd/agentop/cmd_pricing_test.go index df674f4c0..6215ef247 100644 --- a/cmd/agentop/cmd_pricing_test.go +++ b/cmd/agentop/cmd_pricing_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" "github.com/rossoctl/cortex/core/cost/pricing" ) @@ -127,6 +128,38 @@ func TestRunPricing_TableViewListsRowsAndDiscounts(t *testing.T) { } } +// A local install prices from a downloaded list, and the header must say so: naming the +// shipped commit would send a reader to rates the table no longer holds. +func TestRunPricing_TableViewSaysWhenTheListWasDownloaded(t *testing.T) { + all := [pricing.NumTiers]bool{} + for i := range all { + all[i] = true + } + tab, err := pricing.BuildWithList(nil, &pricing.List{ + Entries: []pricing.Entry{{Host: "*", Model: "claude-opus-5-5", Prov: pricing.ProvBundled, + Rates: pricing.Rates{Base: [pricing.NumTiers]float64{pricing.TierInput: 4e-06, pricing.TierCacheWrite: 5e-06, + pricing.TierCacheRead: 2e-07, pricing.TierOutput: 2e-05}, Set: all}}}, + FetchedAt: time.Date(2026, 10, 8, 21, 0, 0, 0, time.UTC), + }) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(pricing.NewRegistry(tab).Handler()) + t.Cleanup(srv.Close) + + var out, errb bytes.Buffer + if code := runPricing([]string{"--stats-url", srv.URL}, &out, &errb); code != 0 { + t.Fatalf("exit %d: %s", code, errb.String()) + } + got := out.String() + if !strings.Contains(got, "rates from litellm's price list, downloaded 2026-10-08 21:00 UTC") { + t.Errorf("output does not say when the list was downloaded:\n%s", got) + } + if strings.Contains(got, "generated from litellm") { + t.Errorf("output names the shipped commit for a downloaded list:\n%s", got) + } +} + // A tier with no rate must render "-", never 0.00: pricing.Cost refuses to price a // request that used such a tier, so a zero would misrepresent a coverage gap as free. func TestRunPricing_UnsetTierRendersAsAbsent(t *testing.T) { diff --git a/cmd/cortex/main.go b/cmd/cortex/main.go index 0579d821d..43133ee1d 100644 --- a/cmd/cortex/main.go +++ b/cmd/cortex/main.go @@ -466,6 +466,9 @@ func main() { // disagree with the plugins about what a request cost. The table is swapped in // place instead; the pointer never changes. See pricing.Registry. pricingRegistry := pricing.NewRegistry(nil) + // Every table goes into the registry through live, which also applies a downloaded price + // list on a local install (runPriceList) without undoing a config reload, or the reverse. + live := &livePricing{reg: pricingRegistry} // This binary is hardcoded to proxy-sidecar. Rejecting other modes // early gives operators a clear boot-time error instead of silently @@ -494,7 +497,7 @@ func main() { // built so a plugin's Configure sees the new table, and swapped in place so // the usage aggregator — which holds this same registry from before the // reload — sees it too. - tab, err := pricing.Build(c.Pricing) + tab, err := live.build(c.Pricing) if err != nil { return nil, nil, nil, fmt.Errorf("pricing: %w", err) } @@ -533,10 +536,10 @@ func main() { // applyPricing puts a prepared table into effect. Called on the reload goroutine, which is // serialised, so the single pending slot needs no lock. applyPricing := func(c *config.Config) { - if pendingPricing != nil { - pricingRegistry.Swap(pendingPricing) - pendingPricing = nil + if pendingPricing != nil && c != nil { + live.Swap(c.Pricing, pendingPricing) } + pendingPricing = nil if c != nil { c.Pricing.WarnIfUnpinned(slog.Default()) } @@ -582,6 +585,10 @@ func main() { if err := rld.Start(ctx); err != nil { log.Fatalf("reloader: %v", err) } + // After the first config is applied, so the list is combined with it from the start. + if localInstall && cfg.Pricing.BundledEnabled() { + go runPriceList(ctx, live) + } var sessions *session.Store var usageAgg *usage.Aggregator diff --git a/cmd/cortex/pricing_list.go b/cmd/cortex/pricing_list.go new file mode 100644 index 000000000..f198da6f1 --- /dev/null +++ b/cmd/cortex/pricing_list.go @@ -0,0 +1,88 @@ +package main + +import ( + "context" + "log/slog" + "path/filepath" + "sync" + "time" + + "github.com/rossoctl/cortex/core/cost/pricing" + "github.com/rossoctl/cortex/core/cost/pricing/pricelist" +) + +// priceListInterval is how often a local install asks whether LiteLLM's price list changed. +// An unchanged list answers 304 with no body, so checking hourly costs nothing; a new model's +// price then reaches the proxy within the hour LiteLLM lists it. +const priceListInterval = time.Hour + +// priceListFile is where a local install keeps the last downloaded list, under ~/.cortex. +const priceListFile = "price-list.json" + +// livePricing keeps the rate table in step with two inputs that change independently: the +// config, which the reloader commits, and LiteLLM's price list, which the downloader applies. +// Either one rebuilds the table from both, under one lock, so neither can undo the other. +type livePricing struct { + reg *pricing.Registry + + mu sync.Mutex + committed bool // a config has been accepted; until then a list only waits + cfg *pricing.Config + list *pricing.List // nil until one is downloaded; the shipped table is used meanwhile +} + +// build prepares a table for a config the reloader has not accepted yet. It applies nothing, +// so a config that is then refused never prices traffic. +func (l *livePricing) build(cfg *pricing.Config) (*pricing.Table, error) { + l.mu.Lock() + list := l.list + l.mu.Unlock() + return pricing.BuildWithList(cfg, list) +} + +// Swap applies an accepted config. prepared is the table build made for it, and is what +// goes live unless a list arrived since: then it is rebuilt with that list, or the reload +// would put an older one back. +// +// Named Swap because that is what the reload closure must do, and what +// TestEveryBinaryInjectsPricing looks for inside it: a binary whose reload never swaps the +// table keeps its boot-time rates forever. +func (l *livePricing) Swap(cfg *pricing.Config, prepared *pricing.Table) { + l.mu.Lock() + defer l.mu.Unlock() + l.committed, l.cfg = true, cfg + tab := prepared + if fresh, err := pricing.BuildWithList(cfg, l.list); err == nil { + tab = fresh + } + l.reg.Swap(tab) +} + +// setList applies a downloaded list to the accepted config. +func (l *livePricing) setList(list *pricing.List) { + l.mu.Lock() + defer l.mu.Unlock() + if !l.committed { + l.list = list + return + } + tab, err := pricing.BuildWithList(l.cfg, list) + if err != nil { + slog.Warn("pricelist: the downloaded price list does not combine with the config; keeping the current prices", + "error", err) + return + } + l.list = list + l.reg.Swap(tab) +} + +// runPriceList keeps a local install's list prices current until ctx ends. Not run outside a +// local install: a sidecar makes no outbound call it was not configured to make, and keeps the +// shipped table. +func runPriceList(ctx context.Context, l *livePricing) { + f := &pricelist.Fetcher{} + if dir, err := defaultCortexDir(); err == nil { + f.CacheFile = filepath.Join(dir, priceListFile) + } + f.Run(ctx, priceListInterval, l.setList) +} diff --git a/cmd/cortex/pricing_list_test.go b/cmd/cortex/pricing_list_test.go new file mode 100644 index 000000000..fd9edc276 --- /dev/null +++ b/cmd/cortex/pricing_list_test.go @@ -0,0 +1,99 @@ +package main + +import ( + "testing" + "time" + + "github.com/rossoctl/cortex/core/cost/pricing" +) + +// opus55 is a downloaded list pricing a model newer than any build's shipped table. +func opus55(input float64) *pricing.List { + all := [pricing.NumTiers]bool{} + for i := range all { + all[i] = true + } + var base [pricing.NumTiers]float64 + base[pricing.TierInput] = input / 1e6 + base[pricing.TierOutput] = 5 * input / 1e6 + base[pricing.TierCacheRead] = input / 20 / 1e6 + base[pricing.TierCacheWrite] = 1.25 * input / 1e6 + return &pricing.List{ + Entries: []pricing.Entry{{Host: "*", Model: "claude-opus-5-5", Prov: pricing.ProvBundled, Rates: pricing.Rates{Base: base, Set: all}}}, + FetchedAt: time.Date(2026, 10, 8, 21, 0, 0, 0, time.UTC), + } +} + +// override prices claude-sonnet-5 on api.anthropic.com, so a test can see the config survive. +func override() *pricing.Config { + return &pricing.Config{Endpoints: []pricing.EndpointConfig{{ + Hosts: []string{"api.anthropic.com"}, + Models: map[string]pricing.ModelConfig{"claude-sonnet-5": {TierRates: pricing.TierRates{InputCostPerMillion: 1}}}, + }}} +} + +func inputRate(t *testing.T, reg *pricing.Registry, model string) (float64, pricing.Provenance) { + t.Helper() + r, p := reg.Resolve("api.anthropic.com", model, 0) + return r.Base[pricing.TierInput] * 1e6, p +} + +func near(a, b float64) bool { return a-b < 1e-9 && b-a < 1e-9 } + +// commitConfig does what a reload does: prepare a table, then swap it in. +func commitConfig(t *testing.T, l *livePricing, cfg *pricing.Config) { + t.Helper() + tab, err := l.build(cfg) + if err != nil { + t.Fatalf("build: %v", err) + } + l.Swap(cfg, tab) +} + +func TestLivePricing_ADownloadedListIsAppliedWithTheConfig(t *testing.T) { + l := &livePricing{reg: pricing.NewRegistry(nil)} + commitConfig(t, l, override()) + l.setList(opus55(4)) + + if got, p := inputRate(t, l.reg, "claude-opus-5-5"); p != pricing.ProvBundled || !near(got, 4) { + t.Errorf("claude-opus-5-5 = %v (%s), want 4 from the downloaded list", got, p) + } + if got, p := inputRate(t, l.reg, "claude-sonnet-5"); p != pricing.ProvConfigured || !near(got, 1) { + t.Errorf("claude-sonnet-5 = %v (%s), want the configured 1: a download must not drop the config", got, p) + } +} + +func TestLivePricing_AReloadDoesNotPutBackAnOlderList(t *testing.T) { + l := &livePricing{reg: pricing.NewRegistry(nil)} + commitConfig(t, l, nil) + l.setList(opus55(4)) + + // A reload prepares its table, a download lands, then the reload commits. + prepared, err := l.build(override()) + if err != nil { + t.Fatalf("build: %v", err) + } + l.setList(opus55(3)) + l.Swap(override(), prepared) + + if got, _ := inputRate(t, l.reg, "claude-opus-5-5"); !near(got, 3) { + t.Errorf("claude-opus-5-5 = %v after the reload, want 3: the list that arrived during it", got) + } + if _, p := inputRate(t, l.reg, "claude-sonnet-5"); p != pricing.ProvConfigured { + t.Errorf("claude-sonnet-5 provenance = %s, want configured from the reloaded config", p) + } +} + +func TestLivePricing_AListBeforeTheFirstConfigWaitsForIt(t *testing.T) { + // Applied on its own, the list would be priced with no config at all — every + // configured rate missing until the first reload. + l := &livePricing{reg: pricing.NewRegistry(nil)} + l.setList(opus55(4)) + if _, p := inputRate(t, l.reg, "claude-opus-5-5"); p != pricing.ProvNone { + t.Errorf("provenance before any config = %s, want none: nothing applied yet", p) + } + commitConfig(t, l, override()) + if got, _ := inputRate(t, l.reg, "claude-opus-5-5"); !near(got, 4) { + t.Errorf("claude-opus-5-5 = %v after the first config, want 4 from the waiting list", got) + } +} diff --git a/core/cost/pricing/config.go b/core/cost/pricing/config.go index 5da8de26d..ee76b1ec3 100644 --- a/core/cost/pricing/config.go +++ b/core/cost/pricing/config.go @@ -211,28 +211,7 @@ func (c *Config) BundledEnabled() bool { // Configured rows are emitted at ProvConfigured so they outrank anything bundled // at equal specificity: an override is an override. func Build(cfg *Config) (*Table, error) { - var entries []Entry - var mults []MultiplierRule - if cfg.BundledEnabled() { - entries = append(entries, Bundled()...) - // So do the free models' zero rates, kept out of Bundled() because that is the - // generated table its golden test pins to the LiteLLM snapshot. - entries = append(entries, bundledFreeRates()...) - // Shipped gateway discounts travel with the shipped rates: the rates are - // vendor list, and for the gateways named here list is a third too high. - // Disabling the bundled table disables both, which is the right pairing — - // a multiplier on rates you did not ship scales somebody else's numbers. - mults = append(mults, bundledMultipliers()...) - } - if cfg != nil { - configured, err := cfg.entries() - if err != nil { - return nil, err - } - entries = append(entries, configured...) - mults = append(mults, cfg.multipliers()...) - } - return NewTable(entries, mults...) + return BuildWithList(cfg, nil) } // multipliers converts the config's endpoint blocks into multiplier rules. diff --git a/core/cost/pricing/describe.go b/core/cost/pricing/describe.go index e6e41808a..b77936fc2 100644 --- a/core/cost/pricing/describe.go +++ b/core/cost/pricing/describe.go @@ -3,6 +3,7 @@ package pricing import ( "sort" "strings" + "time" ) // This file exists because a config file cannot answer the question operators @@ -67,8 +68,10 @@ type MultiplierView struct { // Description is the whole table, unresolved. type Description struct { // UpstreamCommit is the LiteLLM commit the bundled rates were generated from, so a - // figure can be traced to its source without reading the binary. + // figure can be traced to its source without reading the binary. Empty when the rates + // came from a downloaded list instead; ListFetchedAt says when that was. UpstreamCommit string `json:"upstreamCommit,omitempty"` + ListFetchedAt time.Time `json:"listFetchedAt,omitzero"` Rows []RowView `json:"rows"` Multipliers []MultiplierView `json:"multipliers,omitempty"` } @@ -136,6 +139,9 @@ func (t *Table) Describe() Description { if t == nil { return out } + if !t.listFetchedAt.IsZero() { + out.UpstreamCommit, out.ListFetchedAt = "", t.listFetchedAt + } for i := range t.rows { r := &t.rows[i] row := RowView{ diff --git a/core/cost/pricing/internal/pricegen/pricegen.go b/core/cost/pricing/internal/pricegen/pricegen.go index db1d397c1..68143093c 100644 --- a/core/cost/pricing/internal/pricegen/pricegen.go +++ b/core/cost/pricing/internal/pricegen/pricegen.go @@ -1,10 +1,12 @@ // Package pricegen turns LiteLLM's public model_prices_and_context_window.json // into pricing.Entry rows and renders them as Go source. // -// It lives in internal/ because it is build tooling, not runtime code, and it is -// a library rather than living inside the generator command so the golden test can -// run the exact same transform against a committed snapshot. A generator whose -// transform only exists inside a main package cannot be tested without a network. +// It is a library rather than living inside the generator command so the golden test +// can run the exact same transform against a committed snapshot — a generator whose +// transform only exists inside a main package cannot be tested without a network — and +// so pricelist can run it on a list downloaded at runtime. That shared transform is what +// makes a downloaded list a drop-in replacement for the generated table. It stays in +// internal/ because only those two callers should depend on its output shape. package pricegen import ( diff --git a/core/cost/pricing/list.go b/core/cost/pricing/list.go new file mode 100644 index 000000000..ef3aa6efa --- /dev/null +++ b/core/cost/pricing/list.go @@ -0,0 +1,66 @@ +package pricing + +import "time" + +// List is a set of vendor list prices obtained at runtime: LiteLLM's public price map, +// downloaded rather than compiled in. See core/cost/pricing/pricelist. +// +// It exists because the shipped table is frozen at build time, and a model released after +// the build is priced by its family row — the newest member the build knew of. When that +// member costs more, every request to the new model is overstated with nothing to say so: +// claude-opus-5-5 was charged at claude-opus-5's rates for nine days, 1.68x what the gateway +// billed, although LiteLLM had listed its price a week before the first request. +type List struct { + // Entries replace the shipped table's model rows. Produced by the same transform as + // bundled.go, so the two are interchangeable row for row. + Entries []Entry + // FetchedAt is when the list was downloaded, reported by Describe in place of the + // shipped table's upstream commit. + FetchedAt time.Time +} + +// BuildWithList is Build with list's rows in place of the shipped table's. +// +// REPLACED, NOT MERGED: the list is the newer table and stands alone. Merged, the shipped +// family rows would sit beside the list's at the same provenance, and a model the list does +// not name could be priced from a build months old. Everything else Build ships still +// applies — the gateway discounts and the free rates are hand-maintained and the list +// carries neither — and so does `bundled: false`, which turns the list off with the rest. +// +// A nil list is the shipped table: Build(cfg) is BuildWithList(cfg, nil). +func BuildWithList(cfg *Config, list *List) (*Table, error) { + var entries []Entry + var mults []MultiplierRule + bundled := cfg.BundledEnabled() + if bundled { + if list != nil { + entries = append(entries, list.Entries...) + } else { + entries = append(entries, Bundled()...) + } + // So do the free models' zero rates, kept out of Bundled() because that is the + // generated table its golden test pins to the LiteLLM snapshot. + entries = append(entries, bundledFreeRates()...) + // Shipped gateway discounts travel with the shipped rates: the rates are + // vendor list, and for the gateways named here list is a third too high. + // Disabling the bundled table disables both, which is the right pairing — + // a multiplier on rates you did not ship scales somebody else's numbers. + mults = append(mults, bundledMultipliers()...) + } + if cfg != nil { + configured, err := cfg.entries() + if err != nil { + return nil, err + } + entries = append(entries, configured...) + mults = append(mults, cfg.multipliers()...) + } + tab, err := NewTable(entries, mults...) + if err != nil { + return nil, err + } + if bundled && list != nil { + tab.listFetchedAt = list.FetchedAt + } + return tab, nil +} diff --git a/core/cost/pricing/list_test.go b/core/cost/pricing/list_test.go new file mode 100644 index 000000000..c02c75a86 --- /dev/null +++ b/core/cost/pricing/list_test.go @@ -0,0 +1,119 @@ +package pricing + +import ( + "math" + "testing" + "time" +) + +// opus55List is a downloaded list naming one model the shipped table does not: Opus 5.5, which +// costs less than the Opus 5 the shipped table's family row would have charged for it. +func opus55List() *List { + return &List{ + Entries: []Entry{{Host: "*", Model: "claude-opus-5-5", Prov: ProvBundled, Rates: Rates{ + Base: [numTiers]float64{TierInput: 4e-06, TierCacheWrite: 5e-06, TierCacheRead: 2e-07, TierOutput: 2e-05}, + Set: [numTiers]bool{TierInput: true, TierCacheWrite: true, TierCacheRead: true, TierOutput: true}, + }}}, + FetchedAt: time.Date(2026, 10, 8, 21, 0, 0, 0, time.UTC), + } +} + +func near(a, b float64) bool { return math.Abs(a-b) < 1e-9 } + +func TestBuildWithList_PricesFromTheList(t *testing.T) { + tab, err := BuildWithList(nil, opus55List()) + if err != nil { + t.Fatalf("BuildWithList: %v", err) + } + r, p := tab.Resolve("api.anthropic.com", "claude-opus-5-5", 0) + if p != ProvBundled { + t.Fatalf("provenance = %s, want bundled", p) + } + if got := inputPerMillion(r); !near(got, 4.00) { + t.Errorf("input rate = %v, want 4.00 from the list, not the shipped table's Opus family row", got) + } +} + +func TestBuildWithList_ReplacesTheShippedModelRows(t *testing.T) { + // The list is the newer table, so it stands alone. Merged, the shipped table's family rows + // would sit beside the list's at the same provenance and price an unlisted model from a + // build that may be months old. + tab, err := BuildWithList(nil, opus55List()) + if err != nil { + t.Fatalf("BuildWithList: %v", err) + } + if _, p := tab.Resolve("api.anthropic.com", "claude-opus-5", 0); p != ProvNone { + t.Errorf("a model only the shipped table names resolved %s, want none", p) + } +} + +func TestBuildWithList_KeepsTheShippedDiscountAndFreeRates(t *testing.T) { + tab, err := BuildWithList(nil, opus55List()) + if err != nil { + t.Fatalf("BuildWithList: %v", err) + } + r, _ := tab.Resolve("ete-litellm.ai-models.vpc-int.res.ibm.com", "claude-opus-5-5", 0) + if got := inputPerMillion(r); !near(got, 3.04) { + t.Errorf("gateway input rate = %v, want 3.04 (0.76 x the list's 4.00)", got) + } + if _, p := tab.Resolve("opencode.ai", "grok-code-free", 0); p != ProvBundled { + t.Errorf("Zen free model provenance = %s, want bundled: the list carries no free rates", p) + } +} + +func TestBuildWithList_ConfiguredOutranksTheList(t *testing.T) { + tab, err := BuildWithList(mustYAML(t, ` +endpoints: + - hosts: [api.anthropic.com] + models: + claude-opus-5-5: + input_cost_per_million: 1.00 +`), opus55List()) + if err != nil { + t.Fatalf("BuildWithList: %v", err) + } + r, p := tab.Resolve("api.anthropic.com", "claude-opus-5-5", 0) + if p != ProvConfigured || !near(inputPerMillion(r), 1.00) { + t.Errorf("got %s at %v, want configured at 1.00", p, inputPerMillion(r)) + } +} + +func TestBuildWithList_BundledFalseIgnoresTheList(t *testing.T) { + tab, err := BuildWithList(mustYAML(t, "bundled: false\n"), opus55List()) + if err != nil { + t.Fatalf("BuildWithList: %v", err) + } + if _, p := tab.Resolve("api.anthropic.com", "claude-opus-5-5", 0); p != ProvNone { + t.Errorf("provenance = %s, want none: bundled: false turns off every list price", p) + } +} + +func TestBuildWithList_NilListIsTheShippedTable(t *testing.T) { + tab, err := BuildWithList(nil, nil) + if err != nil { + t.Fatalf("BuildWithList: %v", err) + } + if _, p := tab.Resolve("api.anthropic.com", "claude-opus-5", 0); p != ProvBundled { + t.Errorf("provenance = %s, want bundled from the shipped table", p) + } + if d := tab.Describe(); d.UpstreamCommit != BundledUpstreamCommit || !d.ListFetchedAt.IsZero() { + t.Errorf("describe = commit %q fetched %v, want the shipped commit and no fetch time", d.UpstreamCommit, d.ListFetchedAt) + } +} + +func TestBuildWithList_DescribeNamesTheDownload(t *testing.T) { + // agentop pricing prints where the rates came from. Naming the shipped commit for a table + // that no longer holds its rows would send a reader to the wrong source. + l := opus55List() + tab, err := BuildWithList(nil, l) + if err != nil { + t.Fatalf("BuildWithList: %v", err) + } + d := tab.Describe() + if d.UpstreamCommit != "" { + t.Errorf("upstream commit = %q, want empty for a downloaded list", d.UpstreamCommit) + } + if !d.ListFetchedAt.Equal(l.FetchedAt) { + t.Errorf("fetched at = %v, want %v", d.ListFetchedAt, l.FetchedAt) + } +} diff --git a/core/cost/pricing/pricelist/pricelist.go b/core/cost/pricing/pricelist/pricelist.go new file mode 100644 index 000000000..77793350e --- /dev/null +++ b/core/cost/pricing/pricelist/pricelist.go @@ -0,0 +1,224 @@ +// Package pricelist keeps vendor list prices current by downloading LiteLLM's public price +// map while the proxy runs, rather than only when the binary is built. +// +// The shipped table (pricing.Bundled) is generated from the same map, so a downloaded list is +// the shipped table brought up to date: same transform, same rows, newer data. That is the +// whole fix for a model released after the build — LiteLLM listed claude-opus-5-5 on +// 2026-09-22, a week before its first request here, and the build in use priced it as +// claude-opus-5 for nine days. +package pricelist + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "log/slog" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/rossoctl/cortex/core/cost/pricing" + "github.com/rossoctl/cortex/core/cost/pricing/internal/pricegen" +) + +// DefaultURL is LiteLLM's price map, the file bundled.go is generated from. +const DefaultURL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" + +// maxBytes bounds one download. The map is 3.1 MB today; this leaves room for it to grow +// without letting a misbehaving server fill memory. +const maxBytes = 16 << 20 + +// Fetcher downloads the price map and keeps the last good copy on disk. +// +// Not safe for concurrent use. The proxy has one caller, Run. +type Fetcher struct { + // URL is where the price map is read from; DefaultURL when empty. + URL string + // CacheFile holds the last list that downloaded and parsed, so a restart prices from it + // before the network answers, or without the network at all. Empty keeps nothing on disk. + CacheFile string + // Client makes the request; one with a 30s timeout when nil. + Client *http.Client + // Now stamps FetchedAt; time.Now when nil. + Now func() time.Time + + // etag is the version of the list last applied. Sent as If-None-Match, so an unchanged + // list costs a 304 with no body — which is what makes checking every hour free. + etag string +} + +// cacheFile is CacheFile's contents. Prices is the map already reduced to the rows the +// table reads (pricegen.Filter): kilobytes, where the download is megabytes. +type cacheFile struct { + ETag string `json:"etag,omitempty"` + FetchedAt time.Time `json:"fetchedAt"` + Prices json.RawMessage `json:"prices"` +} + +// Cached returns the list saved by an earlier download, or nil when there is none. +// +// A file that cannot be read back is an error rather than nil, so the caller can say the +// saved copy was ignored; the next download replaces it either way. +func (f *Fetcher) Cached() (*pricing.List, error) { + if f.CacheFile == "" { + return nil, nil + } + b, err := os.ReadFile(f.CacheFile) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var c cacheFile + if err := json.Unmarshal(b, &c); err != nil { + return nil, fmt.Errorf("pricelist: %s: %w", f.CacheFile, err) + } + entries, err := pricegen.Entries(c.Prices) + if err != nil { + return nil, fmt.Errorf("pricelist: %s: %w", f.CacheFile, err) + } + f.etag = c.ETag + return &pricing.List{Entries: entries, FetchedAt: c.FetchedAt}, nil +} + +// Fetch downloads the list if it changed since the last one applied. It returns nil and no +// error when it has not. +// +// A list that cannot be used — an error status, a body that is not the price map, one with +// no rows the table reads — is an error and leaves the saved copy alone, so a bad upstream +// day never replaces a working table. +func (f *Fetcher) Fetch(ctx context.Context) (*pricing.List, error) { + url := f.URL + if url == "" { + url = DefaultURL + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + if f.etag != "" { + req.Header.Set("If-None-Match", f.etag) + } + client := f.Client + if client == nil { + client = &http.Client{Timeout: 30 * time.Second} + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusNotModified: + return nil, nil + case http.StatusOK: + default: + return nil, fmt.Errorf("pricelist: %s answered %s", url, resp.Status) + } + raw, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1)) + if err != nil { + return nil, err + } + if len(raw) > maxBytes { + return nil, fmt.Errorf("pricelist: %s is larger than %d bytes", url, maxBytes) + } + prices, err := pricegen.Filter(raw, "") + if err != nil { + return nil, err + } + entries, err := pricegen.Entries(prices) + if err != nil { + return nil, err + } + now := time.Now + if f.Now != nil { + now = f.Now + } + l := &pricing.List{Entries: entries, FetchedAt: now()} + etag := resp.Header.Get("ETag") + if err := f.save(cacheFile{ETag: etag, FetchedAt: l.FetchedAt, Prices: prices}); err != nil { + // The list is good and is applied; only the copy a restart would start from is + // stale. The ETag is not kept, so the next check downloads and tries to save again. + slog.Warn("pricelist: could not save the price list; a restart will start from the previous copy", + "file", f.CacheFile, "error", err) + return l, nil + } + f.etag = etag + return l, nil +} + +// save writes c to CacheFile through a rename, so a crash mid-write leaves the previous copy. +func (f *Fetcher) save(c cacheFile) error { + if f.CacheFile == "" { + return nil + } + b, err := json.Marshal(c) + if err != nil { + return err + } + dir := filepath.Dir(f.CacheFile) + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + tmp, err := os.CreateTemp(dir, ".price-list-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) // a no-op once renamed + if _, err := tmp.Write(b); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), f.CacheFile) +} + +// Run applies the saved list, then checks for a newer one now and every interval until ctx +// ends, applying each change. +// +// A failed check keeps whatever is applied and is logged once per run of failures: a laptop +// offline for an afternoon should not log a warning an hour. +func (f *Fetcher) Run(ctx context.Context, every time.Duration, apply func(*pricing.List)) { + if l, err := f.Cached(); err != nil { + slog.Warn("pricelist: ignoring the saved price list", "file", f.CacheFile, "error", err) + } else if l != nil { + apply(l) + } + tick := time.NewTicker(every) + defer tick.Stop() + failing := false + for { + l, err := f.Fetch(ctx) + switch { + case err != nil && ctx.Err() != nil: + return + case err != nil: + if !failing { + slog.Warn("pricelist: could not download the price list; prices stay as they are until it can", + "error", err) + } + failing = true + default: + if failing { + slog.Info("pricelist: the price list can be downloaded again") + } + failing = false + if l != nil { + apply(l) + slog.Info("pricelist: prices updated from the downloaded list", "models", len(l.Entries)) + } + } + select { + case <-ctx.Done(): + return + case <-tick.C: + } + } +} diff --git a/core/cost/pricing/pricelist/pricelist_test.go b/core/cost/pricing/pricelist/pricelist_test.go new file mode 100644 index 000000000..d91dd391d --- /dev/null +++ b/core/cost/pricing/pricelist/pricelist_test.go @@ -0,0 +1,218 @@ +package pricelist + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/rossoctl/cortex/core/cost/pricing" +) + +// priceMap is LiteLLM's shape in miniature: a non-model row, a first-party Anthropic model +// and the same model through Bedrock, which carries its own rates and must not be read. +const priceMap = `{ + "sample_spec": {"input_cost_per_token": 0}, + "claude-opus-5-5": {"litellm_provider": "anthropic", "input_cost_per_token": 4e-06, + "cache_creation_input_token_cost": 5e-06, "cache_read_input_token_cost": 2e-07, + "output_cost_per_token": 2e-05}, + "us.anthropic.claude-opus-5-5": {"litellm_provider": "bedrock_converse", + "input_cost_per_token": 4.4e-06, "output_cost_per_token": 2.2e-05} +}` + +var fetchTime = time.Date(2026, 10, 8, 21, 0, 0, 0, time.UTC) + +// server serves body under etag and answers 304 to a request carrying it. +type server struct { + mu sync.Mutex + body string + etag string + code int + calls int + sent []string // If-None-Match of each request +} + +func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + s.calls++ + s.sent = append(s.sent, r.Header.Get("If-None-Match")) + if s.code != 0 { + w.WriteHeader(s.code) + return + } + if s.etag != "" && r.Header.Get("If-None-Match") == s.etag { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("ETag", s.etag) + _, _ = w.Write([]byte(s.body)) +} + +func newFetcher(t *testing.T, s *server) (*Fetcher, string) { + t.Helper() + ts := httptest.NewServer(s) + t.Cleanup(ts.Close) + cache := filepath.Join(t.TempDir(), "price-list.json") + return &Fetcher{URL: ts.URL, CacheFile: cache, Now: func() time.Time { return fetchTime }}, cache +} + +func inputPerMillion(t *testing.T, l *pricing.List, model string) float64 { + t.Helper() + tab, err := pricing.BuildWithList(nil, l) + if err != nil { + t.Fatalf("BuildWithList: %v", err) + } + r, p := tab.Resolve("api.anthropic.com", model, 0) + if p == pricing.ProvNone { + t.Fatalf("%s is not priced by the downloaded list", model) + } + return r.Base[pricing.TierInput] * 1e6 +} + +func TestFetch_ReturnsTheAnthropicRates(t *testing.T) { + f, _ := newFetcher(t, &server{body: priceMap, etag: `"v1"`}) + l, err := f.Fetch(context.Background()) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if l == nil { + t.Fatal("Fetch returned no list for a 200") + } + if got := inputPerMillion(t, l, "claude-opus-5-5"); got < 3.999 || got > 4.001 { + t.Errorf("input rate = %v, want 4.00 from the first-party row, not Bedrock's 4.40", got) + } + if !l.FetchedAt.Equal(fetchTime) { + t.Errorf("FetchedAt = %v, want %v", l.FetchedAt, fetchTime) + } +} + +func TestFetch_AnUnchangedListIsNotDownloadedAgain(t *testing.T) { + s := &server{body: priceMap, etag: `"v1"`} + f, _ := newFetcher(t, s) + if _, err := f.Fetch(context.Background()); err != nil { + t.Fatalf("first Fetch: %v", err) + } + l, err := f.Fetch(context.Background()) + if err != nil { + t.Fatalf("second Fetch: %v", err) + } + if l != nil { + t.Error("second Fetch returned a list, want nil: the server said 304") + } + if s.sent[1] != `"v1"` { + t.Errorf("second request sent If-None-Match %q, want the ETag from the first", s.sent[1]) + } +} + +func TestFetch_RefusesAListItCannotUse(t *testing.T) { + for name, s := range map[string]*server{ + "server error": {code: http.StatusInternalServerError}, + "not json": {body: "rate limited"}, + "no anthropic row": {body: `{"gpt-9": {"litellm_provider": "openai", "input_cost_per_token": 1e-06}}`}, + "too large": {body: `{"x": "` + strings.Repeat("a", maxBytes) + `"}`}, + } { + t.Run(name, func(t *testing.T) { + f, cache := newFetcher(t, s) + l, err := f.Fetch(context.Background()) + if err == nil { + t.Fatalf("Fetch returned %v and no error", l) + } + if _, statErr := os.Stat(cache); !os.IsNotExist(statErr) { + t.Errorf("a refused list was written to the cache (stat: %v)", statErr) + } + }) + } +} + +func TestCached_ARestartStartsFromTheLastDownload(t *testing.T) { + s := &server{body: priceMap, etag: `"v1"`} + f, cache := newFetcher(t, s) + if _, err := f.Fetch(context.Background()); err != nil { + t.Fatalf("Fetch: %v", err) + } + + restarted := &Fetcher{URL: f.URL, CacheFile: cache} + l, err := restarted.Cached() + if err != nil { + t.Fatalf("Cached: %v", err) + } + if l == nil { + t.Fatal("Cached returned no list after a successful download") + } + if got := inputPerMillion(t, l, "claude-opus-5-5"); got < 3.999 || got > 4.001 { + t.Errorf("cached input rate = %v, want 4.00", got) + } + if !l.FetchedAt.Equal(fetchTime) { + t.Errorf("cached FetchedAt = %v, want the original download's %v", l.FetchedAt, fetchTime) + } + // The saved ETag goes out with the first request, so a restart costs a 304, not a download. + if l, err := restarted.Fetch(context.Background()); err != nil || l != nil { + t.Errorf("Fetch after restart = %v, %v; want nil, nil (304)", l, err) + } +} + +func TestCached_NoFileIsNoList(t *testing.T) { + f := &Fetcher{CacheFile: filepath.Join(t.TempDir(), "absent.json")} + l, err := f.Cached() + if l != nil || err != nil { + t.Errorf("Cached = %v, %v; want nil, nil", l, err) + } +} + +func TestCached_ACorruptFileIsAnError(t *testing.T) { + cache := filepath.Join(t.TempDir(), "price-list.json") + if err := os.WriteFile(cache, []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + f := &Fetcher{CacheFile: cache} + if l, err := f.Cached(); err == nil { + t.Errorf("Cached = %v and no error for a corrupt file", l) + } +} + +func TestRun_AppliesTheCacheThenEachChange(t *testing.T) { + s := &server{body: priceMap, etag: `"v1"`} + f, cache := newFetcher(t, s) + if _, err := f.Fetch(context.Background()); err != nil { + t.Fatalf("seeding the cache: %v", err) + } + + // Upstream changes while the proxy is down. + s.mu.Lock() + s.body = strings.Replace(priceMap, `"input_cost_per_token": 4e-06`, `"input_cost_per_token": 3e-06`, 1) + s.etag = `"v2"` + s.mu.Unlock() + + var mu sync.Mutex + var applied []float64 + done := make(chan struct{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + restarted := &Fetcher{URL: f.URL, CacheFile: cache, Now: f.Now} + go func() { + restarted.Run(ctx, time.Hour, func(l *pricing.List) { + mu.Lock() + defer mu.Unlock() + applied = append(applied, inputPerMillion(t, l, "claude-opus-5-5")) + if len(applied) == 2 { + close(done) + } + }) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Run did not apply the cached list and then the changed one") + } + mu.Lock() + defer mu.Unlock() + if applied[0] < 3.999 || applied[0] > 4.001 || applied[1] < 2.999 || applied[1] > 3.001 { + t.Errorf("applied input rates = %v, want [4 3]: the cached list first, then the change", applied) + } +} diff --git a/core/cost/pricing/table.go b/core/cost/pricing/table.go index 2fed452a9..303231311 100644 --- a/core/cost/pricing/table.go +++ b/core/cost/pricing/table.go @@ -3,6 +3,7 @@ package pricing import ( "fmt" "strings" + "time" "github.com/gobwas/glob" ) @@ -38,6 +39,9 @@ type Table struct { // pair: one rule scales every model that gateway serves, including ones no row // names explicitly. mults []multRule + // listFetchedAt is set when the model rows came from a downloaded List rather than + // the shipped table; see BuildWithList. + listFetchedAt time.Time } type multRule struct { diff --git a/docs/pricing.md b/docs/pricing.md index 8db6e7786..bb729c454 100644 --- a/docs/pricing.md +++ b/docs/pricing.md @@ -56,6 +56,17 @@ Generated, not hand-written: `core/cost/pricing/bundled.go` carries `model_prices_and_context_window.json`, pinned to one commit that `agentop pricing --json` reports as `upstreamCommit`. +**A local install does not stay on that commit.** It downloads the same file at startup and +then hourly, and replaces the compiled-in rows with what the download produces — the same +transform, so the same shape of table, from newer data. An unchanged file answers `304` with +no body, so checking costs nothing. The last good download is kept in +`~/.cortex/price-list.json`, so a restart, or a laptop with no network, prices from it rather +than from the build. A download that fails, or is not a usable price map, changes nothing. +`agentop pricing` then reports `rates from litellm's price list, downloaded