Feat: Download LiteLLM's price list at runtime on a local install

cmd/agentop/cmd_pricing.go

@@ -20,7 +20,8 @@ import (
 //
 // This exists because no config file can answer the question. `pricing:` shows what the
 // operator wrote; the figures a request is charged come from that PLUS a rate table
-// compiled into the binary PLUS any shipped gateway discount. Storing the table in the
+// compiled into the binary, and on a local install downloaded from LiteLLM, PLUS any
+// shipped gateway discount. Storing the table in the
 // config instead would freeze every install at the rates current on its install date,
 // silently, because that file is written once and never refreshed — which is the exact
 // staleness the pricing work was done to remove.
@@ -42,9 +43,10 @@ Usage:
   agentop pricing --host <gateway>    what that endpoint is charged, discount applied
   agentop pricing --json              raw JSON
 
-The rates come from a table built into the binary (vendor list, refreshed per release),
-your `+"`pricing:`"+` config, and any gateway discount Cortex ships. --host is the useful
-form: it resolves all three the way a request would.
+The rates come from a table built into the binary (vendor list), your `+"`pricing:`"+`
+config, and any gateway discount Cortex ships. A local install updates the built-in table
+from LiteLLM's price list, downloaded hourly, so a new model is priced without a new
+release. --host is the useful form: it resolves all of it the way a request would.
 
 Flags:
 `)
@@ -221,7 +223,8 @@ type thresholdRow struct {
 }
 
 type describeBody struct {
-	UpstreamCommit string `json:"upstreamCommit"`
+	UpstreamCommit string    `json:"upstreamCommit"`
+	ListFetchedAt  time.Time `json:"listFetchedAt"`
 	Rows           []struct {
 		Host       string         `json:"host"`
 		Model      string         `json:"model"`
@@ -250,6 +253,9 @@ func renderTable(body []byte, stdout, stderr io.Writer) int {
 	if d.UpstreamCommit != "" {
 		fmt.Fprintf(stdout, "  bundled rates generated from litellm %s\n", short(d.UpstreamCommit))
 	}
+	if !d.ListFetchedAt.IsZero() {
+		fmt.Fprintf(stdout, "  rates from litellm's price list, downloaded %s\n", d.ListFetchedAt.UTC().Format("2006-01-02 15:04 MST"))
+	}
 	fmt.Fprintf(stdout, "\n  %-22s %-30s %9s %9s %9s %9s  %s\n",
 		"endpoint", "model", "input", "cache-wr", "cache-rd", "output", "from")
 	for _, r := range d.Rows {

cmd/agentop/cmd_pricing_test.go

@@ -6,6 +6,7 @@ import (
 	"net/http/httptest"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/rossoctl/cortex/core/cost/pricing"
 )
@@ -127,6 +128,37 @@ func TestRunPricing_TableViewListsRowsAndDiscounts(t *testing.T) {
 	}
 }
 
+// A local install prices from a downloaded list, and the header must say so.
+func TestRunPricing_TableViewSaysWhenTheListWasDownloaded(t *testing.T) {
+	all := [pricing.NumTiers]bool{}
+	for i := range all {
+		all[i] = true
+	}
+	tab, err := pricing.BuildWithList(nil, &pricing.List{
+		Entries: []pricing.Entry{{Host: "*", Model: "claude-opus-5-5", Prov: pricing.ProvBundled,
+			Rates: pricing.Rates{Base: [pricing.NumTiers]float64{pricing.TierInput: 4e-06, pricing.TierCacheWrite: 5e-06,
+				pricing.TierCacheRead: 2e-07, pricing.TierOutput: 2e-05}, Set: all}}},
+		FetchedAt: time.Date(2026, 10, 8, 21, 0, 0, 0, time.UTC),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	srv := httptest.NewServer(pricing.NewRegistry(tab).Handler())
+	t.Cleanup(srv.Close)
+
+	var out, errb bytes.Buffer
+	if code := runPricing([]string{"--stats-url", srv.URL}, &out, &errb); code != 0 {
+		t.Fatalf("exit %d: %s", code, errb.String())
+	}
+	got := out.String()
+	if !strings.Contains(got, "rates from litellm's price list, downloaded 2026-10-08 21:00 UTC") {
+		t.Errorf("output does not say when the list was downloaded:\n%s", got)
+	}
+	if !strings.Contains(got, "bundled rates generated from litellm "+short(pricing.BundledUpstreamCommit)) {
+		t.Errorf("output does not name the shipped commit, whose rows the list does not name are still in the table:\n%s", got)
+	}
+}
+
 // A tier with no rate must render "-", never 0.00: pricing.Cost refuses to price a
 // request that used such a tier, so a zero would misrepresent a coverage gap as free.
 func TestRunPricing_UnsetTierRendersAsAbsent(t *testing.T) {

cmd/cortex/main.go

@@ -466,6 +466,9 @@ func main() {
 	// disagree with the plugins about what a request cost. The table is swapped in
 	// place instead; the pointer never changes. See pricing.Registry.
 	pricingRegistry := pricing.NewRegistry(nil)
+	// Every table goes into the registry through live, which also applies a downloaded price
+	// list on a local install (runPriceList) without undoing a config reload, or the reverse.
+	live := &livePricing{reg: pricingRegistry}
 
 	// This binary is hardcoded to proxy-sidecar. Rejecting other modes
 	// early gives operators a clear boot-time error instead of silently
@@ -494,7 +497,7 @@ func main() {
 		// built so a plugin's Configure sees the new table, and swapped in place so
 		// the usage aggregator — which holds this same registry from before the
 		// reload — sees it too.
-		tab, err := pricing.Build(c.Pricing)
+		tab, err := live.build(c.Pricing)
 		if err != nil {
 			return nil, nil, nil, fmt.Errorf("pricing: %w", err)
 		}
@@ -533,10 +536,10 @@ func main() {
 	// applyPricing puts a prepared table into effect. Called on the reload goroutine, which is
 	// serialised, so the single pending slot needs no lock.
 	applyPricing := func(c *config.Config) {
-		if pendingPricing != nil {
-			pricingRegistry.Swap(pendingPricing)
-			pendingPricing = nil
+		if pendingPricing != nil && c != nil {
+			live.Swap(c.Pricing, pendingPricing)
 		}
+		pendingPricing = nil
 		if c != nil {
 			c.Pricing.WarnIfUnpinned(slog.Default())
 		}
@@ -582,6 +585,10 @@ func main() {
 	if err := rld.Start(ctx); err != nil {
 		log.Fatalf("reloader: %v", err)
 	}
+	// After the first config is applied, so the list is combined with it from the start.
+	if localInstall && cfg.Pricing.BundledEnabled() {
+		go runPriceList(ctx, live)
+	}
 
 	var sessions *session.Store
 	var usageAgg *usage.Aggregator

cmd/cortex/pricing_list.go

@@ -0,0 +1,88 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/rossoctl/cortex/core/cost/pricing"
+	"github.com/rossoctl/cortex/core/cost/pricing/pricelist"
+)
+
+// priceListInterval is how often a local install asks whether LiteLLM's price list changed.
+// An unchanged list answers 304 with no body, so checking hourly costs nothing; a new model's
+// price then reaches the proxy within the hour LiteLLM lists it.
+const priceListInterval = time.Hour
+
+// priceListFile is where a local install keeps the last downloaded list, under ~/.cortex.
+const priceListFile = "price-list.json"
+
+// livePricing keeps the rate table in step with two inputs that change independently: the
+// config, which the reloader commits, and LiteLLM's price list, which the downloader applies.
+// Either one rebuilds the table from both, under one lock, so neither can undo the other.
+type livePricing struct {
+	reg *pricing.Registry
+
+	mu        sync.Mutex
+	committed bool // a config has been accepted; until then a list only waits
+	cfg       *pricing.Config
+	list      *pricing.List // nil until one is downloaded; the shipped table is used meanwhile
+}
+
+// build prepares a table for a config the reloader has not accepted yet. It applies nothing,
+// so a config that is then refused never prices traffic.
+func (l *livePricing) build(cfg *pricing.Config) (*pricing.Table, error) {
+	l.mu.Lock()
+	list := l.list
+	l.mu.Unlock()
+	return pricing.BuildWithList(cfg, list)
+}
+
+// Swap applies an accepted config. prepared is the table build made for it, and is what
+// goes live unless a list arrived since: then it is rebuilt with that list, or the reload
+// would put an older one back.
+//
+// Named Swap because that is what the reload closure must do, and what
+// TestEveryBinaryInjectsPricing looks for inside it: a binary whose reload never swaps the
+// table keeps its boot-time rates forever.
+func (l *livePricing) Swap(cfg *pricing.Config, prepared *pricing.Table) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.committed, l.cfg = true, cfg
+	tab := prepared
+	if fresh, err := pricing.BuildWithList(cfg, l.list); err == nil {
+		tab = fresh
+	}
+	l.reg.Swap(tab)
+}
+
+// setList applies a downloaded list to the accepted config.
+func (l *livePricing) setList(list *pricing.List) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if !l.committed {
+		l.list = list
+		return
+	}
+	tab, err := pricing.BuildWithList(l.cfg, list)
+	if err != nil {
+		slog.Warn("pricelist: the downloaded price list does not combine with the config; keeping the current prices",
+			"error", err)
+		return
+	}
+	l.list = list
+	l.reg.Swap(tab)
+}
+
+// runPriceList keeps a local install's list prices current until ctx ends. Not run outside a
+// local install: a sidecar makes no outbound call it was not configured to make, and keeps the
+// shipped table.
+func runPriceList(ctx context.Context, l *livePricing) {
+	f := &pricelist.Fetcher{}
+	if dir, err := defaultCortexDir(); err == nil {
+		f.CacheFile = filepath.Join(dir, priceListFile)
+	}
+	f.Run(ctx, priceListInterval, l.setList)
+}

cmd/cortex/pricing_list_test.go

@@ -0,0 +1,99 @@
+package main
+
+import (
+	"testing"
+	"time"
+
+	"github.com/rossoctl/cortex/core/cost/pricing"
+)
+
+// opus55 is a downloaded list pricing a model newer than any build's shipped table.
+func opus55(input float64) *pricing.List {
+	all := [pricing.NumTiers]bool{}
+	for i := range all {
+		all[i] = true
+	}
+	var base [pricing.NumTiers]float64
+	base[pricing.TierInput] = input / 1e6
+	base[pricing.TierOutput] = 5 * input / 1e6
+	base[pricing.TierCacheRead] = input / 20 / 1e6
+	base[pricing.TierCacheWrite] = 1.25 * input / 1e6
+	return &pricing.List{
+		Entries:   []pricing.Entry{{Host: "*", Model: "claude-opus-5-5", Prov: pricing.ProvBundled, Rates: pricing.Rates{Base: base, Set: all}}},
+		FetchedAt: time.Date(2026, 10, 8, 21, 0, 0, 0, time.UTC),
+	}
+}
+
+// override prices claude-sonnet-5 on api.anthropic.com, so a test can see the config survive.
+func override() *pricing.Config {
+	return &pricing.Config{Endpoints: []pricing.EndpointConfig{{
+		Hosts:  []string{"api.anthropic.com"},
+		Models: map[string]pricing.ModelConfig{"claude-sonnet-5": {TierRates: pricing.TierRates{InputCostPerMillion: 1}}},
+	}}}
+}
+
+func inputRate(t *testing.T, reg *pricing.Registry, model string) (float64, pricing.Provenance) {
+	t.Helper()
+	r, p := reg.Resolve("api.anthropic.com", model, 0)
+	return r.Base[pricing.TierInput] * 1e6, p
+}
+
+func near(a, b float64) bool { return a-b < 1e-9 && b-a < 1e-9 }
+
+// commitConfig does what a reload does: prepare a table, then swap it in.
+func commitConfig(t *testing.T, l *livePricing, cfg *pricing.Config) {
+	t.Helper()
+	tab, err := l.build(cfg)
+	if err != nil {
+		t.Fatalf("build: %v", err)
+	}
+	l.Swap(cfg, tab)
+}
+
+func TestLivePricing_ADownloadedListIsAppliedWithTheConfig(t *testing.T) {
+	l := &livePricing{reg: pricing.NewRegistry(nil)}
+	commitConfig(t, l, override())
+	l.setList(opus55(4))
+
+	if got, p := inputRate(t, l.reg, "claude-opus-5-5"); p != pricing.ProvBundled || !near(got, 4) {
+		t.Errorf("claude-opus-5-5 = %v (%s), want 4 from the downloaded list", got, p)
+	}
+	if got, p := inputRate(t, l.reg, "claude-sonnet-5"); p != pricing.ProvConfigured || !near(got, 1) {
+		t.Errorf("claude-sonnet-5 = %v (%s), want the configured 1: a download must not drop the config", got, p)
+	}
+}
+
+func TestLivePricing_AReloadDoesNotPutBackAnOlderList(t *testing.T) {
+	l := &livePricing{reg: pricing.NewRegistry(nil)}
+	commitConfig(t, l, nil)
+	l.setList(opus55(4))
+
+	// A reload prepares its table, a download lands, then the reload commits.
+	prepared, err := l.build(override())
+	if err != nil {
+		t.Fatalf("build: %v", err)
+	}
+	l.setList(opus55(3))
+	l.Swap(override(), prepared)
+
+	if got, _ := inputRate(t, l.reg, "claude-opus-5-5"); !near(got, 3) {
+		t.Errorf("claude-opus-5-5 = %v after the reload, want 3: the list that arrived during it", got)
+	}
+	if _, p := inputRate(t, l.reg, "claude-sonnet-5"); p != pricing.ProvConfigured {
+		t.Errorf("claude-sonnet-5 provenance = %s, want configured from the reloaded config", p)
+	}
+}
+
+func TestLivePricing_AListBeforeTheFirstConfigWaitsForIt(t *testing.T) {
+	// Applied on its own, the list would be priced with no config at all — every
+	// configured rate missing until the first reload.
+	l := &livePricing{reg: pricing.NewRegistry(nil)}
+	l.setList(opus55(4))
+	if _, p := inputRate(t, l.reg, "claude-opus-5-5"); p != pricing.ProvNone {
+		t.Errorf("provenance before any config = %s, want none: nothing applied yet", p)
+	}
+	commitConfig(t, l, override())
+	if got, _ := inputRate(t, l.reg, "claude-opus-5-5"); !near(got, 4) {
+		t.Errorf("claude-opus-5-5 = %v after the first config, want 4 from the waiting list", got)
+	}
+}

core/cost/pricing/config.go

@@ -211,28 +211,7 @@ func (c *Config) BundledEnabled() bool {
 // Configured rows are emitted at ProvConfigured so they outrank anything bundled
 // at equal specificity: an override is an override.
 func Build(cfg *Config) (*Table, error) {
-	var entries []Entry
-	var mults []MultiplierRule
-	if cfg.BundledEnabled() {
-		entries = append(entries, Bundled()...)
-		// So do the free models' zero rates, kept out of Bundled() because that is the
-		// generated table its golden test pins to the LiteLLM snapshot.
-		entries = append(entries, bundledFreeRates()...)
-		// Shipped gateway discounts travel with the shipped rates: the rates are
-		// vendor list, and for the gateways named here list is a third too high.
-		// Disabling the bundled table disables both, which is the right pairing —
-		// a multiplier on rates you did not ship scales somebody else's numbers.
-		mults = append(mults, bundledMultipliers()...)
-	}
-	if cfg != nil {
-		configured, err := cfg.entries()
-		if err != nil {
-			return nil, err
-		}
-		entries = append(entries, configured...)
-		mults = append(mults, cfg.multipliers()...)
-	}
-	return NewTable(entries, mults...)
+	return BuildWithList(cfg, nil)
 }
 
 // multipliers converts the config's endpoint blocks into multiplier rules.

core/cost/pricing/describe.go

@@ -3,6 +3,7 @@ package pricing
 import (
 	"sort"
 	"strings"
+	"time"
 )
 
 // This file exists because a config file cannot answer the question operators
@@ -69,6 +70,7 @@ type Description struct {
 	// UpstreamCommit is the LiteLLM commit the bundled rates were generated from, so a
 	// figure can be traced to its source without reading the binary.
 	UpstreamCommit string           `json:"upstreamCommit,omitempty"`
+	ListFetchedAt  time.Time        `json:"listFetchedAt,omitzero"`
 	Rows           []RowView        `json:"rows"`
 	Multipliers    []MultiplierView `json:"multipliers,omitempty"`
 }
@@ -136,6 +138,7 @@ func (t *Table) Describe() Description {
 	if t == nil {
 		return out
 	}
+	out.ListFetchedAt = t.listFetchedAt
 	for i := range t.rows {
 		r := &t.rows[i]
 		row := RowView{

core/cost/pricing/internal/pricegen/pricegen.go

@@ -1,10 +1,11 @@
 // Package pricegen turns LiteLLM's public model_prices_and_context_window.json
 // into pricing.Entry rows and renders them as Go source.
 //
-// It lives in internal/ because it is build tooling, not runtime code, and it is
-// a library rather than living inside the generator command so the golden test can
-// run the exact same transform against a committed snapshot. A generator whose
-// transform only exists inside a main package cannot be tested without a network.
+// It is a library rather than living inside the generator command so the golden test
+// can run the exact same transform against a committed snapshot — a generator whose
+// transform only exists inside a main package cannot be tested without a network — and
+// so pricelist can run it on a list downloaded at runtime. It stays in internal/ because
+// only those two callers should depend on its output shape.
 package pricegen
 
 import (