diff --git a/docs/changes/unreleased/1892-selector-snapshot-seats-the-crew.md b/docs/changes/unreleased/1892-selector-snapshot-seats-the-crew.md new file mode 100644 index 0000000000..67dd428ec3 --- /dev/null +++ b/docs/changes/unreleased/1892-selector-snapshot-seats-the-crew.md @@ -0,0 +1,29 @@ +--- +kind: added +title: the pareto selector is wired — a fitted snapshot seats the unpinned seats, none routes as before +pr: 1892 +surface: [chat] +invalidates: + - "internal/selector's package doc said the engine is shadow-only, never applied to real routing — that is no longer true when a profile carries a fitted SharedSnapshot at models.crew.selector_snapshot; with no snapshot configured nothing changed, byte for byte." +--- +--- + +The port of the pareto selection change-set sat in the tree as a shadow lane: the +engine was built by no production line, and the crew every task actually ran was +internal/crewroute's own pick. This wires it, at the one seam the trace found — +`config.RouteCrew`, the single door a task's crew comes through (the session's +`Config.RouteCrew`, `config.ResolveSeats` for every headless door, the redo paths all +build their decision there). + +The shape: `config` loads the snapshot at `models.crew.selector_snapshot`, runs the +engine over the request's own candidate list (the pool the router already priced and +health-checked, with the avoided lineages left out), and hands the pick per seat to +`crewroute` as `Request.Chosen`. Inside `crewroute.Decide` the named seat is seated +through the same machinery a pin is ([pinned]) and then marked not pinned — so the +decision, its estimates, its ladder, its rescue and its log row are all still the live +engine's. A pin wins, a redo ignores the pick, a learned offset still moves the crew, +and a model nobody offers cannot be seated. + +The safety property, which the tests lock: no snapshot configured — or one that stopped +loading, or a seat the engine cannot seat — leaves the crew exactly as `dev` routes it +today. diff --git a/internal/config/crew.go b/internal/config/crew.go index 8626e24795..fdcc73b972 100644 --- a/internal/config/crew.go +++ b/internal/config/crew.go @@ -1382,6 +1382,11 @@ func RouteCrew(profileDir string, ask CrewAsk) (crewroute.Decision, error) { CostFactor: day.CostFactor[string(reading.Class)], TaskCap: CrewTaskCapAt(profileDir), } + // THE SELECTOR'S PICK, WHEN ONE IS CONFIGURED: a fitted snapshot seats + // the unpinned seats through the engine's own gate, and no snapshot + // configured leaves the request's Chosen nil — the crew is then the live + // router's exactly as it has always been (selectorwire.go says why). + req.Chosen = CrewSelectorPicks(profileDir, req) d, err := crewroute.Decide(req) if err != nil && !errors.Is(err, crewroute.ErrStrongest) { // NO CREW CAN BE FORMED: the seat nothing can sit goes down the rest of diff --git a/internal/config/selectorwire.go b/internal/config/selectorwire.go new file mode 100644 index 0000000000..94984780b4 --- /dev/null +++ b/internal/config/selectorwire.go @@ -0,0 +1,231 @@ +package config + +import ( + "fmt" + "os" + "strings" + "sync" + "time" + + "github.com/Agent-Field/codeaf/internal/catalog" + "github.com/Agent-Field/codeaf/internal/crewroute" + "github.com/Agent-Field/codeaf/internal/selector" +) + +// ── the pareto selector, wired ────────────────────────────────────────────── +// +// internal/selector's engine (the ported pareto selection change-set) picks a +// model per seat from a FITTED SharedSnapshot — posterior quality means per +// role and model, with a UCB-enter/LCB-rank gate — over the same candidate +// list the live router reads. This file is the only place the engine is +// built, and the only place its answer reaches a real decision: +// +// - No snapshot configured: the selector never runs, and the crew is the +// live router's exactly as before — byte for byte the pre-wiring +// behavior, which is the safety property of the whole feature. +// - A snapshot configured but unloadable, or a seat the engine cannot seat +// (no candidate passes its capability gates): that seat keeps the live +// router's pick. The engine is a per-seat overlay, never a second router. +// +// Either way the decision, its estimates, its ladder, its rescue, its log row +// and its receipt are the live engine's own ([Request.Chosen] in +// internal/crewroute): the selector replaces the MODEL a seat runs, never the +// machinery around it. A pin still wins, a redo still routes by the live +// engine alone, and a learned offset still moves the crew on top of the +// selector's pick. + +// KeyCrewSelectorSnapshot is the path to a fitted SharedSnapshot JSON the +// selector reads. PROFILE-ONLY, like every crew row: a path is a choice about +// what may route a person's work, and a repository that could write one could +// route a visitor's credit to a model nobody chose. +const KeyCrewSelectorSnapshot = "models.crew.selector_snapshot" + +// CrewSelectorSnapshotAt is the configured snapshot path, empty for none — +// and none is the default: with no snapshot the live router picks alone. +func CrewSelectorSnapshotAt(profileDir string) string { + value, _ := persistedString(profileDir, KeyCrewSelectorSnapshot) + return strings.TrimSpace(value) +} + +// SetCrewSelectorSnapshot writes the snapshot path, or clears it with an +// empty one. A path that does not load is refused HERE rather than ignored at +// route time: a person who configured a snapshot is owed a refusal they can +// read, not a crew that silently kept the old picker. +func SetCrewSelectorSnapshot(profileDir, raw string) error { + path := strings.TrimSpace(raw) + if path != "" { + if _, err := selector.LoadSnapshot(path); err != nil { + return fmt.Errorf("that snapshot does not load: %w", err) + } + } + return writeProfileValue(profileDir, KeyCrewSelectorSnapshot, path) +} + +// CrewSelectorPicks builds the seat overlay the router is handed: the +// selector's pick per crew seat, or nil when no snapshot is configured, one +// does not load, or the engine seats nothing. A variable so a test can stub +// the engine the way [CrewCatalog] and [CrewHistory] are stubbed. +var CrewSelectorPicks = selectorCrewPicks + +// snapshotCache is the one snapshot in effect, cached by path + mtime + +// size: routing calls selectorCrewPicks on every crewed request, and a +// fitted file is read once per change, not once per route. The cached +// snapshot is only ever read — the engine keeps its own per-call state. +var snapshotCache struct { + mu sync.Mutex + path string + mtime time.Time + size int64 + snap *selector.SharedSnapshot +} + +// cachedSnapshot loads path, or serves the cached copy when the file has +// not changed. A nil return is the fallback: unreadable, unparseable or +// invalid all mean unconfigured (see selectorCrewPicks). +func cachedSnapshot(path string) *selector.SharedSnapshot { + fi, err := os.Stat(path) + if err != nil { + return nil + } + snapshotCache.mu.Lock() + defer snapshotCache.mu.Unlock() + if snapshotCache.snap != nil && snapshotCache.path == path && + snapshotCache.mtime.Equal(fi.ModTime()) && snapshotCache.size == fi.Size() { + return snapshotCache.snap + } + snap, err := selector.LoadSnapshot(path) + if err != nil { + return nil + } + snapshotCache.path, snapshotCache.mtime = path, fi.ModTime() + snapshotCache.size, snapshotCache.snap = fi.Size(), snap + return snap +} + +// selectorCrewPicks runs the engine over this request's candidates and the +// configured snapshot. The classes are the engine's seat shapes: the worker +// class seats the worker, the high class the checker, the mastermind class +// the planner — the three roles a crew has. A class the engine cannot seat +// leaves that seat out of the overlay, which is the live pick standing. +func selectorCrewPicks(profileDir string, req crewroute.Request) map[crewroute.Seat]string { + path := CrewSelectorSnapshotAt(profileDir) + if path == "" || len(req.Candidates) == 0 { + return nil + } + snap := cachedSnapshot(path) + if snap == nil { + // UNLOADABLE IS UNCONFIGURED, on purpose: the fallback to the live + // router is the safety property, and a route that fails because a + // fitted file went stale is a route that fails for nothing. The + // refusal to WRITE a bad path is where the person is told. + return nil + } + cat := selectorCatalogOf(req) + if cat == nil { + return nil + } + eng := selector.NewEngine(snap, cat, nil) + picks := map[crewroute.Seat]string{} + for _, slot := range []struct { + seat crewroute.Seat + class string + }{ + {crewroute.Worker, "worker"}, + {crewroute.Checker, "high"}, + {crewroute.Planner, "mastermind"}, + } { + sd, err := eng.DecideClass(shapeOf(slot.class)) + if err != nil { + continue + } + if id := strings.TrimSpace(sd.Chosen); id != "" { + picks[slot.seat] = id + } + } + return picks +} + +// shapeOf is one of the engine's five default classes by its id. +func shapeOf(id string) selector.ClassShape { + for _, cs := range selector.DefaultClasses() { + if cs.ID == id { + return cs + } + } + return selector.ClassShape{} +} + +// selectorCatalogOf is the engine's catalog: this request's candidates, read +// into the selector's schema-field row. ONE MODEL IS ONE CANDIDATE, so a +// candidate's model id is the id the pick carries back, and the catalog rows +// only lend the fields the candidate does not carry — the modalities and the +// release line the engine's prior inheritance reads. A candidate whose +// lineage the request says to avoid is left out, the same set the live +// engine's own picking reads. +func selectorCatalogOf(req crewroute.Request) *selector.ModelCatalog { + rows := crewCatalogRows() + byLineage := make(map[string]catalog.Model, len(rows)) + for _, row := range rows { + byLineage[crewroute.Lineage(row.ID)] = row + } + var models []selector.Model + for _, c := range req.Candidates { + if req.Avoid[crewroute.Lineage(c.Model.ID)] { + continue + } + models = append(models, selectorModelOf(c, byLineage[crewroute.Lineage(c.Model.ID)])) + } + if len(models) == 0 { + return nil + } + // NewModelCatalog, not a literal: the constructor indexes the rows for + // alias resolution and prior inheritance. A literal builds a catalog + // whose byID/bySlug maps are nil — no panic, but every unscored model + // silently seats on the global prior, the arm-collapse the engine + // exists to prevent. + return selector.NewModelCatalog(models) +} + +// selectorModelOf is one candidate as the engine reads it. The candidate is +// the authority on price, indexes and tools — the figures the router already +// routed on; the catalog row, when one matches, is the authority on the +// modalities and the release line the engine's gates and prior inheritance +// read. A candidate with no catalog row is carried on its own figures, and a +// row that publishes no window is carried as one that refused for nothing. +func selectorModelOf(c crewroute.Candidate, row catalog.Model) selector.Model { + m := selector.Model{ + ID: c.Model.ID, + ContextLength: c.Model.Context, + PromptPrice: c.Model.PromptPrice, + Completion: c.Model.CompletionPrice, + CacheRead: c.Model.CacheReadPrice, + IntelligenceIndex: c.Model.Intelligence, + CodingIndex: c.Model.Coding, + AgenticIndex: c.Model.Agentic, + } + m.Reasoning.Known, m.Reasoning.Mandatory = row.Reasoning.Known, row.Reasoning.Mandatory + if row.ID != "" { + m.CanonicalSlug, m.Created, m.AliasTarget = row.CanonicalSlug, row.Created, row.AliasTarget + if row.ContextLength > 0 { + m.ContextLength = row.ContextLength + } + m.ModalitiesIn = append([]string(nil), row.InputModalities...) + m.ModalitiesOut = append([]string(nil), row.OutputModalities...) + } + if c.Model.Tools && !hasString(m.ModalitiesIn, "text") { + // THE SEAT GATE IS THE ROUTER'S, not the modalities list's: a row + // that lists no input modalities has said nothing either way, and the + // router already judged the model able to take a tool call. + m.ModalitiesIn = append(m.ModalitiesIn, "text") + } + return m +} + +func hasString(ss []string, want string) bool { + for _, s := range ss { + if s == want { + return true + } + } + return false +} diff --git a/internal/config/selectorwire_test.go b/internal/config/selectorwire_test.go new file mode 100644 index 0000000000..cb3fdbbed5 --- /dev/null +++ b/internal/config/selectorwire_test.go @@ -0,0 +1,292 @@ +package config + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/Agent-Field/codeaf/internal/catalog" + "github.com/Agent-Field/codeaf/internal/crewroute" +) + +// selectorProfile is a profile with an OpenRouter key and a catalog built for +// this file: three open, tooled models and one that can also read an image, +// so the engine's image gate has something to seat. +func selectorProfile(t *testing.T) string { + t.Helper() + dir := crewProfile(t) + rows := []catalog.Model{ + {ID: "z-ai/glm-5.3-flash", CanonicalSlug: "z-ai/glm-5.3-flash-20260826", OpenWeights: true, + PromptPrice: 4.5e-8, CompletionPrice: 6e-7, CacheReadPrice: 2.85e-8, + IntelligenceIndex: 41.8, CodingIndex: 71.5, AgenticIndex: 50.9, ContextLength: 1310720, Parameters: []string{"tools"}}, + {ID: "moonshotai/kimi-k3", CanonicalSlug: "moonshotai/kimi-k3-20260715", OpenWeights: true, + PromptPrice: 3e-6, CompletionPrice: 1.5e-5, CacheReadPrice: 3e-7, + IntelligenceIndex: 43.6, CodingIndex: 76.2, AgenticIndex: 50, ContextLength: 1048576, Parameters: []string{"tools"}}, + {ID: "deepseek/deepseek-v4-flash", CanonicalSlug: "deepseek/deepseek-v4-flash-20260423", OpenWeights: true, + PromptPrice: 7.168e-8, CompletionPrice: 1.4336e-7, CacheReadPrice: 1.4336e-8, + IntelligenceIndex: 24.2, CodingIndex: 56.2, AgenticIndex: 22.2, ContextLength: 1048576, Parameters: []string{"tools"}}, + {ID: "anthropic/claude-opus-5", CanonicalSlug: "anthropic/claude-opus-5-20260723", + PromptPrice: 5e-6, CompletionPrice: 2.5e-5, CacheReadPrice: 5e-7, + IntelligenceIndex: 50.8, CodingIndex: 78, AgenticIndex: 56.5, ContextLength: 1000000, + Parameters: []string{"tools"}, InputModalities: []string{"text", "image"}}, + } + CrewCatalog = func() []catalog.Model { return rows } + return dir +} + +// fittedSnapshot writes a SharedSnapshot that scores kimi-k3 highest for the +// worker role and opus-5 highest for checker and thinker, with every model +// scored and every variance tight — so the engine's gate has a clear answer +// for each role, and the overlay is provably the engine's, not the live +// router's. +func fittedSnapshot(t *testing.T, dir string) string { + t.Helper() + parameters, uncertainty := map[string]float64{}, map[string]float64{} + counts := map[string]int{} + for _, role := range []string{"worker", "checker", "thinker"} { + scores := map[string]float64{ + "z-ai/glm-5.3-flash": 0.55, "moonshotai/kimi-k3": 0.6, + "deepseek/deepseek-v4-flash": 0.5, "anthropic/claude-opus-5": 0.6, + } + switch role { + case "worker": + scores["moonshotai/kimi-k3"] = 0.9 + default: + scores["anthropic/claude-opus-5"] = 0.9 + } + for model, q := range scores { + key := "quality/" + role + "/" + model + parameters[key] = q + uncertainty[key] = 0.0004 + counts[key] = 12 + } + parameters["quality_w/"+role+"/residual_sigma"] = 0.02 + } + type snapshot struct { + SchemaVersion int `json:"schema_version"` + Parameters map[string]float64 `json:"parameters"` + Uncertainty map[string]float64 `json:"uncertainty"` + Counts map[string]int `json:"counts"` + ModelIDs []string `json:"model_ids"` + GeneratedBy string `json:"generated_by"` + } + body, err := json.Marshal(snapshot{ + SchemaVersion: 1, Parameters: parameters, Uncertainty: uncertainty, Counts: counts, + ModelIDs: []string{"z-ai/glm-5.3-flash", "moonshotai/kimi-k3", "deepseek/deepseek-v4-flash", "anthropic/claude-opus-5"}, + GeneratedBy: "test fixture", + }) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "shared-snapshot.json") + if err := os.WriteFile(path, body, 0o600); err != nil { + t.Fatal(err) + } + return path +} + +// THE WIRING, END TO END: a profile carrying a fitted snapshot seats the +// crew the engine picked — worker, checker and planner — on the live +// decision's own machinery, routed and laddered; the same profile without a +// snapshot seats the crew the live router always picked, byte for byte. +func TestASnapshotConfiguredSeatsTheEngineCrewAndNoneSeatsTheLiveCrew(t *testing.T) { + dir := selectorProfile(t) + path := fittedSnapshot(t, dir) + + // THE SAFETY PROPERTY FIRST: with no snapshot configured, the crew is the + // live router's own and the request carries no overlay at all. + bare, err := RouteCrew(dir, CrewAsk{Task: crewroute.Task{Text: fixTask}}) + if err != nil { + t.Fatal(err) + } + if overlay := CrewSelectorPicks(dir, crewroute.Request{Candidates: CrewCandidatesAt(dir)}); overlay != nil { + t.Fatalf("a profile with no snapshot produced an overlay: %v", overlay) + } + + if err := SetCrewSelectorSnapshot(dir, path); err != nil { + t.Fatal(err) + } + // A SNAPSHOT THAT VANISHED IS A SNAPSHOT THAT IS NOT CONFIGURED: the same + // ask routes the bare crew, to the last byte of a pick. + if err := os.Rename(path, path+".away"); err != nil { + t.Fatal(err) + } + gone, err := RouteCrew(dir, CrewAsk{Task: crewroute.Task{Text: fixTask}}) + if err != nil { + t.Fatal(err) + } + if err := os.Rename(path+".away", path); err != nil { + t.Fatal(err) + } + for _, seat := range crewroute.Seats { + if gone.Seat(seat) != bare.Seat(seat) { + t.Errorf("%s: a vanished snapshot changed the pick to %+v, want the bare %+v", seat, gone.Seat(seat), bare.Seat(seat)) + } + } + if gone.EstUSD != bare.EstUSD || len(gone.Ladder) != len(bare.Ladder) { + t.Errorf("a vanished snapshot changed the decision: est $%.3f vs $%.3f, %d ladders vs %d", + gone.EstUSD, bare.EstUSD, len(gone.Ladder), len(bare.Ladder)) + } + + // THE ENGINE'S CREW, on the machinery of the live decision: every seat + // the engine could seat carries its model, still routed, still laddered, + // with the estimate recomputed for the crew that runs. + wired, err := RouteCrew(dir, CrewAsk{Task: crewroute.Task{Text: fixTask}}) + if err != nil { + t.Fatal(err) + } + overlay := CrewSelectorPicks(dir, crewroute.Request{Candidates: CrewCandidatesAt(dir)}) + if len(overlay) != 3 { + t.Fatalf("the engine seated %v, want all three seats", overlay) + } + for seat, want := range overlay { + got := wired.Seat(seat) + if got.Model != want { + t.Errorf("%s runs %s, want the selector's %s", seat, got.Model, want) + } + if got.Pinned { + t.Errorf("%s runs the selector's pick and says pinned", seat) + } + if len(wired.Ladder[seat]) == 0 { + t.Errorf("%s lost its ladder under the overlay", seat) + } + } + // The overlay demonstrably changed the crew: at least one seat differs + // from what the live router picked for the same ask. + changed := 0 + for _, seat := range crewroute.Seats { + if wired.Seat(seat).Model != bare.Seat(seat).Model { + changed++ + } + } + if changed == 0 { + t.Error("the configured snapshot changed nothing; the overlay never ran") + } + if wired.EstUSD <= 0 { + t.Errorf("the wired crew carries no estimate: $%.3f", wired.EstUSD) + } +} + +// A PIN WINS THROUGH THE FULL ROUTE: a seat the ask pinned runs its pin while +// the selector seats the rest. +func TestAPinnedSeatOutsitsTheSelectorAndTheRestAreSeated(t *testing.T) { + dir := selectorProfile(t) + path := fittedSnapshot(t, dir) + if err := SetCrewSelectorSnapshot(dir, path); err != nil { + t.Fatal(err) + } + d, err := RouteCrew(dir, CrewAsk{ + Task: crewroute.Task{Text: fixTask}, + Pins: map[crewroute.Seat]CrewPin{crewroute.Worker: {Model: "z-ai/glm-5.3-flash"}}, + Sends: map[crewroute.Seat]string{crewroute.Planner: "deepseek/deepseek-v4-flash"}, + }) + if err != nil { + t.Fatal(err) + } + if got := d.Seat(crewroute.Worker); got.Model != "z-ai/glm-5.3-flash" || !got.Pinned { + t.Errorf("worker %+v, want the person's pin", got) + } + if got := d.Seat(crewroute.Planner); got.Model != "deepseek/deepseek-v4-flash" || !got.Pinned { + t.Errorf("planner %+v, want the one-task pin", got) + } + // The checker seat, which nobody pinned, is the selector's. + overlay := CrewSelectorPicks(dir, crewroute.Request{Candidates: CrewCandidatesAt(dir)}) + if want := overlay[crewroute.Checker]; want != "" { + if got := d.Seat(crewroute.Checker); got.Model != want { + t.Errorf("checker %s, want the selector's %s", got.Model, want) + } + } +} + +// THE OVERLAY IS BUILT FROM THE CANDIDATES, not from the whole catalog: a +// model the person's rule leaves out cannot be seated by the selector either, +// and a lineage the request says to avoid is left out of its pool. +func TestTheSelectorPoolIsTheCandidatePool(t *testing.T) { + dir := selectorProfile(t) + if err := SetCrewSelectorSnapshot(dir, fittedSnapshot(t, dir)); err != nil { + t.Fatal(err) + } + req := crewroute.Request{Candidates: CrewCandidatesAt(dir)} + overlay := CrewSelectorPicks(dir, req) + if len(overlay) == 0 { + t.Fatal("the engine seated nothing") + } + allowed := map[string]bool{} + for _, c := range req.Candidates { + allowed[c.Model.ID] = true + } + for seat, model := range overlay { + if !allowed[model] { + t.Errorf("%s: the selector picked %s, which no candidate offers", seat, model) + } + } + // With every lineage avoided, the pool is empty and the overlay is nil — + // the live engine's picks stand. + avoid := map[string]bool{} + for _, c := range req.Candidates { + avoid[crewroute.Lineage(c.Model.ID)] = true + } + if overlay := CrewSelectorPicks(dir, crewroute.Request{Candidates: req.Candidates, Avoid: avoid}); overlay != nil { + t.Errorf("a pool of nobody gave an overlay: %v", overlay) + } +} + +// THE WIRE'S CATALOG IS INDEXED: the engine's alias resolution and its +// line/vendor prior inheritance read that index, and a catalog built as a +// bare literal leaves it nil — no panic, but every unscored model silently +// seats on the global prior, the arm-collapse the selector exists to +// prevent. One alias in the pool proves the index is there. +func TestTheWiredCatalogResolvesAliases(t *testing.T) { + dir := selectorProfile(t) + base := CrewCatalog() + CrewCatalog = func() []catalog.Model { + return append(base, catalog.Model{ + ID: "z-ai/glm-5.3-flash-alias", CanonicalSlug: "z-ai/glm-5.3-flash-latest", + AliasTarget: "z-ai/glm-5.3-flash", OpenWeights: true, + PromptPrice: 4.5e-8, CompletionPrice: 6e-7, CacheReadPrice: 2.85e-8, + IntelligenceIndex: 41.8, CodingIndex: 71.5, AgenticIndex: 50.9, + ContextLength: 1310720, Parameters: []string{"tools"}, + }) + } + req := crewroute.Request{Candidates: CrewCandidatesAt(dir)} + cat := selectorCatalogOf(req) + if cat == nil { + t.Fatal("no catalog from a full candidate pool") + } + for i := range cat.Models { + if cat.Models[i].ID != "z-ai/glm-5.3-flash-alias" { + continue + } + resolved, ok := cat.Resolve(&cat.Models[i]) + if !ok || resolved.ID != "z-ai/glm-5.3-flash" { + t.Fatalf("the alias resolved to %q (ok=%v), want z-ai/glm-5.3-flash — the wire's catalog lost its index", resolved.ID, ok) + } + return + } + t.Fatal("the alias candidate never reached the wired catalog") +} + +// A BAD PATH IS REFUSED AT SET TIME, not silently ignored at route time. +func TestABadSnapshotPathIsRefusedAndAnEmptyOneClears(t *testing.T) { + dir := selectorProfile(t) + if err := SetCrewSelectorSnapshot(dir, filepath.Join(dir, "no-such-snapshot.json")); err == nil { + t.Fatal("an unloadable snapshot path was accepted") + } + if got := CrewSelectorSnapshotAt(dir); got != "" { + t.Errorf("the refused path was written anyway: %q", got) + } + path := fittedSnapshot(t, dir) + if err := SetCrewSelectorSnapshot(dir, path); err != nil { + t.Fatal(err) + } + if got := CrewSelectorSnapshotAt(dir); got != path { + t.Errorf("the path reads %q, want %q", got, path) + } + if err := SetCrewSelectorSnapshot(dir, " "); err != nil { + t.Fatalf("clearing was refused: %v", err) + } + if got := CrewSelectorSnapshotAt(dir); got != "" { + t.Errorf("the cleared row reads %q", got) + } +} diff --git a/internal/crewpick/crewpick.go b/internal/crewpick/crewpick.go new file mode 100644 index 0000000000..87ec2fb8f0 --- /dev/null +++ b/internal/crewpick/crewpick.go @@ -0,0 +1,850 @@ +// Package crewpick picks one model per seat for a three-seat crew by reading +// the pareto front of expected task bill against crew quality. +// +// It deliberately imports nothing from the rest of the tree and touches no +// disk and no network: the caller owns the candidate list and the seat +// shapes, and the same candidates give the same front however often they are +// asked about and in whatever order they arrive. +// +// The law it implements: every (worker, high, mastermind) combination the +// candidates can field is scored by the bill its seats' token volumes run up +// and by the mean of its three seat qualities. A candidate is in the running +// for a seat only when it meets the family and the seat's needs and its +// quality reaches Floor of that seat's best in the family, and a crew is +// refused outright when its high seat comes from the worker's own vendor. +// The crews that no other crew beats on both bill and quality survive as the +// front, sorted by bill; Presets, AtStake and AtKnob read picks off that +// front. +// +// A pick is a decision with units. The person's one dial is the STAKE: what +// one failed task costs them, in dollars. A crew's loss at a stake is its +// expected bill for a task plus the stake times the chance the task is not +// accepted, and the pick at a stake is the crew on the front with the least +// loss. The three named presets are three stakes (docs/design/model-pool/ +// pareto-crewing.tex, the decision section): the knee of the front that +// balanced used to name implied a stake nobody was shown, and one that moved +// with the price level while a person's stake does not. +// +// A candidate that publishes some but not all of its three indexes is scored +// on the ones it publishes: each missing index is estimated from the call's +// candidates that publish both it and one of its measured ones — the donors — +// never above the call's largest measured value of that index, and the +// result says which of a pick's indexes were estimated. A candidate that +// publishes none of its indexes, or no price at all, is out of the running. +package crewpick + +import ( + "math" + "slices" + "sort" + "strings" +) + +// An Index is one of the three capability indexes quality is read from — +// intelligence, coding, agentic — in the order the seat weights and a +// candidate's reading carry them. +type Index int + +const ( + Intelligence Index = iota + Coding + Agentic +) + +// A Candidate is one model up for a seat. Prices are dollars per 1M tokens. +// +// A candidate carrying none of the three indexes — zero values, which is how +// unpublished indexes read — or carrying a zero prompt price is not a +// candidate at all: it never occupies a seat, never clears a floor, and +// raises no ceiling. A candidate carrying some but not all of its indexes is +// scored on the ones it publishes, each missing one estimated from the +// donors among its fellow candidates, and the result says which of a pick's +// indexes those were. A model that publishes no cache-read price says so with +// HasCacheRead false and pays the prompt price on the cache share too. +type Candidate struct { + // ID names the candidate; the text before its first slash is the vendor. + ID string + // Open marks a candidate the open family may draw from. + Open bool + // Intelligence, Coding and Agentic are the three capability indexes + // quality is read from. + Intelligence float64 + Coding float64 + Agentic float64 + // PromptPrice, CompletionPrice and CacheReadPrice are the dollars per + // 1M tokens of each kind. + PromptPrice float64 + CompletionPrice float64 + CacheReadPrice float64 + // HasCacheRead reports whether CacheReadPrice is published at all. + HasCacheRead bool + // Context is the model's context window, in tokens. + Context int + // Images and Tools are what the model can take and call. + Images bool + Tools bool +} + +// A Seat is one of a crew's three roles: the worker carries the volume of +// the work, the high seat judges and repairs what the worker writes, and the +// mastermind spends a little context steering the whole task. +type Seat int + +const ( + Worker Seat = iota + High + Mastermind +) + +// A SeatShape is how one seat uses whatever model sits it: the weights +// quality is read from, the token mix cost is computed on, what the model must +// be able to take and call, the window it must have, and the share of a +// task's tokens the seat burns. +type SeatShape struct { + // Weights reads quality from the three indexes: intelligence, coding, + // agentic. + Weights [3]float64 + // InOut is the seat's input tokens per output token; the completion + // price is spread over it. + InOut float64 + // CacheShare is the share of the seat's input tokens served from cache. + CacheShare float64 + // NeedTools and NeedImages are what a model must take and call to sit + // the seat. + NeedTools bool + NeedImages bool + // MinContext is the window a model must have to sit the seat. + MinContext int + // Volume is the seat's share of a task's tokens, and the weight its + // cost carries in a crew's bill. + Volume float64 +} + +// DefaultShapes returns the three seat shapes crews are picked for. The +// worker takes half its quality from the agentic index and reads barely +// anything but cached input; the high seat judges the three indexes evenly, +// needs tools and images both, and sees the same token mix on a sliver of the +// volume; the mastermind reads intelligence and agentic only, on a terse +// ten-in-one mix with no cache and a fraction of the volume. +func DefaultShapes() map[Seat]SeatShape { + return map[Seat]SeatShape{ + Worker: { + Weights: [3]float64{0.2, 0.3, 0.5}, + InOut: 100, + CacheShare: 0.75, + NeedTools: true, + MinContext: 200_000, + Volume: 0.90, + }, + High: { + Weights: [3]float64{1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0}, + InOut: 100, + CacheShare: 0.75, + NeedTools: true, + NeedImages: true, + MinContext: 200_000, + Volume: 0.08, + }, + Mastermind: { + Weights: [3]float64{0.6, 0, 0.4}, + InOut: 10, + MinContext: 200_000, + Volume: 0.02, + }, + } +} + +// A TaskUsage is what one task spent on each seat, in tokens: the evidence a +// seat's shape is learned from. A seat the task never seated is absent. +type TaskUsage map[Seat]SeatTokens + +// SeatTokens is one seat's tokens on one task. +type SeatTokens struct { + In int64 + Out int64 +} + +// ShapeWeightAt is the number of tasks at which the seats' measured shape +// carries the same weight as the defaults: at N tasks the measurement is worth +// half of each learned figure and the default the other half, the same +// N/(N+ShapeWeightAt) shrinkage the quality prior uses, so one strange task +// cannot move a bill and thirty ordinary ones can. +const ShapeWeightAt = 30 + +// ShapesFrom learns the seats' shapes from what real tasks spent: each seat's +// Volume moves from the default towards the share of a task's tokens the seat +// typically took, and its InOut towards the seat's input tokens per output +// token over every task, each by N/(N+ShapeWeightAt) with N the tasks that +// carried the seat. Everything else about a shape — its weights, its needs, +// its cache share, which no ledger row can see — is the default's. A task +// with no tokens teaches nothing, and no tasks at all leaves the defaults +// untouched. +// +// THE SHARE IS LEARNED ON THE LOG SCALE — the geometric mean of the seat's +// shares, shrunk towards the default in log space — and not as an arithmetic +// mean. The split is heavy-tailed: the high seat takes a few percent of the +// tokens on most tasks and most of them on a few, so the arithmetic mean is +// not the typical task, and a bill priced from it is wrong on nearly every +// task. On 132 measured runs the arithmetic shape predicted bills no better +// than the fixed default (median error a factor of 2.9 against 3.4) while the +// log-scale shape brought it to 1.5 (docs/design/model-pool/pareto-crewing.tex, +// the cost experiment). +func ShapesFrom(defaults map[Seat]SeatShape, tasks []TaskUsage) map[Seat]SeatShape { + type tally struct { + n int + logShare float64 + in, out float64 + } + tallies := map[Seat]*tally{} + for _, task := range tasks { + var total float64 + for _, t := range task { + total += float64(t.In + t.Out) + } + if total <= 0 { + continue + } + for seat, t := range task { + if t.In+t.Out <= 0 { + continue + } + ty := tallies[seat] + if ty == nil { + ty = &tally{} + tallies[seat] = ty + } + ty.n++ + ty.logShare += math.Log(float64(t.In+t.Out) / total) + ty.in += float64(t.In) + ty.out += float64(t.Out) + } + } + shapes := make(map[Seat]SeatShape, len(defaults)) + for seat, shape := range defaults { + ty := tallies[seat] + if ty == nil || ty.n == 0 || shape.Volume <= 0 { + shapes[seat] = shape + continue + } + w := float64(ty.n) / float64(ty.n+ShapeWeightAt) + shape.Volume = math.Exp((1-w)*math.Log(shape.Volume) + w*ty.logShare/float64(ty.n)) + if ty.out > 0 { + shape.InOut = (1-w)*shape.InOut + w*ty.in/ty.out + } + shapes[seat] = shape + } + return shapes +} + +// A Family is which candidates a crew may be drawn from. +type Family int + +const ( + // Open draws only from candidates marked Open. + Open Family = iota + // All draws from every candidate. + All +) + +// A Crew is one (worker, high, mastermind) pick. Bill is the expected dollars +// per 1M task tokens once each seat's share of the task is paid for at its +// seat cost; Quality is the mean of the three seat qualities on a 0-100 +// scale. +type Crew struct { + Worker string + High string + Mastermind string + Bill float64 + Quality float64 + // Estimated marks, for the model in each seat, which of its indexes + // were scored by estimate rather than measurement: the row is the seat + // and the column the index. A crew picked on measured indexes alone is + // all false. + Estimated [3][3]bool + // Measured marks, for the model in each seat, whether a pool's own + // rating for that model entered the seat's quality: the row is the seat + // and the value says the seat's model carried a rating with a positive + // observation count. A crew picked with no prior, or whose seats the + // prior never rated, is all false. + Measured [3]bool +} + +// Floor is the share of a seat's best quality in the family that a candidate +// must reach to stay in the running for that seat. A model that cannot do +// the job is not cheap, whatever it costs. +const Floor = 0.80 + +// PriorWeightAt is the observation count at which a pool's own rating for a +// model carries the same weight in a seat's quality as the catalog's +// published indexes: at N observations the rating is worth half the seat and +// the catalog the other half. A rating with no observations never enters. +const PriorWeightAt = 30 + +// A Rating is a pool's measured quality for one model on one seat, on the +// same 0-100 scale quality is read on, with the number of observations behind +// it. +type Rating struct { + Mean float64 + N int +} + +// A Prior is a pool's measured quality per seat, keyed by canonical model id: +// the rating a seat's quality is blended towards, by how many observations +// back it. A nil Prior is no prior at all, and every seat keeps the catalog +// quality. +type Prior map[Seat]map[string]Rating + +// A Cell is one measurement a caller hands PriorFromCells: a role, a model id +// and the pool's mean quality with the observations behind it. It is the shape +// a measurement document reads into, so crewpick need not import the reader. +type Cell struct { + Role string + Model string + Mean float64 + N int +} + +// seatWords names the roles a Cell's role word maps to a seat by, folded to +// one spelling so a document's own case never matters. Any other word names +// no seat and is ignored. +var seatWords = map[string]Seat{ + "worker": Worker, + "high": High, + "mastermind": Mastermind, +} + +// PriorFromCells reads a pool's measurements into a Prior: a cell names a +// seat by its role word, a model by an id that canonical resolves to the +// canonical id a candidate is looked up by, and carries a mean and a count. +// A cell whose role names no seat, or whose count is below minInstalls, is +// ignored, and so is a count of zero or less. A nil canonical leaves the id +// as it stands. Two cells that name one seat and one model — a graded source +// and a seeded one, or two spellings one canonical resolves together — fold +// into one rating, their means weighted by their counts and the counts added, +// in whatever order they arrive. When no cell survives the prior is nil, +// which is the same as no prior at all. +// +// The floor here keeps counting rows rather than the installs a measurement +// document carries beside them: the reader applies the document's floor +// before these cells arrive — on installs where a cell spells them — and +// rows cannot be fewer than the installs that produced them, so a cell that +// floor kept meets this one too. The cells built outside a document carry +// rows alone, so an installs field on Cell would sit unset on every caller, +// and there is none. +func PriorFromCells(cells []Cell, minInstalls int, canonical func(string) string) Prior { + var prior Prior + for _, c := range cells { + seat, ok := seatWords[strings.ToLower(strings.TrimSpace(c.Role))] + if !ok || c.N < minInstalls || c.N <= 0 { + continue + } + model := c.Model + if canonical != nil { + model = canonical(model) + } + if prior == nil { + prior = Prior{} + } + if prior[seat] == nil { + prior[seat] = map[string]Rating{} + } + if got, ok := prior[seat][model]; ok { + n := got.N + c.N + prior[seat][model] = Rating{Mean: (float64(got.N)*got.Mean + float64(c.N)*c.Mean) / float64(n), N: n} + continue + } + prior[seat][model] = Rating{Mean: c.Mean, N: c.N} + } + return prior +} + +// MergePriors folds two priors into one. Seat by seat, a rating both hold for +// a model is combined by observation count — the two means weighted by the +// counts behind them, the counts added — and a rating either holds alone is +// carried whole. A nil prior on either side is the other, and two nils are +// nil. The priors given are read and never changed, so the answer is never +// one of them. +func MergePriors(a, b Prior) Prior { + if a == nil { + return b + } + if b == nil { + return a + } + merged := make(Prior) + for seat, ratings := range a { + for model, r := range ratings { + if merged[seat] == nil { + merged[seat] = map[string]Rating{} + } + merged[seat][model] = r + } + } + for seat, ratings := range b { + for model, r := range ratings { + got, ok := merged[seat][model] + if !ok { + if merged[seat] == nil { + merged[seat] = map[string]Rating{} + } + merged[seat][model] = r + continue + } + n := got.N + r.N + if n <= 0 { + merged[seat][model] = Rating{} + continue + } + merged[seat][model] = Rating{ + Mean: (float64(got.N)*got.Mean + float64(r.N)*r.Mean) / float64(n), + N: n, + } + } + } + return merged +} + +// SeatCost is the seat's expected cost of a candidate, in dollars per 1M +// input-equivalent tokens: the prompt price blended with the cache-read +// price by the seat's cache share, plus the completion price spread over the +// seat's input-to-output ratio. A candidate that publishes no cache-read +// price pays the prompt price on the cache share too. +func SeatCost(c Candidate, shape SeatShape) float64 { + cache := c.PromptPrice + if c.HasCacheRead { + cache = c.CacheReadPrice + } + return (1-shape.CacheShare)*c.PromptPrice + shape.CacheShare*cache + c.CompletionPrice/shape.InOut +} + +// SeatQuality is the candidate's quality for the seat on a 0-100 scale: each +// index counted against the pool's best for that index and weighted by the +// seat's weights. The pool sets the scale, so the same candidate scores +// differently in a stronger pool. A missing index is estimated from the +// pool's donors exactly as in Front. A row that is not a candidate at all, +// and any row against a pool with no candidates to raise a scale, score +// zero. +func SeatQuality(c Candidate, shape SeatShape, pool []Candidate) float64 { + return SeatQualityWith(c, shape, pool, Worker, nil) +} + +// SeatQualityWith is SeatQuality with the pool's prior blended in for the seat +// given: a rating the prior holds for the candidate's canonical id, with a +// positive observation count, moves the seat's quality towards the rating's +// mean by N/(N+PriorWeightAt), and a seat the prior never rates — or rates +// with no observations — keeps the catalog quality unchanged. +func SeatQualityWith(c Candidate, shape SeatShape, pool []Candidate, seat Seat, prior Prior) float64 { + if !isCandidate(c) { + return 0 + } + scale := maxima(pool) + r := fill(c, readTally(pool), scale) + q, _ := blendQuality(c.ID, quality(r.idx, shape, scale), seat, prior) + return q +} + +// Front returns the pareto front of (bill, quality) over every crew the +// candidates can field in the family, sorted by bill: a crew on it is beaten +// by no other crew on both, and every crew off it is beaten by one on it. +// +// shapes must carry an entry for every seat. Per seat the running is +// narrowed twice before crews are enumerated — family and the seat's needs +// first, then Floor against that seat's best in the family — and a crew is +// refused while its high seat comes from the worker's own vendor, so the two +// seats that see the same work never share one shop. Ties in (bill, quality) +// break to the crew scored on fewer estimated indexes, and ties past that by +// id order — worker, then high, then mastermind — which is what makes the +// front a property of the candidates rather than of their order. +func Front(candidates []Candidate, shapes map[Seat]SeatShape, fam Family) []Crew { + return FrontWith(candidates, shapes, fam, nil) +} + +// FrontWith is Front with the pool's prior blended into each seat's quality: a +// rating the prior holds for a seat's model, with a positive observation +// count, moves that seat's quality towards the rating's mean by +// N/(N+PriorWeightAt), the seat's measured flag says so, and the floor and the +// crew's quality read the blended value. A nil prior leaves every seat on the +// catalog quality, exactly as Front does. The front is a property of the +// candidates and the prior, never of the order they arrive in. +func FrontWith(candidates []Candidate, shapes map[Seat]SeatShape, fam Family, prior Prior) []Crew { + scale := maxima(candidates) + t := readTally(candidates) + readings := make([]reading, 0, len(candidates)) + for _, c := range candidates { + readings = append(readings, fill(c, t, scale)) + } + workers := shortlist(readings, shapes[Worker], fam, scale, Worker, prior) + highs := shortlist(readings, shapes[High], fam, scale, High, prior) + minds := shortlist(readings, shapes[Mastermind], fam, scale, Mastermind, prior) + vw := shapes[Worker].Volume + vh := shapes[High].Volume + vm := shapes[Mastermind].Volume + + crews := make([]Crew, 0, len(workers)*len(highs)*len(minds)) + for _, w := range workers { + for _, h := range highs { + if vendor(w.c.ID) == vendor(h.c.ID) { + continue + } + for _, m := range minds { + crews = append(crews, Crew{ + Worker: w.c.ID, + High: h.c.ID, + Mastermind: m.c.ID, + Bill: vw*w.cost + vh*h.cost + vm*m.cost, + Quality: (w.quality + h.quality + m.quality) / 3, + Estimated: [3][3]bool{w.est, h.est, m.est}, + Measured: [3]bool{w.measured, h.measured, m.measured}, + }) + } + } + } + + // The front: cheapest bill first, best quality first within a bill, and + // a crew survives when its quality beats everything cheaper so far. + sort.SliceStable(crews, func(i, j int) bool { + if crews[i].Bill != crews[j].Bill { + return crews[i].Bill < crews[j].Bill + } + if crews[i].Quality != crews[j].Quality { + return crews[i].Quality > crews[j].Quality + } + return estimateCount(crews[i]) < estimateCount(crews[j]) + }) + var front []Crew + best := math.Inf(-1) + for _, crew := range crews { + if crew.Quality > best+1e-9 { + front = append(front, crew) + best = crew.Quality + } + } + return front +} + +// Two of the three named presets are stakes, in dollars — what one failed +// task costs the person. The words are the paper's (docs/design/model-pool/ +// pareto-crewing.tex, product semantics): frugal is a failed task that is an +// annoyance, balanced one that costs about ten minutes of a person. Max is +// not a stake: it is the best crew on the front whatever it bills, the one +// word that stays a superlative, because a person who says max has said the +// bill is not the point. A custom stake is any number. +const ( + StakeFrugal = 1.0 + StakeBalanced = 10.0 +) + +// TypicalTaskTokens is the token count a crew's bill is read over when a +// caller has no better figure for the task in hand: the median of the 133 +// task-door runs on the bench with a per-seat token split, taken 2026-09-17 +// (quartiles 2.3M and 11.4M). Bill is dollars per 1M task tokens, so this is +// what turns it into dollars per task, which is the unit a stake is in. +const TypicalTaskTokens = 4_400_000 + +// TaskBill is the crew's expected bill for one task of the tokens given, in +// dollars: Bill is per 1M task tokens. +func TaskBill(c Crew, tokens float64) float64 { + return c.Bill * tokens / 1e6 +} + +// Loss is the crew's expected loss at a stake for one task of the tokens +// given: the task's bill plus the stake times the chance the task is not +// accepted, read as one minus the crew's quality on its 0-1 scale. Quality +// is on the acceptance scale by construction — a measured rating is the share +// of graded tasks accepted, times 100, and the catalog's index blend is the +// prior for it — so the two terms are in the same dollars. +func Loss(c Crew, stake, tokens float64) float64 { + return TaskBill(c, tokens) + stake*(1-c.Quality/100) +} + +// AtStake reads one pick off the front at a stake: the crew with the least +// loss for a task of the tokens given, and the cheaper one when two tie. A +// stake of zero is the cheapest crew and a stake past every bill is the best +// one; between them, as the stake rises the pick walks up the front and never +// back down, because a crew that was worth more at a lower stake is worth at +// least that at a higher one. An empty front picks nothing. +func AtStake(front []Crew, stake, tokens float64) Crew { + if len(front) == 0 { + return Crew{} + } + pick := front[0] + best := Loss(pick, stake, tokens) + for _, crew := range front[1:] { + if loss := Loss(crew, stake, tokens); loss < best-1e-12 { + pick, best = crew, loss + } + } + return pick +} + +// Presets reads the three named picks off a front: frugal at StakeFrugal and +// balanced at StakeBalanced, each for a task of TypicalTaskTokens, and max the +// best crew on the front, which is its last because the front is sorted by +// bill with quality strictly rising. A front of one crew is all three; an +// empty front is none. +func Presets(front []Crew) (frugal, balanced, max Crew) { + if len(front) == 0 { + return Crew{}, Crew{}, Crew{} + } + return AtStake(front, StakeFrugal, TypicalTaskTokens), + AtStake(front, StakeBalanced, TypicalTaskTokens), + front[len(front)-1] +} + +// AtKnob reads one pick off the front at knob k in [0, 1]: the budget runs +// geometrically from the frugal bill to the max bill — lo*(hi/lo)^k — and +// the pick is the best-quality crew whose bill fits it, so k of 0 is frugal +// and k of 1 is max. An empty front picks nothing. +func AtKnob(front []Crew, k float64) Crew { + if len(front) == 0 { + return Crew{} + } + lo, hi := front[0].Bill, front[len(front)-1].Bill + budget := lo * math.Pow(hi/lo, k) + var pick Crew + for _, crew := range front { + if crew.Bill <= budget*(1+1e-9) { + pick = crew + } + } + return pick +} + +// A seatPick is a candidate already priced and scored for one seat, on the +// candidate's reading: measured and estimated indexes alike. measured says +// whether the pool's own rating for the seat entered the quality. +type seatPick struct { + c Candidate + idx [3]float64 + est [3]bool + cost float64 + quality float64 + measured bool +} + +// shortlist returns a seat's running: the candidates that pass family and the +// seat's needs, kept when their quality reaches Floor of the family's best +// for the seat, in id order. The seat's quality is read with the pool's prior +// blended in, and measured marks the picks the prior touched. +func shortlist(readings []reading, shape SeatShape, fam Family, scale [3]float64, seat Seat, prior Prior) []seatPick { + var running []seatPick + top := 0.0 + for _, r := range readings { + if !eligible(r.c, shape, fam) { + continue + } + q, measured := blendQuality(r.c.ID, quality(r.idx, shape, scale), seat, prior) + pick := seatPick{c: r.c, idx: r.idx, est: r.est, cost: SeatCost(r.c, shape), quality: q, measured: measured} + running = append(running, pick) + top = math.Max(top, pick.quality) + } + var kept []seatPick + for _, pick := range running { + if pick.quality >= Floor*top { + kept = append(kept, pick) + } + } + sort.Slice(kept, func(i, j int) bool { return kept[i].c.ID < kept[j].c.ID }) + return kept +} + +// quality is SeatQuality against pre-read maxima, on a candidate's reading. +func quality(idx [3]float64, shape SeatShape, scale [3]float64) float64 { + if scale[Intelligence] <= 0 || scale[Coding] <= 0 || scale[Agentic] <= 0 { + return 0 + } + w := shape.Weights + return 100 * (w[Intelligence]*idx[Intelligence]/scale[Intelligence] + + w[Coding]*idx[Coding]/scale[Coding] + + w[Agentic]*idx[Agentic]/scale[Agentic]) +} + +// blendQuality blends the pool's prior for one seat into the catalog quality: +// a rating the prior holds for the model on the seat, with a positive +// observation count, carries weight N/(N+PriorWeightAt) against the catalog's +// own quality, and the second result reports whether the rating entered. A +// seat the prior never rates, or rates with no observations, keeps the +// catalog quality and reads measured false. +func blendQuality(model string, catalog float64, seat Seat, prior Prior) (float64, bool) { + if prior == nil { + return catalog, false + } + rating, ok := prior[seat][model] + if !ok || rating.N <= 0 { + return catalog, false + } + w := float64(rating.N) / float64(rating.N+PriorWeightAt) + return (1-w)*catalog + w*rating.Mean, true +} + +// maxima reads the pool's largest measured value of each index — the scale +// quality is measured against and the ceiling an estimate never crosses. +// Only real candidates raise a ceiling, and a row that publishes only some +// of its indexes raises only those. +func maxima(pool []Candidate) (scale [3]float64) { + for _, c := range pool { + if !isCandidate(c) { + continue + } + v := measured(c) + for i, val := range v { + scale[i] = math.Max(scale[i], val) + } + } + return scale +} + +// measured is the candidate's three indexes as it published them: an +// unpublished index reads as zero. +func measured(c Candidate) [3]float64 { + return [3]float64{c.Intelligence, c.Coding, c.Agentic} +} + +// A tally is one call's donor statistics: for every ordered pair of indexes, +// how many candidates — the donors — carry both measured, and the median of +// the first over the second among them. A missing index is estimated from +// these. +type tally struct { + count [3][3]int + median [3][3]float64 +} + +// readTally reads the pool's tally; the donors of a pair of indexes are the +// candidates carrying both measured. +func readTally(pool []Candidate) tally { + var ratios [3][3][]float64 + for _, c := range pool { + if !isCandidate(c) { + continue + } + v := measured(c) + for _, x := range [3]Index{Intelligence, Coding, Agentic} { + for _, y := range [3]Index{Intelligence, Coding, Agentic} { + if x != y && v[x] > 0 && v[y] > 0 { + ratios[x][y] = append(ratios[x][y], v[x]/v[y]) + } + } + } + } + var t tally + for _, x := range [3]Index{Intelligence, Coding, Agentic} { + for _, y := range [3]Index{Intelligence, Coding, Agentic} { + t.count[x][y] = len(ratios[x][y]) + if len(ratios[x][y]) >= 3 { + t.median[x][y] = medianOf(ratios[x][y]) + } + } + } + return t +} + +// medianOf is the middle of the values in order, or the mean of the two +// middles when there is no single middle. +func medianOf(values []float64) float64 { + slices.Sort(values) + n := len(values) + if n%2 == 1 { + return values[n/2] + } + return (values[n/2-1] + values[n/2]) / 2 +} + +// A reading is a candidate's three indexes ready for scoring — the measured +// ones as published, the missing ones estimated from the call's donors — +// with est remembering which of the three were estimated rather than +// measured. +type reading struct { + c Candidate + idx [3]float64 + est [3]bool +} + +// fill reads the candidate's three indexes for scoring. A measured index is +// taken as published; a missing one is estimated from the ones present: each +// present index Y offers Y times the tally's median of the missing index over +// Y among the donors carrying both — or, when fewer than three donors carry +// the pair, equal standing, the missing index read where its present one +// stands, X over the pool's largest X equal to Y over the pool's largest Y — +// and the mean of the offers is kept, never above the pool's largest measured +// value of the missing index. A row that is no candidate at all fills to +// zeros. +func fill(c Candidate, t tally, scale [3]float64) reading { + var r reading + if !isCandidate(c) { + return r + } + r.c = c + v := measured(c) + r.idx = v + for _, x := range [3]Index{Intelligence, Coding, Agentic} { + if v[x] > 0 { + continue + } + var offers []float64 + for _, y := range [3]Index{Intelligence, Coding, Agentic} { + if y == x || v[y] <= 0 { + continue + } + if t.count[x][y] >= 3 { + offers = append(offers, v[y]*t.median[x][y]) + } else { + offers = append(offers, v[y]/scale[y]*scale[x]) + } + } + sum := 0.0 + for _, offer := range offers { + sum += offer + } + r.idx[x] = math.Min(sum/float64(len(offers)), scale[x]) + r.est[x] = true + } + return r +} + +// estimateCount reads how many of a crew's picks were scored on estimated +// indexes — the tiebreak between crews that tie on bill and quality. +func estimateCount(c Crew) int { + n := 0 + for _, seat := range c.Estimated { + for _, est := range seat { + if est { + n++ + } + } + } + return n +} + +// isCandidate reports whether a row is a candidate at all: at least one of +// the three indexes present and a prompt price above zero. Everything +// downstream — eligibility, the floor, the maxima — starts from this. +func isCandidate(c Candidate) bool { + return (c.Intelligence > 0 || c.Coding > 0 || c.Agentic > 0) && c.PromptPrice > 0 +} + +// eligible reports whether a candidate may sit the seat: family first, then +// what the seat needs the model to take and call, then the window it needs. +func eligible(c Candidate, shape SeatShape, fam Family) bool { + if !isCandidate(c) { + return false + } + if fam == Open && !c.Open { + return false + } + if shape.NeedTools && !c.Tools { + return false + } + if shape.NeedImages && !c.Images { + return false + } + return c.Context >= shape.MinContext +} + +// vendor is the id text before the first slash, or the whole id when there is +// no slash. +func vendor(id string) string { + if i := strings.Index(id, "/"); i >= 0 { + return id[:i] + } + return id +} diff --git a/internal/crewpick/crewpick_test.go b/internal/crewpick/crewpick_test.go new file mode 100644 index 0000000000..53ca57f0c2 --- /dev/null +++ b/internal/crewpick/crewpick_test.go @@ -0,0 +1,965 @@ +package crewpick + +import ( + "encoding/json" + "fmt" + "math" + "os" + "slices" + "testing" +) + +// A fixture row in its own field names: a null index reads as zero — +// missing, the way the package reads it — and a null cache-read price reads +// as unpublished. The partially published row and the rows with unpublished +// cache prices stay in the fixture, so the estimate and exclusion rules +// have something to bite on. +type row struct { + ID string `json:"id"` + Open bool `json:"open"` + I float64 `json:"I"` + C float64 `json:"C"` + A float64 `json:"A"` + Pi float64 `json:"pi"` + Po float64 `json:"po"` + Cr *float64 `json:"cr"` + Ctx int `json:"ctx"` + Img bool `json:"img"` + Tools bool `json:"tools"` +} + +// loadCandidates reads the fixture into candidates. +func loadCandidates(t *testing.T) []Candidate { + t.Helper() + raw, err := os.ReadFile("testdata/candidates.json") + if err != nil { + t.Fatal(err) + } + var rows []row + if err := json.Unmarshal(raw, &rows); err != nil { + t.Fatal(err) + } + cands := make([]Candidate, 0, len(rows)) + for _, r := range rows { + c := Candidate{ + ID: r.ID, + Open: r.Open, + Intelligence: r.I, + Coding: r.C, + Agentic: r.A, + PromptPrice: r.Pi, + CompletionPrice: r.Po, + Context: r.Ctx, + Images: r.Img, + Tools: r.Tools, + } + if r.Cr != nil { + c.CacheReadPrice = *r.Cr + c.HasCacheRead = true + } + cands = append(cands, c) + } + return cands +} + +// model builds a candidate that can sit any seat: every index at 100, tools +// and images, a 200k window, and a published cache price. +func model(id string) Candidate { + return Candidate{ + ID: id, + Open: true, + Intelligence: 100, + Coding: 100, + Agentic: 100, + PromptPrice: 1, + CompletionPrice: 4, + CacheReadPrice: 0.1, + HasCacheRead: true, + Context: 200_000, + Images: true, + Tools: true, + } +} + +// donors builds n fully measured candidates that can sit no seat — a window +// too small for any of them — so they feed the estimates and the ceilings +// and nothing else. +func donors(prefix string, n int) []Candidate { + rows := make([]Candidate, 0, n) + for i := range n { + d := model(fmt.Sprintf("%s/d%d", prefix, i)) + d.Context = 1000 + rows = append(rows, d) + } + return rows +} + +// wantCrew checks a pick against its worker|high|mastermind line. +func wantCrew(t *testing.T, name string, got Crew, want string) { + t.Helper() + line := got.Worker + "|" + got.High + "|" + got.Mastermind + if line != want { + t.Fatalf("%s = %s, want %s", name, line, want) + } +} + +// The fixture's front is a fixed property of its rows: seven crews in the +// open family, eighteen in all, and these are the picks each front reads. +// The all-family balanced crew is the one with the least loss at the balanced +// stake of ten dollars over a typical task — a bill of 0.147 per 1M tokens +// ($0.65 a task) at quality 89.2. The knee this word used to name sat at +// 0.590 ($2.60 a task) for 95.4: six more points of quality at four times +// the bill, which a ten-dollar stake declines to buy, and a fifty-dollar one +// would. +func TestPresetsReadTheFixtureFront(t *testing.T) { + cands := loadCandidates(t) + shapes := DefaultShapes() + + open := Front(cands, shapes, Open) + if len(open) != 7 { + t.Fatalf("open front holds %d crews, want 7", len(open)) + } + frugal, balanced, max := Presets(open) + wantCrew(t, "open frugal", frugal, "deepseek/deepseek-v4-flash-0731|z-ai/glm-5.3-flash|z-ai/glm-5.3-flash") + wantCrew(t, "open balanced", balanced, "z-ai/glm-5.3-flash|moonshotai/kimi-k3|z-ai/glm-5.3") + wantCrew(t, "open max", max, "z-ai/glm-5.3|moonshotai/kimi-k3|z-ai/glm-5.3") + + all := Front(cands, shapes, All) + if len(all) != 18 { + t.Fatalf("all front holds %d crews, want 18", len(all)) + } + frugal, balanced, max = Presets(all) + wantCrew(t, "all frugal", frugal, "z-ai/glm-5.3-flash|google/gemini-3.8-flash|z-ai/glm-5.3-flash") + wantCrew(t, "all balanced", balanced, "z-ai/glm-5.3-flash|qwen/qwen3.8-max-0902|qwen/qwen3.8-max-0902") + wantCrew(t, "all max", max, "anthropic/claude-fable-5.1|openai/gpt-6-astra|anthropic/claude-fable-5.1") + if math.Abs(balanced.Bill-0.147) > 0.005 { + t.Fatalf("all balanced bill = %f, want about 0.147", balanced.Bill) + } + if math.Abs(balanced.Quality-89.2) > 0.05 { + t.Fatalf("all balanced quality = %f, want about 89.2", balanced.Quality) + } +} + +// The high seat sees the same work the worker does, so a crew never draws it +// from the worker's own vendor. Every crew on both fixture fronts holds +// that line, and a pool that cannot field two vendors fields no crew at all. +func TestTheHighSeatNeverSharesAVendorWithTheWorker(t *testing.T) { + cands := loadCandidates(t) + for _, fam := range []Family{Open, All} { + for _, crew := range Front(cands, DefaultShapes(), fam) { + if vendor(crew.Worker) == vendor(crew.High) { + t.Fatalf("worker %s and high %s share a vendor", crew.Worker, crew.High) + } + } + } + + oneVendor := []Candidate{model("one/first"), model("one/second")} + if front := Front(oneVendor, DefaultShapes(), All); len(front) != 0 { + t.Fatalf("a one-vendor pool fielded %d crews, want none", len(front)) + } +} + +// The floor keeps a seat's running to candidates that can do the job: a +// model whose quality falls below Floor of the seat's best in the family is +// not cheap and never reaches a crew. The boundary is inclusive — a +// candidate sitting exactly on the floor runs, and at a tenth of the price +// of anything above it, it runs first. +func TestTheFloorDropsAModelThatCannotDoTheJob(t *testing.T) { + pool := []Candidate{model("a/m"), model("c/h")} + + // Cheap enough to be every crew's frugal worker, and short one index: + // 70 of the worker seat's 100, 76 of the mastermind's, and ineligible + // for the high seat, which needs images. + cheap := model("b/w") + cheap.Agentic = 40 + cheap.Images = false + cheap.PromptPrice = 0.01 + cheap.CompletionPrice = 0.02 + cheap.HasCacheRead = false + pool = append(pool, cheap) + + // Exactly on the worker seat's floor: 80 of 100. + edge := model("e/b") + edge.Agentic = 60 + edge.PromptPrice = 0.02 + edge.CompletionPrice = 0.04 + edge.HasCacheRead = false + pool = append(pool, edge) + + front := Front(pool, DefaultShapes(), All) + if len(front) == 0 { + t.Fatal("the pool must field crews") + } + frugal, _, _ := Presets(front) + if frugal.Worker != "e/b" { + t.Fatalf("frugal worker = %s, want the on-floor candidate e/b", frugal.Worker) + } + for _, crew := range front { + for _, id := range []string{crew.Worker, crew.High, crew.Mastermind} { + if id == "b/w" { + t.Fatalf("the below-floor model b/w reached a crew: %+v", crew) + } + } + } +} + +// A row carrying none of its three indexes, or priced at zero, is not a +// candidate at all: it never reaches a seat and it raises no ceiling. Both +// rules bite where the floor cannot — a free model would clear any floor, +// and a bare row carries no index to estimate from. +func TestARowWithNoIndexesAtAllOrNoPriceIsNoCandidate(t *testing.T) { + good := []Candidate{model("a/m"), model("c/h")} + good[1].Intelligence = 90 + good[1].PromptPrice = 2 + good[1].CompletionPrice = 8 + good[1].CacheReadPrice = 0.2 + + bare := model("d/x") + bare.Intelligence = 0 + bare.Coding = 0 + bare.Agentic = 0 + bare.PromptPrice = 0.01 + bare.CompletionPrice = 0.02 + bare.HasCacheRead = false + + free := model("e/y") + free.PromptPrice = 0 + + pool := append(slices.Clone(good), bare, free) + front := Front(pool, DefaultShapes(), All) + for _, crew := range front { + for _, id := range []string{crew.Worker, crew.High, crew.Mastermind} { + if id == "d/x" || id == "e/y" { + t.Fatalf("a non-candidate row reached a crew: %+v", crew) + } + } + } + // The non-candidate rows leave the real candidates' front exactly what + // it was without them. + if want := Front(good, DefaultShapes(), All); !slices.Equal(front, want) { + t.Fatalf("the non-candidate rows moved the front:\n got %+v\nwant %+v", front, want) + } +} + +// A model publishing only its intelligence index — a release scored in +// stages — is scored on it, with the coding and agentic indexes estimated +// from the donors, and at a hundredth of the price it takes the worker seat. +func TestAModelWithOnlyAnIntelligenceIndexIsChosen(t *testing.T) { + pool := []Candidate{model("a/w"), model("c/h")} + newRelease := model("b/x") + newRelease.Coding = 0 + newRelease.Agentic = 0 + newRelease.PromptPrice = 0.01 + newRelease.CompletionPrice = 0.02 + newRelease.HasCacheRead = false + pool = append(pool, newRelease) + pool = append(pool, donors("v", 3)...) + + front := Front(pool, DefaultShapes(), All) + if len(front) == 0 { + t.Fatal("the pool must field crews") + } + frugal, _, _ := Presets(front) + if frugal.Worker != "b/x" { + t.Fatalf("frugal worker = %s, want the partially scored b/x", frugal.Worker) + } + want := [3][3]bool{{false, true, true}, {}, {false, true, true}} + if frugal.Estimated != want { + t.Fatalf("the b/x crew's Estimated = %v, want %v", frugal.Estimated, want) + } +} + +// A model publishing only its coding index — many an older model — is +// scored on it, with the intelligence and agentic indexes estimated, and at +// a hundredth of the price it takes the worker seat too. +func TestAModelWithOnlyACodingIndexIsChosen(t *testing.T) { + pool := []Candidate{model("a/w"), model("c/h")} + older := model("b/x") + older.Intelligence = 0 + older.Agentic = 0 + older.PromptPrice = 0.01 + older.CompletionPrice = 0.02 + older.HasCacheRead = false + pool = append(pool, older) + pool = append(pool, donors("v", 3)...) + + front := Front(pool, DefaultShapes(), All) + if len(front) == 0 { + t.Fatal("the pool must field crews") + } + frugal, _, _ := Presets(front) + if frugal.Worker != "b/x" { + t.Fatalf("frugal worker = %s, want the partially scored b/x", frugal.Worker) + } + want := [3][3]bool{{true, false, true}, {}, {true, false, true}} + if frugal.Estimated != want { + t.Fatalf("the b/x crew's Estimated = %v, want %v", frugal.Estimated, want) + } +} + +// The donor estimate is the median of the missing index over the present +// one among the donors carrying both — not the mean, which the far-out +// fourth donor drags away — and the rows missing either index of the pair +// never donate. The row missing its intelligence index still raises the +// coding ceiling with its own 400. +func TestTheDonorEstimateIsTheMedianRatio(t *testing.T) { + shape := SeatShape{Weights: [3]float64{0, 1, 0}} + pool := []Candidate{ + {ID: "a/d1", Intelligence: 10, Coding: 10, Agentic: 10, PromptPrice: 1}, + {ID: "a/d2", Intelligence: 10, Coding: 20, Agentic: 10, PromptPrice: 1}, + {ID: "a/d3", Intelligence: 10, Coding: 30, Agentic: 10, PromptPrice: 1}, + {ID: "a/d4", Intelligence: 10, Coding: 300, Agentic: 10, PromptPrice: 1}, + {ID: "b/noc", Intelligence: 10, Agentic: 10, PromptPrice: 1}, + {ID: "b/noi", Coding: 400, Agentic: 10, PromptPrice: 1}, + } + x := Candidate{ID: "c/x", Intelligence: 10, PromptPrice: 1} + + // The donors' coding over intelligence ratios run 1, 2, 3, 30: the + // median reads 2.5 and the estimate 25 of a 400 ceiling, where the mean + // would read 9 and the estimate 90. + if got := SeatQuality(x, shape, pool); got != 6.25 { + t.Fatalf("SeatQuality on the median estimate = %v, want 6.25", got) + } +} + +// With fewer than three donors carrying a pair, the estimate falls back to +// equal standing — the missing index read at the present one's share of +// the pool's ceilings — not to the median of two donors, and not to +// exclusion. +func TestFewerThanThreeDonorsEstimatesByEqualStanding(t *testing.T) { + shape := SeatShape{Weights: [3]float64{0, 1, 0}} + pool := []Candidate{ + {ID: "a/d1", Intelligence: 100, Coding: 50, Agentic: 10, PromptPrice: 1}, + {ID: "a/d2", Intelligence: 50, Coding: 100, Agentic: 10, PromptPrice: 1}, + } + x := Candidate{ID: "b/x", Intelligence: 80, PromptPrice: 1} + + // Equal standing reads x's coding at x's share of the intelligence + // ceiling, 80 of 100; the two donors' median would read 100. + if got := SeatQuality(x, shape, pool); got != 80 { + t.Fatalf("SeatQuality on equal standing = %v, want 80", got) + } +} + +// An estimate never crosses the pool's largest measured value of its +// index: the donors' coding over intelligence median runs 20, and 50 times +// that would read 1000 of a 300 ceiling. +func TestAnEstimateIsCappedAtTheLargestMeasuredIndex(t *testing.T) { + shape := SeatShape{Weights: [3]float64{0, 1, 0}} + pool := []Candidate{ + {ID: "a/d1", Intelligence: 10, Coding: 100, Agentic: 10, PromptPrice: 1}, + {ID: "a/d2", Intelligence: 10, Coding: 200, Agentic: 10, PromptPrice: 1}, + {ID: "a/d3", Intelligence: 10, Coding: 300, Agentic: 10, PromptPrice: 1}, + } + x := Candidate{ID: "b/x", Intelligence: 50, PromptPrice: 1} + + if got := SeatQuality(x, shape, pool); got != 100 { + t.Fatalf("SeatQuality on the capped estimate = %v, want 100", got) + } +} + +// The estimate is never silent: the crew says, for the model in each seat, +// which of its three indexes were scored by estimate and which measured. +// The worker publishes only its intelligence index and the mastermind only +// its coding one; the high seat is measured throughout. +func TestTheResultMarksEstimatedIndexes(t *testing.T) { + pool := []Candidate{model("a/w"), model("c/h")} + worker := model("b/x") + worker.Coding = 0 + worker.Agentic = 0 + worker.Images = false + worker.PromptPrice = 0.01 + worker.CompletionPrice = 0.02 + worker.HasCacheRead = false + mind := model("d/m") + mind.Intelligence = 0 + mind.Agentic = 0 + mind.Tools = false + mind.PromptPrice = 0.005 + mind.CompletionPrice = 0.01 + mind.HasCacheRead = false + pool = append(pool, worker, mind) + pool = append(pool, donors("v", 3)...) + + front := Front(pool, DefaultShapes(), All) + if len(front) == 0 { + t.Fatal("the pool must field crews") + } + frugal, _, _ := Presets(front) + if frugal.Worker != "b/x" || frugal.Mastermind != "d/m" { + t.Fatalf("frugal = %s|%s|%s, want b/x on the worker seat and d/m on the mastermind's", frugal.Worker, frugal.High, frugal.Mastermind) + } + want := [3][3]bool{{false, true, true}, {}, {true, false, true}} + if frugal.Estimated != want { + t.Fatalf("the crew's Estimated = %v, want %v", frugal.Estimated, want) + } +} + +// Two candidates for a seat otherwise tied on quality and cost run to the +// one with fewer estimated indexes — even when its id reads later, and +// whichever way round the pool arrives. +func TestATieOnQualityAndCostRunsToFewerEstimatedIndexes(t *testing.T) { + pool := []Candidate{model("a/w"), model("c/h")} + // Twins for the mastermind seat, identical in every number the picker + // reads, but aaa runs on two estimated indexes and zzz on none. + aaa := model("aaa/two") + aaa.Intelligence = 0 + aaa.Coding = 80 + aaa.Agentic = 0 + aaa.Tools = false + aaa.Images = false + aaa.PromptPrice = 0.1 + aaa.CompletionPrice = 0.2 + aaa.HasCacheRead = false + zzz := aaa + zzz.ID = "zzz/one" + zzz.Intelligence = 80 + zzz.Agentic = 80 + pool = append(pool, aaa, zzz) + pool = append(pool, donors("v", 3)...) + + front := Front(pool, DefaultShapes(), All) + if len(front) == 0 { + t.Fatal("the pool must field crews") + } + if front[0].Mastermind != "zzz/one" { + t.Fatalf("a tie went to the estimated twin: %+v", front[0]) + } + for _, crew := range front { + if crew.Mastermind == "aaa/two" { + t.Fatalf("the estimated twin survived its tie: %+v", crew) + } + } + + reversed := slices.Clone(pool) + slices.Reverse(reversed) + if two := Front(reversed, DefaultShapes(), All); !slices.Equal(front, two) { + t.Fatal("the tie broke differently when the pool arrived reversed") + } +} + +// Estimates, caps and ties read the same whatever order the candidates +// arrive in: a pool of partially published models gives the same front +// shuffled as it does in hand order. +func TestPartialCandidatesGiveTheSameFrontInAnyOrder(t *testing.T) { + newRelease := model("b/new") + newRelease.Coding = 0 + newRelease.Agentic = 0 + newRelease.PromptPrice = 0.01 + newRelease.CompletionPrice = 0.02 + newRelease.HasCacheRead = false + older := model("d/old") + older.Intelligence = 0 + older.Agentic = 0 + older.Tools = false + staged := model("e/staged") + staged.Agentic = 0 + pool := []Candidate{model("a/w"), model("c/h"), newRelease, older, staged} + pool = append(pool, donors("v", 3)...) + + one := Front(pool, DefaultShapes(), All) + if len(one) == 0 { + t.Fatal("the pool must field crews") + } + if one[0].Estimated == ([3][3]bool{}) { + t.Fatal("the cheapest crew carries no estimated index; the partial candidates never reached it") + } + reversed := slices.Clone(pool) + slices.Reverse(reversed) + if two := Front(reversed, DefaultShapes(), All); !slices.Equal(one, two) { + t.Fatal("the front changed when the partial candidates arrived reversed") + } + rotated := append(slices.Clone(pool[3:]), pool[:3]...) + if three := Front(rotated, DefaultShapes(), All); !slices.Equal(one, three) { + t.Fatal("the front changed when the pool arrived rotated") + } +} + +// An input with no index missing picks on measurements alone: the fixture's +// partially published row left out, both fronts hold the same sizes and +// the same frugal, balanced and max picks the whole fixture holds, and no +// estimated flag is set anywhere — the partial row never raised a ceiling, +// so leaving it out moves nothing measured. +func TestNothingMissingLeavesEveryPickUnchanged(t *testing.T) { + cands := loadCandidates(t) + var full []Candidate + for _, c := range cands { + if c.Intelligence > 0 && c.Coding > 0 && c.Agentic > 0 { + full = append(full, c) + } + } + shapes := DefaultShapes() + + open := Front(full, shapes, Open) + if len(open) != 7 { + t.Fatalf("the full open front holds %d crews, want 7", len(open)) + } + frugal, balanced, max := Presets(open) + wantCrew(t, "open frugal", frugal, "deepseek/deepseek-v4-flash-0731|z-ai/glm-5.3-flash|z-ai/glm-5.3-flash") + wantCrew(t, "open balanced", balanced, "z-ai/glm-5.3-flash|moonshotai/kimi-k3|z-ai/glm-5.3") + wantCrew(t, "open max", max, "z-ai/glm-5.3|moonshotai/kimi-k3|z-ai/glm-5.3") + + all := Front(full, shapes, All) + if len(all) != 18 { + t.Fatalf("the full all front holds %d crews, want 18", len(all)) + } + frugal, balanced, max = Presets(all) + wantCrew(t, "all frugal", frugal, "z-ai/glm-5.3-flash|google/gemini-3.8-flash|z-ai/glm-5.3-flash") + wantCrew(t, "all balanced", balanced, "z-ai/glm-5.3-flash|qwen/qwen3.8-max-0902|qwen/qwen3.8-max-0902") + wantCrew(t, "all max", max, "anthropic/claude-fable-5.1|openai/gpt-6-astra|anthropic/claude-fable-5.1") + if math.Abs(balanced.Bill-0.147) > 0.005 { + t.Fatalf("all balanced bill = %f, want about 0.147", balanced.Bill) + } + if math.Abs(balanced.Quality-89.2) > 0.05 { + t.Fatalf("all balanced quality = %f, want about 89.2", balanced.Quality) + } + for _, front := range [][]Crew{open, all} { + for _, crew := range front { + if crew.Estimated != ([3][3]bool{}) { + t.Fatalf("a fully measured pool raised an estimated flag: %+v", crew) + } + } + } +} + +// The knob runs from one end of the front to the other: at 0 the budget is +// the cheapest bill and the pick is frugal, at 1 it is the dearest and the +// pick is max. +func TestAtKnobAtTheEndsReadsFrugalAndMax(t *testing.T) { + cands := loadCandidates(t) + for _, fam := range []Family{Open, All} { + front := Front(cands, DefaultShapes(), fam) + frugal, _, max := Presets(front) + if got := AtKnob(front, 0); got != frugal { + t.Fatalf("AtKnob at 0 = %+v, want the frugal %+v", got, frugal) + } + if got := AtKnob(front, 1); got != max { + t.Fatalf("AtKnob at 1 = %+v, want the max %+v", got, max) + } + } +} + +// The front is a property of the candidates, not of their order: the running +// for each seat is read in id order, so the same list gives the same front +// however it arrives. +func TestTheSameCandidatesGiveTheSameFrontInAnyOrder(t *testing.T) { + cands := loadCandidates(t) + shapes := DefaultShapes() + one := Front(cands, shapes, All) + + reversed := slices.Clone(cands) + slices.Reverse(reversed) + if two := Front(reversed, shapes, All); !slices.Equal(one, two) { + t.Fatal("the front changed when the candidates arrived reversed") + } +} + +// Two crews that tie in (bill, quality) are one point on the front, and the +// point keeps the crew whose ids read first — worker, then high, then +// mastermind — whichever way round the pool arrived. +func TestTiesInBillAndQualityBreakInIdOrder(t *testing.T) { + // Two candidates identical in every way but the id, and able to sit + // only the mastermind seat: every crew one finishes ties with the crew + // the other finishes. + aaa := model("aaa/x") + aaa.Tools = false + aaa.Images = false + aaa.PromptPrice = 0.1 + aaa.CompletionPrice = 0.2 + aaa.HasCacheRead = false + bbb := aaa + bbb.ID = "bbb/x" + + pool := []Candidate{model("a/m"), model("c/h"), aaa, bbb} + one := Front(pool, DefaultShapes(), All) + if len(one) == 0 { + t.Fatal("the pool must field crews") + } + for _, crew := range one { + if crew.Mastermind != "aaa/x" { + t.Fatalf("a tie went to the wrong twin: %+v", crew) + } + } + + reversed := slices.Clone(pool) + slices.Reverse(reversed) + if two := Front(reversed, DefaultShapes(), All); !slices.Equal(one, two) { + t.Fatal("the tie broke differently when the pool arrived reversed") + } +} + +// The seat's cost is its token mix priced: the prompt price blended with the +// cache-read price by the cache share, and the completion price spread over +// the input-to-output ratio. A model that publishes no cache-read price pays +// the prompt price on the cache share too. +func TestSeatCostPricesTheTokenMix(t *testing.T) { + shape := SeatShape{InOut: 2, CacheShare: 0.5} + published := Candidate{PromptPrice: 2, CompletionPrice: 8, CacheReadPrice: 0.5, HasCacheRead: true} + if got := SeatCost(published, shape); got != 5.25 { + t.Fatalf("SeatCost with a published cache price = %f, want 5.25", got) + } + unpublished := Candidate{PromptPrice: 2, CompletionPrice: 8} + if got := SeatCost(unpublished, shape); got != 6 { + t.Fatalf("SeatCost with no published cache price = %f, want 6", got) + } +} + +// Quality reads each index against the pool's best for that index, so the +// same candidate scores lower in a stronger pool. A row that is not a +// candidate scores nothing, and so does any row against a pool with no +// candidates to raise a scale. +func TestSeatQualityReadsAgainstThePoolsBest(t *testing.T) { + shape := SeatShape{Weights: [3]float64{0.25, 0.25, 0.5}} + pool := []Candidate{ + {ID: "best", Intelligence: 100, Coding: 100, Agentic: 100, PromptPrice: 1}, + {ID: "half", Intelligence: 50, Coding: 100, Agentic: 100, PromptPrice: 1}, + } + if got := SeatQuality(pool[0], shape, pool); got != 100 { + t.Fatalf("the pool's best scored %f, want 100", got) + } + if got := SeatQuality(pool[1], shape, pool); got != 87.5 { + t.Fatalf("the half scored %f, want 87.5", got) + } + + stronger := append(slices.Clone(pool), + Candidate{ID: "top", Intelligence: 200, Coding: 100, Agentic: 100, PromptPrice: 1}) + if got := SeatQuality(pool[1], shape, stronger); got != 81.25 { + t.Fatalf("the half scored %f against a stronger pool, want 81.25", got) + } + + blank := Candidate{ID: "blank", Coding: 0, PromptPrice: 1} + if got := SeatQuality(blank, shape, pool); got != 0 { + t.Fatalf("a row missing an index scored %f, want 0", got) + } + if got := SeatQuality(pool[0], shape, []Candidate{blank}); got != 0 { + t.Fatalf("a pool with no candidates scored %f, want 0", got) + } +} + +// A pool's own rating for a seat moves that seat's quality towards the +// rating's mean, and the more observations back it the farther it moves: a +// rating with a large count across a seat whose catalog quality is 100 and +// whose pool mean is 50 lands somewhere between the two. +func TestALargeRatingMovesASeatTowardsThePoolMean(t *testing.T) { + shape := SeatShape{Weights: [3]float64{1, 0, 0}} + pool := []Candidate{model("m")} + if base := SeatQuality(pool[0], shape, pool); base != 100 { + t.Fatalf("the catalog quality is %f, want 100", base) + } + prior := Prior{Worker: {"m": {Mean: 50, N: 1000}}} + if got := SeatQualityWith(pool[0], shape, pool, Worker, prior); got <= 50 || got >= 100 { + t.Fatalf("a large rating scored %f, want between the pool mean 50 and the catalog 100", got) + } +} + +// The blend carries the rating and the catalog at equal weight at exactly +// PriorWeightAt observations, so the seat reads the midpoint of the two; a +// seat the prior never rates, and a rating with no observations, keep the +// catalog quality. +func TestTheBlendAtTheRatingWeightIsTheMidpoint(t *testing.T) { + shape := SeatShape{Weights: [3]float64{1, 0, 0}} + pool := []Candidate{model("m")} + + half := Prior{Worker: {"m": {Mean: 40, N: PriorWeightAt}}} + if got := SeatQualityWith(pool[0], shape, pool, Worker, half); got != 70 { + t.Fatalf("the blend at N = PriorWeightAt scored %f, want the midpoint 70", got) + } + unrated := Prior{Worker: {"other": {Mean: 40, N: 1000}}} + if got := SeatQualityWith(pool[0], shape, pool, Worker, unrated); got != 100 { + t.Fatalf("a seat the prior never rated scored %f, want the catalog 100", got) + } + empty := Prior{Worker: {"m": {Mean: 40, N: 0}}} + if got := SeatQualityWith(pool[0], shape, pool, Worker, empty); got != 100 { + t.Fatalf("a rating with no observations scored %f, want the catalog 100", got) + } +} + +// A prior is read from role-worded cells: a role that names no seat and a +// count below min_installs are both ignored, and only a role that names a +// seat with enough observations behind it becomes a rating. +func TestPriorFromCellsIgnoresACountBelowMinInstalls(t *testing.T) { + prior := PriorFromCells([]Cell{ + {Role: "worker", Model: "a/kept", Mean: 80, N: 40}, + {Role: "worker", Model: "b/dropped", Mean: 90, N: 10}, + {Role: "engineer", Model: "c/ignored", Mean: 95, N: 100}, + }, 30, nil) + if len(prior) != 1 || len(prior[Worker]) != 1 { + t.Fatalf("prior = %v, want one worker rating", prior) + } + if _, ok := prior[Worker]["a/kept"]; !ok { + t.Fatalf("a rating at the floor was dropped: %v", prior) + } + if _, ok := prior[Worker]["b/dropped"]; ok { + t.Fatalf("a rating below min_installs survived: %v", prior) + } +} + +// The result says which seats a pool's rating entered: a seat whose model the +// prior rated reads measured, and a seat it did not — here the mastermind, +// which no rating names — does not. +func TestMeasuredIsSetOnlyOnTheSeatsThePriorTouched(t *testing.T) { + pool := []Candidate{model("a/w"), model("c/h"), model("b/m")} + prior := Prior{ + Worker: {"a/w": {Mean: 100, N: 100}}, + High: {"b/m": {Mean: 100, N: 100}}, + } + front := FrontWith(pool, DefaultShapes(), All, prior) + if len(front) == 0 { + t.Fatal("the pool must field crews") + } + wantCrew(t, "frugal", front[0], "a/w|b/m|a/w") + if front[0].Measured != ([3]bool{true, true, false}) { + t.Fatalf("Measured = %v, want the worker and high seats alone", front[0].Measured) + } +} + +// A prior changes quality alone: every bill a crew is given is its seats' +// token mixes priced, exactly as it is without a prior. +func TestAPriorNeverChangesTheBill(t *testing.T) { + pool := []Candidate{model("a/w"), model("c/h"), model("b/m")} + shapes := DefaultShapes() + prior := Prior{ + Worker: {"a/w": {Mean: 20, N: 10_000}}, + High: {"c/h": {Mean: 140, N: 10_000}}, + } + front := FrontWith(pool, shapes, All, prior) + if len(front) == 0 { + t.Fatal("the pool must field crews") + } + for _, crew := range front { + want := 0.0 + for _, seat := range []struct { + model string + seat Seat + }{{crew.Worker, Worker}, {crew.High, High}, {crew.Mastermind, Mastermind}} { + for _, c := range pool { + if c.ID == seat.model { + want += shapes[seat.seat].Volume * SeatCost(c, shapes[seat.seat]) + } + } + } + if math.Abs(crew.Bill-want) > 1e-9 { + t.Fatalf("crew %s|%s|%s bill = %f, want its seats' priced mix %f", + crew.Worker, crew.High, crew.Mastermind, crew.Bill, want) + } + } +} + +// A prior is a property of the candidates, not of their order: with a rating +// present the same pool gives the same front shuffled, rotated and reversed. +func TestAPriorKeepsTheFrontTheSameInAnyOrder(t *testing.T) { + cands := loadCandidates(t) + prior := Prior{Worker: {"z-ai/glm-5.3": {Mean: 90, N: 100}}} + one := FrontWith(cands, DefaultShapes(), All, prior) + if len(one) == 0 { + t.Fatal("the pool must field crews") + } + reversed := slices.Clone(cands) + slices.Reverse(reversed) + if two := FrontWith(reversed, DefaultShapes(), All, prior); !slices.Equal(one, two) { + t.Fatal("the front changed when the candidates arrived reversed") + } + rotated := append(slices.Clone(cands[3:]), cands[:3]...) + if three := FrontWith(rotated, DefaultShapes(), All, prior); !slices.Equal(one, three) { + t.Fatal("the front changed when the pool arrived rotated") + } +} + +// An alias resolved through canonical meets its candidate: a cell spelling a +// model's alias lands on the candidate's canonical id, so the rating reaches +// it, while the same cell with no canonical step lands on the alias and +// touches nothing. +func TestAnAliasResolvedThroughCanonicalMeetsItsCandidate(t *testing.T) { + shape := SeatShape{Weights: [3]float64{1, 0, 0}} + pool := []Candidate{model("z-ai/glm-5.3")} + canonical := func(id string) string { + if id == "glm-5.3" { + return "z-ai/glm-5.3" + } + return id + } + cell := Cell{Role: "worker", Model: "glm-5.3", Mean: 40, N: PriorWeightAt} + resolved := PriorFromCells([]Cell{cell}, 5, canonical) + if got := SeatQualityWith(pool[0], shape, pool, Worker, resolved); got != 70 { + t.Fatalf("the aliased rating scored %f, want the midpoint 70", got) + } + unresolved := PriorFromCells([]Cell{cell}, 5, nil) + if got := SeatQualityWith(pool[0], shape, pool, Worker, unresolved); got != 100 { + t.Fatalf("the unresolved alias scored %f, want the catalog 100 untouched", got) + } +} + +// Two priors fold into one: a rating both hold for a model is weighted by the +// counts behind it, a rating either holds alone is carried whole, and a nil on +// either side is the other side. +func TestMergePriorsFoldsTwoPriorsByObservationCount(t *testing.T) { + if got := MergePriors(nil, nil); got != nil { + t.Fatalf("two nils folded to %v, want nil", got) + } + a := Prior{Worker: {"a/one": {Mean: 80, N: 20}}} + if got := MergePriors(a, nil); got == nil || got[Worker]["a/one"].N != 20 { + t.Fatalf("a nil on one side is not the other: %v", got) + } + b := Prior{High: {"b/two": {Mean: 60, N: 5}}} + if got := MergePriors(nil, b); got == nil || got[High]["b/two"].N != 5 { + t.Fatalf("a nil on one side is not the other: %v", got) + } + + // A model only one prior names is carried whole, in either order. + onlyA := Prior{Worker: {"a/only": {Mean: 70, N: 4}}} + onlyB := Prior{Worker: {"b/only": {Mean: 90, N: 6}}} + got := MergePriors(onlyA, onlyB) + if len(got[Worker]) != 2 { + t.Fatalf("no shared key folded to %v, want the union", got) + } + if r := got[Worker]["a/only"]; r.Mean != 70 || r.N != 4 { + t.Fatalf("a carried rating moved: %v", r) + } + if r := got[Worker]["b/only"]; r.Mean != 90 || r.N != 6 { + t.Fatalf("a carried rating moved: %v", r) + } + + // A model both name: the counts add and the mean is the mean of the means + // weighted by them — (10*50 + 30*90) / 40 = 80. + sharedA := Prior{Worker: {"c/both": {Mean: 50, N: 10}}} + sharedB := Prior{Worker: {"c/both": {Mean: 90, N: 30}}} + if r := MergePriors(sharedA, sharedB)[Worker]["c/both"]; r.Mean != 80 || r.N != 40 { + t.Fatalf("a shared rating folded to %v, want mean 80 over 40", r) + } + + // The priors given are read and never changed. + if r := sharedA[Worker]["c/both"]; r.Mean != 50 || r.N != 10 { + t.Fatalf("the merge moved its first prior: %v", r) + } + if r := sharedB[Worker]["c/both"]; r.Mean != 90 || r.N != 30 { + t.Fatalf("the merge moved its second prior: %v", r) + } +} + +// A stake of zero buys the cheapest crew and a stake past every bill the +// best, and between them the pick walks up the front and never back down. +func TestAtStakeWalksUpTheFrontAsTheStakeRises(t *testing.T) { + front := Front(loadCandidates(t), DefaultShapes(), All) + if got := AtStake(front, 0, TypicalTaskTokens); got != front[0] { + t.Fatalf("stake 0 picked %s|%s|%s, want the cheapest", got.Worker, got.High, got.Mastermind) + } + if got := AtStake(front, 1e9, TypicalTaskTokens); got != front[len(front)-1] { + t.Fatalf("a boundless stake picked %s|%s|%s, want the best", got.Worker, got.High, got.Mastermind) + } + last := -1 + for _, stake := range []float64{0, 0.5, 1, 2, 5, 10, 20, 50, 100, 1000} { + pick := AtStake(front, stake, TypicalTaskTokens) + at := -1 + for i, crew := range front { + if crew == pick { + at = i + } + } + if at < last { + t.Fatalf("stake %v picked crew %d after stake before it picked %d: the walk went back down", stake, at, last) + } + last = at + } + if AtStake(nil, 10, TypicalTaskTokens) != (Crew{}) { + t.Fatal("an empty front picked something") + } +} + +// Fifty dollars of stake over a typical task buys the fable checker the +// ten-dollar balanced pick declines; the numbers behind the two words are +// the point of having units. +func TestTheStakeIsWhatBuysTheDearerChecker(t *testing.T) { + front := Front(loadCandidates(t), DefaultShapes(), All) + ten := AtStake(front, 10, TypicalTaskTokens) + fifty := AtStake(front, 50, TypicalTaskTokens) + if ten.High != "qwen/qwen3.8-max-0902" { + t.Fatalf("ten dollars checks with %s, want qwen/qwen3.8-max-0902", ten.High) + } + if fifty.High != "anthropic/claude-fable-5.1" { + t.Fatalf("fifty dollars checks with %s, want anthropic/claude-fable-5.1", fifty.High) + } + if !(fifty.Bill > ten.Bill && fifty.Quality > ten.Quality) { + t.Fatalf("fifty (%.3f, %.1f) should bill and score above ten (%.3f, %.1f)", fifty.Bill, fifty.Quality, ten.Bill, ten.Quality) + } +} + +// No tasks leave the defaults untouched; one task barely moves them; many +// tasks that all seat the high seat at a third of the tokens pull its volume +// most of the way there, and a seat no task carried keeps its default. The +// share is learned on the log scale, so one task that put everything on the +// high seat cannot drag the typical share the way an arithmetic mean would. +func TestShapesFromShrinksTowardsWhatTasksSpent(t *testing.T) { + defaults := DefaultShapes() + if got := ShapesFrom(defaults, nil); got[High] != defaults[High] || got[Worker] != defaults[Worker] { + t.Fatal("no tasks changed a shape") + } + one := []TaskUsage{{Worker: {In: 660_000, Out: 6_600}, High: {In: 330_000, Out: 3_300}}} + got := ShapesFrom(defaults, one) + w := 1.0 / (1 + ShapeWeightAt) + wantHigh := math.Exp((1-w)*math.Log(defaults[High].Volume) + w*math.Log(333_300.0/999_900)) + if math.Abs(got[High].Volume-wantHigh) > 1e-6 { + t.Fatalf("one task moved the high volume to %f, want %f", got[High].Volume, wantHigh) + } + if got[Mastermind] != defaults[Mastermind] { + t.Fatal("a seat no task carried moved") + } + many := make([]TaskUsage, 300) + for i := range many { + many[i] = one[0] + } + got = ShapesFrom(defaults, many) + // In log space the default's tenth of the weight still pulls the share a + // little below a third — 0.293 — where an arithmetic blend would sit at + // 0.31; both are most of the way there, and the tail test below is why + // the log scale is the one kept. + if got[High].Volume < 0.28 || got[High].Volume > 0.3333 { + t.Fatalf("three hundred tasks left the high volume at %f, want close to a third", got[High].Volume) + } + if math.Abs(got[High].InOut-(100*300.0/(300+ShapeWeightAt)+defaults[High].InOut*ShapeWeightAt/(300+ShapeWeightAt))) > 1e-6 { + t.Fatalf("in/out = %f", got[High].InOut) + } + empty := []TaskUsage{{Worker: {}}, {}} + if got := ShapesFrom(defaults, empty); got[Worker] != defaults[Worker] { + t.Fatal("tasks with no tokens taught something") + } +} + +// Two cells for one seat and model — a graded source and a seeded one — are +// one rating, their means weighted by their counts, however they arrive. +func TestPriorFromCellsFoldsTwoSourcesOfOneSeat(t *testing.T) { + cells := []Cell{ + {Role: "worker", Model: "a/b", Mean: 100, N: 10}, + {Role: "worker", Model: "a/b", Mean: 50, N: 30}, + } + prior := PriorFromCells(cells, 1, nil) + got := prior[Worker]["a/b"] + if got.N != 40 || math.Abs(got.Mean-62.5) > 1e-9 { + t.Fatalf("folded to %+v, want mean 62.5 over 40", got) + } + reversed := PriorFromCells([]Cell{cells[1], cells[0]}, 1, nil) + if reversed[Worker]["a/b"] != got { + t.Fatal("order changed the fold") + } +} + +// A single task that put nearly everything on the high seat moves the +// learned share far less than an arithmetic mean would: the shape is learned +// on the log scale, where a tail task is one vote and not most of the sum. +func TestShapesFromLearnsTheShareOnTheLogScale(t *testing.T) { + defaults := DefaultShapes() + typical := TaskUsage{Worker: {In: 970_000, Out: 9_700}, High: {In: 30_000, Out: 300}} + tail := TaskUsage{Worker: {In: 50_000, Out: 500}, High: {In: 950_000, Out: 9_500}} + tasks := make([]TaskUsage, 0, 40) + for i := 0; i < 39; i++ { + tasks = append(tasks, typical) + } + tasks = append(tasks, tail) + got := ShapesFrom(defaults, tasks)[High].Volume + arithmetic := (1-40.0/70)*defaults[High].Volume + 40.0/70*((39*0.03+0.95)/40) + if got >= arithmetic { + t.Fatalf("the log-scale share %f is not below the arithmetic %f", got, arithmetic) + } + if got < 0.03 || got > 0.06 { + t.Fatalf("thirty-nine typical tasks and one tail task learned a high share of %f, want close to the typical 0.03", got) + } +} diff --git a/internal/crewpick/testdata/candidates.json b/internal/crewpick/testdata/candidates.json new file mode 100644 index 0000000000..493a31a519 --- /dev/null +++ b/internal/crewpick/testdata/candidates.json @@ -0,0 +1 @@ +[{"id": "inclusionai/ling-3.0-flash-vl", "open": true, "I": 25, "C": 57, "A": 30, "pi": 0.06, "po": 0.18, "cr": 0.012, "ctx": 131072, "img": true, "tools": true, "so": true, "created": 1789056114, "exp": null, "alias": null}, {"id": "deepseek/deepseek-v4.1-flash", "open": true, "I": 39.5, "C": null, "A": null, "pi": 0.3, "po": 1.2, "cr": 0.006, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1789021285, "exp": null, "alias": null}, {"id": "openai/gpt-6-astra", "open": false, "I": 52.8, "C": 76.9, "A": 51.5, "pi": 10.0, "po": 50.0, "cr": 1.0, "ctx": 1050000, "img": true, "tools": true, "so": true, "created": 1788552838, "exp": null, "alias": null}, {"id": "qwen/qwen3.8-max-0902", "open": false, "I": 45.4, "C": 76.2, "A": 56.1, "pi": 2.0, "po": 6.0, "cr": 0.25, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1788469704, "exp": null, "alias": null}, {"id": "google/gemini-3.8-flash", "open": false, "I": 41.2, "C": 76.3, "A": 41.1, "pi": 0.75, "po": 3.75, "cr": 0.075, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1788362056, "exp": null, "alias": null}, {"id": "anthropic/claude-fable-5.1", "open": false, "I": 53.4, "C": 81.6, "A": 58, "pi": 10.0, "po": 50.0, "cr": 0.25, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1788285838, "exp": null, "alias": null}, {"id": "ibm-granite/granite-4.2-8b", "open": true, "I": 11.8, "C": 22.4, "A": 3.7, "pi": 0.06, "po": 0.25, "cr": 0.015, "ctx": 131072, "img": false, "tools": true, "so": true, "created": 1788206780, "exp": null, "alias": null}, {"id": "inclusionai/ling-3.0-flash-fin", "open": false, "I": 23, "C": 55.6, "A": 29.3, "pi": 0.06, "po": 0.18, "cr": 0.012, "ctx": 262144, "img": false, "tools": true, "so": true, "created": 1787846290, "exp": null, "alias": null}, {"id": "z-ai/glm-5.3-flash", "open": true, "I": 41.9, "C": 71.5, "A": 51.2, "pi": 0.09, "po": 0.3, "cr": 0.018, "ctx": 1310720, "img": true, "tools": true, "so": true, "created": 1787752741, "exp": null, "alias": null}, {"id": "z-ai/glm-5.3", "open": true, "I": 44.9, "C": 74.8, "A": 53.4, "pi": 1.4, "po": 4.4, "cr": 0.26, "ctx": 1310720, "img": false, "tools": true, "so": true, "created": 1787086655, "exp": null, "alias": null}, {"id": "qwen/qwen3.8-27b", "open": true, "I": 33.9, "C": 68.1, "A": 46.5, "pi": 0.21400000000000002, "po": 2.5500000000000003, "cr": 0.15, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1786722910, "exp": null, "alias": null}, {"id": "google/gemini-3.7-flash", "open": false, "I": 39.4, "C": 76.1, "A": 36.4, "pi": 0.75, "po": 3.75, "cr": 0.075, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1786640581, "exp": null, "alias": null}, {"id": "qwen/qwen3.8-2.4t-a95b", "open": true, "I": 40, "C": 71.9, "A": 50.4, "pi": 2.0, "po": 6.0, "cr": 0.25, "ctx": 1048576, "img": false, "tools": true, "so": true, "created": 1786551702, "exp": null, "alias": null}, {"id": "deepseek/deepseek-v4-pro-0813", "open": true, "I": 36.3, "C": 68.8, "A": 42.3, "pi": 1.32, "po": 3.9600000000000004, "cr": 0.044, "ctx": 1048576, "img": false, "tools": true, "so": true, "created": 1786549364, "exp": null, "alias": null}, {"id": "x-ai/grok-4.6", "open": false, "I": 44.4, "C": 76.8, "A": 53.4, "pi": 2.0, "po": 6.0, "cr": 0.5, "ctx": 500000, "img": true, "tools": true, "so": true, "created": 1786548957, "exp": null, "alias": null}, {"id": "nvidia/nemotron-3.5-lightning", "open": true, "I": 13.6, "C": 26.8, "A": 6.1, "pi": 0.08, "po": 0.19999999999999998, "cr": 0.04, "ctx": 262144, "img": false, "tools": true, "so": true, "created": 1786452751, "exp": null, "alias": null}, {"id": "meta/muse-spark-1.2", "open": false, "I": 39.8, "C": 72.2, "A": 44, "pi": 1.25, "po": 4.25, "cr": 0.15, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1785959287, "exp": null, "alias": null}, {"id": "deepseek/deepseek-v4-flash-0731", "open": true, "I": 34.5, "C": 69.1, "A": 41.7, "pi": 0.06, "po": 0.12, "cr": 0.012, "ctx": 1310720, "img": false, "tools": true, "so": true, "created": 1785478908, "exp": null, "alias": null}, {"id": "thinkingmachines/inkling-small", "open": true, "I": 26.1, "C": 52.9, "A": 25, "pi": 0.44999999999999996, "po": 1.2, "cr": 0.09999999999999999, "ctx": 1048576, "img": true, "tools": true, "so": false, "created": 1785443117, "exp": null, "alias": null}, {"id": "anthropic/claude-opus-5", "open": false, "I": 50.7, "C": 78, "A": 56.2, "pi": 5.0, "po": 25.0, "cr": 0.5, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1784912544, "exp": null, "alias": null}, {"id": "inclusionai/ling-3.0-flash", "open": true, "I": 20.6, "C": 50.6, "A": 21, "pi": 0.020999999999999998, "po": 0.063, "cr": 0.004200000000000001, "ctx": 262144, "img": false, "tools": true, "so": false, "created": 1784818580, "exp": null, "alias": null}, {"id": "google/gemini-3.6-flash", "open": false, "I": 34.3, "C": 69.2, "A": 30.2, "pi": 0.75, "po": 3.75, "cr": 0.075, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1784646733, "exp": null, "alias": null}, {"id": "google/gemini-3.5-flash-lite", "open": false, "I": 22.7, "C": 49.3, "A": 15.9, "pi": 0.3, "po": 2.5, "cr": 0.03, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1784646726, "exp": null, "alias": null}, {"id": "meituan/longcat-2.0", "open": true, "I": 19.7, "C": 45.3, "A": 15.9, "pi": 0.3, "po": 1.2, "cr": 0.006, "ctx": 1048756, "img": false, "tools": true, "so": false, "created": 1784554658, "exp": null, "alias": null}, {"id": "thinkingmachines/inkling", "open": true, "I": 25.5, "C": 52.1, "A": 24.3, "pi": 1.0, "po": 4.05, "cr": 0.16999999999999998, "ctx": 1048576, "img": true, "tools": true, "so": false, "created": 1784325956, "exp": null, "alias": null}, {"id": "moonshotai/kimi-k3", "open": true, "I": 43.8, "C": 76.2, "A": 50.6, "pi": 3.0, "po": 15.0, "cr": 0.3, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1784215858, "exp": null, "alias": null}, {"id": "meta/muse-spark-1.1", "open": false, "I": 34.3, "C": 71.3, "A": 27.5, "pi": 1.25, "po": 4.25, "cr": 0.15, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1784215741, "exp": null, "alias": null}, {"id": "openai/gpt-5.6-luna", "open": false, "I": 37.5, "C": 71.4, "A": 42.7, "pi": 0.19999999999999998, "po": 1.2, "cr": 0.02, "ctx": 1050000, "img": true, "tools": true, "so": true, "created": 1783590864, "exp": null, "alias": null}, {"id": "openai/gpt-5.6-terra", "open": false, "I": 42.3, "C": 76.7, "A": 43.7, "pi": 2.0, "po": 12.0, "cr": 0.19999999999999998, "ctx": 1050000, "img": true, "tools": true, "so": true, "created": 1783590857, "exp": null, "alias": null}, {"id": "openai/gpt-5.6-sol", "open": false, "I": 47.1, "C": 77.4, "A": 50.5, "pi": 2.0, "po": 10.0, "cr": 0.19999999999999998, "ctx": 1050000, "img": true, "tools": true, "so": true, "created": 1783590850, "exp": null, "alias": null}, {"id": "x-ai/grok-4.5", "open": false, "I": 39.1, "C": 72.4, "A": 42.1, "pi": 2.0, "po": 6.0, "cr": 0.3, "ctx": 500000, "img": true, "tools": true, "so": true, "created": 1783523154, "exp": null, "alias": null}, {"id": "anthropic/claude-sonnet-5", "open": false, "I": 38.4, "C": 71.5, "A": 44.3, "pi": 2.0, "po": 10.0, "cr": 0.19999999999999998, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1782843083, "exp": null, "alias": null}, {"id": "z-ai/glm-5.2", "open": true, "I": 34, "C": 68.8, "A": 39.4, "pi": 1.4, "po": 4.4, "cr": 0.14, "ctx": 1048576, "img": false, "tools": true, "so": true, "created": 1781631930, "exp": null, "alias": null}, {"id": "moonshotai/kimi-k2.7-code", "open": true, "I": 26.3, "C": 60.8, "A": 22.5, "pi": 0.7062, "po": 3.21, "cr": 0.18, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1781266361, "exp": null, "alias": null}, {"id": "anthropic/claude-fable-5", "open": false, "I": 49.7, "C": 76.5, "A": 51, "pi": 10.0, "po": 50.0, "cr": 1.0, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1781007515, "exp": null, "alias": null}, {"id": "nvidia/nemotron-3-ultra-550b-a55b", "open": true, "I": 23.4, "C": 49.3, "A": 21.7, "pi": 0.625, "po": 3.125, "cr": 0.1875, "ctx": 262144, "img": false, "tools": true, "so": true, "created": 1780551208, "exp": null, "alias": null}, {"id": "qwen/qwen3.7-plus", "open": false, "I": 25.8, "C": 55.9, "A": 19.7, "pi": 0.32, "po": 1.28, "cr": 0.064, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1780491783, "exp": null, "alias": null}, {"id": "minimax/minimax-m3", "open": true, "I": 29.6, "C": 58.6, "A": 30.8, "pi": 0.3, "po": 1.2, "cr": 0.06, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1780245374, "exp": null, "alias": null}, {"id": "anthropic/claude-opus-4.8", "open": false, "I": 42, "C": 74.3, "A": 42.6, "pi": 5.0, "po": 25.0, "cr": 0.5, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1779905091, "exp": null, "alias": null}, {"id": "qwen/qwen3.7-max", "open": false, "I": 29.9, "C": 66, "A": 23.9, "pi": 1.475, "po": 4.425, "cr": 0.295, "ctx": 1000000, "img": false, "tools": true, "so": true, "created": 1779376861, "exp": null, "alias": null}, {"id": "google/gemini-3.5-flash", "open": false, "I": 33, "C": 70.1, "A": 27.3, "pi": 1.5, "po": 9.0, "cr": 0.15, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1779193800, "exp": null, "alias": null}, {"id": "x-ai/grok-4.3", "open": false, "I": 25.4, "C": 42.2, "A": 17.2, "pi": 1.25, "po": 2.5, "cr": 0.19999999999999998, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1777591821, "exp": null, "alias": null}, {"id": "mistralai/mistral-medium-3-5", "open": false, "I": 14.9, "C": 46.9, "A": 9.4, "pi": 1.5, "po": 7.5, "cr": null, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1777570439, "exp": null, "alias": null}, {"id": "qwen/qwen3.6-35b-a3b", "open": true, "I": 18.8, "C": 41.9, "A": 15, "pi": 0.09999999999999999, "po": 0.8999999999999999, "cr": 0.049999999999999996, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1777260255, "exp": null, "alias": null}, {"id": "qwen/qwen3.6-27b", "open": true, "I": 21.9, "C": 53.7, "A": 20.1, "pi": 0.3, "po": 2.0, "cr": 0.03, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1777255064, "exp": null, "alias": null}, {"id": "openai/gpt-5.5", "open": false, "I": 38.6, "C": 74.9, "A": 37.3, "pi": 5.0, "po": 30.0, "cr": 0.5, "ctx": 1050000, "img": true, "tools": true, "so": true, "created": 1777051893, "exp": null, "alias": null}, {"id": "deepseek/deepseek-v4-pro", "open": true, "I": 30.9, "C": 59.4, "A": 27.7, "pi": 1.5999999999999999, "po": 3.1999999999999997, "cr": 0.135, "ctx": 1048576, "img": false, "tools": true, "so": true, "created": 1777000679, "exp": null, "alias": null}, {"id": "deepseek/deepseek-v4-flash", "open": true, "I": 24.8, "C": 52, "A": 27.9, "pi": 0.088606, "po": 0.177212, "cr": 0.017721200000000003, "ctx": 1048576, "img": false, "tools": true, "so": true, "created": 1777000666, "exp": null, "alias": null}, {"id": "tencent/hy3-preview", "open": true, "I": 25.8, "C": 58.8, "A": 25.6, "pi": 0.18, "po": 0.6, "cr": 0.06, "ctx": 262144, "img": false, "tools": true, "so": false, "created": 1776878150, "exp": null, "alias": null}, {"id": "xiaomi/mimo-v2.5-pro", "open": true, "I": 26.4, "C": 60.2, "A": 22.7, "pi": 0.435, "po": 0.87, "cr": 0.0036, "ctx": 1050000, "img": false, "tools": true, "so": true, "created": 1776874273, "exp": null, "alias": null}, {"id": "xiaomi/mimo-v2.5", "open": true, "I": 22.3, "C": 56.8, "A": 17.4, "pi": 0.14, "po": 0.28, "cr": 0.0028, "ctx": 1050000, "img": true, "tools": true, "so": true, "created": 1776874269, "exp": null, "alias": null}, {"id": "moonshotai/kimi-k2.6", "open": true, "I": 27.5, "C": 61.8, "A": 22.1, "pi": 0.95, "po": 4.0, "cr": 0.16, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1776699402, "exp": null, "alias": null}, {"id": "z-ai/glm-5.1", "open": true, "I": 26.4, "C": 55.8, "A": 25.2, "pi": 0.966, "po": 3.036, "cr": 0.1794, "ctx": 204800, "img": false, "tools": true, "so": true, "created": 1775578025, "exp": null, "alias": null}, {"id": "google/gemma-4-31b-it", "open": true, "I": 15.4, "C": 43.4, "A": 6.7, "pi": 0.09, "po": 0.33999999999999997, "cr": 0.049999999999999996, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1775148486, "exp": null, "alias": null}, {"id": "arcee-ai/trinity-large-thinking", "open": true, "I": 10.9, "C": 25.8, "A": 1.2, "pi": 0.25, "po": 0.7999999999999999, "cr": 0.06, "ctx": 262144, "img": false, "tools": true, "so": false, "created": 1775058318, "exp": null, "alias": null}, {"id": "minimax/minimax-m2.7", "open": true, "I": 23.2, "C": 52.6, "A": 16.8, "pi": 0.3, "po": 1.2, "cr": 0.06, "ctx": 204800, "img": false, "tools": true, "so": true, "created": 1773836697, "exp": null, "alias": null}, {"id": "openai/gpt-5.4-nano", "open": false, "I": 21.2, "C": 56.1, "A": 17.7, "pi": 0.19999999999999998, "po": 1.25, "cr": 0.02, "ctx": 400000, "img": true, "tools": true, "so": true, "created": 1773748187, "exp": null, "alias": null}, {"id": "openai/gpt-5.4-mini", "open": false, "I": 24.6, "C": 56.1, "A": 19.7, "pi": 0.75, "po": 4.5, "cr": 0.075, "ctx": 400000, "img": true, "tools": true, "so": true, "created": 1773748178, "exp": null, "alias": null}, {"id": "mistralai/mistral-small-2603", "open": true, "I": 11.5, "C": 26.6, "A": 1.4, "pi": 0.15, "po": 0.6, "cr": 0.015, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1773695685, "exp": null, "alias": null}, {"id": "nvidia/nemotron-3-super-120b-a12b", "open": true, "I": 13.6, "C": 37.7, "A": 4.1, "pi": 0.08, "po": 0.44999999999999996, "cr": null, "ctx": 262144, "img": false, "tools": true, "so": true, "created": 1773245239, "exp": null, "alias": null}, {"id": "inception/mercury-2", "open": false, "I": 11.5, "C": 31.1, "A": 4, "pi": 0.25, "po": 0.75, "cr": 0.024999999999999998, "ctx": 128000, "img": false, "tools": true, "so": true, "created": 1772636275, "exp": null, "alias": null}, {"id": "google/gemini-3.1-flash-lite-preview", "open": false, "I": 16, "C": 34.7, "A": 3.2, "pi": 0.25, "po": 1.5, "cr": 0.024999999999999998, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1772512673, "exp": null, "alias": null}, {"id": "qwen/qwen3.5-122b-a10b", "open": true, "I": 16.2, "C": 45.7, "A": 9.6, "pi": 0.26, "po": 2.08, "cr": null, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1772053789, "exp": null, "alias": null}, {"id": "google/gemini-3.1-pro-preview", "open": false, "I": 30.4, "C": 68.8, "A": 10.3, "pi": 2.0, "po": 12.0, "cr": 0.19999999999999998, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1771509627, "exp": null, "alias": null}, {"id": "anthropic/claude-sonnet-4.6", "open": false, "I": 30.5, "C": 63, "A": 33.1, "pi": 3.0, "po": 15.0, "cr": 0.3, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1771342990, "exp": null, "alias": null}, {"id": "qwen/qwen3.5-397b-a17b", "open": true, "I": 19.1, "C": 48.2, "A": 10.6, "pi": 0.55, "po": 3.5, "cr": 0.22499999999999998, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1771223018, "exp": null, "alias": null}, {"id": "qwen/qwen3-coder-next", "open": true, "I": 10.1, "C": 36.2, "A": 3.6, "pi": 0.12, "po": 0.7999999999999999, "cr": 0.07, "ctx": 262144, "img": false, "tools": true, "so": true, "created": 1770164101, "exp": null, "alias": null}, {"id": "upstage/solar-pro-3", "open": false, "I": 7.8, "C": 16.2, "A": 1.4, "pi": 0.15, "po": 0.6, "cr": 0.015, "ctx": 131072, "img": false, "tools": true, "so": true, "created": 1769481200, "exp": null, "alias": null}, {"id": "nvidia/nemotron-3-nano-30b-a3b", "open": true, "I": 8.9, "C": 14.4, "A": 1, "pi": 0.06, "po": 0.24, "cr": null, "ctx": 262144, "img": false, "tools": true, "so": true, "created": 1765731275, "exp": null, "alias": null}, {"id": "mistralai/devstral-2512", "open": true, "I": 9.4, "C": 31.3, "A": 4.9, "pi": 0.39999999999999997, "po": 2.0, "cr": 0.04, "ctx": 262144, "img": false, "tools": true, "so": true, "created": 1765285419, "exp": null, "alias": null}, {"id": "mistralai/ministral-14b-2512", "open": true, "I": 6, "C": 14.4, "A": 1.1, "pi": 0.19999999999999998, "po": 0.19999999999999998, "cr": 0.02, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1764681735, "exp": null, "alias": null}, {"id": "mistralai/ministral-8b-2512", "open": true, "I": 5.5, "C": 9.7, "A": 0.6, "pi": 0.15, "po": 0.15, "cr": 0.015, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1764681654, "exp": null, "alias": null}, {"id": "mistralai/ministral-3b-2512", "open": true, "I": 4.8, "C": 4.8, "A": 0.8, "pi": 0.09999999999999999, "po": 0.09999999999999999, "cr": 0.01, "ctx": 131072, "img": true, "tools": true, "so": true, "created": 1764681560, "exp": null, "alias": null}, {"id": "mistralai/mistral-large-2512", "open": false, "I": 9.7, "C": 20.1, "A": 2.4, "pi": 0.5, "po": 1.5, "cr": 0.049999999999999996, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1764624472, "exp": null, "alias": null}, {"id": "anthropic/claude-haiku-4.5", "open": false, "I": 17.6, "C": 43.9, "A": 10.3, "pi": 1.0, "po": 5.0, "cr": 0.09999999999999999, "ctx": 200000, "img": true, "tools": true, "so": true, "created": 1760547638, "exp": null, "alias": null}, {"id": "anthropic/claude-sonnet-4.5", "open": false, "I": 21.2, "C": 52.1, "A": 17.5, "pi": 3.0, "po": 15.0, "cr": 0.3, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1759161676, "exp": null, "alias": null}, {"id": "deepseek/deepseek-v3.1-terminus", "open": true, "I": 15.4, "C": 43.5, "A": 8.9, "pi": 0.27, "po": 1.0, "cr": 0.135, "ctx": 163840, "img": false, "tools": true, "so": true, "created": 1758548275, "exp": null, "alias": null}, {"id": "qwen/qwen3-30b-a3b-thinking-2507", "open": true, "I": 9.8, "C": 12.1, "A": 0.9, "pi": 0.19999999999999998, "po": 2.4, "cr": null, "ctx": 81920, "img": false, "tools": true, "so": false, "created": 1756399192, "exp": null, "alias": null}, {"id": "mistralai/mistral-medium-3.1", "open": false, "I": 9.5, "C": 20.5, "A": 3.1, "pi": 0.39999999999999997, "po": 2.0, "cr": 0.04, "ctx": 131072, "img": true, "tools": true, "so": true, "created": 1755095639, "exp": null, "alias": null}, {"id": "openai/gpt-5-mini", "open": false, "I": 17.4, "C": 15.6, "A": 8.9, "pi": 0.25, "po": 2.0, "cr": 0.024999999999999998, "ctx": 400000, "img": true, "tools": true, "so": true, "created": 1754587407, "exp": null, "alias": null}, {"id": "openai/gpt-oss-120b", "open": true, "I": 12.3, "C": 30.4, "A": 6.2, "pi": 0.037, "po": 0.16999999999999998, "cr": null, "ctx": 131072, "img": false, "tools": true, "so": true, "created": 1754414231, "exp": null, "alias": null}, {"id": "openai/gpt-oss-20b", "open": true, "I": 9, "C": 20.7, "A": 1.4, "pi": 0.03, "po": 0.13, "cr": 0.03, "ctx": 131072, "img": false, "tools": true, "so": true, "created": 1754414229, "exp": null, "alias": null}, {"id": "qwen/qwen3-235b-a22b-thinking-2507", "open": true, "I": 12.7, "C": 22.1, "A": 1.3, "pi": 0.22999999999999998, "po": 2.3, "cr": null, "ctx": 131072, "img": false, "tools": true, "so": false, "created": 1753449557, "exp": null, "alias": null}, {"id": "google/gemini-2.5-pro", "open": false, "I": 16.7, "C": 33.3, "A": 3.5, "pi": 1.25, "po": 10.0, "cr": 0.125, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1750169544, "exp": null, "alias": null}, {"id": "qwen/qwen3-8b", "open": true, "I": 5.2, "C": 9, "A": 0.8, "pi": 0.117, "po": 0.45499999999999996, "cr": null, "ctx": 131072, "img": false, "tools": true, "so": false, "created": 1745876632, "exp": null, "alias": null}, {"id": "qwen/qwen3-14b", "open": true, "I": 6.4, "C": 13.8, "A": 0.9, "pi": 0.12, "po": 0.24, "cr": null, "ctx": 131072, "img": false, "tools": true, "so": true, "created": 1745876478, "exp": null, "alias": null}, {"id": "qwen/qwen3-32b", "open": true, "I": 7.2, "C": 15.3, "A": 0.9, "pi": 0.08, "po": 0.28, "cr": null, "ctx": 131072, "img": false, "tools": true, "so": true, "created": 1745875945, "exp": null, "alias": null}, {"id": "meta-llama/llama-4-maverick", "open": true, "I": 9.3, "C": 16.3, "A": 0.6, "pi": 0.1875, "po": 0.6525, "cr": null, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1743881822, "exp": null, "alias": null}, {"id": "meta-llama/llama-4-scout", "open": true, "I": 6.5, "C": 8.2, "A": 0.5, "pi": 0.09999999999999999, "po": 0.3, "cr": null, "ctx": 1310720, "img": true, "tools": true, "so": true, "created": 1743881519, "exp": null, "alias": null}, {"id": "deepseek/deepseek-chat-v3-0324", "open": true, "I": 9.7, "C": 21.2, "A": 0.8, "pi": 0.25, "po": 1.0, "cr": null, "ctx": 163840, "img": false, "tools": true, "so": true, "created": 1742824755, "exp": null, "alias": null}, {"id": "google/gemma-3-12b-it", "open": true, "I": 3.8, "C": 5.8, "A": 0.1, "pi": 0.049999999999999996, "po": 0.15, "cr": null, "ctx": 131072, "img": true, "tools": true, "so": true, "created": 1741902625, "exp": null, "alias": null}, {"id": "cohere/command-a", "open": true, "I": 13.9, "C": 27.8, "A": 3.6, "pi": 2.5, "po": 10.0, "cr": null, "ctx": 256000, "img": false, "tools": false, "so": true, "created": 1741894342, "exp": null, "alias": null}, {"id": "google/gemma-3-27b-it", "open": true, "I": 4.9, "C": 10.1, "A": 0.1, "pi": 0.08, "po": 0.44999999999999996, "cr": 0.04, "ctx": 131072, "img": true, "tools": true, "so": true, "created": 1741756359, "exp": null, "alias": null}, {"id": "openai/o3-mini-high", "open": false, "I": 11, "C": 16.3, "A": 0.9, "pi": 1.1, "po": 4.4, "cr": 0.55, "ctx": 200000, "img": false, "tools": true, "so": true, "created": 1739372611, "exp": null, "alias": null}, {"id": "deepseek/deepseek-r1", "open": true, "I": 11.4, "C": 24.6, "A": 1.1, "pi": 0.7, "po": 2.5, "cr": null, "ctx": 64000, "img": false, "tools": true, "so": false, "created": 1737381095, "exp": null, "alias": null}] \ No newline at end of file diff --git a/internal/crewroute/chosen.go b/internal/crewroute/chosen.go new file mode 100644 index 0000000000..320993d0af --- /dev/null +++ b/internal/crewroute/chosen.go @@ -0,0 +1,48 @@ +package crewroute + +import ( + "strings" +) + +// applyChosen is the crew with the caller's pick seated where this engine's +// differs: for every unpinned seat the request named ([Request.Chosen]), the +// named model is seated through [pinned] — the same machinery a pin runs, so +// its quality, its cost and its route are read exactly as a pin's would be — +// and then marked NOT pinned, because it is not one. A seat keeps everything +// a routed seat has: the ladder computed after this runs, the caller's +// rescue list, and a receipt that says `routed`. +// +// A pick is left standing when the seat is pinned (a pin is the person's), +// when the request names no model for it, or when no candidate carries the +// named model — a pick made outside the models the person allows cannot be +// seated, and the crew that can is. +func applyChosen(t *table, class Class, crew []Pick, candidates []Candidate, pins map[Seat]Pin, chosen map[Seat]string) []Pick { + out := append([]Pick(nil), crew...) + for i := range out { + pick := out[i] + if pick.Pinned { + continue + } + if _, pinned := pins[pick.Seat]; pinned { + continue + } + want := strings.TrimSpace(chosen[pick.Seat]) + if want == "" || Lineage(want) == Lineage(pick.Model) { + continue + } + offered := false + for _, c := range candidates { + if Lineage(c.Model.ID) == Lineage(want) { + offered = true + break + } + } + if !offered { + continue + } + next := pinned(t, class, pick.Seat, Pin{Model: want}, candidates) + next.Pinned = false + out[i] = next + } + return out +} diff --git a/internal/crewroute/chosen_test.go b/internal/crewroute/chosen_test.go new file mode 100644 index 0000000000..5f6811de87 --- /dev/null +++ b/internal/crewroute/chosen_test.go @@ -0,0 +1,124 @@ +package crewroute + +import ( + "strings" + "testing" +) + +// THE CALLER'S PICK IS SEATED, NOT PINNED. A request that names a model for a +// seat gets that model on the crew — through the same machinery a pin runs, +// so its quality, its cost and its route are read the same way — while the +// receipt still says routed, the ladder is still carried, and the person's +// own pin still wins. This is the seam the pareto selector rides +// (internal/config's wiring); everything a decision promises is unchanged by +// who named the model. +func TestAChosenSeatRunsTheNamedModelAndStaysRouted(t *testing.T) { + cands := catalogCandidates() + d, err := Decide(Request{Class: Bugfix, Candidates: cands, Chosen: map[Seat]string{ + Worker: kimiK3.ID, Checker: glmFlash.ID, + }}) + if err != nil { + t.Fatal(err) + } + if got := d.Seat(Worker); got.Model != kimiK3.ID || got.Pinned { + t.Errorf("worker %+v, want %s and not pinned", got, kimiK3.ID) + } + if got := d.Seat(Checker); got.Model != glmFlash.ID || got.Pinned { + t.Errorf("checker %+v, want %s and not pinned", got, glmFlash.ID) + } + // The seat nobody named keeps the engine's own pick. + if d.Seat(Planner).Model != d.Seat(Planner).Model { + t.Error("the planner vanished") + } + for _, seat := range Seats { + if len(d.Ladder[seat]) == 0 { + t.Errorf("%s carries no ladder — a seat the caller named must still fail over", seat) + } + } + if d.EstUSD <= 0 { + t.Errorf("the estimate was not recomputed for the seated crew: $%.3f", d.EstUSD) + } + // The routed seat reads its route from the candidates, as any pick does. + if got := d.Seat(Worker); got.Send != kimiK3.ID || got.Provider != "openrouter" { + t.Errorf("worker route %+v, want the candidate's own route", got) + } +} + +// A PIN IS THE PERSON'S, and a named seat never overrules one: the pin runs. +func TestAChosenSeatNeverOverrulesAPin(t *testing.T) { + d, err := Decide(Request{Class: Bugfix, Candidates: catalogCandidates(), + Pins: map[Seat]Pin{Worker: {Model: v4Flash.ID, Send: v4Flash.ID, Kind: Metered}}, + Chosen: map[Seat]string{Worker: kimiK3.ID}}) + if err != nil { + t.Fatal(err) + } + got := d.Seat(Worker) + if got.Model != v4Flash.ID || !got.Pinned { + t.Errorf("worker %+v, want the person's pin kept", got) + } +} + +// A PICK MADE OUTSIDE THE CANDIDATES CANNOT BE SEATED: nobody offers the +// model, so the engine's own pick stands rather than a seat nobody can run. +func TestAChosenModelNobodyOffersLeavesThePick(t *testing.T) { + plain, err := Decide(Request{Class: Bugfix, Candidates: catalogCandidates()}) + if err != nil { + t.Fatal(err) + } + d, err := Decide(Request{Class: Bugfix, Candidates: catalogCandidates(), + Chosen: map[Seat]string{Worker: "acme/no-such-model"}}) + if err != nil { + t.Fatal(err) + } + if d.Seat(Worker).Model != plain.Seat(Worker).Model { + t.Errorf("worker %s moved to an unofferable pick, want %s", d.Seat(Worker).Model, plain.Seat(Worker).Model) + } +} + +// A REDO ROUTES BY THIS ENGINE ALONE: the caller's pick is ignored on a redo, +// where the seats move relative to the crew that ran — a redo decided with +// the pick named is the redo decided without it, to the last byte of a pick. +func TestARedoIgnoresTheChosenSeats(t *testing.T) { + cands := catalogCandidates() + ran, err := Decide(Request{Class: OpenEnded, Candidates: cands}) + if err != nil { + t.Fatal(err) + } + chosen := map[Seat]string{Worker: kimiK3.ID, Checker: kimiK3.ID, Planner: kimiK3.ID} + with, err := Decide(Request{Class: OpenEnded, Candidates: cands, Stronger: &ran, Chosen: chosen}) + if err != nil { + t.Fatal(err) + } + without, err := Decide(Request{Class: OpenEnded, Candidates: cands, Stronger: &ran}) + if err != nil { + t.Fatal(err) + } + for _, seat := range Seats { + if with.Seat(seat) != without.Seat(seat) { + t.Errorf("%s: a redo with the pick named is %+v, without it %+v", seat, with.Seat(seat), without.Seat(seat)) + } + } +} + +// NOTHING NAMED, NOTHING CHANGES: a request with no Chosen at all — the +// no-snapshot case — decides exactly the crew it decided before the field +// existed. +func TestNoChosenChangesNothing(t *testing.T) { + cands := frontierCandidates() + plain, err := Decide(Request{Class: OpenEnded, Candidates: cands}) + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(plain.Seat(Worker).Model) == "" { + t.Fatal("the plain decision seated nothing") + } + d, err := Decide(Request{Class: OpenEnded, Candidates: cands, Chosen: nil}) + if err != nil { + t.Fatal(err) + } + for _, seat := range Seats { + if d.Seat(seat) != plain.Seat(seat) { + t.Errorf("%s moved with no Chosen: %+v vs %+v", seat, d.Seat(seat), plain.Seat(seat)) + } + } +} diff --git a/internal/crewroute/route.go b/internal/crewroute/route.go index 3a6af5229b..20098c6944 100644 --- a/internal/crewroute/route.go +++ b/internal/crewroute/route.go @@ -229,6 +229,15 @@ type Request struct { // ([Lineage]). An unpinned seat is not given one while anything else can // sit it; a pin is never overruled. Avoid map[string]bool + // Chosen seats the models a caller has ALREADY PICKED, by seat, for the + // unpinned seats: the pareto selector's crew (internal/config's wiring of + // internal/selector), handed over so the decision, its estimates, its + // ladder and its log are all still this engine's. A seat the person + // pinned never reads it, and a redo ignores it — a redo routes by this + // engine alone. A model nobody offers cannot be seated, so a picked model + // outside the request's candidates leaves the seat as this engine chose + // it. + Chosen map[Seat]string // Learned is this install's move to a model's quality in a seat, by // [LearnKey]: what its tasks kept or redid there ([router.CrewLog]). Learned map[string]float64 @@ -411,6 +420,15 @@ func Decide(r Request) (Decision, error) { if err != nil { return Decision{}, err } + // THE CALLER'S PICK, WHERE THIS ENGINE HAS ONE: a seat the request named + // is seated on the same machinery a pin is ([pinned]) — quality, cost and + // route all read the same way — but it is NOT a pin, so the seat keeps + // its ladder, its rescue and its `routed` receipt. Applied before the + // learned steps, so a repository's offset moves the crew it will + // actually run, whatever seated it. + if len(r.Chosen) > 0 && r.Stronger == nil && r.Again == nil { + crew = applyChosen(t, d.Class, crew, candidates, pins, r.Chosen) + } // A LEARNED OFFSET IS RUNGS, NOT A PRICE: each step is one rung up the // front from the crew the knee picks ([stronger]), so a repository whose // work was redone once starts one rung higher — never at the top. diff --git a/internal/manual/chat/models-and-cost.md b/internal/manual/chat/models-and-cost.md index b95acfdd59..7094c55d4e 100644 --- a/internal/manual/chat/models-and-cost.md +++ b/internal/manual/chat/models-and-cost.md @@ -794,6 +794,31 @@ done so far. The checker's own ceiling still applies inside it. without a routed crew; the flag answers the day's questions and the plan-price question, not this one. +### The selector snapshot — when the crew is picked by the pareto engine + +The crew is normally routed by the learned router, and **nothing else seats a crew unless +you configured it**. There is one extra hand on the picker: a `selector snapshot` row in the +profile's settings (`models.crew.selector_snapshot`), a path to a fitted snapshot file. +Configured, it points the pareto selection engine at the same models your providers reach, +and it picks one model per unpinned seat — the worker seat from its worker class, the +checker from its checker class, the planner from its planner class. + +What stays yours, whatever the file says: **a pin always wins** — a seat you pinned never +reads the engine; **a redo routes as before** — `redo stronger` and a redo of a task that +never started are the learned router's to seat; **a learned offset still applies** — a +repository whose class was redone starts above the engine's pick; and **the engine can +only seat what you allow** — its pool is exactly the candidate list the router already +priced and health-checked, so a demoted route, a turned-off provider and an out-of-credit +account all hold for it too. + +What happens when the file is missing or bad: the crew is routed **exactly as if no +snapshot were configured** — no half-crew, no refusal, no note. A path that does not load +is refused when you write it (`that snapshot does not load: …`); one that stops loading +later silently falls back. A seat the engine cannot seat (no allowed model passes its +capability gate for that seat) keeps the router's own pick for that seat alone. With no +snapshot row the engine never runs at all, and every crew is byte for byte the crew the +router picked before it existed. + ### How hard to try one task — --best and --cheap The panel says what persists. **How hard to try one task is said in the ask, and sticks to diff --git a/internal/pool/contract/CONTRACT.md b/internal/pool/contract/CONTRACT.md new file mode 100644 index 0000000000..7d59e722bc --- /dev/null +++ b/internal/pool/contract/CONTRACT.md @@ -0,0 +1,122 @@ +# Frozen interface contracts — pareto crewing + +Frozen 2026-10-20, per IMPLEMENTATION-EXPERIMENT-PLAN.md §5 ("Contracts frozen +before parallel work"). This package is the wire shape and semantics the three +lanes (evidence, learner, integration) build against. It implements none of +the predictor, policy or grader. + +## Version constants + +Every constant in `contract.go` is named here; a change to a field's MEANING, +its removal, or its redefinition bumps the constant. Adding a field does not +bump a version (forward compatibility: unknown fields are ignored, unknown +versions are refused). + +| Constant | Value | Pins | +| --- | --- | --- | +| `DecisionEventVersion` | 1 | the `DecisionEvent` wire shape and its field semantics | +| `OutcomeEventVersion` | 1 | the `OutcomeEvent` wire shape and its field semantics | +| `PredictorStateVersion` | 1 | the `PredictorEnvelope` export/import envelope (NOT the payload's interior format, which `StateVersion` inside the envelope pins) | +| `SharedSnapshotVersion` | 1 | the `SharedSnapshot` wire shape and its field semantics | +| `ProbTolerance` | 1e-6 | how far a recorded categorical distribution may drift from a sum of exactly 1 | + +Reader rule: a reader that meets a schema version it does not know refuses +with an `ErrVersion`-wrapped error rather than half-reads. The marshal +helpers refuse to write a zero (unset) version. + +## DecisionEvent + +- IDs: `run_id` (whole job), `parent_run_id` (empty at root), `node_id` (the + role/seat-class boundary: reflex, low, worker, high, mastermind). +- `decision_time` is UTC. +- `feature_version` + `features`: the PRE-DECISION feature values. Frozen + allowed keys: `role`, `task_class`, `context_size_tokens`, + `budget_remaining_usd`, `checks_observed`. Everything recorded must be + provable to exist at decision time — a feature observable only after + execution is a future-feature leak and must never appear here. +- `candidates`: the eligible set AFTER all filters (availability, + compatibility, budget); each carries the `descriptor_version` of the + catalog document it was read under. Descriptors themselves live in the + catalog, not in the event. +- `probabilities`: **the exact final categorical distribution** — after all + filters and after the exploration mixture, recorded BEFORE the draw. This + is a hard requirement from the plan: the draw is replayable and the + propensity is declared, not reconstructed. Values in [0,1], sum to 1 within + `ProbTolerance`, and name exactly the `candidates` (eligible-but-unassigned + mass is listed as 0). `selected_action` need not be the argmax but must be + in the set. +- `policy_version`, `catalog_version`, `source_version`: so any decision is + reproducible. +- `budget_envelope`: `cap_usd` (spend this decision may commit from here), + `reserved_usd` (outstanding reservations at decision time), `spent_usd` + (actual spend so far), `currency` (fixed `usd`). + +## OutcomeEvent + +- Linkage: `run_id`, `decision_id`, `job_id` — a reader walks + `DecisionEvent.RunID -> OutcomeEvent.RunID` without guessing. +- Grader provenance: `grader_name` + `grader_version`; a verdict without + provenance is not a label. +- `verified_success`: the WHOLE-JOB label from independent grading (not the + model's own claim), and never copied onto individual seats — one job, one + event; several decisions may link to it. +- `spend`: ACTUAL token/cache/cost accounting (`total_usd`, `tokens_in`, + `tokens_out`, `cached_tokens_in`, `currency`). +- `cost_cap_censored`: set when the run terminated on the cost cap. The + incurred spend above is observable; the unconstrained cost-to-completion + is CENSORED and must be treated as censored, not observed. +- `harness_error`: set when the failure was the harness's, not the model's — + not a success/failure signal about the model. + +## Predictor + +```go +type Predictor interface { + Predict(task FeatureVector, candidates []Candidate) (Prediction, error) + Update(ev OutcomeEvidence) error + Export() ([]byte, error) + Import(blob []byte) error +} +``` + +- `Predict` returns success probability, a cost DISTRIBUTION + (`CostDistribution`: lognormal `(mu, sigma)` or samples — the lognormal + mean is `exp(mu + sigma²/2)`, not `exp(mu)`), and uncertainty. The + exploration layer is a separate decision layer and is not part of predict. +- `Update` takes outcome evidence including the predictor's own recorded + features. **NAMED APPROXIMATION**: a vanilla Gaussian Kalman update on 0/1 + outcomes is NOT an exact posterior. Binary success requires logistic + filtering with an explicitly named approximation — Laplace / + assumed-density / extended filtering — stated in the implementation and its + docs. Continuous log-cost observations admit exact Kalman updates under the + linear model. Censored completion cost is handled as censored. +- `Export`/`Import` move versioned state. `Import` refuses an envelope + version it does not know and a state whose `feature_version` differs from + its own frozen coordinates (old coefficients are invalid in a rotated + basis). + +## SharedSnapshot + +- `coordinate_version` pins the shared representation the parameters are + expressed in; migration on a representation update is a versioned act, + never a merge. +- `contribution_ids`: deduplication keys of contribution batches already + folded in. A duplicate contribution id is REJECTED, not re-counted — + averaging local posteriors as independent observations double-counts common + evidence, and this field is what prevents it. +- `parameters` / `uncertainty` / `counts`: shared values, their uncertainty + (non-negative), and evidence counts, under a common keying. +- `model_ids`: the models the parameters cover; a snapshot that does not + mention a model is never read as scoring it. +- `generated` (UTC) / `generated_by`. +- **Text, prompts and per-task embeddings MUST NOT appear in a snapshot.** + The schema carries no free-text payload field at all; string-typed content + is identity and version naming only. + +## What may never change without a version bump + +Any field's meaning; the frozen allowed feature keys; the requirement that +`probabilities` is the exact final pre-draw categorical distribution; the +whole-job (never per-seat) nature of `verified_success`; the censoring +semantics of `cost_cap_censored`; the no-text rule on `SharedSnapshot` and the +predictor payload; the uniqueness rule on `contribution_ids`. diff --git a/internal/pool/contract/contract.go b/internal/pool/contract/contract.go new file mode 100644 index 0000000000..64d426f705 --- /dev/null +++ b/internal/pool/contract/contract.go @@ -0,0 +1,580 @@ +// Package contract freezes the interface contracts between the three +// pareto-crewing lanes (evidence, learner, integration) before they run in +// parallel, per the approved plan at ~/pareto-handoff/IMPLEMENTATION-EXPERIMENT-PLAN.md +// section 5 and the math in ~/pareto-handoff/PROPOSED-LEARNING-MODEL.md. +// +// WHAT THIS PACKAGE IS NOT: it is not an implementation of the predictor, the +// policy or the grader. It is the wire shape and the semantics those +// implementations must agree on. The types are deliberately boring and +// explicit: they are the load-bearing interface three downstream lanes build +// against, and every field here is one a lane can be held to. +// +// VERSIONING RULE: every event and artifact carries the schema version it was +// written under. A reader that does not know that version refuses rather than +// half-reads (same rule as internal/pool/index). Any change to a field's +// meaning, or the removal/redefinition of a field, is a version bump — see +// CONTRACT.md in this package for the field-by-field pin. +package contract + +import ( + "encoding/json" + "errors" + "fmt" + "math" + "sort" + "time" +) + +// Schema versions frozen 2026-10-20. Bump when a field changes meaning, not +// when a field is added: adding a field is forward compatible, like the index +// document's unknown-field rule. The marshal helpers refuse to emit an unset +// version silently — writing without a version is a caller bug. +const ( + // DecisionEventVersion is the frozen version of DecisionEvent. + DecisionEventVersion = 1 + // OutcomeEventVersion is the frozen version of OutcomeEvent. + OutcomeEventVersion = 1 + // PredictorStateVersion is the frozen version of a Predictor's export + // envelope. The Predictor owns the payload's interior format; this + // version pins only the envelope the export/import contract travels in. + PredictorStateVersion = 1 + // SharedSnapshotVersion is the frozen version of SharedSnapshot. + SharedSnapshotVersion = 1 +) + +// ErrVersion is wrapped by every version-mismatch rejection. +var ErrVersion = errors.New("contract: unsupported schema version") + +// ErrShape is wrapped by structural rejections: a distribution that does not +// sum to one, a probability out of range, a required linkage missing. +var ErrShape = errors.New("contract: structurally invalid document") + +// --------------------------------------------------------------- +// DecisionEvent — what the policy knew at the instant of selection. +// --------------------------------------------------------------- + +// DecisionEvent is recorded BEFORE the draw, once per role/node selection +// boundary. Everything on it must be provable to exist at decision time: +// features carry the version of the feature map they were built under, and a +// feature that was not observable at the decision is a future-feature leak +// that lane A's acceptance checks reject. +type DecisionEvent struct { + // SchemaVersion is DecisionEventVersion at write time. + SchemaVersion int `json:"schema_version"` + + // RunID identifies the whole job; ParentRunID is empty at the root and + // names the run this run was spawned from otherwise. + RunID string `json:"run_id"` + ParentRunID string `json:"parent_run_id,omitempty"` + // NodeID names the role/node boundary this decision was taken at (the + // seat class — one of the five classes the validated picker design + // derives: reflex, low, worker, high, mastermind). + NodeID string `json:"node_id"` + + // DecisionTime is when the decision was taken, UTC. + DecisionTime time.Time `json:"decision_time"` + + // FeatureVersion pins the frozen feature map the event's features were + // built under. A reader must not mix features from two versions. + FeatureVersion string `json:"feature_version"` + + // Features are the allowed pre-decision feature values: role, task class, + // context size, budget remaining, and checks already observed. Every key + // must be one FeatureVersion defines. FROZEN ALLOWED KEYS (see + // CONTRACT.md): "role", "task_class", "context_size_tokens", + // "budget_remaining_usd", "checks_observed". + Features map[string]string `json:"features"` + + // Candidates is the eligible set AFTER all filters (availability, + // compatibility, budget): model ids with the descriptor version each was + // read under. + Candidates []Candidate `json:"candidates"` + + // Probabilities is the exact final categorical distribution over the + // eligible candidates AFTER all filters and after the exploration + // mixture, per the plan's hard requirement: record the exact final + // probabilities before drawing, so the draw is replayable and the + // propensity is declared, not reconstructed. Each value is in [0, 1]; the + // values sum to 1 within tolerance and name exactly the candidates above + // (an eligible candidate assigned no mass is listed with value 0). + // + // SelectedAction does NOT have to be the argmax; it must be drawn + // according to Probabilities. + Probabilities map[string]float64 `json:"probabilities"` + // SelectedAction is the model id assigned to the seat. + SelectedAction string `json:"selected_action"` + + // PolicyVersion, CatalogVersion and SourceVersion are the versions of the + // policy that chose, the model catalog it read and the source SHA the + // selector binary was built from, so any decision is reproducible. + PolicyVersion string `json:"policy_version"` + CatalogVersion string `json:"catalog_version"` + SourceVersion string `json:"source_version"` + + // BudgetEnvelope is the budget this decision operated under. + BudgetEnvelope BudgetEnvelope `json:"budget_envelope"` +} + +// Candidate is one eligible model with the descriptor version it was read +// under. Descriptors themselves (benchmark scores, prices) live in the +// catalog, not here — the event records which version was read so the +// decision replays against that exact catalog document. +type Candidate struct { + ModelID string `json:"model_id"` + DescriptorVersion string `json:"descriptor_version"` +} + +// BudgetEnvelope is the budget state at decision time. +type BudgetEnvelope struct { + // CapUSD is the maximum spend this decision may commit from here on. + CapUSD float64 `json:"cap_usd"` + // ReservedUSD is spend already reserved for this run's outstanding work + // (checks, repairs) at decision time. + ReservedUSD float64 `json:"reserved_usd"` + // SpentUSD is actual spend already incurred on the run at decision time. + SpentUSD float64 `json:"spent_usd"` + // Currency is fixed to "usd"; the field exists so a later currency change + // cannot silently reinterpret the three numbers above. + Currency string `json:"currency"` +} + +// Validate checks the structural rules a writer cannot be allowed to break: +// the version is the frozen one, the distribution sums to one within +// tolerance and covers the candidates, and the selected action is in both. +func (e DecisionEvent) Validate() error { + if e.SchemaVersion != DecisionEventVersion { + return fmt.Errorf("%w: decision event %d, reader knows %d", + ErrVersion, e.SchemaVersion, DecisionEventVersion) + } + if e.RunID == "" || e.NodeID == "" { + return fmt.Errorf("%w: decision event without run or node id", ErrShape) + } + return validateDistribution(e.Probabilities, candidateIDs(e.Candidates), e.SelectedAction) +} + +// MarshalDecisionEvent writes the event, refusing to emit an unset version. +func MarshalDecisionEvent(e DecisionEvent) ([]byte, error) { + if e.SchemaVersion == 0 { + return nil, fmt.Errorf("%w: decision event written without a version", ErrShape) + } + if err := e.Validate(); err != nil { + return nil, err + } + return json.Marshal(e) +} + +// UnmarshalDecisionEvent reads an event and refuses a version this reader +// does not know, then re-validates the structural rules on the wire form. +func UnmarshalDecisionEvent(data []byte) (DecisionEvent, error) { + var e DecisionEvent + if err := json.Unmarshal(data, &e); err != nil { + return e, fmt.Errorf("decode decision event: %w", err) + } + if err := e.Validate(); err != nil { + return e, err + } + return e, nil +} + +// --------------------------------------------------------------- +// OutcomeEvent — what actually happened to the job the decision fed. +// --------------------------------------------------------------- + +// OutcomeEvent is recorded after independent grading. It is a WHOLE-JOB +// record: verified job success is a label of the job, never copied onto +// individual seats. One job produces one OutcomeEvent per graded job; +// several DecisionEvents may link to it. +type OutcomeEvent struct { + // SchemaVersion is OutcomeEventVersion at write time. + SchemaVersion int `json:"schema_version"` + + // RunID and DecisionID link the outcome to the decision that produced the + // executed work. JobID identifies the graded job. A reader must be able + // to walk DecisionEvent.RunID -> OutcomeEvent.RunID without guessing. + RunID string `json:"run_id"` + DecisionID string `json:"decision_id"` + JobID string `json:"job_id"` + + // Grader provenance: which judge produced the verdict and which version + // of that judge. A verdict without provenance is not a label. + GraderName string `json:"grader_name"` + GraderVersion string `json:"grader_version"` + + // VerifiedSuccess is the whole-job label: verified job success after the + // independent checks (and any repairs) the run's pipeline ran. This is + // NOT the model's own success claim. + VerifiedSuccess bool `json:"verified_success"` + + // Spend is the actual token/cache/cost accounting of the executed work. + Spend SpendAccounting `json:"spend"` + + // CostCapCensored is true when the run terminated on the cost cap, which + // censors completion cost: the actual incurred spend above is observable, + // the unconstrained cost-to-completion is not, and the predictor's update + // must treat this outcome's completion cost as censored, not as the true + // cost. + CostCapCensored bool `json:"cost_cap_censored"` + + // HarnessError is true when the run failed on harness/tooling error + // rather than on the model's work. A harness error is not a success or + // failure signal about the model; the exact treatment in the update is + // the learner's declared approximation, not this package's. + HarnessError bool `json:"harness_error"` +} + +// SpendAccounting is the actual spend of the graded job. Every field is +// actual: predictions do not live here. +type SpendAccounting struct { + // TotalUSD is the billed total for the job. + TotalUSD float64 `json:"total_usd"` + // TokensIn and TokensOut are the raw billed token counts. + TokensIn int64 `json:"tokens_in"` + TokensOut int64 `json:"tokens_out"` + // CachedTokensIn is the portion of TokensIn billed at cache terms. + CachedTokensIn int64 `json:"cached_tokens_in"` + // Currency is fixed to "usd", matching BudgetEnvelope. + Currency string `json:"currency"` +} + +// Validate checks version, required linkage, grader provenance and spend. +func (e OutcomeEvent) Validate() error { + if e.SchemaVersion != OutcomeEventVersion { + return fmt.Errorf("%w: outcome event %d, reader knows %d", + ErrVersion, e.SchemaVersion, OutcomeEventVersion) + } + if e.RunID == "" || e.DecisionID == "" || e.JobID == "" { + return fmt.Errorf("%w: outcome event without run/decision/job linkage", ErrShape) + } + if e.GraderName == "" || e.GraderVersion == "" { + return fmt.Errorf("%w: outcome event without grader provenance", ErrShape) + } + if e.Spend.TotalUSD < 0 { + return fmt.Errorf("%w: negative total spend", ErrShape) + } + return nil +} + +// MarshalOutcomeEvent writes the event, refusing to emit an unset version. +func MarshalOutcomeEvent(e OutcomeEvent) ([]byte, error) { + if e.SchemaVersion == 0 { + return nil, fmt.Errorf("%w: outcome event written without a version", ErrShape) + } + if err := e.Validate(); err != nil { + return nil, err + } + return json.Marshal(e) +} + +// UnmarshalOutcomeEvent reads an outcome and refuses a version this reader +// does not know. +func UnmarshalOutcomeEvent(data []byte) (OutcomeEvent, error) { + var e OutcomeEvent + if err := json.Unmarshal(data, &e); err != nil { + return e, fmt.Errorf("decode outcome event: %w", err) + } + if err := e.Validate(); err != nil { + return e, err + } + return e, nil +} + +// --------------------------------------------------------------- +// Predictor — the interface the learner lane implements. This file does NOT +// implement it; it freezes the contract. +// --------------------------------------------------------------- + +// FeatureVector is the compact frozen feature map of the plan: the +// fixed-interaction coordinates plus descriptors. Its exact numeric layout is +// pinned by the feature-map version a DecisionEvent carries, not by this +// struct — the struct exists so the interface below is not stringly typed. +type FeatureVector struct { + // Version is the same string DecisionEvent.FeatureVersion carries. + Version string + // Task are the task-side features known at decision time (role, task + // class, context size, budget remaining, checks already observed). + Task map[string]float64 + // Candidate are the candidate model's descriptor coordinates (benchmark + // values with missingness masks, prices, cache terms) under the catalog + // version the decision recorded. + Candidate map[string]float64 +} + +// Prediction is what predict returns. +type Prediction struct { + // SuccessProb is the personalized probability of verified job success. + SuccessProb float64 + // CostDist is a distribution over total relevant cost. It is deliberately + // a distribution, not a mean: a Gaussian head on log cost implies a + // lognormal mean exp(mu + variance/2), not exp(mu). + CostDist CostDistribution + // Uncertainty is the posterior predictive uncertainty of the success + // head, consumed by the exploration layer — which is a separate decision + // layer, not part of predict. + Uncertainty float64 +} + +// CostDistribution is a parameterized or sampled cost distribution. +// Consumers take the mean and a quantile and MUST NOT assume a mean alone. +type CostDistribution struct { + // LogNormal is the (mu, sigma) of the log bill when the head is Gaussian + // in log cost; nil when the head produced samples instead. + LogNormal *LogNormalParams + // Samples, when non-empty, are draws from the cost distribution. + Samples []float64 +} + +// LogNormalParams parameterizes a lognormal bill. +type LogNormalParams struct { + Mu float64 + Sigma float64 +} + +// Mean returns the lognormal mean exp(mu + sigma^2/2), or the sample mean. +func (c CostDistribution) Mean() float64 { + if c.LogNormal != nil { + return math.Exp(c.LogNormal.Mu + c.LogNormal.Sigma*c.LogNormal.Sigma/2) + } + if len(c.Samples) == 0 { + return 0 + } + s := 0.0 + for _, v := range c.Samples { + s += v + } + return s / float64(len(c.Samples)) +} + +// OutcomeEvidence is the evidence update takes. It carries the outcome event +// plus the feature state the predictor itself recorded for that decision, so +// the update cannot be fed features the predictor never saw. +type OutcomeEvidence struct { + Decision DecisionEvent + Outcome OutcomeEvent + // Features is the predictor's own record of the feature vector it scored. + Features FeatureVector +} + +// Predictor is the interface the learner implements. CONTRACT SUMMARY: +// +// - predict scores a task feature vector against candidate descriptors and +// returns success probability, a cost DISTRIBUTION and uncertainty. +// - update ingests outcome evidence. THE APPROXIMATION IS NAMED, NOT +// HIDDEN: a vanilla Gaussian Kalman update on 0/1 outcomes is NOT an +// exact posterior — the plan requires logistic filtering with an +// explicitly named approximation (Laplace / assumed-density / extended), +// documented in CONTRACT.md and in the implementation. Continuous +// log-cost observations admit exact Kalman updates under the linear +// model. Censored completion cost (OutcomeEvent.CostCapCensored) must be +// handled as censored, not observed. +// - export/import move the predictor's versioned state. The envelope is +// versioned by PredictorStateVersion; the payload is opaque to the +// policy. Import refuses a payload written under a state version this +// build does not know, and refuses a state whose feature-map version +// differs from this predictor's frozen coordinates: old coefficients are +// not valid in a rotated basis. +type Predictor interface { + Predict(task FeatureVector, candidates []Candidate) (Prediction, error) + Update(ev OutcomeEvidence) error + Export() ([]byte, error) // PredictorStateVersion envelope + Import(blob []byte) error +} + +// PredictorEnvelope is the export wire shape: the envelope version, the +// predictor's own state version, and the opaque payload. +type PredictorEnvelope struct { + SchemaVersion int `json:"schema_version"` // PredictorStateVersion + // StateVersion is the interior version the predictor declares. + StateVersion int `json:"state_version"` + // FeatureVersion must match the predictor's frozen feature coordinates. + FeatureVersion string `json:"feature_version"` + // Payload is opaque to everyone but the predictor implementation that + // wrote it. Prompts, per-task texts and embeddings must NOT appear here. + Payload []byte `json:"payload"` +} + +// MarshalPredictorEnvelope wraps a payload for export. +func MarshalPredictorEnvelope(stateVersion int, featureVersion string, payload []byte) ([]byte, error) { + if payload == nil { + return nil, fmt.Errorf("%w: export of a nil predictor payload", ErrShape) + } + return json.Marshal(PredictorEnvelope{ + SchemaVersion: PredictorStateVersion, + StateVersion: stateVersion, + FeatureVersion: featureVersion, + Payload: payload, + }) +} + +// UnmarshalPredictorEnvelope reads the envelope and refuses an envelope +// version this reader does not know. It does not interpret the payload. +func UnmarshalPredictorEnvelope(blob []byte) (PredictorEnvelope, error) { + var env PredictorEnvelope + if err := json.Unmarshal(blob, &env); err != nil { + return env, fmt.Errorf("decode predictor envelope: %w", err) + } + if env.SchemaVersion != PredictorStateVersion { + return env, fmt.Errorf("%w: predictor envelope %d, reader knows %d", + ErrVersion, env.SchemaVersion, PredictorStateVersion) + } + if env.FeatureVersion == "" { + return env, fmt.Errorf("%w: predictor envelope without a feature version", ErrShape) + } + return env, nil +} + +// --------------------------------------------------------------- +// SharedSnapshot — what travels between installations. +// --------------------------------------------------------------- + +// SharedSnapshot is the shared learned artifact: numerical learned structure +// only. TEXT, PROMPTS AND PER-TASK EMBEDDINGS MUST NOT APPEAR IN IT — this is +// stated on the type, in CONTRACT.md, and enforced to the extent a schema +// can: the snapshot carries no free-text payload field at all, and the only +// string-typed content is identity and version naming. +type SharedSnapshot struct { + // SchemaVersion is SharedSnapshotVersion at write time. + SchemaVersion int `json:"schema_version"` + + // CoordinateVersion pins the shared representation the parameters are + // expressed in. Old local posteriors are not valid under a rotated + // coordinate basis, so migration is a versioned act, never a merge. + CoordinateVersion string `json:"coordinate_version"` + + // FeatureVersion of the task feature map the parameters were fit under. + FeatureVersion string `json:"feature_version"` + + // ContributionIDs are the deduplication keys of the contribution batches + // already folded into this snapshot. An aggregator that receives a + // contribution whose id is here REJECTS it rather than counting it + // again: local posteriors and shared state share common evidence, and + // averaging both as independent observations double-counts it. + ContributionIDs []string `json:"contribution_ids"` + + // Parameters are the shared parameter values (shared W and b per the + // model document), keyed by a stable parameter name under + // CoordinateVersion. + Parameters map[string]float64 `json:"parameters"` + + // Uncertainty is the between-install/parameter uncertainty, keyed the + // same way as Parameters (a variance, or a covariance serialized + // parameter-by-parameter as the writer declares). + Uncertainty map[string]float64 `json:"uncertainty"` + + // Counts records evidence counts behind the parameters, keyed the same + // way, so a reader can distinguish "confident" from "barely seen". + Counts map[string]int `json:"counts"` + + // ModelIDs are the model identities the parameters cover, so a snapshot + // that does not mention a model can never be read as scoring it. + ModelIDs []string `json:"model_ids"` + + // Generated is the snapshot's UTC generation time; GeneratedBy names the + // site or aggregator that produced it. + Generated time.Time `json:"generated"` + GeneratedBy string `json:"generated_by"` +} + +// Validate checks version, that contribution ids are unique (the +// deduplication contract is load-bearing), and that uncertainty is +// non-negative where present. +func (s SharedSnapshot) Validate() error { + if s.SchemaVersion != SharedSnapshotVersion { + return fmt.Errorf("%w: shared snapshot %d, reader knows %d", + ErrVersion, s.SchemaVersion, SharedSnapshotVersion) + } + if s.CoordinateVersion == "" || s.FeatureVersion == "" { + return fmt.Errorf("%w: shared snapshot without coordinate or feature version", ErrShape) + } + seen := make(map[string]bool, len(s.ContributionIDs)) + for _, id := range s.ContributionIDs { + if id == "" { + return fmt.Errorf("%w: empty contribution id", ErrShape) + } + if seen[id] { + return fmt.Errorf("%w: duplicate contribution id %q in snapshot", ErrShape, id) + } + seen[id] = true + } + for name, v := range s.Uncertainty { + if v < 0 { + return fmt.Errorf("%w: negative uncertainty on %q", ErrShape, name) + } + } + return nil +} + +// MarshalSharedSnapshot writes the snapshot, refusing to emit an unset +// version. +func MarshalSharedSnapshot(s SharedSnapshot) ([]byte, error) { + if s.SchemaVersion == 0 { + return nil, fmt.Errorf("%w: shared snapshot written without a version", ErrShape) + } + if err := s.Validate(); err != nil { + return nil, err + } + return json.Marshal(s) +} + +// UnmarshalSharedSnapshot reads a snapshot and refuses a version this reader +// does not know. +func UnmarshalSharedSnapshot(data []byte) (SharedSnapshot, error) { + var s SharedSnapshot + if err := json.Unmarshal(data, &s); err != nil { + return s, fmt.Errorf("decode shared snapshot: %w", err) + } + if err := s.Validate(); err != nil { + return s, err + } + return s, nil +} + +// --------------------------------------------------------------- +// shared validation helpers +// --------------------------------------------------------------- + +// ProbTolerance is how far the final categorical distribution may drift from +// an exact sum of one after JSON round trip and float summation. +const ProbTolerance = 1e-6 + +func candidateIDs(cs []Candidate) []string { + ids := make([]string, 0, len(cs)) + for _, c := range cs { + ids = append(ids, c.ModelID) + } + return ids +} + +// validateDistribution is the shared check behind the plan's hard requirement: +// the recorded probabilities are a categorical distribution over the +// candidate set, and the selected action is in it. +func validateDistribution(probs map[string]float64, candidates []string, selected string) error { + set := make(map[string]bool, len(candidates)) + for _, id := range candidates { + set[id] = true + } + sum := 0.0 + for id, p := range probs { + if !set[id] { + return fmt.Errorf("%w: probability names non-candidate %q", ErrShape, id) + } + if p < 0 || p > 1 { + return fmt.Errorf("%w: probability %g out of range on %q", ErrShape, p, id) + } + sum += p + } + if math.Abs(sum-1) > ProbTolerance { + return fmt.Errorf("%w: assignment probabilities sum to %g, not 1", ErrShape, sum) + } + if !set[selected] { + return fmt.Errorf("%w: selected action %q is not in the candidate set", ErrShape, selected) + } + return nil +} + +// SortIDs returns the ids in sorted order, for tests and consumers that need +// a canonical candidate order. +func SortIDs(ids []string) []string { + out := append([]string(nil), ids...) + sort.Strings(out) + return out +} diff --git a/internal/pool/contract/contract_test.go b/internal/pool/contract/contract_test.go new file mode 100644 index 0000000000..88fa1c2214 --- /dev/null +++ b/internal/pool/contract/contract_test.go @@ -0,0 +1,288 @@ +package contract + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + "time" +) + +func sampleDecision() DecisionEvent { + return DecisionEvent{ + SchemaVersion: DecisionEventVersion, + RunID: "run-1", + ParentRunID: "run-0", + NodeID: "worker", + DecisionTime: time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC), + FeatureVersion: "feat-2026-10-20", + Features: map[string]string{ + "role": "worker", + "task_class": "feature", + "context_size_tokens": "48000", + "budget_remaining_usd": "2.10", + "checks_observed": "1", + }, + Candidates: []Candidate{ + {ModelID: "a/model-x", DescriptorVersion: "cat-2026-10-20"}, + {ModelID: "a/model-y", DescriptorVersion: "cat-2026-10-20"}, + }, + Probabilities: map[string]float64{"a/model-x": 0.9, "a/model-y": 0.1}, + SelectedAction: "a/model-x", + PolicyVersion: "policy-1", + CatalogVersion: "cat-2026-10-20", + SourceVersion: "deadbeef", + BudgetEnvelope: BudgetEnvelope{CapUSD: 3.0, ReservedUSD: 0.5, SpentUSD: 0.4, Currency: "usd"}, + } +} + +func sampleOutcome() OutcomeEvent { + return OutcomeEvent{ + SchemaVersion: OutcomeEventVersion, + RunID: "run-1", + DecisionID: "dec-7", + JobID: "job-42", + GraderName: "rubric-judge", + GraderVersion: "3", + VerifiedSuccess: true, + Spend: SpendAccounting{ + TotalUSD: 1.23, TokensIn: 5000, TokensOut: 900, CachedTokensIn: 1200, Currency: "usd", + }, + CostCapCensored: false, + HarnessError: false, + } +} + +func TestDecisionEventRoundTrip(t *testing.T) { + e := sampleDecision() + b, err := MarshalDecisionEvent(e) + if err != nil { + t.Fatalf("marshal: %v", err) + } + got, err := UnmarshalDecisionEvent(b) + if err != nil { + t.Fatalf("unmarshal: %v", err) + } + b2, err := MarshalDecisionEvent(got) + if err != nil { + t.Fatalf("re-marshal: %v", err) + } + if !bytes.Equal(b, b2) { + t.Fatalf("round trip is not byte-stable:\n%s\n%s", b, b2) + } + if got.RunID != e.RunID || got.ParentRunID != e.ParentRunID || got.NodeID != e.NodeID { + t.Fatalf("ids lost: %+v", got) + } + if got.FeatureVersion != e.FeatureVersion || len(got.Features) != len(e.Features) { + t.Fatalf("features lost: %+v", got) + } + if len(got.Candidates) != 2 || got.Candidates[1].ModelID != "a/model-y" { + t.Fatalf("candidates lost: %+v", got.Candidates) + } + if got.Probabilities["a/model-y"] != 0.1 || got.SelectedAction != e.SelectedAction { + t.Fatalf("distribution lost: %+v", got.Probabilities) + } + if got.BudgetEnvelope.CapUSD != 3.0 || got.BudgetEnvelope.Currency != "usd" { + t.Fatalf("budget lost: %+v", got.BudgetEnvelope) + } + if got.PolicyVersion != "policy-1" || got.CatalogVersion != "cat-2026-10-20" || got.SourceVersion != "deadbeef" { + t.Fatalf("versions lost: %+v", got) + } + if !got.DecisionTime.Equal(e.DecisionTime) { + t.Fatalf("time lost: %v", got.DecisionTime) + } +} + +func TestOutcomeEventRoundTrip(t *testing.T) { + e := sampleOutcome() + b, err := MarshalOutcomeEvent(e) + if err != nil { + t.Fatalf("marshal: %v", err) + } + got, err := UnmarshalOutcomeEvent(b) + if err != nil { + t.Fatalf("unmarshal: %v", err) + } + b2, _ := MarshalOutcomeEvent(got) + if !bytes.Equal(b, b2) { + t.Fatalf("round trip is not byte-stable:\n%s\n%s", b, b2) + } + if got.DecisionID != "dec-7" || got.JobID != "job-42" { + t.Fatalf("linkage lost: %+v", got) + } + if got.GraderName != "rubric-judge" || got.GraderVersion != "3" { + t.Fatalf("provenance lost: %+v", got) + } + if got.Spend.CachedTokensIn != 1200 { + t.Fatalf("spend lost: %+v", got.Spend) + } +} + +func TestSharedSnapshotRoundTrip(t *testing.T) { + s := SharedSnapshot{ + SchemaVersion: SharedSnapshotVersion, + CoordinateVersion: "coord-rank2-1", + FeatureVersion: "feat-2026-10-20", + ContributionIDs: []string{"contrib-1", "contrib-2"}, + Parameters: map[string]float64{"b_m/model-x": -0.2, "W_q/f1": 0.03}, + Uncertainty: map[string]float64{"b_m/model-x": 0.5}, + Counts: map[string]int{"b_m/model-x": 30}, + ModelIDs: []string{"a/model-x", "a/model-y"}, + Generated: time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC), + GeneratedBy: "aggregator-1", + } + b, err := MarshalSharedSnapshot(s) + if err != nil { + t.Fatalf("marshal: %v", err) + } + got, err := UnmarshalSharedSnapshot(b) + if err != nil { + t.Fatalf("unmarshal: %v", err) + } + b2, _ := MarshalSharedSnapshot(got) + if !bytes.Equal(b, b2) { + t.Fatalf("round trip is not byte-stable:\n%s\n%s", b, b2) + } + if len(got.ContributionIDs) != 2 || got.Parameters["W_q/f1"] != 0.03 { + t.Fatalf("state lost: %+v", got) + } +} + +func TestPredictorEnvelopeRoundTrip(t *testing.T) { + b, err := MarshalPredictorEnvelope(7, "feat-2026-10-20", []byte{1, 2, 3}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + env, err := UnmarshalPredictorEnvelope(b) + if err != nil { + t.Fatalf("unmarshal: %v", err) + } + if env.StateVersion != 7 || !bytes.Equal(env.Payload, []byte{1, 2, 3}) { + t.Fatalf("envelope lost: %+v", env) + } +} + +func TestVersionMismatchRejection(t *testing.T) { + b, _ := json.Marshal(sampleDecision()) + var bumped map[string]any + if err := json.Unmarshal(b, &bumped); err != nil { + t.Fatal(err) + } + bumped["schema_version"] = DecisionEventVersion + 1 + nb, _ := json.Marshal(bumped) + if _, err := UnmarshalDecisionEvent(nb); err == nil || !strings.Contains(err.Error(), "unsupported schema version") { + t.Fatalf("expected version refusal, got %v", err) + } + + ob, _ := json.Marshal(sampleOutcome()) + var ob2 map[string]any + _ = json.Unmarshal(ob, &ob2) + ob2["schema_version"] = OutcomeEventVersion + 1 + if _, err := UnmarshalOutcomeEvent(mustJSON(t, ob2)); err == nil || !strings.Contains(err.Error(), "unsupported schema version") { + t.Fatalf("expected version refusal, got %v", err) + } + + sb, _ := json.Marshal(SharedSnapshot{SchemaVersion: SharedSnapshotVersion + 1}) + if _, err := UnmarshalSharedSnapshot(sb); err == nil || !strings.Contains(err.Error(), "unsupported schema version") { + t.Fatalf("expected version refusal, got %v", err) + } + + eb, _ := MarshalPredictorEnvelope(1, "feat", []byte{1}) + var env map[string]any + _ = json.Unmarshal(eb, &env) + env["schema_version"] = PredictorStateVersion + 1 + if _, err := UnmarshalPredictorEnvelope(mustJSON(t, env)); err == nil || !strings.Contains(err.Error(), "unsupported schema version") { + t.Fatalf("expected version refusal, got %v", err) + } +} + +func TestStructuralRejection(t *testing.T) { + e := sampleDecision() + e.Probabilities["a/model-x"] = 0.5 // sums to 0.6 + if err := e.Validate(); err == nil || !strings.Contains(err.Error(), "sum to") { + t.Fatalf("expected distribution refusal, got %v", err) + } + e = sampleDecision() + e.SelectedAction = "a/not-a-candidate" + if err := e.Validate(); err == nil || !strings.Contains(err.Error(), "not in the candidate set") { + t.Fatalf("expected action refusal, got %v", err) + } + e = sampleDecision() + e.Probabilities["ghost/model"] = -0.1 // also makes the sum wrong, so delete one + delete(e.Probabilities, "a/model-y") + e.Probabilities["a/model-x"] = 0.9 + if err := e.Validate(); err == nil || !strings.Contains(err.Error(), "non-candidate") { + t.Fatalf("expected non-candidate refusal, got %v", err) + } + + o := sampleOutcome() + o.DecisionID = "" + if err := o.Validate(); err == nil || !strings.Contains(err.Error(), "linkage") { + t.Fatalf("expected linkage refusal, got %v", err) + } + o = sampleOutcome() + o.GraderName = "" + if err := o.Validate(); err == nil || !strings.Contains(err.Error(), "provenance") { + t.Fatalf("expected provenance refusal, got %v", err) + } + + s := SharedSnapshot{SchemaVersion: SharedSnapshotVersion, CoordinateVersion: "c", FeatureVersion: "f", + ContributionIDs: []string{"x", "x"}} + if err := s.Validate(); err == nil || !strings.Contains(err.Error(), "duplicate contribution") { + t.Fatalf("expected duplicate refusal, got %v", err) + } +} + +func TestUnsetVersionRefusedOnWrite(t *testing.T) { + if _, err := MarshalDecisionEvent(sampleDecision()); err != nil { + // The sample carries a version; a zeroed one must refuse. + _ = err + } + e := sampleDecision() + e.SchemaVersion = 0 + if _, err := MarshalDecisionEvent(e); err == nil || !strings.Contains(err.Error(), "without a version") { + t.Fatalf("expected unset-version refusal, got %v", err) + } +} + +func TestUnmarshalKeepsUnknownFieldsForwardCompatible(t *testing.T) { + b, _ := json.Marshal(sampleDecision()) + withExtra := strings.Replace(string(b), `"schema_version":1`, + `"schema_version":1,"future_field":{"x":1}`, 1) + if _, err := UnmarshalDecisionEvent([]byte(withExtra)); err != nil { + t.Fatalf("unknown field must be ignored, got %v", err) + } +} + +func TestCostDistributionMean(t *testing.T) { + c := CostDistribution{LogNormal: &LogNormalParams{Mu: 0, Sigma: 1}} + if m := c.Mean(); m < 1.64 || m > 1.65 { + t.Fatalf("lognormal mean exp(0.5) != %g", m) + } + c2 := CostDistribution{Samples: []float64{1, 2, 3}} + if m := c2.Mean(); m != 2 { + t.Fatalf("sample mean != %g", m) + } +} + +// The Predictor contract itself is not implemented here, but the interface +// must still be satisfiable and documented as such — a compile-time witness. +type stubPredictor struct{} + +func (stubPredictor) Predict(FeatureVector, []Candidate) (Prediction, error) { + return Prediction{}, nil +} +func (stubPredictor) Update(OutcomeEvidence) error { return nil } +func (stubPredictor) Export() ([]byte, error) { return nil, nil } +func (stubPredictor) Import([]byte) error { return nil } + +var _ Predictor = stubPredictor{} + +func mustJSON(t *testing.T, v any) []byte { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return b +} diff --git a/internal/selector/audit_test.go b/internal/selector/audit_test.go new file mode 100644 index 0000000000..3e005b99ce --- /dev/null +++ b/internal/selector/audit_test.go @@ -0,0 +1,97 @@ +package selector + +import ( + "strings" + "testing" +) + +// scopedClass decides one class with a catalog of exactly these models, +// returning the SeatDecision. +func scopedClass(t *testing.T, classID string, snap *SharedSnapshot, mods ...Model) SeatDecision { + t.Helper() + cat := ModelCatalog{Models: mods} + e := NewEngine(snap, &cat, nil) + d, err := e.DecideClass(oneClass(classID, "worker", 1)) + if err != nil { + t.Fatal(err) + } + return d +} + +// TestPresetsNameOnlyVerifiedCrews: a class whose chosen seat has a SCORED +// prior is preset-worthy; a class whose every passing candidate INHERITS its +// prior is not. Presets() must name only the scored crew. +func TestPresetsNameOnlyVerifiedCrews(t *testing.T) { + // Verified class: one scored model -> chosen is scored, not all-inherited. + verified := mkM("aero/verified-20260901", "aero/verified-20260901", 1000, 300_000, true, false, 1) + snapV := mkSnap(nil, + map[string]float64{"quality/worker/aero/verified-20260901": 0.85}, + map[string]float64{"quality/worker/aero/verified-20260901": 0.0001}, + ) + dv := scopedClass(t, "verify", snapV, verified) + if dv.AllInherited { + t.Fatalf("verify class must NOT be all-inherited (chosen %s prior %s)", dv.Chosen, dv.ChosenPrior) + } + + // Inherited class: donor scored, fresh model inherits the vendor mean. + // The donor FAILS the class gate (NeedsTools, no tool mod) so it is not a + // candidate; only the inheritor competes, and it inherits the vendor mean. + donor := mkM("acme/donor-20260801", "acme/donor-20260801", 400, 300_000, false, false, 1) + fresh := mkM("acme/fresh-20260915", "acme/fresh-20260915", 5000, 300_000, true, false, 1) + snapI := mkSnap(nil, + map[string]float64{"quality/worker/acme/donor-20260801": 0.9}, + map[string]float64{"quality/worker/acme/donor-20260801": 0.0001}, + ) + csi := oneClass("inherit", "worker", 1) + csi.NeedsTools = true + // scopedClass uses oneClass default (no tools); build directly. + catI := ModelCatalog{Models: []Model{donor, fresh}} + eI := NewEngine(snapI, &catI, nil) + di, err := eI.DecideClass(csi) + if err != nil { + t.Fatal(err) + } + _ = di + if !di.AllInherited { + t.Fatalf("inherit class must be all-inherited (chosen %s prior %s)", di.Chosen, di.ChosenPrior) + } + + if p := NewEngine(snapI, &ModelCatalog{Models: []Model{donor, fresh}}, nil). + Presets([]SeatDecision{di}); len(p) != 0 { + t.Fatalf("all-inherited class must yield no preset, got %d", len(p)) + } + pv := NewEngine(snapV, &ModelCatalog{Models: []Model{verified}}, nil). + Presets([]SeatDecision{dv}) + if len(pv) != 1 || pv[0].Seats["worker"] != verified.ID { + t.Fatalf("verified class must yield one preset naming %s, got %v", verified.ID, pv) + } +} + +// TestPresetsAuditLine: the audit builds one line per preset proving the seat +// is verified; an all-inherited class names no line. +func TestPresetsAuditLine(t *testing.T) { + verified := mkM("aero/audit-20260901", "aero/audit-20260901", 1000, 300_000, true, false, 1) + snap := mkSnap(nil, + map[string]float64{"quality/worker/aero/audit-20260901": 0.9}, + map[string]float64{"quality/worker/aero/audit-20260901": 0.0001}, + ) + dv := scopedClass(t, "worker", snap, verified) + if dv.AllInherited { + t.Fatalf("worker scored class must not be all inherited (prior %s)", dv.ChosenPrior) + } + + var audit []string + for _, c := range NewEngine(snap, &ModelCatalog{Models: []Model{verified}}, nil). + Presets([]SeatDecision{dv}) { + audit = append(audit, "preset="+c.Class+" seat="+c.Seats["worker"]+" prior="+dv.ChosenPrior) + } + found := false + for _, line := range audit { + if strings.HasPrefix(line, "preset=worker ") && strings.HasSuffix(line, "prior=scored") { + found = true + } + } + if !found { + t.Fatalf("audit must name a verified preset line, got %v", audit) + } +} diff --git a/internal/selector/bill_test.go b/internal/selector/bill_test.go new file mode 100644 index 0000000000..74df63fa1a --- /dev/null +++ b/internal/selector/bill_test.go @@ -0,0 +1,119 @@ +package selector + +import "testing" + +// The volume-weighted bill is mean(passer seat-cost) * class volume. +// The trap: an UNWEIGHTED bill (mean cost alone) reorders the frugal end when +// a cheap-seat, high-volume class outspends a costly-seat, low-volume one in +// real dollars. This test fixes a concrete example where the unweighted order +// (ranked by raw per-seat cost) and the weighted order (ranked by +// mean-Cost*Volume) disagree, and asserts the SELECtor's Bill is the weighted +// one (i.e. that it agrees with the weighted frugal order, not the unweighted +// reorder). +func TestVolumeWeightedBillTrap(t *testing.T) { + // Class A: cheap seat (cost ~2) but very high volume (100) -> bill 200. + // Class B: expensive seat (cost ~18) but low volume (10) -> bill 180. + // + // A raw-cost ranking would call A (cost 2) the most frugal; the weighted + // bill calls B (180 < 200) the frugal end. The selector must report the + // weighted bill. + cheap := mkM("aero/cheap-20260901", "aero/cheap-20260901", 1000, 300_000, true, false, 1) + expensive := mkM("aero/expen-20260901", "aero/expen-20260901", 1000, 300_000, true, false, 30) + + for _, tc := range []struct { + name string + model Model + volume float64 + }{ + {"cheap-highvolume", cheap, 100}, + {"expensive-lowvolume", expensive, 10}, + } { + t.Run(tc.name, func(t *testing.T) { + cat := mkCat(tc.model) + snap := mkSnap(nil, + map[string]float64{"quality/worker/" + tc.model.ID + "-20260901": 1e-6}, + map[string]float64{"quality/worker/" + tc.model.ID + "-20260901": 0.0001}, + ) + // NOTE: model IDs already embed the stamp; the snapshot key must + // use the model ID itself, not an extra stamp. + delete(snap.Parameters, "quality/worker/"+tc.model.ID+"-20260901") + // build fresh with the right keys + snap = mkSnap(nil, + map[string]float64{"quality/worker/" + tc.model.ID: 1e-6}, + map[string]float64{"quality/worker/" + tc.model.ID: 0.0001}, + ) + e := NewEngine(snap, cat, nil) + cs := oneClass("worker", "worker", tc.volume) + cs.NeedsTools = false + cs.NeedsImages = false + d, err := e.DecideClass(cs) + if err != nil { + t.Fatal(err) + } + got := d.Bill + // Bill = mean(passer seat-cost) * volume; single passer -> seatCost*volume. + want := seatCost(&tc.model, cs) * tc.volume + if got != want { + t.Fatalf("%s: weighted bill = %v, want %v (= seatCost %v * volume %v)", tc.name, got, want, seatCost(&tc.model, cs), tc.volume) + } + // the bill must use the WEIGHTED product, not the raw cost alone: + if got == seatCost(&tc.model, cs) { + t.Fatalf("%s: bill equals raw seat cost — unweighted trap", tc.name) + } + }) + } +} + +// TestWeightedBillKeepsOrderUnweightedFlips drives the ordering comparison +// directly through DecideClass over two classes sharing a catalog, showing +// the weighted bill keeps the frugal order that the unweighted bill flips. +func TestWeightedBillKeepsOrderUnweightedFlips(t *testing.T) { + // One scorer model for volume vectors: make two catalogs so seatCost is + // controlled per model without cross-contamination. + cheap := mkM("aero/cheapp-20260901", "aero/cheapp-20260901", 1000, 300_000, true, false, 1) + expen := mkM("aero/spendy-20260901", "aero/spendy-20260901", 1000, 300_000, true, false, 18) + + // classA: cheap seat, volume 100 -> bill 100 + // classB: spendy seat, volume 10 -> bill 180*... cost 18 -> 180 + ca := oneClass("worker", "worker", 100) + cb := oneClass("worker", "worker", 10) + + billA := weightedBillFor(t, cheap, ca) + + rawCostA := seatCost(&cheap, ca) + rawCostB := seatCost(&expen, cb) + + // raw ordering: rawCostA(1) < rawCostB(18): A "frugal" on unweighted. + if !(rawCostA < rawCostB) { + t.Fatalf("test precondition: cheap raw cost %v must be < expen %v", rawCostA, rawCostB) + } + // weighted ordering: 100 vs 180*... billA=100, billB=180 -> A still frugal. + // To FLIP we need billB < billA: volume 10 * cost 18 = 180 > 100, no flip. + // Choose expen cost 8 volume 11 -> 88 < 100: flip. + expen2 := mkM("aero/spendy-20260901", "aero/spendy-20260901", 1000, 300_000, true, false, 8) + billB2 := weightedBillFor(t, expen2, oneClass("worker", "worker", 11)) + // billA=100, billB2=88 -> frugal order flips: B is now frugal on weighted. + if !(billB2 < billA) { + t.Fatalf("weighted bill must flip: got billB=%v billA=%v", billB2, billA) + } + if !(rawCostA < rawCostB) { + t.Fatalf("raw cost order stable: A %v < B %v", rawCostA, rawCostB) + } + t.Logf("unweighted: A(cost %.2f) < B(cost %.2f); weighted: A(bill %.2f) > B(bill %.2f) => weighted flips the frugal end", + rawCostA, rawCostB, billA, billB2) +} + +func weightedBillFor(t *testing.T, m Model, cs ClassShape) float64 { + t.Helper() + cat := mkCat(m) + snap := mkSnap(nil, + map[string]float64{"quality/worker/" + m.ID: 1e-6}, + map[string]float64{"quality/worker/" + m.ID: 0.0001}, + ) + e := NewEngine(snap, cat, nil) + d, err := e.DecideClass(cs) + if err != nil { + t.Fatal(err) + } + return d.Bill +} diff --git a/internal/selector/classes.go b/internal/selector/classes.go new file mode 100644 index 0000000000..e94e8b4372 --- /dev/null +++ b/internal/selector/classes.go @@ -0,0 +1,26 @@ +package selector + +// ClassShape is one of the five task classes the selector seats. The five +// classes are derived from the session findings: reflex (no tools, small +// context), low (tools, moderate context), worker/high/mastermind matching +// the crewpick seat shapes' spirit (worker small, high larger, mastermind +// largest + images). +type ClassShape struct { + ID string // reflex | low | worker | high | mastermind + MinContext int // context gate + NeedsTools bool // tool-use gate + NeedsImages bool // image gate + Volume float64 // seat-volume weight for the bill + Role string // lane B prediction role (worker|checker|thinker) +} + +// DefaultClasses returns the five classes with the session's shapes. +func DefaultClasses() []ClassShape { + return []ClassShape{ + {ID: "reflex", MinContext: 32_000, NeedsTools: false, NeedsImages: false, Volume: 0.60, Role: "worker"}, + {ID: "low", MinContext: 128_000, NeedsTools: true, NeedsImages: false, Volume: 0.28, Role: "worker"}, + {ID: "worker", MinContext: 200_000, NeedsTools: true, NeedsImages: false, Volume: 0.08, Role: "worker"}, + {ID: "high", MinContext: 200_000, NeedsTools: true, NeedsImages: true, Volume: 0.03, Role: "checker"}, + {ID: "mastermind", MinContext: 200_000, NeedsTools: true, NeedsImages: true, Volume: 0.01, Role: "thinker"}, + } +} diff --git a/internal/selector/coldstart_test.go b/internal/selector/coldstart_test.go new file mode 100644 index 0000000000..71fa0ed628 --- /dev/null +++ b/internal/selector/coldstart_test.go @@ -0,0 +1,82 @@ +package selector + +import "testing" + +// A held-out model (deepseek-v4.1-flash-1017 in the replay) has NO scored +// row; it inherits the vendor mean. Mechanism under test: +// - on EVERY seat it passes the UCB entrance gate (ucb >= 0.8*top, because +// the inherited prior has no competitor above it — it is the lone +// candidate, so it is the top), +// - yet because its prior is INHERITED (not scored), the class is +// AllInherited and the model wins NO preset until it is measured. +// +// This is the cold-start verdict: enter on every seat, preset nowhere. +func TestColdStartEntersEverySeatWinsNoPreset(t *testing.T) { + // The held-out model's lineage: a scored sibling (its/vendor donor) that + // fails every class gate except the model's own, so only the held-out + // model passes — mirroring "held out, no indexes, treated as new". + donor := mkM("dcp/donor-20260801", "dcp/donor-20260801", 400, 300_000, false, false, 1) + held := mkM("dcp/deepseek-v4.1-flash-1017", "dcp/deepseek-v4.1-flash-1017", 5000, 300_000, true, false, 1) + snap := mkSnap(nil, + map[string]float64{"quality/worker/dcp/donor-20260801": 0.9}, + map[string]float64{"quality/worker/dcp/donor-20260801": 0.0001}, + ) + cat := ModelCatalog{Models: []Model{donor, held}} + e := NewEngine(snap, &cat, nil) + + // Several seat classes with different credential demands; the held-out + // model must pass the UCB gate on every one and always inherit. + seats := []string{"worker", "coder", "reasoner"} + for _, role := range seats { + cs := oneClass(role, role, 1) + cs.NeedsTools = true // donor (non-tool) gated out; held-out (tool) passes + d, err := e.DecideClass(cs) + if err != nil { + t.Fatal(err) + } + // The held-out model is the only passing candidate, so it IS chosen. + if d.Chosen != held.ID { + t.Fatalf("seat %s: held-out model (%s) must be chosen, got %s", role, held.ID, d.Chosen) + } + // It enters on UCB: PassedCount must include it (1 passer) and + // TopLCB/TopUCB are its own. + if d.PassedCount != 1 { + t.Fatalf("seat %s: expected 1 passer (held-out), got %d", role, d.PassedCount) + } + // gate passed: the chosen (only) candidate's UCB satisfied the floor. + if d.TopUCB <= 0 || d.TopLCB > d.TopUCB { + t.Fatalf("seat %s: gate malformed topUCB %v topLCB %v", role, d.TopUCB, d.TopLCB) + } + // It wins NO preset: its seat is inherited, so the class is + // AllInherited and Presets() names nothing. + if !d.AllInherited { + t.Fatalf("seat %s: held-out prior is inherited, class must be AllInherited (prior %s)", role, d.ChosenPrior) + } + if p := e.Presets([]SeatDecision{d}); len(p) != 0 { + t.Fatalf("seat %s: held-out must win no preset, got %d", role, len(p)) + } + } +} + +// TestColdStartUntilMeasured makes the same held-out model SCORED and shows +// it then wins a preset — the "until measured" half of the verdict. +func TestColdStartUntilMeasured(t *testing.T) { + held := mkM("dcp/deepseek-v4.1-flash-1017", "dcp/deepseek-v4.1-flash-1017", 5000, 300_000, true, false, 1) + // now it has its own scored row + snap := mkSnap(nil, + map[string]float64{"quality/worker/dcp/deepseek-v4.1-flash-1017": 0.88}, + map[string]float64{"quality/worker/dcp/deepseek-v4.1-flash-1017": 0.0001}, + ) + cat := ModelCatalog{Models: []Model{held}} + e := NewEngine(snap, &cat, nil) + d, err := e.DecideClass(oneClass("worker", "worker", 1)) + if err != nil { + t.Fatal(err) + } + if d.AllInherited { + t.Fatalf("measured model must NOT be all-inherited (prior %s)", d.ChosenPrior) + } + if p := e.Presets([]SeatDecision{d}); len(p) != 1 || p[0].Seats["worker"] != held.ID { + t.Fatalf("measured model must win a preset, got %v", p) + } +} diff --git a/internal/selector/decide.go b/internal/selector/decide.go new file mode 100644 index 0000000000..d1613415b3 --- /dev/null +++ b/internal/selector/decide.go @@ -0,0 +1,321 @@ +package selector + +import ( + "fmt" + "math" + "sort" + "time" + + "github.com/Agent-Field/codeaf/internal/pool/contract" +) + +const ( + // UCBAlpha is the credibility half-width for the entrance gate. + UCBAlpha = 1.28 + // CredibilityFloor is the gate fraction of the top UCB. + CredibilityFloor = 0.8 +) + +// Candidate is one model under consideration for a seat, with its prior. +type Candidate struct { + Model *Model + Prior prior + Q float64 // posterior mean + Sigma float64 // posterior sigma + Cost float64 // $ per seat instance (ascending bill basis) + UCB float64 + LCB float64 + Passes bool // capability + credibility gate +} + +// SeatDecision is the selector's output for one seat of one class. +type SeatDecision struct { + Class string + Role string + Chosen string // chosen model id + ChosenQ float64 + ChosenCost float64 + ChosenSigma float64 + ChosenPrior string // scored|line|vendor|global + Candidates int + PassedCount int + TopLCB float64 + TopUCB float64 + Bill float64 // volume-weighted bill for the class + AllInherited bool // every passed candidate has an inherited prior + Event *contract.DecisionEvent +} + +// Engine runs decisions for the five classes against a snapshot + catalog. +type Engine struct { + Snap *SharedSnapshot + Cat *ModelCatalog + Classes []ClassShape + priors *PriorEngine + // eventSink receives every DecisionEvent the engine emits. + eventSink []*contract.DecisionEvent +} + +func NewEngine(s *SharedSnapshot, c *ModelCatalog, classes []ClassShape) *Engine { + if classes == nil { + classes = DefaultClasses() + } + return &Engine{Snap: s, Cat: c, Classes: classes, priors: NewPriorEngine(s, c)} +} + +// Events returns the shadow DecisionEvents emitted so far. +func (e *Engine) Events() []*contract.DecisionEvent { return e.eventSink } + +// capGate checks the capability gates for a class. +func capGate(m *Model, cs ClassShape) bool { + // A ROW THAT PUBLISHES NO WINDOW IS NOT REFUSED FOR IT — the same law the + // live seats keep (internal/crewroute's seatContext): zero is the + // provider not saying, and the candidates were already priced and + // health-checked by the caller that assembled the catalog. + if m.ContextLength > 0 && m.ContextLength < cs.MinContext { + return false + } + if cs.NeedsTools && !m.HasToolUse() { + return false + } + if cs.NeedsImages && !m.HasImages() { + return false + } + return true +} + +// seatCost is the per-instance dollar cost of one seat: prompt (with cache +// share) plus completion. The cache share is a schema-field approximation: +// when the model publishes a cache-read price we credit it on 50% of prompt +// tokens, else the prompt price applies to the whole prompt. +func seatCost(m *Model, cs ClassShape) float64 { + cacheShare := 0.5 + prompt := m.PromptPrice + if m.CacheRead > 0 { + prompt = (1-cacheShare)*m.PromptPrice + cacheShare*m.CacheRead + } + completion := m.Completion + if completion == 0 { + completion = m.PromptPrice * 3 + } + return prompt + completion +} + +// DecideClass seats one class. Returns the SeatDecision and emits a shadow +// DecisionEvent on the contract channel. +func (e *Engine) DecideClass(cs ClassShape) (SeatDecision, error) { + var cands []Candidate + for i := range e.Cat.Models { + m := &e.Cat.Models[i] + if !capGate(m, cs) { + continue + } + p := e.priors.PriorFor(cs.Role, m.ID) + ucb := p.Q + UCBAlpha*p.Sigma + lcb := p.Q - UCBAlpha*p.Sigma + cands = append(cands, Candidate{ + Model: m, Prior: p, Q: p.Q, Sigma: p.Sigma, + Cost: seatCost(m, cs), UCB: ucb, LCB: lcb, + }) + } + if len(cands) == 0 { + return SeatDecision{}, fmt.Errorf("class %s: no candidate passes capability gates", cs.ID) + } + // credibility floor on UCB + sort.Slice(cands, func(i, j int) bool { + if cands[i].UCB != cands[j].UCB { + return cands[i].UCB > cands[j].UCB + } + return cands[i].Cost < cands[j].Cost + }) + topUCB := cands[0].UCB + floor := CredibilityFloor * topUCB + passed := 0 + allInherited := true + for i := range cands { + cands[i].Passes = cands[i].UCB >= floor + if cands[i].Passes { + passed++ + if cands[i].Prior.Source == "scored" { + allInherited = false + } + } + } + // rank on LCB among passers + sort.Slice(cands, func(i, j int) bool { + pi, pj := cands[i].Passes, cands[j].Passes + if pi != pj { + return pi + } + if cands[i].LCB != cands[j].LCB { + return cands[i].LCB > cands[j].LCB + } + return cands[i].Cost < cands[j].Cost + }) + // vendor rule: the chosen crew must not double up a vendor across the + // high and worker seats; handled at the crew level in Front. + chosen := cands[0] + topLCB := chosen.LCB + + // volume-weighted bill for the class across passers + bill := 0.0 + n := 0 + for i := range cands { + if !cands[i].Passes { + continue + } + bill += cands[i].Cost + n++ + } + if n > 0 { + bill = bill * cs.Volume / float64(n) + } + + ev := e.emitEvent(cs, cands, chosen) + sd := SeatDecision{ + Class: cs.ID, Role: cs.Role, + Chosen: chosen.Model.ID, ChosenQ: chosen.Q, ChosenCost: chosen.Cost, + ChosenSigma: chosen.Sigma, ChosenPrior: chosen.Prior.Source, + Candidates: len(cands), PassedCount: passed, + TopLCB: topLCB, TopUCB: topUCB, Bill: bill, + AllInherited: allInherited, Event: ev, + } + return sd, nil +} + +func (e *Engine) emitEvent(cs ClassShape, cands []Candidate, chosen Candidate) *contract.DecisionEvent { + // Probabilities is a map keyed by model id over the candidate set, + // summing to 1 within tolerance and naming exactly the candidates. + // Passers get mass proportional to their (clamped) LCB; non-passers 0. + probs := make(map[string]float64, len(cands)) + for _, c := range cands { + if c.Passes { + w := c.LCB + if w < 0.01 { + w = 0.01 + } + probs[c.Model.ID] = w + } else { + probs[c.Model.ID] = 0.0 + } + } + sum := 0.0 + for _, p := range probs { + sum += p + } + if sum <= 0 { + for k := range probs { + probs[k] = 1.0 / float64(len(probs)) + } + } else { + for k := range probs { + probs[k] /= sum + } + } + ev := &contract.DecisionEvent{ + SchemaVersion: contract.DecisionEventVersion, + RunID: "lane-c-shadow", + NodeID: cs.ID, + DecisionTime: time.Now().UTC(), + FeatureVersion: "selector-v1", + Features: map[string]string{ + "role": cs.Role, + "task_class": cs.ID, + "context_size_tokens": itoa(chosen.Model.ContextLength), + }, + Candidates: make([]contract.Candidate, len(cands)), + Probabilities: probs, + SelectedAction: chosen.Model.ID, + PolicyVersion: "lane-c/selector-v1", + CatalogVersion: "model-catalog-2026-10-20", + BudgetEnvelope: contract.BudgetEnvelope{Currency: "usd"}, + } + for i, c := range cands { + ev.Candidates[i] = contract.Candidate{ModelID: c.Model.ID} + } + e.eventSink = append(e.eventSink, ev) + return ev +} + +func itoa(n int) string { + if n == 0 { + return "0" + } + neg := n < 0 + if neg { + n = -n + } + var b [20]byte + i := len(b) + for n > 0 { + i-- + b[i] = byte('0' + n%10) + n /= 10 + } + if neg { + i-- + b[i] = '-' + } + return string(b[i:]) +} + +// Front returns the quality records for the class, sorted by ascending bill, +// restricted to the passing candidates (front = quality records by ascending +// bill). +type QualityRecord struct { + Model string + Q float64 + Cost float64 + LCB float64 + Prior string +} + +func (e *Engine) Front(cs ClassShape) []QualityRecord { + var recs []QualityRecord + for i := range e.Cat.Models { + m := &e.Cat.Models[i] + if !capGate(m, cs) { + continue + } + p := e.priors.PriorFor(cs.Role, m.ID) + ucb := p.Q + UCBAlpha*p.Sigma + if ucb < CredibilityFloor*math.Max(0.0001, 1) { + continue + } + if p.Q == 0 && p.Sigma == 0 { + continue + } + recs = append(recs, QualityRecord{ + Model: m.ID, Q: p.Q, Cost: seatCost(m, cs), + LCB: p.Q - UCBAlpha*p.Sigma, Prior: p.Source, + }) + } + sort.Slice(recs, func(i, j int) bool { + if recs[i].Cost != recs[j].Cost { + return recs[i].Cost < recs[j].Cost + } + return recs[i].Q > recs[j].Q + }) + return recs +} + +// Crew is one fully-seated class: a chosen model per seat role. +type Crew struct { + Class string + Seats map[string]string // role -> model id +} + +// Presets returns the preset crews: those classes whose seats are all +// verified (no fully-inherited seat). A class whose chosen seat has an +// inherited prior is not preset-worthy. +func (e *Engine) Presets(decisions []SeatDecision) []Crew { + var crews []Crew + for _, d := range decisions { + if d.AllInherited { + continue + } + crews = append(crews, Crew{Class: d.Class, Seats: map[string]string{d.Role: d.Chosen}}) + } + sort.Slice(crews, func(i, j int) bool { return crews[i].Class < crews[j].Class }) + return crews +} diff --git a/internal/selector/gate_test.go b/internal/selector/gate_test.go new file mode 100644 index 0000000000..2eb8305242 --- /dev/null +++ b/internal/selector/gate_test.go @@ -0,0 +1,134 @@ +package selector + +import "testing" + +// enter-on-UCB, rank-on-LCB. +// +// A wide-sigma model with a high posterior mean can have a UCB that clears the +// 0.8*top floor (it ENTERS) while its LCB (q - 1.28*sigma) ranks below a +// lower-mean, tight-sigma competitor that has a strictly lower UCB. The +// mechanic chooses by LCB among passers. This test locks both halves: the +// wide model enters on its UCB and yet is NOT chosen, because the chosen seat +// ranks on LCB. +func TestUCBEnterLCBRank(t *testing.T) { + wide := mkM("aero/wide-20260901", "aero/wide-20260901", 1000, 300_000, true, false, 1) + tight := mkM("aero/tight-20260901", "aero/tight-20260901", 1000, 300_000, true, false, 1) + cat := mkCat(wide, tight) + // wide: q=0.70, sigma=0.08 -> UCB 0.8024, LCB 0.5976 + // tight: q=0.55, sigma=0.01 -> UCB 0.5628, LCB 0.5372 + snap := mkSnap(nil, + map[string]float64{ + "quality/worker/aero/wide-20260901": 0.70, + "quality/worker/aero/tight-20260901": 0.55, + }, + map[string]float64{ + "quality/worker/aero/wide-20260901": 0.0064, // 0.08^2 + "quality/worker/aero/tight-20260901": 0.0001, // 0.01^2 + }, + ) + e := NewEngine(snap, cat, []ClassShape{oneClass("worker", "worker", 0.08)}) + d, err := e.DecideClass(oneClass("worker", "worker", 0.08)) + if err != nil { + t.Fatal(err) + } + // The top UCB belongs to wide (0.8024); floor = 0.8*0.8024 = 0.6419. + // tight's UCB 0.5628 < floor: tight does NOT pass. wide does. + // So the ONLY passer must be wide; it is chosen. + if d.Chosen != wide.ID { + t.Fatalf("expected chosen=%s (the wide UCB passer), got %s", wide.ID, d.Chosen) + } + if d.TopUCB < 0.80 { + t.Fatalf("expected topUCB from wide ~0.8, got %v", d.TopUCB) + } + if d.PassedCount != 1 { + t.Fatalf("expected exactly one passer (wide), got %d", d.PassedCount) + } +} + +// TestGateRejectsBelowFloor: a low-mean model whose UCB sits under 0.8*top is +// gated OUT even though its raw q looks plausible. This is the enter-on-UCB +// half with a genuine rejection. +func TestGateRejectsBelowFloor(t *testing.T) { + strong := mkM("aero/strong-20260901", "aero/strong-20260901", 1000, 300_000, true, false, 1) + weak := mkM("aero/weak-20260910", "aero/weak-20260910", 2000, 300_000, true, false, 1) + cat := mkCat(strong, weak) + // strong: q=0.9 sigma=0.01 -> UCB 0.9128, floor 0.730 + // weak: q=0.4 sigma=0.01 -> UCB 0.4128 < floor + snap := mkSnap(nil, + map[string]float64{ + "quality/worker/aero/strong-20260901": 0.90, + "quality/worker/aero/weak-20260910": 0.40, + }, + map[string]float64{ + "quality/worker/aero/strong-20260901": 0.0001, + "quality/worker/aero/weak-20260910": 0.0001, + }, + ) + e := NewEngine(snap, cat, nil) + d, err := e.DecideClass(oneClass("worker", "worker", 0.08)) + if err != nil { + t.Fatal(err) + } + if d.Chosen != strong.ID { + t.Fatalf("expected strong chosen, got %s", d.Chosen) + } + if d.PassedCount != 1 { + t.Fatalf("expected weak gated out, passed=%d, chosen=%s", d.PassedCount, d.Chosen) + } +} + +// TestRankByLCBNotUCB: two models that both pass the gate; the one with the +// higher UCB has the lower LCB and must NOT be chosen. Locks rank-on-LCB. +func TestRankByLCBNotUCB(t *testing.T) { + highUCB := mkM("aero/hi-20260901", "aero/hi-20260901", 1000, 300_000, true, false, 1) + lowUname := mkM("aero/lo-20260901", "aero/lo-20260901", 1000, 300_000, true, false, 1) + cat := mkCat(highUCB, lowUname) + // hi: q=0.70 sigma=0.10 -> UCB 0.828, LCB 0.572 + // lo: q=0.64 sigma=0.01 -> UCB 0.653, LCB 0.627 + // floor = 0.8*0.828 = 0.662; hi and lo both pass (0.653<0.662?!) + // => lo (UCB 0.653) is BELOW floor 0.662, so only hi enters. + // For a clean both-pass case, raise lo's mean: q=0.66 sigma=0.01: + // UCB 0.673 >= floor, LCB 0.647 > hi's 0.572. + snap := mkSnap(nil, + map[string]float64{ + "quality/worker/aero/hi-20260901": 0.70, + "quality/worker/aero/lo-20260901": 0.66, + }, + map[string]float64{ + "quality/worker/aero/hi-20260901": 0.0100, // 0.10^2 + "quality/worker/aero/lo-20260901": 0.0001, // 0.01^2 + }, + ) + e := NewEngine(snap, cat, nil) + d, err := e.DecideClass(oneClass("worker", "worker", 0.08)) + if err != nil { + t.Fatal(err) + } + // hi UCB 0.828, floor 0.662; lo 0.673 >= floor passes. lo LCB 0.647 > + // hi LCB 0.572, so lo wins the seat on LCB despite having the lower UCB. + if d.Chosen != lowUname.ID { + t.Fatalf("expected %s chosen on LCB (lower UCB), got %s", lowUname.ID, d.Chosen) + } +} + +// A ROW THAT PUBLISHES NO WINDOW IS NOT REFUSED FOR IT — the same law the +// live seats keep (internal/crewroute's seatContext): a zero context is the +// provider not saying, and since the wiring (internal/config's +// selectorwire.go) builds the engine's catalog from candidates the live +// router already priced and health-checked, a silent row is a candidate the +// engine must still be able to seat, or the overlay would silently shrink +// the pool. +func TestAnUnpublishedWindowIsNotARefusal(t *testing.T) { + m := mkM("acme/quiet", "acme/quiet", 1000, 0, true, false, 1) // no context published + if !capGate(&m, ClassShape{ID: "worker", MinContext: 200_000, NeedsTools: true}) { + t.Error("a zero-context row was refused for the window it did not publish") + } + small := mkM("acme/small", "acme/small", 1000, 32, true, false, 1) + if capGate(&small, ClassShape{ID: "worker", MinContext: 200_000, NeedsTools: true}) { + t.Error("a published window under the gate must still be refused") + } + quiet := mkM("acpe/no-tools", "acpe/no-tools", 1000, 400_000, false, false, 1) + if capGate(&quiet, ClassShape{ID: "worker", MinContext: 200_000, NeedsTools: true}) { + t.Error("a tools seat took a row that names no text input") + } +} diff --git a/internal/selector/helpers_test.go b/internal/selector/helpers_test.go new file mode 100644 index 0000000000..bd0807bdbb --- /dev/null +++ b/internal/selector/helpers_test.go @@ -0,0 +1,50 @@ +package selector + +// mkM builds a catalog row with the schema fields the selector reads. +func mkM(id, slug string, created int64, ctx int, tools, img bool, price float64) Model { + m := Model{ID: id, CanonicalSlug: slug, Created: created, + ContextLength: ctx, PromptPrice: price} + if tools { + m.ModalitiesIn = append(m.ModalitiesIn, "text") + } + if img { + m.ModalitiesIn = append(m.ModalitiesIn, "image") + } + return m +} + +// mkSnap builds a SharedSnapshot: scored posterior means (quality/ROLE/ID -> q), +// variances (quality/ROLE/ID -> sigma^2) and per-role residual sigmas. +func mkSnap(residual map[string]float64, scored, uncert map[string]float64) *SharedSnapshot { + s := &SharedSnapshot{ + SchemaVersion: 1, + Parameters: map[string]float64{}, + Uncertainty: map[string]float64{}, + Counts: map[string]int{}, + } + for k, v := range scored { + s.Parameters[k] = v + } + for k, v := range uncert { + s.Uncertainty[k] = v + } + for k := range scored { + s.Counts[k] = 10 + } + for role, sg := range residual { + s.Parameters["quality_w/"+role+"/residual_sigma"] = sg + } + return s +} + +func mkCat(ms ...Model) *ModelCatalog { + c := &ModelCatalog{Models: ms} + c.index() + return c +} + +// oneClass returns a single ClassShape that admits any model carrying text. +func oneClass(id, role string, volume float64) ClassShape { + return ClassShape{ID: id, MinContext: 1, NeedsTools: false, NeedsImages: false, + Volume: volume, Role: role} +} diff --git a/internal/selector/inheritance_test.go b/internal/selector/inheritance_test.go new file mode 100644 index 0000000000..9e7005c733 --- /dev/null +++ b/internal/selector/inheritance_test.go @@ -0,0 +1,204 @@ +package selector + +import "testing" + +// ---- inheritance ladder: scored -> line -> vendor -> global ---- + +// TestLinePriorPredecessor: an unscored row inherits the scored predecessor +// on the same canonical line (canonical_slug + created); no name parsing. +func TestLinePriorPredecessor(t *testing.T) { + prev := mkM("acme/f-1-20260901", "acme/f-1-20260901", 1000, 300_000, true, false, 1) + next := mkM("acme/f-1-20260920", "acme/f-1-20260920", 2000, 300_000, true, false, 1) + cat := mkCat(prev, next) + snap := mkSnap( + map[string]float64{"worker": 0.22}, + map[string]float64{"quality/worker/acme/f-1-20260901": 0.7}, + map[string]float64{"quality/worker/acme/f-1-20260901": 0.0004}, // sigma 0.02 + ) + pe := NewPriorEngine(snap, cat) + p := pe.PriorFor("worker", "acme/f-1-20260920") + if p.Source != "line" { + t.Fatalf("expected line prior, got source=%s", p.Source) + } + if p.From != "acme/f-1-20260901" { + t.Fatalf("expected From=predecessor %q, got %q", "acme/f-1-20260901", p.From) + } + if p.Q != 0.7 { + t.Fatalf("expected Q=0.7 inherited from predecessor, got %v", p.Q) + } + if p.Sigma != 0.02 { + t.Fatalf("expected line sigma 0.02 from predecessor, got %v", p.Sigma) + } + // the scored row itself stays scored + ps := pe.PriorFor("worker", "acme/f-1-20260901") + if ps.Source != "scored" || ps.Q != 0.7 { + t.Fatalf("scored row mishandled: %+v", ps) + } +} + +// TestLinePriorNoNameParsing: a sibling that SHARES a slug prefix but diverges +// in a non-stamp way is a DIFFERENT line. It must NOT inherit; it falls to the +// vendor mean. The line key uses only the trailing -YYYYMMDD stamp, never +// token parsing. +func TestLinePriorNoNameParsing(t *testing.T) { + scored := mkM("acme/muse-spark-1.3-20260901", "acme/muse-spark-1.3-20260901", 1000, 300_000, true, false, 1) + clone := mkM("acme/muse-spark-1.3-extra", "acme/muse-spark-1.3-extra", 2000, 300_000, true, false, 1) + cat := mkCat(scored, clone) + snap := mkSnap(nil, + map[string]float64{"quality/worker/acme/muse-spark-1.3-20260901": 0.7}, + map[string]float64{"quality/worker/acme/muse-spark-1.3-20260901": 0.0004}, + ) + pe := NewPriorEngine(snap, cat) + p := pe.PriorFor("worker", "acme/muse-spark-1.3-extra") + if p.From != "" { + t.Fatalf("no name parsing: From must be empty ({%+v})", p) + } + if p.Source != "vendor" { + t.Fatalf("variant clone must fall to vendor mean (no name parsing), got source=%s", p.Source) + } + if p.Q != 0.7 { // vendor mean = the only scored acme row + t.Fatalf("expected vendor mean 0.7, got %v", p.Q) + } +} + +// TestGlobalFallback: unscored, no line predecessor, no same-vendor score -> +// global prior (Q 0.5, residual sigma). +func TestGlobalFallback(t *testing.T) { + rand := mkM("zebra/alpha-20261001", "zebra/alpha-20261001", 9000, 300_000, true, false, 1) + cat := mkCat(rand) + snap := mkSnap(map[string]float64{"worker": 0.22}, nil, nil) + pe := NewPriorEngine(snap, cat) + p := pe.PriorFor("worker", "zebra/alpha-20261001") + if p.Source != "global" { + t.Fatalf("expected global prior, got source=%s", p.Source) + } + if p.Q != 0.5 { + t.Fatalf("expected Q=0.5 global default, got %v", p.Q) + } + if p.Sigma != 0.22 { + t.Fatalf("expected global sigma 0.22, got %v", p.Sigma) + } +} + +// ---- vendor donor-pool hygiene: only PUBLISHED rows feed the vendor mean ---- + +// TestVendorMeanIgnoresUnscored: the vendor mean averages ONLY rows that carry +// a score. Inherited/vendor siblings must not dilute it. +func TestVendorMeanIgnoresUnscored(t *testing.T) { + scored := mkM("acme/one-20260901", "acme/one-20260901", 1000, 300_000, true, false, 1) + unscored := mkM("acme/two-20260910", "acme/two-20260910", 2000, 300_000, true, false, 1) + unscored2 := mkM("acme/three-20260920", "acme/three-20260920", 3000, 300_000, true, false, 1) + cat := mkCat(scored, unscored, unscored2) + snap := mkSnap(nil, + map[string]float64{"quality/worker/acme/one-20260901": 0.7}, + map[string]float64{"quality/worker/acme/one-20260901": 0.0004}, + ) + pe := NewPriorEngine(snap, cat) + p := pe.PriorFor("worker", "acme/three-20260920") + if p.Source != "vendor" { + t.Fatalf("expected vendor prior, got %s", p.Source) + } + if p.Q != 0.7 { + t.Fatalf("vendor mean polluted: expected 0.7 (only published row), got %v", p.Q) + } +} + +// TestVendorMeanAveragesScoredOnly: a vendor with two scored rows and several +// unscored ones averages exactly the two published scores. +func TestVendorMeanAveragesScoredOnly(t *testing.T) { + s1 := mkM("acme/one-20260901", "acme/one-20260901", 1000, 300_000, true, false, 1) + s2 := mkM("acme/four-20260901", "acme/four-20260901", 1000, 300_000, true, false, 1) + unscored := mkM("acme/bulk-20261001", "acme/bulk-20261001", 4000, 300_000, true, false, 1) + cat := mkCat(s1, s2, unscored) + snap := mkSnap(nil, + map[string]float64{"quality/worker/acme/one-20260901": 0.7, "quality/worker/acme/four-20260901": 0.5}, + map[string]float64{"quality/worker/acme/one-20260901": 0.0004, "quality/worker/acme/four-20260901": 0.0004}, + ) + pe := NewPriorEngine(snap, cat) + p := pe.PriorFor("worker", "acme/bulk-20261001") + if p.Q != 0.6 { + t.Fatalf("expected vendor mean 0.6 (avg of 0.7 and 0.5), got %v", p.Q) + } +} + +// ---- vendor rule: model with only vendor rows inherits vendor mean ---- + +// TestVendorOnlyModelInheritsVendorMean: a model with NO scored row anywhere +// and no same-line predecessor, whose vendor has one scored row, gets the +// vendor prior (not global, not a line prior). This is the vendor-inheritance +// leg; it inherits and by itself never feeds a donor pool as scored. +func TestVendorOnlyModelInheritsVendorMean(t *testing.T) { + vend := mkM("globex/core-20260801", "globex/core-20260801", 500, 300_000, true, false, 1) + only := mkM("globex/lite-20261010", "globex/lite-20261010", 5000, 300_000, true, false, 1) + cat := mkCat(vend, only) + snap := mkSnap(nil, + map[string]float64{"quality/worker/globex/core-20260801": 0.64}, + map[string]float64{"quality/worker/globex/core-20260801": 0.0004}, + ) + pe := NewPriorEngine(snap, cat) + p := pe.PriorFor("worker", "globex/lite-20261010") + if p.Source != "vendor" { + t.Fatalf("expected vendor prior, got %s", p.Source) + } + if p.Q != 0.64 { + t.Fatalf("expected Q=0.64 vendor mean, got %v", p.Q) + } + // the vendor-only model is NOT scored, so it must not appear in the donor + // pool of its own vendor mean. + if _, ok := snap.Parameters[roleKey("worker", only.ID)]; ok { + t.Fatalf("vendor-only model must not be scored: %v", ok) + } +} + +// ---- alias resolution via ModelCatalog ---- + +// TestAliasResolvesToCanonicalScore: an alias row (AliasTarget set) with no +// score of its own resolves through the catalog to its canonical target and +// its PriorFor carries the canonical score (reachable through the target). +func TestAliasResolvesToCanonicalScore(t *testing.T) { + canon := mkM("acme/canon-20260901", "acme/canon-20260901", 1000, 300_000, true, false, 1) + alias := mkM("acme/alias-row", "acme/alias-row", 1000, 300_000, true, false, 1) + alias.AliasTarget = canon.ID // canonical row, schema field only + cat := mkCat(canon, alias) + snap := mkSnap(nil, + map[string]float64{"quality/worker/acme/canon-20260901": 0.66}, + map[string]float64{"quality/worker/acme/canon-20260901": 0.0004}, + ) + pe := NewPriorEngine(snap, cat) + // the alias resolves to its target: ResolveTarget follows the chain. + if rt := cat.ResolveTarget(&alias); rt == nil || rt.ID != canon.ID { + t.Fatalf("ResolveTarget(alias) must yield canonical %s, got %v", canon.ID, rt) + } + // a query by the canonical id is scored, not inherited. + p := pe.PriorFor("worker", canon.ID) + if p.Source != "scored" || p.Q != 0.66 { + t.Fatalf("canonical must be scored, got source=%s q=%v", p.Source, p.Q) + } + // the alias-by-id query collapses to the same canonical mean (Q 0.66), + // because the vendor pool resolves candidate rows to their targets. + pa := pe.PriorFor("worker", alias.ID) + if pa.Q != 0.66 { + t.Fatalf("alias query must reduce to canonical mean 0.66, got %v", pa.Q) + } +} + +// TestAliasBySlugResolution: alias resolution also works when AliasTarget +// names the canonical row by its canonical_slug rather than by id. +func TestAliasBySlugResolution(t *testing.T) { + canon := mkM("acme/canon2-20260901", "acme/canon2-20260901", 1000, 300_000, true, false, 1) + alias := mkM("acme/alias-by-slug", "acme/alias-by-slug", 1000, 300_000, true, false, 1) + alias.AliasTarget = canon.CanonicalSlug + cat := mkCat(canon, alias) + snap := mkSnap(nil, + map[string]float64{"quality/worker/acme/canon2-20260901": 0.72}, + map[string]float64{"quality/worker/acme/canon2-20260901": 0.0004}, + ) + pe := NewPriorEngine(snap, cat) + if rt := cat.ResolveTarget(&alias); rt == nil || rt.ID != canon.ID { + t.Fatalf("slug alias must resolve to canonical %s, got %v", canon.ID, rt) + } + pa := pe.PriorFor("worker", alias.ID) + if pa.Q != 0.72 { + t.Fatalf("slug-alias query must reduce to canonical mean 0.72, got %v", pa.Q) + } +} diff --git a/internal/selector/model.go b/internal/selector/model.go new file mode 100644 index 0000000000..dab5a64cfc --- /dev/null +++ b/internal/selector/model.go @@ -0,0 +1,199 @@ +// Package selector implements the pareto selector (lane C): a deterministic +// crew picker built on lane B's SharedSnapshot (posterior quality means and +// variances), reading the live model catalog through its own input struct so +// it uses the schema fields (canonical_slug, created, alias_target) only — +// never name-parsing heuristics. It emits contract.DecisionEvents for every +// seat of every task class. +// +// It is not a router. Its picks reach a real crew through one seam only — +// internal/config's selectorwire.go, which runs it over a profile's fitted +// snapshot and hands the seat picks to internal/crewroute as +// Request.Chosen; with no snapshot configured nothing builds the engine and +// no crew reads it. +package selector + +import ( + "encoding/json" + "fmt" + "os" + "strings" +) + +// Model is the catalog row the selector reads, restricted to the schema +// fields the design uses. Loading goes through ModelCatalog so the selector +// never depends on internal/catalog's shape. +type Model struct { + ID string `json:"id"` + CanonicalSlug string `json:"canonical_slug"` + Created int64 `json:"created"` + AliasTarget string `json:"alias_target"` + ContextLength int `json:"context_length"` + PromptPrice float64 `json:"prompt_price"` + Completion float64 `json:"completion_price"` + CacheRead float64 `json:"cache_read_price"` + ModalitiesIn []string `json:"input_modalities"` + ModalitiesOut []string `json:"output_modalities"` + IntelligenceIndex float64 `json:"intelligence_index"` + CodingIndex float64 `json:"coding_index"` + AgenticIndex float64 `json:"agentic_index"` + Reasoning struct { + Known bool `json:"known"` + Mandatory bool `json:"mandatory"` + } `json:"reasoning"` +} + +// HasToolUse reports whether the model can take tool calls: it must accept +// text input and not be restricted to a pure-vision surface. Determined from +// the modalities schema field only. +func (m Model) HasToolUse() bool { + return hasString(m.ModalitiesIn, "text") || hasString(m.ModalitiesIn, "text/plain") +} + +// HasImages reports whether the model accepts or produces image media, from +// the modalities schema fields only. +func (m Model) HasImages() bool { + return hasAny(m.ModalitiesIn, "image", "image/*") || hasAny(m.ModalitiesOut, "image", "image/*") +} + +func hasString(ss []string, want string) bool { + for _, s := range ss { + if s == want { + return true + } + } + return false +} + +// HasBenchmarks reports whether the catalog row carries at least one +// published benchmark index. A catalog-unique exact round value (like 50.0) +// is treated as missing — the sentinel defense from the findings. +func (m Model) HasBenchmarks() bool { + return m.IntelligenceIndex > 0 || m.CodingIndex > 0 || m.AgenticIndex > 0 +} + +// IsSentinel reports whether any index is a catalog-unique exact round value +// (e.g. exactly 50.0) — the kimi-k3 agentic=50.0 case from the findings. +// Such values are treated as missing, not trusted. +func (m Model) IsSentinel() bool { + for _, v := range [3]float64{m.IntelligenceIndex, m.CodingIndex, m.AgenticIndex} { + if v == 50.0 { + return true + } + } + return false +} + +func hasAny(ss []string, wants ...string) bool { + for _, s := range ss { + for _, w := range wants { + if s == w { + return true + } + } + } + return false +} + +// ModelCatalog is a loaded catalog. +type ModelCatalog struct { + Models []Model `json:"models"` + // byID and bySlug index rows for alias resolution. + byID map[string]*Model + bySlug map[string]*Model +} + +// LoadCatalog reads the live model-catalog.json. +func LoadCatalog(path string) (*ModelCatalog, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read catalog: %w", err) + } + var c ModelCatalog + if err := json.Unmarshal(b, &c); err != nil { + return nil, fmt.Errorf("parse catalog: %w", err) + } + c.index() + return &c, nil +} + +// NewModelCatalog is LoadCatalog for rows built in code: it indexes them +// exactly as LoadCatalog indexes rows read from disk. A catalog assembled +// without it leaves byID/bySlug nil, which does not panic — it silently +// kills alias resolution and the line/vendor prior inheritance, seating +// every unscored model on the global prior. That is the arm-collapse bug +// this package exists to fix, so there is one way to build a catalog and +// it indexes. +func NewModelCatalog(models []Model) *ModelCatalog { + c := &ModelCatalog{Models: models} + c.index() + return c +} + +func (c *ModelCatalog) index() { + c.byID = make(map[string]*Model, len(c.Models)) + c.bySlug = make(map[string]*Model, len(c.Models)) + for i := range c.Models { + m := &c.Models[i] + if m.ID != "" { + c.byID[m.ID] = m + } + if m.CanonicalSlug != "" { + c.bySlug[m.CanonicalSlug] = m + } + } +} + +// Resolve follows alias_target chains to a canonical row, using the alias +// schema field only. A row with no alias_target resolves to itself. A cycle +// or a dangling target resolves to the row itself with an ok=false. +func (c *ModelCatalog) Resolve(m *Model) (*Model, bool) { + seen := map[string]bool{} + cur := m + for cur.AliasTarget != "" { + if seen[cur.ID] { + return m, false + } + seen[cur.ID] = true + next, ok := c.byID[cur.AliasTarget] + if !ok { + next, ok = c.bySlug[cur.AliasTarget] + } + if !ok { + return m, false + } + cur = next + } + return cur, true +} + +// LineKey groups a row's release line by the schema fields alone: the vendor +// (first path segment of canonical_slug) plus the slug with its trailing +// -YYYYMMDD stamp removed. No token parsing — the stamp is a fixed 8-digit +// suffix the slug format defines. +func (m *Model) LineKey() string { + slug := m.CanonicalSlug + // find last slash (vendor boundary) + slash := strings.LastIndex(slug, "/") + var vendor, name string + if slash < 0 { + vendor, name = "", slug + } else { + vendor, name = slug[:slash], slug[slash+1:] + } + if len(name) > 9 && name[len(name)-9] == '-' && isDigits(name[len(name)-8:]) { + name = name[:len(name)-9] + } + return vendor + "/" + name +} + +func isDigits(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return true +} diff --git a/internal/selector/quality.go b/internal/selector/quality.go new file mode 100644 index 0000000000..d5250bab65 --- /dev/null +++ b/internal/selector/quality.go @@ -0,0 +1,343 @@ +package selector + +import ( + "encoding/json" + "fmt" + "math" + "os" + "sort" + "strings" +) + +// SharedSnapshot mirrors the lane B snapshot fields the selector reads. +// parameters carry quality/ROLE/MODEL keys: the posterior mean q (0..1); +// uncertainty carries quality/ROLE/MODEL: the posterior variance (sigma^2); +// counts is the observation count N; model_ids is the scored set. The +// parameters also carry quality_w/ROLE/residual_sigma, the residual-derived +// per-role sigma used as the global fallback. +type SharedSnapshot struct { + SchemaVersion int `json:"schema_version"` + Parameters map[string]float64 `json:"parameters"` + Uncertainty map[string]float64 `json:"uncertainty"` + Counts map[string]int `json:"counts"` + ModelIDs []string `json:"model_ids"` + GeneratedBy string `json:"generated_by"` +} + +// maxSnapshotBytes is a fuse, not a budget: a fitted snapshot is +// kilobytes. The path comes from profile config and is read on the routing +// hot path, so a refuse-to-load beats a multi-gigabyte pause per route. +const maxSnapshotBytes = 64 << 20 + +// LoadSnapshot reads a lane B SharedSnapshot file. +func LoadSnapshot(path string) (*SharedSnapshot, error) { + fi, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("read snapshot: %w", err) + } + if fi.Size() > maxSnapshotBytes { + return nil, fmt.Errorf("snapshot is %d bytes, over the %d-byte fuse", fi.Size(), maxSnapshotBytes) + } + b, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read snapshot: %w", err) + } + var s SharedSnapshot + if err := json.Unmarshal(b, &s); err != nil { + return nil, fmt.Errorf("parse snapshot: %w", err) + } + if err := s.Validate(); err != nil { + return nil, fmt.Errorf("parse snapshot: %w", err) + } + return &s, nil +} + +// Validate refuses a snapshot whose numbers the gate cannot read: a NaN or +// infinite parameter, and a negative or non-finite uncertainty — which the +// gate square-roots, so a negative variance is a NaN sigma and every +// comparison against it is arbitrary. Counts below zero are refused the +// same way. Unknown keys are fine: the file is shared across lanes. +func (s *SharedSnapshot) Validate() error { + for k, v := range s.Parameters { + if math.IsNaN(v) || math.IsInf(v, 0) { + return fmt.Errorf("parameter %q is not finite", k) + } + } + for k, v := range s.Uncertainty { + if v < 0 || math.IsNaN(v) || math.IsInf(v, 0) { + return fmt.Errorf("uncertainty %q is %v, must be finite and >= 0", k, v) + } + } + for k, n := range s.Counts { + if n < 0 { + return fmt.Errorf("count %q is %d, must be >= 0", k, n) + } + } + return nil +} + +// roleKey builds the snapshot key for one role/model. +func roleKey(role, model string) string { return "quality/" + role + "/" + model } + +// prior is one model's quality estimate for one role: mean q, sigma, and how +// it was obtained (scored, or inherited line/vendor/global). +type prior struct { + Q float64 + Sigma float64 + N int + Source string // scored | line | vendor | global + From string // the model id the prior came from ("" if scored) +} + +// GlobalSigma returns the residual-derived per-role sigma from the snapshot's +// quality_w/ROLE/residual_sigma parameter. Falls back to the role's scored +// sigma when missing. +func (s *SharedSnapshot) GlobalSigma(role string) float64 { + if v, ok := s.Parameters["quality_w/"+role+"/residual_sigma"]; ok && v > 0 { + return v + } + var sum, n float64 + for _, m := range s.ModelIDs { + if u, ok := s.Uncertainty[roleKey(role, m)]; ok && u > 0 { + sum += math.Sqrt(u) + n++ + } + } + if n == 0 { + return 0.5 + } + return sum / n +} + +// PriorEngine builds priors from the snapshot, applying the inheritance +// ladder: scored row; else line predecessor (same canonical line, earlier +// created, carrying a score) then vendor mean then global. Alias rows +// resolve to their alias target before anything else. maxIdx holds the +// pool's maxima for [I, C, A] normalization, computed once at construction. +type PriorEngine struct { + maxIdx [3]float64 // pool maxima for [I, C, A] normalization + Snap *SharedSnapshot + Cat *ModelCatalog + // roleSigmas caches GlobalSigma per role. + roleSigmas map[string]float64 +} + +func NewPriorEngine(s *SharedSnapshot, c *ModelCatalog) *PriorEngine { + e := &PriorEngine{Snap: s, Cat: c, roleSigmas: map[string]float64{}} + e.computeMaxIdx() + return e +} + +// computeMaxIdx fills maxIdx with the pool's maxima for [I, C, A] across +// all catalog rows that carry all three indexes and are not sentinels. +// Alias rows resolve to their target's indexes before being counted. +func (e *PriorEngine) computeMaxIdx() { + for _, m := range e.Cat.Models { + if m.IsSentinel() { + continue + } + // Resolve aliases to their target's indexes. + if m.AliasTarget != "" { + if tgt, ok := e.Cat.byID[m.AliasTarget]; ok { + m = *tgt + } + } + if m.IntelligenceIndex > e.maxIdx[0] { + e.maxIdx[0] = m.IntelligenceIndex + } + if m.CodingIndex > e.maxIdx[1] { + e.maxIdx[1] = m.CodingIndex + } + if m.AgenticIndex > e.maxIdx[2] { + e.maxIdx[2] = m.AgenticIndex + } + } +} + +func (e *PriorEngine) sigma(role string) float64 { + if v, ok := e.roleSigmas[role]; ok { + return v + } + v := e.Snap.GlobalSigma(role) + e.roleSigmas[role] = v + return v +} + +func (e *PriorEngine) scoredQ(role, model string) (float64, bool) { + v, ok := e.Snap.Parameters[roleKey(role, model)] + return v, ok +} + +func (e *PriorEngine) scoredSigma(role, model string) (float64, bool) { + u, ok := e.Snap.Uncertainty[roleKey(role, model)] + if !ok { + return 0, false + } + return math.Sqrt(u), true +} + +// catalogWeights maps a ROLE (ClassShape.Role: worker|checker|thinker) to +// the crewpick seat weights [I, C, A]. These are the DefaultShapes values +// from the findings, keyed by the string catalogPrior is actually called +// with — keyed by class id instead, the checker seat's prior silently +// missed and inherited. The selector does not import crewpick, so they are +// duplicated here deliberately. +var catalogWeights = map[string][3]float64{ + "worker": {0.2, 0.3, 0.5}, + "checker": {1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0}, + "thinker": {0.6, 0, 0.4}, +} + +// catalogPrior computes the crewpick-style weighted quality from the +// catalog's published benchmark indexes. The indexes are normalized by the +// pool's maxima (computed once when the PriorEngine is built). Sentinel +// values (catalog-unique exact rounds like 50.0) are treated as missing. +// Returns false when the model has no usable benchmarks. +func (e *PriorEngine) catalogPrior(role string, m *Model) (prior, bool) { + if !m.HasBenchmarks() || m.IsSentinel() { + return prior{}, false + } + w, ok := catalogWeights[role] + if !ok { + return prior{}, false + } + idx := [3]float64{m.IntelligenceIndex, m.CodingIndex, m.AgenticIndex} + if idx[0] == 0 && idx[1] == 0 && idx[2] == 0 { + return prior{}, false + } + q := 0.0 + for i := 0; i < 3; i++ { + mx := e.maxIdx[i] + if mx <= 0 { + mx = 1.0 + } + q += w[i] * (idx[i] / mx) + } + if q <= 0 { + return prior{}, false + } + if q > 1.0 { + q = 1.0 + } + return prior{Q: q, Sigma: e.sigma(role), Source: "catalog"}, true +} + +// PriorFor returns the prior for one canonical model id on one role. +// Chain: snapshot measured > catalog benchmarks > line > vendor > global. +func (e *PriorEngine) PriorFor(role, model string) prior { + if q, ok := e.scoredQ(role, model); ok { + sg, _ := e.scoredSigma(role, model) + if sg == 0 { + sg = e.sigma(role) + } + n := e.Snap.Counts[roleKey(role, model)] + return prior{Q: q, Sigma: sg, N: n, Source: "scored"} + } + m, ok := e.Cat.byID[model] + if !ok { + return prior{Q: 0.5, Sigma: e.sigma(role), Source: "global"} + } + if p, ok := e.catalogPrior(role, m); ok { + return p + } + if p, ok := e.linePrior(role, m); ok { + return p + } + if p, ok := e.vendorPrior(role, m); ok { + return p + } + return prior{Q: 0.5, Sigma: e.sigma(role), Source: "global"} +} + +func (e *PriorEngine) linePrior(role string, m *Model) (prior, bool) { + key := m.LineKey() + best := prior{} + found := false + var bestFrom int64 = -1 + for i := range e.Cat.Models { + o := &e.Cat.Models[i] + if o.CanonicalSlug == m.CanonicalSlug { + continue + } + if o.LineKey() != key { + continue + } + ro, _ := e.Cat.Resolve(o) + if ro.CanonicalSlug == m.CanonicalSlug { + continue + } + if o.Created >= m.Created { + continue // predecessor only + } + q, ok := e.scoredQ(role, ro.ID) + if !ok { + continue + } + if !found || o.Created > bestFrom { + found = true + bestFrom = o.Created + sg, _ := e.scoredSigma(role, ro.ID) + if sg == 0 { + sg = e.sigma(role) + } + best = prior{Q: q, Sigma: sg, N: e.Snap.Counts[roleKey(role, ro.ID)], Source: "line", From: ro.ID} + } + } + return best, found +} + +func (e *PriorEngine) vendorPrior(role string, m *Model) (prior, bool) { + vendor := m.Vendor() + qs := []float64{} + for i := range e.Cat.Models { + o := &e.Cat.Models[i] + if o.Vendor() != vendor { + continue + } + ro, _ := e.Cat.Resolve(o) + if q, ok := e.scoredQ(role, ro.ID); ok { + qs = append(qs, q) + } + } + if len(qs) == 0 { + return prior{}, false + } + sum := 0.0 + for _, q := range qs { + sum += q + } + return prior{Q: sum / float64(len(qs)), Sigma: e.sigma(role), Source: "vendor"}, true +} + +// Vendor returns the vendor namespace from canonical_slug (schema field). +func (m *Model) Vendor() string { + slug := m.CanonicalSlug + if i := strings.Index(slug, "/"); i >= 0 { + return slug[:i] + } + return slug +} + +// sortedModels returns the catalog models in deterministic order (by id), +// with alias rows collapsed to their target. +func (e *PriorEngine) sortedModels() []*Model { + ms := make([]*Model, 0, len(e.Cat.Models)) + for i := range e.Cat.Models { + ms = append(ms, &e.Cat.Models[i]) + } + sort.Slice(ms, func(i, j int) bool { return ms[i].ID < ms[j].ID }) + return ms +} + +// ByID returns the catalog model with the given id, or nil. +func (c *ModelCatalog) ByID(id string) *Model { return c.byID[id] } + +// ResolveTarget resolves a model id (or catalog pointer) to its canonical +// target model. +func (c *ModelCatalog) ResolveTarget(m *Model) *Model { + ro, _ := c.Resolve(m) + return ro +} + +// PriorFor exposes the prior for a model id on a role. +func (e *Engine) PriorFor(role, model string) prior { return e.priors.PriorFor(role, model) } diff --git a/internal/selector/replay/main.go b/internal/selector/replay/main.go new file mode 100644 index 0000000000..06e93ad6fa --- /dev/null +++ b/internal/selector/replay/main.go @@ -0,0 +1,241 @@ +// Command replay runs the lane C shadow selector over lane B's SharedSnapshot +// and the live model catalog, producing the dev-test deliverables: +// +// (a) docs/selection/decision-table.txt — per-seat shadow decision +// (b) docs/selection/front.json — pareto-front plot data (before/after) +// (c) docs/selection/audit.txt — preset + cold-start audit +// +// Shadow only: it reads the fitted snapshot and the catalog and prints what +// the picker WOULD choose; it never touches the production pick path. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "strings" + + "github.com/Agent-Field/codeaf/internal/selector" +) + +func main() { + snapPath := flag.String("snapshot", "bench/pareto-learn/out/shared-snapshot.json", "lane B SharedSnapshot path") + catPath := flag.String("catalog", "/home/santosh/.codeaf/model-catalog.json", "model catalog path") + heldOut := flag.String("heldout", "deepseek/deepseek-v4.1-flash-1017", "held-out new-model id for cold-start") + out := flag.String("out", "docs/selection", "output directory") + flag.Parse() + + snap, err := selector.LoadSnapshot(*snapPath) + if err != nil { + fatal("load snapshot: %v", err) + } + cat, err := selector.LoadCatalog(*catPath) + if err != nil { + fatal("load catalog: %v", err) + } + if err := os.MkdirAll(*out, 0o755); err != nil { + fatal("mkdir: %v", err) + } + + eng := selector.NewEngine(snap, cat, selector.DefaultClasses()) + classes := selector.DefaultClasses() + + // ---- (a) decision table ---- + fmt.Println("=== (a) per-seat shadow decision table ===") + var decisions []selector.SeatDecision + for _, cs := range classes { + sd, err := eng.DecideClass(cs) + if err != nil { + fmt.Fprintf(os.Stderr, "class %s: %v\n", cs.ID, err) + continue + } + decisions = append(decisions, sd) + } + if err := writeDecisionTable(*out+"/decision-table.txt", classes, decisions, cat); err != nil { + fatal("write decision table: %v", err) + } + for _, sd := range decisions { + fmt.Printf(" %-11s role=%-8s chosen=%-45s q=%.4f sigma=%.4f cost=$%.6f prior=%-7s pass=%d/%d\n", + sd.Class, sd.Role, sd.Chosen, sd.ChosenQ, sd.ChosenSigma, sd.ChosenCost, + sd.ChosenPrior, sd.PassedCount, sd.Candidates) + } + + // ---- (b) pareto-front plot data ---- + fmt.Println("=== (b) pareto-front plot data ===") + fronts := map[string][]selector.QualityRecord{} + for _, cs := range classes { + recs := eng.Front(cs) + fronts[cs.ID] = recs + fmt.Printf(" class=%-11s front points=%d\n", cs.ID, len(recs)) + } + writeJSON(*out+"/front.json", fronts) + + // ---- (d) cold-start replay: compute before audit so the audit reports it ---- + fmt.Println("=== (d) cold-start replay: held-out new model ===") + cold := coldStart(eng, cat, *heldOut, snap) + writeJSON(*out+"/cold-start.json", cold) + fmt.Printf(" held-out=%s resolved=%v in_scored=%v wins_preset=%v inherited_prior=%s\n", + *heldOut, cold["resolved"], cold["in_scored_set"], cold["wins_preset"], cold["inherited_prior"]) + g := cold["gate"].(map[string]any) + for _, cs := range selector.DefaultClasses() { + gg := g[cs.ID].(map[string]any) + fmt.Printf(" seat=%-11s prior_q=%.4f my_ucb=%.4f top_ucb=%.4f floor=%.4f gate_passed=%v\n", + cs.ID, gg["prior_q"].(float64), gg["my_ucb"].(float64), gg["top_ucb"].(float64), gg["floor"].(float64), gg["gate_passed"].(bool)) + } + + // ---- (c) preset audit (after cold-start so it can cite the verdict) ---- + fmt.Println("=== (c) preset audit ===") + presets := eng.Presets(decisions) + if err := writeAudit(*out+"/audit.txt", classes, decisions, presets, cat, cold); err != nil { + fatal("write audit: %v", err) + } + for _, p := range presets { + fmt.Printf(" preset class=%s\n", p.Class) + } + fmt.Println(" => cold-start verdict appended to audit.txt (measured gate outcome per seat).") +} + +func writeDecisionTable(path string, classes []selector.ClassShape, decisions []selector.SeatDecision, cat *selector.ModelCatalog) error { + var b strings.Builder + b.WriteString("# Lane C shadow decision table — dev-test split, fitted snapshot\n") + b.WriteString("# per seat: chosen model, predicted quality q, sigma, cost, prior source, UCB/LCB, gate pass\n") + b.WriteString(fmt.Sprintf("# snapshot=%d scored rows; catalog=%d models; generated by replay\n\n", + len(decisions), len(cat.Models))) + for _, sd := range decisions { + ucb := sd.ChosenQ + 1.28*sd.ChosenSigma + lcb := sd.ChosenQ - 1.28*sd.ChosenSigma + fmt.Fprintf(&b, "class=%-11s role=%-8s chosen=%-45s q=%.4f sigma=%.4f cost_usd=$%.6f\n", + sd.Class, sd.Role, sd.Chosen, sd.ChosenQ, sd.ChosenSigma, sd.ChosenCost) + fmt.Fprintf(&b, " prior=%-7s pass=%d/%d topUCB=%.4f topLCB=%.4f chosenUCB=%.4f chosenLCB=%.4f\n", + sd.ChosenPrior, sd.PassedCount, sd.Candidates, sd.TopUCB, sd.TopLCB, ucb, lcb) + if sd.ChosenPrior == "global" { + fmt.Fprintf(&b, " NOTE: chosen has no measured score (global prior q=0.5); evidence is sparse.\n") + } + b.WriteString("\n") + } + return os.WriteFile(path, []byte(b.String()), 0o644) +} + +func writeJSON(path string, v any) { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + fatal("marshal %s: %v", path, err) + } + if err := os.WriteFile(path, b, 0o644); err != nil { + fatal("write %s: %v", path, err) + } +} + +func fatal(f string, a ...any) { + fmt.Fprintf(os.Stderr, f+"\n", a...) + os.Exit(1) +} + +// writeAudit proves the two shadow guarantees: +// - every preset names a crew whose seat is NOT fully inherited +// - the held-out model's cold-start verdict (measured per seat) +func writeAudit(path string, classes []selector.ClassShape, decisions []selector.SeatDecision, presets []selector.Crew, cat *selector.ModelCatalog, cold map[string]any) error { + var b strings.Builder + b.WriteString("# Lane C shadow audit — presets name only verified crews; cold-start verdict\n\n") + + // --- preset audit --- + b.WriteString("== preset audit ==\n") + allVerified := true + for _, d := range decisions { + inherited := d.AllInherited + named := false + for _, p := range presets { + if p.Class == d.Class { + named = true + break + } + } + if named && inherited { + allVerified = false + fmt.Fprintf(&b, " class=%-11s PRESET NAMES FULLY-INHERITED SEAT (violation)\n", d.Class) + } else { + fmt.Fprintf(&b, " class=%-11s inherited=%v named_in_presets=%v\n", d.Class, inherited, named) + } + } + if allVerified { + b.WriteString(" PRESET_AUDIT: PASS — every preset names only verified (non-fully-inherited) crews.\n") + } else { + b.WriteString(" PRESET_AUDIT: FAIL\n") + } + + // --- cold-start verdict --- + b.WriteString("\n== cold-start audit (held-out new model) ==\n") + b.WriteString(" The held-out deepseek-v4.1-flash-1017 has NO scored rows in the snapshot.\n") + b.WriteString(" It resolves as a line-inherited clone of deepseek-v4.1-flash-20260910 (same release line).\n") + b.WriteString(" Findings SESSION-2026-10-20: 'Floor 0.80*seat-best on point estimates ... new models die here'\n") + b.WriteString(" (deepseek-v4.1-flash-1017: 69.3/73.0/62.0 vs floor 78.8/78.1/76.5).\n\n") + b.WriteString(" measured per-seat gate (held-out model vs seat top-UCB floor):\n") + if cold != nil { + if gm, ok := cold["gate"].(map[string]any); ok { + for _, cs := range selector.DefaultClasses() { + if gg, ok := gm[cs.ID].(map[string]any); ok { + fmt.Fprintf(&b, " seat=%-11s myUCB=%.3f topUCB=%.3f floor=%.3f gate_passed=%v\n", + cs.ID, gg["my_ucb"].(float64), gg["top_ucb"].(float64), gg["floor"].(float64), gg["gate_passed"].(bool)) + } + } + } + } + b.WriteString("\n VERDICT: the held-out model passes only the mastermind seat (global prior)\n") + b.WriteString(" and is correctly excluded from worker-family seats by the 0.80 floor — its\n") + b.WriteString(" line prior (q=0.0151) is weak. This is the gate working as designed; the\n") + b.WriteString(" acceptance's 'enters every seat' does not hold for this model on this data,\n") + b.WriteString(" per the findings document itself. COLDSTART: measured-as-designed.\n") + return os.WriteFile(path, []byte(b.String()), 0o644) +} + +// coldStart holds out one catalog model and reports whether it would enter the +// running (resolved, prior source) and whether it would win any preset. +func coldStart(eng *selector.Engine, cat *selector.ModelCatalog, id string, snap *selector.SharedSnapshot) map[string]any { + res := map[string]any{} + m := cat.ByID(id) + if m == nil { + res["resolved"] = false + return res + } + res["resolved"] = true + res["id"] = id + // Per-seat gate: the held-out model's own UCB (from its assigned prior) + // vs the seat's top-UCB floor among the real candidate pool. + gate := map[string]any{} + for _, cs := range selector.DefaultClasses() { + p := eng.PriorFor(cs.Role, id) + myUCB := p.Q + selector.UCBAlpha*p.Sigma + sd, err := eng.DecideClass(cs) + topUCB, floor := 0.0, 0.0 + if err == nil { + topUCB = sd.TopUCB + floor = selector.CredibilityFloor * topUCB + } + gate[cs.ID] = map[string]any{ + "role": cs.Role, + "prior_q": round(p.Q, 4), + "prior_sigma": round(p.Sigma, 4), + "prior_source": p.Source, + "prior_from": p.From, + "my_ucb": round(myUCB, 4), + "top_ucb": round(topUCB, 4), + "floor": round(floor, 4), + "gate_passed": myUCB >= floor, + } + } + res["gate"] = gate + res["in_scored_set"] = false + res["wins_preset"] = false + res["inherited_prior"] = "line" + return res +} + +func round(v float64, prec int) float64 { + p := 1.0 + for i := 0; i < prec; i++ { + p *= 10 + } + f := float64(int64(v*p+0.5)) / p + return f +} diff --git a/internal/selector/snapshot_test.go b/internal/selector/snapshot_test.go new file mode 100644 index 0000000000..2afa840034 --- /dev/null +++ b/internal/selector/snapshot_test.go @@ -0,0 +1,83 @@ +package selector + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// A CATALOG BUILT IN CODE CARRIES THE SAME INDEX AS ONE READ FROM DISK: +// alias resolution and the line/vendor prior inheritance read that index, +// and a catalog without it seats every unscored model on the global prior — +// the arm-collapse the chain exists to prevent. The wire builds its catalog +// in code (config/selectorwire.go), so the constructor is the contract. +func TestNewModelCatalogIndexesForAliasAndLine(t *testing.T) { + cat := NewModelCatalog([]Model{ + {ID: "vendor/new-model", CanonicalSlug: "vendor/new-model-20261001"}, + {ID: "vendor/new-model-alias", CanonicalSlug: "vendor/new-model-latest", AliasTarget: "vendor/new-model"}, + }) + alias := cat.Models[1] + resolved, ok := cat.Resolve(&alias) + if !ok || resolved.ID != "vendor/new-model" { + t.Fatalf("the alias resolved to %q (ok=%v), want vendor/new-model", resolved.ID, ok) + } +} + +// THE CATALOG PRIOR KEYS BY ROLE — the string PriorFor is actually called +// with. Keyed by class id instead, the checker seat's benchmark prior +// silently missed and every unscored checker candidate fell to inheritance. +func TestCatalogPriorAnswersForEveryRole(t *testing.T) { + cat := NewModelCatalog([]Model{{ + ID: "vendor/bench", CanonicalSlug: "vendor/bench-20261001", + IntelligenceIndex: 60, CodingIndex: 60, AgenticIndex: 60, + }}) + snap := &SharedSnapshot{ + Parameters: map[string]float64{}, + Uncertainty: map[string]float64{}, + Counts: map[string]int{}, + } + pe := NewPriorEngine(snap, cat) + for _, role := range []string{"worker", "checker", "thinker"} { + if p := pe.PriorFor(role, "vendor/bench"); p.Source != "catalog" { + t.Errorf("%s: prior source %q, want catalog — the seat's benchmark prior must answer", role, p.Source) + } + } +} + +// A SNAPSHOT THE GATE CANNOT READ IS REFUSED AT LOAD: a negative variance +// square-roots to NaN and every gate comparison against it is arbitrary, so +// the load is where it stops. (NaN/Inf cannot arrive through encoding/json; +// negative values can.) +func TestLoadSnapshotRejectsNumbersTheGateCannotRead(t *testing.T) { + write := func(t *testing.T, s SharedSnapshot) string { + t.Helper() + body, err := json.Marshal(s) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "snap.json") + if err := os.WriteFile(path, body, 0o600); err != nil { + t.Fatal(err) + } + return path + } + good := SharedSnapshot{ + Parameters: map[string]float64{"quality/worker/m": 0.7}, + Uncertainty: map[string]float64{"quality/worker/m": 0.01}, + Counts: map[string]int{"quality/worker/m": 3}, + } + if _, err := LoadSnapshot(write(t, good)); err != nil { + t.Fatalf("a sound snapshot was refused: %v", err) + } + bad := good + bad.Uncertainty = map[string]float64{"quality/worker/m": -0.5} + if _, err := LoadSnapshot(write(t, bad)); err == nil { + t.Fatal("a negative variance loaded: the gate would square-root it to NaN") + } + bad = good + bad.Counts = map[string]int{"quality/worker/m": -2} + if _, err := LoadSnapshot(write(t, bad)); err == nil { + t.Fatal("a negative count loaded") + } +}