diff --git a/cmd/cc-session/benchmark.go b/cmd/cc-session/benchmark.go index 22ade97..2ded2ea 100644 --- a/cmd/cc-session/benchmark.go +++ b/cmd/cc-session/benchmark.go @@ -30,7 +30,7 @@ func runBenchmark(args []string, out io.Writer, errOut io.Writer, store parser.S days := fs.Int("days", 30, "how far back to scan") minKB := fs.Int("min-kb", 100, "minimum JSONL file size in KB") maxN := fs.Int("n", 10, "max sessions to include") - model := fs.String("model", "opus", "model: opus, opus-4-6, opus-4-7, opus-4-8, or sonnet") + model := fs.String("model", "opus", "model: opus, opus-4-6, opus-4-7, opus-4-8, sonnet, or fable (fable-5-1)") overhead := fs.Int("overhead", 0, "session overhead tokens (system+tools+CLAUDE.md); measure with a 1-turn session") isNoAPI := fs.Bool("no-api", false, "skip API calls; estimate filtered-text tokens with chars/2 (offline fallback)") if err := fs.Parse(reorderArgs(args)); err != nil { diff --git a/docs/benchmark.md b/docs/benchmark.md index 0cec1f1..af5c2a6 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -36,7 +36,7 @@ All other parameters are derived automatically from your real session data. | `--days` | 30 | How far back to scan for sessions | | `--min-kb` | 100 | Minimum JSONL file size in KB | | `--n` | 10 | Max successful session results to report | -| `--model` | opus | Pricing and token-counting model: `opus`, `opus-4-6`, `opus-4-7`, `opus-4-8`, or `sonnet` | +| `--model` | opus | Pricing and token-counting model: `opus`, `opus-4-6`, `opus-4-7`, `opus-4-8`, `sonnet`, or `fable` (`fable-5-1`) | ### Example output @@ -164,10 +164,15 @@ the historical one-shot `NewCtx × CacheWrite` behavior. counting API. The `--model` flag controls both pricing and the tokenizer used by the token counting API. `opus` is an alias for `opus-4-8`; explicit Opus versions map to `claude-opus-4-6`, `claude-opus-4-7`, or `claude-opus-4-8`; `sonnet` maps -to `claude-sonnet-4-6`. Opus 4.6, 4.7, and 4.8 use the same Opus pricing rates. +to `claude-sonnet-4-6`; `fable` and `fable-5-1` both map to `claude-fable-5-1`. +Opus 4.6, 4.7, and 4.8 use the same Opus pricing rates. Fallback constants are used only for behavior that cannot be read directly from transcript usage, such as sparse tool I/O data. +Claude Fable 5.1's cache read is 2.5% of base input, not the 10% every other +model here uses, so its cost-savings numbers are not directly comparable to the +Opus/Sonnet rows. + ### Simplifications - **Uncached input (`input_tokens`) omitted**: with auto-caching the breakpoint sits on the last cacheable block, making the uncached tail near zero. This is an inference from the docs (not an explicit statement). Impact on results: < 2% based on sensitivity analysis. diff --git a/internal/benchmark/model.go b/internal/benchmark/model.go index 44b0db7..11049a0 100644 --- a/internal/benchmark/model.go +++ b/internal/benchmark/model.go @@ -12,11 +12,18 @@ type Pricing struct { var PricingOpus = Pricing{CachedRead: 0.50, CacheWrite: 6.25, BaseInput: 5.00} var PricingSonnet = Pricing{CachedRead: 0.30, CacheWrite: 3.75, BaseInput: 3.00} +// PricingFable is Claude Fable 5.1's pricing. Unlike every other tier here, its +// cache read is 2.5% of base input rather than the usual 10% (source: Anthropic +// pricing docs, "a cache hit costs 2.5% of the standard input price"), so its +// cost-savings numbers are not directly comparable to the Opus/Sonnet rows. +var PricingFable = Pricing{CachedRead: 0.25, CacheWrite: 12.50, BaseInput: 10.00} + const ( TokenCountModelOpus46 = "claude-opus-4-6" TokenCountModelOpus47 = "claude-opus-4-7" TokenCountModelOpus48 = "claude-opus-4-8" TokenCountModelSonnet = "claude-sonnet-4-6" + TokenCountModelFable = "claude-fable-5-1" ) // ModelConfig bundles the pricing and token-counting model for a given model alias. @@ -36,8 +43,10 @@ func ResolveModel(model string) (ModelConfig, error) { return ModelConfig{Pricing: PricingOpus, TokenCountModel: TokenCountModelOpus47}, nil case "opus-4-6": return ModelConfig{Pricing: PricingOpus, TokenCountModel: TokenCountModelOpus46}, nil + case "fable", "fable-5-1": + return ModelConfig{Pricing: PricingFable, TokenCountModel: TokenCountModelFable}, nil default: - return ModelConfig{}, fmt.Errorf("unknown model %q: must be opus, opus-4-6, opus-4-7, opus-4-8, or sonnet", model) + return ModelConfig{}, fmt.Errorf("unknown model %q: must be opus, opus-4-6, opus-4-7, opus-4-8, sonnet, fable, or fable-5-1", model) } } diff --git a/internal/benchmark/model_test.go b/internal/benchmark/model_test.go new file mode 100644 index 0000000..1157b88 --- /dev/null +++ b/internal/benchmark/model_test.go @@ -0,0 +1,93 @@ +package benchmark + +import ( + "strings" + "testing" +) + +func Test_ResolveModel_GivenKnownAlias_ThenReturnsExpectedConfig(t *testing.T) { + tests := []struct { + name string + alias string + wantPricing Pricing + wantTokenCount string + }{ + { + name: "sonnet", + alias: "sonnet", + wantPricing: PricingSonnet, + wantTokenCount: TokenCountModelSonnet, + }, + { + name: "opus", + alias: "opus", + wantPricing: PricingOpus, + wantTokenCount: TokenCountModelOpus48, + }, + { + name: "opus-4-8", + alias: "opus-4-8", + wantPricing: PricingOpus, + wantTokenCount: TokenCountModelOpus48, + }, + { + name: "opus-4-7", + alias: "opus-4-7", + wantPricing: PricingOpus, + wantTokenCount: TokenCountModelOpus47, + }, + { + name: "opus-4-6", + alias: "opus-4-6", + wantPricing: PricingOpus, + wantTokenCount: TokenCountModelOpus46, + }, + { + name: "fable", + alias: "fable", + wantPricing: PricingFable, + wantTokenCount: TokenCountModelFable, + }, + { + name: "fable-5-1", + alias: "fable-5-1", + wantPricing: PricingFable, + wantTokenCount: TokenCountModelFable, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ResolveModel(tt.alias) + if err != nil { + t.Fatalf("ResolveModel(%q) returned error: %v", tt.alias, err) + } + if got.Pricing != tt.wantPricing { + t.Errorf("Pricing = %+v, want %+v", got.Pricing, tt.wantPricing) + } + if got.TokenCountModel != tt.wantTokenCount { + t.Errorf("TokenCountModel = %q, want %q", got.TokenCountModel, tt.wantTokenCount) + } + }) + } +} + +// Regression guard: PricingFable's cache read must stay at 2.5% of base input +// ($0.25/MTok), not the 10% ratio every other tier in this file uses. +func Test_PricingFable_ThenCacheReadIsTwoPointFivePercentOfBaseInput(t *testing.T) { + want := PricingFable.BaseInput * 0.025 + if PricingFable.CachedRead != want { + t.Errorf("PricingFable.CachedRead = %v, want %v (2.5%% of BaseInput %v)", + PricingFable.CachedRead, want, PricingFable.BaseInput) + } +} + +func Test_ResolveModel_GivenUnknownAlias_ThenErrorListsFableAliases(t *testing.T) { + _, err := ResolveModel("nonsense") + if err == nil { + t.Fatal("ResolveModel(\"nonsense\") returned nil error, want unknown model error") + } + if !strings.Contains(err.Error(), "fable") { + t.Errorf("error = %v, want it to list the fable aliases", err) + } +}