Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/cc-session/benchmark.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func runBenchmark(args []string, out io.Writer, errOut io.Writer, store parser.S
days := fs.Int("days", 30, "how far back to scan")
minKB := fs.Int("min-kb", 100, "minimum JSONL file size in KB")
maxN := fs.Int("n", 10, "max sessions to include")
model := fs.String("model", "opus", "model: opus, opus-4-6, opus-4-7, opus-4-8, or sonnet")
model := fs.String("model", "opus", "model: opus, opus-4-6, opus-4-7, opus-4-8, sonnet, or fable (fable-5-1)")
overhead := fs.Int("overhead", 0, "session overhead tokens (system+tools+CLAUDE.md); measure with a 1-turn session")
isNoAPI := fs.Bool("no-api", false, "skip API calls; estimate filtered-text tokens with chars/2 (offline fallback)")
if err := fs.Parse(reorderArgs(args)); err != nil {
Expand Down
9 changes: 7 additions & 2 deletions docs/benchmark.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ All other parameters are derived automatically from your real session data.
| `--days` | 30 | How far back to scan for sessions |
| `--min-kb` | 100 | Minimum JSONL file size in KB |
| `--n` | 10 | Max successful session results to report |
| `--model` | opus | Pricing and token-counting model: `opus`, `opus-4-6`, `opus-4-7`, `opus-4-8`, or `sonnet` |
| `--model` | opus | Pricing and token-counting model: `opus`, `opus-4-6`, `opus-4-7`, `opus-4-8`, `sonnet`, or `fable` (`fable-5-1`) |

### Example output

Expand Down Expand Up @@ -164,10 +164,15 @@ the historical one-shot `NewCtx × CacheWrite` behavior.
counting API. The `--model` flag controls both pricing and the tokenizer used by
the token counting API. `opus` is an alias for `opus-4-8`; explicit Opus versions
map to `claude-opus-4-6`, `claude-opus-4-7`, or `claude-opus-4-8`; `sonnet` maps
to `claude-sonnet-4-6`. Opus 4.6, 4.7, and 4.8 use the same Opus pricing rates.
to `claude-sonnet-4-6`; `fable` and `fable-5-1` both map to `claude-fable-5-1`.
Opus 4.6, 4.7, and 4.8 use the same Opus pricing rates.
Fallback constants are used only for behavior that cannot be read directly from
transcript usage, such as sparse tool I/O data.

Claude Fable 5.1's cache read is 2.5% of base input, not the 10% every other
model here uses, so its cost-savings numbers are not directly comparable to the
Opus/Sonnet rows.

### Simplifications

- **Uncached input (`input_tokens`) omitted**: with auto-caching the breakpoint sits on the last cacheable block, making the uncached tail near zero. This is an inference from the docs (not an explicit statement). Impact on results: < 2% based on sensitivity analysis.
Expand Down
11 changes: 10 additions & 1 deletion internal/benchmark/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,18 @@ type Pricing struct {
var PricingOpus = Pricing{CachedRead: 0.50, CacheWrite: 6.25, BaseInput: 5.00}
var PricingSonnet = Pricing{CachedRead: 0.30, CacheWrite: 3.75, BaseInput: 3.00}

// PricingFable is Claude Fable 5.1's pricing. Unlike every other tier here, its
// cache read is 2.5% of base input rather than the usual 10% (source: Anthropic
// pricing docs, "a cache hit costs 2.5% of the standard input price"), so its
// cost-savings numbers are not directly comparable to the Opus/Sonnet rows.
var PricingFable = Pricing{CachedRead: 0.25, CacheWrite: 12.50, BaseInput: 10.00}

const (
TokenCountModelOpus46 = "claude-opus-4-6"
TokenCountModelOpus47 = "claude-opus-4-7"
TokenCountModelOpus48 = "claude-opus-4-8"
TokenCountModelSonnet = "claude-sonnet-4-6"
TokenCountModelFable = "claude-fable-5-1"
)

// ModelConfig bundles the pricing and token-counting model for a given model alias.
Expand All @@ -36,8 +43,10 @@ func ResolveModel(model string) (ModelConfig, error) {
return ModelConfig{Pricing: PricingOpus, TokenCountModel: TokenCountModelOpus47}, nil
case "opus-4-6":
return ModelConfig{Pricing: PricingOpus, TokenCountModel: TokenCountModelOpus46}, nil
case "fable", "fable-5-1":
return ModelConfig{Pricing: PricingFable, TokenCountModel: TokenCountModelFable}, nil
default:
return ModelConfig{}, fmt.Errorf("unknown model %q: must be opus, opus-4-6, opus-4-7, opus-4-8, or sonnet", model)
return ModelConfig{}, fmt.Errorf("unknown model %q: must be opus, opus-4-6, opus-4-7, opus-4-8, sonnet, fable, or fable-5-1", model)
}
}

Expand Down
93 changes: 93 additions & 0 deletions internal/benchmark/model_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package benchmark

import (
"strings"
"testing"
)

func Test_ResolveModel_GivenKnownAlias_ThenReturnsExpectedConfig(t *testing.T) {
tests := []struct {
name string
alias string
wantPricing Pricing
wantTokenCount string
}{
{
name: "sonnet",
alias: "sonnet",
wantPricing: PricingSonnet,
wantTokenCount: TokenCountModelSonnet,
},
{
name: "opus",
alias: "opus",
wantPricing: PricingOpus,
wantTokenCount: TokenCountModelOpus48,
},
{
name: "opus-4-8",
alias: "opus-4-8",
wantPricing: PricingOpus,
wantTokenCount: TokenCountModelOpus48,
},
{
name: "opus-4-7",
alias: "opus-4-7",
wantPricing: PricingOpus,
wantTokenCount: TokenCountModelOpus47,
},
{
name: "opus-4-6",
alias: "opus-4-6",
wantPricing: PricingOpus,
wantTokenCount: TokenCountModelOpus46,
},
{
name: "fable",
alias: "fable",
wantPricing: PricingFable,
wantTokenCount: TokenCountModelFable,
},
{
name: "fable-5-1",
alias: "fable-5-1",
wantPricing: PricingFable,
wantTokenCount: TokenCountModelFable,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := ResolveModel(tt.alias)
if err != nil {
t.Fatalf("ResolveModel(%q) returned error: %v", tt.alias, err)
}
if got.Pricing != tt.wantPricing {
t.Errorf("Pricing = %+v, want %+v", got.Pricing, tt.wantPricing)
}
if got.TokenCountModel != tt.wantTokenCount {
t.Errorf("TokenCountModel = %q, want %q", got.TokenCountModel, tt.wantTokenCount)
}
})
}
}

// Regression guard: PricingFable's cache read must stay at 2.5% of base input
// ($0.25/MTok), not the 10% ratio every other tier in this file uses.
func Test_PricingFable_ThenCacheReadIsTwoPointFivePercentOfBaseInput(t *testing.T) {
want := PricingFable.BaseInput * 0.025
if PricingFable.CachedRead != want {
t.Errorf("PricingFable.CachedRead = %v, want %v (2.5%% of BaseInput %v)",
PricingFable.CachedRead, want, PricingFable.BaseInput)
}
}

func Test_ResolveModel_GivenUnknownAlias_ThenErrorListsFableAliases(t *testing.T) {
_, err := ResolveModel("nonsense")
if err == nil {
t.Fatal("ResolveModel(\"nonsense\") returned nil error, want unknown model error")
}
if !strings.Contains(err.Error(), "fable") {
t.Errorf("error = %v, want it to list the fable aliases", err)
}
}
Loading