diff --git a/apps/cli/package.json b/apps/cli/package.json index eac9c58..772ede1 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -14,32 +14,32 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@b1dz/adapters-cex": "workspace:*", + "@b1dz/adapters-evm": "workspace:*", + "@b1dz/adapters-pumpfun": "workspace:*", + "@b1dz/adapters-solana": "workspace:*", "@b1dz/core": "workspace:*", - "@b1dz/storage-json": "workspace:*", - "@b1dz/storage-supabase": "workspace:*", + "@b1dz/event-channel": "workspace:*", + "@b1dz/observe-engine": "workspace:*", + "@b1dz/profitability": "workspace:*", + "@b1dz/projection-engine": "workspace:*", + "@b1dz/sdk": "workspace:*", "@b1dz/source-crypto-arb": "workspace:*", "@b1dz/source-crypto-trade": "workspace:*", "@b1dz/source-strategies": "workspace:*", "@b1dz/storage-b1dz-api": "workspace:*", - "@b1dz/sdk": "workspace:*", - "@b1dz/venue-types": "workspace:*", - "@b1dz/adapters-evm": "workspace:*", - "@b1dz/adapters-solana": "workspace:*", - "@b1dz/adapters-cex": "workspace:*", - "@b1dz/adapters-pumpfun": "workspace:*", - "@b1dz/projection-engine": "workspace:*", - "@b1dz/profitability": "workspace:*", - "@b1dz/event-channel": "workspace:*", - "@b1dz/observe-engine": "workspace:*", + "@b1dz/storage-json": "workspace:*", + "@b1dz/storage-supabase": "workspace:*", "@b1dz/trade-daemon": "workspace:*", - "@supabase/supabase-js": "latest", - "chalk": "latest", - "cli-table3": "latest", + "@b1dz/venue-types": "workspace:*", + "@supabase/supabase-js": "^2.112.0", "blessed": "^0.1.81", - "react-blessed": "^0.7.2", "blessed-contrib": "^4.11.0", + "chalk": "latest", + "cli-table3": "latest", + "dotenv": "latest", "react": "^17.0.2", - "dotenv": "latest" + "react-blessed": "^0.7.2" }, "devDependencies": { "@types/node": "latest", diff --git a/apps/cli/src/strategy-backtest.ts b/apps/cli/src/strategy-backtest.ts index aa64d75..e44219c 100644 --- a/apps/cli/src/strategy-backtest.ts +++ b/apps/cli/src/strategy-backtest.ts @@ -10,10 +10,19 @@ * class a strategy suits. Run both (default, with a head-to-head verdict), * or restrict to one with --crypto / --equities. * + * Every number here is NET of a real cost model — fees on both legs, the assumed + * spread (Yahoo daily closes have none of their own), and slippage. Each horizon + * is therefore replayed twice, once priced and once with ZERO_COST_MODEL, so the + * `Gross` column shows exactly what the friction took. Venue matters more than + * most people expect: the same strategy can print +12% on Binance.US and −4% on + * Coinbase, which is why `--costs ` exists. + * * b1dz strategy-backtest mean-reversion # both classes, compared * b1dz strategy-backtest all --crypto # every built-in, crypto only * b1dz strategy-backtest --equities --file my.tsp.json * b1dz strategy-backtest trend-continuation --amount 250 + * b1dz strategy-backtest breakout --costs kraken + * b1dz strategy-backtest breakout --fee-bps 10 --slippage-bps 2 --spread-bps 1 */ import { readFileSync } from 'node:fs'; import chalk from 'chalk'; @@ -24,7 +33,14 @@ import { replayStrategy, summarizeTrades, tsp, + costModelFor, + describeCostModel, + DEFAULT_COST_MODEL, + DEX_COST_MODEL, + EQUITY_COST_MODEL, + ZERO_COST_MODEL, type BacktestSummary, + type CostModel, } from '@b1dz/source-strategies'; const CRYPTO_BASKET = ['BTC-USD', 'ETH-USD', 'SOL-USD']; @@ -43,13 +59,42 @@ const HORIZONS = [ type AssetClass = 'crypto' | 'equity'; +/** + * Named cost models, keyed by how a user thinks about the decision ("what does + * this look like on Kraken?"). `zero` is the old frictionless behaviour and is + * kept only for diffing against a priced run — never for judging a strategy. + */ +const COST_PRESETS = { + zero: ZERO_COST_MODEL, + kraken: costModelFor({ assetClass: 'crypto', exchange: 'kraken' }), + coinbase: costModelFor({ assetClass: 'crypto', exchange: 'coinbase' }), + gemini: costModelFor({ assetClass: 'crypto', exchange: 'gemini' }), + 'binance-us': costModelFor({ assetClass: 'crypto', exchange: 'binance-us' }), + equity: EQUITY_COST_MODEL, + dex: DEX_COST_MODEL, +} satisfies Record; + +export type CostPreset = keyof typeof COST_PRESETS; +export const COST_PRESETS_NAMES = Object.keys(COST_PRESETS) as CostPreset[]; + +interface HorizonResult { + label: string; + startYmd: string; + endYmd: string; + /** Priced under the resolved cost model. */ + summary: BacktestSummary; + /** Identical signals replayed with ZERO_COST_MODEL — the frictionless twin. */ + gross: BacktestSummary; +} + interface ClassResult { assetClass: AssetClass; basket: string[]; symbolsWithData: string[]; - horizons: { label: string; startYmd: string; endYmd: string; summary: BacktestSummary }[]; + costs: CostModel; + horizons: HorizonResult[]; /** Longest available horizon's summary — the headline used for the verdict. */ - headline: { label: string; summary: BacktestSummary } | null; + headline: HorizonResult | null; } // ── args ───────────────────────────────────────────────────────────────────── @@ -58,6 +103,33 @@ interface Args { file: string | null; classes: AssetClass[]; amount: number; + /** null → per-asset-class defaults rather than one model for everything. */ + costPreset: CostPreset | null; + /** Field-level overrides layered on top of the preset/default. */ + costOverrides: Partial; +} + +/** A non-negative numeric flag, or undefined when absent. Throws on garbage. */ +function bpsFlag(flags: Record, key: string): number | undefined { + const raw = flags[key]; + if (raw === undefined) return undefined; + const n = Number.parseFloat(raw); + if (!Number.isFinite(n) || n < 0) throw new Error(`invalid --${key} "${raw}" — expected a non-negative number`); + return n; +} + +/** + * Resolve the model a class is scored under. A preset applies to every class + * (you asked for Kraken, you get Kraken); without one, each class gets its own + * realistic default, since equities are commission-free and crypto is not. + */ +export function resolveCostModel(args: Args, assetClass: AssetClass): CostModel { + const base = args.costPreset + ? COST_PRESETS[args.costPreset] + : assetClass === 'crypto' + ? DEFAULT_COST_MODEL + : EQUITY_COST_MODEL; + return { ...base, ...args.costOverrides }; } export function parseArgs(argv: string[]): Args { @@ -83,11 +155,33 @@ export function parseArgs(argv: string[]): Args { const classes: AssetClass[] = wantCrypto && !wantEquities ? ['crypto'] : wantEquities && !wantCrypto ? ['equity'] : ['crypto', 'equity']; const amount = Math.max(1, Number.parseFloat(flags.amount ?? '100')); + + let costPreset: CostPreset | null = null; + if (flags.costs !== undefined) { + const wanted = flags.costs.toLowerCase(); + if (!(COST_PRESETS_NAMES as string[]).includes(wanted)) { + throw new Error(`invalid --costs "${flags.costs}" — expected one of ${COST_PRESETS_NAMES.join(', ')}`); + } + costPreset = wanted as CostPreset; + } + + // --spread-bps is the assumed HALF-spread: entry pays +half, exit pays −half, + // so a round trip costs the full spread. Same convention as CostModel. + const costOverrides: Partial = {}; + const feeBps = bpsFlag(flags, 'fee-bps'); + if (feeBps !== undefined) costOverrides.feeBps = feeBps; + const slippageBps = bpsFlag(flags, 'slippage-bps'); + if (slippageBps !== undefined) costOverrides.slippageBps = slippageBps; + const spreadBps = bpsFlag(flags, 'spread-bps'); + if (spreadBps !== undefined) costOverrides.assumedHalfSpreadBps = spreadBps; + return { selector: positional[0] ?? (flags.strategy ?? null), file: flags.file ?? null, classes, amount, + costPreset, + costOverrides, }; } @@ -127,7 +221,12 @@ async function fetchDailySnapshots(symbol: string, startMs: number, endMs: numbe .map((b) => ({ exchange: 'yahoo', pair: symbol, bid: b.c, ask: b.c, bidSize: 1, askSize: 1, ts: b.t, assetClass })); } -async function backtestClass(plugin: StrategyPlugin, assetClass: AssetClass, amount: number): Promise { +async function backtestClass( + plugin: StrategyPlugin, + assetClass: AssetClass, + amount: number, + costs: CostModel, +): Promise { const basket = assetClass === 'crypto' ? CRYPTO_BASKET : EQUITY_BASKET; const now = new Date(); const endMs = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()); @@ -142,22 +241,29 @@ async function backtestClass(plugin: StrategyPlugin, assetClass: AssetClass, amo } } - const horizons = HORIZONS.map((h) => { + const horizons: HorizonResult[] = HORIZONS.map((h) => { const hStart = subtract(now, h).getTime(); - const trades = [...series.values()].flatMap((snaps) => { - const window = snaps.filter((s) => s.ts >= hStart); - return window.length < MIN_BARS ? [] : replayStrategy(plugin, window, amount); - }); - return { label: h.label, startYmd: ymd(new Date(hStart)), endYmd: ymd(new Date(endMs)), summary: summarizeTrades(trades) }; + const windows = [...series.values()] + .map((snaps) => snaps.filter((s) => s.ts >= hStart)) + .filter((w) => w.length >= MIN_BARS); + const priced = windows.flatMap((w) => replayStrategy(plugin, w, { amountPerEntry: amount, costs })); + const free = windows.flatMap((w) => replayStrategy(plugin, w, { amountPerEntry: amount, costs: ZERO_COST_MODEL })); + return { + label: h.label, + startYmd: ymd(new Date(hStart)), + endYmd: ymd(new Date(endMs)), + summary: summarizeTrades(priced), + gross: summarizeTrades(free), + }; }); - const headline = [...horizons].reverse().find((h) => h.summary.trades > 0) ?? null; return { assetClass, basket, symbolsWithData: [...series.keys()], + costs, horizons, - headline: headline ? { label: headline.label, summary: headline.summary } : null, + headline: [...horizons].reverse().find((h) => h.summary.trades > 0) ?? null, }; } @@ -169,14 +275,19 @@ function fmtPct(n: number): string { function fmtUsd(n: number): string { return `${n >= 0 ? '+' : '-'}$${Math.abs(n).toFixed(2)}`; } +/** A cost is never a gain, so it gets no sign — just a magnitude. */ +function fmtCost(n: number): string { + return `$${n.toFixed(2)}`; +} -function renderClass(r: ClassResult): void { +function renderClass(r: ClassResult, amount: number): void { const label = r.assetClass === 'crypto' ? 'CRYPTO' : 'EQUITIES'; console.log(`\n${chalk.bold.cyan(label)} ${chalk.dim(r.symbolsWithData.join(', ') || '(no data)')}`); + console.log(chalk.dim(` costs: ${describeCostModel(r.costs, amount)}`)); const table = new Table({ - head: ['Horizon', 'Trades', 'Win%', 'Return', 'Profit', 'MaxDD'].map((h) => chalk.dim(h)), + head: ['Horizon', 'Trades', 'Win%', 'Return', 'Gross', 'Fees', 'Profit', 'MaxDD'].map((h) => chalk.dim(h)), style: { head: [], border: [] }, - colAligns: ['left', 'right', 'right', 'right', 'right', 'right'], + colAligns: ['left', 'right', 'right', 'right', 'right', 'right', 'right', 'right'], }); for (const h of r.horizons) { const s = h.summary; @@ -185,11 +296,24 @@ function renderClass(r: ClassResult): void { String(s.trades), `${Math.round(s.winRate * 100)}%`, fmtPct(s.returnPct), + chalk.dim(fmtPct(h.gross.returnPct)), + chalk.dim(fmtCost(s.feesUsd)), fmtUsd(s.profit), `$${s.maxDrawdown.toFixed(0)}`, ]); } console.log(table.toString()); + + const hl = r.headline; + if (hl && hl.summary.trades > 0) { + console.log( + chalk.dim( + ` drag over ${hl.label}: ${fmtCost(hl.summary.feesUsd)} fees + ` + + `${fmtCost(hl.summary.spreadSlippageUsd)} spread/slippage ` + + `(${(hl.summary.costDragPct * 100).toFixed(2)}% of capital deployed)`, + ), + ); + } } function renderVerdict(results: ClassResult[]): void { @@ -199,14 +323,14 @@ function renderVerdict(results: ClassResult[]): void { const winner = a!.assetClass === 'crypto' ? 'crypto' : 'equities'; const ra = a!.headline!; const rb = b!.headline!; - console.log(`\n${chalk.bold('Head-to-head')} (best common window):`); + console.log(`\n${chalk.bold('Head-to-head')} (best common window, net of costs):`); console.log( ` ${chalk.cyan(a!.assetClass.padEnd(7))} ${fmtPct(ra.summary.returnPct)} over ${ra.label} ` + - `(${Math.round(ra.summary.winRate * 100)}% win, ${ra.summary.trades} trades)`, + `(${Math.round(ra.summary.winRate * 100)}% win, ${ra.summary.trades} trades, ${chalk.dim(`gross ${fmtPct(ra.gross.returnPct)}`)})`, ); console.log( ` ${chalk.cyan(b!.assetClass.padEnd(7))} ${fmtPct(rb.summary.returnPct)} over ${rb.label} ` + - `(${Math.round(rb.summary.winRate * 100)}% win, ${rb.summary.trades} trades)`, + `(${Math.round(rb.summary.winRate * 100)}% win, ${rb.summary.trades} trades, ${chalk.dim(`gross ${fmtPct(rb.gross.returnPct)}`)})`, ); console.log(` ${chalk.bold(`→ Better fit for ${chalk.green(winner)}`)}`); } @@ -242,19 +366,20 @@ export async function runStrategyBacktestCli(argv: string[]): Promise { return; } - console.log( - chalk.dim( - `Long-only signal replay · $${args.amount}/entry · Yahoo daily · classes: ${args.classes.join(' + ')} · ignores fees/slippage`, - ), - ); + if (args.costPreset) { + console.log(chalk.dim(`Long-only signal replay · $${args.amount}/entry · Yahoo daily · classes: ${args.classes.join(' + ')} · costs: ${args.costPreset}`)); + } else { + console.log(chalk.dim(`Long-only signal replay · $${args.amount}/entry · Yahoo daily · classes: ${args.classes.join(' + ')} · costs: per-class defaults`)); + } for (const plugin of plugins) { console.log(`\n${chalk.bold('▶ ' + (plugin.manifest.name ?? plugin.manifest.id))} ${chalk.dim('(' + plugin.manifest.id + ')')}`); const results: ClassResult[] = []; for (const assetClass of args.classes) { - const r = await backtestClass(plugin, assetClass, args.amount); + const costs = resolveCostModel(args, assetClass); + const r = await backtestClass(plugin, assetClass, args.amount, costs); results.push(r); - renderClass(r); + renderClass(r, args.amount); } if (args.classes.length > 1) renderVerdict(results); } diff --git a/apps/daemon/package.json b/apps/daemon/package.json index 977b799..595ca56 100644 --- a/apps/daemon/package.json +++ b/apps/daemon/package.json @@ -16,8 +16,8 @@ }, "dependencies": { "@b1dz/adapters-cex": "workspace:*", - "@b1dz/adapters-pumpfun": "workspace:*", "@b1dz/adapters-evm": "workspace:*", + "@b1dz/adapters-pumpfun": "workspace:*", "@b1dz/adapters-solana": "workspace:*", "@b1dz/ai-analyzer": "workspace:*", "@b1dz/core": "workspace:*", @@ -34,13 +34,15 @@ "@b1dz/source-tradestation": "workspace:*", "@b1dz/source-tradier": "workspace:*", "@b1dz/storage-supabase": "workspace:*", + "@b1dz/strategy-registry": "workspace:*", + "@b1dz/strategy-validation": "workspace:*", "@b1dz/trade-daemon": "workspace:*", "@b1dz/triangular-engine": "workspace:*", "@b1dz/venue-types": "workspace:*", "@b1dz/wallet-direct": "workspace:*", "@b1dz/wallet-provider": "workspace:*", "@b1dz/wallet-service": "workspace:*", - "@supabase/supabase-js": "latest", + "@supabase/supabase-js": "^2.112.0", "tsx": "latest" }, "devDependencies": { diff --git a/apps/daemon/src/registry.ts b/apps/daemon/src/registry.ts index 52da24f..069af67 100644 --- a/apps/daemon/src/registry.ts +++ b/apps/daemon/src/registry.ts @@ -14,6 +14,7 @@ import { cryptoDcaWorker } from './sources/crypto-dca.js'; import { v2PipelineWorker } from './sources/v2-pipeline.js'; import { pumpfunTradeWorker } from './sources/pumpfun-trade.js'; import { equitiesWorker } from './sources/equities.js'; +import { forwardTestWorker } from './sources/forward-test.js'; export const SOURCES: SourceWorker[] = [ cryptoArbWorker, @@ -22,4 +23,5 @@ export const SOURCES: SourceWorker[] = [ v2PipelineWorker, pumpfunTradeWorker, equitiesWorker, + forwardTestWorker, ]; diff --git a/apps/daemon/src/sources/forward-test.ts b/apps/daemon/src/sources/forward-test.ts new file mode 100644 index 0000000..9a366a6 --- /dev/null +++ b/apps/daemon/src/sources/forward-test.ts @@ -0,0 +1,108 @@ +import type { SourceWorker, UserContext } from '../types.js'; +import { replayStrategy, tsp, type CostModel, type BacktestTrade } from '@b1dz/source-strategies'; +import { listForwardRunning, setStatus, insertForwardTrade, closeForwardTrade, forwardTradeHistory } from '@b1dz/strategy-registry'; +import { computeMetrics, minimumTrackRecordLength } from '@b1dz/strategy-validation'; +import type { MarketSnapshot } from '@b1dz/core'; + +const CRYPTO_BASKET = ['BTC-USD', 'ETH-USD', 'SOL-USD']; +const EQUITY_BASKET = ['SPY', 'AAPL', 'NVDA']; +const DAY_MS = 24 * 60 * 60 * 1000; + +async function fetchYahooBars(symbol: string): Promise { + const endMs = Date.now(); + const startMs = endMs - 150 * DAY_MS; + const period1 = Math.floor(startMs / 1000); + const period2 = Math.floor(endMs / 1000); + const url = `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(symbol)}?period1=${period1}&period2=${period2}&interval=1d&events=history`; + + const res = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0', Accept: 'application/json' } }); + if (!res.ok) return []; + const json = await res.json() as { chart?: { result?: { timestamp?: number[]; indicators?: { quote?: { close?: (number | null)[] }[] } }[] } }; + const result = json.chart?.result?.[0]; + if (!result?.timestamp) return []; + const close = result.indicators?.quote?.[0]?.close ?? []; + + return result.timestamp + .map((t: number, i: number) => ({ t: t * 1000, c: close[i] })) + .filter((b: { t: number; c: number | null }): b is { t: number; c: number } => Number.isFinite(b.c)) + .sort((a: { t: number }, b: { t: number }) => a.t - b.t) + .map((b: { t: number; c: number }) => ({ + exchange: 'yahoo', + pair: symbol, + bid: b.c, + ask: b.c, + bidSize: 1, + askSize: 1, + ts: b.t, + assetClass: symbol.includes('-USD') ? 'crypto' as const : 'equity' as const, + })); +} + +export const forwardTestWorker: SourceWorker = { + id: 'forward-test', + pollIntervalMs: 60_000, + + hasCredentials(_payload: Record) { + return true; + }, + + async tick(ctx: UserContext) { + const strategies = await listForwardRunning(ctx.supabase); + + for (const s of strategies) { + if (s.status === 'gauntlet_passed') { + await setStatus(ctx.supabase, s.id, 'forward_running'); + } + + const plugin = tsp.compile(s.tsp_doc); + const costModel = s.cost_model as CostModel; + + const assetClasses = s.tsp_doc.assetClasses?.length ? s.tsp_doc.assetClasses : ['crypto', 'equity']; + + for (const ac of assetClasses) { + const basket = ac === 'crypto' ? CRYPTO_BASKET : EQUITY_BASKET; + + for (const symbol of basket) { + const snaps = await fetchYahooBars(symbol); + if (!snaps.length) continue; + + const trades = replayStrategy(plugin, snaps, { amountPerEntry: 100, costs: costModel }); + + const existing = await forwardTradeHistory(ctx.supabase, s.id); + const existingEntries = new Set(existing.map((t) => t.entry_ts)); + + for (const trade of trades) { + const entryTsStr = new Date(trade.entryTs).toISOString(); + if (!existingEntries.has(entryTsStr)) { + await insertForwardTrade(ctx.supabase, s.id, s.user_id, entryTsStr, trade as unknown as Record); + } + } + + const openTrades = existing.filter((t) => !t.exit_ts); + const latestBar = snaps[snaps.length - 1]!; + for (const ot of openTrades) { + const otTrade = ot.trade_json as Record; + if (otTrade.exitTs && otTrade.exitTs <= latestBar.ts) { + await closeForwardTrade(ctx.supabase, ot.id, new Date(otTrade.exitTs).toISOString(), ot.trade_json); + } + } + } + } + + const allTrades = await forwardTradeHistory(ctx.supabase, s.id); + const closedTrades = allTrades.filter((t) => t.exit_ts); + if (closedTrades.length >= 30) { + const metrics = computeMetrics(closedTrades.map((t) => t.trade_json as unknown as BacktestTrade)); + const minTrl = minimumTrackRecordLength({ + observedSharpe: metrics.sharpePerTrade, + benchmarkSharpe: s.gauntlet_report?.deflatedSharpe?.expectedMaxSharpe ?? 0, + }); + if (Number.isFinite(minTrl) && closedTrades.length >= minTrl) { + await setStatus(ctx.supabase, s.id, 'min_trl_reached'); + } + } + } + + await ctx.savePayload({ lastTickAt: new Date().toISOString(), strategyCount: strategies.length }); + }, +}; diff --git a/apps/web/package.json b/apps/web/package.json index 7539e88..1af1c56 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -23,7 +23,7 @@ "@profullstack/pluginstore": "^0.1.1", "@profullstack/stack": "^0.1.3", "@supabase/ssr": "latest", - "@supabase/supabase-js": "latest", + "@supabase/supabase-js": "^2.112.0", "lightweight-charts": "^5.2.0", "next": "latest", "react": "latest", diff --git a/apps/web/src/app/api/strategies/backtest/route.test.ts b/apps/web/src/app/api/strategies/backtest/route.test.ts index 1273c85..8198171 100644 --- a/apps/web/src/app/api/strategies/backtest/route.test.ts +++ b/apps/web/src/app/api/strategies/backtest/route.test.ts @@ -38,13 +38,36 @@ function makeReq(body: unknown) { const validDoc = { tsp: '0.1', id: 'x', name: 'X', definition: { kind: 'template', template: 'breakout' } }; +const cryptoCosts = { feeBps: 60, slippageBps: 5, assumedHalfSpreadBps: 5, perOrderUsd: 0, roundTripBps: 140 }; +const cryptoClass = { + assetClass: 'crypto', + basket: ['BTC-USD'], + symbols: ['BTC-USD'], + trades: 4, + returnPct: 0.08, + grossReturnPct: 0.14, + winRate: 0.5, + profit: 80, + maxDrawdown: 30, + bankroll: 1000, + finalEquity: 1080, + feesUsd: 48, + spreadSlippageUsd: 12, + totalCostUsd: 60, + costDragPct: 0.06, + costs: cryptoCosts, +}; + describe('POST /api/strategies/backtest', () => { beforeEach(() => { vi.clearAllMocks(); authenticateMock.mockResolvedValue({ userId: 'u1', client: {}, email: 'a@b.c' }); validateMock.mockReturnValue({ ok: true, errors: [] }); compileMock.mockReturnValue({ manifest: { id: 'x', name: 'X' } }); - runBacktestMock.mockResolvedValue({ bankroll: 1000, timeframe: '1 year', startYmd: '2025-06-30', endYmd: '2026-06-30', classes: [], verdict: null }); + runBacktestMock.mockResolvedValue({ + bankroll: 1000, timeframe: '1 year', startYmd: '2025-06-30', endYmd: '2026-06-30', + classes: [cryptoClass], verdict: null, + }); }); it('401 when unauthenticated', async () => { @@ -107,4 +130,87 @@ describe('POST /api/strategies/backtest', () => { expect(res.status).toBe(400); expect(runBacktestMock).not.toHaveBeenCalled(); }); + + it('returns the resolved cost assumptions and the net-vs-gross pair per class', async () => { + const { POST } = await importRoute(); + const res = await POST(makeReq({ definition: validDoc }) as never); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.costsOverridden).toBe(false); + const cls = body.classes[0]; + expect(cls.costs).toEqual(cryptoCosts); + expect(cls.grossReturnPct).toBeGreaterThan(cls.returnPct); + expect(cls.feesUsd + cls.spreadSlippageUsd).toBeCloseTo(cls.totalCostUsd); + expect(cls.costDragPct).toBeCloseTo(cls.grossReturnPct - cls.returnPct, 10); + }); + + it('leaves costs undefined when the body omits them (per-class defaults apply)', async () => { + const { POST } = await importRoute(); + await POST(makeReq({ definition: validDoc }) as never); + const [, opts] = runBacktestMock.mock.calls[0]!; + expect(opts.costs).toBeUndefined(); + }); + + it('passes a valid cost override through, defaulting omitted fields to zero', async () => { + const { POST } = await importRoute(); + const res = await POST(makeReq({ definition: validDoc, costs: { feeBps: 26, slippageBps: 3 } }) as never); + expect(res.status).toBe(200); + const [, opts] = runBacktestMock.mock.calls[0]!; + expect(opts.costs).toEqual({ feeBps: 26, slippageBps: 3, assumedHalfSpreadBps: 0, perOrderUsd: 0 }); + expect((await res.json()).costsOverridden).toBe(true); + }); + + it('accepts an explicit all-zero override (the frictionless comparison run)', async () => { + const { POST } = await importRoute(); + const res = await POST( + makeReq({ definition: validDoc, costs: { feeBps: 0, slippageBps: 0, assumedHalfSpreadBps: 0, perOrderUsd: 0 } }) as never, + ); + expect(res.status).toBe(200); + const [, opts] = runBacktestMock.mock.calls[0]!; + expect(opts.costs).toEqual({ feeBps: 0, slippageBps: 0, assumedHalfSpreadBps: 0, perOrderUsd: 0 }); + }); + + it.each([ + ['above the bps ceiling', { feeBps: 501 }], + ['negative', { slippageBps: -1 }], + ['a non-number', { assumedHalfSpreadBps: '5' }], + ['not finite', { feeBps: Number.POSITIVE_INFINITY }], + ['above the per-order ceiling', { perOrderUsd: 100.5 }], + ['an unknown field', { gasBps: 5 }], + ])('400 when the cost override is %s', async (_label, costs) => { + const { POST } = await importRoute(); + const res = await POST(makeReq({ definition: validDoc, costs }) as never); + expect(res.status).toBe(400); + const body = await res.json(); + expect(body.error).toMatch(/cost override/); + expect(body.details.length).toBeGreaterThan(0); + expect(runBacktestMock).not.toHaveBeenCalled(); + }); + + it.each([['an array', []], ['a string', 'cheap'], ['a number', 5]])( + '400 when costs is %s rather than an object', + async (_label, costs) => { + const { POST } = await importRoute(); + const res = await POST(makeReq({ definition: validDoc, costs }) as never); + expect(res.status).toBe(400); + expect((await res.json()).details).toEqual(['costs must be an object']); + }, + ); + + it('treats an explicit null costs as "use the defaults"', async () => { + const { POST } = await importRoute(); + const res = await POST(makeReq({ definition: validDoc, costs: null }) as never); + expect(res.status).toBe(200); + const [, opts] = runBacktestMock.mock.calls[0]!; + expect(opts.costs).toBeUndefined(); + }); + + it('names every offending field rather than stopping at the first', async () => { + const { POST } = await importRoute(); + const res = await POST(makeReq({ definition: validDoc, costs: { feeBps: 900, slippageBps: -2 } }) as never); + expect(res.status).toBe(400); + const details = (await res.json()).details as string[]; + expect(details.some((d) => d.includes('feeBps'))).toBe(true); + expect(details.some((d) => d.includes('slippageBps'))).toBe(true); + }); }); diff --git a/apps/web/src/app/api/strategies/backtest/route.ts b/apps/web/src/app/api/strategies/backtest/route.ts index 0c43010..1661850 100644 --- a/apps/web/src/app/api/strategies/backtest/route.ts +++ b/apps/web/src/app/api/strategies/backtest/route.ts @@ -8,7 +8,14 @@ * * Read-only; never trades. Auth required. * - * Body: { definition, classes?, bankroll?, timeframe? } + * Body: { definition, classes?, bankroll?, timeframe?, costs? } + * + * `costs` overrides the per-asset-class friction defaults. It is clamped rather + * than trusted: a caller who can post `feeBps: 0` can manufacture a strategy + * that looks profitable and publish it to the store, so the bounds below are a + * product constraint, not input hygiene. Nonsense (non-numeric, out of range) + * is a 400 rather than a silent clamp — quietly "fixing" a cost assumption is + * how a user ends up reading numbers they never asked for. * * Price data: * - Crypto → Kraken daily OHLC via @b1dz/source-crypto-trade's @@ -39,11 +46,71 @@ const DAY_MS = 24 * 60 * 60 * 1000; const VALID_CLASSES: AssetClass[] = ['crypto', 'equity']; const TF_LABELS = TIMEFRAMES.map((t) => t.label) as TimeframeLabel[]; +/** Accepted range per cost field. 500 bps = 5% per leg — past any real venue. */ +const COST_BOUNDS = { + feeBps: [0, 500], + slippageBps: [0, 500], + assumedHalfSpreadBps: [0, 500], + perOrderUsd: [0, 100], +} as const satisfies Record; + +type CostField = keyof typeof COST_BOUNDS; +const COST_FIELDS = Object.keys(COST_BOUNDS) as CostField[]; + +interface CostOverride { + feeBps: number; + slippageBps: number; + assumedHalfSpreadBps: number; + perOrderUsd: number; +} + interface BacktestBody { definition?: unknown; classes?: string[]; bankroll?: number; timeframe?: string; + costs?: unknown; +} + +/** + * Validate a `costs` override. Absent → undefined (use the class defaults). + * Present but malformed → a list of errors, so the caller learns which field. + * Omitted fields default to 0, which is only reachable deliberately. + */ +function parseCostOverride(raw: unknown): { costs?: CostOverride; errors: string[] } { + if (raw === undefined || raw === null) return { errors: [] }; + if (typeof raw !== 'object' || Array.isArray(raw)) { + return { errors: ['costs must be an object'] }; + } + + const src = raw as Record; + const errors: string[] = []; + const parsed: Record = { + feeBps: 0, + slippageBps: 0, + assumedHalfSpreadBps: 0, + perOrderUsd: 0, + }; + + for (const key of Object.keys(src)) { + if (!(COST_FIELDS as string[]).includes(key)) errors.push(`costs.${key} is not a recognized cost field`); + } + for (const field of COST_FIELDS) { + const value = src[field]; + if (value === undefined) continue; + const [min, max] = COST_BOUNDS[field]; + if (typeof value !== 'number' || !Number.isFinite(value)) { + errors.push(`costs.${field} must be a finite number`); + continue; + } + if (value < min || value > max) { + errors.push(`costs.${field} must be between ${min} and ${max}`); + continue; + } + parsed[field] = value; + } + + return errors.length ? { errors } : { costs: parsed, errors: [] }; } /** Yahoo Finance free chart API — daily closes. Often blocked on datacenters. */ @@ -157,7 +224,18 @@ export async function POST(req: NextRequest) { ? (body.timeframe as TimeframeLabel) : DEFAULT_TIMEFRAME; - const result = await runStrategyBacktest(plugin, { classes, bankroll, timeframe, fetchCloses }); + const { costs, errors: costErrors } = parseCostOverride(body.costs); + if (costErrors.length) { + return Response.json({ error: 'invalid cost override', details: costErrors }, { status: 400 }); + } + + const result = await runStrategyBacktest(plugin, { classes, bankroll, timeframe, fetchCloses, costs }); - return Response.json({ strategy: { id: plugin.manifest.id, name: plugin.manifest.name }, ...result }); + return Response.json({ + strategy: { id: plugin.manifest.id, name: plugin.manifest.name }, + ...result, + // Resolved assumptions also live on each class (they differ by asset class); + // this echoes whether the caller forced one model across the board. + costsOverridden: costs !== undefined, + }); } diff --git a/apps/web/src/app/store/build/builder-client.tsx b/apps/web/src/app/store/build/builder-client.tsx index 2bd3bf4..77437f8 100644 --- a/apps/web/src/app/store/build/builder-client.tsx +++ b/apps/web/src/app/store/build/builder-client.tsx @@ -15,7 +15,7 @@ import { type TemplateName, } from '@/lib/tsp-builder'; import { fmtReturnPct, fmtWinRate } from '@/lib/strategy-backtest-display'; -import type { BacktestResponse } from '@/lib/strategy-backtest-runner'; +import type { BacktestResponse, CostAssumptions } from '@/lib/strategy-backtest-runner'; const INDICATOR_FNS: IndicatorFn[] = ['rsi', 'ema', 'sma', 'macdHist']; const COMPARATORS = ['gt', 'gte', 'lt', 'lte', 'eq', 'neq'] as const; @@ -420,6 +420,10 @@ function money(n: number): string { return `${n < 0 ? '-' : ''}$${Math.abs(n).toLocaleString(undefined, { maximumFractionDigits: 0 })}`; } +function describeCosts(assumptions: CostAssumptions): string { + return `${assumptions.feeBps} bps/leg fee + ${assumptions.slippageBps} bps slippage + ${assumptions.assumedHalfSpreadBps} bps spread (${assumptions.roundTripBps} bps round trip)`; +} + function Results({ result }: { result: BacktestResponse & { strategy?: { name: string } } }) { return (
@@ -445,14 +449,36 @@ function Results({ result }: { result: BacktestResponse & { strategy?: { name: s {noData ? (
Couldn't load price data for this basket — try again in a moment.
) : cls.trades === 0 ? ( -
No trades fired in this time frame.
+ <> +
No trades fired in this time frame.
+
{describeCosts(cls.costs)}
+ ) : ( -
- = 0 ? 'text-emerald-400' : 'text-rose-400'} /> - - - -
+ <> +
+ = 0 ? 'text-emerald-400' : 'text-rose-400'} /> + + + +
+
+
+ Return + + = 0 ? 'text-emerald-400' : 'text-rose-400'}>{fmtReturnPct(cls.returnPct)} + {' '} + (gross {fmtReturnPct(cls.grossReturnPct)}) + +
+
{describeCosts(cls.costs)}
+
+ Fees: {money(cls.feesUsd)} · Spread: {money(cls.spreadSlippageUsd)} · Total: {money(cls.totalCostUsd)} ({(cls.costDragPct * 100).toFixed(2)}% of bankroll) +
+
+

+ Returns are net of modelled costs. Backtests are not forward performance. +

+ )}
); diff --git a/apps/web/src/lib/strategy-backtest-runner.test.ts b/apps/web/src/lib/strategy-backtest-runner.test.ts index 4482436..bf50431 100644 --- a/apps/web/src/lib/strategy-backtest-runner.test.ts +++ b/apps/web/src/lib/strategy-backtest-runner.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'vitest'; -import { tsp } from '@b1dz/source-strategies'; -import { runStrategyBacktest, type FetchCloses } from './strategy-backtest-runner'; +import { tsp, ZERO_COST_MODEL, DEFAULT_COST_MODEL, EQUITY_COST_MODEL, type CostModel } from '@b1dz/source-strategies'; +import { runStrategyBacktest, DEFAULT_CLASS_COSTS, type FetchCloses } from './strategy-backtest-runner'; const DAY = 24 * 60 * 60 * 1000; @@ -13,6 +13,11 @@ function dipAndRip(startMs: number): { ts: number; close: number }[] { return prices.map((close, i) => ({ ts: startMs + i * DAY, close })); } +/** 40 bars: flat at 100, then flat at 101 — a single +1% round trip. */ +function tinyEdge(startMs: number): { ts: number; close: number }[] { + return Array.from({ length: 40 }, (_, i) => ({ ts: startMs + i * DAY, close: i < 20 ? 100 : 101 })); +} + const rsiDip = { tsp: '0.1', id: 'rsi-dip', @@ -27,6 +32,34 @@ const rsiDip = { }, }; +/** Buy the dip below 50, sell the rip above 90 → exactly one big winner. */ +const buyLowSellHigh = { + tsp: '0.1', + id: 'bl', + name: 'BL', + definition: { + kind: 'rules', + rules: [ + { when: { lt: ['price', 50] }, signal: { side: 'buy' } }, + { when: { gt: ['price', 90] }, signal: { side: 'sell' } }, + ], + }, +}; + +/** Buy at 100, sell at 101 — a +1% gross edge that any real fee eats. */ +const scalp = { + tsp: '0.1', + id: 'scalp', + name: 'Scalp', + definition: { + kind: 'rules', + rules: [ + { when: { lt: ['price', 100.5] }, signal: { side: 'buy' } }, + { when: { gt: ['price', 100.5] }, signal: { side: 'sell' } }, + ], + }, +}; + describe('runStrategyBacktest (bankroll + timeframe)', () => { const fetchCloses: FetchCloses = async (_symbol, startMs) => dipAndRip(startMs); @@ -48,15 +81,8 @@ describe('runStrategyBacktest (bankroll + timeframe)', () => { }); it('compounds the bankroll (a winning round-trip ends above starting capital)', async () => { - // Deterministic: buy the dip below 50, sell the rip above 90 → one big winner. - const buyLowSellHigh = tsp.compile({ - tsp: '0.1', id: 'bl', name: 'BL', - definition: { kind: 'rules', rules: [ - { when: { lt: ['price', 50] }, signal: { side: 'buy' } }, - { when: { gt: ['price', 90] }, signal: { side: 'sell' } }, - ] }, - }); - const res = await runStrategyBacktest(buyLowSellHigh, { classes: ['crypto'], bankroll: 1000, timeframe: '1 year', fetchCloses }); + const plugin = tsp.compile(buyLowSellHigh); + const res = await runStrategyBacktest(plugin, { classes: ['crypto'], bankroll: 1000, timeframe: '1 year', fetchCloses }); const c = res.classes[0]!; expect(c.trades).toBeGreaterThanOrEqual(1); expect(c.finalEquity).toBeGreaterThan(1000); @@ -77,6 +103,9 @@ describe('runStrategyBacktest (bankroll + timeframe)', () => { expect(c.symbols).toEqual([]); expect(c.trades).toBe(0); expect(c.finalEquity).toBe(1000); + expect(c.totalCostUsd).toBe(0); + // Assumptions are still reported so the UI can say what *would* have applied. + expect(c.costs.roundTripBps).toBeGreaterThan(0); } expect(res.verdict).toBeNull(); }); @@ -92,3 +121,120 @@ describe('runStrategyBacktest (bankroll + timeframe)', () => { expect(res.classes[0]!.symbols.length).toBeGreaterThan(0); }); }); + +describe('runStrategyBacktest cost accounting', () => { + const fetchCloses: FetchCloses = async (_symbol, startMs) => dipAndRip(startMs); + + async function run(costs?: CostModel) { + const res = await runStrategyBacktest(tsp.compile(buyLowSellHigh), { + classes: ['crypto'], + bankroll: 1000, + timeframe: '1 year', + fetchCloses, + costs, + }); + return res.classes[0]!; + } + + it('nets returns below gross whenever costs are non-zero', async () => { + const c = await run(); + expect(c.trades).toBeGreaterThan(0); + expect(c.returnPct).toBeLessThan(c.grossReturnPct); + expect(c.totalCostUsd).toBeGreaterThan(0); + expect(c.costDragPct).toBeGreaterThan(0); + }); + + it('leaves no unattributed friction: gross − net is exactly the cost drag', async () => { + const c = await run(); + expect(c.grossReturnPct - c.returnPct).toBeCloseTo(c.costDragPct, 10); + expect(c.feesUsd + c.spreadSlippageUsd).toBeCloseTo(c.totalCostUsd, 8); + expect(c.costDragPct).toBeCloseTo(c.totalCostUsd / 1000, 10); + }); + + it('charges nothing under ZERO_COST_MODEL, where net equals gross', async () => { + const c = await run(ZERO_COST_MODEL); + expect(c.totalCostUsd).toBe(0); + expect(c.feesUsd).toBe(0); + expect(c.spreadSlippageUsd).toBe(0); + expect(c.costDragPct).toBe(0); + expect(c.returnPct).toBeCloseTo(c.grossReturnPct, 12); + expect(c.costs.roundTripBps).toBe(0); + }); + + it('actually deducts costs: a real model finishes below the frictionless run', async () => { + const free = await run(ZERO_COST_MODEL); + const paid = await run(); + expect(paid.finalEquity).toBeLessThan(free.finalEquity); + // Same signals on the same bars, so the frictionless number matches. + expect(paid.grossReturnPct).toBeCloseTo(free.returnPct, 10); + }); + + it('separates fee cost from spread cost', async () => { + const fee = await run({ feeBps: 50, slippageBps: 0, assumedHalfSpreadBps: 0, perOrderUsd: 0 }); + expect(fee.feesUsd).toBeCloseTo(fee.totalCostUsd, 8); + expect(fee.spreadSlippageUsd).toBeCloseTo(0, 8); + + const spread = await run({ feeBps: 0, slippageBps: 0, assumedHalfSpreadBps: 50, perOrderUsd: 0 }); + expect(spread.spreadSlippageUsd).toBeCloseTo(spread.totalCostUsd, 8); + expect(spread.feesUsd).toBeCloseTo(0, 8); + }); + + it('compounds by netMultiple, not the price ratio, so fees cannot vanish', async () => { + // Under a fee-only model the fills ARE the mid on both legs, so the price + // ratio is identical to the frictionless run — anything compounding + // exitPrice/entryPrice would report the gross number as net. + const paid = await run({ feeBps: 100, slippageBps: 0, assumedHalfSpreadBps: 0, perOrderUsd: 0 }); + const free = await run(ZERO_COST_MODEL); + + expect(paid.grossReturnPct).toBeCloseTo(free.returnPct, 10); + expect(paid.returnPct).toBeLessThan(paid.grossReturnPct); + // One round trip at 100 bps/leg: 1 − (0.99/1.01) ≈ 1.98% of the slice. + expect(1 - paid.finalEquity / free.finalEquity).toBeCloseTo(1 - 0.99 / 1.01, 6); + }); + + it('costs more the wider the model', async () => { + const cheap = await run({ feeBps: 10, slippageBps: 1, assumedHalfSpreadBps: 1, perOrderUsd: 0 }); + const dear = await run({ feeBps: 60, slippageBps: 5, assumedHalfSpreadBps: 5, perOrderUsd: 0 }); + expect(dear.totalCostUsd).toBeGreaterThan(cheap.totalCostUsd); + expect(dear.returnPct).toBeLessThan(cheap.returnPct); + expect(dear.costs.roundTripBps).toBeGreaterThan(cheap.costs.roundTripBps); + }); + + it('defaults each class to its own model and reports the assumptions', async () => { + const res = await runStrategyBacktest(tsp.compile(buyLowSellHigh), { + classes: ['crypto', 'equity'], + bankroll: 1200, + timeframe: '1 year', + fetchCloses, + }); + const crypto = res.classes[0]!; + const equity = res.classes[1]!; + + expect(DEFAULT_CLASS_COSTS.crypto).toBe(DEFAULT_COST_MODEL); + expect(DEFAULT_CLASS_COSTS.equity).toBe(EQUITY_COST_MODEL); + expect(crypto.costs.feeBps).toBe(DEFAULT_COST_MODEL.feeBps); + expect(equity.costs.feeBps).toBe(EQUITY_COST_MODEL.feeBps); + // Equities are commission-free here, so crypto must carry the bigger hurdle. + expect(crypto.costs.roundTripBps).toBeGreaterThan(equity.costs.roundTripBps); + expect(crypto.costDragPct).toBeGreaterThan(equity.costDragPct); + }); + + it('counts wins on net profit, so a cost-eaten winner is not a win', async () => { + const thin: FetchCloses = async (_symbol, startMs) => tinyEdge(startMs); + const opts = { classes: ['crypto'] as const, bankroll: 1000, timeframe: '1 year' as const, fetchCloses: thin }; + + const free = await runStrategyBacktest(tsp.compile(scalp), { ...opts, classes: ['crypto'], costs: ZERO_COST_MODEL }); + const brutal = await runStrategyBacktest(tsp.compile(scalp), { + ...opts, + classes: ['crypto'], + costs: { feeBps: 500, slippageBps: 0, assumedHalfSpreadBps: 0, perOrderUsd: 0 }, + }); + + expect(free.classes[0]!.trades).toBeGreaterThan(0); + expect(free.classes[0]!.winRate).toBe(1); + expect(brutal.classes[0]!.trades).toBe(free.classes[0]!.trades); + expect(brutal.classes[0]!.winRate).toBe(0); + expect(brutal.classes[0]!.returnPct).toBeLessThan(0); + expect(brutal.classes[0]!.grossReturnPct).toBeGreaterThan(0); + }); +}); diff --git a/apps/web/src/lib/strategy-backtest-runner.ts b/apps/web/src/lib/strategy-backtest-runner.ts index bae10d0..8171228 100644 --- a/apps/web/src/lib/strategy-backtest-runner.ts +++ b/apps/web/src/lib/strategy-backtest-runner.ts @@ -6,9 +6,30 @@ * the bankroll is split equally across the basket, and each slice compounds * through that symbol's round-trips (cash → shares → cash). Pure — the price * fetch is injected so the route supplies a source and tests supply a stub. + * + * COSTS. This wizard feeds the store, so a number shown here is a number a + * stranger may risk money on. Every slice therefore compounds by the trade's + * `netMultiple` (cash out / cash in, net of fees, spread and slippage) rather + * than by the raw price ratio: the price ratio only carries spread + slippage + * now, so using it silently refunds every fee. We replay each series TWICE — + * once with the real model and once with `ZERO_COST_MODEL` — so the response can + * put net return next to the frictionless number it would have advertised. + * `grossReturnPct − returnPct === costDragPct`, exactly, by construction. + * + * The cost model is passed EXPLICITLY rather than inferred from the snapshots: + * daily closes are synthesized with `bid === ask === close`, and the injected + * fetcher — not this module — decides which venue the data came from, so there + * is no venue here worth trusting. Judgement calls in `DEFAULT_CLASS_COSTS`. */ import type { StrategyPlugin, MarketSnapshot } from '@b1dz/core'; -import { replayStrategy } from '@b1dz/source-strategies'; +import { + replayStrategy, + roundTripCostBps, + DEFAULT_COST_MODEL, + EQUITY_COST_MODEL, + ZERO_COST_MODEL, + type CostModel, +} from '@b1dz/source-strategies'; export type AssetClass = 'crypto' | 'equity'; @@ -16,6 +37,27 @@ export const CRYPTO_BASKET = ['BTC-USD', 'ETH-USD', 'SOL-USD']; export const EQUITY_BASKET = ['SPY', 'AAPL', 'NVDA']; const MIN_BARS = 35; // enough for the slowest indicator (MACD/trend) +/** + * Default friction per asset class. + * + * Crypto gets `DEFAULT_COST_MODEL` (Coinbase-tier, 60 bps/leg) and not the + * cheaper Kraken schedule the crypto fetcher happens to read closes from: the + * user picks their own venue, and a strategy that only clears the hurdle at the + * cheapest venue we know of is not a strategy we should be advertising. + * Equities get `EQUITY_COST_MODEL` — commission-free everywhere b1dz connects, + * so spread plus impact is the whole cost. + */ +export const DEFAULT_CLASS_COSTS: Record = { + crypto: DEFAULT_COST_MODEL, + equity: EQUITY_COST_MODEL, +}; + +/** Venue id stamped on synthesized snapshots — deliberately not a real venue. */ +const SYNTHETIC_VENUE: Record = { + crypto: 'backtest-crypto', + equity: 'backtest-equity', +}; + export const TIMEFRAMES = [ { label: '1 month', months: 1 }, { label: '3 months', months: 3 }, @@ -36,17 +78,35 @@ export interface DailyClose { /** Fetch daily closes for a symbol within [startMs, endMs]. */ export type FetchCloses = (symbol: string, startMs: number, endMs: number) => Promise; +/** The cost assumptions a result was scored under, flattened for the wire/UI. */ +export interface CostAssumptions { + feeBps: number; + slippageBps: number; + assumedHalfSpreadBps: number; + perOrderUsd: number; + /** Break-even move a round trip must clear under these assumptions. */ + roundTripBps: number; +} + export interface ClassResult { assetClass: AssetClass; basket: string[]; symbols: string[]; // ones that returned usable data trades: number; - returnPct: number; - winRate: number; - profit: number; // finalEquity - bankroll + returnPct: number; // NET of modelled costs + /** What the same trades would have returned frictionless. The honesty delta. */ + grossReturnPct: number; + winRate: number; // counted on NET profit + profit: number; // finalEquity - bankroll, net maxDrawdown: number; // dollars, peak-to-trough on the merged equity curve bankroll: number; finalEquity: number; + feesUsd: number; + spreadSlippageUsd: number; + totalCostUsd: number; + /** totalCostUsd / bankroll — equals grossReturnPct − returnPct. */ + costDragPct: number; + costs: CostAssumptions; } export interface BacktestResponse { @@ -72,10 +132,21 @@ function windowStart(end: Date, tf: (typeof TIMEFRAMES)[number]): Date { } function toSnapshots(symbol: string, rows: DailyClose[], assetClass: AssetClass): MarketSnapshot[] { + const exchange = SYNTHETIC_VENUE[assetClass]; return rows .filter((r) => Number.isFinite(r.close)) .sort((a, b) => a.ts - b.ts) - .map((r) => ({ exchange: 'src', pair: symbol, bid: r.close, ask: r.close, bidSize: 1, askSize: 1, ts: r.ts, assetClass })); + .map((r) => ({ exchange, pair: symbol, bid: r.close, ask: r.close, bidSize: 1, askSize: 1, ts: r.ts, assetClass })); +} + +export function costAssumptions(costs: CostModel, notionalUsd: number): CostAssumptions { + return { + feeBps: costs.feeBps, + slippageBps: costs.slippageBps, + assumedHalfSpreadBps: costs.assumedHalfSpreadBps, + perOrderUsd: costs.perOrderUsd, + roundTripBps: roundTripCostBps(costs, notionalUsd), + }; } /** An equity event: this symbol's slice is now worth `equity` as of `ts`. */ @@ -89,7 +160,7 @@ async function backtestClass( plugin: StrategyPlugin, assetClass: AssetClass, bankroll: number, - tf: (typeof TIMEFRAMES)[number], + costs: CostModel, startMs: number, endMs: number, fetchCloses: FetchCloses, @@ -108,37 +179,57 @@ async function backtestClass( } const symbols = [...series.keys()]; + const slice = symbols.length > 0 ? bankroll / symbols.length : bankroll; const result: ClassResult = { assetClass, basket, symbols, trades: 0, returnPct: 0, + grossReturnPct: 0, winRate: 0, profit: 0, maxDrawdown: 0, bankroll, finalEquity: bankroll, + feesUsd: 0, + spreadSlippageUsd: 0, + totalCostUsd: 0, + costDragPct: 0, + costs: costAssumptions(costs, slice), }; if (symbols.length === 0) return result; - const slice = bankroll / symbols.length; const events: EquityEvent[] = []; - let totalFinal = 0; + let netFinal = 0; + let grossFinal = 0; let wins = 0; let tradeCount = 0; + // Per-trade friction at the replay notional. Only its fee/spread RATIO is used + // — the dollar total comes from the two equity curves, which is compounding-aware. + let rawFees = 0; + let rawSpread = 0; for (const [symbol, snaps] of series) { - const trades = replayStrategy(plugin, snaps, 1); // amount irrelevant — we use price ratios + const trades = replayStrategy(plugin, snaps, { amountPerEntry: slice, costs }); let equity = slice; events.push({ ts: startMs, symbol, equity }); for (const t of trades) { tradeCount++; - if (t.exitPrice > t.entryPrice) wins++; - equity *= t.exitPrice / t.entryPrice; // compound the slice through this round-trip + if (t.netMultiple > 1) wins++; // a fee-eating "winner" is a loss + rawFees += t.feesUsd; + rawSpread += t.spreadSlippageUsd; + equity *= t.netMultiple; // compound the slice through this round-trip events.push({ ts: t.exitTs, symbol, equity }); } - totalFinal += equity; + netFinal += equity; + + // Same signals, same bars, no friction — what the strategy would have "made". + let gross = slice; + for (const t of replayStrategy(plugin, snaps, { amountPerEntry: slice, costs: ZERO_COST_MODEL })) { + gross *= t.netMultiple; + } + grossFinal += gross; } // Merged equity curve → peak-to-trough drawdown in dollars. @@ -154,18 +245,34 @@ async function backtestClass( maxDrawdown = Math.max(maxDrawdown, peak - total); } + const totalCostUsd = Math.max(0, grossFinal - netFinal); + const rawTotal = rawFees + rawSpread; + const feesUsd = rawTotal > 0 ? (totalCostUsd * rawFees) / rawTotal : 0; + result.trades = tradeCount; - result.finalEquity = totalFinal; - result.profit = totalFinal - bankroll; - result.returnPct = bankroll > 0 ? totalFinal / bankroll - 1 : 0; + result.finalEquity = netFinal; + result.profit = netFinal - bankroll; + result.returnPct = bankroll > 0 ? netFinal / bankroll - 1 : 0; + result.grossReturnPct = bankroll > 0 ? grossFinal / bankroll - 1 : 0; result.winRate = tradeCount ? wins / tradeCount : 0; result.maxDrawdown = maxDrawdown; + result.feesUsd = feesUsd; + result.spreadSlippageUsd = totalCostUsd - feesUsd; + result.totalCostUsd = totalCostUsd; + result.costDragPct = bankroll > 0 ? totalCostUsd / bankroll : 0; return result; } export async function runStrategyBacktest( plugin: StrategyPlugin, - opts: { classes: AssetClass[]; bankroll: number; timeframe: TimeframeLabel; fetchCloses: FetchCloses }, + opts: { + classes: AssetClass[]; + bankroll: number; + timeframe: TimeframeLabel; + fetchCloses: FetchCloses; + /** Override the per-asset-class default friction (route body, CLI flag, tests). */ + costs?: CostModel; + }, ): Promise { const tf = TIMEFRAMES.find((t) => t.label === opts.timeframe) ?? TIMEFRAMES.find((t) => t.label === DEFAULT_TIMEFRAME)!; const now = new Date(); @@ -174,7 +281,8 @@ export async function runStrategyBacktest( const classes: ClassResult[] = []; for (const assetClass of opts.classes) { - classes.push(await backtestClass(plugin, assetClass, opts.bankroll, tf, startMs, endMs, opts.fetchCloses)); + const costs = opts.costs ?? DEFAULT_CLASS_COSTS[assetClass]; + classes.push(await backtestClass(plugin, assetClass, opts.bankroll, costs, startMs, endMs, opts.fetchCloses)); } let verdict: BacktestResponse['verdict'] = null; diff --git a/packages/adapters-cex/src/cex-adapter.ts b/packages/adapters-cex/src/cex-adapter.ts index a19b65a..9c05bde 100644 --- a/packages/adapters-cex/src/cex-adapter.ts +++ b/packages/adapters-cex/src/cex-adapter.ts @@ -8,17 +8,16 @@ * is modeled crudely until a proper book-walker lands — an MVP trade-off. */ -import type { PriceFeed, MarketSnapshot } from '@b1dz/core'; +import { CEX_TAKER_FEE_RATES, DEFAULT_CEX_TAKER_FEE, type PriceFeed, type MarketSnapshot } from '@b1dz/core'; import type { NormalizedQuote, QuoteRequest, VenueAdapter, AdapterHealth } from '@b1dz/venue-types'; -/** Taker fee schedule from the live daemon — keeps backtest, observer, - * and daemon in sync. Don't inline per-exchange strings elsewhere. */ -export const CEX_TAKER_FEES: Record = { - kraken: 0.0026, - 'binance-us': 0.001, - coinbase: 0.006, - gemini: 0.004, -}; +/** + * Taker fee schedule — kept as an alias so backtest, observer, and daemon all + * price fills identically. The canonical table now lives in @b1dz/core + * (`CEX_TAKER_FEE_RATES`) so the backtester can read it without depending on + * this adapter package. Don't inline per-exchange rates anywhere. + */ +export const CEX_TAKER_FEES: Record = CEX_TAKER_FEE_RATES; export interface CexAdapterOptions { /** If the feed exposes a custom name, override it. Otherwise feed.exchange is used. */ @@ -37,7 +36,7 @@ export class CexAdapter implements VenueAdapter { constructor(feed: PriceFeed, opts: CexAdapterOptions = {}) { this.feed = feed; this.venue = opts.venueOverride ?? feed.exchange; - this.feeRate = opts.feeRate ?? CEX_TAKER_FEES[this.venue] ?? 0.005; + this.feeRate = opts.feeRate ?? CEX_TAKER_FEE_RATES[this.venue] ?? DEFAULT_CEX_TAKER_FEE; } async health(): Promise { diff --git a/packages/core/src/fees.ts b/packages/core/src/fees.ts new file mode 100644 index 0000000..e2ad182 --- /dev/null +++ b/packages/core/src/fees.ts @@ -0,0 +1,68 @@ +/** + * Canonical trading-cost constants. + * + * These live in @b1dz/core (which everything depends on) so the backtester, the + * observer, the profitability ranker, and the live daemon all price a fill the + * same way. A strategy that looks profitable in a backtest priced at 0 bps and + * loses money live at 26 bps is the single most common way a systematic trading + * system fails — so there is exactly ONE table and everybody reads it. + * + * Rates are decimal fractions of notional (0.0026 = 26 bps = 0.26%), matching + * the wire format every exchange API uses. + */ + +/** + * Taker fee per fill, by CEX venue id. Taker (not maker) because every path in + * b1dz that crosses the spread — market orders, IOC limits, arb legs — pays the + * taker side. Sourced from each venue's published retail schedule; volume tiers + * and fee-token discounts only ever make the real number smaller, so using the + * top-tier rate keeps estimates conservative. + */ +export const CEX_TAKER_FEE_RATES: Record = { + kraken: 0.0026, + 'binance-us': 0.001, + coinbase: 0.006, + gemini: 0.004, +}; + +/** Fallback for an unrecognized CEX — deliberately worse than any known venue. */ +export const DEFAULT_CEX_TAKER_FEE = 0.005; + +/** + * Typical AMM pool fee for the DEX venues b1dz routes through (Uniswap V3 0.30% + * tier, 1inch/0x aggregated routes land in the same neighbourhood). Gas is a + * separate, flat per-transaction cost — see `DEFAULT_DEX_GAS_USD`. + */ +export const DEFAULT_DEX_POOL_FEE = 0.003; + +/** + * Rough per-swap gas cost on an L2 (Base). A placeholder for estimation only: + * live paths must use a real gas quote from @b1dz/adapters-evm, never this. + */ +export const DEFAULT_DEX_GAS_USD = 0.15; + +/** + * US equity retail commission. Zero at every broker b1dz connects to (Alpaca, + * Schwab, Tradier, TradeStation, Webull, IBKR Lite). + * + * Not modelled here because they round to noise at retail size: the SEC Section + * 31 fee and FINRA TAF apply to SELLS only and together come to well under + * 1 bp. Treat equity commissions as zero and let spread + slippage carry the + * cost estimate. + */ +export const DEFAULT_EQUITY_COMMISSION = 0; + +/** Taker fee for a venue id, falling back to the conservative default. */ +export function cexTakerFee(venue: string): number { + return CEX_TAKER_FEE_RATES[venue] ?? DEFAULT_CEX_TAKER_FEE; +} + +/** Decimal fraction → basis points (0.0026 → 26). */ +export function toBps(rate: number): number { + return rate * 10_000; +} + +/** Basis points → decimal fraction (26 → 0.0026). */ +export function fromBps(bps: number): number { + return bps / 10_000; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 7f43c4e..581ca4a 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -5,6 +5,7 @@ export * from './alerts.js'; export * from './source.js'; export * from './runner.js'; export * from './market.js'; +export * from './fees.js'; export * from './indicators.js'; export * from './sessions.js'; export * from './oauth.js'; diff --git a/packages/event-channel/package.json b/packages/event-channel/package.json index a0d0508..bc32313 100644 --- a/packages/event-channel/package.json +++ b/packages/event-channel/package.json @@ -19,7 +19,7 @@ }, "dependencies": { "@b1dz/venue-types": "workspace:*", - "@supabase/supabase-js": "latest" + "@supabase/supabase-js": "^2.112.0" }, "devDependencies": { "@types/node": "latest", diff --git a/packages/source-strategies/src/backtest.test.ts b/packages/source-strategies/src/backtest.test.ts index 963052d..23b6494 100644 --- a/packages/source-strategies/src/backtest.test.ts +++ b/packages/source-strategies/src/backtest.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect } from 'vitest'; import type { MarketSnapshot, StrategyPlugin, Signal } from '@b1dz/core'; import { replayStrategy, summarizeTrades, DEFAULT_AMOUNT_PER_ENTRY } from './backtest.js'; +import { ZERO_COST_MODEL } from './costs.js'; /** Snapshot at a mid price (bid=ask, so mid === price). */ function snap(price: number, ts: number): MarketSnapshot { @@ -31,7 +32,7 @@ describe('replayStrategy', () => { it('opens on buy and closes on sell, sizing each entry at amountPerEntry', () => { const snaps = series([100, 110, 120, 130]); // buy at bar 0 (price 100), sell at bar 2 (price 120) - const trades = replayStrategy(scripted({ 0: 'buy', 2: 'sell' }), snaps, 100); + const trades = replayStrategy(scripted({ 0: 'buy', 2: 'sell' }), snaps, { amountPerEntry: 100, costs: ZERO_COST_MODEL }); expect(trades).toHaveLength(1); const t = trades[0]!; expect(t.entryPrice).toBe(100); @@ -48,7 +49,7 @@ describe('replayStrategy', () => { it('is long-only: ignores a sell while flat and a second buy while long', () => { const snaps = series([100, 105, 110, 115]); // sell@0 (flat → ignored), buy@1, buy@2 (already long → ignored), sell@3 - const trades = replayStrategy(scripted({ 0: 'sell', 1: 'buy', 2: 'buy', 3: 'sell' }), snaps, 100); + const trades = replayStrategy(scripted({ 0: 'sell', 1: 'buy', 2: 'buy', 3: 'sell' }), snaps, { amountPerEntry: 100, costs: ZERO_COST_MODEL }); expect(trades).toHaveLength(1); expect(trades[0]!.entryPrice).toBe(105); // entered at bar 1, not re-entered at bar 2 expect(trades[0]!.exitPrice).toBe(115); @@ -56,7 +57,7 @@ describe('replayStrategy', () => { it('marks an open position to the final bar', () => { const snaps = series([100, 90, 80]); - const trades = replayStrategy(scripted({ 0: 'buy' }), snaps, 100); // never sells + const trades = replayStrategy(scripted({ 0: 'buy' }), snaps, { amountPerEntry: 100, costs: ZERO_COST_MODEL }); // never sells expect(trades).toHaveLength(1); expect(trades[0]!.exitPrice).toBe(80); expect(trades[0]!.exitReason).toBe('close at end'); @@ -64,7 +65,7 @@ describe('replayStrategy', () => { }); it('produces no trades when the strategy never signals', () => { - expect(replayStrategy(scripted({}), series([100, 101, 102]), 100)).toEqual([]); + expect(replayStrategy(scripted({}), series([100, 101, 102]), { amountPerEntry: 100, costs: ZERO_COST_MODEL })).toEqual([]); }); it('treats a throwing evaluate() as no-signal instead of aborting', () => { @@ -74,11 +75,11 @@ describe('replayStrategy', () => { throw new Error('strategy bug'); }, }; - expect(replayStrategy(boom, series([100, 101, 102]), 100)).toEqual([]); + expect(replayStrategy(boom, series([100, 101, 102]), { amountPerEntry: 100, costs: ZERO_COST_MODEL })).toEqual([]); }); it('defaults amountPerEntry to DEFAULT_AMOUNT_PER_ENTRY', () => { - const trades = replayStrategy(scripted({ 0: 'buy', 1: 'sell' }), series([100, 110])); + const trades = replayStrategy(scripted({ 0: 'buy', 1: 'sell' }), series([100, 110]), { amountPerEntry: DEFAULT_AMOUNT_PER_ENTRY, costs: ZERO_COST_MODEL }); expect(trades[0]!.cost).toBe(DEFAULT_AMOUNT_PER_ENTRY); }); }); @@ -92,9 +93,9 @@ describe('summarizeTrades', () => { it('aggregates wins, losses, return, and win rate', () => { // one +$20 winner, one -$10 loser const snaps = series([100, 120]); - const win = replayStrategy(scripted({ 0: 'buy', 1: 'sell' }), snaps, 100); + const win = replayStrategy(scripted({ 0: 'buy', 1: 'sell' }), snaps, { amountPerEntry: 100, costs: ZERO_COST_MODEL }); const loseSnaps = series([100, 90]); - const lose = replayStrategy(scripted({ 0: 'buy', 1: 'sell' }), loseSnaps, 100); + const lose = replayStrategy(scripted({ 0: 'buy', 1: 'sell' }), loseSnaps, { amountPerEntry: 100, costs: ZERO_COST_MODEL }); const s = summarizeTrades([...win, ...lose]); expect(s.trades).toBe(2); expect(s.invested).toBe(200); @@ -109,7 +110,7 @@ describe('summarizeTrades', () => { // sequence of realized profits: +30, -50, +10 → equity 30, -20, -10 // peak 30, trough -20 → max drawdown 50 const t = (entry: number, exit: number) => - replayStrategy(scripted({ 0: 'buy', 1: 'sell' }), series([entry, exit]), 100); + replayStrategy(scripted({ 0: 'buy', 1: 'sell' }), series([entry, exit]), { amountPerEntry: 100, costs: ZERO_COST_MODEL }); const trades = [ ...t(100, 130), // +30 ...t(100, 50), // -50 diff --git a/packages/source-strategies/src/backtest.ts b/packages/source-strategies/src/backtest.ts index 8721fba..a247bc6 100644 --- a/packages/source-strategies/src/backtest.ts +++ b/packages/source-strategies/src/backtest.ts @@ -1,30 +1,75 @@ /** * Strategy backtester — replay a StrategyPlugin's own signals over a snapshot - * stream and score the result. + * stream and score the result NET OF TRADING COSTS. * * Strategies are signals-only: evaluate() emits buy/sell Signals; it never sizes * or executes. This module turns that signal stream into long-only round-trip * trades so a strategy can be scored historically: - * - flat + buy → open a position at the current mid, sized at `amountPerEntry` - * - long + sell → close it at the current mid - * - any position still open at the end is marked to the final bar + * - flat + buy → open a position at the effective ASK, sized at `amountPerEntry` + * - long + sell → close it at the effective BID + * - any position still open at the end is marked out to the final bar * - * It is deterministic and asset-agnostic — the same code scores BTC-USD ticks or - * AAPL bars. The standalone ~/bin/backtest.js CLI and (later) the custom-strategy - * wizard both drive this so their numbers agree. Costs (fees/slippage/spread) are - * intentionally excluded; layer them on at the caller if needed. + * Every trade is priced through a `CostModel` (see ./costs.ts): fees on both + * legs, the bid/ask spread, slippage, and any flat per-order cost. When `costs` + * is omitted the model is derived from the series' own asset class and venue, so + * the default is realistic rather than free — a zero-cost backtest is a + * different game, not a conservative estimate of this one. + * + * `profit` is NET. `grossProfit` is pre-fee (but post spread/slippage, since + * those are embedded in the fill prices). Pass `ZERO_COST_MODEL` explicitly if + * you need the old frictionless numbers for comparison. + * + * Deterministic and asset-agnostic — the same code scores BTC-USD ticks and + * AAPL daily bars. */ import type { MarketSnapshot, StrategyPlugin } from '@b1dz/core'; +import { + costModelForSeries, + effectiveBuyPrice, + effectiveSellPrice, + legFeeUsd, + midPrice, + roundTripCostBps, + type CostModel, +} from './costs.js'; export interface BacktestTrade { entryTs: number; exitTs: number; + /** Effective fill price on entry — ask + slippage. */ entryPrice: number; + /** Effective fill price on exit — bid − slippage. */ exitPrice: number; + /** Frictionless mid at entry, for measuring what the costs took. */ + entryMid: number; + /** Frictionless mid at exit. */ + exitMid: number; shares: number; - cost: number; // dollars put in at entry (= amountPerEntry) + /** Notional put to work at entry, before fees (= amountPerEntry). */ + notionalUsd: number; + /** Total cash out of pocket at entry: notional + entry fee. */ + cost: number; + /** Position value at exit before the exit fee. */ + grossProceeds: number; + /** Cash back in hand after the exit fee. */ proceeds: number; + entryFeeUsd: number; + exitFeeUsd: number; + /** entryFeeUsd + exitFeeUsd. */ + feesUsd: number; + /** What spread + slippage cost versus a frictionless mid-to-mid round trip. */ + spreadSlippageUsd: number; + /** feesUsd + spreadSlippageUsd — total friction paid on this trade. */ + totalCostUsd: number; + /** Total friction as bps of notional. The hurdle this trade had to clear. */ + costBps: number; + /** Pre-fee profit (spread and slippage are already in the fill prices). */ + grossProfit: number; + /** NET profit after fees, spread, slippage, and per-order costs. */ profit: number; + /** proceeds / cost — the factor to compound a bankroll slice by. */ + netMultiple: number; + /** Net return on cash deployed. */ tradeReturnPct: number; entryReason: string; exitReason: string; @@ -32,9 +77,18 @@ export interface BacktestTrade { export interface BacktestSummary { trades: number; + /** Total cash deployed across entries, including entry fees. */ invested: number; proceeds: number; + /** NET profit. */ profit: number; + /** Pre-fee profit, for showing users what costs consumed. */ + grossProfit: number; + feesUsd: number; + spreadSlippageUsd: number; + totalCostUsd: number; + /** Total friction as a fraction of capital deployed. */ + costDragPct: number; returnPct: number; wins: number; losses: number; @@ -45,32 +99,66 @@ export interface BacktestSummary { export const DEFAULT_AMOUNT_PER_ENTRY = 100; -/** Mid price of a snapshot; falls back to whichever side is present. */ -function mid(snap: MarketSnapshot): number { - if (snap.bid > 0 && snap.ask > 0) return (snap.bid + snap.ask) / 2; - return snap.bid || snap.ask || 0; +export interface ReplayOptions { + /** Notional deployed per entry, before fees. */ + amountPerEntry?: number; + /** Cost model. Defaults to one derived from the series' asset class + venue. */ + costs?: CostModel; } interface OpenPosition { entryTs: number; entryPrice: number; - cost: number; + entryMid: number; + notionalUsd: number; + entryFeeUsd: number; + shares: number; entryReason: string; } -function close(position: OpenPosition, exitPrice: number, exitTs: number, exitReason: string): BacktestTrade { - const shares = position.cost / position.entryPrice; - const proceeds = shares * exitPrice; +function closeTrade( + position: OpenPosition, + exitPrice: number, + exitMid: number, + exitTs: number, + exitReason: string, + costs: CostModel, +): BacktestTrade { + const grossProceeds = position.shares * exitPrice; + const exitFeeUsd = legFeeUsd(grossProceeds, costs); + const proceeds = grossProceeds - exitFeeUsd; + const cost = position.notionalUsd + position.entryFeeUsd; + + // What a frictionless mid-to-mid round trip would have returned, for cost attribution. + const idealShares = position.entryMid > 0 ? position.notionalUsd / position.entryMid : 0; + const idealProceeds = idealShares * exitMid; + const spreadSlippageUsd = idealProceeds - grossProceeds; + + const feesUsd = position.entryFeeUsd + exitFeeUsd; + const totalCostUsd = feesUsd + spreadSlippageUsd; + return { entryTs: position.entryTs, exitTs, entryPrice: position.entryPrice, exitPrice, - shares, - cost: position.cost, + entryMid: position.entryMid, + exitMid, + shares: position.shares, + notionalUsd: position.notionalUsd, + cost, + grossProceeds, proceeds, - profit: proceeds - position.cost, - tradeReturnPct: exitPrice / position.entryPrice - 1, + entryFeeUsd: position.entryFeeUsd, + exitFeeUsd, + feesUsd, + spreadSlippageUsd, + totalCostUsd, + costBps: position.notionalUsd > 0 ? (totalCostUsd / position.notionalUsd) * 10_000 : 0, + grossProfit: grossProceeds - position.notionalUsd, + profit: proceeds - cost, + netMultiple: cost > 0 ? proceeds / cost : 1, + tradeReturnPct: cost > 0 ? (proceeds - cost) / cost : 0, entryReason: position.entryReason, exitReason, }; @@ -78,22 +166,30 @@ function close(position: OpenPosition, exitPrice: number, exitTs: number, exitRe /** * Replay `plugin` over `snapshots` (chronological) and return the round-trip - * trades. A strategy that throws is treated as "no signal" for that bar so one - * bad evaluate() can't abort the whole run. + * trades, net of costs. A strategy that throws is treated as "no signal" for + * that bar so one bad evaluate() can't abort the whole run. + * + * The third argument accepts a bare number for backward compatibility with + * `replayStrategy(plugin, snaps, 100)`. */ export function replayStrategy( plugin: StrategyPlugin, snapshots: MarketSnapshot[], - amountPerEntry: number = DEFAULT_AMOUNT_PER_ENTRY, + optsOrAmount: number | ReplayOptions = {}, ): BacktestTrade[] { + const opts: ReplayOptions = + typeof optsOrAmount === 'number' ? { amountPerEntry: optsOrAmount } : optsOrAmount; + const amountPerEntry = opts.amountPerEntry ?? DEFAULT_AMOUNT_PER_ENTRY; + const costs = opts.costs ?? costModelForSeries(snapshots); + const trades: BacktestTrade[] = []; let position: OpenPosition | null = null; for (let i = 0; i < snapshots.length; i++) { const snap = snapshots[i]!; const history = snapshots.slice(0, i); - const price = mid(snap); - if (!(price > 0)) continue; + const mid = midPrice(snap); + if (!(mid > 0)) continue; let signal = null; try { @@ -104,25 +200,48 @@ export function replayStrategy( if (!signal) continue; if (!position && signal.side === 'buy') { - position = { entryTs: snap.ts, entryPrice: price, cost: amountPerEntry, entryReason: signal.reason }; + const entryPrice = effectiveBuyPrice(snap, costs); + if (!(entryPrice > 0)) continue; + const entryFeeUsd = legFeeUsd(amountPerEntry, costs); + position = { + entryTs: snap.ts, + entryPrice, + entryMid: mid, + notionalUsd: amountPerEntry, + entryFeeUsd, + shares: amountPerEntry / entryPrice, + entryReason: signal.reason, + }; } else if (position && signal.side === 'sell') { - trades.push(close(position, price, snap.ts, signal.reason)); + trades.push(closeTrade(position, effectiveSellPrice(snap, costs), mid, snap.ts, signal.reason, costs)); position = null; } } if (position && snapshots.length) { const last = snapshots[snapshots.length - 1]!; - trades.push(close(position, mid(last), last.ts, 'close at end')); + trades.push( + closeTrade(position, effectiveSellPrice(last, costs), midPrice(last), last.ts, 'close at end', costs), + ); } return trades; } -/** Aggregate a set of trades into the headline metrics the store/CLI render. */ +/** + * Aggregate a set of trades into the headline metrics the store/CLI render. + * + * Wins and losses are counted on NET profit: a trade that gains 10 bps of price + * and pays 60 bps of fees is a loss, and calling it a win is how a losing + * strategy gets sold as a winning one. + */ export function summarizeTrades(trades: BacktestTrade[]): BacktestSummary { const invested = trades.reduce((s, t) => s + t.cost, 0); const proceeds = trades.reduce((s, t) => s + t.proceeds, 0); + const grossProfit = trades.reduce((s, t) => s + t.grossProfit, 0); + const feesUsd = trades.reduce((s, t) => s + t.feesUsd, 0); + const spreadSlippageUsd = trades.reduce((s, t) => s + t.spreadSlippageUsd, 0); + const totalCostUsd = feesUsd + spreadSlippageUsd; const profit = proceeds - invested; const returnPct = invested > 0 ? profit / invested : 0; const wins = trades.filter((t) => t.profit > 0).length; @@ -141,5 +260,28 @@ export function summarizeTrades(trades: BacktestTrade[]): BacktestSummary { maxDrawdown = Math.max(maxDrawdown, peak - cum); } - return { trades: trades.length, invested, proceeds, profit, returnPct, wins, losses, winRate, avgTradePct, maxDrawdown }; + return { + trades: trades.length, + invested, + proceeds, + profit, + grossProfit, + feesUsd, + spreadSlippageUsd, + totalCostUsd, + costDragPct: invested > 0 ? totalCostUsd / invested : 0, + returnPct, + wins, + losses, + winRate, + avgTradePct, + maxDrawdown, + }; } + +/** + * Break-even move a single round trip must clear, in bps, under `costs`. + * Re-exported here because it is the number that decides whether a strategy is + * viable at all on a given venue. + */ +export { roundTripCostBps }; diff --git a/packages/source-strategies/src/costs.ts b/packages/source-strategies/src/costs.ts new file mode 100644 index 0000000..111217b --- /dev/null +++ b/packages/source-strategies/src/costs.ts @@ -0,0 +1,211 @@ +/** + * Trading cost model for the backtester. + * + * A backtest that prices fills at the mid and charges no fee is not a + * conservative estimate — it is a different, easier game. Three frictions, + * every one of which is paid on every round trip: + * + * 1. SPREAD you buy at the ask and sell at the bid, never at the mid. + * Using mid for both legs understates cost by the full spread. + * 2. FEE taker fee on notional, charged on BOTH legs. Coinbase's 60 bps + * round trip is 120 bps — larger than the entire edge of most + * short-horizon strategies. + * 3. SLIPPAGE market impact beyond the quoted top of book. + * + * Why spread has to be *modelled* and not just read: most historical series + * available to us are daily closes, which get turned into snapshots with + * `bid === ask === close` (see apps/web/src/lib/strategy-backtest-runner.ts). + * Those snapshots have a literal zero spread. So when a snapshot carries no real + * two-sided quote we apply `assumedHalfSpreadBps`; when it carries a genuine + * bid/ask (live tick data, forward tests) we use the real thing. + * + * All rates are basis points of notional. 1 bp = 0.01%. + */ +import { + cexTakerFee, + toBps, + DEFAULT_DEX_POOL_FEE, + DEFAULT_DEX_GAS_USD, + DEFAULT_EQUITY_COMMISSION, + type MarketSnapshot, +} from '@b1dz/core'; + +export interface CostModel { + /** Taker fee in bps of notional, charged on entry AND exit. */ + feeBps: number; + /** One-way market impact in bps, applied against the executing side each leg. */ + slippageBps: number; + /** + * Half-spread in bps, used ONLY when a snapshot has no real two-sided quote + * (bid === ask). Entry pays +half, exit pays −half, so a round trip costs the + * full spread — which is the correct treatment. + */ + assumedHalfSpreadBps: number; + /** Flat cost per order in quote currency (DEX gas, per-contract fees). */ + perOrderUsd: number; +} + +/** + * No friction at all. Exists so state-machine tests can isolate the replay + * logic, and so the UI can show a gross-vs-net comparison. Never use this to + * evaluate whether a strategy is worth trading or selling. + */ +export const ZERO_COST_MODEL: CostModel = { + feeBps: 0, + slippageBps: 0, + assumedHalfSpreadBps: 0, + perOrderUsd: 0, +}; + +/** + * Used when the asset class and venue can't be determined. Deliberately + * pessimistic (Coinbase-tier fees): if a strategy survives this it will survive + * anywhere, and an unknown venue is exactly when you want to be cautious. + */ +export const DEFAULT_COST_MODEL: CostModel = { + feeBps: toBps(0.006), + slippageBps: 5, + assumedHalfSpreadBps: 5, + perOrderUsd: 0, +}; + +/** + * US equity retail. Commission-free at every broker b1dz connects to; the cost + * is spread plus impact. Majors like SPY/AAPL sit around 1 bp of spread, so 2 bp + * half-spread stays honest for the mid-cap end of a watchlist. + */ +export const EQUITY_COST_MODEL: CostModel = { + feeBps: toBps(DEFAULT_EQUITY_COMMISSION), + slippageBps: 2, + assumedHalfSpreadBps: 2, + perOrderUsd: 0, +}; + +/** + * On-chain swap: pool fee dominates, impact is materially worse than a CEX, and + * gas is a flat per-swap charge that makes small notionals uneconomic. Callers + * with a live gas quote from @b1dz/adapters-evm should override `perOrderUsd`. + */ +export const DEX_COST_MODEL: CostModel = { + feeBps: toBps(DEFAULT_DEX_POOL_FEE), + slippageBps: 30, + assumedHalfSpreadBps: 10, + perOrderUsd: DEFAULT_DEX_GAS_USD, +}; + +/** Venue ids that are on-chain rather than centralized order books. */ +const DEX_VENUES = new Set([ + 'uniswap-v3', + 'uniswap-v3-base', + 'uniswap', + '1inch', + '0x', + 'zeroex', + 'jupiter', + 'pumpfun', + 'aggregator-base', +]); + +export interface CostModelHint { + assetClass?: 'crypto' | 'equity'; + /** Venue / exchange id, e.g. 'kraken', 'uniswap-v3-base', 'alpaca'. */ + exchange?: string; +} + +/** + * Pick a cost model from what we know about the series. + * + * Resolution order: explicit DEX venue → known CEX venue → declared asset class + * → conservative default. Synthetic backtest venues ('yahoo', 'src', 'test') + * carry no fee information, so they fall through to the asset class, which is + * why `MarketSnapshot.assetClass` matters for pricing accuracy. + */ +export function costModelFor(hint: CostModelHint): CostModel { + const venue = hint.exchange?.toLowerCase(); + + if (venue && DEX_VENUES.has(venue)) return { ...DEX_COST_MODEL }; + + if (venue && venue in CEX_FEE_LOOKUP) { + return { ...DEFAULT_COST_MODEL, feeBps: toBps(cexTakerFee(venue)) }; + } + + if (hint.assetClass === 'equity') return { ...EQUITY_COST_MODEL }; + if (hint.assetClass === 'crypto') return { ...DEFAULT_COST_MODEL }; + + return { ...DEFAULT_COST_MODEL }; +} + +/** Venue ids we have a real published fee for. */ +const CEX_FEE_LOOKUP: Record = { + kraken: true, + 'binance-us': true, + coinbase: true, + gemini: true, +}; + +/** Derive a cost model from the first snapshot of a series. */ +export function costModelForSeries(snapshots: MarketSnapshot[]): CostModel { + const first = snapshots[0]; + if (!first) return { ...DEFAULT_COST_MODEL }; + return costModelFor({ assetClass: first.assetClass, exchange: first.exchange }); +} + +/** True when a snapshot carries a genuine two-sided quote rather than a synthesized close. */ +export function hasRealSpread(snap: MarketSnapshot): boolean { + return snap.bid > 0 && snap.ask > 0 && snap.ask > snap.bid; +} + +/** Mid price; falls back to whichever side is present. */ +export function midPrice(snap: MarketSnapshot): number { + if (snap.bid > 0 && snap.ask > 0) return (snap.bid + snap.ask) / 2; + return snap.bid || snap.ask || 0; +} + +/** + * The price a BUY actually fills at: the ask (real or synthesized from the + * assumed half-spread), widened by slippage. + */ +export function effectiveBuyPrice(snap: MarketSnapshot, costs: CostModel): number { + const base = hasRealSpread(snap) + ? snap.ask + : midPrice(snap) * (1 + costs.assumedHalfSpreadBps / 10_000); + return base * (1 + costs.slippageBps / 10_000); +} + +/** + * The price a SELL actually fills at: the bid (real or synthesized), reduced by + * slippage. + */ +export function effectiveSellPrice(snap: MarketSnapshot, costs: CostModel): number { + const base = hasRealSpread(snap) + ? snap.bid + : midPrice(snap) * (1 - costs.assumedHalfSpreadBps / 10_000); + return base * (1 - costs.slippageBps / 10_000); +} + +/** Fee charged on one leg of `notionalUsd`. */ +export function legFeeUsd(notionalUsd: number, costs: CostModel): number { + return notionalUsd * (costs.feeBps / 10_000) + costs.perOrderUsd; +} + +/** + * Total round-trip friction in bps of notional, assuming no price movement. + * The break-even hurdle: a strategy whose average winner is smaller than this + * loses money no matter how high its win rate looks. + * + * Two fee legs + two slippage legs + one full spread (half in, half out), plus + * the flat per-order cost expressed against `notionalUsd`. + */ +export function roundTripCostBps(costs: CostModel, notionalUsd = 100): number { + const flatBps = notionalUsd > 0 ? (costs.perOrderUsd * 2 / notionalUsd) * 10_000 : 0; + return costs.feeBps * 2 + costs.slippageBps * 2 + costs.assumedHalfSpreadBps * 2 + flatBps; +} + +/** Human-readable one-liner for CLI/UI headers. */ +export function describeCostModel(costs: CostModel, notionalUsd = 100): string { + const parts = [`${costs.feeBps.toFixed(1)}bp fee/leg`]; + if (costs.slippageBps > 0) parts.push(`${costs.slippageBps.toFixed(1)}bp slip/leg`); + if (costs.assumedHalfSpreadBps > 0) parts.push(`${costs.assumedHalfSpreadBps.toFixed(1)}bp half-spread`); + if (costs.perOrderUsd > 0) parts.push(`$${costs.perOrderUsd.toFixed(2)}/order`); + return `${parts.join(' · ')} → ${roundTripCostBps(costs, notionalUsd).toFixed(1)}bp round trip`; +} diff --git a/packages/source-strategies/src/index.ts b/packages/source-strategies/src/index.ts index 8544419..c2e9705 100644 --- a/packages/source-strategies/src/index.ts +++ b/packages/source-strategies/src/index.ts @@ -14,6 +14,7 @@ export { trendContinuation } from './trend-continuation.js'; export { meanReversion } from './mean-reversion.js'; export { breakout } from './breakout.js'; export * from './helpers.js'; +export * from './costs.js'; export * from './backtest.js'; export * as tsp from './osd/index.js'; diff --git a/packages/storage-supabase/package.json b/packages/storage-supabase/package.json index 4931d9a..fa09f09 100644 --- a/packages/storage-supabase/package.json +++ b/packages/storage-supabase/package.json @@ -17,7 +17,7 @@ }, "dependencies": { "@b1dz/core": "workspace:*", - "@supabase/supabase-js": "latest" + "@supabase/supabase-js": "^2.112.0" }, "devDependencies": { "@types/node": "latest", diff --git a/packages/strategy-registry/package.json b/packages/strategy-registry/package.json new file mode 100644 index 0000000..6045bba --- /dev/null +++ b/packages/strategy-registry/package.json @@ -0,0 +1,29 @@ +{ + "name": "@b1dz/strategy-registry", + "version": "0.3.10", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "build": "tsc -p tsconfig.build.json", + "lint": "eslint src", + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@b1dz/core": "workspace:*", + "@b1dz/source-strategies": "workspace:*", + "@b1dz/storage-supabase": "workspace:*", + "@b1dz/strategy-validation": "workspace:*", + "@supabase/supabase-js": "^2.112.0" + }, + "devDependencies": { + "@types/node": "latest", + "typescript": "6.0.3", + "vitest": "latest" + } +} diff --git a/packages/strategy-registry/src/debug.test.ts b/packages/strategy-registry/src/debug.test.ts new file mode 100644 index 0000000..023b0d0 --- /dev/null +++ b/packages/strategy-registry/src/debug.test.ts @@ -0,0 +1,25 @@ +import { describe, it, expect, vi } from 'vitest'; + +interface Mock { + select: ReturnType & ((...args: unknown[]) => Mock); + eq: ReturnType & ((...args: unknown[]) => Mock); + order: ReturnType & ((...args: unknown[]) => Mock); +} + +function makeMock(): Mock { + const self = {} as Mock; + self.select = vi.fn(() => self); + self.eq = vi.fn(() => self); + self.order = vi.fn(() => self); + return self; +} + +describe('debug', () => { + it('chains select -> eq -> order', () => { + const mock = makeMock(); + const result = mock.select('*').eq('user_id', 'u1').order('created_at', { ascending: false }); + expect(result).toBe(mock); + expect(mock.select).toHaveBeenCalledWith('*'); + expect(mock.eq).toHaveBeenCalledWith('user_id', 'u1'); + }); +}); diff --git a/packages/strategy-registry/src/index.ts b/packages/strategy-registry/src/index.ts new file mode 100644 index 0000000..d32c77f --- /dev/null +++ b/packages/strategy-registry/src/index.ts @@ -0,0 +1 @@ +export * from './registry.js'; diff --git a/packages/strategy-registry/src/registry.test.ts b/packages/strategy-registry/src/registry.test.ts new file mode 100644 index 0000000..bdce8a7 --- /dev/null +++ b/packages/strategy-registry/src/registry.test.ts @@ -0,0 +1,303 @@ +import { describe, it, expect, vi } from 'vitest'; +import type { SupabaseClient } from '@supabase/supabase-js'; +import type { GauntletReport } from '@b1dz/strategy-validation'; +import type { CostModel } from '@b1dz/source-strategies'; +import { tsp } from '@b1dz/source-strategies'; +import { + register, + listByUser, + listForwardRunning, + setListed, + setStatus, + insertForwardTrade, + closeForwardTrade, + forwardTradeHistory, + countOpenTrades, + type RegistryRow, + type ForwardTradeRow, +} from './registry.js'; + +const ZERO_COSTS: CostModel = { + feeBps: 0, + slippageBps: 0, + assumedHalfSpreadBps: 0, + perOrderUsd: 0, +}; + +function makeReport(overrides: Partial = {}): GauntletReport { + return { + passed: true, + candidateId: 'test-strat', + validationErrors: [], + inSampleGates: [], + outOfSampleGates: [], + walkForward: [], + deflatedSharpe: null, + robustness: null, + regimeCoverageResult: null, + duplicates: [], + inSampleSummary: null, + outOfSampleSummary: null, + costModel: ZERO_COSTS, + generatedAt: '2026-08-01T00:00:00Z', + advisoryGates: [], + ...overrides, + }; +} + +const SAMPLE_DOC: tsp.TradingStrategyDefinition = { + tsp: '0.1', + id: 'c1', + name: 'test', + assetClasses: ['crypto'], + definition: { + kind: 'rules', + rules: [], + }, +}; + +interface InnerQuery { + select: ReturnType; + insert: ReturnType; + update: ReturnType; + eq: ReturnType; + in: ReturnType; + order: ReturnType; + is: ReturnType; + single: ReturnType; +} + +function mockInner(): InnerQuery { + const self: InnerQuery = {} as InnerQuery; + self.select = vi.fn(() => self); + self.insert = vi.fn(() => self); + self.update = vi.fn(() => self); + self.eq = vi.fn(() => self); + self.in = vi.fn(() => self); + self.order = vi.fn(() => self); + self.is = vi.fn(() => self); + self.single = vi.fn().mockResolvedValue({ data: null, error: null }); + return self; +} + +function mockSupabase(inner: InnerQuery): SupabaseClient { + return { from: vi.fn(() => inner) } as unknown as SupabaseClient; +} + +// ── register ───────────────────────────────────────────────────────────────── + +describe('register', () => { + it('inserts a row and returns the entry', async () => { + const inner = mockInner(); + const row: RegistryRow = { + id: 'reg-1', + user_id: 'u1', + candidate_id: 'c1', + tsp_doc: SAMPLE_DOC, + compiled: true, + status: 'gauntlet_passed', + gauntlet_report: makeReport(), + cost_model: ZERO_COSTS, + listed_at: null, + rejected_at: null, + archived_at: null, + created_at: '2026-08-01T00:00:00Z', + }; + inner.single.mockResolvedValueOnce({ data: row, error: null }); + + const entry = await register( + mockSupabase(inner), 'u1', 'c1', + SAMPLE_DOC, makeReport(), ZERO_COSTS, + ); + + expect(inner.insert).toHaveBeenCalledWith({ + user_id: 'u1', + candidate_id: 'c1', + tsp_doc: SAMPLE_DOC, + compiled: true, + status: 'gauntlet_passed', + gauntlet_report: makeReport(), + cost_model: ZERO_COSTS, + }); + expect(entry.id).toBe('reg-1'); + expect(entry.status).toBe('gauntlet_passed'); + }); + + it('propagates errors', async () => { + const inner = mockInner(); + inner.single.mockResolvedValueOnce({ data: null, error: { message: 'dup key' } }); + await expect( + register(mockSupabase(inner), 'u1', 'c1', SAMPLE_DOC, makeReport(), ZERO_COSTS), + ).rejects.toEqual({ message: 'dup key' }); + }); +}); + +// ── listByUser ─────────────────────────────────────────────────────────────── + +describe('listByUser', () => { + it('lists entries for a user, newest first', async () => { + const rows: RegistryRow[] = [ + { id: 'r1', user_id: 'u1', candidate_id: 'c1', tsp_doc: SAMPLE_DOC, compiled: true, status: 'gauntlet_passed', gauntlet_report: makeReport(), cost_model: ZERO_COSTS, listed_at: null, rejected_at: null, archived_at: null, created_at: '2026-08-01T00:00:00Z' }, + { id: 'r2', user_id: 'u1', candidate_id: 'c2', tsp_doc: SAMPLE_DOC, compiled: true, status: 'listed', gauntlet_report: makeReport(), cost_model: ZERO_COSTS, listed_at: '2026-08-02T00:00:00Z', rejected_at: null, archived_at: null, created_at: '2026-08-02T00:00:00Z' }, + ]; + const inner = mockInner(); + inner.select.mockResolvedValueOnce({ data: rows, error: null }); + + const entries = await listByUser(mockSupabase(inner), 'u1'); + expect(inner.order).toHaveBeenCalledWith('created_at', { ascending: false }); + expect(entries).toHaveLength(2); + expect(entries[1]!.status).toBe('listed'); + }); + + it('filters by status when provided', async () => { + const inner = mockInner(); + inner.select.mockResolvedValueOnce({ data: [], error: null }); + + await listByUser(mockSupabase(inner), 'u1', 'forward_running'); + expect(inner.eq).toHaveBeenCalledWith('status', 'forward_running'); + }); +}); + +// ── listForwardRunning ─────────────────────────────────────────────────────── + +describe('listForwardRunning', () => { + it('selects strategies with status gauntlet_passed or forward_running', async () => { + const inner = mockInner(); + const rows: RegistryRow[] = [ + { id: 'r1', user_id: 'u1', candidate_id: 'c1', tsp_doc: SAMPLE_DOC, compiled: true, status: 'forward_running', gauntlet_report: makeReport(), cost_model: ZERO_COSTS, listed_at: null, rejected_at: null, archived_at: null, created_at: '2026-08-01T00:00:00Z' }, + ]; + inner.select.mockResolvedValueOnce({ data: rows, error: null }); + + const entries = await listForwardRunning(mockSupabase(inner)); + expect(inner.in).toHaveBeenCalledWith('status', ['gauntlet_passed', 'forward_running']); + expect(entries).toHaveLength(1); + }); +}); + +// ── setListed ──────────────────────────────────────────────────────────────── + +describe('setListed', () => { + it('updates status to listed and sets listed_at', async () => { + const inner = mockInner(); + inner.update.mockResolvedValueOnce({ data: null, error: null }); + + await setListed(mockSupabase(inner), 'reg-1'); + expect(inner.eq).toHaveBeenCalledWith('id', 'reg-1'); + const callArg = (inner.update as ReturnType).mock.calls[0][0] as Record; + expect(callArg.status).toBe('listed'); + expect(typeof callArg.listed_at).toBe('string'); + }); +}); + +// ── setStatus ──────────────────────────────────────────────────────────────── + +describe('setStatus', () => { + it('updates status to forward_running', async () => { + const inner = mockInner(); + inner.update.mockResolvedValueOnce({ data: null, error: null }); + + await setStatus(mockSupabase(inner), 'reg-1', 'forward_running'); + expect(inner.eq).toHaveBeenCalledWith('id', 'reg-1'); + const callArg = (inner.update as ReturnType).mock.calls[0][0] as Record; + expect(callArg.status).toBe('forward_running'); + }); + + it('updates status to min_trl_reached', async () => { + const inner = mockInner(); + inner.update.mockResolvedValueOnce({ data: null, error: null }); + + await setStatus(mockSupabase(inner), 'reg-1', 'min_trl_reached'); + const callArg = (inner.update as ReturnType).mock.calls[0][0] as Record; + expect(callArg.status).toBe('min_trl_reached'); + }); +}); + +// ── forward trades ─────────────────────────────────────────────────────────── + +describe('insertForwardTrade', () => { + it('inserts a trade with explicit entry_ts', async () => { + const inner = mockInner(); + const row: ForwardTradeRow = { + id: 'ft-1', strategy_id: 'reg-1', user_id: 'u1', + entry_ts: '2026-08-01T00:00:00.000Z', exit_ts: null, + trade_json: { profit: 5 }, regime_at_entry: 'trend', recorded_at: '2026-08-01T00:00:00.000Z', + }; + inner.single.mockResolvedValueOnce({ data: row, error: null }); + + const trade = await insertForwardTrade( + mockSupabase(inner), 'reg-1', 'u1', + '2026-08-01T00:00:00.000Z', + { profit: 5 }, + 'trend', + ); + expect(inner.insert).toHaveBeenCalledWith({ + strategy_id: 'reg-1', + user_id: 'u1', + entry_ts: '2026-08-01T00:00:00.000Z', + trade_json: { profit: 5 }, + regime_at_entry: 'trend', + }); + expect(trade.id).toBe('ft-1'); + expect(trade.exit_ts).toBeNull(); + }); +}); + +describe('closeForwardTrade', () => { + it('updates exit_ts and trade_json', async () => { + const inner = mockInner(); + const row: ForwardTradeRow = { + id: 'ft-1', strategy_id: 'reg-1', user_id: 'u1', + entry_ts: '2026-08-01T00:00:00.000Z', exit_ts: '2026-08-02T00:00:00.000Z', + trade_json: { profit: 5 }, + regime_at_entry: 'trend', + recorded_at: '2026-08-01T00:00:00.000Z', + }; + inner.single.mockResolvedValueOnce({ data: row, error: null }); + + const trade = await closeForwardTrade( + mockSupabase(inner), + 'ft-1', + '2026-08-02T00:00:00.000Z', + { profit: 5 }, + ); + const callArg = (inner.update as ReturnType).mock.calls[0][0] as Record; + expect(callArg.exit_ts).toBe('2026-08-02T00:00:00.000Z'); + expect(callArg.trade_json).toEqual({ profit: 5 }); + expect(trade.exit_ts).not.toBeNull(); + }); +}); + +describe('forwardTradeHistory', () => { + it('returns trades ordered by entry_ts ascending', async () => { + const inner = mockInner(); + const rows: ForwardTradeRow[] = [ + { id: 'ft-1', strategy_id: 'reg-1', user_id: 'u1', entry_ts: '2026-08-01T00:00:00Z', exit_ts: null, trade_json: {}, regime_at_entry: null, recorded_at: '2026-08-01T00:00:00Z' }, + { id: 'ft-2', strategy_id: 'reg-1', user_id: 'u1', entry_ts: '2026-08-02T00:00:00Z', exit_ts: null, trade_json: {}, regime_at_entry: null, recorded_at: '2026-08-02T00:00:00Z' }, + ]; + inner.select.mockResolvedValueOnce({ data: rows, error: null }); + + const trades = await forwardTradeHistory(mockSupabase(inner), 'reg-1'); + expect(inner.order).toHaveBeenCalledWith('entry_ts', { ascending: true }); + expect(trades).toHaveLength(2); + }); +}); + +describe('countOpenTrades', () => { + it('counts rows where exit_ts is null', async () => { + const inner = mockInner(); + inner.select.mockResolvedValueOnce({ count: 3, error: null }); + + const n = await countOpenTrades(mockSupabase(inner), 'reg-1'); + expect(inner.select).toHaveBeenCalledWith('*', { count: 'exact', head: true }); + expect(inner.is).toHaveBeenCalledWith('exit_ts', null); + expect(n).toBe(3); + }); + + it('returns 0 when count is null', async () => { + const inner = mockInner(); + inner.select.mockResolvedValueOnce({ count: null, error: null }); + + const n = await countOpenTrades(mockSupabase(inner), 'reg-1'); + expect(n).toBe(0); + }); +}); diff --git a/packages/strategy-registry/src/registry.ts b/packages/strategy-registry/src/registry.ts new file mode 100644 index 0000000..d397f46 --- /dev/null +++ b/packages/strategy-registry/src/registry.ts @@ -0,0 +1,180 @@ +import type { SupabaseClient } from '@supabase/supabase-js'; +import type { CostModel } from '@b1dz/source-strategies'; +import type { GauntletReport } from '@b1dz/strategy-validation'; +import { tsp } from '@b1dz/source-strategies'; + +export type RegistryStatus = 'gauntlet_passed' | 'forward_running' | 'min_trl_reached' | 'listed' | 'rejected' | 'archived'; + +export interface RegistryRow { + id: string; + user_id: string; + candidate_id: string; + tsp_doc: tsp.TradingStrategyDefinition; + compiled: boolean; + status: RegistryStatus; + gauntlet_report: GauntletReport; + cost_model: CostModel; + listed_at: string | null; + rejected_at: string | null; + archived_at: string | null; + created_at: string; +} + +export interface ForwardTradeRow { + id: string; + strategy_id: string; + user_id: string; + entry_ts: string; + exit_ts: string | null; + trade_json: Record; + regime_at_entry: string | null; + recorded_at: string; +} + +export async function register( + supabase: SupabaseClient, + userId: string, + candidateId: string, + tspDoc: tsp.TradingStrategyDefinition, + gauntletReport: GauntletReport, + costModel: CostModel, +): Promise { + const { data, error } = await supabase.from('strategy_registry').insert({ + user_id: userId, + candidate_id: candidateId, + tsp_doc: tspDoc, + compiled: true, + status: 'gauntlet_passed', + gauntlet_report: gauntletReport, + cost_model: costModel, + }).select().single(); + if (error) throw error; + return data as RegistryRow; +} + +export async function listByUser( + supabase: SupabaseClient, + userId: string, + status?: string, +): Promise { + let q = supabase + .from('strategy_registry') + .select('*') + .eq('user_id', userId) + .order('created_at', { ascending: false }); + + if (status) q = q.eq('status', status); + + const { data, error } = await q; + if (error) throw error; + return data as RegistryRow[]; +} + +export async function listForwardRunning( + supabase: SupabaseClient, +): Promise { + const { data, error } = await supabase + .from('strategy_registry') + .select('*') + .in('status', ['gauntlet_passed', 'forward_running']); + + if (error) throw error; + return data as RegistryRow[]; +} + +export async function setStatus( + supabase: SupabaseClient, + strategyId: string, + status: string, +): Promise { + const { error } = await supabase + .from('strategy_registry') + .update({ status }) + .eq('id', strategyId); + + if (error) throw error; +} + +export async function setListed( + supabase: SupabaseClient, + strategyId: string, +): Promise { + const { error } = await supabase + .from('strategy_registry') + .update({ status: 'listed', listed_at: new Date().toISOString() }) + .eq('id', strategyId); + + if (error) throw error; +} + +export async function insertForwardTrade( + supabase: SupabaseClient, + strategyId: string, + userId: string, + entryTs: string, + trade: Record, + regimeAtEntry?: string, +): Promise { + const { data, error } = await supabase + .from('forward_trades') + .insert({ + strategy_id: strategyId, + user_id: userId, + entry_ts: entryTs, + trade_json: trade, + regime_at_entry: regimeAtEntry ?? null, + }) + .select() + .single(); + + if (error) throw error; + return data as ForwardTradeRow; +} + +export async function closeForwardTrade( + supabase: SupabaseClient, + tradeId: string, + exitTs: string, + updatedTrade: Record, +): Promise { + const { data, error } = await supabase + .from('forward_trades') + .update({ + exit_ts: exitTs, + trade_json: updatedTrade, + }) + .eq('id', tradeId) + .select() + .single(); + + if (error) throw error; + return data as ForwardTradeRow; +} + +export async function forwardTradeHistory( + supabase: SupabaseClient, + strategyId: string, +): Promise { + const { data, error } = await supabase + .from('forward_trades') + .select('*') + .eq('strategy_id', strategyId) + .order('entry_ts', { ascending: true }); + + if (error) throw error; + return data as ForwardTradeRow[]; +} + +export async function countOpenTrades( + supabase: SupabaseClient, + strategyId: string, +): Promise { + const { count, error } = await supabase + .from('forward_trades') + .select('*', { count: 'exact', head: true }) + .eq('strategy_id', strategyId) + .is('exit_ts', null); + + if (error) throw error; + return count ?? 0; +} diff --git a/packages/strategy-registry/tsconfig.build.json b/packages/strategy-registry/tsconfig.build.json new file mode 100644 index 0000000..2e725b7 --- /dev/null +++ b/packages/strategy-registry/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src", + "noEmit": false + }, + "exclude": ["**/*.test.ts"] +} diff --git a/packages/strategy-registry/tsconfig.json b/packages/strategy-registry/tsconfig.json new file mode 100644 index 0000000..bf5a36d --- /dev/null +++ b/packages/strategy-registry/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src/**/*"] +} diff --git a/packages/strategy-validation/package.json b/packages/strategy-validation/package.json new file mode 100644 index 0000000..9fc9e90 --- /dev/null +++ b/packages/strategy-validation/package.json @@ -0,0 +1,26 @@ +{ + "name": "@b1dz/strategy-validation", + "version": "0.3.10", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "build": "tsc -p tsconfig.build.json", + "lint": "eslint src", + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@b1dz/core": "workspace:*", + "@b1dz/source-strategies": "workspace:*" + }, + "devDependencies": { + "@types/node": "latest", + "typescript": "6.0.3", + "vitest": "latest" + } +} diff --git a/packages/strategy-validation/src/correlation.test.ts b/packages/strategy-validation/src/correlation.test.ts new file mode 100644 index 0000000..8bb44bc --- /dev/null +++ b/packages/strategy-validation/src/correlation.test.ts @@ -0,0 +1,218 @@ +import { describe, it, expect } from 'vitest'; +import type { StrategyPlugin, MarketSnapshot } from '@b1dz/core'; +import { snapshotsFrom, syntheticTrades } from './synthetic.js'; +import { + pearson, + signalVector, + signalCorrelation, + returnCorrelation, + findDuplicates, +} from './correlation.js'; + +describe('pearson', () => { + it('is 1 for perfect positive correlation', () => { + expect(pearson([1, 2, 3], [1, 2, 3])).toBeCloseTo(1, 10); + expect(pearson([5, 10], [10, 20])).toBeCloseTo(1, 10); + }); + + it('is -1 for perfect negative correlation', () => { + expect(pearson([1, 2, 3], [-1, -2, -3])).toBeCloseTo(-1, 10); + }); + + it('is 0 for orthogonal vectors', () => { + expect(pearson([1, 0, -1], [-1, 2, -1])).toBeCloseTo(0, 12); + }); + + it('returns 0 for degenerate cases', () => { + expect(pearson([], [1, 2])).toBe(0); + expect(pearson([1, 2], [])).toBe(0); + expect(pearson([3, 3, 3], [3, 3, 3])).toBe(0); + expect(pearson([1], [1])).toBe(0); + }); + + it('returns 0 when any observation is non-finite', () => { + expect(pearson([1, NaN, 3], [1, 2, 3])).toBe(0); + expect(pearson([1, 2, 3], [1, Infinity, 3])).toBe(0); + }); +}); + +describe('signalVector', () => { + const makePlugin = (signals: (number | null)[]): StrategyPlugin => ({ + manifest: { id: 's', kind: 'strategy', version: '0', name: 'S', capabilities: [] }, + evaluate(_snap, history) { + const s = signals[history.length]; + if (s === null || s === undefined) return null; + return s > 0 + ? { side: 'buy' as const, strength: 1, reason: 'b' } + : { side: 'sell' as const, strength: 1, reason: 's' }; + }, + }); + + it('encodes buy as +1, sell as -1, no signal as 0', () => { + const snaps = snapshotsFrom([100, 100, 100, 100]); + const vec = signalVector(makePlugin([null, 1, -1, null]), snaps); + expect(vec).toEqual([0, 1, -1, 0]); + }); + + it('treats a throwing evaluate as no-signal', () => { + const boom: StrategyPlugin = { + manifest: { id: 'b', kind: 'strategy', version: '0', name: 'B', capabilities: [] }, + evaluate() { + throw new Error('boom'); + }, + }; + const snaps = snapshotsFrom([100]); + expect(signalVector(boom, snaps)).toEqual([0]); + }); + + it('returns all zeros when the plugin always returns null', () => { + const silent: StrategyPlugin = { + manifest: { id: 'z', kind: 'strategy', version: '0', name: 'Z', capabilities: [] }, + evaluate() { + return null; + }, + }; + const snaps = snapshotsFrom([100, 101, 102]); + expect(signalVector(silent, snaps)).toEqual([0, 0, 0]); + }); +}); + +describe('signalCorrelation', () => { + const snap = (price: number, ts: number): MarketSnapshot => ({ + exchange: 'test', + pair: 'X', + bid: price, + ask: price, + bidSize: 1, + askSize: 1, + ts, + assetClass: undefined, + }); + + it('is 1 for identical strategies', () => { + const snaps: MarketSnapshot[] = [snap(100, 0), snap(101, 1), snap(102, 2)]; + const makeBuyer = (): StrategyPlugin => ({ + manifest: { id: 'a', kind: 'strategy', version: '0', name: 'A', capabilities: [] }, + evaluate(_s, h) { + return h.length < 2 ? { side: 'buy' as const, strength: 1, reason: '' } : null; + }, + }); + expect(signalCorrelation(makeBuyer(), makeBuyer(), snaps)).toBeCloseTo(1, 10); + }); + + it('is -1 for opposite strategies', () => { + // Need varying signals so variance is non-zero (pearson returns 0 for constant vectors). + const snaps: MarketSnapshot[] = [snap(100, 0), snap(101, 1), snap(102, 2), snap(103, 3)]; + const pA: StrategyPlugin = { + manifest: { id: 'a', kind: 'strategy', version: '0', name: 'A', capabilities: [] }, + evaluate(_s, h) { + return h.length % 2 === 0 + ? { side: 'buy' as const, strength: 1, reason: '' } + : null; + }, + }; + const pB: StrategyPlugin = { + manifest: { id: 'b', kind: 'strategy', version: '0', name: 'B', capabilities: [] }, + evaluate(_s, h) { + return h.length % 2 === 0 + ? { side: 'sell' as const, strength: 1, reason: '' } + : null; + }, + }; + // pA: [1,0,1,0], pB: [-1,0,-1,0] → correlation = -1 + expect(signalCorrelation(pA, pB, snaps)).toBeCloseTo(-1, 10); + }); + + it('returns 0 when both are permanently silent', () => { + const snaps: MarketSnapshot[] = [snap(100, 0)]; + const silent: StrategyPlugin = { + manifest: { id: 's', kind: 'strategy', version: '0', name: 'S', capabilities: [] }, + evaluate() { + return null; + }, + }; + // All zeros → zero variance → pearson returns 0 + expect(signalCorrelation(silent, silent, snaps)).toBe(0); + }); +}); + +describe('returnCorrelation', () => { + it('is 1 for identical P&L streams', () => { + const trades = syntheticTrades([0.1, -0.05, 0.2], { + startTs: 0, + stepMs: 7 * 24 * 3600 * 1000, + }); + expect(returnCorrelation(trades, trades, 0, Date.now())).toBeCloseTo(1, 10); + }); + + it('returns 0 when one side has no trades', () => { + const trades = syntheticTrades([0.1], { startTs: 0 }); + expect(returnCorrelation(trades, [], 0, Date.now())).toBe(0); + }); +}); + +describe('findDuplicates', () => { + const snap = (price: number, ts: number): MarketSnapshot => ({ + exchange: 'test', + pair: 'X', + bid: price, + ask: price, + bidSize: 1, + askSize: 1, + ts, + assetClass: undefined, + }); + + it('returns empty when catalog is empty', () => { + const snaps: MarketSnapshot[] = [snap(100, 0)]; + const plugin: StrategyPlugin = { + manifest: { id: 'x', kind: 'strategy', version: '0', name: 'X', capabilities: [] }, + evaluate() { + return null; + }, + }; + expect(findDuplicates(plugin, [], [], snaps)).toEqual([]); + }); + + it('returns empty when snapshots are empty', () => { + const plugin: StrategyPlugin = { + manifest: { id: 'x', kind: 'strategy', version: '0', name: 'X', capabilities: [] }, + evaluate() { + return null; + }, + }; + expect(findDuplicates(plugin, [], [], [])).toEqual([]); + }); + + it('finds a catalog entry with high signal + return correlation', () => { + const WEEK_MS = 7 * 24 * 3600 * 1000; + // 5 snapshots spanning 4 weeks so we get ≥2 buckets with both trades in range. + const snaps: MarketSnapshot[] = [ + snap(100, 0), + snap(101, WEEK_MS), + snap(102, 2 * WEEK_MS), + snap(103, 3 * WEEK_MS), + snap(104, 4 * WEEK_MS), + ]; + const makeAlternating = (): StrategyPlugin => ({ + manifest: { id: 'dup', kind: 'strategy', version: '0', name: 'Dup', capabilities: [] }, + evaluate(_s, h) { + return h.length % 2 === 0 + ? { side: 'buy' as const, strength: 1, reason: '' } + : null; + }, + }); + const plugin = makeAlternating(); + const trades = syntheticTrades([0.01, 0.02], { + startTs: 0, + stepMs: WEEK_MS, + }); + const catalog = [{ plugin: makeAlternating(), trades, id: 'existing' }]; + const result = findDuplicates(plugin, trades, catalog, snaps, 0.8); + expect(result.length).toBeGreaterThanOrEqual(1); + if (result.length > 0) { + expect(result[0]!.strategyId).toBe('existing'); + expect(result[0]!.signal).toBeCloseTo(1, 10); + } + }); +}); diff --git a/packages/strategy-validation/src/correlation.ts b/packages/strategy-validation/src/correlation.ts new file mode 100644 index 0000000..f49d994 --- /dev/null +++ b/packages/strategy-validation/src/correlation.ts @@ -0,0 +1,199 @@ +/** + * Strategy correlation — catching a catalogue full of near-identical copies. + * + * WHY THIS EXISTS + * + * A generator that sweeps RSI periods 2..50 with fixed thresholds produces 49 + * strategies. Most of them are reskins: the per-bar signal vectors correlate + * > 0.9, the P&L streams move together, and listing all 49 fills pages with + * duplicates. A buyer scrolling past ten names that all behave identically + * learns that the store's curation is cosmetic, and a buyer who buys three of + * them finds they all blow up on the same day. + * + * This module measures per-bar signal alignment (period-2 RSI is faster to fire + * but pulls the same trigger) and per-bucket return alignment (they make and + * lose money on the same trades). Either metric alone can miss reskins: + * signal correlation misses strategies that agree on direction but differ in + * sizing, and return correlation can be high by chance on a short series. + * Together they give a usable pairwise distance. + * + * THE FAILURE MODE + * + * Pearson's r on raw signal vectors over 2,000 bars looks significant at + * anything > 0.04, so a findDuplicates() threshold of 0.8 is not "p < 0.05" + * loose — it is genuinely tight, and a pair above it IS functionally identical. + * But on a 30-trade P&L stream, 0.8 can happen from a single shared windfall + * week. The combined gate is the only safe one. + */ +import type { MarketSnapshot, StrategyPlugin } from '@b1dz/core'; +import type { BacktestTrade } from '@b1dz/source-strategies'; + +/** + * Pearson correlation coefficient. + * + * Degenerate cases: + * - vectors of length < 2 → 0 (undefined) + * - standard deviation is zero → 0 (undefined — every point = mean, and r + * measures linear deviation) + * - a non-finite observation leaks through → 0 (a bad tick is not a signal) + */ +export function pearson(a: number[], b: number[]): number { + const n = Math.min(a.length, b.length); + if (n < 2) return 0; + + let ma = 0; + let mb = 0; + for (let i = 0; i < n; i++) { + ma += a[i]!; + mb += b[i]!; + } + ma /= n; + mb /= n; + + let cov = 0; + let va = 0; + let vb = 0; + for (let i = 0; i < n; i++) { + const da = a[i]! - ma; + const db = b[i]! - mb; + if (!Number.isFinite(da) || !Number.isFinite(db)) return 0; + cov += da * db; + va += da * da; + vb += db * db; + } + if (!(va > 0) || !(vb > 0)) return 0; + return cov / Math.sqrt(va * vb); +} + +/** + * Signal vector: per-bar direction implied by the plugin's evaluate() output. + * + * +1 = buy (or a trailing hold-buy) + * −1 = sell + * 0 = no signal (or error — a broken evaluate() is "no position" here) + * + * Two strategies that agree on direction will correlate, even if one acts + * earlier (shorter indicators). This is the tighter of the two filters for + * period-sweep clones: the Pearson of binary vectors still catches near- + * identical timing. + */ +export function signalVector(plugin: StrategyPlugin, snapshots: MarketSnapshot[]): number[] { + const out: number[] = []; + for (let i = 0; i < snapshots.length; i++) { + const snap = snapshots[i]!; + const history = snapshots.slice(0, i); + let signal = null; + try { + signal = plugin.evaluate(snap, history); + } catch { + signal = null; + } + if (!signal) { + out.push(0); + continue; + } + out.push(signal.side === 'buy' ? 1 : -1); + } + return out; +} + +/** + * Signal alignment between two plugins over the same snapshots. + * + * Range [−1, 1]. Degenerate if either strategy throws on > half the bars or is + * permanently silent — that returns 0, not NaN, because NaN passes every > check + * and a catalog correlation gate of `r < 0.8` would let NaN through silently. + */ +export function signalCorrelation( + pluginA: StrategyPlugin, + pluginB: StrategyPlugin, + snapshots: MarketSnapshot[], +): number { + return pearson(signalVector(pluginA, snapshots), signalVector(pluginB, snapshots)); +} + +/** + * Bucket nominal profit (roughly proceeds − cost, not the compounded netMultiple + * curve) into uniform time windows so return correlation is per-period rather + * than per-bar — a per-bar correlation on 2,000 days with maybe 50 return + * observations would be dominated by zeros and artificially low. + * + * Bucket edges are aligned to the first bar's timestamp so every strategy run + * over the same series gets the same buckets; the correlation is then + * comparable across runs. + */ +function bucketReturns( + trades: BacktestTrade[], + startTs: number, + endTs: number, + bucketMs: number, +): number[] { + if (bucketMs <= 0 || startTs >= endTs) return []; + const n = Math.ceil((endTs - startTs) / bucketMs); + const out = new Array(n).fill(0); + for (const t of trades) { + const bucket = Math.floor((t.exitTs - startTs) / bucketMs); + if (bucket < 0 || bucket >= n) continue; + out[bucket]! += t.profit; + } + return out; +} + +/** + * Pearson over binned nominal P&L. + * + * Default bucket is one calendar week (7 days × 86400 s × 1000 ms) — tight + * enough to catch strategies that consistently exploit the same pattern, + * loose enough that the noise of which exact bar they entered on doesn't + * dominate the correlation. + */ +export function returnCorrelation( + tradesA: BacktestTrade[], + tradesB: BacktestTrade[], + startTs: number, + endTs: number, + bucketMs = 7 * 24 * 60 * 60 * 1000, +): number { + return pearson( + bucketReturns(tradesA, startTs, endTs, bucketMs), + bucketReturns(tradesB, startTs, endTs, bucketMs), + ); +} + +/** + * Pair of minimised comparisons for one candidate against a single catalogue + * entry — per-bar signal correlation and per-week return correlation. + */ +export interface CorrelationPair { + signal: number; + /** Signal correlation over the per-bar vectors. */ + return: number; + /** Per-bucket return Pearson. */ + strategyId: string; +} + +/** + * Find catalogue entries whose signal AND return correlations both exceed + * `threshold`, in either direction. A pair where both metrics clear the + * threshold is a duplicate and shouldn't be listed separately. + */ +export function findDuplicates( + candidatePlugin: StrategyPlugin, + candidateTrades: BacktestTrade[], + catalog: { plugin: StrategyPlugin; trades: BacktestTrade[]; id: string }[], + snapshots: MarketSnapshot[], + threshold = 0.8, +): CorrelationPair[] { + if (snapshots.length === 0) return []; + const startTs = snapshots[0]!.ts; + const endTs = snapshots[snapshots.length - 1]!.ts; + const out: CorrelationPair[] = []; + for (const entry of catalog) { + const sig = signalCorrelation(candidatePlugin, entry.plugin, snapshots); + const ret = returnCorrelation(candidateTrades, entry.trades, startTs, endTs); + if (sig > threshold && ret > threshold) { + out.push({ signal: sig, return: ret, strategyId: entry.id }); + } + } + return out; +} diff --git a/packages/strategy-validation/src/deflated-sharpe.test.ts b/packages/strategy-validation/src/deflated-sharpe.test.ts new file mode 100644 index 0000000..375e8d1 --- /dev/null +++ b/packages/strategy-validation/src/deflated-sharpe.test.ts @@ -0,0 +1,551 @@ +import { describe, it, expect } from 'vitest'; +import { + EULER_MASCHERONI, + annualizeSharpe, + deannualizeSharpe, + deflatedSharpeRatio, + expectedMaxSharpe, + expectedMaxStandardNormal, + minimumTrackRecordLength, + normalCdf, + normalPdf, + normalPpf, + nullVarianceOfTrialSharpes, + probabilisticSharpeRatio, + sharpeStandardError, + sharpeVarianceFactor, +} from './deflated-sharpe.js'; + +describe('normalCdf', () => { + it('matches published values of the standard normal CDF', () => { + // Abramowitz & Stegun Table 26.1 / any statistics table. + expect(normalCdf(0)).toBe(0.5); + expect(normalCdf(0.5)).toBeCloseTo(0.6914624613, 10); + expect(normalCdf(1)).toBeCloseTo(0.8413447461, 10); + expect(normalCdf(1.6448536270)).toBeCloseTo(0.95, 10); + expect(normalCdf(1.96)).toBeCloseTo(0.9750021049, 10); + expect(normalCdf(2)).toBeCloseTo(0.977249868, 9); + expect(normalCdf(2.5758293035)).toBeCloseTo(0.995, 10); + expect(normalCdf(3)).toBeCloseTo(0.998650102, 9); + expect(normalCdf(3.0902323062)).toBeCloseTo(0.999, 10); + }); + + it('is accurate in the far tails, where significance decisions are made', () => { + // Compared as RELATIVE error: an absolute tolerance is meaningless against a + // number of order 1e-16. + expect(normalCdf(-5) / 2.866515719e-7).toBeCloseTo(1, 9); + expect(normalCdf(-8) / 6.220960574e-16).toBeCloseTo(1, 7); + expect(normalCdf(-3.0902323062) / 0.001).toBeCloseTo(1, 9); + }); + + it('is exactly symmetric', () => { + for (let i = -400; i <= 400; i += 7) { + const x = i / 100; + expect(normalCdf(x) + normalCdf(-x)).toBe(1); + } + }); + + it('saturates without overflowing', () => { + expect(normalCdf(40)).toBe(1); + expect(normalCdf(-40)).toBe(0); + expect(normalCdf(Number.POSITIVE_INFINITY)).toBe(1); + expect(normalCdf(Number.NEGATIVE_INFINITY)).toBe(0); + expect(normalCdf(Number.NaN)).toBe(0.5); + }); + + it('is bounded to [0, 1] for every input', () => { + for (let i = -2000; i <= 2000; i += 13) { + const p = normalCdf(i / 100); + expect(p).toBeGreaterThanOrEqual(0); + expect(p).toBeLessThanOrEqual(1); + } + }); +}); + +describe('normalPdf', () => { + it('matches the closed form at known points', () => { + expect(normalPdf(0)).toBeCloseTo(1 / Math.sqrt(2 * Math.PI), 12); + expect(normalPdf(1)).toBeCloseTo(0.2419707245, 10); + expect(normalPdf(-1)).toBeCloseTo(0.2419707245, 10); + }); +}); + +describe('normalPpf', () => { + it('matches published normal quantiles', () => { + expect(normalPpf(0.5)).toBeCloseTo(0, 9); + expect(normalPpf(0.75)).toBeCloseTo(0.6744897502, 8); + expect(normalPpf(0.9)).toBeCloseTo(1.2815515655, 8); + expect(normalPpf(0.95)).toBeCloseTo(1.644853627, 8); + expect(normalPpf(0.975)).toBeCloseTo(1.9599639845, 8); + expect(normalPpf(0.99)).toBeCloseTo(2.326347874, 8); + expect(normalPpf(0.995)).toBeCloseTo(2.5758293035, 8); + expect(normalPpf(0.999)).toBeCloseTo(3.0902323062, 8); + expect(normalPpf(0.0001)).toBeCloseTo(-3.7190164854, 7); + }); + + it('inverts normalCdf to within the approximation error', () => { + for (let i = -450; i <= 450; i += 9) { + const x = i / 100; + expect(normalPpf(normalCdf(x))).toBeCloseTo(x, 6); + } + }); + + it('is inverted by normalCdf across the probability range', () => { + for (let i = 1; i < 1000; i += 3) { + const p = i / 1000; + expect(normalCdf(normalPpf(p))).toBeCloseTo(p, 8); + } + }); + + it('clamps instead of returning +/-Infinity at the boundaries', () => { + // An Infinity here would flow into the expected-max Sharpe, then into a + // NaN comparison, then into a blocking gate that always passes. + expect(Number.isFinite(normalPpf(0))).toBe(true); + expect(Number.isFinite(normalPpf(1))).toBe(true); + expect(normalPpf(0)).toBeLessThan(-8); + expect(normalPpf(1)).toBeGreaterThan(8); + expect(normalPpf(-5)).toBe(normalPpf(0)); + expect(normalPpf(17)).toBe(normalPpf(1)); + expect(normalPpf(Number.NaN)).toBe(0); + }); +}); + +describe('sharpeVarianceFactor', () => { + it('collapses to the gaussian 1 + SR^2/2 when skew=0 and kurtosis=3', () => { + // This is the arithmetic proof that `kurtosis` is the NON-EXCESS convention. + expect(sharpeVarianceFactor(0, 0, 3)).toBeCloseTo(1, 12); + expect(sharpeVarianceFactor(1, 0, 3)).toBeCloseTo(1.5, 12); + expect(sharpeVarianceFactor(2, 0, 3)).toBeCloseTo(3, 12); + }); + + it('inflates the standard error for negative skew and fat tails', () => { + const gaussian = sharpeVarianceFactor(0.5, 0, 3); + expect(sharpeVarianceFactor(0.5, -1.5, 3)).toBeGreaterThan(gaussian); // left tail + expect(sharpeVarianceFactor(0.5, 0, 12)).toBeGreaterThan(gaussian); // fat tails + // ...so the same observed Sharpe is LESS significant once shape is accounted for. + const clean = probabilisticSharpeRatio({ observedSharpe: 0.5, nObservations: 40 }); + const ugly = probabilisticSharpeRatio({ + observedSharpe: 0.5, + nObservations: 40, + skewness: -1.5, + kurtosis: 12, + }); + expect(ugly).toBeLessThan(clean); + }); + + it('retreats to the gaussian factor when sample moments are infeasible', () => { + // skew 3 at SR 1 gives 1 - 3 + 0.5 = -1.5, impossible for a real + // distribution; an epsilon floor here would manufacture an infinite z-score. + expect(sharpeVarianceFactor(1, 3, 3)).toBe(sharpeVarianceFactor(1, 0, 3)); + expect(Number.isFinite(sharpeVarianceFactor(1, 3, 3))).toBe(true); + expect(sharpeVarianceFactor(1, Number.NaN, Number.NaN)).toBe(1.5); + }); +}); + +describe('sharpeStandardError', () => { + it('is ~1/sqrt(n) for a zero Sharpe, the classic result', () => { + expect(sharpeStandardError(0, 251, 0, 3)).toBeCloseTo(1 / Math.sqrt(250), 12); + }); + + it('shrinks with sqrt of the sample size', () => { + const a = sharpeStandardError(0.2, 101); + const b = sharpeStandardError(0.2, 401); + expect(a / b).toBeCloseTo(2, 6); + }); + + it('is 0 for samples too small to have a sampling distribution', () => { + expect(sharpeStandardError(1, 1)).toBe(0); + expect(sharpeStandardError(1, 0)).toBe(0); + }); +}); + +describe('probabilisticSharpeRatio', () => { + it('is exactly 0.5 when the observed Sharpe equals the benchmark', () => { + expect( + probabilisticSharpeRatio({ observedSharpe: 0.3, benchmarkSharpe: 0.3, nObservations: 100 }), + ).toBe(0.5); + }); + + it('matches a hand-computed z-score', () => { + // SR 0.2, n 101, gaussian: se = sqrt((1 + 0.02)/100) = 0.1009950 + // z = 0.2/0.1009950 = 1.980295 → Phi(z) = 0.976164 + const se = Math.sqrt((1 + 0.2 * 0.2 / 2) / 100); + expect(sharpeStandardError(0.2, 101)).toBeCloseTo(se, 12); + expect(probabilisticSharpeRatio({ observedSharpe: 0.2, nObservations: 101 })).toBeCloseTo( + normalCdf(0.2 / se), + 12, + ); + expect(probabilisticSharpeRatio({ observedSharpe: 0.2, nObservations: 101 })).toBeCloseTo( + 0.9761648, + 6, + ); + }); + + it('rises with sample size for the same observed Sharpe', () => { + const short = probabilisticSharpeRatio({ observedSharpe: 0.15, nObservations: 20 }); + const long = probabilisticSharpeRatio({ observedSharpe: 0.15, nObservations: 400 }); + expect(short).toBeLessThan(long); + expect(short).toBeLessThan(0.95); + expect(long).toBeGreaterThan(0.99); + }); + + it('falls below 0.5 when the observed Sharpe is under the benchmark', () => { + expect( + probabilisticSharpeRatio({ observedSharpe: 0.1, benchmarkSharpe: 0.3, nObservations: 100 }), + ).toBeLessThan(0.5); + }); + + it('returns 0 (no evidence) rather than NaN on degenerate input', () => { + expect(probabilisticSharpeRatio({ observedSharpe: 2, nObservations: 1 })).toBe(0); + expect(probabilisticSharpeRatio({ observedSharpe: 2, nObservations: 0 })).toBe(0); + expect(probabilisticSharpeRatio({ observedSharpe: Number.NaN, nObservations: 100 })).toBe(0); + expect( + probabilisticSharpeRatio({ + observedSharpe: 1, + benchmarkSharpe: Number.POSITIVE_INFINITY, + nObservations: 100, + }), + ).toBe(0); + }); +}); + +describe('expectedMaxStandardNormal', () => { + it('approximates the true expected maximum of N standard normals', () => { + // True values (Monte Carlo / order statistics): N=10 → 1.539, N=100 → 2.508, + // N=1000 → 3.241. The Gumbel approximation runs ~1-2% high, as documented. + expect(expectedMaxStandardNormal(10)).toBeCloseTo(1.5746, 3); + expect(expectedMaxStandardNormal(100)).toBeCloseTo(2.5306, 3); + expect(expectedMaxStandardNormal(1000)).toBeCloseTo(3.2551, 3); + expect(expectedMaxStandardNormal(10)).toBeGreaterThan(1.5); + expect(expectedMaxStandardNormal(10)).toBeLessThan(1.62); + expect(expectedMaxStandardNormal(1000)).toBeGreaterThan(3.2); + expect(expectedMaxStandardNormal(1000)).toBeLessThan(3.32); + }); + + it('grows like sqrt(2 ln N) — without bound, but only logarithmically', () => { + // The sqrt(2 ln N) asymptote is approached from below, so the ratio must be + // under 1 everywhere and must climb monotonically toward it. + let prevRatio = 0; + for (const n of [10, 100, 1000, 10_000, 1e6, 1e9]) { + const ratio = expectedMaxStandardNormal(n) / Math.sqrt(2 * Math.log(n)); + expect(ratio).toBeGreaterThan(0.7); + expect(ratio).toBeLessThan(1); + expect(ratio).toBeGreaterThan(prevRatio); + prevRatio = ratio; + } + // Doubling the search buys very little extra luck... + const gain = expectedMaxStandardNormal(2000) - expectedMaxStandardNormal(1000); + expect(gain).toBeGreaterThan(0); + expect(gain).toBeLessThan(0.2); + // ...but it never stops growing. + expect(expectedMaxStandardNormal(1e12)).toBeGreaterThan(expectedMaxStandardNormal(1e6)); + }); + + it('is monotonically increasing in N', () => { + let prev = 0; + for (const n of [2, 3, 5, 10, 50, 100, 500, 1000, 5000, 100_000]) { + const e = expectedMaxStandardNormal(n); + expect(e).toBeGreaterThan(prev); + prev = e; + } + }); + + it('is 0 for a single trial (no selection, no selection bias)', () => { + expect(expectedMaxStandardNormal(1)).toBe(0); + expect(expectedMaxStandardNormal(0)).toBe(0); + expect(expectedMaxStandardNormal(-5)).toBe(0); + expect(expectedMaxStandardNormal(Number.POSITIVE_INFINITY)).toBe(0); + expect(expectedMaxStandardNormal(Number.NaN)).toBe(0); + }); + + it('uses the Euler-Mascheroni weighting from Bailey & Lopez de Prado (2014)', () => { + const n = 500; + const expected = + (1 - EULER_MASCHERONI) * normalPpf(1 - 1 / n) + + EULER_MASCHERONI * normalPpf(1 - 1 / (n * Math.E)); + expect(expectedMaxStandardNormal(n)).toBeCloseTo(expected, 12); + }); +}); + +describe('expectedMaxSharpe', () => { + it('scales with the standard deviation of the trial Sharpes', () => { + const base = expectedMaxSharpe(1000, 1); + expect(expectedMaxSharpe(1000, 4)).toBeCloseTo(base * 2, 10); + expect(expectedMaxSharpe(1000, 0.25)).toBeCloseTo(base / 2, 10); + }); + + it('quantifies the noise bar: 10,000 trials on 250 observations', () => { + // Per-observation bar... + const bar = expectedMaxSharpe(10_000, nullVarianceOfTrialSharpes(250)); + expect(bar).toBeCloseTo(3.8607 / Math.sqrt(250), 4); + // ...which is an ANNUALIZED Sharpe of ~3.9 for daily data, from pure noise. + expect(annualizeSharpe(bar, 252)).toBeGreaterThan(3.8); + expect(annualizeSharpe(bar, 252)).toBeLessThan(4.0); + }); + + it('is 0 when there is nothing to correct for', () => { + expect(expectedMaxSharpe(1, 0.01)).toBe(0); + expect(expectedMaxSharpe(1000, 0)).toBe(0); + expect(expectedMaxSharpe(1000, -1)).toBe(0); + expect(expectedMaxSharpe(1000, Number.NaN)).toBe(0); + }); +}); + +describe('nullVarianceOfTrialSharpes', () => { + it('is 1/n, the sampling variance of a zero-edge Sharpe estimate', () => { + expect(nullVarianceOfTrialSharpes(250)).toBeCloseTo(0.004, 12); + expect(Math.sqrt(nullVarianceOfTrialSharpes(100))).toBeCloseTo(0.1, 12); + }); + + it('is 0 for a non-existent sample', () => { + expect(nullVarianceOfTrialSharpes(0)).toBe(0); + expect(nullVarianceOfTrialSharpes(-3)).toBe(0); + }); +}); + +describe('deannualizeSharpe / annualizeSharpe', () => { + it('round-trips through sqrt(periodsPerYear)', () => { + expect(deannualizeSharpe(2, 252)).toBeCloseTo(2 / Math.sqrt(252), 12); + expect(annualizeSharpe(deannualizeSharpe(2, 252), 252)).toBeCloseTo(2, 12); + }); + + it('passes the value through when the frequency is unknown', () => { + expect(deannualizeSharpe(1.7, 0)).toBe(1.7); + expect(annualizeSharpe(1.7, 0)).toBe(1.7); + }); +}); + +// ───────────────────────────────────────────────────────────────────────────── +// The reason this module exists. +// ───────────────────────────────────────────────────────────────────────────── + +describe('deflatedSharpeRatio — best-of-N deflation', () => { + /** + * An annualized Sharpe of 2.0 on 250 daily observations. Respectable, and the + * kind of number an AI generator produces by the thousand. + */ + const OBSERVATIONS = 250; + const observedSharpe = deannualizeSharpe(2.0, 252); // ≈ 0.126 per observation + + it('PASSES when it is the result of a single honest experiment', () => { + const r = deflatedSharpeRatio({ observedSharpe, nTrials: 1, nObservations: OBSERVATIONS }); + expect(r.expectedMaxSharpe).toBe(0); // nothing to deflate + expect(r.deflatedSharpeRatio).toBe(r.probabilisticSharpeRatio); + expect(r.deflatedSharpeRatio).toBeGreaterThan(0.95); + expect(r.deflatedSharpeRatio).toBeCloseTo(0.9761, 3); + }); + + it('FAILS on the SAME numbers when it was the best of 1,000 candidates', () => { + const r = deflatedSharpeRatio({ observedSharpe, nTrials: 1000, nObservations: OBSERVATIONS }); + + // The luckiest of 1,000 zero-edge strategies posts ~3.26 annualized... + expect(annualizeSharpe(r.expectedMaxSharpe, 252)).toBeGreaterThan(3.2); + // ...so our 2.0 is BELOW the noise bar and the z-score goes negative. + expect(r.expectedMaxSharpe).toBeGreaterThan(r.observedSharpe); + expect(r.zScore).toBeLessThan(0); + + expect(r.deflatedSharpeRatio).toBeLessThan(0.95); + expect(r.deflatedSharpeRatio).toBeLessThan(0.2); + expect(r.deflatedSharpeRatio).toBeCloseTo(0.1043, 3); + + // The undeflated single-hypothesis test still says "significant" — which is + // exactly the mistake this module exists to prevent. + expect(r.probabilisticSharpeRatio).toBeGreaterThan(0.95); + }); + + it('deflates monotonically as the search widens', () => { + let prev = 1; + for (const nTrials of [1, 2, 10, 100, 1000, 10_000, 1_000_000]) { + const r = deflatedSharpeRatio({ observedSharpe, nTrials, nObservations: OBSERVATIONS }); + expect(r.deflatedSharpeRatio).toBeLessThanOrEqual(prev); + prev = r.deflatedSharpeRatio; + } + expect(prev).toBeLessThan(0.01); // a million tries buys any Sharpe you like + }); + + it('lets a genuinely exceptional strategy survive a wide search', () => { + // Sharpe 6 annualized on 250 days is a real anomaly, not a lucky draw... + const exceptional = deflatedSharpeRatio({ + observedSharpe: deannualizeSharpe(6, 252), + nTrials: 1000, + nObservations: OBSERVATIONS, + }); + expect(exceptional.deflatedSharpeRatio).toBeGreaterThan(0.95); + // ...and a package that could only ever say "no" would be a wall, not a store. + }); + + it('needs more observations to justify the same Sharpe after a wide search', () => { + // Same 2.0 annualized Sharpe, best-of-1000, but ten years of data instead of one. + const long = deflatedSharpeRatio({ + observedSharpe, + nTrials: 1000, + nObservations: 2520, + }); + expect(long.deflatedSharpeRatio).toBeGreaterThan(0.95); + // The noise bar falls as 1/sqrt(n): more data, less room for luck. + const short = deflatedSharpeRatio({ observedSharpe, nTrials: 1000, nObservations: 250 }); + expect(long.expectedMaxSharpe).toBeLessThan(short.expectedMaxSharpe); + }); + + it('uses the caller-supplied trial variance when the real spread is known', () => { + const wide = deflatedSharpeRatio({ + observedSharpe, + nTrials: 1000, + nObservations: OBSERVATIONS, + varianceOfTrialSharpes: 4 * nullVarianceOfTrialSharpes(OBSERVATIONS), + }); + const nullish = deflatedSharpeRatio({ + observedSharpe, + nTrials: 1000, + nObservations: OBSERVATIONS, + }); + // A more dispersed candidate pool means a higher bar, so a bigger haircut. + expect(wide.expectedMaxSharpe).toBeCloseTo(nullish.expectedMaxSharpe * 2, 10); + expect(wide.deflatedSharpeRatio).toBeLessThan(nullish.deflatedSharpeRatio); + }); + + it('penalises the short-volatility payoff shape once the Sharpe clears the bar', () => { + const strong = deannualizeSharpe(6, 252); + const clean = deflatedSharpeRatio({ + observedSharpe: strong, + nTrials: 100, + nObservations: OBSERVATIONS, + }); + const skewed = deflatedSharpeRatio({ + observedSharpe: strong, + nTrials: 100, + nObservations: OBSERVATIONS, + skewness: -2.5, + kurtosis: 15, + }); + expect(clean.observedSharpe).toBeGreaterThan(clean.expectedMaxSharpe); + expect(skewed.standardError).toBeGreaterThan(clean.standardError); + expect(skewed.deflatedSharpeRatio).toBeLessThan(clean.deflatedSharpeRatio); + }); + + it('lets extra uncertainty cut both ways below the bar — by design, not by bug', () => { + // A subtlety worth stating explicitly, because it looks wrong at a glance: + // when the observed Sharpe sits BELOW the noise bar, a wider standard error + // pulls the probability back toward 0.5 and therefore UP. That is correct + // Bayesian bookkeeping — a fat-tailed sample is less informative about being + // below the bar too. The result is still a failing DSR either way, so no + // listing decision turns on it. + const clean = deflatedSharpeRatio({ observedSharpe, nTrials: 1000, nObservations: OBSERVATIONS }); + const skewed = deflatedSharpeRatio({ + observedSharpe, + nTrials: 1000, + nObservations: OBSERVATIONS, + skewness: -2.5, + kurtosis: 15, + }); + expect(clean.observedSharpe).toBeLessThan(clean.expectedMaxSharpe); + expect(skewed.deflatedSharpeRatio).toBeGreaterThan(clean.deflatedSharpeRatio); + expect(skewed.deflatedSharpeRatio).toBeLessThan(0.5); + expect(clean.deflatedSharpeRatio).toBeLessThan(0.5); + }); + + it('reports every field finite and JSON-safe on degenerate input', () => { + for (const input of [ + { observedSharpe: 0, nTrials: 0, nObservations: 0 }, + { observedSharpe: Number.NaN, nTrials: 1000, nObservations: 100 }, + { observedSharpe: 5, nTrials: 1, nObservations: 1 }, + { observedSharpe: -1, nTrials: 1e12, nObservations: 3 }, + { observedSharpe: 1, nTrials: 1000, nObservations: 100, varianceOfTrialSharpes: 0 }, + ]) { + const r = deflatedSharpeRatio(input); + for (const [key, value] of Object.entries(r)) { + if (key === 'observedSharpe') continue; // echoed back verbatim + expect(Number.isFinite(value), `${key} finite for ${JSON.stringify(input)}`).toBe(true); + } + expect(r.deflatedSharpeRatio).toBeGreaterThanOrEqual(0); + expect(r.deflatedSharpeRatio).toBeLessThanOrEqual(1); + } + }); +}); + +describe('minimumTrackRecordLength', () => { + it('matches the closed form', () => { + // SR 0.1, gaussian, 95%: 1 + 1.005 * (1.6448536/0.1)^2 = 272.91 + const z = normalPpf(0.95); + const expected = 1 + 1.005 * Math.pow(z / 0.1, 2); + expect(minimumTrackRecordLength({ observedSharpe: 0.1 })).toBeCloseTo(expected, 9); + expect(minimumTrackRecordLength({ observedSharpe: 0.1 })).toBeCloseTo(272.91, 1); + }); + + it('is the exact inverse of probabilisticSharpeRatio', () => { + // If MinTRL is not the inverse of PSR then one of them is lying, and both + // are used to make listing decisions. + for (const [sr, skew, kurt, conf] of [ + [0.1, 0, 3, 0.95], + [0.25, -1.2, 8, 0.99], + [0.05, 0.6, 4.5, 0.9], + [0.4, -0.3, 3.2, 0.999], + ] as const) { + const n = minimumTrackRecordLength({ + observedSharpe: sr, + skewness: skew, + kurtosis: kurt, + targetConfidence: conf, + }); + expect(Number.isFinite(n)).toBe(true); + expect( + probabilisticSharpeRatio({ + observedSharpe: sr, + nObservations: n, + skewness: skew, + kurtosis: kurt, + }), + ).toBeCloseTo(conf, 8); + } + }); + + it('grows with the inverse square of the edge', () => { + const strong = minimumTrackRecordLength({ observedSharpe: 0.2 }); + const half = minimumTrackRecordLength({ observedSharpe: 0.1 }); + // Halving the edge roughly quadruples the required track record. + expect((half - 1) / (strong - 1)).toBeGreaterThan(3.8); + expect((half - 1) / (strong - 1)).toBeLessThan(4.2); + }); + + it('demands more data at higher confidence', () => { + const c90 = minimumTrackRecordLength({ observedSharpe: 0.15, targetConfidence: 0.9 }); + const c99 = minimumTrackRecordLength({ observedSharpe: 0.15, targetConfidence: 0.99 }); + expect(c99).toBeGreaterThan(c90); + }); + + it('demands more data for a negatively skewed, fat-tailed return stream', () => { + const clean = minimumTrackRecordLength({ observedSharpe: 0.15 }); + const ugly = minimumTrackRecordLength({ observedSharpe: 0.15, skewness: -2, kurtosis: 12 }); + expect(ugly).toBeGreaterThan(clean); + // The required length scales exactly with the variance factor: 1.361875 + // versus the gaussian 1.01125, a 35% longer track record for the same Sharpe. + expect((ugly - 1) / (clean - 1)).toBeCloseTo( + sharpeVarianceFactor(0.15, -2, 12) / sharpeVarianceFactor(0.15, 0, 3), + 9, + ); + expect((ugly - 1) / (clean - 1)).toBeCloseTo(1.3467, 3); + }); + + it('answers the forward-test question against the deflated benchmark', () => { + // "How long must we track this live before the best-of-1000 claim holds?" + const observedSharpe = deannualizeSharpe(2.0, 252); + const bar = expectedMaxSharpe(1000, nullVarianceOfTrialSharpes(250)); + // Against a zero benchmark, one year of daily data is already enough... + expect(minimumTrackRecordLength({ observedSharpe })).toBeLessThan(250); + // ...but it can never clear the best-of-1000 bar, because it sits below it. + expect(minimumTrackRecordLength({ observedSharpe, benchmarkSharpe: bar })).toBe( + Number.POSITIVE_INFINITY, + ); + }); + + it('returns Infinity when the Sharpe does not exceed the benchmark', () => { + expect(minimumTrackRecordLength({ observedSharpe: 0 })).toBe(Number.POSITIVE_INFINITY); + expect(minimumTrackRecordLength({ observedSharpe: -0.5 })).toBe(Number.POSITIVE_INFINITY); + expect( + minimumTrackRecordLength({ observedSharpe: 0.2, benchmarkSharpe: 0.2 }), + ).toBe(Number.POSITIVE_INFINITY); + expect(minimumTrackRecordLength({ observedSharpe: Number.NaN })).toBe( + Number.POSITIVE_INFINITY, + ); + }); +}); diff --git a/packages/strategy-validation/src/deflated-sharpe.ts b/packages/strategy-validation/src/deflated-sharpe.ts new file mode 100644 index 0000000..d2a383c --- /dev/null +++ b/packages/strategy-validation/src/deflated-sharpe.ts @@ -0,0 +1,514 @@ +/** + * Deflated Sharpe Ratio — the multiple-testing correction that makes a paid + * strategy store defensible. + * + * THE PROBLEM THIS SOLVES + * + * Generation is free and infinite. We can mint ten thousand TSP documents in a + * minute, backtest them all, and list the best hundred. That process produces + * spectacular-looking equity curves *with no edge whatsoever*, and it does so + * reliably, because picking the maximum of ten thousand noisy estimates is a + * search for luck, not for skill. + * + * Concretely: take 10,000 coin-flip strategies whose TRUE Sharpe is exactly zero + * and score each on 250 observations. Each estimate has a standard error of + * roughly 1/sqrt(250) ≈ 0.063 per observation, ≈ 1.0 annualized. The single best + * of those 10,000 will show an annualized Sharpe near 3.9. Not because it works + * — because we looked 10,000 times. Publish it with a t-test and it clears any + * conventional significance threshold with room to spare, since the t-test + * assumes we ran ONE experiment. + * + * Bailey & López de Prado's answer, and this module's job: don't ask "is this + * Sharpe better than zero", ask "is this Sharpe better than the best you'd + * expect from N tries at nothing". The first question is nearly free to pass; + * the second is the only one a buyer should care about. + * + * REFERENCES + * Bailey, D. H. & López de Prado, M. (2014), "The Deflated Sharpe Ratio: + * Correcting for Selection Bias, Backtest Overfitting, and Non-Normality", + * Journal of Portfolio Management, 40(5), pp. 94–107. + * Bailey, D. H. & López de Prado, M. (2012), "The Sharpe Ratio Efficient + * Frontier", Journal of Risk, 15(2), pp. 3–44. (PSR and MinTRL) + * Mertens, E. (2002), "Comments on variance of the IID estimator in Lo (2002)". + * (the skew/kurtosis-aware standard error used in the PSR denominator) + * + * UNITS — READ THIS BEFORE CALLING ANYTHING HERE + * + * Every Sharpe in this module is a PER-OBSERVATION Sharpe, not an annualized + * one, because `nObservations` and the Sharpe have to describe the same sample. + * Pass an annualized Sharpe with a per-trade observation count and you overstate + * significance by sqrt(periodsPerYear) — a factor of ~16 for daily data. That is + * not a rounding error, it is the difference between "reject" and "list it". + * Use `sharpe(returns, 1)` from ./metrics.js, or `deannualizeSharpe()` below. + */ + +/** Euler–Mascheroni constant, γ. Appears in the Gumbel expected-maximum term. */ +export const EULER_MASCHERONI = 0.5772156649015329; + +/** Below this many observations the sampling distribution is meaningless. */ +export const MIN_OBSERVATIONS_FOR_PSR = 2; + +// ── normal distribution primitives ────────────────────────────────────────── + +/** + * Standard normal CDF, Φ(x). + * + * Hart's (1968) rational approximation in the form given by Graeme West, + * "Better Approximations to Cumulative Normal Functions" (Wilmott, 2005). + * Accurate to roughly double-precision machine epsilon across the whole real + * line, exactly symmetric, and Φ(0) is exactly 0.5. + * + * The commonly copy-pasted Abramowitz & Stegun 26.2.17 polynomial is NOT good + * enough here: its ~7.5e-8 absolute error means Φ(0) ≠ 0.5, and it breaks the + * Φ/Φ⁻¹ round trip at the 1e-4 level in the tails — which is precisely where a + * significance decision at p = 0.95 or p = 0.999 gets made. + */ +export function normalCdf(x: number): number { + if (Number.isNaN(x)) return 0.5; + const a = Math.abs(x); + let tail: number; + + if (a > 37) { + // exp(-37²/2) underflows; the tail is 0 to double precision. + tail = 0; + } else { + const e = Math.exp(-(a * a) / 2); + if (a < 7.07106781186547) { + let n = 3.52624965998911e-2 * a + 0.700383064443688; + n = n * a + 6.37396220353165; + n = n * a + 33.912866078383; + n = n * a + 112.079291497871; + n = n * a + 221.213596169931; + n = n * a + 220.206867912376; + let d = 8.83883476483184e-2 * a + 1.75566716318264; + d = d * a + 16.064177579207; + d = d * a + 86.7807322029461; + d = d * a + 296.564248779674; + d = d * a + 637.333633378831; + d = d * a + 793.826512519948; + d = d * a + 440.413735824752; + tail = (e * n) / d; + } else { + // Continued-fraction tail expansion for the far tail. + let b = a + 0.65; + b = a + 4 / b; + b = a + 3 / b; + b = a + 2 / b; + b = a + 1 / b; + tail = e / (b * 2.506628274631); + } + } + + return x > 0 ? 1 - tail : tail; +} + +/** Standard normal PDF, φ(x). */ +export function normalPdf(x: number): number { + return Math.exp(-(x * x) / 2) / Math.sqrt(2 * Math.PI); +} + +/** + * Widest probability we will invert. Φ⁻¹(1 − 1e-16) ≈ 8.2; beyond that the + * float64 gap between p and 1 is smaller than the representable resolution, so + * clamping is the only alternative to returning ±Infinity — which would then + * propagate into a benchmark Sharpe and poison the report. + */ +const PPF_EPS = 1e-16; + +// Peter Acklam's rational approximation to Φ⁻¹, relative error < 1.15e-9. +const PPF_A = [ + -3.969683028665376e1, 2.209460984245205e2, -2.759285104469687e2, + 1.38357751867269e2, -3.066479806614716e1, 2.506628277459239, +]; +const PPF_B = [ + -5.447609879822406e1, 1.615858368580409e2, -1.556989798598866e2, + 6.680131188771972e1, -1.328068155288572e1, +]; +const PPF_C = [ + -7.784894002430293e-3, -3.223964580411365e-1, -2.400758277161838, + -2.549732539343734, 4.374664141464968, 2.938163982698783, +]; +const PPF_D = [7.784695709041462e-3, 3.224671290700398e-1, 2.445134137142996, 3.754408661907416]; +const PPF_P_LOW = 0.02425; + +/** + * Standard normal inverse CDF (quantile), Φ⁻¹(p). + * + * Acklam's rational approximation, relative error below 1.15e-9 — three orders + * of magnitude tighter than any decision we make with it. `p` is clamped to + * [1e-16, 1−1e-16] so the function is total: `normalPpf(0)` returns a large + * negative finite number rather than −Infinity, because an Infinity here becomes + * an Infinity in the expected-maximum Sharpe, which becomes a NaN comparison, + * which becomes a gate that always passes. + */ +export function normalPpf(p: number): number { + if (Number.isNaN(p)) return 0; + const q0 = Math.min(Math.max(p, PPF_EPS), 1 - PPF_EPS); + + if (q0 < PPF_P_LOW) { + const q = Math.sqrt(-2 * Math.log(q0)); + return ( + (((((PPF_C[0]! * q + PPF_C[1]!) * q + PPF_C[2]!) * q + PPF_C[3]!) * q + PPF_C[4]!) * q + + PPF_C[5]!) / + ((((PPF_D[0]! * q + PPF_D[1]!) * q + PPF_D[2]!) * q + PPF_D[3]!) * q + 1) + ); + } + if (q0 <= 1 - PPF_P_LOW) { + const q = q0 - 0.5; + const r = q * q; + return ( + ((((((PPF_A[0]! * r + PPF_A[1]!) * r + PPF_A[2]!) * r + PPF_A[3]!) * r + PPF_A[4]!) * r + + PPF_A[5]!) * + q) / + (((((PPF_B[0]! * r + PPF_B[1]!) * r + PPF_B[2]!) * r + PPF_B[3]!) * r + PPF_B[4]!) * r + 1) + ); + } + const q = Math.sqrt(-2 * Math.log(1 - q0)); + return -( + (((((PPF_C[0]! * q + PPF_C[1]!) * q + PPF_C[2]!) * q + PPF_C[3]!) * q + PPF_C[4]!) * q + + PPF_C[5]!) / + ((((PPF_D[0]! * q + PPF_D[1]!) * q + PPF_D[2]!) * q + PPF_D[3]!) * q + 1) + ); +} + +// ── standard error of the Sharpe estimator ────────────────────────────────── + +/** + * Mertens' (2002) variance factor for the Sharpe estimator: + * + * 1 − γ3·SR + ((γ4 − 1)/4)·SR² + * + * Divide by (n − 1) and take the root to get the standard error of the estimate. + * `γ3` is skewness, `γ4` is NON-EXCESS kurtosis (Gaussian = 3). For a Gaussian + * sample the expression collapses to the familiar `1 + SR²/2`, which is the + * arithmetic check that the kurtosis convention is right. + * + * Why it must be moment-aware: the term is the price of non-normality. Negative + * skew (`−γ3·SR` becomes positive) and fat tails (`(γ4−1)/4` grows) both INFLATE + * the standard error, so a strategy that wins small and often and loses huge and + * rarely — the classic short-volatility disguise — needs a much higher observed + * Sharpe to clear the same confidence bar. That is the correct treatment, and a + * plain t-test does not do it. + * + * Non-positive results are impossible for any real distribution (the skew– + * kurtosis feasibility bound γ4 ≥ γ3² + 1 forces the expression to at least + * (1 − γ3·SR/2)² ≥ 0), so a non-positive value means the sample moments are + * infeasible — too few observations to estimate a fourth moment. Rather than + * manufacture a near-zero denominator and an infinite z-score, we retreat to the + * normal-theory factor, which is the conservative choice. + */ +export function sharpeVarianceFactor(observedSharpe: number, skewness: number, kurtosis: number): number { + const sr = observedSharpe; + // A non-finite Sharpe would make the factor non-finite, which would make the + // standard error non-finite, which would land a NaN in a stored report. The + // callers that matter (PSR, DSR) reject a non-finite Sharpe outright; this + // just keeps the reported standard error a real number. + if (!Number.isFinite(sr)) return 1; + const gaussian = 1 + (sr * sr) / 2; + if (!Number.isFinite(skewness) || !Number.isFinite(kurtosis)) return gaussian; + const factor = 1 - skewness * sr + ((kurtosis - 1) / 4) * sr * sr; + return factor > 0 ? factor : gaussian; +} + +/** + * Standard error of a per-observation Sharpe estimate over `nObservations`. + * Uses the (n − 1) denominator to match the PSR formulation. + */ +export function sharpeStandardError( + observedSharpe: number, + nObservations: number, + skewness = 0, + kurtosis = 3, +): number { + if (nObservations < MIN_OBSERVATIONS_FOR_PSR) return 0; + return Math.sqrt(sharpeVarianceFactor(observedSharpe, skewness, kurtosis) / (nObservations - 1)); +} + +// ── Probabilistic Sharpe Ratio ────────────────────────────────────────────── + +export interface PsrInput { + /** PER-OBSERVATION Sharpe of the candidate. Not annualized. */ + observedSharpe: number; + /** PER-OBSERVATION Sharpe to beat. 0 asks only "better than nothing". */ + benchmarkSharpe?: number; + /** Number of return observations behind `observedSharpe` (e.g. trade count). */ + nObservations: number; + /** Skewness of those returns. 0 = symmetric. */ + skewness?: number; + /** NON-EXCESS kurtosis of those returns. 3 = Gaussian. */ + kurtosis?: number; +} + +/** + * Probabilistic Sharpe Ratio — P(true Sharpe > benchmark), given the sample. + * + * PSR(SR*) = Φ[ (ŜR − SR*)·sqrt(n − 1) / sqrt(1 − γ3·ŜR + ((γ4−1)/4)·ŜR²) ] + * + * Read it as: "how many standard errors above the benchmark is the observed + * Sharpe, converted to a probability". It is a one-sided confidence level, so + * 0.95 means the usual 5% false-positive tolerance. + * + * On its own, against a benchmark of 0, PSR is easy to pass and therefore not + * worth much — that is exactly the single-hypothesis test that backtest + * overfitting defeats. Its value is as the machinery underneath + * `deflatedSharpeRatio()`, where the benchmark stops being zero and becomes the + * score a lucky coin flip would have posted. + * + * Returns 0 (no evidence) rather than NaN when there are too few observations. + * Output is a probability, so it is always in [0, 1] and always finite. + */ +export function probabilisticSharpeRatio(input: PsrInput): number { + const { observedSharpe, nObservations } = input; + const benchmarkSharpe = input.benchmarkSharpe ?? 0; + const skew = input.skewness ?? 0; + const kurt = input.kurtosis ?? 3; + + if (!Number.isFinite(observedSharpe) || !Number.isFinite(benchmarkSharpe)) return 0; + if (!(nObservations >= MIN_OBSERVATIONS_FOR_PSR)) return 0; + + const se = sharpeStandardError(observedSharpe, nObservations, skew, kurt); + if (!(se > 0)) return 0; + + return normalCdf((observedSharpe - benchmarkSharpe) / se); +} + +// ── expected maximum Sharpe under the null ────────────────────────────────── + +/** + * Expected maximum of N independent standard normal draws, via the Gumbel + * (extreme value type I) limit: + * + * E[max_N] ≈ (1 − γ)·Φ⁻¹(1 − 1/N) + γ·Φ⁻¹(1 − 1/(N·e)) + * + * with γ the Euler–Mascheroni constant. This is the approximation used in Bailey + * & López de Prado (2014). Accurate to ~2% at N = 10 and better as N grows + * (N = 1000 gives 3.255 against a true value of 3.241). + * + * The intuition worth internalising: the maximum of N standard normals grows + * like sqrt(2·ln N). It grows WITHOUT BOUND, but only logarithmically. So + * searching harder always buys you a better-looking backtest, and buying twice + * as much looking gets you almost nothing extra — which is why the honest fix is + * to subtract the expected windfall rather than to search less. + */ +export function expectedMaxStandardNormal(nTrials: number): number { + if (!Number.isFinite(nTrials) || nTrials <= 1) return 0; + const n = nTrials; + return ( + (1 - EULER_MASCHERONI) * normalPpf(1 - 1 / n) + + EULER_MASCHERONI * normalPpf(1 - 1 / (n * Math.E)) + ); +} + +/** + * The intuition-carrying number: the PER-OBSERVATION Sharpe the luckiest of + * `nTrials` genuinely worthless strategies is expected to post. + * + * E[max ŜR] = sqrt(V[ŜR across trials]) · E[max of nTrials standard normals] + * + * This is the bar a candidate has to clear to have said anything at all. If you + * generated 10,000 candidates on 250 observations, `expectedMaxSharpe(10000, + * 1/250)` ≈ 0.244 per observation ≈ 3.9 annualized. Any candidate from that + * search reporting an annualized Sharpe of 3 is BELOW what pure noise delivers. + * + * `varianceOfTrialSharpes` is the variance of the estimated Sharpes ACROSS the + * trials, in per-observation units. Under the null (every strategy truly has + * zero edge) it is 1/nObservations — see `nullVarianceOfTrialSharpes()`. When you + * have the real trial scores, use their sample variance instead: a heterogeneous + * candidate pool disperses more than the null, and more dispersion means a higher + * bar. + * + * Returns 0 for a single trial: with no selection there is no selection bias, and + * the correct benchmark falls back to zero. + */ +export function expectedMaxSharpe(nTrials: number, varianceOfTrialSharpes: number): number { + if (!(varianceOfTrialSharpes > 0) || !Number.isFinite(varianceOfTrialSharpes)) return 0; + const e = expectedMaxStandardNormal(nTrials); + if (!(e > 0)) return 0; + return Math.sqrt(varianceOfTrialSharpes) * e; +} + +/** + * Variance of trial Sharpes under the null hypothesis that every candidate has a + * true Sharpe of exactly zero: Var(ŜR) = (1 + ŜR²/2)/n → 1/n at ŜR = 0. + * + * The right default when the caller only knows HOW MANY candidates it generated, + * not what they each scored. Note the direction of the error: real candidate + * pools are heterogeneous (different templates, different parameters), so their + * Sharpes disperse MORE than the null, which means the true haircut is LARGER + * than this default. Treat it as a floor and pass `trialSharpes` when you have + * them. + */ +export function nullVarianceOfTrialSharpes(nObservations: number): number { + if (!(nObservations > 0)) return 0; + return 1 / nObservations; +} + +/** Convert an annualized Sharpe to the per-observation units this module needs. */ +export function deannualizeSharpe(annualizedSharpe: number, periodsPerYear: number): number { + if (!(periodsPerYear > 0)) return annualizedSharpe; + return annualizedSharpe / Math.sqrt(periodsPerYear); +} + +/** Convert a per-observation Sharpe to annualized units, for display only. */ +export function annualizeSharpe(perObservationSharpe: number, periodsPerYear: number): number { + if (!(periodsPerYear > 0)) return perObservationSharpe; + return perObservationSharpe * Math.sqrt(periodsPerYear); +} + +// ── Deflated Sharpe Ratio ─────────────────────────────────────────────────── + +export interface DsrInput { + /** PER-OBSERVATION Sharpe of the selected candidate. Not annualized. */ + observedSharpe: number; + /** + * How many candidates were generated and scored before this one was selected. + * The single most important input in this package. Pass the REAL number: if a + * generator produced 8,000 documents and kept 40, `nTrials` is 8,000, not 40 + * and not 1. Under-reporting it is how a store lists noise. + */ + nTrials: number; + /** Number of return observations (trades) behind `observedSharpe`. */ + nObservations: number; + /** + * Variance of the trial Sharpes in per-observation units. Defaults to the null + * variance 1/nObservations — see `nullVarianceOfTrialSharpes()`. + */ + varianceOfTrialSharpes?: number; + skewness?: number; + /** NON-EXCESS kurtosis (Gaussian = 3). */ + kurtosis?: number; +} + +export interface DsrResult { + /** + * THE number. P(true Sharpe > best-of-nTrials-under-the-null). A probability + * in [0, 1]; require ≥ 0.95 before making a listing claim. + */ + deflatedSharpeRatio: number; + /** + * PSR against a zero benchmark — what a single-hypothesis test would have + * said. Reported alongside so the size of the selection-bias haircut is + * visible rather than implied. + */ + probabilisticSharpeRatio: number; + /** The benchmark actually used: the Sharpe the luckiest coin flip posts. */ + expectedMaxSharpe: number; + /** Standard error of the Sharpe estimate, in per-observation units. */ + standardError: number; + /** How many standard errors the observed Sharpe sits above the benchmark. */ + zScore: number; + observedSharpe: number; + nTrials: number; + nObservations: number; + varianceOfTrialSharpes: number; + skewness: number; + kurtosis: number; +} + +/** + * Deflated Sharpe Ratio — PSR with the benchmark raised from zero to the + * expected best-of-N under the null. + * + * DSR = PSR( SR* = E[max ŜR over nTrials] ) + * + * Same sample, same observed Sharpe; the ONLY thing that changes is how many + * times we looked. That is the whole idea, and it is why `nTrials` must be the + * true size of the search that produced this candidate — including every + * parameter sweep, every template variation, and every candidate that was + * discarded. Trials you don't count are trials you don't pay for, and the bill + * lands on the buyer. + * + * Never throws; every degenerate path returns a finite probability. + */ +export function deflatedSharpeRatio(input: DsrInput): DsrResult { + const { observedSharpe, nTrials, nObservations } = input; + const skew = input.skewness ?? 0; + const kurt = input.kurtosis ?? 3; + const variance = input.varianceOfTrialSharpes ?? nullVarianceOfTrialSharpes(nObservations); + + const benchmark = expectedMaxSharpe(nTrials, variance); + const se = sharpeStandardError(observedSharpe, nObservations, skew, kurt); + const zScore = se > 0 ? (observedSharpe - benchmark) / se : 0; + + return { + deflatedSharpeRatio: probabilisticSharpeRatio({ + observedSharpe, + benchmarkSharpe: benchmark, + nObservations, + skewness: skew, + kurtosis: kurt, + }), + probabilisticSharpeRatio: probabilisticSharpeRatio({ + observedSharpe, + benchmarkSharpe: 0, + nObservations, + skewness: skew, + kurtosis: kurt, + }), + expectedMaxSharpe: benchmark, + standardError: se, + zScore: Number.isFinite(zScore) ? zScore : 0, + observedSharpe, + nTrials, + nObservations, + varianceOfTrialSharpes: variance, + skewness: skew, + kurtosis: kurt, + }; +} + +// ── Minimum Track Record Length ───────────────────────────────────────────── + +export interface MinTrlInput { + /** PER-OBSERVATION Sharpe. Not annualized. */ + observedSharpe: number; + /** Benchmark to beat, per-observation. Pass `expectedMaxSharpe(...)` to + * answer "how long until this survives the multiple-testing correction". */ + benchmarkSharpe?: number; + skewness?: number; + /** NON-EXCESS kurtosis (Gaussian = 3). */ + kurtosis?: number; + /** One-sided confidence required. Default 0.95. */ + targetConfidence?: number; +} + +/** + * Minimum Track Record Length — how many observations are needed before an + * observed Sharpe of this size and shape becomes statistically credible. + * + * MinTRL = 1 + [1 − γ3·ŜR + ((γ4−1)/4)·ŜR²] · ( Φ⁻¹(confidence) / (ŜR − SR*) )² + * + * It is the exact algebraic inverse of `probabilisticSharpeRatio()`: at exactly + * MinTRL observations, PSR equals `targetConfidence`. (There is a test asserting + * that round trip, because an inverse that isn't one is a silent liar.) + * + * This is the number that sets forward-test duration. Given a candidate's + * observed per-trade Sharpe and its trade frequency, MinTRL converts directly + * into "track this live for N trades / M months before the listing claim is + * defensible". It also encodes the brutal scaling law of this business: required + * length grows with the INVERSE SQUARE of the edge, so halving the Sharpe + * quadruples the wait. + * + * Returns `Number.POSITIVE_INFINITY` when `observedSharpe <= benchmarkSharpe` — + * no amount of data makes a Sharpe credible if it does not exceed the bar in the + * first place. That is the mathematically correct answer and callers must handle + * it (`Number.isFinite` before storing; see `finiteOrNull()` in ./gauntlet.ts). + * Result is fractional; ceil it for a bar or trade count. + */ +export function minimumTrackRecordLength(input: MinTrlInput): number { + const { observedSharpe } = input; + const benchmark = input.benchmarkSharpe ?? 0; + const skew = input.skewness ?? 0; + const kurt = input.kurtosis ?? 3; + const confidence = input.targetConfidence ?? 0.95; + + if (!Number.isFinite(observedSharpe) || !Number.isFinite(benchmark)) { + return Number.POSITIVE_INFINITY; + } + const edge = observedSharpe - benchmark; + if (!(edge > 0)) return Number.POSITIVE_INFINITY; + + const z = normalPpf(Math.min(Math.max(confidence, PPF_EPS), 1 - PPF_EPS)); + const factor = sharpeVarianceFactor(observedSharpe, skew, kurt); + return 1 + factor * Math.pow(z / edge, 2); +} diff --git a/packages/strategy-validation/src/gauntlet.test.ts b/packages/strategy-validation/src/gauntlet.test.ts new file mode 100644 index 0000000..dc083cb --- /dev/null +++ b/packages/strategy-validation/src/gauntlet.test.ts @@ -0,0 +1,218 @@ +import { describe, it, expect } from 'vitest'; +import { ZERO_COST_MODEL, tsp } from '@b1dz/source-strategies'; +import { sinePrices, snapshotsFrom, randomWalkPrices } from './synthetic.js'; +import { runGauntlet, explainReport, DEFAULT_POLICY } from './gauntlet.js'; + +const mrDoc: tsp.TradingStrategyDefinition = { + tsp: '0.1', + id: 'test-mr', + name: 'Test MR', + definition: { + kind: 'template', + template: 'mean-reversion', + params: { period: 14, oversold: 35, overbought: 65 }, + }, +}; + +const sineSnaps = snapshotsFrom( + sinePrices({ bars: 600, period: 20, amplitude: 0.1, noise: 0.01 }), +); + +describe('runGauntlet', () => { + it('runs a mean-reversion strategy on sine data with nTrials=1 without throwing', () => { + const report = runGauntlet({ + definition: mrDoc, + snapshots: sineSnaps, + costs: ZERO_COST_MODEL, + nTrials: 1, + }); + expect(report.validationErrors).toEqual([]); + expect(report.candidateId).toBe('test-mr'); + expect(report.inSampleGates.length).toBeGreaterThan(0); + expect(report.deflatedSharpe).not.toBeNull(); + }); + + it('produces a gauntlet report with all required fields', () => { + const report = runGauntlet({ + definition: mrDoc, + snapshots: sineSnaps, + costs: ZERO_COST_MODEL, + nTrials: 1, + }); + expect(report.candidateId).toBe('test-mr'); + expect(typeof report.generatedAt).toBe('string'); + expect(report.generatedAt).toBeTruthy(); + expect(report.costModel).toBeDefined(); + expect(report.inSampleSummary).not.toBeNull(); + expect(report.robustness).not.toBeNull(); + expect(report.deflatedSharpe).not.toBeNull(); + expect(report.duplicates).toEqual([]); + }); + + it('populates validation errors for an invalid document', () => { + const report = runGauntlet({ + definition: { tsp: '999', id: 'x', name: 'X', definition: {} } as any, + snapshots: sineSnaps, + }); + expect(report.validationErrors.length).toBeGreaterThan(0); + expect(report.passed).toBe(false); + }); + + it('populates validation errors for a doc that fails to compile', () => { + const report = runGauntlet({ + definition: { + tsp: '0.1', + id: 'x', + name: 'X', + definition: { kind: 'template', template: 'nope' }, + } as any, + snapshots: sineSnaps, + }); + expect(report.validationErrors.length).toBeGreaterThan(0); + expect(report.passed).toBe(false); + }); + + it('fails the minBars gate on too few bars', () => { + const tiny = snapshotsFrom(sinePrices({ bars: 10 })); + const report = runGauntlet({ + definition: mrDoc, + snapshots: tiny, + costs: ZERO_COST_MODEL, + }); + const barGate = report.inSampleGates.find((g) => g.name === 'minBars'); + expect(barGate).toBeDefined(); + expect(barGate!.passed).toBe(false); + }); + + it('never throws on degenerate inputs', () => { + expect(() => + runGauntlet({ definition: mrDoc, snapshots: [] }), + ).not.toThrow(); + expect(() => + runGauntlet({ + definition: mrDoc, + snapshots: sineSnaps, + nTrials: -1, + }), + ).not.toThrow(); + expect(() => + runGauntlet({ + definition: mrDoc, + snapshots: sineSnaps, + catalog: [], + }), + ).not.toThrow(); + }); + + it('passes with a lenient policy that lowers every bar', () => { + const lenient = { + minTrades: 1, + minBars: 10, + oosRatio: 0.1, + maxDrawdownPct: 1.0, + minProfitFactor: 0.1, + minDeflatedSharpePValue: 0.01, + minRobustnessFraction: 0.1, + maxRobustnessDegradationPct: 0.99, + minProfitableRegimes: 1, + requireOutOfSampleProfit: false, + walkForwardFolds: 1, + }; + const report = runGauntlet({ + definition: mrDoc, + snapshots: sineSnaps, + costs: ZERO_COST_MODEL, + policy: lenient, + nTrials: 1, + }); + expect(report.validationErrors).toEqual([]); + expect(report.inSampleGates.length).toBeGreaterThan(0); + expect(typeof report.passed).toBe('boolean'); + }); + + it('includes inSampleSummary when bars are sufficient', () => { + const report = runGauntlet({ + definition: mrDoc, + snapshots: sineSnaps, + costs: ZERO_COST_MODEL, + nTrials: 1, + }); + expect(report.inSampleSummary).not.toBeNull(); + if (report.inSampleSummary) { + expect(typeof report.inSampleSummary.trades).toBe('number'); + expect(typeof report.inSampleSummary.returnPct).toBe('number'); + } + }); + + it('includes robustness block when valid', () => { + const report = runGauntlet({ + definition: mrDoc, + snapshots: sineSnaps, + costs: ZERO_COST_MODEL, + nTrials: 1, + }); + expect(report.robustness).not.toBeNull(); + }); + + it('detects duplicate strategies in the catalog', () => { + const report = runGauntlet({ + definition: mrDoc, + snapshots: sineSnaps, + costs: ZERO_COST_MODEL, + catalog: [], + nTrials: 1, + }); + expect(report.duplicates).toEqual([]); + }); +}); + +describe('explainReport', () => { + it('includes PASSED/FAILED, candidate ID, and gate names', () => { + const report = runGauntlet({ + definition: mrDoc, + snapshots: sineSnaps, + costs: ZERO_COST_MODEL, + nTrials: 1, + }); + const text = explainReport(report); + expect(text).toContain(report.candidateId); + expect(text.length).toBeGreaterThan(100); + }); + + it('includes FAILED for a failing report', () => { + const report = runGauntlet({ + definition: mrDoc, + snapshots: [], + costs: ZERO_COST_MODEL, + }); + expect(explainReport(report)).toContain('FAILED'); + }); + + it('includes costs description', () => { + const report = runGauntlet({ + definition: mrDoc, + snapshots: sineSnaps, + costs: ZERO_COST_MODEL, + nTrials: 1, + }); + const text = explainReport(report); + expect(text).toContain('costs'); + }); +}); + +describe('DEFAULT_POLICY', () => { + it('defines sensible defaults for every gate', () => { + expect(DEFAULT_POLICY.minTrades).toBe(30); + expect(DEFAULT_POLICY.minBars).toBe(35); + expect(DEFAULT_POLICY.oosRatio).toBe(0.3); + expect(DEFAULT_POLICY.maxDrawdownPct).toBe(0.25); + expect(DEFAULT_POLICY.minProfitFactor).toBe(1.2); + expect(DEFAULT_POLICY.minDeflatedSharpePValue).toBe(0.95); + expect(DEFAULT_POLICY.minRobustnessFraction).toBe(0.6); + expect(DEFAULT_POLICY.maxRobustnessDegradationPct).toBe(0.5); + expect(DEFAULT_POLICY.minProfitableRegimes).toBe(2); + expect(DEFAULT_POLICY.maxCatalogCorrelation).toBe(0.8); + expect(DEFAULT_POLICY.requireOutOfSampleProfit).toBe(true); + expect(DEFAULT_POLICY.walkForwardFolds).toBe(3); + }); +}); diff --git a/packages/strategy-validation/src/gauntlet.ts b/packages/strategy-validation/src/gauntlet.ts new file mode 100644 index 0000000..c511d8d --- /dev/null +++ b/packages/strategy-validation/src/gauntlet.ts @@ -0,0 +1,431 @@ +/** + * The statistical gauntlet — everything a strategy must survive to earn a listing. + * + * Each gate is a standalone, nameable check with a blocking/non-blocking flag + * and a pass threshold. `runGauntlet()` runs them all, records the results, and + * returns a report whose `passed` field is true ONLY when every blocking gate + * clears. Non-blocking gates (advisory, informational) affect the human-readable + * summary but never block listing. + * + * Every gate is a pure function of its inputs. No gate throws — a caught error + * becomes a FAILED gate with the error message in `detail`, so a single buggy + * gate cannot abort the entire run. The gauntlet's own top-level catch does the + * same: the report always comes back, even when something is profoundly broken. + */ +import type { MarketSnapshot } from '@b1dz/core'; +import { + DEFAULT_AMOUNT_PER_ENTRY, + costModelForSeries, + describeCostModel, + replayStrategy, + summarizeTrades, + tsp, + ZERO_COST_MODEL, + type CostModel, +} from '@b1dz/source-strategies'; +import { computeMetrics, tradeReturns } from './metrics.js'; +import { deflatedSharpeRatio, nullVarianceOfTrialSharpes } from './deflated-sharpe.js'; +import { trainTestSplit, walkForwardSplits } from './splits.js'; +import { robustnessScore } from './robustness.js'; +import { classifyRegimes, regimeBreakdown, regimeCoverage } from './regime.js'; +import { findDuplicates, signalCorrelation } from './correlation.js'; + + +/** A single quality gate — one yes/no decision with supporting numbers. */ +export interface GauntletGate { + /** Short, stable key for reporting and diffing across runs. */ + name: string; + /** true → the gate cleared its threshold. */ + passed: boolean; + /** The measured value (a probability, ratio, count, pct...). */ + value: number; + /** Threshold that gate requires to pass. */ + threshold: number; + /** Human-readable explanation of the result (≤ 200 chars, single line). */ + detail: string; + /** If true and this gate fails, the whole report fails. If false, advisory. */ + blocking: boolean; +} + +/** A composite summary covering every gate for one fold. */ +export interface WalkForwardGateBlock { + foldIndex: number; + gates: GauntletGate[]; + summary: ReturnType; +} + +export interface GauntletPolicy { + /** Minimum trades for the in-sample backtest to be considered meaningful. Default 30. */ + minTrades: number; + /** Fewer bars than this → cannot run. Below minimum warmup for the slowest TSP indicator. */ + minBars: number; + /** Minimum fraction of the series held for out-of-sample. Default 0.3. */ + oosRatio: number; + /** Max drawdown as fraction of peak. Default 0.25. */ + maxDrawdownPct: number; + /** Gross wins / gross losses, net of costs. Default 1.2. */ + minProfitFactor: number; + /** One-sided PSR confidence for the deflated Sharpe gate. Default 0.95. */ + minDeflatedSharpePValue: number; + /** Fraction of effective robustness variants that must stay profitable. Default 0.6. */ + minRobustnessFraction: number; + /** Ceil on how much of the base return the median robust variant may give up. Default 0.5. */ + maxRobustnessDegradationPct: number; + /** Regimes with net profit required. Default 2. */ + minProfitableRegimes: number; + /** Max Pearson across either signal or per-week return correlation with any + * catalogue entry, above which we reject as a duplicate. Default 0.8. */ + maxCatalogCorrelation: number; + /** Require the out-of-sample return to be positive. Default true. */ + requireOutOfSampleProfit: boolean; + /** Walk-forward folds. Default 3. */ + walkForwardFolds: number; +} + +export const DEFAULT_POLICY: GauntletPolicy = { + minTrades: 30, + minBars: 35, + oosRatio: 0.3, + maxDrawdownPct: 0.25, + minProfitFactor: 1.2, + minDeflatedSharpePValue: 0.95, + minRobustnessFraction: 0.6, + maxRobustnessDegradationPct: 0.5, + minProfitableRegimes: 2, + maxCatalogCorrelation: 0.8, + requireOutOfSampleProfit: true, + walkForwardFolds: 3, +}; + +// ── result types ───────────────────────────────────────────────────────────── + +export interface GauntletReport { + passed: boolean; + candidateId: string; + /** Validation errors before any replay was attempted — the document is not executable. */ + validationErrors: string[]; + /** Gates run on the in-sample backtest. Ordered by dependency. */ + inSampleGates: GauntletGate[]; + /** Gates run on the out-of-sample backtest. */ + outOfSampleGates: GauntletGate[]; + /** Per-fold walk-forward gate blocks. Empty if no folds run. */ + walkForward: WalkForwardGateBlock[]; + /** The deflated-Sharpe numbers (computed block). */ + deflatedSharpe: ReturnType | null; + /** Robustness block. */ + robustness: ReturnType | null; + /** Regime coverage. */ + regimeCoverageResult: ReturnType | null; + /** Duplicate matches from the catalogue. */ + duplicates: ReturnType; + /** In-sample summary (all bars). */ + inSampleSummary: ReturnType | null; + /** Out-of-sample summary. */ + outOfSampleSummary: ReturnType | null; + costModel: CostModel; + generatedAt: string; // ISO timestamp + /** Non-blocking gates that passed — evidence, not requirement. */ + advisoryGates: GauntletGate[]; +} + +export interface GauntletInput { + definition: tsp.TradingStrategyDefinition; + snapshots: MarketSnapshot[]; + costs?: CostModel; + policy?: Partial; + /** Catalogue entries to check for duplicates. */ + catalog?: { plugin: ReturnType; trades: ReturnType; id: string }[]; + /** Number of candidates generated before selection. CRITICAL. Defaults to 1. */ + nTrials?: number; + /** Per-observation variance of trial Sharpes (heterogeneous pool). Defaults to the null. */ + trialSharpes?: number[]; + candidateId?: string; +} + +function gate(name: string, value: number, threshold: number, blocking: boolean, detail: string): GauntletGate { + return { name, passed: value >= threshold, value, threshold, blocking, detail }; +} + +function gateLe(name: string, value: number, threshold: number, blocking: boolean, detail: string): GauntletGate { + return { name, passed: value <= threshold, value, threshold, blocking, detail }; +} + +function gateBool(name: string, passed: boolean, blocking: boolean, detail: string): GauntletGate { + // Treat threshold as 1 (must pass); value as 1 if passed, else 0. + return { name, passed, value: passed ? 1 : 0, threshold: 1, blocking, detail }; +} + +function errorText(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +function safeGate(name: string, fn: () => GauntletGate, blocking: boolean): GauntletGate { + try { + return fn(); + } catch (err) { + return gate(name, 0, 1, blocking, `gate failed with error: ${errorText(err)}`); + } +} + +/** Compile the document and return a plugin. Throws on errors (caught upstream). */ +function safeCompile(doc: tsp.TradingStrategyDefinition): ReturnType { + return tsp.compile(doc); +} + +// ── run the gauntlet ───────────────────────────────────────────────────────── + +/* eslint-disable complexity */ +export function runGauntlet(input: GauntletInput): GauntletReport { + const p: GauntletPolicy = { ...DEFAULT_POLICY, ...input.policy }; + const costs = input.costs ?? costModelForSeries(input.snapshots); + const candidateId = input.candidateId ?? input.definition.id ?? 'unknown'; + const now = new Date().toISOString(); + const baseReport = (overrides: Partial = {}): GauntletReport => ({ + passed: false, + candidateId, + validationErrors: [], + inSampleGates: [], + outOfSampleGates: [], + walkForward: [], + deflatedSharpe: null, + robustness: null, + regimeCoverageResult: null, + duplicates: [], + inSampleSummary: null, + outOfSampleSummary: null, + costModel: costs, + generatedAt: now, + advisoryGates: [], + ...overrides, + }); + + // 1. Validate the document. + const validation = tsp.validateDefinition(input.definition); + if (!validation.ok) { + return baseReport({ validationErrors: validation.errors }); + } + + // 2. Compile. + let plugin: ReturnType; + try { + plugin = safeCompile(input.definition); + } catch (err) { + return baseReport({ + validationErrors: [`compile: ${errorText(err)}`], + }); + } + + // 3. Split: train / test. + const { inSample, outOfSample } = trainTestSplit(input.snapshots, p.oosRatio, { minBars: p.minBars }); + if (inSample.length < p.minBars) { + return baseReport({ + inSampleGates: [gate('minBars', inSample.length, p.minBars, true, `only ${inSample.length} bars; need ${p.minBars}`)], + }); + } + + // 4. In-sample replay. + const isTrades = replayStrategy(plugin, inSample, { amountPerEntry: DEFAULT_AMOUNT_PER_ENTRY, costs }); + const isSummary = summarizeTrades(isTrades); + const isMetrics = computeMetrics(isTrades); + + const inSampleGates: GauntletGate[] = []; + inSampleGates.push(safeGate('minTrades', () => + gate('minTrades', isSummary.trades, p.minTrades, true, `${isSummary.trades} trades (need ≥ ${p.minTrades})`), + true)); + inSampleGates.push(safeGate('maxDrawdownPct', () => + gateLe('maxDrawdownPct', isMetrics.maxDrawdownPct, p.maxDrawdownPct, true, + `max drawdown ${(isMetrics.maxDrawdownPct * 100).toFixed(1)}% (limit ${(p.maxDrawdownPct * 100).toFixed(0)}%)`), + true)); + inSampleGates.push(safeGate('minProfitFactor', () => + gate('minProfitFactor', isMetrics.profitFactor, p.minProfitFactor, true, + `profit factor ${isMetrics.profitFactor.toFixed(2)} (need ≥ ${p.minProfitFactor})`), + true)); + + // 5. Deflated Sharpe. + const dsr = deflatedSharpeRatio({ + observedSharpe: isMetrics.sharpePerTrade, + nTrials: input.nTrials ?? 1, + nObservations: isSummary.trades, + skewness: isMetrics.skewness, + kurtosis: isMetrics.kurtosis, + varianceOfTrialSharpes: input.trialSharpes + ? (() => { const m = input.trialSharpes!.reduce((a, b) => a + b, 0) / input.trialSharpes!.length; return input.trialSharpes!.reduce((s, v) => s + (v - m) ** 2, 0) / (input.trialSharpes!.length - 1); })() + : undefined, + }); + + const dsrGate = gate('deflatedSharpeRatio', dsr.deflatedSharpeRatio, p.minDeflatedSharpePValue, true, + `DSR ${dsr.deflatedSharpeRatio.toFixed(3)} (need ≥ ${p.minDeflatedSharpePValue}) — PSR=${dsr.probabilisticSharpeRatio.toFixed(3)} bar=${dsr.expectedMaxSharpe.toFixed(3)} nTrials=${dsr.nTrials}`); + inSampleGates.push(dsrGate); + + // 6. Robustness (in-sample). + let robustness: ReturnType | null = null; + try { + robustness = robustnessScore(input.definition, inSample, { + costs, + minFractionProfitable: p.minRobustnessFraction, + maxDegradationPct: p.maxRobustnessDegradationPct, + }); + } catch (err) { + inSampleGates.push(gate('robustness', 0, 1, true, `robustness error: ${errorText(err)}`)); + } + if (robustness) { + inSampleGates.push(gateBool('robustness', robustness.passed, true, + robustness.detail)); + } + + // 7. Regime coverage (in-sample). + let regimeResult: ReturnType | null = null; + try { + const regimes = classifyRegimes(inSample); + const breakdown = regimeBreakdown(isTrades, regimes, inSample); + regimeResult = regimeCoverage(breakdown, p.minProfitableRegimes); + inSampleGates.push(gateBool('regimeCoverage', regimeResult.passed, true, + `${regimeResult.profitableRegimes.length} profitable regimes (need ≥ ${p.minProfitableRegimes}): ${regimeResult.profitableRegimes.join(', ') || 'none'}`)); + } catch (err) { + inSampleGates.push(gate('regimeCoverage', 0, 1, true, `regime error: ${errorText(err)}`)); + } + + // 8. Out-of-sample. + const outOfSampleGates: GauntletGate[] = []; + let oosSummary: ReturnType | null = null; + if (outOfSample.length >= p.minBars) { + const oosTrades = replayStrategy(plugin, outOfSample, { amountPerEntry: DEFAULT_AMOUNT_PER_ENTRY, costs }); + oosSummary = summarizeTrades(oosTrades); + const oosMetrics = computeMetrics(oosTrades); + + if (p.requireOutOfSampleProfit) { + outOfSampleGates.push(gateBool('outOfSampleProfit', oosSummary.returnPct > 0, true, + `OOS return ${(oosSummary.returnPct * 100).toFixed(2)}% (need > 0)`)); + } + outOfSampleGates.push(gateLe('outOfSampleMaxDrawdownPct', oosMetrics.maxDrawdownPct, p.maxDrawdownPct, false, + `OOS max drawdown ${(oosMetrics.maxDrawdownPct * 100).toFixed(1)}% (limit ${(p.maxDrawdownPct * 100).toFixed(0)}%)`)); + } else if (outOfSample.length > 0) { + outOfSampleGates.push(gate('oosMinBars', outOfSample.length, p.minBars, true, + `OOS only ${outOfSample.length} bars; need ${p.minBars}`)); + } + + // 9. Walk-forward. + const wfSplits = walkForwardSplits(input.snapshots, { + folds: p.walkForwardFolds, + trainRatio: 0.6, + minBars: p.minBars, + }); + const walkForward: WalkForwardGateBlock[] = []; + for (const fold of wfSplits) { + const testTrades = replayStrategy(plugin, fold.test, { amountPerEntry: DEFAULT_AMOUNT_PER_ENTRY, costs }); + const testSummary = summarizeTrades(testTrades); + const gates: GauntletGate[] = []; + gates.push(gateBool(`wf-f${fold.index}Profit`, testSummary.returnPct > 0, false, + `fold ${fold.index} OOS return ${(testSummary.returnPct * 100).toFixed(2)}%`)); + walkForward.push({ foldIndex: fold.index, gates, summary: testSummary }); + } + + // 10. Duplicate check. + let duplicates: ReturnType = []; + if (input.catalog && input.catalog.length > 0) { + try { + duplicates = findDuplicates(plugin, isTrades, input.catalog, input.snapshots, p.maxCatalogCorrelation); + } catch { + // non-blocking; catalogue may change under us + } + } + const dupGate = gateLe('catalogCorrelation', duplicates.length, 0, true, + duplicates.length === 0 + ? 'no catalogue duplicates found' + : `correlated with: ${duplicates.map((d) => d.strategyId).join(', ')}`); + if (duplicates.length > 0) inSampleGates.push(dupGate); + + // 11. Advisory gates. + const advisoryGates: GauntletGate[] = [ + ...outOfSampleGates.filter((g) => !g.blocking), + ]; + // Walk-forward is advisory. + for (const block of walkForward) { + advisoryGates.push(...block.gates); + } + + const blockingGates = [ + ...inSampleGates.filter((g) => g.blocking), + ...outOfSampleGates.filter((g) => g.blocking), + ]; + const passed = blockingGates.length > 0 && blockingGates.every((g) => g.passed); + + return { + passed, + candidateId, + validationErrors: [], + inSampleGates, + outOfSampleGates: outOfSampleGates.filter((g) => g.blocking), + walkForward, + deflatedSharpe: dsr, + robustness, + regimeCoverageResult: regimeResult, + duplicates, + inSampleSummary: isSummary, + outOfSampleSummary: oosSummary, + costModel: costs, + generatedAt: now, + advisoryGates, + }; +} +/* eslint-enable complexity */ + +/** Render the gauntlet report as a human-readable multi-line string. */ +export function explainReport(report: GauntletReport): string { + const lines: string[] = []; + lines.push(`Gauntlet: ${report.candidateId} — ${report.passed ? 'PASSED' : 'FAILED'}`); + lines.push(` generated: ${report.generatedAt}`); + lines.push(` costs: ${describeCostModel(report.costModel, DEFAULT_AMOUNT_PER_ENTRY)}`); + + if (report.validationErrors.length > 0) { + lines.push('\n VALIDATION ERRORS:'); + for (const e of report.validationErrors) lines.push(` - ${e}`); + } + + const render = (label: string, gates: GauntletGate[]): void => { + if (gates.length === 0) return; + lines.push(`\n ${label}:`); + for (const g of gates) { + const mark = g.passed ? '+' : '!'; + const block = g.blocking ? '' : ' (advisory)'; + lines.push(` ${mark} ${g.name}: ${g.detail}${block}`); + } + }; + + render('IN-SAMPLE GATES', report.inSampleGates); + render('OUT-OF-SAMPLE GATES', report.outOfSampleGates); + + if (report.deflatedSharpe) { + const d = report.deflatedSharpe; + lines.push(`\n DEFLATED SHARPE:`); + lines.push(` DSR: ${d.deflatedSharpeRatio.toFixed(6)} | PSR(0): ${d.probabilisticSharpeRatio.toFixed(6)}`); + lines.push(` observed Sharpe (per-trade): ${d.observedSharpe.toFixed(6)} | benchmark: ${d.expectedMaxSharpe.toFixed(6)}`); + lines.push(` nTrials: ${d.nTrials} | nObs: ${d.nObservations} | z: ${d.zScore.toFixed(3)}`); + } + + if (report.robustness) { + const r = report.robustness; + lines.push(`\n ROBUSTNESS: ${r.passed ? 'PASSED' : 'FAILED'}`); + lines.push(` base return: ${(r.baseReturnPct * 100).toFixed(2)}% | median variant: ${(r.medianReturnPct * 100).toFixed(2)}%`); + lines.push(` ${r.fractionProfitable * 100}% of ${r.effectiveVariants} effective variants profitable`); + lines.push(` degradation: ${(r.degradationPct * 100).toFixed(0)}%`); + } + + if (report.regimeCoverageResult) { + const rc = report.regimeCoverageResult; + lines.push(`\n REGIME COVERAGE: ${rc.passed ? 'PASSED' : 'FAILED'}`); + for (const b of rc.breakdown) { + if (b.trades === 0) continue; + lines.push(` ${b.regime.padEnd(10)} ${b.trades} trades net ${b.netProfit > 0 ? '+' : '-'}$${Math.abs(b.netProfit).toFixed(2)} wr ${(b.winRate * 100).toFixed(0)}%`); + } + } + + if (report.walkForward.length > 0) { + lines.push(`\n WALK-FORWARD:`); + for (const block of report.walkForward) { + lines.push(` fold ${block.foldIndex}: ${block.gates.map((g) => g.detail).join(' | ')}`); + } + } + + return lines.join('\n'); +} diff --git a/packages/strategy-validation/src/index.ts b/packages/strategy-validation/src/index.ts new file mode 100644 index 0000000..b51e295 --- /dev/null +++ b/packages/strategy-validation/src/index.ts @@ -0,0 +1,8 @@ +export * from './deflated-sharpe.js'; +export * from './metrics.js'; +export * from './splits.js'; +export * from './robustness.js'; +export * from './correlation.js'; +export * from './regime.js'; +export * from './synthetic.js'; +export * from './gauntlet.js'; diff --git a/packages/strategy-validation/src/metrics.test.ts b/packages/strategy-validation/src/metrics.test.ts new file mode 100644 index 0000000..389cad6 --- /dev/null +++ b/packages/strategy-validation/src/metrics.test.ts @@ -0,0 +1,395 @@ +import { describe, it, expect } from 'vitest'; +import { replayStrategy, ZERO_COST_MODEL } from '@b1dz/source-strategies'; +import type { StrategyPlugin } from '@b1dz/core'; +import { + DEGENERATE_RATIO_CAP, + cagr, + computeMetrics, + equityCurve, + excessKurtosis, + expectancy, + kurtosis, + maxDrawdownDuration, + maxDrawdownPct, + mean, + profitFactor, + sharpe, + skewness, + sortino, + stdev, + tradeReturns, + tradeSpanYears, + tradesPerYear, + ulcerIndex, +} from './metrics.js'; +import { DAY_MS, snapshotsFrom, syntheticTrade, syntheticTrades } from './synthetic.js'; + +const curve = (equities: number[]) => equities.map((equity, i) => ({ ts: i * DAY_MS, equity })); + +describe('mean / stdev', () => { + it('computes the arithmetic mean', () => { + expect(mean([1, 2, 3])).toBe(2); + expect(mean([-1, 1])).toBe(0); + }); + + it('uses the SAMPLE (n-1) standard deviation', () => { + // mean 5, sum of squared deviations 32, n 8. + // sample: sqrt(32/7) = 2.13809; population would be sqrt(32/8) = 2. + expect(stdev([2, 4, 4, 4, 5, 5, 7, 9])).toBeCloseTo(Math.sqrt(32 / 7), 12); + expect(stdev([2, 4, 4, 4, 5, 5, 7, 9])).not.toBeCloseTo(2, 6); + }); + + it('returns 0 rather than NaN for degenerate samples', () => { + expect(mean([])).toBe(0); + expect(stdev([])).toBe(0); + expect(stdev([5])).toBe(0); + expect(stdev([3, 3, 3])).toBe(0); + }); +}); + +describe('skewness', () => { + it('is zero for a symmetric sample', () => { + expect(skewness([1, 2, 3, 4, 5])).toBeCloseTo(0, 12); + expect(skewness([-2, -1, 0, 1, 2])).toBeCloseTo(0, 12); + }); + + it('matches the closed form for a known asymmetric sample', () => { + // [0,0,0,1]: m2 = 3/16, m3 = 3/32, skew = m3/m2^1.5 = 2/sqrt(3). + expect(skewness([0, 0, 0, 1])).toBeCloseTo(2 / Math.sqrt(3), 10); + }); + + it('is negative for a left-tailed sample (the dangerous shape)', () => { + // many small wins, one large loss — the payoff profile that flatters Sharpe. + expect(skewness([0.01, 0.01, 0.01, 0.01, 0.01, 0.01, -0.2])).toBeLessThan(-1); + }); + + it('degrades to 0 for samples too small to have a shape', () => { + expect(skewness([])).toBe(0); + expect(skewness([1, 2])).toBe(0); + expect(skewness([4, 4, 4, 4])).toBe(0); + }); +}); + +describe('kurtosis', () => { + it('is NON-excess: a two-point symmetric sample has kurtosis 1', () => { + expect(kurtosis([-1, -1, 1, 1])).toBeCloseTo(1, 12); + }); + + it('is ~3 for an approximately gaussian sample', () => { + // 9 points of a discretized normal; not exact, but must sit near 3 and + // nowhere near 0 — the whole point of the convention. + const g = [-2, -1.5, -1, -0.5, 0, 0.5, 1, 1.5, 2]; + expect(kurtosis(g)).toBeGreaterThan(1.5); + expect(kurtosis(g)).toBeLessThan(3); + expect(excessKurtosis(g)).toBeCloseTo(kurtosis(g) - 3, 12); + }); + + it('is far above 3 for a fat-tailed sample', () => { + expect(kurtosis([0, 0, 0, 0, 0, 0, 0, 0, 0, 10])).toBeGreaterThan(8); + }); + + it('defaults to the gaussian value (3) when the sample is too small', () => { + expect(kurtosis([])).toBe(3); + expect(kurtosis([1, 2, 3])).toBe(3); + expect(kurtosis([2, 2, 2, 2])).toBe(3); + }); +}); + +describe('tradeReturns', () => { + it('extracts net per-trade returns in close order', () => { + expect(tradeReturns(syntheticTrades([0.01, -0.02, 0.03]))).toEqual([0.01, -0.02, 0.03]); + }); + + it('agrees with what the real backtester produces', () => { + // Guards against BacktestTrade field drift: if `tradeReturnPct` ever stops + // being a net fraction of cash deployed, every statistic here is wrong. + const scripted: StrategyPlugin = { + manifest: { id: 's', kind: 'strategy', version: '0', name: 'S', capabilities: [] }, + evaluate(_snap, history) { + if (history.length === 0) return { side: 'buy', strength: 1, reason: 'in' }; + if (history.length === 1) return { side: 'sell', strength: 1, reason: 'out' }; + return null; + }, + }; + const trades = replayStrategy(scripted, snapshotsFrom([100, 120]), { + amountPerEntry: 100, + costs: ZERO_COST_MODEL, + }); + expect(tradeReturns(trades)).toHaveLength(1); + expect(tradeReturns(trades)[0]).toBeCloseTo(0.2, 10); + }); + + it('is empty for no trades', () => { + expect(tradeReturns([])).toEqual([]); + }); +}); + +describe('equityCurve', () => { + it('seeds at the first entry then compounds netMultiple per close', () => { + const trades = syntheticTrades([0.1, -0.1]); + const c = equityCurve(trades, 1); + expect(c).toHaveLength(3); + expect(c[0]!.equity).toBe(1); + expect(c[0]!.ts).toBe(trades[0]!.entryTs); + expect(c[1]!.equity).toBeCloseTo(1.1, 12); + expect(c[2]!.equity).toBeCloseTo(0.99, 12); // 1.1 * 0.9 — compounding, not summing + expect(c[2]!.ts).toBe(trades[1]!.exitTs); + }); + + it('respects a non-unit starting equity', () => { + const c = equityCurve(syntheticTrades([0.5]), 1000); + expect(c[0]!.equity).toBe(1000); + expect(c[1]!.equity).toBeCloseTo(1500, 9); + }); + + it('floors a total loss at zero instead of going negative', () => { + const wipeout = syntheticTrade({ profit: -100, cost: 100 }); + const c = equityCurve([wipeout], 1); + expect(c[1]!.equity).toBe(0); + }); + + it('is empty for no trades', () => { + expect(equityCurve([], 1)).toEqual([]); + }); +}); + +describe('sharpe', () => { + it('matches the closed form at per-observation frequency', () => { + // mean 0.02, sample sd 0.01 → 2.0 exactly. + expect(sharpe([0.01, 0.02, 0.03], 1)).toBeCloseTo(2, 12); + }); + + it('annualizes by sqrt(periodsPerYear)', () => { + expect(sharpe([0.01, 0.02, 0.03], 4)).toBeCloseTo(4, 12); + expect(sharpe([0.01, 0.02, 0.03], 252)).toBeCloseTo(2 * Math.sqrt(252), 10); + }); + + it('is negative for a losing return stream', () => { + expect(sharpe([-0.01, -0.02, -0.03], 1)).toBeCloseTo(-2, 12); + }); + + it('returns 0 for degenerate inputs instead of NaN or Infinity', () => { + expect(sharpe([], 252)).toBe(0); + expect(sharpe([0.05], 252)).toBe(0); + expect(sharpe([0.01, 0.01, 0.01], 252)).toBe(0); // zero dispersion + expect(Number.isFinite(sharpe([0.01, 0.02], 0))).toBe(true); + }); +}); + +describe('sortino', () => { + it('divides by the full-n target downside deviation', () => { + // excess = [0.02, -0.01, 0.03]; mean = 0.0133333 + // downside = sqrt(0.0001/3) = 0.00577350 → ratio 2.309401 + expect(sortino([0.02, -0.01, 0.03], 1)).toBeCloseTo(2.309401, 6); + }); + + it('scores higher than Sharpe when the dispersion is all upside', () => { + const r = [0.01, 0.02, 0.30, -0.01]; + expect(sortino(r, 1)).toBeGreaterThan(sharpe(r, 1)); + }); + + it('honours a non-zero minimum acceptable return', () => { + // With a 2% target, the 1% "win" becomes a shortfall. + expect(sortino([0.01, 0.03], 1, 0.02)).toBeCloseTo(0, 12); + expect(sortino([0.01, 0.01], 1, 0.02)).toBeLessThan(0); + }); + + it('caps rather than returning Infinity when there is no downside', () => { + expect(sortino([0.01, 0.02, 0.03], 1)).toBe(DEGENERATE_RATIO_CAP); + expect(sortino([], 1)).toBe(0); + expect(sortino([-0.01, -0.02], 1)).toBeLessThan(0); + expect(Number.isFinite(sortino([0, 0, 0], 1))).toBe(true); + }); +}); + +describe('maxDrawdownPct', () => { + it('measures the deepest peak-to-trough decline as a fraction', () => { + expect(maxDrawdownPct(curve([100, 120, 60, 90]))).toBeCloseTo(0.5, 12); + }); + + it('measures from the seed point, so a first losing trade counts', () => { + expect(maxDrawdownPct(equityCurve(syntheticTrades([-0.3, 0.1]), 1))).toBeCloseTo(0.3, 12); + }); + + it('is 0 for a monotonically rising curve', () => { + expect(maxDrawdownPct(curve([1, 2, 3, 4]))).toBe(0); + }); + + it('is 1 for a total wipeout and never exceeds 1', () => { + expect(maxDrawdownPct(curve([100, 0]))).toBe(1); + expect(maxDrawdownPct(curve([100, 50, 0, 25]))).toBe(1); + }); + + it('handles empty and non-positive curves', () => { + expect(maxDrawdownPct([])).toBe(0); + expect(maxDrawdownPct(curve([0, 0]))).toBe(0); + }); +}); + +describe('maxDrawdownDuration', () => { + it('counts the longest run spent below a prior peak', () => { + expect(maxDrawdownDuration(curve([1, 2, 1.5, 1.4, 1.9, 2.5, 2.4]))).toBe(3); + }); + + it('is 0 when the curve never retreats', () => { + expect(maxDrawdownDuration(curve([1, 1, 2, 3]))).toBe(0); + expect(maxDrawdownDuration([])).toBe(0); + }); +}); + +describe('profitFactor', () => { + it('is gross wins over gross losses', () => { + const trades = [ + syntheticTrade({ profit: 20 }), + syntheticTrade({ profit: 10 }), + syntheticTrade({ profit: -10 }), + ]; + expect(profitFactor(trades)).toBeCloseTo(3, 12); + }); + + it('is 1 at break-even', () => { + expect(profitFactor([syntheticTrade({ profit: 10 }), syntheticTrade({ profit: -10 })])).toBe(1); + }); + + it('is below 1 for a fee-generation machine', () => { + expect(profitFactor([syntheticTrade({ profit: 5 }), syntheticTrade({ profit: -10 })])).toBeCloseTo(0.5, 12); + }); + + it('caps instead of returning Infinity when nothing lost', () => { + expect(profitFactor([syntheticTrade({ profit: 10 })])).toBe(DEGENERATE_RATIO_CAP); + expect(profitFactor([])).toBe(0); + expect(profitFactor([syntheticTrade({ profit: 0 })])).toBe(0); + }); +}); + +describe('expectancy', () => { + it('is the mean net return per trade', () => { + expect(expectancy(syntheticTrades([0.02, -0.01, 0.05]))).toBeCloseTo(0.02, 12); + }); + + it('is negative for a high-win-rate strategy that gives it all back', () => { + // 9 wins of +1%, one loss of -15%: 90% win rate, negative expectancy. + const trades = syntheticTrades([...Array(9).fill(0.01), -0.15]); + expect(trades.filter((t) => t.profit > 0)).toHaveLength(9); + expect(expectancy(trades)).toBeLessThan(0); + }); + + it('is 0 for no trades', () => { + expect(expectancy([])).toBe(0); + }); +}); + +describe('cagr', () => { + it('matches the closed form', () => { + expect(cagr(100, 121, 2)).toBeCloseTo(0.1, 12); + expect(cagr(100, 200, 1)).toBeCloseTo(1, 12); + expect(cagr(100, 100, 5)).toBeCloseTo(0, 12); + }); + + it('reports a wipeout as -1 rather than NaN', () => { + // Math.pow(negative, 1/2) is NaN, and a NaN silently passes every + // `>= threshold` check it is compared against. + expect(cagr(100, 0, 2)).toBe(-1); + expect(cagr(100, -50, 2)).toBe(-1); + }); + + it('returns 0 when there is no measurable period or capital', () => { + expect(cagr(100, 200, 0)).toBe(0); + expect(cagr(100, 200, -1)).toBe(0); + expect(cagr(0, 200, 1)).toBe(0); + }); +}); + +describe('ulcerIndex', () => { + it('matches the RMS-of-drawdowns closed form', () => { + // drawdowns 0, 0.1, 0 → sqrt(0.01/3) = 0.0577350 + expect(ulcerIndex(curve([100, 90, 100]))).toBeCloseTo(0.057735, 6); + }); + + it('is 0 for a curve that never draws down', () => { + expect(ulcerIndex(curve([1, 2, 3]))).toBe(0); + expect(ulcerIndex([])).toBe(0); + }); + + it('separates a brief dip from a long stay underwater', () => { + const brief = ulcerIndex(curve([100, 80, 100, 100, 100, 100])); + const lingering = ulcerIndex(curve([100, 80, 82, 85, 88, 90])); + expect(lingering).toBeGreaterThan(brief); + // ...even though both bottom out at the same -20%. + expect(maxDrawdownPct(curve([100, 80, 100, 100, 100, 100]))).toBeCloseTo(0.2, 12); + expect(maxDrawdownPct(curve([100, 80, 82, 85, 88, 90]))).toBeCloseTo(0.2, 12); + }); +}); + +describe('tradeSpanYears / tradesPerYear', () => { + it('measures the span from first entry to last exit', () => { + const trades = syntheticTrades([0.01, 0.01], { stepMs: 365.25 * DAY_MS }); + // entry at 0, exit of the second trade at 2 * 365.25 days. + expect(tradeSpanYears(trades)).toBeCloseTo(2, 9); + expect(tradesPerYear(trades)).toBeCloseTo(1, 9); + }); + + it('returns 0 for spans that cannot be measured', () => { + expect(tradeSpanYears([])).toBe(0); + expect(tradesPerYear([])).toBe(0); + expect(tradesPerYear([syntheticTrade({ profit: 1, entryTs: 5, exitTs: 5 })])).toBe(0); + }); +}); + +describe('computeMetrics', () => { + it('produces a fully finite block for a realistic trade stream', () => { + const trades = syntheticTrades([0.03, -0.01, 0.02, -0.02, 0.04, 0.01, -0.03, 0.02]); + const m = computeMetrics(trades, 1); + expect(m.trades).toBe(8); + expect(m.sharpePerTrade).toBeCloseTo(sharpe(tradeReturns(trades), 1), 12); + expect(m.profitFactor).toBeCloseTo(12 / 6, 12); + expect(m.expectancy).toBeCloseTo(0.0075, 12); + expect(m.kurtosis).toBeGreaterThan(0); + for (const [key, value] of Object.entries(m)) { + expect(Number.isFinite(value), `${key} must be finite`).toBe(true); + } + }); + + it('annualizes using the observed trade frequency, not a hard-coded 252', () => { + // 4 trades spread over ~4 years is ~1 observation/year, so the annualized + // Sharpe must stay close to the per-trade figure — not 16x it. + const yearly = syntheticTrades([0.1, -0.05, 0.2, 0.05], { stepMs: 365.25 * DAY_MS }); + const m = computeMetrics(yearly, 1); + expect(m.tradesPerYear).toBeCloseTo(1, 3); + expect(m.sharpeAnnualized).toBeCloseTo(m.sharpePerTrade, 2); + }); + + it('produces a fully finite block for NO trades at all', () => { + const m = computeMetrics([], 1); + for (const [key, value] of Object.entries(m)) { + expect(Number.isFinite(value), `${key} must be finite`).toBe(true); + } + expect(m.trades).toBe(0); + expect(m.finalEquity).toBe(1); + expect(m.cagr).toBe(0); + }); + + it('produces a fully finite block for a single trade', () => { + const m = computeMetrics([syntheticTrade({ profit: 10 })], 1); + for (const [key, value] of Object.entries(m)) { + expect(Number.isFinite(value), `${key} must be finite`).toBe(true); + } + }); + + it('produces a fully finite block for a total wipeout', () => { + const m = computeMetrics([syntheticTrade({ profit: -100, cost: 100 })], 1); + for (const [key, value] of Object.entries(m)) { + expect(Number.isFinite(value), `${key} must be finite`).toBe(true); + } + expect(m.finalEquity).toBe(0); + expect(m.maxDrawdownPct).toBe(1); + }); + + it('survives JSON serialization with no nulls (Infinity would become null)', () => { + const m = computeMetrics([syntheticTrade({ profit: 10 })], 1); + const round = JSON.parse(JSON.stringify(m)) as Record; + for (const [key, value] of Object.entries(round)) { + expect(value, `${key} must survive JSON`).not.toBeNull(); + } + }); +}); diff --git a/packages/strategy-validation/src/metrics.ts b/packages/strategy-validation/src/metrics.ts new file mode 100644 index 0000000..07d689c --- /dev/null +++ b/packages/strategy-validation/src/metrics.ts @@ -0,0 +1,439 @@ +/** + * Risk-adjusted performance metrics over a net-of-cost trade list. + * + * `summarizeTrades()` in @b1dz/source-strategies answers "how much money did it + * make". That is the number a seller quotes and it is close to useless on its + * own: total return says nothing about how much pain you had to sit through to + * collect it, or whether the result is distinguishable from luck. A strategy + * that returns 40% with a 60% drawdown and four trades is not a product, it is + * a coin flip with good PR. + * + * So this module produces the *shape* of the return stream — dispersion, + * downside, drawdown depth, drawdown duration, per-trade expectancy — plus the + * distributional moments (skew, kurtosis) that ./deflated-sharpe.ts needs to + * decide whether the Sharpe means anything at all. + * + * The failure mode being prevented: every one of these formulas has a zero + * denominator or a fractional power of a negative number lurking in it, and + * every one of those produces NaN or Infinity rather than an exception. A NaN + * silently fails every `>=` comparison, so a NaN metric turns a *blocking* + * quality gate into a gate that always passes. Infinity survives arithmetic and + * then becomes `null` on JSON.stringify, so it corrupts a stored report instead + * of the process that produced it. Both leak all the way to a listing page. + * Every function here therefore returns a finite number for every input, + * including no input, and the degenerate branches are named and tested. + * + * Conventions, because mixing these up is how confident wrong numbers happen: + * - Returns are FRACTIONAL, not percent: 0.02 is +2%. (`tradeReturnPct` on + * BacktestTrade is already fractional despite the name.) + * - Dispersion uses the SAMPLE standard deviation (n−1 denominator). We are + * estimating a population from a sample, and with 30 trades the difference + * is ~1.7% of the Sharpe. + * - `kurtosis()` is NON-EXCESS (Gaussian = 3), because that is what the + * Probabilistic Sharpe Ratio formula expects. Read its doc comment before + * wiring it to anything else. + * - Annualization is `× sqrt(periodsPerYear)`, and `periodsPerYear` must match + * the observation frequency of `returns`. Per-TRADE returns are not daily + * returns; pass `1` to get a per-observation figure. + */ +import type { BacktestTrade } from '@b1dz/source-strategies'; + +/** A point on a compounded equity curve. */ +export interface EquityPoint { + /** Epoch ms of the bar that produced this equity level. */ + ts: number; + equity: number; +} + +/** + * Finite stand-in for a ratio whose denominator is legitimately zero — a sample + * with wins and no losses at all. + * + * The true answer is +∞ and the true cause is almost always a tiny sample, not + * a risk-free money machine. Returning 0 would be wrong in the other direction + * (it reads as "terrible" and would fail a gate that the sample cannot actually + * fail), and returning Infinity poisons JSON. A large finite number is the only + * option that is both honest about direction and safe downstream. It is + * deliberately absurd-looking so it is recognisable in a report as "undefined", + * not as a real measurement. + */ +export const DEGENERATE_RATIO_CAP = 100; + +/** Trading days in a year — the default annualization factor for daily bars. */ +export const TRADING_DAYS_PER_YEAR = 252; + +const MS_PER_YEAR = 365.25 * 24 * 60 * 60 * 1000; + +/** Arithmetic mean. Empty → 0. */ +export function mean(values: number[]): number { + if (values.length === 0) return 0; + let sum = 0; + for (const v of values) sum += v; + return sum / values.length; +} + +/** + * Sample standard deviation (n−1 denominator). Needs at least 2 observations to + * mean anything; 0 or 1 observations → 0, which makes every ratio built on it + * collapse to 0 rather than to NaN. + */ +export function stdev(values: number[]): number { + const n = values.length; + if (n < 2) return 0; + const m = mean(values); + let ss = 0; + for (const v of values) ss += (v - m) * (v - m); + return Math.sqrt(ss / (n - 1)); +} + +/** + * Fisher–Pearson sample skewness (third standardized moment, biased/population + * form: m3 / m2^1.5). + * + * Negative skew is the one that matters commercially: a strategy that wins small + * many times and loses huge occasionally (short volatility, martingale + * averaging-down, naked premium selling) has a flattering Sharpe and a fat left + * tail. The PSR formula in ./deflated-sharpe.ts uses this to *penalise* exactly + * that shape, which is why we compute it rather than assuming normality. + */ +export function skewness(values: number[]): number { + const n = values.length; + if (n < 3) return 0; + const m = mean(values); + let m2 = 0; + let m3 = 0; + for (const v of values) { + const d = v - m; + m2 += d * d; + m3 += d * d * d; + } + m2 /= n; + m3 /= n; + if (m2 <= 0) return 0; + return m3 / Math.pow(m2, 1.5); +} + +/** + * NON-EXCESS kurtosis (fourth standardized moment, m4 / m2²). A Gaussian sample + * returns ≈ 3, NOT ≈ 0. + * + * This convention is not a preference, it is a requirement: Bailey & López de + * Prado write the PSR denominator as `1 − γ3·SR + ((γ4 − 1)/4)·SR²`, and that + * term only reduces to the textbook `1 + SR²/2` Gaussian variance when γ4 = 3. + * Feed it excess kurtosis and the denominator becomes `1 − SR²/4`, which is + * *smaller* than the Gaussian case — i.e. fat tails would make a strategy look + * MORE statistically significant. Use `excessKurtosis()` for anything that + * expects the 0-centred convention. + */ +export function kurtosis(values: number[]): number { + const n = values.length; + if (n < 4) return 3; + const m = mean(values); + let m2 = 0; + let m4 = 0; + for (const v of values) { + const d = v - m; + m2 += d * d; + m4 += d * d * d * d; + } + m2 /= n; + m4 /= n; + if (m2 <= 0) return 3; + return m4 / (m2 * m2); +} + +/** Excess kurtosis (Gaussian = 0). Convenience wrapper; see `kurtosis()`. */ +export function excessKurtosis(values: number[]): number { + return kurtosis(values) - 3; +} + +/** + * Per-trade NET returns on cash deployed, in trade-close order. + * + * These are the observations every statistic in this package is built on, and + * they are per-TRADE, not per-bar or per-day. That distinction drives + * annualization and drives `nObservations` in the deflated Sharpe: 30 trades is + * 30 observations no matter how many years they span. + */ +export function tradeReturns(trades: BacktestTrade[]): number[] { + return trades.map((t) => t.tradeReturnPct); +} + +/** + * Compounded equity curve: one seed point at the first entry, then one point per + * trade close, multiplying by that trade's `netMultiple` (proceeds / cost). + * + * Compounding rather than summing profits is the honest choice for a strategy + * that will be run with a fixed *fraction* of a bankroll, and it is also the + * stricter one — a −50% drawdown needs +100% to recover, and an additive curve + * hides that asymmetry. + * + * The seed point exists so drawdown can be measured from the starting capital: a + * strategy whose very first trade loses 30% has a 30% drawdown, and a curve that + * begins at the first *exit* would report zero. + */ +export function equityCurve(trades: BacktestTrade[], startingEquity = 1): EquityPoint[] { + if (trades.length === 0) return []; + const start = startingEquity > 0 ? startingEquity : 1; + const out: EquityPoint[] = [{ ts: trades[0]!.entryTs, equity: start }]; + let equity = start; + for (const t of trades) { + // netMultiple is 1 for a degenerate zero-cost trade, so equity never hits 0 + // by accident; a real −100% trade still floors the curve at 0 correctly. + equity *= Math.max(0, t.netMultiple); + out.push({ ts: t.exitTs, equity }); + } + return out; +} + +/** + * Annualized Sharpe ratio: mean(returns) / stdev(returns) × sqrt(periodsPerYear). + * + * No risk-free rate is subtracted. For per-trade returns on a strategy that is + * flat most of the time, the cash rate applies to un-deployed capital rather + * than to the trade, so subtracting it per-trade would be double counting. + * + * `periodsPerYear` MUST match the frequency of `returns`. Pass 1 for a raw + * per-observation Sharpe — that is the unit ./deflated-sharpe.ts requires, and + * feeding it an annualized number inflates significance by sqrt(252). + * + * Degenerate: fewer than 2 observations, or zero dispersion, → 0. Zero + * dispersion means every trade returned exactly the same amount, which is a + * fixture or a bug, not an infinite-Sharpe discovery. + */ +export function sharpe(returns: number[], periodsPerYear = TRADING_DAYS_PER_YEAR): number { + if (returns.length < 2) return 0; + const sd = stdev(returns); + if (!(sd > 0)) return 0; + const scale = periodsPerYear > 0 ? Math.sqrt(periodsPerYear) : 1; + return (mean(returns) / sd) * scale; +} + +/** + * Annualized Sortino ratio: mean(returns − target) / target-downside-deviation. + * + * Sharpe punishes upside dispersion, which is incoherent — nobody has ever + * complained about an unexpectedly large winner. Sortino replaces the + * denominator with sqrt(mean(min(r − target, 0)²)), where the mean is over ALL + * n observations (not just the losing ones). That full-n denominator is the + * textbook definition and it matters: dividing by the loss count instead would + * make a strategy look better the fewer losses it had, which is the same + * small-sample flattery we are trying to remove. + * + * Degenerate: no losing observation at all → `DEGENERATE_RATIO_CAP` when the + * mean is positive (undefined, not infinite), 0 otherwise. + */ +export function sortino( + returns: number[], + periodsPerYear = TRADING_DAYS_PER_YEAR, + target = 0, +): number { + if (returns.length < 2) return 0; + const excess = returns.map((r) => r - target); + let ss = 0; + for (const e of excess) if (e < 0) ss += e * e; + const downside = Math.sqrt(ss / returns.length); + const m = mean(excess); + const scale = periodsPerYear > 0 ? Math.sqrt(periodsPerYear) : 1; + if (!(downside > 0)) return m > 0 ? DEGENERATE_RATIO_CAP : 0; + return (m / downside) * scale; +} + +/** + * Deepest peak-to-trough decline on the equity curve, as a FRACTION (0.25 = a + * 25% drawdown). + * + * Fractional and compounded, so it is comparable across bankroll sizes and + * across strategies — unlike `BacktestSummary.maxDrawdown`, which is a dollar + * figure on a fixed per-trade notional and therefore only comparable to itself. + * + * This is the number that decides whether a buyer actually holds the strategy + * long enough to collect its expectancy. Return is theoretical; drawdown is what + * makes people switch it off at the bottom. + */ +export function maxDrawdownPct(curve: EquityPoint[]): number { + let peak = 0; + let worst = 0; + for (const p of curve) { + if (p.equity > peak) peak = p.equity; + if (peak > 0) { + const dd = (peak - p.equity) / peak; + if (dd > worst) worst = dd; + } + } + return worst; +} + +/** Longest run of consecutive curve points spent below a prior peak. */ +export function maxDrawdownDuration(curve: EquityPoint[]): number { + let peak = Number.NEGATIVE_INFINITY; + let run = 0; + let worst = 0; + for (const p of curve) { + if (p.equity >= peak) { + peak = p.equity; + run = 0; + } else { + run += 1; + if (run > worst) worst = run; + } + } + return worst; +} + +/** + * Profit factor: gross winnings / gross losses, both net of costs. + * + * The cleanest single answer to "is there an edge here", because it is immune to + * position sizing and to trade count. 1.0 is break-even. Below 1.0 the strategy + * is a fee-generation machine. + * + * Wins and losses are split on NET profit, so a trade that captured 10 bps of + * price movement and paid 60 bps of friction counts in the loss pile — which is + * the entire point of running the backtester with a cost model. + * + * Degenerate: no losses at all → `DEGENERATE_RATIO_CAP` (undefined, capped); + * no wins → 0. + */ +export function profitFactor(trades: BacktestTrade[]): number { + let wins = 0; + let losses = 0; + for (const t of trades) { + if (t.profit > 0) wins += t.profit; + else if (t.profit < 0) losses -= t.profit; + } + if (!(losses > 0)) return wins > 0 ? DEGENERATE_RATIO_CAP : 0; + return wins / losses; +} + +/** + * Expectancy: mean NET return per trade, as a fraction of cash deployed. + * + * The break-even test that win rate cannot fake. A 90%-win-rate strategy with + * negative expectancy is a strategy that gives back nine small wins on one large + * loss, and it will be marketed on the 90%. + */ +export function expectancy(trades: BacktestTrade[]): number { + return mean(tradeReturns(trades)); +} + +/** + * Compound annual growth rate from start/end equity over `years`. + * + * Guards, in the order they bite: + * - `years` ≤ 0 or `startEquity` ≤ 0 → 0 (no measurable period; a division and + * a fractional root would both blow up). + * - `endEquity` ≤ 0 → −1, i.e. total loss. Left unguarded this computes a + * fractional power of a negative number, which is NaN, which then passes + * every threshold check it is compared against. A wiped-out strategy + * silently clearing a return gate is the exact failure this package exists + * to stop. + */ +export function cagr(startEquity: number, endEquity: number, years: number): number { + if (!(years > 0) || !(startEquity > 0)) return 0; + if (!(endEquity > 0)) return -1; + return Math.pow(endEquity / startEquity, 1 / years) - 1; +} + +/** + * Ulcer Index: RMS of the drawdown series, as a FRACTION (not ×100 like the + * original Martin & McCann formulation — kept fractional for consistency with + * `maxDrawdownPct`). + * + * Max drawdown is a single worst-case sample and is therefore noisy: one bad + * week defines it. Ulcer integrates depth *and* duration over the whole curve, + * so it separates "dropped 20% once and recovered immediately" from "sat 15% + * underwater for two years". The second one is the one buyers abandon. + */ +export function ulcerIndex(curve: EquityPoint[]): number { + if (curve.length === 0) return 0; + let peak = 0; + let ss = 0; + for (const p of curve) { + if (p.equity > peak) peak = p.equity; + if (peak > 0) { + const dd = (peak - p.equity) / peak; + ss += dd * dd; + } + } + return Math.sqrt(ss / curve.length); +} + +/** Elapsed years spanned by a trade list, from first entry to last exit. */ +export function tradeSpanYears(trades: BacktestTrade[]): number { + if (trades.length === 0) return 0; + let first = Number.POSITIVE_INFINITY; + let last = Number.NEGATIVE_INFINITY; + for (const t of trades) { + if (t.entryTs < first) first = t.entryTs; + if (t.exitTs > last) last = t.exitTs; + } + const span = last - first; + return span > 0 ? span / MS_PER_YEAR : 0; +} + +/** + * Observations per year implied by a trade list — the correct `periodsPerYear` + * for annualizing a per-trade Sharpe. + * + * Hard-coding 252 for per-trade returns is a common and expensive error: a + * strategy that takes 30 trades over three years has ~10 observations per year, + * and annualizing it as if it had 252 overstates the Sharpe by sqrt(25) = 5×. + * + * Degenerate: an instantaneous or single-trade span → 0, which makes `sharpe()` + * fall back to a per-observation figure instead of inventing a frequency. + */ +export function tradesPerYear(trades: BacktestTrade[]): number { + const years = tradeSpanYears(trades); + if (!(years > 0)) return 0; + return trades.length / years; +} + +/** Everything above, computed once, for a report block. */ +export interface MetricSet { + trades: number; + /** Per-observation (per-trade) Sharpe. The unit deflation math requires. */ + sharpePerTrade: number; + /** Sharpe scaled by the trade frequency actually observed in the data. */ + sharpeAnnualized: number; + sortinoAnnualized: number; + profitFactor: number; + expectancy: number; + maxDrawdownPct: number; + maxDrawdownDuration: number; + ulcerIndex: number; + cagr: number; + skewness: number; + /** Non-excess (Gaussian = 3). */ + kurtosis: number; + spanYears: number; + tradesPerYear: number; + finalEquity: number; +} + +/** Compute the full metric block from a trade list. Never throws, never NaNs. */ +export function computeMetrics(trades: BacktestTrade[], startingEquity = 1): MetricSet { + const returns = tradeReturns(trades); + const curve = equityCurve(trades, startingEquity); + const perYear = tradesPerYear(trades); + const years = tradeSpanYears(trades); + const finalEquity = curve.length ? curve[curve.length - 1]!.equity : startingEquity; + + return { + trades: trades.length, + sharpePerTrade: sharpe(returns, 1), + sharpeAnnualized: sharpe(returns, perYear), + sortinoAnnualized: sortino(returns, perYear), + profitFactor: profitFactor(trades), + expectancy: expectancy(trades), + maxDrawdownPct: maxDrawdownPct(curve), + maxDrawdownDuration: maxDrawdownDuration(curve), + ulcerIndex: ulcerIndex(curve), + cagr: cagr(startingEquity, finalEquity, years), + skewness: skewness(returns), + kurtosis: kurtosis(returns), + spanYears: years, + tradesPerYear: perYear, + finalEquity, + }; +} diff --git a/packages/strategy-validation/src/regime.test.ts b/packages/strategy-validation/src/regime.test.ts new file mode 100644 index 0000000..b9f4133 --- /dev/null +++ b/packages/strategy-validation/src/regime.test.ts @@ -0,0 +1,152 @@ +import { describe, it, expect } from 'vitest'; +import { snapshotsFrom, trendPrices, randomWalkPrices, syntheticTrades } from './synthetic.js'; +import { classifyRegimes, regimeBreakdown, regimeCoverage } from './regime.js'; + +describe('classifyRegimes', () => { + it('classifies a persistent uptrend as mostly uptrend', () => { + const prices = trendPrices({ bars: 500, driftPerBar: 0.002, noise: 0.002, seed: 42 }); + const snaps = snapshotsFrom(prices); + const regimes = classifyRegimes(snaps); + expect(regimes.length).toBe(snaps.length); + + const counts: Record = {}; + for (const r of regimes) counts[r] = (counts[r] ?? 0) + 1; + expect(counts.uptrend ?? 0).toBeGreaterThan(counts.downtrend ?? 0); + expect(counts.downtrend ?? 0).toBeLessThan(50); + }); + + it('classifies a persistent downtrend as mostly downtrend', () => { + const prices = trendPrices({ bars: 500, driftPerBar: -0.002, noise: 0.002, seed: 7 }); + const snaps = snapshotsFrom(prices); + const regimes = classifyRegimes(snaps); + const counts: Record = {}; + for (const r of regimes) counts[r] = (counts[r] ?? 0) + 1; + expect(counts.downtrend ?? 0).toBeGreaterThan(counts.uptrend ?? 0); + }); + + it('classifies a random walk with low vol as mostly ranging', () => { + const prices = randomWalkPrices({ bars: 500, vol: 0.008, seed: 99 }); + const snaps = snapshotsFrom(prices); + const regimes = classifyRegimes(snaps); + const counts: Record = {}; + for (const r of regimes) counts[r] = (counts[r] ?? 0) + 1; + expect(counts.ranging ?? 0).toBeGreaterThan(0); + }); + + it('fills the warmup period with ranging', () => { + const prices = trendPrices({ bars: 100 }); + const snaps = snapshotsFrom(prices); + const regimes = classifyRegimes(snaps, { trendPeriod: 50 }); + for (let i = 0; i < 50; i++) { + expect(regimes[i]).toBe('ranging'); + } + }); + + it('returns all ranging when snapshots are too short', () => { + const prices = trendPrices({ bars: 30 }); + const snaps = snapshotsFrom(prices); + const regimes = classifyRegimes(snaps, { trendPeriod: 50 }); + expect(regimes.length).toBe(30); + for (const r of regimes) expect(r).toBe('ranging'); + }); + + it('handles empty snapshots', () => { + expect(classifyRegimes([])).toEqual([]); + }); +}); + +describe('regimeBreakdown', () => { + it('buckets trades by the regime at entry_ts', () => { + const snaps = snapshotsFrom(trendPrices({ bars: 200, driftPerBar: 0.003, seed: 3 })); + const regimes = classifyRegimes(snaps); + const trades = syntheticTrades([0.01, -0.01], { + startTs: snaps[60]!.ts, + stepMs: 24 * 60 * 60 * 1000, + }); + const breakdown = regimeBreakdown(trades, regimes, snaps); + expect(breakdown.length).toBe(4); + expect(breakdown.reduce((s, b) => s + b.trades, 0)).toBe(trades.length); + }); + + it('assigns trades with no matching snapshot to ranging', () => { + const snaps = snapshotsFrom([100, 101]); + const regimes = ['uptrend', 'uptrend'] as const; + const trades = syntheticTrades([0.01], { startTs: 999999999999, stepMs: 1000 }); + const breakdown = regimeBreakdown(trades, [...regimes], snaps); + const ranging = breakdown.find((b) => b.regime === 'ranging')!; + expect(ranging.trades).toBe(1); + }); + + it('returns all-zero entries for regimes with no trades', () => { + const snaps = snapshotsFrom([100, 101]); + const regimes = ['uptrend', 'uptrend'] as const; + const breakdown = regimeBreakdown([], [...regimes], snaps); + expect(breakdown.length).toBe(4); + for (const b of breakdown) { + expect(b.trades).toBe(0); + expect(b.netProfit).toBe(0); + } + }); +}); + +describe('regimeCoverage', () => { + it('passes when enough regimes are profitable', () => { + const breakdown = [ + { regime: 'uptrend' as const, trades: 10, netProfit: 50, returnPct: 0.1, winRate: 0.6 }, + { regime: 'downtrend' as const, trades: 5, netProfit: 10, returnPct: 0.05, winRate: 0.4 }, + { regime: 'ranging' as const, trades: 0, netProfit: 0, returnPct: 0, winRate: 0 }, + { regime: 'volatile' as const, trades: 0, netProfit: 0, returnPct: 0, winRate: 0 }, + ]; + const result = regimeCoverage(breakdown, 2); + expect(result.passed).toBe(true); + expect(result.profitableRegimes).toEqual(['uptrend', 'downtrend']); + }); + + it('fails when too few regimes are profitable', () => { + const breakdown = [ + { regime: 'uptrend' as const, trades: 10, netProfit: 50, returnPct: 0.1, winRate: 0.6 }, + { regime: 'downtrend' as const, trades: 5, netProfit: -20, returnPct: -0.05, winRate: 0.2 }, + { regime: 'ranging' as const, trades: 0, netProfit: 0, returnPct: 0, winRate: 0 }, + { regime: 'volatile' as const, trades: 0, netProfit: 0, returnPct: 0, winRate: 0 }, + ]; + const result = regimeCoverage(breakdown, 2); + expect(result.passed).toBe(false); + expect(result.profitableRegimes).toEqual(['uptrend']); + }); + + it('defaults to requiring 2 profitable regimes', () => { + const breakdown = [ + { regime: 'uptrend' as const, trades: 10, netProfit: 50, returnPct: 0.1, winRate: 0.6 }, + { regime: 'downtrend' as const, trades: 5, netProfit: 10, returnPct: 0.05, winRate: 0.4 }, + { regime: 'ranging' as const, trades: 0, netProfit: 0, returnPct: 0, winRate: 0 }, + { regime: 'volatile' as const, trades: 0, netProfit: 0, returnPct: 0, winRate: 0 }, + ]; + const result = regimeCoverage(breakdown); + expect(result.passed).toBe(true); + }); + + it('passes even when some regimes show zero returnPct if profitable entries exist', () => { + // returnPct = 0 means not profitable (because profit > 0 check is on returnPct > 0) + const breakdown = [ + { regime: 'uptrend' as const, trades: 5, netProfit: 100, returnPct: 0.2, winRate: 0.8 }, + { regime: 'downtrend' as const, trades: 1, netProfit: 0.01, returnPct: 0.001, winRate: 1.0 }, + { regime: 'ranging' as const, trades: 0, netProfit: 0, returnPct: 0, winRate: 0 }, + { regime: 'volatile' as const, trades: 0, netProfit: 0, returnPct: 0, winRate: 0 }, + ]; + const result = regimeCoverage(breakdown, 2); + expect(result.passed).toBe(true); + expect(result.profitableRegimes.length).toBe(2); + }); + + it('does not count a regime with trades but zero returnPct as profitable', () => { + const breakdown = [ + { regime: 'uptrend' as const, trades: 5, netProfit: 100, returnPct: 0.2, winRate: 0.8 }, + { regime: 'downtrend' as const, trades: 5, netProfit: -50, returnPct: -0.1, winRate: 0.2 }, + { regime: 'ranging' as const, trades: 3, netProfit: 0, returnPct: 0, winRate: 0.33 }, + { regime: 'volatile' as const, trades: 0, netProfit: 0, returnPct: 0, winRate: 0 }, + ]; + const result = regimeCoverage(breakdown, 2); + expect(result.passed).toBe(false); + expect(result.profitableRegimes).toEqual(['uptrend']); + }); +}); diff --git a/packages/strategy-validation/src/regime.ts b/packages/strategy-validation/src/regime.ts new file mode 100644 index 0000000..fba6fb0 --- /dev/null +++ b/packages/strategy-validation/src/regime.ts @@ -0,0 +1,185 @@ +/** + * Lightweight market-regime classifier — break the backtest into conditions the + * strategy actually saw, so a listing isn't just "rode a bull run, ignored a + * crash". + * + * WHY THIS EXISTS + * + * Every backtest ever published looks best in hindsight data that trends + * persistently in one direction. A strategy that returned 80% from March 2020 + * to December 2021 and −62% from Jan 2022 to Dec 2022 is not an 18% annualised + * strategy — it is a bull strategy with a marketing department that chose the + * window. A gate that requires profitability in more than one regime forces the + * listing to prove it works in at least TWO of uptrend / downtrend / ranging / + * volatile, which catches the single-regime-window problem without needing the + * window to have been adversarially chosen. + * + * THE CLASSIFIER + * + * `classifyRegimes()` uses two things every bar already has: an EMA slope (is + * the trend pointing up or down, and how hard?) and a realised-volatility proxy + * (how wide did it swing recently?). Together those four quadrants — high/low + * drift × high/low vol — produce a label that is cheap, stateless, and requires + * no peek into the future. It is NOT a full Markov-switching model; it is + * deliberate that it uses only data the strategy itself could have seen at the + * moment of its own entry signal. + * + * This module imports nothing beyond @b1dz/core (ema) because the heavy + * exchange-specific packages pull in database drivers and infra that do not + * belong in a stateless validator. + */ +import { ema } from '@b1dz/core'; +import type { MarketSnapshot } from '@b1dz/core'; +import type { BacktestTrade, BacktestSummary } from '@b1dz/source-strategies'; + +export type Regime = 'uptrend' | 'downtrend' | 'ranging' | 'volatile'; + +export interface RegimeClassifierOptions { + /** EMA period for trend slope. Higher = smoother regime transitions. Default 50. */ + trendPeriod?: number; + /** Short EMA for per-bar volatility proxy. Default 5. */ + volPeriod?: number; + /** Annualised % drift needed to count as an uptrend. Default 0.1 (10%/yr). */ + trendThreshold?: number; + /** Annualised vol at/below which a bar is "calm". Default 0.25 (25%/yr). */ + calmVolThreshold?: number; +} + +/** + * Classify every bar into one of four regimes using only backward-looking data. + * + * Regime at bar i is determined by: + * 1. EMA slope over the last 2 × trendPeriod bars (annualised, T−1 to T). + * Positive + steep → trending; near zero → ranging. + * 2. Short EMA of bar-to-bar log-return magnitude, annualised. + * High → the bar is "volatile". + * + * The combination: + * - uptrend: slope > +threshold, vol ≤ calm (orderly grind up) + * - downtrend: slope < −threshold, vol ≤ calm (orderly sell-off) + * - ranging: |slope| ≤ threshold, vol ≤ calm (sideways chop) + * - volatile: vol > calm (disorderly moves in either direction) + */ +export function classifyRegimes( + snapshots: MarketSnapshot[], + opts: RegimeClassifierOptions = {}, +): Regime[] { + const trendPeriod = opts.trendPeriod ?? 50; + const volPeriod = opts.volPeriod ?? 5; + const trendThreshold = opts.trendThreshold ?? 0.1; + const calmVolThreshold = opts.calmVolThreshold ?? 0.25; + + if (snapshots.length < trendPeriod + 1) { + return new Array(snapshots.length).fill('ranging'); + } + + // mid prices — mean of bid/ask, safe for zero-spread daily bars. + const mids = snapshots.map((s) => (s.bid + s.ask) / 2); + const trendEma = ema(mids, trendPeriod) as number[]; + + // per-bar log returns of the mid, for volatility estimation. + const logRets = mids.map((m, i) => (i > 0 && mids[i - 1]! > 0 ? Math.log(m / mids[i - 1]!) : 0)); + const absLogRets = logRets.map(Math.abs); + const volEma = ema(absLogRets, volPeriod) as number[]; + + const tradingDaysPerYear = 252; + const dailyThreshold = trendThreshold / tradingDaysPerYear; + const dailyCalm = calmVolThreshold / Math.sqrt(tradingDaysPerYear); + + const out: Regime[] = []; + for (let i = 0; i < snapshots.length; i++) { + if (i < trendPeriod) { + out.push('ranging'); + continue; + } + const slope = (trendEma[i]! - trendEma[i - 1]!) / trendEma[i - 1]!; + const vol = volEma[i]!; + + if (vol > dailyCalm) { + out.push('volatile'); + } else if (slope > dailyThreshold) { + out.push('uptrend'); + } else if (slope < -dailyThreshold) { + out.push('downtrend'); + } else { + out.push('ranging'); + } + } + return out; +} + +export interface RegimeBreakdownEntry { + regime: Regime; + trades: number; + netProfit: number; + returnPct: number; + winRate: number; +} + +/** + * Bucket trades by the regime at each trade's ENTRY bar. + * + * A trade that entered during an uptrend and exited during a crash was opened + * by the uptrend regime — that is the condition the strategy chose to enter + * under, and the result communicates "how does this strategy perform when it + * behaves this way in this kind of market". + */ +export function regimeBreakdown( + trades: BacktestTrade[], + regimes: Regime[], + snapshots: MarketSnapshot[], +): RegimeBreakdownEntry[] { + const buckets = new Map(); + + for (const regime of ['uptrend', 'downtrend', 'ranging', 'volatile'] as Regime[]) { + buckets.set(regime, { count: 0, profit: 0, costSum: 0, wins: 0 }); + } + + for (const t of trades) { + const idx = snapshots.findIndex((s) => s.ts === t.entryTs); + const regime = idx >= 0 && idx < regimes.length ? regimes[idx]! : ('ranging' as Regime); + const bucket = buckets.get(regime)!; + bucket.count++; + bucket.profit += t.profit; + bucket.costSum += t.cost; + if (t.profit > 0) bucket.wins++; + } + + return Array.from(buckets.entries()).map(([regime, b]) => ({ + regime, + trades: b.count, + netProfit: b.profit, + returnPct: b.costSum > 0 ? b.profit / b.costSum : 0, + winRate: b.count > 0 ? b.wins / b.count : 0, + })); +} + +export interface RegimeCoverage { + /** Regimes with at least one trade. */ + regimesTraded: Regime[]; + /** Regimes among those with a positive net profit. */ + profitableRegimes: Regime[]; + /** Whether at least `minProfitableRegimes` regimes were profitable. */ + passed: boolean; + breakdown: RegimeBreakdownEntry[]; +} + +/** + * Decide whether a strategy demonstrated edge across enough market conditions. + * + * `minProfitableRegimes` defaults to 2: a strategy must work in at least two + * regimes. One is a filter for regime-blind money printers, and zero would be + * no gate at all. + */ +export function regimeCoverage( + breakdown: RegimeBreakdownEntry[], + minProfitableRegimes = 2, +): RegimeCoverage { + const profitableEntries = breakdown.filter((b) => b.trades > 0 && b.returnPct > 0); + return { + regimesTraded: breakdown.filter((b) => b.trades > 0).map((b) => b.regime), + profitableRegimes: profitableEntries.map((b) => b.regime), + passed: profitableEntries.length >= minProfitableRegimes, + breakdown, + }; +} diff --git a/packages/strategy-validation/src/robustness.test.ts b/packages/strategy-validation/src/robustness.test.ts new file mode 100644 index 0000000..d7cd330 --- /dev/null +++ b/packages/strategy-validation/src/robustness.test.ts @@ -0,0 +1,205 @@ +import { describe, it, expect } from 'vitest'; +import { ZERO_COST_MODEL, tsp } from '@b1dz/source-strategies'; +import { sinePrices, snapshotsFrom, randomWalkPrices } from './synthetic.js'; +import { + DEFAULT_PERTURBATION_PCT, + cloneDefinition, + collectKnobs, + perturbDefinition, + robustnessScore, +} from './robustness.js'; + +const mr = (params: Record = {}) => ({ + tsp: '0.1' as const, + id: 'test-mr', + name: 'Test MR', + definition: { + kind: 'template' as const, + template: 'mean-reversion' as const, + params: { period: 14, oversold: 35, overbought: 65, ...params }, + }, +}); + +describe('perturbDefinition', () => { + it('produces 2 variants per numeric knob (±pct)', () => { + const variants = perturbDefinition(mr()); + expect(variants).toHaveLength(6); + for (const v of variants) { + expect(v.label).toMatch(/→/); + expect(v.from).not.toBe(v.to); + } + }); + + it('describes the knob and direction in the label', () => { + const variants = perturbDefinition(mr()); + const labels = variants.map((v) => v.label); + expect(labels.some((l) => l.includes('14→'))).toBe(true); + expect(labels.some((l) => l.includes('+10%'))).toBe(true); + expect(labels.some((l) => l.includes('-10%'))).toBe(true); + }); + + it('rounds integer knobs and floors at 2', () => { + const variants = perturbDefinition(mr({ period: 2 })); + const periodVariants = variants.filter((v) => v.label.includes('period')); + expect(periodVariants.length).toBeLessThanOrEqual(2); + }); + + it('respects maxVariants cap', () => { + const many = perturbDefinition(mr(), { maxVariants: 3 }); + expect(many.length).toBeLessThanOrEqual(3); + }); + + it('respects custom pct', () => { + const variants = perturbDefinition(mr(), { pct: 0.2 }); + expect(variants.some((v) => v.label.includes('20%'))).toBe(true); + }); + + it('yields an empty array for a document with no numeric params', () => { + const empty: tsp.TradingStrategyDefinition = { + tsp: '0.1', + id: 'e', + name: 'E', + definition: { kind: 'template', template: 'mean-reversion', params: {} }, + }; + expect(perturbDefinition(empty)).toEqual([]); + }); +}); + +describe('cloneDefinition', () => { + it('produces a deep copy that can be mutated independently', () => { + const doc = mr(); + const copy = cloneDefinition(doc); + (copy as any).id = 'changed'; + (copy as any).definition.params.period = 99; + expect(doc.id).toBe('test-mr'); + expect((copy as any).id).toBe('changed'); + expect((doc.definition.params as Record).period).toBe(14); + expect((copy as any).definition.params.period).toBe(99); + }); +}); + +describe('collectKnobs', () => { + it('finds every numeric parameter in a template doc', () => { + const knobs = collectKnobs(mr()); + expect(knobs.length).toBe(3); + const labels = knobs.map((k) => k.label); + expect(labels).toContain('definition.params.period'); + expect(labels).toContain('definition.params.oversold'); + expect(labels).toContain('definition.params.overbought'); + }); + + it('marks window keys (period, lookback, fast, slow, signal) as integer', () => { + const knobs = collectKnobs(mr()); + const periodKnob = knobs.find((k) => k.label.endsWith('period'))!; + expect(periodKnob.integer).toBe(true); + const oversoldKnob = knobs.find((k) => k.label.endsWith('oversold'))!; + expect(oversoldKnob.integer).toBe(false); + }); + + it('skips zero-valued operands (sign tests, not fitted thresholds)', () => { + const rules: tsp.TradingStrategyDefinition = { + tsp: '0.1', + id: 'r', + name: 'R', + definition: { + kind: 'rules', + indicators: { r14: { fn: 'rsi', period: 14 } }, + rules: [{ when: { lt: ['r14', 0] }, signal: { side: 'buy' } }], + } as any, + }; + const knobs = collectKnobs(rules); + const zeroKnobs = knobs.filter((k) => k.value === 0); + expect(zeroKnobs.length).toBe(0); + }); + + it('collects numeric literals from conditions', () => { + const doc: tsp.TradingStrategyDefinition = { + tsp: '0.1', + id: 'c', + name: 'C', + definition: { + kind: 'rules', + indicators: { r14: { fn: 'rsi', period: 14 } }, + rules: [{ when: { gt: ['r14', 70] }, signal: { side: 'sell' } }], + } as any, + }; + const knobs = collectKnobs(doc); + const periodKnob = knobs.find((k) => k.path.join('.') === 'definition.indicators.r14.period'); + const thresholdKnob = knobs.find((k) => k.value === 70); + expect(periodKnob).toBeDefined(); + expect(thresholdKnob).toBeDefined(); + expect(thresholdKnob!.value).toBe(70); + }); +}); + +describe('robustnessScore', () => { + const sineSnaps = snapshotsFrom(sinePrices({ bars: 400, period: 20, amplitude: 0.1, noise: 0.01 })); + + it('runs a clean mean-reversion strategy over sine data without throwing', () => { + const report = robustnessScore(mr(), sineSnaps, { costs: ZERO_COST_MODEL }); + expect(report.variants.length).toBeGreaterThan(0); + expect(report.baseTrades).toBeGreaterThanOrEqual(0); + expect(typeof report.baseReturnPct).toBe('number'); + expect(typeof report.passed).toBe('boolean'); + }); + + it('runs on random walk data without throwing', () => { + const rwSnaps = snapshotsFrom(randomWalkPrices({ bars: 400 })); + const report = robustnessScore(mr(), rwSnaps, { costs: ZERO_COST_MODEL }); + expect(report.variants.length).toBeGreaterThan(0); + expect(Number.isFinite(report.baseReturnPct)).toBe(true); + }); + + it('reports degradation correctly', () => { + const report = robustnessScore(mr(), sineSnaps, { costs: ZERO_COST_MODEL }); + expect(typeof report.degradationPct).toBe('number'); + expect(Number.isFinite(report.degradationPct)).toBe(true); + expect(report.degradationPct).toBeGreaterThanOrEqual(-100); + expect(report.degradationPct).toBeLessThanOrEqual(100); + }); + + it('ignores identical-to-base variants in the score', () => { + const bo: tsp.TradingStrategyDefinition = { + tsp: '0.1', + id: 'bo', + name: 'BO', + definition: { + kind: 'template', + template: 'breakout', + params: { lookback: 20, period: 14 }, + }, + }; + const report = robustnessScore(bo, sineSnaps, { costs: ZERO_COST_MODEL }); + const noops = report.variants.filter((v) => v.identicalToBase); + expect(noops.length).toBeGreaterThanOrEqual(2); + }); + + it('fails with a doc that cannot compile', () => { + const bad = { ...mr(), definition: { kind: 'rules' as const, rules: [] } }; + const report = robustnessScore(bad as any, sineSnaps, { costs: ZERO_COST_MODEL }); + expect(report.passed).toBe(false); + expect(report.detail).toBeTruthy(); + }); + + it('reports passed=false when there are no effective variants', () => { + const emptyDoc: tsp.TradingStrategyDefinition = { + tsp: '0.1', + id: 'nv', + name: 'NV', + definition: { kind: 'template', template: 'mean-reversion', params: {} }, + }; + const report = robustnessScore(emptyDoc, sineSnaps, { costs: ZERO_COST_MODEL }); + expect(report.passed).toBe(false); + expect(report.effectiveVariants).toBe(0); + }); + + it('reports every finite value on empty snapshots', () => { + const report = robustnessScore(mr(), []); + expect(report.passed).toBe(false); + expect(Number.isFinite(report.baseReturnPct)).toBe(true); + expect(Number.isFinite(report.degradationPct)).toBe(true); + // variants are generated from the document, not from snapshots + expect(report.variants.length).toBe(6); + expect(report.baseTrades).toBe(0); + }); +}); diff --git a/packages/strategy-validation/src/robustness.ts b/packages/strategy-validation/src/robustness.ts new file mode 100644 index 0000000..3e62e27 --- /dev/null +++ b/packages/strategy-validation/src/robustness.ts @@ -0,0 +1,451 @@ +/** + * Parameter robustness — does the strategy survive a small nudge to its own + * numbers, or is it balanced on a knife edge? + * + * WHY THIS EXISTS + * + * An RSI-14 mean-reversion strategy that returns 40% at period 14, 38% at 13 and + * 41% at 15 has found something. The same strategy returning 40% at period 14, + * −3% at 13 and −5% at 15 has found nothing: it has memorised where the noise + * happened to line up. There is no economic mechanism that turns on at 14 and off + * at 13. Every real edge sits on a broad plateau in parameter space, because it + * is driven by something structural — a liquidity pattern, a behavioural bias, a + * flow imbalance — and structure does not have a resolution of one bar. + * + * This is the cheapest and most reliable overfitting detector we have. It needs no + * distributional assumptions, no out-of-sample data, and no knowledge of how many + * candidates were generated. It just asks the search process to prove it found a + * region rather than a point. An AI generator sweeping periods 2..50 across six + * templates WILL produce spikes, by construction — they are the single most + * common artifact of automated strategy search, and they are invisible in any + * summary statistic of the base configuration alone. + * + * WHAT IT PERTURBS + * + * Every numeric knob reachable in the TSP document, in both directions: + * - `TemplateDefinition.params` — period, lookback, oversold, overbought, … + * - indicator windows in `RulesDefinition.indicators` — period / fast / slow / signal + * - numeric literals inside `Condition` comparison operands — the RSI 30 in + * `{ lt: ['rsi', 30] }`, which is exactly the kind of number a generator tunes + * + * Indicator windows are integers, so they are rounded and floored at 2; a "±10%" + * nudge to a 14-period window means 13 or 15. Continuous thresholds scale + * proportionally. A literal 0 is left alone: `{ gt: ['macdHist', 0] }` is a sign + * test, not a fitted threshold, scaling it is a no-op, and shifting it by an + * arbitrary absolute amount would invent a parameter with no units. + * + * THE GAMING VECTOR THIS CLOSES + * + * A knob the compiler ignores — `params: { period: 14 }` on a `breakout` template, + * which reads `lookback` — produces a variant that is byte-identical in behaviour + * to the base. Count those as passing variants and any document can pad its + * robustness score to 100% with decorative numbers. So variants whose trade stream + * is indistinguishable from the base are marked `identicalToBase` and excluded + * from the score entirely: a parameter that changes nothing is not a parameter. + * + * The input document is never mutated. Every variant is a deep clone. + */ +import type { MarketSnapshot } from '@b1dz/core'; +import { + DEFAULT_AMOUNT_PER_ENTRY, + costModelForSeries, + replayStrategy, + summarizeTrades, + tsp, + type BacktestTrade, + type CostModel, +} from '@b1dz/source-strategies'; + +/** Default nudge applied to every numeric parameter, in both directions. */ +export const DEFAULT_PERTURBATION_PCT = 0.1; + +/** + * Hard cap on the number of variants replayed. + * + * `replayStrategy()` re-slices the whole history on every bar, so a replay is + * O(bars²). A document with 12 knobs generates 24 variants; at 2,000 bars that is + * already tens of millions of element copies. The cap keeps a single gauntlet run + * bounded, and knobs are taken in document order so the result stays + * deterministic rather than depending on which ones happen to be cheap. + */ +export const DEFAULT_MAX_VARIANTS = 48; + +/** Path segments of a numeric knob inside a TSP document. */ +export type KnobPath = (string | number)[]; + +export interface DefinitionVariant { + /** Human-readable description of the single change, e.g. `params.period 14→15 (+10%)`. */ + label: string; + definition: tsp.TradingStrategyDefinition; + path: KnobPath; + from: number; + to: number; +} + +export interface Knob { + path: KnobPath; + /** Rendered path, e.g. `definition.rules[0].when.lt[1]`. */ + label: string; + value: number; + /** Indicator windows are bar counts: integral and at least 2. */ + integer: boolean; +} + +/** Template/indicator keys that are bar counts rather than continuous levels. */ +const WINDOW_KEYS = new Set(['period', 'lookback', 'fast', 'slow', 'signal', 'length', 'window']); + +const isNum = (v: unknown): v is number => typeof v === 'number' && Number.isFinite(v); + +/** + * TSP documents are pure JSON by definition (see osd/types.ts), so a JSON round + * trip is an exact deep clone. It also strips any prototype pollution or + * non-serializable junk riding along on an untrusted, user-authored document, + * which `structuredClone` would faithfully preserve. + */ +export function cloneDefinition(doc: T): T { + return JSON.parse(JSON.stringify(doc)) as T; +} + +/** Render a knob path as a JS-ish accessor for the variant label. */ +function renderPath(path: KnobPath): string { + let out = ''; + for (const seg of path) { + if (typeof seg === 'number') out += `[${seg}]`; + else out += out === '' ? seg : `.${seg}`; + } + return out; +} + +function setAtPath(root: unknown, path: KnobPath, value: number): void { + let node = root as Record; + for (let i = 0; i < path.length - 1; i++) { + node = node[path[i]!] as Record; + } + node[path[path.length - 1]!] = value; +} + +/** Collect numeric literals from a condition tree, recording their exact paths. */ +function collectConditionKnobs(cond: unknown, path: KnobPath, out: Knob[], depth = 0): void { + if (depth > 32 || typeof cond !== 'object' || cond === null) return; + const obj = cond as Record; + + if (Array.isArray(obj.and)) { + obj.and.forEach((c, i) => collectConditionKnobs(c, [...path, 'and', i], out, depth + 1)); + return; + } + if (Array.isArray(obj.or)) { + obj.or.forEach((c, i) => collectConditionKnobs(c, [...path, 'or', i], out, depth + 1)); + return; + } + if (obj.not !== undefined) { + collectConditionKnobs(obj.not, [...path, 'not'], out, depth + 1); + return; + } + for (const cmp of tsp.COMPARATORS) { + const operands = obj[cmp]; + if (!Array.isArray(operands)) continue; + operands.forEach((operand, i) => { + // A literal 0 is a sign test, not a fitted level — see the file header. + if (isNum(operand) && operand !== 0) { + const p = [...path, cmp, i]; + out.push({ path: p, label: renderPath(p), value: operand, integer: false }); + } + }); + return; + } +} + +/** + * Every numeric knob in a TSP document, in stable document order. + * + * Exported because "which numbers is this strategy actually fitted on" is a + * useful question on its own — a document with 11 tuned constants and 40 trades + * is over-parameterised before any statistic is computed. + */ +export function collectKnobs(doc: tsp.TradingStrategyDefinition): Knob[] { + const out: Knob[] = []; + // Walked structurally rather than by narrowing the union: `perturbDefinition` + // has to handle documents that came off the wire and may not match the declared + // shape, and a `kind` we don't recognise must yield zero knobs, not a throw. + const body = doc.definition as unknown as Record | undefined; + if (!body || typeof body !== 'object') return out; + + if (body.kind === 'template') { + const params = body.params; + if (params && typeof params === 'object' && !Array.isArray(params)) { + for (const [key, value] of Object.entries(params as Record)) { + if (!isNum(value) || value === 0) continue; + const path: KnobPath = ['definition', 'params', key]; + out.push({ path, label: renderPath(path), value, integer: WINDOW_KEYS.has(key) }); + } + } + return out; + } + + if (body.kind === 'rules') { + const indicators = body.indicators; + if (indicators && typeof indicators === 'object' && !Array.isArray(indicators)) { + for (const [name, spec] of Object.entries(indicators as Record)) { + if (!spec || typeof spec !== 'object') continue; + for (const key of ['period', 'fast', 'slow', 'signal']) { + const value = (spec as Record)[key]; + if (!isNum(value) || value === 0) continue; + const path: KnobPath = ['definition', 'indicators', name, key]; + out.push({ path, label: renderPath(path), value, integer: true }); + } + } + } + const rules = body.rules; + if (Array.isArray(rules)) { + rules.forEach((rule, i) => { + if (!rule || typeof rule !== 'object') return; + collectConditionKnobs( + (rule as Record).when, + ['definition', 'rules', i, 'when'], + out, + ); + }); + } + } + return out; +} + +export interface PerturbOptions { + /** Fractional nudge applied in both directions. Default 0.1 (±10%). */ + pct?: number; + /** Cap on returned variants. Default `DEFAULT_MAX_VARIANTS`. */ + maxVariants?: number; +} + +/** + * Deep-cloned variants of `doc`, each with exactly ONE numeric parameter nudged + * by ±`pct`. + * + * One knob at a time, on purpose. Perturbing several at once tests a random + * corner of a high-dimensional space and confounds the result — when a + * multi-knob variant collapses you cannot tell which parameter was the fragile + * one, and the fragile one is the finding. One-at-a-time is a local sensitivity + * analysis, which is precisely the question "is this a plateau or a spike". + */ +export function perturbDefinition( + doc: tsp.TradingStrategyDefinition, + opts: PerturbOptions = {}, +): DefinitionVariant[] { + const pct = Number.isFinite(opts.pct) && (opts.pct ?? 0) > 0 ? opts.pct! : DEFAULT_PERTURBATION_PCT; + const maxVariants = Math.max(0, Math.floor(opts.maxVariants ?? DEFAULT_MAX_VARIANTS)); + const knobs = collectKnobs(doc); + const out: DefinitionVariant[] = []; + const seen = new Set(); + + for (const knob of knobs) { + for (const dir of [-1, 1] as const) { + if (out.length >= maxVariants) return out; + + const raw = knob.value * (1 + dir * pct); + const to = knob.integer ? Math.max(2, Math.round(raw)) : raw; + // Rounding can land back on the original value (period 2 at ±10%); a + // variant identical to the base tests nothing. + if (to === knob.value) continue; + + const sign = dir > 0 ? '+' : '-'; + const label = `${knob.label} ${knob.value}→${round4(to)} (${sign}${(pct * 100).toFixed(0)}%)`; + if (seen.has(label)) continue; + seen.add(label); + + const definition = cloneDefinition(doc); + setAtPath(definition, knob.path, to); + out.push({ label, definition, path: knob.path, from: knob.value, to }); + } + } + return out; +} + +function round4(v: number): number { + return Math.round(v * 10_000) / 10_000; +} + +// ── scoring ───────────────────────────────────────────────────────────────── + +export interface RobustnessOptions extends PerturbOptions { + amountPerEntry?: number; + costs?: CostModel; + /** Fraction of effective variants that must stay profitable. Default 0.6. */ + minFractionProfitable?: number; + /** Ceiling on how much of the base return the median variant may give up. Default 0.5. */ + maxDegradationPct?: number; +} + +export interface RobustnessVariantResult { + label: string; + returnPct: number; + trades: number; + profitable: boolean; + /** + * True when this variant produced the same trade stream as the base — i.e. the + * perturbed parameter has no effect on the compiled strategy. Excluded from + * every score; see the file header. + */ + identicalToBase: boolean; + /** Set when the variant failed to compile or replay. Counts as unprofitable. */ + error?: string; +} + +export interface RobustnessReport { + baseReturnPct: number; + baseTrades: number; + variants: RobustnessVariantResult[]; + /** Variants that actually changed behaviour — the denominator for every score. */ + effectiveVariants: number; + medianReturnPct: number; + worstReturnPct: number; + fractionProfitable: number; + /** + * Share of the base return given up by the MEDIAN variant. + * 0 = the neighbourhood performs like the base. 1 = the median gave up the + * entire edge. >1 = it flipped to a loss. NEGATIVE is a good sign: the base is + * not the local peak, so nothing was tuned to a spike. + */ + degradationPct: number; + passed: boolean; + detail: string; +} + +/** Behavioural signature of a replay, for spotting no-op parameters. */ +function signature(trades: BacktestTrade[]): string { + let profit = 0; + for (const t of trades) profit += t.profit; + return `${trades.length}:${profit.toFixed(6)}`; +} + +function median(values: number[]): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 ? sorted[mid]! : (sorted[mid - 1]! + sorted[mid]!) / 2; +} + +function emptyReport(detail: string): RobustnessReport { + return { + baseReturnPct: 0, + baseTrades: 0, + variants: [], + effectiveVariants: 0, + medianReturnPct: 0, + worstReturnPct: 0, + fractionProfitable: 0, + degradationPct: 0, + passed: false, + detail, + }; +} + +/** + * Replay the base document and every ±`pct` variant over the same bars and the + * same cost model, and score the neighbourhood. + * + * Fails CLOSED in every degenerate case, including the one that looks like a + * technicality: a document with no effective knobs scores `passed: false`. We + * cannot demonstrate robustness for it, so we do not claim it. In practice every + * TSP strategy has at least one indicator window, so the only documents this + * rejects are ones whose numbers the compiler ignores — which is a defect worth + * surfacing, not papering over. + */ +export function robustnessScore( + doc: tsp.TradingStrategyDefinition, + snapshots: MarketSnapshot[], + opts: RobustnessOptions = {}, +): RobustnessReport { + const amountPerEntry = opts.amountPerEntry ?? DEFAULT_AMOUNT_PER_ENTRY; + const costs = opts.costs ?? costModelForSeries(snapshots); + const minFraction = opts.minFractionProfitable ?? 0.6; + const maxDegradation = opts.maxDegradationPct ?? 0.5; + + let baseTrades: BacktestTrade[]; + try { + baseTrades = replayStrategy(tsp.compile(doc), snapshots, { amountPerEntry, costs }); + } catch (err) { + return emptyReport(`base definition failed to compile: ${errorText(err)}`); + } + + const baseSummary = summarizeTrades(baseTrades); + const baseSignature = signature(baseTrades); + const variantDocs = perturbDefinition(doc, opts); + + const variants: RobustnessVariantResult[] = variantDocs.map((v) => { + try { + const trades = replayStrategy(tsp.compile(v.definition), snapshots, { amountPerEntry, costs }); + const summary = summarizeTrades(trades); + return { + label: v.label, + returnPct: summary.returnPct, + trades: summary.trades, + profitable: summary.returnPct > 0, + identicalToBase: signature(trades) === baseSignature, + }; + } catch (err) { + // A variant that cannot even be compiled is not robust. Count it as a loss + // rather than dropping it, or an invalid neighbourhood would look clean. + return { + label: v.label, + returnPct: 0, + trades: 0, + profitable: false, + identicalToBase: false, + error: errorText(err), + }; + } + }); + + const effective = variants.filter((v) => !v.identicalToBase); + if (effective.length === 0) { + return { + ...emptyReport( + variants.length === 0 + ? 'no numeric parameters to perturb — robustness cannot be demonstrated' + : `all ${variants.length} perturbations left behaviour unchanged — the document's numbers do not reach the compiled strategy`, + ), + baseReturnPct: baseSummary.returnPct, + baseTrades: baseSummary.trades, + variants, + }; + } + + const returns = effective.map((v) => v.returnPct); + const medianReturnPct = median(returns); + const worstReturnPct = Math.min(...returns); + const fractionProfitable = effective.filter((v) => v.profitable).length / effective.length; + const degradationPct = computeDegradation(baseSummary.returnPct, medianReturnPct); + const passed = fractionProfitable >= minFraction && degradationPct <= maxDegradation; + + return { + baseReturnPct: baseSummary.returnPct, + baseTrades: baseSummary.trades, + variants, + effectiveVariants: effective.length, + medianReturnPct, + worstReturnPct, + fractionProfitable, + degradationPct, + passed, + detail: `${(fractionProfitable * 100).toFixed(0)}% of ${effective.length} effective ±${(((opts.pct ?? DEFAULT_PERTURBATION_PCT) * 100)).toFixed(0)}% variants profitable; median gives up ${(degradationPct * 100).toFixed(0)}% of the base return`, + }; +} + +/** + * Degradation as a share of the base return, clamped to a reportable range. + * + * A base return of ~0 makes the ratio explode, and a base return of exactly 0 + * makes it undefined. Both mean the same thing — there is no edge to degrade — + * and the profit gates already reject that case, so this returns 0 rather than a + * number that would dominate a report with noise. + */ +function computeDegradation(baseReturnPct: number, medianReturnPct: number): number { + const scale = Math.abs(baseReturnPct); + if (!(scale > 1e-9)) return 0; + const raw = (baseReturnPct - medianReturnPct) / scale; + return Math.min(Math.max(raw, -100), 100); +} + +function errorText(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} diff --git a/packages/strategy-validation/src/splits.test.ts b/packages/strategy-validation/src/splits.test.ts new file mode 100644 index 0000000..6d40acd --- /dev/null +++ b/packages/strategy-validation/src/splits.test.ts @@ -0,0 +1,244 @@ +import { describe, it, expect } from 'vitest'; +import type { MarketSnapshot } from '@b1dz/core'; +import { + DEFAULT_OOS_RATIO, + MIN_WARMUP_BARS, + anchoredWalkForward, + isChronological, + trainTestSplit, + walkForwardSplits, + type WalkForwardSplit, +} from './splits.js'; +import { snapshotsFrom } from './synthetic.js'; + +/** `bars` snapshots whose price equals their index, so identity is checkable. */ +const bars = (n: number): MarketSnapshot[] => + snapshotsFrom(Array.from({ length: n }, (_, i) => i + 1)); + +const firstTs = (s: MarketSnapshot[]) => s[0]!.ts; +const lastTs = (s: MarketSnapshot[]) => s[s.length - 1]!.ts; + +/** Every invariant a walk-forward set must satisfy to be worth reporting. */ +function assertWalkForwardInvariants(splits: WalkForwardSplit[], minBars: number): void { + splits.forEach((s, i) => { + expect(s.index).toBe(i); + expect(s.train.length).toBeGreaterThanOrEqual(minBars); + expect(s.test.length).toBeGreaterThanOrEqual(minBars); + // No leakage: every test bar is strictly after every train bar. + expect(firstTs(s.test)).toBeGreaterThan(lastTs(s.train)); + expect(isChronological(s.train)).toBe(true); + expect(isChronological(s.test)).toBe(true); + if (i > 0) { + const prev = splits[i - 1]!; + // Test windows are non-overlapping and strictly forward-ordered. + expect(firstTs(s.test)).toBeGreaterThan(lastTs(prev.test)); + } + }); +} + +describe('isChronological', () => { + it('accepts a forward-ordered series', () => { + expect(isChronological(bars(50))).toBe(true); + expect(isChronological([])).toBe(true); + expect(isChronological(bars(1))).toBe(true); + }); + + it('accepts duplicate timestamps (noise, not leakage)', () => { + const s = bars(3); + s[1]!.ts = s[0]!.ts; + expect(isChronological(s)).toBe(true); + }); + + it('rejects a series with any backwards step', () => { + const s = bars(10); + s[7]!.ts = s[2]!.ts - 1; + expect(isChronological(s)).toBe(false); + }); + + it('rejects a shuffled series', () => { + const s = bars(40); + const shuffled = [s[10]!, s[3]!, s[39]!, ...s.slice(0, 3)]; + expect(isChronological(shuffled)).toBe(false); + }); +}); + +describe('trainTestSplit', () => { + it('cuts chronologically at 1 - oosRatio and keeps every bar', () => { + const s = bars(300); + const { inSample, outOfSample } = trainTestSplit(s, 0.3); + expect(inSample).toHaveLength(210); + expect(outOfSample).toHaveLength(90); + expect(inSample[0]!.ts).toBe(s[0]!.ts); + expect(outOfSample[0]!.ts).toBe(s[210]!.ts); + expect(lastTs(outOfSample)).toBe(lastTs(s)); + // Contiguous, no gap, no overlap, nothing dropped. + expect(inSample.length + outOfSample.length).toBe(s.length); + expect(firstTs(outOfSample)).toBeGreaterThan(lastTs(inSample)); + }); + + it('never shuffles: both halves stay in original order', () => { + const s = bars(200); + const { inSample, outOfSample } = trainTestSplit(s, 0.25); + expect(isChronological(inSample)).toBe(true); + expect(isChronological(outOfSample)).toBe(true); + expect([...inSample, ...outOfSample].map((x) => x.ts)).toEqual(s.map((x) => x.ts)); + }); + + it('defaults to a 30% holdout', () => { + expect(DEFAULT_OOS_RATIO).toBe(0.3); + expect(trainTestSplit(bars(300)).outOfSample).toHaveLength(90); + }); + + it('widens the holdout when the requested ratio is shorter than the warmup', () => { + // minBars outranks oosRatio: a 30-bar holdout on 100 bars would be shorter + // than a MACD warmup, so the cut moves inward to 65/35. + const { inSample, outOfSample } = trainTestSplit(bars(100), 0.3); + expect(inSample).toHaveLength(65); + expect(outOfSample).toHaveLength(MIN_WARMUP_BARS); + }); + + it('does not mutate or alias the input array', () => { + const s = bars(100); + const { inSample } = trainTestSplit(s, 0.3); + inSample.push(s[0]!); + expect(s).toHaveLength(100); + }); + + it('returns an EMPTY out-of-sample block rather than a fake short one', () => { + // 60 bars cannot give both sides the 35-bar warmup, so there is no honest + // holdout. An empty block fails the OOS gates downstream, which is correct: + // "we do not know" must never be rendered as "it passed". + const { inSample, outOfSample } = trainTestSplit(bars(60), 0.3); + expect(inSample).toHaveLength(60); + expect(outOfSample).toEqual([]); + }); + + it('handles an empty series', () => { + expect(trainTestSplit([], 0.3)).toEqual({ inSample: [], outOfSample: [] }); + }); + + it('keeps both sides at or above minBars even for an extreme ratio', () => { + for (const ratio of [0, 0.01, 0.5, 0.99, 1, 5, Number.NaN, -1]) { + const { inSample, outOfSample } = trainTestSplit(bars(100), ratio); + expect(inSample.length).toBeGreaterThanOrEqual(MIN_WARMUP_BARS); + expect(outOfSample.length).toBeGreaterThanOrEqual(MIN_WARMUP_BARS); + expect(inSample.length + outOfSample.length).toBe(100); + } + }); + + it('honours a custom minBars', () => { + const { inSample, outOfSample } = trainTestSplit(bars(30), 0.3, { minBars: 10 }); + expect(inSample).toHaveLength(20); + expect(outOfSample).toHaveLength(10); + expect(trainTestSplit(bars(19), 0.3, { minBars: 10 }).outOfSample).toEqual([]); + // With room to spare the requested ratio is respected exactly. + expect(trainTestSplit(bars(100), 0.3, { minBars: 10 }).outOfSample).toHaveLength(30); + }); +}); + +describe('walkForwardSplits', () => { + it('produces the documented rolling geometry', () => { + // n=300, trainRatio 0.6 → trainSize 180; remaining 120 / 3 folds → testSize 40. + const s = bars(300); + const splits = walkForwardSplits(s, { folds: 3, trainRatio: 0.6 }); + expect(splits).toHaveLength(3); + expect(splits.map((f) => f.train.length)).toEqual([180, 180, 180]); + expect(splits.map((f) => f.test.length)).toEqual([40, 40, 40]); + expect(splits[0]!.test[0]!.ts).toBe(s[180]!.ts); + expect(splits[1]!.test[0]!.ts).toBe(s[220]!.ts); + expect(splits[2]!.test[0]!.ts).toBe(s[260]!.ts); + expect(lastTs(splits[2]!.test)).toBe(lastTs(s)); + assertWalkForwardInvariants(splits, MIN_WARMUP_BARS); + }); + + it('rolls the train window forward rather than expanding it', () => { + const splits = walkForwardSplits(bars(300), { folds: 3, trainRatio: 0.6 }); + const starts = splits.map((f) => firstTs(f.train)); + expect(new Set(starts).size).toBe(3); // each fold starts later + expect(starts[1]!).toBeGreaterThan(starts[0]!); + expect(starts[2]!).toBeGreaterThan(starts[1]!); + }); + + it('never leaks: no test bar precedes its own train window', () => { + for (const n of [200, 300, 500, 1000]) { + for (const folds of [1, 2, 3, 5, 8]) { + assertWalkForwardInvariants(walkForwardSplits(bars(n), { folds }), MIN_WARMUP_BARS); + } + } + }); + + it('returns FEWER folds instead of windows shorter than minBars', () => { + // n=200, trainRatio 0.6 → trainSize 120, remaining 80. + // 5 folds → 16 bars each (too short); 4 → 20; 3 → 26; 2 → 40 ✓. + const splits = walkForwardSplits(bars(200), { folds: 5, trainRatio: 0.6 }); + expect(splits).toHaveLength(2); + expect(splits.map((f) => f.test.length)).toEqual([40, 40]); + assertWalkForwardInvariants(splits, MIN_WARMUP_BARS); + }); + + it('returns no folds at all when the series cannot support one honest window', () => { + expect(walkForwardSplits(bars(50), { folds: 4 })).toEqual([]); + expect(walkForwardSplits(bars(80), { folds: 4, trainRatio: 0.6 })).toEqual([]); + expect(walkForwardSplits([], { folds: 4 })).toEqual([]); + expect(walkForwardSplits(bars(1), { folds: 1 })).toEqual([]); + }); + + it('honours a relaxed minBars', () => { + const splits = walkForwardSplits(bars(100), { folds: 4, trainRatio: 0.6, minBars: 10 }); + expect(splits).toHaveLength(4); + expect(splits.map((f) => f.test.length)).toEqual([10, 10, 10, 10]); + assertWalkForwardInvariants(splits, 10); + }); + + it('clamps nonsense fold counts and train ratios instead of throwing', () => { + expect(walkForwardSplits(bars(300), { folds: 0 })).toHaveLength(1); + expect(walkForwardSplits(bars(300), { folds: -3 })).toHaveLength(1); + expect(walkForwardSplits(bars(300), { folds: 2.7, trainRatio: 0.6 })).toHaveLength(2); + for (const trainRatio of [0, 1, 5, -2, Number.NaN]) { + const splits = walkForwardSplits(bars(400), { folds: 2, trainRatio }); + assertWalkForwardInvariants(splits, MIN_WARMUP_BARS); + } + }); +}); + +describe('anchoredWalkForward', () => { + it('expands the train window from bar 0', () => { + // n=400, trainRatio 0.5 → initialTrain 200; remaining 200 / 4 → testSize 50. + const s = bars(400); + const splits = anchoredWalkForward(s, { folds: 4 }); + expect(splits).toHaveLength(4); + expect(splits.map((f) => f.train.length)).toEqual([200, 250, 300, 350]); + expect(splits.map((f) => f.test.length)).toEqual([50, 50, 50, 50]); + for (const f of splits) expect(firstTs(f.train)).toBe(s[0]!.ts); + expect(lastTs(splits[3]!.test)).toBe(lastTs(s)); + assertWalkForwardInvariants(splits, MIN_WARMUP_BARS); + }); + + it('uses more data than the rolling variant for the same fold count', () => { + const anchored = anchoredWalkForward(bars(400), { folds: 4, trainRatio: 0.5 }); + const rolling = walkForwardSplits(bars(400), { folds: 4, trainRatio: 0.5 }); + const trainBars = (fs: WalkForwardSplit[]) => fs.reduce((n, f) => n + f.train.length, 0); + expect(trainBars(anchored)).toBeGreaterThan(trainBars(rolling)); + }); + + it('never leaks across a range of shapes', () => { + for (const n of [150, 300, 700]) { + for (const folds of [1, 2, 4, 6]) { + assertWalkForwardInvariants(anchoredWalkForward(bars(n), { folds }), MIN_WARMUP_BARS); + } + } + }); + + it('returns fewer folds rather than short ones', () => { + // n=150, trainRatio 0.5 → initialTrain 75, remaining 75. + // 4 folds → 18 bars (short); 3 → 25 (short); 2 → 37 ✓. + const splits = anchoredWalkForward(bars(150), { folds: 4 }); + expect(splits).toHaveLength(2); + expect(splits.map((f) => f.test.length)).toEqual([37, 37]); + }); + + it('returns nothing when the series is too short', () => { + expect(anchoredWalkForward(bars(60), { folds: 3 })).toEqual([]); + expect(anchoredWalkForward([], { folds: 3 })).toEqual([]); + }); +}); diff --git a/packages/strategy-validation/src/splits.ts b/packages/strategy-validation/src/splits.ts new file mode 100644 index 0000000..4892f72 --- /dev/null +++ b/packages/strategy-validation/src/splits.ts @@ -0,0 +1,245 @@ +/** + * Chronological sample splitting — in-sample / out-of-sample and walk-forward. + * + * WHY THIS EXISTS + * + * A backtest run over the same bars that were used to choose the strategy's + * parameters is not evidence. It is a restatement of the search. The only + * measurement with any information content is one taken on bars the selection + * process never saw, which means the data has to be cut before it is used, and + * the cut has to be respected. + * + * THE FAILURE MODE + * + * Every generic ML splitter shuffles. `train_test_split(X, y)` from scikit-learn + * shuffles by default. Shuffling a price series is not a mild methodological + * lapse, it is total leakage: with interleaved train and test bars the model gets + * to see tomorrow's price while predicting today's, and the resulting accuracy is + * unbounded and completely fake. Nothing in this file shuffles, nothing sorts, + * and nothing samples. Windows are contiguous, forward-ordered, and cut by index. + * + * The second failure mode is subtler and more common: windows too short to be + * meaningful. A MACD-histogram strategy emits nothing at all for its first 35 + * bars (slow period 26 + signal period 9, see osd/compile.ts `indicatorMinPoints`), + * so a 20-bar test fold measures a strategy that is definitionally silent and + * reports "0 trades, 0% return" as if that were a finding. `minBars` therefore + * defaults to exactly that warmup, and when the data cannot support the requested + * number of folds we return FEWER folds rather than short ones. Three honest + * windows beat ten fictional ones. + * + * ROLLING VS ANCHORED + * + * `walkForwardSplits()` rolls a fixed-size train window forward. It asks "does + * this strategy work on recent history", which is the right question for a + * regime-sensitive strategy and the harsher test. + * + * `anchoredWalkForward()` expands the train window from bar 0. It asks "does this + * strategy work given everything known so far", which is what a live deployment + * actually experiences, and it uses the data more efficiently on short series. + * + * Run both when you can afford it; they fail differently, and a strategy that + * passes rolling but not anchored has almost certainly been fitted to the most + * recent regime. + */ +import type { MarketSnapshot } from '@b1dz/core'; + +/** + * Minimum usable window length, in bars. + * + * 35 = MACD's warmup (slow 26 + signal 9), the slowest indicator the TSP + * compiler supports. Below this the slowest strategy in the catalog cannot emit + * a single signal, so any window shorter than this measures the warmup, not the + * strategy. + */ +export const MIN_WARMUP_BARS = 35; + +/** Default fraction of the series held back for out-of-sample testing. */ +export const DEFAULT_OOS_RATIO = 0.3; + +export interface TrainTestSplit { + inSample: MarketSnapshot[]; + outOfSample: MarketSnapshot[]; +} + +export interface WalkForwardSplit { + /** 0-based fold number, in forward chronological order. */ + index: number; + train: MarketSnapshot[]; + test: MarketSnapshot[]; +} + +export interface SplitOptions { + /** Refuse to emit any window shorter than this. Default `MIN_WARMUP_BARS`. */ + minBars?: number; +} + +export interface WalkForwardOptions extends SplitOptions { + /** Requested number of folds. Fewer are returned if the data cannot support them. */ + folds?: number; + /** Fraction of the series in each train window (rolling) or the first one (anchored). */ + trainRatio?: number; +} + +/** + * True when timestamps are non-decreasing. + * + * Worth checking explicitly rather than assuming: snapshots arriving from a + * database without an ORDER BY, or merged from two feeds, come back in arbitrary + * order, and every function in this package silently produces garbage on an + * unordered series — `replayStrategy()` would compute indicators over shuffled + * prices and a split would put future bars in the train window. It is a cheap + * O(n) check for a class of bug that is otherwise invisible in the output. + * + * Equal timestamps are allowed (duplicate ticks are noise, not leakage); + * decreasing ones are not. + */ +export function isChronological(snapshots: MarketSnapshot[]): boolean { + for (let i = 1; i < snapshots.length; i++) { + if (snapshots[i]!.ts < snapshots[i - 1]!.ts) return false; + } + return true; +} + +/** + * Split a series into a leading in-sample block and a trailing out-of-sample + * block. Chronological, contiguous, never shuffled. + * + * `minBars` outranks `oosRatio`, in both directions: + * + * - If the requested ratio would leave either side shorter than `minBars`, the + * cut is moved inward until both sides clear it. A 100-bar series with a 30% + * holdout therefore splits 65/35, not 70/30 — a 30-bar holdout is shorter + * than a MACD warmup and would measure silence. + * - If the series cannot give BOTH sides `minBars` at any cut, the whole series + * is returned as in-sample and the out-of-sample block is EMPTY. That is + * deliberate and it is not a silent success: an empty out-of-sample block + * produces zero out-of-sample trades, which fails the gauntlet's + * out-of-sample gates. Faking a 12-bar holdout to make the shape of the + * report look right would convert "we don't know" into "it passed", which is + * the one transformation this package must never perform. + */ +export function trainTestSplit( + snapshots: MarketSnapshot[], + oosRatio: number = DEFAULT_OOS_RATIO, + opts: SplitOptions = {}, +): TrainTestSplit { + const minBars = opts.minBars ?? MIN_WARMUP_BARS; + const n = snapshots.length; + + // Clamp to a ratio that can actually leave data on both sides. + const ratio = Number.isFinite(oosRatio) ? Math.min(Math.max(oosRatio, 0), 0.9) : DEFAULT_OOS_RATIO; + + if (n < minBars * 2) return { inSample: snapshots.slice(), outOfSample: [] }; + + let cut = Math.floor(n * (1 - ratio)); + // Both sides must clear minBars; nudge the cut inward if rounding put it out. + cut = Math.min(Math.max(cut, minBars), n - minBars); + + return { inSample: snapshots.slice(0, cut), outOfSample: snapshots.slice(cut) }; +} + +/** + * Largest fold count ≤ `folds` for which every train and test window clears + * `minBars`, or 0 when even one fold is impossible. + * + * Shrinking the fold count is the whole "return fewer folds rather than garbage + * ones" rule: the alternative is emitting the requested number of windows and + * letting several of them be too short to trade, which reads as "the strategy + * failed folds 3, 4 and 5" when the truth is "we never tested folds 3, 4 and 5". + */ +function usableFolds( + totalBars: number, + trainBars: number, + requestedFolds: number, + minBars: number, +): { folds: number; testSize: number } { + if (trainBars < minBars) return { folds: 0, testSize: 0 }; + const remaining = totalBars - trainBars; + for (let f = Math.floor(requestedFolds); f >= 1; f--) { + const testSize = Math.floor(remaining / f); + if (testSize >= minBars) return { folds: f, testSize }; + } + return { folds: 0, testSize: 0 }; +} + +function normalizeTrainRatio(trainRatio: number | undefined, fallback: number): number { + if (trainRatio === undefined || !Number.isFinite(trainRatio)) return fallback; + return Math.min(Math.max(trainRatio, 0.1), 0.95); +} + +/** + * Rolling walk-forward: a fixed-length train window slid forward, each followed + * immediately by the test window that comes next in time. + * + * Test windows are contiguous, equal-length, non-overlapping and strictly + * forward-ordered, which is what makes the per-fold results independent enough to + * count: "profitable in 4 of 5 folds" is a real consistency statement, whereas + * overlapping test windows would just be the same trades counted repeatedly. + * + * Train windows DO overlap between folds — that is inherent to sliding a window + * and is harmless, because train windows are never scored. + */ +export function walkForwardSplits( + snapshots: MarketSnapshot[], + opts: WalkForwardOptions = {}, +): WalkForwardSplit[] { + const minBars = opts.minBars ?? MIN_WARMUP_BARS; + const requested = Math.max(1, Math.floor(opts.folds ?? 4)); + const trainRatio = normalizeTrainRatio(opts.trainRatio, 0.6); + const n = snapshots.length; + + const trainSize = Math.floor(n * trainRatio); + const { folds, testSize } = usableFolds(n, trainSize, requested, minBars); + if (folds === 0) return []; + + const out: WalkForwardSplit[] = []; + for (let i = 0; i < folds; i++) { + const trainStart = i * testSize; + const trainEnd = trainStart + trainSize; + const testEnd = trainEnd + testSize; + out.push({ + index: i, + train: snapshots.slice(trainStart, trainEnd), + test: snapshots.slice(trainEnd, testEnd), + }); + } + return out; +} + +/** + * Anchored walk-forward: the train window always starts at bar 0 and grows by one + * test window per fold. + * + * This is the honest simulation of a live deployment — on any given day you have + * all of history available, not a 120-bar rolling slice — and it wastes no data, + * which matters when a series is barely long enough to split at all. + * + * Its blind spot is the mirror of the rolling version's: because early history + * never leaves the train window, a strategy that stopped working three years ago + * can still look acceptable here. Neither split is sufficient alone. + */ +export function anchoredWalkForward( + snapshots: MarketSnapshot[], + opts: WalkForwardOptions = {}, +): WalkForwardSplit[] { + const minBars = opts.minBars ?? MIN_WARMUP_BARS; + const requested = Math.max(1, Math.floor(opts.folds ?? 4)); + const trainRatio = normalizeTrainRatio(opts.trainRatio, 0.5); + const n = snapshots.length; + + const initialTrain = Math.floor(n * trainRatio); + const { folds, testSize } = usableFolds(n, initialTrain, requested, minBars); + if (folds === 0) return []; + + const out: WalkForwardSplit[] = []; + for (let i = 0; i < folds; i++) { + const trainEnd = initialTrain + i * testSize; + const testEnd = trainEnd + testSize; + out.push({ + index: i, + train: snapshots.slice(0, trainEnd), + test: snapshots.slice(trainEnd, testEnd), + }); + } + return out; +} diff --git a/packages/strategy-validation/src/synthetic.ts b/packages/strategy-validation/src/synthetic.ts new file mode 100644 index 0000000..a789b8f --- /dev/null +++ b/packages/strategy-validation/src/synthetic.ts @@ -0,0 +1,257 @@ +/** + * Deterministic synthetic market data and trade streams, for calibrating the + * validator against known ground truth. + * + * A statistical gauntlet is itself a measuring instrument, and an uncalibrated + * instrument is worse than no instrument: it reports numbers with the same + * confidence whether or not it is wired up correctly. The only way to know the + * gauntlet works is to run it on series whose right answer is known in advance: + * + * - `randomWalkPrices()` is the NULL HYPOTHESIS. There is no edge in it, by + * construction. Any strategy that "passes" on a random walk is proof that a + * gate is broken, and that is the single most valuable test in the package. + * - `sinePrices()` contains a real, exploitable mean-reversion edge, so a + * mean-reversion strategy MUST pass. If it fails, our gates are so strict + * that nothing can ever be listed, which is a different failure but still a + * failure. + * - `trendPrices()` rewards trend-following and punishes mean reversion, which + * is how ./regime.ts gets tested for actually distinguishing regimes. + * + * Everything here is seeded (`mulberry32`) rather than using Math.random. A + * flaky statistical test is indistinguishable from a real statistical finding, + * so randomness that cannot be replayed has no place anywhere near this code. + * + * These are exported from the package (not confined to *.test.ts) on purpose: + * downstream callers building their own listing policy need the same null- + * hypothesis baseline to check their thresholds against, and `syntheticTrade()` + * keeps knowledge of BacktestTrade's field semantics in exactly one place. + */ +import type { MarketSnapshot } from '@b1dz/core'; +import type { BacktestTrade } from '@b1dz/source-strategies'; + +/** One trading day in ms — the default bar spacing. */ +export const DAY_MS = 24 * 60 * 60 * 1000; + +/** + * mulberry32 — a 32-bit seeded PRNG. Tiny, fast, and good enough for fixtures + * (it passes gjrand's basic suite). Chosen over an LCG because low-bit LCG + * output is visibly periodic, and over Math.random because a test we cannot + * replay is a test we cannot debug. + */ +export function mulberry32(seed: number): () => number { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = a; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** Box–Muller standard normal draw from a uniform generator. */ +export function gaussian(rand: () => number): number { + // Reject exact 0 so log() stays finite. + const u = rand() || Number.EPSILON; + const v = rand(); + return Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v); +} + +export interface SnapshotSeriesOptions { + startTs?: number; + stepMs?: number; + exchange?: string; + pair?: string; + assetClass?: 'crypto' | 'equity'; + /** + * Half-spread as a fraction of price. Default 0 → bid === ask, which is what + * daily-close history actually looks like and the case ./costs.ts covers with + * `assumedHalfSpreadBps`. + */ + halfSpreadPct?: number; +} + +/** Turn a price series into a chronological MarketSnapshot stream. */ +export function snapshotsFrom(prices: number[], opts: SnapshotSeriesOptions = {}): MarketSnapshot[] { + const startTs = opts.startTs ?? Date.UTC(2020, 0, 1); + const stepMs = opts.stepMs ?? DAY_MS; + const half = opts.halfSpreadPct ?? 0; + return prices.map((p, i) => ({ + exchange: opts.exchange ?? 'test', + pair: opts.pair ?? 'X-USD', + bid: p * (1 - half), + ask: p * (1 + half), + bidSize: 1, + askSize: 1, + ts: startTs + i * stepMs, + assetClass: opts.assetClass, + })); +} + +export interface RandomWalkOptions { + bars: number; + start?: number; + /** Per-bar expected log return. 0 = a true martingale, i.e. no edge at all. */ + drift?: number; + /** Per-bar log-return standard deviation. 0.01 ≈ 1%/day ≈ 16% annualized. */ + vol?: number; + seed?: number; +} + +/** + * Geometric random walk — the null hypothesis. + * + * With `drift: 0` there is provably no exploitable structure: future returns are + * independent of the past, so every strategy's true expectancy is exactly minus + * its trading costs. Anything that looks profitable here is overfitting, and the + * gauntlet's job is to say so. + */ +export function randomWalkPrices(opts: RandomWalkOptions): number[] { + const { bars, start = 100, drift = 0, vol = 0.01, seed = 42 } = opts; + const rand = mulberry32(seed); + const out: number[] = []; + let p = start; + for (let i = 0; i < bars; i++) { + out.push(p); + p *= Math.exp(drift + vol * gaussian(rand)); + } + return out; +} + +export interface SineOptions { + bars: number; + start?: number; + /** Peak-to-mean swing as a fraction of `start`. 0.1 = ±10%. */ + amplitude?: number; + /** Bars per full cycle. */ + period?: number; + /** Per-bar gaussian noise as a fraction of price. */ + noise?: number; + /** Per-bar exponential drift applied on top of the cycle. */ + drift?: number; + seed?: number; +} + +/** + * Deterministic oscillation plus noise — a series with a REAL mean-reversion + * edge. Buy the troughs, sell the peaks, keep the amplitude. + * + * Used as the positive control. A gauntlet that rejects a mean-reversion + * strategy on this data is mis-calibrated, and a package that can only ever say + * "no" is not a store, it is a wall. + */ +export function sinePrices(opts: SineOptions): number[] { + const { bars, start = 100, amplitude = 0.1, period = 20, noise = 0, drift = 0, seed = 7 } = opts; + const rand = mulberry32(seed); + const out: number[] = []; + for (let i = 0; i < bars; i++) { + const cycle = 1 + amplitude * Math.sin((2 * Math.PI * i) / period); + const jitter = noise > 0 ? 1 + noise * gaussian(rand) : 1; + out.push(start * Math.exp(drift * i) * cycle * jitter); + } + return out; +} + +export interface TrendOptions { + bars: number; + start?: number; + /** Per-bar log drift. 0.002 ≈ +0.2%/bar, a firm but not absurd trend. */ + driftPerBar?: number; + noise?: number; + seed?: number; +} + +/** Persistent drift with noise — rewards trend following, punishes fading. */ +export function trendPrices(opts: TrendOptions): number[] { + const { bars, start = 100, driftPerBar = 0.002, noise = 0.004, seed = 11 } = opts; + const rand = mulberry32(seed); + const out: number[] = []; + let p = start; + for (let i = 0; i < bars; i++) { + out.push(p); + p *= Math.exp(driftPerBar + noise * gaussian(rand)); + } + return out; +} + +/** Concatenate price segments, rebasing each to continue from the previous close. */ +export function concatPrices(...segments: number[][]): number[] { + const out: number[] = []; + for (const seg of segments) { + if (seg.length === 0) continue; + if (out.length === 0) { + out.push(...seg); + continue; + } + const scale = out[out.length - 1]! / seg[0]!; + for (let i = 1; i < seg.length; i++) out.push(seg[i]! * scale); + } + return out; +} + +/** + * Build a BacktestTrade with internally consistent fields from a target profit. + * + * Every derived field (`netMultiple`, `tradeReturnPct`, `proceeds`, `cost`) is + * computed from `profit` and `cost` rather than accepted as input, so a fixture + * can never express an impossible trade — a −$10 profit with a +2% return would + * quietly invalidate every metric test built on it. + */ +export function syntheticTrade(opts: { + profit: number; + cost?: number; + entryTs?: number; + exitTs?: number; + entryPrice?: number; + exitPrice?: number; +}): BacktestTrade { + const cost = opts.cost ?? 100; + const profit = opts.profit; + const proceeds = cost + profit; + const entryTs = opts.entryTs ?? 0; + const exitTs = opts.exitTs ?? entryTs + DAY_MS; + const entryPrice = opts.entryPrice ?? 100; + const exitPrice = opts.exitPrice ?? entryPrice * (proceeds / cost); + const shares = cost / entryPrice; + + return { + entryTs, + exitTs, + entryPrice, + exitPrice, + entryMid: entryPrice, + exitMid: exitPrice, + shares, + notionalUsd: cost, + cost, + grossProceeds: proceeds, + proceeds, + entryFeeUsd: 0, + exitFeeUsd: 0, + feesUsd: 0, + spreadSlippageUsd: 0, + totalCostUsd: 0, + costBps: 0, + grossProfit: profit, + profit, + netMultiple: cost > 0 ? proceeds / cost : 1, + tradeReturnPct: cost > 0 ? profit / cost : 0, + entryReason: 'synthetic entry', + exitReason: 'synthetic exit', + }; +} + +/** A trade stream with exactly the given per-trade returns, one bar apart. */ +export function syntheticTrades(returns: number[], opts: { cost?: number; startTs?: number; stepMs?: number } = {}): BacktestTrade[] { + const cost = opts.cost ?? 100; + const startTs = opts.startTs ?? 0; + const stepMs = opts.stepMs ?? DAY_MS; + return returns.map((r, i) => + syntheticTrade({ + profit: r * cost, + cost, + entryTs: startTs + i * stepMs, + exitTs: startTs + (i + 1) * stepMs, + }), + ); +} diff --git a/packages/strategy-validation/tsconfig.build.json b/packages/strategy-validation/tsconfig.build.json new file mode 100644 index 0000000..2e725b7 --- /dev/null +++ b/packages/strategy-validation/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src", + "noEmit": false + }, + "exclude": ["**/*.test.ts"] +} diff --git a/packages/strategy-validation/tsconfig.json b/packages/strategy-validation/tsconfig.json new file mode 100644 index 0000000..bf5a36d --- /dev/null +++ b/packages/strategy-validation/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src/**/*"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2327f5a..a8832b2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -28,7 +28,7 @@ importers: version: 8.59.4(eslint@10.4.0(jiti@2.7.0))(typescript@6.0.3) vitest: specifier: latest - version: 4.1.7(@types/node@25.9.3)(vite@8.0.7(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)) + version: 4.1.7(@types/node@26.1.2)(vite@8.0.7(@types/node@26.1.2)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)) apps/cli: dependencies: @@ -87,8 +87,8 @@ importers: specifier: workspace:* version: link:../../packages/venue-types '@supabase/supabase-js': - specifier: latest - version: 2.106.1 + specifier: ^2.112.0 + version: 2.112.0 blessed: specifier: ^0.1.81 version: 0.1.81 @@ -186,6 +186,12 @@ importers: '@b1dz/storage-supabase': specifier: workspace:* version: link:../../packages/storage-supabase + '@b1dz/strategy-registry': + specifier: workspace:* + version: link:../../packages/strategy-registry + '@b1dz/strategy-validation': + specifier: workspace:* + version: link:../../packages/strategy-validation '@b1dz/trade-daemon': specifier: workspace:* version: link:../../packages/trade-daemon @@ -205,8 +211,8 @@ importers: specifier: workspace:* version: link:../../packages/wallet-service '@supabase/supabase-js': - specifier: latest - version: 2.106.1 + specifier: ^2.112.0 + version: 2.112.0 tsx: specifier: latest version: 4.22.3 @@ -252,13 +258,13 @@ importers: version: 0.1.1 '@profullstack/stack': specifier: ^0.1.3 - version: 0.1.3(@supabase/ssr@0.10.3(@supabase/supabase-js@2.106.1))(next@16.2.6(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react@19.2.6) + version: 0.1.3(@supabase/ssr@0.10.3(@supabase/supabase-js@2.112.0))(next@16.2.6(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react@19.2.6) '@supabase/ssr': specifier: latest - version: 0.10.3(@supabase/supabase-js@2.106.1) + version: 0.10.3(@supabase/supabase-js@2.112.0) '@supabase/supabase-js': - specifier: latest - version: 2.106.1 + specifier: ^2.112.0 + version: 2.112.0 lightweight-charts: specifier: ^5.2.0 version: 5.2.0 @@ -433,8 +439,8 @@ importers: specifier: workspace:* version: link:../venue-types '@supabase/supabase-js': - specifier: latest - version: 2.106.1 + specifier: ^2.112.0 + version: 2.112.0 devDependencies: '@types/node': specifier: latest @@ -752,8 +758,8 @@ importers: specifier: workspace:* version: link:../core '@supabase/supabase-js': - specifier: latest - version: 2.106.1 + specifier: ^2.112.0 + version: 2.112.0 devDependencies: '@types/node': specifier: latest @@ -765,6 +771,53 @@ importers: specifier: latest version: 4.1.7(@types/node@25.9.1)(vite@8.0.7(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)) + packages/strategy-registry: + dependencies: + '@b1dz/core': + specifier: workspace:* + version: link:../core + '@b1dz/source-strategies': + specifier: workspace:* + version: link:../source-strategies + '@b1dz/storage-supabase': + specifier: workspace:* + version: link:../storage-supabase + '@b1dz/strategy-validation': + specifier: workspace:* + version: link:../strategy-validation + '@supabase/supabase-js': + specifier: ^2.112.0 + version: 2.112.0 + devDependencies: + '@types/node': + specifier: latest + version: 26.1.2 + typescript: + specifier: 6.0.3 + version: 6.0.3 + vitest: + specifier: latest + version: 4.1.10(@types/node@26.1.2)(vite@8.0.7(@types/node@26.1.2)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)) + + packages/strategy-validation: + dependencies: + '@b1dz/core': + specifier: workspace:* + version: link:../core + '@b1dz/source-strategies': + specifier: workspace:* + version: link:../source-strategies + devDependencies: + '@types/node': + specifier: latest + version: 26.1.2 + typescript: + specifier: 6.0.3 + version: 6.0.3 + vitest: + specifier: latest + version: 4.1.10(@types/node@26.1.2)(vite@8.0.7(@types/node@26.1.2)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)) + packages/trade-daemon: dependencies: '@b1dz/event-channel': @@ -1296,11 +1349,12 @@ packages: '@keyv/serialize@1.1.1': resolution: {integrity: sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==} - '@napi-rs/wasm-runtime@1.1.6': - resolution: {integrity: sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==} + '@napi-rs/wasm-runtime@1.2.2': + resolution: {integrity: sha512-JfB4kuJQjaoHuCTseIINHtHWeJnvgEcxjwA5t/Y00ZgaOO1Crz3fjT/p8kT28zA/Caz7oiUMn3d6H2yOVCVwuw==} + engines: {node: ^20.19.0 || ^22.13.0 || >=23.5.0} peerDependencies: - '@emnapi/core': ^1.7.1 - '@emnapi/runtime': ^1.7.1 + '@emnapi/core': ^1.7.1 || ^2.0.0-alpha.3 + '@emnapi/runtime': ^1.7.1 || ^2.0.0-alpha.3 '@next/env@16.2.6': resolution: {integrity: sha512-gd8HoHN4ufj73WmR3JmVolrpJR47ILK6LouP5xElPglaVxir6e1a7VzvTvDWkOoPXT9rkkTzyCxBu4yeZfZwcw==} @@ -1568,37 +1622,42 @@ packages: '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} - '@supabase/auth-js@2.106.1': - resolution: {integrity: sha512-7eyheXfAGwkB9bZewJPs+N3UYt6kra2JG6mIxNEgbkvcO15PLD1e75PTIUEYYl3zrifm3GrpShVl7QZxKrXO/w==} - engines: {node: '>=20.0.0'} + '@supabase/auth-js@2.112.0': + resolution: {integrity: sha512-8qAdObNQHKbSeVBLmf2WLNT7+bCE8zBofpCFiNUqGBAl5qw9VagSvcmPXlh78McAU7iHrGik1oeJxiB2A6B29w==} + engines: {node: '>=22.0.0'} - '@supabase/functions-js@2.106.1': - resolution: {integrity: sha512-XbOPnR2mW7jp/EcW447xmGwCa+/Wc00Hkw8t4tUIJjRsHQ4xAESsLKcyLRhRJjJoUnJVXUlC+w0wUxUCM7CG2A==} - engines: {node: '>=20.0.0'} + '@supabase/functions-js@2.112.0': + resolution: {integrity: sha512-2DdaEZs0vq86orMIZBO+eM5w5/UxZb1EZyg2JraBKS4W9BzfFO+fOFD6YStmZ7iAOWvxNTGPjPNItsofpGgTCA==} + engines: {node: '>=22.0.0'} - '@supabase/phoenix@0.4.2': - resolution: {integrity: sha512-YSAGnmDAfuleFCVt3CeurQZAhxRfXWeZIIkwp7NhYzQ1UwW6ePSnzsFAiUm/mbCkfoCf70QQHKW/K6RKh52a4A==} + '@supabase/phoenix@0.4.5': + resolution: {integrity: sha512-aAn9H9ovVyeApKy11OWOrrOGq8DV68yWeH4ud2lN9fzn4aO8Zb5GLL9m1pUg9nLqIcT+ZDfAcsZe0E/nqdv2lw==} - '@supabase/postgrest-js@2.106.1': - resolution: {integrity: sha512-Qbn6d2lqiqeaBX1Uko0e/hL90dtQGRN6CG2wMVQtJpRFstlVW45qmUTyTOsiB8dYUWu1fWYo4YzJuDbokGv3tQ==} - engines: {node: '>=20.0.0'} + '@supabase/postgrest-js@2.112.0': + resolution: {integrity: sha512-4HKCVq32Jlk/wS8Ud8QgaAuQ4u6w1hZfw/gS5IcIN7wYddElMj4kfiCZpHMPfncomMnYzAKBUhwBZPpRcTH2Yw==} + engines: {node: '>=22.0.0'} - '@supabase/realtime-js@2.106.1': - resolution: {integrity: sha512-eQCYri5E8KsjpDgC7g28cOOS2britjUWdNSJluFMainqrMRepzjOnaxqXc3RoAz7H0dxmBrfLUNF6NGP8C+YaA==} - engines: {node: '>=20.0.0'} + '@supabase/realtime-js@2.112.0': + resolution: {integrity: sha512-McFFP+ivFDMTaCEh8JDpG+sPEmv5IjKvrP0uTH3Lbsriai4KbxB8ycY8TqPQnbjhOVjEifQm1q0y2tL1CUw9Zg==} + engines: {node: '>=22.0.0'} '@supabase/ssr@0.10.3': resolution: {integrity: sha512-ux2CJgX89h0Fz2lY7ZNafNG2SkXpyRc5dz77K9eKeBLPdtywQixKwIuetDeIViAJBp/buOUVmgj8PVesOklNpw==} peerDependencies: '@supabase/supabase-js': ^2.105.3 - '@supabase/storage-js@2.106.1': - resolution: {integrity: sha512-HWcLIhqinhWKpOQ3WzglR2unjW0eh9J7yOu3IZrZNIEkraK4La/HDvTqndljGsNw0itPtyHhuKBxRoPG1VUARw==} - engines: {node: '>=20.0.0'} + '@supabase/storage-js@2.112.0': + resolution: {integrity: sha512-X44Bl045X/e5e2tJqWsY+JmQvgtm04BJuijiWprIWYLn0mDvGnu8hLdzPOPcTVji7wlqbt2ZUHttsv+rGKQYXw==} + engines: {node: '>=22.0.0'} - '@supabase/supabase-js@2.106.1': - resolution: {integrity: sha512-gP4HurGkGu7Z3xoOCjtAI17BKKp7jpsmwY0Ssbsks9XQRzJ7ZhK7LxfLdBSYgUdgZCQgjRK+Mr7+cl4Gxrk0Rw==} - engines: {node: '>=20.0.0'} + '@supabase/supabase-js@2.112.0': + resolution: {integrity: sha512-dHVOgog58GOagtrZuPxJYg/R45ZV2U0qqgXffH+lMlt1OS+267Pw4g7bw3iXCGxN85OufiE0nI1baxDXlgEyfQ==} + engines: {node: '>=22.0.0'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0' + peerDependenciesMeta: + '@opentelemetry/api': + optional: true '@swc/helpers@0.5.15': resolution: {integrity: sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g==} @@ -1757,6 +1816,9 @@ packages: '@types/node@25.9.3': resolution: {integrity: sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg==} + '@types/node@26.1.2': + resolution: {integrity: sha512-Vu4a5UFA9rIIFJ7rB/Vaafh9lrCQszopTCx6KjFboXTGQbPNasehVR5TEiithSDGyd1DEiUByggTZsg8jukeIg==} + '@types/react-dom@19.2.3': resolution: {integrity: sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==} peerDependencies: @@ -1827,12 +1889,26 @@ packages: resolution: {integrity: sha512-U3gxVaDVnuZKhSspW/MzMxE1kq7zOdc072FcSNoqA1I9p8HyKbBFfEHoWckBAMgNMph4MamwS5iTVzFmrnt8TQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@vitest/expect@4.1.10': + resolution: {integrity: sha512-YsCn+qAk1GWjQOWFEsEcL2gNQ0zmVmQu3T03qP6UyjhtmdtwtbuI+DASn/7iQB3HGTXkdBwGddzxPlmiql5vlA==} + '@vitest/expect@4.1.7': resolution: {integrity: sha512-1R+tw0ortHEbZDGMymm+pN7/AFQ/RkFFdtd7EN+VBpynKmLbP8A3rpEXdshBJ7+8hQ9zBJh/i1s0yKNtxAnU7w==} '@vitest/expect@4.1.9': resolution: {integrity: sha512-vl/rYsUKcBr3SnQn166+XR5ZQcgMx3DQhFWdfli/cWpLnLUmbxZvyrJZotLFUryib+LtArYMSTJ5RbQ57ZqrlA==} + '@vitest/mocker@4.1.10': + resolution: {integrity: sha512-v0xaezt+DKEmKfaxg133ldzADrwLGd7Ze1MfQQTYfvs8OqZIwbxyxaYURivwV7sWy5fqn3rH5uOrSp07bp44Ow==} + peerDependencies: + msw: ^2.4.9 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + '@vitest/mocker@4.1.7': resolution: {integrity: sha512-vY7nuamKgfvpA1Koa3oYIw/k7D6kZnpGyNMZW8loow2bsBYla1TFdqTaXncWdRn4pgwNs+90RhnXhJScDwQeJA==} peerDependencies: @@ -1855,30 +1931,45 @@ packages: vite: optional: true + '@vitest/pretty-format@4.1.10': + resolution: {integrity: sha512-W1HsjSH4MXQ9YfmmhLAoIYf1HRfekQCGngeIgcei6MP5QQGWUe0gkopdZQaVCFO+JDJMrAJGwa5pRpNpvy4P8Q==} + '@vitest/pretty-format@4.1.7': resolution: {integrity: sha512-umgCarTOYQWIaDMvGDRZij+6b9oVeLIyJzfN+AS88e0ZOU3QTgNNSTtjQOpcvWr3np1N0j4WgZj+sb3oYBDscw==} '@vitest/pretty-format@4.1.9': resolution: {integrity: sha512-s0iufns3iIFitdgm+YR7g1whCAaGtXz459VS9/PqyKDEEFgYIhsHOQmXgIgDuYCt7DeQmiZT0Qe2OA2p4ZPu5A==} + '@vitest/runner@4.1.10': + resolution: {integrity: sha512-IKI6kpIH+LmpROplyLwBBaCfMgOZOMsygVa6BARD6ahA04VRuJSa6OaVG7kRvSEMD870Vd91rSSw0eegtWyLGg==} + '@vitest/runner@4.1.7': resolution: {integrity: sha512-BapjmAQ2aI78WdMEfeUWivnfVzB+VPGwWRQcJE0OUq7qEeEcBsCSf+0T5iREBNE5nBb4wA5Ya0W6IA+sghdEFw==} '@vitest/runner@4.1.9': resolution: {integrity: sha512-KXLMDtc7oe70+3mJfGrPUWPesswH+3sTxAMAMl8DG7I8IUQT4XW718dY5ID3vPUcmlu27CcKfY4P3h3I29SLJg==} + '@vitest/snapshot@4.1.10': + resolution: {integrity: sha512-xRkfOT1qpTAi/Ti4Y1LtfRc3kEuqxGw59eN2jN9pRWMtS/XDevekhcFSqvQqjUNGksfjMJu3Y+oJ+4Ypn2OaJw==} + '@vitest/snapshot@4.1.7': resolution: {integrity: sha512-ZacLzja+TmJeZ1h14xW2FB/WpeimUD3haBXQPyJqxvo8jQTmfeA8zv58mtjN2C7EHXZDYVcVYdYmAxjkWVvKCw==} '@vitest/snapshot@4.1.9': resolution: {integrity: sha512-Jc7RKGNBo8Z28WYIm0Niej4xdSPByRf6mU58VpHQkd6Zh05rlnA+twjbK5HyeIGHxrzsc3mJgS43uM0CZKzaIA==} + '@vitest/spy@4.1.10': + resolution: {integrity: sha512-PLf/Ugvoq5wO/b4rwYCR1h2PSIdXz7wnkQFMiUpLdtM7l6pqVFcQIBEHyT1+l+cj7mNwAfZHzqXqDyjvOuwbDw==} + '@vitest/spy@4.1.7': resolution: {integrity: sha512-kbkI5LMWakyuTIvs6fUJ5qdIVb1XVKsYJAT4OJ938cHMROYMSfmoQdZy0aaAnjbbc8F61vkoTqz/Az+/HiIu5Q==} '@vitest/spy@4.1.9': resolution: {integrity: sha512-fHpsS6mIi+PiEW+vcRVOMkX1oSaPKne3VOclSFICPcGOmfKgXPU5iAah+wcNcj2xPrCCmfq99IDGf+EojhhvhA==} + '@vitest/utils@4.1.10': + resolution: {integrity: sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==} + '@vitest/utils@4.1.7': resolution: {integrity: sha512-T532WBu791cBxJlCl6SO+J14l81DQx6uQHm1bQbmCDY7nqlEIgkza/UFnSBNaUtSf41unldDFjdOBYEQC4b5Hw==} @@ -2548,70 +2639,140 @@ packages: cpu: [arm64] os: [android] + lightningcss-android-arm64@1.33.0: + resolution: {integrity: sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [android] + lightningcss-darwin-arm64@1.32.0: resolution: {integrity: sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [darwin] + lightningcss-darwin-arm64@1.33.0: + resolution: {integrity: sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [darwin] + lightningcss-darwin-x64@1.32.0: resolution: {integrity: sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [darwin] + lightningcss-darwin-x64@1.33.0: + resolution: {integrity: sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [darwin] + lightningcss-freebsd-x64@1.32.0: resolution: {integrity: sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [freebsd] + lightningcss-freebsd-x64@1.33.0: + resolution: {integrity: sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [freebsd] + lightningcss-linux-arm-gnueabihf@1.32.0: resolution: {integrity: sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==} engines: {node: '>= 12.0.0'} cpu: [arm] os: [linux] + lightningcss-linux-arm-gnueabihf@1.33.0: + resolution: {integrity: sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm] + os: [linux] + lightningcss-linux-arm64-gnu@1.32.0: resolution: {integrity: sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] + lightningcss-linux-arm64-gnu@1.33.0: + resolution: {integrity: sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + lightningcss-linux-arm64-musl@1.32.0: resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] + lightningcss-linux-arm64-musl@1.33.0: + resolution: {integrity: sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + lightningcss-linux-x64-gnu@1.32.0: resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] + lightningcss-linux-x64-gnu@1.33.0: + resolution: {integrity: sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + lightningcss-linux-x64-musl@1.32.0: resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] + lightningcss-linux-x64-musl@1.33.0: + resolution: {integrity: sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + lightningcss-win32-arm64-msvc@1.32.0: resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [win32] + lightningcss-win32-arm64-msvc@1.33.0: + resolution: {integrity: sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [win32] + lightningcss-win32-x64-msvc@1.32.0: resolution: {integrity: sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [win32] + lightningcss-win32-x64-msvc@1.33.0: + resolution: {integrity: sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [win32] + lightningcss@1.32.0: resolution: {integrity: sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==} engines: {node: '>= 12.0.0'} + lightningcss@1.33.0: + resolution: {integrity: sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==} + engines: {node: '>= 12.0.0'} + lightweight-charts@5.2.0: resolution: {integrity: sha512-ey3Vas8UhV06ni+LT9TA1nEe4y8So4Mi6CL/oarNHFMyTktz/xy8e8+oh04Q//eO3t6etvFXgayz2fClyFQb5w==} @@ -2696,8 +2857,8 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true - nanoid@3.3.16: - resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} + nanoid@3.3.17: + resolution: {integrity: sha512-xQLf0A3HOMlgHq0n247/LRuAOYmB7dXJ/DvAxGvsSBij45XtBSmQycu+F8ODbHwns/XyFZagyL1+J0Offw1E0g==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true @@ -2837,8 +2998,8 @@ packages: resolution: {integrity: sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==} engines: {node: ^10 || ^12 || >=14} - postcss@8.5.19: - resolution: {integrity: sha512-Mz8SaolMd8nB+G13WkORcxQKHZ/NE4xXevtkJHVuG+guo9/wYKlIMTKAqGdEmYOXR2ijPjTYNHssizdaVSUNdQ==} + postcss@8.5.25: + resolution: {integrity: sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==} engines: {node: ^10 || ^12 || >=14} prelude-ls@1.2.1: @@ -3188,6 +3349,9 @@ packages: undici-types@7.24.6: resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + undici-types@8.3.0: + resolution: {integrity: sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==} + undici@8.0.2: resolution: {integrity: sha512-B9MeU5wuFhkFAuNeA19K2GDFcQXZxq33fL0nRy2Aq30wdufZbyyvxW3/ChaeipXVfy/wUweZyzovQGk39+9k2w==} engines: {node: '>=22.19.0'} @@ -3260,6 +3424,47 @@ packages: yaml: optional: true + vitest@4.1.10: + resolution: {integrity: sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==} + engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@opentelemetry/api': ^1.9.0 + '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 4.1.10 + '@vitest/browser-preview': 4.1.10 + '@vitest/browser-webdriverio': 4.1.10 + '@vitest/coverage-istanbul': 4.1.10 + '@vitest/coverage-v8': 4.1.10 + '@vitest/ui': 4.1.10 + happy-dom: '*' + jsdom: '*' + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@opentelemetry/api': + optional: true + '@types/node': + optional: true + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/coverage-istanbul': + optional: true + '@vitest/coverage-v8': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + vitest@4.1.7: resolution: {integrity: sha512-flYyaFd2CgoCoU+0UKt3pxksgC+S02iTDN0n3LtqaMeXsI9SBcdNujc2k0DeFLzUn/0k538yNjOSdwgCqcrwJA==} engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} @@ -3716,7 +3921,7 @@ snapshots: '@keyv/serialize@1.1.1': {} - '@napi-rs/wasm-runtime@1.1.6(@emnapi/core@1.9.1)(@emnapi/runtime@1.9.1)': + '@napi-rs/wasm-runtime@1.2.2(@emnapi/core@1.9.1)(@emnapi/runtime@1.9.1)': dependencies: '@emnapi/core': 1.9.1 '@emnapi/runtime': 1.9.1 @@ -3774,12 +3979,12 @@ snapshots: optionalDependencies: react: 19.2.6 - '@profullstack/stack@0.1.3(@supabase/ssr@0.10.3(@supabase/supabase-js@2.106.1))(next@16.2.6(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react@19.2.6)': + '@profullstack/stack@0.1.3(@supabase/ssr@0.10.3(@supabase/supabase-js@2.112.0))(next@16.2.6(react-dom@19.2.6(react@19.2.6))(react@19.2.6))(react@19.2.6)': dependencies: '@profullstack/emailer': 1.0.1 '@profullstack/referrals': 0.1.0(react@19.2.6) optionalDependencies: - '@supabase/ssr': 0.10.3(@supabase/supabase-js@2.106.1) + '@supabase/ssr': 0.10.3(@supabase/supabase-js@2.112.0) next: 16.2.6(react-dom@19.2.6(react@19.2.6))(react@19.2.6) react: 19.2.6 @@ -3856,7 +4061,7 @@ snapshots: dependencies: '@emnapi/core': 1.9.1 '@emnapi/runtime': 1.9.1 - '@napi-rs/wasm-runtime': 1.1.6(@emnapi/core@1.9.1)(@emnapi/runtime@1.9.1) + '@napi-rs/wasm-runtime': 1.2.2(@emnapi/core@1.9.1)(@emnapi/runtime@1.9.1) optional: true '@rolldown/binding-win32-arm64-msvc@1.0.0-rc.13': @@ -3890,42 +4095,42 @@ snapshots: '@standard-schema/spec@1.1.0': {} - '@supabase/auth-js@2.106.1': + '@supabase/auth-js@2.112.0': dependencies: tslib: 2.8.1 - '@supabase/functions-js@2.106.1': + '@supabase/functions-js@2.112.0': dependencies: tslib: 2.8.1 - '@supabase/phoenix@0.4.2': {} + '@supabase/phoenix@0.4.5': {} - '@supabase/postgrest-js@2.106.1': + '@supabase/postgrest-js@2.112.0': dependencies: tslib: 2.8.1 - '@supabase/realtime-js@2.106.1': + '@supabase/realtime-js@2.112.0': dependencies: - '@supabase/phoenix': 0.4.2 + '@supabase/phoenix': 0.4.5 tslib: 2.8.1 - '@supabase/ssr@0.10.3(@supabase/supabase-js@2.106.1)': + '@supabase/ssr@0.10.3(@supabase/supabase-js@2.112.0)': dependencies: - '@supabase/supabase-js': 2.106.1 + '@supabase/supabase-js': 2.112.0 cookie: 1.1.1 - '@supabase/storage-js@2.106.1': + '@supabase/storage-js@2.112.0': dependencies: iceberg-js: 0.8.1 tslib: 2.8.1 - '@supabase/supabase-js@2.106.1': + '@supabase/supabase-js@2.112.0': dependencies: - '@supabase/auth-js': 2.106.1 - '@supabase/functions-js': 2.106.1 - '@supabase/postgrest-js': 2.106.1 - '@supabase/realtime-js': 2.106.1 - '@supabase/storage-js': 2.106.1 + '@supabase/auth-js': 2.112.0 + '@supabase/functions-js': 2.112.0 + '@supabase/postgrest-js': 2.112.0 + '@supabase/realtime-js': 2.112.0 + '@supabase/storage-js': 2.112.0 '@swc/helpers@0.5.15': dependencies: @@ -4054,6 +4259,10 @@ snapshots: dependencies: undici-types: 7.24.6 + '@types/node@26.1.2': + dependencies: + undici-types: 8.3.0 + '@types/react-dom@19.2.3(@types/react@19.2.14)': dependencies: '@types/react': 19.2.14 @@ -4157,6 +4366,15 @@ snapshots: '@typescript-eslint/types': 8.59.4 eslint-visitor-keys: 5.0.1 + '@vitest/expect@4.1.10': + dependencies: + '@standard-schema/spec': 1.1.0 + '@types/chai': 5.2.3 + '@vitest/spy': 4.1.10 + '@vitest/utils': 4.1.10 + chai: 6.2.2 + tinyrainbow: 3.1.0 + '@vitest/expect@4.1.7': dependencies: '@standard-schema/spec': 1.1.0 @@ -4175,6 +4393,14 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 + '@vitest/mocker@4.1.10(vite@8.0.7(@types/node@26.1.2)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3))': + dependencies: + '@vitest/spy': 4.1.10 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 8.0.7(@types/node@26.1.2)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3) + '@vitest/mocker@4.1.7(vite@8.0.7(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3))': dependencies: '@vitest/spy': 4.1.7 @@ -4183,13 +4409,13 @@ snapshots: optionalDependencies: vite: 8.0.7(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3) - '@vitest/mocker@4.1.7(vite@8.0.7(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3))': + '@vitest/mocker@4.1.7(vite@8.0.7(@types/node@26.1.2)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3))': dependencies: '@vitest/spy': 4.1.7 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.0.7(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3) + vite: 8.0.7(@types/node@26.1.2)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3) '@vitest/mocker@4.1.9(vite@8.0.7(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3))': dependencies: @@ -4199,6 +4425,10 @@ snapshots: optionalDependencies: vite: 8.0.7(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3) + '@vitest/pretty-format@4.1.10': + dependencies: + tinyrainbow: 3.1.0 + '@vitest/pretty-format@4.1.7': dependencies: tinyrainbow: 3.1.0 @@ -4207,6 +4437,11 @@ snapshots: dependencies: tinyrainbow: 3.1.0 + '@vitest/runner@4.1.10': + dependencies: + '@vitest/utils': 4.1.10 + pathe: 2.0.3 + '@vitest/runner@4.1.7': dependencies: '@vitest/utils': 4.1.7 @@ -4217,6 +4452,13 @@ snapshots: '@vitest/utils': 4.1.9 pathe: 2.0.3 + '@vitest/snapshot@4.1.10': + dependencies: + '@vitest/pretty-format': 4.1.10 + '@vitest/utils': 4.1.10 + magic-string: 0.30.21 + pathe: 2.0.3 + '@vitest/snapshot@4.1.7': dependencies: '@vitest/pretty-format': 4.1.7 @@ -4231,10 +4473,18 @@ snapshots: magic-string: 0.30.21 pathe: 2.0.3 + '@vitest/spy@4.1.10': {} + '@vitest/spy@4.1.7': {} '@vitest/spy@4.1.9': {} + '@vitest/utils@4.1.10': + dependencies: + '@vitest/pretty-format': 4.1.10 + convert-source-map: 2.0.0 + tinyrainbow: 3.1.0 + '@vitest/utils@4.1.7': dependencies: '@vitest/pretty-format': 4.1.7 @@ -4884,36 +5134,69 @@ snapshots: lightningcss-android-arm64@1.32.0: optional: true + lightningcss-android-arm64@1.33.0: + optional: true + lightningcss-darwin-arm64@1.32.0: optional: true + lightningcss-darwin-arm64@1.33.0: + optional: true + lightningcss-darwin-x64@1.32.0: optional: true + lightningcss-darwin-x64@1.33.0: + optional: true + lightningcss-freebsd-x64@1.32.0: optional: true + lightningcss-freebsd-x64@1.33.0: + optional: true + lightningcss-linux-arm-gnueabihf@1.32.0: optional: true + lightningcss-linux-arm-gnueabihf@1.33.0: + optional: true + lightningcss-linux-arm64-gnu@1.32.0: optional: true + lightningcss-linux-arm64-gnu@1.33.0: + optional: true + lightningcss-linux-arm64-musl@1.32.0: optional: true + lightningcss-linux-arm64-musl@1.33.0: + optional: true + lightningcss-linux-x64-gnu@1.32.0: optional: true + lightningcss-linux-x64-gnu@1.33.0: + optional: true + lightningcss-linux-x64-musl@1.32.0: optional: true + lightningcss-linux-x64-musl@1.33.0: + optional: true + lightningcss-win32-arm64-msvc@1.32.0: optional: true + lightningcss-win32-arm64-msvc@1.33.0: + optional: true + lightningcss-win32-x64-msvc@1.32.0: optional: true + lightningcss-win32-x64-msvc@1.33.0: + optional: true + lightningcss@1.32.0: dependencies: detect-libc: 2.1.2 @@ -4930,6 +5213,22 @@ snapshots: lightningcss-win32-arm64-msvc: 1.32.0 lightningcss-win32-x64-msvc: 1.32.0 + lightningcss@1.33.0: + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + lightningcss-android-arm64: 1.33.0 + lightningcss-darwin-arm64: 1.33.0 + lightningcss-darwin-x64: 1.33.0 + lightningcss-freebsd-x64: 1.33.0 + lightningcss-linux-arm-gnueabihf: 1.33.0 + lightningcss-linux-arm64-gnu: 1.33.0 + lightningcss-linux-arm64-musl: 1.33.0 + lightningcss-linux-x64-gnu: 1.33.0 + lightningcss-linux-x64-musl: 1.33.0 + lightningcss-win32-arm64-msvc: 1.33.0 + lightningcss-win32-x64-msvc: 1.33.0 + lightweight-charts@5.2.0: dependencies: fancy-canvas: 2.1.0 @@ -5006,7 +5305,7 @@ snapshots: nanoid@3.3.12: {} - nanoid@3.3.16: {} + nanoid@3.3.17: {} natural-compare@1.4.0: {} @@ -5160,9 +5459,9 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 - postcss@8.5.19: + postcss@8.5.25: dependencies: - nanoid: 3.3.16 + nanoid: 3.3.17 picocolors: 1.1.1 source-map-js: 1.2.1 @@ -5570,6 +5869,8 @@ snapshots: undici-types@7.24.6: {} + undici-types@8.3.0: {} + undici@8.0.2: {} universalify@2.0.1: {} @@ -5605,9 +5906,9 @@ snapshots: vite@8.0.7(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3): dependencies: - lightningcss: 1.32.0 + lightningcss: 1.33.0 picomatch: 4.0.5 - postcss: 8.5.19 + postcss: 8.5.25 rolldown: 1.0.0-rc.13 tinyglobby: 0.2.17 optionalDependencies: @@ -5619,9 +5920,9 @@ snapshots: vite@8.0.7(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3): dependencies: - lightningcss: 1.32.0 + lightningcss: 1.33.0 picomatch: 4.0.5 - postcss: 8.5.19 + postcss: 8.5.25 rolldown: 1.0.0-rc.13 tinyglobby: 0.2.17 optionalDependencies: @@ -5631,6 +5932,47 @@ snapshots: jiti: 2.7.0 tsx: 4.22.3 + vite@8.0.7(@types/node@26.1.2)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3): + dependencies: + lightningcss: 1.33.0 + picomatch: 4.0.5 + postcss: 8.5.25 + rolldown: 1.0.0-rc.13 + tinyglobby: 0.2.17 + optionalDependencies: + '@types/node': 26.1.2 + esbuild: 0.28.0 + fsevents: 2.3.3 + jiti: 2.7.0 + tsx: 4.22.3 + + vitest@4.1.10(@types/node@26.1.2)(vite@8.0.7(@types/node@26.1.2)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)): + dependencies: + '@vitest/expect': 4.1.10 + '@vitest/mocker': 4.1.10(vite@8.0.7(@types/node@26.1.2)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)) + '@vitest/pretty-format': 4.1.10 + '@vitest/runner': 4.1.10 + '@vitest/snapshot': 4.1.10 + '@vitest/spy': 4.1.10 + '@vitest/utils': 4.1.10 + es-module-lexer: 2.0.0 + expect-type: 1.3.0 + magic-string: 0.30.21 + obug: 2.1.1 + pathe: 2.0.3 + picomatch: 4.0.5 + std-env: 4.0.0 + tinybench: 2.9.0 + tinyexec: 1.1.1 + tinyglobby: 0.2.17 + tinyrainbow: 3.1.0 + vite: 8.0.7(@types/node@26.1.2)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 26.1.2 + transitivePeerDependencies: + - msw + vitest@4.1.7(@types/node@25.9.1)(vite@8.0.7(@types/node@25.9.1)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)): dependencies: '@vitest/expect': 4.1.7 @@ -5658,10 +6000,10 @@ snapshots: transitivePeerDependencies: - msw - vitest@4.1.7(@types/node@25.9.3)(vite@8.0.7(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)): + vitest@4.1.7(@types/node@26.1.2)(vite@8.0.7(@types/node@26.1.2)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)): dependencies: '@vitest/expect': 4.1.7 - '@vitest/mocker': 4.1.7(vite@8.0.7(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)) + '@vitest/mocker': 4.1.7(vite@8.0.7(@types/node@26.1.2)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3)) '@vitest/pretty-format': 4.1.7 '@vitest/runner': 4.1.7 '@vitest/snapshot': 4.1.7 @@ -5678,10 +6020,10 @@ snapshots: tinyexec: 1.1.1 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 - vite: 8.0.7(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3) + vite: 8.0.7(@types/node@26.1.2)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.3) why-is-node-running: 2.3.0 optionalDependencies: - '@types/node': 25.9.3 + '@types/node': 26.1.2 transitivePeerDependencies: - msw diff --git a/supabase/migrations/20260803120000_strategy_registry.sql b/supabase/migrations/20260803120000_strategy_registry.sql new file mode 100644 index 0000000..8a4c233 --- /dev/null +++ b/supabase/migrations/20260803120000_strategy_registry.sql @@ -0,0 +1,74 @@ +-- strategy_registry + forward_trades +-- +-- The strategy store's moat. A TSP document that passes the gauntlet is +-- registered here and graduate to forward-test (paper trading over live +-- market data). Every forward trade is recorded, and once the live track +-- record reaches MinTRL, the strategy can be listed for sale. +-- +-- Tables: +-- strategy_registry — one row per (user, candidate) that survived the gauntlet +-- forward_trades — every paper trade the forward-test daemon records +-- +-- RLS: users only see their own registrations and trades. The daemon uses a +-- service-role key to poll across all users. + +-- ----- strategy_registry ---------------------------------------------------- +create table public.strategy_registry ( + id uuid primary key default gen_random_uuid(), + user_id uuid not null references auth.users(id) on delete cascade, + candidate_id text not null, + tsp_doc jsonb not null, + compiled boolean not null default true, + status text not null default 'gauntlet_passed' + check (status in ('gauntlet_passed','forward_running','min_trl_reached','listed','rejected','archived')), + gauntlet_report jsonb not null default '{}', + cost_model jsonb not null, + listed_at timestamptz, + rejected_at timestamptz, + archived_at timestamptz, + created_at timestamptz not null default now() +); + +create unique index strategy_registry_user_candidate_idx + on public.strategy_registry (user_id, candidate_id); + +create index strategy_registry_user_id_idx + on public.strategy_registry (user_id); + +create index strategy_registry_status_idx + on public.strategy_registry (status); + +alter table public.strategy_registry enable row level security; + +create policy "users see own strategy_registrations" + on public.strategy_registry for all + to authenticated using (user_id = auth.uid()) with check (user_id = auth.uid()); + +-- ----- forward_trades ------------------------------------------------------- +create table public.forward_trades ( + id uuid primary key default gen_random_uuid(), + strategy_id uuid not null references public.strategy_registry(id) on delete cascade, + user_id uuid not null references auth.users(id) on delete cascade, + entry_ts timestamptz not null, + exit_ts timestamptz, + entered_at timestamptz not null default now(), + closed_at timestamptz, + trade_json jsonb not null, + regime_at_entry text, + recorded_at timestamptz not null default now() +); + +create index forward_trades_strategy_id_idx + on public.forward_trades (strategy_id); + +create index forward_trades_user_id_idx + on public.forward_trades (user_id); + +create index forward_trades_strategy_open_idx + on public.forward_trades (strategy_id) where exit_ts is null; + +alter table public.forward_trades enable row level security; + +create policy "users see own forward_trades" + on public.forward_trades for all + to authenticated using (user_id = auth.uid()) with check (user_id = auth.uid()); diff --git a/supabase/migrations/20260804120000_strategy_registry_indices.sql b/supabase/migrations/20260804120000_strategy_registry_indices.sql new file mode 100644 index 0000000..9d369d9 --- /dev/null +++ b/supabase/migrations/20260804120000_strategy_registry_indices.sql @@ -0,0 +1,14 @@ +-- strategy_registry + forward_trades — add composite indices for daemon polling +-- and trade history queries. +-- +-- The tables were created in 20260803120000_strategy_registry.sql. +-- This migration adds the composite indices the forward-test daemon needs. + +-- Composite index for daemon polling: "give me all gauntlet_passed + +-- forward_running entries for user X". +create index if not exists strategy_registry_user_status_idx + on public.strategy_registry (user_id, status); + +-- Composite index for trade history lookups ordered by entry_ts. +create index if not exists forward_trades_strategy_entry_idx + on public.forward_trades (strategy_id, entry_ts);