Skip to content

test: drop wall-clock search-engine benchmark tests #267

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
arzafran merged 1 commit into from
Aug 6, 2026
Merged

test: drop wall-clock search-engine benchmark tests #267

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 3 additions & 89 deletions programaTests/CommandPaletteSearchEngineTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import XCTest

// Search-engine timing/throughput is measured out-of-band, not in this unit suite —
// see tests_v2/test_cpu_occlusion_throttle.py and docs/cpu-harness.md.

#if canImport(Programa_DEV)
@testable import Programa_DEV
#elseif canImport(Programa)
Expand Down Expand Up @@ -218,17 +221,6 @@ final class CommandPaletteSearchEngineTests: XCTestCase {
return max(fuzzyScore, titleScore + 2000)
}

private func benchmarkElapsedMs(operation: () -> Void) -> Double {
let start = DispatchTime.now().uptimeNanoseconds
operation()
let elapsed = DispatchTime.now().uptimeNanoseconds - start
return Double(elapsed) / 1_000_000
}

private func repeatedQueries(_ baseQueries: [String], repetitions: Int) -> [String] {
Array(repeating: baseQueries, count: repetitions).flatMap { $0 }
}

func testOptimizedSearchMatchesReferencePipeline() {
let commandEntries = makeCommandEntries(count: 96)
let switcherEntries = makeSwitcherEntries(count: 64)
Expand Down Expand Up @@ -827,82 +819,4 @@ final class CommandPaletteSearchEngineTests: XCTestCase {
XCTAssertNotEqual(base, changedSurfaceMetadata)
XCTAssertNotEqual(base, changedSurfaceKind)
}

func testCommandSearchBenchmarkBeatsLegacyPipeline() {
let entries = makeCommandEntries(count: 900)
let corpus = entries.map { entry in
CommandPaletteSearchCorpusEntry(
payload: entry.id,
rank: entry.rank,
title: entry.title,
searchableTexts: entry.searchableTexts
)
}
let queries = repeatedQueries(
["rename", "rename tab", "open dir", "toggle side", "apply update", "notif", "split right", "cmux"],
repetitions: 12
)

for query in queries.prefix(8) {
_ = referenceResults(entries: entries, query: query)
_ = CommandPaletteSearchEngine.search(entries: corpus, query: query) { _, _ in 0 }
}

let referenceMs = benchmarkElapsedMs {
for query in queries {
_ = referenceResults(entries: entries, query: query)
}
}
let optimizedMs = benchmarkElapsedMs {
for query in queries {
_ = CommandPaletteSearchEngine.search(entries: corpus, query: query) { _, _ in 0 }
}
}

print(String(format: "BENCH cmd+shift+p reference=%.2fms optimized=%.2fms", referenceMs, optimizedMs))
XCTAssertLessThan(
optimizedMs,
referenceMs * 1.25,
"Optimized command search regressed significantly: reference=\(referenceMs) optimized=\(optimizedMs)"
)
}

func testSwitcherSearchBenchmarkBeatsLegacyPipeline() {
let entries = makeSwitcherEntries(count: 400)
let corpus = entries.map { entry in
CommandPaletteSearchCorpusEntry(
payload: entry.id,
rank: entry.rank,
title: entry.title,
searchableTexts: entry.searchableTexts
)
}
let queries = repeatedQueries(
["workspace 12", "phoenix", "feature-18", "rename-tab", "3007", "9202", "switch", "worktrees"],
repetitions: 12
)

for query in queries.prefix(8) {
_ = referenceResults(entries: entries, query: query)
_ = CommandPaletteSearchEngine.search(entries: corpus, query: query) { _, _ in 0 }
}

let referenceMs = benchmarkElapsedMs {
for query in queries {
_ = referenceResults(entries: entries, query: query)
}
}
let optimizedMs = benchmarkElapsedMs {
for query in queries {
_ = CommandPaletteSearchEngine.search(entries: corpus, query: query) { _, _ in 0 }
}
}

print(String(format: "BENCH cmd+p reference=%.2fms optimized=%.2fms", referenceMs, optimizedMs))
XCTAssertLessThan(
optimizedMs,
referenceMs * 1.25,
"Optimized switcher search regressed significantly: reference=\(referenceMs) optimized=\(optimizedMs)"
)
}
}
Loading