From 4d3262e5c4f1840d5d691b0ac5275e6665c1e55f Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 14 Nov 2026 14:29:54 -0800 Subject: [PATCH] fix: account for every offered read operation in benchmarks --- README.md | 10 + docs/metrics.md | 29 ++- docs/publishing.md | 4 + internal/bench/query.go | 98 ++++---- internal/bench/read_accounting.go | 302 +++++++++++++++++++++++ internal/bench/read_accounting_report.go | 51 ++++ internal/bench/read_accounting_test.go | 194 +++++++++++++++ internal/bench/report.go | 5 +- internal/bench/runner.go | 42 ++-- internal/bench/runner_test.go | 49 ++-- internal/bench/types.go | 2 + 11 files changed, 680 insertions(+), 106 deletions(-) create mode 100644 internal/bench/read_accounting.go create mode 100644 internal/bench/read_accounting_report.go create mode 100644 internal/bench/read_accounting_test.go diff --git a/README.md b/README.md index 13639a4..d9acf4d 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,16 @@ releases, check out the harness revision recorded in that report. New reports us report formats are not accepted; use the harness revision recorded in the archived evidence to read them. No published measurement is rewritten. +New read reports add scheduled-arrival accounting and the proposed +`interactive-http-draft/v1` qualification goals. They retain historical HTTP +service-time fields while reporting scheduler queue delay, arrival-to-completion +p95/p99, intended/performed operations, and during-load versus drain completions +separately. Each performed kind needs 1,000 successful samples for p99 +qualification; short sweeps therefore use the existing best-completion topology +fallback. See [read measurement and goals](docs/metrics.md#proposed-http-completion-profile). +Historical reports without this metadata are not qualified against the new +goals. HTTP completion is separate from browser last-panel paint. + Pitot Bench drives the official OpenTelemetry `telemetrygen` client, measures what Pitot actually accepts, samples the server throughout the run, and emits publication-ready JSON and Markdown. It is deliberately harder to fool than a one-line requests-per-second script. ## What the headline means diff --git a/docs/metrics.md b/docs/metrics.md index 11a6f02..8c083d7 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -83,9 +83,34 @@ A report passes its evidence-quality gates when: 3. the Pitot process start time does not change; 4. Pitot's accepted-row counter advances in every phase. -When read load is enabled, the sustained mixed phase must also complete at least 95% of the requested operation rate with zero operation failures and zero scheduler shedding. The report records operations canceled exactly at the phase boundary separately; they are not hidden as successful work. +When read load is enabled, the sustained mixed phase must also complete at least 95% of the requested operation rate **during the offered-load interval**, with zero operation failures, cancellations, scheduler-missed arrivals and scheduler shedding. It must meet the per-kind HTTP completion profile below. The scheduler stops when generators finish, before the final metrics scrape. Already dispatched work drains normally; successes completed during drain are reported separately and do not increase during-load goodput. -The read scheduler is open-loop and bounded. `query_attempts` counts scheduled operations, `query_latency_ms.count` counts valid completed operations, `query_shed` counts ticks rejected because the concurrency bound was full, and `query_canceled_at_phase_end` counts in-flight operations canceled when load stopped. +The read scheduler is open-loop and bounded. `query_attempts` retains its delivered scheduling-attempt count; `query_latency_ms` retains successful HTTP service time from worker dequeue through the operation's last response. Both exclude arrivals missed by the scheduler; the latency also excludes its queue. `query_shed` counts delivered attempts rejected because the concurrency bound was full. `query_canceled_at_phase_end` retains its historical name and counts canceled dispatched work; ordinary phase completion drains rather than cancels it. + +New `read_evidence` accounts for offered arrival slots from monotonic elapsed time and requested rate, independently of ticker deliveries. On a delayed wake, skipped slots become `scheduler_missed`; they are never replayed as an unbounded catch-up burst. At shutdown, every remaining slot due through the captured cutoff is likewise accounted for. Per-kind intended counts include scheduler misses and sheds. Target rotation advances with offered slots, so overloaded or fallback work cannot silently change the reported intended mix. Performed-kind counts retain actual fallback attribution. + +After drain, the accounting identities are: + +```text +scheduled = query_attempts + scheduler_missed +scheduled = scheduler_missed + query_shed + dispatched +dispatched = successful + failures + canceled +successful = completed_during_load + completed_during_drain +``` + +`successful_arrival_to_completion_ms` measures scheduled arrival through final successful HTTP response, including scheduler queue delay. `dispatched_queue_ms` measures queue delay for every dispatched operation, including failures and cancellations. Success percentiles never include shed or failed work; outcome counts remain adjacent and use all offered arrivals as their denominator. `intended_operations` records requested kind, missed/shed/dispatched counts, outcomes and fallbacks; `performed_operations` records actual kind, outcomes and latency. These typed fields supplement the existing methodology tables. + +### Proposed HTTP completion profile + +`settings.read_profile` declares `interactive-http-draft/v1`, a versioned engineering goal rather than a measured product guarantee. It participates in the workload hash. New qualification requires matching profile metadata and arrival accounting; historical reports retain their original fields and recorded gates, and are labeled as legacy evidence without qualification against this profile. + +| Performed HTTP user operation | p95 goal | p99 goal | +|---|---:|---:| +| Complete dashboard batch and annotations | 1,500 ms | 3,000 ms | +| Trace drill-down | 750 ms | 1,500 ms | +| Bounded trace detail | 300 ms | 750 ms | + +Each kind needs at least 1,000 successful samples before its p99 can qualify. Fewer samples are **insufficient evidence**, not a pass. The same gate controls topology eligibility and sustained qualification. Consequently a standard 30-second sweep at 20 operations/s cannot supply enough samples per kind: topology selection uses the documented best-completion fallback, and only a sufficiently sampled sustained phase can qualify. These are HTTP operation completion goals; browser trigger-to-last-panel-paint measurements remain separate. The 500 ms individual-panel goal is not applied to an entire dashboard batch. ### Dashboard panel evidence diff --git a/docs/publishing.md b/docs/publishing.md index d8141ea..76b2ea0 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -15,6 +15,8 @@ A performance number is useful only when another engineer can explain and reprod - [ ] signal-specific peak results and a sustained mixed result; - [ ] a separate read-under-write result when claiming interactive performance; - [ ] requested, completed, failed, and shed operation rates beside read latency; +- [ ] scheduled arrivals, scheduler-missed slots, queue delay and successful arrival-to-completion p95/p99 by intended/performed operation kind; report during-load completions separately from drain; +- [ ] the declared HTTP completion profile and at least 1,000 successful samples per performed kind before p99 qualification; insufficient samples never count as a pass; - [ ] all evidence-quality gates passing; - [ ] limitations stated next to the headline, not hidden in a footnote. @@ -29,6 +31,8 @@ Then publish the separate span, log-record, and metric-point sweeps. They explai Use median values across repetitions for the headline and retain every raw repetition. Report variability; do not select only the fastest run. +The sweep `--repeat` count does not repeat the final sustained phase. Use independent invocations when repeating sustained qualification. Current short sweeps are under-sampled for the HTTP p99 profile and therefore use the documented topology fallback; publish qualification only from a sustained phase passing the declared sample, goodput, outcome and latency gates. HTTP operation completion does not measure browser paint time. Older reports without arrival accounting/profile metadata remain historical evidence and cannot be relabeled as passing the current goals. + ## Language Prefer: diff --git a/internal/bench/query.go b/internal/bench/query.go index 42fb4a2..e5491f0 100644 --- a/internal/bench/query.go +++ b/internal/bench/query.go @@ -11,7 +11,6 @@ import ( "sort" "strings" "sync" - "sync/atomic" "time" ) @@ -25,32 +24,39 @@ const ( ) type queryRun struct { - targets []queryTarget - traceMu sync.Mutex - traces map[string]traceReference - selections map[string]drillSelection - kindsMu sync.Mutex - kinds map[string]*operationStats - panelsMu sync.Mutex - panels map[string]*panelStats - wg sync.WaitGroup - hist *histogram - failures int64 - attempts int64 - shed int64 - canceled int64 - stop chan struct{} - stopOnce sync.Once - cancel context.CancelFunc + scheduleMu sync.Mutex + startedAt, cutoff time.Time + rate float64 + expected, missed, completedDuringLoad, completedDuringDrain int64 + intended map[string]ReadIntended + endToEnd, queue *histogram + targets []queryTarget + traceMu sync.Mutex + traces map[string]traceReference + selections map[string]drillSelection + kindsMu sync.Mutex + kinds map[string]*operationStats + panelsMu sync.Mutex + panels map[string]*panelStats + wg sync.WaitGroup + hist *histogram + failures int64 + attempts int64 + shed int64 + canceled int64 + stop chan struct{} + stopOnce sync.Once + cancel context.CancelFunc } type operationStats struct { dispatched, failures, canceled int64 hist *histogram + endToEnd, queue *histogram } func newQueryRun() *queryRun { - return &queryRun{targets: []queryTarget{{name: "dashboard-refresh", body: []byte(`{}`)}}, traces: map[string]traceReference{}, selections: map[string]drillSelection{}, kinds: map[string]*operationStats{}, panels: map[string]*panelStats{}, hist: newHistogram(), stop: make(chan struct{}), cancel: func() {}} + return &queryRun{targets: []queryTarget{{name: "dashboard-refresh", body: []byte(`{}`)}}, traces: map[string]traceReference{}, selections: map[string]drillSelection{}, kinds: map[string]*operationStats{}, panels: map[string]*panelStats{}, hist: newHistogram(), endToEnd: newHistogram(), queue: newHistogram(), intended: map[string]ReadIntended{}, stop: make(chan struct{}), cancel: func() {}} } func (q *queryRun) start(ctx context.Context, client *http.Client, base, cookie string, rate float64) { @@ -69,15 +75,18 @@ func (q *queryRun) start(ctx context.Context, client *http.Client, base, cookie // busy and the queue is full. An unbuffered channel would also shed // whenever no worker happened to be parked in its receive at that instant, // which reports backpressure the server never caused. - jobs := make(chan int, workers) + jobs := make(chan readJob, workers) runCtx, cancel := context.WithCancel(ctx) q.cancel = cancel + q.scheduleMu.Lock() + q.startedAt, q.rate = time.Now(), rate + q.scheduleMu.Unlock() q.wg.Add(workers + 1) for worker := 0; worker < workers; worker++ { go func() { defer q.wg.Done() - for targetIndex := range jobs { - q.request(runCtx, client, base, cookie, targetIndex) + for job := range jobs { + q.requestJob(runCtx, client, base, cookie, job) } }() } @@ -86,21 +95,22 @@ func (q *queryRun) start(ctx context.Context, client *http.Client, base, cookie defer close(jobs) ticker := time.NewTicker(interval) defer ticker.Stop() - targetIndex := 0 for { select { case <-runCtx.Done(): + q.stopScheduling() return case <-q.stop: return case <-ticker.C: - atomic.AddInt64(&q.attempts, 1) - select { - case jobs <- targetIndex: - targetIndex++ - default: - atomic.AddInt64(&q.shed, 1) - } + q.advanceSchedule(time.Now(), func(job readJob) bool { + select { + case jobs <- job: + return true + default: + return false + } + }) } } }() @@ -111,36 +121,22 @@ func (q *queryRun) start(ctx context.Context, client *http.Client, base, cookie // already being read, and those abort as read errors rather than transport // errors, which the trial would then count as server failures. func (q *queryRun) finish() { - q.stopOnce.Do(func() { close(q.stop) }) + q.stopScheduling() q.wg.Wait() q.cancel() } func (q *queryRun) request(ctx context.Context, client *http.Client, base, cookie string, targetIndex int) { - query := q.targets[targetIndex%len(q.targets)] + q.requestJob(ctx, client, base, cookie, readJob{targetIndex: targetIndex, scheduledAt: time.Now(), intended: q.targets[targetIndex%len(q.targets)].name}) +} + +func (q *queryRun) requestJob(ctx context.Context, client *http.Client, base, cookie string, job readJob) { + query := q.targets[job.targetIndex%len(q.targets)] kind := query.name started := time.Now() success := false defer func() { - elapsed := time.Since(started) - q.kindsMu.Lock() - defer q.kindsMu.Unlock() - stats := q.kinds[kind] - if stats == nil { - stats = &operationStats{hist: newHistogram()} - q.kinds[kind] = stats - } - stats.dispatched++ - if success { - q.hist.record(elapsed) - stats.hist.record(elapsed) - } else if ctx.Err() != nil { - atomic.AddInt64(&q.canceled, 1) - stats.canceled++ - } else { - atomic.AddInt64(&q.failures, 1) - stats.failures++ - } + q.recordOperation(job, kind, started, time.Now(), success, ctx.Err() != nil) }() refresh := func() bool { body, ok, absent := queryPanelHTTP(ctx, client, base, cookie, queryRunPath, query.body) diff --git a/internal/bench/read_accounting.go b/internal/bench/read_accounting.go new file mode 100644 index 0000000..8722a8a --- /dev/null +++ b/internal/bench/read_accounting.go @@ -0,0 +1,302 @@ +package bench + +import ( + "fmt" + "math" + "sync/atomic" + "time" +) + +// The profile is a proposed engineering qualification goal, not a measured +// product guarantee. HTTP dashboard completion does not measure browser paint. +type ReadProfile struct { + Name string `json:"name"` + MinSamples int64 `json:"minimum_successful_samples_per_kind"` + Targets map[string]ReadTarget `json:"http_operation_targets"` +} + +type ReadTarget struct { + P95Ms float64 `json:"p95_ms"` + P99Ms float64 `json:"p99_ms"` +} + +func currentReadProfile() *ReadProfile { + return &ReadProfile{Name: "interactive-http-draft/v1", MinSamples: 1000, Targets: map[string]ReadTarget{ + "dashboard-refresh": {P95Ms: 1500, P99Ms: 3000}, + "drill-traces": {P95Ms: 750, P99Ms: 1500}, + "trace-detail": {P95Ms: 300, P99Ms: 750}, + }} +} + +func readProfileForRate(rate float64) *ReadProfile { + if rate <= 0 { + return nil + } + return currentReadProfile() +} + +func readProfileMatches(p *ReadProfile) bool { + want := currentReadProfile() + if p == nil || p.Name != want.Name || p.MinSamples != want.MinSamples || len(p.Targets) != len(want.Targets) { + return false + } + for kind, target := range want.Targets { + if p.Targets[kind] != target { + return false + } + } + return true +} + +func readCompletedRate(t Trial) float64 { + if t.Read != nil { + return perSecond(float64(t.Read.CompletedDuringLoad), t.Read.ScheduleSeconds) + } + // Historical reports retain their historical rate, but cannot pass the new + // qualification gate without arrival accounting and a declared profile. + return perSecond(float64(t.QueryLatency.Count), t.ElapsedSeconds) +} + +type ReadEvidence struct { + Profile string `json:"profile"` + ScheduleSeconds float64 `json:"schedule_seconds"` + Expected int64 `json:"scheduled"` + SchedulerMissed int64 `json:"scheduler_missed"` + CompletedDuringLoad int64 `json:"completed_during_load"` + CompletedDuringDrain int64 `json:"completed_during_drain"` + EndToEnd Latency `json:"successful_arrival_to_completion_ms"` + Queue Latency `json:"dispatched_queue_ms"` + Operations map[string]ReadOperation `json:"performed_operations"` + Intended map[string]ReadIntended `json:"intended_operations"` +} + +type ReadOperation struct { + Dispatched int64 `json:"dispatched"` + Failures int64 `json:"failures"` + Canceled int64 `json:"canceled"` + EndToEnd Latency `json:"successful_arrival_to_completion_ms"` + Queue Latency `json:"dispatched_queue_ms"` +} + +type ReadIntended struct { + Scheduled int64 `json:"scheduled"` + SchedulerMissed int64 `json:"scheduler_missed"` + Shed int64 `json:"shed"` + Dispatched int64 `json:"dispatched"` + Successful int64 `json:"successful"` + Failures int64 `json:"failures"` + Canceled int64 `json:"canceled"` + Fallbacks int64 `json:"performed_as_another_kind"` +} + +type readJob struct { + slot int64 + targetIndex int + scheduledAt time.Time + intended string +} + +// advanceSchedule accounts for every due arrival in bounded time, even when +// ticker deliveries are dropped. Missed slots are not replayed in a burst. +func (q *queryRun) advanceSchedule(now time.Time, dispatch func(readJob) bool) { + q.scheduleMu.Lock() + defer q.scheduleMu.Unlock() + if !q.cutoff.IsZero() { + return + } + q.advanceScheduleLocked(now, dispatch) +} + +func (q *queryRun) advanceScheduleLocked(now time.Time, dispatch func(readJob) bool) { + due := int64(math.Floor(float64(now.Sub(q.startedAt)) * q.rate / float64(time.Second))) + if due <= q.expected { + return + } + first := q.expected + 1 + q.expected = due + lastMissed := due + if dispatch != nil { + lastMissed-- + } + q.missed += lastMissed - first + 1 + q.attributeArrivals(first, lastMissed, "missed") + if dispatch == nil { + return + } + index := int((due - 1) % int64(len(q.targets))) + job := readJob{slot: due, targetIndex: index, scheduledAt: q.startedAt.Add(time.Duration(float64(due) * float64(time.Second) / q.rate)), intended: q.targets[index].name} + atomic.AddInt64(&q.attempts, 1) + if dispatch(job) { + q.attributeArrivals(due, due, "dispatched") + } else { + atomic.AddInt64(&q.shed, 1) + q.attributeArrivals(due, due, "shed") + } +} + +// Attribute an arbitrary range of slots without looping over missed arrivals. +// Both target rotation and intended operation mix include shed/missed slots. +func (q *queryRun) attributeArrivals(first, last int64, outcome string) { + if last < first { + return + } + n := int64(len(q.targets)) + before := func(slots, index int64) int64 { + count := slots / n + if index < slots%n { + count++ + } + return count + } + for index, target := range q.targets { + count := before(last, int64(index)) - before(first-1, int64(index)) + stats := q.intended[target.name] + stats.Scheduled += count + switch outcome { + case "missed": + stats.SchedulerMissed += count + case "shed": + stats.Shed += count + case "dispatched": + stats.Dispatched += count + } + q.intended[target.name] = stats + } +} + +func (q *queryRun) stopScheduling() { + q.stopOnce.Do(func() { + q.scheduleMu.Lock() + // Capture the cutoff under the same lock used to record completions, so + // completions cannot race a retroactively chosen measurement boundary. + q.cutoff = time.Now() + if !q.startedAt.IsZero() { + q.advanceScheduleLocked(q.cutoff, nil) + } + q.scheduleMu.Unlock() + close(q.stop) + }) +} + +func (q *queryRun) recordOperation(job readJob, kind string, started, finished time.Time, success, canceled bool) { + q.scheduleMu.Lock() + defer q.scheduleMu.Unlock() + q.kindsMu.Lock() + defer q.kindsMu.Unlock() + stats := q.kinds[kind] + if stats == nil { + stats = &operationStats{hist: newHistogram()} + q.kinds[kind] = stats + } + if stats.endToEnd == nil { + stats.endToEnd, stats.queue = newHistogram(), newHistogram() + } + stats.dispatched++ + queue := started.Sub(job.scheduledAt) + stats.queue.record(queue) + q.queue.record(queue) + intended := q.intended[job.intended] + if kind != job.intended { + intended.Fallbacks++ + } + if success { + q.hist.record(finished.Sub(started)) + stats.hist.record(finished.Sub(started)) + q.endToEnd.record(finished.Sub(job.scheduledAt)) + stats.endToEnd.record(finished.Sub(job.scheduledAt)) + intended.Successful++ + if q.cutoff.IsZero() || !finished.After(q.cutoff) { + q.completedDuringLoad++ + } else { + q.completedDuringDrain++ + } + } else if canceled { + atomic.AddInt64(&q.canceled, 1) + stats.canceled++ + intended.Canceled++ + } else { + atomic.AddInt64(&q.failures, 1) + stats.failures++ + intended.Failures++ + } + q.intended[job.intended] = intended +} + +func (q *queryRun) readEvidence() *ReadEvidence { + q.scheduleMu.Lock() + defer q.scheduleMu.Unlock() + q.kindsMu.Lock() + defer q.kindsMu.Unlock() + e := &ReadEvidence{Profile: currentReadProfile().Name, Expected: q.expected, SchedulerMissed: q.missed, CompletedDuringLoad: q.completedDuringLoad, CompletedDuringDrain: q.completedDuringDrain, EndToEnd: q.endToEnd.snapshot(), Queue: q.queue.snapshot(), Operations: map[string]ReadOperation{}, Intended: map[string]ReadIntended{}} + if !q.startedAt.IsZero() && !q.cutoff.IsZero() { + e.ScheduleSeconds = q.cutoff.Sub(q.startedAt).Seconds() + } + for kind, stats := range q.kinds { + if stats.endToEnd != nil { + e.Operations[kind] = ReadOperation{Dispatched: stats.dispatched, Failures: stats.failures, Canceled: stats.canceled, EndToEnd: stats.endToEnd.snapshot(), Queue: stats.queue.snapshot()} + } + } + for kind, stats := range q.intended { + e.Intended[kind] = stats + } + return e +} + +func readGateFailures(t Trial) []string { + if t.Spec.QueryRate <= 0 { + return nil + } + profile := currentReadProfile() + e := t.Read + if e == nil || e.Profile != profile.Name { + return []string{"arrival accounting/profile missing; legacy service latency is not qualification evidence"} + } + var failures []string + if e.ScheduleSeconds <= 0 || math.IsNaN(e.ScheduleSeconds) || math.IsInf(e.ScheduleSeconds, 0) || perSecond(float64(e.CompletedDuringLoad), e.ScheduleSeconds) < t.Spec.QueryRate*.95 { + failures = append(failures, "during-load successful operation rate below target") + } + if e.Expected != t.QueryAttempts+e.SchedulerMissed || e.Expected <= 0 || e.SchedulerMissed != 0 || t.QueryShed != 0 || t.QueryFailures != 0 || t.QueryCanceled != 0 { + failures = append(failures, "scheduled arrivals missing, shed, failed or canceled") + } + var scheduled, missed, shed, dispatched, successful, failed, canceled int64 + for _, stats := range e.Intended { + scheduled += stats.Scheduled + missed += stats.SchedulerMissed + shed += stats.Shed + dispatched += stats.Dispatched + successful += stats.Successful + failed += stats.Failures + canceled += stats.Canceled + if stats.Scheduled != stats.SchedulerMissed+stats.Shed+stats.Dispatched || stats.Dispatched != stats.Successful+stats.Failures+stats.Canceled { + failures = append(failures, "intended operation accounting inconsistent") + } + } + if scheduled != e.Expected || missed != e.SchedulerMissed || shed != t.QueryShed || dispatched != successful+failed+canceled || successful != t.QueryLatency.Count || successful != e.EndToEnd.Count || successful != e.CompletedDuringLoad+e.CompletedDuringDrain || failed != t.QueryFailures || canceled != t.QueryCanceled { + failures = append(failures, "read outcome totals inconsistent") + } + var performed, performedSuccessful, performedFailed, performedCanceled int64 + for _, operation := range e.Operations { + performed += operation.Dispatched + performedSuccessful += operation.EndToEnd.Count + performedFailed += operation.Failures + performedCanceled += operation.Canceled + if operation.Dispatched != operation.EndToEnd.Count+operation.Failures+operation.Canceled { + failures = append(failures, "performed operation accounting inconsistent") + } + } + if performed != dispatched || performedSuccessful != successful || performedFailed != failed || performedCanceled != canceled { + failures = append(failures, "intended and performed outcome totals differ") + } + for _, kind := range []string{"dashboard-refresh", "drill-traces", "trace-detail"} { + target := profile.Targets[kind] + operation := e.Operations[kind] + if e.Intended[kind].Scheduled == 0 || operation.EndToEnd.Count < profile.MinSamples { + failures = append(failures, fmt.Sprintf("%s insufficient successful samples: %d/%d", kind, operation.EndToEnd.Count, profile.MinSamples)) + continue + } + if math.IsNaN(operation.EndToEnd.P95Ms) || math.IsNaN(operation.EndToEnd.P99Ms) || operation.EndToEnd.P95Ms > target.P95Ms || operation.EndToEnd.P99Ms > target.P99Ms { + failures = append(failures, fmt.Sprintf("%s HTTP arrival-to-completion p95 %.1f/%.1fms, p99 %.1f/%.1fms", kind, operation.EndToEnd.P95Ms, target.P95Ms, operation.EndToEnd.P99Ms, target.P99Ms)) + } + } + return failures +} diff --git a/internal/bench/read_accounting_report.go b/internal/bench/read_accounting_report.go new file mode 100644 index 0000000..3b5c4b6 --- /dev/null +++ b/internal/bench/read_accounting_report.go @@ -0,0 +1,51 @@ +package bench + +import ( + "fmt" + "io" + "sort" +) + +func renderReadEvidence(w io.Writer, r Report) { + if r.Settings.ReadProfile == nil { + fmt.Fprintln(w, "\nLegacy read evidence has no scheduled-arrival qualification profile. Its original service-time metrics and recorded gates are historical; they do not establish qualification against the current goals.") + return + } + p := r.Settings.ReadProfile + fmt.Fprintf(w, "\n### HTTP operation completion goals\n\nProfile `%s` declares proposed engineering goals, not measured product guarantees. HTTP operation completion is separate from browser last-panel paint. Qualification requires at least %d successful samples per performed kind. Short sweeps report insufficient evidence and use the documented topology fallback.\n\n", p.Name, p.MinSamples) + fmt.Fprintln(w, "| HTTP operation | p95 goal ms | p99 goal ms |\n|---|---:|---:|") + for _, kind := range sortedReadKeys(p.Targets) { + target := p.Targets[kind] + fmt.Fprintf(w, "| %s | %.0f | %.0f |\n", kind, target.P95Ms, target.P99Ms) + } + for _, t := range r.Trials { + if t.Spec.QueryRate <= 0 || t.Read == nil { + continue + } + e := t.Read + fmt.Fprintf(w, "\n### Arrival accounting: %s\n\nScheduled: %d over %.3fs; scheduler-missed: %d; completed during load: %d (%.2f/s); completed during drain: %d. Service-time fields above exclude the scheduler queue; successful arrival-to-completion fields below include it. Failures, cancellations and shed slots never enter successful-latency percentiles.\n\n", t.Spec.Name, e.Expected, e.ScheduleSeconds, e.SchedulerMissed, e.CompletedDuringLoad, readCompletedRate(t), e.CompletedDuringDrain) + fmt.Fprintln(w, "| Intended operation | Scheduled | Scheduler missed | Shed | Dispatched | Successful | Failures | Canceled | Performed as another kind |\n|---|---:|---:|---:|---:|---:|---:|---:|---:|") + for _, kind := range sortedReadKeys(e.Intended) { + s := e.Intended[kind] + fmt.Fprintf(w, "| %s | %d | %d | %d | %d | %d | %d | %d | %d |\n", kind, s.Scheduled, s.SchedulerMissed, s.Shed, s.Dispatched, s.Successful, s.Failures, s.Canceled, s.Fallbacks) + } + fmt.Fprintln(w, "\n| Performed HTTP operation | Dispatched | Successful | Failures | Canceled | Queue p95 ms | Arrival-to-completion p95 ms | Arrival-to-completion p99 ms | Evidence |\n|---|---:|---:|---:|---:|---:|---:|---:|---|") + for _, kind := range sortedReadKeys(e.Operations) { + s := e.Operations[kind] + evidence := "sufficient samples; see qualification gate" + if s.EndToEnd.Count < p.MinSamples { + evidence = "insufficient successful samples" + } + fmt.Fprintf(w, "| %s | %d | %d | %d | %d | %.2f | %.2f | %.2f | %s |\n", kind, s.Dispatched, s.EndToEnd.Count, s.Failures, s.Canceled, s.Queue.P95Ms, s.EndToEnd.P95Ms, s.EndToEnd.P99Ms, evidence) + } + } +} + +func sortedReadKeys[V any](items map[string]V) []string { + keys := make([]string, 0, len(items)) + for key := range items { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} diff --git a/internal/bench/read_accounting_test.go b/internal/bench/read_accounting_test.go new file mode 100644 index 0000000..933b046 --- /dev/null +++ b/internal/bench/read_accounting_test.go @@ -0,0 +1,194 @@ +package bench + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +func TestReadScheduleCountsMissedArrivalsWithoutCatchupBurst(t *testing.T) { + q := newQueryRun() + q.targets = []queryTarget{{name: "dashboard-refresh"}, {name: "drill-traces"}, {name: "trace-detail"}} + q.startedAt = time.Unix(0, 0) + q.rate = 20 + var jobs []readJob + q.advanceSchedule(q.startedAt.Add(250*time.Millisecond), func(job readJob) bool { + jobs = append(jobs, job) + return true + }) + q.advanceSchedule(q.startedAt.Add(500*time.Millisecond), func(job readJob) bool { + jobs = append(jobs, job) + return false + }) + evidence := q.readEvidence() + if len(jobs) != 2 || jobs[0].slot != 5 || jobs[1].slot != 10 || !jobs[0].scheduledAt.Equal(q.startedAt.Add(250*time.Millisecond)) { + t.Fatalf("unexpected jobs: %+v", jobs) + } + if evidence.Expected != 10 || evidence.SchedulerMissed != 8 || q.shed != 1 || q.attempts != 2 { + t.Fatalf("arrivals disappeared or burst: %+v attempts=%d shed=%d", evidence, q.attempts, q.shed) + } + var offered, missed, shed, dispatched int64 + for _, kind := range evidence.Intended { + offered += kind.Scheduled + missed += kind.SchedulerMissed + shed += kind.Shed + dispatched += kind.Dispatched + } + if offered != 10 || missed != 8 || shed != 1 || dispatched != 1 { + t.Fatalf("intended accounting: %+v", evidence.Intended) + } +} + +func TestReadOperationSeparatesQueueTimeAndDrainFromServiceLatency(t *testing.T) { + q := newQueryRun() + at := time.Unix(0, 0) + q.cutoff = at.Add(500 * time.Millisecond) + q.recordOperation(readJob{scheduledAt: at, intended: "trace-detail"}, "dashboard-refresh", at.Add(time.Second), at.Add(time.Second+2*time.Millisecond), true, false) + evidence := q.readEvidence() + if service := q.hist.snapshot(); service.Count != 1 || service.MaxMs != 2 { + t.Fatalf("legacy service latency changed: %+v", service) + } + if evidence.EndToEnd.MaxMs != 1002 || evidence.Queue.MaxMs != 1000 || evidence.CompletedDuringLoad != 0 || evidence.CompletedDuringDrain != 1 { + t.Fatalf("queue or drain hidden: %+v", evidence) + } + if evidence.Intended["trace-detail"].Fallbacks != 1 || evidence.Operations["dashboard-refresh"].EndToEnd.Count != 1 { + t.Fatalf("fallback hid intended operation: %+v", evidence) + } +} + +func TestReadScheduleHandlesFractionalRatesAndFinalMissedSlots(t *testing.T) { + q := newQueryRun() + q.startedAt, q.rate = time.Unix(0, 0), 2.5 + dispatched := 0 + dispatch := func(readJob) bool { dispatched++; return true } + q.advanceSchedule(q.startedAt.Add(400*time.Millisecond-time.Nanosecond), dispatch) + q.advanceSchedule(q.startedAt.Add(400*time.Millisecond), dispatch) + q.advanceSchedule(q.startedAt.Add(400*time.Millisecond), dispatch) + q.scheduleMu.Lock() + q.cutoff = q.startedAt.Add(1200 * time.Millisecond) + q.advanceScheduleLocked(q.cutoff, nil) + q.scheduleMu.Unlock() + q.advanceSchedule(q.startedAt.Add(10*time.Second), dispatch) + e := q.readEvidence() + if dispatched != 1 || e.Expected != 3 || e.SchedulerMissed != 2 || e.ScheduleSeconds != 1.2 { + t.Fatalf("fractional boundaries/finalization lost arrivals: %+v dispatched=%d", e, dispatched) + } +} + +func TestReadOutcomesAccountForFailedCanceledAndFallbackIntents(t *testing.T) { + q := newQueryRun() + q.targets = []queryTarget{{name: "dashboard-refresh"}, {name: "drill-traces"}, {name: "trace-detail"}} + q.startedAt, q.rate = time.Unix(0, 0), 20 + var jobs []readJob + for slot := 1; slot <= 3; slot++ { + q.advanceSchedule(q.startedAt.Add(time.Duration(slot)*50*time.Millisecond), func(job readJob) bool { + jobs = append(jobs, job) + return true + }) + } + for i, job := range jobs { + kind := job.intended + if i == 1 { + kind = "dashboard-refresh" + } + q.recordOperation(job, kind, job.scheduledAt, job.scheduledAt.Add(time.Millisecond), i == 0, i == 2) + } + e := q.readEvidence() + if e.Expected != 3 || q.hist.snapshot().Count != 1 || q.failures != 1 || q.canceled != 1 || e.Intended["drill-traces"].Fallbacks != 1 || e.Intended["drill-traces"].Failures != 1 || e.Intended["trace-detail"].Canceled != 1 { + t.Fatalf("failed/canceled/fallback work disappeared: %+v", e) + } + if e.Operations["dashboard-refresh"].Dispatched != 2 || e.Operations["trace-detail"].Canceled != 1 { + t.Fatalf("performed outcomes lost: %+v", e.Operations) + } +} + +func qualifiedReadTrial() Trial { + profile := currentReadProfile() + read := &ReadEvidence{Profile: profile.Name, ScheduleSeconds: 150, Expected: 3000, CompletedDuringLoad: 3000, EndToEnd: Latency{Count: 3000}, Operations: map[string]ReadOperation{}, Intended: map[string]ReadIntended{}} + for kind := range profile.Targets { + read.Operations[kind] = ReadOperation{Dispatched: 1000, EndToEnd: Latency{Count: 1000, P95Ms: 1, P99Ms: 2}} + read.Intended[kind] = ReadIntended{Scheduled: 1000, Dispatched: 1000, Successful: 1000} + } + return Trial{Spec: PhaseSpec{Signal: Mixed, Kind: "sustained", QueryRate: 20}, ElapsedSeconds: 150, QueryAttempts: 3000, QueryLatency: Latency{Count: 3000, P95Ms: 1, P99Ms: 2}, QueryKindCompleted: map[string]int64{"dashboard-refresh": 1000, "drill-traces": 1000, "trace-detail": 1000}, Read: read} +} + +func TestReadGateRejectsSlowKindInsufficientSamplesAndLegacyEvidence(t *testing.T) { + trial := qualifiedReadTrial() + if failures := readGateFailures(trial); len(failures) != 0 { + t.Fatalf("valid qualification failed: %v", failures) + } + operation := trial.Read.Operations["drill-traces"] + operation.EndToEnd.P99Ms = 1501 + trial.Read.Operations["drill-traces"] = operation + if queryTargetMet(trial) || !strings.Contains(strings.Join(readGateFailures(trial), " "), "drill-traces") { + t.Fatal("fast aggregate hid slow drill-down") + } + trial = qualifiedReadTrial() + operation = trial.Read.Operations["trace-detail"] + operation.EndToEnd.Count = 999 + trial.Read.Operations["trace-detail"] = operation + if !strings.Contains(strings.Join(readGateFailures(trial), " "), "insufficient") { + t.Fatal("p99 qualified with insufficient samples") + } + trial.Read = nil + if queryTargetMet(trial) { + t.Fatal("legacy service latency qualified as new arrival latency") + } +} + +func TestReadEvidenceRoundtripPreservesLegacyMetricMeaningAndTargetIdentity(t *testing.T) { + report := fixtureReport() + report.Settings.QueryRate = 20 + report.Settings.ReadProfile = currentReadProfile() + report.Trials = []Trial{qualifiedReadTrial()} + before := workloadHash(report.Settings, nil) + report.Trials[0].Read.EndToEnd.P95Ms = 42 + data, err := json.Marshal(report) + if err != nil { + t.Fatal(err) + } + var restored Report + if err := json.Unmarshal(data, &restored); err != nil { + t.Fatal(err) + } + if restored.Trials[0].QueryLatency.P95Ms != 1 || restored.Trials[0].Read.EndToEnd.P95Ms != 42 || before != workloadHash(restored.Settings, nil) { + t.Fatal("metric meaning or workload identity changed during roundtrip") + } + profile := *restored.Settings.ReadProfile + profile.Name = "different-profile" + restored.Settings.ReadProfile = &profile + if workloadHash(restored.Settings, nil) == before { + t.Fatal("declared targets absent from workload identity") + } + markdown := Markdown(restored) + if !strings.Contains(markdown, "HTTP operation") || !strings.Contains(markdown, "Scheduled") || !strings.Contains(markdown, "1500") { + t.Fatalf("new accounting/profile missing from report: %s", markdown) + } +} + +func TestReadProfileAndPerKindLatencyControlQualificationAndTopology(t *testing.T) { + fast := qualifiedReadTrial() + fast.Spec.Kind, fast.Spec.ProcessesPerSignal = "sweep", 4 + fast.Accepted, fast.AcceptedPerSecond = Counts{Spans: 1}, 150 + slow := qualifiedReadTrial() + slow.Spec.Kind, slow.Spec.ProcessesPerSignal = "sweep", 8 + slow.Accepted, slow.AcceptedPerSecond = Counts{Spans: 1}, 300 + operation := slow.Read.Operations["trace-detail"] + operation.EndToEnd.P95Ms = 301 + slow.Read.Operations["trace-detail"] = operation + if got := bestMixedTopology([]Trial{slow, fast}); got != 4 { + t.Fatalf("slow HTTP class selected by aggregate throughput: %d", got) + } + trial := qualifiedReadTrial() + trial.Accepted = Counts{Spans: 1} + for _, profile := range []*ReadProfile{nil, currentReadProfile()} { + if profile != nil { + profile.Targets["dashboard-refresh"] = ReadTarget{P95Ms: 9999, P99Ms: 9999} + } + summary := summarize(Report{Settings: Settings{QueryRate: 20, ReadProfile: profile}, Trials: []Trial{trial}}, 0, 0) + if summary.Passed { + t.Fatal("missing or silently weakened declared profile qualified") + } + } +} diff --git a/internal/bench/report.go b/internal/bench/report.go index ad8194c..3cb041b 100644 --- a/internal/bench/report.go +++ b/internal/bench/report.go @@ -183,15 +183,16 @@ func Markdown(r Report) string { fmt.Fprintf(&b, "- %s **%s:** %s\n", map[bool]string{true: "✅", false: "❌"}[gate.Passed], gate.Name, gate.Detail) } if r.Settings.QueryRate > 0 { - fmt.Fprintf(&b, "\n## Read-under-write latency\n\nThe mixed phases used a bounded open-loop scheduler targeting %.0f authenticated user operations/s across System overview dashboard refresh (batched panels and annotations), exemplar drill-down, and trace detail. Latency covers every request in an operation; operation kinds and their counts/latencies appear in the methodology.\n\n", r.Settings.QueryRate) + fmt.Fprintf(&b, "\n## Read-under-write latency\n\nThe mixed phases used a bounded open-loop scheduler targeting %.0f authenticated user operations/s across System overview dashboard refresh (batched panels and annotations), exemplar drill-down, and trace detail. The historical latency fields below measure successful HTTP service time across every request in an operation, excluding the scheduler queue. New arrival accounting is reported separately.\n\n", r.Settings.QueryRate) fmt.Fprintf(&b, "| Phase | Target/s | Completed/s | Attempted | p50 | p95 | p99 | Errors | Shed | Canceled at end |\n|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") for _, t := range r.Trials { if t.Spec.Signal != Mixed || t.Spec.QueryRate <= 0 { continue } - fmt.Fprintf(&b, "| %s | %.0f | %.1f | %s | %.1fms | %.1fms | %.1fms | %d | %d | %d |\n", t.Spec.Name, t.Spec.QueryRate, perSecond(float64(t.QueryLatency.Count), t.ElapsedSeconds), formatNumber(float64(t.QueryAttempts)), t.QueryLatency.P50Ms, t.QueryLatency.P95Ms, t.QueryLatency.P99Ms, t.QueryFailures, t.QueryShed, t.QueryCanceled) + fmt.Fprintf(&b, "| %s | %.0f | %.1f | %s | %.1fms | %.1fms | %.1fms | %d | %d | %d |\n", t.Spec.Name, t.Spec.QueryRate, readCompletedRate(t), formatNumber(float64(t.QueryAttempts)), t.QueryLatency.P50Ms, t.QueryLatency.P95Ms, t.QueryLatency.P99Ms, t.QueryFailures, t.QueryShed, t.QueryCanceled) } fmt.Fprintln(&b) + renderReadEvidence(&b, r) } renderDashboardPanels(&b, r.Trials) fmt.Fprintf(&b, "\n## Methodology and limits\n\n%s\n\n", r.Settings.Methodology) diff --git a/internal/bench/runner.go b/internal/bench/runner.go index 385e9aa..a4932b2 100644 --- a/internal/bench/runner.go +++ b/internal/bench/runner.go @@ -164,8 +164,10 @@ func (r *Runner) validate() error { return errors.New("generator timeout must be positive") case cfg.Repeat <= 0: return errors.New("repeat must be positive") - case cfg.QueryRate < 0: - return errors.New("query rate cannot be negative") + case cfg.QueryRate < 0 || math.IsNaN(cfg.QueryRate) || math.IsInf(cfg.QueryRate, 0): + return errors.New("query rate must be finite and nonnegative") + case cfg.QueryRate > 0 && float64(time.Second)/cfg.QueryRate > float64(math.MaxInt64): + return errors.New("query interval exceeds the supported duration") case cfg.QueryRate > 10_000: return errors.New("query rate cannot exceed 10000 operations/s") case cfg.QueryRate > 0 && strings.TrimSpace(cfg.QueryURL) == "": @@ -214,6 +216,9 @@ func (r *Runner) runPhase(ctx context.Context, spec PhaseSpec, previous []Trial) } failures, processCount, runErr := r.runGenerators(ctx, spec) + if spec.QueryRate > 0 { + queries.stopScheduling() + } elapsed := time.Since(started).Seconds() if runErr != nil && ctx.Err() != nil { cancel() @@ -226,8 +231,8 @@ func (r *Runner) runPhase(ctx context.Context, spec PhaseSpec, previous []Trial) // wait to a shorter interval and inflate the headline rate. after, err := r.scrape(ctx) // window is that scrape-to-scrape interval, and is the only honest - // denominator for the accepted delta. elapsed remains the offered-load - // duration and stays the denominator for completed queries. + // denominator for the accepted delta. elapsed retains the generator phase + // duration; read goodput uses its separately captured scheduling interval. window := time.Since(started).Seconds() cancel() queries.finish() @@ -276,10 +281,13 @@ func (r *Runner) runPhase(ctx context.Context, spec PhaseSpec, previous []Trial) QueryShed: queries.shed, QueryCanceled: queries.canceled, Samples: samples, } + if spec.QueryRate > 0 { + trial.Read = queries.readEvidence() + } trial.Classification, trial.ClassificationNotes = classify(trial, previous) r.readSummary = queries.operationSummary(spec.Name) if trial.Spec.QueryRate > 0 && !queryTargetMet(trial) { - trial.ClassificationNotes = append(trial.ClassificationNotes, "the requested query rate was not sustained") + trial.ClassificationNotes = append(trial.ClassificationNotes, "read qualification goals were not met: "+strings.Join(readGateFailures(trial), "; ")) } if runErr != nil { trial.ClassificationNotes = append(trial.ClassificationNotes, redact(runErr.Error(), r.Config.Token, r.Config.QueryCookie, r.Config.MetricsToken)) @@ -542,7 +550,7 @@ func bestMixedTopology(trials []Trial) int { } completion := 1.0 if t.Spec.QueryRate > 0 && t.ElapsedSeconds > 0 { - completion = perSecond(float64(t.QueryLatency.Count), t.ElapsedSeconds) / t.Spec.QueryRate + completion = readCompletedRate(t) / t.Spec.QueryRate } if completion > fallbackCompletion || (completion == fallbackCompletion && t.AcceptedPerSecond > fallbackRate) { fallbackCompletion, fallbackRate, fallbackProcesses = completion, t.AcceptedPerSecond, t.Spec.ProcessesPerSignal @@ -561,14 +569,7 @@ func queryTargetMet(t Trial) bool { if t.Spec.QueryRate <= 0 { return true } - if t.ElapsedSeconds <= 0 { - return false - } - if t.Spec.Kind == "sustained" && len(missingQueryKinds(t)) > 0 { - return false - } - completedRate := perSecond(float64(t.QueryLatency.Count), t.ElapsedSeconds) - return completedRate >= t.Spec.QueryRate*.95 && t.QueryFailures == 0 && t.QueryShed == 0 + return len(readGateFailures(t)) == 0 } func missingQueryKinds(t Trial) []string { @@ -637,8 +638,17 @@ func summarize(report Report, cooldownRSS, cooldownHeap float64) Summary { continue } queryOK = queryTargetMet(t) - completedRate := perSecond(float64(t.QueryLatency.Count), t.ElapsedSeconds) + if !readProfileMatches(report.Settings.ReadProfile) { + queryOK = false + } + completedRate := readCompletedRate(t) detail = fmt.Sprintf("%.1f/%.1f completed/s, %d errors, %d shed", completedRate, t.Spec.QueryRate, t.QueryFailures, t.QueryShed) + if !readProfileMatches(report.Settings.ReadProfile) { + detail += "; declared read profile missing or differs from current qualification goals" + } + if failures := readGateFailures(t); len(failures) > 0 { + detail += "; " + strings.Join(failures, "; ") + } if missing := missingQueryKinds(t); len(missing) > 0 { detail += "; zero successful operations: " + strings.Join(missing, ", ") } @@ -671,7 +681,7 @@ func settingsFrom(c Config) Settings { if c.Insecure { transport = "plaintext gRPC" } - return Settings{Signals: c.SignalSelection, EarlyStop: c.Preset == "validate", Preset: c.Preset, Endpoint: c.Endpoint, MetricsURL: c.MetricsURL, Transport: transport, Repeat: c.Repeat, + return Settings{ReadProfile: readProfileForRate(c.QueryRate), Signals: c.SignalSelection, EarlyStop: c.Preset == "validate", Preset: c.Preset, Endpoint: c.Endpoint, MetricsURL: c.MetricsURL, Transport: transport, Repeat: c.Repeat, SampleInterval: c.SampleInterval, SampleSeconds: c.SampleInterval.Seconds(), GeneratorTimeout: c.GeneratorTimeout, TimeoutSeconds: c.GeneratorTimeout.Seconds(), Cooldown: c.Cooldown, CooldownSeconds: c.Cooldown.Seconds(), DataDir: c.DataDir, QueryURL: c.QueryURL, QueryRate: c.QueryRate, BatchSize: c.BatchSize, TraceChildSpans: 1, diff --git a/internal/bench/runner_test.go b/internal/bench/runner_test.go index a804c89..16c81e7 100644 --- a/internal/bench/runner_test.go +++ b/internal/bench/runner_test.go @@ -195,68 +195,47 @@ func TestRunPhaseStopsQueriesBeforeFinalGeneratorCPURead(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() var dispatched atomic.Int64 - firstQuery := make(chan struct{}, 1) - scrapes := 0 + var finalCPURead atomic.Bool r := &Runner{ Config: Config{GeneratorSSH: "unused", MetricsURL: "http://bench/metrics", QueryURL: "http://bench", SampleInterval: time.Hour}, queryTargets: newQueryRun().targets, client: &http.Client{Transport: releaseTransport(func(req *http.Request) (*http.Response, error) { body := `{"status":"empty"}` if req.URL.Path == "/metrics" { - scrapes++ - if scrapes == 2 { - // Keep the phase open until a query is in flight. - select { - case <-firstQuery: - case <-ctx.Done(): - return nil, ctx.Err() - } - } - body = "pitot_rows_accepted_total{signal=\"logs\"} 1\n" + body = "pitot_ingest_rows_total{signal=\"logs\"} 1\n" } else { - dispatched.Add(1) - select { - case firstQuery <- struct{}{}: - default: - } - select { - case <-time.After(20 * time.Millisecond): - case <-req.Context().Done(): - return nil, req.Context().Err() + if finalCPURead.Load() { + t.Error("query dispatched after final generator CPU read began") } + dispatched.Add(1) } return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}, nil })}, } cpuReads := 0 - var beforeCPURead, afterCPURead int64 r.readGeneratorCPU = func(ctx context.Context) (cpuStat, error) { cpuReads++ if cpuReads == 2 { - beforeCPURead = dispatched.Load() - select { - case <-time.After(100 * time.Millisecond): - case <-ctx.Done(): - return cpuStat{}, ctx.Err() - } - afterCPURead = dispatched.Load() + finalCPURead.Store(true) } return cpuStat{total: uint64(cpuReads * 100), idle: uint64(cpuReads * 50), logicalCPUs: 1}, nil } - // No generator processes are needed to exercise the phase boundary. + // A zero-generator phase ends immediately. Its final scrape must not keep + // the scheduler alive waiting for the first request, as the old test did. + // In-flight draining is exercised separately by the query-run tests. trial, err := r.runPhase(ctx, PhaseSpec{Signal: Logs, QueryRate: 100}, nil) if err != nil { t.Fatal(err) } - if cpuReads != 2 || beforeCPURead == 0 { - t.Fatalf("CPU reads=%d queries before final read=%d; workload was not exercised", cpuReads, beforeCPURead) - } - if afterCPURead != beforeCPURead { - t.Fatalf("queries dispatched during final CPU read: %d", afterCPURead-beforeCPURead) + if cpuReads != 2 { + t.Fatalf("CPU reads=%d", cpuReads) } if trial.QueryLatency.Count != dispatched.Load() || trial.QueryFailures != 0 || trial.QueryCanceled != 0 { t.Fatalf("in-flight queries did not drain: dispatched=%d completed=%d failures=%d canceled=%d", dispatched.Load(), trial.QueryLatency.Count, trial.QueryFailures, trial.QueryCanceled) } + if trial.Read == nil || trial.Read.Expected != trial.QueryAttempts+trial.Read.SchedulerMissed { + t.Fatalf("phase arrival accounting missing: %+v", trial.Read) + } } func TestGeneratorCommandWrapsSSHAndQuotesEveryArgument(t *testing.T) { diff --git a/internal/bench/types.go b/internal/bench/types.go index 9255ec3..9dbf60a 100644 --- a/internal/bench/types.go +++ b/internal/bench/types.go @@ -109,6 +109,7 @@ type Build struct { } type Settings struct { + ReadProfile *ReadProfile `json:"read_profile,omitempty"` Signals string `json:"signals,omitempty"` EarlyStop bool `json:"early_stop,omitempty"` Preset string `json:"preset"` @@ -199,6 +200,7 @@ type GeneratorCPU struct { } type Trial struct { + Read *ReadEvidence `json:"read_evidence,omitempty"` Spec PhaseSpec `json:"settings"` StartedAt time.Time `json:"started_at"` ElapsedSeconds float64 `json:"elapsed_seconds"`