From ac22481466fed0b18752503ef84a524a6c27b2d3 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Thu, 12 Nov 2026 07:03:24 -0800 Subject: [PATCH] chore!: rename benchmark harness to Pitot --- .gitignore | 3 + .gitleaks.toml | 4 +- README.md | 127 +++++++++--------- cmd/{fanout-bench => pitot-bench}/main.go | 50 +++---- .../main_test.go | 30 ++--- docs/metrics.md | 64 ++++----- docs/publishing.md | 36 ++--- docs/telemetrygen.md | 6 +- go.mod | 2 +- internal/bench/doctor.go | 8 +- internal/bench/hetzner_test.go | 6 +- internal/bench/managed.go | 87 ++++++------ internal/bench/managed_test.go | 45 ++++--- internal/bench/panel_evidence.go | 8 +- internal/bench/panel_evidence_test.go | 30 ++--- internal/bench/prom.go | 6 +- internal/bench/prom_test.go | 16 +-- internal/bench/query.go | 2 +- internal/bench/query_workload.go | 2 +- internal/bench/query_workload_test.go | 26 ++-- internal/bench/remote.go | 118 ++++++++-------- internal/bench/remote_test.go | 104 +++++++------- internal/bench/report.go | 36 ++--- internal/bench/report_test.go | 23 ++-- internal/bench/runner.go | 62 ++++----- internal/bench/runner_test.go | 90 ++++++------- .../testdata/fanout-v2026.10.1-alpha-env.txt | 46 ------- .../testdata/pitot-v2026.11.5-alpha-env.txt | 58 ++++++++ internal/bench/types.go | 20 +-- justfile | 4 +- scripts/hetzner.sh | 60 ++++----- 31 files changed, 606 insertions(+), 573 deletions(-) rename cmd/{fanout-bench => pitot-bench}/main.go (77%) rename cmd/{fanout-bench => pitot-bench}/main_test.go (79%) delete mode 100644 internal/bench/testdata/fanout-v2026.10.1-alpha-env.txt create mode 100644 internal/bench/testdata/pitot-v2026.11.5-alpha-env.txt diff --git a/.gitignore b/.gitignore index 806000b..dfbb73f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ *.prof .env + +# Local source index. +.codegraph/ diff --git a/.gitleaks.toml b/.gitleaks.toml index 95ba52a..8c282cb 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -17,7 +17,7 @@ regex = '''(/Users|/home)/[A-Za-z0-9_.-]+/''' keywords = ["/users/", "/home/"] [[rules]] -id = "fanout-bearer-secret" -description = "Fanout opaque bearer secret" +id = "pitot-bearer-secret" +description = "Pitot opaque bearer secret" regex = '''fo[iacr]_[A-Za-z0-9_-]{43}\b''' keywords = ["foi_", "foa_", "for_", "foc_"] diff --git a/README.md b/README.md index 5444615..13639a4 100644 --- a/README.md +++ b/README.md @@ -1,89 +1,96 @@ -# Fanout Bench +# Pitot Bench -Reproducible OpenTelemetry performance evidence for [Fanout](https://github.com/labstack/fanout). +Reproducible OpenTelemetry performance evidence for [Pitot](https://github.com/labstack/pitot). -Published evidence: [Fanout v2026.11.0-alpha on two Hetzner ccx33s](published/2026-11-06-ccx33x2-v2026.11.0-alpha/README.md). +Published evidence: [Fanout v2026.11.0-alpha on two Hetzner ccx33s](published/2026-11-06-ccx33x2-v2026.11.0-alpha/README.md). These pre-rename measurements retain their original product identity. -Fanout Bench drives the official OpenTelemetry `telemetrygen` client, measures what Fanout actually accepts, samples the server throughout the run, and emits publication-ready JSON and Markdown. It is deliberately harder to fool than a one-line requests-per-second script. +The harness now uses `pitot-bench`, `--pitot-*` flags, `PITOT_BENCH_*` settings, +and Pitot release assets, environment settings, headers, cookies and metrics. +Use a renamed product release (`v2026.11.4-alpha` or later). To reproduce older +releases, check out the harness revision recorded in that report. New reports use `pitot-bench/v1` and `pitot_*` JSON field names. Previous +report formats are not accepted; use the harness revision recorded in the +archived evidence to read them. No published measurement is rewritten. + +Pitot Bench drives the official OpenTelemetry `telemetrygen` client, measures what Pitot actually accepts, samples the server throughout the run, and emits publication-ready JSON and Markdown. It is deliberately harder to fool than a one-line requests-per-second script. ## What the headline means The headline is: ```text -Fanout accepted rows ÷ the interval between the counter scrapes bracketing the phase +Pitot accepted rows ÷ the interval between the counter scrapes bracketing the phase ``` -Accepted rows come from `fanout_ingest_rows_total{signal=…}` at the Fanout server. They do **not** come from the offered rate, telemetrygen's intended count, or a client-side queue. The three units are named precisely: +Accepted rows come from `pitot_ingest_rows_total{signal=…}` at the Pitot server. They do **not** come from the offered rate, telemetrygen's intended count, or a client-side queue. The three units are named precisely: - traces are reported as accepted **spans**; - logs are reported as accepted **log records**; - metrics are reported as accepted **metric data points**. -With pinned telemetrygen v0.157, each generated trace exports one parent and one child span. The generator's limiter charges once for each span, so its configured rate is already aggregate spans/s; Fanout Bench does not apply a hidden multiplier. +With pinned telemetrygen v0.157, each generated trace exports one parent and one child span. The generator's limiter charges once for each span, so its configured rate is already aggregate spans/s; Pitot Bench does not apply a hidden multiplier. ## Quick start Prerequisites: [hcloud](https://github.com/hetznercloud/cli) with an active context (or `HCLOUD_TOKEN`), a registered Hetzner SSH key available to your SSH client, Go 1.27+, `ssh-keygen`, [just](https://github.com/casey/just), and SSH/SCP. ```sh -export FANOUT_BENCH_HCLOUD_SSH_KEY='benchmark-key' -just hetzner --fanout-version v2026.10.1-alpha --preset quick +export PITOT_BENCH_HCLOUD_SSH_KEY='benchmark-key' +just hetzner --pitot-version v2026.11.5-alpha --preset quick ``` The script runs `just build` before provisioning, creates a private network and two fresh Hetzner Cloud VMs with unique names, waits for SSH on both, runs the harness for each group, and downloads the evidence. Cleanup attempts to delete both servers and then the network on every exit, including failures and Ctrl-C. After three failed attempts it names each resource requiring manual deletion and exits nonzero. Hetzner bills every started hour for each VM, so grouping runs shares the billing time of both machines. -The defaults are a `ccx33` for Fanout (8 dedicated vCPUs, 32 GB RAM) and a separate `ccx33` for generators (8 dedicated vCPUs, 32 GB RAM), both in `fsn1` on `ubuntu-24.04`. OTLP travels over a private network (`10.77.0.0/16`, cloud subnet `10.77.0.0/24`) in that location’s network zone. Override the VM types with `FANOUT_BENCH_HCLOUD_TYPE` and `FANOUT_BENCH_HCLOUD_GENERATOR_TYPE`, and location/image with `FANOUT_BENCH_HCLOUD_LOCATION` and `FANOUT_BENCH_HCLOUD_IMAGE`. The hourly VM cost is **twice the ccx33 hourly rate**, plus applicable public IPv4 charges and tax; budget for both machines for every started hour. Check the selected location’s current rates before provisioning. `FANOUT_BENCH_HCLOUD_SSH_KEY` also accepts comma-separated key names. Each invocation shares one temporary known-hosts file across its groups, leaving your SSH known-hosts file untouched. Pass `--ssh-identity ~/.ssh/benchmark` if the registered key is not available through your SSH agent. +The defaults are a `ccx33` for Pitot (8 dedicated vCPUs, 32 GB RAM) and a separate `ccx33` for generators (8 dedicated vCPUs, 32 GB RAM), both in `fsn1` on `ubuntu-24.04`. OTLP travels over a private network (`10.77.0.0/16`, cloud subnet `10.77.0.0/24`) in that location’s network zone. Override the VM types with `PITOT_BENCH_HCLOUD_TYPE` and `PITOT_BENCH_HCLOUD_GENERATOR_TYPE`, and location/image with `PITOT_BENCH_HCLOUD_LOCATION` and `PITOT_BENCH_HCLOUD_IMAGE`. The hourly VM cost is **twice the ccx33 hourly rate**, plus applicable public IPv4 charges and tax; budget for both machines for every started hour. Check the selected location’s current rates before provisioning. `PITOT_BENCH_HCLOUD_SSH_KEY` also accepts comma-separated key names. Each invocation shares one temporary known-hosts file across its groups, leaving your SSH known-hosts file untouched. Pass `--ssh-identity ~/.ssh/benchmark` if the registered key is not available through your SSH agent. -Separate complete sets of `fanout-bench run` flags with a standalone `--`. Groups must be nonempty and use matching SSH port and identity settings (defaults: port 22 and the SSH client's default identity). All groups run in order even if one fails, with a one-line summary per group at the end and a nonzero exit if any failed. They share both VMs, the private network, and both host descriptions; each harness run creates fresh stages and a fresh Fanout data directory. Both VMs and the network are deleted after the last group. +Separate complete sets of `pitot-bench run` flags with a standalone `--`. Groups must be nonempty and use matching SSH port and identity settings (defaults: port 22 and the SSH client's default identity). All groups run in order even if one fails, with a one-line summary per group at the end and a nonzero exit if any failed. They share both VMs, the private network, and both host descriptions; each harness run creates fresh stages and a fresh Pitot data directory. Both VMs and the network are deleted after the last group. -A plain `--host` run on any clean SSH-accessible Linux host runs Fanout and telemetrygen together on that host: +A plain `--host` run on any clean SSH-accessible Linux host runs Pitot and telemetrygen together on that host: ```sh just build -./bin/fanout-bench run \ +./bin/pitot-bench run \ --host 'root@' \ --host-description 'Linux benchmark host' \ - --fanout-version v2026.10.1-alpha \ + --pitot-version v2026.11.5-alpha \ --preset quick ``` Replace `` with the address of your host. The remote harness: 1. detects the host's Linux architecture over SSH; -2. downloads the matching Fanout release and verifies it against the release `SHA256SUMS`; +2. downloads the matching Pitot release and verifies it against the release `SHA256SUMS`; 3. cross-builds pinned telemetrygen and the benchmark agent for the host; -4. uploads all three binaries to a private, isolated `/tmp/fanout-bench-*` directory; -5. starts a fresh Fanout data directory on one loopback-only random port; +4. uploads all three binaries to a private, isolated `/tmp/pitot-bench-*` directory; +5. starts a fresh Pitot data directory on one loopback-only random port; 6. creates one-time admin, ingest, and query credentials without printing them; -7. runs every phase and then `fanout repair verify`; +7. runs every phase and then `pitot repair verify`; 8. downloads `report.json`, `report.md`, and `samples.jsonl` locally; -9. stops Fanout and removes the remote run directory. +9. stops Pitot and removes the remote run directory. -For an existing host, pass `--ssh-identity ~/.ssh/benchmark`, `--ssh-port 2222`, or `--ssh-known-hosts ` when needed. `--host-description ` records the provider or machine configuration beside the detected CPU model. Reports redact the SSH address and omit the run-scoped known-hosts path from reproduction commands. `--keep-remote` preserves the Fanout directory on an existing host for investigation; split runs still remove the generator stage and ephemeral SSH key. The Hetzner script always deletes both VMs and the network. +For an existing host, pass `--ssh-identity ~/.ssh/benchmark`, `--ssh-port 2222`, or `--ssh-known-hosts ` when needed. `--host-description ` records the provider or machine configuration beside the detected CPU model. Reports redact the SSH address and omit the run-scoped known-hosts path from reproduction commands. `--keep-remote` preserves the Pitot directory on an existing host for investigation; split runs still remove the generator stage and ephemeral SSH key. The Hetzner script always deletes both VMs and the network. To use two existing clean Linux hosts, add all three split flags together: ```sh -./bin/fanout-bench run \ - --host 'root@' \ +./bin/pitot-bench run \ + --host 'root@' \ --generator-host 'root@' \ --generator-address 10.77.0.3 \ - --fanout-listen 10.77.0.2 \ - --host-description 'Fanout host' \ + --pitot-listen 10.77.0.2 \ + --host-description 'Pitot host' \ --generator-host-description 'Generator host' \ - --fanout-version v2026.10.1-alpha --preset quick + --pitot-version v2026.11.5-alpha --preset quick ``` -The public SSH targets are used by your local machine for provisioning. The two IP flags must be reachable between the hosts, with SSH from Fanout to the generator and OTLP from the generator to Fanout. Provisioning detects both Linux architectures, builds the harness for Fanout and telemetrygen for the generator, and uploads each binary only where it runs. A local temporary ed25519 key grants the Fanout host access to the generator for that run; its public key has a unique authorized-keys marker, and the private key is uploaded with mode `0600` into the Fanout stage. Both stages use mode `0700`. +The public SSH targets are used by your local machine for provisioning. The two IP flags must be reachable between the hosts, with SSH from Pitot to the generator and OTLP from the generator to Pitot. Provisioning detects both Linux architectures, builds the harness for Pitot and telemetrygen for the generator, and uploads each binary only where it runs. A local temporary ed25519 key grants the Pitot host access to the generator for that run; its public key has a unique authorized-keys marker, and the private key is uploaded with mode `0600` into the Pitot stage. Both stages use mode `0700`. -`managed` remains the sole orchestrator on the Fanout host: it binds Fanout to the supplied private IP on a free port, performs setup, samples metrics, issues read load, verifies storage, and writes the report. Every telemetrygen process runs through SSH on the generator host, bounded by a remote timeout. Cleanup removes the marked authorized-key line, the private key, and both stages. Reports record both host descriptions and `generator_on_target_host: false`; reproduction commands redact both public targets and omit the private IP flags. +`managed` remains the sole orchestrator on the Pitot host: it binds Pitot to the supplied private IP on a free port, performs setup, samples metrics, issues read load, verifies storage, and writes the report. Every telemetrygen process runs through SSH on the generator host, bounded by a remote timeout. Cleanup removes the marked authorized-key line, the private key, and both stages. Reports record both host descriptions and `generator_on_target_host: false`; reproduction commands redact both public targets and omit the private IP flags. -To measure an unreleased build, pass `--fanout-binary ` with a local Linux Fanout binary for the remote architecture and a non-release label such as `--fanout-version explore-fdfbf7f5`. The binary is identified by its SHA-256 as `unreleased-local-binary`, and the report warns that the run is diagnostic and must not be published. +To measure an unreleased build, pass `--pitot-binary ` with a local Linux Pitot binary for the remote architecture and a non-release label such as `--pitot-version explore-fdfbf7f5`. The binary is identified by its SHA-256 as `unreleased-local-binary`, and the report warns that the run is diagnostic and must not be published. -Nothing is installed system-wide. Existing Fanout services and data directories are not touched. Neither remote host needs a Go toolchain or Docker daemon, and Fanout needs no preconfigured account. The Fanout host needs an SSH client; the generator host needs an SSH server and GNU `timeout` (included in the default Ubuntu image). +Nothing is installed system-wide. Existing Pitot services and data directories are not touched. Neither remote host needs a Go toolchain or Docker daemon, and Pitot needs no preconfigured account. The Pitot host needs an SSH client; the generator host needs an SSH server and GNU `timeout` (included in the default Ubuntu image). -> `telemetrygen` currently accepts OTLP headers through a command-line flag and prints those headers in its startup configuration. Fanout Bench suppresses and redacts child output, but the short-lived ingest token can still be visible to local process-listing tools while a generator is running. Use a dedicated token and revoke it after the run. +> `telemetrygen` currently accepts OTLP headers through a command-line flag and prints those headers in its startup configuration. Pitot Bench suppresses and redacts child output, but the short-lived ingest token can still be visible to local process-listing tools while a generator is running. Use a dedicated token and revoke it after the run. ## Suites @@ -97,20 +104,20 @@ Nothing is installed system-wide. Existing Fanout services and data directories The default topology is 8 workers per process at 2,500 items/s per worker with 1,000-item OTLP batches. A mixed step launches the same topology independently for traces, logs, and metrics. Without read load, the sustained proof reruns the mixed topology with the highest **accepted** throughput. With `--query-rate`, it chooses the highest-throughput topology that also completed at least 95% of the requested reads with no errors or scheduler shedding. If no sweep point qualifies, it repeats the best read-completion point and fails the sustained-read quality gate rather than publishing a write-only peak as mixed capacity. Both reports record the selected sustained process count per signal. The wider publish sweep adds independent exporters to get past telemetrygen's per-process limits; see [the source analysis](docs/telemetrygen.md). -For publication runs, dedicate the default ccx33 to Fanout and use the separate ccx33 for generators, with three repetitions. Run ingest-only and read-under-write as two groups sharing the same fresh pair of VMs; see [the publication commands](docs/publishing.md#publication-runs). For example: +For publication runs, dedicate the default ccx33 to Pitot and use the separate ccx33 for generators, with three repetitions. Run ingest-only and read-under-write as two groups sharing the same fresh pair of VMs; see [the publication commands](docs/publishing.md#publication-runs). For example: ```sh scripts/hetzner.sh \ - --fanout-version v2026.10.1-alpha \ + --pitot-version v2026.11.5-alpha \ --preset publish --repeat 3 \ --output results/publish-ingest-2026-10-14 \ -- \ - --fanout-version v2026.10.1-alpha \ + --pitot-version v2026.11.5-alpha \ --preset publish --repeat 3 --query-rate 20 \ --output results/publish-read-write-2026-10-14 ``` -Publication runs measure Fanout’s ingest ceiling with all eight ccx33 vCPUs available to Fanout; telemetrygen runs on the separate ccx33 over the private network. A plain `--host` run without the three split flags remains a same-host run, and its report warns about generator contention. +Publication runs measure Pitot’s ingest ceiling with all eight ccx33 vCPUs available to Pitot; telemetrygen runs on the separate ccx33 over the private network. A plain `--host` run without the three split flags remains a same-host run, and its report warns about generator contention. ## Faster Hetzner validation @@ -118,14 +125,14 @@ Run ingest and both read groups concurrently on three separate VM pairs: ```sh scripts/hetzner.sh --parallel-groups \ - --fanout-version v2026.10.1-alpha --preset validate --repeat 3 \ + --pitot-version v2026.11.5-alpha --preset validate --repeat 3 \ --output results/validate-ingest \ -- \ - --fanout-version v2026.10.1-alpha --preset validate --repeat 1 \ + --pitot-version v2026.11.5-alpha --preset validate --repeat 1 \ --signals mixed --query-rate 20 --rate 210 \ --output results/validate-reads-70 \ -- \ - --fanout-version v2026.10.1-alpha --preset validate --repeat 1 \ + --pitot-version v2026.11.5-alpha --preset validate --repeat 1 \ --signals mixed --query-rate 20 \ --output results/validate-reads-saturated ``` @@ -136,16 +143,16 @@ Set the Hetzner SSH-key environment variable as in Quick start. Expect **about t ## Read and write together -The write-only suite answers “how quickly can Fanout ingest?” It does not answer “what happens to dashboard latency while ingesting?” Add automatically authenticated read load to mixed phases: +The write-only suite answers “how quickly can Pitot ingest?” It does not answer “what happens to dashboard latency while ingesting?” Add automatically authenticated read load to mixed phases: ```sh scripts/hetzner.sh \ - --fanout-version v2026.10.1-alpha \ + --pitot-version v2026.11.5-alpha \ --preset standard \ --query-rate 50 ``` -The harness targets only the current Fanout API: `POST /api/queries/run`, `POST /api/queries/exemplars`, and `POST /api/annotations/query`. It uses a bounded open-loop scheduler targeting user operations per second at 15-minute, 1-hour, 6-hour, 12-hour, and 24-hour windows. Operations rotate through dashboard refresh, service-map trace drill-down, and trace detail. A refresh sends the full System overview panel batch exactly as the UI's initial all-visible load does, then requests annotations over the returned time-panel bounds. Layout widths model a fixed 1664-pixel viewport and are rounded as in the UI. Drill-down uses `POST /api/queries/exemplars` with the selected service and the panel response's absolute window. Trace detail uses `GET /api/traces/:id` with a discovered ID, namespace, and that same window, retained separately for each range; an empty detail is a failure. Until a trace is discovered, detail slots perform drill-down; until a selectable panel is cached, those slots perform dashboard refresh instead. Each slot performs one user operation. The current System overview has no log drill or standalone log/trace search operation. Each scheduled read measures every request in its operation, and any request or panel error fails the operation. The report records per-operation counts and latency in its methodology alongside the dashboard definition hash; aggregate fields and quality gates retain their schema. +The harness targets only the current Pitot API: `POST /api/queries/run`, `POST /api/queries/exemplars`, and `POST /api/annotations/query`. It uses a bounded open-loop scheduler targeting user operations per second at 15-minute, 1-hour, 6-hour, 12-hour, and 24-hour windows. Operations rotate through dashboard refresh, service-map trace drill-down, and trace detail. A refresh sends the full System overview panel batch exactly as the UI's initial all-visible load does, then requests annotations over the returned time-panel bounds. Layout widths model a fixed 1664-pixel viewport and are rounded as in the UI. Drill-down uses `POST /api/queries/exemplars` with the selected service and the panel response's absolute window. Trace detail uses `GET /api/traces/:id` with a discovered ID, namespace, and that same window, retained separately for each range; an empty detail is a failure. Until a trace is discovered, detail slots perform drill-down; until a selectable panel is cached, those slots perform dashboard refresh instead. Each slot performs one user operation. The current System overview has no log drill or standalone log/trace search operation. Each scheduled read measures every request in its operation, and any request or panel error fails the operation. The report records per-operation counts and latency in its methodology alongside the dashboard definition hash; aggregate fields and quality gates retain their schema. It reports attempted and completed rates, phase-end cancellations, explicit scheduler shedding, failures, mean, p50, p95, p99, and maximum latency. The concurrency bound prevents a stalled server from creating unbounded requests. @@ -165,9 +172,9 @@ Every run produces one self-contained directory: results/20261014T190000Z/ ├── report.json authoritative schema-versioned result ├── report.md readable, publication-ready report -├── samples.jsonl per-second Fanout ingest, resource, and background-work samples -├── fanout-repair.txt post-run authoritative-storage verification -└── fanout.log setup-token-scrubbed server log +├── samples.jsonl per-second Pitot ingest, resource, and background-work samples +├── pitot-repair.txt post-run authoritative-storage verification +└── pitot.log setup-token-scrubbed server log ``` `samples.jsonl` includes cumulative `ingest_rows_total` by signal (`spans`, `logs`, `metrics`), `rollup_seconds` by rollup (`service`, `edge`, `version`, `read_cache`), `write_gate_hold_seconds` and `write_gate_wait_seconds` by operation, `compaction_seconds_total`, `compactions_total`, `publish_wait_seconds_total`, `ingest_backpressure_seconds_total`, `ingest_throttled_total` by reason, `ingest_admission_rows_per_second`, `compaction_retired_batches_total`, and `compaction_pass_seconds_total`, plus the current `compaction_backlog_batches` gauge. Derive per-second accepted-row rates from consecutive samples' `ingest_rows_total` deltas divided by their `at_seconds` difference. Missing metrics leave these fields absent or zero. See [sample definitions](docs/metrics.md#background-work-samples) for their sources and interpretation. @@ -176,12 +183,12 @@ The terminal output is optimized for decisions: ```text ╭────────────────────────────────────────────────────────────────────╮ -│ FANOUT BENCH • accepted-row performance, with evidence │ +│ PITOT BENCH • accepted-row performance, with evidence │ ╰────────────────────────────────────────────────────────────────────╯ Target 10.77.0.2:39281 (plaintext gRPC) Generator telemetrygen v0.157.0 same-host=false Suite standard repeat=1 18 phases ~5.2 min load time - Evidence Fanout counters @ /metrics samples every 1.0s + Evidence Pitot counters @ /metrics samples every 1.0s ◇ mixed-p4-r1 offered 240,000 items/s for 15s … accepted …/s ratio …% CPU … cores peak RSS … @@ -197,7 +204,7 @@ The terminal output is optimized for decisions: ✓ generator-processes all exited successfully ✓ server-dropped-rows zero ✓ server-restarts none - ✓ authoritative-counters Fanout accepted rows in every phase + ✓ authoritative-counters Pitot accepted rows in every phase ✓ sustained-query-load …/… completed/s, 0 errors, 0 shed ✓ storage-repair-verify authoritative Parquet storage verified ╰─ PASS — report is internally valid ────────────────────────────────╯ @@ -208,7 +215,7 @@ The terminal output is optimized for decisions: Regenerate Markdown without rerunning load: ```sh -./bin/fanout-bench render \ +./bin/pitot-bench render \ --input results/publish-2026-10-14/report.json \ --output results/publish-2026-10-14/report.md ``` @@ -219,8 +226,8 @@ For every phase the report includes: - offered and accepted items/s, exact accepted counts, and acceptance ratio; - accepted spans, log records, and metric points separately; -- generator process count/failures and Fanout dropped-row/restart checks; -- average Fanout CPU cores, peak/end RSS, peak/end Go heap, allocations, GC pause; +- generator process count/failures and Pitot dropped-row/restart checks; +- average Pitot CPU cores, peak/end RSS, peak/end Go heap, allocations, GC pause; - peak ingest queue depth, Parquet file count/bytes, and optional local data-directory growth; - a raw time series in `samples.jsonl` for independent charting, including accepted rows by signal, rollup durations, write-gate hold/wait durations, compaction duration/count/backlog, publish wait duration, and ingest backpressure duration; - attempted/completed read rate, latency, failures, scheduler shedding, and phase-end cancellations when query load is enabled. @@ -232,7 +239,7 @@ See [docs/metrics.md](docs/metrics.md) for definitions and [docs/publishing.md]( Presets make comparisons easy. Overrides are intended for diagnostics and ceiling exploration: ```sh -./bin/fanout-bench run \ +./bin/pitot-bench run \ --preset standard \ --processes 16 \ --workers 8 \ @@ -246,9 +253,9 @@ Presets make comparisons easy. Overrides are intended for diagnostics and ceilin 3 signals × processes-per-signal × workers-per-process × rate-per-worker ``` -Retryable throttling is valid when telemetrygen retries and exits successfully. Accepted throughput still comes only from Fanout counters. Any nonzero generator exit, including exhaustion of its export retry timeout, invalidates the phase and fails the generator-processes gate; throttling never excuses exporter death. +Retryable throttling is valid when telemetrygen retries and exits successfully. Accepted throughput still comes only from Pitot counters. Any nonzero generator exit, including exhaustion of its export retry timeout, invalidates the phase and fails the generator-processes gate; throttling never excuses exporter death. -Avoid unbounded telemetrygen runs. Its SDK queues may drop locally while the console continues to count generated data. Fanout Bench always uses bounded rates and treats Fanout's accepted counter as authoritative. +Avoid unbounded telemetrygen runs. Its SDK queues may drop locally while the console continues to count generated data. Pitot Bench always uses bounded rates and treats Pitot's accepted counter as authoritative. ## Development @@ -258,17 +265,17 @@ just test The project uses only the Go standard library at runtime. The generator is a separately pinned official OpenTelemetry binary, making its version visible and replaceable without pulling the Collector dependency graph into the harness. -## Existing local Fanout process +## Existing local Pitot process -Fresh Hetzner Cloud VMs are the publication path. For local development against an already-running Fanout, install telemetrygen, provide short-lived credentials through environment variables, and omit `--host`: +Fresh Hetzner Cloud VMs are the publication path. For local development against an already-running Pitot, install telemetrygen, provide short-lived credentials through environment variables, and omit `--host`: ```sh just telemetrygen -export FANOUT_BENCH_INGEST_TOKEN='short-lived-benchmark-token' -export FANOUT_BENCH_METRICS_TOKEN='matching-Fanout-metrics-token' +export PITOT_BENCH_INGEST_TOKEN='short-lived-benchmark-token' +export PITOT_BENCH_METRICS_TOKEN='matching-Pitot-metrics-token' -./bin/fanout-bench doctor --telemetrygen ./bin/telemetrygen -./bin/fanout-bench run \ +./bin/pitot-bench doctor --telemetrygen ./bin/telemetrygen +./bin/pitot-bench run \ --preset quick \ --telemetrygen ./bin/telemetrygen \ --endpoint localhost:7520 \ @@ -277,6 +284,6 @@ export FANOUT_BENCH_METRICS_TOKEN='matching-Fanout-metrics-token' --same-host ``` -Managed runs set `FANOUT_ADDR` to one random port shared by OTLP/gRPC, OTLP/HTTP, the API, and `/metrics`. Split runs bind the private listen IP; same-host runs bind loopback. Metrics stay private: a per-run `FANOUT_METRICS_TOKEN` is sent as `Authorization: Bearer …` and never written to artifacts. MCP, alerts, and chat stay disabled. +Managed runs set `PITOT_ADDR` to one random port shared by OTLP/gRPC, OTLP/HTTP, the API, and `/metrics`. Split runs bind the private listen IP; same-host runs bind loopback. Metrics stay private: a per-run `PITOT_METRICS_TOKEN` is sent as `Authorization: Bearer …` and never written to artifacts. MCP, alerts, and chat stay disabled. -Private local metrics endpoints use `FANOUT_BENCH_METRICS_TOKEN`. Local authenticated read load uses `FANOUT_BENCH_QUERY_COOKIE`. Secrets have no value flags and are never persisted. telemetrygen does expose its OTLP header in its process arguments while running, so use only a dedicated short-lived token and revoke it afterward. Managed remote runs create and destroy this token automatically. +Private local metrics endpoints use `PITOT_BENCH_METRICS_TOKEN`. Local authenticated read load uses `PITOT_BENCH_QUERY_COOKIE`. Secrets have no value flags and are never persisted. telemetrygen does expose its OTLP header in its process arguments while running, so use only a dedicated short-lived token and revoke it afterward. Managed remote runs create and destroy this token automatically. diff --git a/cmd/fanout-bench/main.go b/cmd/pitot-bench/main.go similarity index 77% rename from cmd/fanout-bench/main.go rename to cmd/pitot-bench/main.go index f144c28..9c33d10 100644 --- a/cmd/fanout-bench/main.go +++ b/cmd/pitot-bench/main.go @@ -11,7 +11,7 @@ import ( "syscall" "time" - "github.com/labstack/fanout-bench/internal/bench" + "github.com/labstack/pitot-bench/internal/bench" ) var version = "dev" @@ -50,12 +50,12 @@ func main() { } func usage() { - fmt.Fprintln(os.Stderr, `fanout-bench — reproducible OpenTelemetry performance evidence + fmt.Fprintln(os.Stderr, `pitot-bench — reproducible OpenTelemetry performance evidence Usage: - fanout-bench doctor [flags] verify telemetrygen, authentication, and metrics - fanout-bench run [flags] provision a host and execute a benchmark suite - fanout-bench render [flags] regenerate Markdown from report.json + pitot-bench doctor [flags] verify telemetrygen, authentication, and metrics + pitot-bench run [flags] provision a host and execute a benchmark suite + pitot-bench render [flags] regenerate Markdown from report.json Secrets are accepted only through environment variables and are never persisted.`) } @@ -70,36 +70,36 @@ func commonFlags(name string, args []string) (bench.Config, error) { fs.StringVar(&cfg.HostDescription, "host-description", "", "host description recorded alongside hardware evidence") if name == "run" { fs.StringVar(&cfg.GeneratorHost, "generator-host", "", "SSH target to provision as a separate generator host") - fs.StringVar(&cfg.GeneratorAddress, "generator-address", "", "generator host IP reachable from the Fanout host") - fs.StringVar(&cfg.FanoutListen, "fanout-listen", "", "Fanout host IP reachable from the generator host") + fs.StringVar(&cfg.GeneratorAddress, "generator-address", "", "generator host IP reachable from the Pitot host") + fs.StringVar(&cfg.PitotListen, "pitot-listen", "", "Pitot host IP reachable from the generator host") fs.StringVar(&cfg.GeneratorHostDescription, "generator-host-description", "", "generator host description recorded in the report") - fs.StringVar(&cfg.FanoutBinary, "fanout-binary", "", "local Linux Fanout binary for a remote run instead of a release; --fanout-version is then a non-release label and the report is marked not publishable") + fs.StringVar(&cfg.PitotBinary, "pitot-binary", "", "local Linux Pitot binary for a remote run instead of a release; --pitot-version is then a non-release label and the report is marked not publishable") } fs.IntVar(&cfg.SSHPort, "ssh-port", 22, "SSH port") fs.BoolVar(&cfg.KeepRemote, "keep-remote", false, "preserve the isolated remote run directory after downloading artifacts") fs.StringVar(&cfg.SignalSelection, "signals", "all", "sweeps: all or mixed (skips single-signal sweeps)") fs.StringVar(&cfg.Preset, "preset", "standard", "suite: quick, standard, local, validate, or publish") - fs.StringVar(&cfg.Endpoint, "endpoint", "localhost:7520", "Fanout OTLP/gRPC endpoint") - fs.StringVar(&cfg.MetricsURL, "metrics-url", "http://localhost:7520/metrics", "Fanout Prometheus metrics URL (required evidence source)") + fs.StringVar(&cfg.Endpoint, "endpoint", "localhost:7520", "Pitot OTLP/gRPC endpoint") + fs.StringVar(&cfg.MetricsURL, "metrics-url", "http://localhost:7520/metrics", "Pitot Prometheus metrics URL (required evidence source)") fs.BoolVar(&cfg.Insecure, "insecure", false, "use plaintext OTLP/gRPC") fs.StringVar(&cfg.Telemetrygen, "telemetrygen", "./bin/telemetrygen", "path to pinned telemetrygen binary") fs.StringVar(&cfg.TelemetrygenVersion, "telemetrygen-version", "v0.157.0", "version recorded for the generator binary") fs.StringVar(&cfg.OutputDir, "output", "", "artifact directory (run command only)") - fs.StringVar(&tokenEnv, "token-env", "FANOUT_BENCH_INGEST_TOKEN", "environment variable containing the ingest bearer token") - fs.StringVar(&metricsTokenEnv, "metrics-token-env", "FANOUT_BENCH_METRICS_TOKEN", "environment variable containing an optional metrics bearer token") - fs.StringVar(&cfg.FanoutVersion, "fanout-version", "unknown", "Fanout release under test") - fs.StringVar(&cfg.FanoutCommit, "fanout-commit", "", "Fanout git commit under test") - fs.StringVar(&cfg.FanoutArtifact, "fanout-artifact", "", "Fanout binary or release artifact identity under test") - fs.StringVar(&cfg.DataDir, "data-dir", "", "local Fanout data directory for disk-growth evidence") + fs.StringVar(&tokenEnv, "token-env", "PITOT_BENCH_INGEST_TOKEN", "environment variable containing the ingest bearer token") + fs.StringVar(&metricsTokenEnv, "metrics-token-env", "PITOT_BENCH_METRICS_TOKEN", "environment variable containing an optional metrics bearer token") + fs.StringVar(&cfg.PitotVersion, "pitot-version", "unknown", "Pitot release under test") + fs.StringVar(&cfg.PitotCommit, "pitot-commit", "", "Pitot git commit under test") + fs.StringVar(&cfg.PitotArtifact, "pitot-artifact", "", "Pitot binary or release artifact identity under test") + fs.StringVar(&cfg.DataDir, "data-dir", "", "local Pitot data directory for disk-growth evidence") fs.DurationVar(&cfg.SampleInterval, "sample-interval", time.Second, "server resource sampling interval") fs.DurationVar(&cfg.GeneratorTimeout, "export-timeout", 30*time.Second, "telemetrygen export timeout") fs.DurationVar(&cfg.Cooldown, "cooldown", 15*time.Second, "post-load retained-memory observation window") fs.IntVar(&cfg.Repeat, "repeat", 1, "repeat every sweep topology") - fs.BoolVar(&cfg.SameHost, "same-host", false, "declare that Fanout and generators share a host") + fs.BoolVar(&cfg.SameHost, "same-host", false, "declare that Pitot and generators share a host") fs.IntVar(&cfg.BatchSize, "batch-size", 1000, "telemetrygen OTLP batch size") - fs.StringVar(&cfg.QueryURL, "query-url", "", "Fanout HTTP base URL for authenticated read-under-write load") + fs.StringVar(&cfg.QueryURL, "query-url", "", "Pitot HTTP base URL for authenticated read-under-write load") fs.Float64Var(&cfg.QueryRate, "query-rate", 0, "aggregate UI operations/s during mixed phases") - fs.StringVar(&queryCookieEnv, "query-cookie-env", "FANOUT_BENCH_QUERY_COOKIE", "environment variable containing the authenticated session Cookie header") + fs.StringVar(&queryCookieEnv, "query-cookie-env", "PITOT_BENCH_QUERY_COOKIE", "environment variable containing the authenticated session Cookie header") fs.IntVar(&cfg.ProcessOverride, "processes", 0, "override sweep with one process count per signal") fs.IntVar(&cfg.WorkerOverride, "workers", 0, "override workers per telemetrygen process") fs.Float64Var(&cfg.RateOverride, "rate", 0, "override items/s per telemetrygen worker") @@ -113,8 +113,8 @@ func commonFlags(name string, args []string) (bench.Config, error) { if err := bench.ValidateGeneratorHosts(cfg); err != nil { return cfg, err } - if cfg.FanoutBinary != "" && cfg.Host == "" { - return cfg, errors.New("--fanout-binary requires a remote run with --host") + if cfg.PitotBinary != "" && cfg.Host == "" { + return cfg, errors.New("--pitot-binary requires a remote run with --host") } cfg.Token = os.Getenv(tokenEnv) cfg.MetricsToken = os.Getenv(metricsTokenEnv) @@ -161,16 +161,16 @@ func run(ctx context.Context, args []string) error { func managed(ctx context.Context, args []string) error { fs := flag.NewFlagSet("managed", flag.ContinueOnError) var cfg bench.ManagedConfig - fs.StringVar(&cfg.FanoutPath, "fanout", "./fanout", "provisioned Fanout binary") + fs.StringVar(&cfg.PitotPath, "pitot", "./pitot", "provisioned Pitot binary") fs.StringVar(&cfg.TelemetrygenPath, "telemetrygen", "./telemetrygen", "provisioned telemetrygen binary") - fs.StringVar(&cfg.Listen, "listen", "127.0.0.1", "Fanout listen IP for OTLP, metrics, and queries") + fs.StringVar(&cfg.Listen, "listen", "127.0.0.1", "Pitot listen IP for OTLP, metrics, and queries") fs.StringVar(&cfg.GeneratorSSH, "generator-ssh", "", "SSH target for telemetrygen launches") fs.IntVar(&cfg.GeneratorSSHPort, "generator-ssh-port", 22, "SSH port for telemetrygen launches") fs.StringVar(&cfg.GeneratorSSHIdentity, "generator-ssh-identity", "", "ephemeral SSH private key for the generator host") fs.StringVar(&cfg.GeneratorTelemetrygen, "generator-telemetrygen", "", "telemetrygen path on the generator host") fs.StringVar(&cfg.OutputDir, "output", "./results", "remote evidence directory") - fs.StringVar(&cfg.FanoutVersion, "fanout-version", "unknown", "Fanout version") - fs.StringVar(&cfg.FanoutArtifact, "fanout-artifact", "", "verified release artifact identity") + fs.StringVar(&cfg.PitotVersion, "pitot-version", "unknown", "Pitot version") + fs.StringVar(&cfg.PitotArtifact, "pitot-artifact", "", "verified release artifact identity") fs.StringVar(&cfg.SignalSelection, "signals", "all", "sweeps: all or mixed (skips single-signal sweeps)") fs.StringVar(&cfg.Preset, "preset", "standard", "suite preset") fs.IntVar(&cfg.Repeat, "repeat", 1, "sweep repetitions") diff --git a/cmd/fanout-bench/main_test.go b/cmd/pitot-bench/main_test.go similarity index 79% rename from cmd/fanout-bench/main_test.go rename to cmd/pitot-bench/main_test.go index c3e2fe7..b3a092e 100644 --- a/cmd/fanout-bench/main_test.go +++ b/cmd/pitot-bench/main_test.go @@ -7,10 +7,10 @@ import ( "strings" "testing" - "github.com/labstack/fanout-bench/internal/bench" + "github.com/labstack/pitot-bench/internal/bench" ) -func TestLocalDefaultsUseSharedFanoutListener(t *testing.T) { +func TestLocalDefaultsUseSharedPitotListener(t *testing.T) { cfg, err := commonFlags("test", nil) if err != nil { t.Fatal(err) @@ -27,7 +27,7 @@ func TestSplitReproductionCommandPassesValidationAfterSubstitution(t *testing.T) } cfg, err := commonFlags("run", []string{ "--host", "root@192.0.2.10", "--generator-host", "root@192.0.2.11", - "--generator-address", "10.77.0.3", "--fanout-listen", "10.77.0.2", + "--generator-address", "10.77.0.3", "--pitot-listen", "10.77.0.2", "--ssh-port", "2222", "--telemetrygen", executable, "--preset", "validate", "--signals", "mixed", }) if err != nil { @@ -46,7 +46,7 @@ func TestSplitReproductionCommandPassesValidationAfterSubstitution(t *testing.T) } command := strings.NewReplacer( "", "192.0.2.20", "", "192.0.2.21", - "", "10.77.0.13", "", "10.77.0.12", + "", "10.77.0.13", "", "10.77.0.12", ).Replace(report.ReproductionCommand) // Parse shell quoting without executing the reproduction command. output, err := exec.Command("bash", "-c", "set -- "+command+"; printf '%s\\0' \"$@\"").Output() @@ -62,7 +62,7 @@ func TestSplitReproductionCommandPassesValidationAfterSubstitution(t *testing.T) t.Fatal("validation flags lost") } if reproduced.Host != "root@192.0.2.20" || reproduced.GeneratorHost != "root@192.0.2.21" || - reproduced.GeneratorAddress != "10.77.0.13" || reproduced.FanoutListen != "10.77.0.12" || reproduced.SSHPort != 2222 { + reproduced.GeneratorAddress != "10.77.0.13" || reproduced.PitotListen != "10.77.0.12" || reproduced.SSHPort != 2222 { t.Fatalf("reproduction lost topology: %+v", reproduced) } } @@ -78,12 +78,12 @@ func TestHostFlags(t *testing.T) { } func TestSplitFlagsRequiredTogetherAndRemoteOnly(t *testing.T) { - flags := [][]string{{"--generator-host", "root@generator.example"}, {"--generator-address", "10.77.0.3"}, {"--fanout-listen", "10.77.0.2"}} + flags := [][]string{{"--generator-host", "root@generator.example"}, {"--generator-address", "10.77.0.3"}, {"--pitot-listen", "10.77.0.2"}} for mask := 0; mask < 8; mask++ { for _, remote := range []bool{false, true} { var args []string if remote { - args = append(args, "--host", "root@fanout.example") + args = append(args, "--host", "root@pitot.example") } for i, pair := range flags { if mask&(1<