diff --git a/.bazelignore b/.bazelignore index 1b5d52ff9e..15b2d8aab0 100644 --- a/.bazelignore +++ b/.bazelignore @@ -9,6 +9,5 @@ target .codex-worktrees .claude-worktrees .claude/worktrees -evals mockups vendor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a95f886c89..efa293bd50 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,6 +114,9 @@ jobs: working-directory: benchmark_data/runtime run: python3 -m unittest discover -s tests -p 'test_*.py' + - name: Test benchmark result publication + run: python3 scripts/test-claude-observation-benchmark-publication.py + # The cheap repository gates in one job: commit lint, cache lineage, the # release guards, and rustfmt. Each used to hold its own ubuntu-latest # runner for a couple of minutes of work; on 20 free slots that is three @@ -189,21 +192,17 @@ jobs: bash tests/release_pr_integrity_test.sh bash tests/dashboard_bundle_check_test.sh - # Judge formatting with the toolchain `rust-toolchain.toml` pins, not - # whatever `stable` is on the runner: rustfmt's output changes between - # releases (1.98.1 breaks a chain 1.97.1 keeps inline), and - # `dtolnay/rust-toolchain@stable` exports `RUSTUP_TOOLCHAIN`, which - # overrides the file. Installing through rustup with no override keeps - # the pin in one place. - - name: Install the repository's pinned toolchain - run: rustup show active-toolchain || rustup toolchain install + - uses: ./.github/actions/setup-bazel + with: + cache-key: ci-format-linux + - name: Check formatting - run: cargo fmt --all -- --check + run: bazel build --config=ci //:rustfmt_check # Manifest edits that skip a gen_builds.py rerun leave stale BUILD # files; compare the committed files against a fresh generation. - name: Check generated Bazel BUILD files are current - run: python3 scripts/bazel/gen_builds.py --check + run: bazel run //scripts:bazel/gen_builds.py -- --check - name: Test and check dev skill host copies run: | diff --git a/.github/workflows/distribution-acceptance.yml b/.github/workflows/distribution-acceptance.yml index f00b04b594..34b0e92f3a 100644 --- a/.github/workflows/distribution-acceptance.yml +++ b/.github/workflows/distribution-acceptance.yml @@ -35,6 +35,7 @@ jobs: timeout-minutes: 240 env: TARGET: x86_64-unknown-linux-gnu + AST_GREP_VERSION: "0.44.0" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -46,6 +47,11 @@ jobs: - uses: ./.github/actions/setup-pnpm + - uses: ./.github/actions/setup-bazel + with: + cache-key: distribution-acceptance-linux + test-tools: "true" + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.14.8" diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 528f022951..ceab90b486 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -145,6 +145,11 @@ jobs: pnpm-lock.yaml Cargo.lock + - uses: ./.github/actions/setup-bazel + if: steps.release.outputs.prs_created == 'true' + with: + cache-key: release-metadata-ubuntu-latest + - name: Update release PR generated metadata if: steps.release.outputs.prs_created == 'true' env: diff --git a/AGENTS.md b/AGENTS.md index 50826b2fa8..67e3d93144 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -79,7 +79,9 @@ unauthorized external action after completing independent, authorized work. - Bazel is the sole workspace build, test, check, run, benchmark, and code-generation authority. Run `bazel build //...`, `bazel test //... --test_output=errors`, `bazel run //crates/...`, and - `bazel test //... --test_arg=--bench` for benchmarks. Edition 2024, + `bazel test //crates/: --test_arg=--bench` for an explicit + benchmark target. Benchmarks with Cargo `test = false` remain compiled by + default builds but run only when selected explicitly. Edition 2024, resolver 3, and the toolchain pinned in `rust-toolchain.toml` remain canonical inputs. Never invoke `cargo` directly; route every build, test, check, run, bench, and codegen through Bazel. diff --git a/BUILD.bazel b/BUILD.bazel index 0aceb3f83c..72f76d5c29 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,4 +1,11 @@ load("@npm//:defs.bzl", "npm_link_all_packages") +load("@rules_rust//rust:toolchain.bzl", "current_rustfmt_toolchain") + +current_rustfmt_toolchain( + name = "rustfmt_toolchain", + tags = ["manual"], + visibility = ["//visibility:public"], +) npm_link_all_packages(name = "node_modules") @@ -6,6 +13,7 @@ exports_files( [ "version.txt", "Cargo.toml", + "rustfmt.toml", "pnpm-lock.yaml", "pnpm-lock.bazel.yaml", ], @@ -28,3 +36,11 @@ config_setting( values = {"compilation_mode": "opt"}, visibility = ["//visibility:public"], ) + +# BEGIN generated by scripts/bazel/gen_builds.py +filegroup( + name = "rustfmt_check", + srcs = ["//crates/tracedecay-agent-hosts:rustfmt_check_2024", "//crates/tracedecay-api:rustfmt_check_2024", "//crates/tracedecay-application:rustfmt_check_2024", "//crates/tracedecay-automation-runtime:rustfmt_check_2024", "//crates/tracedecay-automation:rustfmt_check_2024", "//crates/tracedecay-capture:rustfmt_check_2024", "//crates/tracedecay-cli:rustfmt_check_2024", "//crates/tracedecay-code-extraction:rustfmt_check_2024", "//crates/tracedecay-code-index-retention:rustfmt_check_2024", "//crates/tracedecay-code-index-runtime:rustfmt_check_2024", "//crates/tracedecay-code-index:rustfmt_check_2024", "//crates/tracedecay-configuration:rustfmt_check_2024", "//crates/tracedecay-contracts:rustfmt_check_2024", "//crates/tracedecay-daemon-control:rustfmt_check_2024", "//crates/tracedecay-daemon-identity:rustfmt_check_2024", "//crates/tracedecay-daemon-protocol:rustfmt_check_2024", "//crates/tracedecay-daemon-service:rustfmt_check_2024", "//crates/tracedecay-dashboard-api:rustfmt_check_2024", "//crates/tracedecay-domain:rustfmt_check_2024", "//crates/tracedecay-framing:rustfmt_check_2024", "//crates/tracedecay-global-db:rustfmt_check_2024", "//crates/tracedecay-graph-db:rustfmt_check_2024", "//crates/tracedecay-graph-query:rustfmt_check_2024", "//crates/tracedecay-hooks:rustfmt_check_2024", "//crates/tracedecay-host-admission:rustfmt_check_2024", "//crates/tracedecay-host-integration:rustfmt_check_2024", "//crates/tracedecay-lcm:rustfmt_check_2024", "//crates/tracedecay-lsp:rustfmt_check_2024", "//crates/tracedecay-maintenance:rustfmt_check_2024", "//crates/tracedecay-mcp-catalog:rustfmt_check_2024", "//crates/tracedecay-mcp:rustfmt_check_2024", "//crates/tracedecay-policy:rustfmt_check_2024", "//crates/tracedecay-privacy:rustfmt_check_2024", "//crates/tracedecay-private-fs:rustfmt_check_2024", "//crates/tracedecay-project:rustfmt_check_2024", "//crates/tracedecay-query:rustfmt_check_2024", "//crates/tracedecay-runtime-core:rustfmt_check_2024", "//crates/tracedecay-rusqlite-runtime:rustfmt_check_2024", "//crates/tracedecay-sdk:rustfmt_check_2024", "//crates/tracedecay-search-eval:rustfmt_check_2024", "//crates/tracedecay-session-memory:rustfmt_check_2024", "//crates/tracedecay-session-runtime:rustfmt_check_2024", "//crates/tracedecay-session-temporal-store:rustfmt_check_2024", "//crates/tracedecay-sessions:rustfmt_check_2024", "//crates/tracedecay-source-edit:rustfmt_check_2024", "//crates/tracedecay-store-runtime:rustfmt_check_2024", "//crates/tracedecay-store:rustfmt_check_2024", "//crates/tracedecay-temporal-query:rustfmt_check_2024", "//crates/tracedecay-tool-catalog:rustfmt_check_2024", "//crates/tracedecay:rustfmt_check_2024", "//scripts:rustfmt_check_2024", "//sdks/codegen:rustfmt_check_2024"], + tags = ["manual"], +) +# END generated by scripts/bazel/gen_builds.py diff --git a/Cargo.lock b/Cargo.lock index 75ee39a4e5..7d0171c222 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8225,7 +8225,6 @@ dependencies = [ "indexmap", "memchr", "typed-path", - "zopfli", ] [[package]] @@ -8240,18 +8239,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" -[[package]] -name = "zopfli" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" -dependencies = [ - "bumpalo", - "crc32fast", - "log", - "simd-adler32", -] - [[package]] name = "zvariant" version = "5.15.0" diff --git a/MODULE.bazel b/MODULE.bazel index f2aff004cb..b1d96ddd1a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,9 +6,14 @@ bazel_dep(name = "rules_rust", version = "0.74.0") # Rustc adds every dependency search directory to PATH on Windows; large # graphs exceed its environment limit (rust-lang/rust#110889). Native # shared symlinks preserve artifact identities with one search directory. +# Build-script flags use response files to keep the compiler wrapper's own +# command line within Windows' separate CreateProcessW limit. single_version_override( module_name = "rules_rust", - patches = ["//scripts:bazel/patches/rules-rust-windows-dependency-search.patch"], + patches = [ + "//scripts:bazel/patches/rules-rust-windows-dependency-search.patch", + "//scripts:bazel/patches/rules-rust-windows-build-flags.patch", + ], patch_strip = 1, ) bazel_dep(name = "aspect_rules_js", version = "3.5.1") @@ -27,6 +32,7 @@ rust.toolchain( "opt": "3", }, versions = ["1.97.1"], + rustfmt_version = "1.97.1", ) use_repo(rust, "rust_toolchains") register_toolchains("@rust_toolchains//:all") diff --git a/benchmark_data/BUILD.bazel b/benchmark_data/BUILD.bazel index 809b9b68ee..9aaa30ac94 100644 --- a/benchmark_data/BUILD.bazel +++ b/benchmark_data/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py # BEGIN generated by scripts/bazel/gen_builds.py filegroup( @@ -31,4 +31,14 @@ filegroup( exports_files(["runtime/fixtures/project/tests/catalog.rs"], visibility = ["//visibility:public"], ) + +filegroup( + name = "session-temporal", + srcs = glob(["session-temporal/**"]), + visibility = ["//visibility:public"], +) + +exports_files(["session-temporal/workload-v1.json"], + visibility = ["//visibility:public"], +) # END generated by scripts/bazel/gen_builds.py diff --git a/benchmark_data/claude-observation/README.md b/benchmark_data/claude-observation/README.md index 017c6526d6..0f07d771e3 100644 --- a/benchmark_data/claude-observation/README.md +++ b/benchmark_data/claude-observation/README.md @@ -63,8 +63,10 @@ The legacy schema records real SHA-256 identities for workload members and native fixtures plus the host target, Rust/Cargo versions, kernel, and hardware. Those source/content digests remain useful provenance. Do not recreate the former clean-checkout archive, tracked-source snapshot, compiler-input -attestation, or evidence-only commit workflow. Run ordinary Cargo against the -working checkout and report the source revision and dirty state truthfully. +attestation, or evidence-only commit workflow. The runner uses Bazel `--config=release` in the current clean checkout and +records the source revision before and after measurement, the Bazel and Rust +versions, and the measured executable identity. It validates new output before +publishing the result and index, without archive or backup copies. Linux `/proc` is the explicit measurement platform contract. Preflight requires all measured interfaces, a successful write of `5` to diff --git a/benchmark_data/claude-observation/workload-v1.json b/benchmark_data/claude-observation/workload-v1.json index 63705aadbe..54ed38a105 100644 --- a/benchmark_data/claude-observation/workload-v1.json +++ b/benchmark_data/claude-observation/workload-v1.json @@ -652,6 +652,6 @@ ] } }, - "command": "cargo test --quiet --release -p tracedecay --features test-helpers --test claude_observation_benchmark production_observation_pipeline_baseline -- --ignored --exact --nocapture --test-threads=1", + "command": "scripts/run-claude-observation-benchmark.sh", "evidence_runner": "scripts/run-claude-observation-benchmark.sh" } diff --git a/benchmark_data/session-temporal/README.md b/benchmark_data/session-temporal/README.md index 930ce9263b..7745d20acb 100644 --- a/benchmark_data/session-temporal/README.md +++ b/benchmark_data/session-temporal/README.md @@ -37,10 +37,9 @@ Runtime sanitizer provenance for those fixtures is in scripts/run-session-temporal-benchmark.sh --dry-run scripts/run-session-temporal-benchmark.sh --run # diagnostic on Linux or macOS scripts/run-session-temporal-benchmark.sh --refresh-contract # Linux only -cargo bench --bench session_temporal --all-features -- --run ``` -Dry-run is Cargo-free. `--run` isolates `HOME` and `TRACEDECAY_DATA_DIR` and +Dry-run performs no build. The measurement runner uses Bazel `--config=release`. `--run` isolates `HOME` and `TRACEDECAY_DATA_DIR` and measures: `rebuild_activate`, `exact_replay`, `compact_rank`, and `late_hydrate`. `--run` prints diagnostic samples but never changes checked-in evidence. `--refresh-contract` is Linux-only and the only publishing path: it requires a @@ -57,7 +56,7 @@ record counts. Recorded only when executed in the same capture window: ```bash -cargo test --test session_suite --all-features temporal_derived_evidence:: -- --test-threads=1 +bazel test --config=ci //crates/tracedecay:session_suite --test_arg=temporal_derived_evidence:: --test_arg=--test-threads=1 ``` Quantiles are descriptive nearest-rank sample labels, not inferential claims. diff --git a/benchmark_data/session-temporal/workload-v1.json b/benchmark_data/session-temporal/workload-v1.json index 9b4c53203f..7b5683fc25 100644 --- a/benchmark_data/session-temporal/workload-v1.json +++ b/benchmark_data/session-temporal/workload-v1.json @@ -49,15 +49,6 @@ "SessionRetrievalService with GlobalDbSessionTemporalExecution and AllSessionsInAuthorizedRoot hydration" ] }, - "profile": { - "cargo_profile": "bench", - "debug": false, - "debug_assertions": false, - "incremental": false, - "manifest_path": "Cargo.toml", - "opt_level": 3, - "overflow_checks": false - }, "runner": { "path": "scripts/run-session-temporal-benchmark.sh" }, diff --git a/crates/tracedecay-agent-hosts/BUILD.bazel b/crates/tracedecay-agent-hosts/BUILD.bazel index 1b69c1b40f..cf949d3a0e 100644 --- a/crates/tracedecay-agent-hosts/BUILD.bazel +++ b/crates/tracedecay-agent-hosts/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -91,7 +91,7 @@ cargo_build_script( rust_test( name = "context_scout_evidence", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/context_scout_evidence.rs", edition = crate_edition(), version = "0.1.0", @@ -106,7 +106,7 @@ rust_test( rust_test( name = "hooks_boundary", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/hooks_boundary.rs", edition = crate_edition(), version = "0.1.0", @@ -135,6 +135,24 @@ rust_test( data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(["build.rs", "src/product_version/root_manifest.rs"] + glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) + filegroup( name = "src_agents_hermes_templates", srcs = glob(["src/agents/hermes/templates/**"]), diff --git a/crates/tracedecay-agent-hosts/src/hooks/dispatch.rs b/crates/tracedecay-agent-hosts/src/hooks/dispatch.rs index 68b5dca53f..6303180aa0 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/dispatch.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/dispatch.rs @@ -154,20 +154,26 @@ pub fn publish_daemon_bindings( // admission waits behind that cold filesystem work. A live callback // already appending to a prepared spool is a legitimate peer writer, // so publication waits the same bounded budget instead of failing. - drop( - tracedecay_hooks::HookSpoolV1::open_within( - tracedecay_hooks::hook_v2_spool_root(&layout.data_root, *host), - tracedecay_hooks::HookSpoolConfigV1::stock(*host), - now, - tracedecay_hooks::HOOK_SYNCHRONOUS_BUDGET, - ) + let (capture, _) = tracedecay_hooks::HookSpoolV1::open_within( + tracedecay_hooks::hook_v2_spool_root(&layout.data_root, *host), + tracedecay_hooks::HookSpoolConfigV1::stock(*host), + now, + tracedecay_hooks::HOOK_SYNCHRONOUS_BUDGET, + ) + .map_err(|error| tracedecay_domain::errors::TraceDecayError::Config { + message: format!( + "failed to prepare {} Hook capture spool: {error}", + host.hook_key() + ), + })?; + capture + .prepare() .map_err(|error| tracedecay_domain::errors::TraceDecayError::Config { message: format!( - "failed to prepare {} Hook capture spool: {error}", + "failed to prepare {} Hook capture durability: {error}", host.hook_key() ), - })?, - ); + })?; drop( tracedecay_hooks::HookDeliveryReceiptWriterV1::open_within( tracedecay_hooks::hook_delivery_receipt_spool_root(&layout.data_root, *host), diff --git a/crates/tracedecay-agent-hosts/src/hooks/dispatch/tests.rs b/crates/tracedecay-agent-hosts/src/hooks/dispatch/tests.rs index aabdbd99b9..9eec61abfc 100644 --- a/crates/tracedecay-agent-hosts/src/hooks/dispatch/tests.rs +++ b/crates/tracedecay-agent-hosts/src/hooks/dispatch/tests.rs @@ -859,6 +859,25 @@ fn binding_publication_waits_for_a_live_callback_holding_the_spool() { publish_daemon_bindings(&runtime, &layout).unwrap(); let host = NativeHostIdentityV1::ClaudeCode; + let capture_root = tracedecay_hooks::hook_v2_spool_root(&layout.data_root, host); + let (prepared, report) = + HookSpoolV1::open(&capture_root, HookSpoolConfigV1::stock(host), UtcMicros(1)).unwrap(); + assert_eq!((report.pending_records, report.next_sequence), (0, 1)); + assert!( + !report.checkpoint_rewritten, + "publication prepares the first callback's checkpoint" + ); + let barriers = tracedecay_private_fs::framed_log::sync_latency::inject( + &capture_root, + std::time::Duration::ZERO, + ); + prepared.prepare().unwrap(); + assert_eq!( + barriers.syncs(), + 0, + "publication already prepared this exact file" + ); + drop(barriers); let held = std::time::Duration::from_millis(20); let (capture, _) = HookSpoolV1::open( tracedecay_hooks::hook_v2_spool_root(&layout.data_root, host), diff --git a/crates/tracedecay-api/BUILD.bazel b/crates/tracedecay-api/BUILD.bazel index 483a0f18a6..0474e1e202 100644 --- a/crates/tracedecay-api/BUILD.bazel +++ b/crates/tracedecay-api/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -44,7 +44,7 @@ rust_library( rust_test( name = "dashboard_presentations", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/dashboard_presentations.rs", edition = crate_edition(), version = "0.1.0", @@ -59,7 +59,7 @@ rust_test( rust_test( name = "handoff_routes", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/handoff_routes.rs", edition = crate_edition(), version = "0.1.0", @@ -74,7 +74,7 @@ rust_test( rust_test( name = "multi_root_read_model", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/multi_root_read_model.rs", edition = crate_edition(), version = "0.1.0", @@ -101,3 +101,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-application/BUILD.bazel b/crates/tracedecay-application/BUILD.bazel index 9104752908..93bc090870 100644 --- a/crates/tracedecay-application/BUILD.bazel +++ b/crates/tracedecay-application/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -93,7 +93,7 @@ rust_test( rust_test( name = "pr_tracking", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/pr_tracking.rs", edition = crate_edition(), version = "0.1.0", @@ -108,6 +108,8 @@ rust_test( rust_test( name = "work_rollup", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]) + glob(["tests/application_suite/**/*.rs", "tests/application_suite/github_stack_coordinator/**/*.rs"]), crate_root = "benches/work_rollup.rs", edition = crate_edition(), @@ -121,6 +123,12 @@ rust_test( data = glob(["benches/**", "assets/**", "fixtures/**", "data/**", "resources/**", "vendor/**", "tests/**"], allow_empty = True) + ["//tests:fixtures"], ) +filegroup( + name = "benchmark_binaries", + testonly = True, + srcs = [":work_rollup__binary"], +) + rust_test( name = "unit_test", crate = ":tracedecay-application__+test-helpers_+test-transport__dev", @@ -136,6 +144,24 @@ rust_test( data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["benches/**/*.rs"]) + glob(["tests/application_suite/**/*.rs", "tests/application_suite/github_stack_coordinator/**/*.rs"]) + glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + glob(["tests/application_suite/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) + filegroup( name = "src_advisory_fixtures_provider_branch_review", srcs = glob(["src/advisory/fixtures/provider_branch_review/**"]), diff --git a/crates/tracedecay-application/src/primitives/production/extended_primitive.rs b/crates/tracedecay-application/src/primitives/production/extended_primitive.rs index 3bf4ebfc27..c071819d76 100644 --- a/crates/tracedecay-application/src/primitives/production/extended_primitive.rs +++ b/crates/tracedecay-application/src/primitives/production/extended_primitive.rs @@ -5,7 +5,9 @@ use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, OnceLock, PoisonError}; use sha2::{Digest, Sha256}; -use tracedecay_code_index::graph_projection::{CodeGraphReadCostMeter, CodeGraphSymbolSummaryV1}; +use tracedecay_code_index::graph_projection::{ + CodeGraphReadCostMeter, CodeGraphReadinessRequirement, CodeGraphSymbolSummaryV1, +}; use tracedecay_contracts::retrieval::{ HealthDeltaRequest, HealthDeltaResult, PrimitiveFailureKind, RetrievalPortContext, SymbolPrimitiveRecord, @@ -567,14 +569,24 @@ impl ExtendedPrimitivePort for TraceDecayExtendedPrimitivePortV1 { Box::pin(tracing::Instrument::instrument( async move { let cancellation = request_graph_cancellation(context.request); - let reader = match open_code_graph( - self.code_graph.as_ref(), - context.request, - now_observed(), - Arc::clone(&cancellation), - ) - .await - { + let opened = self + .code_graph + .open( + CodeGraphReadRequest::new( + context.request, + now_observed(), + Arc::clone(&cancellation), + ) + .with_readiness(CodeGraphReadinessRequirement::Engine), + ) + .await; + let reader = match opened.and_then(|graph| { + graph.reader_with_cancellation( + context.request, + now_observed(), + Arc::clone(&cancellation), + ) + }) { Ok(reader) => reader, Err(error) => { return graph_read_outcome(&error, EvidenceDomain::Source, now_observed()); diff --git a/crates/tracedecay-automation-runtime/BUILD.bazel b/crates/tracedecay-automation-runtime/BUILD.bazel index af2e4385fb..32d181d17b 100644 --- a/crates/tracedecay-automation-runtime/BUILD.bazel +++ b/crates/tracedecay-automation-runtime/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -107,3 +107,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-automation/BUILD.bazel b/crates/tracedecay-automation/BUILD.bazel index 78fe2ba49f..1f6dc10858 100644 --- a/crates/tracedecay-automation/BUILD.bazel +++ b/crates/tracedecay-automation/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -56,3 +56,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-capture/BUILD.bazel b/crates/tracedecay-capture/BUILD.bazel index 0de9dbc400..031434ad39 100644 --- a/crates/tracedecay-capture/BUILD.bazel +++ b/crates/tracedecay-capture/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -44,7 +44,7 @@ rust_library( rust_test( name = "canonical_envelope_allocations", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/canonical_envelope_allocations.rs", edition = crate_edition(), version = "0.1.0", @@ -86,3 +86,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + glob(["tests/capture_suite/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-cli/BUILD.bazel b/crates/tracedecay-cli/BUILD.bazel index 60c544dd71..658aa9605d 100644 --- a/crates/tracedecay-cli/BUILD.bazel +++ b/crates/tracedecay-cli/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -85,7 +85,7 @@ rust_binary( rust_test( name = "c_library_allocator", - srcs = glob(["tests/**/*.rs"]) + glob(["build-support/**/*.rs", "src/**/*.rs"]) + ["//crates/tracedecay:tests_common", "//tests:support"], + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + glob(["build-support/**/*.rs", "src/**/*.rs"]) + ["//crates/tracedecay:tests_common_sources", "//tests:support_sources"], crate_root = "tests/c_library_allocator.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -102,7 +102,7 @@ rust_test( rust_test( name = "core_cli_suite", - srcs = glob(["tests/core_cli_suite/**/*.rs"]) + glob(["build-support/**/*.rs", "tests/support/**/*.rs"]) + ["//crates/tracedecay:tests_common", "//tests:support"], + srcs = glob(["tests/core_cli_suite/**/*.rs"]) + glob(["build-support/**/*.rs", "tests/support/**/*.rs"]) + ["//crates/tracedecay:tests_common_sources", "//tests:support_sources"], crate_root = "tests/core_cli_suite/main.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -119,7 +119,7 @@ rust_test( rust_test( name = "host_journeys_suite", - srcs = glob(["tests/host_journeys_suite/**/*.rs"]) + glob(["tests/support/**/*.rs"]) + ["//tests:support"], + srcs = glob(["tests/host_journeys_suite/**/*.rs"]) + glob(["tests/support/**/*.rs"]) + ["//tests:support_sources"], crate_root = "tests/host_journeys_suite/main.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -136,7 +136,7 @@ rust_test( rust_test( name = "process_allocator_heaps", - srcs = glob(["tests/**/*.rs"]) + glob(["build-support/**/*.rs", "src/**/*.rs"]) + ["//crates/tracedecay:tests_common", "//tests:support"], + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + glob(["build-support/**/*.rs", "src/**/*.rs"]) + ["//crates/tracedecay:tests_common_sources", "//tests:support_sources"], crate_root = "tests/process_allocator_heaps.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -153,7 +153,7 @@ rust_test( rust_test( name = "refresh_memory_journey", - srcs = glob(["tests/**/*.rs"]) + glob(["build-support/**/*.rs", "src/**/*.rs"]) + ["//crates/tracedecay:tests_common", "//tests:support"], + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + glob(["build-support/**/*.rs", "src/**/*.rs"]) + ["//crates/tracedecay:tests_common_sources", "//tests:support_sources"], crate_root = "tests/refresh_memory_journey.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -170,7 +170,7 @@ rust_test( rust_test( name = "retained_parse_heap", - srcs = glob(["tests/**/*.rs"]) + glob(["build-support/**/*.rs", "src/**/*.rs"]) + ["//crates/tracedecay:tests_common", "//tests:support"], + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + glob(["build-support/**/*.rs", "src/**/*.rs"]) + ["//crates/tracedecay:tests_common_sources", "//tests:support_sources"], crate_root = "tests/retained_parse_heap.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -187,7 +187,7 @@ rust_test( rust_test( name = "work_loop_journey", - srcs = glob(["tests/**/*.rs"]) + glob(["build-support/**/*.rs", "src/**/*.rs"]) + ["//crates/tracedecay:tests_common", "//tests:support"], + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + glob(["build-support/**/*.rs", "src/**/*.rs"]) + ["//crates/tracedecay:tests_common_sources", "//tests:support_sources"], crate_root = "tests/work_loop_journey.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -204,7 +204,7 @@ rust_test( rust_test( name = "work_route_exposure_conformance", - srcs = glob(["tests/**/*.rs"]) + glob(["build-support/**/*.rs", "src/**/*.rs"]) + ["//crates/tracedecay:tests_common", "//tests:support"], + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + glob(["build-support/**/*.rs", "src/**/*.rs"]) + ["//crates/tracedecay:tests_common_sources", "//tests:support_sources"], crate_root = "tests/work_route_exposure_conformance.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -218,3 +218,21 @@ rust_test( rustc_env = {"CARGO_BIN_EXE_tracedecay": "$(rootpath :tracedecay__test)", "CARGO_TARGET_TMPDIR": "/tmp"}, runfiles_env = {"CARGO_BIN_EXE_tracedecay": "//crates/tracedecay-cli:tracedecay__test", "TRACEDECAY_HOST_CLI_FIXTURE": "//crates/tracedecay-cli:tracedecay-host-cli-fixture", "TRACEDECAY_TEST_BIN": "//crates/tracedecay-cli:tracedecay__test"}, ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(["build.rs"] + glob(["build-support/*.rs"]) + glob(["build-support/**/*.rs"]) + glob(["src/**/*.rs"]) + glob(["build-support/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + glob(["build-support/**/*.rs", "src/**/*.rs"]) + ["//crates/tracedecay:tests_common_sources", "//tests:support_sources"] + glob(["tests/core_cli_suite/**/*.rs"]) + glob(["build-support/**/*.rs", "tests/support/**/*.rs"]) + ["//crates/tracedecay:tests_common_sources", "//tests:support_sources"] + glob(["tests/host_journeys_suite/**/*.rs"]) + glob(["tests/support/**/*.rs"]) + ["//tests:support_sources"]).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-cli/Cargo.toml b/crates/tracedecay-cli/Cargo.toml index 896c8e2343..16b82cab84 100644 --- a/crates/tracedecay-cli/Cargo.toml +++ b/crates/tracedecay-cli/Cargo.toml @@ -203,7 +203,9 @@ mimalloc = { version = "0.1.52", optional = true, features = ["no_thp"] } # Windows release archives are `.zip` (Unix ships `.tar.gz` via flate2/tar); # consumed by the self-update extraction in src/upgrade.rs. [target.'cfg(windows)'.dependencies] -zip = { version = "8", default-features = false, features = ["deflate"] } +# Keep the flate2 backend selected by the workspace: sealed segment digests +# must not change when Windows archive support is enabled. +zip = { version = "8", default-features = false, features = ["deflate-flate2"] } # sigstore-verify links aws-lc; on x86_64 MSVC it needs NASM unless its # prebuilt assembly objects are allowed, and release runners carry no NASM. aws-lc-rs = { version = "1.18", features = ["prebuilt-nasm"] } diff --git a/crates/tracedecay-cli/src/global.rs b/crates/tracedecay-cli/src/global.rs index dd02f2a7c1..61f13ebcc4 100644 --- a/crates/tracedecay-cli/src/global.rs +++ b/crates/tracedecay-cli/src/global.rs @@ -371,6 +371,7 @@ mod gather_tests { use std::fs; #[cfg(unix)] use std::os::unix::fs::symlink; + use tracedecay_runtime_core::path_safety::{plain_git_args, plain_host_path}; fn make_enrolled_project(profile: &ProfileRoot, root: &Path, project_id: &str) { tracedecay_runtime_core::storage::pin_fixture_repository_identity(root, project_id) @@ -384,8 +385,8 @@ mod gather_tests { fn git(dir: &Path, args: &[&str]) { let status = std::process::Command::new("git") - .args(args) - .current_dir(dir) + .args(plain_git_args(args)) + .current_dir(plain_host_path(dir)) .status() .unwrap(); assert!(status.success(), "git {args:?}"); diff --git a/crates/tracedecay-cli/tests/core_cli_suite/test_profile_isolation_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/test_profile_isolation_test.rs index 4ad0997712..6c05809f3a 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/test_profile_isolation_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/test_profile_isolation_test.rs @@ -124,7 +124,7 @@ fn fixture_children_run_only_admitted_fake_hosts_never_ambient_ones() { } fn canonical(path: &Path) -> PathBuf { - path.canonicalize().unwrap_or_else(|_| path.to_path_buf()) + tracedecay_runtime_core::path_safety::canonical_root_identity(path) } #[test] diff --git a/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs index b5aec7e824..16d01971c5 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs @@ -1305,8 +1305,9 @@ fn capture_spool_refusal_names_its_typed_cause_on_stderr() { enroll_native_capture_project(&home_path, &project_path, "proj_capture_typed_cause"); // A directory where the spool's records file belongs is a spool the hook // must refuse; the refusal must say which spool fault it hit. - std::fs::create_dir_all(native_capture_spool_root(&data_root, host).join("records.v1.bin")) - .unwrap(); + let records_path = native_capture_spool_root(&data_root, host).join("records.v1.bin"); + std::fs::remove_file(&records_path).unwrap(); + std::fs::create_dir(&records_path).unwrap(); let output = run_native_capture_hook( &home_path, diff --git a/crates/tracedecay-code-extraction/BUILD.bazel b/crates/tracedecay-code-extraction/BUILD.bazel index af95b4e37a..0f51132fa6 100644 --- a/crates/tracedecay-code-extraction/BUILD.bazel +++ b/crates/tracedecay-code-extraction/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -122,7 +122,7 @@ rust_binary( rust_test( name = "extract_alloc", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/extract_alloc.rs", edition = crate_edition(), version = "0.1.0", @@ -152,7 +152,7 @@ rust_test( rust_test( name = "retained_parse_heap", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/retained_parse_heap.rs", edition = crate_edition(), version = "0.1.0", @@ -179,3 +179,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(["build.rs"] + glob(["examples/**/*.rs"]) + glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + glob(["tests/main/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-code-index-retention/BUILD.bazel b/crates/tracedecay-code-index-retention/BUILD.bazel index e4b868121c..c02f6ddecf 100644 --- a/crates/tracedecay-code-index-retention/BUILD.bazel +++ b/crates/tracedecay-code-index-retention/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -107,3 +107,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-code-index-runtime/BUILD.bazel b/crates/tracedecay-code-index-runtime/BUILD.bazel index 0aeb58c380..6fa5fad3d1 100644 --- a/crates/tracedecay-code-index-runtime/BUILD.bazel +++ b/crates/tracedecay-code-index-runtime/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -90,3 +90,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/residency.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/residency.rs index 462343ba2e..1f789df72e 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/residency.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/residency.rs @@ -59,8 +59,10 @@ async fn an_idle_worktree_gives_back_its_decode_and_search_still_answers_fresh() .expect("the seated generation answers"); assert!(!fresh.served_stale); let generation = fresh.generation.as_str().to_owned(); - // A pass still finishing the mount can re-seat the decode it measures. + // Attribution can hold the decode while awaiting admission after the + // source pass settles. Wait for the worker to release that last handle. wait_for_settled_owner(®istry, fixture.path()).await; + wait_for_worker_phase(®istry, fixture.path(), CodeIndexWorkerPhaseV1::Parked).await; let used = owners.report(Instant::now()); assert_eq!( diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/text_artifact_carry_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/text_artifact_carry_tests.rs index 9f75d31ec5..dfc036ad79 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/text_artifact_carry_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/text_artifact_carry_tests.rs @@ -315,6 +315,9 @@ fn assert_edit_over_damaged_parent_builds_cold(damage: impl FnOnce(&Path)) { ); published(scheduler.reconcile_now().expect("publish the parent")); drain_text(&scheduler); + // Corruption is an offline store change: Windows refuses to mutate an + // artifact while the scheduler still owns its mapped serving pages. + drop(scheduler); damage(&active_text_artifact_path(store.path())); fixture.edit( @@ -322,6 +325,11 @@ fn assert_edit_over_damaged_parent_builds_cold(damage: impl FnOnce(&Path)) { &module(4, "").replace("wrapping_mul(31)", "wrapping_mul(37)"), ); fixture.commit_all("edit over a damaged parent"); + let mut scheduler = super::scheduler( + &fixture, + store.path().to_path_buf(), + Arc::new(SharedCodeIndexBytePoolV1::default()), + ); published(scheduler.reconcile_now().expect("publish the successor")); drain_text(&scheduler); let built = active_artifact(store.path()); diff --git a/crates/tracedecay-code-index-runtime/src/project_reads.rs b/crates/tracedecay-code-index-runtime/src/project_reads.rs index 622711a1a8..9a885ffedf 100644 --- a/crates/tracedecay-code-index-runtime/src/project_reads.rs +++ b/crates/tracedecay-code-index-runtime/src/project_reads.rs @@ -60,12 +60,19 @@ fn refuse_projection_wait(request: &CodeGraphReadRequest<'_>) -> Result<(), Code /// the read waits for it within what is left of its budget, so a re-warm /// that finishes in time serves the read instead of refusing it. Past the /// budget it answers the measured warm-up still needed. +#[tracing::instrument( + name = "daemon.code_index.read.await_rewarm", + level = "trace", + skip_all, + fields(readiness = ?request.readiness) +)] async fn await_rewarm( store: &Arc, request: &CodeGraphReadRequest<'_>, ) -> Result<(), CodeGraphReadError> { // A zero budget never blocks: it starts the re-warm and reports it. - if store.await_rewarm(Duration::ZERO).is_ok() { + let requirement = request.readiness; + if store.await_rewarm_for(Duration::ZERO, requirement).is_ok() { return Ok(()); } let now = now_micros(); @@ -81,7 +88,7 @@ async fn await_rewarm( let budget = Duration::from_micros(u64::try_from(expires_at.0.saturating_sub(now.0)).unwrap_or(0)); let waiting = Arc::clone(store); - let wait = tokio::task::spawn_blocking(move || waiting.await_rewarm(budget)); + let wait = tokio::task::spawn_blocking(move || waiting.await_rewarm_for(budget, requirement)); let settled = match request.live_cancellation { Some(signal) => tokio::select! { biased; diff --git a/crates/tracedecay-code-index/BUILD.bazel b/crates/tracedecay-code-index/BUILD.bazel index d8b8a82152..6cbb689054 100644 --- a/crates/tracedecay-code-index/BUILD.bazel +++ b/crates/tracedecay-code-index/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -144,7 +144,7 @@ rust_test( rust_test( name = "resident_accounting", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/resident_accounting.rs", edition = crate_edition(), version = "0.1.0", @@ -159,6 +159,8 @@ rust_test( rust_test( name = "code_index_chunks", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/code_index_chunks.rs", edition = crate_edition(), @@ -174,6 +176,8 @@ rust_test( rust_test( name = "git_ancestry", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/git_ancestry.rs", edition = crate_edition(), @@ -189,6 +193,8 @@ rust_test( rust_test( name = "partitioned_codec", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/partitioned_codec.rs", edition = crate_edition(), @@ -204,6 +210,8 @@ rust_test( rust_test( name = "restore_generation", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/restore_generation.rs", edition = crate_edition(), @@ -219,6 +227,8 @@ rust_test( rust_test( name = "retained_parse", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/retained_parse.rs", edition = crate_edition(), @@ -234,6 +244,8 @@ rust_test( rust_test( name = "sealed_storage", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/sealed_storage.rs", edition = crate_edition(), @@ -249,6 +261,8 @@ rust_test( rust_test( name = "source_search", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/source_search.rs", edition = crate_edition(), @@ -262,6 +276,12 @@ rust_test( data = glob(["benches/**", "assets/**", "fixtures/**", "data/**", "resources/**", "vendor/**", "tests/**"], allow_empty = True) + ["//tests:fixtures"], ) +filegroup( + name = "benchmark_binaries", + testonly = True, + srcs = [":code_index_chunks__binary", ":git_ancestry__binary", ":partitioned_codec__binary", ":restore_generation__binary", ":retained_parse__binary", ":sealed_storage__binary", ":source_search__binary"], +) + rust_test( name = "unit_test", crate = ":tracedecay-code-index__+eval-helpers_+lang-dockerfile_+test-helpers__dev", @@ -276,3 +296,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["benches/**/*.rs"]) + glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + glob(["tests/code_index_suite/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-code-index/benches/sealed_storage.rs b/crates/tracedecay-code-index/benches/sealed_storage.rs index 73a58ba58a..9bee93bff9 100644 --- a/crates/tracedecay-code-index/benches/sealed_storage.rs +++ b/crates/tracedecay-code-index/benches/sealed_storage.rs @@ -134,6 +134,7 @@ impl CodeIndexExecutionControlV1 for ActiveControl { struct Sealed { manifest: Vec, file_segments: BTreeMap>, + segment_kinds: BTreeMap, evidence: (String, Vec), } @@ -288,8 +289,11 @@ fn main() -> Result<(), Box> { let successor_restore_ms = digest_decoded(&successor_sealed, &store_segments, &mut decoded)?; let mut sections = BTreeMap::new(); - for bytes in store_segments.values() { - add_sections(bytes, &mut sections)?; + let mut segment_kinds = clean_sealed.segment_kinds.clone(); + segment_kinds.extend(successor_sealed.segment_kinds.clone()); + for (digest, bytes) in &store_segments { + let kind = segment_kinds.get(digest).ok_or("segment kind is missing")?; + add_sections(bytes, kind, &mut sections)?; } for sealed in [&clean_sealed, &successor_sealed] { add_evidence_sections(&sealed.evidence.1, &mut sections)?; @@ -484,13 +488,11 @@ fn request( content_digest: digest, disposition: SnapshotFileDispositionV1::Present, }); - if changed.contains(&source.logical_path) { - captured_files.push(CodeIndexCapturedFileV1 { - file_occurrence_id: occurrence, - sanitized_bytes: Arc::clone(&source.bytes), - sensitivity_level: SensitivityLevelV1::Public, - }); - } + captured_files.push(CodeIndexCapturedFileV1 { + file_occurrence_id: occurrence, + sanitized_bytes: Arc::clone(&source.bytes), + sensitivity_level: SensitivityLevelV1::Public, + }); } Ok(CodeIndexBuildRequestV1 { snapshot: SanitizedCodeSnapshotV1 { @@ -540,19 +542,28 @@ fn seal( parent: Option<&[u8]>, ) -> Result { let mut file_segments = BTreeMap::new(); + let mut segment_kinds = BTreeMap::new(); let mut pack = Vec::new(); let mut evidence = None; let manifest = publication.encode(parent, |publication| { match publication { - SealedGenerationSegmentPublicationV1::File { digest, bytes } - | SealedGenerationSegmentPublicationV1::FileEvidence { digest, bytes } - | SealedGenerationSegmentPublicationV1::ResolutionIndex { digest, bytes } => { + SealedGenerationSegmentPublicationV1::File { digest, bytes } => { + file_segments.insert(digest.as_str().to_owned(), bytes.to_vec()); + segment_kinds.insert(digest.as_str().to_owned(), "file"); + } + SealedGenerationSegmentPublicationV1::FileEvidence { digest, bytes } => { file_segments.insert(digest.as_str().to_owned(), bytes.to_vec()); + segment_kinds.insert(digest.as_str().to_owned(), "file_evidence"); + } + SealedGenerationSegmentPublicationV1::ResolutionIndex { digest, bytes } => { + file_segments.insert(digest.as_str().to_owned(), bytes.to_vec()); + segment_kinds.insert(digest.as_str().to_owned(), "resolution_index"); } SealedGenerationSegmentPublicationV1::CodeGraphPage { page_digest, bytes, .. } => { file_segments.insert(page_digest.as_str().to_owned(), bytes.to_vec()); + segment_kinds.insert(page_digest.as_str().to_owned(), "code_graph"); } SealedGenerationSegmentPublicationV1::GenerationEvidencePage { bytes, .. } => { pack.extend_from_slice(bytes); @@ -572,6 +583,7 @@ fn seal( Ok(Sealed { manifest, file_segments, + segment_kinds, evidence: evidence.ok_or_else(|| { CodeIndexProductionErrorV1::Contract("sealed generation has no evidence".to_owned()) })?, @@ -654,10 +666,11 @@ fn digest_decoded( } } -/// Split one stored file segment by payload section. The stored bytes are +/// Split one stored segment by its production publication kind and payload section. The stored bytes are /// either the canonical JSON itself or its raw DEFLATE stream. fn add_sections( bytes: &[u8], + kind: &str, sections: &mut BTreeMap, ) -> Result<(), Box> { let json = if bytes.first() == Some(&b'{') { @@ -669,6 +682,10 @@ fn add_sections( }; *sections.entry("total".to_owned()).or_default() += json.len(); let value: serde_json::Value = serde_json::from_slice(&json)?; + if kind != "file" { + *sections.entry(kind.to_owned()).or_default() += json.len(); + return Ok(()); + } let file = value.get("file").ok_or("segment has no file payload")?; for (key, section) in file.as_object().ok_or("file payload is not an object")? { if key == "artifacts" { diff --git a/crates/tracedecay-code-index/benches/support/git.rs b/crates/tracedecay-code-index/benches/support/git.rs index e3314cf530..32554117e7 100644 --- a/crates/tracedecay-code-index/benches/support/git.rs +++ b/crates/tracedecay-code-index/benches/support/git.rs @@ -7,8 +7,9 @@ use sha2::{Digest, Sha256}; use tempfile::TempDir; use tracedecay_domain::UtcMicros; use tracedecay_graph_db::{ - GraphDbRegistration, GraphDbRegistry, GraphDbRegistryConfig, GraphGenerationManifest, - GraphIdempotencyKey, NeverCancelled, VerifiedGraphSnapshot, + GraphDbOwnerAttachmentV1, GraphDbOwnerRegistrationV1, GraphDbRegistration, GraphDbRegistry, + GraphDbRegistryConfig, GraphGenerationManifest, GraphIdempotencyKey, NeverCancelled, + VerifiedGraphSnapshot, }; use tracedecay_rusqlite_runtime::{ ExistingWriterLocator, PersistentWriter, StorageOperationExecutor, @@ -19,13 +20,15 @@ use tracedecay_rusqlite_runtime::{ use tracedecay_store::{ AdmissionConfigV1, BrainId, GraphProjectionIdentityV1, GraphPublicationInputDigestV1, GraphPublicationOperationContextV1, GraphPublicationStoreV1, ProjectId, - RetainedGraphStoreLeaseV1, RuntimeCancellationIdV1, RuntimeCancellationIdentityV1, - RuntimeDeadlineIdV1, RuntimeDeadlineV1, RuntimeInterruptionV1, RuntimeRequestControlV1, - RuntimeRequestProbeV1, StoreAuthorityEpochV1, StoreIncarnationV1, StoreRuntimeBindingV1, - StoreShardIdV1, UserProfileId, VerifiedStoreLocatorV1, canonical_store_locator_digest, + RetainedGraphStoreLeaseV1, RetainedGraphStoreOwnerAttachmentV1, + RetainedGraphStoreOwnerOperationLeaseErrorV1, RuntimeCancellationIdV1, + RuntimeCancellationIdentityV1, RuntimeDeadlineIdV1, RuntimeDeadlineV1, RuntimeInterruptionV1, + RuntimeRequestControlV1, RuntimeRequestProbeV1, StoreAuthorityEpochV1, StoreIncarnationV1, + StoreRuntimeBindingV1, StoreShardIdV1, UserProfileId, VerifiedStoreLocatorV1, + canonical_store_locator_digest, }; -#[derive(Debug)] +#[derive(Clone, Debug)] struct BenchmarkGraphLease { binding: StoreRuntimeBindingV1, verified_locator: VerifiedStoreLocatorV1, @@ -46,6 +49,27 @@ impl RetainedGraphStoreLeaseV1 for BenchmarkGraphLease { } } +impl RetainedGraphStoreOwnerAttachmentV1 for BenchmarkGraphLease { + fn binding(&self) -> &StoreRuntimeBindingV1 { + &self.binding + } + + fn verified_locator(&self) -> &VerifiedStoreLocatorV1 { + &self.verified_locator + } + + fn canonical_path(&self) -> &Path { + &self.canonical_path + } + + fn issue_operation_lease( + &self, + ) -> Result, RetainedGraphStoreOwnerOperationLeaseErrorV1> + { + Ok(Arc::new(self.clone())) + } +} + struct NoWrites; impl StorageOperationExecutor for NoWrites { @@ -130,6 +154,7 @@ pub struct PersistentGitGraph { _writer: PersistentWriter, _readers: ReaderPool, registry: GraphDbRegistry, + owner_attachment: Option, binding: StoreRuntimeBindingV1, graph_path: PathBuf, authority: GraphPublicationExactSqlStorage, @@ -197,6 +222,7 @@ impl PersistentGitGraph { Self { registry: GraphDbRegistry::new(GraphDbRegistryConfig { max_open: 1 }) .expect("benchmark graph registry config is valid"), + owner_attachment: None, authority: GraphPublicationExactSqlStorage::from_authorized_handle(handle) .expect("benchmark graph publication authority attaches"), binding, @@ -211,6 +237,7 @@ impl PersistentGitGraph { } pub fn publish(&mut self, manifest: GraphGenerationManifest) -> VerifiedGraphSnapshot { + self.mount_owner(); self.sequence += 1; let (append_control, append_probe) = operation_control(self.sequence); let append_context = @@ -259,6 +286,7 @@ impl PersistentGitGraph { } pub fn recover_snapshot(&mut self) -> VerifiedGraphSnapshot { + self.owner_attachment.take(); let registration = self.registration(); assert!( self.registry @@ -266,6 +294,7 @@ impl PersistentGitGraph { .expect("benchmark graph store closes"), "benchmark recovery must close an open graph store", ); + self.mount_owner(); self.sequence += 1; let (control, probe) = operation_control(self.sequence); let context = GraphPublicationOperationContextV1::new(&control, &probe) @@ -284,6 +313,26 @@ impl PersistentGitGraph { .expect("benchmark verified snapshot recovers") } + fn mount_owner(&mut self) { + if self.owner_attachment.is_some() { + return; + } + let operation = self.registration(); + let authority_attachment = Box::new(BenchmarkGraphLease { + binding: operation.authority_lease.binding().clone(), + verified_locator: operation.authority_lease.verified_locator().clone(), + canonical_path: operation.authority_lease.canonical_path().to_path_buf(), + }); + self.owner_attachment = Some( + self.registry + .resolve_owner_attachment(GraphDbOwnerRegistrationV1 { + operation, + authority_attachment, + }) + .expect("benchmark graph map owner mounts"), + ); + } + fn registration(&self) -> GraphDbRegistration { let canonical_path = self.graph_path.clone(); let verified_locator = VerifiedStoreLocatorV1::new( diff --git a/crates/tracedecay-code-index/src/graph_projection.rs b/crates/tracedecay-code-index/src/graph_projection.rs index d85a15b945..9e6776f827 100644 --- a/crates/tracedecay-code-index/src/graph_projection.rs +++ b/crates/tracedecay-code-index/src/graph_projection.rs @@ -91,6 +91,15 @@ const ADJACENCY_SEED_CHUNK: usize = 4_096; /// v12 stores oversized records as bounded DEFLATE bytes, preserving every field. pub const CODE_GRAPH_PROJECTOR_REVISION: &str = "code-graph-projector.v12"; +/// Resident graph data a read actually needs before opening its reader. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CodeGraphReadinessRequirement { + /// Occurrence-seeded point reads do not consult the interactive catalog. + Engine, + /// Name, file, and census reads need the engine and interactive catalog. + Catalog, +} + /// Every semantic edge kind, at its [`relation_edge_kind_index`]. const RELATION_EDGE_KINDS: [RelationEdgeKindV1; 9] = [ RelationEdgeKindV1::Calls, @@ -602,11 +611,20 @@ impl CodeGraphProjectionStore { /// once when nothing released is warming; the reader then answers the /// store's state as before. pub fn await_rewarm(&self, budget: Duration) -> Result<(), CodeGraphRewarmPendingV1> { + self.await_rewarm_for(budget, CodeGraphReadinessRequirement::Catalog) + } + + /// Waits only for the resident data required by this read. + pub fn await_rewarm_for( + &self, + budget: Duration, + requirement: CodeGraphReadinessRequirement, + ) -> Result<(), CodeGraphRewarmPendingV1> { let started = Instant::now(); let mut woke = false; loop { let epoch = self.warm_clock.epoch(); - let Some(pending) = self.rewarm_in_flight(woke) else { + let Some(pending) = self.rewarm_in_flight(woke, requirement) else { return Ok(()); }; let left = budget.saturating_sub(started.elapsed()); @@ -622,7 +640,11 @@ impl CodeGraphProjectionStore { /// measured warm-up it still needs; `None` when no released owner is /// warming. An engine still cold after a settled warm failed it, which /// the reader answers. - fn rewarm_in_flight(&self, woke: bool) -> Option { + fn rewarm_in_flight( + &self, + woke: bool, + requirement: CodeGraphReadinessRequirement, + ) -> Option { let engine = match self.snapshot.serving_engine_resident() { Ok(true) => None, Ok(false) if woke || !self.released.load(AtomicOrdering::Acquire) => return None, @@ -632,6 +654,11 @@ impl CodeGraphProjectionStore { } Err(_) => return None, }; + if requirement == CodeGraphReadinessRequirement::Engine { + return engine.map(|remaining| CodeGraphRewarmPendingV1 { + retry_after: remaining.max(Duration::from_millis(1)), + }); + } let catalog = match (self.interactive_catalog.residency(), engine) { (CatalogResidency::Resident | CatalogResidency::Unreleased, None) => return None, (CatalogResidency::Resident | CatalogResidency::Unreleased, Some(_)) => Duration::ZERO, diff --git a/crates/tracedecay-code-index/src/graph_projection/interactive.rs b/crates/tracedecay-code-index/src/graph_projection/interactive.rs index f012e95810..8053446ec4 100644 --- a/crates/tracedecay-code-index/src/graph_projection/interactive.rs +++ b/crates/tracedecay-code-index/src/graph_projection/interactive.rs @@ -272,6 +272,17 @@ impl CodeGraphReadCostMeter { } impl CodeGraphProjectionStore { + /// Holds the existing catalog-build gate while a fixture drives real reads. + #[cfg(any(test, feature = "test-helpers"))] + pub fn hold_catalog_build_for_test( + &self, + ) -> Result, CodeGraphProjectionError> { + self.interactive_catalog + .build + .lock() + .map_err(|_| catalog_lock_poisoned()) + } + /// Builds and validates the generation-pinned interactive catalog before /// serving latency-bounded reads. Only a fully built immutable catalog is /// published into the store's shared slot. diff --git a/crates/tracedecay-configuration/BUILD.bazel b/crates/tracedecay-configuration/BUILD.bazel index e1a4f2e9da..fd0623173f 100644 --- a/crates/tracedecay-configuration/BUILD.bazel +++ b/crates/tracedecay-configuration/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -107,3 +107,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-contracts/BUILD.bazel b/crates/tracedecay-contracts/BUILD.bazel index af5ad57458..528685348f 100644 --- a/crates/tracedecay-contracts/BUILD.bazel +++ b/crates/tracedecay-contracts/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -71,3 +71,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"]) + glob(["tests/contracts_suite/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-daemon-control/BUILD.bazel b/crates/tracedecay-daemon-control/BUILD.bazel index 241e7e23a0..dc60cc3d0c 100644 --- a/crates/tracedecay-daemon-control/BUILD.bazel +++ b/crates/tracedecay-daemon-control/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -56,3 +56,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-daemon-identity/BUILD.bazel b/crates/tracedecay-daemon-identity/BUILD.bazel index 081e9f5c16..e7e4e107e3 100644 --- a/crates/tracedecay-daemon-identity/BUILD.bazel +++ b/crates/tracedecay-daemon-identity/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -56,3 +56,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-daemon-protocol/BUILD.bazel b/crates/tracedecay-daemon-protocol/BUILD.bazel index a559f4d133..9d51e6b21c 100644 --- a/crates/tracedecay-daemon-protocol/BUILD.bazel +++ b/crates/tracedecay-daemon-protocol/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -56,3 +56,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-daemon-service/BUILD.bazel b/crates/tracedecay-daemon-service/BUILD.bazel index 62c0ca14e4..dcc239eba0 100644 --- a/crates/tracedecay-daemon-service/BUILD.bazel +++ b/crates/tracedecay-daemon-service/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -78,7 +78,7 @@ rust_library( rust_test( name = "cli_dispatch_schema_bodies", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/cli_dispatch_schema_bodies.rs", edition = crate_edition(), version = "0.1.0", @@ -93,7 +93,7 @@ rust_test( rust_test( name = "shutdown_coordination", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/shutdown_coordination.rs", edition = crate_edition(), version = "0.1.0", @@ -121,3 +121,21 @@ rust_test( + ["//crates/tracedecay-hooks:fixtures_host_events_kimi"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-dashboard-api/BUILD.bazel b/crates/tracedecay-dashboard-api/BUILD.bazel index c8fcc9ad7f..179fad95d0 100644 --- a/crates/tracedecay-dashboard-api/BUILD.bazel +++ b/crates/tracedecay-dashboard-api/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -90,3 +90,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-dashboard-api/src/lcm_api.rs b/crates/tracedecay-dashboard-api/src/lcm_api.rs index 1a3a008438..ea848d2a56 100644 --- a/crates/tracedecay-dashboard-api/src/lcm_api.rs +++ b/crates/tracedecay-dashboard-api/src/lcm_api.rs @@ -573,14 +573,10 @@ where match outcome { DashboardLcmReadOutcomeV1::Ready(page) => { let timeline_coverage = aggregates::timeline_view_coverage(&request, &page); - let coverage = if aggregates::is_aggregate_request(&request) { - DashboardCoverageV1::complete( - aggregates::returned_count(&page), - "canonical hydrated records", - ) - } else { - DashboardCoverageV1::unknown() - }; + let coverage = DashboardCoverageV1::complete( + aggregates::returned_count(&page), + "canonical hydrated records", + ); match aggregates::render_canonical_payload( request, page, diff --git a/crates/tracedecay-dashboard-api/src/lcm_api/aggregates.rs b/crates/tracedecay-dashboard-api/src/lcm_api/aggregates.rs index 2f15f7321a..02051603f4 100644 --- a/crates/tracedecay-dashboard-api/src/lcm_api/aggregates.rs +++ b/crates/tracedecay-dashboard-api/src/lcm_api/aggregates.rs @@ -472,13 +472,6 @@ fn timeline_json( }) } -pub(super) fn is_aggregate_request(request: &DashboardLcmReadRequestV1) -> bool { - matches!( - request, - DashboardLcmReadRequestV1::Overview { .. } | DashboardLcmReadRequestV1::Timeline { .. } - ) -} - pub(super) fn timeline_view_coverage( request: &DashboardLcmReadRequestV1, page: &DashboardLcmCanonicalPageV1, diff --git a/crates/tracedecay-domain/BUILD.bazel b/crates/tracedecay-domain/BUILD.bazel index 99d4e34f7c..a48ee57632 100644 --- a/crates/tracedecay-domain/BUILD.bazel +++ b/crates/tracedecay-domain/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -44,7 +44,7 @@ rust_library( rust_test( name = "canonical_scratch_release", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/canonical_scratch_release.rs", edition = crate_edition(), version = "0.1.0", @@ -74,7 +74,7 @@ rust_test( rust_test( name = "session_contract", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/session_contract.rs", edition = crate_edition(), version = "0.1.0", @@ -102,6 +102,24 @@ rust_test( data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + glob(["tests/domain_suite/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) + filegroup( name = "tests", srcs = glob(["tests/**"]), @@ -114,8 +132,20 @@ filegroup( visibility = ["//visibility:public"], ) +filegroup( + name = "tests_domain_suite_work_product_contract_sources", + srcs = glob(["tests/domain_suite/work_product_contract/**/*.rs"]), + visibility = ["//visibility:public"], +) + filegroup( name = "tests_fixtures", srcs = glob(["tests/fixtures/**"]), visibility = ["//visibility:public"], ) + +filegroup( + name = "tests_sources", + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-framing/BUILD.bazel b/crates/tracedecay-framing/BUILD.bazel index b4afeee2c5..b70290465d 100644 --- a/crates/tracedecay-framing/BUILD.bazel +++ b/crates/tracedecay-framing/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -56,3 +56,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-global-db/BUILD.bazel b/crates/tracedecay-global-db/BUILD.bazel index e8c2a40126..a9260dbb67 100644 --- a/crates/tracedecay-global-db/BUILD.bazel +++ b/crates/tracedecay-global-db/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -95,7 +95,7 @@ rust_library( rust_test( name = "registered_schema_fail_closed", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/registered_schema_fail_closed.rs", edition = crate_edition(), version = "0.1.0", @@ -110,7 +110,7 @@ rust_test( rust_test( name = "session_relation_graph", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/session_relation_graph.rs", edition = crate_edition(), version = "0.1.0", @@ -125,6 +125,8 @@ rust_test( rust_test( name = "lcm_expansion", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/lcm_expansion.rs", edition = crate_edition(), @@ -140,6 +142,8 @@ rust_test( rust_test( name = "session_activity_reads", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/session_activity_reads.rs", edition = crate_edition(), @@ -155,6 +159,8 @@ rust_test( rust_test( name = "stack_delivery_batch", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/stack_delivery_batch.rs", edition = crate_edition(), @@ -168,6 +174,12 @@ rust_test( data = glob(["benches/**", "assets/**", "fixtures/**", "data/**", "resources/**", "vendor/**", "tests/**"], allow_empty = True) + ["//tests:fixtures"], ) +filegroup( + name = "benchmark_binaries", + testonly = True, + srcs = [":lcm_expansion__binary", ":session_activity_reads__binary", ":stack_delivery_batch__binary"], +) + rust_test( name = "unit_test", crate = ":tracedecay-global-db__+test-helpers__dev", @@ -182,3 +194,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["benches/**/*.rs"]) + glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-global-db/benches/lcm_expansion.rs b/crates/tracedecay-global-db/benches/lcm_expansion.rs index f7a63cbb2d..fca3a34a82 100644 --- a/crates/tracedecay-global-db/benches/lcm_expansion.rs +++ b/crates/tracedecay-global-db/benches/lcm_expansion.rs @@ -17,9 +17,12 @@ use tracedecay_lcm::{ LcmContentSlice, LcmExpandRequest, LcmExpandResponse, LcmExpandTarget, LcmSourceRef, LcmSummaryNodeDraft, }; -use tracedecay_session_temporal_store::RegisteredGlobalDbSessionTemporalExecution; +use tracedecay_session_temporal_store::{ + RegisteredGlobalDbSessionTemporalExecution, SessionTemporalStore, +}; use tracedecay_store::{ AnchoredObservationWrite, ObservationProjectionStore, ObservationStore, ObservationWrite, + SessionRetrievalStore, SessionTemporalSnapshotRequestV1, build_observation_resolution_authorization_v1, build_observation_retrieval_anchor, }; use tracedecay_temporal_query::execution::{BindingDigest, ExecutionControl}; @@ -261,12 +264,20 @@ async fn build_fixture() -> Fixture { .expect("publish benchmark leaf through production authority"); expected_source_content.push(leaf_summary_text(index)); } - let root_receipt = database + database .lcm_publish_immutable_summary_guarded(root_publication(), &publication_control, || Ok(())) .await .expect("publish benchmark root through production authority"); - let generation = u64::try_from(root_receipt.generation) - .expect("published benchmark generation is non-negative"); + let temporal = SessionTemporalStore::new(database); + temporal + .materialize_pending_session_refresh_for_test(&session_id) + .await + .expect("publish benchmark temporal rows and summary relations"); + let frozen = temporal + .freeze_session_temporal_snapshot(SessionTemporalSnapshotRequestV1::new(session_id.clone())) + .await + .expect("freeze benchmark canonical temporal watermarks"); + let watermarks = frozen.watermarks(); let snapshot = TemporalExecutionSnapshot::new_authorized( TemporalSnapshotRequest::new( session_id, @@ -278,11 +289,11 @@ async fn build_fixture() -> Fixture { ) .expect("benchmark temporal request is valid"), TemporalWatermarks { - generation, - source: 0, - projection: 0, - index: 0, - summary: generation, + generation: watermarks.active_generation().value(), + source: watermarks.source_frontier(), + projection: watermarks.projection_frontier(), + index: watermarks.projection_frontier(), + summary: watermarks.summary_frontier(), }, KernelVersions { schema: 1, diff --git a/crates/tracedecay-global-db/src/registered_source_rewrite.rs b/crates/tracedecay-global-db/src/registered_source_rewrite.rs index b59ddf4ecb..2393bc63e3 100644 --- a/crates/tracedecay-global-db/src/registered_source_rewrite.rs +++ b/crates/tracedecay-global-db/src/registered_source_rewrite.rs @@ -15,6 +15,12 @@ use tracedecay_store::{ProjectionStoreError, ProjectionStoreResult}; use super::RegisteredGlobalDb; use super::observation_projection::{ensure_projection_output_state_cache, retire_source_record}; +/// Unoffered-record reads are keyset-paginated at this page size. The exact- +/// SQL engine refuses a query that materializes more rows than its hard cap, +/// and one long host session can hold more unoffered records than that cap, +/// which made rewrite completion fail deterministically for those sources. +const UNOFFERED_RECORD_PAGE_SIZE: i64 = 1_000; + /// One record a source layout offered, under the identity admission derived. pub struct ObservationSourcePresenceV1 { pub observation_id: CanonicalObservationIdV1, @@ -207,31 +213,57 @@ impl RegisteredGlobalDb { .map_err(|error| storage("release source rewrite completion", error))?; return Ok(0); } - let mut rows = transaction - .query( - "SELECT observation_id FROM observation_source_presence - WHERE source_key = ?1 AND generation <> ?2", - params![key.as_str(), generation.as_str()], - ) - .await - .map_err(|error| storage("read unoffered source records", error))?; - let mut unoffered = Vec::new(); - while let Some(row) = rows - .next() - .await - .map_err(|error| storage("read unoffered source records", error))? - { - unoffered.push( - row.get::(0) - .map_err(|error| storage("read unoffered source records", error))?, - ); - } - drop(rows); ensure_projection_output_state_cache(&transaction).await?; let mut retired = 0_u64; - for observation_id in &unoffered { - if retire_source_record(&transaction, observation_id).await? { - retired += 1; + // Keyset-paginate the unoffered ids instead of materializing the whole + // set in one statement: a single rewritten source can hold more + // unoffered records than the engine's per-query materialization cap, + // and one oversized read wedged every completion attempt forever. + // `retire_source_record` never touches `observation_source_presence`, + // so the ordered id cursor stays stable across pages; the settle + // deletes below remain the authoritative cleanup. + let mut after_observation_id = String::new(); + loop { + let mut rows = transaction + .query( + "SELECT observation_id FROM observation_source_presence + WHERE source_key = ?1 AND generation <> ?2 + AND observation_id > ?3 + ORDER BY observation_id + LIMIT ?4", + params![ + key.as_str(), + generation.as_str(), + after_observation_id, + UNOFFERED_RECORD_PAGE_SIZE + ], + ) + .await + .map_err(|error| storage("read unoffered source records", error))?; + let mut page = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|error| storage("read unoffered source records", error))? + { + page.push( + row.get::(0) + .map_err(|error| storage("read unoffered source records", error))?, + ); + } + drop(rows); + if page.is_empty() { + break; + } + let page_len = page.len(); + after_observation_id = page[page_len - 1].clone(); + for observation_id in &page { + if retire_source_record(&transaction, observation_id).await? { + retired += 1; + } + } + if (page_len as i64) < UNOFFERED_RECORD_PAGE_SIZE { + break; } } for sql in [ diff --git a/crates/tracedecay-global-db/src/tests.rs b/crates/tracedecay-global-db/src/tests.rs index dc9488ff86..f9602f7467 100644 --- a/crates/tracedecay-global-db/src/tests.rs +++ b/crates/tracedecay-global-db/src/tests.rs @@ -17,6 +17,8 @@ mod payload_gc_cost; mod schema_install_cancellation; #[cfg(test)] mod session_sync; +#[cfg(test)] +mod source_rewrite; #[cfg(test)] use super::registered_analytics::ANALYTICS_INSERT_ROWS_PER_STATEMENT; diff --git a/crates/tracedecay-global-db/src/tests/source_rewrite.rs b/crates/tracedecay-global-db/src/tests/source_rewrite.rs new file mode 100644 index 0000000000..9881d2a643 --- /dev/null +++ b/crates/tracedecay-global-db/src/tests/source_rewrite.rs @@ -0,0 +1,322 @@ +//! Regression tests for registered source-rewrite completion. +//! +//! Rewrite completion retires every record the rewritten source no longer +//! offers. One long host session can hold more unoffered records than the +//! exact-SQL engine's per-query materialization cap, so completion must read +//! them in keyset pages rather than one unbounded statement; the unbounded +//! read failed deterministically for exactly those sources and wedged the +//! host catch-up in a retry loop that never converged. + +use tempfile::TempDir; + +use tracedecay_domain::{ + ObservationScopeV1, ObservationSourceGenerationV1, ObservationSourceIdentityV1, ProviderId, + SessionId, +}; +use tracedecay_runtime_core::db::engine::{QueryExecutor, params}; +use tracedecay_store::{ProjectionSkipReason, ProjectionStoreError}; + +use crate::ObservationSourcePresenceV1; +use crate::tests::harness::{ + HostAdmissionScope, HostAdmissionTestRuntimeV1, RegisteredGlobalDbHarness, + seed_projected_messages, +}; + +/// More unoffered records than one keyset page *and* more than the engine's +/// per-query materialization cap, so the test fails against the unbounded +/// read and exercises repeated pagination against the fix. +const UNOFFERED_RECORDS: i64 = 10_003; + +#[tokio::test] +async fn rewrite_completion_retires_unoffered_records_beyond_the_query_cap() { + let harness = RegisteredGlobalDbHarness::open("source-rewrite-unoffered-cap").await; + let source = + ObservationSourceIdentityV1::new(SessionId::new("session.rewrite-cap").unwrap()).unwrap(); + let scope = ObservationScopeV1::Profile; + let previous = ObservationSourceGenerationV1::new(1).unwrap(); + let generation = ObservationSourceGenerationV1::new(2).unwrap(); + + let key = serde_json::to_string(&(source.clone(), scope.clone())).unwrap(); + let writer = harness.registered.writer_connection().unwrap(); + for index in 0..UNOFFERED_RECORDS { + writer + .execute( + "INSERT INTO observation_source_presence ( + source_key, observation_id, generation, start_offset + ) VALUES (?1, ?2, ?3, 0)", + params![ + key.as_str(), + format!("sha256:unoffered-{index:06}"), + previous.generation_id().to_string() + ], + ) + .await + .unwrap(); + } + + harness + .registered + .begin_observation_source_rewrite(&source, &scope, previous, generation, 0) + .await + .unwrap(); + + // None of the seeded records are re-offered under the new generation, so + // completion must retire-path all of them. The `observations` table is + // empty, so each retire is a no-op returning false; what matters is that + // the reads stay under the materialization cap and the settle deletes + // still clear the source. + let retired = harness + .registered + .complete_observation_source_rewrite(&source, &scope, generation) + .await + .unwrap_or_else(|error| panic!("rewrite completion must not exceed query caps: {error}")); + assert_eq!(retired, 0); + + let mut remaining = writer + .query( + "SELECT COUNT(*) FROM observation_source_presence WHERE source_key = ?1", + params![key.as_str()], + ) + .await + .unwrap(); + let count = remaining.next().await.unwrap().unwrap(); + let presence: i64 = count.get(0).unwrap(); + assert_eq!(presence, 0, "unoffered presence rows must be settled"); + + let mut pending = writer + .query( + "SELECT COUNT(*) FROM observation_source_rewrites WHERE source_key = ?1", + params![key.as_str()], + ) + .await + .unwrap(); + let row = pending.next().await.unwrap().unwrap(); + let rewrites: i64 = row.get(0).unwrap(); + assert_eq!(rewrites, 0, "completed rewrite row must be removed"); +} + +async fn table_count(conn: &impl QueryExecutor, sql: &str) -> i64 { + let mut rows = conn.query(sql, ()).await.unwrap(); + let row = rows.next().await.unwrap().unwrap(); + row.get(0).unwrap() +} + +/// Retiring an unoffered record is real work, not a presence-row cleanup: +/// its provenance and workflow facts are removed, a `source_record_retired` +/// disposition takes their place, and a sole-owned projected output is +/// deleted. A completion that dies mid-transaction must roll every earlier +/// retirement back; the pending rewrite survives, so a retry after the +/// failure clears converges — the retry loop the paged read exists to keep +/// alive would otherwise stay wedged. +/// +/// The fixture plants an inconsistent provenance row (a surviving co-owner +/// that claims an output its durable record does not derive) on the +/// record the keyset order retires last, so the first attempt retires real +/// records across pages and then fails inside +/// `retire_source_record` with `ProvenanceCollision`. Removing the planted +/// row and completing again retires every unoffered record for real. +#[tokio::test] +async fn rewrite_completion_retires_projected_records_and_rolls_back_a_failed_attempt() { + // More real projected records than one keyset page, so a failed attempt + // leaves proven retirements to roll back on every page. + const RECORDS: usize = 1_005; + + let directory = TempDir::new().unwrap(); + let runtime = HostAdmissionTestRuntimeV1::profile(directory.path()) + .await + .unwrap(); + let registered = runtime + .registered_database(HostAdmissionScope::Profile) + .expect("registered profile database"); + let seeded = seed_projected_messages(&runtime, 0..RECORDS).await; + + let source = ObservationSourceIdentityV1::for_provider( + ProviderId::new("codex").unwrap(), + SessionId::new("session.audit-batch").unwrap(), + ) + .unwrap(); + let scope = ObservationScopeV1::Profile; + let previous = ObservationSourceGenerationV1::new(1).unwrap(); + let generation = ObservationSourceGenerationV1::new(2).unwrap(); + + let offered: Vec = seeded + .iter() + .map(|(observation, _)| ObservationSourcePresenceV1 { + observation_id: observation.observation_id().clone(), + generation: previous, + start_offset: observation.identity().position().start(), + }) + .collect(); + registered + .record_observation_source_presence(&source, &scope, &offered) + .await + .unwrap(); + registered + .begin_observation_source_rewrite(&source, &scope, previous, generation, 0) + .await + .unwrap(); + + // The rewrite re-offers the first record under the new generation; its + // presence, projection, and outputs must survive completion untouched. + let survivor = &seeded[0].0; + registered + .record_observation_source_presence( + &source, + &scope, + &[ObservationSourcePresenceV1 { + observation_id: survivor.observation_id().clone(), + generation, + start_offset: survivor.identity().position().start(), + }], + ) + .await + .unwrap(); + + // The unoffered record the keyset order retires last, plus the output + // its projection owns. + let collided_index = seeded + .iter() + .enumerate() + .skip(1) + .max_by(|left, right| { + left.1 + .0 + .observation_id() + .as_str() + .cmp(right.1.0.observation_id().as_str()) + }) + .map(|(index, _)| index) + .unwrap(); + let collided_message_id = format!("message.record.audit-batch-{collided_index}"); + + // Plant a provenance row making the re-offered record co-own the last + // record's output. When the last record retires, the surviving owner is + // read back and its stored rendering derives a different message, which + // the retire path refuses with `ProvenanceCollision`. + let writer = registered.writer_connection().unwrap(); + writer + .execute( + "INSERT INTO observation_projection_provenance ( + projector_version, observation_id, output_ordinal, receipt_id, + output_provider, output_message_id, output_digest, message_created, + retrieval_anchor_id + ) + SELECT projector_version, observation_id, 1, receipt_id, + output_provider, ?1, output_digest, message_created, + retrieval_anchor_id + FROM observation_projection_provenance + WHERE observation_id = ?2", + params![ + collided_message_id.as_str(), + survivor.observation_id().as_str() + ], + ) + .await + .unwrap(); + + let error = registered + .complete_observation_source_rewrite(&source, &scope, generation) + .await + .unwrap_err(); + assert!( + matches!(error, ProjectionStoreError::ProvenanceCollision), + "colliding output ownership must fail completion, got {error:?}" + ); + + // Every retirement the attempt already performed must be rolled back: + // provenance and projected outputs restored, no dispositions recorded, + // and the pending rewrite still waiting for a converging retry. + assert_eq!( + table_count( + &writer, + "SELECT COUNT(*) FROM observation_projection_dispositions" + ) + .await, + 0, + "retirements before the collision must roll back with the transaction" + ); + assert_eq!( + table_count( + &writer, + "SELECT COUNT(*) FROM observation_projection_provenance" + ) + .await, + i64::try_from(RECORDS).unwrap() + 1, + "rolled-back retirements must restore provenance" + ); + assert_eq!( + table_count(&writer, "SELECT COUNT(*) FROM lcm_raw_messages").await, + i64::try_from(RECORDS).unwrap(), + "rolled-back retirements must restore projected outputs" + ); + assert_eq!( + table_count(&writer, "SELECT COUNT(*) FROM observation_source_rewrites").await, + 1, + "the failed attempt must leave the rewrite pending" + ); + + writer + .execute( + "DELETE FROM observation_projection_provenance + WHERE observation_id = ?1 AND output_ordinal = 1", + params![survivor.observation_id().as_str()], + ) + .await + .unwrap(); + + let retired = registered + .complete_observation_source_rewrite(&source, &scope, generation) + .await + .unwrap_or_else(|error| { + panic!("retry after clearing the collision must converge: {error}") + }); + assert_eq!(retired, u64::try_from(RECORDS - 1).unwrap()); + + let retired_reason = ProjectionSkipReason::SourceRecordRetired.as_str(); + assert_eq!( + table_count( + &writer, + &format!( + "SELECT COUNT(*) FROM observation_projection_dispositions + WHERE reason = '{retired_reason}'" + ) + ) + .await, + i64::try_from(RECORDS - 1).unwrap(), + "every unoffered record must take a source_record_retired disposition" + ); + assert_eq!( + table_count( + &writer, + "SELECT COUNT(*) FROM observation_projection_provenance" + ) + .await, + 1, + "only the re-offered record's provenance survives" + ); + assert_eq!( + table_count(&writer, "SELECT COUNT(*) FROM lcm_raw_messages").await, + 1, + "only the re-offered record's projected output survives" + ); + assert_eq!( + table_count(&writer, "SELECT COUNT(*) FROM observation_source_presence").await, + 1, + "only the re-offered record's presence row survives" + ); + assert_eq!( + table_count(&writer, "SELECT COUNT(*) FROM observation_source_rewrites").await, + 0, + "completed rewrite row must be removed" + ); + assert_eq!( + table_count( + &writer, + "SELECT COUNT(*) FROM session_temporal_resets + WHERE session_id = 'session.audit-batch'" + ) + .await, + 1, + "retired records must request their session's temporal rebuild" + ); +} diff --git a/crates/tracedecay-graph-db/BUILD.bazel b/crates/tracedecay-graph-db/BUILD.bazel index 84fc32b408..39ee20c0ac 100644 --- a/crates/tracedecay-graph-db/BUILD.bazel +++ b/crates/tracedecay-graph-db/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -61,7 +61,7 @@ rust_library( rust_test( name = "at_rest_compact_mutation_probe", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/at_rest_compact_mutation_probe.rs", edition = crate_edition(), version = "0.1.0", @@ -76,7 +76,7 @@ rust_test( rust_test( name = "at_rest_grafeo_probe", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/at_rest_grafeo_probe.rs", edition = crate_edition(), version = "0.1.0", @@ -91,7 +91,7 @@ rust_test( rust_test( name = "at_rest_snapshot", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/at_rest_snapshot.rs", edition = crate_edition(), version = "0.1.0", @@ -106,7 +106,7 @@ rust_test( rust_test( name = "generation_cycle_rss", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/generation_cycle_rss.rs", edition = crate_edition(), version = "0.1.0", @@ -136,7 +136,7 @@ rust_test( rust_test( name = "replay_hydrate_probe", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/replay_hydrate_probe.rs", edition = crate_edition(), version = "0.1.0", @@ -151,7 +151,7 @@ rust_test( rust_test( name = "tiered_storage_rss", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/tiered_storage_rss.rs", edition = crate_edition(), version = "0.1.0", @@ -166,6 +166,8 @@ rust_test( rust_test( name = "code_traversal", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/code_traversal.rs", edition = crate_edition(), @@ -181,6 +183,8 @@ rust_test( rust_test( name = "mixed_read_write", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/mixed_read_write.rs", edition = crate_edition(), @@ -196,6 +200,8 @@ rust_test( rust_test( name = "native_state_100k", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/native_state_100k.rs", edition = crate_edition(), @@ -211,6 +217,8 @@ rust_test( rust_test( name = "publication_scale", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/publication_scale.rs", edition = crate_edition(), @@ -226,6 +234,8 @@ rust_test( rust_test( name = "replay_after_mismatch", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/replay_after_mismatch.rs", edition = crate_edition(), @@ -239,6 +249,12 @@ rust_test( data = glob(["benches/**", "assets/**", "fixtures/**", "data/**", "resources/**", "vendor/**", "tests/**"], allow_empty = True) + ["//tests:fixtures"], ) +filegroup( + name = "benchmark_binaries", + testonly = True, + srcs = [":code_traversal__binary", ":mixed_read_write__binary", ":native_state_100k__binary", ":publication_scale__binary", ":replay_after_mismatch__binary"], +) + rust_test( name = "unit_test", crate = ":tracedecay-graph-db__+eval-helpers_+test-helpers__dev", @@ -253,3 +269,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["benches/**/*.rs"]) + glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + glob(["tests/graph_db_suite/**/*.rs"]) + glob(["tests/support/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-graph-query/BUILD.bazel b/crates/tracedecay-graph-query/BUILD.bazel index 01a4dff1b1..53b25bcb0f 100644 --- a/crates/tracedecay-graph-query/BUILD.bazel +++ b/crates/tracedecay-graph-query/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -107,3 +107,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-graph-query/src/projection.rs b/crates/tracedecay-graph-query/src/projection.rs index 26a6bb12de..d5c165b781 100644 --- a/crates/tracedecay-graph-query/src/projection.rs +++ b/crates/tracedecay-graph-query/src/projection.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use thiserror::Error; use tracedecay_code_index::graph_projection::{ CodeGraphInteractiveReader, CodeGraphProjectionError, CodeGraphProjectionStore, + CodeGraphReadinessRequirement, }; pub use tracedecay_contracts::retrieval::CodeGraphReadFreshnessV1; use tracedecay_contracts::{ @@ -67,6 +68,7 @@ pub struct CodeGraphReadRequest<'a> { pub context: &'a RequestContext, pub observed_at: UtcMicros, pub cancellation: Arc, + pub readiness: CodeGraphReadinessRequirement, pub deadline: Option, pub live_cancellation: Option<&'a CancellationSignal>, } @@ -81,11 +83,17 @@ impl<'a> CodeGraphReadRequest<'a> { context, observed_at, cancellation, + readiness: CodeGraphReadinessRequirement::Catalog, deadline: None, live_cancellation: None, } } + pub fn with_readiness(mut self, readiness: CodeGraphReadinessRequirement) -> Self { + self.readiness = readiness; + self + } + pub fn with_deadline(mut self, deadline: Deadline) -> Self { self.deadline = Some(deadline); self diff --git a/crates/tracedecay-hooks/BUILD.bazel b/crates/tracedecay-hooks/BUILD.bazel index 4d2728d179..8271233b8b 100644 --- a/crates/tracedecay-hooks/BUILD.bazel +++ b/crates/tracedecay-hooks/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -44,7 +44,7 @@ rust_library( rust_test( name = "envelope_identity_allocations", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/envelope_identity_allocations.rs", edition = crate_edition(), version = "0.1.0", @@ -72,6 +72,24 @@ rust_test( data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) + filegroup( name = "fixtures_host_events", srcs = glob(["fixtures/host_events/**"]), diff --git a/crates/tracedecay-hooks/src/spool/mod.rs b/crates/tracedecay-hooks/src/spool/mod.rs index e4392d0fad..984c34ac7d 100644 --- a/crates/tracedecay-hooks/src/spool/mod.rs +++ b/crates/tracedecay-hooks/src/spool/mod.rs @@ -583,6 +583,31 @@ impl HookSpoolV1 { Ok(record) } + /// Prepare the exact records file before publishing a daemon binding. + /// No event is appended: the empty file and its directory entry become + /// durable here so the first callback only commits its own record data. + #[tracing::instrument(name = "hooks.spool.prepare", level = "trace", skip_all)] + pub fn prepare(self) -> Result<(), HookSpoolError> { + self.ensure_healthy()?; + if self.recovery_required { + return Err(HookSpoolError::RecoveryRequired); + } + // append_frame writes its bytes verbatim through append_unsynced; + // an empty slice creates the file without inventing a frame/sequence. + append_frame(&records_path(&self.root), &[])?; + let revision = + records_file_revision(&self.root)?.ok_or(HookSpoolError::MetadataCorrupted)?; + if self.observed_records_revision.is_none() { + // Opening a new spool checkpointed the absent records file. Bind + // that empty checkpoint to the file just created, so the first + // callback can reuse it rather than rebuilding under its lease. + write_checkpoint(&self.root, self.config, &self.pending, &self.records_prefix)?; + } + let root = self.root.clone(); + drop(self); + commit_records(&root, revision.identity, revision.length, COMMIT_WAIT) + } + /// Release the writer lease, then make every frame this handle appended or /// deduplicated against durable. Concurrent committers share one sync of /// the records file, so the lease covers only writing frames; a caller diff --git a/crates/tracedecay-hooks/src/spool/tests.rs b/crates/tracedecay-hooks/src/spool/tests.rs index efdbe49b21..467da8adc0 100644 --- a/crates/tracedecay-hooks/src/spool/tests.rs +++ b/crates/tracedecay-hooks/src/spool/tests.rs @@ -170,6 +170,62 @@ fn checkpoint_header_json_len(bytes: &[u8]) -> usize { ) as usize } +#[test] +fn preparation_commits_empty_records_and_reset_invalidates_its_extent() { + let root = TestDir::new("prepared-records"); + let (spool, report) = HookSpoolV1::open(&root.0, config(), UtcMicros(10)).unwrap(); + assert_eq!(report.next_sequence, 1); + spool.prepare().unwrap(); + let revision = records_file_revision(&root.0).unwrap().unwrap(); + assert_eq!(revision.length, 0); + assert_eq!( + CommitLockV1::acquire(&root.0, COMMIT_WAIT) + .unwrap() + .synced_through(revision.identity) + .unwrap(), + Some(0) + ); + let (mut spool, report) = HookSpoolV1::open(&root.0, config(), UtcMicros(11)).unwrap(); + assert_eq!((report.pending_records, report.next_sequence), (0, 1)); + assert!( + !report.checkpoint_rewritten, + "first callback must reuse the prepared records checkpoint" + ); + let barriers = tracedecay_private_fs::framed_log::sync_latency::inject(&root.0, Duration::ZERO); + let record = spool + .append(envelope(1, 9), &binding(), UtcMicros(11)) + .unwrap(); + assert_eq!(record.sequence, 1); + spool.commit().unwrap(); + assert_eq!( + barriers.syncs(), + 1, + "prepared capture only syncs record data" + ); + drop(barriers); + + HookSpoolV1::reset(&root.0, config(), UtcMicros(12)).unwrap(); + assert_eq!( + CommitLockV1::acquire(&root.0, COMMIT_WAIT) + .unwrap() + .synced_through(revision.identity) + .unwrap(), + None, + "reset must invalidate the prepared file's extent" + ); + let (spool, report) = HookSpoolV1::open(&root.0, config(), UtcMicros(13)).unwrap(); + assert_eq!((report.pending_records, report.next_sequence), (0, 1)); + spool.prepare().unwrap(); + let revision = records_file_revision(&root.0).unwrap().unwrap(); + assert_eq!( + CommitLockV1::acquire(&root.0, COMMIT_WAIT) + .unwrap() + .synced_through(revision.identity) + .unwrap(), + Some(0) + ); +} + #[test] fn checksum_is_real_sha256() { assert_eq!( diff --git a/crates/tracedecay-host-admission/BUILD.bazel b/crates/tracedecay-host-admission/BUILD.bazel index 74a9016a6b..7311301fbc 100644 --- a/crates/tracedecay-host-admission/BUILD.bazel +++ b/crates/tracedecay-host-admission/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -95,7 +95,7 @@ rust_library( rust_test( name = "host_capture_background_cpu_admission", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/host_capture_background_cpu_admission.rs", edition = crate_edition(), version = "0.1.0", @@ -122,3 +122,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-host-integration/BUILD.bazel b/crates/tracedecay-host-integration/BUILD.bazel index 47d58361dd..ef2076b7ab 100644 --- a/crates/tracedecay-host-integration/BUILD.bazel +++ b/crates/tracedecay-host-integration/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -56,3 +56,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-lcm/BUILD.bazel b/crates/tracedecay-lcm/BUILD.bazel index c568914227..8c8a1317c1 100644 --- a/crates/tracedecay-lcm/BUILD.bazel +++ b/crates/tracedecay-lcm/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -88,3 +88,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"]) + glob(["tests/lcm_suite/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-lsp/BUILD.bazel b/crates/tracedecay-lsp/BUILD.bazel index 735f9451e9..d4d038f332 100644 --- a/crates/tracedecay-lsp/BUILD.bazel +++ b/crates/tracedecay-lsp/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -88,3 +88,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"]) + glob(["tests/lsp_suite/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-maintenance/BUILD.bazel b/crates/tracedecay-maintenance/BUILD.bazel index faa7e29260..11a2698adf 100644 --- a/crates/tracedecay-maintenance/BUILD.bazel +++ b/crates/tracedecay-maintenance/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -90,3 +90,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-mcp-catalog/BUILD.bazel b/crates/tracedecay-mcp-catalog/BUILD.bazel index 60c34b3616..b7edf16eef 100644 --- a/crates/tracedecay-mcp-catalog/BUILD.bazel +++ b/crates/tracedecay-mcp-catalog/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -56,3 +56,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-mcp/BUILD.bazel b/crates/tracedecay-mcp/BUILD.bazel index 7fe0bb72dc..c9a899cf69 100644 --- a/crates/tracedecay-mcp/BUILD.bazel +++ b/crates/tracedecay-mcp/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -61,7 +61,7 @@ rust_library( rust_test( name = "broker_stream_transport", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/broker_stream_transport.rs", edition = crate_edition(), version = "0.1.0", @@ -76,7 +76,7 @@ rust_test( rust_test( name = "server_lifecycle", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/server_lifecycle.rs", edition = crate_edition(), version = "0.1.0", @@ -91,7 +91,7 @@ rust_test( rust_test( name = "server_protocol", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/server_protocol.rs", edition = crate_edition(), version = "0.1.0", @@ -106,7 +106,7 @@ rust_test( rust_test( name = "server_read_coalescing", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/server_read_coalescing.rs", edition = crate_edition(), version = "0.1.0", @@ -121,7 +121,7 @@ rust_test( rust_test( name = "server_rmcp", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/server_rmcp.rs", edition = crate_edition(), version = "0.1.0", @@ -136,7 +136,7 @@ rust_test( rust_test( name = "server_settlement", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/server_settlement.rs", edition = crate_edition(), version = "0.1.0", @@ -163,3 +163,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-mcp/src/handlers/dashboard_lcm.rs b/crates/tracedecay-mcp/src/handlers/dashboard_lcm.rs index f64d818932..61159cec75 100644 --- a/crates/tracedecay-mcp/src/handlers/dashboard_lcm.rs +++ b/crates/tracedecay-mcp/src/handlers/dashboard_lcm.rs @@ -11,7 +11,7 @@ use tracedecay_domain::{ use tracedecay_session_memory::session::{ SessionCursorRequest, SessionRetrievalScope, SessionTemporalQuery, }; -use tracedecay_temporal_query::context::ContextBudget; +use tracedecay_temporal_query::context::{ContextBudget, MAX_CONTEXT_RECORDS}; use tracedecay_temporal_query::ranking::DiversityLimits; use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; @@ -33,7 +33,6 @@ use tracedecay_session_runtime::session_retrieval::{ const SUMMARY_DESCRIBE_CONCURRENCY: usize = 8; const DASHBOARD_AGGREGATE_PAGE_LIMIT: usize = 16; -const ADMITTED_RETRIEVAL_PAGE_LIMIT: i64 = 100; const ADMITTED_RETRIEVAL_BYTE_LIMIT: usize = 64 * 1024; #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -923,8 +922,8 @@ fn retrieval_query( ) } }; - let limit = usize::try_from(limit.clamp(1, ADMITTED_RETRIEVAL_PAGE_LIMIT)).ok()?; - // The admitted application port caps each request at 100 records. Larger + let limit = usize::try_from(limit.max(1)).ok()?.min(MAX_CONTEXT_RECORDS); + // Selection must fit the canonical context assembler before hydration. Larger // dashboard windows advance only through its opaque cursor, preserving the // same immutable authorization and frozen participant manifest per page. let execution_limits = admitted_execution_limits(limit); @@ -1124,7 +1123,7 @@ mod tests { } #[test] - fn dashboard_session_page_preserves_temporal_cursor_and_exact_limit() { + fn dashboard_session_page_preserves_cursor_with_context_bounded_selection() { let request = DashboardLcmReadRequestV1::Session { session_id: "session.dashboard.cursor".to_owned(), limit: 100, @@ -1133,7 +1132,7 @@ mod tests { let query = retrieval_query(&request, initial_cursor(&request)) .expect("cursor-backed dashboard page"); - assert_eq!(query.limit(), 100); + assert_eq!(query.limit(), MAX_CONTEXT_RECORDS); assert_eq!(query.cursor(), Some("opaque-temporal-cursor")); assert!(query.semantic_filter().include_summaries); assert_eq!( @@ -1153,7 +1152,7 @@ mod tests { let query = retrieval_query(&request, Some("opaque-frozen-manifest-cursor".to_owned())) .expect("aggregate continuation"); - assert_eq!(query.limit(), 100); + assert_eq!(query.limit(), MAX_CONTEXT_RECORDS); assert_eq!(query.cursor(), Some("opaque-frozen-manifest-cursor")); assert_eq!( query.retrieval_scope(), diff --git a/crates/tracedecay-policy/BUILD.bazel b/crates/tracedecay-policy/BUILD.bazel index 4123f1252c..c29eff46f0 100644 --- a/crates/tracedecay-policy/BUILD.bazel +++ b/crates/tracedecay-policy/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -71,3 +71,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"]) + glob(["tests/policy_suite/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-privacy/BUILD.bazel b/crates/tracedecay-privacy/BUILD.bazel index 2e2e35eedc..b90efbdaf3 100644 --- a/crates/tracedecay-privacy/BUILD.bazel +++ b/crates/tracedecay-privacy/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -44,7 +44,7 @@ rust_library( rust_test( name = "search_cache_retention", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/search_cache_retention.rs", edition = crate_edition(), version = "0.1.0", @@ -71,3 +71,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-private-fs/BUILD.bazel b/crates/tracedecay-private-fs/BUILD.bazel index 7b824d1db8..2083784af0 100644 --- a/crates/tracedecay-private-fs/BUILD.bazel +++ b/crates/tracedecay-private-fs/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -44,7 +44,7 @@ rust_library( rust_test( name = "atomic_publication", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/atomic_publication.rs", edition = crate_edition(), version = "0.1.0", @@ -60,7 +60,7 @@ rust_test( rust_test( name = "framed_log", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/framed_log.rs", edition = crate_edition(), version = "0.1.0", @@ -88,3 +88,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-project/BUILD.bazel b/crates/tracedecay-project/BUILD.bazel index e56a7710ae..0f98722a1f 100644 --- a/crates/tracedecay-project/BUILD.bazel +++ b/crates/tracedecay-project/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -90,3 +90,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-project/src/test_support/host_admission.rs b/crates/tracedecay-project/src/test_support/host_admission.rs index 426a005ff2..93629f1173 100644 --- a/crates/tracedecay-project/src/test_support/host_admission.rs +++ b/crates/tracedecay-project/src/test_support/host_admission.rs @@ -402,6 +402,34 @@ impl HostAdmissionTestRuntimeV1 { Arc::clone(&self.session_registry) } + /// Joins this profile's owners and closes their stores before an offline + /// fixture mutation. Other runtimes sharing the profile must be dropped first. + pub async fn shutdown(self) -> Result<()> { + let Self { + session_registry, + profile_database, + profile_registered, + project_registered, + _database_scope: database_scope, + .. + } = self; + drop((profile_database, profile_registered, project_registered)); + session_registry.cancel_store_opens_for_shutdown(); + session_registry.cancel_terminal_tasks(); + session_registry.cancel_memory_graph_reconciliation_tasks(); + let terminal = session_registry.shutdown_terminal_tasks().await; + let reconciliation = session_registry + .shutdown_memory_graph_reconciliation_tasks() + .await; + terminal.map_err(|message| TraceDecayError::Config { message })?; + reconciliation.map_err(|message| TraceDecayError::Config { message })?; + session_registry + .close_retained_graph_runtimes_for_shutdown() + .await?; + drop((session_registry, database_scope)); + Ok(()) + } + #[doc(hidden)] pub async fn checkpoint_session_database_for_test( &self, diff --git a/crates/tracedecay-query/BUILD.bazel b/crates/tracedecay-query/BUILD.bazel index 534e7e5a73..b0cb4db3fc 100644 --- a/crates/tracedecay-query/BUILD.bazel +++ b/crates/tracedecay-query/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -191,7 +191,7 @@ rust_binary( rust_test( name = "canonical_execution_equivalence", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/canonical_execution_equivalence.rs", edition = crate_edition(), version = "0.1.0", @@ -208,7 +208,7 @@ rust_test( rust_test( name = "index_bench_workload", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/index_bench_workload.rs", edition = crate_edition(), version = "0.1.0", @@ -225,7 +225,7 @@ rust_test( rust_test( name = "retrieval_contract_spine", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/retrieval_contract_spine.rs", edition = crate_edition(), version = "0.1.0", @@ -257,6 +257,8 @@ rust_test( rust_test( name = "code_lexical_catchup", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/code_lexical_catchup.rs", edition = crate_edition(), @@ -270,6 +272,12 @@ rust_test( data = glob(["benches/**", "assets/**", "fixtures/**", "data/**", "resources/**", "vendor/**", "tests/**"], allow_empty = True) + ["//tests:fixtures"], ) +filegroup( + name = "benchmark_binaries", + testonly = True, + srcs = [":code_lexical_catchup__binary"], +) + rust_test( name = "unit_test", crate = ":tracedecay-query__+production_+test-helpers__dev", @@ -284,3 +292,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["benches/**/*.rs"]) + glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + glob(["tests/search_quality_suite/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-runtime-core/BUILD.bazel b/crates/tracedecay-runtime-core/BUILD.bazel index 8da14bc252..92e9855fd7 100644 --- a/crates/tracedecay-runtime-core/BUILD.bazel +++ b/crates/tracedecay-runtime-core/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -61,7 +61,7 @@ rust_library( rust_test( name = "git_discovery_authority", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/git_discovery_authority.rs", edition = crate_edition(), version = "0.1.0", @@ -76,7 +76,7 @@ rust_test( rust_test( name = "git_repository_authority", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/git_repository_authority.rs", edition = crate_edition(), version = "0.1.0", @@ -106,7 +106,7 @@ rust_test( rust_test( name = "sqlite_foreign_snapshot_cost", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/sqlite_foreign_snapshot_cost.rs", edition = crate_edition(), version = "0.1.0", @@ -121,7 +121,7 @@ rust_test( rust_test( name = "tracing_capture_keepalive", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/tracing_capture_keepalive.rs", edition = crate_edition(), version = "0.1.0", @@ -136,6 +136,8 @@ rust_test( rust_test( name = "exact_sql_writer_workload", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/exact_sql_writer_workload.rs", edition = crate_edition(), @@ -151,6 +153,8 @@ rust_test( rust_test( name = "git_repository_authority_bin", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/git_repository_authority.rs", edition = crate_edition(), @@ -164,6 +168,12 @@ rust_test( data = glob(["benches/**", "assets/**", "fixtures/**", "data/**", "resources/**", "vendor/**", "tests/**"], allow_empty = True) + ["//tests:fixtures"], ) +filegroup( + name = "benchmark_binaries", + testonly = True, + srcs = [":exact_sql_writer_workload__binary", ":git_repository_authority_bin__binary"], +) + rust_test( name = "unit_test", crate = ":tracedecay-runtime-core__+test-helpers_+test-transport__dev", @@ -178,3 +188,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["benches/**/*.rs"]) + glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + glob(["tests/runtime_core_suite/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-rusqlite-runtime/BUILD.bazel b/crates/tracedecay-rusqlite-runtime/BUILD.bazel index afacce46b2..8150ded139 100644 --- a/crates/tracedecay-rusqlite-runtime/BUILD.bazel +++ b/crates/tracedecay-rusqlite-runtime/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -44,7 +44,7 @@ rust_library( rust_test( name = "rusqlite_suite", - srcs = glob(["tests/rusqlite_suite/**/*.rs"]) + glob(["tests/support/**/*.rs"]) + ["//tests:storage_runtime_rusqlite_suite"], + srcs = glob(["tests/rusqlite_suite/**/*.rs"]) + glob(["tests/support/**/*.rs"]) + ["//tests:storage_runtime_rusqlite_suite_sources"], crate_root = "tests/rusqlite_suite/main.rs", edition = crate_edition(), version = "0.1.0", @@ -59,7 +59,7 @@ rust_test( rust_test( name = "work_product_read_scaling", - srcs = glob(["tests/**/*.rs"]) + ["//tests:storage_runtime_rusqlite_suite"], + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + ["//tests:storage_runtime_rusqlite_suite_sources"], crate_root = "tests/work_product_read_scaling.rs", edition = crate_edition(), version = "0.1.0", @@ -86,3 +86,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + ["//tests:storage_runtime_rusqlite_suite_sources"] + glob(["tests/rusqlite_suite/**/*.rs"]) + glob(["tests/support/**/*.rs"]) + ["//tests:storage_runtime_rusqlite_suite_sources"]).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-rusqlite-runtime/tests/work_product_read_scaling.rs b/crates/tracedecay-rusqlite-runtime/tests/work_product_read_scaling.rs index d1f09bb18d..f02528946b 100644 --- a/crates/tracedecay-rusqlite-runtime/tests/work_product_read_scaling.rs +++ b/crates/tracedecay-rusqlite-runtime/tests/work_product_read_scaling.rs @@ -1,4 +1,4 @@ -//! Current-head read scaling runs alone so other storage tests cannot skew its ratio. +//! Compare current-head reads against short and long published histories. #[path = "support/work_product_graph.rs"] mod work_product_graph; @@ -6,15 +6,13 @@ mod work_product_graph; pub mod work_registered_store; use tracedecay_contracts::{AddWorkTaskRequestV1, WorkGraphReadV1, WorkProductExpectedAuthorityV1}; -use tracedecay_domain::UtcMicros; +use tracedecay_domain::{UtcMicros, WorkGraphVersionV1}; use work_product_graph::{ binding, context, create, item, mutation, mutations, read_current, repository_selection, }; use work_registered_store::RegisteredWorkStore; -/// Publish `versions` graph versions, one task per version, and return the -/// fastest of several current reads of the head. -fn fastest_current_read_after(versions: u64) -> std::time::Duration { +fn published_history(versions: u64) -> (RegisteredWorkStore, WorkGraphVersionV1) { let store = RegisteredWorkStore::start("work-product-long-history"); let mut head = create( &store, @@ -43,27 +41,24 @@ fn fastest_current_read_after(versions: u64) -> std::time::Duration { ) .expect("publish the next graph version"); } - (0..5) - .map(|_| { - let started = std::time::Instant::now(); - let WorkGraphReadV1::Current { snapshot, .. } = - read_current(&store).expect("read the current head") - else { - panic!("a current read must answer with a current snapshot"); - }; - let elapsed = started.elapsed(); - assert_eq!( - snapshot.graph().version(), - head.verified_graph_version().graph_version() - ); - assert_eq!( - snapshot.graph().items().len(), - usize::try_from(versions).expect("versions fit") - ); - elapsed - }) - .min() - .expect("at least one read") + let version = head.verified_graph_version().graph_version(); + (store, version) +} + +fn current_read(store: &RegisteredWorkStore, version: WorkGraphVersionV1) -> std::time::Duration { + let started = std::time::Instant::now(); + let WorkGraphReadV1::Current { snapshot, .. } = + read_current(store).expect("read the current head") + else { + panic!("a current read must answer with a current snapshot"); + }; + let elapsed = started.elapsed(); + assert_eq!(snapshot.graph().version(), version); + assert_eq!( + snapshot.graph().items().len(), + usize::try_from(version.get()).expect("versions fit") + ); + elapsed } /// Every Work mutation reads its current head first, so a current read that @@ -75,9 +70,27 @@ fn fastest_current_read_after(versions: u64) -> std::time::Duration { /// about 136x, so 48x separates them by roughly threefold either way. #[test] fn a_current_read_folds_a_long_history_once_rather_than_once_per_version() { - let short = fastest_current_read_after(16); - let long = fastest_current_read_after(128); - let growth = long.as_secs_f64() / short.as_secs_f64(); + let (short_store, short_version) = published_history(16); + let (long_store, long_version) = published_history(128); + current_read(&short_store, short_version); + current_read(&long_store, long_version); + + // Prepare both histories before timing so publication and changing compiler + // load do not separate the short and long measurement windows. + let mut samples: Vec<_> = (0..5) + .map(|sample| { + let (short, long) = if sample % 2 == 0 { + let short = current_read(&short_store, short_version); + (short, current_read(&long_store, long_version)) + } else { + let long = current_read(&long_store, long_version); + (current_read(&short_store, short_version), long) + }; + (long.as_secs_f64() / short.as_secs_f64(), short, long) + }) + .collect(); + samples.sort_by(|left, right| left.0.total_cmp(&right.0)); + let (growth, short, long) = samples[samples.len() / 2]; assert!( growth < 48.0, "an 8x longer history made a current read {growth:.0}x slower \ diff --git a/crates/tracedecay-sdk/BUILD.bazel b/crates/tracedecay-sdk/BUILD.bazel index 4fe803b47e..13bd736257 100644 --- a/crates/tracedecay-sdk/BUILD.bazel +++ b/crates/tracedecay-sdk/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -44,7 +44,7 @@ rust_library( rust_test( name = "remote_client_proxy", - srcs = glob(["tests/**/*.rs"]) + ["//tests:support"], + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + ["//tests:support_sources"], crate_root = "tests/remote_client_proxy.rs", edition = crate_edition(), version = "0.1.0", @@ -60,7 +60,7 @@ rust_test( rust_test( name = "sdk_suite", - srcs = glob(["tests/sdk_suite/**/*.rs"]) + ["//tests:support"], + srcs = glob(["tests/sdk_suite/**/*.rs"]) + ["//tests:support_sources"], crate_root = "tests/sdk_suite/main.rs", edition = crate_edition(), version = "0.1.0", @@ -92,3 +92,21 @@ rust_test( exports_files(["src/bin/generate.rs", "src/codegen.rs", "src/operations.rs"], visibility = ["//visibility:public"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + ["//tests:support_sources"] + glob(["tests/sdk_suite/**/*.rs"]) + ["//tests:support_sources"]).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-search-eval/BUILD.bazel b/crates/tracedecay-search-eval/BUILD.bazel index 5b1412e47d..6ab3b9bcad 100644 --- a/crates/tracedecay-search-eval/BUILD.bazel +++ b/crates/tracedecay-search-eval/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -148,3 +148,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-session-memory/BUILD.bazel b/crates/tracedecay-session-memory/BUILD.bazel index 4fb7b0fb91..a3c6c5b1cc 100644 --- a/crates/tracedecay-session-memory/BUILD.bazel +++ b/crates/tracedecay-session-memory/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -108,6 +108,24 @@ rust_test( data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) + filegroup( name = "src", srcs = glob(["src/**"]), diff --git a/crates/tracedecay-session-runtime/BUILD.bazel b/crates/tracedecay-session-runtime/BUILD.bazel index 7958e3b5da..ef5b2164c5 100644 --- a/crates/tracedecay-session-runtime/BUILD.bazel +++ b/crates/tracedecay-session-runtime/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -78,7 +78,7 @@ rust_library( rust_test( name = "session_backlog_read_cost", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/session_backlog_read_cost.rs", edition = crate_edition(), version = "0.1.0", @@ -93,7 +93,7 @@ rust_test( rust_test( name = "session_drain_read_cost", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/session_drain_read_cost.rs", edition = crate_edition(), version = "0.1.0", @@ -108,7 +108,7 @@ rust_test( rust_test( name = "session_idle_history_commits", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/session_idle_history_commits.rs", edition = crate_edition(), version = "0.1.0", @@ -123,7 +123,7 @@ rust_test( rust_test( name = "session_ingest_commit_cost", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/session_ingest_commit_cost.rs", edition = crate_edition(), version = "0.1.0", @@ -138,7 +138,7 @@ rust_test( rust_test( name = "session_ingest_index_read_cost", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/session_ingest_index_read_cost.rs", edition = crate_edition(), version = "0.1.0", @@ -153,7 +153,7 @@ rust_test( rust_test( name = "session_live_ingest_work", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/session_live_ingest_work.rs", edition = crate_edition(), version = "0.1.0", @@ -168,7 +168,7 @@ rust_test( rust_test( name = "session_refresh_read_cost", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/session_refresh_read_cost.rs", edition = crate_edition(), version = "0.1.0", @@ -183,7 +183,7 @@ rust_test( rust_test( name = "session_retention_read_cost", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/session_retention_read_cost.rs", edition = crate_edition(), version = "0.1.0", @@ -198,7 +198,7 @@ rust_test( rust_test( name = "session_retention_write_cost", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/session_retention_write_cost.rs", edition = crate_edition(), version = "0.1.0", @@ -213,7 +213,7 @@ rust_test( rust_test( name = "session_wal_streaming", - srcs = glob(["tests/**/*.rs"]), + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]), crate_root = "tests/session_wal_streaming.rs", edition = crate_edition(), version = "0.1.0", @@ -240,3 +240,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"]) + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-session-runtime/src/lcm_summary_convergence.rs b/crates/tracedecay-session-runtime/src/lcm_summary_convergence.rs index 7c0e0124dd..5d87545c05 100644 --- a/crates/tracedecay-session-runtime/src/lcm_summary_convergence.rs +++ b/crates/tracedecay-session-runtime/src/lcm_summary_convergence.rs @@ -136,6 +136,8 @@ async fn backfill_queue( // An idle pass must not take the writer. The probe is only a hint: the // page re-reads the authoritative frontier under the writer transaction. let snapshot = database + .read_connection() + .background() .read_snapshot() .await .map_err(|error| LcmError::Db(error.to_string()))?; @@ -171,6 +173,8 @@ async fn requeue_parked_sessions( ) -> Result { let binding = crate::lcm_summarization::summarizer_binding_identity(database)?; let snapshot = database + .read_connection() + .background() .read_snapshot() .await .map_err(|error| LcmError::Db(error.to_string()))?; @@ -208,6 +212,8 @@ async fn rewrite_predecessor_ranges( database: &RegisteredGlobalDbLeaseV1, ) -> Result { let snapshot = database + .read_connection() + .background() .read_snapshot() .await .map_err(|error| LcmError::Db(error.to_string()))?; @@ -238,6 +244,8 @@ async fn load_candidate( now_unix_ms: i64, ) -> Result, LcmError> { let snapshot = database + .read_connection() + .background() .read_snapshot() .await .map_err(|error| LcmError::Db(error.to_string()))?; @@ -250,6 +258,8 @@ async fn load_session_candidate( session_id: &str, ) -> Result, LcmError> { let snapshot = database + .read_connection() + .background() .read_snapshot() .await .map_err(|error| LcmError::Db(error.to_string()))?; @@ -259,6 +269,8 @@ async fn load_session_candidate( async fn load_next_retry(database: &RegisteredGlobalDbLeaseV1) -> Result, LcmError> { let snapshot = database + .read_connection() + .background() .read_snapshot() .await .map_err(|error| LcmError::Db(error.to_string()))?; @@ -621,8 +633,48 @@ fn duration_millis(duration: Duration) -> Result { #[cfg(test)] mod tests { - use super::is_retryable; + use super::{is_retryable, load_candidate}; + use tracedecay_global_db::tests::harness::RegisteredGlobalDbHarness; use tracedecay_lcm::LcmError; + use tracedecay_runtime_core::db::engine::{Error as EngineError, params}; + + #[tokio::test] + async fn convergence_reads_leave_foreground_reader_capacity_available() { + let harness = RegisteredGlobalDbHarness::open("lcm-convergence-reader-admission").await; + let database = &harness.registered; + assert!(load_candidate(database, 0).await.unwrap().is_none()); + + let background = database.read_connection().background(); + let mut held = Vec::new(); + loop { + match background.read_snapshot().await { + Ok(snapshot) => held.push(snapshot), + Err(EngineError::Runtime(reason)) => { + assert_eq!(reason, "exact SQL reader lane is saturated"); + break; + } + Err(error) => panic!("unexpected background admission failure: {error}"), + } + } + assert!(!held.is_empty(), "idle background reads must be admitted"); + + let foreground = database.read_snapshot().await.unwrap(); + let mut rows = foreground.query("SELECT 1", params![]).await.unwrap(); + assert_eq!( + rows.next().await.unwrap().unwrap().get::(0).unwrap(), + 1 + ); + drop(rows); + foreground.commit().await.unwrap(); + assert!( + matches!(load_candidate(database, 0).await, Err(LcmError::Db(reason)) if reason.contains("exact SQL reader lane is saturated")), + "background convergence must not consume the foreground reservation" + ); + for snapshot in held { + snapshot.commit().await.unwrap(); + } + assert!(load_candidate(database, 0).await.unwrap().is_none()); + } #[test] fn convergence_error_classification_preserves_retry_truth() { diff --git a/crates/tracedecay-session-runtime/src/session_retrieval/tests.rs b/crates/tracedecay-session-runtime/src/session_retrieval/tests.rs index 598ab7c1e9..1898dc6540 100644 --- a/crates/tracedecay-session-runtime/src/session_retrieval/tests.rs +++ b/crates/tracedecay-session-runtime/src/session_retrieval/tests.rs @@ -170,8 +170,34 @@ async fn seed_real_page_fixture_in_session( session_id: String, finalize: bool, ) -> RealPageFixture { - let message_id = format!("message.page.{rank:02}"); - let text = format!("canonical content {rank}"); + seed_real_page_fixtures_in_session( + database, + root, + rank..rank + 1, + provider, + session_id, + finalize, + ) + .await + .pop() + .expect("one seeded page fixture") +} + +async fn seed_real_page_fixtures_in_session( + database: &tracedecay_global_db::RegisteredGlobalDb, + root: &TemporalAuthorizedRoot, + ranks: std::ops::Range, + provider: String, + session_id: String, + finalize: bool, +) -> Vec { + // Match the host admission projector's bounded transaction window. + const BATCH_ITEMS: usize = 32; + let rank = ranks + .clone() + .next_back() + .expect("at least one fixture rank"); + let scalar = ranks.len() == 1; assert!( database .upsert_session(&SessionRecord { @@ -197,97 +223,144 @@ async fn seed_real_page_fixture_in_session( let session = SessionId::new(session_id.clone()).expect("session"); let source = ObservationSourceIdentityV1::for_provider(provider_id.clone(), session.clone()) .expect("observation source"); - let ordinal = u64::try_from(rank).expect("ordinal"); - let range = ObservationSourceRangeV1::new(ordinal, ordinal + 1).expect("source range"); - let record_id = ObservationId::new(format!("record.page.{rank:02}")).expect("record id"); - let projected_message_id = if provider == "claude" { - record_id.as_str().to_owned() - } else { - message_id.clone() - }; - let relations = CanonicalObservationRelationsV1::new(session) - .with_message_id(ObservationId::new(message_id.clone()).expect("message id")); - let envelope = CanonicalObservationEnvelopeV1::new( - provider_id, - "message", - record_id.clone(), - relations, - vec![CanonicalObservationFactV1::Message { - role: CanonicalMessageRoleV1::Assistant, - content: json!(text), - model: Some("fixture-model".to_owned()), - timestamp: Some(i64::try_from(rank).expect("timestamp")), - }], - CanonicalObservationEvidenceV1::new(ObservationOrderingDomainV1::SnapshotOrder, range), - ) - .expect("canonical message envelope"); - let payload = serde_json::to_value(envelope).expect("canonical payload"); - let observation = DurableObservationV1::new( - ObservationIdentityMaterialV1::for_native_record( - source, - ObservationScopeV1::Profile, - ObservationSourceGenerationV1::new(1).expect("generation"), - range, - ObservationOrderingDomainV1::SnapshotOrder, - record_id, - ) - .expect("observation identity"), - SanitizationReceiptV1::new( - SanitizationReceiptRefV1::new( - SanitizationReceiptId::new(format!("receipt.page.{rank:02}")).expect("receipt id"), - tracedecay_domain::ComponentVersion::new("sanitizer.page-fixture.v1") - .expect("sanitizer"), - ) - .expect("receipt reference"), - SanitizerDispositionV1::Accepted, - SensitivityV1::NonSensitive, - Some(PayloadReferenceV1::for_payload(&payload).expect("payload reference")), - ) - .expect("receipt"), - RetentionClass::new("retention.page-fixture").expect("retention"), - payload, - ) - .expect("durable observation"); let store = database.observation_store(); - let previous_cursor = store - .get_source_cursor(observation.source(), observation.scope()) + let mut previous_cursor = store + .get_source_cursor(&source, &ObservationScopeV1::Profile) .await .expect("source cursor"); - let next_cursor = ObservationSourceCursorV1::for_ordering( - observation.source().clone(), - observation.scope().clone(), - observation.identity().generation(), - observation.identity().ordering_domain(), - observation.identity().position().end(), - ) - .expect("next cursor"); - let write = ObservationWrite::new(observation.clone(), previous_cursor, next_cursor) + let last_rank = rank; + let mut pending_writes = Vec::new(); + let mut fixtures = Vec::with_capacity(ranks.len()); + for rank in ranks { + let message_id = format!("message.page.{rank:02}"); + let text = format!("canonical content {rank}"); + let ordinal = u64::try_from(rank).expect("ordinal"); + let range = ObservationSourceRangeV1::new(ordinal, ordinal + 1).expect("source range"); + let record_id = ObservationId::new(format!("record.page.{rank:02}")).expect("record id"); + let projected_message_id = if provider == "claude" { + record_id.as_str().to_owned() + } else { + message_id.clone() + }; + let relations = CanonicalObservationRelationsV1::new(session.clone()) + .with_message_id(ObservationId::new(message_id.clone()).expect("message id")); + let envelope = CanonicalObservationEnvelopeV1::new( + provider_id.clone(), + "message", + record_id.clone(), + relations, + vec![CanonicalObservationFactV1::Message { + role: CanonicalMessageRoleV1::Assistant, + content: json!(text), + model: Some("fixture-model".to_owned()), + timestamp: Some(i64::try_from(rank).expect("timestamp")), + }], + CanonicalObservationEvidenceV1::new(ObservationOrderingDomainV1::SnapshotOrder, range), + ) + .expect("canonical message envelope"); + let payload = serde_json::to_value(envelope).expect("canonical payload"); + let observation = DurableObservationV1::new( + ObservationIdentityMaterialV1::for_native_record( + source.clone(), + ObservationScopeV1::Profile, + ObservationSourceGenerationV1::new(1).expect("generation"), + range, + ObservationOrderingDomainV1::SnapshotOrder, + record_id, + ) + .expect("observation identity"), + SanitizationReceiptV1::new( + SanitizationReceiptRefV1::new( + SanitizationReceiptId::new(format!("receipt.page.{rank:02}")) + .expect("receipt id"), + tracedecay_domain::ComponentVersion::new("sanitizer.page-fixture.v1") + .expect("sanitizer"), + ) + .expect("receipt reference"), + SanitizerDispositionV1::Accepted, + SensitivityV1::NonSensitive, + Some(PayloadReferenceV1::for_payload(&payload).expect("payload reference")), + ) + .expect("receipt"), + RetentionClass::new("retention.page-fixture").expect("retention"), + payload, + ) + .expect("durable observation"); + let next_cursor = ObservationSourceCursorV1::for_ordering( + observation.source().clone(), + observation.scope().clone(), + observation.identity().generation(), + observation.identity().ordering_domain(), + observation.identity().position().end(), + ) + .expect("next cursor"); + let write = ObservationWrite::new( + observation.clone(), + previous_cursor.take(), + next_cursor.clone(), + ) .expect("observation write"); - let projection_generation = - ProjectionGenerationId::new("projection.page-fixture.v1").expect("projection generation"); - let authorization = build_observation_resolution_authorization_v1( - write.observation(), - tracedecay_store::OBSERVATION_CAPTURE_AUTHORITY_V1, - ) - .expect("resolution authorization"); - let anchor = build_observation_retrieval_anchor( - write.observation(), - projection_generation.clone(), - UtcMicros(1), - authorization, - ) - .expect("retrieval anchor"); - store - .persist_observation( - AnchoredObservationWrite::new(write, anchor, projection_generation) - .expect("anchored observation"), + let projection_generation = ProjectionGenerationId::new("projection.page-fixture.v1") + .expect("projection generation"); + let authorization = build_observation_resolution_authorization_v1( + write.observation(), + tracedecay_store::OBSERVATION_CAPTURE_AUTHORITY_V1, ) - .await - .expect("persist canonical observation"); - store - .project_observation(observation.observation_id()) - .await - .expect("project canonical observation"); + .expect("resolution authorization"); + let anchor = build_observation_retrieval_anchor( + write.observation(), + projection_generation.clone(), + UtcMicros(1), + authorization, + ) + .expect("retrieval anchor"); + previous_cursor = Some(next_cursor); + let write = AnchoredObservationWrite::new(write, anchor, projection_generation) + .expect("anchored observation"); + if scalar { + store + .persist_observation(write) + .await + .expect("persist canonical observation"); + store + .project_observation(observation.observation_id()) + .await + .expect("project canonical observation"); + } else { + pending_writes.push(write); + if pending_writes.len() == BATCH_ITEMS || rank == last_rank { + let count = pending_writes.len(); + let outcomes = store + .persist_observations(std::mem::take(&mut pending_writes)) + .await + .expect("persist canonical observation page"); + assert_eq!(outcomes.len(), count); + let projected = store + .project_queued_observations(count) + .await + .expect("project canonical observation page") + .expect("registered batched projector"); + assert_eq!(projected.items.len(), count); + assert!( + !projected.has_more, + "fixture page must fully project before the next page" + ); + } + } + fixtures.push(RealPageFixture { + provider: provider.clone(), + session_id: session_id.clone(), + message_id, + projected_message_id, + text, + anchor_id: derive_exact_observation_anchor_id( + observation.scope(), + observation.observation_id(), + ) + .expect("canonical anchor id"), + active_generation: 0, + }); + } let active_generation = if finalize { database .lcm_protect_session_raw_messages(&provider, &session_id) @@ -312,19 +385,10 @@ async fn seed_real_page_fixture_in_session( 0 }; - RealPageFixture { - provider, - session_id, - message_id, - projected_message_id, - text, - anchor_id: derive_exact_observation_anchor_id( - observation.scope(), - observation.observation_id(), - ) - .expect("canonical anchor id"), - active_generation, + for fixture in &mut fixtures { + fixture.active_generation = active_generation; } + fixtures } #[tokio::test] @@ -1556,19 +1620,48 @@ async fn small_lookup_reads_a_session_larger_than_the_response_budget() { .await; let root = real_page_root("root.page"); let session_id = "session.page.large".to_owned(); - let mut expected_messages = BTreeMap::new(); - for rank in 0..RECORDS { - let fixture = seed_real_page_fixture_in_session( + let mut fixtures = seed_real_page_fixtures_in_session( + harness.registered.as_ref(), + &root, + 0..RECORDS - 1, + "codex".to_owned(), + session_id.clone(), + false, + ) + .await; + // The scalar fixture replaces the session row before its last projection; + // retain that final metadata instead of merging all message timestamps. + fixtures.push( + seed_real_page_fixture_in_session( harness.registered.as_ref(), &root, - rank, + RECORDS - 1, "codex".to_owned(), session_id.clone(), - rank + 1 == RECORDS, + true, ) - .await; - expected_messages.insert(fixture.projected_message_id, fixture.text); - } + .await, + ); + let stored_session = harness + .registered + .get_session("codex", &session_id) + .await + .expect("fixture session read") + .expect("fixture session"); + assert_eq!(stored_session.started_at, Some(1499)); + assert_eq!(stored_session.ended_at, Some(1499)); + assert_eq!( + stored_session.title.as_deref(), + Some("fixture session 1499") + ); + assert_eq!( + stored_session.project_path, + tracedecay_sessions::runtime::shared::durable_project_path_key("/fixture/1499"), + ); + let expected_messages = fixtures + .into_iter() + .map(|fixture| (fixture.projected_message_id, fixture.text)) + .collect::>(); let root = registered_profile_retrieval_root(&harness.registered); let scope = root .identity() diff --git a/crates/tracedecay-session-temporal-store/BUILD.bazel b/crates/tracedecay-session-temporal-store/BUILD.bazel index 33929ba1c2..35d5ada7a9 100644 --- a/crates/tracedecay-session-temporal-store/BUILD.bazel +++ b/crates/tracedecay-session-temporal-store/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -107,3 +107,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-session-temporal-store/src/operations/generation.rs b/crates/tracedecay-session-temporal-store/src/operations/generation.rs index a91e6f68f7..7ab1cadfc9 100644 --- a/crates/tracedecay-session-temporal-store/src/operations/generation.rs +++ b/crates/tracedecay-session-temporal-store/src/operations/generation.rs @@ -735,10 +735,18 @@ mod tests { let mut affected = 0_usize; let mut final_rewind = None; for _ in 0..32 { - let page = - invalidate_raw_summary_revision(&conn, "cursor", "large-closure", 100, PAGE_LIMIT) - .await - .unwrap(); + // Production commits one bounded invalidation page atomically. + let transaction = conn.transaction().await.unwrap(); + let page = invalidate_raw_summary_revision( + &transaction, + "cursor", + "large-closure", + 100, + PAGE_LIMIT, + ) + .await + .unwrap(); + transaction.commit().await.unwrap(); assert!(page.stale_summary_count <= PAGE_LIMIT); affected = affected.saturating_add(page.stale_summary_count); final_rewind = Some(page.rewind_frontier_store_id); diff --git a/crates/tracedecay-session-temporal-store/src/projection/tests.rs b/crates/tracedecay-session-temporal-store/src/projection/tests.rs index ce07d4334f..5ad4ac4286 100644 --- a/crates/tracedecay-session-temporal-store/src/projection/tests.rs +++ b/crates/tracedecay-session-temporal-store/src/projection/tests.rs @@ -467,6 +467,265 @@ async fn relation_batch_persists_restarts_and_completes_without_duplicates() { } } +#[tokio::test] +async fn all_skipped_refresh_persists_reopens_and_completes_idempotently() { + let tmp = TempDir::new().unwrap(); + let session_id = fixture_session("session.projector.all-skipped"); + let request; + { + let runtime = HostAdmissionTestRuntimeV1::profile(tmp.path()) + .await + .unwrap(); + let (observation, write) = fixture_observation(&session_id, 0, None, false); + Box::pin(persist_fixture(&runtime, observation, write)).await; + let store = temporal_store(&runtime); + refresh_through(&store, &session_id, 1, 0).await; + for ordinal in 1..=2 { + let (observation, write) = fixture_observation_from_facts( + &session_id, + ordinal, + ProviderId::new(format!("projector-test-{ordinal}")).unwrap(), + ObservationId::new(format!("record.projector.{ordinal}")).unwrap(), + CanonicalObservationRelationsV1::new(session_id.clone()), + vec![CanonicalObservationFactV1::Boundary { + boundary_kind: tracedecay_domain::CanonicalBoundaryKindV1::TurnStart, + }], + None, + ); + assert_eq!( + tracedecay_store::derive_canonical_projection(&observation) + .unwrap() + .skip_reason(), + Some(ProjectionSkipReason::NonConversationalRecord) + ); + Box::pin(persist_fixture(&runtime, observation, write)).await; + } + store + .begin_or_join_session_refresh(SessionRefreshBeginOrJoinRequestV1::new( + session_id.clone(), + SessionRefreshFrontierV1::new(3, 1).unwrap(), + )) + .await + .unwrap(); + let recovery = store + .session_refresh_recovery(&session_id) + .await + .unwrap() + .unwrap(); + let (progress, batch) = store + .materialize_session_temporal_refresh_batch_for_test(&recovery) + .await + .unwrap() + .unwrap(); + assert!(batch.occurrences().is_empty()); + assert_eq!(batch.item_count(), 0); + assert_eq!(progress.frontier().committed_through(), 3); + request = SessionRefreshCompletionRequestV1::new( + progress.operation_id().clone(), + session_id.clone(), + progress.frontier(), + *progress.coverage(), + ) + .unwrap(); + store + .persist_session_refresh_projection_batch(progress, batch) + .await + .unwrap(); + } + let receipt = { + let runtime = HostAdmissionTestRuntimeV1::profile(tmp.path()) + .await + .unwrap(); + let store = temporal_store(&runtime); + let recovery = store + .session_refresh_recovery(&session_id) + .await + .unwrap() + .unwrap(); + assert_eq!( + recovery.restart_state(), + SessionRefreshRestartStateV1::ReadyToComplete + ); + let receipt = store + .complete_session_refresh(request.clone(), ExecutionControl::default()) + .await + .unwrap(); + assert_eq!(receipt.state(), SessionRefreshTerminalStateV1::Complete); + receipt + }; + let runtime = HostAdmissionTestRuntimeV1::profile(tmp.path()) + .await + .unwrap(); + assert_eq!( + temporal_store(&runtime) + .complete_session_refresh(request, ExecutionControl::default()) + .await + .unwrap(), + receipt + ); + for (kind, expected) in [ + (SessionTemporalFixtureCountV1::Occurrences, 1), + (SessionTemporalFixtureCountV1::RefreshReceipts, 2), + ] { + assert_eq!( + runtime + .session_temporal_fixture_count_for_test(HostAdmissionScope::Profile, kind) + .await + .unwrap(), + expected + ); + } +} + +#[tokio::test] +async fn refresh_at_committed_frontier_needs_no_new_observation_effect() { + let tmp = TempDir::new().unwrap(); + let runtime = HostAdmissionTestRuntimeV1::profile(tmp.path()) + .await + .unwrap(); + let session_id = fixture_session("session.projector.unchanged-frontier"); + let (observation, write) = fixture_observation(&session_id, 0, None, false); + Box::pin(persist_fixture(&runtime, observation, write)).await; + let store = temporal_store(&runtime); + let baseline = refresh_through(&store, &session_id, 1, 0).await; + let refreshed = refresh_through(&store, &session_id, 1, 1).await; + assert_ne!(refreshed, baseline); + assert_eq!( + runtime + .session_temporal_fixture_count_for_test( + HostAdmissionScope::Profile, + SessionTemporalFixtureCountV1::Occurrences + ) + .await + .unwrap(), + 1 + ); +} + +#[tokio::test] +async fn refresh_frontier_without_observation_effects_cannot_activate() { + assert_unsupported_frontier_cannot_activate(FrontierPrefix::Absent, false).await; +} + +#[tokio::test] +async fn skipped_prefix_cannot_prove_an_unsupported_target_frontier() { + for foreign_target in [false, true] { + assert_unsupported_frontier_cannot_activate(FrontierPrefix::Skipped, foreign_target).await; + } +} + +#[tokio::test] +async fn message_prefix_cannot_prove_an_unsupported_target_frontier() { + assert_unsupported_frontier_cannot_activate(FrontierPrefix::Message, false).await; +} + +enum FrontierPrefix { + Absent, + Skipped, + Message, +} + +async fn assert_unsupported_frontier_cannot_activate(prefix: FrontierPrefix, foreign_target: bool) { + let tmp = TempDir::new().unwrap(); + let runtime = HostAdmissionTestRuntimeV1::profile(tmp.path()) + .await + .unwrap(); + let session_id = fixture_session("session.projector.unsupported-frontier"); + let (observation, write) = fixture_observation(&session_id, 0, None, false); + Box::pin(persist_fixture(&runtime, observation, write)).await; + let store = temporal_store(&runtime); + refresh_through(&store, &session_id, 1, 0).await; + let target = if matches!(prefix, FrontierPrefix::Skipped) { + let (observation, write) = fixture_observation_from_facts( + &session_id, + 1, + ProviderId::new("projector-test-1").unwrap(), + ObservationId::new("record.projector.1").unwrap(), + CanonicalObservationRelationsV1::new(session_id.clone()), + vec![CanonicalObservationFactV1::Boundary { + boundary_kind: tracedecay_domain::CanonicalBoundaryKindV1::TurnStart, + }], + None, + ); + assert_eq!( + tracedecay_store::derive_canonical_projection(&observation) + .unwrap() + .skip_reason(), + Some(ProjectionSkipReason::NonConversationalRecord) + ); + Box::pin(persist_fixture(&runtime, observation, write)).await; + 3 + } else if matches!(prefix, FrontierPrefix::Message) { + let (observation, write) = fixture_observation(&session_id, 1, None, false); + Box::pin(persist_fixture(&runtime, observation, write)).await; + 3 + } else { + 2 + }; + if foreign_target { + let foreign_session = fixture_session("session.projector.foreign-frontier"); + let (observation, write) = fixture_observation(&foreign_session, 2, None, false); + Box::pin(persist_fixture(&runtime, observation, write)).await; + } + store + .begin_or_join_session_refresh(SessionRefreshBeginOrJoinRequestV1::new( + session_id.clone(), + SessionRefreshFrontierV1::new(target, 1).unwrap(), + )) + .await + .unwrap(); + let recovery = store + .session_refresh_recovery(&session_id) + .await + .unwrap() + .unwrap(); + let (progress, batch) = store + .materialize_session_temporal_refresh_batch_for_test(&recovery) + .await + .unwrap() + .unwrap(); + assert_eq!( + batch.occurrences().len(), + usize::from(matches!(prefix, FrontierPrefix::Message)) + ); + assert_eq!(progress.frontier().committed_through(), target); + let request = SessionRefreshCompletionRequestV1::new( + progress.operation_id().clone(), + session_id.clone(), + progress.frontier(), + *progress.coverage(), + ) + .unwrap(); + store + .persist_session_refresh_projection_batch(progress, batch) + .await + .unwrap(); + let error = store + .complete_session_refresh(request, ExecutionControl::default()) + .await + .expect_err("a prefix batch cannot prove an unsupported source frontier"); + assert!(matches!(error, SessionStoreError::Storage { .. })); + assert_eq!( + store + .session_refresh_recovery(&session_id) + .await + .unwrap() + .unwrap() + .restart_state(), + SessionRefreshRestartStateV1::ReadyToComplete + ); + assert_eq!( + runtime + .session_temporal_fixture_count_for_test( + HostAdmissionScope::Profile, + SessionTemporalFixtureCountV1::RefreshReceipts + ) + .await + .unwrap(), + 1 + ); +} + #[tokio::test] async fn terminal_receipt_rejects_corrupted_derived_evidence() { let tmp = TempDir::new().unwrap(); diff --git a/crates/tracedecay-session-temporal-store/src/rebuild.rs b/crates/tracedecay-session-temporal-store/src/rebuild.rs index 3f1652516b..614a6a49d7 100644 --- a/crates/tracedecay-session-temporal-store/src/rebuild.rs +++ b/crates/tracedecay-session-temporal-store/src/rebuild.rs @@ -232,11 +232,43 @@ pub(super) async fn validate_candidate_frontier( } expected.insert(occurrence_id.clone()); } - if expected.is_empty() && source_frontier != base_frontier { - return Err(storage_message( - ACTIVATE_OPERATION, - "candidate generation has no canonical message outputs past its base frontier", - )); + if source_frontier != base_frontier { + // Discovery targets an effect belonging to this session. Skipped + // effects prove an empty advance, but an earlier effect or another + // session's effect cannot prove the requested endpoint. + let mut target_effect = conn + .query( + "SELECT EXISTS ( + SELECT 1 FROM session_temporal_observation_effects + WHERE session_id = ?1 AND observation_sequence = ?2 + )", + params![ + session_id, + i64::try_from(source_frontier) + .map_err(|error| storage(ACTIVATE_OPERATION, error))? + ], + ) + .await + .map_err(|error| storage(ACTIVATE_OPERATION, error))?; + let supported = match target_effect + .next() + .await + .map_err(|error| storage(ACTIVATE_OPERATION, error))? + { + Some(row) => { + row.get::(0) + .map_err(|error| storage(ACTIVATE_OPERATION, error))? + != 0 + } + None => false, + }; + drop(target_effect); + if !supported { + return Err(storage_message( + ACTIVATE_OPERATION, + "candidate source frontier has no observation effect for this session", + )); + } } require_unsettled_message_ids(conn, session_id, generation, &parent_resolver).await?; diff --git a/crates/tracedecay-session-temporal-store/src/test_registered_impls.rs b/crates/tracedecay-session-temporal-store/src/test_registered_impls.rs index caf537f245..c83afa73e1 100644 --- a/crates/tracedecay-session-temporal-store/src/test_registered_impls.rs +++ b/crates/tracedecay-session-temporal-store/src/test_registered_impls.rs @@ -16,6 +16,7 @@ use tracedecay_global_db::{ use tracedecay_runtime_core::db::DatabaseEngineReadSnapshot; use tracedecay_runtime_core::db::engine::{ Connection, Error as EngineError, Executor, IntoParams, QueryExecutor, Rows, TestConnection, + Transaction, }; use tracedecay_store::StoreShardScopeV1; @@ -84,6 +85,36 @@ impl SessionTemporalExec for TestConnection { } } +impl SessionTemporalQuery for Transaction { + fn query

( + &self, + sql: &str, + params: P, + ) -> impl Future> + Send + where + P: IntoParams + Send, + { + Transaction::query(self, sql, params) + } +} + +impl SessionTemporalExec for Transaction { + fn execute

( + &self, + sql: &str, + params: P, + ) -> impl Future> + Send + where + P: IntoParams + Send, + { + Transaction::execute(self, sql, params) + } + + fn execute_batch(&self, sql: &str) -> impl Future> + Send { + Transaction::execute_batch(self, sql) + } +} + impl SessionTemporalQuery for RegisteredGlobalDbWriteTransaction<'_> { fn query

( &self, diff --git a/crates/tracedecay-sessions/BUILD.bazel b/crates/tracedecay-sessions/BUILD.bazel index 79a3fcfefe..2f971e4912 100644 --- a/crates/tracedecay-sessions/BUILD.bazel +++ b/crates/tracedecay-sessions/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -61,6 +61,8 @@ rust_library( rust_test( name = "cursor_dispatch_model", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/cursor_dispatch_model.rs", edition = crate_edition(), @@ -76,6 +78,8 @@ rust_test( rust_test( name = "host_transcript_io", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/host_transcript_io.rs", edition = crate_edition(), @@ -89,6 +93,12 @@ rust_test( data = glob(["benches/**", "assets/**", "fixtures/**", "data/**", "resources/**", "vendor/**", "tests/**"], allow_empty = True) + ["//tests:fixtures"], ) +filegroup( + name = "benchmark_binaries", + testonly = True, + srcs = [":cursor_dispatch_model__binary", ":host_transcript_io__binary"], +) + rust_test( name = "unit_test", crate = ":tracedecay-sessions__+test-helpers__dev", @@ -103,3 +113,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["benches/**/*.rs"]) + glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-source-edit/BUILD.bazel b/crates/tracedecay-source-edit/BUILD.bazel index 9fae365ead..837023281e 100644 --- a/crates/tracedecay-source-edit/BUILD.bazel +++ b/crates/tracedecay-source-edit/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -90,3 +90,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-store-runtime/BUILD.bazel b/crates/tracedecay-store-runtime/BUILD.bazel index 8fb9940478..985d3dae86 100644 --- a/crates/tracedecay-store-runtime/BUILD.bazel +++ b/crates/tracedecay-store-runtime/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -90,3 +90,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-store-runtime/src/session_registry/code_graph/layered_refresh_tests.rs b/crates/tracedecay-store-runtime/src/session_registry/code_graph/layered_refresh_tests.rs index 2ff64920ff..b269feffc7 100644 --- a/crates/tracedecay-store-runtime/src/session_registry/code_graph/layered_refresh_tests.rs +++ b/crates/tracedecay-store-runtime/src/session_registry/code_graph/layered_refresh_tests.rs @@ -22,6 +22,7 @@ use tracedecay_graph_db::{ GraphGenerationRowSpill, GraphGenerationRows, GraphProjectorRevision, NeverCancelled, VerifiedGraphSnapshot, }; +use tracedecay_runtime_core::path_safety::{plain_git_args, plain_host_path}; use super::super::DaemonSessionRuntimeRegistryV1; @@ -29,8 +30,8 @@ const MODULES: usize = 120; fn git(root: &Path, args: &[&str]) { let output = Command::new("git") - .args(args) - .current_dir(root) + .args(plain_git_args(args)) + .current_dir(plain_host_path(root)) .output() .expect("run git fixture command"); assert!( diff --git a/crates/tracedecay-store/BUILD.bazel b/crates/tracedecay-store/BUILD.bazel index 95674f0367..f33ca1c23d 100644 --- a/crates/tracedecay-store/BUILD.bazel +++ b/crates/tracedecay-store/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -71,3 +71,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"]) + glob(["test-support/**/*.rs"]) + glob(["tests/store_suite/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-temporal-query/BUILD.bazel b/crates/tracedecay-temporal-query/BUILD.bazel index aafe9a1776..530d249a24 100644 --- a/crates/tracedecay-temporal-query/BUILD.bazel +++ b/crates/tracedecay-temporal-query/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -56,3 +56,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay-temporal-query/src/context.rs b/crates/tracedecay-temporal-query/src/context.rs index 26f2685e44..ecbe830484 100644 --- a/crates/tracedecay-temporal-query/src/context.rs +++ b/crates/tracedecay-temporal-query/src/context.rs @@ -16,7 +16,9 @@ use super::resolution::summary::SummaryOmission; use crate::execution::TemporalPortError; const CANONICAL_CONTEXT_FORMAT: &str = "tracedecay.compact_context.v1"; -const MAX_CONTEXT_RECORDS: usize = 64; +/// Maximum hydrated records accepted by one compact context assembly. +/// Paged consumers must bound selection before hydration and retain continuation. +pub const MAX_CONTEXT_RECORDS: usize = 64; const MAX_CONTEXT_ANCHORS: usize = 256; const MAX_CONTEXT_FRAME_ITEMS: usize = 256; const MAX_CONTEXT_OUTPUT_BYTES: u64 = 1024 * 1024; diff --git a/crates/tracedecay-tool-catalog/BUILD.bazel b/crates/tracedecay-tool-catalog/BUILD.bazel index b6b81a25ba..e87f356ac6 100644 --- a/crates/tracedecay-tool-catalog/BUILD.bazel +++ b/crates/tracedecay-tool-catalog/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -71,3 +71,21 @@ rust_test( + ["//tests:fixtures"], data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"], ) + +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(glob(["src/**/*.rs"]) + glob(["tests/tool_catalog_suite/**/*.rs"])).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay/BUILD.bazel b/crates/tracedecay/BUILD.bazel index d16ba05942..0eb5a502a8 100644 --- a/crates/tracedecay/BUILD.bazel +++ b/crates/tracedecay/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py package(default_visibility = ["//visibility:public"]) @@ -10,7 +10,7 @@ load("//scripts:bazel/rust_test.bzl", "perf_rustc_flags", "rust_test") rust_library( name = "tracedecay", - srcs = glob(["src/**/*.rs"]) + glob(["benches/rmcp/**/*.rs", "benches/session_temporal/**/*.rs", "tests/common/**/*.rs"]) + ["//tests:support"], + srcs = glob(["src/**/*.rs"]) + glob(["benches/rmcp/**/*.rs", "benches/session_temporal/**/*.rs", "tests/common/**/*.rs"]) + ["//tests:support_sources"], crate_root = "src/lib.rs", crate_name = "tracedecay", edition = crate_edition(), @@ -27,7 +27,7 @@ rust_library( rust_library( name = "tracedecay__+rmcp-benchmark_+test-helpers_+test-transport__dev", - srcs = glob(["src/**/*.rs"]) + glob(["benches/rmcp/**/*.rs", "benches/session_temporal/**/*.rs", "tests/common/**/*.rs"]) + ["//tests:support"], + srcs = glob(["src/**/*.rs"]) + glob(["benches/rmcp/**/*.rs", "benches/session_temporal/**/*.rs", "tests/common/**/*.rs"]) + ["//tests:support_sources"], crate_root = "src/lib.rs", crate_name = "tracedecay", edition = crate_edition(), @@ -44,7 +44,7 @@ rust_library( rust_library( name = "tracedecay__-default__d2de7b7dcda", - srcs = glob(["src/**/*.rs"]) + glob(["benches/rmcp/**/*.rs", "benches/session_temporal/**/*.rs", "tests/common/**/*.rs"]) + ["//tests:support"], + srcs = glob(["src/**/*.rs"]) + glob(["benches/rmcp/**/*.rs", "benches/session_temporal/**/*.rs", "tests/common/**/*.rs"]) + ["//tests:support_sources"], crate_root = "src/lib.rs", crate_name = "tracedecay", edition = crate_edition(), @@ -61,7 +61,7 @@ rust_library( rust_library( name = "tracedecay__bare__d18d185dadf", - srcs = glob(["src/**/*.rs"]) + glob(["benches/rmcp/**/*.rs", "benches/session_temporal/**/*.rs", "tests/common/**/*.rs"]) + ["//tests:support"], + srcs = glob(["src/**/*.rs"]) + glob(["benches/rmcp/**/*.rs", "benches/session_temporal/**/*.rs", "tests/common/**/*.rs"]) + ["//tests:support_sources"], crate_root = "src/lib.rs", crate_name = "tracedecay", edition = crate_edition(), @@ -92,7 +92,7 @@ rust_binary( rust_test( name = "agent_suite", - srcs = glob(["tests/agent_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support"], + srcs = glob(["tests/agent_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"], crate_root = "tests/agent_suite/main.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -109,7 +109,7 @@ rust_test( rust_test( name = "automation_runner_test", - srcs = glob(["tests/automation_runner_test/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support"], + srcs = glob(["tests/automation_runner_test/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"], crate_root = "tests/automation_runner_test/main.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -136,12 +136,12 @@ rust_test( rustc_flags = perf_rustc_flags(1), crate_features = ["default", "full", "lite", "production", "rmcp-benchmark", "test-helpers", "test-transport", "token-counting"], compile_data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"] + ["//benchmark_data:claude-observation"], - data = glob(["tests/claude_observation_benchmark/**", "assets/**", "fixtures/**", "data/**", "resources/**", "vendor/**", "tests/**"], allow_empty = True) + ["//tests:fixtures"], + data = glob(["tests/claude_observation_benchmark/**", "assets/**", "fixtures/**", "data/**", "resources/**", "vendor/**", "tests/**"], allow_empty = True) + ["//tests:fixtures"] + ["//benchmark_data:claude-observation"], ) rust_test( name = "cursor_observation_identity_fallback", - srcs = glob(["tests/**/*.rs"]) + ["//crates/tracedecay-domain:tests", "//crates/tracedecay-domain:tests_domain_suite_work_product_contract", "//tests:support"], + srcs = glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + ["//crates/tracedecay-domain:tests_domain_suite_work_product_contract_sources", "//crates/tracedecay-domain:tests_sources", "//tests:support_sources"], crate_root = "tests/cursor_observation_identity_fallback.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -151,14 +151,14 @@ rust_test( rustc_flags = perf_rustc_flags(1), crate_features = ["default", "full", "lite", "production", "rmcp-benchmark", "test-helpers", "test-transport", "token-counting"], compile_data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"] + ["//benchmark_data:claude-observation", "//crates/tracedecay-agent-hosts:src_agents_hermes_templates", "//crates/tracedecay-application:src_advisory_fixtures_provider_branch_review", "//crates/tracedecay-domain:tests_fixtures", "//crates/tracedecay-hooks:fixtures_host_events", "//crates/tracedecay-hooks:fixtures_host_events_claude", "//crates/tracedecay-hooks:fixtures_host_events_codex", "//crates/tracedecay-hooks:fixtures_host_events_cursor", "//crates/tracedecay-hooks:fixtures_host_events_droid", "//crates/tracedecay-hooks:fixtures_host_events_hermes", "//crates/tracedecay-hooks:fixtures_host_events_kimi", "//crates/tracedecay-hooks:fixtures_host_events_opencode", "//crates/tracedecay-hooks:fixtures_host_events_pi", "//crates/tracedecay-session-memory:src", "//plugin:skills_managing-session-context", "//plugin:skills_managing-session-context_references"], - data = glob(["tests/**", "assets/**", "fixtures/**", "data/**", "resources/**", "vendor/**"], allow_empty = True) + ["//crates/tracedecay-domain:tests", "//crates/tracedecay-domain:tests_domain_suite_work_product_contract", "//tests:support"] + ["//tests:fixtures"], + data = glob(["tests/**", "assets/**", "fixtures/**", "data/**", "resources/**", "vendor/**"], allow_empty = True) + ["//crates/tracedecay-domain:tests_domain_suite_work_product_contract", "//crates/tracedecay-domain:tests", "//tests:support"] + ["//tests:fixtures"], rustc_env = {"CARGO_TARGET_TMPDIR": "/tmp"}, runfiles_env = {"TRACEDECAY_SEARCH_EVAL_DIRECT_TEST_BIN": "//crates/tracedecay-search-eval:tracedecay-search-eval-direct__test", "TRACEDECAY_SEARCH_EVAL_TEST_BIN": "//crates/tracedecay-search-eval:tracedecay-search-eval_bin__test", "TRACEDECAY_TEST_BIN": "//crates/tracedecay-cli:tracedecay__test", "TRACEDECAY_TYPESCRIPT_SDK_DIST": "//sdks/typescript:dist", "TRACEDECAY_TYPESCRIPT_SDK_RUNNER": "//sdks/typescript:typed_terminal_live"}, ) rust_test( name = "daemon_suite", - srcs = glob(["tests/daemon_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support"], + srcs = glob(["tests/daemon_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"], crate_root = "tests/daemon_suite/main.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -175,7 +175,7 @@ rust_test( rust_test( name = "dashboard_api_test", - srcs = glob(["tests/dashboard_api_test/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support"], + srcs = glob(["tests/dashboard_api_test/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"], crate_root = "tests/dashboard_api_test/main.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -192,7 +192,7 @@ rust_test( rust_test( name = "graph_suite", - srcs = glob(["tests/graph_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support"], + srcs = glob(["tests/graph_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"], crate_root = "tests/graph_suite/main.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -209,7 +209,7 @@ rust_test( rust_test( name = "hermes_suite", - srcs = glob(["tests/hermes_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support"], + srcs = glob(["tests/hermes_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"], crate_root = "tests/hermes_suite/main.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -226,7 +226,7 @@ rust_test( rust_test( name = "hooks_lsp_suite", - srcs = glob(["tests/hooks_lsp_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support"], + srcs = glob(["tests/hooks_lsp_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"], crate_root = "tests/hooks_lsp_suite/main.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -244,7 +244,7 @@ rust_test( rust_test( name = "mcp_suite", size = "large", - srcs = glob(["tests/mcp_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support"], + srcs = glob(["tests/mcp_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"], crate_root = "tests/mcp_suite/main.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -261,7 +261,7 @@ rust_test( rust_test( name = "memory_suite", - srcs = glob(["tests/memory_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support"], + srcs = glob(["tests/memory_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"], crate_root = "tests/memory_suite/main.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -271,7 +271,7 @@ rust_test( rustc_flags = perf_rustc_flags(1), crate_features = ["default", "full", "lite", "production", "rmcp-benchmark", "test-helpers", "test-transport", "token-counting"], compile_data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"] + ["//crates/tracedecay-agent-hosts:src_agents_hermes_templates"], - data = glob(["tests/memory_suite/**", "assets/**", "fixtures/**", "data/**", "resources/**", "vendor/**", "tests/**"], allow_empty = True) + ["//tests:support"] + ["//tests:fixtures"], + data = glob(["tests/memory_suite/**", "assets/**", "fixtures/**", "data/**", "resources/**", "vendor/**", "tests/**"], allow_empty = True) + ["//tests:support"] + ["//tests:fixtures"] + ["//evals/memory:scenarios"], rustc_env = {"CARGO_TARGET_TMPDIR": "/tmp"}, runfiles_env = {"TRACEDECAY_TEST_BIN": "//crates/tracedecay-cli:tracedecay__test"}, ) @@ -294,7 +294,7 @@ rust_test( rust_test( name = "runtime_acceptance_suite", - srcs = glob(["tests/runtime_acceptance_suite/**/*.rs"]) + glob(["tests/common/**/*.rs", "tests/session_suite/**/*.rs", "tests/session_suite/temporal_projection/**/*.rs", "tests/storage_suite/**/*.rs"]) + ["//crates/tracedecay-domain:tests", "//crates/tracedecay-domain:tests_domain_suite_work_product_contract", "//tests:support"], + srcs = glob(["tests/runtime_acceptance_suite/**/*.rs"]) + glob(["tests/common/**/*.rs", "tests/session_suite/**/*.rs", "tests/session_suite/temporal_projection/**/*.rs", "tests/storage_suite/**/*.rs"]) + ["//crates/tracedecay-domain:tests_domain_suite_work_product_contract_sources", "//crates/tracedecay-domain:tests_sources", "//tests:support_sources"], crate_root = "tests/runtime_acceptance_suite/main.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -304,14 +304,14 @@ rust_test( rustc_flags = perf_rustc_flags(1), crate_features = ["default", "full", "lite", "production", "rmcp-benchmark", "test-helpers", "test-transport", "token-counting"], compile_data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"] + ["//crates/tracedecay-agent-hosts:src_agents_hermes_templates", "//crates/tracedecay-application:src_advisory_fixtures_provider_branch_review", "//crates/tracedecay-domain:tests_fixtures", "//crates/tracedecay-hooks:fixtures_host_events_claude", "//crates/tracedecay-hooks:fixtures_host_events_codex", "//crates/tracedecay-hooks:fixtures_host_events_cursor"], - data = glob(["tests/runtime_acceptance_suite/**", "assets/**", "fixtures/**", "data/**", "resources/**", "vendor/**", "tests/**"], allow_empty = True) + ["//crates/tracedecay-domain:tests", "//crates/tracedecay-domain:tests_domain_suite_work_product_contract", "//tests:support"] + ["//tests:fixtures"], + data = glob(["tests/runtime_acceptance_suite/**", "assets/**", "fixtures/**", "data/**", "resources/**", "vendor/**", "tests/**"], allow_empty = True) + ["//crates/tracedecay-domain:tests_domain_suite_work_product_contract", "//crates/tracedecay-domain:tests", "//tests:support"] + ["//tests:fixtures"], rustc_env = {"CARGO_TARGET_TMPDIR": "/tmp"}, runfiles_env = {"TRACEDECAY_SEARCH_EVAL_TEST_BIN": "//crates/tracedecay-search-eval:tracedecay-search-eval_bin__test", "TRACEDECAY_TEST_BIN": "//crates/tracedecay-cli:tracedecay__test"}, ) rust_test( name = "session_suite", - srcs = glob(["tests/session_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support"], + srcs = glob(["tests/session_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"], crate_root = "tests/session_suite/main.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -328,7 +328,7 @@ rust_test( rust_test( name = "storage_runtime_rusqlite_suite", - srcs = ["//tests:storage_runtime_rusqlite_suite"], + srcs = ["//tests:storage_runtime_rusqlite_suite_sources"], crate_root = "//tests:storage_runtime_rusqlite_suite/main.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -343,7 +343,7 @@ rust_test( rust_test( name = "storage_suite", - srcs = glob(["tests/storage_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support"], + srcs = glob(["tests/storage_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"], crate_root = "tests/storage_suite/main.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -360,7 +360,7 @@ rust_test( rust_test( name = "transcript_ingest_suite", - srcs = glob(["tests/transcript_ingest_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support"], + srcs = glob(["tests/transcript_ingest_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"], crate_root = "tests/transcript_ingest_suite/main.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -377,7 +377,7 @@ rust_test( rust_test( name = "transport_acceptance_suite", - srcs = glob(["tests/transport_acceptance_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support"], + srcs = glob(["tests/transport_acceptance_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"], crate_root = "tests/transport_acceptance_suite/main.rs", edition = crate_edition(), version = "1.0.0-beta.76", @@ -394,6 +394,8 @@ rust_test( rust_test( name = "large_repos", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/large_repos.rs", edition = crate_edition(), @@ -409,6 +411,8 @@ rust_test( rust_test( name = "session_temporal", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/session_temporal.rs", edition = crate_edition(), @@ -419,11 +423,13 @@ rust_test( rustc_flags = perf_rustc_flags(1), crate_features = ["default", "full", "lite", "production", "rmcp-benchmark", "test-helpers", "test-transport", "token-counting"], compile_data = glob(["tests/**","assets/**","fixtures/**","data/**","resources/**","vendor/**"], allow_empty = True) + ["//tests:fixtures"] + ["//benchmark_data:runtime_fixtures_project_src", "//benchmark_data:runtime_fixtures_project_tests", "//crates/tracedecay-hooks:fixtures_host_events_kimi"], - data = glob(["benches/**", "assets/**", "fixtures/**", "data/**", "resources/**", "vendor/**", "tests/**"], allow_empty = True) + ["//tests:fixtures"], + data = glob(["benches/**", "assets/**", "fixtures/**", "data/**", "resources/**", "vendor/**", "tests/**"], allow_empty = True) + ["//tests:fixtures"] + ["//benchmark_data:session-temporal"], ) rust_test( name = "transcript_ingest", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/transcript_ingest.rs", edition = crate_edition(), @@ -439,6 +445,8 @@ rust_test( rust_test( name = "rmcp_connection_pipeline", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/rmcp_connection_pipeline.rs", edition = crate_edition(), @@ -454,6 +462,8 @@ rust_test( rust_test( name = "mcp_server_construction", + tags = ["manual"], + use_libtest_harness = False, srcs = glob(["benches/**/*.rs"]), crate_root = "benches/mcp_server_construction.rs", edition = crate_edition(), @@ -467,6 +477,12 @@ rust_test( data = glob(["benches/**", "assets/**", "fixtures/**", "data/**", "resources/**", "vendor/**", "tests/**"], allow_empty = True) + ["//tests:fixtures"], ) +filegroup( + name = "benchmark_binaries", + testonly = True, + srcs = [":large_repos__binary", ":session_temporal__binary", ":transcript_ingest__binary", ":rmcp_connection_pipeline__binary", ":mcp_server_construction__binary"], +) + rust_test( name = "unit_test", size = "large", @@ -485,8 +501,32 @@ rust_test( runfiles_env = {"TRACEDECAY_TEST_BIN": "//crates/tracedecay-cli:tracedecay__test"}, ) +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(["//tests:storage_runtime_rusqlite_suite_sources"] + glob(["benches/**/*.rs"]) + glob(["examples/**/*.rs"]) + glob(["src/**/*.rs"]) + glob(["benches/rmcp/**/*.rs", "benches/session_temporal/**/*.rs", "tests/common/**/*.rs"]) + ["//tests:support_sources"] + glob(["tests/**/*.rs"], exclude = ["tests/fixtures/**"]) + ["//crates/tracedecay-domain:tests_domain_suite_work_product_contract_sources", "//crates/tracedecay-domain:tests_sources", "//tests:support_sources"] + glob(["tests/agent_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"] + glob(["tests/automation_runner_test/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"] + glob(["tests/claude_observation_benchmark/**/*.rs"]) + glob(["tests/daemon_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"] + glob(["tests/dashboard_api_test/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"] + glob(["tests/graph_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"] + glob(["tests/hermes_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"] + glob(["tests/hooks_lsp_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"] + glob(["tests/mcp_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"] + glob(["tests/memory_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"] + glob(["tests/product_surface_suite/**/*.rs"]) + glob(["tests/runtime_acceptance_suite/**/*.rs"]) + glob(["tests/common/**/*.rs", "tests/session_suite/**/*.rs", "tests/session_suite/temporal_projection/**/*.rs", "tests/storage_suite/**/*.rs"]) + ["//crates/tracedecay-domain:tests_domain_suite_work_product_contract_sources", "//crates/tracedecay-domain:tests_sources", "//tests:support_sources"] + glob(["tests/session_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"] + glob(["tests/storage_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"] + glob(["tests/transcript_ingest_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"] + glob(["tests/transport_acceptance_suite/**/*.rs"]) + glob(["tests/common/**/*.rs"]) + ["//tests:support_sources"]).to_list(), + tags = ["manual"], +) + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) + filegroup( name = "tests_common", srcs = glob(["tests/common/**"]), visibility = ["//visibility:public"], ) + +filegroup( + name = "tests_common_sources", + srcs = glob(["tests/common/**/*.rs"]), + visibility = ["//visibility:public"], +) diff --git a/crates/tracedecay/benches/session_temporal.rs b/crates/tracedecay/benches/session_temporal.rs index 9790cc3c7b..890041b341 100644 --- a/crates/tracedecay/benches/session_temporal.rs +++ b/crates/tracedecay/benches/session_temporal.rs @@ -37,8 +37,7 @@ fn main() { }) } _ => Err( - "usage: cargo bench -p tracedecay --bench session_temporal --features test-helpers [-- --run|--refresh-contract]" - .to_owned(), + "usage: scripts/run-session-temporal-benchmark.sh --run|--refresh-contract".to_owned(), ), }; if let Err(error) = result { diff --git a/crates/tracedecay/benches/session_temporal/harness.rs b/crates/tracedecay/benches/session_temporal/harness.rs index 602b744705..85871ea278 100644 --- a/crates/tracedecay/benches/session_temporal/harness.rs +++ b/crates/tracedecay/benches/session_temporal/harness.rs @@ -320,7 +320,6 @@ pub fn validate_contract() -> BenchResult<()> { json!(RUNNER_PATH), "runner path", )?; - validate_bench_profile(&root)?; validate_sanitization_receipt(&root)?; let index = read_json(&root.join(EVIDENCE_INDEX_PATH))?; @@ -519,7 +518,6 @@ fn validate_refresh_inputs(root: &Path, workload: &Value) -> BenchResult<()> { json!(true), "production path availability", )?; - validate_bench_profile(root)?; validate_sanitization_receipt(root) } @@ -535,7 +533,7 @@ pub async fn run_measurement() -> BenchResult { ); } validate_contract()?; - validate_bench_profile_enforced()?; + validate_measurement_configuration()?; let root = repository_root(); let measurement = Box::pin(capture_measurement()).await?; @@ -556,7 +554,7 @@ pub async fn refresh_contract() -> BenchResult { if !host_policy.allows_contract_refresh() { return Err("Session-temporal contract refresh is Linux-hosted".to_owned()); } - validate_bench_profile_enforced()?; + validate_measurement_configuration()?; let root = repository_root(); let workload = read_json(&root.join(WORKLOAD_PATH))?; validate_refresh_inputs(&root, &workload)?; @@ -789,7 +787,6 @@ async fn prepare_repetition(repetition: usize) -> BenchResult BenchResult<()> { Ok(()) } -fn validate_bench_profile(root: &Path) -> BenchResult<()> { - let manifest = fs::read_to_string(root.join("Cargo.toml")) - .map_err(|error| format!("read Cargo.toml: {error}"))?; - let profile = manifest - .split_once("[profile.bench]") - .map(|(_, profile)| profile.split("\n[").next().unwrap_or(profile)) - .ok_or_else(|| "Cargo.toml is missing [profile.bench]".to_owned())?; - for line in [ - "opt-level = 3", - "debug = false", - "debug-assertions = false", - "overflow-checks = false", - "incremental = false", - ] { - if !profile.lines().any(|candidate| candidate.trim() == line) { - return Err(format!("bench profile is missing {line:?}")); - } - } - Ok(()) -} - -fn validate_bench_profile_enforced() -> BenchResult<()> { +fn validate_measurement_configuration() -> BenchResult<()> { if cfg!(debug_assertions) { return Err( - "optimized bench profile required: debug_assertions are enabled (use cargo bench)" + "optimized bench profile required: debug_assertions are enabled (use Bazel --config=release)" .to_owned(), ); } @@ -1160,11 +1136,16 @@ fn require_json_value(actual: &Value, expected: Value, label: &str) -> BenchResu fn repository_root() -> PathBuf { // The product package sits at `crates/tracedecay`; this benchmark reads git // metadata and fixtures that live at the workspace root above it. - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .and_then(Path::parent) - .expect("workspace root above crates/tracedecay") - .to_owned() + if let Some(root) = std::env::var_os("TRACEDECAY_BENCHMARK_REPO_ROOT") { + return PathBuf::from(root); + } + PathBuf::from( + std::env::var_os("CARGO_MANIFEST_DIR").unwrap_or_else(|| env!("CARGO_MANIFEST_DIR").into()), + ) + .parent() + .and_then(Path::parent) + .expect("workspace root above crates/tracedecay") + .to_owned() } fn current_commit(root: &Path) -> BenchResult { diff --git a/crates/tracedecay/benches/session_temporal/root_relation_fixture.rs b/crates/tracedecay/benches/session_temporal/root_relation_fixture.rs index 5f7fd1215d..dfbd5b9c2d 100644 --- a/crates/tracedecay/benches/session_temporal/root_relation_fixture.rs +++ b/crates/tracedecay/benches/session_temporal/root_relation_fixture.rs @@ -1,6 +1,6 @@ //! Production root-wide relation-hydration fixture for the session benchmark. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use tracedecay_contracts::RequestContext; use tracedecay_domain::{RetrievalGrainV1, SessionId, TemporalModeV1}; @@ -10,15 +10,15 @@ use tracedecay_session_memory::session::{ SessionRefreshTarget, SessionRequestBinding, SessionRetrievalOutcome, SessionRetrievalScope, SessionTemporalQuery, }; -use tracedecay_store::{ - SessionRefreshCompletionRequestV1, SessionRefreshFrontierV1, SessionRefreshProgressV1, -}; +use tracedecay_store::{SessionRefreshCompletionRequestV1, SessionRefreshProgressV1}; use tracedecay_temporal_query::context::ContextBudget; use tracedecay_temporal_query::execution::ExecutionControl; use tracedecay_temporal_query::ranking::DiversityLimits; use super::{AllowAuthorizer, BenchResult, CONFIG_VERSION, PROJECTOR_VERSION}; -use tracedecay_session_temporal_store::{SessionTemporalAccess, SessionTemporalStore}; +use tracedecay_session_temporal_store::{ + SessionTemporalAccess, SessionTemporalRefreshDiscoveryCursor, SessionTemporalStore, +}; pub(super) const ROOT_RELATION_PARTICIPANT_COUNT: usize = 64; @@ -47,7 +47,6 @@ pub(super) async fn refresh_sessions( db: &RegisteredGlobalDb, request: impl Fn(&SessionId) -> BenchResult<(RequestContext, SessionRequestBinding)>, sessions: Vec, - observation_count: u64, ) -> BenchResult { if sessions.len() != ROOT_RELATION_PARTICIPANT_COUNT { return Err(format!( @@ -55,6 +54,22 @@ pub(super) async fn refresh_sessions( sessions.len() )); } + let mut frontiers = BTreeMap::new(); + let mut cursor = SessionTemporalRefreshDiscoveryCursor::default(); + loop { + let (requests, next, has_more) = SessionTemporalAccess::new(db) + .pending_session_temporal_refresh_page_result(sessions.len(), 1, &cursor) + .await + .map_err(|error| format!("discover root refresh frontiers: {error}"))? + .into_parts(); + for request in requests { + frontiers.insert(request.session_id().clone(), request.target_frontier()); + } + cursor = next; + if !has_more { + break; + } + } let refresh = SessionRefreshService::new( AllowAuthorizer, SessionTemporalStore::new(db), @@ -71,8 +86,9 @@ pub(super) async fn refresh_sessions( Some("codex".to_owned()), TemporalModeV1::Current, RetrievalGrainV1::LogicalMessage, - SessionRefreshFrontierV1::new(observation_count, 0) - .map_err(|error| format!("root refresh frontier: {error}"))?, + frontiers.remove(session_id).ok_or_else(|| { + format!("root session {session_id} has no pending canonical refresh") + })?, ) .map_err(|error| format!("root refresh target: {error}"))?; let (context, binding) = request(session_id)?; diff --git a/crates/tracedecay/src/daemon/production_harness.rs b/crates/tracedecay/src/daemon/production_harness.rs index cdae3b7619..47e4f4aabd 100644 --- a/crates/tracedecay/src/daemon/production_harness.rs +++ b/crates/tracedecay/src/daemon/production_harness.rs @@ -21,6 +21,8 @@ use super::*; #[cfg(unix)] use tracedecay_application::pr_tracking::acquire_manual_branch_lifecycle; use tracedecay_code_index_runtime::CodeIndexSchedulerRegistryV1; +#[cfg(feature = "test-helpers")] +use tracedecay_code_index_runtime::code_index_scheduler::CodeIndexWorktreeSchedulerV1; #[cfg(all(unix, feature = "test-transport"))] use tracedecay_code_index_runtime::git_transactions; use tracedecay_daemon_identity::profile_identity; @@ -888,6 +890,29 @@ impl ProductionProjectCompositionHarnessV1 { .await) } + /// Uses the registry's existing test handle to fence source-proof renewal + /// while a production read observes the previous publication. + #[cfg(feature = "test-helpers")] + pub async fn project_scheduler_handle( + &self, + project_root: &Path, + ) -> Result>> { + let resources = self + .resources + .as_ref() + .ok_or_else(|| TraceDecayError::Config { + message: "production-composition harness is shut down".to_owned(), + })?; + resources + .invocation + .code_index_schedulers + .scheduler_handle(project_root) + .await + .ok_or_else(|| TraceDecayError::Config { + message: "production-composition project scheduler is not mounted".to_owned(), + }) + } + pub async fn project_data_root(&self, project_root: impl AsRef) -> Result { Ok(self .server(project_root)? @@ -1729,3 +1754,6 @@ mod lcm_preserved_profile_journey_test; #[cfg(all(test, target_os = "linux", target_env = "gnu"))] mod idle_history_reads_journey_test; + +#[cfg(test)] +mod source_body_readiness_journey_test; diff --git a/crates/tracedecay/src/daemon/production_harness/source_body_readiness_journey_test.rs b/crates/tracedecay/src/daemon/production_harness/source_body_readiness_journey_test.rs new file mode 100644 index 0000000000..acb9b21475 --- /dev/null +++ b/crates/tracedecay/src/daemon/production_harness/source_body_readiness_journey_test.rs @@ -0,0 +1,100 @@ +use std::sync::{Arc, mpsc}; +use std::time::Duration; + +use tempfile::TempDir; +use tracedecay_code_index::graph_projection::CodeGraphCatalogReleaseV1; +use tracedecay_domain::{ProjectId, SymbolOccurrenceId, UtcMicros}; +use tracedecay_graph_db::NeverCancelled; + +use super::ProductionProjectCompositionHarnessV1; +use super::journey_test_support::{git, tool_payload}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn source_body_serves_while_the_released_catalog_cannot_rewarm() { + let isolation = TempDir::new().unwrap(); + let project = isolation.path().join("project"); + std::fs::create_dir_all(&project).unwrap(); + let source = "pub fn source_anchor() -> u32 { 7 }"; + std::fs::write(project.join("lib.rs"), source).unwrap(); + git(&project, &["init", "-q", "-b", "main"]); + git(&project, &["add", "."]); + git(&project, &["commit", "-qm", "seed source body"]); + let harness = ProductionProjectCompositionHarnessV1::open(isolation.path(), [project.clone()]) + .await + .unwrap(); + let project_id = ProjectId::new(harness.project_id(&project).await.unwrap()).unwrap(); + let scope = + tracedecay_code_index_runtime::resolved_scope_for_project(&project, &project_id).unwrap(); + let serving = harness + .resources + .as_ref() + .unwrap() + .invocation + .code_index_schedulers + .latest_text_serving_for_scope(&scope) + .await + .unwrap(); + let store = serving.interactive_graph_store().unwrap(); + store + .warm_interactive_catalog_with_cancellation(None, Arc::new(NeverCancelled)) + .unwrap(); + let lookup = tool_payload( + &harness + .call_tool( + &project, + "tracedecay_find_exact_symbol", + serde_json::json!({"name": "source_anchor", "format": "json"}), + ) + .await + .unwrap(), + ); + let node_id = lookup["matches"][0]["id"].as_str().unwrap(); + let occurrence = SymbolOccurrenceId::new(node_id).unwrap(); + let reader = store + .interactive_reader_with_cancellation(store.generation(), Arc::new(NeverCancelled)) + .unwrap(); + assert!(matches!( + store.release_interactive_catalog(), + CodeGraphCatalogReleaseV1::Released { .. } + )); + let held_store = Arc::clone(&store); + let (held_tx, held_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = mpsc::channel::<()>(); + let holding = std::thread::spawn(move || { + let _guard = held_store.hold_catalog_build_for_test().unwrap(); + held_tx.send(()).unwrap(); + let _ = release_rx.recv(); + }); + held_rx.await.unwrap(); + // These point reads prove that the retained engine and occurrence are ready. + let point = reader.symbol_summary(&occurrence, Arc::new(NeverCancelled)); + let catalog_pending = store.await_rewarm(Duration::ZERO); + let deadline = UtcMicros(tracedecay_contracts::now_micros().0 + 1_000_000); + let response = harness + .call_tool_with_deadline( + &project, + "tracedecay_source_body", + serde_json::json!({"node_id": node_id, "format": "json"}), + deadline, + ) + .await; + let still_pending = store.await_rewarm(Duration::ZERO); + drop(release_tx); + holding.join().unwrap(); + let warmed = store.await_rewarm(Duration::from_secs(5)); + drop(reader); + drop(store); + drop(serving); + harness.shutdown().await; + + assert!(point.unwrap().is_some()); + assert!(catalog_pending.is_err()); + assert!( + still_pending.is_err(), + "source read must finish before the held catalog warms" + ); + assert_eq!(warmed, Ok(())); + let payload = tool_payload(&response.unwrap()); + assert_eq!(payload["outcome"]["outcome"], "evidence", "{payload}"); + assert_eq!(payload["outcome"]["value"]["payload"]["body"], source); +} diff --git a/crates/tracedecay/tests/claude_observation_benchmark/artifact.rs b/crates/tracedecay/tests/claude_observation_benchmark/artifact.rs index f63658b160..84d5f7338f 100644 --- a/crates/tracedecay/tests/claude_observation_benchmark/artifact.rs +++ b/crates/tracedecay/tests/claude_observation_benchmark/artifact.rs @@ -19,16 +19,13 @@ use super::model::{ ProviderBenchmarkSuiteResult, ProviderPhaseResult, RawPhaseSample, WorkloadIdentity, }; use super::{ - BENCHMARK_COMMAND, BUILD_CARGO_CONFIG_IDENTITY, BUILD_CARGO_VERSION, BUILD_COMMIT, - BUILD_PROFILE, BUILD_RUSTC_VERSION, BUILD_RUSTC_WORKSPACE_WRAPPER, BUILD_RUSTC_WRAPPER, - BUILD_RUSTFLAGS, BUILD_SOURCE_MANIFEST_SHA256, BUILD_SOURCE_MODE, BUILD_TARGET_TRIPLE, - BUILD_TREE, HARNESS_SOURCES, MEASURED_REPETITIONS, NATIVE_PROVIDER_FIXTURES, + BENCHMARK_COMMAND, HARNESS_SOURCES, MEASURED_REPETITIONS, NATIVE_PROVIDER_FIXTURES, PROVIDER_PIPELINE_SCOPE, RECORDS_PER_REPETITION, RESULT_SCHEMA_VERSION, RETIRED_WORKLOAD_ID, WARMUP_REPETITIONS, WORKLOAD_ID, WORKLOAD_MANIFEST, WORKLOAD_MANIFEST_PATH, WORKLOAD_SCHEMA_VERSION, }; -pub(super) struct AttestedBuild { +pub(super) struct MeasuredBuild { pub(super) evidence_status: EvidenceStatus, pub(super) build_identity: BuildIdentity, } @@ -241,24 +238,18 @@ fn validate_acceptance_result(result: &BenchmarkResult) -> Result<(), String> { || !is_lower_hex(&result.build_identity.commit, 40) || !is_lower_hex(&result.build_identity.tree, 40) || result.build_identity.profile != "release" - || result.build_identity.source_mode != "git_archive_read_only_v1" - || !is_lower_hex(&result.build_identity.source_manifest_sha256, 64) - || result.build_identity.source_file_count == 0 + || result.build_identity.source_mode != "clean_git_worktree_v1" || result.build_identity.target_triple.is_empty() || result.build_identity.rustc_version != result.rustc - || result.build_identity.cargo_version != result.cargo - || result.build_identity.rustflags != "normalized-empty" - || result.build_identity.rustc_wrapper.is_empty() - || result.build_identity.rustc_workspace_wrapper.is_empty() - || !is_lower_hex(&result.build_identity.cargo_config_identity, 40) - || result.build_identity.data_root_basis != "current_executable_parent" + || result.build_identity.bazel_version != result.bazel + || result.build_identity.data_root_basis != "TEST_TMPDIR" || !is_lower_hex(&result.build_identity.executable_sha256, 64) || result.build_identity.executable_size_bytes == 0 { - return Err("acceptance result build attestation is invalid".to_string()); + return Err("acceptance result build identity is invalid".to_string()); } if result.rustc.is_empty() - || result.cargo.is_empty() + || result.bazel.is_empty() || result.kernel.is_empty() || result.cpu_identity.is_empty() || result.logical_cpu_count == 0 @@ -689,70 +680,41 @@ pub(super) fn sha256_file(path: &Path) -> std::io::Result<(String, u64)> { Ok((hex::encode(digest.finalize()), size)) } -pub(super) fn validate_release_profile( - debug_assertions: bool, - profile: Option<&str>, -) -> Result<(), String> { - if debug_assertions { - return Err("benchmark evidence cannot run with debug assertions".to_string()); - } - if profile != Some("release") { - return Err("benchmark evidence requires release build attestation".to_string()); +pub(super) fn measure_build_identity(git: &GitSnapshot) -> Result { + if cfg!(debug_assertions) { + return Err("benchmark requires Bazel --config=release".to_owned()); } - Ok(()) -} - -pub(super) fn attest_build(git: &GitSnapshot) -> AttestedBuild { - validate_release_profile(cfg!(debug_assertions), BUILD_PROFILE) - .expect("benchmark must use the release evidence runner"); - let commit = BUILD_COMMIT.expect("missing build-time Git commit attestation"); - let tree = BUILD_TREE.expect("missing build-time Git tree attestation"); - assert_eq!(commit, git.commit, "build commit differs from runtime HEAD"); - assert_eq!(tree, git.tree, "build tree differs from runtime HEAD tree"); let executable = fs::canonicalize(std::env::current_exe().expect("resolve executable")) .expect("canonicalize executable"); let (executable_sha256, executable_size_bytes) = sha256_file(&executable).expect("hash benchmark executable"); - let rustc_version = BUILD_RUSTC_VERSION.expect("missing build-time rustc identity"); - let cargo_version = BUILD_CARGO_VERSION.expect("missing build-time Cargo identity"); - assert_eq!(rustc_version, command_output("rustc", &["-Vv"])); - assert_eq!(cargo_version, command_output("cargo", &["-V"])); - let (_, source_file_count) = validate_source_archive(); - AttestedBuild { + let rustc_version = command_output("rustc", &["-Vv"]); + let target_triple = rustc_version + .lines() + .find_map(|line| line.strip_prefix("host: ")) + .expect("rustc host target") + .to_owned(); + assert!( + std::env::var_os("TEST_TMPDIR").is_some(), + "benchmark requires Bazel test isolation" + ); + let bazel_version = std::env::var("TRACEDECAY_BENCHMARK_BAZEL_VERSION") + .expect("run benchmark through its Bazel runner"); + Ok(MeasuredBuild { evidence_status: EvidenceStatus::Acceptance, build_identity: BuildIdentity { - commit: commit.to_string(), - tree: tree.to_string(), + commit: git.commit.clone(), + tree: git.tree.clone(), profile: "release".to_string(), - source_mode: BUILD_SOURCE_MODE - .expect("missing immutable source mode") - .to_string(), - source_manifest_sha256: BUILD_SOURCE_MANIFEST_SHA256 - .expect("missing source manifest identity") - .to_string(), - source_file_count, - target_triple: BUILD_TARGET_TRIPLE - .expect("missing build target triple") - .to_string(), - rustc_version: rustc_version.to_string(), - cargo_version: cargo_version.to_string(), - rustflags: BUILD_RUSTFLAGS - .expect("missing normalized Rust flags") - .to_string(), - rustc_wrapper: BUILD_RUSTC_WRAPPER - .expect("missing rustc wrapper identity") - .to_string(), - rustc_workspace_wrapper: BUILD_RUSTC_WORKSPACE_WRAPPER - .expect("missing workspace wrapper identity") - .to_string(), - cargo_config_identity: BUILD_CARGO_CONFIG_IDENTITY - .expect("missing Cargo configuration identity") - .to_string(), - data_root_basis: "current_executable_parent".to_string(), + source_mode: "clean_git_worktree_v1".to_string(), + target_triple, + rustc_version, + bazel_version, + data_root_basis: "TEST_TMPDIR".to_string(), executable_sha256, executable_size_bytes, }, - } + }) } pub(super) fn validate_git_snapshots( @@ -869,9 +831,6 @@ pub(super) fn command_output(command: &str, args: &[&str]) -> String { } pub(super) fn git_snapshot() -> GitSnapshot { - if BUILD_SOURCE_MODE == Some("git_archive_read_only_v1") { - return validate_source_archive().0; - } verify_git_toplevel(); GitSnapshot { commit: git_output(&["rev-parse", "HEAD"]), @@ -880,56 +839,6 @@ pub(super) fn git_snapshot() -> GitSnapshot { } } -fn validate_source_archive() -> (GitSnapshot, usize) { - let manifest_path = repository_root().join(".tracedecay-benchmark-source-manifest"); - let manifest = fs::read(&manifest_path).expect("read immutable source manifest"); - assert_eq!( - sha256_hex(&manifest), - BUILD_SOURCE_MANIFEST_SHA256.expect("missing source manifest identity"), - "source manifest differs from build attestation" - ); - let manifest = String::from_utf8(manifest).expect("source manifest is UTF-8"); - let mut source_file_count = 0; - for line in manifest.lines() { - let mut fields = line.splitn(3, '\t'); - let mode = fields.next().expect("source manifest mode"); - let digest = fields.next().expect("source manifest digest"); - let relative = fields.next().expect("source manifest path"); - if mode == "160000" { - continue; - } - let path = repository_root().join(relative); - let bytes = fs::read(&path) - .unwrap_or_else(|error| panic!("read immutable compiler input {relative}: {error}")); - assert_eq!( - sha256_hex(&bytes), - digest, - "compiler input changed: {relative}" - ); - assert!( - fs::metadata(&path) - .expect("compiler input metadata") - .permissions() - .readonly(), - "compiler input is writable: {relative}" - ); - source_file_count += 1; - } - assert!(source_file_count > 0, "source manifest contains no files"); - ( - GitSnapshot { - commit: BUILD_COMMIT - .expect("missing source commit identity") - .to_string(), - tree: BUILD_TREE - .expect("missing source tree identity") - .to_string(), - dirty: false, - }, - source_file_count, - ) -} - fn worktree_is_dirty() -> bool { let output = Command::new("git") .args([ @@ -952,17 +861,21 @@ pub(super) fn status_output_is_dirty(output: &[u8]) -> bool { /// The product package directory. `HARNESS_SOURCES` records paths relative to /// the package (`tests/claude_observation_benchmark/...`), a different anchor from the /// workspace-level fixtures and benchmark data below. -pub(super) fn package_root() -> &'static Path { - Path::new(env!("CARGO_MANIFEST_DIR")) +pub(super) fn package_root() -> PathBuf { + repository_root().join("crates/tracedecay") } -pub(super) fn repository_root() -> &'static Path { - // The product package sits at `crates/tracedecay`; the fixtures, benchmark - // data, and git metadata this module reads all live at the workspace root. - Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .and_then(Path::parent) - .expect("workspace root above crates/tracedecay") +pub(super) fn repository_root() -> PathBuf { + if let Some(root) = std::env::var_os("TRACEDECAY_BENCHMARK_REPO_ROOT") { + return PathBuf::from(root); + } + PathBuf::from( + std::env::var_os("CARGO_MANIFEST_DIR").unwrap_or_else(|| env!("CARGO_MANIFEST_DIR").into()), + ) + .parent() + .and_then(Path::parent) + .expect("workspace root above crates/tracedecay") + .to_owned() } fn git_output(args: &[&str]) -> String { diff --git a/crates/tracedecay/tests/claude_observation_benchmark/main.rs b/crates/tracedecay/tests/claude_observation_benchmark/main.rs index d8a18f8b31..f885d8763b 100644 --- a/crates/tracedecay/tests/claude_observation_benchmark/main.rs +++ b/crates/tracedecay/tests/claude_observation_benchmark/main.rs @@ -19,7 +19,7 @@ const WARMUP_REPETITIONS: usize = 3; const MEASURED_REPETITIONS: usize = 30; const RECORDS_PER_REPETITION: usize = 64; const CONCURRENCY: usize = 1; -const BENCHMARK_COMMAND: &str = "cargo test --quiet --release -p tracedecay --features test-helpers --test claude_observation_benchmark production_observation_pipeline_baseline -- --ignored --exact --nocapture --test-threads=1"; +const BENCHMARK_COMMAND: &str = "scripts/run-claude-observation-benchmark.sh"; const EVIDENCE_RUNNER: &str = "scripts/run-claude-observation-benchmark.sh"; const WORKLOAD_IMPLEMENTATION: &str = "tests/claude_observation_benchmark/main.rs"; const WORKLOAD_MANIFEST_PATH: &str = "benchmark_data/claude-observation/workload-v1.json"; @@ -113,22 +113,6 @@ const HARNESS_SOURCES: &[(&str, &str)] = &[ include_str!("tests.rs"), ), ]; -const BUILD_COMMIT: Option<&str> = option_env!("TRACEDECAY_BENCHMARK_BUILD_COMMIT"); -const BUILD_TREE: Option<&str> = option_env!("TRACEDECAY_BENCHMARK_BUILD_TREE"); -const BUILD_PROFILE: Option<&str> = option_env!("TRACEDECAY_BENCHMARK_BUILD_PROFILE"); -const BUILD_SOURCE_MODE: Option<&str> = option_env!("TRACEDECAY_BENCHMARK_BUILD_SOURCE_MODE"); -const BUILD_SOURCE_MANIFEST_SHA256: Option<&str> = - option_env!("TRACEDECAY_BENCHMARK_SOURCE_MANIFEST_SHA256"); -const BUILD_TARGET_TRIPLE: Option<&str> = option_env!("TRACEDECAY_BENCHMARK_BUILD_TARGET_TRIPLE"); -const BUILD_RUSTC_VERSION: Option<&str> = option_env!("TRACEDECAY_BENCHMARK_BUILD_RUSTC_VERSION"); -const BUILD_CARGO_VERSION: Option<&str> = option_env!("TRACEDECAY_BENCHMARK_BUILD_CARGO_VERSION"); -const BUILD_RUSTFLAGS: Option<&str> = option_env!("TRACEDECAY_BENCHMARK_BUILD_RUSTFLAGS"); -const BUILD_RUSTC_WRAPPER: Option<&str> = option_env!("TRACEDECAY_BENCHMARK_BUILD_RUSTC_WRAPPER"); -const BUILD_RUSTC_WORKSPACE_WRAPPER: Option<&str> = - option_env!("TRACEDECAY_BENCHMARK_BUILD_RUSTC_WORKSPACE_WRAPPER"); -const BUILD_CARGO_CONFIG_IDENTITY: Option<&str> = - option_env!("TRACEDECAY_BENCHMARK_BUILD_CARGO_CONFIG_IDENTITY"); - #[tokio::test] #[ignore = "release-mode claude-observation performance baseline; run the documented exact command"] async fn production_observation_pipeline_baseline() { diff --git a/crates/tracedecay/tests/claude_observation_benchmark/model.rs b/crates/tracedecay/tests/claude_observation_benchmark/model.rs index b64001cbae..205c5d65b0 100644 --- a/crates/tracedecay/tests/claude_observation_benchmark/model.rs +++ b/crates/tracedecay/tests/claude_observation_benchmark/model.rs @@ -233,15 +233,9 @@ pub(super) struct BuildIdentity { pub(super) tree: String, pub(super) profile: String, pub(super) source_mode: String, - pub(super) source_manifest_sha256: String, - pub(super) source_file_count: usize, pub(super) target_triple: String, pub(super) rustc_version: String, - pub(super) cargo_version: String, - pub(super) rustflags: String, - pub(super) rustc_wrapper: String, - pub(super) rustc_workspace_wrapper: String, - pub(super) cargo_config_identity: String, + pub(super) bazel_version: String, pub(super) data_root_basis: String, pub(super) executable_sha256: String, pub(super) executable_size_bytes: u64, @@ -259,7 +253,7 @@ pub(super) struct BenchmarkResult { pub(super) git_after: GitSnapshot, pub(super) command: String, pub(super) rustc: String, - pub(super) cargo: String, + pub(super) bazel: String, pub(super) kernel: String, pub(super) cpu_identity: String, pub(super) logical_cpu_count: usize, diff --git a/crates/tracedecay/tests/claude_observation_benchmark/runner.rs b/crates/tracedecay/tests/claude_observation_benchmark/runner.rs index 7cdc6eac2e..50d6c87e6d 100644 --- a/crates/tracedecay/tests/claude_observation_benchmark/runner.rs +++ b/crates/tracedecay/tests/claude_observation_benchmark/runner.rs @@ -40,7 +40,7 @@ fn ensure_background_cpu_authority() { } use super::artifact::{ - attest_build, command_output, git_snapshot, validate_git_snapshots, workload_identity, + command_output, git_snapshot, measure_build_identity, validate_git_snapshots, workload_identity, }; use super::metrics::{ aggregate_samples, cpu_identity, elapsed_ns, memory_total_kib, preflight_platform, @@ -1267,7 +1267,8 @@ pub(super) async fn run() { "benchmark evidence requires a clean worktree before execution" ); let identity_before = workload_identity(); - let attested_build = attest_build(&git_before); + let measured_build = measure_build_identity(&git_before) + .expect("benchmark must use the release evidence runner"); for repetition in 0..WARMUP_REPETITIONS { let fixture = Fixture::new(repetition).await; let source = fixture.source(); @@ -1373,14 +1374,15 @@ pub(super) async fn run() { let result = BenchmarkResult { schema_version: RESULT_SCHEMA_VERSION, workload_id: WORKLOAD_ID.to_string(), - evidence_status: attested_build.evidence_status, + evidence_status: measured_build.evidence_status, workload_identity: identity_before, - build_identity: attested_build.build_identity, + build_identity: measured_build.build_identity, git_before, git_after, command: BENCHMARK_COMMAND.to_string(), rustc: command_output("rustc", &["-Vv"]), - cargo: command_output("cargo", &["-V"]), + bazel: std::env::var("TRACEDECAY_BENCHMARK_BAZEL_VERSION") + .expect("benchmark Bazel version"), kernel: command_output("uname", &["-srmo"]), cpu_identity: cpu_identity(), logical_cpu_count: std::thread::available_parallelism() diff --git a/crates/tracedecay/tests/daemon_suite/git_watch_test.rs b/crates/tracedecay/tests/daemon_suite/git_watch_test.rs index 9eac00a122..a0e03ff37f 100644 --- a/crates/tracedecay/tests/daemon_suite/git_watch_test.rs +++ b/crates/tracedecay/tests/daemon_suite/git_watch_test.rs @@ -120,7 +120,8 @@ fn symbol_count(payload: &Value, name: &str) -> usize { .count() } fn generation_index_len(data_root: &Path, project: &Path) -> usize { - let scope = scoped_code_index_store_root(&data_root.join("code-index-v1"), project); + let project = tracedecay_runtime_core::path_safety::canonical_root_identity(project); + let scope = scoped_code_index_store_root(&data_root.join("code-index-v1"), &project); let pointer: DurablePublicationPointerV1 = serde_json::from_slice( &fs::read(scope.join("active-code-generation-v1.json")).expect("active generation pointer"), ) diff --git a/crates/tracedecay/tests/dashboard_api_test/api.rs b/crates/tracedecay/tests/dashboard_api_test/api.rs index 1ebd1477c2..6cd62d8899 100644 --- a/crates/tracedecay/tests/dashboard_api_test/api.rs +++ b/crates/tracedecay/tests/dashboard_api_test/api.rs @@ -905,3 +905,122 @@ fn lcm_project_store_wins_over_global_accounting_override() { server.stop(); }); } + +#[test] +fn lcm_large_session_pages_preserve_continuation_and_timeline_counts() { + let runtime = create_runtime(); + runtime.block_on(async { + let fixture = start_dashboard_fixture_without_memory().await; + let session_id = "session-dashboard-paged"; + let session = SessionRecord { + provider: "cursor".to_owned(), + session_id: session_id.to_owned(), + project_key: fixture.host_runtime.project_id().as_str().to_owned(), + project_path: fixture.project_root.display().to_string(), + title: Some("Paged dashboard session".to_owned()), + started_at: Some(1_700_001_000), + ended_at: None, + transcript_path: None, + metadata_json: None, + parent_session_id: None, + is_subagent: false, + agent_id: None, + parent_tool_use_id: None, + }; + assert!( + fixture + .host_runtime + .upsert_session_for_test(HostAdmissionScope::Project, &session) + .await + .unwrap() + ); + for ordinal in 1..=70_u64 { + fixture + .host_runtime + .seed_session_message_observation_for_test( + tracedecay::dashboard::observation_seed::DashboardSessionMessageSeedV1 { + project_id: fixture.host_runtime.project_id().as_str(), + provider: "cursor", + session_id, + message_id: &format!("paged-message-{ordinal}"), + role: "user", + content: "A small canonical message.", + model: None, + timestamp: 1_700_001_000 + ordinal as i64, + ordinal, + }, + ) + .await + .unwrap(); + } + fixture + .host_runtime + .materialize_session_temporal_refresh_for_test(session_id) + .await + .unwrap(); + let agent = http_agent(); + let base = format!( + "{}/api/plugins/hermes-lcm/session/{session_id}?limit=100", + fixture.base_url + ); + let (status, first) = get_json(&agent, &base); + assert_eq!(status, 200); + let first_messages = first["payload"]["messages"] + .as_array() + .unwrap_or_else(|| panic!("large session must serve a bounded page: {first}")); + assert_eq!(first_messages.len(), 64); + assert_eq!(first["coverage"]["completeness"], "partial", "{first}"); + assert_eq!(first["coverage"]["examined"], 64, "{first}"); + assert_eq!(first["payload"]["counts"]["message_count"], 70, "{first}"); + let cursor = first["payload"]["next_cursor"] + .as_str() + .expect("bounded page continuation"); + let second_url = url::Url::parse_with_params(&base, &[("cursor", cursor)]).unwrap(); + let (status, second) = get_json(&agent, second_url.as_str()); + assert_eq!(status, 200); + let second_messages = second["payload"]["messages"] + .as_array() + .unwrap_or_else(|| panic!("continuation must serve remaining messages: {second}")); + assert_eq!(second_messages.len(), 6); + assert!(second["payload"]["next_cursor"].is_null()); + assert_eq!(second["coverage"]["completeness"], "complete", "{second}"); + for field in ["eligible", "examined", "matched"] { + assert_eq!(second["coverage"][field], 6, "{second}"); + } + assert_eq!(second["coverage"]["omitted"], 0, "{second}"); + assert_eq!(second["payload"]["counts"]["message_count"], 70, "{second}"); + let ids = first_messages + .iter() + .chain(second_messages) + .map(|message| message["message_id"].as_str().unwrap()) + .collect::>(); + assert_eq!( + ids.len(), + 70, + "continuation must neither skip nor repeat messages" + ); + let (status, timeline) = get_json( + &agent, + &format!( + "{}/api/plugins/hermes-lcm/timeline?bucket=day&limit=400", + fixture.base_url, + ), + ); + assert_eq!(status, 200); + assert!( + timeline["payload"].is_object(), + "timeline must drain bounded pages: {timeline}" + ); + assert_eq!(timeline["coverage"]["examined"], 70, "{timeline}"); + let counted: u64 = timeline["payload"]["buckets"] + .as_array() + .unwrap() + .iter() + .map(|bucket| bucket["count"].as_u64().unwrap()) + .sum(); + assert_eq!( + counted, 70, + "timeline buckets must include both canonical pages" + ); + }); +} diff --git a/crates/tracedecay/tests/dashboard_api_test/automation.rs b/crates/tracedecay/tests/dashboard_api_test/automation.rs index 811f259c26..e62e65ab7c 100644 --- a/crates/tracedecay/tests/dashboard_api_test/automation.rs +++ b/crates/tracedecay/tests/dashboard_api_test/automation.rs @@ -417,7 +417,6 @@ fn final_self_improvement_smoke_covers_autonomous_curation_and_skill_deployment( /// admission with the run's receipt, and the run settles on the daemon. #[test] fn fact_store_curate_answers_with_its_receipt_while_a_slow_curator_runs() { - const CURATOR_TURN: Duration = Duration::from_secs(3); let runtime = create_runtime(); runtime.block_on(async { let tmp = tempdir_or_panic(); @@ -430,10 +429,9 @@ fn fact_store_curate_answers_with_its_receipt_while_a_slow_curator_runs() { let (cg, host_runtime) = setup_project(&profile, &project_root).await; let fixture = seed_memory_fixture(&cg).await; - let slow_codex = FakeCodexAppServer::new_memory_curator_answering_after( + let slow_codex = FakeCodexAppServer::new_memory_curator_held( fixture.near_duplicate_fact_id.clone(), fixture.near_duplicate_last_event_id.clone(), - CURATOR_TURN, ); let project_id = cg .configuration_runtime() @@ -471,7 +469,6 @@ fn fact_store_curate_answers_with_its_receipt_while_a_slow_curator_runs() { assert_eq!(status, 200, "automation config patch failed: {config}"); let run_id = "request.dashboard.slow-curator-receipt"; - let started = Instant::now(); let response = agent .post(format!( "{base_url}/api/application/retained/fact_store_curate" @@ -482,7 +479,6 @@ fn fact_store_curate_answers_with_its_receipt_while_a_slow_curator_runs() { "min_confidence_millionths": 500_000 })) .unwrap_or_else(|error| panic!("POST fact_store_curate failed: {error}")); - let answered = started.elapsed(); let (status, receipt) = response_to_json(response); assert_eq!(status, 200, "fact_store_curate failed: {receipt}"); let payload = &receipt["value"]["outcome"]["value"]["payload"]; @@ -499,16 +495,11 @@ fn fact_store_curate_answers_with_its_receipt_while_a_slow_curator_runs() { receipt["value"]["outcome"]["value"]["reconciliation"], "pending" ); - assert!( - answered < Duration::from_millis(250), - "fact_store_curate answered after {answered:?}, behind a {CURATOR_TURN:?} curator" - ); - + // The backend cannot complete before release, so receiving this receipt + // proves admission does not await the curator, independent of CPU load. + slow_codex.wait_for_turn(); + slow_codex.release_turn(); let settled = wait_for_settled_run(&agent, &base_url, run_id); - assert!( - started.elapsed() >= CURATOR_TURN, - "the run settled before its curator answered" - ); assert_eq!(settled["status"], "succeeded", "{settled}"); assert_eq!(settled["trigger"], "application"); assert_eq!(settled["accepted_count"], 1); diff --git a/crates/tracedecay/tests/dashboard_api_test/dashboard_api_support.rs b/crates/tracedecay/tests/dashboard_api_test/dashboard_api_support.rs index 7da09a0c40..005a6fd66c 100644 --- a/crates/tracedecay/tests/dashboard_api_test/dashboard_api_support.rs +++ b/crates/tracedecay/tests/dashboard_api_test/dashboard_api_support.rs @@ -912,15 +912,13 @@ pub(crate) struct FakeCodexAppServer { impl FakeCodexAppServer { pub(crate) fn new_memory_curator(fact_id: FactId, last_event_id: FactEventId) -> Self { - Self::new_memory_curator_answering_after(fact_id, last_event_id, Duration::ZERO) + let server = Self::new_memory_curator_held(fact_id, last_event_id); + server.release_turn(); + server } - /// A curator backend that holds each turn for `delay` before answering. - pub(crate) fn new_memory_curator_answering_after( - fact_id: FactId, - last_event_id: FactEventId, - delay: Duration, - ) -> Self { + /// A curator backend whose answer waits for the test's explicit release. + pub(crate) fn new_memory_curator_held(fact_id: FactId, last_event_id: FactEventId) -> Self { let temp = tempdir_or_panic(); let script_path = temp.path().join("codex.py"); let bin = fake_codex_bin(temp.path()); @@ -948,7 +946,12 @@ for line in sys.stdin: "result": {"thread": {"id": "thread-dashboard", "model": "dashboard-fake-model"}} }), flush=True) elif method == "turn/start": - time.sleep(__TURN_DELAY_SECONDS__) + with open(__TURN_STARTED__, "w") as started: + started.write("started") + while not os.path.exists(__TURN_RELEASE__): + if not os.path.isdir(os.path.dirname(__TURN_RELEASE__)): + sys.exit(44) + time.sleep(0.01) payload = { "ops": [{ "op": "normalize_tags", @@ -981,11 +984,38 @@ for line in sys.stdin: &serde_json::to_string(last_event_id.as_str()) .unwrap_or_else(|error| panic!("encode fake curator event id: {error}")), ) - .replace("__TURN_DELAY_SECONDS__", &delay.as_secs_f64().to_string()); + .replace( + "__TURN_STARTED__", + &serde_json::to_string(&temp.path().join("turn-started")).unwrap(), + ) + .replace( + "__TURN_RELEASE__", + &serde_json::to_string(&temp.path().join("turn-release")).unwrap(), + ); write_file(&script_path, &script); install_fake_codex_launcher(&script_path, &bin); Self { _temp: temp, bin } } + + pub(crate) fn wait_for_turn(&self) { + crate::common::poll_until( + Instant::now() + Duration::from_secs(60), + Duration::from_millis(25), + || { + self._temp + .path() + .join("turn-started") + .try_exists() + .expect("read curator turn marker") + .then_some(()) + }, + || "curator did not reach its held turn".to_owned(), + ); + } + + pub(crate) fn release_turn(&self) { + write_file(&self._temp.path().join("turn-release"), "release"); + } } pub(crate) async fn start_dashboard_fixture(seed_lcm: bool) -> DashboardFixture { diff --git a/crates/tracedecay/tests/dashboard_api_test/runtime.rs b/crates/tracedecay/tests/dashboard_api_test/runtime.rs index feb1c5cd12..e4620d38f4 100644 --- a/crates/tracedecay/tests/dashboard_api_test/runtime.rs +++ b/crates/tracedecay/tests/dashboard_api_test/runtime.rs @@ -22,6 +22,8 @@ use tracedecay_graph_query::{ VerifiedCodeGraphRead, }; use tracedecay_host_admission::session_ingest_authority::GlobalDbSessionIngestAuthority; +use tracedecay_lcm::payload::PayloadFileRollback; +use tracedecay_lcm::raw::{commit_staged_raw_message, stage_raw_message_with_payload_tracked}; use tracedecay_project::project::{TraceDecay, TraceDecayOpenOptions}; use tracedecay_project::test_support::host_admission::ensure_process_background_cpu_authority; use tracedecay_runtime_core::config::ProfileRoot; @@ -519,19 +521,46 @@ impl DashboardTestRuntimeV1 { operation: "seed dashboard test session message".to_owned(), message: "registered session database has no storage root".to_owned(), })?; - for message in messages { - database - .lcm_ingest_raw_message(storage_root, message) - .await + // Keep fixture write leases bounded like the host projection drain. + for chunk in messages.chunks(32) { + let mut rollback = PayloadFileRollback::begin_cancellation_safe(storage_root); + let staged = chunk + .iter() + .map(|message| { + stage_raw_message_with_payload_tracked(storage_root, message, &mut rollback) + }) + .collect::, _>>() .map_err(|error| TraceDecayError::Database { operation: "seed dashboard test session message".to_owned(), message: error.to_string(), })?; + let transaction = database.begin_write_transaction().await?; + for (message, staged) in chunk.iter().zip(staged) { + commit_staged_raw_message(&transaction, message, staged) + .await + .map_err(|error| TraceDecayError::Database { + operation: "seed dashboard test session message".to_owned(), + message: error.to_string(), + })?; + } + transaction + .commit() + .await + .map_err(|error| TraceDecayError::Database { + operation: "commit dashboard test session messages".to_owned(), + message: error.to_string(), + })?; + rollback.disarm(); } let mut store_ids = Vec::with_capacity(messages.len()); for message in messages { - let raw = load_registered_raw_message(database, &message.provider, &message.message_id) + let store_id = database + .lcm_raw_message_store_id(&message.provider, &message.message_id) .await + .map_err(|error| TraceDecayError::Database { + operation: "read dashboard test transcript store id".to_owned(), + message: error.to_string(), + })? .ok_or_else(|| TraceDecayError::Database { operation: "read dashboard test transcript store id".to_owned(), message: format!( @@ -539,7 +568,7 @@ impl DashboardTestRuntimeV1 { message.provider, message.message_id ), })?; - store_ids.push(raw.store_id); + store_ids.push(store_id); } Ok(store_ids) } @@ -687,20 +716,6 @@ fn dashboard_test_graph_error(operation: &str, error: impl Display) -> TraceDeca } } -async fn load_registered_raw_message( - database: &RegisteredGlobalDb, - provider: &str, - message_id: &str, -) -> Option { - let snapshot = database - .read_snapshot() - .await - .expect("dashboard test raw-message snapshot must remain registered"); - tracedecay_lcm::schema::load_raw_message(&snapshot, provider, message_id) - .await - .expect("dashboard test raw-message load must not hide database or receipt failure") -} - /// Gives the fixture root the same registered-repository identity a /// production project gains at registration: a real git repository whose /// common directory carries the authoritative repository identity marker. diff --git a/crates/tracedecay/tests/hooks_lsp_suite/hook_branch_routing_test.rs b/crates/tracedecay/tests/hooks_lsp_suite/hook_branch_routing_test.rs index fff30a04eb..abb7e7fc7c 100644 --- a/crates/tracedecay/tests/hooks_lsp_suite/hook_branch_routing_test.rs +++ b/crates/tracedecay/tests/hooks_lsp_suite/hook_branch_routing_test.rs @@ -32,6 +32,7 @@ fn git(project: &Path, args: &[&str]) { ); } +#[cfg(unix)] fn git_head_oid(project: &Path) -> String { let output = Command::new("git") .args(["rev-parse", "HEAD"]) @@ -49,6 +50,7 @@ fn git_head_oid(project: &Path) -> String { /// anywhere. A subsequent write on another branch rolls the store to a newer /// generation under the new ref while the first branch keeps its sealed /// provenance record. +#[cfg(unix)] #[tokio::test] async fn hook_branch_write_lands_in_a_sealed_single_store_generation() { let dir = TempDir::new().unwrap(); @@ -299,3 +301,37 @@ async fn hook_branch_write_lands_in_a_sealed_single_store_generation() { "no write may create a per-branch database" ); } + +#[cfg(not(unix))] +#[tokio::test] +async fn hook_branch_write_refuses_when_scheduler_authority_is_unavailable() { + let dir = TempDir::new().unwrap(); + let temp_root = canonical_temp_path(dir.path()); + let project = temp_root.join("project"); + std::fs::create_dir_all(&project).unwrap(); + git(&project, &["init", "-b", "main"]); + let harness = ProductionProjectCompositionHarnessV1::open(&temp_root, [project.clone()]) + .await + .unwrap(); + let shard_root = harness.project_data_root(&project).await.unwrap(); + let before = tracedecay_runtime_core::branch_meta::load_branch_meta(&shard_root); + let error = harness + .track_worktree_branch(&project, &project, "feature/hook") + .await + .expect_err("branch activation requires the scheduler authority"); + assert!(matches!( + error, + tracedecay_domain::TraceDecayError::ProjectRoute { + reason_code, + retryable: true, + .. + } if reason_code == "code_index_scheduler_unavailable" + )); + let after = tracedecay_runtime_core::branch_meta::load_branch_meta(&shard_root); + assert_eq!( + serde_json::to_value(before).unwrap(), + serde_json::to_value(after).unwrap() + ); + assert!(!shard_root.join("branches").exists()); + harness.shutdown().await; +} diff --git a/crates/tracedecay/tests/hooks_lsp_suite/hook_lifecycle_lease_test.rs b/crates/tracedecay/tests/hooks_lsp_suite/hook_lifecycle_lease_test.rs index 608ee40df3..bd59095cb2 100644 --- a/crates/tracedecay/tests/hooks_lsp_suite/hook_lifecycle_lease_test.rs +++ b/crates/tracedecay/tests/hooks_lsp_suite/hook_lifecycle_lease_test.rs @@ -3,6 +3,10 @@ use std::path::Path; use std::process::{Command, Output, Stdio}; use std::time::{SystemTime, UNIX_EPOCH}; +use tracedecay_runtime_core::lifecycle_lease::{ + ExclusiveLeaseAttempt, try_acquire_exclusive_for_profile, +}; + use super::common::{ apply_tracedecay_home_env, git_program, spawn_tracedecay_daemon, tracedecay_command_with_home, }; @@ -499,7 +503,17 @@ fn native_hook_captures_only_bound_transport_spool_records() { assert_eq!(replay.stdout, expected_stdout, "{hook} replay: {replay:?}"); assert!(replay.stderr.is_empty(), "{hook} replay: {replay:?}"); } - assert!(!home.join(".tracedecay/lifecycle.lock").exists()); + assert!(home.join(".tracedecay/lifecycle.lock").is_file(), "{hook}"); + let maintenance = try_acquire_exclusive_for_profile( + &home.join(".tracedecay"), + "verify completed hook released its lease", + ) + .unwrap(); + assert!( + matches!(maintenance, ExclusiveLeaseAttempt::Acquired(_)), + "{hook} retained its lifecycle lease after exit" + ); + drop(maintenance); assert!(!home.join(".tracedecay/global.db").exists()); assert!(!data_root.join("tracedecay.db").exists()); assert!(!data_root.join("sessions.db").exists()); diff --git a/crates/tracedecay/tests/mcp_suite/diagnostics_read_test.rs b/crates/tracedecay/tests/mcp_suite/diagnostics_read_test.rs index 6621a80664..ac6dd468d3 100644 --- a/crates/tracedecay/tests/mcp_suite/diagnostics_read_test.rs +++ b/crates/tracedecay/tests/mcp_suite/diagnostics_read_test.rs @@ -193,6 +193,20 @@ async fn moved_source_proof_reads_the_publication_as_stale_until_renewed() { let published = await_published_diagnostics(&server, arguments.clone()).await; assert_eq!(published_records(&published).len(), 1, "{published}"); + let scheduler = fixture + .harness + .project_scheduler_handle(&fixture.project_root) + .await + .expect("mounted project scheduler"); + let (held_tx, held_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel::<()>(); + let holding = tokio::task::spawn_blocking(move || { + let _guard = scheduler.lock().expect("hold source-proof renewal"); + held_tx.send(()).expect("scheduler held"); + let _ = release_rx.blocking_recv(); + }); + held_rx.await.expect("source-proof renewal is fenced"); + let index = std::fs::File::options() .write(true) .open(fixture.project_root.join(".git/index")) @@ -202,7 +216,12 @@ async fn moved_source_proof_reads_the_publication_as_stale_until_renewed() { .expect("move the git index mtime"); drop(index); - let mut codes = Vec::new(); + let stale = + handle_real_server_tool_call(&server, "tracedecay_diagnostics", arguments.clone()).await; + drop(release_tx); + holding.await.expect("release source-proof renewal"); + assert_eq!(stale["isError"], json!(true), "{stale}"); + let mut codes = vec![stale["structuredContent"]["problem"]["code"].clone()]; let renewed = loop { let result = handle_real_server_tool_call(&server, "tracedecay_diagnostics", arguments.clone()) diff --git a/crates/tracedecay/tests/mcp_suite/fixture.rs b/crates/tracedecay/tests/mcp_suite/fixture.rs index 9ae8078581..5e0b732a45 100644 --- a/crates/tracedecay/tests/mcp_suite/fixture.rs +++ b/crates/tracedecay/tests/mcp_suite/fixture.rs @@ -92,7 +92,7 @@ pub fn write_indexed_fixture_sources(project: &Path) { fs::create_dir_all(project.join("src")).unwrap(); fs::write( - project.join("src/main.rs"), + project.join("src").join("main.rs"), r#" use crate::utils::helper; mod utils; @@ -106,7 +106,7 @@ fn main() { .unwrap(); fs::write( - project.join("src/utils.rs"), + project.join("src").join("utils.rs"), r#" /// Returns a greeting string. pub fn helper() -> String { @@ -123,7 +123,7 @@ fn format_greeting(name: &str) -> String { // Test file so affected-tests can find something fs::create_dir_all(project.join("tests")).unwrap(); fs::write( - project.join("tests/test_utils.rs"), + project.join("tests").join("test_utils.rs"), r#" use crate::utils::helper; @@ -188,7 +188,7 @@ impl SeedTargets { /// rewrites the absolute paths embedded in `store_manifest.json`. fn seed_store(flavor: &Path, project_root: &Path, targets: &SeedTargets) -> io::Result<()> { fs::create_dir_all(project_root)?; - let src_home = flavor.join("home/.tracedecay"); + let src_home = flavor.join("home").join(".tracedecay"); let src_data = sole_subdir(&src_home.join("projects"))?; let project_id = default_profile_project_id(project_root); @@ -303,7 +303,7 @@ async fn build_template(dest: &Path) -> io::Result<()> { let project = root.join("project"); fs::create_dir_all(&project)?; - let profile_root = root.join("home/.tracedecay"); + let profile_root = root.join("home").join(".tracedecay"); let global_db_path = profile_root.join("global.db"); let options = TraceDecayOpenOptions { profile_root: Some(profile_root.clone()), @@ -558,7 +558,9 @@ async fn changing_a_schema_column_constant_does_not_reuse_the_template() { let global_db = selected .join(EMPTY_FLAVOR) - .join("home/.tracedecay/global.db"); + .join("home") + .join(".tracedecay") + .join("global.db"); let connection = Connection::open(&global_db).unwrap(); let mut statement = connection .prepare("SELECT name FROM pragma_table_info('graph_scopes') ORDER BY cid") @@ -594,9 +596,15 @@ async fn template_recorded_at_an_older_git_correlation_version_is_rebuilt() { // What a build before the Git correlation bump left in a warm target. let stale = ensure_template(&tmp_root, older).await.unwrap(); - let stale_sessions = sole_subdir(&stale.join(EMPTY_FLAVOR).join("home/.tracedecay/projects")) - .unwrap() - .join(tracedecay_runtime_core::storage::SESSIONS_DB_FILENAME); + let stale_sessions = sole_subdir( + &stale + .join(EMPTY_FLAVOR) + .join("home") + .join(".tracedecay") + .join("projects"), + ) + .unwrap() + .join(tracedecay_runtime_core::storage::SESSIONS_DB_FILENAME); let downgraded = Connection::open(&stale_sessions) .unwrap() .execute( @@ -610,7 +618,7 @@ async fn template_recorded_at_an_older_git_correlation_version_is_rebuilt() { ); let template = ensure_template(&tmp_root, current).await.unwrap(); - let profile_root = tmp_root.join("home/.tracedecay"); + let profile_root = tmp_root.join("home").join(".tracedecay"); let cg = match init_project_from_template_root( Some(&template), &tmp_root.join("project"), @@ -649,7 +657,9 @@ async fn template_seed_opens_the_copied_store_for_the_requested_project() { let template_store = sole_subdir( &template .join(EMPTY_FLAVOR) - .join("home/.tracedecay/projects"), + .join("home") + .join(".tracedecay") + .join("projects"), ) .unwrap(); let template_manifest: Value = serde_json::from_str( @@ -668,7 +678,7 @@ async fn template_seed_opens_the_copied_store_for_the_requested_project() { let project = tmp_root.join("requested-project"); fs::create_dir_all(&project).unwrap(); - let profile_root = tmp_root.join("profile/.tracedecay"); + let profile_root = tmp_root.join("profile").join(".tracedecay"); let cg = init_project_from_template_root( Some(&template), &project, @@ -713,7 +723,7 @@ async fn template_seed_does_not_fall_back_when_the_destination_exists() { .expect("template builds from a real init"); let project = tmp_root.join("requested-project"); fs::create_dir_all(&project).unwrap(); - let profile_root = tmp_root.join("profile/.tracedecay"); + let profile_root = tmp_root.join("profile").join(".tracedecay"); let project_id = default_profile_project_id(&project); fs::create_dir_all(profile_root.join("projects").join(&project_id)).unwrap(); diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/active_project_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/active_project_test.rs index e0528ea3e4..d321729e50 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/active_project_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/active_project_test.rs @@ -231,10 +231,8 @@ async fn open_checkout() -> OpenedCheckout { ); let project_root = canonical_existing_identity(&project_root).expect("canonical project"); - let isolation_root = isolation - .path() - .canonicalize() - .expect("canonical isolation"); + let isolation_root = + canonical_existing_identity(isolation.path()).expect("canonical isolation"); let harness = ProductionProjectCompositionHarnessV1::open_with_scope_prefix( &isolation_root, [project_root.clone()], diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_describe_behavior.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_describe_behavior.rs index ffbabeebf1..c1c616316d 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_describe_behavior.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/lcm_describe_behavior.rs @@ -570,24 +570,24 @@ fn session_temporal() -> Value { fn source_coverage(mode: &str) -> Value { json!({ - "committed_frontier": 3, + "committed_frontier": 2, "covered_intervals": [], "missing_intervals": [], - "observed_frontier": 3, + "observed_frontier": 2, "reason": {"kind": "caught_up"}, "request": {"mode": {"kind": mode}}, "source_id": "orchard-describe:cursor", "state": "fresh", - "target_watermark": 3 + "target_watermark": 2 }) } fn watermarks() -> Value { json!({ "generation": 3, - "index": 3, - "projection": 3, - "source": 3, + "index": 2, + "projection": 2, + "source": 2, "summary": 1 }) } diff --git a/crates/tracedecay/tests/mcp_suite/support.rs b/crates/tracedecay/tests/mcp_suite/support.rs index 84c694abab..41e68f172d 100644 --- a/crates/tracedecay/tests/mcp_suite/support.rs +++ b/crates/tracedecay/tests/mcp_suite/support.rs @@ -1693,8 +1693,8 @@ pub(crate) async fn persist_temporal_lcm_observation_with_access( }; let source = ObservationSourceIdentityV1::for_provider(provider.clone(), session_id.clone()).unwrap(); - let source_frontier = u64::try_from(ordinal).unwrap().saturating_add(1); - let range = ObservationSourceRangeV1::new(source_frontier - 1, source_frontier).unwrap(); + let source_end = u64::try_from(ordinal).unwrap().saturating_add(1); + let range = ObservationSourceRangeV1::new(source_end - 1, source_end).unwrap(); let stable_record_id = ObservationId::new(format!("record.mcp.{session_id}.{message_id}")).unwrap(); let relations = CanonicalObservationRelationsV1::new(session_id.clone()) @@ -1795,7 +1795,7 @@ pub(crate) async fn persist_temporal_lcm_observation_with_access( durability: base_anchor.durability().clone(), }) .unwrap(); - observation_store + let committed = observation_store .persist_observation( AnchoredObservationWrite::new(write, anchor.clone(), projection_generation).unwrap(), ) @@ -1847,7 +1847,7 @@ pub(crate) async fn persist_temporal_lcm_observation_with_access( .unwrap(); TemporalLcmProjectionInput { occurrence, - source_frontier, + source_frontier: committed.receipt().sequence(), } } diff --git a/crates/tracedecay/tests/session_suite/temporal_benchmark_runner.rs b/crates/tracedecay/tests/session_suite/temporal_benchmark_runner.rs index aac629998f..ee864acc7d 100644 --- a/crates/tracedecay/tests/session_suite/temporal_benchmark_runner.rs +++ b/crates/tracedecay/tests/session_suite/temporal_benchmark_runner.rs @@ -7,13 +7,11 @@ use tracedecay_runtime_core::test_executable::write_executable_script; use tempfile::TempDir; const RUNNER_PATH: &str = "scripts/run-session-temporal-benchmark.sh"; -const FAKE_CARGO_STATUS: i32 = 47; +const FAKE_BAZEL_STATUS: i32 = 47; struct RunnerInvocation { output: Output, - cargo_receipt: Option, - parent_data: PathBuf, - parent_home: PathBuf, + bazel_receipt: Option, } fn write_executable(path: &Path, body: &str) { @@ -24,15 +22,15 @@ fn invoke_runner(uname: &str, mode: &str) -> RunnerInvocation { let temp = TempDir::new().expect("runner tempdir"); let fake_bin = temp.path().join("bin"); fs::create_dir_all(&fake_bin).expect("create fake bin"); - let cargo_receipt_path = temp.path().join("cargo-receipt.txt"); + let bazel_receipt_path = temp.path().join("bazel-receipt.txt"); write_executable( &fake_bin.join("uname"), "#!/bin/sh\nprintf '%s\\n' \"$FAKE_UNAME\"\n", ); write_executable( - &fake_bin.join("cargo"), - "#!/bin/sh\n{\n printf 'argv='\n printf '<%s>' \"$@\"\n printf '\\nHOME=<%s>\\n' \"$HOME\"\n printf 'TRACEDECAY_DATA_DIR=<%s>\\n' \"$TRACEDECAY_DATA_DIR\"\n} >\"$FAKE_CARGO_RECEIPT\"\nexit 47\n", + &fake_bin.join("bazel"), + "#!/bin/sh\n{\n printf 'argv='\n printf '<%s>' \"$@\"\n printf '\\nHOME=<%s>\\n' \"$HOME\"\n printf 'TRACEDECAY_DATA_DIR=<%s>\\n' \"$TRACEDECAY_DATA_DIR\"\n} >\"$FAKE_BAZEL_RECEIPT\"\nexit 47\n", ); let parent_home = temp.path().join("parent-home"); @@ -49,7 +47,7 @@ fn invoke_runner(uname: &str, mode: &str) -> RunnerInvocation { .arg(mode) .current_dir(repository_root()) .env("CARGO_HOME", temp.path().join("cargo-home")) - .env("FAKE_CARGO_RECEIPT", &cargo_receipt_path) + .env("FAKE_BAZEL_RECEIPT", &bazel_receipt_path) .env("FAKE_UNAME", uname) .env("HOME", &parent_home) .env("PATH", fake_path) @@ -59,64 +57,44 @@ fn invoke_runner(uname: &str, mode: &str) -> RunnerInvocation { .output() .expect("run session-temporal runner"); - let cargo_receipt = match fs::read_to_string(&cargo_receipt_path) { + let bazel_receipt = match fs::read_to_string(&bazel_receipt_path) { Ok(receipt) => Some(receipt), Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, - Err(error) => panic!("read cargo receipt: {error}"), + Err(error) => panic!("read Bazel receipt: {error}"), }; RunnerInvocation { output, - cargo_receipt, - parent_data, - parent_home, + bazel_receipt, } } fn repository_root() -> PathBuf { - crate::common::repository_root().to_path_buf() + crate::common::repository_root() + .canonicalize() + .expect("canonical fixture repository root") } #[test] -fn diagnostic_runner_reaches_cargo_on_linux_and_macos() { +fn diagnostic_runner_reaches_bazel_on_linux_and_macos() { for (uname, platform) in [("Linux", "linux"), ("Darwin", "macos")] { let invocation = invoke_runner(uname, "--run"); assert_eq!( invocation.output.status.code(), - Some(FAKE_CARGO_STATUS), + Some(FAKE_BAZEL_STATUS), "{platform} runner output: {}", String::from_utf8_lossy(&invocation.output.stderr) ); let receipt = invocation - .cargo_receipt - .expect("diagnostic runner must execute cargo"); - assert!( - // `eebf0957a` retargeted the shipped runner after the crate move: - // the repository root is a virtual workspace, so `cargo bench` must - // name `-p tracedecay` to resolve the harness at all. - receipt.starts_with( - "argv=<-p><--bench><--all-features><--><--run>\n" + .bazel_receipt + .expect("diagnostic runner must execute Bazel"); + assert_eq!( + receipt.lines().next().expect("Bazel argument receipt"), + format!( + "argv=<--config=release><--config=ci><--test_strategy=standalone><--nocache_test_results><--test_output=all><--test_env=TRACEDECAY_BENCHMARK_REPO_ROOT={}><--test_arg=--bench><--test_arg=--run>", + repository_root().display() ), - "{platform} cargo receipt: {receipt}" - ); - assert!( - receipt.contains("HOME=<"), - "{platform} cargo receipt: {receipt}" - ); - assert!( - !receipt.contains(&format!("HOME=<{}>", invocation.parent_home.display())), - "{platform} runner must isolate HOME: {receipt}" - ); - assert!( - receipt.contains("TRACEDECAY_DATA_DIR=<"), - "{platform} cargo receipt: {receipt}" - ); - assert!( - !receipt.contains(&format!( - "TRACEDECAY_DATA_DIR=<{}>", - invocation.parent_data.display() - )), - "{platform} runner must isolate TRACEDECAY_DATA_DIR: {receipt}" + "{platform} Bazel receipt: {receipt}" ); } } diff --git a/crates/tracedecay/tests/transcript_ingest_suite/projector_upgrade.rs b/crates/tracedecay/tests/transcript_ingest_suite/projector_upgrade.rs index 5d1360e104..1f1fa4a4f1 100644 --- a/crates/tracedecay/tests/transcript_ingest_suite/projector_upgrade.rs +++ b/crates/tracedecay/tests/transcript_ingest_suite/projector_upgrade.rs @@ -102,7 +102,11 @@ fn reset_store(db_path: &Path) { for suffix in ["-wal", "-shm"] { let mut sidecar = db_path.as_os_str().to_owned(); sidecar.push(suffix); - std::fs::remove_file(sidecar).ok(); + match std::fs::remove_file(sidecar) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => panic!("remove closed store {suffix}: {error}"), + } } } @@ -153,9 +157,16 @@ async fn shipped_store_is_reset_required_and_a_reset_store_serves_both_messages( search_texts(&db, "claude", "claudeprobe").await, [CLAUDE_TEXT] ); - drop(db); + db.shutdown() + .await + .expect("close stores before shaping the shipped schema"); shape_store_as_shipped(&db_path); + // Retain the profile owner across the refused project mount so its + // partially opened workers can be joined before the offline reset. + let profile = HostAdmissionTestRuntimeV1::profile(&profile_root) + .await + .expect("mount the current profile around the shipped project store"); let refused = HostAdmissionTestRuntimeV1::project(&profile_root, &project, project_id.clone()) .await .err() @@ -184,6 +195,10 @@ async fn shipped_store_is_reset_required_and_a_reset_store_serves_both_messages( "a refused store is left for the operator's reset, never rewritten" ); + profile + .shutdown() + .await + .expect("close refused project mount before reset"); reset_store(&db_path); let reset = open_project_session_db(&project).await.unwrap(); ingest_both(&reset, &home, &project, &cursor_transcript).await; diff --git a/crates/tracedecay/tests/transcript_ingest_suite/restart_atomicity.rs b/crates/tracedecay/tests/transcript_ingest_suite/restart_atomicity.rs index d5677097c6..09958db2b1 100644 --- a/crates/tracedecay/tests/transcript_ingest_suite/restart_atomicity.rs +++ b/crates/tracedecay/tests/transcript_ingest_suite/restart_atomicity.rs @@ -70,6 +70,10 @@ pub(super) struct ProjectSessionTestRuntime { } impl ProjectSessionTestRuntime { + pub(super) async fn shutdown(self) -> tracedecay_domain::errors::Result<()> { + self.runtime.shutdown().await + } + pub(super) fn runtime(&self) -> &HostAdmissionTestRuntimeV1 { &self.runtime } diff --git a/crates/tracedecay/tests/transcript_ingest_suite/support.rs b/crates/tracedecay/tests/transcript_ingest_suite/support.rs index 5cd111a6f1..44e76ad13a 100644 --- a/crates/tracedecay/tests/transcript_ingest_suite/support.rs +++ b/crates/tracedecay/tests/transcript_ingest_suite/support.rs @@ -48,8 +48,9 @@ pub fn assert_metadata_path_eq(actual: &serde_json::Value, expected: &Path) { pub fn assert_sanitized_path_text_eq(actual: &str, expected: &Path) { const REDACTED: &str = "[TraceDecay redacted: high-entropy token]"; let expected_text = normalize_path_text(&expected.to_string_lossy()); + let actual_text = normalize_path_text(actual); let mut cursor = 0; - for part in actual.split(REDACTED).filter(|part| !part.is_empty()) { + for part in actual_text.split(REDACTED).filter(|part| !part.is_empty()) { let found = expected_text[cursor..] .find(part) .map(|index| cursor + index); diff --git a/crates/tracedecay/tests/transport_acceptance_suite/v2_surface_mount_conformance.rs b/crates/tracedecay/tests/transport_acceptance_suite/v2_surface_mount_conformance.rs index 33c096826b..8d8d21dbb5 100644 --- a/crates/tracedecay/tests/transport_acceptance_suite/v2_surface_mount_conformance.rs +++ b/crates/tracedecay/tests/transport_acceptance_suite/v2_surface_mount_conformance.rs @@ -57,7 +57,7 @@ use tracedecay_api::{ is_http_application_operation_exposed, }; use tracedecay_contracts::catalog_composition::build_application_catalog_snapshot; -use tracedecay_runtime_core::ast_grep::AST_GREP_BIN_ENV; +use tracedecay_runtime_core::ast_grep::{AST_GREP_BIN_ENV, ast_grep_command}; use tracedecay_session_memory::event_lane::ActivityFamilyV1; use tracedecay_tool_catalog::{ ApplicationSurfaceOperation, BindingSurface, CapabilityManifestV1, CatalogSnapshotV1, @@ -193,7 +193,7 @@ impl MountFixture { common::initialize_tracedecay_cli_project(&home, &project); let daemon = common::spawn_tracedecay_daemon_with(&home, |command| { - command.env(AST_GREP_BIN_ENV, ast_grep_executable()); + command.env(AST_GREP_BIN_ENV, ast_grep_command().get_program()); }); let authority = wait_for_http_authority(&common::daemon_authority_path(&profile)); @@ -244,29 +244,14 @@ impl MountFixture { fn isolated_command(home: &Path) -> Command { let mut command = common::tracedecay_command_with_home(home); + // Isolated children retain only system PATH entries. Pass the executable + // selected by the shared resolver, including npm's native Windows binary. command .env("TRACEDECAY_TEST_ALLOW_INCOMPLETE_HOLDER_SCAN", "1") - .env(AST_GREP_BIN_ENV, ast_grep_executable()); + .env(AST_GREP_BIN_ENV, ast_grep_command().get_program()); command } -/// The ast-grep executable the source-edit bindings are graded against. -/// -/// A fixture child's PATH is the system directories only, and an npm-installed -/// ast-grep never lives there, so the child is handed the test process's own -/// executable through the product's explicit override instead. -fn ast_grep_executable() -> PathBuf { - let name = format!("ast-grep{}", std::env::consts::EXE_SUFFIX); - std::env::var_os(AST_GREP_BIN_ENV) - .map(PathBuf::from) - .or_else(|| { - std::env::split_paths(&std::env::var_os("PATH")?) - .map(|dir| dir.join(&name)) - .find(|path| path.is_file()) - }) - .expect("ast-grep must be on the test PATH to grade the ast-grep bindings") -} - /// The dashboard server's closed operation->route table, restated in the test /// so the probe does not derive its routes from the catalog it grades. /// diff --git a/dashboard/src/data/query/envelope.ts b/dashboard/src/data/query/envelope.ts index 0eaf93c35f..8aa6f31b88 100644 --- a/dashboard/src/data/query/envelope.ts +++ b/dashboard/src/data/query/envelope.ts @@ -7,7 +7,7 @@ import { readOnlyScopeRefusal } from '../scope/store.ts'; * states rather than exceptions: the UI always has a state to render. */ export type EnvelopeResult = | { outcome: 'envelope'; envelope: DashboardEnvelopeV1 } - | { outcome: 'transport'; state: DashboardDomainStateV1; detail?: string }; + | { outcome: 'transport'; state: DashboardDomainStateV1; detail?: string; responseReceived?: true }; /** Fetches and decodes a DashboardEnvelopeV1 from a daemon API route. * - network failure → offline @@ -81,6 +81,7 @@ function decodeEnvelopeBody( outcome: 'transport', state: envelope.domain_state, detail: envelope.coverage.omission_reasons[0], + responseReceived: true, }; } return { outcome: 'envelope', envelope: envelope as DashboardEnvelopeV1 }; diff --git a/dashboard/src/ui/ReadSection.dom.test.tsx b/dashboard/src/ui/ReadSection.dom.test.tsx new file mode 100644 index 0000000000..1c17149d17 --- /dev/null +++ b/dashboard/src/ui/ReadSection.dom.test.tsx @@ -0,0 +1,57 @@ +import { cleanup, render, screen } from '@testing-library/react'; +import { afterEach, expect, it, vi } from 'vitest'; +import { z } from 'zod'; + +import { fetchEnvelope } from '../data/query/envelope.ts'; +import { fixtureEnvelope } from '../test/fixtureEnvelope.ts'; +import { envelopeReadState, ReadSection } from './ReadSection.tsx'; + +afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); +}); + +it.each(['lcm_temporal_budget_execution_work_exhausted', undefined])( + 'renders a received unknown response with reason %s without claiming lost connectivity', async (reason) => { + const envelope = fixtureEnvelope(null, 'unknown'); + envelope['coverage'] = { + ...(envelope['coverage'] as Record), + omission_reasons: reason ? [reason] : [], + }; + vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify(envelope)))); + const result = await fetchEnvelope('/api/sessions/timeline', z.unknown()); + const child = vi.fn(() =>

Timeline records
); + render( + + {child} + , + ); + + expect(screen.getByText('Unknown')).toBeTruthy(); + expect(screen.getByText(`· ${reason ?? 'no usable result was provided'}`)).toBeTruthy(); + expect(screen.getByText(/The daemon responded without a usable result/)).toBeTruthy(); + expect(screen.queryByText(/No response has been recorded/)).toBeNull(); + expect(screen.queryByText(/Refresh once the daemon is serving/)).toBeNull(); + expect(screen.queryByText(/daemon unreachable/)).toBeNull(); + expect(child).not.toHaveBeenCalled(); + }, +); + +it('keeps an unobserved read distinct from a received unknown response', () => { + render( + + {() =>
Timeline records
} + , + ); + + expect(screen.getByText(/No response has been recorded/)).toBeTruthy(); + expect(screen.queryByText(/The daemon responded without a usable result/)).toBeNull(); +}); diff --git a/dashboard/src/ui/ReadSection.tsx b/dashboard/src/ui/ReadSection.tsx index 573c7f4a29..1b0cee6837 100644 --- a/dashboard/src/ui/ReadSection.tsx +++ b/dashboard/src/ui/ReadSection.tsx @@ -31,6 +31,8 @@ export type ReadState state: BlockedState; /** Whatever the source said about this state, and nothing where it said nothing. */ detail?: string | undefined; + /** A decoded response can report unknown without supplying a payload. */ + responseReceived?: boolean | undefined; /** * The decoded body a *source-level* refusal still carried. * @@ -112,7 +114,10 @@ export function envelopeReadState( return { kind: 'blocked', state: result.state, - detail: result.detail ?? details.transport ?? 'daemon unreachable', + detail: result.detail ?? (result.responseReceived + ? 'no usable result was provided' + : details.transport ?? 'daemon unreachable'), + responseReceived: result.responseReceived, }; } return { kind: 'ready', value: result.envelope }; @@ -150,7 +155,7 @@ export function ReadSection({ return state.kind === 'ready' ? ( <>{children(state.value)} ) : ( - + ); } return ( @@ -315,12 +320,19 @@ export function CenteredState({ title, kind, detail, + responseReceived, }: { title: string; kind: DomainStateKind; detail?: string | undefined; + responseReceived?: boolean | undefined; }) { - const guidance = STATE_GUIDANCE[kind]; + const guidance = kind === 'unknown' && responseReceived + ? { + sentence: 'The daemon responded without a usable result for this read.', + action: 'The reported reason describes why no result is shown.', + } + : STATE_GUIDANCE[kind]; // A dead channel on an instrument still shows its ruled field and its bezel: // the reader can see the surface is present and simply carrying no signal. // Sized by its CONTAINER, not the viewport: this plate renders both on diff --git a/evals/memory/BUILD.bazel b/evals/memory/BUILD.bazel new file mode 100644 index 0000000000..af098d6a49 --- /dev/null +++ b/evals/memory/BUILD.bazel @@ -0,0 +1,5 @@ +filegroup( + name = "scenarios", + srcs = glob(["scenarios/*.json"]), + visibility = ["//crates/tracedecay:__pkg__"], +) diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 8f85a21177..db3d4721cc 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -3,6 +3,7 @@ load("//scripts:bazel/rust_test.bzl", "rust_test") exports_files( [ "bazel/rust_test.bzl", + "bazel/gen_builds.py", "bazel/rust_test_environment_probe.txt", "check-dashboard-bundle.py", ], @@ -18,3 +19,25 @@ rust_test( }, use_libtest_harness = False, ) + +# BEGIN generated by scripts/bazel/gen_builds.py +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(["bazel/rust_test_environment_probe.rs"]).to_list(), + tags = ["manual"], +) + + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) + +# END generated by scripts/bazel/gen_builds.py diff --git a/scripts/bazel/gen_builds.py b/scripts/bazel/gen_builds.py old mode 100644 new mode 100755 index a9dd631da3..10ffeb9eff --- a/scripts/bazel/gen_builds.py +++ b/scripts/bazel/gen_builds.py @@ -42,7 +42,7 @@ BANNER = ( "# Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock.\n" - "# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py\n" + "# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py\n" ) # Generated filegroups embedded in a handwritten BUILD.bazel live inside @@ -663,7 +663,7 @@ def owner_pkg(repo_rel): return str(Path(*parts[:2])), str(Path(*parts[2:-1])) return parts[0], str(Path(*parts[1:-1])) - def group_label(repo_rel): + def group_label(repo_rel, *, sources=False): """filegroup label + pkg-relative file label for an out-of-package file, registering the group for emission in the owning package.""" pkg_dir, group_rel = owner_pkg(repo_rel) @@ -673,6 +673,12 @@ def group_label(repo_rel): gname = "shared_fixtures" entry = out_of_pkg.setdefault((pkg_dir, gname), {"files": set(), "glob": group_rel}) entry["files"].add(pkg_rel) + if sources: + gname += "_sources" + entry = out_of_pkg.setdefault( + (pkg_dir, gname), {"files": set(), "glob": group_rel, "sources": True} + ) + entry["files"].add(pkg_rel) return f"//{pkg_dir}:{gname}", pkg_dir, pkg_rel def srcs_for(p, t): @@ -682,10 +688,13 @@ def srcs_for(p, t): rel = rel_src(p, t) if rel.startswith(".."): repo_rel = Path(t["src_path"]).resolve().relative_to(REPO) - glabel, pkg_dir, pkg_rel = group_label(repo_rel) + glabel, pkg_dir, pkg_rel = group_label(repo_rel, sources=True) return (f'["{glabel}"]', f'"//{pkg_dir}:{pkg_rel}"') d = Path(rel).parent - return (f'glob(["{d}/**/*.rs"])', f'"{rel}"') + # Package-root fixtures are data, including deliberately invalid or + # vendored Rust. Nested support/fixtures modules remain source inputs. + exclude = ', exclude = ["tests/fixtures/**"]' if d == Path("tests") else "" + return (f'glob(["{d}/**/*.rs"]{exclude})', f'"{rel}"') def extra_mod_srcs(p, t, covered=()): """(pkg_relative_srcs, filegroup_labels) for `#[path = "..."]` modules @@ -722,7 +731,7 @@ def extra_mod_srcs(p, t, covered=()): + "/**/*.rs" ) continue - glabel, _, _ = group_label(repo_rel) + glabel, _, _ = group_label(repo_rel, sources=True) labels.add(glabel) return sorted(local), sorted(labels), sorted(pulled_dirs) @@ -791,8 +800,52 @@ def needs_env_tmpdir(p, t, extra_dirs=()): for _, text in target_sources(t, tuple(extra_dirs)) ) + formatting_targets = [] + + def formatting_rules(package, sources_by_edition): + rules = [] + for edition, expressions in sorted(sources_by_edition.items()): + sources = f"rustfmt_sources_{edition}" + check = f"rustfmt_check_{edition}" + formatting_targets.append(f"//{package}:{check}") + args = ( + '--config-path "$(location //:rustfmt.toml)" ' + f'--edition {edition} --config skip_children=true --check' + ) + command = f'"$(RUSTFMT)" {args} $(locations :{sources}) && touch "$@"' + # PowerShell executes the generated command as a script. Invoke + # the formatter per file there to stay below Windows' argv limit. + powershell = ( + f"foreach ($$source in ('$(locations :{sources})' -split ' ')) {{ " + f'& "$(RUSTFMT)" {args} $$source; ' + 'if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; ' + 'New-Item -ItemType File -Path "$@" -Force | Out-Null' + ) + rules.extend([ + 'filegroup(\n' + f' name = "{sources}",\n' + ' srcs = depset(' + " + ".join(sorted(expressions)) + ').to_list(),\n' + ' tags = ["manual"],\n)\n', + 'genrule(\n' + f' name = "{check}",\n' + f' srcs = [":{sources}", "//:rustfmt.toml"],\n' + f' outs = ["{check}.ok"],\n' + f' cmd = {json.dumps(command)},\n' + f' cmd_ps = {json.dumps(powershell)},\n' + ' tools = ["//:rustfmt_toolchain"],\n' + ' toolchains = ["//:rustfmt_toolchain"],\n' + ' tags = ["manual"],\n' + ' visibility = ["//visibility:public"],\n)\n', + ]) + return rules + def render_build(p): name = p["name"] + manifest = tomllib.loads(Path(p["manifest_path"]).read_text()) + formatting_sources = {} + + def format_sources(expression, edition): + formatting_sources.setdefault(edition, set()).add(expression) test_sizes = (p.get("metadata") or {}).get("bazel", {}).get("test_sizes", {}) if not isinstance(test_sizes, dict) or any( size not in ("small", "medium", "large", "enormous") @@ -872,6 +925,7 @@ def unique_name(want): srcs_expr_lib += " + glob([" + q(lib_extra) + "])" if lib_mod_labels: srcs_expr_lib += " + [" + q(lib_mod_labels) + "]" + format_sources(srcs_expr_lib, lib_t["edition"]) out += [ "rust_library(", f' name = "{tgt}",', @@ -899,6 +953,7 @@ def unique_name(want): if has_build: srcs = srcs_expr(["build.rs"] + BUILD_SCRIPT_SRCS_EXTRA.get(name, [])) + format_sources(srcs, p["edition"]) build_edges = dedup( label(d, build_tree) for d in md["build"] if d in build_tree ) @@ -945,6 +1000,7 @@ def unique_name(want): srcs += " + glob([" + q(bin_local) + "])" if bin_labels: srcs += " + [" + q(bin_labels) + "]" + format_sources(srcs, t["edition"]) req = frozenset(t.get("required-features") or []) # Cargo builds examples against the dev resolution, and builds the # binaries a test run spawns with that run's features too (test @@ -1044,10 +1100,16 @@ def unique_name(want): ] out = [l for l in out if l is not None] + benchmark_binaries = [] for t in p["targets"]: kinds = t["kind"] if not ({"test", "bench"} & set(kinds)): continue + declared_target = next( + (target for kind in kinds for target in manifest.get(kind, []) + if target.get("name") == t["name"]), + {}, + ) req = frozenset(t.get("required-features") or []) fmap = contexts[(name, "normal,build,dev", req)] deps = dedup( @@ -1070,17 +1132,28 @@ def unique_name(want): srcs += " + glob([" + q(mod_local) + "])" if mod_labels: srcs += " + [" + q(mod_labels) + "]" + format_sources(srcs, t["edition"]) runtime_globs = dedup([ f"{Path(rel_src(p, t)).parent}/**", *[f"{directory}/**" for directory in RESOURCE_DIRS if directory != "src"], ]) - data = srcs if srcs.startswith("[") else ( + # Source groups keep separate full resource groups for runfiles. + data = srcs.replace("_sources\"", "\"") if srcs.startswith("[") else ( f"glob([{q(runtime_globs)}], allow_empty = True)" - + (" + [" + q(mod_labels) + "]" if mod_labels else "") + + (" + [" + q([label.removesuffix("_sources") for label in mod_labels]) + "]" if mod_labels else "") ) # Suites read the shared fixtures as ../../tests/fixtures from the # crate directory, where the launcher starts them. data += ' + ["//tests:fixtures"]' + if name == "tracedecay" and t["name"] == "memory_suite": + data += ' + ["//evals/memory:scenarios"]' + if name == "tracedecay" and t["name"] == "claude_observation_benchmark": + data += ' + ["//benchmark_data:claude-observation"]' + if name == "tracedecay" and t["name"] == "session_temporal": + workload, _, _ = group_label( + Path("benchmark_data/session-temporal/workload-v1.json") + ) + data += " + [" + q([workload]) + "]" # include_str!/include_bytes! are compile-time inputs. They cover crate-root # resource dirs beside tests/ and repo-root fixtures under //tests. compile_data = ( @@ -1110,10 +1183,18 @@ def unique_name(want): runfiles_env.setdefault(variable, target) env_attr = " rustc_env = " + env_dict(rustc_env) + "," if rustc_env else None test_name = unique_name(t["name"]) + manual_benchmark = "bench" in kinds and not t["test"] + if manual_benchmark: + benchmark_binaries.append(f":{test_name}__binary") out += [ "rust_test(", f' name = "{test_name}",', test_size_attr(test_name), + # Cargo metadata distinguishes opt-in measurements from default tests. + ' tags = ["manual"],' + if manual_benchmark else None, + " use_libtest_harness = False," + if not declared_target.get("harness", True) else None, " srcs = " + srcs + ",", f' crate_root = {crate_root},', ' edition = crate_edition(),', @@ -1133,6 +1214,15 @@ def unique_name(want): ] out = [l for l in out if l is not None] + if benchmark_binaries: + out += [ + "filegroup(", + ' name = "benchmark_binaries",', + " testonly = True,", + " srcs = [" + q(benchmark_binaries) + "],", + ")\n", + ] + if lib_t: lib_runfiles_env = runtime_binaries(p, lib_t, lib_pulled) dev_edges = md["normal"] + md["dev"] @@ -1182,6 +1272,7 @@ def unique_name(want): ' visibility = ["//visibility:public"],', ')\n', ] + out.extend(formatting_rules(dir_of[name], formatting_sources)) return "\n".join(out) outputs = {} @@ -1196,13 +1287,19 @@ def package_rules(pkg_dir, exports): name member crate roots, so they need none.""" rules = [] for gname, entry in sorted(by_pkg.get(pkg_dir, {}).items()): + pattern = f'{entry["glob"]}/**' + exclude = "" + if entry.get("sources"): + pattern += "/*.rs" + if entry["glob"] == "tests": + exclude = ', exclude = ["tests/fixtures/**"]' rules.append( f'filegroup(\n name = "{gname}",\n' - f' srcs = glob(["{entry["glob"]}/**"]),\n' + f' srcs = glob(["{pattern}"]{exclude}),\n' ' visibility = ["//visibility:public"],\n)' ) crate_roots = sorted(entry["files"]) - if exports and crate_roots: + if exports and crate_roots and not entry.get("sources"): rules.append( "exports_files([" + q(crate_roots) + "],\n" ' visibility = ["//visibility:public"],\n)' @@ -1307,10 +1404,27 @@ def render_shared(path, pkg_dir): obsolete.append(path) else: outputs[path] = rendered + # Handwritten Rust targets outside Cargo members use the same source-only + # checks. SDK generator sources are already covered by tracedecay-sdk. + for package, sources in { + "scripts": {"2024": {'["bazel/rust_test_environment_probe.rs"]'}}, + "sdks/codegen": {"2024": {'["dashboard_schema.rs"]'}}, + }.items(): + path = REPO / package / "BUILD.bazel" + rules = formatting_rules(package, sources) + block = GEN_BEGIN + "\n" + "\n\n".join(rules) + "\n" + GEN_END + outputs[path] = splice_generated(path.read_text(), block, path) + root_build = REPO / "BUILD.bazel" + format_aggregate = ( + GEN_BEGIN + '\nfilegroup(\n name = "rustfmt_check",\n' + + " srcs = [" + q(sorted(formatting_targets)) + "],\n" + + ' tags = ["manual"],\n)\n' + GEN_END + ) + outputs[root_build] = splice_generated(root_build.read_text(), format_aggregate, root_build) # A generated block or fully generated file can also go stale when its # last Cargo reference disappears; re-render the marked ones too. for path in sorted(REPO.glob("*/BUILD.bazel")): - if path in outputs or not path.exists(): + if path.resolve() in outputs or not path.exists(): continue text = path.read_text() if ( @@ -1338,7 +1452,7 @@ def render_shared(path, pkg_dir): print("generated BUILD.bazel files are out of date:") for path in stale: print(f" {path.relative_to(REPO)}") - print("rerun: python3 scripts/bazel/gen_builds.py") + print("rerun: bazel run //scripts:bazel/gen_builds.py") # Show the first diffs so CI output explains the divergence. import difflib shown = 0 diff --git a/scripts/bazel/patches/rules-rust-windows-build-flags.patch b/scripts/bazel/patches/rules-rust-windows-build-flags.patch new file mode 100644 index 0000000000..df11b8eb37 --- /dev/null +++ b/scripts/bazel/patches/rules-rust-windows-build-flags.patch @@ -0,0 +1,29 @@ +diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl +--- a/rust/private/rustc.bzl ++++ b/rust/private/rustc.bzl +@@ -1108,7 +1108,8 @@ + for build_env_file in build_env_files: + process_wrapper_flags.add("--env-file", build_env_file) + +- process_wrapper_flags.add_all(build_flags_files, before_each = "--arg-file") ++ if toolchain.exec_triple.system != "windows": ++ process_wrapper_flags.add_all(build_flags_files, before_each = "--arg-file") + + all_allowed_unstable_features = [] + if getattr(ctx.attr, "unstable_rust_features_config", None): +@@ -1591,6 +1592,15 @@ + all_args = [process_wrapper_flags, rustc_path, rustc_flags] + if rust_flags_args != None: + all_args.append(rust_flags_args) ++ if toolchain.exec_triple.system == "windows": ++ # Build-script flags follow authored flags, just as --arg-file does. ++ # The wrapper expands nested response files before starting rustc; ++ # keeping their paths in one response file also bounds its own argv. ++ build_flags = ctx.actions.args() ++ build_flags.set_param_file_format("multiline") ++ build_flags.use_param_file("@%s", use_always = True) ++ build_flags.add_all(build_flags_files, format_each = "@%s") ++ all_args.append(build_flags) + + # Path mapping must be disabled whenever any input string carries a + # `$(location ...)` / `$(execpath ...)` macro: location expansion diff --git a/scripts/bazel/rust_test.bzl b/scripts/bazel/rust_test.bzl index 75e71985b2..934530c5f8 100644 --- a/scripts/bazel/rust_test.bzl +++ b/scripts/bazel/rust_test.bzl @@ -96,13 +96,18 @@ def _isolated_rust_test_impl(ctx): } for variable, file in {"CARGO": toolchain.cargo, "RUSTC": toolchain.rustc, "RUSTDOC": toolchain.rust_doc}.items(): paths[variable] = _runfiles_path(file, workspace) + # Distribution acceptance explicitly supplies its extracted CLI through + # --test_env. Every other runfile, and the default CLI, stays declared here. if windows: runfiles_env = "\n".join([ + ('if not defined {} '.format(variable) if variable == "TRACEDECAY_TEST_BIN" else "") + 'set "{}=%TEST_SRCDIR%/{}"'.format(variable, path) for variable, path in paths.items() ]) else: runfiles_env = " ".join([ + '"{}=${{{}:-$TEST_SRCDIR/{}}}"'.format(variable, variable, path) + if variable == "TRACEDECAY_TEST_BIN" else '"{}=$TEST_SRCDIR/{}"'.format(variable, path) for variable, path in paths.items() ]) diff --git a/scripts/bench-hot-paths.py b/scripts/bench-hot-paths.py index a9879f2a6b..decb758652 100755 --- a/scripts/bench-hot-paths.py +++ b/scripts/bench-hot-paths.py @@ -27,7 +27,8 @@ Linux instead of reported as zero. Usage: - scripts/bench-hot-paths.py --bin target/release/tracedecay + bazel build --config=release //crates/tracedecay-cli:tracedecay + scripts/bench-hot-paths.py --bin bazel-bin/crates/tracedecay-cli/tracedecay scripts/bench-hot-paths.py --bin ./tracedecay --samples 40 --out target/bench-hot-paths/run1 Exit status: 0 when the run completed (whatever the numbers), 2 on a harness diff --git a/scripts/check-distribution-acceptance.sh b/scripts/check-distribution-acceptance.sh index bbd1b9c701..52e9e81568 100755 --- a/scripts/check-distribution-acceptance.sh +++ b/scripts/check-distribution-acceptance.sh @@ -688,21 +688,35 @@ cargo nextest run \ # graph. Query, root-library, and LSP source suites run in exhaustive Linux CI; # rerunning them here added no archive assertion. The MCP harness stays focused # on the modules that spawn TRACEDECAY_TEST_BIN, so it proves the packaged CLI -# without rerunning all 573 source-library tests. Run it from the verified -# checkout so unchanged source-graph units retain their original Cargo paths. +# without rerunning unrelated source-library tests. Run it from the verified +# checkout through the canonical isolated Bazel launcher. echo "distribution acceptance: checking packaged CLI integration behavior" resolve_clean_source_head "$repo" "$source_git_sha" >/dev/null -TRACEDECAY_TEST_BIN="$packaged_cli_bin" \ - CARGO_NET_OFFLINE=true cargo nextest run \ - --manifest-path "$repo/Cargo.toml" \ - --release \ - -p tracedecay \ - --test mcp_suite \ - --features tracedecay/test-transport \ - --no-fail-fast \ - --retries 2 \ - -E 'test(/^mcp_cli_serve_test::/) | test(/^serve_template_path_test::/) | test(=mcp_handler_test::lcm_test::lcm_status_cli_bridge_accepts_json_args)' \ - --no-tests=fail +# The extracted binary is outside Bazel's declared runfiles. Run locally and +# disable result caching so its current bytes are exercised on every invocation. +# Check each filter independently: libtest otherwise accepts zero matches. +require_command bazel +for filter in \ + 'mcp_cli_serve_test::' \ + 'serve_template_path_test::' \ + 'mcp_handler_test::lcm_test::lcm_status_cli_bridge_accepts_json_args'; do + ( + cd "$repo" + test_args=(--test_arg="$filter") + expected_count=nonzero + if [[ $filter == mcp_handler_test::* ]]; then + test_args+=(--test_arg=--exact) + expected_count=1 + fi + REQUIRE_EXACT_TEST_COUNT="$expected_count" scripts/require-exact-test.sh \ + bazel test --config=ci //crates/tracedecay:mcp_suite \ + --test_strategy=standalone \ + --nocache_test_results \ + --test_output=all \ + --test_env="TRACEDECAY_TEST_BIN=$packaged_cli_bin" \ + "${test_args[@]}" + ) +done resolve_clean_source_head "$repo" "$source_git_sha" >/dev/null install_root="$work/install" diff --git a/scripts/ci-pr-dogfood-smoke.sh b/scripts/ci-pr-dogfood-smoke.sh index 77657b73a7..5c016e9de7 100755 --- a/scripts/ci-pr-dogfood-smoke.sh +++ b/scripts/ci-pr-dogfood-smoke.sh @@ -519,7 +519,7 @@ run_smoke() { main() { local binary project_root base_ref head_ref base_branch head_branch checked_out_oid head_oid started_ms status - binary="${TRACEDECAY_BIN:-$REPO_ROOT/target/debug/tracedecay}" + binary="${TRACEDECAY_BIN:-$REPO_ROOT/bazel-bin/crates/tracedecay-cli/tracedecay}" project_root="${TRACEDECAY_DOGFOOD_PROJECT:-$REPO_ROOT}" base_ref="${TRACEDECAY_DOGFOOD_BASE_REF:-}" head_ref="${TRACEDECAY_DOGFOOD_HEAD_REF:-HEAD}" diff --git a/scripts/claude_stock_integration.sh b/scripts/claude_stock_integration.sh index 7e91266b09..55bdce368d 100755 --- a/scripts/claude_stock_integration.sh +++ b/scripts/claude_stock_integration.sh @@ -7,11 +7,11 @@ # `claude-integration` CI job and runnable locally: # # npm install --global @anthropic-ai/claude-code@ -# cargo build -p tracedecay-cli --bin tracedecay +# bazel build //crates/tracedecay-cli:tracedecay # scripts/claude_stock_integration.sh # # Environment: -# TRACEDECAY_BIN tracedecay binary to install/test (default: target/debug/tracedecay) +# TRACEDECAY_BIN tracedecay binary to install/test (default: bazel-bin/crates/tracedecay-cli/tracedecay) # CLAUDE_BIN stock claude binary (default: claude on PATH) # # Everything runs in a throwaway HOME and a throwaway project. No model calls diff --git a/scripts/efficiency-scorecard.py b/scripts/efficiency-scorecard.py index 681eba44aa..b6cd1d9e40 100755 --- a/scripts/efficiency-scorecard.py +++ b/scripts/efficiency-scorecard.py @@ -7,7 +7,7 @@ diffing two files. scripts/efficiency-scorecard.py # build + run (3 runs) - scripts/efficiency-scorecard.py --binary target/release/tracedecay + scripts/efficiency-scorecard.py --binary /path/to/tracedecay scripts/efficiency-scorecard.py --quick # 1 run smoke (~1 min) scripts/efficiency-scorecard.py --label b9181367ba \ --binary /path/to/old/tracedecay --output target/effscore-base @@ -1205,28 +1205,29 @@ def flatten_store_bytes(run: dict) -> None: def build_binary() -> Path: - jobs = os.environ.get("CARGO_BUILD_JOBS", "8") - print(f"scorecard: building tracedecay (release, jobs={jobs})", file=sys.stderr) - env = dict(os.environ, CARGO_BUILD_JOBS=jobs, TRACEDECAY_SKIP_DASHBOARD_BUILD="1") + print("scorecard: building tracedecay (Bazel release)", file=sys.stderr) completed = subprocess.run( ( - "cargo", + "bazel", "build", - "--locked", - "--release", - "-p", - "tracedecay-cli", - "--bin", - "tracedecay", + "--config=release", + "//crates/tracedecay-cli:tracedecay", ), cwd=REPO_ROOT, - env=env, check=False, ) if completed.returncode != 0: raise HarnessError("the tracedecay release build failed") - target_dir = Path(os.environ.get("CARGO_TARGET_DIR", REPO_ROOT / "target")) - binary = target_dir / "release" / "tracedecay" + output = subprocess.run( + ("bazel", "info", "--config=release", "bazel-bin"), + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + if output.returncode != 0 or not output.stdout.strip(): + raise HarnessError(f"could not resolve the Bazel output directory: {output.stderr.strip()}") + binary = Path(output.stdout.strip()) / "crates" / "tracedecay-cli" / "tracedecay" if not binary.is_file(): raise HarnessError(f"expected a binary at {binary} after the build") return binary @@ -1243,7 +1244,7 @@ def main() -> int: parser = argparse.ArgumentParser( description=__doc__.splitlines()[0], formatter_class=argparse.RawDescriptionHelpFormatter ) - parser.add_argument("--binary", help="prebuilt tracedecay binary (default: cargo build)") + parser.add_argument("--binary", help="prebuilt tracedecay binary (default: Bazel release build)") parser.add_argument( "--output", default=str(REPO_ROOT / "target" / "efficiency-scorecard"), diff --git a/scripts/hermes_plugin_unit_check.py b/scripts/hermes_plugin_unit_check.py index 71c07d0bf5..45c5d6e4f2 100644 --- a/scripts/hermes_plugin_unit_check.py +++ b/scripts/hermes_plugin_unit_check.py @@ -26,7 +26,8 @@ python3 scripts/hermes_plugin_unit_check.py [plugin_dir] plugin_dir defaults to a fresh install generated into a temp HOME via the -tracedecay binary named by $TRACEDECAY_BIN (default: target/debug/tracedecay). +tracedecay binary named by $TRACEDECAY_BIN +(default: bazel-bin/crates/tracedecay-cli/tracedecay). """ import copy @@ -54,7 +55,7 @@ def generate_plugin(work: Path) -> Path: """Generates the plugin into a throwaway HOME using the real installer.""" repo_root = Path(__file__).resolve().parent.parent bin_path = Path( - os.environ.get("TRACEDECAY_BIN", repo_root / "target" / "debug" / "tracedecay") + os.environ.get("TRACEDECAY_BIN", repo_root / "bazel-bin" / "crates" / "tracedecay-cli" / "tracedecay") ) assert bin_path.is_file(), f"tracedecay binary not found at {bin_path}" home = work / "home" diff --git a/scripts/hermes_stock_integration.sh b/scripts/hermes_stock_integration.sh index a39081c4dd..638ae0c736 100755 --- a/scripts/hermes_stock_integration.sh +++ b/scripts/hermes_stock_integration.sh @@ -5,11 +5,11 @@ # # git clone https://github.com/NousResearch/hermes-agent.git /tmp/hermes-upstream # git -C /tmp/hermes-upstream checkout -# cargo build -p tracedecay-cli --bin tracedecay +# bazel build //crates/tracedecay-cli:tracedecay # scripts/hermes_stock_integration.sh # # Environment: -# TRACEDECAY_BIN tracedecay binary to install/test (default: target/debug/tracedecay) +# TRACEDECAY_BIN tracedecay binary to install/test (default: bazel-bin/crates/tracedecay-cli/tracedecay) # HERMES_UPSTREAM_DIR stock hermes-agent checkout (default: /tmp/hermes-upstream) # # Everything is offline and deterministic: a throwaway user HOME (and its diff --git a/scripts/lib/stock-host.sh b/scripts/lib/stock-host.sh index 579c917108..c10bc32771 100644 --- a/scripts/lib/stock-host.sh +++ b/scripts/lib/stock-host.sh @@ -16,15 +16,15 @@ readonly STOCK_HOST_REPO_ROOT STOCK_HOST_TRACEDECAY_BIN="" # Prints the absolutized tracedecay binary under test ($TRACEDECAY_BIN, or the -# default cargo debug output) and requires it to be executable. +# default Bazel CLI output) and requires it to be executable. resolve_tracedecay_bin() { local bin - bin="${TRACEDECAY_BIN:-$STOCK_HOST_REPO_ROOT/target/debug/tracedecay}" - bin="$(cd "$(dirname "$bin")" && pwd)/$(basename "$bin")" + bin="${TRACEDECAY_BIN:-$STOCK_HOST_REPO_ROOT/bazel-bin/crates/tracedecay-cli/tracedecay}" if [[ ! -x "$bin" ]]; then - echo "error: tracedecay binary not found at $bin (build with: cargo build -p tracedecay-cli --bin tracedecay)" >&2 + echo "error: tracedecay binary not found at $bin (build with: bazel build //crates/tracedecay-cli:tracedecay)" >&2 return 1 fi + bin="$(cd "$(dirname "$bin")" && pwd)/$(basename "$bin")" printf '%s\n' "$bin" } diff --git a/scripts/mcp-conformance-smoke.sh b/scripts/mcp-conformance-smoke.sh index 3049df0d68..e4593f3050 100755 --- a/scripts/mcp-conformance-smoke.sh +++ b/scripts/mcp-conformance-smoke.sh @@ -19,8 +19,9 @@ # into the temp dir, so it never touches the user's real tracedecay state. # # Usage: -# scripts/mcp-conformance-smoke.sh # auto-detect binary -# TRACEDECAY_BIN=target/debug/tracedecay scripts/mcp-conformance-smoke.sh +# bazel build //crates/tracedecay-cli:tracedecay +# scripts/mcp-conformance-smoke.sh # use the Bazel output +# TRACEDECAY_BIN=/path/to/tracedecay scripts/mcp-conformance-smoke.sh set -euo pipefail @@ -292,30 +293,12 @@ cleanup() { exit "$status" } -find_tracedecay_bin() { - local target_dir candidate - - if [[ -n "${TRACEDECAY_BIN:-}" ]]; then - printf '%s\n' "$TRACEDECAY_BIN" - return - fi - - target_dir="${CARGO_TARGET_DIR:-$REPO_ROOT/target}" - for candidate in "$target_dir/debug/tracedecay" "$target_dir/release/tracedecay"; do - if [[ -x "$candidate" ]]; then - printf '%s\n' "$candidate" - return - fi - done - command -v tracedecay || true -} - main() { local tracedecay_bin npm_cache fixture status - tracedecay_bin="$(find_tracedecay_bin)" + tracedecay_bin="${TRACEDECAY_BIN:-$REPO_ROOT/bazel-bin/crates/tracedecay-cli/tracedecay}" if [[ -z "$tracedecay_bin" || ! -x "$tracedecay_bin" ]]; then - echo "error: no tracedecay binary found; build one or set TRACEDECAY_BIN" >&2 + echo "error: tracedecay binary not executable at $tracedecay_bin; run bazel build //crates/tracedecay-cli:tracedecay or set TRACEDECAY_BIN" >&2 return 2 fi tracedecay_bin="$(readlink -f "$tracedecay_bin")" diff --git a/scripts/mcp_probe/README.md b/scripts/mcp_probe/README.md index c570acd433..c5816a617a 100644 --- a/scripts/mcp_probe/README.md +++ b/scripts/mcp_probe/README.md @@ -31,8 +31,8 @@ Each module exposes a `probes_for(discovered)` function that returns ## Quick run ```sh -# 1. Build the release binary (driver shells out to ../../target/release/tracedecay) -cargo build -p tracedecay-cli --release --bin tracedecay +# 1. Build the release binary (driver uses bazel-bin/crates/tracedecay-cli/tracedecay) +bazel build --config=release //crates/tracedecay-cli:tracedecay # 2. Run the matrix python3 scripts/mcp_probe/probe.py diff --git a/scripts/mcp_probe/probe.py b/scripts/mcp_probe/probe.py index 5948869067..958f3b2ff9 100755 --- a/scripts/mcp_probe/probe.py +++ b/scripts/mcp_probe/probe.py @@ -34,7 +34,7 @@ ROOT = Path(__file__).resolve().parent REPO_ROOT = ROOT.parents[1] -DEFAULT_BIN = REPO_ROOT / "target" / "release" / "tracedecay" +DEFAULT_BIN = REPO_ROOT / "bazel-bin" / "crates" / "tracedecay-cli" / "tracedecay" DEFAULT_LOG = "/tmp/tracedecay_matrix.log" DEFAULT_REPOS = ROOT / "repos.toml" DEFAULT_STDERR_DIR = "/tmp/tracedecay_matrix_stderr" @@ -347,7 +347,7 @@ def load_probe_set(languages: list[str], discovered: dict) -> dict: def main() -> int: if not BIN.exists(): print(f"error: tracedecay binary not found at {BIN}", file=sys.stderr) - print(f"hint: run `cargo build --release` first", file=sys.stderr) + print("hint: run `bazel build --config=release //crates/tracedecay-cli:tracedecay` first", file=sys.stderr) return 2 sys.path.insert(0, str(ROOT)) diff --git a/scripts/opencode_stock_integration.sh b/scripts/opencode_stock_integration.sh index 5d80845f45..29238982c1 100755 --- a/scripts/opencode_stock_integration.sh +++ b/scripts/opencode_stock_integration.sh @@ -6,11 +6,11 @@ # Used by the `host-stock-integration` CI job and runnable locally: # # npm install --global @opencode/cli@ -# cargo build -p tracedecay-cli --bin tracedecay +# bazel build //crates/tracedecay-cli:tracedecay # scripts/opencode_stock_integration.sh # # Environment: -# TRACEDECAY_BIN tracedecay binary to install/test (default: target/debug/tracedecay) +# TRACEDECAY_BIN tracedecay binary to install/test (default: bazel-bin/crates/tracedecay-cli/tracedecay) # OPENCODE_BIN stock opencode binary (default: opencode on PATH) # # Everything runs in a throwaway HOME and a throwaway initialized project; no diff --git a/scripts/perf-gate.sh b/scripts/perf-gate.sh index c55b1492b4..9f46cb93db 100755 --- a/scripts/perf-gate.sh +++ b/scripts/perf-gate.sh @@ -99,7 +99,6 @@ PERF_REINDEX_WORKTREES="${PERF_REINDEX_WORKTREES:-0}" # Load shape. Overridable so a laptop can run a shorter pass than CI. PERF_WORKERS="${PERF_WORKERS:-6}" PERF_DURATION_SECONDS="${PERF_DURATION_SECONDS:-60}" -PERF_CARGO_PROFILE="${PERF_CARGO_PROFILE:-release}" PERF_DAEMON_READY_TIMEOUT="${PERF_DAEMON_READY_TIMEOUT:-300}" PERF_INDEX_TIMEOUT="${PERF_INDEX_TIMEOUT:-$((PERF_BUDGET_INDEX_SECONDS + 120))}" @@ -227,23 +226,20 @@ mkdir -p "$RUN_DIR/home/.local/share" "$RUN_DIR/home/.config" "$RUN_DIR/profile" mkdir -p "$PERF_OUTPUT_DIR" # ── PHASE BUILD ────────────────────────────────────────────────────────────── -# Runs before the environment is redirected so cargo keeps the real HOME, and -# with it the real ~/.cargo registry and toolchain. +# Build before redirecting the environment so Bazel keeps its configured cache. if [[ -n "${TRACEDECAY_PERF_BIN:-}" ]]; then [[ -x "$TRACEDECAY_PERF_BIN" ]] || die "TRACEDECAY_PERF_BIN '$TRACEDECAY_PERF_BIN' is not executable" BIN="$(cd "$(dirname "$TRACEDECAY_PERF_BIN")" && pwd)/$(basename "$TRACEDECAY_PERF_BIN")" log "==> PHASE BUILD: using prebuilt binary $BIN" else - command -v cargo >/dev/null 2>&1 || die "cargo is required unless TRACEDECAY_PERF_BIN is set" - log "==> PHASE BUILD: cargo build -p tracedecay-cli --profile $PERF_CARGO_PROFILE --bin tracedecay" - # No CARGO_TARGET_DIR override: .cargo/config.toml already pins a repo-local - # target dir, which is exactly what the CI cache restores. - (cd "$REPO_ROOT" && cargo build --locked -p tracedecay-cli --profile "$PERF_CARGO_PROFILE" --bin tracedecay) >&2 || + command -v bazel >/dev/null 2>&1 || die "bazel is required unless TRACEDECAY_PERF_BIN is set" + log "==> PHASE BUILD: bazel build --config=release //crates/tracedecay-cli:tracedecay" + (cd "$REPO_ROOT" && bazel build --config=release //crates/tracedecay-cli:tracedecay) >&2 || die "the tracedecay binary failed to build" - BUILD_DIR="$PERF_CARGO_PROFILE" - [[ "$BUILD_DIR" == "dev" ]] && BUILD_DIR="debug" - BIN="$REPO_ROOT/target/$BUILD_DIR/tracedecay" + BUILD_DIR="$(cd "$REPO_ROOT" && bazel info --config=release bazel-bin)" || + die "could not resolve the Bazel output directory" + BIN="$BUILD_DIR/crates/tracedecay-cli/tracedecay" [[ -x "$BIN" ]] || die "expected a binary at $BIN after the build" fi BUILD_VERSION="$("$BIN" --version 2>/dev/null | tr -d '\n' || echo unknown)" diff --git a/scripts/require-exact-test.sh b/scripts/require-exact-test.sh index e9e50f298a..f0dcd7caf5 100755 --- a/scripts/require-exact-test.sh +++ b/scripts/require-exact-test.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -# Run one name-filtered cargo test and require that it actually ran. +# Run one name-filtered libtest target and require that it actually ran. # -# `cargo test -- --exact` exits 0 when the filter matches NOTHING: +# `bazel test TARGET --test_arg=NAME --test_arg=--exact` exits 0 when the filter matches NOTHING: # # running 0 tests # test result: ok. 0 passed; 0 failed; 0 ignored; 118 filtered out @@ -12,7 +12,8 @@ # nextest has --no-tests=fail for this; libtest does not, so assert on the count # libtest reports instead of on the exit code alone. # -# usage: scripts/require-exact-test.sh cargo test --test foo the_test -- --exact +# usage: scripts/require-exact-test.sh bazel test //crates/tracedecay-hooks:unit_test \ +# --test_output=all --test_arg=spool::tests::checksum_is_real_sha256 --test_arg=--exact # # Set REQUIRE_EXACT_TEST_COUNT to a positive integer, or to `nonzero` for a # single-target suite whose membership may evolve but must never become empty. @@ -27,7 +28,7 @@ fi if [ "$#" -eq 0 ]; then echo "usage: $0 " >&2 - echo "example: $0 cargo test --test foo the_test -- --exact" >&2 + echo "example: $0 bazel test //crates/tracedecay-hooks:unit_test --test_output=all --test_arg=spool::tests::checksum_is_real_sha256 --test_arg=--exact" >&2 exit 2 fi diff --git a/scripts/run-claude-observation-benchmark.sh b/scripts/run-claude-observation-benchmark.sh index 5532409402..3f60de70d1 100755 --- a/scripts/run-claude-observation-benchmark.sh +++ b/scripts/run-claude-observation-benchmark.sh @@ -6,7 +6,7 @@ cd "$repo_root" git_root=$(git rev-parse --show-toplevel) if [[ $(cd "$git_root" && pwd -P) != "$repo_root" ]]; then - echo "benchmark runner must execute from the CARGO_MANIFEST_DIR Git worktree" >&2 + echo "benchmark runner must execute from the current Git worktree" >&2 exit 1 fi if [[ -n $(git status --porcelain=v1 --untracked-files=normal --ignore-submodules=none) ]]; then @@ -15,7 +15,6 @@ if [[ -n $(git status --porcelain=v1 --untracked-files=normal --ignore-submodule fi commit=$(git rev-parse HEAD) -tree=$(git rev-parse 'HEAD^{tree}') short_commit=${commit:0:8} result_name="result-$(date -u +%F)-${short_commit}.json" result_path="benchmark_data/claude-observation/$result_name" @@ -29,117 +28,49 @@ if ! grep -q '"current_acceptance": null' "$index_path"; then exit 1 fi -scratch_root="$repo_root/target/claude-observation-source" -mkdir -p "$scratch_root" -build_root=$(mktemp -d "$scratch_root/archive.XXXXXXXX") -capture=$(mktemp) -index_backup=$(mktemp) -cp "$index_path" "$index_backup" -complete=false +staging=$(mktemp -d "${TMPDIR:-/tmp}/claude-observation-bench.XXXXXXXX") +result_move_started=false cleanup() { - cd "$repo_root" - rm -f "$capture" - if [[ $complete != true ]]; then - rm -f "$result_path" - cp "$index_backup" "$index_path" - fi - rm -f "$index_backup" - if [[ -d ${build_root:-} ]]; then - chmod -R u+w "$build_root" || true - rm -rf "$build_root" + local status=$? index_status=0 + trap - EXIT INT TERM + if [[ $result_move_started == true ]]; then + # The index is the publication authority, including when a signal arrives + # after its rename but before the shell can record another state change. + python3 - "$index_path" "$result_name" <<'PY' || index_status=$? +import json +import sys + +try: + with open(sys.argv[1]) as source: + selected = json.load(source)["current_acceptance"] + if selected is not None and not isinstance(selected, str): + raise ValueError("invalid current_acceptance") +except (OSError, ValueError, KeyError, TypeError) as error: + print(f"cannot determine benchmark publication; preserving result: {error}", file=sys.stderr) + sys.exit(2) +sys.exit(1 if selected == sys.argv[2] else 0) +PY + case "$index_status" in + 0) rm -f "$result_path" || status=1 ;; + 1) ;; + *) [[ $status -ne 0 ]] || status=1 ;; + esac fi + rm -rf "$staging" || status=1 + exit "$status" } trap cleanup EXIT -git archive "$commit" | tar -x -C "$build_root" - -write_source_manifest() { - local destination=$1 - : >"$destination" - while IFS= read -r -d '' relative; do - mode=$(git ls-files -s -- "$relative" | awk '{print $1}') - if [[ $mode == 160000 ]]; then - digest=$(git rev-parse "$commit:$relative") - else - digest=$(sha256sum "$build_root/$relative" | awk '{print $1}') - fi - printf '%s\t%s\t%s\n' "$mode" "$digest" "$relative" >>"$destination" - done < <(git ls-files -z) -} - -source_manifest="$build_root/.tracedecay-benchmark-source-manifest" -write_source_manifest "$source_manifest" -source_manifest_sha256=$(sha256sum "$source_manifest" | awk '{print $1}') - -host_target=$(rustc -Vv | sed -n 's/^host: //p') -[[ -n $host_target ]] || { echo "could not resolve rustc host target" >&2; exit 1; } -export CARGO_BUILD_TARGET=$host_target -unset RUSTFLAGS -export CARGO_ENCODED_RUSTFLAGS= - -config_identity=$({ - for config in "$build_root/.cargo/config.toml" \ - "${CARGO_HOME:-${HOME:-}/.cargo}/config" \ - "${CARGO_HOME:-${HOME:-}/.cargo}/config.toml"; do - if [[ -f $config ]]; then - printf '%s\t%s\n' "$(basename "$config")" "$(git hash-object "$config")" - else - printf '%s\tmissing\n' "$(basename "$config")" - fi - done - env | LC_ALL=C sort | grep -E '^(AR|CC|CFLAGS|CXX|CXXFLAGS|LDFLAGS|RUSTC|RUSTUP_TOOLCHAIN|CARGO_BUILD_TARGET|CARGO_PROFILE_RELEASE_)=' || true -} | git hash-object --stdin) - -wrapper_identity() { - local wrapper=$1 - if [[ -z $wrapper ]]; then - printf 'environment:none;cargo_config:%s' "$config_identity" - return - fi - local version - version=$($wrapper --version 2>&1 | head -n 1) || version=version-unavailable - printf 'environment:%s;%s' "$(basename "$wrapper")" "$version" -} - -export TRACEDECAY_BENCHMARK_BUILD_COMMIT=$commit -export TRACEDECAY_BENCHMARK_BUILD_TREE=$tree -export TRACEDECAY_BENCHMARK_BUILD_PROFILE=release -export TRACEDECAY_BENCHMARK_BUILD_SOURCE_MODE=git_archive_read_only_v1 -export TRACEDECAY_BENCHMARK_SOURCE_MANIFEST_SHA256=$source_manifest_sha256 -export TRACEDECAY_BENCHMARK_BUILD_TARGET_TRIPLE=$host_target -export TRACEDECAY_BENCHMARK_BUILD_RUSTC_VERSION="$(rustc -Vv)" -export TRACEDECAY_BENCHMARK_BUILD_CARGO_VERSION="$(cargo -V)" -export TRACEDECAY_BENCHMARK_BUILD_RUSTFLAGS=normalized-empty -export TRACEDECAY_BENCHMARK_BUILD_RUSTC_WRAPPER="$(wrapper_identity "${RUSTC_WRAPPER:-}")" -export TRACEDECAY_BENCHMARK_BUILD_RUSTC_WORKSPACE_WRAPPER="$(wrapper_identity "${RUSTC_WORKSPACE_WRAPPER:-}")" -export TRACEDECAY_BENCHMARK_BUILD_CARGO_CONFIG_IDENTITY=$config_identity - -# A fresh Git archive intentionally contains no generated dashboard assets. -# Generate them before freezing the source tree, then prove pnpm did not mutate -# any tracked input. The measured Cargo build can remain fully read-only. -( - cd "$build_root" - pnpm install --frozen-lockfile - cd dashboard - pnpm run build -) -post_dashboard_manifest="$build_root/.tracedecay-benchmark-source-manifest.post-dashboard" -write_source_manifest "$post_dashboard_manifest" -if ! cmp -s "$source_manifest" "$post_dashboard_manifest"; then - echo "dashboard build modified tracked benchmark source" >&2 - exit 1 -fi -rm -f "$post_dashboard_manifest" - -mkdir -p "$build_root/target" -chmod -R a-w "$build_root" -chmod u+rwx "$build_root/target" - -( - cd "$build_root" - cargo test --quiet --release -p tracedecay --features test-helpers \ - --test claude_observation_benchmark production_observation_pipeline_baseline -- \ - --ignored --exact --nocapture --test-threads=1 -) 2>&1 | tee "$capture" +trap 'exit 130' INT +trap 'exit 143' TERM +capture="$staging/capture.log" + +bazel test --config=release --config=ci //crates/tracedecay:claude_observation_benchmark \ + --test_strategy=standalone --nocache_test_results --test_output=all \ + --test_env=TRACEDECAY_BENCHMARK_REPO_ROOT="$repo_root" \ + --test_env=TRACEDECAY_BENCHMARK_BAZEL_VERSION="$(bazel --version)" \ + --test_arg=production_observation_pipeline_baseline \ + --test_arg=--ignored --test_arg=--exact --test_arg=--quiet --test_arg=--nocapture --test_arg=--test-threads=1 \ + 2>&1 | tee "$capture" if [[ $(grep -c '^TRACEDECAY_CLAUDE_OBSERVATION_BENCHMARK_RESULT=' "$capture") -ne 1 ]]; then echo "benchmark did not emit exactly one result" >&2 @@ -150,19 +81,23 @@ if [[ -z $result_json ]]; then echo "benchmark result marker was malformed" >&2 exit 1 fi -printf '%s\n' "$result_json" >"$result_path" - +# Stage only the new output. Historical evidence is read through symlinks, +# without creating a second copy of any existing artifact. +for historical in "$repo_root"/benchmark_data/claude-observation/result-*.json; do + ln -s "$historical" "$staging/$(basename "$historical")" +done +printf '%s\n' "$result_json" >"$staging/$result_name" sed "s/\"current_acceptance\": null/\"current_acceptance\": \"$result_name\"/" \ - "$index_backup" >"$index_path" -( - cd "$build_root" - TRACEDECAY_BENCHMARK_REQUIRE_ACCEPTANCE=1 \ - TRACEDECAY_BENCHMARK_EVIDENCE_DIR="$repo_root/benchmark_data/claude-observation" \ - cargo test --quiet --release -p tracedecay --features test-helpers \ - --test claude_observation_benchmark evidence_directory_matches_index_contract -- \ - --exact --test-threads=1 -) - -complete=true + "$index_path" >"$staging/evidence-index.json" +scripts/require-exact-test.sh bazel test --config=release --config=ci //crates/tracedecay:claude_observation_benchmark \ + --test_strategy=standalone --nocache_test_results --test_output=all \ + --test_env=TRACEDECAY_BENCHMARK_REPO_ROOT="$repo_root" \ + --test_env=TRACEDECAY_BENCHMARK_REQUIRE_ACCEPTANCE=1 \ + --test_env=TRACEDECAY_BENCHMARK_EVIDENCE_DIR="$staging" \ + --test_arg=evidence_directory_matches_index_contract --test_arg=--exact \ + --test_arg=--test-threads=1 + +result_move_started=true +mv "$staging/$result_name" "$result_path" +mv "$staging/evidence-index.json" "$index_path" echo "validated $result_path" -echo "commit only the result, evidence index, and README summary as the evidence follow-up" diff --git a/scripts/run-session-temporal-benchmark.sh b/scripts/run-session-temporal-benchmark.sh index d892c47231..fcad7d5f1c 100755 --- a/scripts/run-session-temporal-benchmark.sh +++ b/scripts/run-session-temporal-benchmark.sh @@ -5,12 +5,12 @@ usage() { cat <<'EOF' Usage: scripts/run-session-temporal-benchmark.sh --dry-run|--run|--refresh-contract - --dry-run Read-only, Cargo-free validation of harness artifacts and + --dry-run Read-only, build-free validation of harness artifacts and Codex fixture provenance. Does not mutate the checkout. - --run Diagnostic measurement through the optimized bench profile on + --run Diagnostic measurement through the Bazel release configuration on Linux and macOS. Isolates HOME and TRACEDECAY_DATA_DIR for the child process. - Windows CI proves temporal durability via nextest. + Windows CI proves temporal durability via Bazel. --refresh-contract Run the same real measurement from a clean source commit, then publish the result with its clean-commit provenance (Linux-hosted). @@ -37,7 +37,6 @@ validate_harness_evidence() { import json import pathlib import sys -import tomllib root = pathlib.Path(sys.argv[1]) benchmark_root = root / "benchmark_data/session-temporal" @@ -165,17 +164,6 @@ require("attestation" not in json.dumps(historical_result).lower(), require("attestation" not in json.dumps(workload).lower(), "deleted attestation terminology remains in workload") -with (root / "Cargo.toml").open("rb") as handle: - cargo = tomllib.load(handle) -profile = cargo.get("profile", {}).get("bench", {}) -require(profile == { - "opt-level": 3, - "debug": False, - "debug-assertions": False, - "overflow-checks": False, - "incremental": False, -}, "optimized bench profile mismatch") - storage = workload.get("storage_isolation") or {} require("HOME" in storage.get("required_environment", []), "HOME isolation required") require("TRACEDECAY_DATA_DIR" in storage.get("required_environment", []), @@ -194,23 +182,16 @@ run_benchmark() { exit 64 ;; *) - printf '%s\n' "Session-temporal ${mode} measurement harness supports Linux and macOS; use CI nextest durable coverage on other platforms" >&2 + printf '%s\n' "Session-temporal ${mode} measurement harness supports Linux and macOS; use CI Bazel durable coverage on other platforms" >&2 exit 64 ;; esac - isolation_root="$(mktemp -d "${TMPDIR:-/tmp}/session-temporal-bench.XXXXXX")" - cargo_home="${CARGO_HOME:-$HOME/.cargo}" - rustup_home="${RUSTUP_HOME:-$HOME/.rustup}" - cleanup() { - rm -rf "$isolation_root" - } - trap cleanup EXIT - export HOME="$isolation_root/home" - export TRACEDECAY_DATA_DIR="$isolation_root/tracedecay-data" - export CARGO_HOME="$cargo_home" - export RUSTUP_HOME="$rustup_home" - mkdir -p "$HOME" "$TRACEDECAY_DATA_DIR" - cargo bench -p tracedecay --bench session_temporal --all-features -- "$mode" + # The Bazel test launcher owns HOME and TraceDecay profile isolation. + # Local execution lets contract refresh publish into this exact checkout. + bazel test --config=release --config=ci //crates/tracedecay:session_temporal \ + --test_strategy=standalone --nocache_test_results --test_output=all \ + --test_env=TRACEDECAY_BENCHMARK_REPO_ROOT="$repo_root" \ + --test_arg=--bench --test_arg="$mode" } if [[ $# -ne 1 ]]; then @@ -224,7 +205,7 @@ cd "$repo_root" case "$1" in --dry-run) validate_harness_evidence - printf 'OK: session-temporal dry-run validated harness_ready evidence (Cargo-free, no mutation)\n' + printf 'OK: session-temporal dry-run validated harness_ready evidence (build-free, no mutation)\n' ;; --run) run_benchmark --run diff --git a/scripts/test-claude-observation-benchmark-publication.py b/scripts/test-claude-observation-benchmark-publication.py new file mode 100644 index 0000000000..5aa81bd8bb --- /dev/null +++ b/scripts/test-claude-observation-benchmark-publication.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Exercise benchmark publication and cleanup without running measurements.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parent.parent +RUNNER = "run-claude-observation-benchmark.sh" + + +class BenchmarkPublicationTests(unittest.TestCase): + def setUp(self) -> None: + self.scratch = tempfile.TemporaryDirectory(prefix="benchmark-publication-") + self.addCleanup(self.scratch.cleanup) + root = Path(self.scratch.name) + self.repo = root / "repo" + scripts = self.repo / "scripts" + scripts.mkdir(parents=True) + for name in (RUNNER, "require-exact-test.sh"): + shutil.copy2(ROOT / "scripts" / name, scripts / name) + self.evidence = self.repo / "benchmark_data/claude-observation" + self.evidence.mkdir(parents=True) + self.historical = self.evidence / "result-historical.json" + self.historical.write_text('{"evidence_status":"historical_stale"}\n') + self.historical_bytes = self.historical.read_bytes() + self.index = self.evidence / "evidence-index.json" + self.index.write_text(json.dumps({ + "schema_version": 1, + "current_acceptance": None, + "historical_stale": [self.historical.name], + }) + "\n") + self.index_bytes = self.index.read_bytes() + git_env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} + git_env.update(GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_NOSYSTEM="1") + for args in ( + ("init", "-q"), + ("add", "."), + ("-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", + "-c", "core.hooksPath=/dev/null", "-c", "commit.gpgsign=false", + "commit", "-qm", "fixture"), + ): + subprocess.run(["git", *args], cwd=self.repo, env=git_env, check=True) + commit = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=self.repo, env=git_env, text=True + ).strip() + self.result = self.evidence / f"result-2030-01-01-{commit[:8]}.json" + bin_dir = root / "bin" + bin_dir.mkdir() + self.tmp = root / "tmp" + self.tmp.mkdir() + self.env = dict(git_env, PATH=f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + TMPDIR=str(self.tmp), REAL_MV=shutil.which("mv") or "") + self.assertTrue(self.env["REAL_MV"]) + self.write_command(bin_dir, "date", "#!/bin/sh\nprintf '2030-01-01\\n'\n") + self.write_command(bin_dir, "bazel", '''#!/usr/bin/env python3 +import json +from pathlib import Path +import sys + +if sys.argv[1:] == ["--version"]: + print("bazel fixture") +elif "--test_arg=production_observation_pipeline_baseline" in sys.argv: + print('TRACEDECAY_CLAUDE_OBSERVATION_BENCHMARK_RESULT={"measured":true} ') +elif "--test_arg=evidence_directory_matches_index_contract" in sys.argv: + prefix = "--test_env=TRACEDECAY_BENCHMARK_EVIDENCE_DIR=" + directory = Path(next(arg[len(prefix):] for arg in sys.argv if arg.startswith(prefix))) + index = json.loads((directory / "evidence-index.json").read_text()) + assert json.loads((directory / index["current_acceptance"]).read_text()) == {"measured": True} + for name in index["historical_stale"]: + assert (directory / name).is_symlink() + assert json.loads((directory / name).read_text())["evidence_status"] == "historical_stale" + print("test result: ok. 1 passed; 0 failed; 0 ignored") +else: + raise SystemExit("unexpected Bazel command") +''') + self.write_command(bin_dir, "mv", '''#!/usr/bin/env python3 +import os +from pathlib import Path +import signal +import subprocess +import sys + +index_move = Path(sys.argv[1]).name == "evidence-index.json" +mode = os.environ.get("PUBLICATION_FAULT", "") +if index_move and mode == "index_move_failure": + raise SystemExit(73) +if index_move and mode == "unreadable_index": + Path(sys.argv[2]).unlink() + raise SystemExit(74) +if index_move and mode == "malformed_index": + Path(sys.argv[2]).write_text("{invalid") + raise SystemExit(74) +result = subprocess.run([os.environ["REAL_MV"], *sys.argv[1:]], check=False) +if result.returncode: + raise SystemExit(result.returncode) +boundary = "after_index" if index_move else "after_result" +if mode == boundary: + os.kill(os.getppid(), getattr(signal, os.environ["PUBLICATION_SIGNAL"])) +''') + + @staticmethod + def write_command(directory: Path, name: str, body: str) -> None: + path = directory / name + path.write_text(body) + path.chmod(0o755) + + def run_publication(self, fault: str = "", signal: str = "SIGTERM"): + completed = subprocess.run( + ["bash", str(self.repo / "scripts" / RUNNER)], cwd=self.repo, + env=dict(self.env, PUBLICATION_FAULT=fault, PUBLICATION_SIGNAL=signal), + capture_output=True, text=True, timeout=15, + ) + self.assertEqual(self.historical.read_bytes(), self.historical_bytes) + self.assertEqual(list(self.tmp.iterdir()), [], "publication staging was retained") + return completed + + def assert_published(self) -> None: + self.assertEqual(json.loads(self.result.read_text()), {"measured": True}) + self.assertEqual(json.loads(self.index.read_text())["current_acceptance"], self.result.name) + + def test_failed_index_move_removes_new_result_and_allows_retry(self) -> None: + failed = self.run_publication("index_move_failure") + self.assertEqual(failed.returncode, 73, failed.stderr) + self.assertFalse(self.result.exists()) + self.assertEqual(self.index.read_bytes(), self.index_bytes) + retried = self.run_publication() + self.assertEqual(retried.returncode, 0, retried.stderr) + self.assert_published() + + def test_interrupt_after_result_removes_only_unpublished_output(self) -> None: + for sig, code in (("SIGINT", 130), ("SIGTERM", 143)): + with self.subTest(signal=sig): + interrupted = self.run_publication("after_result", sig) + self.assertEqual(interrupted.returncode, code, interrupted.stderr) + self.assertFalse(self.result.exists()) + self.assertEqual(self.index.read_bytes(), self.index_bytes) + retried = self.run_publication() + self.assertEqual(retried.returncode, 0, retried.stderr) + self.assert_published() + + def test_interrupt_after_index_preserves_published_result(self) -> None: + interrupted = self.run_publication("after_index", "SIGTERM") + self.assertEqual(interrupted.returncode, 143, interrupted.stderr) + self.assert_published() + + def test_int_after_index_preserves_published_result(self) -> None: + interrupted = self.run_publication("after_index", "SIGINT") + self.assertEqual(interrupted.returncode, 130, interrupted.stderr) + self.assert_published() + + def test_unreadable_index_preserves_result_and_reports_cleanup_failure(self) -> None: + failed = self.run_publication("unreadable_index") + self.assertEqual(failed.returncode, 74, failed.stderr) + self.assertEqual(json.loads(self.result.read_text()), {"measured": True}) + self.assertIn("cannot determine benchmark publication; preserving result", failed.stderr) + + def test_malformed_index_preserves_result_and_reports_cleanup_failure(self) -> None: + failed = self.run_publication("malformed_index") + self.assertEqual(failed.returncode, 74, failed.stderr) + self.assertEqual(json.loads(self.result.read_text()), {"measured": True}) + self.assertIn("cannot determine benchmark publication; preserving result", failed.stderr) + + def test_preexisting_result_is_preserved(self) -> None: + original = b'{"preexisting":true}\n' + self.result.write_bytes(original) + # Ignore this fixture artifact so the real overwrite refusal, rather + # than the earlier dirty-worktree guard, is exercised. + (self.repo / ".git/info/exclude").write_text(f"{self.result.relative_to(self.repo)}\n") + refused = self.run_publication() + self.assertNotEqual(refused.returncode, 0) + self.assertIn("refusing to overwrite", refused.stderr) + self.assertEqual(self.result.read_bytes(), original) + self.assertEqual(self.index.read_bytes(), self.index_bytes) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/scripts/test-efficiency-scorecard.py b/scripts/test-efficiency-scorecard.py index 8055885ee9..dc3540db43 100644 --- a/scripts/test-efficiency-scorecard.py +++ b/scripts/test-efficiency-scorecard.py @@ -2,7 +2,9 @@ from __future__ import annotations import importlib.util +import subprocess import sys +import tempfile import unittest from pathlib import Path from types import ModuleType @@ -22,6 +24,55 @@ def load_scorecard() -> ModuleType: return module +class ScorecardBuildTests(unittest.TestCase): + def test_resolves_binary_from_configured_bazel_output(self) -> None: + scorecard = load_scorecard() + with tempfile.TemporaryDirectory(prefix="scorecard build ") as scratch: + output = Path(scratch) + binary = output / "crates" / "tracedecay-cli" / "tracedecay" + binary.parent.mkdir(parents=True) + binary.touch() + with mock.patch.object( + scorecard.subprocess, + "run", + side_effect=[ + subprocess.CompletedProcess([], 0), + subprocess.CompletedProcess([], 0, f"{output}\n", ""), + ], + ) as run: + self.assertEqual(scorecard.build_binary(), binary) + self.assertEqual( + [call.args[0] for call in run.call_args_list], + [ + ("bazel", "build", "--config=release", "//crates/tracedecay-cli:tracedecay"), + ("bazel", "info", "--config=release", "bazel-bin"), + ], + ) + + def test_failed_build_does_not_use_an_old_binary(self) -> None: + scorecard = load_scorecard() + with mock.patch.object( + scorecard.subprocess, "run", return_value=subprocess.CompletedProcess([], 1) + ) as run: + with self.assertRaisesRegex(scorecard.HarnessError, "release build failed"): + scorecard.build_binary() + self.assertEqual(run.call_count, 1) + + def test_failed_or_empty_output_discovery_is_an_error(self) -> None: + scorecard = load_scorecard() + for discovery in ( + subprocess.CompletedProcess([], 1, "", "configuration failed"), + subprocess.CompletedProcess([], 0, "", ""), + ): + with self.subTest(discovery=discovery), mock.patch.object( + scorecard.subprocess, + "run", + side_effect=[subprocess.CompletedProcess([], 0), discovery], + ): + with self.assertRaisesRegex(scorecard.HarnessError, "resolve the Bazel output"): + scorecard.build_binary() + + class FakeStatusObserver: def __init__( self, diff --git a/scripts/test-neutralize-release-pr-closing-keywords.py b/scripts/test-neutralize-release-pr-closing-keywords.py index 1d8691b785..9f827ce98b 100644 --- a/scripts/test-neutralize-release-pr-closing-keywords.py +++ b/scripts/test-neutralize-release-pr-closing-keywords.py @@ -229,6 +229,12 @@ def git(*args: str, cwd: Path = main) -> None: edited_path = scratch_path / "edited.md" stubs = { "cargo": cargo_stub.format(lockfile=checkout / "Cargo.lock"), + "bazel": ( + "#!/bin/sh\n" + '[ "$#" -eq 2 ] && [ "$1" = run ] && ' + '[ "$2" = //scripts:bazel/gen_builds.py ] || exit 64\n' + 'exec python3 scripts/bazel/gen_builds.py\n' + ), "gh": ( "#!/bin/sh\n" 'case "$1 $2" in\n' diff --git a/scripts/tool-sweep.sh b/scripts/tool-sweep.sh index 532a7f05af..4100b5ac6e 100755 --- a/scripts/tool-sweep.sh +++ b/scripts/tool-sweep.sh @@ -20,7 +20,7 @@ reported as failed rather than skipped. EOF } -BIN="${TRACEDECAY_BIN:-$REPO_ROOT/target/release/tracedecay}" +BIN="${TRACEDECAY_BIN:-$REPO_ROOT/bazel-bin/crates/tracedecay-cli/tracedecay}" OUT="" WHOLE_RUN_DEADLINE_MS="${TRACEDECAY_SWEEP_WHOLE_RUN_DEADLINE_MS:-1800000}" diff --git a/scripts/update-release-pr-lockfile.sh b/scripts/update-release-pr-lockfile.sh index 5cc8474415..c1363d7c50 100755 --- a/scripts/update-release-pr-lockfile.sh +++ b/scripts/update-release-pr-lockfile.sh @@ -33,7 +33,7 @@ repository_root=$PWD # pnpm validates workspace crate versions even for a frozen npm installation. # Repair the release's Cargo lock before installing generated-build inputs. pnpm install --frozen-lockfile -python3 scripts/bazel/gen_builds.py +bazel run //scripts:bazel/gen_builds.py pnpm --dir sdks/typescript run build pnpm --dir plugin/chatgpt-extension run build diff --git a/scripts/with-isolated-tracedecay-daemon.sh b/scripts/with-isolated-tracedecay-daemon.sh index 7f52042c9b..83e0563a92 100755 --- a/scripts/with-isolated-tracedecay-daemon.sh +++ b/scripts/with-isolated-tracedecay-daemon.sh @@ -9,7 +9,6 @@ usage() { cat >&2 <<'EOF' Usage: with-isolated-tracedecay-daemon.sh --bin PATH [options] -- COMMAND [ARG...] - with-isolated-tracedecay-daemon.sh --cargo DIR [options] -- COMMAND [ARG...] Options: --ready-timeout SECONDS Daemon readiness deadline (default: 60) @@ -27,15 +26,13 @@ EOF ready_timeout=60 stop_timeout=5 lifecycle_label="" -daemon_mode="" daemon_value="" while (($# > 0)); do case "$1" in - --bin | --cargo) + --bin) (($# >= 2)) || usage - [[ -z "$daemon_mode" ]] || usage - daemon_mode="${1#--}" + [[ -z "$daemon_value" ]] || usage daemon_value="$2" shift 2 ;; @@ -64,27 +61,16 @@ while (($# > 0)); do esac done -[[ -n "$daemon_mode" && $# -gt 0 ]] || usage +[[ -n "$daemon_value" && $# -gt 0 ]] || usage for value in "$ready_timeout" "$stop_timeout"; do [[ "$value" =~ ^[1-9][0-9]*$ ]] || usage done -case "$daemon_mode" in - bin) - [[ -x "$daemon_value" ]] || { - echo "error: tracedecay binary is not executable: $daemon_value" >&2 - exit 2 - } - daemon_value="$(cd "$(dirname "$daemon_value")" && pwd)/$(basename "$daemon_value")" - ;; - cargo) - [[ -f "$daemon_value/Cargo.toml" ]] || { - echo "error: Cargo.toml not found under: $daemon_value" >&2 - exit 2 - } - daemon_value="$(cd "$daemon_value" && pwd)" - ;; -esac +[[ -x "$daemon_value" ]] || { + echo "error: tracedecay binary is not executable: $daemon_value" >&2 + exit 2 +} +daemon_value="$(cd "$(dirname "$daemon_value")" && pwd)/$(basename "$daemon_value")" command -v python3 >/dev/null 2>&1 || { echo "error: python3 is required for the bounded daemon socket probe" >&2 @@ -163,17 +149,9 @@ trap 'exit 130' INT trap 'exit 143' TERM [[ -z "$lifecycle_label" ]] || echo "== starting $lifecycle_label" -if [[ "$daemon_mode" == "bin" ]]; then - python3 -S "$PROCESS_HELPER" exec-session --parent-pid "$$" -- \ - "$daemon_value" daemon run --socket "$TRACEDECAY_DAEMON_SOCKET" \ - >"$daemon_log" 2>&1 & -else - ( - cd "$daemon_value" - exec python3 -S "$PROCESS_HELPER" exec-session --parent-pid "$$" -- \ - cargo run -- daemon run --socket "$TRACEDECAY_DAEMON_SOCKET" - ) >"$daemon_log" 2>&1 & -fi +python3 -S "$PROCESS_HELPER" exec-session --parent-pid "$$" -- \ + "$daemon_value" daemon run --socket "$TRACEDECAY_DAEMON_SOCKET" \ + >"$daemon_log" 2>&1 & daemon_pid=$! # `exec-session` execs into the daemon, so this is the daemon's own PID. The # smoke command samples its resident memory per readiness probe; a timeout or diff --git a/sdks/codegen/BUILD.bazel b/sdks/codegen/BUILD.bazel index 80dae49629..e201407e05 100644 --- a/sdks/codegen/BUILD.bazel +++ b/sdks/codegen/BUILD.bazel @@ -37,3 +37,25 @@ rust_binary( edition = "2024", deps = ["//crates/tracedecay-dashboard-api:tracedecay-dashboard-api"], ) + +# BEGIN generated by scripts/bazel/gen_builds.py +filegroup( + name = "rustfmt_sources_2024", + srcs = depset(["dashboard_schema.rs"]).to_list(), + tags = ["manual"], +) + + +genrule( + name = "rustfmt_check_2024", + srcs = [":rustfmt_sources_2024", "//:rustfmt.toml"], + outs = ["rustfmt_check_2024.ok"], + cmd = "\"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $(locations :rustfmt_sources_2024) && touch \"$@\"", + cmd_ps = "foreach ($$source in ('$(locations :rustfmt_sources_2024)' -split ' ')) { & \"$(RUSTFMT)\" --config-path \"$(location //:rustfmt.toml)\" --edition 2024 --config skip_children=true --check $$source; if ($$LASTEXITCODE -ne 0) { exit $$LASTEXITCODE } }; New-Item -ItemType File -Path \"$@\" -Force | Out-Null", + tools = ["//:rustfmt_toolchain"], + toolchains = ["//:rustfmt_toolchain"], + tags = ["manual"], + visibility = ["//visibility:public"], +) + +# END generated by scripts/bazel/gen_builds.py diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel index aa23c045dd..861baa535c 100644 --- a/tests/BUILD.bazel +++ b/tests/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. python3 scripts/bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py # BEGIN generated by scripts/bazel/gen_builds.py filegroup( @@ -12,6 +12,12 @@ exports_files(["storage_runtime_rusqlite_suite/main.rs", "storage_runtime_rusqli visibility = ["//visibility:public"], ) +filegroup( + name = "storage_runtime_rusqlite_suite_sources", + srcs = glob(["storage_runtime_rusqlite_suite/**/*.rs"]), + visibility = ["//visibility:public"], +) + filegroup( name = "support", srcs = glob(["support/**"]), @@ -22,6 +28,12 @@ exports_files(["support/isolated_profile.rs"], visibility = ["//visibility:public"], ) +filegroup( + name = "support_sources", + srcs = glob(["support/**/*.rs"]), + visibility = ["//visibility:public"], +) + filegroup( name = "fixtures", srcs = glob(["fixtures/**"]),