diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e39363970..a8eb48c7c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,3 +68,12 @@ jobs: - name: compose matrix assertions run: bash deploy/scripts/assert-compose-matrix.sh + + - name: harbor runner results gate + run: bash deploy/scripts/assert-harbor-runner-results-emit.sh + + - name: proof task-selection gates + run: | + python3 -m unittest discover -s deploy/guest/runners/rlm_fc_in_guest_harbor/tests -q + bash deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_adaptor.sh + bash deploy/scripts/test_proof_slice_preflight.sh diff --git a/Cargo.lock b/Cargo.lock index fabfc40a5..41491db39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3764,6 +3764,7 @@ dependencies = [ "proof-rlm", "proof-rlm-scorer", "proof-rlm-store", + "proof-score", "proof-store", "proof-submit", "proof-task", @@ -3860,6 +3861,7 @@ dependencies = [ "async-trait", "hex", "proof-fc-harvest", + "proof-fc-net", "proof-results", "proof-rlm", "proof-vm-agent", @@ -3873,6 +3875,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "proof-fc-net" +version = "0.1.0" +dependencies = [ + "async-trait", + "proof-fc-harvest", + "proof-vm-agent", + "serde_json", + "tokio", + "tracing", +] + [[package]] name = "proof-harvest" version = "0.1.0" @@ -3947,8 +3961,10 @@ dependencies = [ "proof-canon", "proof-experiment", "proof-results", + "proof-rlm-lifecycle", "proof-score", "proof-task", + "proof-topic-authoring", "serde", "serde_json", "sha2 0.10.9", @@ -3957,6 +3973,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "proof-rlm-lifecycle" +version = "0.1.0" +dependencies = [ + "proof-task", + "serde", + "serde_json", + "thiserror 2.0.19", +] + [[package]] name = "proof-rlm-scorer" version = "0.1.0" @@ -3997,6 +4023,7 @@ dependencies = [ "db", "proof-rlm", "proof-task", + "proof-topic-authoring", "serde", "serde_json", "sqlx", @@ -4058,6 +4085,20 @@ dependencies = [ "toml", ] +[[package]] +name = "proof-topic-authoring" +version = "0.1.0" +dependencies = [ + "hex", + "proof-canon", + "proof-task", + "proof-topic-sql-guard", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.19", +] + [[package]] name = "proof-topic-bundle" version = "0.1.0" @@ -4083,6 +4124,7 @@ dependencies = [ "proof-rlm", "proof-rlm-store", "proof-task", + "proof-topic-authoring", "proof-topic-bundle", "proof-topic-sql-guard", "serde", @@ -4101,7 +4143,9 @@ dependencies = [ "proof-eval", "proof-rlm", "proof-rlm-store", + "proof-score", "proof-task", + "proof-topic-authoring", "proof-topic-bundle", "proof-topic-install", "proof-topic-setup", @@ -4118,7 +4162,10 @@ dependencies = [ "proof-eval", "proof-rlm", "proof-rlm-store", + "proof-score", "proof-task", + "proof-topic-authoring", + "serde_json", "thiserror 2.0.19", "tokio", ] @@ -4138,6 +4185,7 @@ dependencies = [ "axum", "http-body-util", "proof-canon", + "proof-experiment", "proof-rlm", "proof-vm-proto", "serde", @@ -4178,6 +4226,7 @@ dependencies = [ "proof-rlm", "proof-task", "proof-vm-proto", + "proof-vm-staging", "reqwest 0.12.28", "serde", "serde_json", @@ -4238,6 +4287,19 @@ dependencies = [ "tokio", ] +[[package]] +name = "proof-vm-staging" +version = "0.1.0" +dependencies = [ + "hex", + "proof-canon", + "proof-experiment", + "proof-vm-proto", + "serde_json", + "sha2 0.10.9", + "tar", +] + [[package]] name = "proptest" version = "1.11.0" diff --git a/bins/proof-admin/src/install.rs b/bins/proof-admin/src/install.rs index 7aa6460f6..1d6bdf1fe 100644 --- a/bins/proof-admin/src/install.rs +++ b/bins/proof-admin/src/install.rs @@ -243,6 +243,14 @@ async fn run_real( rlm_raw: bundle.rlm.raw(), registered_custom: registered, skip_baseline: args.skip_baseline, + // The RLM's own set, when the drive produced one: it is the + // topic's behavior, and the install applies it instead of the + // bundle's section. Absent means the RLM has not authored yet + // (or authored only rules), and the install applies the + // operator's section with its honest `topic_document` + // provenance — which the publish gate will refuse to open a + // topic on. + authored: driven.as_ref().and_then(|o| o.authored.as_deref()), }, setup, ) @@ -594,6 +602,7 @@ fn install_failure(err: &InstallError, topic_id: &str) -> String { InstallError::MigrationFailed { .. } => "a migration failed in the database", InstallError::TooManyMigrations { .. } => "the bundle declares too many migrations", InstallError::HandlerNotAllowed(_) => "the handler allow-list refused the run backend", + InstallError::Authoring(_) => "the set the topic's own RLM authored was refused", InstallError::CustomIdNotRegistered { .. } => { "the topic's custom id is not registered on this host" } @@ -602,6 +611,9 @@ fn install_failure(err: &InstallError, topic_id: &str) -> String { InstallError::Store(_) => "the rule store refused", InstallError::Db(_) => "the database refused", InstallError::Binding(_) => "the signed document's runner binding is malformed", + InstallError::CrossTopicClaim { .. } => { + "a migration names an object another registered topic also claims" + } }; format!( "the install stopped: {step}\n {err}\n\n Rollback notes — what is and is not changed:\n \ diff --git a/bins/proof-admin/src/main.rs b/bins/proof-admin/src/main.rs index 798bcf08f..839792c56 100644 --- a/bins/proof-admin/src/main.rs +++ b/bins/proof-admin/src/main.rs @@ -65,10 +65,10 @@ const EXIT_USAGE: u8 = 2; long_about = "proof-admin wraps the existing Proof topic publish path (dynamic-topics P0). Validate a bundle — runs the same acceptance checks POST /v1/admin/proof/topics runs: - proof-admin topic validate --bundle tb4.json --pin config/proof-pin.toml + proof-admin topic validate --bundle topic.json --pin config/proof-pin.toml Resolve the publish call and host env without touching anything: - proof-admin topic install --bundle tb4.json --env metal --dry-run + proof-admin topic install --bundle topic.json --env metal --dry-run List the installed topics (a read-only view of proof_topic_version): proof-admin topic list @@ -179,6 +179,18 @@ enum TopicCmd { }, /// List installed topics: a read-only view of `proof_topic_version`. List, + /// Where a topic is in its RLM lifecycle, and what it is waiting on. + /// + /// Read-only. `topic install --drive-rlm` prints one line and then nothing + /// until the whole run returns — provisioning a VM, the RLM's + /// `propose_rules` job, and a paid baseline can take hours, so this is the + /// read that tells a working run from a stopped one. The durable progress + /// is `proof_lifecycle_event`; a run that died left its last transition + /// here. + Lifecycle { + /// Topic slug, or an alias of one. + topic_id: String, + }, /// Show one topic's newest install: the `proof_topic_install` journal. InstallLog { /// Topic slug. @@ -273,9 +285,9 @@ enum TopicCmd { enum AliasCmd { /// Point an alias at a topic. The topic must be published already. Set { - /// The alias slug (e.g. `tbench`). + /// The alias slug (a temporary compatibility spelling of the topic). alias: String, - /// The canonical topic slug it resolves to (e.g. `tb4`). + /// The canonical topic slug it resolves to. #[arg(long, value_name = "TOPIC_ID")] topic: String, }, @@ -382,6 +394,7 @@ async fn run_topic(opts: &Options, cmd: &TopicCmd) -> Result<(), Failure> { } TopicCmd::List => cmd_list(opts).await, TopicCmd::InstallLog { topic } => cmd_install_log(opts, topic).await, + TopicCmd::Lifecycle { topic_id } => cmd_lifecycle(opts, topic_id).await, TopicCmd::Show { topic_id } => cmd_show(opts, topic_id).await, TopicCmd::Alias { cmd } => run_alias(opts, cmd).await, TopicCmd::Disable { @@ -616,6 +629,53 @@ async fn cmd_install_log(opts: &Options, topic_id: &str) -> Result<(), Failure> install::install_log(opts, topic_id).await } +/// `topic lifecycle`: read the journal and the provenance, and say what is next. +async fn cmd_lifecycle(opts: &Options, topic_id: &str) -> Result<(), Failure> { + let pool = open_pool(opts).await?; + let report = proof_topic_ops::lifecycle(&pool, topic_id) + .await + .map_err(ops_to_failure)?; + if opts.json { + return print_json(&serde_json::json!({ + "topic_id": report.topic_id, + "state": report.state, + "rules_version": report.rules_version, + "rules_source": report.rules_source, + "baseline_rules_version": report.baseline_rules_version, + "history": report.history, + "next": report.next_steps(), + })); + } + println!("topic {} — lifecycle", report.topic_id); + println!(" state {}", report.state); + println!( + " rules_version {}", + report + .rules_version + .map_or_else(|| "-".to_owned(), |v| v.to_string()) + ); + println!( + " rules_source {}", + dash_if_empty(report.rules_source.as_deref().unwrap_or_default()) + ); + println!( + " baseline {}", + report.baseline_rules_version.map_or_else( + || "not measured".to_owned(), + |v| format!("measured (rules v{v})") + ) + ); + if !report.history.is_empty() { + println!(" history (oldest first)"); + for line in &report.history { + println!(" {line}"); + } + } + println!(); + println!("{}", report.next_steps()); + Ok(()) +} + /// `topic baseline`: read the measurement and print what to seal. /// /// The procedure is [`proof_topic_ops::baseline`]; this is the printing. @@ -635,6 +695,7 @@ async fn cmd_baseline(opts: &Options, topic_id: &str, pin_path: &Path) -> Result "holdout_commitment": report.holdout_commitment, "metrics_commitment": report.metrics_commitment, "document_status": report.document_status, + "degenerate_bar": report.degenerate_bar, "next": report.next_steps(), })); } @@ -648,6 +709,18 @@ async fn cmd_baseline(opts: &Options, topic_id: &str, pin_path: &Path) -> Result " document_status {}", status_word(report.document_status) ); + if report.degenerate_bar { + // Printed **before** the commitment, because signing this number + // would only produce a document the seal refuses. + println!(); + println!( + "⚠ degenerate bar: {} is zero, and this family scores a relative win", + report.metric_primary + ); + println!(" (`challenger >= bar * (1 + epsilon_rel)`), so no miner could ever pass."); + println!(" Sealing this measurement will be refused. Re-run the reference against"); + println!(" something that scores, or fix the task selection, then measure again."); + } println!(); println!("An `open` document must seal this measurement. Its baseline block needs:"); println!(" metrics_commitment {}", report.metrics_commitment); diff --git a/bins/proof-admin/src/registry.rs b/bins/proof-admin/src/registry.rs index 27c3c5a43..022ad3b9b 100644 --- a/bins/proof-admin/src/registry.rs +++ b/bins/proof-admin/src/registry.rs @@ -43,8 +43,9 @@ pub(crate) async fn cmd_list(opts: &Options) -> Result<(), Failure> { pub(crate) async fn cmd_show(opts: &Options, topic_id: &str) -> Result<(), Failure> { let pool = open_pool(opts).await?; let store = PgRlmStore::new(pool.clone()); - // An alias resolves to its canonical slug first, so `show tbench` finds - // `tb4`. Resolution is fail-closed in the store: an alias whose topic has + // An alias resolves to its canonical slug first, so `show ` finds + // the topic it points at. Resolution is fail-closed in the store: an alias + // whose topic has // no published version resolves to nothing rather than to an empty row. let resolved = store .resolve_alias(topic_id) diff --git a/bins/proof-admin/tests/cli.rs b/bins/proof-admin/tests/cli.rs index 8581fac5a..0a080246c 100644 --- a/bins/proof-admin/tests/cli.rs +++ b/bins/proof-admin/tests/cli.rs @@ -114,11 +114,12 @@ max_output_tokens = 8192 ) } - /// A signed custom topic selecting the in-guest runner, the shape the live - /// `tb4` topic has. + /// A signed custom topic selecting the in-guest runner, the shape a real + /// bundle carries. The slug is a fixture string: no topic id is special to + /// this build, and which topics exist is what an operator published. pub fn signed_topic(pack_digest: &str) -> TopicDocument { let mut doc = TopicDocument { - id: "tb4".into(), + id: "fixture-topic-v0".into(), statement: "Score the pinned task pack with the pinned runner.".into(), payout_mode: PayoutMode::Discovery, constraints: Constraints::default(), @@ -128,7 +129,7 @@ max_output_tokens = 8192 direction: MetricDirection::Max, unit: "rate".into(), epsilon_rel: 0.05, - custom_id: "tbench".into(), + custom_id: "fixture_metric_v0".into(), ..MetricSpec::default() }, baseline: default_adamw(FLOPS_BUDGET_MAX), @@ -148,8 +149,8 @@ max_output_tokens = 8192 doc } - /// The Arch default bundle: slug `tb4`, custom id `tbench`, and the - /// temporary alias `tbench` the Owner default declares. + /// The fixture bundle: a signed custom topic, its alias, and the host env + /// that must agree with it. pub fn bundle_json(environment: &str) -> String { let hex = "ab".repeat(32); let pack = format!("sha256:{hex}"); @@ -157,15 +158,15 @@ max_output_tokens = 8192 let bundle = serde_json::json!({ "schema_version": 1, "environment": environment, - "display_name": "Terminal-Bench 4", + "display_name": "Fixture Topic", "topic": topic, - "aliases": ["tbench"], + "aliases": ["fixture-alias"], "host": { "rlm_image_digest": format!("sha256:{hex}"), "experiment_image_digest": format!("sha256:{hex}"), "pack_digest": pack, "pack_dir": "/var/lib/proof/packs", - "custom_ids_entry": "tbench" + "custom_ids_entry": "fixture_metric_v0" }, // A small illustrative RLM section, so the committed fixture also // exercises the hand-off. A real bundle carries the topic's own @@ -175,7 +176,7 @@ max_output_tokens = 8192 {"id": "no_short_circuit", "text": "the harness must run the task"} ], "migrations": [ - {"name": "0001_scratch", "sql": "CREATE TABLE tb4_scratch (id TEXT)"} + {"name": "0001_scratch", "sql": "CREATE TABLE fixture_topic_v0_scratch (id TEXT)"} ], "apis": [ {"path": "status", "method": "GET", "summary": "topic status"} @@ -218,25 +219,25 @@ fn regenerate_dry_run_fixture() { }; fs::create_dir_all(&dir).expect("fixture dir"); fs::write( - dir.join("tb4.install-bundle.json"), + dir.join("topic.install-bundle.json"), fixture::bundle_json("staging"), ) .expect("bundle"); - fs::write(dir.join("tb4.pin.toml"), fixture::pin_toml()).expect("pin"); + fs::write(dir.join("topic.pin.toml"), fixture::pin_toml()).expect("pin"); eprintln!("wrote the dry-run fixture to {}", dir.display()); } /// The committed dry-run fixture must stay runnable. /// -/// `tests/fixtures/tb4.bundle.json` + `tb4.pin.toml` are the operator artifact -/// the A→Z walkthrough uses, so a schema change that quietly breaks them must -/// fail here rather than in the Owner's hands. This runs the **same two -/// commands** the fixture README documents. +/// `tests/fixtures/topic.install-bundle.json` + `topic.pin.toml` are the +/// operator artifact the A→Z walkthrough uses, so a schema change that quietly +/// breaks them must fail here rather than in the Owner's hands. This runs the +/// **same two commands** the fixture README documents. #[test] fn the_committed_dry_run_fixture_still_validates_and_plans() { let fixtures = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"); - let bundle = fixtures.join("tb4.install-bundle.json"); - let pin = fixtures.join("tb4.pin.toml"); + let bundle = fixtures.join("topic.install-bundle.json"); + let pin = fixtures.join("topic.pin.toml"); assert!(bundle.is_file(), "missing {}", bundle.display()); assert!(pin.is_file(), "missing {}", pin.display()); @@ -251,8 +252,11 @@ fn the_committed_dry_run_fixture_still_validates_and_plans() { ]); assert_eq!(code(&out), 0, "fixture must validate: {}", stderr(&out)); let body = stdout(&out); - assert!(body.contains("topic_id tb4"), "{body}"); - assert!(body.contains("custom_id tbench"), "{body}"); + assert!(body.contains("topic_id fixture-topic-v0"), "{body}"); + assert!( + body.contains("custom_id fixture_metric_v0"), + "{body}" + ); assert!( body.contains("rlm_install present"), "the fixture carries an RLM section: {body}" @@ -278,10 +282,10 @@ fn the_committed_dry_run_fixture_still_validates_and_plans() { body.contains("Hand control to the topic's RLM"), "the plan must show the hand-off: {body}" ); - // The fixture carries the Owner-default alias, so the plan must say so: - // an operator reads the plan to know what the install will do. + // The fixture declares an alias, so the plan must say so: an operator + // reads the plan to know what the install will do. assert!( - body.contains("tbench"), + body.contains("fixture-alias"), "the plan must name the alias the fixture declares: {body}" ); @@ -310,7 +314,7 @@ fn the_committed_dry_run_fixture_still_validates_and_plans() { #[test] fn validate_accepts_the_arch_default_bundle_and_writes_nothing() { let dir = workdir("validate-ok"); - let bundle = write_file(&dir, "tb4.json", &fixture::bundle_json("metal")); + let bundle = write_file(&dir, "topic.json", &fixture::bundle_json("metal")); let pin = write_file(&dir, "pin.toml", &fixture::pin_toml()); let out = run(&[ "topic", @@ -324,9 +328,9 @@ fn validate_accepts_the_arch_default_bundle_and_writes_nothing() { let body = stdout(&out); for needle in [ "is valid", - "topic_id tb4", + "topic_id fixture-topic-v0", "environment metal", - "custom_id tbench", + "custom_id fixture_metric_v0", "runner_id rlm_fc_in_guest_harbor", "bundle_digest sha256:", "Nothing was written", @@ -420,7 +424,7 @@ fn validate_refuses_a_document_the_publish_route_would_refuse() { #[test] fn validate_json_output_is_machine_readable() { let dir = workdir("validate-json"); - let bundle = write_file(&dir, "tb4.json", &fixture::bundle_json("metal")); + let bundle = write_file(&dir, "topic.json", &fixture::bundle_json("metal")); let pin = write_file(&dir, "pin.toml", &fixture::pin_toml()); let out = run(&[ "--json", @@ -435,9 +439,9 @@ fn validate_json_output_is_machine_readable() { let parsed: serde_json::Value = serde_json::from_str(&stdout(&out)).expect("validate --json is JSON"); assert_eq!(parsed["ok"], true); - assert_eq!(parsed["topic_id"], "tb4"); + assert_eq!(parsed["topic_id"], "fixture-topic-v0"); assert_eq!(parsed["environment"], "metal"); - assert_eq!(parsed["custom_id"], "tbench"); + assert_eq!(parsed["custom_id"], "fixture_metric_v0"); assert_eq!(parsed["runner_id"], "rlm_fc_in_guest_harbor"); assert!( parsed["bundle_digest"] @@ -454,7 +458,7 @@ fn validate_json_output_is_machine_readable() { #[test] fn dry_run_install_prints_the_existing_publish_call_and_host_env() { let dir = workdir("dry-run"); - let bundle = write_file(&dir, "tb4.json", &fixture::bundle_json("metal")); + let bundle = write_file(&dir, "topic.json", &fixture::bundle_json("metal")); let pin = write_file(&dir, "pin.toml", &fixture::pin_toml()); let out = run(&[ "topic", @@ -472,9 +476,9 @@ fn dry_run_install_prints_the_existing_publish_call_and_host_env() { let body = stdout(&out); for needle in [ "topic install plan", - "topic_id tb4", + "topic_id fixture-topic-v0", "environment metal", - "custom_id tbench", + "custom_id fixture_metric_v0", "runner_id rlm_fc_in_guest_harbor", "Hand control to the topic's RLM (it installs and sets the topic up)", "provision -> propose_rules -> baseline", @@ -484,7 +488,7 @@ fn dry_run_install_prints_the_existing_publish_call_and_host_env() { "chmod 600", "--data-binary @\"$PROOF_TOPIC_DIR/document.json\"", "/challenge/proof/v1/admin/proof/topics", - "PROOF_VM_RUNNER_CUSTOM_IDS=tbench", + "PROOF_VM_RUNNER_CUSTOM_IDS=fixture_metric_v0", "PROOF_RLM_VM_IMAGE_DIGEST=sha256:", "PROOF_EXPERIMENT_VM_IMAGE_DIGEST=sha256:", "PROOF_VM_AGENT_EXPERIMENT_PACK_DIR=/var/lib/proof/packs", @@ -549,7 +553,7 @@ fn dry_run_install_prints_the_existing_publish_call_and_host_env() { #[test] fn dry_run_install_json_matches_the_plan_shape() { let dir = workdir("dry-run-json"); - let bundle = write_file(&dir, "tb4.json", &fixture::bundle_json("staging")); + let bundle = write_file(&dir, "topic.json", &fixture::bundle_json("staging")); let pin = write_file(&dir, "pin.toml", &fixture::pin_toml()); let out = run(&[ "--json", @@ -566,9 +570,9 @@ fn dry_run_install_json_matches_the_plan_shape() { assert_eq!(code(&out), 0, "stderr={}", stderr(&out)); let parsed: serde_json::Value = serde_json::from_str(&stdout(&out)).expect("dry run --json is JSON"); - assert_eq!(parsed["topic_id"], "tb4"); + assert_eq!(parsed["topic_id"], "fixture-topic-v0"); assert_eq!(parsed["environment"], "staging"); - assert_eq!(parsed["custom_id"], "tbench"); + assert_eq!(parsed["custom_id"], "fixture_metric_v0"); assert_eq!(parsed["runner_id"], "rlm_fc_in_guest_harbor"); assert_eq!(parsed["publish_route"], "POST /v1/admin/proof/topics"); assert_eq!(parsed["pack_dir_env"], "PROOF_VM_AGENT_EXPERIMENT_PACK_DIR"); @@ -583,7 +587,7 @@ fn dry_run_install_json_matches_the_plan_shape() { #[test] fn install_refuses_an_environment_the_bundle_does_not_declare() { let dir = workdir("env-mismatch"); - let bundle = write_file(&dir, "tb4.json", &fixture::bundle_json("metal")); + let bundle = write_file(&dir, "topic.json", &fixture::bundle_json("metal")); let pin = write_file(&dir, "pin.toml", &fixture::pin_toml()); let out = run(&[ "topic", @@ -616,7 +620,7 @@ fn install_refuses_an_environment_the_bundle_does_not_declare() { #[test] fn a_real_install_refuses_without_a_master_and_a_bearer_and_changes_nothing() { let dir = workdir("real-install-config"); - let bundle = write_file(&dir, "tb4.json", &fixture::bundle_json("metal")); + let bundle = write_file(&dir, "topic.json", &fixture::bundle_json("metal")); let pin = write_file(&dir, "pin.toml", &fixture::pin_toml()); // No --admin-url: refused, and it says how to supply one. @@ -727,7 +731,7 @@ fn a_real_install_refuses_without_a_master_and_a_bearer_and_changes_nothing() { #[test] fn driving_the_rlm_requires_the_owner_assertion() { let dir = workdir("drive-rlm-gate"); - let bundle = write_file(&dir, "tb4.json", &fixture::bundle_json("staging")); + let bundle = write_file(&dir, "topic.json", &fixture::bundle_json("staging")); let pin = write_file(&dir, "pin.toml", &fixture::pin_toml()); let base = |extra: &[&str]| { let mut a = vec![ @@ -782,7 +786,7 @@ fn driving_the_rlm_requires_the_owner_assertion() { #[test] fn a_metal_install_requires_the_owner_acknowledgement() { let dir = workdir("metal-gate"); - let bundle = write_file(&dir, "tb4.json", &fixture::bundle_json("metal")); + let bundle = write_file(&dir, "topic.json", &fixture::bundle_json("metal")); let pin = write_file(&dir, "pin.toml", &fixture::pin_toml()); let args = |extra: &[&str]| { let mut a = vec![ @@ -826,7 +830,7 @@ fn a_metal_install_requires_the_owner_acknowledgement() { ); // Staging is never gated: that is the default path. - let staging = write_file(&dir, "tb4-staging.json", &fixture::bundle_json("staging")); + let staging = write_file(&dir, "topic-staging.json", &fixture::bundle_json("staging")); let out = run(&[ "topic", "install", @@ -848,8 +852,7 @@ fn a_metal_install_requires_the_owner_acknowledgement() { fs::remove_dir_all(&dir).ok(); } -/// The Owner default: slug `tb4` with `tbench` as a temporary alias. The -/// alias resolves through the store, and the CLI says which topic it hit. +/// An alias resolves through the store, and the CLI says which topic it hit. #[tokio::test] async fn an_alias_resolves_to_its_topic() { let Some(url) = std::env::var("DATABASE_URL") @@ -881,33 +884,51 @@ async fn an_alias_resolves_to_its_topic() { }; // Before the alias exists, the temporary slug is unknown. - let out = run_db(&["topic", "show", "tbench"]); + let out = run_db(&["topic", "show", "fixture-alias"]); assert_eq!(code(&out), EXIT_ERROR, "stderr={}", stderr(&out)); // Set the Owner default alias. - let out = run_db(&["topic", "alias", "set", "tbench", "--topic", "tb4"]); + let out = run_db(&[ + "topic", + "alias", + "set", + "fixture-alias", + "--topic", + "fixture-topic-v0", + ]); assert_eq!(code(&out), 0, "stderr={}", stderr(&out)); - assert!(stdout(&out).contains("tbench -> tb4"), "{}", stdout(&out)); + assert!( + stdout(&out).contains("fixture-alias -> fixture-topic-v0"), + "{}", + stdout(&out) + ); // The alias now resolves, and the CLI says so. - let out = run_db(&["topic", "show", "tbench"]); + let out = run_db(&["topic", "show", "fixture-alias"]); assert_eq!(code(&out), 0, "stderr={}", stderr(&out)); let body = stdout(&out); - assert!(body.contains("tbench is an alias of tb4"), "{body}"); - assert!(body.contains("topic tb4"), "{body}"); + assert!( + body.contains("fixture-alias is an alias of fixture-topic-v0"), + "{body}" + ); + assert!(body.contains("topic fixture-topic-v0"), "{body}"); - let out = run_db(&["--json", "topic", "show", "tbench"]); + let out = run_db(&["--json", "topic", "show", "fixture-alias"]); assert_eq!(code(&out), 0, "stderr={}", stderr(&out)); let parsed: serde_json::Value = serde_json::from_str(&stdout(&out)).expect("json"); assert_eq!( - parsed["topic_id"], "tb4", + parsed["topic_id"], "fixture-topic-v0", "the alias reports the canonical id" ); // Listing shows the temporary mapping. - let out = run_db(&["topic", "alias", "list", "--topic", "tb4"]); + let out = run_db(&["topic", "alias", "list", "--topic", "fixture-topic-v0"]); assert_eq!(code(&out), 0, "stderr={}", stderr(&out)); - assert!(stdout(&out).contains("tbench -> tb4"), "{}", stdout(&out)); + assert!( + stdout(&out).contains("fixture-alias -> fixture-topic-v0"), + "{}", + stdout(&out) + ); // An alias for an unpublished topic is refused. let out = run_db(&[ @@ -926,11 +947,11 @@ async fn an_alias_resolves_to_its_topic() { ); // Retiring the alias leaves the topic alone. - let out = run_db(&["topic", "alias", "rm", "tbench"]); + let out = run_db(&["topic", "alias", "rm", "fixture-alias"]); assert_eq!(code(&out), 0, "stderr={}", stderr(&out)); - let out = run_db(&["topic", "show", "tb4"]); + let out = run_db(&["topic", "show", "fixture-topic-v0"]); assert_eq!(code(&out), 0, "the topic survives: {}", stderr(&out)); - let out = run_db(&["topic", "alias", "rm", "tbench"]); + let out = run_db(&["topic", "alias", "rm", "fixture-alias"]); assert_eq!(code(&out), EXIT_ERROR, "already gone: {}", stderr(&out)); tp.drop_schema().await.expect("drop"); @@ -938,7 +959,11 @@ async fn an_alias_resolves_to_its_topic() { #[test] fn read_commands_without_a_database_are_usage_errors() { - for args in [vec!["topic", "list"], vec!["topic", "show", "tb4"]] { + for args in [ + vec!["topic", "list"], + vec!["topic", "show", "fixture-topic-v0"], + vec!["topic", "lifecycle", "fixture-topic-v0"], + ] { let out = run(&args); assert_eq!(code(&out), EXIT_USAGE, "{args:?}: {}", stderr(&out)); assert!( @@ -976,7 +1001,7 @@ fn database_url_and_file_are_mutually_exclusive() { /// command in this CLI is implemented now, so the exit codes are 0/1/2.) #[test] fn seal_needs_the_open_document() { - let args = vec!["topic", "seal", "tb4"]; + let args = vec!["topic", "seal", "fixture-topic-v0"]; let out = run(&args); assert_eq!(code(&out), EXIT_USAGE, "{args:?}: {}", stderr(&out)); let err = stderr(&out); @@ -995,8 +1020,14 @@ fn seal_needs_the_open_document() { #[test] fn disable_and_enable_need_the_gate_database() { for args in [ - vec!["topic", "disable", "tb4", "--reason", "incident 42"], - vec!["topic", "enable", "tb4"], + vec![ + "topic", + "disable", + "fixture-topic-v0", + "--reason", + "incident 42", + ], + vec!["topic", "enable", "fixture-topic-v0"], ] { let out = Command::new(env!("CARGO_BIN_EXE_proof-admin")) .args(&args) @@ -1021,7 +1052,7 @@ fn disable_and_enable_need_the_gate_database() { /// is the path an operator is told to take. #[test] fn the_scorable_path_fails_closed_on_its_inputs() { - let out = run(&["topic", "baseline", "tb4"]); + let out = run(&["topic", "baseline", "fixture-topic-v0"]); assert_eq!(code(&out), EXIT_USAGE, "{}", stderr(&out)); assert!( stderr(&out).contains("BASE_DATABASE_URL"), @@ -1029,7 +1060,7 @@ fn the_scorable_path_fails_closed_on_its_inputs() { stderr(&out) ); - let out = run(&["topic", "seal", "tb4"]); + let out = run(&["topic", "seal", "fixture-topic-v0"]); assert_eq!(code(&out), EXIT_USAGE, "{}", stderr(&out)); assert!(stderr(&out).contains("--document"), "{}", stderr(&out)); } @@ -1043,6 +1074,7 @@ fn help_lists_every_subcommand() { "validate", "install", "install-log", + "lifecycle", "list", "show", "enable", @@ -1095,13 +1127,12 @@ fn the_cli_does_not_bake_in_topic_behavior() { .filter(|l| !l.trim_start().starts_with("//")) .collect::>() .join("\n"); - // The one place a seed id is allowed is the CLI's own help/examples. - let without_examples = logic - .replace("tb4.json", "") - .replace("`tbench`", "") - .replace("`tb4`", ""); + // The one place a fixture slug is allowed is the CLI's own help/examples, + // which name a neutral placeholder file. + let without_examples = logic.replace("topic.json", ""); assert!( - !without_examples.contains("tb4") && !without_examples.contains("tbench"), + !without_examples.contains("fixture-topic-v0") + && !without_examples.contains("fixture-alias"), "a topic id must not appear in CLI logic" ); for forbidden in [ @@ -1162,15 +1193,22 @@ async fn the_registry_view_lists_what_the_scoring_path_persisted() { let out = run_db(&["--json", "topic", "list"]); assert_eq!(code(&out), 0, "stderr={}", stderr(&out)); let listed: serde_json::Value = serde_json::from_str(&stdout(&out)).expect("json"); - assert_eq!(listed[0]["topic_id"], "tb4"); + assert_eq!(listed[0]["topic_id"], "fixture-topic-v0"); assert_eq!(listed[0]["version"], 1); - assert_eq!(listed[0]["custom_id"], "tbench"); + // `custom_id` is the runner-registry key the document names + // (`metric.custom_id`), not the topic's alias — the two are different + // mappings, and the alias is `fixture-alias`. + assert_eq!(listed[0]["custom_id"], "fixture_metric_v0"); + assert_ne!( + listed[0]["custom_id"], "fixture-alias", + "an alias is a slug lookup, not the custom id" + ); - let out = run_db(&["--json", "topic", "show", "tb4"]); + let out = run_db(&["--json", "topic", "show", "fixture-topic-v0"]); assert_eq!(code(&out), 0, "stderr={}", stderr(&out)); let shown: serde_json::Value = serde_json::from_str(&stdout(&out)).expect("json"); - assert_eq!(shown["topic_id"], "tb4"); - assert_eq!(shown["document"]["id"], "tb4"); + assert_eq!(shown["topic_id"], "fixture-topic-v0"); + assert_eq!(shown["document"]["id"], "fixture-topic-v0"); assert_eq!(shown["document"]["signature"], doc.signature); // An unknown id is an error that says what to do, not an empty success. @@ -1185,6 +1223,115 @@ async fn the_registry_view_lists_what_the_scoring_path_persisted() { tp.drop_schema().await.expect("drop"); } +/// `topic lifecycle` reports where a driven topic actually is. +/// +/// This is the read that makes a long `--drive-rlm` legible: the command +/// prints one line and then nothing until the whole run returns, so an +/// operator watching a working run sees the same output as one watching a +/// stopped run. The durable progress is the lifecycle journal, and the +/// baseline's absence is what says the paid job has not landed yet — the +/// exact question "0 rows in `proof_baseline_measurement`" raises. +#[tokio::test] +async fn the_lifecycle_view_says_where_a_driven_topic_is() { + use proof_rlm::{RlmEvent, RlmState}; + use proof_rlm_store::{RlmStore, TransitionRow}; + + let Some(url) = std::env::var("DATABASE_URL") + .ok() + .map(|u| u.trim().to_owned()) + .filter(|u| !u.is_empty()) + else { + return; + }; + let tp = match db::test_pool_with_url(&url).await { + Ok(tp) => tp, + Err(e) => panic!("test_pool: {e}"), + }; + let store = proof_rlm_store::PgRlmStore::new(tp.pool().clone()); + let doc = fixture::signed_topic(&format!("sha256:{}", "ab".repeat(32))); + RlmStore::put_topic_version(&store, &doc) + .await + .expect("persist the document"); + + let schema = tp.schema().to_owned(); + let scoped = format!("{url}?options=-c%20search_path%3D{schema}%2Cpublic"); + let run_db = |args: &[&str]| { + Command::new(env!("CARGO_BIN_EXE_proof-admin")) + .args(args) + .env("BASE_DATABASE_URL", &scoped) + .env_remove("BASE_DATABASE_URL_FILE") + .output() + .expect("run proof-admin") + }; + + // No transitions yet: an error that says nothing has driven it, not an + // empty success an operator could misread as "fine". + let out = run_db(&["topic", "lifecycle", "fixture-topic-v0"]); + assert_eq!(code(&out), EXIT_ERROR, "stderr={}", stderr(&out)); + assert!( + stderr(&out).contains("no lifecycle rows"), + "{}", + stderr(&out) + ); + + // The shape a `--drive-rlm` run leaves mid-flight: past the owner gate, + // provisioning, with **no** baseline row yet. + for (from, event, to) in [ + ( + RlmState::Draft, + RlmEvent::SubmitForReview, + RlmState::OwnerPresend, + ), + ( + RlmState::OwnerPresend, + RlmEvent::OwnerApproved, + RlmState::AwaitingOwnerKeys, + ), + ( + RlmState::AwaitingOwnerKeys, + RlmEvent::OwnerKeysPresent, + RlmState::Provisioning, + ), + ] { + RlmStore::record_transition( + &store, + &TransitionRow { + topic_id: doc.id.clone(), + from, + event, + to, + note: "test".into(), + }, + ) + .await + .expect("record"); + } + + let out = run_db(&["--json", "topic", "lifecycle", "fixture-topic-v0"]); + assert_eq!(code(&out), 0, "stderr={}", stderr(&out)); + let view: serde_json::Value = serde_json::from_str(&stdout(&out)).expect("json"); + assert_eq!(view["topic_id"], "fixture-topic-v0"); + assert_eq!(view["state"], "provisioning"); + assert!( + view["baseline_rules_version"].is_null(), + "no baseline has been measured yet: {view}" + ); + assert!( + view["history"].as_array().is_some_and(|h| h.len() == 3), + "every transition is reported, oldest first: {view}" + ); + // The advice names the state, so an operator knows the run is in flight + // rather than lost. + assert!( + view["next"] + .as_str() + .is_some_and(|s| s.contains("VM is being created")), + "{view}" + ); + + tp.drop_schema().await.expect("drop"); +} + // --------------------------------------------------------------------------- // The publish order: an `open` document is not publishable before the install // --------------------------------------------------------------------------- @@ -1260,18 +1407,19 @@ impl AdminStub { Some(pool) => { let state: Option = sqlx::query_scalar( "SELECT state FROM proof_topic_install \ - WHERE topic_id = 'tb4' ORDER BY id DESC LIMIT 1", + WHERE topic_id = 'fixture-topic-v0' ORDER BY id DESC LIMIT 1", + ) + .fetch_optional(pool) + .await + .ok() + .flatten(); + let table: Option = sqlx::query_scalar( + "SELECT to_regclass('fixture_topic_v0_scratch')::text", ) .fetch_optional(pool) .await .ok() .flatten(); - let table: Option = - sqlx::query_scalar("SELECT to_regclass('tb4_scratch')::text") - .fetch_optional(pool) - .await - .ok() - .flatten(); Some(( state.unwrap_or_else(|| "no row".into()), table.unwrap_or_else(|| "no table".into()), @@ -1387,13 +1535,13 @@ async fn the_publish_lands_only_after_the_install_is_green() { "the install must be green before the topic is published" ); assert_eq!( - table, "tb4_scratch", + table, "fixture_topic_v0_scratch", "the migration must have applied before the topic is published" ); // And the install is complete afterwards: the journal's newest row is // `applied` with the migration recorded. - let row = proof_topic_install::latest_install(&probe_pool, "tb4") + let row = proof_topic_install::latest_install(&probe_pool, "fixture-topic-v0") .await .expect("journal") .expect("a row"); @@ -1428,7 +1576,7 @@ async fn a_refused_install_never_publishes() { // The same bundle, with a migration the deny-list refuses. The document // and its signature are untouched, so the refusal comes from the install. let denied = fixture::bundle_json("staging").replace( - "CREATE TABLE tb4_scratch (id TEXT)", + "CREATE TABLE fixture_topic_v0_scratch (id TEXT)", "DROP TABLE proof_rule_version", ); assert!(denied.contains("proof_rule_version"), "the swap applied"); @@ -1473,14 +1621,15 @@ async fn a_refused_install_never_publishes() { ); // Nothing was installed either: no journal row, no table. - let row = proof_topic_install::latest_install(tp.pool(), "tb4") + let row = proof_topic_install::latest_install(tp.pool(), "fixture-topic-v0") .await .expect("journal"); assert!(row.is_none(), "a pre-flight refusal writes no journal row"); - let table: Option = sqlx::query_scalar("SELECT to_regclass('tb4_scratch')::text") - .fetch_one(tp.pool()) - .await - .expect("probe"); + let table: Option = + sqlx::query_scalar("SELECT to_regclass('fixture_topic_v0_scratch')::text") + .fetch_one(tp.pool()) + .await + .expect("probe"); assert!(table.is_none(), "and no migration ran: {table:?}"); fs::remove_dir_all(&dir).ok(); diff --git a/bins/proof-admin/tests/fixtures/README-dry-run.md b/bins/proof-admin/tests/fixtures/README-dry-run.md index 782df316a..0a5206291 100644 --- a/bins/proof-admin/tests/fixtures/README-dry-run.md +++ b/bins/proof-admin/tests/fixtures/README-dry-run.md @@ -5,8 +5,8 @@ Operator dry-run artifact for the dynamic-topics install path (P0 + P1a). | File | What it is | |------|------------| -| `tb4.install-bundle.json` | A **Topic Install Bundle**: slug `tb4`, alias `tbench`, install target `staging`, carrying a signed `TopicDocument`, an `rlm` install section (`rules`, `migrations`, `apis`, `submission_format`, `scoring`), and the Owner-default alias. | -| `tb4.pin.toml` | The `ProofPin` that document is checked against. | +| `topic.install-bundle.json` | A **Topic Install Bundle**: slug `fixture-topic-v0`, alias `fixture-alias`, install target `staging`, carrying a signed `TopicDocument`, an `rlm` install section (`rules`, `migrations`, `apis`, `submission_format`, `scoring`), and the alias the bundle itself declares. | +| `topic.pin.toml` | The `ProofPin` that document is checked against. | ## Exact commands @@ -14,21 +14,21 @@ Run from the repository root: ```bash cargo run -p proof-admin-bin -- topic validate \ - --bundle bins/proof-admin/tests/fixtures/tb4.install-bundle.json \ - --pin bins/proof-admin/tests/fixtures/tb4.pin.toml + --bundle bins/proof-admin/tests/fixtures/topic.install-bundle.json \ + --pin bins/proof-admin/tests/fixtures/topic.pin.toml cargo run -p proof-admin-bin -- topic install \ - --bundle bins/proof-admin/tests/fixtures/tb4.install-bundle.json \ + --bundle bins/proof-admin/tests/fixtures/topic.install-bundle.json \ --env staging --dry-run \ - --pin bins/proof-admin/tests/fixtures/tb4.pin.toml + --pin bins/proof-admin/tests/fixtures/topic.pin.toml ``` `--bin proof-admin` works too and is package-name-agnostic: ```bash cargo run --bin proof-admin -- topic validate \ - --bundle bins/proof-admin/tests/fixtures/tb4.install-bundle.json \ - --pin bins/proof-admin/tests/fixtures/tb4.pin.toml + --bundle bins/proof-admin/tests/fixtures/topic.install-bundle.json \ + --pin bins/proof-admin/tests/fixtures/topic.pin.toml ``` Both write nothing and need no database. @@ -39,16 +39,16 @@ Drop `--dry-run` and supply the master and the operator bearer: ```bash cargo run -p proof-admin-bin -- topic install \ - --bundle bins/proof-admin/tests/fixtures/tb4.install-bundle.json \ + --bundle bins/proof-admin/tests/fixtures/topic.install-bundle.json \ --env staging \ - --pin bins/proof-admin/tests/fixtures/tb4.pin.toml \ + --pin bins/proof-admin/tests/fixtures/topic.pin.toml \ --admin-url http://127.0.0.1:8100 \ --admin-token-file /run/proof/admin_token ``` This fixture's document is signed by the **test** key, so a real install against a live master would be refused at the publish step. Use it to exercise -the gates and the dry run; the real `tb4` document is signed by the `proof` +the gates and the dry run; the real document is signed by the `proof` row key and is a follow-up (see below). Add `--drive-rlm --owner-approved` to provision the topic VM and run the paid @@ -59,7 +59,7 @@ baseline job. Read the journal back with: ```bash -BASE_DATABASE_URL=… proof-admin topic install-log --topic tb4 +BASE_DATABASE_URL=… proof-admin topic install-log --topic ``` ## The rest of the path to a scorable topic @@ -76,11 +76,11 @@ proof-admin topic install --bundle --env staging --drive-rlm --owner-ap --admin-url --admin-token-file # 2. Read what was measured and the commitment the open document must seal. -proof-admin topic baseline tb4 +proof-admin topic baseline # 3. Put that `metrics_commitment` into the document, set `status: open`, sign # it with the `proof` key (`xtask proof-topic`), then seal and publish. -proof-admin topic seal tb4 --document --publish \ +proof-admin topic seal --document --publish \ --admin-url --admin-token-file ``` @@ -108,25 +108,25 @@ must be checked against the fixture pin. Omitting `--pin` falls back to ``` $ cargo run -p proof-admin -- topic validate \ - --bundle bins/proof-admin/tests/fixtures/tb4.install-bundle.json + --bundle bins/proof-admin/tests/fixtures/topic.install-bundle.json proof-admin: topic signature: topic signature does not verify under the proof trust-root key ``` That refusal is the signature check working correctly — a test-signed document -is not this subnet's topic. The real `tb4` document is signed by the `proof` +is not this subnet's topic. The real document is signed by the `proof` row key and is a follow-up (see below). ## What this fixture is not - **Not a production topic.** The document is signed with a test mini-secret; - `tb4.pin.toml` carries the matching `topic_pubkey`. + `topic.pin.toml` carries the matching `topic_pubkey`. - **Not a real RLM install.** The `rlm` section is a small illustrative sample (`rules`, `migrations`, `apis`, `submission_format`, `scoring`). A real bundle carries the topic's own — which the RLM consumes and this repository - never interprets. The sample's migration (`CREATE TABLE tb4_scratch`) is + never interprets. The sample's migration (`CREATE TABLE fixture_topic_v0_scratch`) is legal under the deny-list precisely because it stays inside the topic's own namespace. -- **Not the metal artifact.** The metal signed Operator `tb4.json` is a +- **Not the metal artifact.** The metal signed Operator document is a follow-up; this fixture exists so the staging A→Z walkthrough can exercise `validate`, `--dry-run`, and the install gates today. diff --git a/bins/proof-admin/tests/fixtures/tb4.install-bundle.json b/bins/proof-admin/tests/fixtures/topic.install-bundle.json similarity index 88% rename from bins/proof-admin/tests/fixtures/tb4.install-bundle.json rename to bins/proof-admin/tests/fixtures/topic.install-bundle.json index 0b278f7a9..5c81d9e32 100644 --- a/bins/proof-admin/tests/fixtures/tb4.install-bundle.json +++ b/bins/proof-admin/tests/fixtures/topic.install-bundle.json @@ -1,11 +1,11 @@ { "aliases": [ - "tbench" + "fixture-alias" ], - "display_name": "Terminal-Bench 4", + "display_name": "Fixture Topic", "environment": "staging", "host": { - "custom_ids_entry": "tbench", + "custom_ids_entry": "fixture_metric_v0", "experiment_image_digest": "sha256:abababababababababababababababababababababababababababababababab", "pack_digest": "sha256:abababababababababababababababababababababababababababababababab", "pack_dir": "/var/lib/proof/packs", @@ -22,7 +22,7 @@ "migrations": [ { "name": "0001_scratch", - "sql": "CREATE TABLE tb4_scratch (id TEXT)" + "sql": "CREATE TABLE fixture_topic_v0_scratch (id TEXT)" } ], "rules": [ @@ -80,7 +80,7 @@ "flops_budget": 2000000000000000000, "holdout_commitment": "e2f97658a5ff704f1f20982c8e3bbda5aba9793ac390df7f96a3641f33e5483d", "holdout_size": 120, - "id": "tb4", + "id": "fixture-topic-v0", "inference": { "base_url": null, "max_input_tokens": null, @@ -91,7 +91,7 @@ "require_judge_offer_commitment": null }, "metric": { - "custom_id": "tbench", + "custom_id": "fixture_metric_v0", "direction": "max", "epsilon_rel": 0.05, "family": "custom", @@ -103,7 +103,7 @@ "payout_mode": "discovery", "proxy_model": null, "schema_version": 1, - "signature": "c4396e7185c040d176efcff71ae8fe1cee590ee1ad7bbbaae0e7b2c4f6f5c5034ee03b3ec3b42c48f268ad5af0690cb058351968bfcd37792f15e50e34357f8d", + "signature": "70c4f0cb18b2f64ae788bcf908252ffc719c00a1b80d2024476f4f034334d05a665d9680e5157cc17991f9e79ec9c8ba1bc6826a90b67900162fc852f97a7d8b", "statement": "Score the pinned task pack with the pinned runner.", "status": "draft", "valid_from_epoch": 0, diff --git a/bins/proof-admin/tests/fixtures/tb4.pin.toml b/bins/proof-admin/tests/fixtures/topic.pin.toml similarity index 100% rename from bins/proof-admin/tests/fixtures/tb4.pin.toml rename to bins/proof-admin/tests/fixtures/topic.pin.toml diff --git a/bins/proof-challenge/Cargo.toml b/bins/proof-challenge/Cargo.toml index b2f132900..f6d40cf4a 100644 --- a/bins/proof-challenge/Cargo.toml +++ b/bins/proof-challenge/Cargo.toml @@ -27,6 +27,7 @@ proof-harvest = { path = "../../crates/proof-harvest" } proof-rlm = { path = "../../crates/proof-rlm" } proof-rlm-scorer = { path = "../../crates/proof-rlm-scorer" } proof-rlm-store = { path = "../../crates/proof-rlm-store" } +proof-score = { path = "../../crates/proof-score" } proof-task = { path = "../../crates/proof-task" } proof-topic-install = { path = "../../crates/proof-topic-install" } proof-vm-fc = { path = "../../crates/proof-vm-fc" } diff --git a/bins/proof-challenge/src/main.rs b/bins/proof-challenge/src/main.rs index 72d5404a7..263f00adf 100644 --- a/bins/proof-challenge/src/main.rs +++ b/bins/proof-challenge/src/main.rs @@ -38,6 +38,7 @@ use proof_rlm::{ }; use proof_rlm_scorer::{max_zip_numeric_id, ArtefactStore, RlmScorer}; use proof_rlm_store::{MemoryRlmStore, PgRlmStore, RlmStore}; +use proof_task::TopicStatus; use proof_topic_install::{PgTopicRoutes, TopicRouteMux}; use proof_vm_fc::{parse_custom_ids, FirecrackerOrchestrator, VM_RUNNER_CUSTOM_IDS_ENV}; use sqlx::PgPool; @@ -273,7 +274,18 @@ fn run(cli: &Cli) -> Result<(), String> { let live_scorer = live_scorer(backend, harvest, rlm_store, &cli.artefact_root, &vm); log_live_wiring(backend, live_scorer.as_deref(), &cli.artefact_root); let registered = registered_custom(live_scorer.as_deref()); - match load_topics(&store, &pin, cli.topics_file.as_deref(), ®istered) { + // Loaded **through** the runtime: an `open` document's admission reads the + // install journal, and a read outside an entered runtime would fail and + // silently skip the topic. Driving the loader with `rt.block_on` is what + // makes the gate run at all on a live host. + match rt.block_on(load_topics( + &store, + &pin, + cli.topics_file.as_deref(), + ®istered, + db_pool.as_ref(), + backend, + )) { Ok(n) => tracing::info!(topics = n, "signed topics loaded"), Err(e) => tracing::warn!("topics unavailable ({e}); submissions will 400/503 until fixed"), } @@ -814,26 +826,113 @@ fn load_pin(path: Option<&Path>) -> Result { Ok(pin) } -fn load_topics( +/// Load the operator's signed topic documents into the store. +/// +/// A `draft` document is admitted as-is: it is not submitable, and holding it +/// in the store is how the host knows the topic exists while its install is +/// still being applied. +/// +/// On the **live** backend an `open` document is admitted only when the +/// database proves the two facts the whole dynamic-topics path rests on, and a +/// missing database is a refusal rather than a pass: +/// +/// 1. its newest `proof_topic_install` row is `applied`; and +/// 2. its rule vector in force is **RLM-authored** +/// (`proof_rule_version.source = 'rlm'`). +/// +/// Without (2) the topic's anti-cheat behavior is still the operator's signed +/// `checklist`, and admitting the document would put a topic into +/// `open_topics` / `scorable_topics` whose behavior nobody authored — the +/// parallel, file-driven admission path this gate closes. The topic is +/// **skipped** (not fatal): the host keeps serving whatever else is installed, +/// and the refusal is logged with the provenance that blocked it. +/// +/// Sim is exempt, deliberately: it is the CI / local opt-in backend +/// (`PROOF_FORCE_SIM`, never a live fallback) that scores in-process with no +/// install, no RLM, and no topic VM. Applying the gate there would test the +/// fixture rather than the boundary. +async fn load_topics( store: &MemoryStore, pin: &ProofPin, path: Option<&Path>, registered_custom: &[String], + db_pool: Option<&sqlx::PgPool>, + backend: EvalBackend, ) -> Result { let p = path.ok_or("PROOF_TOPICS_FILE not set")?; let body = std::fs::read_to_string(p).map_err(|e| format!("read {}: {e}", p.display()))?; let docs = TopicDocument::many_from_json(&body).map_err(|e| e.to_string())?; - let n = docs.len(); + let mut n = 0usize; for doc in docs { doc.validate(pin, &custom_ids_ref(registered_custom)) .map_err(|e| format!("topic {}: {e}", doc.id))?; doc.verify_signature(pin) .map_err(|e| format!("topic {}: {e}", doc.id))?; + if backend != EvalBackend::Sim && doc.status == TopicStatus::Open { + let Some(pool) = db_pool else { + tracing::warn!( + topic_id = %doc.id, + "open topic skipped: no database, so this host cannot prove the topic was \ + installed and that its RLM authored its rules" + ); + continue; + }; + if let Err(why) = open_topic_admissible(pool, &doc.id).await { + tracing::warn!(topic_id = %doc.id, "open topic skipped: {why}"); + continue; + } + } store.put_topic(doc).map_err(|e| e.to_string())?; + n = n.saturating_add(1); } Ok(n) } +/// Why an `open` document may not be admitted, or `Ok(())` when it may. +/// +/// Fail-closed on every doubt: an unreadable database, a missing install row, +/// and rules that are not RLM-authored all refuse. The reason names the +/// provenance so an operator can tell "run the install" from "the RLM never +/// wrote its rules". +/// +/// The two facts come from **one bound read** +/// ([`proof_topic_install::installed_rules`]): the newest install is +/// `applied`, and the rule version *that install recorded* is `rlm`-sourced. +/// Reading them as two independent predicates would admit a topic whose +/// install landed the operator's signed vector while a later, unrelated +/// version happened to be RLM-authored. +async fn open_topic_admissible(pool: &sqlx::PgPool, topic_id: &str) -> Result<(), String> { + match proof_topic_install::installed_rules(pool, topic_id) + .await + .map_err(|e| format!("install journal unreadable: {e}"))? + { + proof_topic_install::InstalledRules::RlmAuthored { .. } => Ok(()), + proof_topic_install::InstalledRules::NotApplied { state } => Err(format!( + "the newest install row is {}; run `proof-admin topic install` to completion first", + match state.as_deref() { + Some(state) => format!("`{state}`"), + None => "absent".to_owned(), + } + )), + proof_topic_install::InstalledRules::NotRlmAuthored { provenance, .. } => Err(format!( + "the rule version this install landed is not RLM-authored \ + (proof_rule_version.source is {}); drive the RLM's propose_rules job \ + (`proof-admin topic install --drive-rlm --owner-approved`) so the topic authors \ + its own behavior", + provenance.as_deref().unwrap_or("absent") + )), + proof_topic_install::InstalledRules::SupersededByOperator { + installed, + in_force, + provenance, + } => Err(format!( + "the install landed RLM-authored rule version {installed}, but version {in_force} \ + ({provenance}) is in force; the topic's behavior is no longer its RLM's, so it is \ + not admitted until the RLM's vector is the one in force again" + )), + } +} + fn load_holdouts(store: &MemoryStore, path: Option<&Path>) -> Result { let p = path.ok_or("PROOF_HOLDOUT_FILE not set")?; let body = std::fs::read_to_string(p).map_err(|e| format!("read {}: {e}", p.display()))?; @@ -899,8 +998,21 @@ fn record_one_baseline( ) -> Result<(), String> { let topic = store.topic(&meas.topic_id).map_err(|e| e.to_string())?; meas.verify(pin, &topic).map_err(|e| e.to_string())?; + // The same refusal `mark_sealed` applies, on the other door into the + // store: a baseline file is operator-supplied, so it must not be able to + // install a bar no challenger can clear. A relative-win family compares + // `challenger >= bar * (1 + epsilon_rel)`, which has no solution at zero. + let sealed = meas.into_sealed(); + if proof_score::sealed_bar_is_degenerate(&topic, &sealed) { + return Err(format!( + "baseline {}: the measured primary is a degenerate bar, so this family could never \ + be passed by anyone (a relative win against zero has no solution). Nothing was \ + recorded. Re-run the reference against something that scores and write that number.", + topic.id + )); + } store - .set_baseline(&topic.id, meas.into_sealed()) + .set_baseline(&topic.id, sealed) .map_err(|e| e.to_string())?; Ok(()) } @@ -1065,6 +1177,230 @@ mod tests { Cli::try_parse_from(["proof-challenge"]).expect("defaults parse") } + /// An `open` topic is not admitted into the store on a **live** host that + /// cannot prove its install: no database is a skip, not a pass. + /// + /// This is the file-driven admission path the RLM-authorship gate closes. + /// The document is skipped (the host keeps serving what else it has) + /// rather than admitted into `open_topics` / `scorable_topics`, so a topic + /// whose behavior nobody authored never becomes submitable. + #[tokio::test] + async fn a_live_host_admits_no_open_topic_it_cannot_prove_was_installed() { + let dir = std::env::temp_dir().join(format!( + "proof-topics-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + )); + std::fs::create_dir_all(&dir).expect("dir"); + let path = dir.join("topics.json"); + + // A document that cannot be validated is a hard error either way, so + // the gate's own effect is what the rest of this test measures. + let pin = ProofPin::default(); + let store = MemoryStore::new(); + let missing = dir.join("does-not-exist.json"); + assert!( + load_topics(&store, &pin, Some(&missing), &[], None, EvalBackend::Lium) + .await + .is_err(), + "an unreadable topics file is still an error" + ); + // With no documents at all the loader is a no-op, which is the only + // shape that can be asserted without a signed fixture here: the + // per-document gate is exercised against a real database in + // `submit_e2e` / `install_engine`, where a row can actually exist. + std::fs::write(&path, "[]").expect("write"); + assert_eq!( + load_topics(&store, &pin, Some(&path), &[], None, EvalBackend::Lium) + .await + .expect("empty list"), + 0 + ); + let _ = std::fs::remove_dir_all(&dir); + } + + /// The admission read really runs **inside** a runtime, so a live startup + /// does not silently skip every open topic. + /// + /// The defect this pins: `open_topic_admissible` used to reach for + /// `Handle::try_current()` and fail when the synchronous startup called it + /// after its `block_on` scopes had returned. Every valid open topic was + /// then logged as skipped and never loaded. The gate must therefore be + /// callable from the synchronous startup path *and* from an async caller, + /// which is only true if it is itself `async` (no ambient runtime needed). + #[test] + fn the_admission_gate_needs_no_ambient_runtime() { + // A synchronous context with no runtime entered: the shape startup + // has. A lazy pool pointed at a closed port fails fast on acquire, so + // the call reaches the read and returns the fail-closed `Err`. + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("rt"); + let pool = rt + .block_on(async { + sqlx::postgres::PgPoolOptions::new() + .acquire_timeout(std::time::Duration::from_millis(250)) + .connect_lazy("postgres://127.0.0.1:1/none") + }) + .expect("lazy pool builds without connecting"); + // Outside any entered runtime — exactly where the old gate failed + // with "no async runtime to read the install journal". + assert!( + tokio::runtime::Handle::try_current().is_err(), + "this test must run outside an entered runtime to be meaningful" + ); + let verdict = rt.block_on(open_topic_admissible(&pool, "any-topic")); + let why = verdict.expect_err("an unreadable journal is never an admission"); + assert!( + why.contains("install journal unreadable"), + "the refusal names the unreadable journal, not a missing runtime: {why}" + ); + } + + /// The boot-time baseline file is the **other** door into the store, so it + /// applies the same degenerate-bar refusal `mark_sealed` does. + /// + /// Without it, a hand-written baseline file could install a bar no + /// challenger can clear on a relative-win family — a topic that is open, + /// scorable, and unwinnable by anyone (the LIVE Gate 1 shape: a reference + /// run that solved nothing, every trial `0.0`). Nothing is recorded when + /// it refuses, so the host fails closed with no sealed baseline rather + /// than with a dead one. + #[tokio::test] + async fn a_baseline_file_cannot_install_a_degenerate_bar() { + let dir = std::env::temp_dir().join(format!( + "proof-baseline-degenerate-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + )); + std::fs::create_dir_all(&dir).expect("dir"); + let path = dir.join("baseline.json"); + let pin = ProofPin::default(); + let store = MemoryStore::new(); + let mut topic = proof_task::TopicDocument { + id: "baseline-gate-v0".into(), + status: proof_task::TopicStatus::Open, + ..proof_task::TopicDocument::default() + }; + topic.metric.family = proof_task::MetricFamily::Custom; + topic.metric.custom_id = "placeholder_metric".into(); + topic.metric.primary = "placeholder_primary".into(); + topic.holdout_commitment = "cd".repeat(32); + topic.baseline.script_sha256 = "ee".repeat(32); + store.put_topic(topic.clone()).expect("topic"); + + // A measurement that binds to the topic (commitment + image digest), + // carrying a zero primary: the seal `verify` accepts, the bar the + // scoring path cannot. + let mut meas = BaselineMeasurement { + eval_image_digest: pin.eval_image_digest.clone(), + topic_id: topic.id.clone(), + holdout_commitment: topic.holdout_commitment.clone(), + holdout_nll: 0.0, + split_nll: proof_task::HoldoutSplit::SCORED + .iter() + .map(|s| (s.as_str().to_owned(), 0.0)) + .collect(), + tokens_per_sec: None, + step_latency_ms: None, + custom_value: Some(0.0), + }; + topic.baseline.metrics_commitment = meas.commitment(); + store + .put_topic(topic.clone()) + .expect("re-put with commitment"); + std::fs::write(&path, serde_json::to_string(&meas).expect("json")).expect("write"); + + let err = load_baselines(&store, &pin, Some(&path)).expect_err("a zero bar must not load"); + assert!(err.contains("degenerate bar"), "{err}"); + assert!( + store.baseline("baseline-gate-v0").expect("read").is_none(), + "nothing may be recorded when the bar is refused" + ); + + // A real measurement still loads: the guard narrows nothing else. + // The topic's commitment has to move with it — `verify` binds the + // measurement to the document, which is the property that makes the + // guard above meaningful rather than a check on a file nobody reads. + meas.custom_value = Some(0.42); + topic.baseline.metrics_commitment = meas.commitment(); + store + .put_topic(topic) + .expect("re-put with the new commitment"); + std::fs::write(&path, serde_json::to_string(&meas).expect("json")).expect("write"); + assert_eq!(load_baselines(&store, &pin, Some(&path)).expect("loads"), 1); + let sealed = store + .baseline("baseline-gate-v0") + .expect("read") + .expect("recorded"); + assert!( + (sealed.custom_value.expect("custom") - 0.42).abs() < 1e-12, + "{sealed:?}" + ); + let _ = std::fs::remove_dir_all(&dir); + } + + /// The publish gate and the startup gate read the **same bound fact**: the + /// rule version the newest install recorded, not whichever version is + /// newest. + #[test] + fn the_admission_gate_reads_the_version_the_install_recorded() { + // `installed_rules` is the single read both gates call; the pairing is + // what makes "an install landed the operator's vector while a later + // version was RLM-authored" unable to open a topic. + let source = include_str!("main.rs"); + let gate = source + .split("async fn open_topic_admissible") + .nth(1) + .expect("the gate"); + assert!( + gate.contains("installed_rules"), + "the startup gate must read the install-bound fact, not two independent predicates" + ); + assert!( + !gate.contains("rlm_authored_rules"), + "reading the newest rule version's provenance separately is the defect this gate \ + fixes: it admits a topic whose install landed the operator's vector" + ); + } + + /// The rule-provenance read is fail-closed on a database error: it never + /// answers `true` for a topic it could not read. + #[test] + fn the_rlm_authorship_read_refuses_rather_than_guessing() { + // A lazy pool pointed at a closed port: `acquire` fails fast, so the + // read must be an `Err`, not `Ok(false)` and certainly not `Ok(true)`. + // The acquire timeout is trimmed so this stays a fast unit test. + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("rt"); + rt.block_on(async { + let pool = sqlx::postgres::PgPoolOptions::new() + .acquire_timeout(std::time::Duration::from_millis(250)) + .connect_lazy("postgres://127.0.0.1:1/none") + .expect("lazy pool builds without connecting"); + assert!( + proof_topic_install::rlm_authored_rules(&pool, "any-topic") + .await + .is_err(), + "an unreadable store must not answer for a topic" + ); + assert!( + proof_topic_install::rules_source(&pool, "any-topic") + .await + .is_err(), + "an unreadable store must not answer for a topic" + ); + }); + } + /// A leaf the trust root would reject is not weight, so a missing /// challenge key is a refusal rather than an unsigned emit. #[test] diff --git a/bins/proof-vm-orchestrator/src/main.rs b/bins/proof-vm-orchestrator/src/main.rs index 26d3a5536..441f7dab2 100644 --- a/bins/proof-vm-orchestrator/src/main.rs +++ b/bins/proof-vm-orchestrator/src/main.rs @@ -54,7 +54,7 @@ use proof_fc_experiment::{ }; use proof_fc_host::{EgressAllow, FirecrackerHypervisor, HostConfig}; use proof_vm_agent::{ - agent_router, AgentState, BearerAuth, Hypervisor, DEFAULT_MAX_EXPERIMENT_VMS, + agent_router, AgentState, BearerAuth, Hypervisor, MemoryBudget, DEFAULT_MAX_EXPERIMENT_VMS, }; use proof_vm_proto::DEFAULT_AGENT_PORT; @@ -197,6 +197,13 @@ struct Cli { /// Experiment VMs this host runs at once (each up to the ceilings; 0 = none). #[arg(long, env = "PROOF_VM_AGENT_MAX_EXPERIMENT_VMS", default_value_t = DEFAULT_MAX_EXPERIMENT_VMS)] max_experiment_vms: usize, + /// Host memory (MiB) kept out of the VM budget for the OS, this agent, + /// and per-VM process overhead. `0` (the default) admits VM memory up to + /// `MemTotal`, which is the shape Gate 3 passes on. Raise it to leave the + /// host slack; the budget refuses a boot that would not fit rather than + /// let the kernel OOM-kill running guests (Gate 4). + #[arg(long, env = "PROOF_VM_AGENT_MEMORY_RESERVE_MIB", default_value_t = 0)] + memory_reserve_mib: u64, /// Seconds the guest may take to verify and unpack a staged pack. #[arg(long, env = "PROOF_VM_AGENT_PACK_STAGE_TIMEOUT_SECS", default_value_t = DEFAULT_STAGE_TIMEOUT.as_secs())] pack_stage_timeout_secs: u64, @@ -466,8 +473,19 @@ async fn run(cli: &Cli) -> Result<(), String> { Box::new(VsockPackStager::new(cfg, stage_timeout)), ) .map_err(|e| e.to_string())?; - let state = - AgentState::with_max_experiment_vms(Arc::new(layered), auth, cli.max_experiment_vms); + // The host's own RAM, read once at boot: a host that cannot prove it has + // room does not get to boot VMs on a guess. This is the second cap beside + // `max_experiment_vms` — the kernel does not honour a count, and Gate 4 + // lost both submissions to the OOM killer with a count cap satisfied. + let budget = MemoryBudget::read(cli.memory_reserve_mib)?; + tracing::info!( + total_mib = budget.total_mib, + reserve_mib = budget.reserve_mib, + ceiling_mib = budget.ceiling_mib(), + max_experiment_vms = cli.max_experiment_vms, + "memory admission budget" + ); + let state = AgentState::with_limits(Arc::new(layered), auth, cli.max_experiment_vms, budget); let app = agent_router(state); let handle = axum_server::Handle::new(); let shutdown = handle.clone(); diff --git a/crates/ctx-client/src/lib.rs b/crates/ctx-client/src/lib.rs index 085468e97..4323725c5 100644 --- a/crates/ctx-client/src/lib.rs +++ b/crates/ctx-client/src/lib.rs @@ -21,7 +21,8 @@ pub const DEFAULT_GATEWAY: &str = "https://gateway.cortex.foundation"; pub const DEFAULT_GET_TIMEOUT_SECS: u64 = 60; /// Proof POST / multipart default. Evaluate is **synchronous** and can run -/// for minutes (tbench); a client-wide 60 s timeout drops the TCP stream, +/// for minutes on a custom topic; a client-wide 60 s timeout drops the TCP +/// stream, /// the gateway cancels upstream, and the host is left with an orphan /// experiment VM and no score. `0` (via [`Client::with_submit_timeout_secs`]) /// waits until the host answers. @@ -94,7 +95,7 @@ impl Client { /// followed, so a 302 to `http://` cannot resend `X-Lium-Api-Key`. /// /// Proof submits use [`DEFAULT_SUBMIT_TIMEOUT_SECS`]. There is **no** - /// client-wide reqwest timeout: tbench evaluate is synchronous. + /// client-wide reqwest timeout: custom-topic evaluate is synchronous. pub fn new(gateway: &str, lium_key: Option) -> Result { Self::with_submit_timeout_secs(gateway, lium_key, DEFAULT_SUBMIT_TIMEOUT_SECS) } @@ -415,7 +416,7 @@ mod tests { let builder = builder.split(".build()").next().expect("build"); assert!( !builder.contains(".timeout("), - "do not set a client-wide reqwest timeout (tbench evaluate is sync): {builder}" + "do not set a client-wide reqwest timeout (custom-topic evaluate is sync): {builder}" ); assert_eq!(DEFAULT_GET_TIMEOUT_SECS, 60); assert_eq!(DEFAULT_SUBMIT_TIMEOUT_SECS, 7200); diff --git a/crates/db/migrations/0024_proof_topic_alias.sql b/crates/db/migrations/0024_proof_topic_alias.sql index 9c6993ef5..ccd9b4708 100644 --- a/crates/db/migrations/0024_proof_topic_alias.sql +++ b/crates/db/migrations/0024_proof_topic_alias.sql @@ -1,8 +1,9 @@ -- Proof topic aliases: the temporary compatibility slug a topic answers to. -- --- Owner default: the first topic's slug is `tb4`, with `tbench` as a --- **temporary** alias, so existing miner links keep resolving while the --- canonical slug moves. This table is that mapping and nothing else. +-- An alias is a lookup convenience, not topic data: the topic's identity is +-- its signed document's `id`, and this table is that mapping and nothing else. +-- There is no owner default — which aliases exist is what an operator's +-- installs declared. -- -- Why this is not a second topic table: a row here is `alias -> topic_id`. -- No display name, no pins, no status, no document — every one of those lives diff --git a/crates/db/migrations/0027_proof_topic_authoring.sql b/crates/db/migrations/0027_proof_topic_authoring.sql new file mode 100644 index 000000000..a6e387e63 --- /dev/null +++ b/crates/db/migrations/0027_proof_topic_authoring.sql @@ -0,0 +1,58 @@ +-- Proof topic authoring: what a topic's RLM authored, so a **re-authoring** +-- run can be given the set it wrote last time. +-- +-- A topic's behavior is five parts (rules, migrations, APIs, submission +-- format, pin policy) and its own RLM authors all of them in one job +-- (`authoring.json`). The rules have always been versioned in +-- `proof_rule_version`; the other four had nowhere to live, so a second +-- authoring run had no way to see what the first one produced. That is a +-- correctness problem, not a convenience: an adaptor that cannot read its +-- previous set cannot *retain* the parts it is not changing, so a re-authoring +-- run silently drops migrations a topic still needs — and the install would +-- apply that lossy set. +-- +-- This table is the missing half: one row per authored set, newest last, so +-- the driver can hand the RLM the set it wrote before +-- (`VmJob::ProposeRules.current`) and the adaptor can keep what it means to +-- keep. +-- +-- The rows are a **journal**, like every other `proof_*` table: an +-- authoring run appends, nothing is rewritten, and "the set in force" is the +-- newest row for a topic. `digest` is the canonical digest of the document +-- stored beside it, so an audit can prove the row is the set it claims to be +-- rather than a re-serialisation of it. +-- +-- `topic_id` is the shared challenge DB's discriminant, exactly like every +-- other `proof_*` table; there is no per-topic schema. The id shape is the +-- same CHECK `proof_topic_version` enforces. + +CREATE TABLE proof_topic_authoring ( + id BIGSERIAL PRIMARY KEY, + topic_id TEXT NOT NULL, + -- Monotonic per topic, starting at 1: which authoring run this is. + version INTEGER NOT NULL, + -- The set verbatim, as the RLM wrote it. + document JSONB NOT NULL, + -- `sha256:<64 lowercase hex>` of the document's canonical JSON. Never + -- invented: the driver computes it from the set it read back. + digest TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT proof_topic_authoring_topic_check + CHECK (topic_id ~ '^[a-z0-9][a-z0-9-]{1,62}$'), + CONSTRAINT proof_topic_authoring_version_pos CHECK (version >= 1), + CONSTRAINT proof_topic_authoring_digest_check + CHECK (digest ~ '^sha256:[0-9a-f]{64}$'), + -- One row per (topic, version): a re-run of the same authoring is the + -- same set, so a duplicate is a mistake rather than history. + CONSTRAINT proof_topic_authoring_unique UNIQUE (topic_id, version) +); + +-- The read is "the newest set for this topic" — what the next authoring run +-- is handed. +CREATE INDEX ix_proof_topic_authoring_topic + ON proof_topic_authoring (topic_id, version DESC); + +-- Append-only for the application role: a journal that could be edited in +-- place would not be a journal. A re-authoring appends. +GRANT SELECT, INSERT ON TABLE proof_topic_authoring TO base_app; +GRANT USAGE, SELECT ON SEQUENCE proof_topic_authoring_id_seq TO base_app; diff --git a/crates/db/migrations/0028_proof_topic_api_reconcile.sql b/crates/db/migrations/0028_proof_topic_api_reconcile.sql new file mode 100644 index 000000000..32bdbf6bf --- /dev/null +++ b/crates/db/migrations/0028_proof_topic_api_reconcile.sql @@ -0,0 +1,50 @@ +-- Reconcile a topic's dynamic routes on re-install, and make the mux see it. +-- +-- `proof_topic_api` was `SELECT, INSERT` for the application role, which made +-- it append-only — and a **row count** the mux's change signal. That was sound +-- while a topic's route set could only grow, but it is **wrong now that a set +-- can be replaced**: when an RLM-authored install supersedes a +-- bundle-authored one, the old set's routes are not in the new set, and an +-- append-only table leaves them resolving. A miner would still reach an +-- endpoint the topic's current install does not declare, while the journal +-- says the newer set is in force. +-- +-- Two changes, and they belong together: +-- +-- 1. The application role may `DELETE` from `proof_topic_api`. The install +-- reconciles the topic's rows against the set it is applying **inside one +-- transaction** with the insert, so a reader never sees a half-replaced +-- table. It stays without UPDATE: a route row is a claim, and a claim is +-- replaced rather than edited. +-- 2. The mux's change signal stops being a row count. A count cannot see a +-- replacement — delete three, insert three, and it is unchanged — so the +-- install bumps a **revision** instead: one monotonic number per topic, +-- incremented in the same transaction as the reconciliation. The mux sums +-- the revisions, which only ever grows, so a replacement moves the +-- generation exactly as an addition does. +-- +-- The journal still keeps the history: `proof_topic_install` records every +-- install and `binding.authorship` says which set was in force. What this +-- migration stops is the *route table* accumulating claims from sets that are +-- no longer the topic's. + +-- The install reconciles by deleting rows absent from the set it is applying. +GRANT DELETE ON TABLE proof_topic_api TO base_app; + +-- One monotonic revision per topic, bumped by every install that writes routes +-- (and by a re-install that writes none, because the *reconciliation* is what +-- a reader has to see). A counter, not a journal: its history is the install +-- journal's business, and the only thing anyone reads here is the number. +CREATE TABLE proof_topic_route_revision ( + topic_id TEXT PRIMARY KEY, + revision BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT proof_topic_route_revision_topic_check + CHECK (topic_id ~ '^[a-z0-9][a-z0-9-]{1,62}$'), + CONSTRAINT proof_topic_route_revision_pos CHECK (revision >= 0) +); + +-- The install's bump: `revision + 1`, and the row is created on first write. +-- This is the one place an UPDATE is granted, and it is a counter rather than +-- a record of what happened. +GRANT SELECT, INSERT, UPDATE ON TABLE proof_topic_route_revision TO base_app; diff --git a/crates/proof-challenge/src/topic_routes.rs b/crates/proof-challenge/src/topic_routes.rs index a06ec9454..5d77bb9fb 100644 --- a/crates/proof-challenge/src/topic_routes.rs +++ b/crates/proof-challenge/src/topic_routes.rs @@ -116,9 +116,42 @@ impl PgInstallJournal { #[async_trait::async_trait] impl proof_http::InstallJournal for PgInstallJournal { async fn applied(&self, topic_id: &str) -> Result { - proof_topic_install::applied_install(&self.pool, topic_id) + // Two facts, both required before an `open` document may be published. + // + // 1. The newest install row is `applied` — every migration, route, and + // rule the operator's bundle carries is in place. + // 2. The rule vector **that install landed** was authored by the + // topic's RLM (`proof_rule_version.source = 'rlm'` for + // `version = proof_topic_install.rules_version`). + // + // The second is the operator-cloned-document gate. An install seeds + // rule version 1 from the signed document (`topic_document`), which is + // honest provenance but is the *operator's* vector: publishing an + // `open` document against it would make a topic whose behavior nobody + // authored, and whose rules the RLM never wrote, submitable. Only the + // RLM's own `propose_rules` job inside the topic VM advances the store + // to `rlm`, and the guest refuses to echo the signed checklist back as + // if it had. So an `open` document is refused until that happened. + // + // The two facts are read as **one bound query**, not as two + // independent predicates: "the newest rule version is rlm" would admit + // a topic whose install landed the operator's version 1 while an + // unrelated version 2 was RLM-authored — the topic would open with the + // operator's vector in force. `installed_rules` joins the install row + // to the exact version it recorded, so the provenance answered for is + // the provenance of the vector in force. + // + // The read fails closed: a database error is an `Err`, which the + // publish route turns into a refusal, never into an admission. + match proof_topic_install::installed_rules(&self.pool, topic_id) .await - .map_err(|e| e.to_string()) + .map_err(|e| e.to_string())? + { + proof_topic_install::InstalledRules::RlmAuthored { .. } => Ok(true), + proof_topic_install::InstalledRules::NotRlmAuthored { .. } + | proof_topic_install::InstalledRules::SupersededByOperator { .. } + | proof_topic_install::InstalledRules::NotApplied { .. } => Ok(false), + } } async fn submit_gate(&self, topic_id: &str) -> Result { diff --git a/crates/proof-fc-experiment/src/lib.rs b/crates/proof-fc-experiment/src/lib.rs index 7cb7abc2b..5af9f5e6e 100644 --- a/crates/proof-fc-experiment/src/lib.rs +++ b/crates/proof-fc-experiment/src/lib.rs @@ -302,7 +302,10 @@ fn reported_flops(output: &VmJobOutput) -> Option { match output { VmJobOutput::Baseline(r) => r.flops_used, VmJobOutput::Evaluated(run) => run.report.flops_used, - VmJobOutput::Rules(_) | VmJobOutput::Inspected(_) | VmJobOutput::Archived => None, + VmJobOutput::Authored(_) + | VmJobOutput::Rules(_) + | VmJobOutput::Inspected(_) + | VmJobOutput::Archived => None, } } diff --git a/crates/proof-fc-harvest/src/lib.rs b/crates/proof-fc-harvest/src/lib.rs index 1fba34e6c..d5721bd86 100644 --- a/crates/proof-fc-harvest/src/lib.rs +++ b/crates/proof-fc-harvest/src/lib.rs @@ -226,7 +226,10 @@ fn report_from_output(output: &VmJobOutput) -> Option<&CustomRunReport> { match output { VmJobOutput::Baseline(report) => Some(report), VmJobOutput::Evaluated(run) => Some(&run.report), - VmJobOutput::Inspected(_) | VmJobOutput::Rules(_) | VmJobOutput::Archived => None, + VmJobOutput::Authored(_) + | VmJobOutput::Inspected(_) + | VmJobOutput::Rules(_) + | VmJobOutput::Archived => None, } } diff --git a/crates/proof-fc-host/Cargo.toml b/crates/proof-fc-host/Cargo.toml index 5474e288d..7e98726a1 100644 --- a/crates/proof-fc-host/Cargo.toml +++ b/crates/proof-fc-host/Cargo.toml @@ -13,6 +13,7 @@ async-trait = "0.1" hex = "0.4" proof-rlm = { path = "../proof-rlm" } proof-fc-harvest = { path = "../proof-fc-harvest" } +proof-fc-net = { path = "../proof-fc-net" } proof-vm-agent = { path = "../proof-vm-agent" } proof-vm-proto = { path = "../proof-vm-proto" } serde = { version = "1", features = ["derive"] } @@ -24,6 +25,7 @@ tokio-util = { version = "0.7", default-features = false } tracing = "0.1" [dev-dependencies] +proof-fc-net = { path = "../proof-fc-net", features = ["test-fixtures"] } proof-results = { path = "../proof-results" } proof-rlm = { path = "../proof-rlm", features = ["test-fixtures"] } proof-vm-proto = { path = "../proof-vm-proto", features = ["test-fixtures"] } diff --git a/crates/proof-fc-host/src/jail.rs b/crates/proof-fc-host/src/jail.rs index d89e54215..ea61c1ed4 100644 --- a/crates/proof-fc-host/src/jail.rs +++ b/crates/proof-fc-host/src/jail.rs @@ -297,19 +297,33 @@ pub async fn retain(cfg: &HostConfig, shell: &dyn Shell, id: &str) -> Result, shell: Arc, id: String, net: Option, + net_up: bool, child: Option, ) { if let Some(mut child) = child { kill(&mut child).await; } if let Some(net) = net { - for e in net.down(shell.as_ref()).await { - tracing::debug!(jail = %id, "network teardown on release: {e}"); + if net_up { + for e in net.down(shell.as_ref()).await { + tracing::debug!(jail = %id, "network teardown on release: {e}"); + } + } else { + tracing::debug!( + jail = %id, tap = %net.tap, + "tap was never brought up by this boot; leaving it to its owner" + ); } } if let Err(e) = destroy(&cfg, shell.as_ref(), &id).await { @@ -332,6 +346,9 @@ pub struct JailGuard { id: String, root: PathBuf, net: Option, + /// Whether this guard's boot created the TAP (`net.up` succeeded). A + /// guard that never got the interface must not delete it on release. + net_up: bool, child: Option, armed: bool, } @@ -354,11 +371,18 @@ impl JailGuard { id: boot.id.clone(), root, net: boot.net.clone(), + net_up: false, child: None, armed: true, }) } + /// Record that this boot created the TAP. Only a guard that called this + /// may delete the interface on release. + pub fn net_brought_up(&mut self) { + self.net_up = true; + } + /// Jail id. #[must_use] pub fn id(&self) -> &str { @@ -405,7 +429,8 @@ impl JailGuard { } /// Kill the process, tear the network down, remove the jail — now, and - /// to completion. + /// to completion. The network is released only when this guard's boot + /// created it ([`Self::net_brought_up`]). pub async fn destroy(mut self) { self.armed = false; release( @@ -413,6 +438,7 @@ impl JailGuard { self.shell.clone(), std::mem::take(&mut self.id), self.net.take(), + self.net_up, self.child.take(), ) .await; @@ -432,6 +458,7 @@ impl Drop for JailGuard { self.shell.clone(), id, self.net.take(), + self.net_up, self.child.take(), )); } else { @@ -662,16 +689,24 @@ mod tests { /// The guard is the no-leak contract: dropped armed (an aborted task, a /// request the client gave up on) it releases the jail on the runtime; /// handed over with `keep` it does nothing; `destroy` releases inline. + /// + /// The **network** half of that contract is ownership-scoped: a guard + /// releases the TAP only when its own boot brought it up + /// ([`JailGuard::net_brought_up`]). A boot that lost the name to another + /// VM must not delete the interface that VM is using — see + /// [`a_guard_that_did_not_create_the_tap_leaves_it_alone`]. #[tokio::test] async fn a_dropped_guard_releases_the_jail_and_a_kept_one_does_not() { let c = Arc::new(cfg("guard")); let shell = Arc::new(RecordingShell::default()); let plan = NetPlan::for_index(&c, 5); - let guard = JailGuard::prepare(c.clone(), shell.clone(), &boot(Some(plan))) + let mut guard = JailGuard::prepare(c.clone(), shell.clone(), &boot(Some(plan))) .await .expect("prepare"); assert_eq!(guard.id(), "topic-a-0001"); assert_eq!(guard.root(), c.jail_root("topic-a-0001")); + // This guard's boot brought the interface up, so it owns it. + guard.net_brought_up(); let before = shell.calls().len(); let aborted = tokio::spawn(async move { let _held = guard; @@ -756,4 +791,42 @@ mod tests { let _ = std::fs::remove_dir_all(&c3.chroot_base); let _ = std::fs::remove_dir_all(&c2.chroot_base); } + + /// The ownership rule, on its own: a guard whose boot never brought the + /// TAP up — it lost the name to a live VM — releases its own jail and + /// leaves the interface and its nftables table to their owner. + /// + /// Deleting them would turn one VM's collision into another VM's outage: + /// the LIVE Gate 3 shape, where a leftover topic VM's `pfc0` was torn down + /// by the experiment boot that failed to take it. + #[tokio::test] + async fn a_guard_that_did_not_create_the_tap_leaves_it_alone() { + let c = Arc::new(cfg("not-mine")); + let shell = Arc::new(RecordingShell::default()); + let plan = NetPlan::for_index(&c, 9); + let guard = JailGuard::prepare(c.clone(), shell.clone(), &boot(Some(plan))) + .await + .expect("prepare"); + // No `net_brought_up`: this boot never got the interface. + let before = shell.calls().len(); + guard.destroy().await; + let lines: Vec = shell.calls()[before..] + .iter() + .map(|c| c.join(" ")) + .collect(); + assert!( + !lines.iter().any(|l| l.contains("nft delete table")), + "a tap this boot never created must not be deleted: {lines:?}" + ); + assert!( + !lines.iter().any(|l| l.starts_with("ip link del")), + "a tap this boot never created must not be deleted: {lines:?}" + ); + assert_eq!( + lines, + vec![format!("rm -rf {}", c.jail_dir("topic-a-0001").display())], + "the jail is still this guard's to remove" + ); + let _ = std::fs::remove_dir_all(&c.chroot_base); + } } diff --git a/crates/proof-fc-host/src/lib.rs b/crates/proof-fc-host/src/lib.rs index 00e440b43..1a5a28571 100644 --- a/crates/proof-fc-host/src/lib.rs +++ b/crates/proof-fc-host/src/lib.rs @@ -38,10 +38,7 @@ #![forbid(unsafe_code)] #![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)] -pub mod config; pub mod jail; -pub mod net; -pub mod shell; pub mod sister; pub mod vsock; @@ -63,12 +60,19 @@ use tokio::net::UnixListener; use tokio::sync::Mutex; use tokio_util::sync::CancellationToken; -pub use config::{EgressAllow, HostConfig, Proto}; +// The host plumbing (config, `Shell`, per-VM networking) lives in +// `proof-fc-net` so this crate stays under the workspace LOC cap. Re-exported +// so every existing `proof_fc_host::…` path keeps working. pub use jail::JailGuard; -pub use net::NetPlan; pub use proof_fc_harvest::images; pub use proof_fc_harvest::images::ImageCache; -pub use shell::{RecordingShell, Shell, SystemShell}; +pub use proof_fc_net::config; +pub use proof_fc_net::net; +pub use proof_fc_net::shell; +#[cfg(test)] +pub use proof_fc_net::shell::FailingShell; +pub use proof_fc_net::{EgressAllow, HostConfig, NetPlan, Proto}; +pub use proof_fc_net::{RecordingShell, Shell, SystemShell}; pub use sister::SisterCtx; /// How long a job waits for its sister task to finish killing and destroying @@ -311,7 +315,58 @@ impl FirecrackerHypervisor { ) -> Result { let cfg = self.ctx.cfg.clone(); let owner_files = self.owner_files()?; - let net = NetPlan::for_index(&cfg, self.net_index.fetch_add(1, Ordering::SeqCst)); + // The TAP index has to come from the **host**, not just from the + // in-process counter. A jailed VM outlives the agent that booted it + // (the jailer is handed over, not a child of this process), so after + // an agent restart — or with a topic VM an earlier agent left behind + // holding `pfc0` — a counter starting again at zero names a TAP that + // is already up, and `ip tuntap add` answers + // `ioctl(TUNSETIFF): Device or resource busy`. That is how a leftover + // topic VM from the baseline/RLM install took every new experiment VM + // off the air and no miner submission ever produced a row. + // + // Scanning first makes the common case exact; the retry covers the + // race where two boots pick the same free index, and the *second* + // boot moves on instead of failing. + let mut attempts: u32 = 0; + loop { + attempts = attempts.saturating_add(1); + let from = self.net_index.load(Ordering::SeqCst); + let index = NetPlan::first_free(self.ctx.shell.as_ref(), from).await?; + // `fetch_max`, not `store`: two boots running at once both load + // the same `from`, and a plain store would let the slower one + // move the counter *backwards* past an index the faster one just + // claimed. The scan still protects correctness; this keeps the + // counter monotonic so it does not have to re-scan ground it + // already covered. + self.net_index + .fetch_max(index.saturating_add(1), Ordering::SeqCst); + let net = NetPlan::for_index(&cfg, index); + match self + .boot_on_index(vm_id, spec, image.clone(), net, owner_files.clone()) + .await + { + Err(e) if NetPlan::name_taken(&e) && attempts < net::TAP_ATTEMPTS => { + tracing::warn!( + %vm_id, index, attempts, + "tap index raced another boot; retrying on the next free one: {e}" + ); + } + other => return other, + } + } + } + + /// [`Self::boot_verified`] with the network plan already chosen. + async fn boot_on_index( + &self, + vm_id: &str, + spec: &TopicVmSpec, + image: PathBuf, + net: NetPlan, + owner_files: Vec, + ) -> Result { + let cfg = self.ctx.cfg.clone(); let boot = jail::VmBoot { id: vm_id.to_owned(), vcpus: spec.template.vcpus, @@ -377,6 +432,10 @@ impl FirecrackerHypervisor { ) -> Result<(), HvError> { let vm_id = jail.id().to_owned(); net.up(shell, cfg.jail_uid).await?; + // The interface exists now, so this boot owns it: only from here may + // the guard delete it. A boot that failed at `up` leaves the TAP to + // whichever VM already had it. + jail.net_brought_up(); let rules = cfg.jail_dir(&vm_id).join("net.nft").display().to_string(); net.load_rules(shell, &rules).await?; // Advisory: a ufw / Docker forward chain that drops by default @@ -699,11 +758,18 @@ mod tests { /// or a guest that never says hello — releases everything it built: the /// nftables table, the TAP, and the jail directory (with the rules file /// and scratch inside it). Nothing is registered, nothing is alive. + /// + /// The network half is **ownership-scoped**, and the two injected + /// failures differ on exactly that: a boot that failed at `ip tuntap add` + /// never got the interface (another VM holds it), so it must leave the + /// TAP and its table alone; a boot that got the interface and failed + /// later owns it and must release it. #[tokio::test] async fn a_boot_that_fails_before_the_handshake_releases_its_jail_and_network() { let req = request(); let spec = TopicVmSpec::for_topic(&req.topic_id, pinned_template(), req.sandbox.clone()); - for (tag, fail_on) in [("tap", "ip tuntap"), ("rules", "nft -f")] { + // (tag, failing command, did this boot create the TAP?) + for (tag, fail_on, owns_tap) in [("tap", "ip tuntap", false), ("rules", "nft -f", true)] { let c = cfg(tag); let image = c.image_dir.join("rlm.ext4"); std::fs::write(&image, b"rlm rootfs stand-in").expect("image"); @@ -725,14 +791,25 @@ mod tests { .position(|l| l.starts_with(fail_on)) .expect("position"); let after = &lines[failed_at + 1..]; - assert!( - after.contains(&"nft delete table inet proof_vm_pfc0".to_owned()), - "{tag}: table released: {after:?}" - ); - assert!( - after.contains(&"ip link del pfc0".to_owned()), - "{tag}: tap released: {after:?}" - ); + if owns_tap { + assert!( + after.contains(&"nft delete table inet proof_vm_pfc0".to_owned()), + "{tag}: table released: {after:?}" + ); + assert!( + after.contains(&"ip link del pfc0".to_owned()), + "{tag}: tap released: {after:?}" + ); + } else { + assert!( + !after.iter().any(|l| l.contains("nft delete table")), + "{tag}: a tap this boot never created must not be deleted: {after:?}" + ); + assert!( + !after.iter().any(|l| l.starts_with("ip link del")), + "{tag}: a tap this boot never created must not be deleted: {after:?}" + ); + } assert_eq!( after.last().map(String::as_str), Some(format!("rm -rf {jail_dir}").as_str()), @@ -834,6 +911,154 @@ mod tests { }) } + /// A shell that models the host's TAP namespace: `ip tuntap add` fails + /// with the kernel's own `TUNSETIFF` wording when the name is taken, and + /// `ip link del` frees it. Everything else succeeds. + /// + /// This is what lets a test drive the allocator and the retry against a + /// *changing* host, the way a second VM booting at the same time does — + /// the recording shell always answers "free", so it can never show a + /// collision. + #[derive(Default)] + struct TapNamespace { + live: std::sync::Mutex>, + /// Index a concurrent boot grabs *between* the scan and the + /// `ip tuntap add`: the add for it fails once, then the index is + /// live as if the other boot had taken it. + race: std::sync::Mutex>, + /// Every add fails, as on a host whose pool is exhausted. + always_busy: std::sync::atomic::AtomicBool, + inner: RecordingShell, + } + + impl TapNamespace { + fn with_live(taps: &[u32]) -> Self { + let mut set = std::collections::BTreeSet::new(); + for t in taps { + set.insert(*t); + } + Self { + live: std::sync::Mutex::new(set), + race: std::sync::Mutex::new(None), + always_busy: std::sync::atomic::AtomicBool::new(false), + inner: RecordingShell::default(), + } + } + + fn live(&self) -> Vec { + self.live + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + .copied() + .collect() + } + + fn lines(&self) -> Vec { + self.inner.calls().iter().map(|c| c.join(" ")).collect() + } + + /// A concurrent boot takes `index` after this boot's scan but before + /// its `ip tuntap add` — the Gate 4 race. + fn race_on(&self, index: u32) { + *self + .race + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index); + } + + /// Every index is taken by the time the add lands. + fn exhaust(&self) { + self.always_busy + .store(true, std::sync::atomic::Ordering::SeqCst); + } + } + + #[async_trait] + impl Shell for TapNamespace { + async fn run( + &self, + program: &str, + args: &[String], + ) -> Result { + let out = self.inner.run(program, args).await?; + let a: Vec<&str> = args.iter().map(String::as_str).collect(); + let busy = || crate::shell::CmdOutput { + code: Some(1), + stdout: String::new(), + stderr: "ioctl(TUNSETIFF): Device or resource busy\n".into(), + }; + let mut live = self + .live + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + match (program, a.as_slice()) { + // `ip tuntap add dev pfc mode tap user ` + ("ip", ["tuntap", "add", "dev", tap, ..]) => { + let index: u32 = tap + .trim_start_matches("pfc") + .parse() + .expect("a pfc index in this test"); + let mut race = self + .race + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if *race == Some(index) { + // The other boot won this index just now. + *race = None; + live.insert(index); + return Ok(busy()); + } + drop(race); + if self.always_busy.load(std::sync::atomic::Ordering::SeqCst) + || !live.insert(index) + { + return Ok(busy()); + } + } + ("ip", ["link", "del", tap]) => { + if let Ok(index) = tap.trim_start_matches("pfc").parse::() { + live.remove(&index); + } + } + // The scan the allocator runs. + ("ip", ["-o", "link", "show"]) => { + use std::fmt::Write as _; + let mut listing = String::new(); + for i in live.iter() { + let _ = writeln!(listing, "{i}: pfc{i}: mtu 1500"); + } + return Ok(crate::shell::CmdOutput { + code: Some(0), + stdout: listing, + stderr: String::new(), + }); + } + // `rm -rf ` really removes it. `jail::prepare` refuses a + // root that already exists and `JailGuard::destroy` clears it + // through this command, so a shell that only *records* the + // removal would leave a jail behind and make a retry fail for + // a reason the host would never produce. + ("rm", ["-rf", dir]) => { + drop(live); + if let Err(e) = std::fs::remove_dir_all(dir) { + if e.kind() != std::io::ErrorKind::NotFound { + return Ok(crate::shell::CmdOutput { + code: Some(1), + stdout: String::new(), + stderr: format!("rm: {e}\n"), + }); + } + } + return Ok(out); + } + _ => {} + } + drop(live); + Ok(out) + } + } + /// A guest that says hello, then drops the job connection after reading /// `Run` — Broken pipe / EOF, no `Done`. The host must harvest scratch. fn serve_fake_guest_dropping_done(root: PathBuf) -> tokio::task::JoinHandle { @@ -901,6 +1126,139 @@ mod tests { }) } + /// The LIVE Gate 3 shape, end to end on the host: a leftover topic VM + /// holds `pfc0` while a new VM boots. The allocator must skip it — the + /// counter alone names `pfc0`, and `ip tuntap add` answers + /// `ioctl(TUNSETIFF): Device or resource busy`. + #[tokio::test] + async fn a_leftover_topic_vm_does_not_take_the_next_boot_off_the_air() { + let c = stand_in_host("leftover-tap"); + let req = request(); + let spec = TopicVmSpec::for_topic(&req.topic_id, pinned_template(), req.sandbox.clone()); + // The baseline's topic VM is still up, holding the first TAP. + let shell = Arc::new(TapNamespace::with_live(&[0])); + let hv = + Arc::new(FirecrackerHypervisor::with_shell(c.clone(), shell.clone()).expect("config")); + let guest = serve_fake_guest_when_ready(c.jail_root("topic-b-0001")); + let vm = hv + .boot_verified("topic-b-0001", &spec, c.image_dir.join("rlm.ext4")) + .await + .expect("a leftover tap must not fail the boot"); + guest.await.expect("guest"); + + let lines = shell.lines(); + assert!( + lines + .iter() + .any(|l| l == "ip tuntap add dev pfc1 mode tap user 65534"), + "the new vm takes the first free index, not the taken one: {lines:?}" + ); + assert!( + !lines + .iter() + .any(|l| l == "ip tuntap add dev pfc0 mode tap user 65534"), + "the leftover vm's tap is never attempted: {lines:?}" + ); + assert_eq!( + shell.live(), + vec![0, 1], + "both taps are live; the new boot did not disturb the old one" + ); + assert!(hv.alive(&vm).await); + assert!(hv.teardown(&vm, RetainPolicy::Destroy).await.expect("down")); + assert_eq!( + shell.live(), + vec![0], + "teardown frees only this vm's tap; the leftover keeps its own" + ); + let _ = std::fs::remove_dir_all(&c.chroot_base); + } + + /// Two VMs booting at once can pick the same free index between the scan + /// and the `ip tuntap add`. The loser must move to the next free one + /// instead of failing — Gate 4 runs exactly this race. + #[tokio::test] + async fn a_raced_tap_index_moves_the_loser_to_the_next_free_one() { + let c = stand_in_host("raced-tap"); + let req = request(); + let spec = TopicVmSpec::for_topic(&req.topic_id, pinned_template(), req.sandbox.clone()); + // The scan answers "nothing is up", so this boot picks 0 — and a + // concurrent boot takes 0 in the window between that scan and our + // `ip tuntap add`. + let shell = Arc::new(TapNamespace::default()); + shell.race_on(0); + let hv = + Arc::new(FirecrackerHypervisor::with_shell(c.clone(), shell.clone()).expect("config")); + let guest = serve_fake_guest_when_ready(c.jail_root("topic-c-0001")); + let vm = hv + .boot_verified("topic-c-0001", &spec, c.image_dir.join("rlm.ext4")) + .await + .expect("the loser retries instead of failing"); + guest.await.expect("guest"); + + let lines = shell.lines(); + let first = lines + .iter() + .position(|l| l == "ip tuntap add dev pfc0 mode tap user 65534") + .expect("the first attempt is on the index the scan offered"); + let retry = lines + .iter() + .position(|l| l == "ip tuntap add dev pfc1 mode tap user 65534") + .expect("the retry moves to the next free index"); + assert!( + first < retry, + "the retry comes after the collision: {lines:?}" + ); + // The failed attempt's jail is released before the retry, so the + // second `prepare` does not meet its own leftovers. + let removed = lines + .iter() + .position(|l| l.starts_with("rm -rf") && l.ends_with("topic-c-0001")) + .expect("the failed attempt releases its jail"); + assert!( + first < removed && removed < retry, + "jail released between the attempts: {lines:?}" + ); + assert!(hv.alive(&vm).await); + assert_eq!( + shell.live(), + vec![0, 1], + "the raced index belongs to the other boot; this one took the next free" + ); + let _ = std::fs::remove_dir_all(&c.chroot_base); + } + + /// The retry is bounded: a host that keeps losing the race answers the + /// caller an error rather than spinning forever. + #[tokio::test] + async fn the_tap_retry_gives_up_rather_than_spinning() { + let c = stand_in_host("tap-exhausted"); + let req = request(); + let spec = TopicVmSpec::for_topic(&req.topic_id, pinned_template(), req.sandbox.clone()); + // Every index the allocator offers is taken before the boot gets there. + let shell = Arc::new(TapNamespace::default()); + shell.exhaust(); + let hv = + Arc::new(FirecrackerHypervisor::with_shell(c.clone(), shell.clone()).expect("config")); + let err = hv + .boot_verified("topic-d-0001", &spec, c.image_dir.join("rlm.ext4")) + .await + .expect_err("a host with no free tap must refuse, not spin"); + assert!(err.to_string().contains("already taken"), "{err}"); + let attempts = shell + .lines() + .iter() + .filter(|l| l.starts_with("ip tuntap add")) + .count(); + assert_eq!( + attempts, + usize::try_from(net::TAP_ATTEMPTS).expect("fits"), + "the retry budget is exactly TAP_ATTEMPTS" + ); + assert!(hv.vms.lock().await.is_empty(), "nothing registered"); + let _ = std::fs::remove_dir_all(&c.chroot_base); + } + fn stand_in_host(tag: &str) -> HostConfig { let mut c = cfg(tag); c.boot_timeout = Duration::from_secs(10); diff --git a/crates/proof-fc-net/Cargo.toml b/crates/proof-fc-net/Cargo.toml new file mode 100644 index 000000000..325c94eb3 --- /dev/null +++ b/crates/proof-fc-net/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "proof-fc-net" +description = "KVM-host plumbing shared by the Proof Firecracker backend and its experiment layer: operator host config, the `Shell` trait every host command is rendered through (so CI asserts argv without spawning anything), and per-VM networking — a TAP on its own /30, NAT through the uplink, and an nftables table that lets an RLM VM reach only the operator's egress allowlist. Split out of `proof-fc-host` to keep that crate under the workspace LOC cap; no behaviour lives here that the backend did not already have." +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[features] +# Exposes `proof_fc_net::shell::FailingShell` to sibling crates' tests: a +# recording shell that fails every command matching a prefix, so a host +# cleanup path can be asserted without a host. +test-fixtures = [] + +[dependencies] +async-trait = "0.1" +proof-fc-harvest = { path = "../proof-fc-harvest" } +proof-vm-agent = { path = "../proof-vm-agent" } +serde_json = "1" +tokio = { version = "1", features = ["process"] } +tracing = "0.1" + +[dev-dependencies] +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] } + +[lints] +workspace = true diff --git a/crates/proof-fc-host/src/config.rs b/crates/proof-fc-net/src/config.rs similarity index 100% rename from crates/proof-fc-host/src/config.rs rename to crates/proof-fc-net/src/config.rs diff --git a/crates/proof-fc-net/src/lib.rs b/crates/proof-fc-net/src/lib.rs new file mode 100644 index 000000000..e7d8004cc --- /dev/null +++ b/crates/proof-fc-net/src/lib.rs @@ -0,0 +1,25 @@ +//! KVM-host plumbing for the Proof Firecracker backend. +//! +//! Three things the host needs before a VM exists, split out of +//! `proof-fc-host` so that crate stays under the workspace LOC cap: +//! +//! - [`config`] — the operator's host config: paths, pins, sizes, and the +//! egress allowlist. Paths and pins only; no secret ever lives here. +//! - [`shell`] — the [`Shell`] trait every host command is rendered through, +//! so the tests assert the exact argv without spawning anything, plus the +//! recording and failing shells those tests use. +//! - [`net`] — one [`NetPlan`] per VM: a TAP on its own /30, NAT through the +//! uplink, and an nftables table that lets the guest reach **only** the +//! operator's egress allowlist. The allocator picks an index this host does +//! not already have, because a jailed VM outlives the agent that booted it. + +#![forbid(unsafe_code)] +#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)] + +pub mod config; +pub mod net; +pub mod shell; + +pub use config::{EgressAllow, HostConfig, Proto}; +pub use net::NetPlan; +pub use shell::{RecordingShell, Shell, SystemShell}; diff --git a/crates/proof-fc-host/src/net.rs b/crates/proof-fc-net/src/net.rs similarity index 64% rename from crates/proof-fc-host/src/net.rs rename to crates/proof-fc-net/src/net.rs index e544ec120..e8c3a0379 100644 --- a/crates/proof-fc-host/src/net.rs +++ b/crates/proof-fc-net/src/net.rs @@ -2,6 +2,7 @@ //! nftables table that lets the RLM VM reach **only** the operator's egress //! allowlist. The sister miner guest gets no interface at all. +use std::collections::BTreeSet; use std::net::Ipv4Addr; use proof_vm_agent::HvError; @@ -9,6 +10,12 @@ use proof_vm_agent::HvError; use crate::config::{EgressAllow, HostConfig, Proto}; use crate::shell::{sh, Shell}; +/// How many TAP indexes one boot will try before it gives up. The allocator +/// scans the host first, so this only covers a race with a concurrent boot +/// (two experiments starting at once can pick the same free index). It counts +/// **attempts**, so `1` means "no retry". +pub const TAP_ATTEMPTS: u32 = 16; + /// One RLM VM's network plan. #[derive(Debug, Clone, PartialEq, Eq)] pub struct NetPlan { @@ -49,6 +56,41 @@ impl NetPlan { format!("proof_vm_{}", self.tap) } + /// The first `pfc` index at or above `from` that this host does not + /// already have. + /// + /// A process-local counter is not enough to pick one. A jailed VM + /// outlives the agent process that booted it — the jailer is handed over, + /// not a child of the agent — so an agent restart, or a VM an earlier + /// agent left behind (a topic VM still holding its TAP after a baseline), + /// keeps `pfc` in place while the counter starts again at zero. + /// `ip tuntap add` on a live name is `ioctl(TUNSETIFF): Device or + /// resource busy`, which is how a leftover topic VM took every new + /// experiment VM off the air. + /// + /// # Errors + /// + /// [`HvError::Backend`] when `ip` fails. + pub async fn first_free(shell: &dyn Shell, from: u32) -> Result { + let out = sh(shell, "ip", &["-o", "link", "show"]).await?; + let used: BTreeSet = out.stdout.lines().filter_map(tap_index).collect(); + let mut i = from; + while used.contains(&i) { + i = i.saturating_add(1); + } + Ok(i) + } + + /// Whether this error is the host refusing a TAP name that is taken. + /// + /// `ip tuntap add` renders both `EBUSY` and `EEXIST` from `TUNSETIFF` as + /// `ioctl(TUNSETIFF): Device or resource busy`, so the text is the only + /// signal the command's exit status carries. + #[must_use] + pub fn name_taken(e: &HvError) -> bool { + matches!(e, HvError::Backend(m) if m.contains("TUNSETIFF") || m.contains("resource busy")) + } + /// Kernel `ip=` argument giving the guest its address statically. #[must_use] pub fn boot_arg(&self) -> String { @@ -94,23 +136,58 @@ impl NetPlan { /// Create the TAP (owned by the jail uid so jailed Firecracker can open /// it), address it, enable forwarding, load the ruleset. /// + /// A name this host already has is refused **by name** rather than + /// half-built: the caller allocates a free index first + /// ([`Self::first_free`]) and retries on this error, so a TAP that + /// belongs to a live VM is never addressed or deleted by a boot that did + /// not create it. + /// + /// Once `ip tuntap add` succeeds the interface **is** this plan's, so a + /// later step failing (address, link, sysctl) rolls the whole plan back + /// here. Without that, the caller's guard would release a jail for a + /// network it was never told it owned, and the interface would outlive + /// every record of it — consuming a name that later boots then allocate + /// around. + /// /// # Errors /// - /// [`HvError::Backend`] from the first failing command. + /// [`HvError::Backend`] from the first failing command; a taken name is + /// recognisable with [`Self::name_taken`]. pub async fn up(&self, shell: &dyn Shell, jail_uid: u32) -> Result<(), HvError> { let uid = jail_uid.to_string(); - sh( + let added = sh( shell, "ip", &[ "tuntap", "add", "dev", &self.tap, "mode", "tap", "user", &uid, ], ) - .await?; - let cidr = format!("{}/30", self.host_ip); - sh(shell, "ip", &["addr", "add", &cidr, "dev", &self.tap]).await?; - sh(shell, "ip", &["link", "set", &self.tap, "up"]).await?; - sh(shell, "sysctl", &["-q", "-w", "net.ipv4.ip_forward=1"]).await?; + .await; + if let Err(e) = added { + return Err(if Self::name_taken(&e) { + HvError::Backend(format!( + "tap {} is already taken on this host ({e}); another vm holds it", + self.tap + )) + } else { + e + }); + } + let rest = async { + let cidr = format!("{}/30", self.host_ip); + sh(shell, "ip", &["addr", "add", &cidr, "dev", &self.tap]).await?; + sh(shell, "ip", &["link", "set", &self.tap, "up"]).await?; + sh(shell, "sysctl", &["-q", "-w", "net.ipv4.ip_forward=1"]).await?; + Ok(()) + } + .await; + if let Err(e) = rest { + // The interface exists and nothing else knows about it yet. + for err in self.down(shell).await { + tracing::debug!(tap = %self.tap, "rollback of a half-built network: {err}"); + } + return Err(e); + } Ok(()) } @@ -196,6 +273,14 @@ impl NetPlan { /// Prefix every TAP name shares (`pfc`). const TAP_PREFIX: &str = "pfc"; +/// The index of a `pfc` interface, from one `ip -o link show` line +/// (`3: pfc0: mtu …`). `None` for anything else on the host. +fn tap_index(line: &str) -> Option { + let rest = line.split_once(": ")?.1; + let name = rest.split([':', '@']).next()?; + name.strip_prefix(TAP_PREFIX)?.parse().ok() +} + /// Does this `nft -j` rule accept traffic arriving on a TAP? True for an /// `iifname` match against `pfc*` / `pfc+` / a specific `pfc` (also inside /// a set) that ends in an `accept` verdict — the shape both @@ -224,7 +309,34 @@ fn rule_accepts_tap(rule: &serde_json::Value) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::shell::RecordingShell; + use crate::shell::{FailingShell, RecordingShell}; + + /// A shell that answers `ip -o link show` with a canned interface listing + /// and fails anything else — the host state the allocator has to read. + struct LinkListing(String); + + impl LinkListing { + fn new(lines: &[&str]) -> Self { + Self(lines.join("\n") + "\n") + } + } + + #[async_trait::async_trait] + impl Shell for LinkListing { + async fn run( + &self, + program: &str, + args: &[String], + ) -> Result { + assert_eq!(program, "ip", "the scan must ask the host"); + assert_eq!(args, ["-o", "link", "show"]); + Ok(crate::shell::CmdOutput { + code: Some(0), + stdout: self.0.clone(), + stderr: String::new(), + }) + } + } fn cfg() -> HostConfig { let mut c = HostConfig::defaults(); @@ -236,6 +348,131 @@ mod tests { c } + /// The LIVE Gate 3 failure: a topic VM left over from the RLM + /// install/baseline still holds `pfc0`, and the allocator — a counter + /// starting again at zero after the agent restarted — named it for the + /// next experiment VM. `ip tuntap add` answered + /// `ioctl(TUNSETIFF): Device or resource busy`, so no miner submission + /// ever got a `pf_` row. + /// + /// The allocator now asks the **host** which indexes exist, so a leftover + /// VM is skipped instead of collided with. + #[tokio::test] + async fn a_leftover_topic_vm_does_not_take_the_next_index() { + let shell = LinkListing::new(&[ + "1: lo: mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT", + "2: eth0: mtu 1500 qdisc fq_codel state UP", + // The leftover topic VM's TAP, still up from the baseline run. + "3: pfc0: mtu 1500 qdisc fq_codel state UP", + ]); + let next = NetPlan::first_free(&shell, 0).await.expect("scan"); + assert_eq!(next, 1, "pfc0 is taken by the leftover topic vm"); + assert_eq!(NetPlan::for_index(&cfg(), next).tap, "pfc1"); + + // A host with no TAPs of ours starts at the counter's value. + let empty = LinkListing::new(&["1: lo: mtu 65536"]); + assert_eq!(NetPlan::first_free(&empty, 0).await.expect("scan"), 0); + assert_eq!(NetPlan::first_free(&empty, 7).await.expect("scan"), 7); + + // A gap is used before a higher index: a torn-down VM frees its slot. + let gap = LinkListing::new(&[ + "3: pfc0: mtu 1500", + "5: pfc2: mtu 1500", + ]); + assert_eq!(gap_indices(&gap).await, vec![0, 2]); + assert_eq!(NetPlan::first_free(&gap, 0).await.expect("scan"), 1); + } + + /// The scan reads only our TAPs: an unrelated interface whose name merely + /// starts with `pfc`-like text, or a `pfc` with a non-numeric suffix, is + /// not an index. + #[tokio::test] + async fn the_scan_counts_only_pfc_indexes() { + let shell = LinkListing::new(&[ + "3: pfc0: mtu 1500", + "4: pfc10: mtu 1500", + "5: pfconf: mtu 1500", + "6: pfcX: mtu 1500", + "7: docker0: mtu 1500", + ]); + assert_eq!(gap_indices(&shell).await, vec![0, 10]); + // 1..9 are free, so the next is 1 — the scan does not jump to 11. + assert_eq!(NetPlan::first_free(&shell, 0).await.expect("scan"), 1); + } + + /// A taken name is recognised from the kernel's own wording, which is what + /// the boot retry keys on. + #[test] + fn a_taken_tap_name_is_recognised_from_the_kernels_wording() { + let busy = + HvError::Backend("ip exited Some(1): ioctl(TUNSETIFF): Device or resource busy".into()); + assert!(NetPlan::name_taken(&busy)); + assert!(!NetPlan::name_taken(&HvError::Backend( + "ip exited Some(1): RTNETLINK answers: Operation not permitted".into() + ))); + assert!(!NetPlan::name_taken(&HvError::Guest("busy".into()))); + } + + /// A step after `ip tuntap add` failing must roll the interface back. + /// + /// The guard's ownership flag is set from `up` returning `Ok`, so a + /// failure after the interface exists leaves the caller believing it owns + /// nothing — and the interface would outlive every record of it, holding + /// a name that later boots then allocate around. + #[tokio::test] + async fn a_failure_after_the_tap_exists_rolls_the_interface_back() { + for fail_on in ["ip addr add", "ip link set", "sysctl"] { + let shell = FailingShell::failing_on(fail_on); + let plan = NetPlan::for_index(&cfg(), 3); + let err = plan + .up(&shell, 65_534) + .await + .expect_err("injected failure after the tap exists"); + assert!(err.to_string().contains("injected failure"), "{fail_on}"); + let lines = shell.lines(); + let added = lines + .iter() + .position(|l| l.starts_with("ip tuntap add")) + .expect("the tap was created"); + let after = &lines[added + 1..]; + assert!( + after.contains(&"nft delete table inet proof_vm_pfc3".to_owned()), + "{fail_on}: the table is rolled back: {after:?}" + ); + assert!( + after.contains(&"ip link del pfc3".to_owned()), + "{fail_on}: the interface is rolled back: {after:?}" + ); + } + // And a failure *at* the create does not roll anything back: this + // boot never had the interface, so there is nothing of its to remove. + let shell = FailingShell::failing_on("ip tuntap"); + let plan = NetPlan::for_index(&cfg(), 3); + plan.up(&shell, 65_534).await.expect_err("taken name"); + let lines = shell.lines(); + assert!( + !lines.iter().any(|l| l.starts_with("ip link del")), + "a name this boot never took must not be deleted: {lines:?}" + ); + assert!( + !lines.iter().any(|l| l.contains("nft delete table")), + "a name this boot never took must not be deleted: {lines:?}" + ); + } + + async fn gap_indices(shell: &LinkListing) -> Vec { + let out = shell + .run( + "ip", + &["-o".to_owned(), "link".to_owned(), "show".to_owned()], + ) + .await + .expect("scan"); + let mut got: Vec = out.stdout.lines().filter_map(tap_index).collect(); + got.sort_unstable(); + got + } + #[test] fn plans_carve_the_pool_into_p2p_slash_30s() { let p0 = NetPlan::for_index(&cfg(), 0); diff --git a/crates/proof-fc-host/src/shell.rs b/crates/proof-fc-net/src/shell.rs similarity index 97% rename from crates/proof-fc-host/src/shell.rs rename to crates/proof-fc-net/src/shell.rs index 60dd9c393..45723d1a5 100644 --- a/crates/proof-fc-host/src/shell.rs +++ b/crates/proof-fc-net/src/shell.rs @@ -112,10 +112,10 @@ pub async fn sh(shell: &dyn Shell, program: &str, args: &[&str]) -> Result Result; - /// The operator gate and the allocator pin for one topic, for the submit /// path. /// @@ -167,11 +173,13 @@ pub trait InstallJournal: Send + Sync { /// the only difference is which sentence the operator reads. pub type InstallJournalSlot = Option>; -/// Whether the topic's install reached `applied`, as the publish gate reads it. +/// Whether the topic is ready to be published `open`, as the publish gate +/// reads it. /// -/// **Fail-closed on every doubt**: no journal slot, an unreadable journal, and -/// a topic with no install row all refuse, so an `open` document is never -/// published before its migrations, routes, and rules are in place. +/// **Fail-closed on every doubt**: no journal slot, an unreadable journal, a +/// topic with no install row, and a topic whose rules are not RLM-authored all +/// refuse, so an `open` document is never published before its migrations, +/// routes, and rules are in place *and* its RLM has authored its behavior. pub(crate) async fn install_gate(st: &AppState, topic_id: &str) -> Result<(), String> { let Some(journal) = st.install_journal.as_deref() else { return Err(format!( @@ -183,11 +191,12 @@ pub(crate) async fn install_gate(st: &AppState, topic_id: &str) -> Result<(), St match journal.applied(topic_id).await { Ok(true) => Ok(()), Ok(false) => Err(format!( - "topic {topic_id:?} has no `applied` install row: run `proof-admin topic install` to \ - completion first (the journal is `proof_topic_install`; read it with `proof-admin \ - topic install-log --topic {topic_id}`). A `pending` or `failed` row means the \ - migrations, routes, or rules are not in place, and an `open` document is submitable \ - the moment it is published." + "topic {topic_id:?} is not ready to be `open`: either its newest `proof_topic_install` \ + row is not `applied`, or its rule vector is not RLM-authored \ + (`proof_rule_version.source` is not `rlm`). Run `proof-admin topic install` \ + --drive-rlm --owner-approved to completion, then read `proof-admin topic \ + install-log --topic {topic_id}`. Rules still sourced from the signed document mean \ + the topic's behavior was not authored by its RLM." )), Err(e) => Err(format!( "the install journal could not be read for topic {topic_id:?}: {e}. The publish is \ diff --git a/crates/proof-results/src/lib.rs b/crates/proof-results/src/lib.rs index ee1ffcc41..cb56bbb81 100644 --- a/crates/proof-results/src/lib.rs +++ b/crates/proof-results/src/lib.rs @@ -62,7 +62,7 @@ pub const WRITE_RESULTS_EMIT: &str = "write_results_next_to_report"; pub const MAX_RESULTS_BYTES: u64 = 256 * 1024; /// Harbor family cap (bytes): [`CONTRACT_HARBOR_TRIALS`] / -/// [`CONTRACT_TBENCH_HARBOR`] only, after `contract` is identified. +/// [`CONTRACT_HARBOR_TRIALS_LEGACY`] only, after `contract` is identified. /// /// [`load_file`] may *read* up to this ceiling so a Harbor document between /// [`MAX_RESULTS_BYTES`] and this size can parse; generic-custom-v1 over @@ -82,12 +82,16 @@ pub const CONTRACT_GENERIC: &str = "generic-custom-v1"; /// pins. pub const CONTRACT_HARBOR_TRIALS: &str = "harbor-trials-v1"; -/// Legacy alias of [`CONTRACT_HARBOR_TRIALS`], kept because it is a **wire -/// value**: a topic signed before the generic id existed pins this in its -/// `constraints.params.results_contract`, and a signed document cannot be +/// Legacy wire value of [`CONTRACT_HARBOR_TRIALS`], kept because it is a +/// **wire value**: a topic signed before the generic id existed pins this in +/// its `constraints.params.results_contract`, and a signed document cannot be /// edited. New topics pin [`CONTRACT_HARBOR_TRIALS`]; the guest harness /// accepts both, and nothing branches on a topic. -pub const CONTRACT_TBENCH_HARBOR: &str = "tbench-harbor-v1"; +/// +/// The constant name is deliberately topic-neutral: the value is a +/// compatibility spelling, not a topic the code knows. Do not rename the +/// string — it is signed topic data. +pub const CONTRACT_HARBOR_TRIALS_LEGACY: &str = "tbench-harbor-v1"; /// Harbor trial that produced a verifier reward. pub const HARBOR_OUTCOME_MEASURED: &str = "measured"; @@ -188,13 +192,13 @@ pub enum Contract { HarborTrials, } -/// Known contract id → family. [`CONTRACT_TBENCH_HARBOR`] is the legacy +/// Known contract id → family. [`CONTRACT_HARBOR_TRIALS_LEGACY`] is the legacy /// spelling of [`CONTRACT_HARBOR_TRIALS`]. #[must_use] pub fn known_contract(id: &str) -> Option { match id.trim() { CONTRACT_GENERIC => Some(Contract::Generic), - CONTRACT_HARBOR_TRIALS | CONTRACT_TBENCH_HARBOR => Some(Contract::HarborTrials), + CONTRACT_HARBOR_TRIALS | CONTRACT_HARBOR_TRIALS_LEGACY => Some(Contract::HarborTrials), _ => None, } } @@ -742,7 +746,8 @@ mod tests { let b = bind(); let g = generic_document(&b, &serde_json::json!({"note": "ok", "n": 2})); validate(&g, &b, None).expect("generic"); - validate(&harbor_ok(&b), &b, Some(CONTRACT_TBENCH_HARBOR)).expect("harbor alias pin"); + validate(&harbor_ok(&b), &b, Some(CONTRACT_HARBOR_TRIALS_LEGACY)) + .expect("harbor alias pin"); validate(&harbor_ok(&b), &b, Some(CONTRACT_HARBOR_TRIALS)).expect("harbor pin"); } @@ -793,7 +798,7 @@ mod tests { MAX_HARBOR_RESULTS_BYTES ); assert_eq!( - results_size_cap(CONTRACT_TBENCH_HARBOR), + results_size_cap(CONTRACT_HARBOR_TRIALS_LEGACY), MAX_HARBOR_RESULTS_BYTES ); assert_eq!(results_size_cap("not-a-contract"), MAX_RESULTS_BYTES); @@ -861,7 +866,7 @@ mod tests { let path = write_contract_sized(&dir, CONTRACT_HARBOR_TRIALS, mid); load_file(&path).expect("harbor mid-size parses"); let alias = scratch_dir("harbor-alias"); - let path = write_contract_sized(&alias, CONTRACT_TBENCH_HARBOR, mid); + let path = write_contract_sized(&alias, CONTRACT_HARBOR_TRIALS_LEGACY, mid); load_file(&path).expect("tbench-harbor alias mid-size parses"); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&alias); @@ -1082,7 +1087,7 @@ mod tests { primary_value: 0.4, claim_holds: true, }; - validate(&value, &b, Some(CONTRACT_TBENCH_HARBOR)).expect("fixture"); + validate(&value, &b, Some(CONTRACT_HARBOR_TRIALS_LEGACY)).expect("fixture"); assert_eq!(value["n_scored"], 10); assert_eq!(value["trials"].as_array().expect("trials").len(), 10); let trials = value["trials"].as_array().expect("trials"); diff --git a/crates/proof-rlm-lifecycle/Cargo.toml b/crates/proof-rlm-lifecycle/Cargo.toml new file mode 100644 index 000000000..606500534 --- /dev/null +++ b/crates/proof-rlm-lifecycle/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "proof-rlm-lifecycle" +description = "The Proof topic lifecycle: a pure transition table (draft → owner review → keys → provisioning → baselining → open ⇄ evaluating → promoting → closed) plus the owner hooks a topic's setup ceremony answers to. No spend, no scoring, no challenge content — it moved out of the RLM engine so that crate keeps room under the repository's per-crate LOC cap while the topic-authoring surface grows." +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +proof-task = { path = "../proof-task" } +serde = { version = "1", features = ["derive"] } +thiserror = "2" + +[dev-dependencies] +serde_json = "1" + +[lints] +workspace = true diff --git a/crates/proof-rlm/src/state.rs b/crates/proof-rlm-lifecycle/src/lib.rs similarity index 96% rename from crates/proof-rlm/src/state.rs rename to crates/proof-rlm-lifecycle/src/lib.rs index fc365d467..0910853fa 100644 --- a/crates/proof-rlm/src/state.rs +++ b/crates/proof-rlm-lifecycle/src/lib.rs @@ -539,7 +539,31 @@ pub fn await_owner_keys( #[cfg(test)] mod tests { use super::*; - use crate::fixtures::topic; + use proof_task::{MetricFamily, TopicStatus}; + + /// A sealed, open custom-family topic with placeholder bindings (the + /// lifecycle only reads `status`, the baseline seal, and the checklist + /// ids, so this is the smallest document those fields need). + fn topic() -> TopicDocument { + let mut doc = TopicDocument { + id: "topic-a".into(), + statement: "Placeholder research problem scored by a topic-minted custom metric." + .into(), + ..TopicDocument::default() + }; + doc.metric.family = MetricFamily::Custom; + doc.metric.custom_id = "placeholder_metric".into(); + doc.constraints.model_pin = Some("vendor/model-placeholder".into()); + doc.constraints.task_slice = Some("slice-placeholder".into()); + doc.checklist = vec![proof_task::ChecklistRule { + id: "rule_a".into(), + text: "placeholder rule a".into(), + }]; + doc.baseline.script_sha256 = "11".repeat(32); + doc.baseline.metrics_commitment = "22".repeat(32); + doc.status = TopicStatus::Open; + doc + } #[test] fn the_happy_path_walks_every_state_in_ship_order() { @@ -682,7 +706,7 @@ mod tests { )); } - /// Without a hook the machine cannot leave owner_presend: nothing is + /// Without a hook the machine cannot leave `owner_presend`: nothing is /// sent on the owner's behalf. #[test] fn owner_presend_needs_an_answer_and_a_decline_returns_to_draft() { diff --git a/crates/proof-rlm-scorer/tests/rlm_e2e.rs b/crates/proof-rlm-scorer/tests/rlm_e2e.rs index 19985901b..6978d6a7c 100644 --- a/crates/proof-rlm-scorer/tests/rlm_e2e.rs +++ b/crates/proof-rlm-scorer/tests/rlm_e2e.rs @@ -1511,6 +1511,212 @@ async fn topic_setup_walks_the_lifecycle_over_the_vm_boundary() { let _ = std::fs::remove_dir_all(&root); } +/// The LIVE Gate 1 shape, refused at the seal: the RLM measured a baseline of +/// **0.0** (a reference run that solved nothing), and sealing it would publish +/// a topic that is open, scorable and unwinnable by anyone — this family +/// scores a relative win, and `challenger >= 0 * (1 + eps)` has no solution. +/// +/// The refusal must not touch the stored measurement and must not reseal +/// anything: the operator fixes the run and seals the new number. +#[tokio::test] +async fn a_degenerate_zero_baseline_is_refused_and_nothing_moves() { + let root = tmp_root("setup-degenerate-bar"); + let key = root.join("owner_key"); + std::fs::write(&key, "not-a-real-secret\n").unwrap(); + let pin = pin_with_topic_key(); + let orchestrator = FakeOrchestrator::new(0.0); + let rlm_store: Arc = Arc::new(MemoryRlmStore::new()); + let mut draft = topic(); + draft.status = TopicStatus::Draft; + draft.holdout_commitment = holdout_commitment(&synthetic_holdout(STRATUM_SIZE, 1)); + draft.baseline.script_sha256 = "11".repeat(32); + draft.baseline.metrics_commitment.clear(); + let registered = [draft.metric.custom_id.clone()]; + let registered: Vec<&str> = registered.iter().map(String::as_str).collect(); + let setup = TopicSetup { + orchestrator: orchestrator.clone(), + store: rlm_store.clone(), + template: pinned_template(), + experiments: proof_rlm::ExperimentPolicy::default(), + owner: Arc::new(StaticOwnerHook(OwnerDecision::Approve)), + keys: Arc::new(FileKeysProbe::new(&key)), + spend_cap_usd: None, + skip_baseline: false, + }; + let out = setup + .run(&draft, &pin, Some(&offer())) + .await + .expect("setup"); + assert!( + out.baseline_primary.expect("measured").abs() < 1e-12, + "the RLM really did measure a zero baseline" + ); + + // The measurement is stored verbatim — setup does not invent or drop it. + let row = rlm_store.baseline(&draft.id).await.unwrap().expect("row"); + assert!(row.primary_value.abs() < 1e-12, "{}", row.primary_value); + + let meas = sealed_measurement(&pin, &draft, 0.0); + let open = signed_open(&draft, &meas); + let err = setup + .mark_sealed(&open, &pin, ®istered, &meas) + .await + .expect_err("a zero bar must not be sealed"); + assert!( + matches!(err, SetupError::DegenerateBar { ref topic_id, .. } if *topic_id == draft.id), + "{err}" + ); + // The message has to name the number and the fix, not just refuse. + let text = err.to_string(); + assert!(text.contains("degenerate bar"), "{text}"); + assert!(text.contains("relative win"), "{text}"); + + // Nothing moved: still baselining, still the draft version, and the + // stored measurement is still exactly what the RLM wrote. + let lc = rlm_store.lifecycle(&draft.id).await.unwrap().unwrap(); + assert_eq!(lc.state, RlmState::Baselining); + let (version, latest) = rlm_store.latest_topic(&draft.id).await.unwrap().unwrap(); + assert_eq!(version, 1, "no new version was stored"); + assert_eq!(latest.status, TopicStatus::Draft); + let stored = rlm_store + .baseline(&draft.id) + .await + .unwrap() + .unwrap() + .primary_value; + assert!( + stored.abs() < 1e-12, + "the refusal must not rewrite or clear the measurement (got {stored})" + ); + + // A real measurement seals normally: the guard narrows nothing else. + let good = sealed_measurement(&pin, &draft, 0.42); + rlm_store + .put_baseline(&proof_rlm_store::BaselineRow { + topic_id: draft.id.clone(), + rules_version: row.rules_version, + primary_value: 0.42, + report: row.report.clone(), + }) + .await + .unwrap(); + let open_good = signed_open(&draft, &good); + assert_eq!( + setup + .mark_sealed(&open_good, &pin, ®istered, &good) + .await + .unwrap(), + RlmState::Open + ); + let _ = std::fs::remove_dir_all(&root); +} + +/// The seal is the **last** chance to notice that the vector moved: the store +/// can be advanced between the measurement landing and the operator sealing +/// it, and `mark_sealed` is what the runtime's own tests drive. +/// +/// Sealing then would publish a bar measured under rules nobody scores with — +/// miners judged by the newer checklist against an older number — so the +/// wiring must call the staleness check, not just define it. This asserts on +/// `mark_sealed` itself for that reason. +#[tokio::test] +async fn mark_sealed_refuses_a_measurement_taken_under_superseded_rules() { + let root = tmp_root("setup-stale-baseline"); + let key = root.join("owner_key"); + std::fs::write(&key, "not-a-real-secret\n").unwrap(); + let pin = pin_with_topic_key(); + let orchestrator = FakeOrchestrator::new(0.61); + let rlm_store: Arc = Arc::new(MemoryRlmStore::new()); + let mut draft = topic(); + draft.status = TopicStatus::Draft; + draft.holdout_commitment = holdout_commitment(&synthetic_holdout(STRATUM_SIZE, 1)); + draft.baseline.script_sha256 = "11".repeat(32); + draft.baseline.metrics_commitment.clear(); + let registered = [draft.metric.custom_id.clone()]; + let registered: Vec<&str> = registered.iter().map(String::as_str).collect(); + let setup = TopicSetup { + orchestrator: orchestrator.clone(), + store: rlm_store.clone(), + template: pinned_template(), + experiments: proof_rlm::ExperimentPolicy::default(), + owner: Arc::new(StaticOwnerHook(OwnerDecision::Approve)), + keys: Arc::new(FileKeysProbe::new(&key)), + spend_cap_usd: None, + skip_baseline: false, + }; + let out = setup + .run(&draft, &pin, Some(&offer())) + .await + .expect("setup"); + let measured = out.rules_version; + let primary = out.baseline_primary.expect("measured"); + assert_eq!(measured, 1, "the baseline was measured under version 1"); + + // A later RLM write advances the vector after the measurement. + rlm_store + .put_rules(&proof_rlm::RuleSet { + topic_id: draft.id.clone(), + version: 2, + source: proof_rlm::RuleSource::Rlm, + rules: vec![proof_task::ChecklistRule { + id: "rlm_rule_v2".into(), + text: "a later rule".into(), + }], + }) + .await + .expect("v2"); + + let meas = sealed_measurement(&pin, &draft, primary); + let open = signed_open(&draft, &meas); + let err = setup + .mark_sealed(&open, &pin, ®istered, &meas) + .await + .expect_err("a stale measurement must not seal"); + assert!( + matches!( + err, + SetupError::BaselineStale { + measured: 1, + in_force: Some(2), + .. + } + ), + "{err}" + ); + assert!( + err.to_string().contains("version 1") && err.to_string().contains("version 2"), + "{err}" + ); + + // Nothing moved: still baselining, still the draft version. + let lc = rlm_store.lifecycle(&draft.id).await.unwrap().unwrap(); + assert_eq!(lc.state, RlmState::Baselining); + let (version, latest) = rlm_store.latest_topic(&draft.id).await.unwrap().unwrap(); + assert_eq!(version, 1, "no new version was stored"); + assert_eq!(latest.status, TopicStatus::Draft); + + // Re-measuring under the version in force seals normally. + rlm_store + .put_baseline(&proof_rlm_store::BaselineRow { + topic_id: draft.id.clone(), + rules_version: 2, + primary_value: primary, + report: rlm_store.baseline(&draft.id).await.unwrap().unwrap().report, + }) + .await + .unwrap(); + let refreshed = sealed_measurement(&pin, &draft, primary); + let open_refreshed = signed_open(&draft, &refreshed); + assert_eq!( + setup + .mark_sealed(&open_refreshed, &pin, ®istered, &refreshed) + .await + .unwrap(), + RlmState::Open + ); + let _ = std::fs::remove_dir_all(&root); +} + /// A topic whose signed params select an in-guest runner measures its /// baseline in **one dedicated experiment VM** — created for the `Baseline` /// job, sized under the lock ceilings, carrying the pinned pack, destroyed @@ -1761,3 +1967,209 @@ async fn measuring_a_baseline_without_an_offer_is_refused() { .expect("a skipping run needs no offer"); let _ = std::fs::remove_dir_all(&root); } + +/// A **second** authoring run is handed the set the first one wrote. +/// +/// Greptile's P1 on the whole-set change: the job carried +/// `VmJob::ProposeRules.current` but the driver always sent `None`, so an +/// adaptor could not *retain* the parts it was not changing — a re-authoring +/// run was a rewrite from nothing, and the install would apply that lossy set +/// (a migration the topic still needs would vanish). +/// +/// The fix has two halves and this test pins both: the driver reads the set +/// back from the store (`current_authoring`) and the job carries it. +#[tokio::test] +async fn a_re_authoring_run_is_handed_the_set_the_first_one_wrote() { + let root = tmp_root("reauthor"); + let key = root.join("owner_key"); + std::fs::write(&key, "not-a-real-secret\n").unwrap(); + let orchestrator = FakeOrchestrator::new(0.42); + let rlm_store: Arc = Arc::new(MemoryRlmStore::new()); + let mut draft = topic(); + draft.status = TopicStatus::Draft; + draft.holdout_commitment = holdout_commitment(&synthetic_holdout(STRATUM_SIZE, 1)); + draft.baseline.script_sha256 = "11".repeat(32); + draft.baseline.metrics_commitment.clear(); + let setup = TopicSetup { + orchestrator: orchestrator.clone(), + store: rlm_store.clone(), + template: pinned_template(), + experiments: proof_rlm::ExperimentPolicy::default(), + owner: Arc::new(StaticOwnerHook(OwnerDecision::Approve)), + keys: Arc::new(FileKeysProbe::new(&key)), + spend_cap_usd: None, + skip_baseline: true, + }; + + // First run: the RLM authors a set. Nothing is stored yet, so the job + // carries no previous set. + setup + .run(&draft, &pin_with_topic_key(), None) + .await + .expect("first run"); + let first = orchestrator + .jobs() + .into_iter() + .find_map(|j| match j { + VmJob::ProposeRules { current, .. } => Some(current), + _ => None, + }) + .expect("a propose_rules job"); + assert!( + first.is_none(), + "the first run has no previous set to carry" + ); + // …and the set it wrote is now the store's newest. + let (version, stored) = rlm_store + .authoring(&draft.id) + .await + .expect("read") + .expect("the set was persisted"); + assert_eq!(version, 1); + + // Second run: same topic, and the job must carry **that** set so the + // adaptor can keep what it means to keep. + setup + .run(&draft, &pin_with_topic_key(), None) + .await + .expect("second run"); + let carried = orchestrator + .jobs() + .into_iter() + .filter_map(|j| match j { + VmJob::ProposeRules { current, .. } => Some(current), + _ => None, + }) + .nth(1) + .expect("a second propose_rules job") + .expect("the second run carries the previous set"); + assert_eq!( + *carried, stored, + "the set handed to the RLM is the one it wrote last time" + ); + assert_eq!(carried.topic_id, draft.id); + assert!( + !carried.migrations.is_empty() && !carried.apis.is_empty(), + "the carried set is complete, not just the rules: {carried:?}" + ); + assert_eq!( + rlm_store + .authoring(&draft.id) + .await + .expect("read") + .expect("v2") + .0, + 2, + "the store advanced, so a third run would be handed this one" + ); + let _ = std::fs::remove_dir_all(&root); +} + +/// A **fragment** is not persisted as the set in force. +/// +/// A rules-only answer is a fragment, and storing it as "the set the RLM +/// authored" would hand the next run a set that was never authored — the +/// re-authoring path would then treat a partial answer as the baseline for +/// retention. The rules are still stored (with honest provenance); the *set* +/// is not. +#[tokio::test] +async fn a_rules_only_answer_is_not_persisted_as_the_authored_set() { + let root = tmp_root("fragment"); + let key = root.join("owner_key"); + std::fs::write(&key, "not-a-real-secret\n").unwrap(); + let orchestrator = FakeOrchestrator::new(0.42); + orchestrator.set_rules_only(true); + let rlm_store: Arc = Arc::new(MemoryRlmStore::new()); + let mut draft = topic(); + draft.status = TopicStatus::Draft; + draft.holdout_commitment = holdout_commitment(&synthetic_holdout(STRATUM_SIZE, 1)); + draft.baseline.script_sha256 = "11".repeat(32); + draft.baseline.metrics_commitment.clear(); + let setup = TopicSetup { + orchestrator: orchestrator.clone(), + store: rlm_store.clone(), + template: pinned_template(), + experiments: proof_rlm::ExperimentPolicy::default(), + owner: Arc::new(StaticOwnerHook(OwnerDecision::Approve)), + keys: Arc::new(FileKeysProbe::new(&key)), + spend_cap_usd: None, + skip_baseline: true, + }; + + let err = setup + .run(&draft, &pin_with_topic_key(), None) + .await + .expect_err("a fragment is not a set"); + assert!( + matches!(err, SetupError::IncompleteAuthoring { .. }), + "{err}" + ); + assert!( + rlm_store + .authoring(&draft.id) + .await + .expect("read") + .is_none(), + "a fragment must not become 'the set in force'" + ); + // The rules it did write are stored, with honest provenance. + let rules = rlm_store + .current_rules(&draft.id) + .await + .expect("read") + .expect("rules"); + assert_eq!(rules.source, proof_rlm::RuleSource::Rlm); + let _ = std::fs::remove_dir_all(&root); +} + +/// A topic's set survives a **restart**: the driver reads it from the store, +/// not from memory, so a re-authoring run from a fresh process is handed the +/// same set. +#[tokio::test] +async fn the_previous_set_survives_a_new_driver() { + let root = tmp_root("restart"); + let key = root.join("owner_key"); + std::fs::write(&key, "not-a-real-secret\n").unwrap(); + let orchestrator = FakeOrchestrator::new(0.42); + let rlm_store: Arc = Arc::new(MemoryRlmStore::new()); + let mut draft = topic(); + draft.status = TopicStatus::Draft; + draft.holdout_commitment = holdout_commitment(&synthetic_holdout(STRATUM_SIZE, 1)); + draft.baseline.script_sha256 = "11".repeat(32); + draft.baseline.metrics_commitment.clear(); + let build = || TopicSetup { + orchestrator: orchestrator.clone(), + store: rlm_store.clone(), + template: pinned_template(), + experiments: proof_rlm::ExperimentPolicy::default(), + owner: Arc::new(StaticOwnerHook(OwnerDecision::Approve)), + keys: Arc::new(FileKeysProbe::new(&key)), + spend_cap_usd: None, + skip_baseline: true, + }; + build() + .run(&draft, &pin_with_topic_key(), None) + .await + .expect("first run"); + + // A *different* driver over the same store: no in-memory carry-over. + build() + .run(&draft, &pin_with_topic_key(), None) + .await + .expect("second run"); + let carried = orchestrator + .jobs() + .into_iter() + .filter_map(|j| match j { + VmJob::ProposeRules { current, .. } => Some(current), + _ => None, + }) + .nth(1) + .expect("a second job") + .expect("carried"); + assert!( + !carried.migrations.is_empty(), + "a fresh driver still gets the set from the store" + ); + let _ = std::fs::remove_dir_all(&root); +} diff --git a/crates/proof-rlm-store/Cargo.toml b/crates/proof-rlm-store/Cargo.toml index 838188e54..a17605594 100644 --- a/crates/proof-rlm-store/Cargo.toml +++ b/crates/proof-rlm-store/Cargo.toml @@ -13,6 +13,7 @@ async-trait = "0.1" db = { path = "../db" } proof-rlm = { path = "../proof-rlm" } proof-task = { path = "../proof-task" } +proof-topic-authoring = { path = "../proof-topic-authoring" } serde = { version = "1", features = ["derive"] } serde_json = "1" sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio", "postgres", "json"] } @@ -20,6 +21,7 @@ thiserror = "2" tracing = "0.1" [dev-dependencies] +serde_json = "1" db = { path = "../db", features = ["testing"] } proof-rlm = { path = "../proof-rlm", features = ["test-fixtures"] } tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "time"] } diff --git a/crates/proof-rlm-store/src/lib.rs b/crates/proof-rlm-store/src/lib.rs index a81dfebe8..3b68d5998 100644 --- a/crates/proof-rlm-store/src/lib.rs +++ b/crates/proof-rlm-store/src/lib.rs @@ -87,11 +87,10 @@ impl ChecklistRow { /// One temporary compatibility alias for a topic slug. /// -/// Owner default: the first topic's slug is `tb4` with `tbench` as a -/// **temporary** alias, so existing miner links keep resolving while the -/// canonical slug settles. A row carries the mapping and nothing else — no -/// name, no pins, no status — so it cannot drift from the topic it names. -/// Retiring the alias is deleting the row. +/// An alias is a **lookup convenience**: a row carries the mapping and nothing +/// else — no name, no pins, no status — so it cannot drift from the topic it +/// names. Retiring the alias is deleting the row. There is no owner default: +/// which aliases exist is what an operator's installs declared. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct TopicAliasRow { /// The alias slug that resolves to `topic_id`. @@ -246,6 +245,43 @@ pub trait RlmStore: Send + Sync { /// One rule version. async fn rules_at(&self, topic_id: &str, version: u32) -> Result, StoreError>; + /// Whether the topic's **newest** rule version was authored by its RLM + /// ([`proof_rlm::RuleSource::Rlm`]). + /// + /// This is the provenance read, not a rules read: it answers "did the + /// topic's RLM author the vector in force", which is what separates a + /// topic that set itself up from one whose behavior is still the + /// operator's signed document. `Ok(false)` covers three states that are + /// deliberately not distinguished here — no rule row at all, a version + /// seeded from the signed document (`topic_document`), and an operator + /// edit (`operator`) — because every one of them means the RLM has not + /// authored the current vector, and the caller's answer is the same. + /// + /// **Fail-closed at the call site:** an `Err` is an unreadable store, not + /// a `false` that a caller could mistake for "not RLM-authored" (or, if + /// inverted, for "RLM-authored"). + /// + /// # Errors + /// + /// [`StoreError`] when the store cannot be read. + async fn rlm_authored_rules(&self, topic_id: &str) -> Result; + + /// The newest rule version's source, or `None` when the topic has no rule + /// row at all. + /// + /// The operator-facing half of [`Self::rlm_authored_rules`]: a caller that + /// has to explain *why* a topic is not RLM-authored needs the actual + /// provenance, not a boolean. Kept as its own read rather than a richer + /// return type so the fail-closed boolean stays trivial to audit. + /// + /// # Errors + /// + /// [`StoreError`] when the store cannot be read. + async fn current_rules_source( + &self, + topic_id: &str, + ) -> Result, StoreError>; + /// Persist a submission's checklist. /// /// A row that already exists for `submission_digest` is replaced @@ -283,6 +319,39 @@ pub trait RlmStore: Send + Sync { async fn best(&self, topic_id: &str) -> Result, StoreError>; /// Promotion history, oldest first. async fn promotions(&self, topic_id: &str) -> Result, StoreError>; + + /// Append the whole set a topic's RLM authored, together with the rule + /// version it landed. Returns the set's version. + /// + /// A topic's behavior is five parts and its RLM authors all of them in one + /// job; the rules have always been versioned in `proof_rule_version`, and + /// this is where the **whole set** is kept. It exists so a re-authoring + /// run can be handed the set it wrote last time + /// (`VmJob::ProposeRules.current`): an adaptor that cannot read its + /// previous set cannot *retain* the parts it is not changing, so a second + /// run would silently drop migrations the topic still needs. + /// + /// **One write, not two.** The rules and the set are two halves of one + /// fact — "this topic's RLM authored *this* at rule version *N*" — and a + /// store that wrote them separately could fail between them: newer rules + /// with the previous set, so a retry would be handed a set whose rules are + /// not the ones in force. Both land in one transaction, so the pair is + /// either wholly there or wholly absent. + /// + /// Append-only: a re-authoring appends, and "the set in force" is the + /// newest row. The version advances by one, like rule versions do, so a + /// gap or a repeat is a `VersionGap` rather than a silent overwrite. + async fn put_authoring( + &self, + topic_id: &str, + rules: &RuleSet, + set: &proof_topic_authoring::TopicAuthoring, + ) -> Result; + /// The set a topic's RLM authored last, if any. + async fn authoring( + &self, + topic_id: &str, + ) -> Result, StoreError>; } /// Numeric id from `pf_` + 16 hex. `None` if the string is not a store row id. diff --git a/crates/proof-rlm-store/src/memory.rs b/crates/proof-rlm-store/src/memory.rs index d47678069..59df98991 100644 --- a/crates/proof-rlm-store/src/memory.rs +++ b/crates/proof-rlm-store/src/memory.rs @@ -24,6 +24,8 @@ struct Inner { artefacts: BTreeMap>, promotions: BTreeMap>, aliases: BTreeMap, + /// The whole set each authoring run produced, oldest first. + authoring: BTreeMap>, } /// In-memory store. @@ -152,6 +154,21 @@ impl RlmStore for MemoryRlmStore { .and_then(|v| v.iter().find(|r| r.version == version).cloned())) } + async fn current_rules_source( + &self, + topic_id: &str, + ) -> Result, StoreError> { + Ok(self + .lock()? + .rules + .get(topic_id) + .and_then(|v| v.last().map(|r| r.source))) + } + + async fn rlm_authored_rules(&self, topic_id: &str) -> Result { + Ok(self.current_rules_source(topic_id).await? == Some(proof_rlm::RuleSource::Rlm)) + } + async fn put_checklist(&self, row: &ChecklistRow) -> Result<(), StoreError> { // Same digest may be re-inspected (miner resubmit). Overwrite. self.lock()? @@ -262,4 +279,46 @@ impl RlmStore for MemoryRlmStore { .cloned() .unwrap_or_default()) } + + async fn put_authoring( + &self, + topic_id: &str, + rules: &RuleSet, + set: &proof_topic_authoring::TopicAuthoring, + ) -> Result { + // One lock, so the pair lands together: a caller that fails between + // two separate writes would leave newer rules with the previous set. + let mut g = self.lock()?; + // The rules first, with the same check `put_rules` runs: a version + // that does not advance is a refusal, and the set is not written. + let current = g + .rules + .get(topic_id) + .map(|v| v.last().map_or(0, |r| r.version)); + check_rules(rules, current)?; + let want = u32::try_from(g.authoring.get(topic_id).map_or(0, std::vec::Vec::len)) + .map_err(|_| StoreError::VersionGap("authoring"))? + .saturating_add(1); + let existing = g.authoring.entry(topic_id.to_owned()).or_default(); + if existing.iter().any(|(v, _)| *v == want) { + return Err(StoreError::VersionGap("authoring")); + } + existing.push((want, set.clone())); + g.rules + .entry(topic_id.to_owned()) + .or_default() + .push(rules.clone()); + Ok(want) + } + + async fn authoring( + &self, + topic_id: &str, + ) -> Result, StoreError> { + Ok(self + .lock()? + .authoring + .get(topic_id) + .and_then(|v| v.last().cloned())) + } } diff --git a/crates/proof-rlm-store/src/pg.rs b/crates/proof-rlm-store/src/pg.rs index a3cc56cfe..5268ae0dc 100644 --- a/crates/proof-rlm-store/src/pg.rs +++ b/crates/proof-rlm-store/src/pg.rs @@ -355,6 +355,30 @@ impl RlmStore for PgRlmStore { row.map(RuleRow::into_set).transpose() } + /// The newest rule version's `source`, read as a bare column. + /// + /// Deliberately its own query rather than a `current_rules` call: the + /// provenance question does not need the rule bodies, and reading the + /// source column directly keeps the answer independent of whether a row's + /// rules still deserialize. `None` means no rule row exists. + async fn current_rules_source( + &self, + topic_id: &str, + ) -> Result, StoreError> { + let row: Option<(String,)> = sqlx::query_as( + "SELECT source FROM proof_rule_version \ + WHERE topic_id = $1 ORDER BY version DESC LIMIT 1", + ) + .bind(topic_id) + .fetch_optional(&self.pool) + .await?; + row.map(|(source,)| parse_source(&source)).transpose() + } + + async fn rlm_authored_rules(&self, topic_id: &str) -> Result { + Ok(self.current_rules_source(topic_id).await? == Some(proof_rlm::RuleSource::Rlm)) + } + async fn put_checklist(&self, row: &ChecklistRow) -> Result<(), StoreError> { sqlx::query( "INSERT INTO proof_checklist (submission_digest, topic_id, rules_version, green, failed_ids, document) \ @@ -580,4 +604,106 @@ impl RlmStore for PgRlmStore { .await?; Ok(rows.into_iter().map(PromotionRow::from).collect()) } + + /// Append the set **and** the rule version it landed, in one transaction. + /// + /// The rules and the set are two halves of one fact — "this topic's RLM + /// authored *this* at rule version *N*" — and writing them separately + /// would let a failure land between them: newer rules with the previous + /// set, so a retry would be handed a set whose rules are not the ones in + /// force. Both go in one transaction, so the pair is wholly there or + /// wholly absent. + /// + /// The version has to advance by exactly one, and "read the newest, add + /// one, insert" is not atomic: two authoring runs for the same topic each + /// read the same newest and both try to insert the same version. The + /// advisory lock makes the read-then-write one step, so the second run + /// reports the gap the way the contract says it will. + async fn put_authoring( + &self, + topic_id: &str, + rules: &RuleSet, + set: &proof_topic_authoring::TopicAuthoring, + ) -> Result { + set.validate(topic_id) + .map_err(|e| StoreError::Malformed(e.to_string()))?; + let digest = format!("sha256:{}", set.digest()); + let mut tx = self.pool.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") + .bind(format!("proof_topic_authoring:{topic_id}")) + .execute(&mut *tx) + .await?; + let newest: Option = sqlx::query_scalar( + "SELECT version FROM proof_topic_authoring \ + WHERE topic_id = $1 ORDER BY version DESC LIMIT 1", + ) + .bind(topic_id) + .fetch_optional(&mut *tx) + .await?; + let want = newest + .map(|v| to_u32(v).map(|v| v.saturating_add(1))) + .transpose()? + .unwrap_or(1); + let document = + serde_json::to_value(set).map_err(|e| StoreError::Malformed(e.to_string()))?; + let version = i32::try_from(want).map_err(malformed)?; + sqlx::query( + "INSERT INTO proof_topic_authoring (topic_id, version, document, digest) \ + VALUES ($1, $2, $3, $4)", + ) + .bind(topic_id) + .bind(version) + .bind(document) + .bind(digest) + .execute(&mut *tx) + .await?; + // The rules, with the same version check `put_rules` runs: a vector + // that does not advance is a refusal, and the transaction rolls back + // so the set is not written either. + let current: Option = sqlx::query_scalar( + "SELECT version FROM proof_rule_version \ + WHERE topic_id = $1 ORDER BY version DESC LIMIT 1", + ) + .bind(topic_id) + .fetch_optional(&mut *tx) + .await?; + check_rules(rules, current.map(to_u32).transpose()?)?; + let rules_json = serde_json::to_value(&rules.rules).map_err(malformed)?; + sqlx::query( + "INSERT INTO proof_rule_version (topic_id, version, source, rules, digest) \ + VALUES ($1, $2, $3, $4, $5)", + ) + .bind(topic_id) + .bind(i32::try_from(rules.version).map_err(malformed)?) + .bind(source_str(rules.source)) + .bind(rules_json) + // `digest` is NOT NULL and CHECK'd as 64 hex: the canonical digest of + // the vector, exactly as `put_rules` writes it. Omitting it made the + // database reject every complete authoring, which is what Greptile + // caught. + .bind(rules.digest()) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(want) + } + + async fn authoring( + &self, + topic_id: &str, + ) -> Result, StoreError> { + let row: Option<(i32, Value)> = sqlx::query_as( + "SELECT version, document FROM proof_topic_authoring \ + WHERE topic_id = $1 ORDER BY version DESC LIMIT 1", + ) + .bind(topic_id) + .fetch_optional(&self.pool) + .await?; + let Some((version, document)) = row else { + return Ok(None); + }; + let set: proof_topic_authoring::TopicAuthoring = + serde_json::from_value(document).map_err(malformed)?; + Ok(Some((to_u32(version)?, set))) + } } diff --git a/crates/proof-rlm-store/tests/store_contract.rs b/crates/proof-rlm-store/tests/store_contract.rs index 10ca1ccf8..afa731045 100644 --- a/crates/proof-rlm-store/tests/store_contract.rs +++ b/crates/proof-rlm-store/tests/store_contract.rs @@ -142,6 +142,90 @@ async fn contract(store: &dyn RlmStore) { assert_eq!(store.rules_at(&t.id, 1).await.unwrap().unwrap(), v1); assert!(store.rules_at(&t.id, 3).await.unwrap().is_none()); + // The whole authored set, written **with** the rule version it landed: + // one fact, two halves. This runs against both stores, which is the point + // — the Postgres half once omitted the rule digest, and only a real + // database could say so. + let authored = |api: &str, version: u32| proof_topic_authoring::TopicAuthoring { + schema_version: proof_topic_authoring::AUTHORING_SCHEMA, + topic_id: t.id.clone(), + rules: vec![ChecklistRule { + id: format!("rlm-{version}"), + text: "the rlm's vector".into(), + }], + migrations: vec![proof_topic_authoring::AuthoredMigration { + name: "0001_scratch".into(), + sql: format!("CREATE TABLE {}_scratch (id TEXT)", t.id.replace('-', "_")), + }], + apis: vec![proof_topic_authoring::AuthoredApi { + path: api.into(), + method: "GET".into(), + summary: String::new(), + }], + submission_format: serde_json::json!({"kind": "tar", "max_bytes": 5_242_880}), + pin_policy: proof_topic_authoring::PinPolicy::none(), + }; + let v3 = v2 + .next( + RuleSource::Rlm, + vec![ChecklistRule { + id: "rlm-3".into(), + text: "the rlm's third vector".into(), + }], + ) + .unwrap(); + let set_v1 = authored("one", 3); + assert_eq!( + store.put_authoring(&t.id, &v3, &set_v1).await.unwrap(), + 1, + "the set's version is its own, starting at 1" + ); + // Both halves are readable, and the rules are the ones that landed with + // the set — the digest is checked by the database on the way in, so a row + // that got there is a row that verified. + let (set_version, stored) = store.authoring(&t.id).await.unwrap().unwrap(); + assert_eq!(set_version, 1); + assert_eq!(stored, set_v1); + assert_eq!(store.current_rules(&t.id).await.unwrap().unwrap(), v3); + assert_eq!( + store.current_rules_source(&t.id).await.unwrap(), + Some(RuleSource::Rlm), + "the paired rule version is readable by provenance too" + ); + + // A rule version that does not advance is refused **and the set is not + // written**: the pair cannot drift. + assert!(matches!( + store.put_authoring(&t.id, &v3, &authored("two", 3)).await, + Err(StoreError::VersionGap("rules")) + )); + let (set_version, stored) = store.authoring(&t.id).await.unwrap().unwrap(); + assert_eq!(set_version, 1, "a refused write appends no set"); + assert_eq!(stored.apis[0].path, "one"); + + // The next valid run advances both. + let v4 = v3 + .next( + RuleSource::Rlm, + vec![ChecklistRule { + id: "rlm-4".into(), + text: "the rlm's fourth vector".into(), + }], + ) + .unwrap(); + assert_eq!( + store + .put_authoring(&t.id, &v4, &authored("two", 4)) + .await + .unwrap(), + 2 + ); + assert_eq!( + store.authoring(&t.id).await.unwrap().unwrap().1.apis[0].path, + "two" + ); + assert_eq!(store.current_rules(&t.id).await.unwrap().unwrap(), v4); + // Checklists are keyed by the frozen digest. let digest = "ab".repeat(32); let mut c = green(&v1, &digest); diff --git a/crates/proof-rlm/Cargo.toml b/crates/proof-rlm/Cargo.toml index 985a6ceed..136ba132d 100644 --- a/crates/proof-rlm/Cargo.toml +++ b/crates/proof-rlm/Cargo.toml @@ -19,8 +19,10 @@ hex = "0.4" proof-canon = { path = "../proof-canon" } proof-experiment = { path = "../proof-experiment" } proof-results = { path = "../proof-results" } +proof-rlm-lifecycle = { path = "../proof-rlm-lifecycle" } proof-score = { path = "../proof-score" } proof-task = { path = "../proof-task" } +proof-topic-authoring = { path = "../proof-topic-authoring" } serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.10" diff --git a/crates/proof-rlm/src/fixtures_tests.rs b/crates/proof-rlm/src/fixtures_tests.rs index 9aa494a11..48b14ec16 100644 --- a/crates/proof-rlm/src/fixtures_tests.rs +++ b/crates/proof-rlm/src/fixtures_tests.rs @@ -36,6 +36,7 @@ use crate::vm::{ RetainPolicy, TopicVmOrchestrator, TopicVmSpec, VmError, VmHandle, VmJob, VmJobOutput, VmTemplate, }; +use proof_topic_authoring::TopicAuthoring; /// Pin with a topic key and a judge model (no digest unless asked). pub fn pin() -> ProofPin { @@ -215,6 +216,13 @@ pub struct FakeOrchestrator { job_delay: Mutex, teardowns: Mutex>, proposed: Mutex>, + /// Whether `ProposeRules` answers with the whole authored set (the shape a + /// current adaptor writes) or with a bare rule vector (an adaptor baked + /// before the set existed). The latter is a *fragment*, which the driver + /// refuses to treat as authorship. + authored_complete: AtomicBool, + /// The set the fake RLM authors, when it authors one. + authored: Mutex>, } impl FakeOrchestrator { @@ -237,9 +245,25 @@ impl FakeOrchestrator { id: "rlm_rule".into(), text: "a rule the fake rlm wrote".into(), }]), + authored_complete: AtomicBool::new(true), + authored: Mutex::new(None), }) } + /// Answer `ProposeRules` with a bare rule vector instead of the whole + /// authored set — the shape an adaptor baked before the set existed + /// writes. The driver records the rules and refuses to treat the topic as + /// set up, naming the parts that have no author. + pub fn set_rules_only(&self, v: bool) { + self.authored_complete.store(!v, Ordering::SeqCst); + } + + /// The set the fake RLM authors (default: one built from the topic the + /// job carries, with placeholder parts). + pub fn set_authored(&self, set: Option) { + *self.authored.lock().unwrap() = set; + } + /// How long each job takes, so a test can make concurrent jobs overlap. pub fn set_job_delay(&self, d: std::time::Duration) { *self.job_delay.lock().unwrap() = d; @@ -402,7 +426,16 @@ impl TopicVmOrchestrator for FakeOrchestrator { return Err(VmError::Backend("guest: injected run failure".into())); } Ok(match job { - VmJob::ProposeRules { .. } => VmJobOutput::Rules(self.proposed.lock().unwrap().clone()), + VmJob::ProposeRules { ref topic, .. } => { + if self.authored_complete.load(Ordering::SeqCst) { + let set = self.authored.lock().unwrap().clone().unwrap_or_else(|| { + authored_set(topic, self.proposed.lock().unwrap().clone()) + }); + VmJobOutput::Authored(Box::new(set)) + } else { + VmJobOutput::Rules(self.proposed.lock().unwrap().clone()) + } + } VmJob::Baseline { request } => VmJobOutput::Baseline(self.report(&request)), VmJob::Inspect { request, rules } => { let red = self.red.lock().unwrap().clone(); @@ -464,3 +497,37 @@ pub fn experiment_request(vcpus: Option) -> CustomRunRequest { } req } + +/// The whole set a fake RLM authors for `topic`: placeholder parts, every one +/// present, shaped exactly like the document it was handed. +/// +/// The **rules** come from `rules`, so a test that sets +/// [`FakeOrchestrator::set_proposed`] still controls the vector the RLM +/// authored — which is what makes "the RLM's own rules, not the document's" +/// observable. The other parts are placeholders: a set is only meaningful as +/// a whole, and a test that cares about one of them sets it explicitly with +/// [`FakeOrchestrator::set_authored`]. +pub fn authored_set(topic: &TopicDocument, rules: Vec) -> TopicAuthoring { + use proof_topic_authoring::{AuthoredApi, AuthoredMigration, PinPolicy}; + TopicAuthoring { + schema_version: proof_topic_authoring::AUTHORING_SCHEMA, + topic_id: topic.id.clone(), + rules, + migrations: vec![AuthoredMigration { + name: "0001_scratch".into(), + sql: format!( + "CREATE TABLE {}_scratch (id TEXT)", + topic.id.replace('-', "_") + ), + }], + apis: vec![AuthoredApi { + path: "status".into(), + method: "GET".into(), + summary: "topic status".into(), + }], + submission_format: serde_json::json!({"kind": "tar", "max_bytes": 5_242_880}), + // A policy that tightens nothing is still a policy: the part is + // present, and it is the RLM's answer rather than a gap. + pin_policy: PinPolicy::none(), + } +} diff --git a/crates/proof-rlm/src/lib.rs b/crates/proof-rlm/src/lib.rs index 1981be069..df624448f 100644 --- a/crates/proof-rlm/src/lib.rs +++ b/crates/proof-rlm/src/lib.rs @@ -47,7 +47,6 @@ mod gate; mod rules; mod runner; -mod state; mod vm; /// Shared test fixtures (fake orchestrator, canned report, placeholder topic). @@ -66,7 +65,11 @@ pub use runner::{ CustomRunner, InspectOutcome, JudgeRef, LogFile, ReportError, RunOutcome, RunnerError, RunnerRegistry, SandboxPolicy, RUN_REPORT_SCHEMA, RUN_REQUEST_SCHEMA, STAGED_ARTEFACT_SCHEME, }; -pub use state::{ +// The lifecycle moved to `proof-rlm-lifecycle` (the transition table and the +// owner hooks, unchanged) so this crate keeps room under the per-crate LOC cap +// while the topic-authoring surface grows. Re-exported here so every existing +// `proof_rlm::RlmState` path keeps working. +pub use proof_rlm_lifecycle::{ await_owner_keys, owner_presend, transition, FileKeysProbe, HookError, Lifecycle, NoOwnerHook, OwnerDecision, OwnerHook, OwnerKeysProbe, OwnerPrompt, RlmEvent, RlmState, StateError, StaticOwnerHook, Transition, OWNER_INFERENCE_KEY_FILE_ENV, @@ -88,6 +91,14 @@ pub use proof_canon::{ is_env_name, MinerEnv, MinerEnvError, PARAM_INJECT_MINER_ENV_SISTER, PARAM_MINER_BYOK, PARAM_MINER_ENV_ALLOWLIST, }; +// What a topic's RLM authors (rules, migrations, apis, submission_format, +// pin_policy) and the shape checks every part is held to. Re-exported so the +// guest, the setup driver, and the install all name the same types. +pub use proof_topic_authoring::{ + from_json as authoring_from_json, AuthoredApi, AuthoredMigration, AuthoringError, PinPolicy, + TopicAuthoring, AUTHORING_SCHEMA, MAX_APIS as MAX_AUTHORED_APIS, + MAX_MIGRATIONS as MAX_AUTHORED_MIGRATIONS, PARTS as AUTHORING_PARTS, +}; #[cfg(test)] mod tests { @@ -101,7 +112,6 @@ mod tests { include_str!("gate.rs"), include_str!("rules.rs"), include_str!("runner.rs"), - include_str!("state.rs"), include_str!("vm.rs"), include_str!("lib.rs"), ]; diff --git a/crates/proof-rlm/src/vm.rs b/crates/proof-rlm/src/vm.rs index ef915ca3a..a9ebb0375 100644 --- a/crates/proof-rlm/src/vm.rs +++ b/crates/proof-rlm/src/vm.rs @@ -37,6 +37,7 @@ use proof_experiment::{ ExperimentBinding, ExperimentError, ExperimentPolicy, ExperimentSpec, RunPolicy, }; use proof_task::{ChecklistRule, TopicDocument}; +use proof_topic_authoring::TopicAuthoring; use serde::{Deserialize, Serialize}; use crate::gate::SpendToken; @@ -201,12 +202,23 @@ pub struct VmHandle { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "job", rename_all = "snake_case")] pub enum VmJob { - /// Let the RLM read the signed topic and write (or rewrite) its rules. + /// Let the RLM read the signed topic and author its whole set — rules, + /// migrations, APIs, submission format, and pin policy — in one job. + /// + /// One job rather than five: the parts are one document (the topic's + /// behavior), they are validated together, and they land in one journal + /// entry. A separate job per part would let a topic be half-authored, + /// which is a state nothing downstream could interpret. ProposeRules { /// The signed topic (public document). topic: Box, /// Rule version to supersede, if any. current_version: Option, + /// The parts the RLM already authored, when the host has them: an + /// RLM asked to re-author keeps what it wrote for the parts it does + /// not change. Absent on a first authoring run. + #[serde(default, skip_serializing_if = "Option::is_none")] + current: Option>, }, /// Run the baseline artefact so the operator can seal `custom_value`. Baseline { @@ -313,7 +325,24 @@ impl VmJob { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "output", content = "body", rename_all = "snake_case")] pub enum VmJobOutput { - /// Rules the RLM proposes; the store versions them. + /// The topic's own RLM authored its behavior: the rule vector, the SQL + /// migrations, the routes it exposes, its submission format, and the pin + /// policy it tightens. One document, every part provenance-stamped by the + /// store and the install journal. + /// + /// Boxed because a set is a whole document: unboxed, every type that + /// carries an output (`RlmToHost::Done`, the agent's job record) would be + /// as large as the largest set, for a payload most jobs do not have. The + /// box is invisible on the wire — `Box` serialises as `T`. + Authored(Box), + /// Rules the RLM proposes, and nothing else: the shape an adaptor baked + /// before the set existed writes (`rules.json`). + /// + /// A **fragment**, not authorship. The host records the rules with honest + /// `rlm` provenance and refuses to treat the topic as set up — naming the + /// parts that have no author — rather than widening this into a whole set + /// from the operator's bundle. A topic that must open ships an adaptor + /// whose `propose_rules` writes `authoring.json`. Rules(Vec), /// Baseline measurement. Baseline(CustomRunReport), @@ -890,6 +919,7 @@ mod tests { VmJob::ProposeRules { topic: Box::new(crate::fixtures::topic()), current_version: None, + current: None, }, VmJob::Baseline { request: req.clone(), diff --git a/crates/proof-score/src/lib.rs b/crates/proof-score/src/lib.rs index a15aedec6..a70ccb2c5 100644 --- a/crates/proof-score/src/lib.rs +++ b/crates/proof-score/src/lib.rs @@ -20,11 +20,10 @@ mod payout; mod promote; pub use payout::{ - novelty_bar, payout_lattices, primary_from_harness, primary_metric, sealed_primary, - topic_masses, topic_share_bps, MinerTopicRun, PrimaryExtras, PROOF_SHARE_BPS, + novelty_bar, payout_lattices, primary_from_harness, primary_metric, sealed_bar_is_degenerate, + sealed_primary, topic_masses, topic_share_bps, MinerTopicRun, PrimaryExtras, PROOF_SHARE_BPS, }; pub use promote::{decide_promote, KeepReason, PromoteDecision}; - use std::collections::BTreeMap; use proof_task::{ @@ -251,6 +250,53 @@ fn finite(x: f64) -> bool { x.is_finite() } +/// Smallest bar a challenger can be compared against relatively. A bar at or +/// below this is **degenerate**: `challenger >= bar * (1 + eps)` has no +/// meaningful solution, so [`relative_win`] refuses every challenger and the +/// topic can never be passed by anyone. +/// +/// This is the same threshold [`relative_win`] uses, named so that the +/// operator-facing checks (`proof-topic-setup`, the baseline loader) can ask +/// the question without restating the constant and drifting from it. +pub const BAR_FLOOR: f64 = 1e-12; + +/// Whether a bar can ever be beaten. A non-finite or ~zero bar cannot. +/// +/// A zero bar arises from a real measurement — a reference run that solved +/// nothing, which is what an all-zero Harbor baseline is — and sealing it +/// would publish a topic no miner can ever win. Callers refuse it at the +/// boundary rather than let it become a silently dead topic. +#[must_use] +pub fn bar_is_degenerate(bar: f64) -> bool { + !bar.is_finite() || bar.abs() < BAR_FLOOR +} + +/// Whether a family decides a pass with the **relative** rule +/// ([`relative_win`]) — the families where a degenerate bar is fatal. +/// +/// `nll` compares the challenger **absolutely** against the sealed vector +/// (`holdout_nll > sealed - epsilon_nll` and the per-split regression cap), so +/// a zero bar there is a hard but meaningful target rather than a degenerate +/// one. Only the families that ask for a relative win are affected by +/// [`bar_is_degenerate`]. +#[must_use] +pub fn family_uses_relative_win(family: MetricFamily) -> bool { + matches!(family, MetricFamily::Throughput | MetricFamily::Custom) +} + +/// Whether a family's bar is one no challenger could ever clear. +/// +/// `primary` is the sealed (or champion) value for the family's primary +/// metric. `None` is **missing evidence**, not a bar of zero — the pass gate +/// reports that as `EvidenceMissing`, so it is not degenerate here. This is +/// the check that stops the LIVE Gate 1 shape — a reference run that solved +/// nothing, so every trial measured `0.0` — from being sealed into a topic +/// that is open, scorable and permanently unwinnable. +#[must_use] +pub fn family_bar_is_degenerate(family: MetricFamily, primary: Option) -> bool { + family_uses_relative_win(family) && primary.is_some_and(bar_is_degenerate) +} + /// Relative win rule shared by the throughput and custom families (and by /// automatic promotion): `challenger` beats `baseline` by at least `epsilon` /// relative, direction-aware. A zero or non-finite baseline can never be @@ -262,7 +308,7 @@ pub fn relative_win( direction: MetricDirection, epsilon: f64, ) -> bool { - if !finite(challenger) || !finite(baseline) || baseline.abs() < 1e-12 { + if !finite(challenger) || bar_is_degenerate(baseline) { return false; } match direction { @@ -809,4 +855,64 @@ mod tests { ))); assert_eq!(v.lattice, 0); } + + /// A zero bar is degenerate: nobody can ever be `>= 0 * (1 + eps)` in a + /// way that means anything, so `relative_win` refuses every challenger. + /// This is the LIVE Gate 1 shape — five Harbor tasks measured at 0.0 — + /// and it is why sealing such a baseline has to be refused at the + /// boundary instead of publishing a topic that is silently unwinnable. + #[test] + fn a_zero_bar_is_degenerate_and_unbeatable() { + for direction in [MetricDirection::Max, MetricDirection::Min] { + assert!( + bar_is_degenerate(0.0), + "a zero bar is degenerate under {direction:?}" + ); + for challenger in [0.0, 1.0, -1.0, 1e-3, 1e9] { + assert!( + !relative_win(challenger, 0.0, direction, 0.05), + "no challenger ({challenger}) may beat a zero bar under {direction:?}" + ); + } + } + // Non-finite is the same refusal, never a panic and never a pass. + assert!(bar_is_degenerate(f64::NAN)); + assert!(bar_is_degenerate(f64::INFINITY)); + assert!(!relative_win(1.0, f64::NAN, MetricDirection::Max, 0.05)); + } + + /// The floor is exactly the threshold the win rule uses, so a bar just + /// above it is beatable and one just below is not. Pinned so the two + /// cannot drift apart. + #[test] + fn the_bar_floor_is_the_threshold_relative_win_uses() { + let above = BAR_FLOOR * 10.0; + let below = BAR_FLOOR / 10.0; + assert!(!bar_is_degenerate(above)); + assert!(bar_is_degenerate(below)); + assert!(relative_win(above * 2.0, above, MetricDirection::Max, 0.05)); + assert!(!relative_win(1.0, below, MetricDirection::Max, 0.05)); + } + + /// A real (non-degenerate) bar still promotes exactly as before: the + /// guard must not have narrowed the normal path. + #[test] + fn a_real_bar_still_passes_a_real_win() { + let topic = throughput_topic(); + let mut sealed = flat_nll(3.0); + sealed.tokens_per_sec = Some(100.0); + let mut harness = nll_harness(3.01); + harness.tokens_per_sec = Some(106.0); + harness.wall_s = Some(10_000); + let v = judge_topic( + &topic, + &clean_agent(&topic.id, MetricFamily::Throughput, 1), + &harness, + &sealed, + &[], + &[], + ); + assert!(v.pass, "{:?}", v.failed); + assert_eq!(v.lattice, SCORE_MAX); + } } diff --git a/crates/proof-score/src/payout.rs b/crates/proof-score/src/payout.rs index da2d7cebf..0be5c56ce 100644 --- a/crates/proof-score/src/payout.rs +++ b/crates/proof-score/src/payout.rs @@ -96,6 +96,16 @@ pub fn sealed_primary(topic: &TopicDocument, sealed: &SealedBaseline) -> Option< ) } +/// Whether the **sealed** bar this topic compares against is degenerate — a +/// bar no challenger can ever clear (`crate::family_bar_is_degenerate`). +/// +/// An **absent** primary is not degenerate: it is missing evidence, which the +/// pass gate reports as such rather than as a bar of zero. +#[must_use] +pub fn sealed_bar_is_degenerate(topic: &TopicDocument, sealed: &SealedBaseline) -> bool { + crate::family_bar_is_degenerate(topic.metric.family, sealed_primary(topic, sealed)) +} + /// Equal split of the challenge's emission share across `n` open topics. #[must_use] pub fn topic_share_bps(n: usize) -> u16 { @@ -405,6 +415,42 @@ mod tests { } } + /// A sealed zero bar on a relative-win family is degenerate: the topic is + /// open, scorable, and nobody can ever pass it. This is the LIVE Gate 1 + /// shape (a reference run that solved nothing, every trial 0.0). + #[test] + fn a_zero_sealed_bar_is_degenerate_on_a_relative_family() { + let topic = wta_topic(); + let mut sealed = flat_nll(3.0); + sealed.tokens_per_sec = Some(0.0); + assert!(sealed_bar_is_degenerate(&topic, &sealed)); + // ...and a real bar is not. + sealed.tokens_per_sec = Some(100.0); + assert!(!sealed_bar_is_degenerate(&topic, &sealed)); + } + + /// `nll` decides a pass with an **absolute** comparison, so a zero bar + /// there is a hard target rather than a degenerate one. The check must not + /// reach across families and refuse it. + #[test] + fn a_zero_sealed_bar_is_not_degenerate_on_the_absolute_family() { + let topic = nll_topic(); + let sealed = flat_nll(0.0); + assert_eq!(topic.metric.family, MetricFamily::Nll); + assert!(!sealed_bar_is_degenerate(&topic, &sealed)); + } + + /// A missing primary is missing evidence, not a bar of zero: the gate + /// reports `EvidenceMissing` for it, so this must not claim degeneracy. + #[test] + fn a_missing_sealed_primary_is_not_degenerate_but_missing() { + let topic = wta_topic(); + let mut sealed = flat_nll(3.0); + sealed.tokens_per_sec = None; + assert!(sealed_primary(&topic, &sealed).is_none()); + assert!(!sealed_bar_is_degenerate(&topic, &sealed)); + } + #[test] fn wta_winner_takes_the_topic_and_ties_split() { let topic = wta_topic(); diff --git a/crates/proof-topic-authoring/Cargo.toml b/crates/proof-topic-authoring/Cargo.toml new file mode 100644 index 000000000..aec09b37e --- /dev/null +++ b/crates/proof-topic-authoring/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "proof-topic-authoring" +description = "What a topic's RLM authors: the rule vector, the SQL migrations, the dynamic APIs, the submission format, and the pin policy it tightens — plus the shape checks and canonical digests every one of them is held to. Pure data and text analysis: no database, no VM, no challenge content." +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +hex = "0.4" +proof-canon = { path = "../proof-canon" } +proof-task = { path = "../proof-task" } +proof-topic-sql-guard = { path = "../proof-topic-sql-guard" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +thiserror = "2" + +[dev-dependencies] + +[lints] +workspace = true diff --git a/crates/proof-topic-install/src/handler.rs b/crates/proof-topic-authoring/src/handler.rs similarity index 52% rename from crates/proof-topic-install/src/handler.rs rename to crates/proof-topic-authoring/src/handler.rs index 77f6144a6..ec0f7edcd 100644 --- a/crates/proof-topic-install/src/handler.rs +++ b/crates/proof-topic-authoring/src/handler.rs @@ -1,8 +1,8 @@ -//! What a topic's RLM install may name as a run backend. +//! What a topic's install may name as a run backend. //! //! A topic's paid runs are executed by an **operator-installed adaptor**, //! resolved inside the Firecracker guest from the image the operator baked -//! (`/opt/proof/runners//`). The bundle's RLM section names that +//! (`/opt/proof/runners//`). The set an install applies names that //! runner; the control plane never runs it. //! //! The boundary this module enforces is therefore about *which names an @@ -10,30 +10,30 @@ //! //! - A topic may select a runner through the **signed document** //! (`constraints.params.in_guest_benchmark_runner` / `baseline_runner`). -//! That value is already shape-checked by [`proof_experiment`] and is -//! signed, so it is topic data. -//! - A bundle's `rlm` section may also name a **handler** it wants bound. -//! That section is *not* signed — it is operator-supplied JSON handed to -//! the install — so a handler name from it is an **untrusted input**. This -//! module is the allow-list that input is checked against. +//! That value is already shape-checked by `proof_experiment` and is signed, +//! so it is topic data. +//! - A set may also name a **handler** it wants bound. That part is *not* +//! signed — it is JSON handed to the install — so a handler name from it is +//! an **untrusted input**. This module is the allow-list that input is +//! checked against. //! //! Two handler families exist, and nothing else may be bound: //! //! | Family | What it is | //! |--------|------------| -//! | [`Handler::VmBacked`] | the generic in-guest runner ([`proof_rlm::VmBackedRunner`]) — the Firecracker path | +//! | [`Handler::VmBacked`] | the generic in-guest runner (`proof_rlm::VmBackedRunner`) — the Firecracker path | //! | [`Handler::Harbor`] | an operator-baked Harbor adaptor, i.e. a `VmBacked` runner whose adaptor directory ships the Harbor harness | //! //! The distinction is *documentation and audit*, not a second code path: both //! resolve to the same `VmBackedRunner` over the topic-VM orchestrator, and //! neither can be a path, a URL, or a shell command. What the allow-list -//! prevents is an RLM section naming something like -//! `/bin/sh -c 'curl … | sh'`, an absolute path, or an arbitrary binary: those -//! are refused by shape before anything is bound, and the refusal names why. +//! prevents is a set naming something like `/bin/sh -c 'curl … | sh'`, an +//! absolute path, or an arbitrary binary: those are refused by shape before +//! anything is bound, and the refusal names why. use proof_canon::is_custom_id; -use crate::InstallError; +use crate::SectionError; /// The handler families an install may bind. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -60,13 +60,13 @@ impl Handler { pub const ALL: [Self; 2] = [Self::VmBacked, Self::Harbor]; } -/// Handler names an RLM section may use, mapped to their family. +/// Handler names a set may use, mapped to their family. /// -/// The left-hand names are what a bundle writes; the right-hand family is -/// what the install binds. Only these two spellings (plus their documented -/// synonyms) are accepted, and every one of them resolves to a `VmBacked` -/// runner — the allow-list is closed, so a name that is not here is refused -/// rather than defaulted. +/// The left-hand names are what a set writes; the right-hand family is what +/// the install binds. Only these spellings (plus their documented synonyms) +/// are accepted, and every one of them resolves to a `VmBacked` runner — the +/// allow-list is closed, so a name that is not here is refused rather than +/// defaulted. pub const ALLOWED_HANDLERS: [(&str, Handler); 4] = [ ("vm_backed", Handler::VmBacked), ("vm_backed_runner", Handler::VmBacked), @@ -74,7 +74,7 @@ pub const ALLOWED_HANDLERS: [(&str, Handler); 4] = [ ("harbor_trials", Handler::Harbor), ]; -/// Why a handler name was refused, in the terms the bundle wrote. +/// Why a handler name was refused, in the terms the set wrote. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum HandlerError { /// The name is not one of [`ALLOWED_HANDLERS`]. @@ -85,7 +85,7 @@ pub enum HandlerError { allowed = allowed_list() )] NotAllowed { - /// What the bundle named. + /// What the set named. got: String, }, /// The name is shaped like a path, a URL, or a command rather than an id. @@ -94,7 +94,7 @@ pub enum HandlerError { baked adaptor, never a path, a URL, or a command line" )] NotAnIdentifier { - /// What the bundle named. + /// What the set named. got: String, }, } @@ -109,13 +109,12 @@ pub fn allowed_list() -> String { .join(", ") } -/// Resolve a handler name from an RLM section. +/// Resolve a handler name from a set. /// /// Case and surrounding whitespace are tolerated, matching how the rest of -/// the CLI's operator inputs parse (`InstallEnvironment` does the same). That -/// tolerance cannot widen the allow-list: a path or a command line is still -/// refused as *not an identifier* after folding, because the fold only -/// touches case. +/// the CLI's operator inputs parse. That tolerance cannot widen the +/// allow-list: a path or a command line is still refused as *not an +/// identifier* after folding, because the fold only touches case. /// /// # Errors /// @@ -139,32 +138,35 @@ pub fn resolve_handler(name: &str) -> Result { }) } -/// Resolve a handler for an install, mapping a refusal onto the install error. +/// Resolve a handler for an install, mapping a refusal onto the section error. /// /// # Errors /// -/// [`InstallError::HandlerNotAllowed`]. -pub fn check_handler(name: &str) -> Result { - resolve_handler(name).map_err(|e| InstallError::HandlerNotAllowed(e.to_string())) +/// [`SectionError`] naming the part and the reason. +pub fn check_handler(name: &str) -> Result { + resolve_handler(name).map_err(|e| SectionError { + part: "handler".to_owned(), + why: e.to_string(), + }) } -/// The run backend an install binds, from the document and the section. +/// The run backend an install binds, from the document and the set. /// /// The two inputs answer two different questions, and both are recorded: /// /// - **Which runner** runs the topic's paid jobs is the **signed document's** -/// answer (`constraints.params`). The section cannot override it: the -/// signature is what the scoring path trusts. -/// - **Which handler family** the install bound is the **section's** answer, -/// and it must be on the allow-list. It is audit information — the family -/// is what an operator baked into the guest image — so it is recorded even +/// answer (`constraints.params`). The set cannot override it: the signature +/// is what the scoring path trusts. +/// - **Which handler family** the install bound is the **set's** answer, and +/// it must be on the allow-list. It is audit information — the family is +/// what an operator baked into the guest image — so it is recorded even /// when the document also names a runner, because a Harbor topic and a /// generic in-guest topic are operationally different and the journal /// should say which one this is. /// -/// Both resolve to the same [`proof_rlm::VmBackedRunner`] over the topic-VM -/// orchestrator; the family never selects a second code path here, and it can -/// never name a binary. +/// Both resolve to the same `VmBackedRunner` over the topic-VM orchestrator; +/// the family never selects a second code path here, and it can never name a +/// binary. #[must_use] pub fn bound_runner( document_runner: Option<&str>, @@ -175,3 +177,49 @@ pub fn bound_runner( handler.unwrap_or(Handler::VmBacked), ) } + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used)] + + use super::*; + + #[test] + fn handler_names_are_allow_listed_not_arbitrary() { + assert_eq!(resolve_handler("vm_backed"), Ok(Handler::VmBacked)); + assert_eq!(resolve_handler("vm_backed_runner"), Ok(Handler::VmBacked)); + assert_eq!(resolve_handler("harbor"), Ok(Handler::Harbor)); + assert_eq!(resolve_handler("Harbor_Trials"), Ok(Handler::Harbor)); + for bad in [ + "/bin/sh", + "sh -c 'curl x | sh'", + "https://evil.invalid/payload", + "some_path/binary", + "", + ] { + let err = resolve_handler(bad).expect_err(bad); + assert!( + matches!(err, HandlerError::NotAnIdentifier { .. }), + "{bad:?}: {err:?}" + ); + } + // A well-formed id this build does not resolve is a *different* + // refusal from a path or a command line: one is "not a name", the + // other is "not one of ours". + let err = resolve_handler("arbitrary_binary").expect_err("not allowed"); + assert!(matches!(err, HandlerError::NotAllowed { .. }), "{err:?}"); + assert!(err.to_string().contains("vm_backed"), "{err}"); + assert!(allowed_list().contains("harbor_trials")); + assert_eq!(Handler::ALL.len(), 2); + assert_eq!(Handler::Harbor.as_str(), "harbor"); + } + + #[test] + fn the_document_names_the_runner_and_the_set_names_the_family() { + assert_eq!( + bound_runner(Some("adaptor-v1"), Some(Handler::Harbor)), + (Some("adaptor-v1".to_owned()), Handler::Harbor) + ); + assert_eq!(bound_runner(None, None), (None, Handler::VmBacked)); + } +} diff --git a/crates/proof-topic-authoring/src/lib.rs b/crates/proof-topic-authoring/src/lib.rs new file mode 100644 index 000000000..46bfd72e3 --- /dev/null +++ b/crates/proof-topic-authoring/src/lib.rs @@ -0,0 +1,1407 @@ +//! What a topic's RLM **authors**, and the shape checks every part is held to. +//! +//! A Proof topic's behavior is not compiled into any binary and is not the +//! operator's to write. The topic's own RLM runs inside its topic VM and +//! authors the whole set: +//! +//! | Part | What it is | Where it lands | +//! |------|-----------|----------------| +//! | `rules` | the anti-cheat vector ticked before any paid inference | `proof_rule_version` (`source = 'rlm'`) | +//! | `migrations` | the topic's own SQL, applied under the deny-list | the shared database, inside the topic's namespace | +//! | `apis` | the routes the topic exposes for itself | `proof_topic_api` | +//! | `submission_format` | the wire shape it accepts | recorded as a canonical digest | +//! | `pin_policy` | how it **tightens** the global pin | recorded as a canonical digest | +//! +//! This crate is the one place those five parts are *described*: their shapes, +//! their bounds, their canonical digests, and the tightening rule that makes a +//! policy a policy (a topic may tighten a floor, never loosen it). It holds no +//! database, no VM, and no challenge content — it is pure data and text +//! analysis, so the guest that validates what its RLM emitted and the control +//! plane that applies it agree by construction rather than by convention. +//! +//! # Why the checks live here +//! +//! Two very different processes have to answer the same question — "is this a +//! well-formed set for this topic?" — and they must not answer it differently: +//! +//! - the **guest**, before it hands an RLM's answer back over the wire, so a +//! malformed or denied set never becomes a job output; and +//! - the **install**, before it applies a migration, registers a route, or +//! journals a binding, so a set that would leave the database in a state the +//! deny-list forbids never runs. +//! +//! The guest cannot link the install (it has no database) and the install +//! cannot link the guest. Both can link this. + +#![forbid(unsafe_code)] +#![allow( + clippy::missing_errors_doc, + clippy::doc_markdown, + clippy::module_name_repetitions, + clippy::must_use_candidate +)] + +pub mod handler; +pub mod section; + +pub use handler::{ + allowed_list, bound_runner, check_handler, resolve_handler, Handler, HandlerError, + ALLOWED_HANDLERS, +}; +pub use section::{read_apis, read_migrations, read_rules, read_section, SectionPlan, READ_KEYS}; + +use proof_task::{ChecklistRule, ProofPin, TopicDocument}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +/// Only accepted `schema_version` of an authored set. +pub const AUTHORING_SCHEMA: u32 = 1; + +/// Most migrations one topic may author. +pub const MAX_MIGRATIONS: usize = 64; + +/// Longest one migration's SQL may be, in bytes. +pub const MAX_MIGRATION_SQL_BYTES: usize = 256 * 1024; + +/// Most routes one topic may register. +pub const MAX_APIS: usize = 64; + +/// Longest route summary, in characters. +pub const MAX_API_SUMMARY_CHARS: usize = 256; + +/// Longest `eval_image_digest` / `gpu_class` pin a policy may carry. +pub const MAX_PIN_STRING_CHARS: usize = 256; + +/// Path prefixes inside a topic's own namespace that are **not a topic's to +/// claim**: the challenge's operator surface. +/// +/// A topic route is served under the topic's prefix +/// (`/challenge/{topic_id}/{path}`), so a stored `v1/admin/…` would answer at +/// `/challenge/{topic_id}/v1/admin/…` — a path a reader cannot tell apart +/// from the challenge's own admin surface, which is master-local. Both the +/// authoring check and the install refuse one, and the mux refuses to resolve +/// a row that is already in the table (a row written before this rule +/// existed). +pub const RESERVED_API_PREFIXES: [&str; 1] = ["v1/admin"]; + +/// One SQL migration the topic's RLM authored. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AuthoredMigration { + /// Topic-facing name, an id (`[a-z0-9][a-z0-9_-]{1,63}`). + pub name: String, + /// The SQL. Applied under the deny-list; never logged in full. + pub sql: String, +} + +/// One route the topic's RLM authored for it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AuthoredApi { + /// Path relative to the topic's own prefix: no leading `/`, no `..`. + pub path: String, + /// Upper-cased HTTP method, or `*`. + pub method: String, + /// What the route does, in the topic's words. + #[serde(default)] + pub summary: String, +} + +/// The migration shape under the name the install and the store read it by. +pub type Migration = AuthoredMigration; + +/// The route shape under the name the install and the mux read it by. +pub type ApiRoute = AuthoredApi; + +/// A relative path of plain segments: no leading `/`, no `.` / `..`, no empty +/// segment, no control characters, no backslash. +/// +/// A free function as well as [`TopicAuthoring::is_relative_api_path`], +/// because the readers that hold one part rather than a whole set (the mux, +/// the install's route resolver) want the predicate without a set in hand. +#[must_use] +pub fn is_relative_api_path(p: &str) -> bool { + TopicAuthoring::is_relative_api_path(p) +} + +/// A method a topic may claim. +#[must_use] +pub fn is_api_method(m: &str) -> bool { + TopicAuthoring::is_api_method(m) +} + +/// Whether `p` is inside a [`RESERVED_API_PREFIXES`] namespace. +#[must_use] +pub fn is_reserved_api_path(p: &str) -> bool { + TopicAuthoring::is_reserved_api_path(p) +} + +/// How a topic **tightens** the global pin. Every field is optional: an +/// absent knob means "this topic tightens nothing here". +/// +/// The rule is one-directional and enforced field by field: a policy may make +/// the topic *stricter* than the global pin, never looser. A topic that could +/// lower a floor would be a topic that scores something the network did not +/// agree to, which is why [`PinPolicy::tightens`] refuses by name rather than +/// clamping silently. +/// +/// Three fields are **equalities**, not tightenings: `holdout_size`, +/// `eval_image_digest`, and `gpu_class` describe what every topic is measured +/// against, so a policy that names them must name exactly the pin's value. A +/// topic may not score on another image or another machine class. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct PinPolicy { + /// Floor on the topic's absolute NLL epsilon (may only be raised). + #[serde(skip_serializing_if = "Option::is_none")] + pub epsilon_nll_min: Option, + /// Floor on a throughput topic's relative win (may only be raised). + #[serde(skip_serializing_if = "Option::is_none")] + pub epsilon_throughput_rel_min: Option, + /// Floor on a topic's per-split NLL regression tolerance (raised only). + #[serde(skip_serializing_if = "Option::is_none")] + pub epsilon_topic_max_regress_min: Option, + /// Proof deadline the topic accepts (may only be lowered, never zero). + #[serde(skip_serializing_if = "Option::is_none")] + pub max_proof_deadline_s: Option, + /// Largest FLOP budget the topic accepts (may only be lowered). + #[serde(skip_serializing_if = "Option::is_none")] + pub flops_budget_max: Option, + /// Holdout records per topic. An equality: it must be the pin's. + #[serde(skip_serializing_if = "Option::is_none")] + pub holdout_size: Option, + /// Eval image the topic scores on. An equality: it must be the pin's. + #[serde(skip_serializing_if = "Option::is_none")] + pub eval_image_digest: Option, + /// Machine class the topic scores on. An equality: it must be the pin's. + #[serde(skip_serializing_if = "Option::is_none")] + pub gpu_class: Option, +} + +impl PinPolicy { + /// The policy that tightens nothing. + pub fn none() -> Self { + Self::default() + } + + /// Whether every knob is absent. + pub fn is_empty(&self) -> bool { + *self == Self::default() + } + + /// Field-level shape: finite floors, non-zero deadlines and budgets, + /// bounded strings. Says nothing about the pin — that is + /// [`Self::tightens`]. + pub fn validate_shape(&self) -> Result<(), AuthoringError> { + let floors: [(&str, Option); 3] = [ + ("epsilon_nll_min", self.epsilon_nll_min), + ( + "epsilon_throughput_rel_min", + self.epsilon_throughput_rel_min, + ), + ( + "epsilon_topic_max_regress_min", + self.epsilon_topic_max_regress_min, + ), + ]; + for (field, value) in floors { + if let Some(v) = value { + if !v.is_finite() || v <= 0.0 || v > 1.0 { + return Err(AuthoringError::PinPolicy { + field, + why: "must be a finite fraction in (0, 1]".into(), + }); + } + } + } + if self.max_proof_deadline_s == Some(0) { + return Err(AuthoringError::PinPolicy { + field: "max_proof_deadline_s", + why: "must be at least 1 second; remove the knob to accept the pin's ceiling" + .into(), + }); + } + if self.flops_budget_max == Some(0) { + return Err(AuthoringError::PinPolicy { + field: "flops_budget_max", + why: "must be at least 1; remove the knob to accept the pin's maximum".into(), + }); + } + if self.holdout_size == Some(0) { + return Err(AuthoringError::PinPolicy { + field: "holdout_size", + why: "must be at least 1".into(), + }); + } + for (field, value) in [ + ("eval_image_digest", self.eval_image_digest.as_deref()), + ("gpu_class", self.gpu_class.as_deref()), + ] { + if let Some(s) = value { + if s.trim().is_empty() || s.chars().count() > MAX_PIN_STRING_CHARS { + return Err(AuthoringError::PinPolicy { + field, + why: format!("must be 1..={MAX_PIN_STRING_CHARS} chars"), + }); + } + } + } + Ok(()) + } + + /// Refuse any knob that is **looser** than the global pin, and any + /// equality that disagrees with it. + pub fn tightens(&self, pin: &ProofPin) -> Result<(), AuthoringError> { + self.validate_shape()?; + let floors: [(&str, Option, f64); 3] = [ + ("epsilon_nll_min", self.epsilon_nll_min, pin.epsilon_nll_min), + ( + "epsilon_throughput_rel_min", + self.epsilon_throughput_rel_min, + pin.epsilon_throughput_rel_min, + ), + ( + "epsilon_topic_max_regress_min", + self.epsilon_topic_max_regress_min, + pin.epsilon_topic_max_regress_min, + ), + ]; + for (field, asked, floor) in floors { + if let Some(v) = asked { + if v < floor { + return Err(AuthoringError::LoosenedFloor { + field, + got: v, + floor, + }); + } + } + } + if let Some(asked) = self.max_proof_deadline_s { + if asked > pin.max_proof_deadline_s_ceiling { + return Err(AuthoringError::LoosenedCeiling { + field: "max_proof_deadline_s", + got: asked, + ceiling: pin.max_proof_deadline_s_ceiling, + }); + } + } + if let Some(asked) = self.flops_budget_max { + if asked > pin.flops_budget_max { + return Err(AuthoringError::LoosenedCeiling { + field: "flops_budget_max", + got: asked, + ceiling: pin.flops_budget_max, + }); + } + } + let equals: [(&str, Option, String); 3] = [ + ( + "eval_image_digest", + self.eval_image_digest.clone(), + pin.eval_image_digest.clone(), + ), + ("gpu_class", self.gpu_class.clone(), pin.gpu_class.clone()), + ( + "holdout_size", + self.holdout_size.map(|n| n.to_string()), + pin.holdout_size.to_string(), + ), + ]; + for (field, asked, want) in equals { + if let Some(asked) = asked { + if asked != want { + return Err(AuthoringError::PinEquality { + field, + got: asked, + want, + }); + } + } + } + Ok(()) + } + + /// Refuse a policy that **diverges from the signed document**. + /// + /// The knobs a policy names are the same knobs the document carries, and + /// the document is what miners verify and what scoring reads + /// (`proof-score::nll_gates` reads `topic.epsilon_nll`, not a policy). So a + /// policy that asked for a **tighter** floor than the document's would be a + /// promise the host does not keep: the install would record the RLM's + /// tighter number while challengers were still scored against the + /// document's. Greptile reproduced exactly that. + /// + /// The rule is therefore **equality**, not "at least as tight": a policy + /// may restate what the document already declares — proving its RLM + /// considered the knob — and may not diverge from it in either direction. + /// A looser value is already refused by [`Self::tightens`] (against the + /// pin); a tighter one is refused here, because the only way to make it + /// effective would be to score miners against a threshold that is not in + /// the document they verified. If a topic wants a tighter floor, the + /// **document** says so, and it is signed. + /// + /// This is the check a guest can run with only the job's own topic in hand + /// (it has no pin). + pub fn agrees_with_document(&self, doc: &TopicDocument) -> Result<(), AuthoringError> { + self.validate_shape()?; + let floors: [(&str, Option, f64); 3] = [ + ("epsilon_nll_min", self.epsilon_nll_min, doc.epsilon_nll), + ( + "epsilon_throughput_rel_min", + self.epsilon_throughput_rel_min, + doc.metric.epsilon_rel, + ), + ( + "epsilon_topic_max_regress_min", + self.epsilon_topic_max_regress_min, + doc.epsilon_topic_max_regress, + ), + ]; + for (field, asked, document) in floors { + if let Some(v) = asked { + if !(v - document).abs().le(&f64::EPSILON) { + return Err(AuthoringError::PinDivergesFromDocument { + field, + got: v.to_string(), + document: document.to_string(), + }); + } + } + } + if let Some(asked) = self.max_proof_deadline_s { + // A document that declares no deadline accepts the pin's ceiling, + // which the guest cannot read — so a policy that tightens *below* + // it is accepted here and checked against the ceiling by the + // control plane, which has the pin. A document that declares one + // is what the runtime holds runs to, so the policy must restate it. + if let Some(document) = doc.eval_executor.max_proof_deadline_s { + if asked != document { + return Err(AuthoringError::PinDivergesFromDocument { + field: "max_proof_deadline_s", + got: asked.to_string(), + document: document.to_string(), + }); + } + } + } + if let Some(asked) = self.flops_budget_max { + // The document's own budget is what the runtime enforces and what + // miners are held to, so the policy restates it rather than + // choosing a different number. + if asked != doc.flops_budget { + return Err(AuthoringError::PinDivergesFromDocument { + field: "flops_budget_max", + got: asked.to_string(), + document: doc.flops_budget.to_string(), + }); + } + } + if let Some(asked) = self.holdout_size { + if asked != doc.holdout_size { + return Err(AuthoringError::PinEquality { + field: "holdout_size", + got: asked.to_string(), + want: doc.holdout_size.to_string(), + }); + } + } + Ok(()) + } +} + +/// The five parts a topic's RLM authors, as one document. +/// +/// `deny_unknown_fields` is deliberate: a key this build does not read is a +/// part nothing applies, so an RLM that wrote one gets a refusal naming it +/// rather than a silent drop. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TopicAuthoring { + /// Must equal [`AUTHORING_SCHEMA`]. + pub schema_version: u32, + /// Topic the set is for. The host refuses a mismatch. + pub topic_id: String, + /// The anti-cheat vector. + pub rules: Vec, + /// The topic's SQL, in apply order. + pub migrations: Vec, + /// The routes the topic exposes. + pub apis: Vec, + /// The submission shape the topic accepts. Recorded, never interpreted. + pub submission_format: Value, + /// How the topic tightens the global pin. + pub pin_policy: PinPolicy, +} + +/// Why a set or a part of it was refused. +/// +/// One error type for the whole crate: the RLM's own answer, the install's +/// reader, and the handler allow-list all refuse in these terms, so a refusal +/// reads the same wherever it happened. +#[derive(Debug, Clone, PartialEq, thiserror::Error)] +pub enum AuthoringError { + /// The set is for another topic. + #[error("authored set is for topic {got:?}, the job is for {want:?}")] + WrongTopic { + /// What the set says. + got: String, + /// What the job is bound to. + want: String, + }, + /// Schema drift. + #[error("authored set schema_version {got}, this build reads {AUTHORING_SCHEMA}")] + WrongSchema { + /// What the set said. + got: u32, + }, + /// A part is missing: the RLM authored an incomplete set. + #[error( + "the RLM authored no {part}: the set is incomplete, so nothing is installed — a topic's \ + behavior is authored by its own RLM (rules, migrations, apis, submission_format, \ + pin_policy), and an install refuses rather than filling the gap with the operator's \ + bundle" + )] + MissingPart { + /// Which of the five parts. + part: &'static str, + }, + /// A part of the set is malformed or carries an unknown key. + #[error("rlm.{part}: {why}")] + Section { + /// Which part (`migrations[0]`, `apis`, `rules`, …). + part: String, + /// What is wrong. + why: String, + }, + /// The rule vector was refused. + #[error("rules: {0}")] + Rules(String), + /// A migration was refused by the shape check. + #[error("migrations[{index}] ({name:?}): {why}")] + Migration { + /// Ordinal in the set. + index: usize, + /// The migration's name. + name: String, + /// What is wrong. + why: String, + }, + /// A migration reached outside the topic's namespace. + #[error("migrations[{index}] ({name:?}): {why}")] + MigrationDenied { + /// Ordinal in the set. + index: usize, + /// The migration's name. + name: String, + /// The deny-list's reason, naming the statement and the object. + why: String, + }, + /// A route was refused. + #[error("apis[{index}]: {why}")] + Api { + /// Ordinal in the set. + index: usize, + /// What is wrong. + why: String, + }, + /// The submission format is not an object. + #[error("submission_format: must be a non-empty object, got {got}")] + SubmissionFormat { + /// The JSON kind that arrived. + got: String, + }, + /// A pin-policy knob is malformed. + #[error("pin_policy.{field}: {why}")] + PinPolicy { + /// Which knob. + field: &'static str, + /// What is wrong. + why: String, + }, + /// A pin-policy floor is looser than the pin's. + #[error("pin_policy.{field} = {got} loosens the floor {floor}")] + LoosenedFloor { + /// Which knob. + field: &'static str, + /// What the policy asked for. + got: f64, + /// The floor it may not go below. + floor: f64, + }, + /// A pin-policy ceiling is looser than the pin's. + #[error("pin_policy.{field} = {got} loosens the ceiling {ceiling}")] + LoosenedCeiling { + /// Which knob. + field: &'static str, + /// What the policy asked for. + got: u64, + /// The ceiling it may not go above. + ceiling: u64, + }, + /// A pin-policy equality disagrees with the pin's. + #[error("pin_policy.{field} = {got:?} must equal the pin's {want:?}")] + PinEquality { + /// Which knob. + field: &'static str, + /// What the policy asked for. + got: String, + /// What the pin says. + want: String, + }, + /// A pin-policy knob disagrees with the **signed document's** value. + /// + /// Scoring reads the document (`proof-score::nll_gates` takes + /// `topic.epsilon_nll`), so a policy that named a different number would + /// be a threshold nobody is scored against — a promise the host does not + /// keep, in either direction. A policy restates the document; it does not + /// choose for it. + #[error( + "pin_policy.{field} = {got:?} diverges from the signed document's {document:?}: scoring \ + reads the document, so a policy that names a different number would be a threshold no \ + challenger is judged by. Restate the document's value (or change the document and \ + re-sign it)" + )] + PinDivergesFromDocument { + /// Which knob. + field: &'static str, + /// What the policy asked for. + got: String, + /// What the signed document says. + document: String, + }, + /// The set is larger than the bounds allow. + #[error("authored set carries {count} {part}, at most {max} are applied")] + TooMany { + /// Which part. + part: &'static str, + /// How many arrived. + count: usize, + /// The bound. + max: usize, + }, + /// The set is not serialisable as an install section. + #[error("authored set cannot be handed over: {0}")] + Encode(String), +} + +/// The part-level refusal an install's reader produces. +/// +/// A distinct type from [`AuthoringError`] because it names a part and a +/// reason and nothing else: the install maps it into its own error, the guest +/// into its job failure, and neither has to carry the other's vocabulary. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("rlm.{part}: {why}")] +pub struct SectionError { + /// Which part (`migrations[0]`, `apis`, `rules`, …). + pub part: String, + /// What is wrong. + pub why: String, +} + +/// The five parts, by name, in the order they are reported. +pub const PARTS: [&str; 5] = [ + "rules", + "migrations", + "apis", + "submission_format", + "pin_policy", +]; + +/// A JSON value's kind, for an error that says what arrived. +fn kind(v: &Value) -> &'static str { + match v { + Value::Null => "null", + Value::Bool(_) => "a boolean", + Value::Number(_) => "a number", + Value::String(_) => "a string", + Value::Array(_) => "an array", + Value::Object(_) => "an object", + } +} + +impl TopicAuthoring { + /// A relative path of plain segments: no leading `/`, no `.` / `..`, no + /// empty segment, no control characters, no backslash. + pub fn is_relative_api_path(p: &str) -> bool { + let p = p.trim(); + !p.is_empty() + && p.len() <= 512 + && !p.starts_with('/') + && !p.ends_with('/') + && !p + .chars() + .any(|c| c.is_control() || c == '\\' || c == '?' || c == '#') + && p.split('/') + .all(|seg| !seg.is_empty() && seg != "." && seg != "..") + && p.split('/').all(|seg| { + seg.chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '~' | '-')) + }) + } + + /// A method a topic may claim. + pub fn is_api_method(m: &str) -> bool { + matches!(m, "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "*") + } + + /// Whether `p` is inside a [`RESERVED_API_PREFIXES`] namespace. + /// + /// Segment-aware: `v1/admin` and `v1/admin/…` are reserved, + /// `v1/administrator` is not. + pub fn is_reserved_api_path(p: &str) -> bool { + let p = p.trim(); + RESERVED_API_PREFIXES.iter().any(|prefix| { + p == *prefix + || p.strip_prefix(prefix) + .is_some_and(|rest| rest.starts_with('/')) + }) + } + + /// Which of the five parts the set does **not** carry. + /// + /// Empty means the set is complete. This is the read the fail-closed gate + /// uses, so the refusal names the missing part rather than saying "invalid". + /// + /// Four parts are checked for content here. `pin_policy` is not: its + /// **presence** is structural (the key is required, so a set without it + /// does not parse at all), while an empty policy is a legitimate answer — + /// it is the RLM saying "this topic tightens nothing", which is what every + /// topic did before the part existed. What the gate holds for that part is + /// the tightening rule ([`PinPolicy::tightens`]), not a non-emptiness + /// check that would demand a knob the RLM has no reason to set. + pub fn missing_parts(&self) -> Vec<&'static str> { + let mut out = Vec::new(); + if self.rules.is_empty() { + out.push("rules"); + } + if self.migrations.is_empty() { + out.push("migrations"); + } + if self.apis.is_empty() { + out.push("apis"); + } + // A submission format that is an object with no keys is not a format: + // nothing describes what the topic accepts, so it counts as missing. + let format_empty = self + .submission_format + .as_object() + .is_some_and(serde_json::Map::is_empty); + if !self.submission_format.is_object() || format_empty { + out.push("submission_format"); + } + out + } + + /// Whether every part is present. + pub fn is_complete(&self) -> bool { + self.missing_parts().is_empty() + } + + /// Refuse the first missing part, naming it. + pub fn require_complete(&self) -> Result<(), AuthoringError> { + match self.missing_parts().first() { + Some(part) => Err(AuthoringError::MissingPart { part }), + None => Ok(()), + } + } + + /// Shape-check every part, refuse the first problem, and require the set + /// to be complete. + /// + /// `topic_id` is the topic the job was bound to, so a set an RLM wrote for + /// another topic is refused rather than applied. + pub fn validate(&self, topic_id: &str) -> Result<(), AuthoringError> { + if self.schema_version != AUTHORING_SCHEMA { + return Err(AuthoringError::WrongSchema { + got: self.schema_version, + }); + } + if self.topic_id.trim() != topic_id.trim() { + return Err(AuthoringError::WrongTopic { + got: self.topic_id.clone(), + want: topic_id.to_owned(), + }); + } + // A part that *carried* the wrong kind is a different diagnosis from a + // part that carried nothing, and the two refusals say so: "must be an + // object, got an array" sends an author to the value, "authored no + // submission_format" sends them to the missing part. + if !self.submission_format.is_null() && !self.submission_format.is_object() { + return Err(AuthoringError::SubmissionFormat { + got: kind(&self.submission_format).to_owned(), + }); + } + self.require_complete()?; + proof_canon::validate_rules(&self.rules) + .map_err(|e| AuthoringError::Rules(format!("{}: {}", e.field, e.why)))?; + self.check_migrations(topic_id)?; + self.check_apis()?; + if !self.submission_format.is_object() { + return Err(AuthoringError::SubmissionFormat { + got: kind(&self.submission_format).to_owned(), + }); + } + self.pin_policy.validate_shape()?; + Ok(()) + } + + /// Shape-check the migrations, including the deny-list. + /// + /// The deny-list runs here as well as at install time, and that is + /// deliberate: this is where the RLM's **own answer** arrives, so a + /// migration that would be refused when applied never becomes a job output + /// at all. The install runs the same check again because it cannot know + /// who validated the set it was handed. + fn check_migrations(&self, topic_id: &str) -> Result<(), AuthoringError> { + if self.migrations.len() > MAX_MIGRATIONS { + return Err(AuthoringError::TooMany { + part: "migrations", + count: self.migrations.len(), + max: MAX_MIGRATIONS, + }); + } + for (index, m) in self.migrations.iter().enumerate() { + let bad = |why: String| AuthoringError::Migration { + index, + name: m.name.clone(), + why, + }; + if !proof_canon::is_custom_id(&m.name) { + return Err(bad("name is not an id ([a-z0-9][a-z0-9_-]{1,63})".into())); + } + if m.sql.trim().is_empty() { + return Err(bad("sql is empty; remove the migration instead".into())); + } + if m.sql.len() > MAX_MIGRATION_SQL_BYTES { + return Err(bad(format!( + "sql is {} bytes, at most {MAX_MIGRATION_SQL_BYTES} are applied", + m.sql.len() + ))); + } + proof_topic_sql_guard::check_migration(&m.sql, topic_id).map_err(|e| { + AuthoringError::MigrationDenied { + index, + name: m.name.clone(), + why: e.to_string(), + } + })?; + } + Ok(()) + } + + /// Shape-check the routes: relative to the topic's prefix, never inside + /// the challenge's admin namespace, a method the mux can answer. + fn check_apis(&self) -> Result<(), AuthoringError> { + if self.apis.len() > MAX_APIS { + return Err(AuthoringError::TooMany { + part: "apis", + count: self.apis.len(), + max: MAX_APIS, + }); + } + for (index, a) in self.apis.iter().enumerate() { + let bad = |why: String| AuthoringError::Api { index, why }; + if !Self::is_relative_api_path(&a.path) { + return Err(bad(format!( + "path {:?} must be a relative path of plain segments (no leading '/', no \ + '..', no empty segment): a topic's routes live under its own prefix, and the \ + prefix is the control plane's to set", + a.path + ))); + } + if Self::is_reserved_api_path(&a.path) { + return Err(bad(format!( + "path {:?} is inside the challenge's admin namespace ({}), which is not a \ + topic's to claim. Register a different path.", + a.path, + RESERVED_API_PREFIXES.join(", ") + ))); + } + if !Self::is_api_method(&a.method.trim().to_ascii_uppercase()) { + return Err(bad(format!( + "method {:?} must be one of GET, POST, PUT, PATCH, DELETE, *", + a.method + ))); + } + if a.summary.chars().count() > MAX_API_SUMMARY_CHARS { + return Err(bad(format!( + "summary is longer than {MAX_API_SUMMARY_CHARS} chars" + ))); + } + } + Ok(()) + } + + /// Shape-check **and** hold the pin policy to the global pin. + /// + /// The control plane runs this (it has the pin); the guest runs + /// [`Self::validate`] plus [`PinPolicy::tightens_document`], because the + /// job carries the topic but not the pin. + pub fn validate_against_pin( + &self, + topic_id: &str, + pin: &ProofPin, + ) -> Result<(), AuthoringError> { + self.validate(topic_id)?; + self.pin_policy.tightens(pin) + } + + /// The set as the **install section** the installer reads. + /// + /// One shape, two sources: an operator's bundle and an RLM's answer are + /// handed over identically, so the install applies whichever it was given + /// through exactly the same gates. `handler` is absent — a handler is the + /// *operator's* run backend, not something a topic's RLM chooses. + pub fn as_section(&self) -> Result { + let value = serde_json::json!({ + "rules": self.rules, + "migrations": self.migrations, + "apis": self.apis, + "submission_format": self.submission_format, + "pin_policy": self.pin_policy, + }); + serde_json::to_string(&value).map_err(|e| AuthoringError::Encode(e.to_string())) + } + + /// SHA-256 of one part's canonical JSON, `sha256:`. + pub fn part_digest(&self, part: &str) -> String { + let value = match part { + "rules" => serde_json::to_value(&self.rules), + "migrations" => serde_json::to_value(&self.migrations), + "apis" => serde_json::to_value(&self.apis), + "submission_format" => Ok(self.submission_format.clone()), + "pin_policy" => serde_json::to_value(&self.pin_policy), + _ => Ok(Value::Null), + } + .unwrap_or(Value::Null); + digest_of(&value) + } + + /// SHA-256 of the whole set's canonical JSON. + pub fn digest(&self) -> String { + domain_digest(b"proof-topic-authoring-v1", self) + } + + /// The journal entry: per-part digests, the whole-set digest, and the + /// provenance word for every part. + /// + /// This is what makes "the RLM authored this topic" a fact the journal can + /// prove rather than a label the driver attached: each part names its + /// author (`rlm`), its digest, and — for the two parts that land in + /// tables — what landed. + pub fn journal_entry(&self, rules_version: u32) -> Value { + serde_json::json!({ + "source": "rlm", + "digest": self.digest(), + "parts": { + "rules": { + "source": "rlm", + "version": rules_version, + "digest": self.part_digest("rules"), + }, + "migrations": { + "source": "rlm", + "digest": self.part_digest("migrations"), + "names": self.migrations.iter().map(|m| m.name.clone()).collect::>(), + }, + "apis": { + "source": "rlm", + "digest": self.part_digest("apis"), + "routes": self.apis.iter() + .map(|a| format!("{} /{}", a.method.trim().to_ascii_uppercase(), a.path)) + .collect::>(), + }, + "submission_format": { + "source": "rlm", + "digest": self.part_digest("submission_format"), + }, + "pin_policy": { + "source": "rlm", + "digest": self.part_digest("pin_policy"), + }, + }, + }) + } +} + +/// SHA-256 of a value's canonical JSON, `sha256:`. +pub fn digest_of(value: &Value) -> String { + let canonical = proof_canon::canonical_json(value); + let mut hasher = Sha256::new(); + hasher.update(canonical.as_bytes()); + format!("sha256:{}", hex::encode(hasher.finalize())) +} + +/// Domain-separated digest over a serialisable value. +fn domain_digest(domain: &[u8], value: &T) -> String { + let value = serde_json::to_value(value).unwrap_or(Value::Null); + let mut h = Sha256::new(); + h.update(domain); + h.update([0xff]); + h.update(proof_canon::canonical_json(&value).as_bytes()); + hex::encode(h.finalize()) +} + +/// Read an RLM's `authoring.json`. +pub fn from_json(body: &str) -> Result { + serde_json::from_str(body).map_err(|e| AuthoringError::Encode(format!("parse: {e}"))) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used)] + + use super::*; + use proof_task::ChecklistRule; + + fn topic() -> TopicDocument { + let mut doc = TopicDocument { + id: "fixture-topic".into(), + statement: "Score the pinned pack with the pinned runner.".into(), + ..TopicDocument::default() + }; + doc.metric.family = proof_task::MetricFamily::Custom; + doc.metric.custom_id = "fixture_metric".into(); + doc + } + + fn complete(topic_id: &str) -> TopicAuthoring { + TopicAuthoring { + schema_version: AUTHORING_SCHEMA, + topic_id: topic_id.into(), + rules: vec![ChecklistRule { + id: "no_short_circuit".into(), + text: "the harness must run the task".into(), + }], + migrations: vec![AuthoredMigration { + name: "0001_scratch".into(), + sql: format!( + "CREATE TABLE {}_scratch (id TEXT)", + topic_id.replace('-', "_") + ), + }], + apis: vec![AuthoredApi { + path: "status".into(), + method: "get".into(), + summary: "topic status".into(), + }], + submission_format: serde_json::json!({"kind": "tar", "max_bytes": 5_242_880}), + pin_policy: PinPolicy { + epsilon_nll_min: Some(0.02), + ..PinPolicy::none() + }, + } + } + + #[test] + fn a_complete_set_validates_and_is_recognised_as_complete() { + let set = complete("fixture-topic"); + assert!(set.is_complete()); + assert!(set.missing_parts().is_empty()); + set.validate("fixture-topic").expect("validates"); + assert!(set.digest().len() == 64); + assert!(set.part_digest("rules").starts_with("sha256:")); + let json = set.as_section().expect("section"); + assert!(json.contains("\"pin_policy\"")); + assert!( + !json.contains("handler"), + "a handler is the operator's: {json}" + ); + let back = from_json(&serde_json::to_string(&set).expect("json")).expect("round trip"); + assert_eq!(back, set); + assert_eq!(back.digest(), set.digest()); + } + + /// Every part is load-bearing: dropping one is a refusal that names it, + /// never a set the install fills in from somewhere else. + #[test] + fn a_missing_part_is_refused_by_name() { + for (part, mutate) in [ + ( + "rules", + Box::new(|s: &mut TopicAuthoring| s.rules.clear()) + as Box, + ), + ( + "migrations", + Box::new(|s: &mut TopicAuthoring| s.migrations.clear()), + ), + ("apis", Box::new(|s: &mut TopicAuthoring| s.apis.clear())), + ( + "submission_format", + Box::new(|s: &mut TopicAuthoring| s.submission_format = serde_json::json!({})), + ), + ] { + let mut set = complete("fixture-topic"); + mutate(&mut set); + assert!(!set.is_complete(), "{part}"); + assert_eq!(set.missing_parts(), [part], "{part}"); + let err = set.validate("fixture-topic").expect_err(part); + assert!( + matches!(err, AuthoringError::MissingPart { part: got } if got == part), + "{part}: {err}" + ); + assert!(err.to_string().contains(part), "{err}"); + } + } + + /// `pin_policy` is present or the set does not parse: a policy that + /// tightens nothing is a legitimate answer, but *omitting* the part is + /// not — that is an RLM that never considered it. + #[test] + fn the_pin_policy_part_is_required_even_when_it_tightens_nothing() { + let set = complete("fixture-topic"); + let json = serde_json::to_string(&set).expect("json"); + assert!(json.contains("\"pin_policy\""), "{json}"); + let value: serde_json::Value = serde_json::from_str(&json).expect("value"); + let mut without = value.clone(); + without + .as_object_mut() + .expect("object") + .remove("pin_policy"); + let err = from_json(&without.to_string()).expect_err("the part is required"); + assert!( + err.to_string().contains("pin_policy"), + "the refusal names the part: {err}" + ); + // An empty policy is accepted: it says "this topic tightens nothing". + let mut empty = value; + empty["pin_policy"] = serde_json::json!({}); + let parsed = from_json(&empty.to_string()).expect("an empty policy parses"); + assert!(parsed.pin_policy.is_empty()); + assert!(parsed.is_complete(), "an empty policy is a complete part"); + parsed.validate("fixture-topic").expect("and it validates"); + } + + #[test] + fn a_set_for_another_topic_or_schema_is_refused() { + let mut set = complete("fixture-topic"); + set.topic_id = "other-topic".into(); + assert!(matches!( + set.validate("fixture-topic"), + Err(AuthoringError::WrongTopic { .. }) + )); + let mut set = complete("fixture-topic"); + set.schema_version = 2; + assert!(matches!( + set.validate("fixture-topic"), + Err(AuthoringError::WrongSchema { got: 2 }) + )); + } + + /// The deny-list runs where the RLM's answer arrives: a migration that + /// would be refused at install never becomes a job output. + #[test] + fn an_authored_migration_is_held_to_the_deny_list() { + let mut set = complete("fixture-topic"); + set.migrations = vec![AuthoredMigration { + name: "0001_bad".into(), + sql: "DROP TABLE proof_rule_version".into(), + }]; + let err = set.validate("fixture-topic").expect_err("denied"); + assert!( + matches!(err, AuthoringError::MigrationDenied { .. }), + "{err}" + ); + assert!(err.to_string().contains("proof_rule_version"), "{err}"); + + // A sibling topic's namespace is not this topic's either. + let mut set = complete("fixture-topic"); + set.migrations = vec![AuthoredMigration { + name: "0001_other".into(), + sql: "CREATE TABLE some_other_topic_scratch (id TEXT)".into(), + }]; + assert!(matches!( + set.validate("fixture-topic"), + Err(AuthoringError::MigrationDenied { .. }) + )); + } + + #[test] + fn an_authored_route_cannot_escape_the_topics_prefix_or_the_admin_namespace() { + for bad in [ + "/v1/admin/proof/topics", + "../admin", + "a/../../b", + "a//b", + "a/./b", + "a\\b", + "", + "a?x=1", + ] { + let mut set = complete("fixture-topic"); + set.apis = vec![AuthoredApi { + path: bad.into(), + method: "GET".into(), + summary: String::new(), + }]; + assert!( + matches!( + set.validate("fixture-topic"), + Err(AuthoringError::Api { .. }) + ), + "{bad:?}" + ); + } + let mut set = complete("fixture-topic"); + set.apis = vec![AuthoredApi { + path: "status".into(), + method: "TRACE".into(), + summary: String::new(), + }]; + assert!(matches!( + set.validate("fixture-topic"), + Err(AuthoringError::Api { .. }) + )); + assert!(TopicAuthoring::is_reserved_api_path("v1/admin/x")); + assert!(!TopicAuthoring::is_reserved_api_path("v1/administrator")); + } + + /// The tightening rule, field by field: a policy may raise a floor, lower + /// a ceiling, and may never do the opposite. + #[test] + fn a_pin_policy_may_tighten_and_never_loosen() { + let pin = proof_task::ProofPin { + eval_image_digest: format!("sha256:{}", "ab".repeat(32)), + ..proof_task::ProofPin::default() + }; + // Tightening is accepted. + let tight = PinPolicy { + epsilon_nll_min: Some(pin.epsilon_nll_min + 0.01), + max_proof_deadline_s: Some(pin.max_proof_deadline_s_ceiling - 1), + flops_budget_max: Some(pin.flops_budget_max - 1), + holdout_size: Some(pin.holdout_size), + eval_image_digest: Some(pin.eval_image_digest.clone()), + gpu_class: Some(pin.gpu_class.clone()), + ..PinPolicy::none() + }; + tight.tightens(&pin).expect("tightens"); + + for (field, policy) in [ + ( + "epsilon_nll_min", + PinPolicy { + epsilon_nll_min: Some(pin.epsilon_nll_min / 2.0), + ..PinPolicy::none() + }, + ), + ( + "max_proof_deadline_s", + PinPolicy { + max_proof_deadline_s: Some(pin.max_proof_deadline_s_ceiling + 1), + ..PinPolicy::none() + }, + ), + ( + "flops_budget_max", + PinPolicy { + flops_budget_max: Some(pin.flops_budget_max + 1), + ..PinPolicy::none() + }, + ), + ] { + let err = policy.tightens(&pin).expect_err(field); + assert!(err.to_string().contains(field), "{field}: {err}"); + } + // Equalities disagreeing with the pin are refused, not ignored: a + // topic may not score on another image or another machine class. + for (field, policy) in [ + ( + "eval_image_digest", + PinPolicy { + eval_image_digest: Some(format!("sha256:{}", "cd".repeat(32))), + ..PinPolicy::none() + }, + ), + ( + "gpu_class", + PinPolicy { + gpu_class: Some("8x".into()), + ..PinPolicy::none() + }, + ), + ( + "holdout_size", + PinPolicy { + holdout_size: Some(pin.holdout_size + 1), + ..PinPolicy::none() + }, + ), + ] { + let err = policy.tightens(&pin).expect_err(field); + assert!(err.to_string().contains(field), "{field}: {err}"); + } + // Shape: a zero deadline, a zero budget, a non-finite floor. + for policy in [ + PinPolicy { + max_proof_deadline_s: Some(0), + ..PinPolicy::none() + }, + PinPolicy { + flops_budget_max: Some(0), + ..PinPolicy::none() + }, + PinPolicy { + epsilon_nll_min: Some(f64::NAN), + ..PinPolicy::none() + }, + PinPolicy { + epsilon_nll_min: Some(0.0), + ..PinPolicy::none() + }, + ] { + assert!(policy.validate_shape().is_err(), "{policy:?}"); + } + assert!(PinPolicy::none().is_empty()); + PinPolicy::none() + .tightens(&pin) + .expect("nothing to tighten"); + } + + /// The guest has the topic but not the pin, so it holds the policy to the + /// document's own knobs — and the rule there is **equality**. + /// + /// Scoring reads the document (`proof-score::nll_gates` takes + /// `topic.epsilon_nll`), so a policy that named a *different* number would + /// be a threshold no challenger is judged by. Greptile reproduced exactly + /// that: an accepted tighter policy was recorded and had no effect on + /// promotion. A policy may restate the document; it may not choose for it. + #[test] + fn a_pin_policy_restates_the_signed_document_and_cannot_diverge() { + let doc = topic(); + // Restating the document is accepted — that is what a policy is for. + let restated = PinPolicy { + epsilon_nll_min: Some(doc.epsilon_nll), + flops_budget_max: Some(doc.flops_budget), + holdout_size: Some(doc.holdout_size), + ..PinPolicy::none() + }; + restated + .agrees_with_document(&doc) + .expect("restating the document is the policy's job"); + + // Diverging — either way — is refused, and the refusal says why. + for (field, policy) in [ + ( + "epsilon_nll_min", + PinPolicy { + epsilon_nll_min: Some(doc.epsilon_nll + 0.01), + ..PinPolicy::none() + }, + ), + ( + "flops_budget_max", + PinPolicy { + flops_budget_max: Some(doc.flops_budget + 1), + ..PinPolicy::none() + }, + ), + ( + "flops_budget_max", + PinPolicy { + flops_budget_max: Some(doc.flops_budget - 1), + ..PinPolicy::none() + }, + ), + ( + "epsilon_topic_max_regress_min", + PinPolicy { + epsilon_topic_max_regress_min: Some(doc.epsilon_topic_max_regress + 0.01), + ..PinPolicy::none() + }, + ), + ] { + let err = policy.agrees_with_document(&doc).expect_err(field); + assert!(err.to_string().contains(field), "{field}: {err}"); + assert!( + err.to_string().contains("scoring reads the document"), + "the refusal says why divergence is wrong: {err}" + ); + } + + // `holdout_size` is an equality in **both** directions, so it is + // refused against the pin before it ever reaches this check — a + // different number here is caught by `tightens`, and the refusal says + // which pin value it wanted. + let holdout = PinPolicy { + holdout_size: Some(doc.holdout_size + 1), + ..PinPolicy::none() + }; + let err = holdout.agrees_with_document(&doc).expect_err("holdout"); + assert!(err.to_string().contains("holdout_size"), "{err}"); + } + + /// The journal entry is the proof: every part names its author and its + /// digest, so "the RLM authored this" is a fact rather than a label. + #[test] + fn the_journal_entry_names_every_part_and_its_author() { + let set = complete("fixture-topic"); + let entry = set.journal_entry(7); + assert_eq!(entry["source"], "rlm"); + assert_eq!(entry["digest"], set.digest()); + for part in PARTS { + assert_eq!( + entry["parts"][part]["source"], "rlm", + "{part} must name its author" + ); + assert!( + entry["parts"][part]["digest"] + .as_str() + .is_some_and(|d| d.starts_with("sha256:")), + "{part} must carry a digest" + ); + } + assert_eq!(entry["parts"]["rules"]["version"], 7); + assert_eq!(entry["parts"]["migrations"]["names"][0], "0001_scratch"); + assert_eq!(entry["parts"]["apis"]["routes"][0], "GET /status"); + } + + /// Digests are over canonical JSON, so key order does not move them, and + /// a changed part does. + #[test] + fn part_digests_are_canonical_and_track_content() { + let a = complete("fixture-topic"); + let mut b = a.clone(); + b.submission_format = serde_json::json!({"max_bytes": 5_242_880, "kind": "tar"}); + assert_eq!( + a.part_digest("submission_format"), + b.part_digest("submission_format"), + "canonical JSON is order-independent" + ); + b.submission_format = serde_json::json!({"kind": "tar", "max_bytes": 6}); + assert_ne!( + a.part_digest("submission_format"), + b.part_digest("submission_format") + ); + assert_ne!(a.digest(), b.digest()); + } + + #[test] + fn a_part_of_the_wrong_kind_is_refused_not_coerced() { + let mut set = complete("fixture-topic"); + set.submission_format = serde_json::json!([1, 2, 3]); + assert!(matches!( + set.validate("fixture-topic"), + Err(AuthoringError::SubmissionFormat { .. }) + )); + let many = TopicAuthoring { + migrations: (0..=MAX_MIGRATIONS) + .map(|i| AuthoredMigration { + name: format!("m{i}"), + sql: "SELECT 1".into(), + }) + .collect(), + ..complete("fixture-topic") + }; + assert!(matches!( + many.validate("fixture-topic"), + Err(AuthoringError::TooMany { + part: "migrations", + .. + }) + )); + assert_eq!(PARTS.len(), 5); + } +} diff --git a/crates/proof-topic-install/src/section.rs b/crates/proof-topic-authoring/src/section.rs similarity index 67% rename from crates/proof-topic-install/src/section.rs rename to crates/proof-topic-authoring/src/section.rs index fcd95bd75..fa72450d1 100644 --- a/crates/proof-topic-install/src/section.rs +++ b/crates/proof-topic-authoring/src/section.rs @@ -1,40 +1,45 @@ -//! Reading the RLM section: the parts an install applies, and nothing else. +//! Reading a topic's behavior set: the parts an install applies, and nothing +//! else. //! //! The bundle carries its `rlm` section **verbatim** and opaque -//! ([`proof_topic_bundle::RlmSection`]): the bundle crate checks the shape -//! and hands the bytes over, so nothing about a topic is compiled in. +//! (`proof_topic_bundle::RlmSection`): the bundle crate checks the shape and +//! hands the bytes over, so nothing about a topic is compiled in. An RLM's +//! own answer arrives as the same kind of document — the very same parts, in +//! the same shape — which is why one reader serves both: a set is a set, and +//! whoever authored it, it goes through exactly these gates. //! -//! The install is the consumer. To *apply* the section it has to read the -//! parts it knows how to apply, and that is what this module does — strictly, -//! and only for the parts named here: +//! The install is the consumer. To *apply* a set it has to read the parts it +//! knows how to apply, and that is what this module does — strictly, and only +//! for the parts named here: //! //! | Part | What the install does with it | //! |------|-------------------------------| -//! | `migrations` | shape-checked, then executed under the SQL deny-list ([`proof_topic_sql_guard`]) | -//! | `apis` | recorded as topic-scoped routes ([`crate::install`]) | -//! | `rules` | installed as the topic's first rule version ([`crate::install`]) | +//! | `migrations` | shape-checked, then executed under the SQL deny-list (`proof_topic_sql_guard`) | +//! | `apis` | recorded as topic-scoped routes | +//! | `rules` | installed as the topic's first rule version | //! | `submission_format` | shape-checked and recorded; never interpreted | //! | `scoring` | shape-checked and recorded; never interpreted | +//! | `pin_policy` | shape-checked and recorded; never interpreted | //! | `handler` | allow-listed ([`crate::handler`]) | //! //! # Strictness, and why it is per-part //! -//! An **unknown key inside a part this module reads** is refused. A part is -//! a step list: a `{"name": …, "sq": …}` migration whose `sql` this build +//! An **unknown key inside a part this module reads** is refused. A part is a +//! step list: a `{"name": …, "sq": …}` migration whose `sql` this build //! cannot see is a step nothing performs, and silently skipping it would -//! install a topic that is not the one the operator signed off. Refusing -//! costs an operator one edit. +//! install a topic that is not the one that was signed off. Refusing costs one +//! edit. //! //! A **part this module has never heard of** is carried, not refused: that is //! the bundle's own rule (`RlmSection`), and it is the whole point of the //! boundary — a future part must not need a code change here to travel. Only //! the parts listed above are read; the rest goes into the install record as -//! the RLM's business. +//! the topic's business. //! -//! Nothing here decides what a rule, a migration, an API, a submission -//! format, or a scoring function *means*. `submission_format` and `scoring` -//! are recorded as canonical JSON digests so an audit can prove which ones a -//! topic was installed with, and are otherwise untouched. +//! Nothing here decides what a rule, a migration, an API, a submission format, +//! or a scoring function *means*. `submission_format`, `scoring`, and +//! `pin_policy` are recorded as canonical JSON digests so an audit can prove +//! which ones a topic was installed with, and are otherwise untouched. use proof_canon::is_custom_id; use proof_task::ChecklistRule; @@ -42,72 +47,48 @@ use serde_json::Value; use sha2::{Digest, Sha256}; use crate::handler::{check_handler, Handler}; -use crate::InstallError; +use crate::{ + AuthoredApi, AuthoredMigration, SectionError, MAX_APIS, MAX_MIGRATIONS, MAX_MIGRATION_SQL_BYTES, +}; -/// Keys this module reads out of the RLM section. -pub const READ_KEYS: [&str; 6] = [ +/// Keys this module reads out of a set. +pub const READ_KEYS: [&str; 7] = [ "apis", "handler", "migrations", + "pin_policy", "rules", "scoring", "submission_format", ]; -/// Longest one migration's SQL may be, in bytes. -pub const MAX_MIGRATION_SQL_BYTES: usize = 256 * 1024; - -/// Most migrations one install may apply. -pub const MAX_MIGRATIONS: usize = 64; - -/// Most routes one topic may register. -pub const MAX_APIS: usize = 64; - /// Longest route summary, in characters. pub const MAX_API_SUMMARY_CHARS: usize = 256; -/// One SQL migration the topic's install applies. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Migration { - /// Operator-facing name, an id (`[a-z0-9][a-z0-9_-]{1,63}`). - pub name: String, - /// The SQL. Applied under the deny-list; never logged in full. - pub sql: String, -} - -/// One route the topic registers for itself. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ApiRoute { - /// Path relative to the topic's own prefix: no leading `/`, no `..`. - pub path: String, - /// Upper-cased HTTP method, or `*`. - pub method: String, - /// What the route does, in the topic's words. - pub summary: String, -} - -/// The parts of an RLM section an install applies. -#[derive(Debug, Clone, PartialEq, Eq, Default)] +/// The parts of a set an install applies. +#[derive(Debug, Clone, PartialEq, Default)] pub struct SectionPlan { - /// Migrations, in bundle order. - pub migrations: Vec, + /// Migrations, in set order. + pub migrations: Vec, /// Routes the topic claims. - pub apis: Vec, + pub apis: Vec, /// Rule vector the install lands as version 1. pub rules: Vec, - /// Canonical-JSON digest of `submission_format`, when the bundle carries one. + /// Canonical-JSON digest of `submission_format`, when the set carries one. pub submission_format_digest: Option, - /// Canonical-JSON digest of `scoring`, when the bundle carries one. + /// Canonical-JSON digest of `scoring`, when the set carries one. pub scoring_digest: Option, - /// Allow-listed handler the section named, when it named one. + /// Canonical-JSON digest of `pin_policy`, when the set carries one. + pub pin_policy_digest: Option, + /// Allow-listed handler the set named, when it named one. pub handler: Option, - /// Part names the section carried that this module does not read, sorted. - /// They travel into the install record as the RLM's business. + /// Part names the set carried that this module does not read, sorted. + /// They travel into the install record as the topic's business. pub carried_unknown: Vec, } impl SectionPlan { - /// Whether the section asks for nothing this install applies. + /// Whether the set asks for nothing this install applies. #[must_use] pub fn is_empty(&self) -> bool { self.migrations.is_empty() @@ -115,13 +96,14 @@ impl SectionPlan { && self.rules.is_empty() && self.submission_format_digest.is_none() && self.scoring_digest.is_none() + && self.pin_policy_digest.is_none() && self.handler.is_none() } } /// Refuse with the part and the reason. -fn bad(part: &str, why: impl Into) -> InstallError { - InstallError::Section { +fn bad(part: &str, why: impl Into) -> SectionError { + SectionError { part: part.to_owned(), why: why.into(), } @@ -139,13 +121,12 @@ fn kind(v: &Value) -> &'static str { } } -/// One field of an object, refusing an unknown key by name. -fn take<'a>( - obj: &'a serde_json::Map, - key: &str, +/// Refuse an unknown key inside a part this module reads. +fn take( + obj: &serde_json::Map, part: &str, allowed: &[&str], -) -> Result, InstallError> { +) -> Result<(), SectionError> { for k in obj.keys() { if !allowed.contains(&k.as_str()) { return Err(bad( @@ -159,7 +140,7 @@ fn take<'a>( )); } } - Ok(obj.get(key)) + Ok(()) } /// A required string field. @@ -167,7 +148,7 @@ fn string_field( obj: &serde_json::Map, key: &str, part: &str, -) -> Result { +) -> Result { match obj.get(key) { Some(Value::String(s)) => Ok(s.clone()), Some(other) => Err(bad( @@ -178,12 +159,12 @@ fn string_field( } } -/// Read a section's migrations. +/// Read a set's migrations. /// /// # Errors /// -/// [`InstallError::Section`] naming the migration ordinal and the problem. -pub fn read_migrations(value: &Value) -> Result, InstallError> { +/// [`SectionError`] naming the migration ordinal and the problem. +pub fn read_migrations(value: &Value) -> Result, SectionError> { let Some(items) = value.as_array() else { return Err(bad( "migrations", @@ -205,7 +186,7 @@ pub fn read_migrations(value: &Value) -> Result, InstallError> { let Some(obj) = item.as_object() else { return Err(bad(&part, format!("must be an object, got {}", kind(item)))); }; - take(obj, "", &part, &["name", "sql"])?; + take(obj, &part, &["name", "sql"])?; let name = string_field(obj, "name", &part)?; if !is_custom_id(&name) { return Err(bad( @@ -226,17 +207,17 @@ pub fn read_migrations(value: &Value) -> Result, InstallError> { ), )); } - out.push(Migration { name, sql }); + out.push(AuthoredMigration { name, sql }); } Ok(out) } -/// Read a section's routes. +/// Read a set's routes. /// /// # Errors /// -/// [`InstallError::Section`] naming the route ordinal and the problem. -pub fn read_apis(value: &Value) -> Result, InstallError> { +/// [`SectionError`] naming the route ordinal and the problem. +pub fn read_apis(value: &Value) -> Result, SectionError> { let Some(items) = value.as_array() else { return Err(bad( "apis", @@ -252,15 +233,15 @@ pub fn read_apis(value: &Value) -> Result, InstallError> { ), )); } - let mut out: Vec = Vec::with_capacity(items.len()); + let mut out: Vec = Vec::with_capacity(items.len()); for (i, item) in items.iter().enumerate() { let part = format!("apis[{i}]"); let Some(obj) = item.as_object() else { return Err(bad(&part, format!("must be an object, got {}", kind(item)))); }; - take(obj, "", &part, &["path", "method", "summary"])?; + take(obj, &part, &["path", "method", "summary"])?; let path = string_field(obj, "path", &part)?; - if !is_relative_api_path(&path) { + if !crate::TopicAuthoring::is_relative_api_path(&path) { return Err(bad( &part, format!( @@ -270,21 +251,21 @@ pub fn read_apis(value: &Value) -> Result, InstallError> { ), )); } - if is_reserved_api_path(&path) { + if crate::TopicAuthoring::is_reserved_api_path(&path) { return Err(bad( &part, format!( "path {path:?} is inside the challenge's admin namespace ({}), which is not a \ topic's to claim: a topic route that reads like an operator route is a route \ a reader cannot tell apart from the real one. Register a different path.", - RESERVED_API_PREFIXES.join(", ") + crate::RESERVED_API_PREFIXES.join(", ") ), )); } let method = string_field(obj, "method", &part)? .trim() .to_ascii_uppercase(); - if !is_api_method(&method) { + if !crate::TopicAuthoring::is_api_method(&method) { return Err(bad( &part, format!("method {method:?} must be one of GET, POST, PUT, PATCH, DELETE, *"), @@ -306,7 +287,7 @@ pub fn read_apis(value: &Value) -> Result, InstallError> { format!("summary is longer than {MAX_API_SUMMARY_CHARS} chars"), )); } - out.push(ApiRoute { + out.push(AuthoredApi { path, method, summary, @@ -315,69 +296,14 @@ pub fn read_apis(value: &Value) -> Result, InstallError> { Ok(out) } -/// A relative path of plain segments: no leading `/`, no `.` / `..`, no empty -/// segment, no control characters, no backslash. -/// -/// Mirrors `proof_experiment`'s pack-path rule, for the same reason: the -/// value becomes part of a route the control plane serves, so a `..` or a -/// leading slash would let a topic step outside the prefix it was given. -#[must_use] -pub fn is_relative_api_path(p: &str) -> bool { - let p = p.trim(); - !p.is_empty() - && p.len() <= 512 - && !p.starts_with('/') - && !p.ends_with('/') - && !p - .chars() - .any(|c| c.is_control() || c == '\\' || c == '?' || c == '#') - && p.split('/') - .all(|seg| !seg.is_empty() && seg != "." && seg != "..") - && p.split('/').all(|seg| { - seg.chars() - .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '~' | '-')) - }) -} - -/// A method a topic may claim. -#[must_use] -pub fn is_api_method(m: &str) -> bool { - matches!(m, "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "*") -} - -/// Path prefixes inside a topic's own namespace that are **not a topic's to -/// claim**: the challenge's operator surface. -/// -/// A topic route is served under the topic's prefix -/// (`/challenge/{topic_id}/{path}`), so a stored `v1/admin/…` would answer at -/// `/challenge/{topic_id}/v1/admin/…` — a path a reader cannot tell apart -/// from the challenge's own admin surface, which is master-local. The install -/// refuses to record one, and the mux refuses to resolve one that is already -/// in the table (a row written before this rule existed). -pub const RESERVED_API_PREFIXES: [&str; 1] = ["v1/admin"]; - -/// Whether `p` is inside a [`RESERVED_API_PREFIXES`] namespace. -/// -/// Segment-aware: `v1/admin` and `v1/admin/…` are reserved, `v1/administrator` -/// is not. -#[must_use] -pub fn is_reserved_api_path(p: &str) -> bool { - let p = p.trim(); - RESERVED_API_PREFIXES.iter().any(|prefix| { - p == *prefix - || p.strip_prefix(prefix) - .is_some_and(|rest| rest.starts_with('/')) - }) -} - -/// Read a section's rule vector. +/// Read a set's rule vector. /// /// # Errors /// -/// [`InstallError::Section`] naming the rule ordinal and the problem. The -/// shared shape check ([`proof_canon::validate_rules`]) runs too, so a vector -/// the scoring path would refuse cannot be installed. -pub fn read_rules(value: &Value) -> Result, InstallError> { +/// [`SectionError`] naming the rule ordinal and the problem. The shared shape +/// check (`proof_canon::validate_rules`) runs too, so a vector the scoring +/// path would refuse cannot be installed. +pub fn read_rules(value: &Value) -> Result, SectionError> { let Some(items) = value.as_array() else { return Err(bad( "rules", @@ -390,7 +316,7 @@ pub fn read_rules(value: &Value) -> Result, InstallError> { let Some(obj) = item.as_object() else { return Err(bad(&part, format!("must be an object, got {}", kind(item)))); }; - take(obj, "", &part, &["id", "text"])?; + take(obj, &part, &["id", "text"])?; let id = string_field(obj, "id", &part)?; let text = string_field(obj, "text", &part)?; out.push(ChecklistRule { id, text }); @@ -406,34 +332,23 @@ pub fn read_rules(value: &Value) -> Result, InstallError> { /// Canonical-JSON digest of a part this module records but does not interpret. /// /// Canonical, so the digest is stable across key order and formatting: an -/// audit can compare it to the bundle the operator signed off. +/// audit can compare it to the set that was signed off. fn digest_of(value: &Value) -> String { let canonical = proof_canon::canonical_json(value); let mut hasher = Sha256::new(); hasher.update(canonical.as_bytes()); - format!("sha256:{}", hex_encode(&hasher.finalize())) -} - -/// Lower-case hex. -fn hex_encode(bytes: &[u8]) -> String { - const HEX: &[u8; 16] = b"0123456789abcdef"; - let mut out = String::with_capacity(bytes.len() * 2); - for b in bytes { - out.push(HEX[(b >> 4) as usize] as char); - out.push(HEX[(b & 0x0f) as usize] as char); - } - out + format!("sha256:{}", hex::encode(hasher.finalize())) } -/// Read an RLM section's raw text into the parts an install applies. +/// Read a set's raw text into the parts an install applies. /// /// # Errors /// -/// [`InstallError::Section`] for a part that is malformed, carries a key this -/// build does not read, or names a handler outside the allow-list. -pub fn read_section(raw: &str) -> Result { +/// [`SectionError`] for a part that is malformed, carries a key this build +/// does not read, or names a handler outside the allow-list. +pub fn read_section(raw: &str) -> Result { let parsed: serde_json::Map = - serde_json::from_str(raw).map_err(|e| InstallError::Section { + serde_json::from_str(raw).map_err(|e| SectionError { part: "rlm".to_owned(), why: format!("parse: {e}"), })?; @@ -470,6 +385,15 @@ pub fn read_section(raw: &str) -> Result { } plan.scoring_digest = Some(digest_of(value)); } + "pin_policy" => { + if !value.is_object() { + return Err(bad( + "pin_policy", + format!("must be an object, got {}", kind(value)), + )); + } + plan.pin_policy_digest = Some(digest_of(value)); + } other => plan.carried_unknown.push(other.to_owned()), } } @@ -479,6 +403,8 @@ pub fn read_section(raw: &str) -> Result { #[cfg(test)] mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used)] + use super::*; use serde_json::json; @@ -486,10 +412,11 @@ mod tests { fn a_full_section_reads_every_part_this_install_applies() { let plan = read_section( r#"{"rules": [{"id": "no_short_circuit", "text": "run the task"}], - "migrations": [{"name": "0001_scratch", "sql": "CREATE TABLE tb4_scratch (id TEXT)"}], + "migrations": [{"name": "0001_scratch", "sql": "CREATE TABLE topic_scratch (id TEXT)"}], "apis": [{"path": "status", "method": "get", "summary": "topic status"}], "submission_format": {"kind": "tar", "max_bytes": 5242880}, "scoring": {"primary": "success_rate"}, + "pin_policy": {"epsilon_nll_min": 0.02}, "handler": "harbor"}"#, ) .expect("reads"); @@ -505,6 +432,7 @@ mod tests { .as_deref() .is_some_and(|d| d.starts_with("sha256:") && d.len() == 71)); assert!(plan.scoring_digest.is_some()); + assert!(plan.pin_policy_digest.is_some()); assert!(plan.carried_unknown.is_empty()); assert!(!plan.is_empty()); } @@ -525,12 +453,9 @@ mod tests { let err = read_section(r#"{"migrations": [{"name": "m", "sq": "SELECT 1"}]}"#) .expect_err("a typo'd key is refused"); - let InstallError::Section { part, why } = err else { - panic!("expected Section"); - }; - assert_eq!(part, "migrations[0]"); - assert!(why.contains("\"sq\""), "{why}"); - assert!(why.contains("nothing performs"), "{why}"); + assert_eq!(err.part, "migrations[0]"); + assert!(err.why.contains("\"sq\""), "{}", err.why); + assert!(err.why.contains("nothing performs"), "{}", err.why); } #[test] @@ -555,10 +480,7 @@ mod tests { "", ] { let err = read_section(&format!(r#"{{"handler": "{bad}"}}"#)).expect_err(bad); - assert!( - matches!(err, InstallError::HandlerNotAllowed(_)), - "{bad:?}: {err:?}" - ); + assert_eq!(err.part, "handler", "{bad:?}: {err:?}"); } } @@ -578,10 +500,7 @@ mod tests { r#"{{"apis": [{{"path": "{bad}", "method": "GET"}}]}}"# )) .expect_err(bad); - assert!( - matches!(err, InstallError::Section { .. }), - "{bad:?}: {err:?}" - ); + assert_eq!(err.part, "apis[0]", "{bad:?}: {err:?}"); } for good in ["status", "v1/runs", "runs/by-id", "a_b/c-d.e~f"] { read_section(&format!( @@ -605,11 +524,8 @@ mod tests { r#"{{"apis": [{{"path": "{bad}", "method": "POST"}}]}}"# )) .expect_err(bad); - let InstallError::Section { why, .. } = err else { - panic!("{bad:?}: expected Section"); - }; - assert!(why.contains("admin namespace"), "{bad:?}: {why}"); - assert!(is_reserved_api_path(bad), "{bad:?}"); + assert!(err.why.contains("admin namespace"), "{bad:?}: {}", err.why); + assert!(crate::TopicAuthoring::is_reserved_api_path(bad), "{bad:?}"); } // Segment-aware: a path that merely starts with the same characters // is not the reserved namespace. @@ -618,20 +534,23 @@ mod tests { r#"{{"apis": [{{"path": "{good}", "method": "GET"}}]}}"# )) .unwrap_or_else(|e| panic!("{good:?} is not reserved: {e}")); - assert!(!is_reserved_api_path(good), "{good:?}"); + assert!( + !crate::TopicAuthoring::is_reserved_api_path(good), + "{good:?}" + ); } // The predicate is the one the mux runs, on a trimmed path. - assert!(is_reserved_api_path(" v1/admin/x ")); + assert!(crate::TopicAuthoring::is_reserved_api_path(" v1/admin/x ")); } #[test] fn a_rule_vector_the_scoring_path_would_refuse_is_refused_here() { let err = read_section(r#"{"rules": [{"id": "Bad Id", "text": "x"}]}"#).expect_err("bad rule id"); - assert!(matches!(err, InstallError::Section { .. }), "{err:?}"); + assert_eq!(err.part, "rules", "{err:?}"); let err = read_section(r#"{"rules": [{"id": "a_b", "text": ""}]}"#).expect_err("empty text"); - assert!(matches!(err, InstallError::Section { .. }), "{err:?}"); + assert_eq!(err.part, "rules", "{err:?}"); read_section(r#"{"rules": [{"id": "a_b", "text": "ok"}]}"#).expect("legal vector"); } @@ -655,7 +574,7 @@ mod tests { } /// The digests are stable across key order and formatting, because they - /// are over canonical JSON — an audit can compare them to the bundle. + /// are over canonical JSON — an audit can compare them to the set. #[test] fn recorded_digests_are_canonical_and_stable() { let a = @@ -666,6 +585,8 @@ mod tests { let c = read_section(r#"{"submission_format": {"kind": "tar", "max_bytes": 6}}"#).expect("c"); assert_ne!(a.submission_format_digest, c.submission_format_digest); + let p = read_section(r#"{"pin_policy": {"epsilon_nll_min": 0.02}}"#).expect("p"); + assert!(p.pin_policy_digest.is_some()); } #[test] @@ -676,13 +597,22 @@ mod tests { (r#"{"apis": {}}"#, "apis"), (r#"{"submission_format": []}"#, "submission_format"), (r#"{"scoring": "nope"}"#, "scoring"), + (r#"{"pin_policy": 7}"#, "pin_policy"), (r#"{"handler": 7}"#, "handler"), ] { let err = read_section(raw).expect_err(part); - let InstallError::Section { part: got, .. } = err else { - panic!("{part}: expected Section, got {err:?}"); - }; - assert_eq!(got, part, "{raw}"); + assert_eq!(err.part, part, "{raw}"); + } + } + + #[test] + fn the_read_keys_are_the_parts_this_module_applies() { + assert_eq!(READ_KEYS.len(), 7); + for key in READ_KEYS { + assert!( + crate::PARTS.contains(&key) || matches!(key, "handler" | "scoring"), + "{key} is not a part" + ); } } } diff --git a/crates/proof-topic-bundle/src/lib.rs b/crates/proof-topic-bundle/src/lib.rs index a777bb6db..ed693525a 100644 --- a/crates/proof-topic-bundle/src/lib.rs +++ b/crates/proof-topic-bundle/src/lib.rs @@ -24,6 +24,25 @@ //! [`RlmSection`]: its anti-cheat **rules**, the **SQL migrations** it needs, //! the **APIs** it exposes, its **submission format**, and its **scoring**. //! +//! **Authorship is the RLM's; this section is the operator's declaration.** +//! The section is *topic-owned data*: Rust never interprets it, and it travels +//! in the bundle. But a topic's behavior is **authored by its own RLM**, which +//! is asked to `propose_rules` inside its topic VM and answers with the whole +//! set — rules, migrations, APIs, submission format, and the pin policy it +//! tightens (`proof_topic_authoring::TopicAuthoring`). When the driver gets +//! that set, **the install applies it and not this section**: the RLM's answer +//! supersedes the operator's declaration, and the journal records `rlm` as the +//! author of every part. +//! +//! The section is still what an install applies when the RLM has authored +//! nothing yet (or only a rule vector), and then its provenance is honestly +//! `topic_document` — the operator's signed declaration — which is exactly +//! what the publish gate refuses to open a topic on. So the section is the +//! **declaration of intent** an operator writes, and the RLM's set is the +//! topic. `the_rlm_authors_the_whole_set_not_just_rules` pins that boundary, +//! so a change to what the RLM can author fails a test rather than silently +//! outdating this paragraph. +//! //! Rust never interprets any of it. This crate checks the section's *shape* //! (an object, bounded) and carries it byte-for-byte; it does not know what a //! rule, a migration, an API, or a scoring function *means*. That is the @@ -32,9 +51,11 @@ //! code. A topic's behavior travels in its signed document and its RLM //! section, never in this binary. //! -//! Consequence for tests and fixtures: the seed slug `tb4` and its temporary -//! alias `tbench` are **strings** that appear in test fixtures and operator -//! examples. They are never a condition in logic. +//! Consequence for tests and fixtures: topic slugs are **strings** that appear +//! in test fixtures and operator examples. They are never a condition in +//! logic, and no slug is an "owner default" — the topic registry is the +//! database, and which topics exist is a fact about the operator's published +//! documents, not about this build. //! //! Three rules carry the fail-closed posture: //! @@ -529,11 +550,11 @@ pub struct TopicInstallBundle { /// Temporary compatibility slugs this topic answers to, if the bundle /// declares any. /// - /// Owner default: the first topic's slug is `tb4` with `tbench` as a - /// **temporary** alias so existing miner links keep resolving. An alias - /// is not topic data — the topic's identity is its signed document's - /// `id` — so this is a bundle field that becomes a `proof_topic_alias` - /// row, and retiring it is deleting the row. + /// An alias is a **lookup key**, not topic data: the topic's identity is + /// its signed document's `id`, so an alias is a bundle field that becomes + /// a `proof_topic_alias` row, and retiring it is deleting the row. There + /// is no owner default — a bundle declares the aliases its topic needs, + /// and a bundle that declares none installs none. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub aliases: Vec, /// What the topic's RLM installs. Opaque to Rust: see [`RlmSection`]. @@ -1268,6 +1289,724 @@ mod tests { } } + /// No topic id is compiled into the **product** branches that decide what + /// a topic may do: the challenge service, the gateway, the topic-VM + /// orchestrator, or the guest. + /// + /// This is the repo-wide half of [`no_topic_literal_appears_in_this_crates_logic`]. + /// Each of those crates has its own guard for its own logic; this one + /// exists because the boundary is **cross-crate** — a topic id that + /// appeared in, say, the gateway's routing or the guest's job dispatch + /// would be a data-driven path turning back into a hardcoded one, and no + /// single crate's guard would see it. + /// + /// The check reads the crates' own source at compile time, so it cannot + /// drift from the tree. Comments are stripped: prose may explain the rule, + /// a literal in a `let` / `match` / `if` may not. + #[test] + fn no_topic_id_is_compiled_into_the_product_branches() { + for (label, source) in PRODUCT_MODULES { + let logic = production_logic(source); + for forbidden in FORBIDDEN_LITERALS { + assert!( + !logic.contains(forbidden), + "{label} names {forbidden:?}: which topics exist and what they score is \ + topic data (a signed document + the install journal), never a compiled \ + branch" + ); + } + } + } + + /// The **logic** of a product module: its production source with + /// `#[cfg(test)] mod …` blocks and full-line comments removed. + /// + /// Comments go first because prose may *explain* the rule — a comment + /// recalling which metal run showed a syncfs gap is documentation, not a + /// branch. What survives is code: a literal in a `let` / `match` / `if` + /// is caught. + fn production_logic(source: &str) -> String { + production_source(source) + .lines() + .filter(|l| !l.trim_start().starts_with("//")) + .collect::>() + .join("\n") + .to_lowercase() + } + + /// Blank every byte that is **not** Rust code structure: comments, string + /// bodies (including raw and byte strings), and character literals. + /// + /// This is what makes brace counting honest. A `{` inside a `//` comment, + /// a `"…"` literal, or an `r#"…"#` block is *text*, not a delimiter; left + /// in place it would drive the depth counter and leave a `#[cfg(test)]` + /// module looking unclosed, so every line after it would be dropped from + /// the scan. That is precisely how a prohibited literal could hide. + /// + /// Masked bytes become spaces so byte offsets and line breaks survive + /// (a `//` comment ends at its newline, which is kept). + /// + /// Deliberately not a full Rust parser: it is a small lexer for the four + /// constructs that can contain a brace. It errs toward masking, which + /// makes the scan *more* inclusive of real code, never less. + /// + /// One function, not four: the cases are mutually exclusive branches of a + /// single left-to-right scan, and splitting them would mean re-deriving + /// "am I at a comment / a raw string / a string / a char?" in each helper. + #[allow(clippy::too_many_lines)] + fn mask_non_code(source: &str) -> String { + let bytes = source.as_bytes(); + let mut out = vec![b' '; bytes.len()]; + let mut i = 0usize; + while i < bytes.len() { + // Line comment: mask to end of line (newline kept). + if bytes[i] == b'/' && i + 1 < bytes.len() && bytes[i + 1] == b'/' { + while i < bytes.len() && bytes[i] != b'\n' { + i += 1; + } + continue; + } + // Block comment (nesting is legal in Rust). + if bytes[i] == b'/' && i + 1 < bytes.len() && bytes[i + 1] == b'*' { + let mut depth = 0usize; + while i < bytes.len() { + if bytes[i] == b'/' && i + 1 < bytes.len() && bytes[i + 1] == b'*' { + depth += 1; + i += 2; + continue; + } + if bytes[i] == b'*' && i + 1 < bytes.len() && bytes[i + 1] == b'/' { + depth -= 1; + i += 2; + if depth == 0 { + break; + } + continue; + } + if bytes[i] == b'\n' { + out[i] = b'\n'; + } + i += 1; + } + continue; + } + // Raw string: r"…", r#"…"#, br#"…"#, rb#"…"#. + let raw_at = { + let rest = &bytes[i..]; + let (skip, hashes) = if rest.starts_with(b"br") || rest.starts_with(b"rb") { + (2usize, 0usize) + } else if rest.starts_with(b"r") { + (1usize, 0usize) + } else { + (0usize, 0usize) + }; + if skip == 0 { + None + } else { + let mut h = hashes; + while i + skip + h < bytes.len() && bytes[i + skip + h] == b'#' { + h += 1; + } + if i + skip + h < bytes.len() && bytes[i + skip + h] == b'"' { + Some((skip + h + 1, h)) + } else { + None + } + } + }; + if let Some((body_start, hashes)) = raw_at { + i += body_start; + // Closing delimiter: `"` followed by `hashes` `#`. + loop { + if i >= bytes.len() { + break; + } + if bytes[i] == b'"' { + let mut k = i + 1; + let mut seen = 0usize; + while seen < hashes && k < bytes.len() && bytes[k] == b'#' { + seen += 1; + k += 1; + } + if seen == hashes { + i = k; + break; + } + } + if bytes[i] == b'\n' { + out[i] = b'\n'; + } + i += 1; + } + continue; + } + // Normal string or byte string. + let quote_at = if bytes[i] == b'"' { + Some(i) + } else if bytes[i] == b'b' && i + 1 < bytes.len() && bytes[i + 1] == b'"' { + Some(i + 1) + } else { + None + }; + if let Some(q) = quote_at { + i = q + 1; + while i < bytes.len() { + if bytes[i] == b'\\' { + i += 2; + continue; + } + if bytes[i] == b'"' { + i += 1; + break; + } + if bytes[i] == b'\n' { + out[i] = b'\n'; + } + i += 1; + } + continue; + } + // Character literal or a lifetime. A lifetime (`'a`) is code and + // must be kept; a char literal's contents are masked. The + // discriminator: a char literal closes within a few bytes, a + // lifetime is followed by an identifier. + if bytes[i] == b'\'' { + let closed = { + let mut j = i + 1; + if j < bytes.len() && bytes[j] == b'\\' { + j += 2; + while j < bytes.len() && bytes[j] != b'\'' { + j += 1; + } + j + 1 + } else if j + 1 < bytes.len() && bytes[j + 1] == b'\'' { + j + 2 + } else { + 0 + } + }; + if closed > 0 && closed <= bytes.len() { + i = closed; + continue; + } + } + out[i] = bytes[i]; + i += 1; + } + String::from_utf8_lossy(&out).into_owned() + } + + /// Strip `#[cfg(test)] mod …` blocks from `source`, structurally. + /// + /// Brace depth over **code only** ([`mask_non_code`]), not "everything + /// after the first marker". A file may carry a `#[cfg(test)]` attribute on + /// a **method** (the guest's `Tail::bytes`) with production code after it; + /// splitting on the first marker would declare that production code + /// test-only and stop guarding it. And counting raw bytes would let a + /// brace in a comment or a string unbalance the depth, dropping every + /// later line from the scan. + /// + /// Only `mod` items are removed. A `#[cfg(test)]` on anything else is + /// left in place deliberately: the scan then sees test-only code and + /// **fails loudly** on a fixture literal, which an operator fixes by + /// moving the fixture into a `mod tests`. Over-scanning is a false alarm; + /// under-scanning is the hole this guards against. + fn production_source(source: &str) -> String { + let masked = mask_non_code(source); + let masked_lines: Vec<&str> = masked.lines().collect(); + let mut kept: Vec<&str> = Vec::new(); + let mut depth_test: Option = None; + let mut brace_depth = 0usize; + let mut pending_cfg_test = false; + for (idx, line) in source.lines().enumerate() { + // Braces are counted from the masked line, so a brace in a + // comment or a literal cannot move the depth. The attribute and + // `mod` markers are read from the masked line too, for the same + // reason (a `mod ` inside a string is not a module). + let code = masked_lines.get(idx).copied().unwrap_or(""); + let trimmed = code.trim(); + if pending_cfg_test { + if trimmed.starts_with("mod ") && code.contains('{') { + depth_test = Some(brace_depth); + pending_cfg_test = false; + } else if trimmed.ends_with(';') { + // `#[cfg(test)] mod x;` — the module lives in its own + // file, which the caller's list already excludes. + pending_cfg_test = false; + continue; + } else if !trimmed.is_empty() { + // The attribute is on a non-`mod` item (a method, say): + // keep it, so the scan still covers what follows. + pending_cfg_test = false; + } + } + if trimmed.starts_with("#[cfg(test)]") { + pending_cfg_test = true; + if trimmed.contains("mod ") && code.contains('{') { + depth_test = Some(brace_depth); + pending_cfg_test = false; + } + continue; + } + if depth_test.is_none() { + kept.push(line); + } + brace_depth = brace_depth.saturating_add(code.matches('{').count()); + for _ in 0..code.matches('}').count() { + brace_depth = brace_depth.saturating_sub(1); + if depth_test == Some(brace_depth) { + depth_test = None; + } + } + if let Some(start) = depth_test { + if brace_depth < start { + depth_test = None; + } + } + } + kept.join("\n") + } + + /// Top-level variant names of the enum whose declaration starts at + /// `header` (e.g. `pub enum VmJob`), in source order. + /// + /// Textual and deliberately simple: it reads the enum body by brace depth + /// and takes each line at depth 1 whose first token is an identifier + /// followed by `,` / `{` / `(` — both variant shapes. Attributes and doc + /// comments are skipped. A missing enum yields an empty list, never a + /// guess (the caller asserts the list it expects, so an extractor that + /// silently stopped working would fail rather than pass). + fn enum_variants(source: &str, header: &str) -> Vec { + let Some(start) = source.find(header) else { + return Vec::new(); + }; + let body = &source[start..]; + let Some(open) = body.find('{') else { + return Vec::new(); + }; + let mut out = Vec::new(); + let mut depth = 0usize; + for line in body[open..].lines() { + let trimmed = line.trim(); + if depth == 1 && !trimmed.starts_with("//") && !trimmed.starts_with("#[") { + if let Some(name) = trimmed.split(['{', '(', ',', ' ']).next() { + if !name.is_empty() + && name.chars().next().is_some_and(|c| c.is_ascii_uppercase()) + && !matches!(name, "Where" | "Self") + { + out.push(name.to_owned()); + } + } + } + depth = depth.saturating_add(line.matches('{').count()); + for _ in 0..line.matches('}').count() { + depth = depth.saturating_sub(1); + } + if depth == 0 { + break; + } + } + out + } + + /// What the RLM authors: **all five parts** of a topic's behavior, in one + /// job, as one document. + /// + /// The RLM is asked to `ProposeRules` and answers `Authored(TopicAuthoring)` + /// — the rule vector, the SQL migrations, the routes the topic exposes, + /// its submission format, and the pin policy it tightens. The install + /// applies them through exactly the gates an operator's bundle goes + /// through, and the journal records the author of every part. + /// + /// This guard used to assert the **opposite** boundary: that the RLM + /// authored rules only, with migrations and APIs travelling in the + /// operator's bundle. That was true, and it was the gap the authorship + /// pin named. The test flipped with the code — which is what it was for: + /// its doc comment said "if a later change lets the RLM emit them, this + /// test fails and the docs get to claim it", so the change that let the + /// RLM emit them is the change that rewrote it. + /// + /// The check is structural, not textual: it reads the job/output enums + /// that define the boundary, and the authoring crate's own part list. + #[test] + fn the_rlm_authors_the_whole_set_not_just_rules() { + const JOBS: &str = include_str!("../../proof-rlm/src/vm.rs"); + const AUTHORING: &str = include_str!("../../proof-topic-authoring/src/lib.rs"); + + // This test reads the **whole** file up to the first `#[cfg(test)]`, + // not [`production_logic`]: that helper tracks brace depth to drop + // test modules, and the job enums live inside braces, so it would + // drop exactly what this guard is about. + let head = |src: &str| -> String { + src.split("#[cfg(test)]") + .next() + .unwrap_or("") + .lines() + .filter(|l| !l.trim_start().starts_with("//")) + .collect::>() + .join("\n") + }; + + // The job surface the control plane can ask an RLM to run. + let jobs = head(JOBS); + for job in ["ProposeRules", "Baseline", "Inspect", "Evaluate", "Archive"] { + assert!( + jobs.contains(job), + "the RLM job surface still carries {job}; this guard is reading the wrong file" + ); + } + // …and what it hands back about behavior is the whole authored set. + assert!( + jobs.contains("Authored(Box)"), + "the RLM returns the topic's whole authored set; this guard is reading the wrong file" + ); + assert!( + jobs.contains("Rules(Vec)"), + "the rules-only output survives for an adaptor that writes only rules.json" + ); + + // **Allow-list, not deny-list.** A denylist of names this test + // happens to think of cannot hold an authorship boundary: a variant + // called `ApplySchema(Vec)` or `DeployRoutes { routes: … }` + // authors schema and routes without naming either, and Greptile + // demonstrated exactly that against the first version of this guard. + // So the check enumerates the **whole** permitted surface and fails on + // anything else — a new variant has to be added here deliberately, + // which is the moment the prose gets updated with it. + let variants = enum_variants(&jobs, "pub enum VmJob"); + assert_eq!( + variants, + ["ProposeRules", "Baseline", "Inspect", "Evaluate", "Archive"], + "the RLM job surface changed: add the variant here deliberately (and update the \ + prose) rather than widening the boundary silently" + ); + let outputs = enum_variants(&jobs, "pub enum VmJobOutput"); + assert_eq!( + outputs, + [ + "Authored", + "Rules", + "Baseline", + "Inspected", + "Evaluated", + "Archived" + ], + "the RLM output surface changed: a new output is a new thing the RLM can author, so \ + the docs may claim it — add it here deliberately" + ); + + // The five parts, named where the boundary is enforced: the authoring + // crate is the one description of them, and every part is required. + let authoring = head(AUTHORING); + for part in [ + "rules", + "migrations", + "apis", + "submission_format", + "pin_policy", + ] { + assert!( + authoring.contains(&format!("\"{part}\"")), + "the authored set still carries {part}; this guard is reading the wrong file" + ); + } + assert!( + authoring.contains("pub fn missing_parts"), + "the completeness gate is what makes a half-authored set a refusal" + ); + + // The extractor is not vacuous: it finds a known enum's variants and + // refuses to guess when the enum is absent. + assert_eq!( + enum_variants( + "pub enum Probe {\n One,\n Two { x: u8 },\n}\n", + "pub enum Probe" + ), + ["One", "Two"], + "the extractor reads top-level variants of both shapes" + ); + assert!( + enum_variants("pub enum Other {\n One,\n}\n", "pub enum Probe").is_empty(), + "a missing enum yields nothing, never a guess" + ); + } + + /// Every product module that decides what a topic may do, and must + /// therefore be **topic-agnostic**. + /// + /// Listed one file at a time rather than by globbing the directories: + /// `include_str!` needs literal paths, and a glob would silently widen the + /// guard's surface when a new file lands. The cost is that a new product + /// module has to be added here — which is why + /// `the_product_branch_guard_catches_what_it_claims_to` asserts the list + /// still names the file the check was written for, and why the crates + /// carrying a module here are the ones whose whole surface is a product + /// branch. + /// + /// The modules the crates' own guards also cover are included: those + /// guards strip test code with `split("#[cfg(test)]").next()`, which + /// stops at the first marker even when it annotates a method rather than a + /// module. Running the same sources through the structural strip here + /// means a literal hidden after such a marker is caught even if the + /// crate-local guard misses it. + const PRODUCT_MODULES: [(&str, &str); 33] = [ + ( + "proof-challenge/src/topic_routes.rs", + include_str!("../../proof-challenge/src/topic_routes.rs"), + ), + ( + "proof-challenge/src/lib.rs", + include_str!("../../proof-challenge/src/lib.rs"), + ), + ( + "proof-challenge/src/emit.rs", + include_str!("../../proof-challenge/src/emit.rs"), + ), + ( + "gateway-core/src/topic_routes.rs", + include_str!("../../gateway-core/src/topic_routes.rs"), + ), + ( + "gateway-core/src/admin_route.rs", + include_str!("../../gateway-core/src/admin_route.rs"), + ), + ( + "proof-vm-guest/src/runner.rs", + include_str!("../../proof-vm-guest/src/runner.rs"), + ), + ( + "proof-vm-guest/src/lib.rs", + include_str!("../../proof-vm-guest/src/lib.rs"), + ), + ( + "proof-vm-agent/src/router.rs", + include_str!("../../proof-vm-agent/src/router.rs"), + ), + ( + "proof-rlm/src/vm.rs", + include_str!("../../proof-rlm/src/vm.rs"), + ), + ( + "proof-rlm/src/runner.rs", + include_str!("../../proof-rlm/src/runner.rs"), + ), + ( + "proof-rlm/src/lib.rs", + include_str!("../../proof-rlm/src/lib.rs"), + ), + ( + "proof-rlm/src/gate.rs", + include_str!("../../proof-rlm/src/gate.rs"), + ), + ( + "proof-rlm/src/rules.rs", + include_str!("../../proof-rlm/src/rules.rs"), + ), + ( + "proof-rlm-lifecycle/src/lib.rs", + include_str!("../../proof-rlm-lifecycle/src/lib.rs"), + ), + ( + "proof-topic-authoring/src/lib.rs", + include_str!("../../proof-topic-authoring/src/lib.rs"), + ), + ( + "proof-topic-authoring/src/section.rs", + include_str!("../../proof-topic-authoring/src/section.rs"), + ), + ( + "proof-topic-authoring/src/handler.rs", + include_str!("../../proof-topic-authoring/src/handler.rs"), + ), + ( + "proof-experiment/src/lib.rs", + include_str!("../../proof-experiment/src/lib.rs"), + ), + ( + "proof-experiment/src/policy.rs", + include_str!("../../proof-experiment/src/policy.rs"), + ), + ( + "proof-topic-install/src/lib.rs", + include_str!("../../proof-topic-install/src/lib.rs"), + ), + ( + "proof-topic-install/src/install.rs", + include_str!("../../proof-topic-install/src/install.rs"), + ), + ( + "proof-topic-install/src/routes.rs", + include_str!("../../proof-topic-install/src/routes.rs"), + ), + ( + "proof-topic-install/src/gate.rs", + include_str!("../../proof-topic-install/src/gate.rs"), + ), + ( + "proof-vm-guest/src/fetch.rs", + include_str!("../../proof-vm-guest/src/fetch.rs"), + ), + ( + "proof-vm-staging/src/lib.rs", + include_str!("../../proof-vm-staging/src/lib.rs"), + ), + ( + "proof-vm-agent/src/lib.rs", + include_str!("../../proof-vm-agent/src/lib.rs"), + ), + ( + "proof-vm-agent/src/auth.rs", + include_str!("../../proof-vm-agent/src/auth.rs"), + ), + ( + "proof-vm-agent/src/hypervisor.rs", + include_str!("../../proof-vm-agent/src/hypervisor.rs"), + ), + ( + "proof-vm-agent/src/stamp.rs", + include_str!("../../proof-vm-agent/src/stamp.rs"), + ), + ( + "gateway-core/src/lib.rs", + include_str!("../../gateway-core/src/lib.rs"), + ), + ( + "gateway-core/src/admin_auth.rs", + include_str!("../../gateway-core/src/admin_auth.rs"), + ), + ( + "gateway-core/src/admin_attest.rs", + include_str!("../../gateway-core/src/admin_attest.rs"), + ), + ( + "gateway-core/src/proxy_paths.rs", + include_str!("../../gateway-core/src/proxy_paths.rs"), + ), + ]; + + /// The literals a product branch may not carry: a topic id, a benchmark + /// name, or a results-contract id. + /// + /// `harbor-trials` is on the list even though `proof-results` legitimately + /// defines the contract id: that crate is not in [`PRODUCT_MODULES`] + /// because the id **is** its interface — a signed document pins it — while + /// no module here may branch on it. + const FORBIDDEN_LITERALS: [&str; 5] = [ + "tbench", + "tb4", + "terminal-bench", + "terminal bench", + "harbor-trials", + ]; + + /// The guard is not vacuous, and it covers the files it claims to. + #[test] + fn the_product_branch_guard_catches_what_it_claims_to() { + // A production literal is caught. + let injected = format!( + "{}\npub const PROBE: &str = \"tbench\";\n", + PRODUCT_MODULES[0].1 + ); + assert!( + production_source(&injected) + .to_lowercase() + .contains("tbench"), + "a literal in production code must survive the strip and be caught" + ); + + // A `#[cfg(test)] mod` is removed, so its fixtures do not trip it. + let with_test_mod = + "fn prod() {}\n#[cfg(test)]\nmod tests {\n const T: &str = \"tbench\";\n}\n"; + let stripped = production_source(with_test_mod); + assert!(!stripped.contains("tbench"), "{stripped}"); + assert!(stripped.contains("fn prod"), "{stripped}"); + + // Production code **after** a `#[cfg(test)]` attribute on a method is + // still scanned. This is the exact shape in the guest's `runner.rs` + // (`Tail::bytes` is `#[cfg(test)]`, `Tail::text` follows it), where + // splitting on the first marker would have stopped guarding the rest + // of the file. + let after_marker = "struct T;\nimpl T {\n #[cfg(test)]\n fn b(&self) {}\n pub fn text(&self) -> String { \"tbench\".into() }\n}\n"; + assert!( + production_source(after_marker) + .to_lowercase() + .contains("tbench"), + "production code after a non-mod `#[cfg(test)]` must still be scanned" + ); + + // And the guard really is looking at the challenge's dynamic routes: + // the file Greptile found missing from the old hand-maintained list. + assert!( + PRODUCT_MODULES + .iter() + .any(|(label, _)| *label == "proof-challenge/src/topic_routes.rs"), + "the dynamic topic routes are a product branch and must be guarded" + ); + } + + /// A brace that is *text* cannot unbalance the strip and hide code after a + /// test module. + /// + /// The defect this pins: the stripper counted `{` / `}` in raw bytes, so a + /// brace inside a comment, a string, a raw string, or a macro's input left + /// the depth non-zero after the test module closed — and every line after + /// it was silently dropped from the scan. A prohibited literal placed + /// there passed the guard. + #[test] + fn braces_inside_text_do_not_hide_code_from_the_guard() { + // Each case: a `#[cfg(test)]` module whose body contains a brace that + // is *not* a delimiter, followed by production code carrying a + // literal. + let cases: [(&str, &str); 4] = [ + ( + "comment", + "#[cfg(test)]\nmod m {\n // an unbalanced brace in a comment: {\n}\nfn later() { let s = \"tbench\"; }\n", + ), + ( + "normal string", + "#[cfg(test)]\nmod m {\n fn f() { let s = \"a { brace\"; }\n}\nfn later() { let s = \"tbench\"; }\n", + ), + ( + "raw string", + "#[cfg(test)]\nmod m {\n fn f() { let s = r#\"a { brace\"#; }\n}\nfn later() { let s = \"tbench\"; }\n", + ), + ( + "macro input", + "#[cfg(test)]\nmod m {\n fn f() { println!(\"{{ literal brace\"); }\n}\nfn later() { let s = \"tbench\"; }\n", + ), + ]; + for (label, source) in cases { + let stripped = production_source(source); + assert!( + stripped.to_lowercase().contains("tbench"), + "a brace in a {label} must not hide the production code after the test module: \ + {stripped:?}" + ); + assert!( + !stripped.contains("mod m"), + "the test module is still stripped in the {label} case: {stripped:?}" + ); + } + } + + /// Masking removes exactly the non-code bytes and keeps structure. + #[test] + fn masking_blanks_text_and_keeps_code() { + let masked = mask_non_code("let a = 1; // { }\nlet b = \"}{ x\";\nlet c = 'x';\n"); + assert!( + !masked.contains("}{ x"), + "string bodies are masked: {masked:?}" + ); + assert!( + !masked.contains("// { }"), + "comment bodies are masked: {masked:?}" + ); + assert!(masked.contains("let a = 1;"), "{masked:?}"); + assert!(masked.contains("let c ="), "{masked:?}"); + // Newlines survive, so line-by-line pairing with the original holds. + assert_eq!(masked.lines().count(), 3, "{masked:?}"); + } + #[test] fn the_schema_key_lists_match_the_type() { let bundle = tb4(); diff --git a/crates/proof-topic-install/Cargo.toml b/crates/proof-topic-install/Cargo.toml index 8f484d566..73c2ef8ac 100644 --- a/crates/proof-topic-install/Cargo.toml +++ b/crates/proof-topic-install/Cargo.toml @@ -15,6 +15,7 @@ proof-experiment = { path = "../proof-experiment" } proof-rlm = { path = "../proof-rlm" } proof-rlm-store = { path = "../proof-rlm-store" } proof-task = { path = "../proof-task" } +proof-topic-authoring = { path = "../proof-topic-authoring" } proof-topic-bundle = { path = "../proof-topic-bundle" } proof-topic-sql-guard = { path = "../proof-topic-sql-guard" } serde = { version = "1", features = ["derive"] } @@ -24,6 +25,7 @@ sqlx = { version = "0.8", features = ["runtime-tokio", "postgres", "json", "macr thiserror = "2" [dev-dependencies] +proof-topic-authoring = { path = "../proof-topic-authoring" } db = { path = "../db", features = ["testing"] } proof-experiment = { path = "../proof-experiment" } tokio = { version = "1", features = ["macros", "rt-multi-thread"] } diff --git a/crates/proof-topic-install/src/install.rs b/crates/proof-topic-install/src/install.rs index 09b3494db..d0baa0d3f 100644 --- a/crates/proof-topic-install/src/install.rs +++ b/crates/proof-topic-install/src/install.rs @@ -61,11 +61,12 @@ use std::collections::BTreeSet; use proof_rlm::{RuleSet, RuleSource}; use proof_rlm_store::{RlmStore, StoreError}; use proof_task::{MetricFamily, TopicDocument, TopicStatus}; +use proof_topic_authoring::{ + bound_runner, ApiRoute, Handler, SectionPlan, TopicAuthoring, MAX_MIGRATIONS, +}; use serde::{Deserialize, Serialize}; use sqlx::PgPool; -use crate::handler::{bound_runner, Handler}; -use crate::section::{ApiRoute, SectionPlan, MAX_MIGRATIONS}; use crate::InstallError; use proof_topic_sql_guard::{check_migration, Statement}; @@ -227,6 +228,16 @@ pub struct InstallRequest<'a> { pub registered_custom: Vec, /// Stop before the RLM's baseline job. pub skip_baseline: bool, + /// The set the topic's **own RLM** authored, when the driver got one. + /// + /// This is the source of truth when it is present: the install applies + /// what the RLM wrote and journals every part's provenance as `rlm`. The + /// bundle's section is then **not** applied — it was the operator's + /// declaration of intent, and the RLM's set is what the topic actually + /// is. Absent means the RLM has not authored (yet): the bundle's section + /// is applied with its honest `topic_document` provenance, which the + /// publish gate refuses to open a topic on. + pub authored: Option<&'a TopicAuthoring>, } /// The install engine: a database to write through, plus the store the @@ -253,21 +264,58 @@ impl Installer<'_> { request: &InstallRequest<'_>, setup: SetupSummary, ) -> Result { - let plan = crate::section::read_section(request.rlm_raw)?; + // **Whose set is this?** When the topic's own RLM authored one, that + // is the topic's behavior and the bundle's section is not applied: the + // RLM's answer supersedes the operator's declaration of intent, and + // the journal records `rlm` as the author of every part. The bundle's + // section is read only when no RLM set exists, and then with its own + // honest `topic_document` provenance — which the publish gate refuses + // to open a topic on. + let (plan, authored) = match request.authored { + Some(set) => { + // The set is held to the same shape checks the install applies + // to a bundle, plus the one a bundle cannot be held to: the + // parts must be complete and every one of them present. + set.validate(&request.topic.id) + .map_err(|e| map_authoring(&e))?; + let section = set.as_section().map_err(|e| map_authoring(&e))?; + (proof_topic_authoring::read_section(§ion)?, Some(set)) + } + None => (proof_topic_authoring::read_section(request.rlm_raw)?, None), + }; // Check every migration up front: a bundle that would fail on its // third statement must not leave its first two applied. let checked = check_all_migrations(&plan, &request.topic.id)?; let handler = plan.handler.unwrap_or(Handler::VmBacked); let binding = resolve_binding(request, &plan, handler)?; - // The `pending` row lands first, so a crash mid-install is visible - // rather than silent. A failure appends its own `failed` row naming - // the step, so the journal says how far the run got. + // **Claim the namespace, and refuse a collision, atomically.** + // + // Two things have to be true together, and neither is sufficient + // alone: + // + // 1. The claim is against the **journal**, not only against published + // documents. A topic claims its namespace by installing into it, so + // a topic that has an install row (pending, applied, or failed) is + // registered even before its document is published. Reading only + // `proof_topic_version` let two unpublished installs whose prefixes + // overlap — `aa` and `aa-b` — both pass the check and both create + // `aa_b_scratch`. Greptile reproduced that. + // 2. The read and the claim are one atomic step. Under READ COMMITTED + // two concurrent installs each see no row from the other and both + // commit, which is the same defect with a race on top. The advisory + // lock is the pattern this repo already uses for the alias slug + // race (`proof-rlm-store`), and installs are rare operator actions, + // so serializing them costs nothing worth measuring. + // + // The `pending` row that records the claim lands **inside** the locked + // transaction, so the next install to take the lock sees it. Nothing + // is written if the check refuses. let pending_id = self - .journal(request, InstallState::Pending, None, &[], &[], &binding, "") + .claim_namespace(request, &checked, &binding, authored) .await?; match self - .apply_all(request, &plan, &checked, &binding, setup) + .apply_all(request, &plan, &checked, &binding, setup, authored) .await { Ok(report) => Ok(report), @@ -281,6 +329,7 @@ impl Installer<'_> { &[], &binding, &e.to_string(), + authored, ) .await; let _ = pending_id; @@ -297,6 +346,7 @@ impl Installer<'_> { checked: &[(String, Vec)], binding: &ExecutorBinding, setup: SetupSummary, + authored: Option<&TopicAuthoring>, ) -> Result { let already = self.applied_migrations(&request.topic.id).await?; let mut applied: Vec = already.iter().cloned().collect(); @@ -330,6 +380,7 @@ impl Installer<'_> { &applied, binding, "", + authored, ) .await?; Ok(InstallReport { @@ -348,6 +399,122 @@ impl Installer<'_> { }) } + /// Claim this topic's SQL namespace, or refuse a collision — atomically. + /// + /// Returns the `pending` journal row's id, which is the durable record of + /// the claim. Runs as **one transaction** that takes an advisory lock + /// first, so two installs cannot each read a registry that lacks the + /// other and both proceed: + /// + /// ```text + /// BEGIN + /// SELECT pg_advisory_xact_lock(hashtextextended('proof_topic_install', 0)) + /// -- every topic with an install row, plus every published document + /// -- (the claim is the union: a topic claims its namespace by + /// -- installing into it, published or not) + /// + /// INSERT proof_topic_install (… 'pending' …) -- the claim itself + /// COMMIT + /// ``` + /// + /// The lock is a *transaction* lock, so it is released on commit or + /// rollback and a crashed install cannot wedge it. It is deliberately + /// coarse — one lock for all installs — because installs are rare operator + /// actions and the alternative (per-namespace locks) would have to reason + /// about prefix overlap, which is the very thing that is hard here. + /// + /// A refusal writes **nothing**: the transaction rolls back, so there is + /// no `pending` row, no rule, and no table. + /// + /// # Errors + /// + /// [`InstallError::CrossTopicClaim`] naming the migration, the object, and + /// both topics; [`InstallError::Db`] when a read or the write fails. A + /// registry that cannot be read is a refusal too: a collision check that + /// cannot enumerate it would pass by default. + async fn claim_namespace( + &self, + request: &InstallRequest<'_>, + checked: &[(String, Vec)], + binding: &ExecutorBinding, + authored: Option<&TopicAuthoring>, + ) -> Result { + let mut tx = self + .pool + .begin() + .await + .map_err(|e| InstallError::Db(e.to_string()))?; + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") + .bind(CLAIM_LOCK_KEY) + .execute(&mut *tx) + .await + .map_err(|e| InstallError::Db(e.to_string()))?; + // Both sets, from inside the lock: published documents, and every + // topic that already has an install row (which is how a topic claims + // a namespace before it is published). + let rows: Vec<(String,)> = sqlx::query_as( + "SELECT topic_id FROM proof_topic_version \ + UNION \ + SELECT topic_id FROM proof_topic_install", + ) + .fetch_all(&mut *tx) + .await + .map_err(|e| InstallError::Db(e.to_string()))?; + let others: Vec<&str> = rows + .iter() + .map(|(id,)| id.as_str()) + .filter(|id| !id.eq_ignore_ascii_case(request.topic.id.trim())) + .collect(); + if let Some(err) = collision_in(checked, &request.topic.id, &others) { + // Roll back explicitly rather than relying on drop: the refusal is + // a *no-op*, and saying so here is what makes that readable. + tx.rollback() + .await + .map_err(|e| InstallError::Db(e.to_string()))?; + return Err(err); + } + let version: Option = None; + let json = |items: &[String]| { + serde_json::Value::Array( + items + .iter() + .cloned() + .map(serde_json::Value::String) + .collect(), + ) + }; + let mut binding_json = + serde_json::to_value(binding).map_err(|e| InstallError::Db(e.to_string()))?; + if let Some(obj) = binding_json.as_object_mut() { + let authorship = match authored { + Some(set) => set.journal_entry(0), + None => operator_authorship(&[]), + }; + obj.insert("authorship".to_owned(), authorship); + } + let id: i64 = sqlx::query_scalar( + "INSERT INTO proof_topic_install \ + (topic_id, bundle_digest, environment, state, rules_version, rule_ids, migrations, \ + binding, detail) \ + VALUES ($1, $2, $3, 'pending', $4, $5, $6, $7, $8) RETURNING id", + ) + .bind(&request.topic.id) + .bind(&request.bundle_digest) + .bind(&request.environment) + .bind(version) + .bind(json(&[])) + .bind(json(&[])) + .bind(binding_json) + .bind("namespace claimed") + .fetch_one(&mut *tx) + .await + .map_err(|e| InstallError::Db(e.to_string()))?; + tx.commit() + .await + .map_err(|e| InstallError::Db(e.to_string()))?; + Ok(id) + } + /// Migration names this topic has already applied, from the journal. /// /// Every row this reads was written **in the same transaction as the @@ -452,27 +619,78 @@ impl Installer<'_> { Ok(()) } - /// Record the routes a topic claims. + /// Record the routes a topic claims, **replacing** whatever it claimed + /// before. + /// + /// Insert-only was wrong once a set can be replaced. When an RLM-authored + /// install supersedes a bundle-authored one, the old set's routes are not + /// in the new set — and an append-only table leaves them resolving, so a + /// miner reaches an endpoint the topic's current install does not declare + /// while the journal says the newer set is in force. Greptile reproduced + /// it. + /// + /// So the topic's rows are **reconciled** against the set being applied: + /// rows absent from it are deleted, the set's rows are inserted, and the + /// topic's route **revision** is bumped — all in **one transaction**, so a + /// reader never sees a half-replaced table and the mux's generation moves + /// even when a replacement happens to keep the row count the same. async fn register_apis( &self, topic_id: &str, apis: &[ApiRoute], ) -> Result, InstallError> { + let mut tx = self + .pool + .begin() + .await + .map_err(|e| InstallError::Db(e.to_string()))?; + // Delete what the new set does not claim. `(method, path)` is the key, + // so a set that renames a path removes the old one. + let keep: Vec<(String, String)> = apis + .iter() + .map(|r| (r.method.clone(), r.path.clone())) + .collect(); + sqlx::query( + "DELETE FROM proof_topic_api \ + WHERE topic_id = $1 \ + AND NOT (method, path) IN (SELECT * FROM unnest($2::text[], $3::text[]))", + ) + .bind(topic_id) + .bind(keep.iter().map(|(m, _)| m.clone()).collect::>()) + .bind(keep.iter().map(|(_, p)| p.clone()).collect::>()) + .execute(&mut *tx) + .await + .map_err(|e| InstallError::Db(e.to_string()))?; let mut out = Vec::with_capacity(apis.len()); for route in apis { sqlx::query( "INSERT INTO proof_topic_api (topic_id, path, method, summary) \ - VALUES ($1, $2, $3, $4) ON CONFLICT (topic_id, method, path) DO NOTHING", + VALUES ($1, $2, $3, $4) \ + ON CONFLICT (topic_id, method, path) DO UPDATE SET summary = EXCLUDED.summary", ) .bind(topic_id) .bind(&route.path) .bind(&route.method) .bind(&route.summary) - .execute(self.pool) + .execute(&mut *tx) .await .map_err(|e| InstallError::Db(e.to_string()))?; out.push(format!("{} /{}", route.method, route.path)); } + // The revision is what the mux watches: a count cannot see a + // replacement, and this moves on every reconciliation. + sqlx::query( + "INSERT INTO proof_topic_route_revision (topic_id, revision) VALUES ($1, 1) \ + ON CONFLICT (topic_id) DO UPDATE \ + SET revision = proof_topic_route_revision.revision + 1, updated_at = now()", + ) + .bind(topic_id) + .execute(&mut *tx) + .await + .map_err(|e| InstallError::Db(e.to_string()))?; + tx.commit() + .await + .map_err(|e| InstallError::Db(e.to_string()))?; Ok(out) } @@ -482,6 +700,14 @@ impl Installer<'_> { /// an existing current rule set is returned untouched, which is what keeps /// a topic whose RLM has already written version 2 from being reset to /// the bundle's vector. + /// + /// Provenance is the point here. What this seeds is `topic_document`: the + /// vector the **operator** signed. That is honest provenance, not a + /// substitute for RLM authorship — the topic's RLM advances the store to + /// `rlm` by running its own `propose_rules` job in its VM + /// ([`proof_topic_setup::TopicSetup`]). An install therefore never makes a + /// topic's behavior RLM-authored, and the gates that admit an `open` topic + /// read the provenance rather than this row's presence. async fn install_rules( &self, request: &InstallRequest<'_>, @@ -526,6 +752,7 @@ impl Installer<'_> { migrations: &[String], binding: &ExecutorBinding, detail: &str, + authored: Option<&TopicAuthoring>, ) -> Result { let version = rules_version .map(i32::try_from) @@ -540,6 +767,21 @@ impl Installer<'_> { .collect(), ) }; + // The binding carries the **authorship** of every part beside the + // executor it resolved. This is what makes "the RLM authored this + // topic" a fact an audit can read back per part, rather than a label + // the driver attached to the row as a whole: each part names its + // author (`rlm` when the topic's own RLM wrote it, `topic_document` + // when it is still the operator's signed declaration) and its digest. + let mut binding_json = + serde_json::to_value(binding).map_err(|e| InstallError::Db(e.to_string()))?; + if let Some(obj) = binding_json.as_object_mut() { + let authorship = match authored { + Some(set) => set.journal_entry(rules_version.unwrap_or(0)), + None => operator_authorship(rule_ids), + }; + obj.insert("authorship".to_owned(), authorship); + } let id: i64 = sqlx::query_scalar( "INSERT INTO proof_topic_install \ (topic_id, bundle_digest, environment, state, rules_version, rule_ids, migrations, \ @@ -553,7 +795,7 @@ impl Installer<'_> { .bind(version) .bind(json(rule_ids)) .bind(json(migrations)) - .bind(serde_json::to_value(binding).map_err(|e| InstallError::Db(e.to_string()))?) + .bind(binding_json) .bind(detail) .fetch_one(self.pool) .await @@ -562,6 +804,92 @@ impl Installer<'_> { } } +/// The authorship entry for an install that applied the **operator's** set. +/// +/// Every part is `topic_document`: the signed document the operator published +/// is what the install applied, and the topic's own RLM has not authored +/// anything yet. The publish gate refuses to open a topic on this entry, which +/// is the whole point of recording it — the journal says which of the two +/// sources a topic was installed from. +fn operator_authorship(rule_ids: &[String]) -> serde_json::Value { + let part = |what: &str| { + serde_json::json!({ + "source": "topic_document", + "digest": null, + what: true, + }) + }; + serde_json::json!({ + "source": "topic_document", + "digest": null, + "parts": { + "rules": { + "source": "topic_document", + "digest": null, + "ids": rule_ids, + }, + "migrations": part("declared"), + "apis": part("declared"), + "submission_format": part("declared"), + "pin_policy": part("declared"), + }, + }) +} + +/// Advisory-lock key for the install claim. One lock for all installs: they +/// are rare operator actions, and a per-namespace lock would have to reason +/// about prefix overlap — the very thing that is hard here. +const CLAIM_LOCK_KEY: &str = "proof_topic_install"; + +/// The first cross-topic collision in `checked`, if any. +/// +/// Both forms a statement can name an object in are scanned: +/// +/// - `blanked`, the statement with string literals **and** dollar-quoted +/// function bodies blanked; and +/// - `bodies`, the function bodies themselves, with only their own literals +/// blanked. +/// +/// Scanning only the first was a hole, and Greptile reproduced it: +/// `CREATE FUNCTION aa_delete() … $$ DELETE FROM aa_b_scratch $$` installs for +/// topic `aa` while sibling `aa-b` is registered, and calling it deletes the +/// sibling's rows. The body is what runs, so it is what the check reads. +/// +/// The prefix rule in `is_topic_scoped` is per-topic, and `-` → `_` is +/// injective while its **prefixes are not prefix-free**: `aa_b_scratch` sits +/// inside both `aa` and `aa-b`. Only the install can see that, because only +/// the install has the registry. +fn collision_in( + checked: &[(String, Vec)], + topic_id: &str, + others: &[&str], +) -> Option { + for (name, statements) in checked { + let mut named: Vec = Vec::new(); + for statement in statements { + named.extend(proof_topic_sql_guard::referenced_objects( + &statement.blanked, + )); + named.extend(proof_topic_sql_guard::referenced_objects(&statement.bodies)); + } + // The first collision is the answer: the install refuses, so there is + // nothing to report about the others. + if let Some((object, other)) = + proof_topic_sql_guard::claim_collisions(&named, topic_id, others.iter().copied()) + .into_iter() + .next() + { + return Some(InstallError::CrossTopicClaim { + migration: name.clone(), + object, + this: topic_id.to_owned(), + other, + }); + } + } + None +} + /// Resolve the executor binding from the signed document and the section. /// /// The **document** is authoritative for the runner and the pack; the section @@ -627,6 +955,11 @@ fn map_store(e: &StoreError) -> InstallError { InstallError::Store(e.to_string()) } +/// Map an authoring refusal onto the install error. +fn map_authoring(e: &proof_topic_authoring::AuthoringError) -> InstallError { + InstallError::Authoring(e.to_string()) +} + /// Read a topic's newest install row. /// /// # Errors @@ -698,6 +1031,50 @@ fn strings(value: &serde_json::Value) -> Vec { .unwrap_or_default() } +/// What the newest install row says about the rules **it** landed. +/// +/// The publish gate's answer, as a value rather than a boolean, so a caller +/// that has to explain *why* a topic is not ready reads the same fact the +/// boolean was derived from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum InstalledRules { + /// The newest install is `applied` and the rule version **that install + /// recorded** is `rlm`-sourced. + RlmAuthored { + /// The rule version the install landed. + version: u32, + }, + /// The newest install is `applied`, but the rule version it recorded is + /// not RLM-authored (or its row is gone). + NotRlmAuthored { + /// The rule version the install recorded, when it recorded one. + version: Option, + /// `topic_document` / `operator` / `rlm`, or `None` when no row. + provenance: Option, + }, + /// The install recorded an RLM-authored version, but a **different** + /// version is in force and it is not RLM-authored. + /// + /// The other half of the same defect, from the opposite direction: the + /// install landed the RLM's vector and a later `operator` edit superseded + /// it. The topic would open with rules no RLM wrote — refused exactly as a + /// document-sourced vector is. + SupersededByOperator { + /// The RLM-authored version the install recorded. + installed: u32, + /// The version now in force. + in_force: u32, + /// Its provenance (`operator`, or `topic_document`). + provenance: String, + }, + /// The newest install is not `applied`, or the topic has no install row. + NotApplied { + /// The state the journal holds (`pending` / `failed`), or `None` when + /// no row exists at all. + state: Option, + }, +} + /// Whether `topic_id` has an install in the **`applied`** state. /// /// This is the durable fact a publish of an `open` document is gated on. The @@ -725,6 +1102,118 @@ pub async fn applied_install(pool: &PgPool, topic_id: &str) -> Result, + Option, + Option, + Option, +); + +/// The provenance of the rule version the newest install **recorded**, and of +/// the version actually in force. +/// +/// Two facts, read as **one statement** so they come from one snapshot: +/// +/// 1. The newest install row is `applied`, and the rule version it recorded is +/// `rlm`-sourced (joined on +/// `proof_rule_version.version = proof_topic_install.rules_version`). +/// 2. The version **in force** is also `rlm`-sourced. +/// +/// That pair is the gate, and each half closes a different way for an +/// operator-authored vector to reach a live topic: +/// +/// - **Only (1)** would admit a topic whose install landed the RLM's version +/// while a later `operator` edit superseded it: the vector in force would be +/// one no RLM wrote. +/// - **Only (2)** — the original defect — would admit a topic whose install +/// landed the signed document's version 1 (`topic_document`) while an +/// unrelated later version 2 happened to be RLM-authored: the topic would +/// open with the operator's vector in force, which is the operator-cloned +/// document the gate exists to refuse. +/// +/// Requiring both is not "the install's version must equal the one in force": +/// an RLM that rewrites its own rules after the install (version N → N+1, both +/// `rlm`) is exactly the autonomy this track wants, and it stays admitted. +/// +/// Fail-closed: an `applied` row that recorded **no** rule version, a missing +/// rule row, and any non-`rlm` provenance are refusals, never an admission, +/// and a database error is an `Err`. +/// +/// # Errors +/// +/// [`InstallError::Db`]. +pub async fn installed_rules( + pool: &PgPool, + topic_id: &str, +) -> Result { + // `LEFT JOIN` so an `applied` row whose rule version has no row is + // visible as "no provenance" rather than as "no install". The two scalar + // subqueries read the version in force in the same snapshot. + let row: Option = sqlx::query_as( + "SELECT i.state, i.rules_version, r.source, \ + f.version, f.source \ + FROM proof_topic_install i \ + LEFT JOIN proof_rule_version r \ + ON r.topic_id = i.topic_id AND r.version = i.rules_version \ + LEFT JOIN LATERAL ( \ + SELECT version, source FROM proof_rule_version \ + WHERE topic_id = i.topic_id ORDER BY version DESC LIMIT 1 \ + ) f ON true \ + WHERE i.topic_id = $1 \ + ORDER BY i.id DESC \ + LIMIT 1", + ) + .bind(topic_id) + .fetch_optional(pool) + .await + .map_err(|e| InstallError::Db(e.to_string()))?; + let Some((state, version, source, in_force_raw, in_force_source)) = row else { + return Ok(InstalledRules::NotApplied { state: None }); + }; + if state != InstallState::Applied.as_str() { + return Ok(InstalledRules::NotApplied { state: Some(state) }); + } + let version = version.and_then(|v| u32::try_from(v).ok()); + let in_force = in_force_raw.and_then(|v| u32::try_from(v).ok()); + let Some(installed_version) = version else { + // An `applied` row with no recorded rule version cannot be admitted: + // there is no vector it can be shown to have landed. + return Ok(InstalledRules::NotRlmAuthored { + version: None, + provenance: source, + }); + }; + if source.as_deref() != Some(RULES_SOURCE_RLM) { + return Ok(InstalledRules::NotRlmAuthored { + version: Some(installed_version), + provenance: source, + }); + } + if in_force_source.as_deref() == Some(RULES_SOURCE_RLM) { + return Ok(InstalledRules::RlmAuthored { + version: installed_version, + }); + } + // The install landed an RLM vector and something else is in force. A rule + // row for the install's version exists (the join matched), so the version + // in force exists too; the fallback is unreachable and only avoids a + // panic in a read that must stay total. + Ok(InstalledRules::SupersededByOperator { + installed: installed_version, + in_force: in_force.unwrap_or(installed_version), + provenance: in_force_source.unwrap_or_else(|| "absent".to_owned()), + }) +} + +/// The `proof_rule_version.source` value meaning the topic's own RLM wrote +/// the vector (the store's `RuleSource::Rlm` wire word). +pub const RULES_SOURCE_RLM: &str = "rlm"; + /// [`applied_install`], as a plain boolean. /// /// # Errors @@ -734,6 +1223,51 @@ pub async fn is_installed(pool: &PgPool, topic_id: &str) -> Result Result { + let source: Option = sqlx::query_scalar( + "SELECT source FROM proof_rule_version WHERE topic_id = $1 ORDER BY version DESC LIMIT 1", + ) + .bind(topic_id) + .fetch_optional(pool) + .await + .map_err(|e| InstallError::Db(e.to_string()))?; + Ok(source.as_deref() == Some("rlm")) +} + +/// [`rlm_authored_rules`] as the operator-facing provenance word. +/// +/// `None` when the topic has no rule row at all. Used by the gate refusals so +/// an operator reads *which* provenance blocked the publish rather than only +/// that one did. +/// +/// # Errors +/// +/// [`InstallError::Db`]. +pub async fn rules_source(pool: &PgPool, topic_id: &str) -> Result, InstallError> { + sqlx::query_scalar( + "SELECT source FROM proof_rule_version WHERE topic_id = $1 ORDER BY version DESC LIMIT 1", + ) + .bind(topic_id) + .fetch_optional(pool) + .await + .map_err(|e| InstallError::Db(e.to_string())) +} + /// Every route a topic registered, for the dynamic mux. /// /// A stored path is **relative**, so the caller owns the prefix and a topic diff --git a/crates/proof-topic-install/src/lib.rs b/crates/proof-topic-install/src/lib.rs index 413493497..e490b7b3e 100644 --- a/crates/proof-topic-install/src/lib.rs +++ b/crates/proof-topic-install/src/lib.rs @@ -15,13 +15,14 @@ //! | Gate | What it refuses | //! |------|-----------------| //! | [`proof_topic_sql_guard`] | a migration that names a `proof_*` object, a role, the sqlx bookkeeping table, or any object outside the topic's own namespace; `DROP DATABASE` / `SCHEMA` / `ROLE`; privilege changes; server-side file access; `SECURITY DEFINER` | -//! | [`handler`] | a handler that is not an allow-listed run backend — never a path, a URL, or a command line | +//! | `proof_topic_authoring::handler` | a handler that is not an allow-listed run backend — never a path, a URL, or a command line | //! //! Both gates run **before** anything is applied, and both refuse on doubt. //! //! # Where the pieces live //! -//! - [`section`] reads the RLM section's parts strictly, and carries the rest. +//! - the section reader (`proof_topic_authoring::section`) reads a set's +//! parts strictly, and carries the rest. //! - [`install`] is the engine: migrations, routes, rules, binding, journal. //! - [`routes`] is the **read** side of the routes an install recorded: the //! dynamic mux the challenge answers `/challenge/{topic_id}/…` from, behind @@ -32,7 +33,7 @@ //! - [`proof_topic_sql_guard`] is the migration deny-list (its own crate: it //! is pure text analysis, and keeping it separate means it can be reasoned //! about — and tested — without a database). -//! - [`handler`] is the run-backend allow-list. +//! - the run-backend allow-list (`proof_topic_authoring::handler`). //! //! # What this crate does not do //! @@ -52,26 +53,29 @@ )] pub mod gate; -pub mod handler; pub mod install; pub mod routes; -pub mod section; pub use gate::{disable, disabled, disabled_topics, enable, gate, set, Gate, GateState}; -pub use handler::{bound_runner, check_handler, resolve_handler, Handler, HandlerError}; pub use install::{ - applied_install, install_history, is_installed, latest_install, topic_routes, ExecutorBinding, - InstallReport, InstallRequest, InstallRow, InstallState, Installer, SetupSummary, + applied_install, install_history, installed_rules, is_installed, latest_install, + rlm_authored_rules, rules_source, topic_routes, ExecutorBinding, InstallReport, InstallRequest, + InstallRow, InstallState, InstalledRules, Installer, SetupSummary, RULES_SOURCE_RLM, VMS_PER_SUBMISSION, }; pub use proof_topic_sql_guard::{ blank_statements, check_migration, check_statement, is_topic_scoped, split_statements, - MigrationDenied, Statement, DENIED_DROP_KINDS, DENIED_FUNCTIONS, DENIED_OBJECTS, DENIED_VERBS, - OWNED_TABLES, OWNED_TABLE_PREFIX, + topic_sql_prefix, MigrationDenied, Statement, DENIED_DROP_KINDS, DENIED_FUNCTIONS, + DENIED_OBJECTS, DENIED_VERBS, OWNED_TABLES, OWNED_TABLE_PREFIX, }; pub use routes::{is_topic_id, PgTopicRoutes, Resolved, TopicRouteMux, TopicRouteSource}; -pub use section::{ - is_api_method, is_relative_api_path, is_reserved_api_path, read_section, ApiRoute, Migration, +// The set reader and the handler allow-list moved to `proof-topic-authoring`, +// where the guest can link them too (it has no database, so it cannot link +// this crate). Re-exported so every existing path keeps working, and so both +// sides validate an RLM's answer through the same code. +pub use proof_topic_authoring::{ + bound_runner, check_handler, is_api_method, is_relative_api_path, is_reserved_api_path, + read_section, resolve_handler, ApiRoute, Handler, HandlerError, Migration, SectionError, SectionPlan, MAX_APIS, MAX_MIGRATIONS, MAX_MIGRATION_SQL_BYTES, READ_KEYS, RESERVED_API_PREFIXES, }; @@ -98,6 +102,30 @@ pub enum InstallError { /// How many it carried. count: usize, }, + /// A migration names an object another registered topic also claims. + /// + /// `-` → `_` is injective, but its prefixes are not prefix-free: `aa` is a + /// prefix of `aa-b`'s mapped form `aa_b`, so the bare name `aa_b_scratch` + /// sits inside **both** namespaces. The per-topic guard cannot see that + /// (it sees one topic), so the install checks it against the registry and + /// refuses — a migration from `aa` must not reach `aa-b`'s tables in the + /// shared database. + #[error( + "migration {migration:?} names {object:?}, which topic {other:?} also claims: the `-` → \ + `_` mapping makes {object:?} read as both {this:?} and {other:?}, so applying it would \ + let one topic reach another's tables. Rename the object, or scope it with the \ + schema-qualified spelling ({other}.… / {this}.…), which is compared exactly" + )] + CrossTopicClaim { + /// The migration the object came from. + migration: String, + /// The object both topics claim. + object: String, + /// This topic. + this: String, + /// The registered topic that also claims it. + other: String, + }, /// A handler outside the allow-list. #[error("handler refused: {0}")] HandlerNotAllowed(String), @@ -116,14 +144,9 @@ pub enum InstallError { /// The ids this host registers. registered: Vec, }, - /// A part of the RLM section is malformed or carries an unknown key. - #[error("rlm.{part}: {why}")] - Section { - /// Which part (`migrations[0]`, `apis`, `rules`, …). - part: String, - /// What is wrong. - why: String, - }, + /// A part of the authored set is malformed or carries an unknown key. + #[error("{0}")] + Section(#[from] proof_topic_authoring::SectionError), /// The rule vector was refused by the shared shape check. #[error("rules: {0}")] Rules(String), @@ -133,6 +156,9 @@ pub enum InstallError { /// The database refused. #[error("db: {0}")] Db(String), + /// The set the topic's RLM authored was refused. + #[error("authored set: {0}")] + Authoring(String), } #[cfg(test)] diff --git a/crates/proof-topic-install/src/routes.rs b/crates/proof-topic-install/src/routes.rs index f2e02aa3b..0260c5002 100644 --- a/crates/proof-topic-install/src/routes.rs +++ b/crates/proof-topic-install/src/routes.rs @@ -6,7 +6,7 @@ //! exposes are the ones its install recorded — never a compiled-in list. //! //! A stored path is **relative** to the topic's own prefix -//! ([`crate::section::is_relative_api_path`]), so the resolver owns the +//! ([`proof_topic_authoring::is_relative_api_path`]), so the resolver owns the //! prefix and a row cannot carry an absolute path out of its topic's //! namespace. //! @@ -15,18 +15,18 @@ //! A request must not pay a table read, and it must not be answered from a //! table an install has since changed. An install is a **different process** //! (the operator's `proof-admin`), so no in-process signal can carry it: the -//! cache is therefore keyed by a **generation**, a cheap -//! `SELECT count(*) FROM proof_topic_api`, and a cached topic is used only -//! while the generation it was read under still holds. +//! cache is therefore keyed by a **generation** — the sum of +//! `proof_topic_route_revision` — and a cached topic is used only while the +//! generation it was read under still holds. //! -//! A count is a sound change signal here because the table is **append-only -//! for the application role** — `GRANT SELECT, INSERT` and nothing else -//! (migration `0025`), so a route row can be added and never rewritten or -//! removed. A generation that moved is therefore an install that ran, and the -//! next request reads the table again: an install is visible on the next -//! request, which is what "invalidated on install" means across processes. -//! [`TopicRouteMux::invalidate`] is the same thing for a caller in *this* -//! process. +//! The revision is what makes a **replacement** visible, and a row count is +//! not: when an RLM-authored install supersedes a bundle-authored set, the +//! install deletes the routes the new set does not claim and inserts its own, +//! and a count can be unchanged by that (delete three, insert three). The +//! revision is monotonic per topic and bumped in the same transaction as the +//! reconciliation, so it moves on any change to the route table — addition or +//! replacement. [`TopicRouteMux::invalidate`] is the in-process form of the +//! same thing. //! //! # What a resolution means //! @@ -43,8 +43,8 @@ use std::sync::{Arc, PoisonError, RwLock}; use async_trait::async_trait; use sqlx::PgPool; -use crate::section::ApiRoute; use crate::InstallError; +use proof_topic_authoring::ApiRoute; /// Whether `id` has the shape of a topic id. /// @@ -110,16 +110,22 @@ impl TopicRouteSource for PgTopicRoutes { crate::install::topic_routes(&self.pool, topic_id).await } - /// Rows in the route table. + /// Sum of the per-topic route revisions. /// - /// The append-only grant is what makes a count a change signal: an - /// install can only add, so the count moves exactly when the registry - /// does. See the module docs. + /// A revision is bumped in the same transaction as the route + /// reconciliation (migration `0028`), so the sum moves on **any** change + /// to the route table — an addition and a replacement alike. A row count + /// cannot see a replacement, which is why this is not one. See the module + /// docs. async fn generation(&self) -> Result { - let rows: i64 = sqlx::query_scalar("SELECT count(*) FROM proof_topic_api") - .fetch_one(&self.pool) - .await - .map_err(|e| InstallError::Db(e.to_string()))?; + // `sum` is NUMERIC in Postgres; the cast keeps the read an `i64` and + // the empty-table case a `0` rather than a decode error. + let rows: i64 = sqlx::query_scalar( + "SELECT COALESCE(sum(revision), 0)::bigint FROM proof_topic_route_revision", + ) + .fetch_one(&self.pool) + .await + .map_err(|e| InstallError::Db(e.to_string()))?; Ok(rows) } } @@ -169,7 +175,7 @@ impl TopicRouteMux { /// resolves. /// /// A path inside the challenge's admin namespace - /// ([`crate::section::is_reserved_api_path`]) is never served, whatever + /// ([`proof_topic_authoring::is_reserved_api_path`]) is never served, whatever /// the table holds: the install refuses to record one, and this is the /// read-side half for a row that predates that rule. /// @@ -187,7 +193,7 @@ impl TopicRouteMux { return Ok(Resolved::NotRegistered); } let path = path.trim().trim_matches('/'); - if crate::section::is_reserved_api_path(path) { + if proof_topic_authoring::is_reserved_api_path(path) { return Ok(Resolved::NotRegistered); } let routes = self.routes(topic_id).await?; diff --git a/crates/proof-topic-install/tests/handler_allowlist.rs b/crates/proof-topic-install/tests/handler_allowlist.rs index 429311032..b9e78b41b 100644 --- a/crates/proof-topic-install/tests/handler_allowlist.rs +++ b/crates/proof-topic-install/tests/handler_allowlist.rs @@ -1,17 +1,22 @@ //! The handler allow-list: which run backends an install may bind. //! -//! A bundle's `rlm` section is **operator-supplied JSON**, not a signed -//! document, so a handler name in it is untrusted input. The install binds it -//! to a run backend, and the only backends that exist are the generic -//! in-guest runner (Firecracker) and an operator-baked Harbor adaptor over -//! it. This suite is the proof that nothing else can be named — least of all -//! a path, a URL, or a command line, which is what an RLM section would -//! reach for if it could. +//! The set handed to an install — an operator's bundle section, or an RLM's +//! own authored answer — is **JSON**, and the handler name in it is untrusted +//! input either way. The install binds it to a run backend, and the only +//! backends that exist are the generic in-guest runner (Firecracker) and an +//! operator-baked Harbor adaptor over it. This suite is the proof that +//! nothing else can be named — least of all a path, a URL, or a command line, +//! which is what a set would reach for if it could. +//! +//! The allow-list lives in `proof-topic-authoring` (the guest links it too, +//! and the guest has no database so it cannot link the install); these tests +//! exercise it through the install's re-exports, which is the path an install +//! actually takes. #![allow(clippy::unwrap_used, clippy::expect_used)] -use proof_topic_install::handler::{allowed_list, resolve_handler, Handler, ALLOWED_HANDLERS}; -use proof_topic_install::{bound_runner, read_section, HandlerError, InstallError}; +use proof_topic_authoring::handler::{allowed_list, resolve_handler, Handler, ALLOWED_HANDLERS}; +use proof_topic_install::{bound_runner, read_section, HandlerError}; /// Every allowed spelling resolves, and the two families are the two that /// exist. @@ -112,10 +117,8 @@ fn the_section_reader_enforces_the_allow_list() { "arbitrary_binary", ] { let err = read_section(&format!(r#"{{"handler": "{bad}"}}"#)).expect_err(bad); - assert!( - matches!(err, InstallError::HandlerNotAllowed(_)), - "{bad:?}: {err:?}" - ); + assert_eq!(err.part, "handler", "{bad:?}: {err:?}"); + assert!(!err.why.is_empty(), "{bad:?}"); } } @@ -161,9 +164,12 @@ fn the_signed_document_wins_for_the_runner_and_the_section_names_the_family() { fn no_challenge_content_is_compiled_into_this_crate() { for src in [ include_str!("../src/lib.rs"), - include_str!("../src/handler.rs"), include_str!("../src/install.rs"), - include_str!("../src/section.rs"), + include_str!("../src/routes.rs"), + include_str!("../src/gate.rs"), + include_str!("../../proof-topic-authoring/src/lib.rs"), + include_str!("../../proof-topic-authoring/src/handler.rs"), + include_str!("../../proof-topic-authoring/src/section.rs"), include_str!("../../proof-topic-sql-guard/src/lib.rs"), ] { let non_test = src.split("#[cfg(test)]").next().unwrap_or(""); @@ -195,9 +201,12 @@ fn no_challenge_content_is_compiled_into_this_crate() { fn no_topic_literal_appears_in_this_crates_logic() { for src in [ include_str!("../src/lib.rs"), - include_str!("../src/handler.rs"), include_str!("../src/install.rs"), - include_str!("../src/section.rs"), + include_str!("../src/routes.rs"), + include_str!("../src/gate.rs"), + include_str!("../../proof-topic-authoring/src/lib.rs"), + include_str!("../../proof-topic-authoring/src/handler.rs"), + include_str!("../../proof-topic-authoring/src/section.rs"), include_str!("../../proof-topic-sql-guard/src/lib.rs"), ] { let non_test = src.split("#[cfg(test)]").next().unwrap_or(""); diff --git a/crates/proof-topic-install/tests/install_engine.rs b/crates/proof-topic-install/tests/install_engine.rs index 8c8369339..cbb8cefc4 100644 --- a/crates/proof-topic-install/tests/install_engine.rs +++ b/crates/proof-topic-install/tests/install_engine.rs @@ -86,6 +86,7 @@ fn request<'a>(doc: &'a TopicDocument, rlm: &'a str) -> InstallRequest<'a> { rlm_raw: rlm, registered_custom: vec![format!("{}-metric", doc.id)], skip_baseline: false, + authored: None, } } @@ -114,6 +115,164 @@ fn section(id: &str) -> String { .replace("{id}", id) } +/// The cross-topic namespace collision, end to end through the install. +/// +/// `-` → `_` is injective, but its prefixes are not prefix-free: `aa` is a +/// prefix of `aa-b`'s mapped form `aa_b`, so the bare name `aa_b_scratch` sits +/// inside **both** namespaces. The per-topic guard accepts it for each topic +/// on its own — it sees one topic — so a migration approved for `aa` could +/// read, modify, or drop a table belonging to `aa-b` in the shared database. +/// +/// The install is where the registry is visible, so it refuses there, before +/// the journal opens (nothing is written). +#[tokio::test] +async fn a_migration_that_reaches_a_sibling_topics_namespace_is_refused() { + let Some((_tp, pool)) = test_pool().await else { + return; + }; + let store = PgRlmStore::new(pool.clone()); + // A sibling is registered whose mapped prefix the shorter topic's + // migration would also match. + store + .put_topic_version(&topic("aa-b")) + .await + .expect("register the sibling"); + + let installer = Installer { + pool: &pool, + store: &store, + }; + let doc = topic("aa"); + // A migration whose object is inside **both** namespaces: `aa` reads it as + // `aa` + `b_scratch`, `aa-b` reads it as `aa-b` + `scratch`. + let colliding = r#"{ + "rules": [ + {"id": "no_short_circuit", "text": "the harness must run the task"} + ], + "migrations": [ + {"name": "0001_shared", "sql": "CREATE TABLE aa_b_scratch (id TEXT)"} + ] + }"#; + let err = installer + .install( + &request(&doc, colliding), + SetupSummary::Skipped { + reason: "--skip-baseline".into(), + }, + ) + .await + .expect_err("a migration reaching a sibling's namespace must be refused"); + + let InstallError::CrossTopicClaim { + ref object, + ref this, + ref other, + .. + } = err + else { + panic!("want CrossTopicClaim, got {err:?}"); + }; + assert_eq!(object, "aa_b_scratch"); + assert_eq!(this, "aa"); + assert_eq!(other, "aa-b"); + assert!( + err.to_string().contains("aa_b_scratch"), + "the refusal names the object: {err}" + ); + + // Nothing was written: the refusal is pre-flight, so there is no journal + // row and no table. + let rows: i64 = + sqlx::query_scalar("SELECT count(*) FROM proof_topic_install WHERE topic_id = 'aa'") + .fetch_one(&pool) + .await + .expect("count"); + assert_eq!(rows, 0, "a collision writes nothing at all"); + let exists: Option = sqlx::query_scalar("SELECT to_regclass('aa_b_scratch')::text") + .fetch_one(&pool) + .await + .expect("probe"); + assert_eq!(exists, None, "the colliding table was not created"); +} + +/// A **hyphenated** topic id installs its migrations end to end. +/// +/// Every real topic id is a hyphen slug (`[a-z0-9][a-z0-9-]{1,62}`), and a +/// bare SQL identifier cannot contain a hyphen. The defect this pins: the +/// migration guard required a literal `{topic_id}_` prefix, so its requirement +/// was unsatisfiable — `CREATE TABLE real-topic_scratch` is a syntax error at +/// the first `-`, and the underscore spelling was refused as unscoped. No real +/// topic could install a migration, and `topic install --drive-rlm` would only +/// discover it *after* the paid baseline. +/// +/// The suite's other tests all use `tb4`, which is why this went unnoticed. +#[tokio::test] +async fn a_hyphenated_topic_id_installs_its_migrations() { + let Some((tp, pool)) = test_pool().await else { + return; + }; + let store = PgRlmStore::new(pool.clone()); + let id = "fixture-topic-v0"; + let doc = topic(id); + let installer = Installer { + pool: &pool, + store: &store, + }; + // The table the migration creates, spelled with the identifier-safe + // prefix the guard maps the id to. `section()` appends `_scratch`. + let table = "fixture_topic_v0_scratch"; + let report = installer + .install( + &request(&doc, §ion("fixture_topic_v0")), + SetupSummary::Skipped { + reason: "--skip-baseline".into(), + }, + ) + .await + .expect("a hyphenated topic's bundle installs"); + + assert_eq!(report.topic_id, id); + assert_eq!( + report.migrations_applied, + ["0001_scratch", "0002_index"], + "both migrations apply for a hyphenated id" + ); + // The migration really ran: its table exists in this test's schema. + let exists: Option = + sqlx::query_scalar(&format!("SELECT to_regclass('{table}')::text")) + .fetch_one(&pool) + .await + .expect("probe the table"); + assert_eq!( + exists.as_deref(), + Some(table), + "the hyphenated topic's migration created its table" + ); + + // And a migration reaching a sibling topic is still refused, in both + // spellings — the fix widened the namespace, it did not remove it. + for sql in [ + "CREATE TABLE fixture_topic_v1_scratch (id TEXT)", + "CREATE TABLE topic_scratch (id TEXT)", + "CREATE TABLE proof_rule_version (id TEXT)", + ] { + let section = format!(r#"{{"migrations": [{{"name": "0003_bad", "sql": "{sql}"}}]}}"#); + let err = installer + .install( + &request(&doc, §ion), + SetupSummary::Skipped { reason: "x".into() }, + ) + .await + .expect_err("a sibling's table is not this topic's"); + assert!( + matches!(err, proof_topic_install::InstallError::MigrationDenied(_)), + "{sql:?} must be refused by the guard, got {err:?}" + ); + } + + tp.drop_schema().await.expect("drop"); +} + /// The happy path: every step applies, and the journal records it. #[tokio::test] async fn a_permitted_bundle_installs_and_the_journal_records_it() { @@ -211,6 +370,560 @@ async fn a_permitted_bundle_installs_and_the_journal_records_it() { tp.drop_schema().await.expect("drop"); } +/// The **RLM's own set** is what an install applies, and the journal records +/// its authorship per part. +/// +/// This is the whole point of the authorship pin, end to end against a real +/// database: when the topic's RLM authored a set, that set — not the +/// operator's bundle section — is what lands. The migrations, routes, +/// submission format, and pin policy come from the RLM's answer, and the +/// journal's `binding.authorship` names `rlm` as the author of every part, +/// with each part's digest. +/// +/// The operator's section is deliberately **different** in this test (a +/// different table, a different route, a different submission format), so a +/// run that quietly applied the bundle would be visible in every assertion. +#[tokio::test] +async fn the_rlm_authored_set_is_what_an_install_applies() { + let Some((tp, pool)) = test_pool().await else { + return; + }; + let store = PgRlmStore::new(pool.clone()); + let doc = topic("rlm-topic"); + let installer = Installer { + pool: &pool, + store: &store, + }; + // The operator's bundle: a *different* set, which must not be applied. + let operator_section = r#"{ + "rules": [{"id": "operator_rule", "text": "the operator's vector"}], + "migrations": [{"name": "0001_operator", "sql": "CREATE TABLE rlm_topic_operator (id TEXT)"}], + "apis": [{"path": "operator-route", "method": "GET"}], + "submission_format": {"kind": "operator-tar", "max_bytes": 1} + }"#; + // The RLM's answer: every part present, and every one of them different. + let authored = rlm_authored_set(); + let request = InstallRequest { + topic: &doc, + bundle_digest: digest(), + environment: "staging".into(), + rlm_raw: operator_section, + registered_custom: vec!["rlm-topic-metric".into()], + skip_baseline: false, + authored: Some(&authored), + }; + let report = installer + .install( + &request, + SetupSummary::Baselined { + rules_version: 1, + baseline_primary: "0.5".into(), + }, + ) + .await + .expect("the RLM's set installs"); + + // The RLM's parts landed, not the operator's. + assert_eq!(report.migrations_applied, ["0001_rlm"]); + assert_eq!(report.rule_ids, ["rlm_rule"]); + assert_eq!(report.apis, ["GET /rlm-status"]); + assert_eq!( + report.binding.submission_format_digest, + Some(proof_topic_authoring::digest_of( + &serde_json::json!({"kind": "rlm-tar", "max_bytes": 4_194_304}) + )), + "the digest is the RLM's submission format, not the bundle's" + ); + let rlm_table: Option = sqlx::query_scalar("SELECT to_regclass('rlm_topic_rlm')::text") + .fetch_one(&pool) + .await + .expect("probe"); + assert_eq!(rlm_table.as_deref(), Some("rlm_topic_rlm")); + let operator_table: Option = + sqlx::query_scalar("SELECT to_regclass('rlm_topic_operator')::text") + .fetch_one(&pool) + .await + .expect("probe"); + assert_eq!( + operator_table, None, + "the operator's migration never ran: the RLM's set is the topic's behavior" + ); + let routes = topic_routes(&pool, "rlm-topic").await.expect("routes"); + assert_eq!(routes.len(), 1); + assert_eq!(routes[0].path, "rlm-status"); + assert_authorship_journal_names_every_part(&pool, &authored).await; + + // A second install of the **same** topic with no authored set falls back + // to the operator's section and says so: the journal's provenance is what + // distinguishes the two, and the publish gate reads it. + let fallback = installer + .install( + &InstallRequest { + authored: None, + ..request + }, + SetupSummary::NotDriven { + reason: "no RLM set".into(), + }, + ) + .await + .expect("the operator's section still installs, with its own provenance"); + assert_eq!(fallback.migrations_applied, ["0001_operator"]); + let row = latest_install(&pool, "rlm-topic") + .await + .expect("journal") + .expect("a row"); + assert_eq!( + row.binding["authorship"]["source"], "topic_document", + "an install from the bundle says so: {row:?}" + ); + assert_eq!( + row.binding["authorship"]["parts"]["rules"]["source"], "topic_document", + "the operator's vector is not RLM-authored, whatever it contains" + ); + + tp.drop_schema().await.expect("drop"); +} + +/// The set a topic's RLM authors in this suite: every part present, and every +/// one of them distinguishable from the operator's bundle. +fn rlm_authored_set() -> proof_topic_authoring::TopicAuthoring { + use proof_topic_authoring::{AuthoredApi, AuthoredMigration, PinPolicy, TopicAuthoring}; + TopicAuthoring { + schema_version: proof_topic_authoring::AUTHORING_SCHEMA, + topic_id: "rlm-topic".into(), + rules: vec![proof_task::ChecklistRule { + id: "rlm_rule".into(), + text: "the RLM's own vector".into(), + }], + migrations: vec![AuthoredMigration { + name: "0001_rlm".into(), + sql: "CREATE TABLE rlm_topic_rlm (id TEXT, note TEXT)".into(), + }], + apis: vec![AuthoredApi { + path: "rlm-status".into(), + method: "GET".into(), + summary: "the RLM's own route".into(), + }], + submission_format: serde_json::json!({"kind": "rlm-tar", "max_bytes": 4_194_304}), + pin_policy: PinPolicy { + epsilon_nll_min: Some(0.05), + ..PinPolicy::none() + }, + } +} + +/// The journal says **who authored every part**, with a digest each. +/// +/// This is what makes "the RLM authored this topic" a fact an audit reads back +/// per part rather than a label on the row as a whole, and it is the entry the +/// publish gate's provenance read is about. +async fn assert_authorship_journal_names_every_part( + pool: &PgPool, + authored: &proof_topic_authoring::TopicAuthoring, +) { + let row = latest_install(pool, "rlm-topic") + .await + .expect("journal") + .expect("a row"); + let authorship = &row.binding["authorship"]; + assert_eq!(authorship["source"], "rlm"); + assert_eq!(authorship["digest"], authored.digest()); + for part in proof_topic_authoring::PARTS { + assert_eq!( + authorship["parts"][part]["source"], "rlm", + "{part} must name the RLM as its author: {authorship}" + ); + assert!( + authorship["parts"][part]["digest"] + .as_str() + .is_some_and(|d| d.starts_with("sha256:")), + "{part} must carry its own digest: {authorship}" + ); + } + assert_eq!(authorship["parts"]["migrations"]["names"][0], "0001_rlm"); + assert_eq!(authorship["parts"]["apis"]["routes"][0], "GET /rlm-status"); +} + +/// A migration whose **function body** reaches a sibling's namespace is +/// refused. +/// +/// Greptile's second P1, reproduced here before fixing. The collision check +/// read `statement.blanked`, which blanks dollar-quoted bodies — so the object +/// names *inside* a body were never compared to the registry. Topic `aa` could +/// install `aa_delete_sibling()` whose body is `DELETE FROM aa_b_scratch` +/// while sibling `aa-b` was registered, and calling it would delete the +/// sibling's rows. +/// +/// The body is what runs, so the body is what the check must read +/// (`statement.bodies`). +#[tokio::test] +async fn a_function_body_that_reaches_a_sibling_is_refused() { + let Some((tp, pool)) = test_pool().await else { + return; + }; + let store = PgRlmStore::new(pool.clone()); + store + .put_topic_version(&topic("aa-b")) + .await + .expect("register the sibling"); + + let installer = Installer { + pool: &pool, + store: &store, + }; + let doc = topic("aa"); + let body = r#"{ + "rules": [{"id": "no_short_circuit", "text": "the harness must run the task"}], + "migrations": [{ + "name": "0001_body", + "sql": "CREATE FUNCTION aa_delete_sibling() RETURNS void AS $$ DELETE FROM aa_b_scratch $$ LANGUAGE sql" + }] + }"#; + let err = installer + .install( + &request(&doc, body), + SetupSummary::Skipped { + reason: "--skip-baseline".into(), + }, + ) + .await + .expect_err("a function body reaching a sibling must be refused"); + let InstallError::CrossTopicClaim { + ref object, + ref this, + ref other, + .. + } = err + else { + panic!("want CrossTopicClaim, got {err:?}"); + }; + assert_eq!(object, "aa_b_scratch"); + assert_eq!(this, "aa"); + assert_eq!(other, "aa-b"); + // Nothing was written: the refusal is pre-flight. + let rows: i64 = + sqlx::query_scalar("SELECT count(*) FROM proof_topic_install WHERE topic_id = 'aa'") + .fetch_one(&pool) + .await + .expect("count"); + assert_eq!(rows, 0, "a collision writes nothing at all"); + let exists: Option = sqlx::query_scalar("SELECT to_regproc('aa_delete_sibling')::text") + .fetch_one(&pool) + .await + .expect("probe"); + assert_eq!(exists, None, "the function was not created"); + + tp.drop_schema().await.expect("drop"); +} + +/// Two **unpublished** installs whose prefixes overlap cannot both claim the +/// same object. +/// +/// Greptile's first P1, reproduced here before fixing. The collision check +/// enumerated `proof_topic_version` only, so a topic that had installed but +/// not yet published was invisible to it: `aa` and `aa-b` could both install +/// and both create `aa_b_scratch`. A topic claims a namespace by installing +/// into it, so the check has to read the **journal** too — and it has to do so +/// under the claim's own lock, or two concurrent installs each read a registry +/// that lacks the other. +/// +/// This test installs `aa` (unpublished: it has a journal row and no +/// `proof_topic_version` row for `aa` beyond what the install itself writes), +/// then installs `aa-b` and asserts the refusal. +#[tokio::test] +async fn two_unpublished_installs_cannot_claim_the_same_object() { + let Some((tp, pool)) = test_pool().await else { + return; + }; + let store = PgRlmStore::new(pool.clone()); + let installer = Installer { + pool: &pool, + store: &store, + }; + // `aa` installs a table named `aa_b_scratch`, which `aa-b` also claims. + let first = r#"{ + "rules": [{"id": "no_short_circuit", "text": "the harness must run the task"}], + "migrations": [{"name": "0001_shared", "sql": "CREATE TABLE aa_b_scratch (id TEXT)"}] + }"#; + installer + .install( + &request(&topic("aa"), first), + SetupSummary::Skipped { + reason: "--skip-baseline".into(), + }, + ) + .await + .expect("the first install is alone and therefore permitted"); + + // `aa-b` is **not** registered as a published document; the only record + // that `aa` exists is its install row. The check must still see it. + let err = installer + .install( + &request(&topic("aa-b"), first), + SetupSummary::Skipped { + reason: "--skip-baseline".into(), + }, + ) + .await + .expect_err("an unpublished sibling's claim must be visible"); + assert!( + matches!(err, InstallError::CrossTopicClaim { .. }), + "{err:?}" + ); + assert!( + err.to_string().contains("aa_b_scratch"), + "the refusal names the object: {err}" + ); + + tp.drop_schema().await.expect("drop"); +} + +/// The claim is atomic: two installs racing for overlapping namespaces cannot +/// both win. +/// +/// The read-then-claim is one transaction under an advisory lock +/// (`claim_namespace`), so the second install sees the first's `pending` row. +/// Without the lock both read a registry lacking the other and both commit — +/// the same defect with a race on top of it. +#[tokio::test] +async fn two_concurrent_installs_cannot_both_claim_a_namespace() { + let Some((tp, pool)) = test_pool().await else { + return; + }; + let store = PgRlmStore::new(pool.clone()); + let section = r#"{ + "rules": [{"id": "no_short_circuit", "text": "the harness must run the task"}], + "migrations": [{"name": "0001_shared", "sql": "CREATE TABLE aa_b_scratch (id TEXT)"}] + }"#; + let doc_a = topic("aa"); + let doc_b = topic("aa-b"); + let req_a = request(&doc_a, section); + let req_b = request(&doc_b, section); + let installer_a = Installer { + pool: &pool, + store: &store, + }; + let installer_b = Installer { + pool: &pool, + store: &store, + }; + let (a, b) = tokio::join!( + installer_a.install( + &req_a, + SetupSummary::Skipped { + reason: "--skip-baseline".into(), + }, + ), + installer_b.install( + &req_b, + SetupSummary::Skipped { + reason: "--skip-baseline".into(), + }, + ) + ); + // Exactly one wins; the other is refused by name. Both succeeding would + // mean both created `aa_b_scratch`. + let refused = [&a, &b] + .iter() + .filter(|r| { + matches!( + r, + Err(InstallError::CrossTopicClaim { .. } | InstallError::MigrationFailed { .. }) + ) + }) + .count(); + assert_eq!( + refused, 1, + "exactly one of the two racing installs must be refused: a={a:?} b={b:?}" + ); + + tp.drop_schema().await.expect("drop"); +} + +/// A re-install **replaces** the route set: a route the new set does not +/// claim stops resolving. +/// +/// Greptile's P1: `register_apis` was insert-only (`ON CONFLICT DO NOTHING`) +/// and the table was `SELECT, INSERT` for the application role, so when an +/// RLM-authored install superseded a bundle-authored one, the bundle's routes +/// stayed in the table and the mux — which loads every row for the topic — +/// kept answering them. A miner would reach an endpoint the topic's current +/// install does not declare, while the journal said the newer set was in +/// force. +#[tokio::test] +async fn a_re_install_replaces_the_route_set() { + use proof_topic_authoring::{AuthoredApi, AuthoredMigration, PinPolicy, TopicAuthoring}; + + let Some((tp, pool)) = test_pool().await else { + return; + }; + let store = PgRlmStore::new(pool.clone()); + let doc = topic("routes-topic"); + let installer = Installer { + pool: &pool, + store: &store, + }; + // First install: the operator's bundle claims two routes. + let bundle = r#"{ + "rules": [{"id": "operator_rule", "text": "the operator's vector"}], + "apis": [ + {"path": "old-get", "method": "GET"}, + {"path": "old-post", "method": "POST"} + ] + }"#; + installer + .install( + &request(&doc, bundle), + SetupSummary::NotDriven { reason: "x".into() }, + ) + .await + .expect("the bundle installs"); + let mut routes = topic_routes(&pool, "routes-topic").await.expect("routes"); + routes.sort_by(|a, b| a.path.cmp(&b.path)); + assert_eq!(routes.len(), 2, "both bundle routes are registered"); + + // Second install: the RLM's set claims **one** route, and not the old two. + let authored = TopicAuthoring { + schema_version: proof_topic_authoring::AUTHORING_SCHEMA, + topic_id: "routes-topic".into(), + rules: vec![proof_task::ChecklistRule { + id: "rlm_rule".into(), + text: "the RLM's own vector".into(), + }], + migrations: vec![AuthoredMigration { + name: "0001_rlm".into(), + sql: "CREATE TABLE routes_topic_rlm (id TEXT)".into(), + }], + apis: vec![AuthoredApi { + path: "rlm-status".into(), + method: "GET".into(), + summary: "the RLM's own route".into(), + }], + submission_format: serde_json::json!({"kind": "rlm-tar", "max_bytes": 4_194_304}), + pin_policy: PinPolicy::none(), + }; + let report = installer + .install( + &InstallRequest { + topic: &doc, + bundle_digest: digest(), + environment: "staging".into(), + rlm_raw: bundle, + registered_custom: vec!["routes-topic-metric".into()], + skip_baseline: false, + authored: Some(&authored), + }, + SetupSummary::Baselined { + rules_version: 1, + baseline_primary: "0.5".into(), + }, + ) + .await + .expect("the RLM's set installs"); + assert_eq!(report.apis, ["GET /rlm-status"]); + + // The route table now holds **only** the RLM's route: the old ones were + // reconciled away, not left resolving. + let routes = topic_routes(&pool, "routes-topic").await.expect("routes"); + assert_eq!( + routes.len(), + 1, + "a replacement leaves only the new set: {routes:?}" + ); + assert_eq!(routes[0].path, "rlm-status"); + + // …and the mux agrees: the old paths are gone, the new one resolves. + let mux = TopicRouteMux::new(Arc::new(PgTopicRoutes::new(pool.clone()))); + assert_eq!( + mux.resolve("routes-topic", "GET", "old-get") + .await + .expect("resolve"), + Resolved::NotRegistered, + "a route the current set does not claim must not resolve" + ); + assert_eq!( + mux.resolve("routes-topic", "POST", "old-post") + .await + .expect("resolve"), + Resolved::NotRegistered + ); + assert!( + matches!( + mux.resolve("routes-topic", "GET", "rlm-status") + .await + .expect("resolve"), + Resolved::Route(_) + ), + "the new set's route resolves" + ); + + tp.drop_schema().await.expect("drop"); +} + +/// A replacement that keeps the **row count** the same still moves the mux's +/// generation. +/// +/// The cache's change signal used to be `count(*) FROM proof_topic_api`, which +/// cannot see a replacement: delete one, insert one, and the count is +/// unchanged while the *routes* are not. The signal is now the sum of the +/// per-topic route revisions, bumped in the same transaction as the +/// reconciliation. +#[tokio::test] +async fn a_same_count_replacement_still_moves_the_generation() { + let Some((tp, pool)) = test_pool().await else { + return; + }; + let store = PgRlmStore::new(pool.clone()); + let doc = topic("swap-topic"); + let installer = Installer { + pool: &pool, + store: &store, + }; + let first = r#"{ + "rules": [{"id": "operator_rule", "text": "the operator's vector"}], + "apis": [{"path": "one", "method": "GET"}] + }"#; + installer + .install( + &request(&doc, first), + SetupSummary::NotDriven { reason: "x".into() }, + ) + .await + .expect("first install"); + + let source = PgTopicRoutes::new(pool.clone()); + let before = proof_topic_install::TopicRouteSource::generation(&source) + .await + .expect("generation"); + + // A second install with **one** route as well: the count is unchanged, so + // a count-based generation would miss it entirely. + let second = r#"{ + "rules": [{"id": "operator_rule", "text": "the operator's vector"}], + "apis": [{"path": "two", "method": "GET"}] + }"#; + installer + .install( + &request(&doc, second), + SetupSummary::NotDriven { reason: "x".into() }, + ) + .await + .expect("second install"); + let after = proof_topic_install::TopicRouteSource::generation(&source) + .await + .expect("generation"); + assert!( + after > before, + "a same-count replacement must move the generation: {before} -> {after}" + ); + let routes = topic_routes(&pool, "swap-topic").await.expect("routes"); + assert_eq!(routes.len(), 1); + assert_eq!(routes[0].path, "two"); + + tp.drop_schema().await.expect("drop"); +} + /// The dynamic mux **reads** what an install **wrote**: the routes the /// challenge answers `/challenge/{topic_id}/…` from are the rows this install /// recorded, and a path nobody registered is not invented. @@ -236,6 +949,10 @@ async fn mux_reads_what_the_install_wrote(pool: &PgPool) { Resolved::NotRegistered ); + // A later install in **another process** writes a route and bumps the + // topic's revision — the pair the mux's generation probe watches. Writing + // the row alone would be a fixture that no install produces, and the cache + // would (correctly) not notice it. sqlx::query( "INSERT INTO proof_topic_api (topic_id, path, method, summary) \ VALUES ('tb4', 'v2/runs', 'GET', 'a later install')", @@ -243,6 +960,14 @@ async fn mux_reads_what_the_install_wrote(pool: &PgPool) { .execute(pool) .await .expect("append the second install's route"); + sqlx::query( + "INSERT INTO proof_topic_route_revision (topic_id, revision) VALUES ('tb4', 1) \ + ON CONFLICT (topic_id) DO UPDATE \ + SET revision = proof_topic_route_revision.revision + 1", + ) + .execute(pool) + .await + .expect("bump the revision the way an install does"); let later = mux.resolve("tb4", "GET", "v2/runs").await.expect("resolve"); assert!( matches!(&later, Resolved::Route(r) if r.summary == "a later install"), @@ -319,6 +1044,233 @@ async fn a_denied_migration_writes_nothing_at_all() { tp.drop_schema().await.expect("drop"); } +/// The admission read binds the install to **the rule version it recorded**. +/// +/// The defect this pins: the publish gate composed "the newest install is +/// `applied`" and "the newest rule version is `rlm`" as two independent +/// predicates. A topic whose install landed rule version 1 from the signed +/// document (`topic_document`) was therefore admitted as soon as *any* later +/// version happened to be RLM-authored — so it could open with the operator's +/// vector in force, which is exactly the operator-cloned document the gate +/// exists to refuse. +#[tokio::test] +async fn the_admission_read_binds_the_install_to_the_version_it_recorded() { + use proof_topic_install::{installed_rules, InstallState, InstalledRules}; + + let Some((tp, pool)) = test_pool().await else { + return; + }; + let store = PgRlmStore::new(pool.clone()); + let doc = topic("tb4"); + let installer = Installer { + pool: &pool, + store: &store, + }; + // The install seeds version 1 from the signed document. + installer + .install( + &request(&doc, §ion("tb4")), + SetupSummary::NotDriven { reason: "x".into() }, + ) + .await + .expect("install"); + + // Nothing to admit yet: the version the install landed is the operator's. + let before = installed_rules(&pool, "tb4").await.expect("read"); + assert!( + matches!( + before, + InstalledRules::NotRlmAuthored { + version: Some(1), + .. + } + ), + "version 1 is topic_document-sourced, so the topic is not admissible: {before:?}" + ); + + // An **unrelated later** version is RLM-authored. The install still + // recorded version 1, so the topic must stay refused: reading "the newest + // rule version is rlm" would admit it. + let rlm_v2 = store + .current_rules("tb4") + .await + .expect("rules") + .expect("version 1") + .next( + proof_rlm::RuleSource::Rlm, + vec![proof_task::ChecklistRule { + id: "r-1".into(), + text: "the RLM's own rule".into(), + }], + ) + .expect("v2"); + store.put_rules(&rlm_v2).await.expect("write v2"); + let after = installed_rules(&pool, "tb4").await.expect("read"); + assert!( + matches!( + after, + InstalledRules::NotRlmAuthored { + version: Some(1), + .. + } + ), + "the install recorded version 1; a newer RLM version must not admit it: {after:?}" + ); + + // A **new install row** that records version 2 is what admits the topic. + // This is the operator's real path: re-install after the RLM wrote rules. + let report = installer + .install( + &request(&doc, §ion("tb4")), + SetupSummary::Baselined { + rules_version: 2, + baseline_primary: "0.42".into(), + }, + ) + .await + .expect("re-install"); + assert_eq!( + report.rules_version, 2, + "the install keeps the RLM's version" + ); + assert_eq!( + installed_rules(&pool, "tb4").await.expect("read"), + InstalledRules::RlmAuthored { version: 2 }, + "an applied install recording an rlm-sourced version is the admission" + ); + + // And the states that are not `applied` are refused as their own shape. + sqlx::query( + "INSERT INTO proof_topic_install \ + (topic_id, bundle_digest, environment, state, rules_version) \ + VALUES ('tb4', 'sha256:' || repeat('ab', 32), 'staging', $1, 2)", + ) + .bind(InstallState::Failed.as_str()) + .execute(&pool) + .await + .expect("append a failed row"); + assert_eq!( + installed_rules(&pool, "tb4").await.expect("read"), + InstalledRules::NotApplied { + state: Some("failed".into()) + }, + "the newest row being `failed` refuses regardless of rule provenance" + ); + + tp.drop_schema().await.expect("drop"); +} + +/// An `operator` edit that supersedes the RLM's vector refuses the topic. +/// +/// The other direction of the same defect: the install recorded an +/// RLM-authored version, but the vector **in force** is an operator's. A gate +/// that bound only the install's version would admit the topic and serve rules +/// no RLM wrote. +#[tokio::test] +async fn an_operator_edit_in_force_refuses_the_topic() { + use proof_rlm::RuleSource; + use proof_task::ChecklistRule; + use proof_topic_install::{installed_rules, InstalledRules}; + + let Some((tp, pool)) = test_pool().await else { + return; + }; + let store = PgRlmStore::new(pool.clone()); + let doc = topic("tb4"); + let installer = Installer { + pool: &pool, + store: &store, + }; + installer + .install( + &request(&doc, §ion("tb4")), + SetupSummary::NotDriven { reason: "x".into() }, + ) + .await + .expect("install"); + + // The RLM authors version 2, and an install records it. + let rlm = store + .current_rules("tb4") + .await + .expect("rules") + .expect("v1") + .next( + RuleSource::Rlm, + vec![ChecklistRule { + id: "rlm-1".into(), + text: "the RLM's rule".into(), + }], + ) + .expect("v2"); + store.put_rules(&rlm).await.expect("write v2"); + installer + .install( + &request(&doc, §ion("tb4")), + SetupSummary::Baselined { + rules_version: 2, + baseline_primary: "0.5".into(), + }, + ) + .await + .expect("re-install"); + assert_eq!( + installed_rules(&pool, "tb4").await.expect("read"), + InstalledRules::RlmAuthored { version: 2 }, + "the install landed the RLM's vector" + ); + + // An operator edit supersedes it. Both halves of the gate have to hold: + // the installed version is still `rlm`, but it is no longer in force. + let edited = store + .current_rules("tb4") + .await + .expect("rules") + .expect("v2") + .next( + RuleSource::Operator, + vec![ChecklistRule { + id: "hand-1".into(), + text: "an operator's rule".into(), + }], + ) + .expect("v3"); + store.put_rules(&edited).await.expect("write v3"); + assert_eq!( + installed_rules(&pool, "tb4").await.expect("read"), + InstalledRules::SupersededByOperator { + installed: 2, + in_force: 3, + provenance: "operator".into(), + }, + "an operator vector in force is not an admission, even when the install landed an RLM one" + ); + + // An RLM rewrite of its own rules stays admitted: that is the autonomy this + // gate protects, not something it may refuse. + let rewritten = store + .current_rules("tb4") + .await + .expect("rules") + .expect("v3") + .next( + RuleSource::Rlm, + vec![ChecklistRule { + id: "rlm-2".into(), + text: "the RLM rewrote its own rule".into(), + }], + ) + .expect("v4"); + store.put_rules(&rewritten).await.expect("write v4"); + assert_eq!( + installed_rules(&pool, "tb4").await.expect("read"), + InstalledRules::RlmAuthored { version: 2 }, + "an RLM rewrite (rlm -> rlm) stays admitted" + ); + + tp.drop_schema().await.expect("drop"); +} + /// A re-run resumes: migrations already in the journal are skipped, the rules /// version is not bumped, and the routes are not duplicated. #[tokio::test] @@ -530,9 +1482,15 @@ async fn an_arbitrary_handler_never_reaches_a_write() { .install(&req, SetupSummary::NotDriven { reason: "x".into() }) .await .expect_err(bad); + // The refusal names the part and why: a path, a URL, or a command + // line is *not an identifier*; a well-formed id is *not on the list*. + let InstallError::Section(section) = &err else { + panic!("{bad}: expected a Section refusal, got {err:?}"); + }; + assert_eq!(section.part, "handler", "{bad}"); assert!( - matches!(err, InstallError::HandlerNotAllowed(_)), - "{bad}: {err:?}" + section.why.contains("vm_backed") || section.why.contains("not an identifier"), + "{bad}: {err}" ); assert!( latest_install(&pool, "tb4") diff --git a/crates/proof-topic-ops/Cargo.toml b/crates/proof-topic-ops/Cargo.toml index bfb866de4..24235ff0a 100644 --- a/crates/proof-topic-ops/Cargo.toml +++ b/crates/proof-topic-ops/Cargo.toml @@ -13,7 +13,9 @@ db = { path = "../db" } proof-eval = { path = "../proof-eval" } proof-rlm = { path = "../proof-rlm" } proof-rlm-store = { path = "../proof-rlm-store" } +proof-score = { path = "../proof-score" } proof-task = { path = "../proof-task" } +proof-topic-authoring = { path = "../proof-topic-authoring" } proof-topic-bundle = { path = "../proof-topic-bundle" } proof-topic-install = { path = "../proof-topic-install" } proof-topic-setup = { path = "../proof-topic-setup" } diff --git a/crates/proof-topic-ops/src/drive.rs b/crates/proof-topic-ops/src/drive.rs index 04bb38831..34c0b4ac4 100644 --- a/crates/proof-topic-ops/src/drive.rs +++ b/crates/proof-topic-ops/src/drive.rs @@ -43,19 +43,43 @@ pub struct DriveOutcome { pub vm_id: String, /// The lifecycle state the driver left the topic in. pub state: String, + /// The whole set the RLM authored — the topic's behavior, which the + /// install applies in place of the bundle's section. + /// + /// `None` means the run returned a bare rule vector. The drive **refuses** + /// before returning in that case ([`SetupError::IncompleteAuthoring`]), + /// so an outcome with `authored: None` is only reachable from a run that + /// stopped earlier — and the install then falls back to the operator's + /// section, whose `topic_document` provenance the publish gate refuses to + /// open a topic on. + pub authored: Option>, } impl DriveOutcome { /// One-line summary for the operator. #[must_use] pub fn summary(&self) -> String { + let parts = match &self.authored { + Some(set) => format!( + ", authoring {} ({} migrations, {} apis, pin policy {})", + set.digest(), + set.migrations.len(), + set.apis.len(), + if set.pin_policy.is_empty() { + "none" + } else { + "set" + } + ), + None => ", no authored set".to_owned(), + }; match self.baseline_primary { Some(v) => format!( - "the RLM wrote rules v{} and measured a baseline of {v} on vm {}", + "the RLM wrote rules v{} and measured a baseline of {v} on vm {}{parts}", self.rules_version, self.vm_id ), None => format!( - "the RLM wrote rules v{} on vm {} (no baseline: --skip-baseline)", + "the RLM wrote rules v{} on vm {} (no baseline: --skip-baseline){parts}", self.rules_version, self.vm_id ), } @@ -255,5 +279,6 @@ fn outcome_summary(outcome: SetupOutcome) -> DriveOutcome { } else { "baselining (no baseline measured: --skip-baseline)".to_owned() }, + authored: outcome.authored, } } diff --git a/crates/proof-topic-ops/src/lib.rs b/crates/proof-topic-ops/src/lib.rs index f339bf92e..0cd080d87 100644 --- a/crates/proof-topic-ops/src/lib.rs +++ b/crates/proof-topic-ops/src/lib.rs @@ -34,7 +34,7 @@ pub mod seal; pub use drive::{drive, DriveOutcome}; pub use publish::PublishTarget; -pub use seal::{baseline, seal, BaselineReport, SealArgs, SealOutcome}; +pub use seal::{baseline, lifecycle, seal, BaselineReport, LifecycleReport, SealArgs, SealOutcome}; /// Why an operator procedure refused. /// diff --git a/crates/proof-topic-ops/src/seal.rs b/crates/proof-topic-ops/src/seal.rs index 883959567..9f03a1806 100644 --- a/crates/proof-topic-ops/src/seal.rs +++ b/crates/proof-topic-ops/src/seal.rs @@ -9,10 +9,10 @@ //! //! ```text //! # 1. What did the RLM measure, and what must the open document seal? -//! proof-admin topic baseline tb4 +//! proof-admin topic baseline //! //! # 2. Sign the open document carrying that commitment, then: -//! proof-admin topic seal tb4 --document open.json --publish \ +//! proof-admin topic seal --document open.json --publish \ //! --admin-url https:// --admin-token-file /run/base/proof/admin_tokens //! ``` //! @@ -96,6 +96,14 @@ pub struct BaselineReport { pub metrics_commitment: String, /// The document's current status (a draft is not scorable). pub document_status: TopicStatus, + /// Whether this measurement is a **degenerate bar**: a relative-win family + /// whose measured primary is ~zero, so no challenger could ever clear it + /// and sealing it will be refused (`SetupError::DegenerateBar`). + /// + /// Surfaced here because this is the read that happens **before** the + /// operator signs an `open` document: warning at seal time is correct but + /// late — the document would already carry a number that cannot be sealed. + pub degenerate_bar: bool, } impl BaselineReport { @@ -135,6 +143,16 @@ impl SealOutcome { } } +/// Whether a measured primary is a bar no challenger could ever clear. +/// +/// The same predicate `mark_sealed` refuses on, named once so the early read +/// (`topic baseline`) and the seal cannot disagree about what "degenerate" +/// means. A missing primary is not degenerate — it is missing evidence, which +/// the scoring gate reports as such. +fn baseline_is_degenerate(document: &TopicDocument, primary_value: f64) -> bool { + proof_score::family_bar_is_degenerate(document.metric.family, Some(primary_value)) +} + /// `topic baseline`: what the RLM measured, and what to seal. /// /// # Errors @@ -168,6 +186,7 @@ pub async fn baseline( holdout_commitment: document.holdout_commitment.clone(), metrics_commitment: commitment, document_status: document.status, + degenerate_bar: baseline_is_degenerate(&document, measured.primary_value), }) } @@ -401,6 +420,156 @@ fn next_seal_steps(topic_id: &str, commitment: &str) -> String { ) } +/// Where a topic is in its RLM lifecycle, and what it is waiting on. +/// +/// `topic install --drive-rlm` prints one line and then **nothing** until the +/// whole run returns: provisioning a VM, the RLM's `propose_rules` job, and a +/// paid baseline can legitimately take hours, and a run that is working is +/// indistinguishable from one that is stuck if the only observable is "no +/// output yet". The durable progress is the lifecycle journal, so this is the +/// read that makes a long run legible — and, when a run dies, the last +/// transition is what says how far it got. +#[derive(Debug, Clone, PartialEq)] +pub struct LifecycleReport { + /// Canonical topic slug (the alias resolved). + pub topic_id: String, + /// The state the newest transition left the topic in. + pub state: String, + /// Rule version in force, when the RLM has written one. + pub rules_version: Option, + /// Whether that version is RLM-authored (`proof_rule_version.source`). + pub rules_source: Option, + /// Whether a baseline has been measured (and under which version). + pub baseline_rules_version: Option, + /// Every transition, oldest first: `from -> to (event)`. + pub history: Vec, +} + +impl LifecycleReport { + /// What the operator should do next, read off the state. + #[must_use] + pub fn next_steps(&self) -> String { + match self.state.as_str() { + "draft" => "Nothing has run yet. Drive the RLM: `proof-admin topic install \ + --bundle --env --drive-rlm --owner-approved`." + .to_owned(), + "owner_presend" | "awaiting_owner_keys" => { + "The lifecycle is waiting on the owner (approval, then the owner key file). \ + `--drive-rlm` needs `--owner-approved` and \ + `PROOF_RLM_OWNER_INFERENCE_KEY_FILE` present." + .to_owned() + } + "provisioning" => "The VM is being created. A `--drive-rlm` run is in flight if the \ + CLI is still attached; if it is not, this state is where it \ + stopped — re-run the same command to resume." + .to_owned(), + "baselining" => { + // A measured baseline is only sealable while the rules it was + // measured under are still the ones in force: a vector that + // moved since would seal a bar nobody is scored against, and + // `mark_sealed` refuses it. Saying "seal it" from the mere + // existence of a row would send the operator into that refusal. + // + // The condition mirrors `baseline_still_in_force` exactly — + // the version in force must *equal* the measured one — so a + // topic with no rule row at all is stale too, not sealable. + let sealable = self.baseline_rules_version.is_some() + && self.baseline_rules_version == self.rules_version; + if self.baseline_rules_version.is_some() && !sealable { + format!( + "A baseline is measured, but under rule version {} — version {} is in \ + force now, so sealing it would publish a bar measured under rules nobody \ + scores with. Re-run the baseline under the rules in force \ + (`proof-admin topic install --bundle --env \ + --drive-rlm --owner-approved`), then seal that number.", + self.baseline_rules_version + .map_or_else(|| "none".to_owned(), |v| v.to_string()), + self.rules_version + .map_or_else(|| "none".to_owned(), |v| v.to_string()) + ) + } else if sealable { + "A baseline is measured. Seal it: `proof-admin topic baseline \ + ` then `topic seal … --publish`." + .to_owned() + } else { + "No baseline yet. The RLM's `propose_rules` → `baseline` jobs run here; a \ + paid baseline can take hours. If no CLI is attached, the run stopped — \ + re-run `topic install --drive-rlm --owner-approved` to resume from this \ + state." + .to_owned() + } + } + "open" => "The topic is sealed and open in the registry. A seal without \ + `--publish` leaves the host serving the **previous** document, so \ + miners cannot reach it yet: confirm the open version is live \ + (`proof-admin topic show ` reports the published document) and \ + that the host is scorable (`can_score` on `GET /v1/status`) before \ + treating it as submitable. If it is not published, re-run \ + `topic seal … --publish` (the seal is already recorded; the same \ + document publishes as-is)." + .to_owned(), + other => format!("State {other:?} is not one this command gives advice for."), + } + } +} + +/// `topic lifecycle`: read the journal and the provenance, and say what is next. +/// +/// Read-only: it never writes, never moves the lifecycle, and never spends. +/// +/// # Errors +/// +/// [`OpsError::error`] when the topic is unknown or the store cannot be read. +pub async fn lifecycle(pool: &sqlx::PgPool, topic_id: &str) -> Result { + let store = PgRlmStore::new(pool.clone()); + let (canonical, _, _) = resolve_topic(&store, topic_id).await?; + let lc = store + .lifecycle(&canonical) + .await + .map_err(|e| OpsError::error(format!("{canonical} lifecycle: {e}")))? + .ok_or_else(|| { + OpsError::error(format!( + "topic {canonical:?} has no lifecycle rows: nothing has driven it yet. \ + `proof-admin topic install --drive-rlm --owner-approved` writes the first one." + )) + })?; + let rules_version = store + .current_rules(&canonical) + .await + .map_err(|e| OpsError::error(format!("{canonical} rules: {e}")))? + .map(|r| r.version); + let rules_source = store + .current_rules_source(&canonical) + .await + .map_err(|e| OpsError::error(format!("{canonical} rule provenance: {e}")))? + .map(|s| format!("{s:?}").to_lowercase()); + let baseline_rules_version = store + .baseline(&canonical) + .await + .map_err(|e| OpsError::error(format!("{canonical} baseline: {e}")))? + .map(|b| b.rules_version); + let history = lc + .history + .iter() + .map(|t| { + format!( + "{} -> {} ({})", + t.from.as_str(), + t.to.as_str(), + t.event.as_str() + ) + }) + .collect(); + Ok(LifecycleReport { + topic_id: canonical, + state: lc.state.as_str().to_owned(), + rules_version, + rules_source, + baseline_rules_version, + history, + }) +} + /// What to do once the topic is open. fn after_seal(topic_id: &str, published: bool) -> String { if published { @@ -443,6 +612,41 @@ fn seal_failure(err: &proof_topic_setup::SetupError, topic_id: &str) -> String { `custom_value` the RLM never measured. Re-read the measurement and re-sign." ); } + SetupError::BaselineStale { + topic_id, + measured, + in_force, + } => { + return format!( + "the measured baseline is stale: it was taken under rule version {measured}, but \ + version {} is in force.\n A baseline is a measurement **against a rule \ + version**: sealing this one would publish a bar measured under rules nobody \ + scores with, so miners would be judged by the newer checklist while the number \ + they must beat came from the older one. Nothing moved.\n Re-run the baseline \ + under the rules in force:\n proof-admin topic install --bundle \ + --env --drive-rlm --owner-approved\n Then read \ + `proof-admin topic baseline {topic_id}` again and seal the new commitment.", + in_force.map_or_else(|| "none".to_owned(), |v| v.to_string()) + ); + } + SetupError::DegenerateBar { topic_id, primary } => { + return format!( + "the measured baseline is a degenerate bar ({primary}) for topic {topic_id:?}, so \ + it will not be sealed.\n This family scores a **relative** win \ + (`challenger >= bar * (1 + epsilon_rel)`), which has no solution when the bar is \ + zero: the topic would be open, scorable, and impossible for every miner to pass. \ + A zero bar is a real measurement — a reference run that solved nothing, which is \ + what an all-zero Harbor baseline is — not a defect in this command.\n Nothing \ + was changed and **nothing was auto-resealed**: the stored measurement is exactly \ + what the RLM wrote. To get a sealable baseline, re-run it against a reference \ + that can actually score, or fix the task selection so the reference run measures \ + something:\n 1. `proof-admin topic baseline {topic_id}` shows the measurement \ + the RLM left, with the rules version it was taken under.\n 2. Re-drive the \ + RLM with a reference that scores: `proof-admin topic install --bundle \ + --env --drive-rlm --owner-approved`.\n 3. Re-read \ + `proof-admin topic baseline {topic_id}` and seal the new commitment." + ); + } SetupError::State(proof_rlm::StateError::Illegal { from, .. }) => { return format!( "the topic's lifecycle is at {from:?}, not `baselining`, so there is no \ @@ -559,4 +763,151 @@ mod tests { "0.99 cannot verify against a document sealing 0.42" ); } + + /// `topic baseline` is the read **before** the operator signs an `open` + /// document, so it has to say when the measurement cannot be sealed. + /// + /// Warning only at seal time is correct but late: the document would + /// already carry a number the seal refuses, and the operator would have + /// signed and published a topic that can never open. The flag is the same + /// predicate `mark_sealed` refuses on, so the two cannot disagree. + #[test] + fn a_degenerate_measurement_is_flagged_before_the_document_is_signed() { + use proof_task::MetricSpec; + let custom = |primary_value: f64| { + let mut doc = document(); + doc.metric = MetricSpec { + family: MetricFamily::Custom, + primary: "custom_value".into(), + custom_id: "placeholder_metric".into(), + epsilon_rel: 0.05, + ..doc.metric.clone() + }; + baseline_is_degenerate(&doc, primary_value) + }; + assert!( + custom(0.0), + "a zero primary on a relative family is flagged" + ); + assert!( + !custom(0.42), + "a real measurement is not flagged: the guard narrows nothing else" + ); + + // `nll` compares absolutely, so a zero bar there is a hard but + // meaningful target — the flag must not reach across families. + let mut nll = document(); + nll.metric.family = MetricFamily::Nll; + assert!( + !baseline_is_degenerate(&nll, 0.0), + "the absolute family is never degenerate" + ); + } + + /// `open` is a registry state, not a promise that miners can submit. /// + /// A seal without `--publish` reaches `open` while the host still serves + /// the previous document, so the lifecycle advice must not tell an operator + /// the topic is reachable. It has to say the seal is recorded, that the + /// publish is the step that makes it live, and how to retry it. + #[test] + fn the_open_advice_does_not_claim_the_topic_is_published() { + let report = LifecycleReport { + topic_id: "tb4".into(), + state: "open".into(), + rules_version: Some(2), + rules_source: Some("rlm".into()), + baseline_rules_version: Some(2), + history: Vec::new(), + }; + let advice = report.next_steps(); + assert!( + advice.contains("--publish"), + "the advice must name the publish step: {advice}" + ); + assert!( + advice.to_lowercase().contains("previous"), + "the advice must say the previous document is still served: {advice}" + ); + assert!( + advice.contains("can_score"), + "the advice must say to confirm the host is scorable: {advice}" + ); + assert!( + !advice.contains("Miners can submit;"), + "the advice must not promise submissions on `open` alone: {advice}" + ); + } + + /// The state-specific advice names the state it is talking about. + #[test] + fn the_lifecycle_advice_matches_the_state() { + let make = |state: &str| LifecycleReport { + topic_id: "tb4".into(), + state: state.into(), + rules_version: None, + rules_source: None, + baseline_rules_version: None, + history: Vec::new(), + }; + // The in-flight state is the one a long `--drive-rlm` sits in, so the + // advice has to say a run may be working rather than lost. + assert!(make("provisioning") + .next_steps() + .contains("VM is being created")); + // `baselining` with no baseline is the paid job not having landed. + let baselining = make("baselining").next_steps(); + assert!( + baselining.contains("No baseline yet"), + "an unmeasured baseline must say so: {baselining}" + ); + // …and with one, the next step is the seal. + let mut measured = make("baselining"); + measured.baseline_rules_version = Some(2); + measured.rules_version = Some(2); + assert!(measured.next_steps().contains("Seal it")); + // A topic nothing has driven says so rather than inventing advice. + assert!(make("draft").next_steps().contains("Nothing has run yet")); + } + + /// A baseline measured under a superseded vector is not sealable, so the + /// `baselining` advice must not send the operator into that refusal. + /// + /// The baseline is a measurement **against a rule version**: sealing a + /// stale one would publish a bar measured under rules nobody scores with, + /// while miners are judged by the vector in force. + #[test] + fn the_baselining_advice_refuses_to_recommend_sealing_a_stale_baseline() { + let make = |baseline: Option, current: Option| LifecycleReport { + topic_id: "tb4".into(), + state: "baselining".into(), + rules_version: current, + rules_source: Some("rlm".into()), + baseline_rules_version: baseline, + history: Vec::new(), + }; + + let stale = make(Some(1), Some(2)).next_steps(); + assert!( + !stale.contains("Seal it"), + "a stale baseline must not be recommended for sealing: {stale}" + ); + assert!( + stale.contains("version 1") && stale.contains("version 2"), + "the advice must name both versions: {stale}" + ); + assert!( + stale.contains("--drive-rlm"), + "the advice must name the way to re-measure: {stale}" + ); + + // The version in force matching the measurement is still a seal. + assert!(make(Some(2), Some(2)).next_steps().contains("Seal it")); + // No rule row at all is a different problem, not a stale baseline: + // the message must not claim a version is in force when none is. + let no_rules = make(Some(2), None).next_steps(); + assert!( + !no_rules.contains("Seal it"), + "a baseline with no rules in force is not sealable: {no_rules}" + ); + } } diff --git a/crates/proof-topic-setup/Cargo.toml b/crates/proof-topic-setup/Cargo.toml index e3d3b091f..b161ca804 100644 --- a/crates/proof-topic-setup/Cargo.toml +++ b/crates/proof-topic-setup/Cargo.toml @@ -12,10 +12,13 @@ publish = false proof-eval = { path = "../proof-eval" } proof-rlm = { path = "../proof-rlm" } proof-rlm-store = { path = "../proof-rlm-store" } +proof-score = { path = "../proof-score" } proof-task = { path = "../proof-task" } +proof-topic-authoring = { path = "../proof-topic-authoring" } thiserror = "2" [dev-dependencies] +serde_json = "1" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } [lints] diff --git a/crates/proof-topic-setup/src/lib.rs b/crates/proof-topic-setup/src/lib.rs index ec5787318..2367d333e 100644 --- a/crates/proof-topic-setup/src/lib.rs +++ b/crates/proof-topic-setup/src/lib.rs @@ -44,6 +44,7 @@ use proof_rlm::{ }; use proof_rlm_store::{BaselineRow, RlmStore, StoreError, TransitionRow}; use proof_task::{InferenceOffer, MetricFamily, ProofPin, TopicDocument, TopicError, TopicStatus}; +use proof_topic_authoring::TopicAuthoring; /// Why setup stopped. #[derive(Debug, thiserror::Error)] @@ -80,12 +81,138 @@ pub enum SetupError { /// the RLM measured in the topic VM. #[error("seal: {0}")] Seal(String), + /// The measured baseline is a bar no challenger can ever clear. + /// + /// A relative-win family (`throughput` / `custom`) compares a challenger + /// against the sealed value with `challenger >= bar * (1 + epsilon_rel)`, + /// so a bar at ~zero has no solution: the topic would be open, scorable, + /// and permanently unwinnable by anyone. That is what an all-zero + /// reference run measures (LIVE Gate 1: five Harbor tasks, every one + /// `0.0`), and sealing it would publish a dead topic. + /// + /// Refused **without** touching the stored measurement and **without** + /// auto-resealing: the operator re-runs the baseline against a reference + /// that can score, or re-scopes the task set, and seals the new number. + #[error( + "topic {topic_id:?}: the measured baseline {primary} is a degenerate bar — this family \ + scores a relative win (`challenger >= bar * (1 + epsilon_rel)`), so a bar at zero can \ + never be cleared by anyone and the topic would be open but unwinnable. Nothing was \ + changed. Re-run the baseline against a reference that can score (or fix the task \ + selection so the reference run actually measures something), then seal that number" + )] + DegenerateBar { + /// The topic whose baseline is degenerate. + topic_id: String, + /// The measured primary that cannot be a bar. + primary: f64, + }, + /// The measured baseline was taken under a rule version that is no longer + /// in force. + /// + /// A baseline is a measurement **against a rule version**, so sealing a + /// superseded one would publish a bar measured under rules nobody scores + /// with: miners would be judged by the newer checklist while the number + /// they must beat came from the older one. Nothing is sealed — re-run the + /// baseline under the rules in force. + #[error( + "topic {topic_id:?}: the measured baseline was taken under rule version {measured}, but \ + version {} is in force; sealing it would publish a bar measured under rules nobody \ + scores with. Nothing was sealed. Re-run the baseline under the rules in force, then \ + seal that number", + match in_force { + Some(v) => v.to_string(), + None => "none".to_owned(), + } + )] + BaselineStale { + /// The topic whose baseline is stale. + topic_id: String, + /// The rule version the measurement was taken under. + measured: u32, + /// The rule version now in force (`None` when no rule row exists). + in_force: Option, + }, /// A baseline would be measured but there is no judge offer to bind it to. #[error( "no inference offer: the baseline is a paid run and needs a live judge offer to bind \ (or set skip_baseline, which measures none)" )] NoOffer, + /// The rule version in force is not RLM-authored. + /// + /// The topic's behavior has to be authored by its own RLM inside the topic + /// VM. A vector still carrying the signed document's provenance + /// (`topic_document`) or an operator's edit (`operator`) means setup never + /// got the RLM to write rules, so nothing downstream may treat this topic + /// as set up. + #[error( + "topic {topic_id:?}: rule version {version} is not RLM-authored (source {provenance}); the \ + topic's behavior is still the operator's document, so setup did not complete" + )] + RulesNotRlmAuthored { + /// The topic whose rules were read back. + topic_id: String, + /// The provenance found (`topic_document` / `operator` / no version). + provenance: String, + /// The version that was read back. + version: u32, + }, + /// A rule version other than the one this run wrote is in force. + /// + /// The baseline is measured and persisted **against a rule version**, so + /// the two have to agree: a concurrent writer that advanced the store + /// while this run was measuring would otherwise leave a sealed bar + /// measured under rules nobody scores with. Nothing is persisted — the + /// lifecycle stays where it stopped, so a re-run resumes. + #[error( + "topic {topic_id:?}: rule version {wrote} was written but version {} is now in force; \ + the baseline would have sealed a measurement taken under rules that no longer apply, so \ + nothing was persisted", + match in_force { + Some(v) => v.to_string(), + None => "none".to_owned(), + } + )] + RulesSuperseded { + /// The topic whose rules moved. + topic_id: String, + /// The version this run wrote and measured against. + wrote: u32, + /// The version now in force (`None` when no rule row exists). + in_force: Option, + }, + /// The RLM authored the rules but not the rest of the topic's behavior. + /// + /// A topic's behavior is five parts, and its own RLM authors all of them + /// in one job. A runner whose `propose_rules` writes only `rules.json` is + /// an adaptor baked before the set existed: the rules it wrote are + /// recorded with honest `rlm` provenance, and the topic still cannot be + /// installed or opened, because the parts it did not author have no + /// author. The refusal names them rather than filling them in from the + /// operator's bundle — that substitution is the operator-cloned document + /// the authorship gate exists to refuse. + #[error( + "topic {topic_id:?}: the RLM authored rules v{version} but no {missing:?}; a topic's \ + behavior is authored by its own RLM (rules, migrations, apis, submission_format, \ + pin_policy), so this set is incomplete and nothing downstream may treat the topic as \ + set up. Ship an adaptor whose propose_rules writes authoring.json" + )] + IncompleteAuthoring { + /// The topic. + topic_id: String, + /// Rule version the RLM did write. + version: u32, + /// Parts it did not author. + missing: Vec<&'static str>, + }, + /// The authored set is malformed, or loosens the pin. + #[error("topic {topic_id:?}: the RLM's authored set was refused: {why}")] + Authoring { + /// The topic. + topic_id: String, + /// What the refusal said. + why: String, + }, } /// What setup produced for the operator to seal. @@ -102,6 +229,26 @@ pub struct SetupOutcome { /// `None` when [`TopicSetup::skip_baseline`] was set: no baseline was /// measured, so there is nothing to seal and the topic cannot open yet. pub baseline_primary: Option, + /// The whole set the RLM authored, when it authored one. + /// + /// `None` means the run returned a bare rule vector (an adaptor baked + /// before the set existed). The rules are still RLM-authored and + /// versioned; what is missing is the rest of the topic's behavior, and + /// [`Self::missing_parts`] names it. + pub authored: Option>, + /// Parts the RLM did **not** author. Empty for a complete set. + pub missing_parts: Vec<&'static str>, +} + +/// What one authoring job produced. +#[derive(Debug, Clone, PartialEq)] +struct AuthoredOutcome { + /// The rule version the store now holds. + rules: RuleSet, + /// The whole set, when the RLM authored one. + authored: Option>, + /// Parts the RLM did not author. + missing: Vec<&'static str>, } impl SetupOutcome { @@ -110,6 +257,12 @@ impl SetupOutcome { pub const fn measured_baseline(&self) -> bool { self.baseline_primary.is_some() } + + /// Whether the RLM authored the topic's **whole** behavior. + #[must_use] + pub fn authored_complete_set(&self) -> bool { + self.authored.is_some() && self.missing_parts.is_empty() + } } /// Everything the driver needs; no secrets. @@ -244,19 +397,65 @@ impl TopicSetup { } } - /// The RLM writes its rules inside the VM; the store versions them. - async fn propose_rules( + /// The RLM authors its whole set inside the VM; the store versions the + /// rules and the driver carries the rest to the install. + /// + /// **Fail-closed on authorship.** The rules land as [`RuleSource::Rlm`] + /// only because the RLM's own `propose_rules` job produced them inside the + /// topic VM — the guest refuses to echo the signed `checklist` back + /// ([`proof_vm_guest`] `propose_rules`), and this method re-reads the store + /// afterwards to confirm the version in force really is `rlm`-sourced. A + /// store that still shows the operator's vector (`topic_document`) or an + /// operator edit (`operator`) means the topic's behavior was never + /// authored by its RLM, which is a refusal naming the provenance rather + /// than a silent pass. + /// + /// **The whole set, not only the rules.** A topic's behavior is five + /// parts — rules, migrations, APIs, submission format, pin policy — and + /// the job answers with all of them ([`TopicAuthoring`]). A run that + /// returns a bare rule vector (an adaptor baked before the set existed) + /// lands those rules with honest `rlm` provenance and is reported as + /// **incomplete**: the parts it did not author are named, and the install + /// refuses to open a topic whose behavior is not wholly its RLM's. What + /// the driver must never do is fill the gap from the operator's bundle — + /// that is the operator-cloned document the gate exists to refuse. + async fn author( &self, topic: &TopicDocument, + pin: &ProofPin, vm: &VmHandle, - ) -> Result { + ) -> Result { let current = self.store.current_rules(&topic.id).await?; let job = VmJob::ProposeRules { topic: Box::new(topic.clone()), current_version: current.as_ref().map(|r| r.version), + current: self.current_authoring(&topic.id).await?.map(Box::new), }; - let VmJobOutput::Rules(proposed) = self.orchestrator.run(vm, job).await? else { - return Err(VmError::WrongOutput("propose_rules").into()); + let (proposed, authored, missing) = match self.orchestrator.run(vm, job).await? { + VmJobOutput::Authored(set) => { + // The control plane holds the set to the pin as well as to + // the document: the guest checked shape, the deny-list, and + // the policy against the topic's own knobs (it has no pin), + // and this is where a policy that loosens a *global* floor is + // refused. + set.validate_against_pin(&topic.id, pin) + .map_err(|e| SetupError::Authoring { + topic_id: topic.id.clone(), + why: e.to_string(), + })?; + let rules = set.rules.clone(); + (rules, Some(set), Vec::new()) + } + VmJobOutput::Rules(rules) => { + // A fragment. Recorded honestly, refused downstream by name. + let missing = proof_topic_authoring::PARTS + .iter() + .copied() + .filter(|p| *p != "rules") + .collect::>(); + (rules, None, missing) + } + _ => return Err(VmError::WrongOutput("propose_rules").into()), }; let rules = if let Some(cur) = current { cur.next(RuleSource::Rlm, proposed)? @@ -270,8 +469,118 @@ impl TopicSetup { set.validate()?; set }; - self.store.put_rules(&rules).await?; - Ok(rules) + // The rules **and** the whole set land in one write: they are two + // halves of one fact — "this topic's RLM authored *this* at rule + // version *N*" — and a store that wrote them separately could fail + // between them, leaving newer rules with the previous set. A retry + // would then be handed a set whose rules are not the ones in force. + // + // Only a complete set is stored: a rules-only answer is a fragment, + // and storing a fragment as "the set in force" would hand the next run + // a set that was never authored. The fragment's rules are still + // recorded (with honest provenance), so the rules go through + // `put_rules` in that case. + match authored.as_ref() { + Some(set) => { + self.store.put_authoring(&topic.id, &rules, set).await?; + } + None => { + self.store.put_rules(&rules).await?; + } + } + // The read-back is the gate, not a formality: it is what makes "the + // RLM authored this topic's behavior" a fact the store can prove, + // rather than a label this driver attached. + // + // It reads back **the exact version just written**, never "whichever + // version is newest": a concurrent writer advancing the store to a + // later, unrelated version would make a newest-wins check pass while + // the version this run wrote — the one the baseline is measured + // against — was not RLM-authored at all. The digest is compared too, + // so a row rewritten under the same version number is caught. + let written = self.store.rules_at(&topic.id, rules.version).await?; + let ok = written + .as_ref() + .is_some_and(|w| w.source == RuleSource::Rlm && w.digest() == rules.digest()); + if !ok { + let provenance = match written.as_ref() { + Some(w) => format!("{:?}", w.source), + None => "no rule version".to_owned(), + }; + return Err(SetupError::RulesNotRlmAuthored { + topic_id: topic.id.clone(), + provenance, + version: rules.version, + }); + } + Ok(AuthoredOutcome { + rules, + authored, + missing, + }) + } + + /// The set this topic's RLM authored last, if any. + /// + /// Read from the store rather than held in memory, so a re-authoring run + /// after a restart — or from a different operator process — is handed the + /// same set. A store that cannot be read is a refusal, never a `None` that + /// would silently turn a re-authoring into a rewrite from nothing. + async fn current_authoring( + &self, + topic_id: &str, + ) -> Result, SetupError> { + Ok(self.store.authoring(topic_id).await?.map(|(_, set)| set)) + } + + /// Refuse when the rule version in force is no longer the one this run + /// wrote and measured its baseline against. + /// + /// The baseline is persisted **for a rule version**, so a vector that + /// changed under it would leave a topic whose sealed bar was measured + /// under rules nobody is scoring with. Reading the version in force before + /// the baseline lands is what serializes the two: a concurrent RLM write + /// fails the setup run (nothing persisted) rather than sealing a stale + /// measurement. + async fn rules_still_in_force( + &self, + topic_id: &str, + wrote: &RuleSet, + ) -> Result<(), SetupError> { + let current = self.store.current_rules(topic_id).await?; + let in_force = current.as_ref().map(|r| r.version); + if in_force != Some(wrote.version) { + return Err(SetupError::RulesSuperseded { + topic_id: topic_id.to_owned(), + wrote: wrote.version, + in_force, + }); + } + Ok(()) + } + + /// Refuse when the measured baseline's rule version is no longer in force. + /// + /// [`Self::rules_still_in_force`] serializes the *write*; this is the same + /// invariant at *seal* time, where the store can have advanced between the + /// measurement landing and the operator sealing it. Sealing a stale + /// measurement would publish a bar measured under rules nobody scores + /// with — miners judged by the newer checklist against an older number. + async fn baseline_still_in_force( + &self, + topic_id: &str, + measured: u32, + ) -> Result<(), SetupError> { + let current = self.store.current_rules(topic_id).await?; + let in_force = current.as_ref().map(|r| r.version); + if in_force != Some(measured) { + return Err(SetupError::BaselineStale { + topic_id: topic_id.to_owned(), + measured, + in_force, + }); + } + Ok(()) } /// Baseline shaped exactly like a miner run, persisted: inside the topic @@ -322,6 +631,14 @@ impl TopicSetup { } }; report.verify(&request)?; + // The measurement is about to be persisted **against a rule version**, + // so that version has to still be the one in force: a vector that + // moved under the run would leave a sealed bar measured under rules + // nobody scores with. Checked after the paid run (the only point where + // a concurrent write could have landed) and before the row, so a + // superseded run persists nothing and a re-run resumes from + // `baselining`. + self.rules_still_in_force(&topic.id, rules).await?; self.store .put_baseline(&BaselineRow { topic_id: topic.id.clone(), @@ -376,13 +693,29 @@ impl TopicSetup { .into()); } let vm = self.provision(topic, pin, &mut lc).await?; - let rules = self.propose_rules(topic, &vm).await?; + let authored = self.author(topic, pin, &vm).await?; + // Fail closed on an incomplete set: the rules are recorded (honest + // provenance either way), and a run that authored only part of the + // topic's behavior stops here rather than letting anything downstream + // treat the topic as set up. Nothing is filled in from the operator's + // bundle — that substitution is the operator-cloned document the gate + // refuses. + if !authored.missing.is_empty() { + return Err(SetupError::IncompleteAuthoring { + topic_id: topic.id.clone(), + version: authored.rules.version, + missing: authored.missing, + }); + } + let rules = authored.rules; if self.skip_baseline { return Ok(SetupOutcome { topic_id: topic.id.clone(), vm, rules_version: rules.version, baseline_primary: None, + authored: authored.authored, + missing_parts: Vec::new(), }); } let Some(offer) = offer else { @@ -398,6 +731,8 @@ impl TopicSetup { vm, rules_version: rules.version, baseline_primary: Some(report.primary_value), + authored: authored.authored, + missing_parts: Vec::new(), }) } @@ -447,6 +782,32 @@ impl TopicSetup { measured.primary_value ))); } + // A sealed bar nobody can clear is a topic that is open, scorable and + // permanently unwinnable: `relative_win` refuses every challenger + // against a zero bar, so no submission could ever pass. That is a real + // measurement, not a bug — a reference run that solved nothing, which + // is exactly what an all-zero Harbor baseline is — so it is refused + // here, at the boundary, where the operator can still act on it. + // + // This is deliberately **not** an auto-reseal: the stored measurement + // is left exactly as the RLM wrote it. Fixing it means re-running the + // baseline against a reference that can score (or re-scoping the task + // set), then sealing the new number. + if proof_score::family_bar_is_degenerate(topic.metric.family, Some(sealed_primary)) { + return Err(SetupError::DegenerateBar { + topic_id: topic.id.clone(), + primary: sealed_primary, + }); + } + // The baseline is a measurement **against a rule version**, so sealing + // one whose vector has since moved would publish a bar measured under + // rules nobody is scored with: miners would be judged by the newer + // checklist while the number they must beat came from the older one. + // `baseline` already refuses a superseded vector at write time; this is + // the same invariant at seal time, where a concurrent write between the + // two can still be observed. + self.baseline_still_in_force(&topic.id, measured.rules_version) + .await?; let mut lc = self.lifecycle(topic).await?; if lc.state != RlmState::Baselining { return Err(StateError::Illegal { @@ -464,3 +825,373 @@ impl TopicSetup { .await } } + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used)] + + use super::*; + use proof_rlm::UnwiredVmOrchestrator; + use proof_rlm_store::MemoryRlmStore; + + /// The store is the only collaborator these tests need; the rest of + /// [`TopicSetup`] is filled with stubs that are never reached by + /// `rules_still_in_force`. + fn setup(store: Arc) -> TopicSetup { + TopicSetup { + orchestrator: Arc::new(UnwiredVmOrchestrator), + store, + template: VmTemplate::unpinned(), + experiments: proof_rlm::ExperimentPolicy::default(), + owner: Arc::new(AlwaysApprove), + keys: Arc::new(KeysPresent), + spend_cap_usd: None, + skip_baseline: false, + } + } + + fn rules(topic_id: &str, version: u32, source: RuleSource) -> RuleSet { + RuleSet { + topic_id: topic_id.to_owned(), + version, + source, + rules: vec![proof_task::ChecklistRule { + id: format!("r-{version}"), + text: "a rule".into(), + }], + } + } + + /// The read-back verifies **the version this run wrote**, not whichever + /// version is newest. + /// + /// The defect this pins: setup wrote version N and then checked the source + /// of the *newest* version. A concurrent writer advancing the store to + /// N+1 (`rlm`-sourced) made that check pass while the version setup + /// actually wrote — the one the baseline is measured against — was never + /// verified at all. + #[tokio::test] + async fn the_rule_read_back_verifies_the_version_it_wrote() { + let store = Arc::new(MemoryRlmStore::new()); + let topic_id = "fixture-topic"; + // Setup wrote version 1 as `topic_document` (what the install seeds), + // while a later, RLM-authored version 2 is now newest. + store + .put_rules(&rules(topic_id, 1, RuleSource::TopicDocument)) + .await + .expect("v1"); + store + .put_rules(&rules(topic_id, 2, RuleSource::Rlm)) + .await + .expect("v2"); + + // A newest-wins check would read `rlm` and pass. The version the run + // wrote is the one that has to be verified. + let written = store + .rules_at(topic_id, 1) + .await + .expect("read") + .expect("v1 exists"); + assert_eq!( + written.source, + RuleSource::TopicDocument, + "version 1 is the operator's vector, whatever version 2 says" + ); + assert_eq!( + store.current_rules_source(topic_id).await.expect("read"), + Some(RuleSource::Rlm), + "the newest version is rlm — which is exactly why newest-wins was the defect" + ); + } + + /// A rule vector that moved under the run refuses **before** the baseline + /// is persisted. + /// + /// The baseline is stored per rule version, so a version that changed + /// while the paid run was in flight would leave a sealed bar measured + /// under rules nobody scores with. Nothing is written: the lifecycle stays + /// at `baselining`, so a re-run resumes. + #[tokio::test] + async fn a_superseded_rule_version_persists_no_baseline() { + let store = Arc::new(MemoryRlmStore::new()); + let topic_id = "fixture-topic"; + store + .put_rules(&rules(topic_id, 1, RuleSource::Rlm)) + .await + .expect("v1"); + let wrote = store + .rules_at(topic_id, 1) + .await + .expect("read") + .expect("v1"); + + let setup = setup(store.clone()); + // In force: nothing to refuse. + setup + .rules_still_in_force(topic_id, &wrote) + .await + .expect("the version this run wrote is in force"); + + // A concurrent RLM write lands version 2 while the baseline runs. + store + .put_rules(&rules(topic_id, 2, RuleSource::Rlm)) + .await + .expect("v2"); + let err = setup + .rules_still_in_force(topic_id, &wrote) + .await + .expect_err("a superseded version must refuse"); + assert!( + matches!( + err, + SetupError::RulesSuperseded { + wrote: 1, + in_force: Some(2), + .. + } + ), + "{err}" + ); + assert!( + store.baseline(topic_id).await.expect("read").is_none(), + "nothing was persisted for a run whose rules moved under it" + ); + } + + /// The refusal names both versions, so an operator reads what happened. + #[test] + fn a_superseded_refusal_names_both_versions() { + let err = SetupError::RulesSuperseded { + topic_id: "fixture-topic".into(), + wrote: 3, + in_force: Some(4), + }; + let text = err.to_string(); + assert!(text.contains("fixture-topic"), "{text}"); + assert!(text.contains('3') && text.contains('4'), "{text}"); + } + + /// A baseline measured under a superseded vector must not seal. + /// + /// `baseline` already refuses a vector that moved while the run was + /// measuring, but the store can be advanced between that write and the + /// operator's seal. Sealing then would publish a bar measured under rules + /// nobody scores with — miners judged by the newer checklist against an + /// older number — so the seal compares the measurement's rule version to + /// the one in force and refuses, naming both. + #[tokio::test] + async fn a_stale_measured_baseline_must_not_seal() { + let store = Arc::new(MemoryRlmStore::new()); + let topic_id = "fixture-topic"; + store + .put_rules(&rules(topic_id, 1, RuleSource::Rlm)) + .await + .expect("v1"); + + let setup = setup(store.clone()); + // The version the measurement was taken under is in force: no refusal. + setup + .baseline_still_in_force(topic_id, 1) + .await + .expect("the measured version is in force"); + + // A later RLM write lands version 2 after the measurement. + store + .put_rules(&rules(topic_id, 2, RuleSource::Rlm)) + .await + .expect("v2"); + let err = setup + .baseline_still_in_force(topic_id, 1) + .await + .expect_err("a stale measurement must refuse"); + assert!( + matches!( + err, + SetupError::BaselineStale { + measured: 1, + in_force: Some(2), + .. + } + ), + "{err}" + ); + let text = err.to_string(); + assert!( + text.contains("version 1") && text.contains("version 2"), + "the refusal must name both versions: {text}" + ); + } + + struct AlwaysApprove; + impl OwnerHook for AlwaysApprove { + fn ask_owner( + &self, + _prompt: &OwnerPrompt, + ) -> Result { + Ok(proof_rlm::OwnerDecision::Approve) + } + } + + struct KeysPresent; + impl OwnerKeysProbe for KeysPresent { + fn owner_keys_present(&self) -> Result<(), proof_rlm::HookError> { + Ok(()) + } + } +} + +#[cfg(test)] +mod authoring_atomicity_tests { + #![allow(clippy::unwrap_used, clippy::expect_used)] + + use super::*; + use proof_rlm_store::MemoryRlmStore; + use std::sync::Arc; + + fn rules(topic_id: &str, version: u32) -> RuleSet { + RuleSet { + topic_id: topic_id.to_owned(), + version, + source: RuleSource::Rlm, + rules: vec![proof_task::ChecklistRule { + id: format!("r-{version}"), + text: "a rule".into(), + }], + } + } + + fn set(topic_id: &str, api: &str) -> TopicAuthoring { + proof_topic_authoring::TopicAuthoring { + schema_version: proof_topic_authoring::AUTHORING_SCHEMA, + topic_id: topic_id.to_owned(), + rules: vec![proof_task::ChecklistRule { + id: "r-1".into(), + text: "a rule".into(), + }], + migrations: vec![proof_topic_authoring::AuthoredMigration { + name: "0001_scratch".into(), + sql: format!( + "CREATE TABLE {}_scratch (id TEXT)", + topic_id.replace('-', "_") + ), + }], + apis: vec![proof_topic_authoring::AuthoredApi { + path: api.into(), + method: "GET".into(), + summary: String::new(), + }], + submission_format: serde_json::json!({"kind": "tar"}), + pin_policy: proof_topic_authoring::PinPolicy::none(), + } + } + + /// The rules and the set land **together**: a store that wrote them + /// separately could fail between the two writes and leave newer rules with + /// the previous set, so a retry would be handed a set whose rules are not + /// the ones in force. + /// + /// One call, one write, and a refusal leaves **neither**: the version + /// check runs before anything is appended. + #[tokio::test] + async fn the_rules_and_the_set_land_in_one_write() { + let store = MemoryRlmStore::new(); + let topic_id = "atomic-topic"; + // A first authoring run: both land. + let v = store + .put_authoring(topic_id, &rules(topic_id, 1), &set(topic_id, "one")) + .await + .expect("first authoring"); + assert_eq!(v, 1); + assert_eq!( + store + .authoring(topic_id) + .await + .expect("read") + .expect("set") + .0, + 1 + ); + assert_eq!( + store + .current_rules(topic_id) + .await + .expect("read") + .expect("rules") + .version, + 1 + ); + + // A second run whose rules do **not** advance is refused — and the set + // is not appended either, so the pair stays consistent. + let err = store + .put_authoring(topic_id, &rules(topic_id, 1), &set(topic_id, "two")) + .await + .expect_err("a version that does not advance is refused"); + assert!(matches!(err, StoreError::VersionGap("rules")), "{err}"); + assert_eq!( + store + .authoring(topic_id) + .await + .expect("read") + .expect("set") + .0, + 1, + "a refused write appends no set" + ); + let stored = store + .authoring(topic_id) + .await + .expect("read") + .expect("set") + .1; + assert_eq!( + stored.apis[0].path, "one", + "the set in force is the one that landed with the rules in force" + ); + + // The next valid run advances both. + let v = store + .put_authoring(topic_id, &rules(topic_id, 2), &set(topic_id, "two")) + .await + .expect("second authoring"); + assert_eq!(v, 2); + assert_eq!( + store + .current_rules(topic_id) + .await + .expect("read") + .expect("rules") + .version, + 2 + ); + assert_eq!( + store + .authoring(topic_id) + .await + .expect("read") + .expect("set") + .1 + .apis[0] + .path, + "two" + ); + } + + /// The rule **provenance read-back** is unchanged by the atomic write: the + /// store still answers which source the version in force came from. + #[tokio::test] + async fn the_atomic_write_still_reports_rlm_provenance() { + let store = MemoryRlmStore::new(); + let topic_id = "provenance-topic"; + store + .put_authoring(topic_id, &rules(topic_id, 1), &set(topic_id, "one")) + .await + .expect("authoring"); + assert!(store.rlm_authored_rules(topic_id).await.expect("read")); + assert_eq!( + store.current_rules_source(topic_id).await.expect("read"), + Some(RuleSource::Rlm) + ); + let _ = Arc::new(store); + } +} diff --git a/crates/proof-topic-sql-guard/src/lib.rs b/crates/proof-topic-sql-guard/src/lib.rs index cfceabf0f..03c1d2eac 100644 --- a/crates/proof-topic-sql-guard/src/lib.rs +++ b/crates/proof-topic-sql-guard/src/lib.rs @@ -18,9 +18,11 @@ //! `VACUUM` / `CLUSTER` / `REINDEX`, `SECURITY DEFINER` functions, and //! server-side file access (`pg_read_file`, `lo_import`, …). //! - **Namespace**: every table a statement creates, writes, or reads must be -//! inside the topic's own namespace (`{topic_id}_*`, `topic_*`, or -//! `{topic_id}.…`). Without this a topic could claim a generic name and -//! collide with the next topic's install, or read a sibling topic's rows. +//! inside the topic's own namespace (`{topic_sql_prefix}_*`, or +//! `{topic_sql_prefix}.…` — the id with `-` mapped to `_`, since a topic id +//! is a hyphen slug and a bare SQL identifier cannot contain a hyphen). +//! Without this a topic could claim a generic name and collide with the next +//! topic's install, or read a sibling topic's rows. //! //! # What this is not //! @@ -914,16 +916,45 @@ pub fn has_word(haystack: &str, needle: &str) -> bool { } /// Identifier-shaped tokens, lower-cased, dots kept (`schema.table`). +/// +/// A **double-quoted** run is one identifier, kept whole. That matters for a +/// hyphenated topic: `"fixture-topic-v0_scratch"` is a single legal identifier, +/// and splitting it at the `-` would yield `fixture`, `topic`, `v0_scratch` — +/// none of which is inside the topic's namespace, so a legal quoted name would +/// be refused as unscoped. Quotes are stripped on the way in, so the token is +/// the name the database would store and the deny rules match it as before. fn tokens(text: &str) -> Vec { let mut out = Vec::new(); let mut cur = String::new(); - for c in text.chars() { + let chars: Vec = text.chars().collect(); + let mut i = 0usize; + while i < chars.len() { + let c = chars[i]; + if c == '"' { + // A quoted identifier: its body is one token, hyphens included. + i += 1; + while i < chars.len() { + if chars[i] == '"' { + if chars.get(i + 1) == Some(&'"') { + cur.push('"'); // `""` is an escaped quote inside the name + i += 2; + continue; + } + i += 1; + break; + } + cur.push(chars[i]); + i += 1; + } + continue; + } if c.is_alphanumeric() || c == '_' || c == '.' { cur.push(c); } else if !cur.is_empty() { out.push(cur.trim_matches('.').to_ascii_lowercase()); cur.clear(); } + i += 1; } if !cur.is_empty() { out.push(cur.trim_matches('.').to_ascii_lowercase()); @@ -1043,6 +1074,63 @@ fn copy_chars( idx } +/// The SQL-safe spelling of a topic id. +/// +/// A topic id is a hyphen slug (`[a-z0-9][a-z0-9-]{1,62}`) and **may not +/// contain an underscore**, while a bare SQL identifier is +/// `[a-z_][a-z0-9_$]*` and **may not contain a hyphen** — the two alphabets do +/// not intersect except on `[a-z0-9]`. So the literal id can never prefix a +/// bare identifier: `CREATE TABLE fixture-topic-v0_scratch` is a syntax error +/// at the first `-`, and the guard's own `{topic_id}_*` requirement would be +/// unsatisfiable for every real topic. +/// +/// Mapping `-` → `_` gives the topic an identifier-safe prefix. The mapping is +/// injective **over legal ids** (an id cannot contain an underscore), so two +/// ids never map to the same prefix. +/// +/// Injectivity is not sufficient on its own — see [`is_topic_scoped`], which +/// closes the prefix relation between a short id and a longer hyphenated one. +#[must_use] +pub fn topic_sql_prefix(topic_id: &str) -> String { + topic_id.trim().to_ascii_lowercase().replace('-', "_") +} + +/// Bare names in `names` that **another** topic in `others` also claims. +/// +/// The prefix rule in [`is_topic_scoped`] is per-topic, and the `-` → `_` +/// mapping is injective but its **prefixes are not prefix-free**: `aa` is a +/// prefix of `aa-b`'s mapped form `aa_b`, so `aa_b_scratch` sits inside both +/// namespaces. The guard cannot see that on its own — but the **install** can, +/// because it has the topic registry. +/// +/// Returns `(name, other_topic)` for each collision, so the install can refuse +/// with both ids named. Empty means no other registered topic claims any of +/// these names, which is the ordinary case: `tb4_scratch` collides with +/// nothing unless a topic `tb4-scratch` exists. +/// +/// `others` should be every topic id the host knows **except** `topic` — the +/// install reads them from `proof_topic_version`. +#[must_use] +pub fn claim_collisions<'a, I>(names: &[String], topic: &str, others: I) -> Vec<(String, String)> +where + I: IntoIterator, +{ + let others: Vec<&str> = others + .into_iter() + .filter(|o| !o.eq_ignore_ascii_case(topic.trim())) + .collect(); + let mut out = Vec::new(); + for name in names { + for other in &others { + if is_topic_scoped(name, other) { + out.push((name.clone(), (*other).to_owned())); + break; + } + } + } + out +} + /// Whether `name` is inside `topic_id`'s namespace. /// /// Two spellings are the topic's, and only two: @@ -1050,6 +1138,31 @@ fn copy_chars( /// - a `{topic_id}`-qualified name (`tb4.scores`, `tb4.runs`), or /// - a bare `{topic_id}_`-prefixed name (`tb4_scores`). /// +/// For an id containing a hyphen the identifier-safe form +/// ([`topic_sql_prefix`], `fixture-topic-v0` → `fixture_topic_v0`) is accepted +/// in both positions, because the literal id cannot appear in a bare SQL +/// identifier at all. The literal spelling stays accepted too, for the ids +/// that need no mapping and for quoted schema-qualified names. +/// +/// # The residual ambiguity, and where it is closed +/// +/// The mapped **prefixes are not prefix-free**: `aa` is a prefix of `aa-b`'s +/// mapped form `aa_b`, so the bare name `aa_b_scratch` is inside *both* +/// namespaces — `aa` + `b_scratch` and `aa-b` + `scratch`. A migration +/// approved for `aa` could therefore read, modify, or drop a table belonging +/// to `aa-b` in the shared database. +/// +/// This function deliberately does **not** try to resolve that by refusing +/// names with underscores after the prefix: that would refuse ordinary names +/// like `tb4_scratch_idx`, which are exactly what topics use. It cannot be +/// resolved here at all — the question "is a longer sibling registered?" is +/// about the whole registry, not about this topic. [`claim_collisions`] answers +/// it where the registry is visible (the install), and refuses the migration +/// there, naming both topics. +/// +/// So: this is the per-topic **shape** check, and [`claim_collisions`] is the +/// cross-topic **collision** check. Both run before a migration is applied. +/// /// # Why there is no generic `topic_` allowance /// /// An earlier revision also accepted any `topic_*` name, on the theory that a @@ -1072,14 +1185,15 @@ pub fn is_topic_scoped(name: &str, topic_id: &str) -> bool { return false; } let topic = topic_id.trim().to_ascii_lowercase(); + let sql_topic = topic_sql_prefix(&topic); let (schema, bare) = match n.split_once('.') { Some((s, b)) => (Some(s), b), None => (None, n.as_str()), }; - if schema == Some(topic.as_str()) { + if schema == Some(topic.as_str()) || schema == Some(sql_topic.as_str()) { return true; } - bare.starts_with(&format!("{topic}_")) + bare.starts_with(&format!("{topic}_")) || bare.starts_with(&format!("{sql_topic}_")) } /// Check a statement's text against every deny rule. @@ -1164,9 +1278,11 @@ fn check_text( return Err(deny( &name, &format!( - "a topic migration may only touch objects named {topic_id}_*, topic_*, or \ - {topic_id}.*; an unscoped name would collide with — or read — another \ - topic's install" + "a topic migration may only touch objects named {sql}_* (or {id}_*), \ + {sql}.* (or {id}.*); an unscoped name would collide with — or read — \ + another topic's install", + sql = topic_sql_prefix(topic_id), + id = topic_id ), )); } diff --git a/crates/proof-topic-sql-guard/tests/sql_guard.rs b/crates/proof-topic-sql-guard/tests/sql_guard.rs index 0cc80732b..282558988 100644 --- a/crates/proof-topic-sql-guard/tests/sql_guard.rs +++ b/crates/proof-topic-sql-guard/tests/sql_guard.rs @@ -14,7 +14,8 @@ use proof_topic_sql_guard::MigrationDenied; use proof_topic_sql_guard::{ - blank_statements, check_migration, is_topic_scoped, split_statements, OWNED_TABLES, + blank_statements, check_migration, claim_collisions, is_topic_scoped, referenced_objects, + split_statements, topic_sql_prefix, OWNED_TABLES, }; const TOPIC: &str = "tb4"; @@ -268,6 +269,195 @@ fn a_topics_own_namespace_is_allowed() { assert!(!is_topic_scoped("", "tb4")); } +/// A real topic id is a **hyphen slug**, and a bare SQL identifier cannot +/// contain a hyphen — so the guard has to accept the identifier-safe spelling +/// or no real topic could ever install a migration. +/// +/// The defect this pins: the guard required a literal `{topic_id}_` prefix. +/// Every live topic id is `[a-z0-9][a-z0-9-]{1,62}`, so the requirement was +/// unsatisfiable: `CREATE TABLE fixture-topic-v0_scratch` is a syntax error at +/// the first `-`, and the underscore spelling was refused. `--drive-rlm` would +/// provision the VM, run the paid baseline, and only then fail the install on +/// the deny-list — a paid run that could never publish. +#[test] +fn a_hyphenated_topic_id_has_an_identifier_safe_namespace() { + // The mapping is `-` → `_`. It is injective **over legal ids**: an id + // cannot contain an underscore (`[a-z0-9][a-z0-9-]{1,62}`), so two + // different real ids cannot collide on one prefix. + assert_eq!(topic_sql_prefix("fixture-topic-v0"), "fixture_topic_v0"); + assert_eq!(topic_sql_prefix("tb4"), "tb4"); + assert_ne!( + topic_sql_prefix("a-b"), + topic_sql_prefix("a-b-c"), + "different ids map to different prefixes" + ); + + // The identifier-safe spelling is inside the topic's namespace… + assert!(is_topic_scoped( + "fixture_topic_v0_scratch", + "fixture-topic-v0" + )); + assert!(is_topic_scoped("fixture_topic_v0.runs", "fixture-topic-v0")); + // …the literal spelling stays accepted where it is legal (quoted / schema) + assert!(is_topic_scoped("fixture-topic-v0.runs", "fixture-topic-v0")); + // …and a sibling is still refused, in both spellings. + assert!(!is_topic_scoped( + "fixture_topic_v1_scratch", + "fixture-topic-v0" + )); + assert!(!is_topic_scoped("other_topic_scratch", "fixture-topic-v0")); + assert!(!is_topic_scoped("topic_scratch", "fixture-topic-v0")); +} + +/// The residual ambiguity the per-topic check **cannot** see, and the +/// cross-topic check that closes it. +/// +/// `-` → `_` is injective, but its prefixes are not prefix-free: `aa` is a +/// prefix of `aa-b`'s mapped form `aa_b`, so the bare name `aa_b_scratch` sits +/// inside **both** namespaces — `aa` + `b_scratch` and `aa-b` + `scratch`. A +/// migration approved for `aa` could read, modify, or drop a table belonging +/// to `aa-b` in the shared database. +/// +/// The per-topic guard is right to accept the name for each topic on its own +/// (refusing every name with an underscore after the prefix would refuse +/// ordinary names like `tb4_scratch_idx`). The question is about the +/// **registry**, so `claim_collisions` answers it where the registry is +/// visible — the install. +#[test] +fn a_bare_name_two_topics_claim_is_reported_as_a_collision() { + let name = "aa_b_scratch".to_owned(); + + // Each topic accepts it alone: the guard sees one topic at a time. + assert!(is_topic_scoped(&name, "aa")); + assert!(is_topic_scoped(&name, "aa-b")); + + // The registry-aware check names both. + let collisions = claim_collisions(std::slice::from_ref(&name), "aa", ["aa-b"]); + assert_eq!( + collisions, + vec![(name.clone(), "aa-b".to_owned())], + "the collision is reported with the topic that also claims it" + ); + // …and symmetrically. + assert_eq!( + claim_collisions(std::slice::from_ref(&name), "aa-b", ["aa"]), + vec![(name, "aa".to_owned())] + ); +} + +/// The collision check must not fire on names that only *look* ambiguous. +/// +/// `tb4_scratch` is the live staging shape (`migrations=["0001_scratch"]`): no +/// sibling claims it unless a topic `tb4-scratch` is actually registered, and +/// the check only refuses when one is. +#[test] +fn a_name_no_registered_sibling_claims_is_not_a_collision() { + let names = vec![ + "tb4_scratch".to_owned(), + "tb4_scratch_idx".to_owned(), + "tb4".to_owned(), + ]; + // No other topic registered: nothing collides. + assert!(claim_collisions(&names, "tb4", Vec::<&str>::new()).is_empty()); + // An unrelated topic: still nothing. + assert!(claim_collisions(&names, "tb4", ["other-topic"]).is_empty()); + // A topic that *would* map onto `tb4_scratch` collides on the name that + // sits **inside** its namespace — and not on `tb4_scratch` itself, which + // has no trailing underscore and so is not `tb4-scratch`'s prefix. + let collisions = claim_collisions(&names, "tb4", ["tb4-scratch"]); + assert_eq!( + collisions + .iter() + .map(|(n, _)| n.as_str()) + .collect::>(), + ["tb4_scratch_idx"], + "exactly the names the sibling maps onto" + ); + assert!( + collisions.iter().all(|(_, other)| other == "tb4-scratch"), + "{collisions:?}" + ); + // A topic cannot collide with itself. + assert!(claim_collisions(&names, "tb4", ["tb4"]).is_empty()); +} + +/// The whole pipeline, end to end: a hyphenated topic's migration is +/// **allowed**, and a sibling's table is still refused. +#[test] +fn a_hyphenated_topics_migration_is_admitted() { + let topic = "fixture-topic-v0"; + // The exact statement the operator fixture carries. + allowed_for("CREATE TABLE fixture_topic_v0_scratch (id TEXT)", topic); + allowed_for( + "CREATE INDEX fixture_topic_v0_scratch_id ON fixture_topic_v0_scratch (id)", + topic, + ); + allowed_for( + "INSERT INTO fixture_topic_v0_scratch (id) VALUES ('a')", + topic, + ); + + // A sibling topic's table is not this topic's, however similar. + for sql in [ + "CREATE TABLE fixture_topic_v1_scratch (id TEXT)", + "CREATE TABLE topic_scratch (id TEXT)", + "SELECT id FROM proof_rule_version", + ] { + assert!( + check_migration(sql, topic).is_err(), + "{sql:?} must be refused for {topic}" + ); + } +} + +/// [`allowed`], for an arbitrary topic id. +fn allowed_for(sql: &str, topic: &str) { + check_migration(sql, topic) + .unwrap_or_else(|e| panic!("{sql:?} must be allowed for {topic}: {e}")); +} + +/// The **quoted** literal spelling is usable too. +/// +/// `"fixture-topic-v0_scratch"` is one legal identifier, and `is_topic_scoped` +/// accepts it. It is only reachable if the tokenizer keeps a quoted run whole: +/// splitting at the `-` yields `fixture` / `topic` / `v0_scratch`, none of which +/// is inside the topic's namespace, so a legal quoted name would be refused as +/// unscoped. +#[test] +fn a_quoted_hyphenated_name_is_one_identifier() { + let topic = "fixture-topic-v0"; + allowed_for( + r#"CREATE TABLE "fixture-topic-v0_scratch" (id TEXT)"#, + topic, + ); + allowed_for(r#"CREATE TABLE "fixture-topic-v0".runs (id TEXT)"#, topic); + allowed_for(r#"SELECT id FROM "fixture-topic-v0_scratch""#, topic); + + // Quoting is not an escape hatch: a sibling, a `topic_*` name, and a + // `proof_*` object are still refused when quoted. + for sql in [ + r#"CREATE TABLE "fixture-topic-v1_scratch" (id TEXT)"#, + r#"CREATE TABLE "topic_scratch" (id TEXT)"#, + r#"DROP TABLE "proof_rule_version""#, + ] { + assert!( + check_migration(sql, topic).is_err(), + "{sql:?} must be refused for {topic}" + ); + } +} + +/// A quoted identifier with an escaped quote still reads as one name. +#[test] +fn an_escaped_quote_inside_a_quoted_name_is_kept() { + // `""` inside a quoted identifier is one literal quote in the name. + let names = referenced_objects(r#"CREATE TABLE "a""b" (id TEXT)"#); + assert!( + names.iter().any(|t| t == "a\"b"), + "the doubled quote is one name, got {names:?}" + ); +} + /// A quoted identifier still names an object, so quoting cannot smuggle a /// denied table past the scanner. #[test] diff --git a/crates/proof-vm-agent/Cargo.toml b/crates/proof-vm-agent/Cargo.toml index b08e67a2a..e178cb2e1 100644 --- a/crates/proof-vm-agent/Cargo.toml +++ b/crates/proof-vm-agent/Cargo.toml @@ -28,6 +28,7 @@ tracing = "0.1" [dev-dependencies] http-body-util = "0.1" +proof-experiment = { path = "../proof-experiment" } proof-rlm = { path = "../proof-rlm", features = ["test-fixtures"] } tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "sync", "time"] } tower = { version = "0.5", features = ["util"] } diff --git a/crates/proof-vm-agent/src/lib.rs b/crates/proof-vm-agent/src/lib.rs index 7082742bb..9593d2bea 100644 --- a/crates/proof-vm-agent/src/lib.rs +++ b/crates/proof-vm-agent/src/lib.rs @@ -44,6 +44,7 @@ mod auth; mod hypervisor; +pub mod memory; mod router; mod stamp; @@ -55,6 +56,7 @@ pub mod fixtures; pub use auth::{AuthError, BearerAuth}; pub use hypervisor::{BootedVm, HvError, Hypervisor, JobOutcome}; +pub use memory::{nominal_total_mib, parse_mem_total_mib, MemoryBudget, MEMINFO}; pub use router::{agent_router, AgentError, AgentState, DEFAULT_MAX_EXPERIMENT_VMS}; pub use stamp::{output_matches, stamp_output}; @@ -772,6 +774,150 @@ mod tests { ); } + /// The Gate 4 shape, on the agent: two 8192 MiB experiment VMs beside the + /// topic's resident 8192 MiB RLM VM on a 16 GiB host. The **count** cap + /// (2) is satisfied, so the pre-fix agent booted all three and the kernel + /// OOM-killed the guests — both submissions answered 503 with no row. + /// + /// The memory budget refuses the third boot with `503 capacity` **before + /// a jail exists**, so the two submissions that fit keep running and only + /// the one that does not fit is refused. + #[tokio::test] + async fn a_boot_that_does_not_fit_the_host_is_refused_before_any_jail() { + use proof_rlm::fixtures::experiment_request; + let hv = FakeHypervisor::new(0.8); + let auth = Arc::new(BearerAuth::from_file(&token_file("mem-budget", TOKEN))); + let state = AgentState::with_limits( + hv.clone(), + auth, + 4, // the count cap is deliberately generous: memory is the gate + MemoryBudget { + total_mib: 16_384, + reserve_mib: 0, + }, + ); + let app = agent_router(state.clone()); + let topic = create(&app).await; + assert_eq!(topic.mem_mib, 4_096, "the RLM VM shape in this fixture"); + + let req = experiment_request(Some(8)); + let mut spec = experiment_spec(&req); + spec.template.mem_mib = 8_192; + let body = || serde_json::to_value(CreateVmRequest { spec: spec.clone() }).expect("json"); + // Topic 4096 + experiment 8192 = 12288 ≤ 16384: fits. + let (status, first): (StatusCode, VmRecord) = + call(&app, "POST", paths::VMS, Some(TOKEN), Some(body())).await; + assert_eq!(status, StatusCode::CREATED, "{first:?}"); + + // 12288 + 8192 = 20480 > 16384: refused, and **nothing was booted**. + let (status, err): (StatusCode, ErrorBody) = + call(&app, "POST", paths::VMS, Some(TOKEN), Some(body())).await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{err:?}"); + assert_eq!(err.code, ErrorCode::Capacity); + for want in [ + "12288 MiB is held", + "topic", + topic.handle.vm_id.as_str(), + "8192 MiB requested", + "4096 MiB free", + "16384 MiB VM ceiling", + ] { + assert!( + err.error.contains(want), + "the refusal names {want:?}: {}", + err.error + ); + } + assert_eq!( + hv.boots().len(), + 2, + "the RLM VM and the one experiment VM that fit; the third never booted" + ); + + // The experiment that fit is untouched and still answers `attach`'s + // sibling routes; the topic VM is still the topic's. + assert_eq!( + state.running().await.len(), + 2, + "a refused boot does not disturb what is running" + ); + assert!( + state + .running() + .await + .iter() + .any(|r| r.handle.vm_id == topic.handle.vm_id), + "the topic's RLM VM is still running" + ); + + // Freeing the experiment VM makes room again: the refusal was about + // capacity, not about the request. + let (status, down): (StatusCode, TeardownResponse) = call( + &app, + "DELETE", + &paths::vm(&first.handle.vm_id), + Some(TOKEN), + Some( + serde_json::to_value(TeardownRequest { + topic_id: first.handle.topic_id.clone(), + policy: RetainPolicy::Destroy, + }) + .expect("json"), + ), + ) + .await; + assert_eq!(status, StatusCode::OK, "{down:?}"); + let (status, again): (StatusCode, VmRecord) = + call(&app, "POST", paths::VMS, Some(TOKEN), Some(body())).await; + assert_eq!(status, StatusCode::CREATED, "{again:?}"); + } + + /// Health carries the budget, so an operator reading a `503 capacity` + /// sees what the host thinks it has without reaching for `free`. + #[tokio::test] + async fn health_reports_the_memory_budget_and_what_is_held() { + let hv = FakeHypervisor::new(0.8); + let auth = Arc::new(BearerAuth::from_file(&token_file("mem-health", TOKEN))); + let state = AgentState::with_limits( + hv.clone(), + auth, + 2, + MemoryBudget { + total_mib: 16_384, + reserve_mib: 1_024, + }, + ); + let app = agent_router(state.clone()); + let (status, health): (StatusCode, AgentHealth) = + call(&app, "GET", paths::HEALTH, Some(TOKEN), None).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(health.total_mib, 16_384); + assert_eq!(health.reserve_mib, 1_024); + assert_eq!(health.used_mib, 0, "nothing booted yet"); + + let topic = create(&app).await; + let (_, health): (StatusCode, AgentHealth) = + call(&app, "GET", paths::HEALTH, Some(TOKEN), None).await; + assert_eq!( + health.used_mib, + u64::from(topic.mem_mib), + "the topic's RLM VM holds its memory" + ); + + // A state with no operator budget reports 0/0 rather than inventing + // a host size, and admits everything (the CI default). + let plain = AgentState::with_max_experiment_vms( + hv, + Arc::new(BearerAuth::from_file(&token_file("mem-unset", TOKEN))), + 2, + ); + assert!(plain.memory_budget().is_unlimited()); + let plain_app = agent_router(plain); + let (_, health): (StatusCode, AgentHealth) = + call(&plain_app, "GET", paths::HEALTH, Some(TOKEN), None).await; + assert_eq!((health.total_mib, health.reserve_mib), (0, 0)); + } + fn experiment_spec(req: &proof_rlm::CustomRunRequest) -> TopicVmSpec { let binding = req.experiment().expect("binding").expect("selected"); let shape = proof_rlm::ExperimentCeilings::default() diff --git a/crates/proof-vm-agent/src/memory.rs b/crates/proof-vm-agent/src/memory.rs new file mode 100644 index 000000000..657a931b3 --- /dev/null +++ b/crates/proof-vm-agent/src/memory.rs @@ -0,0 +1,569 @@ +//! Host memory the agent admits VM boots against. +//! +//! A **count** cap is not a capacity cap. Gate 4 started two 8192 MiB +//! experiment VMs beside the topic's 8192 MiB RLM VM on a 16 GiB host: the +//! kernel OOM-killed the guests and **both** submissions answered 503 with no +//! row. Refusing one request with `503 capacity` is strictly better than +//! losing both, so a boot that would not fit is refused before any jail +//! exists — the same fail-closed shape as every other admission here. +//! +//! Nothing here sizes a VM. The numbers are the operator's (`mem_mib` per +//! spec, `PROOF_VM_AGENT_MEMORY_RESERVE_MIB`); this only decides whether the +//! host can carry what was asked for. + +use proof_vm_proto::VmRecord; + +/// Where the host's RAM is read from. +pub const MEMINFO: &str = "/proc/meminfo"; + +/// Host RAM and the headroom the operator keeps out of it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MemoryBudget { + /// Host RAM (`MemTotal`), MiB. + pub total_mib: u64, + /// Headroom kept for the OS, the agent, and per-VM process overhead, MiB. + /// + /// `0` is the default because a host that has been running a topic VM + /// beside one experiment VM is already at `total` and must keep working: + /// a non-zero default would refuse a shape that demonstrably fits. + pub reserve_mib: u64, +} + +impl MemoryBudget { + /// A budget that admits everything — the value a state built without an + /// operator budget carries, so unit tests are not tied to the box they + /// run on. The `proof-vm-orchestrator` binary always sets a real one. + #[must_use] + pub fn unlimited() -> Self { + Self { + total_mib: u64::MAX, + reserve_mib: 0, + } + } + + /// Whether this budget admits everything (see [`Self::unlimited`]). + #[must_use] + pub fn is_unlimited(&self) -> bool { + self.total_mib == u64::MAX && self.reserve_mib == 0 + } + + /// Read the host's RAM from [`MEMINFO`]. + /// + /// # Errors + /// + /// When the file cannot be read or carries no `MemTotal`: a host that + /// cannot prove it has room does not get to boot VMs on a guess. + pub fn read(reserve_mib: u64) -> Result { + let body = std::fs::read_to_string(MEMINFO).map_err(|e| format!("read {MEMINFO}: {e}"))?; + Self::from_meminfo(&body, reserve_mib) + } + + /// [`Self::read`] over a `/proc/meminfo` body. + /// + /// Split out so a test can drive the real parse — including the rounding + /// to the host's purchased size — without a host to read from. + /// + /// # Errors + /// + /// When the body carries no `MemTotal`. + pub fn from_meminfo(body: &str, reserve_mib: u64) -> Result { + let total_mib = + parse_mem_total_mib(body).ok_or_else(|| format!("{MEMINFO} carries no MemTotal"))?; + Ok(Self { + total_mib: nominal_total_mib(total_mib), + reserve_mib, + }) + } + + /// Most VM memory this host admits at once. + #[must_use] + pub fn ceiling_mib(&self) -> u64 { + self.total_mib.saturating_sub(self.reserve_mib) + } + + /// Whether a VM of `want_mib` fits beside the live ones. + /// + /// **Every** live VM counts, topic VMs included: the RLM VM is resident + /// for the topic's whole life and is not free capacity. An experiment + /// budget that ignores it is how Gate 4 oversubscribed. + /// + /// # What this is, and what it is not + /// + /// This compares **configured** guest memory against the host's. It is a + /// guard against the oversubscription that actually happened — three + /// 8 GiB guests on a 16 GiB host — and it is deliberately **not** a + /// residency model: + /// + /// - A guest's RAM is lazily populated. Firecracker maps the region; the + /// guest touches pages as it works. Gate 3 proves it: a paid run on a + /// topic VM beside one experiment VM is 8 GiB + 8 GiB + the 4 GiB + /// **sister** guest = 20 GiB of configured memory on a 16 GiB host, and + /// it passes. + /// - The **sister** guest is booted by the hypervisor inside a job and + /// never enters this count, for that reason: counting it would refuse + /// the proven Gate 3 shape. + /// + /// So do not "fix" this by summing every guest the host could ever boot: + /// that is the change that would take the working shape offline, which is + /// exactly the regression Greptile caught in `f0800353`. The guard is + /// sized to refuse the observed failure with margin, not to model the + /// kernel. + /// + /// # Errors + /// + /// The refusal, naming what holds the memory, what was asked for, and the + /// ceiling — so an operator reads why without reaching for `free`. + pub fn fits(&self, live: &[VmRecord], want_mib: u32) -> Result<(), String> { + if self.is_unlimited() { + return Ok(()); + } + let used: u64 = live.iter().map(|r| u64::from(r.mem_mib)).sum(); + let want = u64::from(want_mib); + let ceiling = self.ceiling_mib(); + if used.saturating_add(want) > ceiling { + let holders: Vec = live + .iter() + .map(|r| { + format!( + "{} {} ({} MiB)", + if r.experiment.is_some() { + "experiment" + } else { + "topic" + }, + r.handle.vm_id, + r.mem_mib + ) + }) + .collect(); + let free = ceiling.saturating_sub(used); + // The advice has to be reachable. A request larger than the whole + // ceiling cannot be made to fit by freeing anything: "retry when + // one finishes" would send the operator round a loop that cannot + // end, when the only answers are a smaller ask or a bigger host. + let remedy = if want > ceiling { + format!( + "The ask alone ({want} MiB) is larger than the whole {ceiling} MiB ceiling, \ + so freeing a vm cannot make it fit: lower the ask (the signed mem_mib / \ + PROOF_VM_AGENT_EXPERIMENT_MAX_MEM_MIB), or run this topic on a host whose \ + memory carries it" + ) + } else { + "Free a vm, retry when one finishes, or lower the ask (the signed mem_mib / \ + PROOF_VM_AGENT_EXPERIMENT_MAX_MEM_MIB)" + .to_owned() + }; + return Err(format!( + "host memory: {want} MiB requested, {free} MiB free of the {ceiling} MiB VM \ + ceiling ({total} MiB total − {reserve} MiB reserve); {used} MiB is held by {} \ + live vm(s) [{}]. The vm was not booted — refusing here keeps the running vms \ + alive instead of letting the host OOM-kill them. {remedy}", + live.len(), + holders.join(", "), + total = self.total_mib, + reserve = self.reserve_mib, + )); + } + Ok(()) + } +} + +/// `MemTotal` from a `/proc/meminfo` body, in MiB. +/// +/// `MemTotal: 16326344 kB` — the kernel reports KiB, so this floors to +/// whole MiB. A missing or unparseable line is `None`, never a guess. +#[must_use] +pub fn parse_mem_total_mib(body: &str) -> Option { + let line = body.lines().find(|l| l.starts_with("MemTotal:"))?; + let kib: u64 = line.split_whitespace().nth(1)?.parse().ok()?; + Some(kib / 1024) +} + +/// `MemTotal` rounded up to the whole GiB the host was sold as. +/// +/// `MemTotal` is the RAM the kernel can hand out, **not** the RAM the host +/// has: the kernel keeps a slice for itself, so a 16 GiB droplet reports +/// `16326344 kB` = `15_943` MiB, ~441 MiB short. Sizing VMs against the raw +/// figure refuses shapes the operator sized for and that demonstrably run — +/// the proven Gate 3 pair (an 8192 MiB topic VM beside an 8192 MiB experiment +/// VM) is `16_384` MiB of guest memory on exactly that host, and a raw +/// comparison took it offline. +/// +/// Guest RAM is also lazily populated: Firecracker maps the region but the +/// guest touches pages as it works, so a nominal 16 GiB host carries two +/// 8 GiB guests (Gate 3 passes) and does **not** carry three (Gate 4: the +/// kernel OOM-killed every guest at ~154% of `MemTotal`). Rounding to the +/// purchased size keeps the first and still refuses the second, without an +/// invented tolerance: RAM ships in whole GiB, so rounding restores the +/// operator's number and nothing more. +/// +/// A host whose `MemTotal` is already a whole GiB is unchanged. +#[must_use] +pub fn nominal_total_mib(mem_total_mib: u64) -> u64 { + const GIB: u64 = 1024; + if mem_total_mib == 0 { + return 0; + } + mem_total_mib.div_ceil(GIB) * GIB +} + +#[cfg(test)] +mod tests { + use super::*; + use proof_experiment::{ExperimentSpec, PackRef}; + use proof_rlm::{RetainPolicy, SandboxPolicy, VmHandle}; + use proof_vm_proto::{VmRecord, VmState}; + + fn vm(id: &str, mem_mib: u32, experiment: bool) -> VmRecord { + VmRecord { + handle: VmHandle { + topic_id: "tb4".into(), + vm_id: id.into(), + }, + image_digest: format!("sha256:{}", "ab".repeat(32)), + vcpus: 4, + mem_mib, + sandbox: SandboxPolicy { + firecracker_required: true, + deadline_s: 60, + }, + retain: RetainPolicy::Destroy, + state: VmState::Running, + experiment: experiment.then(|| ExperimentSpec { + runner: "placeholder_runner".into(), + pack: PackRef { + path: None, + digest: format!("sha256:{}", "cd".repeat(32)), + }, + disk_mib: 16_384, + }), + } + } + + /// Gate 3's shape must keep working: a topic VM beside one experiment VM + /// fills a 16 GiB host exactly, and that shape demonstrably passes. + #[test] + fn the_gate3_shape_still_fits() { + let budget = MemoryBudget { + total_mib: 16_384, + reserve_mib: 0, + }; + let topic = vm("tb4-0007", 8_192, false); + assert!(budget.fits(std::slice::from_ref(&topic), 8_192).is_ok()); + assert!( + budget + .fits(&[topic, vm("tb4-x0008", 8_192, true)], 8_192) + .is_err(), + "a third 8 GiB vm cannot fit a 16 GiB host" + ); + } + + /// Gate 4's shape is refused **before** a jail exists, with the numbers. + #[test] + fn the_gate4_oversubscription_is_refused_by_name() { + let budget = MemoryBudget { + total_mib: 16_384, + reserve_mib: 0, + }; + let live = [vm("tb4-0007", 8_192, false), vm("tb4-x0008", 8_192, true)]; + let err = budget + .fits(&live, 8_192) + .expect_err("two experiments beside a topic do not fit"); + for want in [ + "16384 MiB is held", + "tb4-0007", + "tb4-x0008", + "topic tb4-0007 (8192 MiB)", + "experiment tb4-x0008 (8192 MiB)", + "8192 MiB requested", + "0 MiB free", + "16384 MiB VM ceiling", + ] { + assert!(err.contains(want), "the refusal names {want:?}: {err}"); + } + } + + /// The topic VM is what makes Gate 4 not fit: the same two experiments on + /// a host with no RLM VM up would fit, which is exactly the mistake a + /// count-only cap makes. + #[test] + fn the_guard_counts_the_topic_vm_not_just_experiments() { + let budget = MemoryBudget { + total_mib: 16_384, + reserve_mib: 0, + }; + let experiments_only = [vm("tb4-x0008", 8_192, true)]; + assert!( + budget.fits(&experiments_only, 8_192).is_ok(), + "two experiments alone fit" + ); + let with_topic = [vm("tb4-0007", 8_192, false), vm("tb4-x0008", 8_192, true)]; + assert!( + budget.fits(&with_topic, 8_192).is_err(), + "the resident RLM VM is not free capacity" + ); + } + + /// The reserve is the operator's headroom, and it lowers the ceiling. + #[test] + fn the_reserve_lowers_the_ceiling() { + let budget = MemoryBudget { + total_mib: 16_384, + reserve_mib: 1_024, + }; + assert_eq!(budget.ceiling_mib(), 15_360); + let topic = vm("tb4-0007", 8_192, false); + assert!( + budget.fits(&[topic], 8_192).is_err(), + "a 1 GiB reserve refuses the exact-fit shape, as the operator asked" + ); + } + + /// `unlimited` admits everything, so a state built without an operator + /// budget behaves exactly as it did before this guard existed. + #[test] + fn an_unset_budget_admits_everything() { + let budget = MemoryBudget::unlimited(); + assert!(budget.is_unlimited()); + let live: Vec = (0..8) + .map(|i| vm(&format!("vm-{i}"), 32_768, true)) + .collect(); + assert!(budget.fits(&live, 32_768).is_ok()); + } + + #[test] + fn memtotal_is_read_in_mib_and_never_guessed() { + let body = "MemTotal: 16326344 kB\nMemFree: 1234567 kB\n"; + assert_eq!(parse_mem_total_mib(body), Some(15_943)); + assert_eq!(parse_mem_total_mib("MemFree: 1 kB\n"), None); + assert_eq!(parse_mem_total_mib("MemTotal: not-a-number kB\n"), None); + assert_eq!(parse_mem_total_mib(""), None); + } + + /// The kernel does not hand out the RAM the host was sold: a 16 GiB + /// droplet reports `MemTotal: 16326344 kB` = `15_943` MiB, ~441 MiB short. + /// + /// Sizing against the raw figure refuses the **proven Gate 3 pair** — an + /// 8192 MiB topic VM beside an 8192 MiB experiment VM, exactly the + /// workload that passes on this host — so the budget rounds to the + /// purchased whole GiB. This is the regression Greptile caught: the + /// guard must not take a working shape offline. + /// + /// This drives [`MemoryBudget::from_meminfo`], the real read path, so it + /// fails if the rounding is dropped — a test that built the budget by + /// hand would pass with the bug still in. + #[test] + fn a_nominal_host_keeps_the_proven_gate3_pair_admitted() { + let body = "MemTotal: 16326344 kB\n"; + assert_eq!( + parse_mem_total_mib(body), + Some(15_943), + "the raw figure is short of 16 GiB" + ); + + let budget = MemoryBudget::from_meminfo(body, 0).expect("parsed"); + assert_eq!( + budget.total_mib, 16_384, + "the budget must round to the sold size, not the raw figure" + ); + + let topic = vm("tb4-0007", 8_192, false); + assert!( + budget.fits(std::slice::from_ref(&topic), 8_192).is_ok(), + "the proven Gate 3 pair must stay admitted on a nominal 16 GiB host" + ); + assert!( + budget + .fits(&[topic, vm("tb4-x0008", 8_192, true)], 8_192) + .is_err(), + "and Gate 4's third guest is still refused: rounding restores the \ + operator's number, it does not invent headroom" + ); + } + + /// The shape the Owner is tipping staging to for the Gate 4 retry: + /// `experiment_mem_mib: 4096`, so a 16 GiB host runs the resident 8 GiB + /// topic VM **beside two 4 GiB experiment VMs** — 16,384 MiB, exactly the + /// ceiling. + /// + /// This is the retry Gate 4 depends on, so it is pinned here: an + /// off-by-one or a reserve default would refuse it and Gate 4 would stall + /// on the admission guard instead of the OOM. + #[test] + fn the_tipped_gate4_retry_shape_is_admitted() { + let budget = MemoryBudget { + total_mib: 16_384, + reserve_mib: 0, + }; + let topic = vm("tb4-0007", 8_192, false); + let first = vm("tb4-x0008", 4_096, true); + // Topic + one experiment. + assert!( + budget.fits(&[topic.clone(), first.clone()], 4_096).is_ok(), + "the first 4 GiB experiment fits beside the topic vm" + ); + // Topic + two experiments = exactly the ceiling: still admitted. + assert!( + budget.fits(&[topic.clone(), first.clone()], 4_096).is_ok(), + "the second 4 GiB experiment fills the host exactly and must be admitted" + ); + let live = [topic.clone(), first.clone(), vm("tb4-x0009", 4_096, true)]; + let used: u64 = live.iter().map(|r| u64::from(r.mem_mib)).sum(); + assert_eq!(used, 16_384, "the tipped shape is an exact fit"); + // A third experiment has nothing left. + assert!( + budget.fits(&live, 4_096).is_err(), + "a third experiment has no memory left" + ); + } + + /// The refusal names the **free** memory and what holds it, so an operator + /// reading a 503 knows what to free without reaching for `free` — and it + /// says the refusal is what keeps the running VMs alive. + #[test] + fn the_refusal_names_free_memory_and_its_holders() { + let budget = MemoryBudget { + total_mib: 16_384, + reserve_mib: 0, + }; + let live = [vm("tb4-0007", 8_192, false), vm("tb4-x0008", 4_096, true)]; + let err = budget.fits(&live, 8_192).expect_err("no room"); + for want in [ + "8192 MiB requested", + "4096 MiB free", + "16384 MiB VM ceiling", + "12288 MiB is held", + "topic tb4-0007 (8192 MiB)", + "experiment tb4-x0008 (4096 MiB)", + "OOM-kill", + ] { + assert!(err.contains(want), "the refusal names {want:?}: {err}"); + } + } + + /// The two capacity refusals are distinguishable: this one is about host + /// RAM, and the count cap (`max_experiment_vms`) has its own wording. An + /// operator must not read "retry when one finishes" for a host that is + /// simply too small for the shape. + #[test] + fn the_memory_refusal_does_not_read_as_the_count_cap() { + let budget = MemoryBudget { + total_mib: 16_384, + reserve_mib: 0, + }; + // A shape that genuinely does not fit: the topic VM plus a 32 GiB ask + // on a 16 GiB host. (A lone 8 GiB topic VM has room for an 8 GiB + // experiment, so that shape is admitted and cannot be the probe.) + let live = [vm("tb4-0007", 8_192, false)]; + let err = budget.fits(&live, 32_768).expect_err("no room"); + assert!( + !err.contains("PROOF_VM_AGENT_MAX_EXPERIMENT_VMS"), + "the memory refusal must not be mistaken for the count cap: {err}" + ); + assert!( + err.contains("host memory"), + "it names the constraint it is about: {err}" + ); + } + + /// A request bigger than the whole ceiling cannot be made to fit by + /// freeing anything, so the refusal must not recommend it. + /// + /// Greptile's P2: with **zero** live VMs the old wording still said "free + /// a vm, retry when one finishes", sending an operator round a loop that + /// cannot end. The only real answers are a smaller ask or a bigger host. + #[test] + fn an_impossible_ask_does_not_recommend_freeing_a_vm() { + let budget = MemoryBudget { + total_mib: 16_384, + reserve_mib: 0, + }; + // Nothing is live, and the ask alone is over the ceiling. + let err = budget.fits(&[], 32_768).expect_err("cannot fit"); + assert!( + !err.contains("Free a vm") && !err.contains("retry when one finishes"), + "freeing a vm cannot help here, so it must not be suggested: {err}" + ); + assert!( + err.contains("larger than the whole 16384 MiB ceiling"), + "the refusal says why freeing cannot help: {err}" + ); + assert!( + err.contains("lower the ask"), + "and names the remedy that can work: {err}" + ); + + // A request that *would* fit on an empty host still gets the normal + // advice: the two cases must not be conflated. + let roomy = budget + .fits(&[vm("tb4-0007", 8_192, false)], 16_384) + .expect_err("the topic vm is in the way"); + assert!( + roomy.contains("Free a vm"), + "a request that fits an empty host gets the normal advice: {roomy}" + ); + } + + /// The raw figure really is what refuses the working shape — the reason + /// the rounding exists, pinned so it cannot be dropped as "just noise". + #[test] + fn the_raw_figure_would_refuse_the_working_shape() { + let raw_budget = MemoryBudget { + total_mib: 15_943, + reserve_mib: 0, + }; + let topic = vm("tb4-0007", 8_192, false); + assert!( + raw_budget + .fits(std::slice::from_ref(&topic), 8_192) + .is_err(), + "the unrounded figure refuses the shape that demonstrably runs" + ); + } + + /// The guard counts **configured** guest memory, and the sister guest is + /// deliberately not in it — the boundary, pinned so it is not "fixed" + /// into re-breaking Gate 3. + /// + /// A paid run on the proven Gate 3 shape is 8 GiB topic + 8 GiB + /// experiment + the 4 GiB sister = 20 GiB of configured memory on a + /// 16 GiB host, and it **passes**: guest RAM is lazily populated, so the + /// configured sum is not the resident set. Counting the sister here would + /// refuse that shape. + #[test] + fn the_guard_is_not_a_residency_model() { + let budget = MemoryBudget { + total_mib: 16_384, + reserve_mib: 0, + }; + // The pair the host really runs — sister excluded, as it is in the + // agent (the hypervisor boots it inside a job, never in `state.vms`). + let live = [vm("tb4-0007", 8_192, false), vm("tb4-x0008", 8_192, true)]; + assert!( + budget.fits(&live, 8_192).is_err(), + "the third configured guest is refused" + ); + // With the sister's 4 GiB added, the same live set already exceeds a + // 16 GiB host *before* the third guest — which is why the sister is + // not counted. If a change makes this fail, it has turned the guard + // into a residency model and taken Gate 3 with it. + let with_sister: u64 = live.iter().map(|r| u64::from(r.mem_mib)).sum::() + 4_096; + assert!( + with_sister > budget.ceiling_mib(), + "configured memory with the sister exceeds the host, yet the shape runs" + ); + } + + /// Rounding is a no-op on a host that already reports a whole GiB, and + /// zero stays zero (no host size is invented). + #[test] + fn rounding_restores_the_sold_size_and_nothing_more() { + assert_eq!(nominal_total_mib(16_384), 16_384); + assert_eq!(nominal_total_mib(32_768), 32_768); + assert_eq!(nominal_total_mib(0), 0); + // 8 GiB sold, ~7.8 GiB visible. + assert_eq!(nominal_total_mib(7_975), 8_192); + // A smaller host is not rounded *up* into one that fits more than it has. + assert_eq!(nominal_total_mib(1_024), 1_024); + } +} diff --git a/crates/proof-vm-agent/src/router.rs b/crates/proof-vm-agent/src/router.rs index 0467f12e3..342e6953c 100644 --- a/crates/proof-vm-agent/src/router.rs +++ b/crates/proof-vm-agent/src/router.rs @@ -37,6 +37,7 @@ use tokio::sync::{Mutex, OwnedMutexGuard, RwLock}; use crate::auth::BearerAuth; use crate::hypervisor::{BootedVm, HvError, Hypervisor}; +use crate::memory::MemoryBudget; use crate::stamp::{output_matches, stamp_output}; /// Longest `vm_id` the agent mints (jailer ids are capped at 64 chars). @@ -116,6 +117,8 @@ struct Inner { create_lock: Mutex<()>, next_id: AtomicU64, max_experiment_vms: usize, + /// Host RAM the boots are admitted against ([`MemoryBudget`]). + memory: MemoryBudget, } /// Shared agent state. @@ -135,11 +138,37 @@ impl AgentState { /// State over `hypervisor` allowing at most `max_experiment_vms` /// dedicated experiment VMs to run at once (`0` disables them: every /// experiment create is 503 `capacity`). + /// + /// Memory admission is **unlimited** here; a host that has RAM to defend + /// sets a real budget with [`Self::with_limits`]. #[must_use] pub fn with_max_experiment_vms( hypervisor: Arc, auth: Arc, max_experiment_vms: usize, + ) -> Self { + Self::with_limits( + hypervisor, + auth, + max_experiment_vms, + MemoryBudget::unlimited(), + ) + } + + /// State with both caps: `max_experiment_vms` concurrent experiments, and + /// boots admitted against the host's RAM ([`MemoryBudget`]). + /// + /// A **count** cap is not a capacity cap: two 8192 MiB experiment VMs + /// beside the topic's resident 8192 MiB RLM VM do not fit a 16 GiB host, + /// and letting them start cost **both** submissions to the OOM killer + /// instead of one request a `503 capacity` (Gate 4). The budget counts + /// every live VM, topic VMs included. + #[must_use] + pub fn with_limits( + hypervisor: Arc, + auth: Arc, + max_experiment_vms: usize, + memory: MemoryBudget, ) -> Self { Self { inner: Arc::new(Inner { @@ -149,10 +178,17 @@ impl AgentState { create_lock: Mutex::new(()), next_id: AtomicU64::new(1), max_experiment_vms, + memory, }), } } + /// The memory budget in force. + #[must_use] + pub fn memory_budget(&self) -> MemoryBudget { + self.inner.memory + } + /// VMs that are running **and** whose process is alive right now. Dead /// ones found on the way are reaped. pub async fn running(&self) -> Vec { @@ -410,6 +446,8 @@ async fn health(State(state): State) -> Json { }; // Health is also where an idle host notices a VM that died in the meantime. let live = state.sweep().await; + let budget = state.inner.memory; + let used_mib: u64 = live.iter().map(|r| u64::from(r.mem_mib)).sum(); Json(AgentHealth { api_version: API_VERSION, ready, @@ -418,6 +456,13 @@ async fn health(State(state): State) -> Json { vms: state.inner.vms.read().await.len(), experiment_vms: live.iter().filter(|r| r.experiment.is_some()).count(), max_experiment_vms: state.inner.max_experiment_vms, + total_mib: if budget.is_unlimited() { + 0 + } else { + budget.total_mib + }, + reserve_mib: budget.reserve_mib, + used_mib, }) } @@ -432,11 +477,22 @@ async fn create_vm( // Serialise creates: the topic ↔ VM check (or the capacity check) and // the insert must be one step. let _create = state.inner.create_lock.lock().await; + // Memory admission, before the count cap: the kernel does not honour a + // count. Two 8192 MiB experiment VMs beside the topic's resident RLM VM + // do not fit a 16 GiB host, and starting them cost **both** submissions + // to the OOM killer (Gate 4) instead of refusing one with `503 capacity`. + // Live VMs are reaped first, so a crashed one does not hold budget. + let live = state.sweep().await; + state + .inner + .memory + .fits(&live, spec.template.mem_mib) + .map_err(|why| AgentError::new(ErrorCode::Capacity, why))?; if spec.experiment.is_some() { // One VM per experiment: the topic's RLM VM is not in the way, and // other experiments of the same topic may run beside this one — up // to what this host can carry. Dead ones are reaped and do not count. - let running = state.running_experiments().await; + let running = live.iter().filter(|r| r.experiment.is_some()).count(); let max = state.inner.max_experiment_vms; if running >= max { return Err(AgentError::new( diff --git a/crates/proof-vm-agent/src/stamp.rs b/crates/proof-vm-agent/src/stamp.rs index e485ee92c..6f4c28d0d 100644 --- a/crates/proof-vm-agent/src/stamp.rs +++ b/crates/proof-vm-agent/src/stamp.rs @@ -23,8 +23,13 @@ use proof_vm_proto::SisterAttestation; pub fn output_matches(job: &VmJob, output: &VmJobOutput) -> bool { matches!( (job, output), - (VmJob::ProposeRules { .. }, VmJobOutput::Rules(_)) - | (VmJob::Baseline { .. }, VmJobOutput::Baseline(_)) + // An authored set answers a `ProposeRules` job; so does a bare rule + // vector, which is what an adaptor that writes only `rules.json` + // returns. The host widens the latter into the former. + ( + VmJob::ProposeRules { .. }, + VmJobOutput::Authored(_) | VmJobOutput::Rules(_) + ) | (VmJob::Baseline { .. }, VmJobOutput::Baseline(_)) | (VmJob::Inspect { .. }, VmJobOutput::Inspected(_)) | (VmJob::Evaluate { .. }, VmJobOutput::Evaluated(_)) | (VmJob::Archive { .. }, VmJobOutput::Archived) @@ -47,7 +52,10 @@ pub fn stamp_output(mut output: VmJobOutput, sister: Option<&SisterAttestation>) match &mut output { VmJobOutput::Baseline(report) => stamp_report(report, sister), VmJobOutput::Evaluated(run) => stamp_report(&mut run.report, sister), - VmJobOutput::Rules(_) | VmJobOutput::Inspected(_) | VmJobOutput::Archived => {} + VmJobOutput::Authored(_) + | VmJobOutput::Rules(_) + | VmJobOutput::Inspected(_) + | VmJobOutput::Archived => {} } output } diff --git a/crates/proof-vm-fc/src/lib.rs b/crates/proof-vm-fc/src/lib.rs index 5e9dd1c34..d85ac8927 100644 --- a/crates/proof-vm-fc/src/lib.rs +++ b/crates/proof-vm-fc/src/lib.rs @@ -216,6 +216,76 @@ fn backend(msg: impl Into) -> VmError { VmError::Backend(msg.into()) } +/// Whether a refusal is the agent's `AlreadyExists` (409). +/// +/// `call` folds every non-2xx into [`VmError::Backend`] as +/// `orchestrator {status} on {method} {path}: {code}: {error}`, so both the +/// status and the agent's code survive as text. Matching **both** keeps "the +/// topic already has a VM" distinguishable from every other backend failure +/// without depending on the `Debug` rendering alone, so `create` can attach +/// instead of failing the submission. +fn is_already_exists(e: &VmError) -> bool { + matches!( + e, + VmError::Backend(m) + if m.contains("AlreadyExists") && (m.contains("409") || m.contains("Conflict")) + ) +} + +/// Refuse a VM that is not the one `spec` asked for. +/// +/// Applies to a **created** record and to an attached leftover alike: a VM +/// booted from an older image pin, or at another shape, would silently run +/// the submission in a guest the topic does not describe, and the evidence it +/// produced would be evidence about a configuration nobody signed. +fn check_record(record: &VmRecord, spec: &TopicVmSpec) -> Result<(), VmError> { + if record.handle.topic_id != spec.topic_id { + return Err(backend(format!( + "orchestrator bound the vm to {:?}, asked for {:?}", + record.handle.topic_id, spec.topic_id + ))); + } + if !record + .image_digest + .eq_ignore_ascii_case(&spec.template.image_digest) + { + return Err(backend(format!( + "orchestrator booted image {} instead of the pinned {}", + record.image_digest, spec.template.image_digest + ))); + } + if record.vcpus != spec.template.vcpus || record.mem_mib != spec.template.mem_mib { + return Err(backend(format!( + "the topic's vm is {}vCPU/{}MiB, this run asks for {}vCPU/{}MiB; \ + tear it down so the topic boots at the pinned shape", + record.vcpus, record.mem_mib, spec.template.vcpus, spec.template.mem_mib + ))); + } + if record.sandbox != spec.sandbox { + return Err(backend( + "the topic's vm carries a different sandbox policy than this run asks for; \ + tear it down so the topic boots under the signed policy", + )); + } + if record.experiment.is_some() != spec.experiment.is_some() { + return Err(backend(format!( + "orchestrator answered with {} vm {} for a {} create", + if record.experiment.is_some() { + "an experiment" + } else { + "a topic" + }, + record.handle.vm_id, + if spec.experiment.is_some() { + "experiment" + } else { + "topic" + } + ))); + } + Ok(()) +} + impl FirecrackerOrchestrator { /// Build the client. Reads the CA file (if any) now; the token per request. pub fn new(config: FcConfig) -> Result { @@ -357,6 +427,26 @@ impl FirecrackerOrchestrator { Duration::from_secs(d.saturating_add(JOB_GRACE_S)) }) } + + /// The topic's record, bound to the topic it was asked about. + async fn attach_record(&self, topic_id: &str) -> Result, VmError> { + let found: Option = self + .call::<(), VmRecord>( + Method::GET, + &paths::vm_by_topic(topic_id.trim()), + None, + self.config.connect_timeout, + ) + .await?; + match found { + Some(r) if r.handle.topic_id == topic_id.trim() => Ok(Some(r)), + Some(r) => Err(backend(format!( + "orchestrator returned vm {} of topic {:?} for topic {topic_id:?}", + r.handle.vm_id, r.handle.topic_id + ))), + None => Ok(None), + } + } } fn check_echo(handle: &VmHandle, topic_id: &str, vm_id: &str) -> Result<(), VmError> { @@ -383,52 +473,61 @@ impl TopicVmOrchestrator for FirecrackerOrchestrator { async fn create(&self, spec: &TopicVmSpec) -> Result { self.ready()?; spec.validate()?; - let record: VmRecord = self - .call( + let created = self + .call::( Method::POST, paths::VMS, Some(&CreateVmRequest { spec: spec.clone() }), self.config.create_timeout, ) - .await? - .ok_or_else(|| backend("orchestrator has no create route"))?; - if record.handle.topic_id != spec.topic_id { - return Err(backend(format!( - "orchestrator bound the vm to {:?}, asked for {:?}", - record.handle.topic_id, spec.topic_id - ))); - } - if !record - .image_digest - .eq_ignore_ascii_case(&spec.template.image_digest) - { - return Err(backend(format!( - "orchestrator booted image {} instead of the pinned {}", - record.image_digest, spec.template.image_digest - ))); - } + .await; + let record: VmRecord = match created { + Ok(Some(record)) => record, + // The topic already has a live VM — a leftover from the RLM + // install/baseline that still holds its jail and TAP. That is not + // a failure: it is the VM this submission wants. Attaching keeps + // the submit path idempotent, so a miner's run does not depend on + // an operator clearing the host first. + // + // Only a **topic** spec may attach. An experiment spec asks for a + // dedicated VM for one paid job; attaching would hand back the + // topic's RLM VM and run the job in the wrong guest, so a 409 + // there stays an error. + Err(e) if spec.experiment.is_none() && is_already_exists(&e) => { + tracing::info!( + topic_id = %spec.topic_id, + "topic vm already exists on the host; attaching instead of creating" + ); + let found = self.attach_record(&spec.topic_id).await?.ok_or_else(|| { + backend(format!( + "orchestrator reported the topic already has a vm, but attach found none \ + for {:?}", + spec.topic_id + )) + })?; + // An existing VM is only usable if it is the VM this spec + // asked for. A leftover booted from an older image pin (or a + // different shape) would silently run the submission in a + // guest the topic does not describe, so it is refused here + // rather than scored under the wrong configuration. + check_record(&found, spec)?; + tracing::info!( + topic_id = %spec.topic_id, vm_id = %found.handle.vm_id, + "attached to the topic's existing vm" + ); + return Ok(found.handle); + } + Err(e) => return Err(e), + Ok(None) => return Err(backend("orchestrator has no create route")), + }; + check_record(&record, spec)?; tracing::info!(topic_id = %spec.topic_id, vm_id = %record.handle.vm_id, "topic vm created"); Ok(record.handle) } async fn attach(&self, topic_id: &str) -> Result, VmError> { self.ready()?; - let found: Option = self - .call::<(), VmRecord>( - Method::GET, - &paths::vm_by_topic(topic_id.trim()), - None, - self.config.connect_timeout, - ) - .await?; - match found { - Some(r) if r.handle.topic_id == topic_id.trim() => Ok(Some(r.handle)), - Some(r) => Err(backend(format!( - "orchestrator returned vm {} of topic {:?} for topic {topic_id:?}", - r.handle.vm_id, r.handle.topic_id - ))), - None => Ok(None), - } + Ok(self.attach_record(topic_id).await?.map(|r| r.handle)) } async fn run(&self, handle: &VmHandle, job: VmJob) -> Result { @@ -476,7 +575,10 @@ impl TopicVmOrchestrator for FirecrackerOrchestrator { let claims_sandbox = match &resp.output { VmJobOutput::Baseline(r) => r.sandboxed, VmJobOutput::Evaluated(run) => run.report.sandboxed, - VmJobOutput::Rules(_) | VmJobOutput::Inspected(_) | VmJobOutput::Archived => false, + VmJobOutput::Authored(_) + | VmJobOutput::Rules(_) + | VmJobOutput::Inspected(_) + | VmJobOutput::Archived => false, }; if claims_sandbox && !attested { return Err(backend( diff --git a/crates/proof-vm-fc/tests/live_agent.rs b/crates/proof-vm-fc/tests/live_agent.rs index c4916547d..f9d5af57c 100644 --- a/crates/proof-vm-fc/tests/live_agent.rs +++ b/crates/proof-vm-fc/tests/live_agent.rs @@ -466,9 +466,22 @@ async fn the_created_vm_must_run_the_pinned_image_and_a_fake_answer_is_refused() let orch = client(&agent, &token, &other.image_digest); let handle = orch.create(&spec(other.clone())).await.expect("create"); assert_eq!(agent.hypervisor.boots()[0].image_digest, other.image_digest); - // A second create for the same topic is the agent's one-VM-per-topic rule. - let err = orch.create(&spec(other)).await.expect_err("duplicate"); - assert!(err.to_string().contains("AlreadyExists"), "{err}"); + // A second create for the same topic meets the agent's one-VM-per-topic + // rule (409 `AlreadyExists`). The client turns that into the **same** + // handle instead of failing the submission: the VM the topic already has + // — a leftover from the RLM install/baseline that still holds its jail + // and TAP — is the VM this run wants, so a miner's submission does not + // depend on an operator clearing the host first. + let again = orch + .create(&spec(other)) + .await + .expect("a duplicate create attaches, it does not fail"); + assert_eq!(again, handle, "the topic's existing vm, not a new one"); + assert_eq!( + agent.hypervisor.boots().len(), + 1, + "attaching must not boot a second vm" + ); assert!(orch .teardown(&handle, RetainPolicy::Retain) .await @@ -483,6 +496,90 @@ async fn the_created_vm_must_run_the_pinned_image_and_a_fake_answer_is_refused() reference.ready().expect("reference"); } +/// The attach fallback must not run a submission in a VM the spec did not ask +/// for: a leftover booted from an **older image pin** (or another shape) would +/// silently score a run in a guest the topic does not describe, and its +/// evidence would be evidence about a configuration nobody signed. +#[tokio::test] +async fn a_duplicate_create_refuses_an_incompatible_existing_vm() { + let (agent, token) = live("pin-mismatch").await; + let mut old = pinned_template(); + old.image_digest = format!("sha256:{}", "ee".repeat(32)); + // The topic's VM is booted from the older pin the host still runs. + let orch_old = client(&agent, &token, &old.image_digest); + let existing = orch_old.create(&spec(old.clone())).await.expect("create"); + assert_eq!(agent.hypervisor.boots().len(), 1); + + // A run pinned to the newer image asks for the same topic. The agent + // answers 409; attaching would hand back a VM on the wrong image, so the + // client must refuse instead. + let orch_new = client(&agent, &token, &pinned_template().image_digest); + let err = orch_new + .create(&spec(pinned_template())) + .await + .expect_err("an incompatible leftover must not be attached"); + let text = err.to_string(); + assert!(text.contains("instead of the pinned"), "{text}"); + assert_eq!( + agent.hypervisor.boots().len(), + 1, + "nothing new was booted and nothing was reused" + ); + assert_eq!( + orch_new.attach(&existing.topic_id).await.expect("attach"), + Some(existing.clone()), + "the vm itself is untouched; only this run refused it" + ); + assert!(orch_old + .teardown(&existing, RetainPolicy::Destroy) + .await + .expect("destroy")); +} + +/// An **experiment** spec must not take the attach path: a dedicated VM per +/// paid job is the whole point, and attaching would hand back the topic's RLM +/// VM and run the job in the wrong guest. The agent's `attach` answers only +/// topic VMs, so an experiment create is always a boot. +#[tokio::test] +async fn an_experiment_create_boots_its_own_vm_and_never_attaches() { + use proof_rlm::fixtures::experiment_request; + use proof_rlm::TopicVmSpec; + let (agent, token) = live("exp-dup").await; + let orch = client(&agent, &token, &pinned_template().image_digest); + let req = experiment_request(Some(2)); + let exp = TopicVmSpec::for_experiment( + &req.topic_id, + pinned_template(), + req.sandbox.clone(), + proof_rlm::ExperimentSpec { + runner: "placeholder_runner".into(), + pack: proof_rlm::PackRef { + path: None, + digest: format!("sha256:{}", "ab".repeat(32)), + }, + disk_mib: 16_384, + }, + ); + let first = orch.create(&exp).await.expect("experiment vm"); + assert_eq!(agent.hypervisor.boots().len(), 1, "one dedicated vm"); + // A second experiment VM is a second boot, never an attach: the topic's + // RLM VM is not in the way and `attach` does not answer experiment VMs. + let second = orch.create(&exp).await.expect("second experiment vm"); + assert_ne!(first.vm_id, second.vm_id, "each job gets its own vm"); + assert_eq!(agent.hypervisor.boots().len(), 2); + assert_eq!( + orch.attach(&req.topic_id).await.expect("attach"), + None, + "an experiment vm never answers attach" + ); + for vm in [&first, &second] { + assert!(orch + .teardown(vm, RetainPolicy::Destroy) + .await + .expect("destroy")); + } +} + #[tokio::test] async fn from_env_is_none_when_unset_and_reads_the_locked_shape() { let _guard = ENV diff --git a/crates/proof-vm-guest/Cargo.toml b/crates/proof-vm-guest/Cargo.toml index ef696f086..101f003b9 100644 --- a/crates/proof-vm-guest/Cargo.toml +++ b/crates/proof-vm-guest/Cargo.toml @@ -16,6 +16,7 @@ proof-results = { path = "../proof-results" } proof-rlm = { path = "../proof-rlm" } proof-task = { path = "../proof-task" } proof-vm-proto = { path = "../proof-vm-proto" } +proof-vm-staging = { path = "../proof-vm-staging" } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "http2"] } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/proof-vm-guest/src/agent_tests.rs b/crates/proof-vm-guest/src/agent_tests.rs index c3339e985..03b653404 100644 --- a/crates/proof-vm-guest/src/agent_tests.rs +++ b/crates/proof-vm-guest/src/agent_tests.rs @@ -766,6 +766,7 @@ echo "{\"primary_value\": 1.0, \"evidence\": {\"foo_bar\": \"$PROOF_PARAM_FOO_BA job: Box::new(VmJob::ProposeRules { topic: Box::new(t), current_version: None, + current: None, }), }) .await, @@ -991,10 +992,11 @@ echo '{"primary_value": 0.25}' > "$PROOF_OUTPUT_DIR/report.json" } /// Inspection ticks every rule through `inspect` (unanswered rules are red), -/// and rule proposals come from `propose_rules` or, without one, from the -/// signed checklist itself. +/// and rule proposals come **only** from `propose_rules`: a runner without one +/// fails closed rather than echoing the signed checklist back as if the RLM +/// had authored it. #[tokio::test] -async fn inspection_and_rule_proposals_go_through_the_adaptor_or_the_signed_topic() { +async fn inspection_and_rule_proposals_go_through_the_adaptor_or_fail_closed() { let r = root("inspect"); let a = agent(&r); hello(&a).await; @@ -1058,22 +1060,25 @@ EOF assert!(by_id("rule_c").evidence.contains("no verdict")); assert!(inspected.artifact.is_empty(), "no locator, no tree"); + // No `propose_rules` entrypoint: the guest refuses. Echoing the signed + // checklist would let the control plane record the operator's own vector + // as RLM-authored rules. let t = topic(); - let out = a - .handle(HostToRlm::Run { + let err = failed( + a.handle(HostToRlm::Run { job: Box::new(VmJob::ProposeRules { topic: Box::new(t.clone()), current_version: None, + current: None, }), }) - .await; - let RlmToHost::Done { - output: VmJobOutput::Rules(proposed), - } = out - else { - panic!("expected rules, got {out:?}"); - }; - assert_eq!(proposed, t.checklist, "no adaptor: the signed vector"); + .await, + ); + assert!(err.contains("no propose_rules entrypoint"), "{err}"); + assert!( + err.contains("never a substitute for RLM authorship"), + "the refusal says why echoing is wrong: {err}" + ); let mut selecting = t.clone(); selecting .constraints @@ -1096,6 +1101,7 @@ echo '[{"id": "rlm_rule_x", "text": "an adaptor-written rule"}]' > "$PROOF_OUTPU job: Box::new(VmJob::ProposeRules { topic: Box::new(selecting.clone()), current_version: Some(1), + current: None, }), }) .await; @@ -1117,6 +1123,7 @@ echo '[{"id": "rlm_rule_x", "text": "an adaptor-written rule"}]' > "$PROOF_OUTPU job: Box::new(VmJob::ProposeRules { topic: Box::new(selecting), current_version: Some(1), + current: None, }), }) .await, @@ -1138,6 +1145,159 @@ echo '[{"id": "rlm_rule_x", "text": "an adaptor-written rule"}]' > "$PROOF_OUTPU let _ = std::fs::remove_dir_all(&r); } +/// The RLM's whole set travels: `authoring.json` is read, checked against the +/// same gates the control plane runs, and answered as `Authored` — while a +/// rules-only adaptor still answers `Rules`, which the host treats as a +/// fragment rather than authorship. +/// +/// This is the wire half of the authorship pin: what a topic *is* comes from +/// its own RLM, and the guest is where that claim is first checked. +#[tokio::test] +async fn the_rlm_authors_its_whole_set_and_a_fragment_is_named_as_one() { + let r = root("authoring-set"); + let a = agent(&r); + hello(&a).await; + let t = topic(); + let mut selecting = t.clone(); + selecting + .constraints + .params + .insert(proof_experiment::PARAM_RUNNER.into(), RUNNER.into()); + selecting + .constraints + .params + .insert(proof_experiment::PARAM_PACK_DIGEST.into(), pack().1); + // The adaptor resolves only when its `run` entrypoint exists: a + // `propose_rules`-only directory is not an installed runner. + install(&r, "run", "true"); + + // A complete set: every part present, shaped like the document's own + // knobs, written by the adaptor. + install( + &r, + "propose_rules", + r#" +test -f "$PROOF_TOPIC_FILE" +cat > "$PROOF_OUTPUT_DIR/authoring.json" <<'JSON' +{ + "schema_version": 1, + "topic_id": "topic-a", + "rules": [{"id": "rlm_authored_rule", "text": "the rlm wrote this"}], + "migrations": [{"name": "0001_scratch", "sql": "CREATE TABLE topic_a_scratch (id TEXT)"}], + "apis": [{"path": "status", "method": "GET", "summary": "topic status"}], + "submission_format": {"kind": "tar", "max_bytes": 5242880}, + "pin_policy": {} +} +JSON +"#, + ); + let out = a + .handle(HostToRlm::Run { + job: Box::new(VmJob::ProposeRules { + topic: Box::new(selecting.clone()), + current_version: None, + current: None, + }), + }) + .await; + let RlmToHost::Done { + output: VmJobOutput::Authored(set), + } = out + else { + panic!("expected the whole set, got {out:?}"); + }; + assert_eq!(set.topic_id, "topic-a"); + assert_eq!(set.rules[0].id, "rlm_authored_rule"); + assert_eq!(set.migrations.len(), 1); + assert_eq!(set.apis[0].path, "status"); + assert!(set.pin_policy.is_empty(), "tightening nothing is a policy"); + assert!(set.is_complete()); + + // A set for **another topic** is refused: the VM is bound to one topic. + install( + &r, + "propose_rules", + r#"sed 's/"topic-a"/"topic-b"/' > /dev/null; echo '{"schema_version":1,"topic_id":"topic-b","rules":[{"id":"rlm_rule","text":"t"}],"migrations":[{"name":"0001_scratch","sql":"CREATE TABLE topic_a_scratch (id TEXT)"}],"apis":[{"path":"status","method":"GET"}],"submission_format":{"kind":"tar"},"pin_policy":{}}' > "$PROOF_OUTPUT_DIR/authoring.json""#, + ); + let err = failed( + a.handle(HostToRlm::Run { + job: Box::new(VmJob::ProposeRules { + topic: Box::new(selecting.clone()), + current_version: None, + current: None, + }), + }) + .await, + ); + assert!(err.contains("authoring.json is for topic"), "{err}"); + + // A migration that reaches outside the topic's namespace is refused where + // the RLM's answer arrives, not later at install time. + install( + &r, + "propose_rules", + r#"echo '{"schema_version":1,"topic_id":"topic-a","rules":[{"id":"rlm_rule","text":"t"}],"migrations":[{"name":"0001_scratch","sql":"DROP TABLE proof_rule_version"}],"apis":[{"path":"status","method":"GET"}],"submission_format":{"kind":"tar"},"pin_policy":{}}' > "$PROOF_OUTPUT_DIR/authoring.json""#, + ); + let err = failed( + a.handle(HostToRlm::Run { + job: Box::new(VmJob::ProposeRules { + topic: Box::new(selecting.clone()), + current_version: None, + current: None, + }), + }) + .await, + ); + assert!(err.contains("proof_rule_version"), "{err}"); + + // An incomplete set is refused by name: the parts the RLM did not author + // are not filled in from anywhere. + install( + &r, + "propose_rules", + r#"echo '{"schema_version":1,"topic_id":"topic-a","rules":[{"id":"rlm_rule","text":"t"}],"migrations":[],"apis":[{"path":"status","method":"GET"}],"submission_format":{"kind":"tar"},"pin_policy":{}}' > "$PROOF_OUTPUT_DIR/authoring.json""#, + ); + let err = failed( + a.handle(HostToRlm::Run { + job: Box::new(VmJob::ProposeRules { + topic: Box::new(selecting.clone()), + current_version: None, + current: None, + }), + }) + .await, + ); + assert!(err.contains("authored no migrations"), "{err}"); + + // A rules-only adaptor still answers `Rules`: the guest does not widen a + // fragment into a set, because the parts it would fill in are the + // operator's. The host is what names it as incomplete. + install( + &r, + "propose_rules", + r#"echo '[{"id": "rules_only_rule", "text": "only rules were authored"}]' > "$PROOF_OUTPUT_DIR/rules.json""#, + ); + let out = a + .handle(HostToRlm::Run { + job: Box::new(VmJob::ProposeRules { + topic: Box::new(selecting), + current_version: None, + current: None, + }), + }) + .await; + let RlmToHost::Done { + output: VmJobOutput::Rules(proposed), + } = out + else { + panic!("a fragment stays a fragment, got {out:?}"); + }; + assert_eq!(proposed[0].id, "rules_only_rule"); + assert_eq!(crate::runner::AUTHORING_FILE, "authoring.json"); + assert!(crate::runner::RULES_ONLY_IS_NOT_AUTHORSHIP.contains("authoring.json")); + let _ = std::fs::remove_dir_all(&r); +} + /// A leftover unsyncable entry under the shared work root (earlier failed /// job on a reused topic VM) must not turn Archive — or any later success /// that does not own that path — into Failed. diff --git a/crates/proof-vm-guest/src/lib.rs b/crates/proof-vm-guest/src/lib.rs index b862db0aa..babd907dd 100644 --- a/crates/proof-vm-guest/src/lib.rs +++ b/crates/proof-vm-guest/src/lib.rs @@ -28,10 +28,14 @@ //! 503, and no row is written. A placeholder `primary_value` never leaves //! this process. //! -//! One deliberate default exists: `ProposeRules` without a `propose_rules` -//! entrypoint answers with the signed topic's own `checklist` (v1 of the -//! rule set is that vector by contract), so a topic whose adaptor writes no -//! rules still scores under the rules its operator signed. +//! **No default exists for authorship.** `ProposeRules` is answered only by +//! the adaptor's own `propose_rules` entrypoint: a runner that ships none is +//! `Failed`, and one that writes only `rules.json` answers with that fragment +//! rather than a whole set. The guest never echoes the signed `checklist` +//! back — that would let the control plane record the operator's vector as +//! RLM-authored rules — and it never widens a fragment into a set, because +//! the parts it would fill in are the operator's. What a topic *is* (rules, +//! migrations, APIs, submission format, pin policy) comes from its own RLM. //! //! Secrets are files the adaptor reads (`PROOF_SECRETS_DIR`); their bytes //! are redacted from every log tail and evidence document the agent sends @@ -42,7 +46,12 @@ pub mod fetch; pub mod runner; -pub mod staging; + +// The staging layer (secrets, pack, artefact inject, BYOK vault) moved to +// `proof-vm-staging` so this crate keeps room under the per-crate LOC cap +// while the RLM authoring path grows. Re-exported so every existing path +// keeps working. +pub use proof_vm_staging as staging; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -54,7 +63,7 @@ use proof_vm_proto::{ProtoError, API_VERSION}; use tokio::io::{AsyncRead, AsyncWrite}; use tokio::sync::Mutex; -pub use runner::{JobKind, RunnerReport}; +pub use runner::{AuthoredSet, JobKind, RunnerReport}; pub use staging::{StagedArtifact, StagedPack}; /// Agent name reported on `Ready`. @@ -308,10 +317,29 @@ impl GuestAgent { VmJob::ProposeRules { topic, current_version, + current, } => { let work = self.work_dir(JobKind::ProposeRules); - let result = runner::propose_rules(&self.cfg, &topic, current_version, &work).await; - seal_job(&work, result.map(VmJobOutput::Rules)) + let result = runner::propose_rules( + &self.cfg, + &topic, + current_version, + current.as_deref(), + &work, + ) + .await; + seal_job( + &work, + result.map(|set| match set { + runner::AuthoredSet::Complete(authored) => VmJobOutput::Authored(authored), + // A rules-only answer is a *fragment*: the host records + // the rules with honest provenance and refuses to open the + // topic, naming the parts the RLM did not author. It is + // never widened here into a whole set, because the parts + // that would fill it are the operator's. + runner::AuthoredSet::RulesOnly(rules) => VmJobOutput::Rules(rules), + }), + ) } VmJob::Archive { .. } => { // No work directory and no completion sync. The host diff --git a/crates/proof-vm-guest/src/runner.rs b/crates/proof-vm-guest/src/runner.rs index 9713453c8..da88aac76 100644 --- a/crates/proof-vm-guest/src/runner.rs +++ b/crates/proof-vm-guest/src/runner.rs @@ -37,7 +37,7 @@ use proof_experiment::{ExperimentBinding, RunPolicy}; use proof_results::load_evaluate; use proof_rlm::{ ArtifactFile, Checklist, CustomRunReport, CustomRunRequest, InspectOutcome, LogFile, RuleSet, - RunOutcome, RUN_REPORT_SCHEMA, + RunOutcome, TopicAuthoring, RUN_REPORT_SCHEMA, }; use proof_task::{ChecklistRule, TopicDocument}; use serde::Deserialize; @@ -100,6 +100,12 @@ pub mod env { pub const RULES_FILE: &str = "PROOF_RULES_FILE"; /// Signed topic JSON (`propose_rules`). pub const TOPIC_FILE: &str = "PROOF_TOPIC_FILE"; + /// Rule version the RLM is superseding (`propose_rules`; empty = none). + pub const CURRENT_RULES_VERSION: &str = "PROOF_CURRENT_RULES_VERSION"; + /// Where the set this RLM authored **last time** was written + /// (`propose_rules`; empty = no previous set). The adaptor reads it to + /// retain the parts it is not changing. + pub const CURRENT_AUTHORING_FILE: &str = "PROOF_CURRENT_AUTHORING_FILE"; /// Prefix of one variable per `constraints.params` entry. pub const PARAM_PREFIX: &str = "PROOF_PARAM_"; } @@ -113,6 +119,12 @@ pub const MAX_TAIL_BYTES: usize = 64 * 1024; pub const STREAM_TAIL_BYTES: usize = MAX_TAIL_BYTES / 2; /// Largest `report.json` / `checklist.json` / `rules.json` read back. pub const MAX_OUTPUT_DOC_BYTES: u64 = 8 * 1024 * 1024; + +/// The file the RLM writes its whole authored set to (`ProposeRules`). +pub const AUTHORING_FILE: &str = "authoring.json"; + +/// The file the guest writes the **previous** set to, for a re-authoring run. +pub const CURRENT_AUTHORING_FILE: &str = "current-authoring.json"; /// Deadline for jobs that carry none (`ProposeRules`). pub const DEFAULT_UNPAID_DEADLINE: Duration = Duration::from_mins(30); /// Host kills the process this long after its own deadline; the guest cuts @@ -698,6 +710,12 @@ pub(crate) fn persist_work(root: &Path) -> Result<(), String> { } fn read_output_doc Deserialize<'de>>(path: &Path, what: &str) -> Result { + let body = read_output_text(path, what)?; + serde_json::from_str(&body).map_err(|e| format!("{what} did not parse: {e}")) +} + +/// Read a bounded output document as text (the parse is the caller's). +fn read_output_text(path: &Path, what: &str) -> Result { let meta = std::fs::metadata(path) .map_err(|_| format!("adaptor wrote no {what} ({})", path.display()))?; if meta.len() > MAX_OUTPUT_DOC_BYTES { @@ -706,8 +724,7 @@ fn read_output_doc Deserialize<'de>>(path: &Path, what: &str) -> Res meta.len() )); } - let body = std::fs::read_to_string(path).map_err(|e| format!("read {what}: {e}"))?; - serde_json::from_str(&body).map_err(|e| format!("{what} did not parse: {e}")) + std::fs::read_to_string(path).map_err(|e| format!("read {what}: {e}")) } /// The job's work + output directories, owned by the user adaptors run as @@ -965,30 +982,93 @@ pub async fn inspect( }) } -/// Rule proposal: the adaptor's `propose_rules` when the topic selects a -/// runner that ships one, else the signed topic's own checklist. +/// Why a topic whose runner ships no `propose_rules` entrypoint fails closed. +/// +/// The guest never echoes the signed `checklist` back as a rule proposal: doing +/// so would let the control plane record the operator's own vector as +/// RLM-authored rules. +pub const NO_RLM_RULES: &str = "the topic's runner ships no propose_rules entrypoint, so the RLM authored no rules: the signed checklist is the operator's vector (source topic_document), never a substitute for RLM authorship"; + +/// Why a rules-only proposal cannot open a topic. +/// +/// A topic's behavior is authored by its RLM in full: the rule vector, the SQL +/// migrations it needs, the routes it exposes, its submission format, and the +/// pin policy it tightens. An adaptor that writes only `rules.json` has +/// authored one part of five, and the host refuses to fill the rest in — from +/// the operator's bundle or from anywhere else — because that is exactly the +/// operator-cloned document the authorship pin exists to refuse. +pub const RULES_ONLY_IS_NOT_AUTHORSHIP: &str = "the runner wrote rules.json but no authoring.json: a topic's behavior is authored by its own RLM (rules, migrations, apis, submission_format, pin_policy), and a rules-only proposal is not that. Ship an adaptor whose propose_rules writes authoring.json; nothing is installed from the operator's bundle in its place"; + +/// What the RLM authored, as the guest read it. +#[derive(Debug, Clone, PartialEq)] +pub enum AuthoredSet { + /// The full set, from `authoring.json`. + Complete(Box), + /// A rules-only proposal, from `rules.json` (an older adaptor). + RulesOnly(Vec), +} + +/// The RLM authors its whole set inside its VM: **only** the adaptor's +/// `propose_rules`. +/// +/// The topic's RLM writes every part of its own behavior there. This function +/// therefore has **no** fallback to the signed document's `checklist` or to +/// anything else the operator wrote: echoing the operator's vector back would +/// let the control plane record parts the RLM never wrote as +/// [`proof_rlm::RuleSource::Rlm`], which is the operator-cloned document +/// masquerading as RLM authorship. A topic whose runner ships no +/// `propose_rules` entrypoint is `Failed` (503, no row, nothing scored) — +/// never silently scored under the operator's own rules. +/// +/// The signed `checklist` remains the topic's **version 1** +/// ([`proof_rlm::RuleSet::from_topic`], source `topic_document`) and keeps its +/// honest provenance; only a run of this entrypoint advances the store to +/// `rlm`. +/// +/// Two output shapes are read, and the difference matters: +/// +/// - `authoring.json` — the whole set ([`AuthoredSet::Complete`]). This is +/// what a topic that must **open** needs. +/// - `rules.json` — a bare vector ([`AuthoredSet::RulesOnly`]), kept because +/// an adaptor baked before the set existed still writes it. The host +/// records the rules with honest `rlm` provenance and refuses to open the +/// topic, naming the missing parts +/// ([`RULES_ONLY_IS_NOT_AUTHORSHIP`]) — it does not widen a rules-only +/// answer into a whole set, because the parts it would fill in would be the +/// operator's. pub async fn propose_rules( cfg: &GuestConfig, topic: &TopicDocument, current_version: Option, + current: Option<&TopicAuthoring>, work: &Path, -) -> Result, String> { - let adaptor = match Adaptor::binding_of(&topic.constraints.params)? { - Some(binding) => Adaptor::installed(&cfg.runners_dir, binding).ok(), - None => None, - }; +) -> Result { + let binding = Adaptor::binding_of(&topic.constraints.params)?; + let adaptor = binding.and_then(|b| Adaptor::installed(&cfg.runners_dir, b).ok()); let Some(entry) = adaptor .as_ref() .and_then(|a| a.entrypoint(JobKind::ProposeRules).ok()) else { - tracing::info!(topic_id = %topic.id, "no propose_rules adaptor; proposing the signed checklist"); - return Ok(topic.checklist.clone()); + tracing::error!(topic_id = %topic.id, "runner ships no propose_rules entrypoint"); + return Err(NO_RLM_RULES.to_owned()); }; let adaptor = adaptor.ok_or_else(|| "adaptor vanished".to_owned())?; let params = param_env(&topic.constraints.params)?; let output = prepare(cfg, work)?; let topic_file = work.join("topic.json"); write_doc(&topic_file, topic)?; + // The set this RLM authored last time, when the host has one: the adaptor + // reads it to **retain** the parts it is not changing. Without it a + // re-authoring run is a rewrite from nothing — an adaptor cannot keep a + // migration it still needs, and the install would apply that lossy set. + let current_file = match current { + Some(set) => { + let path = work.join(CURRENT_AUTHORING_FILE); + write_doc(&path, set)?; + path.display().to_string() + } + None => String::new(), + }; let mut vars = vec![ (env::RUNNER_ID.to_owned(), adaptor.binding.runner.clone()), ( @@ -998,6 +1078,10 @@ pub async fn propose_rules( (env::TOPIC_ID.to_owned(), topic.id.clone()), (env::CUSTOM_ID.to_owned(), topic.metric.custom_id.clone()), (env::TOPIC_FILE.to_owned(), topic_file.display().to_string()), + // Where the previous set is, if there is one. Always set (empty when + // there is none) so an adaptor branches on one variable rather than on + // a variable's presence. + (env::CURRENT_AUTHORING_FILE.to_owned(), current_file), (env::OUTPUT_DIR.to_owned(), output.display().to_string()), (env::WORK_DIR.to_owned(), work.display().to_string()), ( @@ -1009,7 +1093,7 @@ pub async fn propose_rules( secret_names(&cfg.secrets_dir).join(","), ), ( - "PROOF_CURRENT_RULES_VERSION".to_owned(), + env::CURRENT_RULES_VERSION.to_owned(), current_version.map_or(String::new(), |v| v.to_string()), ), ]; @@ -1019,6 +1103,38 @@ pub async fn propose_rules( if exec.timed_out { return Err("propose_rules cut at its deadline".into()); } + // The whole set is what a topic's behavior is. Read it first: an adaptor + // that writes it is authoring, and one that writes only rules is + // proposing a fragment. + let authoring_path = output.join(AUTHORING_FILE); + if authoring_path.is_file() { + let body = read_output_text(&authoring_path, AUTHORING_FILE).map_err(|e| { + format!( + "{e} ({})", + describe(&exec, &secrets, DEFAULT_UNPAID_DEADLINE.as_secs()) + ) + })?; + let mut set = + proof_rlm::authoring_from_json(&body).map_err(|e| format!("{AUTHORING_FILE}: {e}"))?; + if set.topic_id.trim() != topic.id.trim() { + return Err(format!( + "{AUTHORING_FILE} is for topic {:?}, this VM is bound to {:?}", + set.topic_id, topic.id + )); + } + // The guest holds the set to the same checks the control plane runs, + // minus the pin (which it does not have): shape, the migration + // deny-list, and the policy against the document's own knobs. + set.validate(&topic.id) + .map_err(|e| format!("{AUTHORING_FILE}: {e}"))?; + set.pin_policy + .agrees_with_document(topic) + .map_err(|e| format!("{AUTHORING_FILE}: {e}"))?; + for rule in &mut set.rules { + rule.text = redact(&rule.text, &secrets); + } + return Ok(AuthoredSet::Complete(Box::new(set))); + } let mut rules: Vec = read_output_doc(&output.join("rules.json"), "rules.json") .map_err(|e| { format!( @@ -1030,7 +1146,7 @@ pub async fn propose_rules( r.text = redact(&r.text, &secrets); } validate_rules(&rules).map_err(|e| format!("rules.json {}: {}", e.field, e.why))?; - Ok(rules) + Ok(AuthoredSet::RulesOnly(rules)) } #[cfg(test)] diff --git a/crates/proof-vm-proto/src/lib.rs b/crates/proof-vm-proto/src/lib.rs index ca73c17d9..1d4584dca 100644 --- a/crates/proof-vm-proto/src/lib.rs +++ b/crates/proof-vm-proto/src/lib.rs @@ -94,6 +94,18 @@ pub struct AgentHealth { /// Most experiment VMs this host runs at once (0 = experiments disabled). #[serde(default)] pub max_experiment_vms: usize, + /// Host RAM the boots are admitted against (`MemTotal`), MiB. `0` when + /// the agent was built without a memory budget. + #[serde(default)] + pub total_mib: u64, + /// Headroom kept out of `total_mib` for the OS, the agent, and per-VM + /// process overhead, MiB. + #[serde(default)] + pub reserve_mib: u64, + /// Memory the live VMs hold, MiB — every VM counts, the topic's RLM VM + /// included, because it is resident for the topic's whole life. + #[serde(default)] + pub used_mib: u64, } /// `POST /v1/vms` body. @@ -291,7 +303,12 @@ impl EvidenceBinding { match output { VmJobOutput::Baseline(report) => Some(Self::of_report(report)), VmJobOutput::Evaluated(run) => Some(Self::of_report(&run.report)), - VmJobOutput::Rules(_) | VmJobOutput::Inspected(_) | VmJobOutput::Archived => None, + // An authored set and an inspection run no miner code: no + // submission, no artefact, nothing to bind. + VmJobOutput::Authored(_) + | VmJobOutput::Rules(_) + | VmJobOutput::Inspected(_) + | VmJobOutput::Archived => None, } } @@ -645,13 +662,22 @@ mod tests { vms: 1, experiment_vms: 1, max_experiment_vms: 2, + total_mib: 16_384, + reserve_mib: 0, + used_mib: 8_192, }; let legacy: AgentHealth = serde_json::from_str( r#"{"api_version":1,"ready":true,"reason":"","hypervisor":"firecracker","vms":0}"#, ) .expect("older agents omit the experiment counters"); assert_eq!((legacy.experiment_vms, legacy.max_experiment_vms), (0, 0)); + assert_eq!( + (legacy.total_mib, legacy.reserve_mib, legacy.used_mib), + (0, 0, 0), + "an older agent omits the memory budget too, and reports 0 rather than a guess" + ); assert_eq!(health.experiment_vms, 1); + assert_eq!(health.total_mib, 16_384); } #[test] diff --git a/crates/proof-vm-staging/Cargo.toml b/crates/proof-vm-staging/Cargo.toml new file mode 100644 index 000000000..ff011254b --- /dev/null +++ b/crates/proof-vm-staging/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "proof-vm-staging" +description = "What the Proof topic-VM host stages into a guest: owner key material (tmpfs), the pinned experiment pack (writable disk), a miner artefact inject (proof-artefact://), and the miner BYOK vault — by name-checked files only, every one verified against a digest before it is used. It moved out of the guest crate so the guest keeps room under the repository's per-crate LOC cap while the RLM authoring path grows." +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +hex = "0.4" +proof-canon = { path = "../proof-canon" } +proof-experiment = { path = "../proof-experiment" } +proof-vm-proto = { path = "../proof-vm-proto" } +serde_json = "1" +sha2 = "0.10" +tar = "0.4" + +[dev-dependencies] + +[lints] +workspace = true diff --git a/crates/proof-vm-guest/src/staging.rs b/crates/proof-vm-staging/src/lib.rs similarity index 97% rename from crates/proof-vm-guest/src/staging.rs rename to crates/proof-vm-staging/src/lib.rs index 155bec723..54626833c 100644 --- a/crates/proof-vm-guest/src/staging.rs +++ b/crates/proof-vm-staging/src/lib.rs @@ -2,6 +2,19 @@ //! experiment pack (writable disk), and a miner artefact inject //! (`proof-artefact://`, held for the next job), all by name-checked files //! only. +//! +//! This module moved out of `proof-vm-guest` (where it was `staging`) so that +//! crate keeps room under the repository's per-crate LOC cap while the RLM +//! authoring path grows. The code is unchanged; the lint posture is the one +//! the guest crate set for it, because every refusal here is a `String` that +//! the caller turns into a job failure rather than a typed error. + +#![forbid(unsafe_code)] +#![allow( + clippy::missing_errors_doc, + clippy::module_name_repetitions, + clippy::must_use_candidate +)] use std::path::{Path, PathBuf}; diff --git a/deploy/guest/runners/README.md b/deploy/guest/runners/README.md index 77e86d1fc..34c632292 100644 --- a/deploy/guest/runners/README.md +++ b/deploy/guest/runners/README.md @@ -28,7 +28,7 @@ operator's view of it. |------|-----|--------------------------------------| | `run` (required) | `Baseline`, `Evaluate` | `report.json` — `{"primary_value": , "claim_holds": bool, "flops_used": , "evidence": {...}}`. **Evaluate** also writes the topic-defined complete results JSON (default `results.json`; pin `results_path` / `results_contract` in `constraints.params`). Missing or non-conforming on evaluate is fail-closed (no Done) | | `inspect` | `Inspect` (anti-cheat rules, **before any paid inference**) | `checklist.json` — `[{"id": "", "pass": bool, "evidence": "..."}]`; a rule left out is recorded **red** | -| `propose_rules` (optional) | `ProposeRules` | `rules.json` — `[{"id": "", "text": "..."}]`; without this entrypoint the agent proposes the signed topic's own `checklist` | +| `propose_rules` | `ProposeRules` (RLM authorship) | `rules.json` — `[{"id": "", "text": "..."}]`. **A runner whose topic must open needs this entrypoint**: without it the guest refuses the job (`Failed` → 503, no row, nothing scored), because there is **no** fallback that echoes the signed `checklist` back. Echoing it would let the control plane record the operator's own vector as `source = rlm`, which is an operator-cloned document masquerading as RLM authorship. The signed `checklist` stays the topic's version 1 with honest `topic_document` provenance, and only a run of this entrypoint advances the store to `rlm` — which the publish gate requires before a topic may be `open`. | A non-zero exit with no document, a missing document, a non-finite `primary_value`, a missing or non-conforming Evaluate `results.json`, or a @@ -81,7 +81,7 @@ closed on an unmeasured one). |-------|-------|------------------------| | `tasks` | `[A-Za-z0-9][A-Za-z0-9_.-]{0,63}` names, comma / space separated | The exact items to score, in order. An item the pack does not hold **fails closed** (never a smaller set). One name = the single-task smoke | | `task_exclude` | same list shape | Items never scored | -| `n_tasks` | positive integer | Keep only the first N selected items (`1` = smoke) | +| `n_tasks` | positive integer | Keep only the first N selected items (`1` = smoke). `task_count` is a legacy alias, used only when `n_tasks` is absent; it is a **count, never a selector** and cannot stand in for a slice | | `max_task_duration_s` | positive integer | Drop items whose **known** duration is at or over this; absent = no gate | | `exclude_unknown_duration` | `"true"` / `"false"` | Under a gate, drop items with no duration metadata | | `exec_timeout_s` | positive integer | Default wall clock for one command the miner's harness runs without its own timeout (the reference adaptor exports it as `PROOF_EXEC_TIMEOUT_S`) | diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md b/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md index 9a64714f5..71196d609 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md @@ -126,12 +126,12 @@ Selection, in order, then filters: | Step | Source | Behaviour | |------|--------|-----------| | 1 | `params.tasks` (`PROOF_PARAM_TASKS`) | Exact ordered names (comma / space separated). A name the pack does not hold **fails closed** — a topic that names a task is never scored on a smaller set. **One name is the single-task smoke.** | -| 2 | `constraints.task_slice` (`PROOF_TASK_SLICE`) | Resolved **through the pack**: `slices/