From e1d7062e7ebd70dd5581ea6adf0da725f30239a5 Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Thu, 8 Oct 2026 02:13:27 +0000 Subject: [PATCH 1/9] test(api): pin append validation against deleted and live identities (#1902) Records the outcomes of the existing-identity check on the membership-index path before it is removed, so the removal can show which outcomes it kept and which it changed. Co-Authored-By: Claude Sonnet 5.5 --- .../src/bulk_construction/publication.rs | 3 + .../publication/deleted_identity_tests.rs | 145 ++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 crates/graphforge-api/src/bulk_construction/publication/deleted_identity_tests.rs diff --git a/crates/graphforge-api/src/bulk_construction/publication.rs b/crates/graphforge-api/src/bulk_construction/publication.rs index b503fe3ec..2bdce4c74 100644 --- a/crates/graphforge-api/src/bulk_construction/publication.rs +++ b/crates/graphforge-api/src/bulk_construction/publication.rs @@ -547,3 +547,6 @@ impl GraphForge { #[cfg(test)] mod tests; + +#[cfg(test)] +mod deleted_identity_tests; diff --git a/crates/graphforge-api/src/bulk_construction/publication/deleted_identity_tests.rs b/crates/graphforge-api/src/bulk_construction/publication/deleted_identity_tests.rs new file mode 100644 index 000000000..e6107be89 --- /dev/null +++ b/crates/graphforge-api/src/bulk_construction/publication/deleted_identity_tests.rs @@ -0,0 +1,145 @@ +//! Append validation against deleted and live identities (#1902). +//! +//! These pin what a bulk append may do with a UUID the graph once held, so the +//! source of the existing-identity check can change without changing answers. + +use super::super::tests::edge_batch; +use super::super::tests::node_batch; +use super::super::tests::operation; +use super::super::tests::uuid; +use super::super::*; +use graphforge_ir::IrLiteral; +use std::collections::HashMap; + +fn param(id: Uuid) -> HashMap { + HashMap::from([("id".to_owned(), IrLiteral::Uuid(*id.as_bytes()))]) +} + +fn delete_node(graph: &GraphForge, id: Uuid) { + graph + .execute_with_params("MATCH (n) WHERE n.node_uuid = $id DELETE n", ¶m(id)) + .unwrap(); +} + +fn delete_edge(graph: &GraphForge, source: Uuid, target: Uuid) { + graph + .execute_with_params( + "MATCH (a)-[r]->(b) WHERE a.node_uuid = $a AND b.node_uuid = $b DELETE r", + &HashMap::from([ + ("a".to_owned(), IrLiteral::Uuid(*source.as_bytes())), + ("b".to_owned(), IrLiteral::Uuid(*target.as_bytes())), + ]), + ) + .unwrap(); +} + +fn project() -> (tempfile::TempDir, GraphForge) { + let directory = tempfile::tempdir().unwrap(); + let graph = GraphForge::new(Some(directory.path().to_str().unwrap())).unwrap(); + (directory, graph) +} + +fn outcome(result: Result) -> String { + match result { + Ok(_) => "accepted".to_owned(), + Err(error) => format!("{:?}: {error}", error.reason), + } +} + +#[test] +fn probe_outcomes_for_deleted_and_live_identities() { + let (_directory, graph) = project(); + let (a, b, c) = (uuid(1_001), uuid(1_002), uuid(1_003)); + let (e1, e2) = (uuid(2_001), uuid(2_002)); + graph + .publish_bulk_nodes( + operation(10), + &[node_batch(&[a, b, c], &["P", "P", "P"], &[None, None, None])], + ) + .unwrap(); + graph + .publish_bulk_edges( + operation(11), + &[edge_batch(&[e1, e2], &["R", "R"], &[a, b], &[b, c])], + ) + .unwrap(); + delete_edge(&graph, a, b); + delete_node(&graph, c); + let empty = graph.validate_bulk_nodes(operation(14), &[]).unwrap(); + let rows = [ + ( + "deleted node re-add", + outcome(graph.validate_bulk_nodes(operation(12), &[node_batch(&[c], &["P"], &[None])])), + ), + ( + "live node dup", + outcome(graph.validate_bulk_nodes(operation(13), &[node_batch(&[a], &["P"], &[None])])), + ), + ( + "deleted edge re-add", + outcome(graph.validate_bulk_edges( + operation(15), + &[edge_batch(&[e1], &["R"], &[a], &[b])], + &empty, + )), + ), + ( + "live edge dup", + outcome(graph.validate_bulk_edges( + operation(16), + &[edge_batch(&[e2], &["R"], &[a], &[b])], + &empty, + )), + ), + ( + "edge uuid == live node", + outcome(graph.validate_bulk_edges( + operation(17), + &[edge_batch(&[a], &["R"], &[a], &[b])], + &empty, + )), + ), + ( + "edge uuid == deleted node", + outcome(graph.validate_bulk_edges( + operation(18), + &[edge_batch(&[c], &["R"], &[a], &[b])], + &empty, + )), + ), + ( + "node uuid == live edge", + outcome(graph.validate_bulk_nodes(operation(19), &[node_batch(&[e2], &["P"], &[None])])), + ), + ( + "node uuid == deleted edge", + outcome(graph.validate_bulk_nodes(operation(20), &[node_batch(&[e1], &["P"], &[None])])), + ), + ( + "edge to deleted node", + outcome(graph.validate_bulk_edges( + operation(21), + &[edge_batch(&[uuid(2_003)], &["R"], &[a], &[c])], + &empty, + )), + ), + ( + "edge from never-seen node", + outcome(graph.validate_bulk_edges( + operation(24), + &[edge_batch(&[uuid(2_004)], &["R"], &[uuid(9_999)], &[a])], + &empty, + )), + ), + ]; + for (name, result) in &rows { + eprintln!("PIN {name}: {result}"); + } + let publish = graph.publish_bulk_nodes(operation(22), &[node_batch(&[c], &["P"], &[None])]); + eprintln!("PIN publish deleted node re-add: {:?}", publish.map(|_| ()).map_err(|e| e.to_string())); + let publish = graph.publish_bulk_edges( + operation(23), + &[edge_batch(&[e1], &["R"], &[a], &[b])], + ); + eprintln!("PIN publish deleted edge re-add: {:?}", publish.map(|_| ()).map_err(|e| e.to_string())); +} From 6df5dde4ea99cab4edbf4830341814fe780fc792 Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Thu, 8 Oct 2026 02:15:32 +0000 Subject: [PATCH 2/9] test(api): pin deleted-identity validation and commit refusals on the current path (#1902) Co-Authored-By: Claude Sonnet 5.5 --- .../publication/deleted_identity_tests.rs | 88 ++++++++++++++++--- 1 file changed, 74 insertions(+), 14 deletions(-) diff --git a/crates/graphforge-api/src/bulk_construction/publication/deleted_identity_tests.rs b/crates/graphforge-api/src/bulk_construction/publication/deleted_identity_tests.rs index e6107be89..852906bc6 100644 --- a/crates/graphforge-api/src/bulk_construction/publication/deleted_identity_tests.rs +++ b/crates/graphforge-api/src/bulk_construction/publication/deleted_identity_tests.rs @@ -17,7 +17,7 @@ fn param(id: Uuid) -> HashMap { fn delete_node(graph: &GraphForge, id: Uuid) { graph - .execute_with_params("MATCH (n) WHERE n.node_uuid = $id DELETE n", ¶m(id)) + .execute_with_params("MATCH (n) WHERE n.node_uuid = $id DETACH DELETE n", ¶m(id)) .unwrap(); } @@ -50,7 +50,7 @@ fn outcome(result: Result) -> String { fn probe_outcomes_for_deleted_and_live_identities() { let (_directory, graph) = project(); let (a, b, c) = (uuid(1_001), uuid(1_002), uuid(1_003)); - let (e1, e2) = (uuid(2_001), uuid(2_002)); + let (e1, e2, e3) = (uuid(2_001), uuid(2_002), uuid(2_005)); graph .publish_bulk_nodes( operation(10), @@ -60,7 +60,7 @@ fn probe_outcomes_for_deleted_and_live_identities() { graph .publish_bulk_edges( operation(11), - &[edge_batch(&[e1, e2], &["R", "R"], &[a, b], &[b, c])], + &[edge_batch(&[e1, e2, e3], &["R", "R", "R"], &[a, b, b], &[b, c, a])], ) .unwrap(); delete_edge(&graph, a, b); @@ -87,7 +87,7 @@ fn probe_outcomes_for_deleted_and_live_identities() { "live edge dup", outcome(graph.validate_bulk_edges( operation(16), - &[edge_batch(&[e2], &["R"], &[a], &[b])], + &[edge_batch(&[e3], &["R"], &[a], &[b])], &empty, )), ), @@ -109,12 +109,20 @@ fn probe_outcomes_for_deleted_and_live_identities() { ), ( "node uuid == live edge", - outcome(graph.validate_bulk_nodes(operation(19), &[node_batch(&[e2], &["P"], &[None])])), + outcome(graph.validate_bulk_nodes(operation(19), &[node_batch(&[e3], &["P"], &[None])])), ), ( "node uuid == deleted edge", outcome(graph.validate_bulk_nodes(operation(20), &[node_batch(&[e1], &["P"], &[None])])), ), + ( + "edge re-add cascade-deleted by DETACH", + outcome(graph.validate_bulk_edges( + operation(25), + &[edge_batch(&[e2], &["R"], &[a], &[b])], + &empty, + )), + ), ( "edge to deleted node", outcome(graph.validate_bulk_edges( @@ -132,14 +140,66 @@ fn probe_outcomes_for_deleted_and_live_identities() { )), ), ]; - for (name, result) in &rows { - eprintln!("PIN {name}: {result}"); + let conflict = |kind: &str, field: &str| { + format!( + "IdentityConflict: GF_BULK_VALIDATION(identity_conflict): bulk {kind} row 0 field \"{field}\": duplicate or existing UUID" + ) + }; + let missing = |field: &str| { + format!( + "MissingEndpoint: GF_BULK_VALIDATION(missing_endpoint): bulk edge row 0 field \"{field}\": endpoint does not exist" + ) + }; + let expected = [ + ("deleted node re-add", "accepted".to_owned()), + ("live node dup", conflict("node", "node_uuid")), + ("deleted edge re-add", "accepted".to_owned()), + ("live edge dup", conflict("edge", "edge_uuid")), + ("edge uuid == live node", conflict("edge", "edge_uuid")), + ("edge uuid == deleted node", "accepted".to_owned()), + ("node uuid == live edge", conflict("node", "node_uuid")), + ("node uuid == deleted edge", "accepted".to_owned()), + ("edge to deleted node", missing("target_uuid")), + ("edge from never-seen node", missing("source_uuid")), + ]; + let observed = rows + .iter() + .filter(|(name, _)| *name != "edge re-add cascade-deleted by DETACH") + .map(|(name, result)| (*name, result.clone())) + .collect::>(); + assert_eq!(observed.len(), expected.len()); + for ((name, got), (expected_name, want)) in observed.iter().zip(expected.iter()) { + assert_eq!(name, expected_name); + assert_eq!(got, want, "{name}"); } - let publish = graph.publish_bulk_nodes(operation(22), &[node_batch(&[c], &["P"], &[None])]); - eprintln!("PIN publish deleted node re-add: {:?}", publish.map(|_| ()).map_err(|e| e.to_string())); - let publish = graph.publish_bulk_edges( - operation(23), - &[edge_batch(&[e1], &["R"], &[a], &[b])], - ); - eprintln!("PIN publish deleted edge re-add: {:?}", publish.map(|_| ()).map_err(|e| e.to_string())); +} + +/// Validation looks at live state only, but the commit refuses to reuse the +/// identity of a deleted entity: the UUID is spent for good. +#[test] +fn deleted_identities_are_not_reused_at_commit() { + let (_directory, graph) = project(); + let (a, b, c) = (uuid(1_001), uuid(1_002), uuid(1_003)); + let (e1, e2) = (uuid(2_001), uuid(2_002)); + graph + .publish_bulk_nodes( + operation(10), + &[node_batch(&[a, b, c], &["P", "P", "P"], &[None, None, None])], + ) + .unwrap(); + graph + .publish_bulk_edges( + operation(11), + &[edge_batch(&[e1, e2], &["R", "R"], &[a, b], &[b, c])], + ) + .unwrap(); + delete_edge(&graph, a, b); + delete_node(&graph, c); + let node = graph.publish_bulk_nodes(operation(22), &[node_batch(&[c], &["P"], &[None])]); + assert!(node.is_err(), "deleted node UUID must stay spent"); + let edge = graph.publish_bulk_edges(operation(23), &[edge_batch(&[e1], &["R"], &[a], &[b])]); + assert!(edge.is_err(), "deleted edge UUID must stay spent"); + let cascaded = + graph.publish_bulk_edges(operation(24), &[edge_batch(&[e2], &["R"], &[a], &[b])]); + assert!(cascaded.is_err(), "cascade-deleted edge UUID must stay spent"); } From 3a143877e7040f2895bf9e9ffcf47b8cda56db2a Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:19:27 +0000 Subject: [PATCH 3/9] perf(storage): stop producing and reading the UUID membership index (#1902) The index (`topology/uuid-membership/manifest.json`, `identities-v5-*.uuidx`, `node-surrogates-v5-*.uuidx`, `topology-receipt.json`) duplicated the UUID-ordered `node_uuid`/`edge_uuid` columns of the published Parquet at 17-25 B per edge and was rewritten with every generation. Under the pre-v1 policy the format changes in place. - Neither the bulk nor the staged encoder writes it. The bulk builder no longer spills the sorted edge UUIDs to scratch for it, and the staged encoder keeps no retained parent index. Only the ordinal node-identity facet is encoded. - `TopologyIdentityProbe` answers every identity question from the published Parquet: a sorted, deduplicated candidate set prunes row groups by `min/max`, then pages by the column index, and decodes only the selected pages of `node_uuid` (and `node_id`) or `edge_uuid`. `UuidProbeMetrics` reports `pages_considered` and `pages_read`. Footers are cached by file identity, and a hit is re-pointed at the path asked for, because hydration hard-links one object under many workspace paths. - Append validation and the writer commit refuse a UUID that is a live node, a live edge or a deleted entity (one namespace), and a missing endpoint. - Deleted UUIDs are never reusable. `topology/deleted_identities.parquet`, a sorted unique column carried forward by each generation that deletes, replaces the index's tombstones; a graph that never deletes never has one. - Readers moved off the index: writer endpoint lookup and commit, the topology rewrite participant, label lookup by UUID, graph publication, branch import, composite publish, bulk validation, resumable construction, search node identity, and the ordinal discovery freshness check. Projects that still carry the files open; the files are ignored. - The G500 certification evidence drops the index's fsync and write counters, and its storage category policy now treats the identity category as node-bearing. Co-Authored-By: Claude Sonnet 5.5 --- .../src/branches/import_graph.rs | 6 - .../graphforge-api/src/bulk_construction.rs | 153 +- .../src/bulk_construction/normalization.rs | 17 +- .../src/bulk_construction/publication.rs | 36 +- .../publication/deleted_identity_tests.rs | 68 +- .../graphforge-api/src/composite_publish.rs | 26 +- crates/graphforge-api/src/construction.rs | 4 +- .../graphforge-api/src/embedding_refresh.rs | 2 +- .../graphforge-api/src/graph_publication.rs | 14 - crates/graphforge-api/src/lib.rs | 6 +- crates/graphforge-api/src/maintenance.rs | 8 +- .../src/mutation_transaction.rs | 2 +- .../src/ontology_composition_lifecycle.rs | 2 +- .../src/resumable_construction.rs | 10 +- .../src/resumable_construction/codec_tests.rs | 4 +- .../src/workspace_hydration/tests.rs | 10 +- .../graphforge-api/src/workspace_ontology.rs | 2 +- .../tests/bounded_query_shapes.rs | 16 +- .../tests/file_backed_graph_generation.rs | 6 +- .../tests/identity_first_touch.rs | 181 +- .../tests/lifecycle_io_attribution.rs | 6 +- .../tests/permanent_storage_budgets.rs | 111 +- .../graphforge-api/tests/scale_g500_ladder.rs | 63 +- crates/graphforge-search/src/node_identity.rs | 36 +- crates/graphforge-search/src/source.rs | 2 +- .../graphforge-search/src/vector_lifecycle.rs | 2 +- .../src/adjacency/classification_tests.rs | 2 +- crates/graphforge-storage/src/catalog.rs | 18 +- .../src/construction_bulk_tests.rs | 44 +- .../src/construction_chunk_spool_tests.rs | 6 +- .../src/construction_directory.rs | 11 - .../graphforge-storage/src/durable_rewrite.rs | 36 + .../src/graph_construction.rs | 36 +- .../encoding_publication.rs | 17 +- .../encoding_publication/tests.rs | 193 +- .../src/graph_construction/io_evidence.rs | 39 +- .../src/graph_construction/shape.rs | 20 +- .../src/graph_construction/shape/tests.rs | 1 - .../src/graph_construction/supersession.rs | 115 +- .../src/graph_construction/tests.rs | 22 +- .../src/graph_construction_encoding.rs | 140 +- .../src/graph_construction_encoding/bulk.rs | 105 +- .../bulk/identities.rs | 144 -- .../graph_construction_encoding/bulk/plan.rs | 2 +- .../bulk/scratch_edges.rs | 9 - .../graph_construction_encoding/inventory.rs | 57 +- .../src/graph_construction_encoding/tests.rs | 15 +- .../src/label_membership_counts.rs | 7 +- crates/graphforge-storage/src/lib.rs | 25 +- crates/graphforge-storage/src/mutator.rs | 125 +- .../src/ordinal_identity_v4.rs | 10 +- .../src/ordinal_identity_v4/tests.rs | 29 +- .../src/project_generation.rs | 23 +- .../src/runtime_entity_labels.rs | 27 +- .../src/topology_identity.rs | 944 +++++++++ .../src/topology_identity/tests.rs | 284 +++ .../graphforge-storage/src/uuid_membership.rs | 1095 +---------- .../src/uuid_membership/construction.rs | 1741 ++--------------- .../src/uuid_membership/construction/tests.rs | 307 --- .../src/uuid_membership/identity_codec.rs | 154 -- .../src/uuid_membership/maintenance.rs | 179 +- .../src/uuid_membership/maintenance/tests.rs | 132 +- .../src/uuid_membership/ordinal_artifacts.rs | 9 +- .../ordinal_artifacts/tests.rs | 26 +- .../src/uuid_membership/probing.rs | 1306 ------------- .../src/uuid_membership/probing/tests.rs | 525 ----- .../src/uuid_membership/rebuild.rs | 1065 ++-------- .../src/uuid_membership/rebuild/tests.rs | 155 +- .../src/uuid_membership/tests.rs | 388 +--- .../src/uuid_membership/topology_delta.rs | 1521 ++------------ .../uuid_membership/topology_delta/tests.rs | 193 -- crates/graphforge-storage/src/writer.rs | 137 +- crates/graphforge-storage/src/writer/tests.rs | 267 +-- .../ci/schemas/g500-certification.schema.json | 8 - .../ci/test-validate-g500-certification.py | 2 - 75 files changed, 2636 insertions(+), 9873 deletions(-) delete mode 100644 crates/graphforge-storage/src/graph_construction_encoding/bulk/identities.rs create mode 100644 crates/graphforge-storage/src/topology_identity.rs create mode 100644 crates/graphforge-storage/src/topology_identity/tests.rs delete mode 100644 crates/graphforge-storage/src/uuid_membership/construction/tests.rs delete mode 100644 crates/graphforge-storage/src/uuid_membership/identity_codec.rs delete mode 100644 crates/graphforge-storage/src/uuid_membership/probing.rs delete mode 100644 crates/graphforge-storage/src/uuid_membership/probing/tests.rs diff --git a/crates/graphforge-api/src/branches/import_graph.rs b/crates/graphforge-api/src/branches/import_graph.rs index dd4795bd5..2e8149012 100644 --- a/crates/graphforge-api/src/branches/import_graph.rs +++ b/crates/graphforge-api/src/branches/import_graph.rs @@ -55,12 +55,6 @@ fn incorporate_with_policy( return Err(conflict()); } } - if !graphforge_storage::uuid_membership_index_present(&destination.dir()) { - graphforge_storage::rebuild_uuid_membership_indexes( - &destination.dir(), - graphforge_storage::UuidIndexBuildLimits::default(), - )?; - } let mut labels = BTreeMap::>::new(); for (kind, query) in [ ( diff --git a/crates/graphforge-api/src/bulk_construction.rs b/crates/graphforge-api/src/bulk_construction.rs index 535845e6a..d3c7b9538 100644 --- a/crates/graphforge-api/src/bulk_construction.rs +++ b/crates/graphforge-api/src/bulk_construction.rs @@ -282,64 +282,42 @@ impl ValidatedBulkEdges { } } -fn open_membership_index( +/// The identity probe for the committed topology generation, cached on the +/// facade until the generation moves. +fn open_identity_probe( graph: &GraphForge, input_kind: BulkInputKind, ) -> Result< - std::sync::MutexGuard<'_, Option>, + std::sync::MutexGuard<'_, Option>, BulkValidationError, > { - let current_generation = - graphforge_storage::read_topology_generation(&graph.dir()).map_err(|error| { - contract_error( - input_kind, - BulkValidationReason::ProjectState, - &error.to_string(), - ) - })?; - let mut cached = graph.uuid_membership_index.lock().map_err(|error| { + let project_state = |error: &dyn std::fmt::Display| { contract_error( input_kind, BulkValidationReason::ProjectState, &error.to_string(), ) - })?; + }; + let current_generation = graphforge_storage::read_topology_generation(&graph.dir()) + .map_err(|error| project_state(&error))?; + let mut cached = graph + .identity_probe + .lock() + .map_err(|error| project_state(&error))?; if cached .as_ref() - .is_some_and(|index| index.topology_generation() != current_generation) + .is_some_and(|probe| probe.topology_generation() != current_generation) { *cached = None; } - if !graphforge_storage::uuid_membership_index_present(&graph.dir()) { - let has_nodes = - graphforge_storage::node_topology_present(&graph.dir()).map_err(|error| { - contract_error( - input_kind, - BulkValidationReason::ProjectState, - &error.to_string(), - ) - })?; - let has_edges = std::fs::read_dir(graph.dir().join("topology/edges")) - .ok() - .is_some_and(|mut entries| entries.any(|entry| entry.is_ok())); - if has_nodes || has_edges { - return Err(contract_error( - input_kind, - BulkValidationReason::ProjectState, - "UUID membership index is missing; run the bounded storage rebuild before ingest", - )); - } - return Ok(cached); - } if cached.is_none() { + let dir = graph.dir(); + let files = dir + .topology_files() + .map_err(|error| project_state(&error))?; *cached = Some( - graphforge_storage::UuidMembershipIndex::open(&graph.dir()).map_err(|error| { - contract_error( - input_kind, - BulkValidationReason::ProjectState, - &error.to_string(), - ) - })?, + graphforge_storage::TopologyIdentityProbe::open(&dir, &files, current_generation) + .map_err(|error| project_state(&error))?, ); } Ok(cached) @@ -350,55 +328,65 @@ fn existing_edge_context( endpoint_candidates: &[Uuid], edge_candidates: Option<&[Uuid]>, ) -> Result<(HashSet, HashSet), BulkValidationError> { - let mut index = open_membership_index(graph, BulkInputKind::Edge)?; - let known_nodes = indexed_existing( - index.as_mut(), - endpoint_candidates, - graphforge_storage::UuidIndexKind::Node, - BulkInputKind::Edge, - )?; + let mut probe = open_identity_probe(graph, BulkInputKind::Edge)?; + let known_nodes = live_nodes(probe.as_mut(), endpoint_candidates, BulkInputKind::Edge)?; let Some(edge_candidates) = edge_candidates else { return Ok((known_nodes, HashSet::new())); }; - let mut existing = indexed_existing( - index.as_mut(), - edge_candidates, - graphforge_storage::UuidIndexKind::Edge, - BulkInputKind::Edge, - )?; - existing.extend(indexed_existing( - index.as_mut(), - edge_candidates, - graphforge_storage::UuidIndexKind::Node, - BulkInputKind::Edge, - )?); + let existing = taken_identities(probe.as_mut(), edge_candidates, BulkInputKind::Edge)?; Ok((known_nodes, existing)) } -/// Probe `candidates` (sorted, deduplicated) and return the subset the index -/// already holds. Membership only: callers never iterate the result. -fn indexed_existing( - index: Option<&mut graphforge_storage::UuidMembershipIndex>, +/// Probe `candidates` (sorted, deduplicated) and return the live nodes among +/// them. Membership only: callers never iterate the result. +fn live_nodes( + probe: Option<&mut graphforge_storage::TopologyIdentityProbe>, candidates: &[Uuid], - index_kind: graphforge_storage::UuidIndexKind, input_kind: BulkInputKind, ) -> Result, BulkValidationError> { - let Some(index) = index else { + let Some(probe) = probe else { return Ok(HashSet::new()); }; - let (found, _) = index.probe(index_kind, candidates).map_err(|error| { + let (found, _) = probe + .probe(graphforge_storage::UuidIndexKind::Node, candidates) + .map_err(|error| { + contract_error( + input_kind, + BulkValidationReason::ProjectState, + &error.to_string(), + ) + })?; + Ok(selected(candidates, &found)) +} + +/// The subset of `candidates` that is already spent: a live node, a live edge +/// or a deleted entity. Node and edge UUIDs share one namespace and a deleted +/// UUID is never reused, exactly as the commit enforces. +fn taken_identities( + probe: Option<&mut graphforge_storage::TopologyIdentityProbe>, + candidates: &[Uuid], + input_kind: BulkInputKind, +) -> Result, BulkValidationError> { + let Some(probe) = probe else { + return Ok(HashSet::new()); + }; + let (found, _) = probe.taken(candidates).map_err(|error| { contract_error( input_kind, BulkValidationReason::ProjectState, &error.to_string(), ) })?; - Ok(candidates + Ok(selected(candidates, &found)) +} + +fn selected(candidates: &[Uuid], found: &[bool]) -> HashSet { + candidates .iter() .copied() .zip(found) .filter_map(|(uuid, present)| present.then_some(uuid)) - .collect()) + .collect() } /// Non-null UUIDs of `field`, sorted and deduplicated: the same set, in the @@ -444,27 +432,21 @@ fn candidate_endpoint_uuids(batches: &[RecordBatch]) -> Result, BulkVa #[cfg(test)] fn indexed_uuid_count(graph: &GraphForge, kind: graphforge_storage::UuidIndexKind) -> u64 { - graphforge_storage::UuidMembershipIndex::open(&graph.dir()) - .expect("published graph has an authenticated UUID membership index") - .count(kind) + let dir = graph.dir(); + let files = dir.topology_files().expect("published topology files"); + graphforge_storage::TopologyIdentityProbe::open( + &dir, + &files, + graphforge_storage::read_topology_generation(&dir).expect("topology generation"), + ) + .expect("published graph has readable topology") + .count(kind) } pub(crate) fn register_existing_endpoints( writer: &mut graphforge_storage::GraphWriter, - dir: &std::path::Path, endpoints: &BTreeSet, ) -> Result<(), super::GfError> { - if !graphforge_storage::uuid_membership_index_present(dir) { - graphforge_storage::rebuild_uuid_membership_indexes( - dir, - graphforge_storage::UuidIndexBuildLimits::default(), - )?; - } - if !graphforge_storage::uuid_membership_index_is_fresh(dir)? { - return Err(super::GfError::Storage( - "bulk endpoint UUID index is stale".into(), - )); - } let requested = endpoints.iter().copied().collect::>(); writer .register_existing_endpoints(&requested) @@ -580,8 +562,7 @@ mod tests { .unwrap(); let missing = uuid(70_001); let failure = - register_existing_endpoints(&mut writer, &graph.dir(), &BTreeSet::from([missing])) - .unwrap_err(); + register_existing_endpoints(&mut writer, &BTreeSet::from([missing])).unwrap_err(); assert_eq!( failure.to_string(), "validation error: bulk edge endpoint disappeared before publication" diff --git a/crates/graphforge-api/src/bulk_construction/normalization.rs b/crates/graphforge-api/src/bulk_construction/normalization.rs index b6624409d..00c8a1f32 100644 --- a/crates/graphforge-api/src/bulk_construction/normalization.rs +++ b/crates/graphforge-api/src/bulk_construction/normalization.rs @@ -9,7 +9,7 @@ use super::{ RecordBatch, Schema, SchemaRef, Sha256, StringArray, StructArray, Time64NanosecondArray, TimestampMicrosecondArray, UInt8Array, UInt16Array, UInt32Array, Uuid, ValidatedBulkEdges, ValidatedBulkNodes, batch_error, candidate_endpoint_uuids, candidate_uuids, contract_error, - existing_edge_context, field_error, indexed_existing, open_membership_index, row_error, + existing_edge_context, field_error, open_identity_probe, row_error, taken_identities, }; const NODE_REQUIRED: [(&str, DataType, bool); 2] = [ @@ -1095,19 +1095,8 @@ impl GraphForge { let mut existing = HashSet::new(); if reject_existing { let candidates = candidate_uuids(batches, BulkInputKind::Node, "node_uuid")?; - let mut index = open_membership_index(self, BulkInputKind::Node)?; - existing = indexed_existing( - index.as_mut(), - &candidates, - graphforge_storage::UuidIndexKind::Node, - BulkInputKind::Node, - )?; - existing.extend(indexed_existing( - index.as_mut(), - &candidates, - graphforge_storage::UuidIndexKind::Edge, - BulkInputKind::Node, - )?); + let mut probe = open_identity_probe(self, BulkInputKind::Node)?; + existing = taken_identities(probe.as_mut(), &candidates, BulkInputKind::Node)?; } let mut observed = HashSet::new(); let mut rows = Vec::new(); diff --git a/crates/graphforge-api/src/bulk_construction/publication.rs b/crates/graphforge-api/src/bulk_construction/publication.rs index 2bdce4c74..aa9c3ecad 100644 --- a/crates/graphforge-api/src/bulk_construction/publication.rs +++ b/crates/graphforge-api/src/bulk_construction/publication.rs @@ -4,8 +4,8 @@ use super::{ Arc, BTreeSet, BulkEdgePublicationError, BulkEdgeRow, BulkInputKind, BulkNodePublicationError, BulkNodeRow, BulkValidationReason, DataType, Digest, Field, FixedSizeBinaryArray, GraphForge, HashMap, OperationId, RecordBatch, Schema, SchemaRef, Sha256, StringArray, UInt64Array, Uuid, - ValidatedBulkNodes, contract_metadata, indexed_existing, open_membership_index, - register_existing_endpoints, row_error, + ValidatedBulkNodes, contract_metadata, open_identity_probe, register_existing_endpoints, + row_error, taken_identities, }; fn bulk_node_generation_uuid(operation_uuid: OperationId, rows: &[BulkNodeRow]) -> Uuid { @@ -221,19 +221,8 @@ impl GraphForge { let mut candidates = normalized.identities().collect::>(); candidates.sort_unstable(); candidates.dedup(); - let mut index = open_membership_index(self, BulkInputKind::Node)?; - let mut existing = indexed_existing( - index.as_mut(), - &candidates, - graphforge_storage::UuidIndexKind::Node, - BulkInputKind::Node, - )?; - existing.extend(indexed_existing( - index.as_mut(), - &candidates, - graphforge_storage::UuidIndexKind::Edge, - BulkInputKind::Node, - )?); + let mut probe = open_identity_probe(self, BulkInputKind::Node)?; + let existing = taken_identities(probe.as_mut(), &candidates, BulkInputKind::Node)?; if normalized .rows .iter() @@ -408,19 +397,8 @@ impl GraphForge { .collect::>(); candidates.sort_unstable(); candidates.dedup(); - let mut index = open_membership_index(self, BulkInputKind::Edge)?; - let mut existing = indexed_existing( - index.as_mut(), - &candidates, - graphforge_storage::UuidIndexKind::Edge, - BulkInputKind::Edge, - )?; - existing.extend(indexed_existing( - index.as_mut(), - &candidates, - graphforge_storage::UuidIndexKind::Node, - BulkInputKind::Edge, - )?); + let mut probe = open_identity_probe(self, BulkInputKind::Edge)?; + let existing = taken_identities(probe.as_mut(), &candidates, BulkInputKind::Edge)?; if let Some(row) = normalized .rows .iter() @@ -457,7 +435,7 @@ impl GraphForge { .iter() .flat_map(|row| [row.source_uuid, row.target_uuid]) .collect::>(); - register_existing_endpoints(&mut writer, &self.dir(), &endpoints)?; + register_existing_endpoints(&mut writer, &endpoints)?; for row in &normalized.rows { next_catalog.intern_relation_type(&row.rel_type)?; writer.create_edge( diff --git a/crates/graphforge-api/src/bulk_construction/publication/deleted_identity_tests.rs b/crates/graphforge-api/src/bulk_construction/publication/deleted_identity_tests.rs index 852906bc6..07be6695a 100644 --- a/crates/graphforge-api/src/bulk_construction/publication/deleted_identity_tests.rs +++ b/crates/graphforge-api/src/bulk_construction/publication/deleted_identity_tests.rs @@ -17,7 +17,10 @@ fn param(id: Uuid) -> HashMap { fn delete_node(graph: &GraphForge, id: Uuid) { graph - .execute_with_params("MATCH (n) WHERE n.node_uuid = $id DETACH DELETE n", ¶m(id)) + .execute_with_params( + "MATCH (n) WHERE n.node_uuid = $id DETACH DELETE n", + ¶m(id), + ) .unwrap(); } @@ -54,13 +57,22 @@ fn probe_outcomes_for_deleted_and_live_identities() { graph .publish_bulk_nodes( operation(10), - &[node_batch(&[a, b, c], &["P", "P", "P"], &[None, None, None])], + &[node_batch( + &[a, b, c], + &["P", "P", "P"], + &[None, None, None], + )], ) .unwrap(); graph .publish_bulk_edges( operation(11), - &[edge_batch(&[e1, e2, e3], &["R", "R", "R"], &[a, b, b], &[b, c, a])], + &[edge_batch( + &[e1, e2, e3], + &["R", "R", "R"], + &[a, b, b], + &[b, c, a], + )], ) .unwrap(); delete_edge(&graph, a, b); @@ -109,11 +121,15 @@ fn probe_outcomes_for_deleted_and_live_identities() { ), ( "node uuid == live edge", - outcome(graph.validate_bulk_nodes(operation(19), &[node_batch(&[e3], &["P"], &[None])])), + outcome( + graph.validate_bulk_nodes(operation(19), &[node_batch(&[e3], &["P"], &[None])]), + ), ), ( "node uuid == deleted edge", - outcome(graph.validate_bulk_nodes(operation(20), &[node_batch(&[e1], &["P"], &[None])])), + outcome( + graph.validate_bulk_nodes(operation(20), &[node_batch(&[e1], &["P"], &[None])]), + ), ), ( "edge re-add cascade-deleted by DETACH", @@ -151,14 +167,14 @@ fn probe_outcomes_for_deleted_and_live_identities() { ) }; let expected = [ - ("deleted node re-add", "accepted".to_owned()), + ("deleted node re-add", conflict("node", "node_uuid")), ("live node dup", conflict("node", "node_uuid")), - ("deleted edge re-add", "accepted".to_owned()), + ("deleted edge re-add", conflict("edge", "edge_uuid")), ("live edge dup", conflict("edge", "edge_uuid")), ("edge uuid == live node", conflict("edge", "edge_uuid")), - ("edge uuid == deleted node", "accepted".to_owned()), + ("edge uuid == deleted node", conflict("edge", "edge_uuid")), ("node uuid == live edge", conflict("node", "node_uuid")), - ("node uuid == deleted edge", "accepted".to_owned()), + ("node uuid == deleted edge", conflict("node", "node_uuid")), ("edge to deleted node", missing("target_uuid")), ("edge from never-seen node", missing("source_uuid")), ]; @@ -172,10 +188,16 @@ fn probe_outcomes_for_deleted_and_live_identities() { assert_eq!(name, expected_name); assert_eq!(got, want, "{name}"); } + let cascaded = rows + .iter() + .find(|(name, _)| *name == "edge re-add cascade-deleted by DETACH") + .unwrap(); + assert_eq!(cascaded.1, conflict("edge", "edge_uuid")); } -/// Validation looks at live state only, but the commit refuses to reuse the -/// identity of a deleted entity: the UUID is spent for good. +/// A UUID is spent for good once an entity holds it: validation refuses the +/// identity of a deleted entity with the same typed conflict the commit gives, +/// so validation never accepts what the commit rejects. #[test] fn deleted_identities_are_not_reused_at_commit() { let (_directory, graph) = project(); @@ -184,7 +206,11 @@ fn deleted_identities_are_not_reused_at_commit() { graph .publish_bulk_nodes( operation(10), - &[node_batch(&[a, b, c], &["P", "P", "P"], &[None, None, None])], + &[node_batch( + &[a, b, c], + &["P", "P", "P"], + &[None, None, None], + )], ) .unwrap(); graph @@ -201,5 +227,21 @@ fn deleted_identities_are_not_reused_at_commit() { assert!(edge.is_err(), "deleted edge UUID must stay spent"); let cascaded = graph.publish_bulk_edges(operation(24), &[edge_batch(&[e2], &["R"], &[a], &[b])]); - assert!(cascaded.is_err(), "cascade-deleted edge UUID must stay spent"); + assert!( + cascaded.is_err(), + "cascade-deleted edge UUID must stay spent" + ); + // The refusal is durable: a fresh open sees the same spent identities. + drop(graph); + let reopened = GraphForge::new(Some(_directory.path().to_str().unwrap())).unwrap(); + assert!( + reopened + .publish_bulk_nodes(operation(25), &[node_batch(&[c], &["P"], &[None])]) + .is_err() + ); + assert!( + reopened + .publish_bulk_edges(operation(26), &[edge_batch(&[e1], &["R"], &[a], &[b])]) + .is_err() + ); } diff --git a/crates/graphforge-api/src/composite_publish.rs b/crates/graphforge-api/src/composite_publish.rs index c8904b37b..bda09222f 100644 --- a/crates/graphforge-api/src/composite_publish.rs +++ b/crates/graphforge-api/src/composite_publish.rs @@ -10,7 +10,6 @@ pub(crate) use rebase::administrative_contract; use rebase::{capture_rebase_baseline, ensure_rebase_compatible}; use std::collections::{BTreeSet, HashMap, HashSet}; -use std::path::Path; use arrow::array::{Array, FixedSizeBinaryArray}; use arrow::record_batch::RecordBatch; @@ -802,7 +801,6 @@ fn build_validation_snapshot( fn register_existing_endpoints( writer: &mut graphforge_storage::GraphWriter, - dir: &Path, endpoints: &BTreeSet, same_request_nodes: &BTreeSet, ) -> Result<(), GfError> { @@ -813,11 +811,6 @@ fn register_existing_endpoints( if existing.is_empty() { return Ok(()); } - if !graphforge_storage::uuid_membership_index_is_fresh(dir)? { - return Err(GfError::Storage( - "composite endpoint resolution requires a fresh authenticated UUID index; run the explicit bounded index migration first".into(), - )); - } writer.register_existing_endpoints(&existing)?; Ok(()) } @@ -872,7 +865,7 @@ fn apply_graph_mutations( _ => None, }) .collect::>(); - register_existing_endpoints(&mut writer, &graph.dir(), &endpoints, &same_request_nodes)?; + register_existing_endpoints(&mut writer, &endpoints, &same_request_nodes)?; let mut node_sets: HashMap>> = HashMap::new(); let mut edge_sets: HashMap>> = @@ -1512,15 +1505,6 @@ mod tests { seed.create_node(existing, graphforge_value::EntityTypeId::decode(0).unwrap()) .unwrap(); seed.flush().unwrap(); - graphforge_storage::rebuild_uuid_membership_indexes( - directory.path(), - graphforge_storage::UuidIndexBuildLimits { - scan_batch_rows: 1, - run_records: 1, - merge_fan_in: 2, - }, - ) - .unwrap(); let same_request = uuid7(181); let endpoints = BTreeSet::from([existing, same_request]); @@ -1530,13 +1514,7 @@ mod tests { let mut writer = graphforge_storage::GraphWriter::open_at(directory.path(), OntologyMode::Strict, 2) .unwrap(); - register_existing_endpoints( - &mut writer, - directory.path(), - &endpoints, - &same_request_nodes, - ) - .unwrap(); + register_existing_endpoints(&mut writer, &endpoints, &same_request_nodes).unwrap(); let io = graphforge_storage::io_stats::snapshot().expect("requested I/O statistics"); assert_eq!(io.node_full_reads, 0); assert_eq!(io.node_filtered_reads, 0); diff --git a/crates/graphforge-api/src/construction.rs b/crates/graphforge-api/src/construction.rs index 1a4ed2805..10c2cebc2 100644 --- a/crates/graphforge-api/src/construction.rs +++ b/crates/graphforge-api/src/construction.rs @@ -539,7 +539,9 @@ mod tests { "{name} hydration verification reads" ); assert!(work.io.rewrite_commits > 0, "{name} topology rewrite work"); - assert!(work.io.uuid_files_synced > 0, "{name} UUID sync work"); + // Identity is answered from the published Parquet: appending an + // edge stages and syncs no identity file (#1902). + assert_eq!(work.io.uuid_files_synced, 0, "{name} UUID sync work"); } } diff --git a/crates/graphforge-api/src/embedding_refresh.rs b/crates/graphforge-api/src/embedding_refresh.rs index 93ded6209..51b9fa407 100644 --- a/crates/graphforge-api/src/embedding_refresh.rs +++ b/crates/graphforge-api/src/embedding_refresh.rs @@ -261,7 +261,7 @@ impl GraphForge { ordinal_identities: Arc::clone(&self.ordinal_identities), read_only: self.read_only, current_generation_uuid: Arc::clone(&self.current_generation_uuid), - uuid_membership_index: std::sync::Mutex::new(None), + identity_probe: std::sync::Mutex::new(None), #[cfg(feature = "knowledge")] epistemic_ledger_cache: std::sync::Mutex::new(None), clock: std::sync::Mutex::new(Arc::clone( diff --git a/crates/graphforge-api/src/graph_publication.rs b/crates/graphforge-api/src/graph_publication.rs index d3f041cbb..64f124165 100644 --- a/crates/graphforge-api/src/graph_publication.rs +++ b/crates/graphforge-api/src/graph_publication.rs @@ -184,13 +184,6 @@ impl GraphForge { )); } - if !graphforge_storage::uuid_membership_index_is_fresh(&self.dir())? { - graphforge_storage::rebuild_uuid_membership_indexes_with_topology( - &self.dir(), - graphforge_storage::UuidIndexBuildLimits::default(), - std::sync::Arc::clone(&self.dir().topology), - )?; - } let (graph, graph_objects) = compact_graph_participant( &self.dir(), &parent, @@ -281,13 +274,6 @@ impl GraphForge { "project generation changed before graph publication".into(), )); } - if !graphforge_storage::uuid_membership_index_is_fresh(&self.dir())? { - graphforge_storage::rebuild_uuid_membership_indexes_with_topology( - &self.dir(), - graphforge_storage::UuidIndexBuildLimits::default(), - std::sync::Arc::clone(&self.dir().topology), - )?; - } let (graph, graph_objects) = compact_graph_participant(&self.dir(), &parent, false, &self.dir().topology_files()?)?; let provenance_enabled = parent.capability("provenance")?.is_some(); diff --git a/crates/graphforge-api/src/lib.rs b/crates/graphforge-api/src/lib.rs index e35e0e8fe..23cc364da 100644 --- a/crates/graphforge-api/src/lib.rs +++ b/crates/graphforge-api/src/lib.rs @@ -666,7 +666,7 @@ pub struct GraphForge { /// Generation UUID whose graph snapshot was hydrated into `dir`. current_generation_uuid: Arc>, /// Authenticated UUID index handle cached for one topology generation. - uuid_membership_index: Mutex>, + identity_probe: Mutex>, /// Decoded epistemic ledgers cached for one immutable read generation. /// See `epistemic_snapshot::EpistemicLedgerCache` for the invalidation /// contract: keyed by `(generation_uuid, manifest_sha256)`, so a publish @@ -904,7 +904,7 @@ impl GraphForge { })), read_only: false, current_generation_uuid: Arc::new(Mutex::new(generation_uuid)), - uuid_membership_index: Mutex::new(None), + identity_probe: Mutex::new(None), #[cfg(feature = "knowledge")] epistemic_ledger_cache: Mutex::new(None), ordinal_identities, @@ -1185,7 +1185,7 @@ impl GraphForge { })), read_only, current_generation_uuid: Arc::new(Mutex::new(generation_uuid)), - uuid_membership_index: Mutex::new(None), + identity_probe: Mutex::new(None), #[cfg(feature = "knowledge")] epistemic_ledger_cache: Mutex::new(None), ordinal_identities, diff --git a/crates/graphforge-api/src/maintenance.rs b/crates/graphforge-api/src/maintenance.rs index 1c95d3136..da4219014 100644 --- a/crates/graphforge-api/src/maintenance.rs +++ b/crates/graphforge-api/src/maintenance.rs @@ -191,7 +191,7 @@ impl GraphForge { .lock() .expect("semantic storage binding lock poisoned") = bindings; *self - .uuid_membership_index + .identity_probe .lock() .expect("UUID membership index lock poisoned") = None; drop(old_workspace); @@ -478,8 +478,8 @@ mod tests { &graphforge_storage::resolve_project_generation(&root).unwrap(), ) .unwrap(); - *graph.uuid_membership_index.lock().unwrap() = - Some(graphforge_storage::UuidMembershipIndex::open(&graph.dir()).unwrap()); + *graph.identity_probe.lock().unwrap() = + Some(graphforge_storage::TopologyIdentityProbe::open_dir(&graph.dir()).unwrap()); // Observe reclamation without adding another workspace owner. let old_dir = graph.dir().to_path_buf(); let snapshot = graph @@ -499,7 +499,7 @@ mod tests { assert!(report.cleanup.is_some()); assert_ne!(graph.dir().path(), old_dir.as_path()); - assert!(graph.uuid_membership_index.lock().unwrap().is_none()); + assert!(graph.identity_probe.lock().unwrap().is_none()); assert!(old_dir.exists(), "active stream retains old workspace"); drop(snapshot); assert!(!old_dir.exists(), "last stream releases old workspace"); diff --git a/crates/graphforge-api/src/mutation_transaction.rs b/crates/graphforge-api/src/mutation_transaction.rs index 53b50547d..f743f2e69 100644 --- a/crates/graphforge-api/src/mutation_transaction.rs +++ b/crates/graphforge-api/src/mutation_transaction.rs @@ -210,7 +210,7 @@ impl FacadeMutationLifecycle<'_> { } *self .graph - .uuid_membership_index + .identity_probe .lock() .expect("UUID membership index lock poisoned") = None; self.graph.adjacency_provider_for_session().invalidate(); diff --git a/crates/graphforge-api/src/ontology_composition_lifecycle.rs b/crates/graphforge-api/src/ontology_composition_lifecycle.rs index 001cdaaf0..b5f26d20a 100644 --- a/crates/graphforge-api/src/ontology_composition_lifecycle.rs +++ b/crates/graphforge-api/src/ontology_composition_lifecycle.rs @@ -551,7 +551,7 @@ impl GraphForge { self.replace_workspace_owner(crate::GraphWorkspace::new(dir, owner, &inventory)?); self.graph_open_evidence = evidence; *self - .uuid_membership_index + .identity_probe .lock() .expect("UUID membership index lock poisoned") = None; } diff --git a/crates/graphforge-api/src/resumable_construction.rs b/crates/graphforge-api/src/resumable_construction.rs index 5243890cf..3027c9104 100644 --- a/crates/graphforge-api/src/resumable_construction.rs +++ b/crates/graphforge-api/src/resumable_construction.rs @@ -511,7 +511,7 @@ impl GraphConstructionSession<'_> { ); *self .graph - .uuid_membership_index + .identity_probe .lock() .expect("UUID membership lock poisoned") = None; // Release old reader handles before their workspace; streams own their pins. @@ -1181,7 +1181,7 @@ mod tests { drop(resumed); assert_construction_relationships(&graph, &original_relationships); - let index = graphforge_storage::UuidMembershipIndex::open(&graph.dir()).unwrap(); + let index = graphforge_storage::TopologyIdentityProbe::open_dir(&graph.dir()).unwrap(); assert_eq!(index.count(graphforge_storage::UuidIndexKind::Node), 3); assert_eq!(index.count(graphforge_storage::UuidIndexKind::Edge), 2); let catalog = graph.runtime_catalog.lock().unwrap(); @@ -1315,7 +1315,8 @@ mod tests { }) })); - let child_index = graphforge_storage::UuidMembershipIndex::open(&graph.dir()).unwrap(); + let child_index = + graphforge_storage::TopologyIdentityProbe::open_dir(&graph.dir()).unwrap(); assert_eq!( child_index.count(graphforge_storage::UuidIndexKind::Node), 4 @@ -1359,7 +1360,8 @@ mod tests { ); drop(historical_replay); assert_construction_relationships(&graph, ¤t_relationships); - let current_index = graphforge_storage::UuidMembershipIndex::open(&graph.dir()).unwrap(); + let current_index = + graphforge_storage::TopologyIdentityProbe::open_dir(&graph.dir()).unwrap(); assert_eq!( current_index.count(graphforge_storage::UuidIndexKind::Node), 4 diff --git a/crates/graphforge-api/src/resumable_construction/codec_tests.rs b/crates/graphforge-api/src/resumable_construction/codec_tests.rs index 3c01b6aa1..10a31faa1 100644 --- a/crates/graphforge-api/src/resumable_construction/codec_tests.rs +++ b/crates/graphforge-api/src/resumable_construction/codec_tests.rs @@ -8,7 +8,7 @@ use arrow::record_batch::RecordBatch; use graphforge_core::portable::{ PortableV2Limits, PortableV2Mode, PortableV2Output, PortableV2SelectionProfile, }; -use graphforge_storage::UuidMembershipIndex; +use graphforge_storage::TopologyIdentityProbe; use uuid::Uuid; use crate::{ @@ -91,7 +91,7 @@ fn graph_rows(graph: &GraphForge) -> (Vec, Vec) { } fn ordinals(graph: &GraphForge, ids: &[Uuid]) -> Vec> { - UuidMembershipIndex::open(&graph.dir()) + TopologyIdentityProbe::open_dir(&graph.dir()) .unwrap() .lookup_node_surrogates(ids) .unwrap() diff --git a/crates/graphforge-api/src/workspace_hydration/tests.rs b/crates/graphforge-api/src/workspace_hydration/tests.rs index 23a24ad86..10f27eb78 100644 --- a/crates/graphforge-api/src/workspace_hydration/tests.rs +++ b/crates/graphforge-api/src/workspace_hydration/tests.rs @@ -277,13 +277,9 @@ fn compact_graph_root_reopens_through_ordinary_api_and_rematerializes() { .execute("MATCH (n:Person) RETURN n.name AS name") .expect("ordinary query over compact-root generation"); assert_eq!(result.stats.rows_produced, 1); - // Mutable route and UUID controls are private; immutable payloads stay shared. - let controls = [ - "semantic-routes.json", - "topology/uuid-membership/manifest.json", - "topology/uuid-membership/topology-receipt.json", - ]; - assert_eq!(reopened.graph_open_evidence().files_copied, 3); + // The mutable route control is private; immutable payloads stay shared. + let controls = ["semantic-routes.json"]; + assert_eq!(reopened.graph_open_evidence().files_copied, 1); let mut control_bytes = 0; for relative in controls { let file = std::fs::File::open(reopened.dir().join(relative)).unwrap(); diff --git a/crates/graphforge-api/src/workspace_ontology.rs b/crates/graphforge-api/src/workspace_ontology.rs index 7dc35be99..84bad85dd 100644 --- a/crates/graphforge-api/src/workspace_ontology.rs +++ b/crates/graphforge-api/src/workspace_ontology.rs @@ -341,7 +341,7 @@ impl GraphForge { self.replace_workspace_owner(crate::GraphWorkspace::new(dir, workspace, &inventory)?); self.graph_open_evidence = evidence; *self - .uuid_membership_index + .identity_probe .lock() .expect("UUID membership index lock poisoned") = None; } else { diff --git a/crates/graphforge-api/tests/bounded_query_shapes.rs b/crates/graphforge-api/tests/bounded_query_shapes.rs index f9aaa807b..5fcca5558 100644 --- a/crates/graphforge-api/tests/bounded_query_shapes.rs +++ b/crates/graphforge-api/tests/bounded_query_shapes.rs @@ -1054,15 +1054,17 @@ fn undeclared_topology_files_do_not_change_readers_or_writes() { return Ok("search feature disabled".into()); } "uuid" => { - let metrics = graphforge_storage::rebuild_uuid_membership_indexes_with_topology( - &workspace, - graphforge_storage::UuidIndexBuildLimits::default(), - topology, - ) - .map_err(|error| error.to_string())?; + let files = graphforge_storage::enumerate_topology_files(&topology, None) + .map_err(|error| error.to_string())?; + let generation = graphforge_storage::read_topology_generation(&workspace) + .map_err(|error| error.to_string())?; + let probe = + graphforge_storage::TopologyIdentityProbe::open(&workspace, &files, generation) + .map_err(|error| error.to_string())?; return Ok(format!( "nodes={}, edges={}", - metrics.node_count, metrics.edge_count + probe.count(graphforge_storage::UuidIndexKind::Node), + probe.count(graphforge_storage::UuidIndexKind::Edge) )); } _ => unreachable!(), diff --git a/crates/graphforge-api/tests/file_backed_graph_generation.rs b/crates/graphforge-api/tests/file_backed_graph_generation.rs index 8479ae8ce..51babcf50 100644 --- a/crates/graphforge-api/tests/file_backed_graph_generation.rs +++ b/crates/graphforge-api/tests/file_backed_graph_generation.rs @@ -29,7 +29,7 @@ use graphforge_storage::{ PortableV2Limits, PortableV2Mode, PortableV2Output, PortableV2PackageClass, PortableV2PropertyProjection, PortableV2SelectionProfile, PortableV2SubsetClosure, PortableV2SubsetRequest, ProjectCapability, ProjectGenerationRequest, ProjectStageOutcome, - UuidIndexKind, UuidMembershipIndex, capture_graph_files, empty_workspace_participants, + TopologyIdentityProbe, UuidIndexKind, capture_graph_files, empty_workspace_participants, resolve_project_generation, stage_project_generation_with_graph_tree, }; use sha2::{Digest, Sha256}; @@ -106,7 +106,7 @@ fn property_digests(root: &Path) -> BTreeMap { /// The UUID-membership index of a project's current compact generation, read /// from a private materialization that outlives it. -fn membership_index(project: &Path) -> (tempfile::TempDir, UuidMembershipIndex) { +fn membership_index(project: &Path) -> (tempfile::TempDir, TopologyIdentityProbe) { let inventory = resolve_project_generation(project) .unwrap() .graph_files_inventory() @@ -114,7 +114,7 @@ fn membership_index(project: &Path) -> (tempfile::TempDir, UuidMembershipIndex) .unwrap(); let workspace = tempfile::tempdir_in(project).unwrap(); graphforge_storage::materialize_graph_objects(project, &inventory, workspace.path()).unwrap(); - let index = UuidMembershipIndex::open(workspace.path()).unwrap(); + let index = TopologyIdentityProbe::open_dir(workspace.path()).unwrap(); (workspace, index) } diff --git a/crates/graphforge-api/tests/identity_first_touch.rs b/crates/graphforge-api/tests/identity_first_touch.rs index 1b693bf8c..89d529e52 100644 --- a/crates/graphforge-api/tests/identity_first_touch.rs +++ b/crates/graphforge-api/tests/identity_first_touch.rs @@ -7,18 +7,18 @@ //! reads a forward run: the commit that builds the next identity generation on //! it does, and refuses a flipped one. //! -//! The small UUID-membership controls (the manifests, the receipts, the -//! tombstone runs and the lock) are different: hydration copies each into a +//! The small ordinal controls (the manifest, the receipt, the tombstone runs +//! and the lock) are different: hydration copies each into a //! private single-link file and checks the copy against the manifest, so the //! open is the operation that touches them, and it refuses a flipped one. The //! lock is published empty, so it has no byte to flip; it is opened and //! flocked, never read. //! -//! The UUID-membership (v3) runs, `identities-v5-*` and `node-surrogates-v5-*`, -//! are hard-linked like the ordinal runs and read by no query. Every topology -//! commit pins the authenticated UUID snapshot it builds the next generation -//! on, which checks every block of every run against the manifest, so the -//! commit refuses a flipped one. +//! Append validation reads the node and edge Parquet that identity is answered +//! from (#1902). Those objects are hard-linked like the ordinal runs and read by +//! no query that does not need their columns; the commit's identity probe is the +//! operation that touches them, and it refuses a flipped one before it names a +//! new digest. use std::io::{Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; @@ -417,8 +417,8 @@ fn inert_swap(relative: &str, bytes: &[u8]) -> (u64, fn(u8) -> u8) { ((digest + 63) as u64, other_hex_digit) } -/// The copied UUID-membership controls of a project that has been mutated, so -/// that a topology receipt and a non-empty tombstone run are published too. +/// The copied ordinal controls of a project that has been mutated, so that a +/// non-empty tombstone run is published too. #[test] fn flipped_identity_controls_are_refused_by_the_open_that_copies_them() { let (_root, path) = project(2_048); @@ -447,10 +447,16 @@ fn flipped_identity_controls_are_refused_by_the_open_that_copies_them() { }) .expect("a DELETE publishes a non-empty tombstone run") .clone(); + // The membership manifest and topology receipt are no longer published. + for retired in ["manifest.json", "topology-receipt.json"] { + let relative = format!("topology/uuid-membership/{retired}"); + assert!( + objects.iter().all(|(path, ..)| *path != relative), + "{relative} must not be published" + ); + } let cases = [ - control("manifest.json"), control("ordinal-v4-manifest.json"), - control("topology-receipt.json"), control("ordinal-v4-receipt.json"), tombstones, ]; @@ -484,132 +490,41 @@ fn flipped_identity_controls_are_refused_by_the_open_that_copies_them() { assert_eq!(rows(&forge, ORDERED), Ok(3)); } -/// The descriptor of one v3 run, from the project's published UUID manifest: -/// `(offset, length)` of each block. -fn v3_run_blocks(path: &Path, run: &str) -> Vec<(usize, usize)> { - let manifest = objects(path) - .into_iter() - .find(|(relative, ..)| relative == "topology/uuid-membership/manifest.json") - .expect("a published UUID manifest") - .1; - let manifest: serde_json::Value = - serde_json::from_slice(&std::fs::read(manifest).unwrap()).unwrap(); - let record = manifest["runs"] - .as_array() - .unwrap() - .iter() - .flat_map(|descriptor| [&descriptor["identities"], &descriptor["node_surrogates"]]) - .find(|record| record["name"] == run) - .unwrap_or_else(|| panic!("{run} is not in the UUID manifest")); - record["blocks"] - .as_array() - .unwrap() - .iter() - .map(|block| { - ( - usize::try_from(block["offset"].as_u64().unwrap()).unwrap(), - usize::try_from(block["len"].as_u64().unwrap()).unwrap(), - ) - }) - .collect() -} - -/// Record boundaries of an identity block: `UUID[16] kind[1]`, then a -/// surrogate[8] unless the record is a live edge (kind 1). -fn identity_records(block: &[u8]) -> Vec<(usize, u8)> { - let mut records = Vec::new(); - let mut at = 0; - while at < block.len() { - let kind = block[at + 16]; - records.push((at, kind)); - at += if kind == 1 { 17 } else { 25 }; - } - assert_eq!(at, block.len(), "identity records tile the block"); - records -} - -/// A byte of a v3 run no structural check reads, so that only a checksum can -/// refuse its change. Neither the first nor the last record of a block (their -/// keys are the block's fences) is touched, and nothing the decoder validates -/// (UUID order, kind, nonzero node surrogate, surrogate order) changes: -/// -/// - identities: the high byte of a middle node record's surrogate, which -/// stays nonzero; -/// - node surrogates (`surrogate[8] UUID[16]`): the last UUID byte of a middle -/// record. -fn inert_v3_flip(path: &Path, prefix: &str) -> (PathBuf, u64) { - let (relative, object, ..) = objects(path) +/// The commit that appends builds its identity probe over the published node +/// Parquet. The objects are hard-linked from the content store, so a flipped byte +/// is visible through the workspace; the probe's first touch checks the whole +/// object against its inventory checksum and refuses the commit before it names +/// a new digest. +#[test] +fn flipped_node_parquet_is_refused_by_the_commit_that_probes_it() { + let (_root, path) = project(NODES); + let (relative, object, _, length) = objects(&path) .into_iter() .find(|(relative, .., length)| { - relative.starts_with(&format!("topology/uuid-membership/{prefix}")) && *length > 0 + relative.starts_with("topology/nodes/") && relative.ends_with(".parquet") && *length > 0 }) - .unwrap_or_else(|| panic!("no non-empty {prefix} run")); - let run = relative.rsplit('/').next().unwrap(); - let bytes = std::fs::read(&object).unwrap(); - let (offset, length) = v3_run_blocks(path, run)[0]; - let block = &bytes[offset..offset + length]; - let at = if prefix.starts_with("identities") { - let records = identity_records(block); - let inner = &records[1..records.len() - 1]; - let middle = inner[inner.len() / 2..] - .iter() - .chain(inner) - .find(|(_, kind)| *kind == 0) - .expect("a live node record inside the block") - .0; - middle + 17 - } else { - assert_eq!(length % 24, 0, "node surrogate records are 24 bytes"); - (length / 24 / 2) * 24 + 23 - }; - (object, (offset + at) as u64) -} - -/// No query reads a v3 run; the commit that builds the next topology -/// generation authenticates every block of every run, and refuses a flipped -/// one before it names a new digest. -#[test] -fn flipped_uuid_membership_run_is_refused_by_the_commit_that_builds_on_it() { - // Every class is tried before asserting, so one run reports each run the - // commit fails to refuse. - let mut unrefused = Vec::new(); - for prefix in ["identities-v5-", "node-surrogates-v5-"] { - let (_root, path) = project(NODES); - let (object, offset) = inert_v3_flip(&path, prefix); - let _swap = ByteSwap::apply(&object, offset, |byte| byte ^ 0x80); - let published = generation_uuid(&path); + .expect("a published node fragment"); + // Inside the UUID column pages, well clear of the footer a reader parses. + let _swap = ByteSwap::apply(&object, length / 4, |byte| byte ^ 0x80); + let published = generation_uuid(&path); - let forge = GraphForge::new(Some(path.to_str().unwrap())).unwrap(); - // Open and reads never touch a v3 run. - assert_eq!(rows(&forge, HEALTHY), Ok(3), "{prefix}"); - assert_eq!(rows(&forge, ORDERED), Ok(3), "{prefix}"); - assert_eq!(rows(&forge, WHOLE), Ok(NODES * 4), "{prefix}"); - // The v3 reader reports the refusal as `GF_IO`; the message is the - // integrity check's own, which no decoder emits. - match forge.execute("CREATE (:Entity)") { - Ok(_) => unrefused.push(format!("{prefix}: the commit was served")), - Err(error) - if error - .to_string() - .contains("UUID run block authentication failed") => - { - eprintln!("{prefix}: refused: {} {error}", error.code()); - } - Err(error) => unrefused.push(format!( - "{prefix}: refused for the wrong reason: {} {error}", - error.code() - )), - } - drop(forge); - assert_eq!( - generation_uuid(&path), - published, - "{prefix}: a refused commit published a generation" - ); - } + let forge = GraphForge::new(Some(path.to_str().unwrap())).unwrap(); + let refused = forge + .execute("CREATE (:Entity)") + .map(drop) + .map_err(|error| format!("{} {error}", error.code())) + .expect_err("a commit probing a flipped node fragment must be refused"); + eprintln!("{relative}: refused: {refused}"); assert!( - unrefused.is_empty(), - "the commit that builds on a flipped UUID-membership run must refuse it:\n{}", - unrefused.join("\n") + refused.to_lowercase().contains("checksum") + || refused.to_lowercase().contains("do not match") + || refused.to_lowercase().contains("authenticat"), + "refused for the wrong reason: {refused}" + ); + drop(forge); + assert_eq!( + generation_uuid(&path), + published, + "a refused commit published a generation" ); } diff --git a/crates/graphforge-api/tests/lifecycle_io_attribution.rs b/crates/graphforge-api/tests/lifecycle_io_attribution.rs index 5ac2e74ea..196c9f4bf 100644 --- a/crates/graphforge-api/tests/lifecycle_io_attribution.rs +++ b/crates/graphforge-api/tests/lifecycle_io_attribution.rs @@ -534,9 +534,11 @@ fn open_reads_control_bytes_not_payload_bytes() { total_read_bytes(&cost.second_query) ); } - // The comparison is meaningful only if the payload really grew. + // The comparison is meaningful only if the payload really grew. The edge + // count grows 16x; the membership index that once added 17-25 B per edge to + // the payload is gone (#1902), so the payload grows by less. assert!( - large.payload_bytes > 6 * small.payload_bytes, + large.payload_bytes > 4 * small.payload_bytes, "edge payload did not grow: {} -> {}", small.payload_bytes, large.payload_bytes diff --git a/crates/graphforge-api/tests/permanent_storage_budgets.rs b/crates/graphforge-api/tests/permanent_storage_budgets.rs index 3c252b104..20033d526 100644 --- a/crates/graphforge-api/tests/permanent_storage_budgets.rs +++ b/crates/graphforge-api/tests/permanent_storage_budgets.rs @@ -593,42 +593,6 @@ fn parquet_experiment(source: &Path) -> Value { "encode_elapsed_ns":encode_ns, "decode_elapsed_ns":decode_ns}) } -fn pack_identity_records(original: &[u8]) -> Vec { - assert_eq!(original.len() % 32, 0); - let mut packed = Vec::with_capacity(original.len() / 32 * 25); - for record in original.chunks_exact(32) { - assert_eq!( - &record[17..24], - &[0; 7], - "only reserved zero padding is omitted" - ); - packed.extend_from_slice(&record[..17]); - packed.extend_from_slice(&record[24..32]); - } - packed -} - -#[test] -fn packed_identity_candidate_preserves_full_width_and_tombstone_boundaries() { - for uuid in [[0; 16], [255; 16]] { - for kind in 0..=3_u8 { - for surrogate in [0_u64, 1, u32::MAX.into(), u64::from(u32::MAX) + 1, u64::MAX] { - let mut original = [0; 32]; - original[..16].copy_from_slice(&uuid); - original[16] = kind; - original[24..32].copy_from_slice(&surrogate.to_be_bytes()); - let packed = pack_identity_records(&original); - assert_eq!(&packed[..16], &uuid); - assert_eq!(packed[16], kind); - assert_eq!( - u64::from_be_bytes(packed[17..25].try_into().unwrap()), - surrogate - ); - } - } - } -} - fn adjacency_codec_experiment(source: &Path) -> Value { use arrow::ipc::{ CompressionType, @@ -724,70 +688,6 @@ fn adjacency_codec_experiment(source: &Path) -> Value { "largest_decoded_batch_array_bytes":largest_decoded_batch,"production_format_changed":true}) } -fn identity_padding_experiment(source: &Path) -> Value { - let selected = graphforge_storage::resolve_project_generation(source).unwrap(); - let inventory = selected.graph_files_inventory().unwrap().unwrap(); - let mut records = 0_u64; - let mut edges = 0_u64; - let mut runs = 0_u64; - let mut current_bytes = 0_u64; - for entry in &inventory.files { - let name = Path::new(&entry.relative_path) - .file_name() - .unwrap() - .to_str() - .unwrap(); - if !entry.relative_path.starts_with("topology/uuid-membership/") - || !name.starts_with("identities-") - { - continue; - } - let bytes = std::fs::read( - graphforge_storage::graph_object_path(source, &entry.content_sha256).unwrap(), - ) - .unwrap(); - let mut cursor = bytes.as_slice(); - let mut prior: Option<[u8; 16]> = None; - while !cursor.is_empty() { - assert!(cursor.len() >= 17); - let uuid: [u8; 16] = cursor[..16].try_into().unwrap(); - assert!(prior.is_none_or(|prior| prior < uuid)); - prior = Some(uuid); - let kind = cursor[16]; - assert!(kind <= 3); - let width = if kind == 1 { 17 } else { 25 }; - assert!(cursor.len() >= width); - let surrogate = if kind == 1 { - 0 - } else { - u64::from_be_bytes(cursor[17..25].try_into().unwrap()) - }; - // Expand only in the test to quantify the previous physical representation. - // This is not a legacy reader or production migration path. - let mut expanded = [0_u8; 32]; - expanded[..17].copy_from_slice(&cursor[..17]); - expanded[24..].copy_from_slice(&surrogate.to_be_bytes()); - let packed = pack_identity_records(&expanded); - assert_eq!(&packed[..width], &cursor[..width]); - if kind == 1 { - assert_eq!(&packed[17..], &[0; 8]); - edges += 1; - } - records += 1; - cursor = &cursor[width..]; - } - current_bytes += bytes.len() as u64; - runs += 1; - } - assert!(records > 0); - assert_eq!(current_bytes, records * 25 - edges * 8); - assert!(current_bytes <= records * 25); - json!({"identity_runs":runs,"records":records,"live_edge_records":edges, - "current_record_bytes":current_bytes,"previous_32_byte_baseline":records * 32, - "reserved_padding_saved_bytes":records * 7,"defined_zero_edge_saved_bytes":edges * 8, - "production_format_changed":true}) -} - #[test] fn permanent_sequential_single_route() { assess(Fixture { @@ -1156,7 +1056,6 @@ fn assess(f: Fixture) { "random-identity fixtures must retain the measured at-least-20% Parquet reduction" ); } - let identity_experiment = identity_padding_experiment(&source); let manifest_experiment = bucket_manifest_experiment(&source); if f.heterogeneous { // 352 before ADR 0037. Construction now publishes the adjacency CSR with @@ -1164,7 +1063,9 @@ fn assess(f: Fixture) { // Merkle tree covers them, and the authenticated walk visits their leaves // and the interior nodes above them. The count is a pinned observation of // that tree's shape, not an invariant, so it moves when the file set does. - assert_eq!(manifest_experiment["authenticated_lookups_checked"], 373); + // 373 while the fixture still published the membership index files + // (manifest, topology receipt, identity and surrogate runs; #1902). + assert_eq!(manifest_experiment["authenticated_lookups_checked"], 368); assert!( manifest_experiment["source_manifest_allocated_bytes"] .as_u64() @@ -1202,7 +1103,7 @@ fn assess(f: Fixture) { json!({"fixture":f.name,"nodes":f.nodes,"edges":f.edges,"routes":f.routes,"random_ids":matches!(f.identifiers, Identifiers::Random),"properties":f.properties,"heterogeneous_schemas":f.heterogeneous,"adjacency_built":f.adjacency,"constructed_allocated_bytes":constructed.allocated_bytes,"constructed_categories":constructed.categories, "construction_elapsed_ns":construction_ns,"semantic_fingerprint":fingerprint,"whole_project":whole_project, "permanent": {"logical_bytes":storage.logical_bytes,"physical_logical_bytes":storage.physical_logical_bytes,"allocated_bytes":storage.allocated_bytes,"physical_objects":storage.physical_objects,"categories":storage.categories}, - "adjacency_codec_experiment":adjacency_experiment,"parquet_experiment":experiment,"identity_padding_experiment":identity_experiment,"manifest_bucket_experiment":manifest_experiment}) + "adjacency_codec_experiment":adjacency_experiment,"parquet_experiment":experiment,"manifest_bucket_experiment":manifest_experiment}) ); } @@ -3400,7 +3301,7 @@ fn cas_uuid_hydration_budget(root: &Path, source: &Path) { graphforge_storage::graph_object_path(source, &entry.content_sha256).unwrap(), ) .unwrap(); - if matches!(name, "manifest.json" | "topology-receipt.json") { + if matches!(name, "ordinal-v4-manifest.json" | "ordinal-v4-receipt.json") { assert_eq!(graphforge_filesystem::file_link_count(&file).unwrap(), 1); assert_ne!( graphforge_filesystem::file_identity(&file).unwrap(), @@ -3410,7 +3311,7 @@ fn cas_uuid_hydration_budget(root: &Path, source: &Path) { control_allocated += graphforge_filesystem::file_space_usage(&file) .unwrap() .allocated_bytes; - } else if name.starts_with("identities-v5") || name.starts_with("node-surrogates-v5") { + } else if name.starts_with("forward-v4-") || name.starts_with("ordinal-v4-") { assert!(file.metadata().unwrap().permissions().readonly()); assert_eq!( graphforge_filesystem::file_identity(&file).unwrap(), diff --git a/crates/graphforge-api/tests/scale_g500_ladder.rs b/crates/graphforge-api/tests/scale_g500_ladder.rs index 957d9e2da..940f15107 100644 --- a/crates/graphforge-api/tests/scale_g500_ladder.rs +++ b/crates/graphforge-api/tests/scale_g500_ladder.rs @@ -3619,16 +3619,15 @@ fn run_integrated_certification_config( let manifest_bytes = committed_inventory .files .iter() - .find(|entry| entry.relative_path == "topology/uuid-membership/manifest.json") - .expect("constructed UUID manifest") + .find(|entry| entry.relative_path == "topology/uuid-membership/ordinal-v4-manifest.json") + .expect("constructed ordinal manifest") .byte_length; - // Fresh construction has no topology-mutation receipt yet. If present, - // its authenticated bytes are also copied privately during hydration. let receipt_bytes = committed_inventory .files .iter() - .find(|entry| entry.relative_path == "topology/uuid-membership/topology-receipt.json") - .map_or(0, |entry| entry.byte_length); + .find(|entry| entry.relative_path == "topology/uuid-membership/ordinal-v4-receipt.json") + .expect("constructed ordinal receipt") + .byte_length; let hydration_uuid_control_bytes = manifest_bytes + receipt_bytes; journal.replace_project_owner("source_project", &committed_generation); journal.pass("ingest", phase, Some(input_fingerprint)); @@ -4434,7 +4433,7 @@ struct LifecycleLinearityObservation { /// Bytes of every canonical artifact the publication installed or reused. canonical_output_bytes: u64, cas_publication_io: graphforge_storage::GraphPublicationIo, - encode_fsync_components: [u64; 4], + encode_fsync_components: [u64; 3], hydration_files_copied: u64, hydration_uuid_control_bytes: u64, /// Files the final source manifest declares, and the catalog objects that @@ -4445,7 +4444,7 @@ struct LifecycleLinearityObservation { hydration_directory_fsync_operations: u64, shape_read_component_calls: [u64; 6], shape_write_component_calls: [u64; 2], - encode_write_component_calls: [u64; 5], + encode_write_component_calls: [u64; 4], category_metrics: BTreeMap, category_authority_metrics: BTreeMap, phase_disk_peaks: BTreeMap, @@ -4600,9 +4599,6 @@ fn lifecycle_linearity_observation(evidence: &Value) -> LifecycleLinearityObserv construction["encode_source_spool_fsync_operations"] .as_u64() .expect("encode spool fsyncs"), - construction["encode_membership_fsync_operations"] - .as_u64() - .expect("encode membership fsyncs"), construction["encode_ordinal_fsync_operations"] .as_u64() .expect("encode ordinal fsyncs"), @@ -4648,9 +4644,6 @@ fn lifecycle_linearity_observation(evidence: &Value) -> LifecycleLinearityObserv construction["encode_output_write_operations"] .as_u64() .expect("encode output writes"), - construction["encode_membership_write_operations"] - .as_u64() - .expect("encode membership writes"), construction["encode_source_spool_write_operations"] .as_u64() .expect("encode spool writes"), @@ -5000,12 +4993,17 @@ fn validate_retained_reconciliation(evidence: &Value) -> Result<(), String> { let topology_rows = live_nodes .checked_add(live_edges) .ok_or_else(|| "live topology denominator overflow".to_owned())?; - for category in ["uuid_and_surrogates", "adjacency"] { + // The ordinal node-identity facet holds one record per node; the UUID + // membership index that held one per edge too is gone (#1902). + for (category, denominator) in [ + ("uuid_and_surrogates", live_nodes), + ("adjacency", topology_rows), + ] { let logical_bytes = evidence_u64( evidence, &format!("/storage/{owner}/categories/{category}/logical_bytes"), )?; - if logical_bytes < topology_rows { + if logical_bytes < denominator { return Err(format!( "{owner}.{category} is below the topology denominator" )); @@ -5395,7 +5393,7 @@ enum PhaseMetricPolicy { /// `materialize_graph_objects` reads the route table once to /// authenticate routes, then for every object that /// `requires_single_link_materialization` (the route table itself, the - /// UUID-membership controls and the private ordinal-v4 authority) reads + /// private ordinal-v4 controls) reads /// it once while copying (`copy_and_authenticate_materialized_object`) /// and once more to verify the installed copy (`verify_file_counted`), /// writing it exactly once. Every other object is hard-linked from the @@ -5572,7 +5570,7 @@ const HYDRATION_ROUTE_TABLE_CONTROL_BYTES: u64 = 2 * 1024; // ceil(bytes / staged block) ..= bytes); object count and fsyncs are zero. // encode_write_postwrite_authentication: bytes and read calls are data- // proportional; write calls and fsyncs reconcile to the native encoder -// components (output, spool, membership, ordinal barriers). +// components (output, spool, ordinal barriers). // publication_preauthentication: the encoded-inventory control read is // structure-bounded by one encoding buffer, and its call count derives from // those bytes; every other field is zero. @@ -6031,7 +6029,6 @@ fn validate_positive_normalized_ceiling( enum CategoryBehavior { NodeBearing, EdgeBearing, - Mixed, FixedInventory, StructurallyZero, } @@ -6124,7 +6121,9 @@ fn category_behavior(category: &str) -> Result { // CSR shards do not append trailing rows without edges; its adjacency // payload therefore grows on the edge axis. This is fixture-specific. "topology_edges" | "adjacency" => Ok(CategoryBehavior::EdgeBearing), - "uuid_and_surrogates" => Ok(CategoryBehavior::Mixed), + // The ordinal node-identity facet is all that remains; it scales with + // nodes only (#1902). + "uuid_and_surrogates" => Ok(CategoryBehavior::NodeBearing), "catalog_and_manifests" => Ok(CategoryBehavior::FixedInventory), "properties" | "construction_staging" @@ -6209,9 +6208,6 @@ fn validate_category_taxonomy( { validate_affine_metric(&name, values, denominators)?; } - (CategoryBehavior::Mixed, 1 | 3) => { - validate_affine_metric(&name, values, denominators)?; - } (CategoryBehavior::NodeBearing, 4 | 5) if matches!(axis, LinearityAxis::Nodes) => { @@ -6222,9 +6218,6 @@ fn validate_category_taxonomy( { validate_quantized_allocation(&name, values, denominators)?; } - (CategoryBehavior::Mixed, 4 | 5) => { - validate_quantized_allocation(&name, values, denominators)?; - } (_, 1 | 3 | 4 | 5) if values[0] == values[1] && values[0] == values[2] => {} _ => { return Err(format!("{name} changed on its controlled axis: {values:?}")); @@ -6418,9 +6411,12 @@ fn validate_lifecycle_metric_policies_for_axis( .checked_add(observation.live_edges) .ok_or_else(|| "live topology denominator overflow".to_owned())?; for owner in ["source", "clean_import"] { - for category in ["uuid_and_surrogates", "adjacency"] { + for (category, denominator) in [ + ("uuid_and_surrogates", observation.live_nodes), + ("adjacency", topology_rows), + ] { let key = format!("{owner}.{category}"); - if observation.category_metrics[&key][1] < topology_rows { + if observation.category_metrics[&key][1] < denominator { return Err(format!( "{key} logical bytes are below the authoritative topology-row denominator at rung {rung}" )); @@ -6761,7 +6757,7 @@ fn validate_lifecycle_metric_policies_for_axis( .ok_or_else(|| format!("{name} component sum overflows"))?; if values[rung] != expected { return Err(format!( - "{name} does not reconcile output/spool/membership/ordinal barriers at rung {rung}" + "{name} does not reconcile output/spool/ordinal barriers at rung {rung}" )); } // Floor: a non-empty graph always writes at least one @@ -6926,7 +6922,6 @@ fn synthetic_category_metrics(axis: LinearityAxis, factor: u64) -> BTreeMap matches!(axis, LinearityAxis::Nodes), CategoryBehavior::EdgeBearing => matches!(axis, LinearityAxis::Edges), - CategoryBehavior::Mixed => true, CategoryBehavior::FixedInventory => false, CategoryBehavior::StructurallyZero => { metrics.insert(format!("{owner}.{category}"), [0; 6]); @@ -7001,7 +6996,7 @@ fn synthetic_linearity_observations_for_axis( factor, 0, 0, - 43, + 35, ], ), ( @@ -7153,7 +7148,7 @@ fn synthetic_linearity_observations_for_axis( ..Default::default() }, }, - encode_fsync_components: [10, 5, 8, 20], + encode_fsync_components: [10, 5, 20], hydration_files_copied: 19, hydration_uuid_control_bytes: 100, manifest_files: SYNTHETIC_MANIFEST_FILES, @@ -7162,7 +7157,7 @@ fn synthetic_linearity_observations_for_axis( hydration_directory_fsync_operations: 19, shape_read_component_calls: [factor, 0, 0, 0, 0, 0], shape_write_component_calls: [factor, 0], - encode_write_component_calls: [factor, 0, 0, 0, 0], + encode_write_component_calls: [factor, 0, 0, 0], category_metrics: synthetic_category_metrics(axis, factor), category_authority_metrics: synthetic_category_metrics(axis, factor), phase_disk_peaks: CERTIFICATION_PHASES @@ -8084,7 +8079,7 @@ fn encode_fsync_inventory_rejects_barriers_that_stop_being_counted() { .position(|field| *field == "fsync_calls") .unwrap(); for observation in &mut no_barriers { - observation.encode_fsync_components = [0; 4]; + observation.encode_fsync_components = [0; 3]; observation .phases .get_mut("encode_write_postwrite_authentication") diff --git a/crates/graphforge-search/src/node_identity.rs b/crates/graphforge-search/src/node_identity.rs index 976530df4..1c0de0b4d 100644 --- a/crates/graphforge-search/src/node_identity.rs +++ b/crates/graphforge-search/src/node_identity.rs @@ -26,8 +26,8 @@ use std::path::Path; use std::sync::Mutex; use arrow::array::{Array, FixedSizeBinaryArray, UInt64Array}; +use graphforge_storage::SearchArtifactError; use graphforge_storage::ordinal_identity_v4::{V4OrdinalIdentityError, V4OrdinalIdentityHandle}; -use graphforge_storage::{SearchArtifactError, UuidIndexKind, UuidMembershipIndex}; /// A facade's generation-pinned ordinal identity authority, the same handle its /// queries resolve destination identities through. @@ -40,7 +40,6 @@ pub(crate) struct NodeIdentityCheck<'a> { enum Authority<'a> { Ordinal(Box>), - Membership(Box), Unindexed(BTreeSet<[u8; 16]>), } @@ -66,18 +65,13 @@ enum Distinct { impl<'a> NodeIdentityCheck<'a> { /// Select the authority for `project_dir`. A session ordinal authority is - /// preferred; without one, the v3 index is used when present. + /// preferred; without one, rows are checked for repeats only. pub(crate) fn open( project_dir: &Path, ordinal: Option<&'a SessionOrdinalIdentity>, ) -> Result { let authority = if let Some(handle) = ordinal { Authority::Ordinal(Box::new(OrdinalCheck::open(project_dir, handle)?)) - } else if graphforge_storage::uuid_membership_index_present(project_dir) { - Authority::Membership(Box::new( - UuidMembershipIndex::open(project_dir) - .map_err(|error| source(error.to_string()))?, - )) } else { Authority::Unindexed(BTreeSet::new()) }; @@ -110,16 +104,6 @@ impl<'a> NodeIdentityCheck<'a> { Authority::Ordinal(check) => { check.resolve_batch(&batch_uuids, surrogates, checkpoint)? } - Authority::Membership(index) => { - let (values, _) = index - .lookup_node_surrogates(&batch_uuids) - .map_err(|error| source(error.to_string()))?; - values - .iter() - .enumerate() - .map(|(row, value)| *value == Some(surrogates.value(row))) - .collect() - } Authority::Unindexed(_) => vec![true; batch_uuids.len()], }; // Callers zip the verdicts with the rows; one per row, or none pass. @@ -133,8 +117,6 @@ impl<'a> NodeIdentityCheck<'a> { /// already checked that this row's `node_id` exceeds the previous one. pub(crate) fn distinct(&mut self, node_uuid: [u8; 16]) -> bool { match &mut self.authority { - // The authenticated index maps each UUID to one surrogate. - Authority::Membership(_) => true, Authority::Unindexed(seen) => seen.insert(node_uuid), Authority::Ordinal(check) => match &mut check.distinct { Distinct::Ascending(last) => { @@ -148,23 +130,11 @@ impl<'a> NodeIdentityCheck<'a> { } /// Prove after the last row that no live node was left out. - pub(crate) fn finish( - self, - rows: usize, - checkpoint: &mut C, - ) -> Result<(), SearchArtifactError> + pub(crate) fn finish(self, checkpoint: &mut C) -> Result<(), SearchArtifactError> where C: FnMut() -> Result<(), SearchArtifactError>, { match self.authority { - Authority::Membership(index) => { - if rows as u64 != index.count(UuidIndexKind::Node) { - return Err(source( - "topology row count disagrees with authenticated UUID index", - )); - } - Ok(()) - } Authority::Unindexed(_) => Ok(()), Authority::Ordinal(mut check) => { if let Some(last) = check.ranges.last().map(|range| *range.end()) { diff --git a/crates/graphforge-search/src/source.rs b/crates/graphforge-search/src/source.rs index adacdd102..a7e5223c8 100644 --- a/crates/graphforge-search/src/source.rs +++ b/crates/graphforge-search/src/source.rs @@ -363,7 +363,7 @@ where return Err(error); } *source_bytes = admitted; - identity.finish(topology_rows, checkpoint)?; + identity.finish(checkpoint)?; Ok(eligible) } diff --git a/crates/graphforge-search/src/vector_lifecycle.rs b/crates/graphforge-search/src/vector_lifecycle.rs index 1a5cc6c24..263b84848 100644 --- a/crates/graphforge-search/src/vector_lifecycle.rs +++ b/crates/graphforge-search/src/vector_lifecycle.rs @@ -396,7 +396,7 @@ where if let Some(error) = failure { return Err(error); } - identity.finish(rows, &mut checkpoint)?; + identity.finish(&mut checkpoint)?; let snapshot = SearchSourceSnapshot::from_admitted_files( project_dir, source_generation, diff --git a/crates/graphforge-storage/src/adjacency/classification_tests.rs b/crates/graphforge-storage/src/adjacency/classification_tests.rs index 70657f8e5..c084d2ba3 100644 --- a/crates/graphforge-storage/src/adjacency/classification_tests.rs +++ b/crates/graphforge-storage/src/adjacency/classification_tests.rs @@ -51,7 +51,7 @@ fn unusable_index_objects_are_classified_by_cause() { // Authenticated bytes (the manifest records their checksum) that are // not a shard: a correct writer never produces them. - let (_dir, path, mut reader, shard) = fixture(); + let (_dir, _path, mut reader, shard) = fixture(); let length = std::fs::metadata(&shard).unwrap().len() as usize; let garbage = vec![7_u8; length]; std::fs::write(&shard, &garbage).unwrap(); diff --git a/crates/graphforge-storage/src/catalog.rs b/crates/graphforge-storage/src/catalog.rs index 46d3975b6..e43c3bbe4 100644 --- a/crates/graphforge-storage/src/catalog.rs +++ b/crates/graphforge-storage/src/catalog.rs @@ -787,6 +787,17 @@ const MAX_ADMITTED_COLUMN_BYTES: i64 = 64 * 1024 * 1024; /// Open one path through the storage-wide fail-closed Parquet admission policy. pub(crate) fn admitted_parquet( path: &Path, +) -> Result, DataFusionError> { + admitted_parquet_with_options( + path, + parquet::arrow::arrow_reader::ArrowReaderOptions::new(), + ) +} + +/// [`admitted_parquet`] with explicit reader options, e.g. to load the page index. +pub(crate) fn admitted_parquet_with_options( + path: &Path, + reader_options: parquet::arrow::arrow_reader::ArrowReaderOptions, ) -> Result, DataFusionError> { let mut options = std::fs::OpenOptions::new(); options.read(true); @@ -804,8 +815,11 @@ pub(crate) fn admitted_parquet( ))); } preflight_parquet_handle(&mut file, metadata.len())?; - let builder = - ParquetRecordBatchReaderBuilder::try_new(admitted_path_file(file)?).map_err(parquet_err)?; + let builder = ParquetRecordBatchReaderBuilder::try_new_with_options( + admitted_path_file(file)?, + reader_options, + ) + .map_err(parquet_err)?; admit_decoded_parquet(&builder)?; Ok(builder) } diff --git a/crates/graphforge-storage/src/construction_bulk_tests.rs b/crates/graphforge-storage/src/construction_bulk_tests.rs index e0a7bef0f..e3a77eb4d 100644 --- a/crates/graphforge-storage/src/construction_bulk_tests.rs +++ b/crates/graphforge-storage/src/construction_bulk_tests.rs @@ -827,10 +827,7 @@ mod bulk_builder { "bulk.after_tables", "bulk.after_adjacency", "encode.after_inventory_pinned", - "uuid_encode.after_intent", - "uuid_encode.after_delta_runs", - "uuid_encode.after_manifest", - "bulk.after_membership", + "bulk.after_ordinal", "bulk.before_inventory", "bulk.after_inventory_before_intent_removal", ] { @@ -1204,9 +1201,10 @@ mod bulk_builder { /// Base scatter plus explicitly accounted bounded refinement/spools. /// Every successful scratch block is consumed exactly once. fn assert_scratch_traffic(report: &crate::BulkBuildReport, edges: u64) { - // 28-byte edge records, two 16-byte adjacency entries and a 16-byte - // identity per edge, plus an 8-byte header per block. - let payload = edges * (28 + 2 * 16 + 16); + // 28-byte edge records and two 16-byte adjacency entries per edge, + // plus an 8-byte header per block. Edge UUIDs are not spilled: no + // index is built from them (#1902). + let payload = edges * (28 + 2 * 16); let extra = report.edge_refinement_write_bytes + report.csr_spool_write_bytes; let base = report.scratch_write_bytes - extra; assert!( @@ -1260,6 +1258,36 @@ mod bulk_builder { } } + /// The membership index duplicated the published Parquet UUID columns and + /// is no longer produced (#1902). The ordinal node facet beside it is. + #[test] + fn neither_bulk_route_publishes_a_membership_index() { + let (nodes, edges) = graph(1_021, 3_001, 700, scattered); + let memory = bulk_with(&nodes, &edges, 2, 4).unwrap(); + let scratch = scratch_run(&nodes, &edges, 2, 4, (7, 5)).unwrap().inventory; + for (route, inventory) in [("memory", &memory), ("scratch", &scratch)] { + let paths = inventory + .iter() + .map(|(path, _, _)| path.as_str()) + .collect::>(); + assert!( + paths.contains(&"topology/uuid-membership/ordinal-v4-manifest.json"), + "{route}: the ordinal facet is the one artifact kept: {paths:?}" + ); + let index = paths + .iter() + .filter(|path| { + path.starts_with("topology/uuid-membership/") + && (path.ends_with("/manifest.json") + || path.ends_with("/topology-receipt.json") + || path.contains("identities-v5") + || path.contains("node-surrogates-v5")) + }) + .collect::>(); + assert!(index.is_empty(), "{route} published {index:?}"); + } + } + #[test] fn edge_windows_that_straddle_partitions_publish_the_in_memory_files() { // Several canonical edge files, and partition boundaries that fall @@ -1504,7 +1532,7 @@ mod bulk_builder { "bulk.after_edges", "bulk.after_ranks", "bulk.after_tables", - "bulk.after_membership", + "bulk.after_ordinal", "bulk.after_adjacency", "bulk.before_inventory", ] { diff --git a/crates/graphforge-storage/src/construction_chunk_spool_tests.rs b/crates/graphforge-storage/src/construction_chunk_spool_tests.rs index 44f36f9ab..3575f8b25 100644 --- a/crates/graphforge-storage/src/construction_chunk_spool_tests.rs +++ b/crates/graphforge-storage/src/construction_chunk_spool_tests.rs @@ -588,9 +588,9 @@ fn a_process_killed_during_the_spooled_build_reruns_to_identical_artifacts() { "bulk.after_tables", "bulk.after_adjacency", "encode.after_inventory_pinned", - "uuid_encode.after_intent", - "uuid_encode.after_manifest", - "bulk.after_membership", + "bulk.after_ordinal", + "v4_publish.after_artifacts", + "v4_publish.after_manifest_install", "bulk.before_inventory", "bulk.after_inventory_before_intent_removal", ] { diff --git a/crates/graphforge-storage/src/construction_directory.rs b/crates/graphforge-storage/src/construction_directory.rs index 3ed14c3b6..ee7ee41df 100644 --- a/crates/graphforge-storage/src/construction_directory.rs +++ b/crates/graphforge-storage/src/construction_directory.rs @@ -22,17 +22,6 @@ pub(crate) struct ConstructionDirectory { commits: Arc>, } impl ConstructionDirectory { - pub(crate) fn from_physical( - directory: &StableDirectory, - allocation: Option<&StorageAllocationOperation>, - ) -> io::Result { - Ok(Self { - directory: Arc::new(directory.try_clone()?), - allocation: allocation.cloned(), - commits: Arc::default(), - }) - } - pub(crate) fn open(path: &Path) -> io::Result { Ok(Self { directory: Arc::new(StableDirectory::open(path)?), diff --git a/crates/graphforge-storage/src/durable_rewrite.rs b/crates/graphforge-storage/src/durable_rewrite.rs index 3eb653249..7ff7d589c 100644 --- a/crates/graphforge-storage/src/durable_rewrite.rs +++ b/crates/graphforge-storage/src/durable_rewrite.rs @@ -200,6 +200,42 @@ pub(crate) fn reconcile_auxiliary( } } +/// Decide whether a rewrite that carried no auxiliary receipt committed. With +/// the project rewrite lock held and any durable intent rolled forward, the +/// generation state is either the prior pair (nothing happened) or the next +/// pair (the intent completed); anything else is ambiguous. +pub(crate) fn reconcile_generation_transition( + root: &Path, + prior: GenerationPair, + next: GenerationPair, +) -> Result { + let guard = acquire(root)?; + guard.revalidate()?; + let raw = crate::generation::read_generation_state_raw(root)?; + recover_locked( + root, + &guard.directory, + GenerationPair { + topology: raw.topology, + search: raw.search, + property: raw.property, + }, + )?; + let current = crate::generation::read_generation_state_raw(root)?; + let current = GenerationPair { + topology: current.topology, + search: current.search, + property: current.property, + }; + if current == next { + Ok(AuxiliaryReconcileOutcome::Committed) + } else if current == prior { + Ok(AuxiliaryReconcileOutcome::NotCommitted) + } else { + Err(storage("rewrite outcome is ambiguous or substituted")) + } +} + fn storage(error: impl std::fmt::Display) -> GfError { GfError::Storage(error.to_string()) } diff --git a/crates/graphforge-storage/src/graph_construction.rs b/crates/graphforge-storage/src/graph_construction.rs index 0591d89da..6dc271947 100644 --- a/crates/graphforge-storage/src/graph_construction.rs +++ b/crates/graphforge-storage/src/graph_construction.rs @@ -123,9 +123,19 @@ use serde::{Deserialize, Serialize}; use sha2::Digest; use uuid::Uuid; +use crate::TopologyIdentityProbe; use crate::UuidIndexKind; use crate::construction_detail_codec::{DetailCodec, DetailValidator}; -use crate::uuid_membership::{AuthenticatedUuidIndexSnapshot, UuidConstructionSnapshotWork}; + +/// What the construction session needs to know about its parent topology. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct UuidConstructionSnapshotWork { + pub authentication_bytes: u64, + pub authentication_blocks: u64, + pub live_nodes: u64, + pub live_edges: u64, + pub max_node_surrogate: u64, +} use crate::construction_record_layout::{ BASE_IDENTITY_WIDTH, ENDPOINT_WIDTH, FORMAT_VERSION, IDENTITY_SURROGATE_OFFSET, @@ -650,9 +660,6 @@ pub struct ConstructionShape { pub semantic_authority_sha256: Option, /// Parent generation retained by the publisher; zero denotes an empty base. pub parent_topology_generation: u64, - /// Authenticated parent UUID-manifest authority. The shaped identities file - /// contains only this session's delta and never copies the parent payload. - pub parent_uuid_manifest_sha256: Option, /// UUID-sorted node/edge identity records with assigned surrogates. pub identities: String, /// UUID-sorted node type records, when nodes were staged. @@ -740,8 +747,8 @@ impl ConstructionSemanticAuthority { pub use crate::graph_construction_encoding::{ BulkBatchReader, BulkBuildPlan, BulkBuildReport, BulkPassReport, BulkRoute, BulkSource, - BulkStagedReason, ConstructionRetainedArtifact, GraphConstructionEncoding, - GraphConstructionEncodingEvidence, GraphConstructionEncodingInvocationEvidence, + BulkStagedReason, GraphConstructionEncoding, GraphConstructionEncodingEvidence, + GraphConstructionEncodingInvocationEvidence, }; #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -1046,7 +1053,7 @@ pub struct GraphConstructionSession { project: StableDirectory, root: StableDirectory, checkpoint: Checkpoint, - base_snapshot: Option, + base_snapshot: Option, parent_catalog: RuntimeCatalog, compact_parent: Option, semantic_authority: Option, @@ -1619,25 +1626,22 @@ impl GraphConstructionSession { } (None, UuidConstructionSnapshotWork::default()) } else { - let mut snapshot = if let Some(inventory) = &compact_inventory { - AuthenticatedUuidIndexSnapshot::open_from_compact_inventory( + let snapshot = if let Some(inventory) = &compact_inventory { + TopologyIdentityProbe::open_compact( project_dir, inventory, parent_topology_generation, )? } else { - AuthenticatedUuidIndexSnapshot::open_at_generation( - graph_source_dir, - parent_topology_generation, - )? + let files = crate::TopologyFiles::discover_legacy(graph_source_dir)?; + TopologyIdentityProbe::open(graph_source_dir, &files, parent_topology_generation)? }; let max_node_surrogate = crate::writer::read_surrogate_tails(graph_source_dir)? .ok_or_else(|| storage("nonempty parent lacks surrogate tails"))? .0; - let (authentication_bytes, authentication_blocks) = snapshot.take_authentication_work(); let work = UuidConstructionSnapshotWork { - authentication_bytes, - authentication_blocks, + authentication_bytes: snapshot.authenticated_bytes(), + authentication_blocks: snapshot.authenticated_objects(), live_nodes: snapshot.count(UuidIndexKind::Node), live_edges: snapshot.count(UuidIndexKind::Edge), max_node_surrogate, diff --git a/crates/graphforge-storage/src/graph_construction/encoding_publication.rs b/crates/graphforge-storage/src/graph_construction/encoding_publication.rs index bec3d9542..2e4824fad 100644 --- a/crates/graphforge-storage/src/graph_construction/encoding_publication.rs +++ b/crates/graphforge-storage/src/graph_construction/encoding_publication.rs @@ -376,7 +376,7 @@ impl GraphConstructionSession { let encoding = self.prepare_canonical_encoding_with_cancellation(generation, cancelled)?; let nodes = encoding.evidence.ordinal_records; - let edges = encoding.evidence.membership_records.saturating_sub(nodes); + let edges = encoding.evidence.edge_records; if let Ok(mut report) = self.bulk_report.lock() { *report = crate::graph_construction_encoding::BulkBuildReport { nodes, @@ -445,7 +445,7 @@ impl GraphConstructionSession { shape, generation, self.checkpoint.ontology_mode, - self.base_snapshot.as_ref(), + self.checkpoint.base_work.live_nodes, parent.as_ref(), self.semantic_authority.as_ref(), &shape_outputs, @@ -650,19 +650,6 @@ impl GraphConstructionSession { .publication_application_read_operations .checked_add(manifest_read_calls) .ok_or_else(|| storage("publication manifest read call count overflows"))?; - for retained in &encoding.retained_artifacts { - let entry = manifest_state - .entries() - .find(|entry| entry.relative_path == retained.target_path) - .ok_or_else(|| storage("retained construction object is absent from parent"))?; - if entry.byte_length != retained.bytes - || entry.content_sha256 != retained.sha256 - || entry.content_xxh64 != retained.xxh64 - { - return Err(storage("retained construction object authority changed")); - } - } - let workspace = self .project_path .join(PRIVATE_ROOT) diff --git a/crates/graphforge-storage/src/graph_construction/encoding_publication/tests.rs b/crates/graphforge-storage/src/graph_construction/encoding_publication/tests.rs index 9b23d1cd9..ccb64fe0d 100644 --- a/crates/graphforge-storage/src/graph_construction/encoding_publication/tests.rs +++ b/crates/graphforge-storage/src/graph_construction/encoding_publication/tests.rs @@ -249,7 +249,7 @@ fn canonical_encoder_outputs_feed_ordinary_readers_index_and_adjacency() { let encoding = session.encode_canonical(&shape, 1).unwrap(); assert_eq!(encoding.evidence.prior_topology_rows_decoded, 0); assert_eq!(encoding.evidence.retained_topology_bytes_copied, 0); - assert_eq!(encoding.evidence.membership_records, 5); + assert_eq!(encoding.evidence.edge_records, 2); assert_eq!(encoding.evidence.ordinal_records, 3); assert_eq!(encoding.evidence.ordinal_artifact_write_bytes, 120); assert_eq!(encoding.evidence.ordinal_artifact_write_operations, 2); @@ -350,7 +350,7 @@ fn canonical_encoder_outputs_feed_ordinary_readers_index_and_adjacency() { .sum::(), 2 ); - let index = crate::UuidMembershipIndex::open(&graph).unwrap(); + let index = crate::TopologyIdentityProbe::open_dir(&graph).unwrap(); assert_eq!(index.count(crate::UuidIndexKind::Node), 3); assert_eq!(index.count(crate::UuidIndexKind::Edge), 2); // ADR 0037: the encoder publishes the adjacency CSR with the generation, so @@ -838,7 +838,6 @@ fn canonical_encoder_reuse_accounts_only_second_invocation_io() { assert!(!second.invocation.performed); assert!(second.invocation.reused); assert_eq!(second.invocation.evidence.output_write_bytes, 0); - assert_eq!(second.invocation.evidence.membership_total_write_bytes, 0); assert_eq!( session.evidence().encode_application_write_bytes, writes_after_first @@ -1417,11 +1416,8 @@ fn canonical_routing_is_checkpoint_bound_in_all_ontology_modes() { } #[test] -fn generation_two_parent_index_is_structurally_referenced_without_payload_copy() { +fn generation_two_encoding_publishes_no_membership_index() { let project = nonempty_project_generation_two(); - assert!(crate::has_runtime_entity_label_encoding_marker( - &project.path().join("fixture-graph") - )); let operation = Uuid::from_u128(9_340); let mut session = GraphConstructionSession::open( project.path(), @@ -1436,41 +1432,17 @@ fn generation_two_parent_index_is_structurally_referenced_without_payload_copy() session.seal().unwrap(); let shape = session.shape_canonical_with_cancellation(|| false).unwrap(); let encoded = session.encode_canonical(&shape, 3).unwrap(); - assert_eq!(encoded.evidence.retained_index_payload_bytes, 0); assert_eq!(encoded.evidence.retained_topology_bytes_copied, 0); assert_eq!(encoded.evidence.prior_topology_rows_decoded, 0); - assert_eq!(encoded.evidence.retained_index_runs, 2); - let index_outputs = encoded - .artifacts - .iter() - .filter(|artifact| { - artifact.path.contains("uuid-membership") - && !artifact.path.contains("ordinal-v4") - && !artifact.path.contains("forward-v4") - && !artifact.path.contains("tombstones-v4") - }) - .count(); - // New identity + reverse runs and the new manifest. The retained base - // and level-one descriptors remain structural references. - assert_eq!(index_outputs, 3); - assert!(!encoded.retained_artifacts.is_empty()); - let assembled = project - .path() - .join(PRIVATE_ROOT) - .join(operation.simple().to_string()) - .join(&encoded.root) - .join("graph"); - for retained in &encoded.retained_artifacts { - let target = assembled.join(&retained.target_path); - std::fs::create_dir_all(target.parent().unwrap()).unwrap(); - std::fs::hard_link( - std::path::Path::new(&retained.source_root).join(&retained.source_path), - target, - ) - .unwrap(); - } - let opened = crate::UuidMembershipIndex::open(&assembled).unwrap(); - assert_eq!(opened.count(crate::UuidIndexKind::Node), 4); + // Only the ordinal node-identity facet lives under uuid-membership/; the + // membership index (manifest, runs, receipt) is no longer encoded. + assert!(encoded.artifacts.iter().all(|artifact| { + !artifact.path.contains("uuid-membership") + || ["ordinal-v4", "forward-v4", "tombstones-v4"] + .iter() + .any(|facet| artifact.path.contains(facet)) + })); + let _ = (operation, PRIVATE_ROOT); } #[test] @@ -1574,118 +1546,7 @@ fn parent_phase_observations_survive_staging_sealed_and_shaped_resumes() { } #[test] -fn completed_encoding_replay_reauthenticates_retained_parent_payload() { - let project = nonempty_project_generation_two(); - let operation = Uuid::from_u128(9_343); - let mut session = GraphConstructionSession::open( - project.path(), - operation, - 2, - GraphConstructionBudgets::default(), - ) - .unwrap(); - session - .append(ConstructionChunkKind::Node, "delta", &node_batch(4, 1)) - .unwrap(); - session.seal().unwrap(); - let shape = session.shape_canonical_with_cancellation(|| false).unwrap(); - let encoded = session.encode_canonical(&shape, 3).unwrap(); - let retained = encoded - .retained_artifacts - .iter() - .find(|artifact| artifact.bytes > 0) - .unwrap(); - let path = std::path::Path::new(&retained.source_root).join(&retained.source_path); - let original_metadata = std::fs::metadata(&path).unwrap(); - let original_identity = - graphforge_filesystem::file_identity(&std::fs::File::open(&path).unwrap()).unwrap(); - let original_permissions = original_metadata.permissions(); - let mut permissions = original_permissions.clone(); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - permissions.set_mode(0o600); - } - #[cfg(not(unix))] - permissions.set_readonly(false); - std::fs::set_permissions(&path, permissions).unwrap(); - let mut file = std::fs::OpenOptions::new().write(true).open(&path).unwrap(); - use std::io::{Seek as _, SeekFrom}; - file.seek(SeekFrom::Start(0)).unwrap(); - file.write_all(&[0xff]).unwrap(); - file.sync_all().unwrap(); - drop(file); - std::fs::set_permissions(&path, original_permissions).unwrap(); - assert_eq!( - std::fs::metadata(&path).unwrap().len(), - original_metadata.len() - ); - assert_eq!( - graphforge_filesystem::file_identity(&std::fs::File::open(&path).unwrap()).unwrap(), - original_identity, - ); - let error = session.encode_canonical(&shape, 3).unwrap_err(); - assert!( - error - .to_string() - .contains("graph payload XXH64 checksum does not match its inventory"), - "{error}" - ); -} - -#[test] -fn generation_one_parent_uses_streamed_binary_carry_and_authenticates_result() { - let project = nonempty_project_with_nodes(2); - let operation = Uuid::from_u128(9_341); - let mut session = GraphConstructionSession::open( - project.path(), - operation, - 1, - GraphConstructionBudgets::default(), - ) - .unwrap(); - session - .append(ConstructionChunkKind::Node, "delta", &node_batch(3, 1)) - .unwrap(); - session.seal().unwrap(); - let shape = session.shape_canonical_with_cancellation(|| false).unwrap(); - let encoded = session.encode_canonical(&shape, 2).unwrap(); - assert!(encoded.evidence.retained_index_payload_bytes > 0); - assert!(encoded.evidence.membership_read_bytes > 0); - assert!( - encoded.evidence.membership_total_write_bytes > encoded.evidence.membership_write_bytes - ); - - let graph = project - .path() - .join(PRIVATE_ROOT) - .join(operation.simple().to_string()) - .join(&encoded.root) - .join("graph"); - let parent_index = project - .path() - .join("fixture-graph/topology/uuid-membership"); - let encoded_index = graph.join("topology/uuid-membership"); - let parent_manifest: serde_json::Value = - serde_json::from_slice(&std::fs::read(parent_index.join("manifest.json")).unwrap()) - .unwrap(); - for run in parent_manifest["runs"].as_array().unwrap() { - if !run["base"].as_bool().unwrap() { - continue; - } - for field in ["identities", "node_surrogates"] { - let name = run[field]["name"].as_str().unwrap(); - assert_eq!(std::fs::metadata(parent_index.join(name)).unwrap().len(), 0); - std::fs::copy(parent_index.join(name), encoded_index.join(name)).unwrap(); - } - } - let index = crate::UuidMembershipIndex::open(&graph).unwrap(); - assert_eq!(index.count(crate::UuidIndexKind::Node), 3); - assert_eq!(index.count(crate::UuidIndexKind::Edge), 1); -} - -#[test] -fn parent_uuid_path_substitution_is_rejected_before_encoding() { +fn parent_topology_path_substitution_is_rejected_before_encoding() { let project = nonempty_project_with_nodes(2); let operation = Uuid::from_u128(9_342); let mut session = GraphConstructionSession::open( @@ -1708,14 +1569,17 @@ fn parent_uuid_path_substitution_is_rejected_before_encoding() { let victim = inventory .files .iter() - .find(|entry| entry.relative_path.contains("/identities-") && entry.byte_length != 0) + .find(|entry| entry.relative_path.starts_with("topology/nodes/") && entry.byte_length != 0) .unwrap(); let victim = crate::graph_object_path(project.path(), &victim.content_sha256).unwrap(); - let saved = victim.with_extension("uuidx.saved"); + let saved = victim.with_extension("saved"); std::fs::rename(&victim, &saved).unwrap(); std::fs::copy(&saved, &victim).unwrap(); let error = session.encode_canonical(&shape, 2).unwrap_err(); - assert!(error.to_string().contains("identity changed")); + assert!( + error.to_string().contains("changed under a retained probe"), + "{error}" + ); } fn encoded_publication_session(root: &TempDir, operation: Uuid) -> GraphConstructionSession { @@ -1930,7 +1794,7 @@ fn construction_append_publishes_current_complete_ordinal_authority() { .map(|id| Some(Uuid::from_u128(u128::from(*id)))) .collect::>() ); - let mut membership = crate::UuidMembershipIndex::open(&graph).unwrap(); + let mut membership = crate::TopologyIdentityProbe::open_dir(&graph).unwrap(); let uuids = ids .iter() .map(|id| Uuid::from_u128(u128::from(*id))) @@ -2122,12 +1986,15 @@ fn canonical_publication_installs_compact_graph_and_advances_current_once() { .iter() .any(|entry| entry.relative_path.starts_with("topology/nodes/")) ); - assert!( - inventory - .files - .iter() - .any(|entry| { entry.relative_path == "topology/uuid-membership/manifest.json" }) - ); + assert!(inventory.files.iter().any(|entry| { + entry.relative_path == "topology/uuid-membership/ordinal-v4-manifest.json" + })); + // The membership index (manifest, receipt, runs) is not published. + assert!(inventory.files.iter().all(|entry| { + entry.relative_path != "topology/uuid-membership/manifest.json" + && entry.relative_path != "topology/uuid-membership/topology-receipt.json" + && !entry.relative_path.contains("-v5-") + })); let current_path = root.path().join("CURRENT"); let current_bytes = std::fs::read(¤t_path).unwrap(); std::fs::write(¤t_path, b"concurrently-advanced-current\n").unwrap(); @@ -2141,7 +2008,7 @@ fn canonical_publication_installs_compact_graph_and_advances_current_once() { let materialized_graph = materialized.path().join("graph"); std::fs::create_dir(&materialized_graph).unwrap(); crate::materialize_graph_objects(root.path(), &inventory, &materialized_graph).unwrap(); - let uuid_index = crate::UuidMembershipIndex::open(&materialized_graph).unwrap(); + let uuid_index = crate::TopologyIdentityProbe::open_dir(&materialized_graph).unwrap(); assert_eq!(uuid_index.count(crate::UuidIndexKind::Node), 2); assert_eq!(uuid_index.count(crate::UuidIndexKind::Edge), 0); drop(current); diff --git a/crates/graphforge-storage/src/graph_construction/io_evidence.rs b/crates/graphforge-storage/src/graph_construction/io_evidence.rs index 0cb47ffee..2669a010a 100644 --- a/crates/graphforge-storage/src/graph_construction/io_evidence.rs +++ b/crates/graphforge-storage/src/graph_construction/io_evidence.rs @@ -102,9 +102,6 @@ pub struct GraphConstructionEvidence { /// Canonical output writer submissions. #[serde(default)] pub encode_output_write_operations: u64, - /// UUID-membership writer submissions, including carry. - #[serde(default)] - pub encode_membership_write_operations: u64, /// Authenticated source-spool writer submissions. #[serde(default)] pub encode_source_spool_write_operations: u64, @@ -126,9 +123,6 @@ pub struct GraphConstructionEvidence { /// Authenticated source-spool durability barriers. #[serde(default)] pub encode_source_spool_fsync_operations: u64, - /// UUID-membership durability barriers. - #[serde(default)] - pub encode_membership_fsync_operations: u64, /// Ordinal-index payload and publication durability barriers. #[serde(default)] pub encode_ordinal_fsync_operations: u64, @@ -908,7 +902,6 @@ pub(super) fn copy_post_shape_io( target.encode_application_write_bytes = source.encode_application_write_bytes; target.encode_application_write_operations = source.encode_application_write_operations; target.encode_output_write_operations = source.encode_output_write_operations; - target.encode_membership_write_operations = source.encode_membership_write_operations; target.encode_source_spool_write_operations = source.encode_source_spool_write_operations; target.encode_ordinal_artifact_write_operations = source.encode_ordinal_artifact_write_operations; @@ -918,7 +911,6 @@ pub(super) fn copy_post_shape_io( target.canonical_artifact_objects = source.canonical_artifact_objects; target.encode_output_fsync_operations = source.encode_output_fsync_operations; target.encode_source_spool_fsync_operations = source.encode_source_spool_fsync_operations; - target.encode_membership_fsync_operations = source.encode_membership_fsync_operations; target.encode_ordinal_fsync_operations = source.encode_ordinal_fsync_operations; target.publication_application_read_bytes = source.publication_application_read_bytes; target.publication_application_read_operations = source.publication_application_read_operations; @@ -1110,14 +1102,13 @@ pub(super) fn record_encoding_io_evidence( evidence.encode_application_read_bytes = checked_evidence_sum( "encoding read bytes", evidence.encode_application_read_bytes, - &[measured.input_read_bytes, measured.membership_read_bytes], + &[measured.input_read_bytes], )?; evidence.encode_application_read_operations = checked_evidence_sum( "encoding read operations", evidence.encode_application_read_operations, &[ measured.input_read_operations, - measured.membership_read_operations, measured.source_spool_read_operations, ], )?; @@ -1126,7 +1117,6 @@ pub(super) fn record_encoding_io_evidence( evidence.encode_application_write_bytes, &[ measured.output_write_bytes, - measured.membership_total_write_bytes, measured.source_spool_write_bytes, measured.ordinal_artifact_write_bytes, measured.ordinal_publication_write_bytes, @@ -1137,7 +1127,6 @@ pub(super) fn record_encoding_io_evidence( evidence.encode_application_write_operations, &[ measured.output_write_operations, - measured.membership_write_operations, measured.source_spool_write_operations, measured.ordinal_artifact_write_operations, measured.ordinal_publication_write_operations, @@ -1148,7 +1137,6 @@ pub(super) fn record_encoding_io_evidence( evidence.encode_fsync_operations, &[ measured.fsync_operations, - measured.membership_fsync_operations, measured.source_spool_fsync_operations, measured.ordinal_fsync_operations, ], @@ -1164,18 +1152,7 @@ pub(super) fn record_encoding_io_evidence( .checked_add(artifact.bytes) .ok_or_else(|| storage("canonical output inventory overflows")) })?; - let retained_bytes = encoded - .retained_artifacts - .iter() - .try_fold(0_u64, |total, artifact| { - total - .checked_add(artifact.bytes) - .ok_or_else(|| storage("retained artifact inventory overflows")) - })?; - evidence.staged_and_retained_disk_bytes = evidence - .write_bytes - .checked_add(retained_bytes) - .ok_or_else(|| storage("staged and retained inventory overflows"))?; + evidence.staged_and_retained_disk_bytes = evidence.write_bytes; Ok(()) } @@ -1189,11 +1166,6 @@ fn record_encoding_components( measured.output_write_operations, "encode_output_write_operations", ), - ( - &mut evidence.encode_membership_write_operations, - measured.membership_write_operations, - "encode_membership_write_operations", - ), ( &mut evidence.encode_source_spool_write_operations, measured.source_spool_write_operations, @@ -1219,11 +1191,6 @@ fn record_encoding_components( measured.source_spool_fsync_operations, "encode_source_spool_fsync_operations", ), - ( - &mut evidence.encode_membership_fsync_operations, - measured.membership_fsync_operations, - "encode_membership_fsync_operations", - ), ( &mut evidence.encode_ordinal_fsync_operations, measured.ordinal_fsync_operations, @@ -1514,7 +1481,7 @@ pub(super) fn read_run_record( } pub(super) fn account_probe_work( - work: &crate::uuid_membership::UuidProbeMetrics, + work: &crate::UuidProbeMetrics, evidence: &mut GraphConstructionEvidence, ) -> Result<(), GfError> { evidence.retained_probe_read_bytes = checked_evidence_sum( diff --git a/crates/graphforge-storage/src/graph_construction/shape.rs b/crates/graphforge-storage/src/graph_construction/shape.rs index c5c63c10e..a74f113cd 100644 --- a/crates/graphforge-storage/src/graph_construction/shape.rs +++ b/crates/graphforge-storage/src/graph_construction/shape.rs @@ -15,15 +15,15 @@ use super::recovery::{ is_shape_scoped_name, reconcile_shape_artifact_removal, unlink_shape_artifact_files, }; use super::{ - ArtifactReceipt, AuthenticatedShapeSource, AuthenticatedUuidIndexSnapshot, BASE_IDENTITY_WIDTH, - BLOCK_BYTES, BTreeMap, BufReader, BufWriter, CatalogSource, Checkpoint, ConstructionChunkKind, + ArtifactReceipt, AuthenticatedShapeSource, BASE_IDENTITY_WIDTH, BLOCK_BYTES, BTreeMap, + BufReader, BufWriter, CatalogSource, Checkpoint, ConstructionChunkKind, ConstructionChunkReceipt, ConstructionPublicationState, ConstructionShape, CountingRead, DetailCodec, DetailValidator, Digest, EDGE_DETAIL_WIDTH, ENDPOINT_WIDTH, File, GfError, GraphConstructionEvidence, GraphConstructionSession, GraphConstructionState, HashingWriter, IDENTITY_SURROGATE_OFFSET, IDENTITY_WIDTH, IoCounter, NODE_DETAIL_WIDTH, OsStr, RESOLVED_ENDPOINT_WIDTH, RESOLVED_SURROGATE_OFFSET, Read, ReadWork, SHAPE_INTENT, - SealDirectoryBatch, ShapeIntent, StableDirectory, Uuid, UuidIndexKind, Write, - account_cache_release, account_fixed_read_operations, account_fixed_write_operations, + SealDirectoryBatch, ShapeIntent, StableDirectory, TopologyIdentityProbe, Uuid, UuidIndexKind, + Write, account_cache_release, account_fixed_read_operations, account_fixed_write_operations, account_merge_read, account_merge_write, account_probe_work, account_sequential_read, account_sequential_write, artifact_temp, authenticate_artifact, build_runtime_catalog, canonical_artifact_target, checked_evidence_sum, combine_cache_cleanup, @@ -1071,10 +1071,6 @@ impl GraphConstructionSession { ontology_mode: self.checkpoint.ontology_mode, semantic_authority_sha256: self.checkpoint.semantic_authority_sha256.clone(), parent_topology_generation: self.checkpoint.parent_topology_generation, - parent_uuid_manifest_sha256: self - .base_snapshot - .as_ref() - .map(|snapshot| snapshot.manifest_sha256().to_owned()), identities, node_details, edge_details, @@ -1990,7 +1986,7 @@ fn validate_staged_details( fn reject_staged_base_conflicts( root: &StableDirectory, identities_name: &str, - base: &mut AuthenticatedUuidIndexSnapshot, + base: &mut TopologyIdentityProbe, window_rows: usize, cancelled: &mut impl FnMut() -> bool, evidence: &mut GraphConstructionEvidence, @@ -2338,7 +2334,7 @@ pub(super) fn resolve_endpoint_surrogates( plan: &PartitionPlan, identities_name: &str, endpoints_name: Option<&str>, - base: Option<&mut AuthenticatedUuidIndexSnapshot>, + base: Option<&mut TopologyIdentityProbe>, window_rows: usize, max_partition_bytes: u64, max_external_partition_bytes: u64, @@ -2396,7 +2392,7 @@ fn route_resolved_endpoints( plan: &PartitionPlan, identities_name: &str, endpoints_name: &str, - mut base: Option<&mut AuthenticatedUuidIndexSnapshot>, + mut base: Option<&mut TopologyIdentityProbe>, window_rows: usize, resolved: &mut FixedRangePartitioner<'_, RESOLVED_ENDPOINT_WIDTH>, cancelled: &mut impl FnMut() -> bool, @@ -2653,7 +2649,7 @@ fn resolve_endpoint_stages( plan: &PartitionPlan, identities_name: &str, endpoints_name: Option<&str>, - base: Option<&mut AuthenticatedUuidIndexSnapshot>, + base: Option<&mut TopologyIdentityProbe>, boundary: u64, retain_segments: bool, cpu_admission: Option<&std::sync::Arc>, diff --git a/crates/graphforge-storage/src/graph_construction/shape/tests.rs b/crates/graphforge-storage/src/graph_construction/shape/tests.rs index 45341b118..bba9c43d0 100644 --- a/crates/graphforge-storage/src/graph_construction/shape/tests.rs +++ b/crates/graphforge-storage/src/graph_construction/shape/tests.rs @@ -479,7 +479,6 @@ fn nonempty_base_rejects_duplicate_cross_kind_and_missing_endpoint_without_copy( ); assert!(!operation_root.join("staged-identities.run").exists()); assert_eq!(shape.parent_topology_generation, 1); - assert!(shape.parent_uuid_manifest_sha256.is_some()); drop(delta); let mut retained_endpoints = diff --git a/crates/graphforge-storage/src/graph_construction/supersession.rs b/crates/graphforge-storage/src/graph_construction/supersession.rs index 724206b56..9968d21d6 100644 --- a/crates/graphforge-storage/src/graph_construction/supersession.rs +++ b/crates/graphforge-storage/src/graph_construction/supersession.rs @@ -75,7 +75,7 @@ impl GraphConstructionSession { { return Err(storage("supersession private authority changed")); } - let _retained_successor_leases = if self.has_encoding_successor() { + if self.has_encoding_successor() { let output = self .root .open_child_directory(OsStr::new("encoded-v1")) @@ -108,7 +108,6 @@ impl GraphConstructionSession { &self.checkpoint.evidence, cancelled, )?; - self.authenticate_retained_successors(&inventory, cancelled)? } else { // The shape manifest is the successor authority here. Its retained // payloads were verified at the replay/recovery boundary by @@ -121,8 +120,7 @@ impl GraphConstructionSession { for receipt in read_completed_shape_outputs(&self.root, &self.checkpoint)? { authenticate_retained_identity(&self.root, &receipt)?; } - None - }; + } // Authenticate the complete immutable receipt chain BEFORE any removal. let mut previous = None; @@ -230,97 +228,6 @@ impl GraphConstructionSession { Ok(()) } - fn authenticate_retained_successors( - &mut self, - encoding: &GraphConstructionEncoding, - cancelled: &mut impl FnMut() -> bool, - ) -> Result< - Option<( - crate::ResolvedProjectGeneration, - crate::graph_object_store::GraphObjectReadLease, - )>, - GfError, - > { - if encoding.retained_artifacts.is_empty() { - if encoding.evidence.retained_index_runs != 0 { - return Err(storage("retained-index evidence lacks references")); - } - return Ok(None); - } - let parent = crate::resolve_generation_by_uuid( - &self.project_path, - self.checkpoint.parent_generation_uuid, - )?; - if hex(&parent.manifest_sha256()) != self.checkpoint.parent_generation_manifest_sha256 { - return Err(storage("supersession parent manifest changed")); - } - let lease = crate::graph_object_store::begin_graph_object_read(&self.project_path)?; - let (inventory, work) = compact_parent_inventory(&parent)?; - self.record_supersession_reads(work.bytes, work.operations)?; - let inventory = - inventory.ok_or_else(|| storage("supersession compact parent is absent"))?; - let manifest = inventory - .files - .iter() - .find(|entry| entry.relative_path == "topology/uuid-membership/manifest.json") - .ok_or_else(|| storage("supersession parent UUID manifest is absent"))?; - let (_manifest, work, released) = lease.open_for_construction( - &manifest.content_sha256, - manifest.byte_length, - manifest.content_xxh64, - cancelled, - )?; - self.record_supersession_reads(work.read_bytes, work.read_calls)?; - account_cache_release(released, &mut self.checkpoint.evidence)?; - let mut previous = None; - for retained in &encoding.retained_artifacts { - if previous.is_some_and(|name: &str| name >= retained.target_path.as_str()) - || !retained - .target_path - .starts_with("topology/uuid-membership/") - || retained.source_root != self.project_path.to_string_lossy() - || retained.source_root_volume != self.project.identity().volume_serial - || retained.source_root_file_id != hex(&self.project.identity().file_id) - || retained.parent_manifest_sha256 != manifest.content_sha256 - { - return Err(storage("supersession retained parent authority changed")); - } - previous = Some(retained.target_path.as_str()); - let entry = inventory - .files - .iter() - .find(|entry| entry.relative_path == retained.target_path) - .ok_or_else(|| storage("supersession retained parent artifact is absent"))?; - let source = crate::graph_object_path(&self.project_path, &entry.content_sha256)?; - if entry.content_sha256 != retained.sha256 - || entry.content_xxh64 != retained.xxh64 - || entry.byte_length != retained.bytes - || source - .strip_prefix(&self.project_path) - .map_err(storage)? - .to_string_lossy() - != retained.source_path - { - return Err(storage("supersession retained parent artifact changed")); - } - let (file, work, released) = lease.open_for_construction( - &entry.content_sha256, - entry.byte_length, - entry.content_xxh64, - cancelled, - )?; - let identity = file_identity(file.as_ref()).map_err(storage)?; - if identity.volume_serial != retained.source_volume - || hex(&identity.file_id) != retained.source_file_id - { - return Err(storage("supersession retained parent identity changed")); - } - self.record_supersession_reads(work.read_bytes, work.read_calls)?; - account_cache_release(released, &mut self.checkpoint.evidence)?; - } - Ok(Some((parent, lease))) - } - fn checkpoint_supersession(&mut self) -> Result<(), GfError> { let mut next = self.checkpoint.clone(); next.evidence.recovery_checkpoint_fsync_operations = next @@ -333,24 +240,6 @@ impl GraphConstructionSession { Ok(()) } - fn record_supersession_reads(&mut self, bytes: u64, operations: u64) -> Result<(), GfError> { - self.checkpoint.evidence.recovery_application_read_bytes = self - .checkpoint - .evidence - .recovery_application_read_bytes - .checked_add(bytes) - .ok_or_else(|| storage("supersession read bytes overflow"))?; - self.checkpoint - .evidence - .recovery_application_read_operations = self - .checkpoint - .evidence - .recovery_application_read_operations - .checked_add(operations) - .ok_or_else(|| storage("supersession read operations overflow"))?; - Ok(()) - } - /// Unlink one consumed payload against its receipt, reconciling the /// allocation ledgers. /// diff --git a/crates/graphforge-storage/src/graph_construction/tests.rs b/crates/graphforge-storage/src/graph_construction/tests.rs index 97ce49df7..2c3e284fa 100644 --- a/crates/graphforge-storage/src/graph_construction/tests.rs +++ b/crates/graphforge-storage/src/graph_construction/tests.rs @@ -891,22 +891,17 @@ fn publication_crash_after_current_finalizes_same_target_on_reopen() { let graph = materialized.path().join("graph"); std::fs::create_dir(&graph).unwrap(); crate::materialize_graph_objects(root.path(), &inventory, &graph).unwrap(); - let uuid_index = crate::UuidMembershipIndex::open(&graph).unwrap(); + let uuid_index = crate::TopologyIdentityProbe::open_dir(&graph).unwrap(); assert_eq!(uuid_index.count(crate::UuidIndexKind::Node), 2); } #[test] -fn uuid_encoding_crashes_recover_every_durable_boundary() { +fn ordinal_encoding_crashes_recover_every_durable_boundary() { for failpoint in [ "encode.parquet.after_temp_fsync.topology/nodes/00000000000000000001-00000000000000000008.parquet", "encode.parquet.after_install.topology/nodes/00000000000000000001-00000000000000000008.parquet", "encode.copy.after_temp_fsync.topology/runtime_catalog.parquet", "encode.copy.after_install.topology/runtime_catalog.parquet", - "uuid_encode.after_intent", - "uuid_encode.after_temps", - "uuid_encode.after_delta_runs", - "uuid_encode.after_manifest", - "uuid_encode.after_intent_removal", "v4_publish.after_artifacts", "v4_publish.after_artifacts_fsync", "v4_publish.after_receipt_temp_fsync", @@ -943,7 +938,11 @@ fn uuid_encoding_crashes_recover_every_durable_boundary() { .unwrap(); let shape = resumed.shape_canonical_with_cancellation(|| false).unwrap(); let encoded = resumed.encode_canonical(&shape, 1).unwrap(); - assert_eq!(encoded.evidence.membership_records, 8, "{failpoint}"); + assert_eq!( + encoded.evidence.ordinal_records + encoded.evidence.edge_records, + 8, + "{failpoint}" + ); let membership = root .path() .join(PRIVATE_ROOT) @@ -1048,7 +1047,10 @@ fn over_bound_encoded_inventory_is_refused_before_pinning_and_resumes() { resumed.checkpoint.encoding_inventory_sha256, Some(crate::graph_construction_encoding::inventory_authority_sha256(&encoding).unwrap()) ); - assert_eq!(encoding.evidence.membership_records, 8); + assert_eq!( + encoding.evidence.ordinal_records + encoding.evidence.edge_records, + 8 + ); drop(resumed); let encoded = StableDirectory::open(&encoded_root).unwrap(); @@ -1106,7 +1108,7 @@ fn over_bound_encoded_inventory_is_refused_before_pinning_and_resumes() { assert_eq!(io.read_bytes, inventory_bytes); } let mut changed = encoding.clone(); - changed.evidence.membership_records += 1; + changed.evidence.edge_records += 1; let error = open() .reclaim_superseded_payloads_with_successor(Some(&changed), &mut || false) .unwrap_err(); diff --git a/crates/graphforge-storage/src/graph_construction_encoding.rs b/crates/graphforge-storage/src/graph_construction_encoding.rs index be9b02d7f..dd7bfab6c 100644 --- a/crates/graphforge-storage/src/graph_construction_encoding.rs +++ b/crates/graphforge-storage/src/graph_construction_encoding.rs @@ -48,9 +48,7 @@ use crate::property_overlay::{ use crate::schemas::{ TOPOLOGY_NODES_SCHEMA, TYPED_EDGE_SCHEMA, uuid_field, with_semantic_route_metadata, }; -use crate::uuid_membership::{ - AuthenticatedUuidIndexSnapshot, ConstructionIndexOutput, ConstructionIndexReference, -}; +use crate::uuid_membership::ConstructionIndexOutput; use crate::{SemanticRouteKind, SemanticStorageBindings}; mod adjacency; @@ -354,35 +352,6 @@ pub struct ConstructionEncodedArtifact { pub xxh64: u64, } -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -/// Exact authenticated parent object that a publisher must structurally retain. -pub struct ConstructionRetainedArtifact { - /// Stable authenticated parent directory supplied to the publisher. - pub source_root: String, - /// Device identity of the authenticated parent directory. - pub source_root_volume: u64, - /// File identity of the authenticated parent directory. - pub source_root_file_id: String, - /// Parent-root-relative immutable object name. - pub source_path: String, - /// Device identity of the retained immutable object. - pub source_volume: u64, - /// File identity of the retained immutable object. - pub source_file_id: String, - /// Generation-root-relative target name for structural installation. - pub target_path: String, - /// Exact retained object length. - pub bytes: u64, - /// Exact retained object digest. - pub sha256: String, - /// Required seed-zero checksum of the exact artifact bytes. - #[serde(with = "crate::corruption_checksum::wire_hex")] - pub xxh64: u64, - /// Authenticated parent index manifest that authorized this reference. - pub parent_manifest_sha256: String, -} - #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] /// Measured bounded work for canonical encoding. pub struct GraphConstructionEncodingEvidence { @@ -418,22 +387,6 @@ pub struct GraphConstructionEncodingEvidence { pub peak_batch_bytes: u64, /// Largest number of simultaneously live durable Parquet writers. Always one. pub peak_open_writers: u64, - /// New identity records streamed into the v3 index. - pub membership_records: u64, - /// New v3 membership bytes written. - pub membership_write_bytes: u64, - /// Retained v3 run descriptors structurally reused. - pub retained_index_runs: u64, - /// Retained v3 payload bytes read only for required binary-carry compaction. - pub retained_index_payload_bytes: u64, - /// Actual block reads performed by v3 construction encoding and carry. - pub membership_read_bytes: u64, - /// Actual block reads performed by v3 construction encoding and carry. - pub membership_read_operations: u64, - /// Actual block writes, including outputs superseded by carry. - pub membership_write_operations: u64, - /// All v3 bytes written, including superseded carry inputs. - pub membership_total_write_bytes: u64, /// Largest number of decoded shaped-row readers simultaneously live. pub peak_open_input_readers: u64, /// Authenticated source bytes written to one private random-access spool. @@ -448,17 +401,12 @@ pub struct GraphConstructionEncodingEvidence { pub source_spool_fsync_operations: u64, /// Peak owned authenticated source spool bytes (one source at a time). pub source_spool_peak_temporary_bytes: u64, - /// v3 durability barriers. - pub membership_fsync_operations: u64, - /// Newly created immutable v3 runs, including superseded carry inputs. - pub membership_created_runs: u64, - /// Peak live v3 transform/merge buffer bytes. - pub membership_peak_buffer_bytes: u64, - /// Peak bytes in owned temporary v3 outputs. - pub membership_peak_temporary_bytes: u64, /// Canonical node identities streamed into the v4 ordinal index. #[serde(default)] pub ordinal_records: u64, + /// Canonical edge identities encoded into the edge topology. + #[serde(default)] + pub edge_records: u64, /// Immutable v4 payload bytes written before publication metadata. #[serde(default)] pub ordinal_artifact_write_bytes: u64, @@ -510,8 +458,6 @@ pub struct GraphConstructionEncoding { pub shape_authority_sha256: String, /// Sorted, unique canonical artifact records. pub artifacts: Vec, - /// Authenticated retained-parent objects required to assemble this graph. - pub retained_artifacts: Vec, /// Measured bounded work. pub evidence: GraphConstructionEncodingEvidence, /// Work performed by the invocation that returned this inventory. @@ -549,7 +495,7 @@ pub(crate) fn encode( shape: &ConstructionShape, generation: u64, ontology_mode: OntologyMode, - parent_index: Option<&AuthenticatedUuidIndexSnapshot>, + parent_nodes: u64, parent_generation: Option<&crate::ResolvedProjectGeneration>, semantic_authority: Option<&ConstructionSemanticAuthority>, shape_outputs: &[ArtifactReceipt], @@ -604,7 +550,7 @@ pub(crate) fn encode( let actual_authority = inventory_authority_sha256(&existing)?; match expected_inventory_sha256 { Some(expected) if expected == actual_authority => { - let authentication = authenticate_inventory(&output, &existing, parent_index)?; + let authentication = authenticate_inventory(&output, &existing)?; remove_encoding_intent(&output)?; existing.invocation = GraphConstructionEncodingInvocationEvidence { performed: false, @@ -717,26 +663,15 @@ pub(crate) fn encode( "ordinal control calls", )?; - let identities_xxh64 = shaped_output_xxh64(shape_outputs, &shape.identities)?; - let membership_region = - crate::concurrency_attribution::RegionScope::named("membership_encoding"); - let mut index = crate::uuid_membership::encode_construction_index( - crate::uuid_membership::ConstructionIdentityInput::Shaped { - source: source.physical(), - name: &shape.identities, - xxh64: identities_xxh64, - }, - output.physical(), - generation, - shape.parent_topology_generation, - parent_index, - shape.node_count, - shape.edge_count, - cancelled, - output.allocation(), - )?; - - drop(membership_region); + let _identities_xxh64 = shaped_output_xxh64(shape_outputs, &shape.identities)?; + // The ordinal node-identity facet is published under uuid-membership/. A + // crashed attempt's private residue goes first. + crate::uuid_membership::clear_private_ordinal_residue(&output)?; + output + .create_child_directory(OsStr::new("graph")) + .and_then(|graph| graph.create_child_directory(OsStr::new("topology"))) + .and_then(|topology| topology.create_child_directory(OsStr::new("uuid-membership"))) + .map_err(storage)?; let nodes_region = crate::concurrency_attribution::RegionScope::named("node_encoding"); let mut encoding_lanes = lanes::ParquetLanes::new(admission, budgets.max_batch_bytes); let v4 = encode_nodes( @@ -762,7 +697,7 @@ pub(crate) fn encode( let metrics = &bundle.metrics; let expected_delta_nodes = shape .node_count - .checked_sub(parent_index.map_or(0, |parent| parent.count(crate::UuidIndexKind::Node))) + .checked_sub(parent_nodes) .ok_or_else(|| storage("shaped node count is smaller than parent"))?; if metrics.input_records != expected_delta_nodes { return Err(storage("v4 construction count differs from shaped nodes")); @@ -779,7 +714,6 @@ pub(crate) fn encode( output.physical(), bundle, generation, - &index.source_sha256, parent_ordinal .as_ref() .and_then(|(_, manifest)| parent_generation.map(|parent| (parent, manifest))), @@ -810,7 +744,7 @@ pub(crate) fn encode( .max(publication.peak_temporary_bytes); account_cache_release(metrics.cache_release, &mut evidence)?; crate::graph_construction::construction_failpoint("encode.after_v4_before_inventory"); - index.artifacts.extend(v4_artifacts); + artifacts.extend(v4_artifacts.into_iter().map(index_artifact)); } drop(nodes_region); let edges_region = crate::concurrency_attribution::RegionScope::named("edge_encoding"); @@ -874,26 +808,7 @@ pub(crate) fn encode( )?; drop(adjacency_region); - evidence.membership_records = index.input_records; - evidence.membership_read_bytes = index.read_bytes; - evidence.membership_write_bytes = index.final_write_bytes; - evidence.membership_total_write_bytes = index.write_bytes; - evidence.membership_read_operations = index.read_operations; - evidence.membership_write_operations = index.write_operations; - evidence.membership_fsync_operations = index.fsync_operations; - evidence.membership_created_runs = index.created_runs; - evidence.membership_peak_buffer_bytes = index.peak_buffer_bytes; - evidence.membership_peak_temporary_bytes = index.peak_temporary_bytes; - account_cache_release(index.cache_release, &mut evidence)?; - evidence.retained_index_runs = index.retained_runs; - evidence.retained_index_payload_bytes = index.retained_payload_bytes; - let retained_artifacts = index - .retained_references - .into_iter() - .map(retained_artifact) - .collect(); - artifacts.extend(index.artifacts.into_iter().map(index_artifact)); - + evidence.edge_records = shape.edge_count; artifacts.sort_unstable_by(|left, right| left.path.cmp(&right.path)); if artifacts .windows(2) @@ -910,12 +825,11 @@ pub(crate) fn encode( shape_inputs_sha256: shape.runtime_catalog_inputs_sha256.clone(), shape_authority_sha256: shape_authority_sha256.to_owned(), artifacts, - retained_artifacts, evidence, invocation: GraphConstructionEncodingInvocationEvidence::default(), }; install_json(&output, INVENTORY, &completed)?; - authenticate_inventory_control(&completed, parent_index)?; + authenticate_inventory_control(&completed)?; remove_encoding_intent(&output)?; let invocation = completed.evidence.clone(); completed.invocation = GraphConstructionEncodingInvocationEvidence { @@ -1003,22 +917,6 @@ fn construction_parent_routes( Ok(table) } -fn retained_artifact(value: ConstructionIndexReference) -> ConstructionRetainedArtifact { - ConstructionRetainedArtifact { - source_root: value.source_root, - source_root_volume: value.source_root_volume, - source_root_file_id: value.source_root_file_id, - source_path: value.source_path, - source_volume: value.source_volume, - source_file_id: value.source_file_id, - target_path: value.target_path, - bytes: value.bytes, - sha256: value.sha256, - xxh64: value.xxh64, - parent_manifest_sha256: value.parent_manifest_sha256, - } -} - fn index_artifact(value: ConstructionIndexOutput) -> ConstructionEncodedArtifact { ConstructionEncodedArtifact { path: format!("topology/uuid-membership/{}", value.name), diff --git a/crates/graphforge-storage/src/graph_construction_encoding/bulk.rs b/crates/graphforge-storage/src/graph_construction_encoding/bulk.rs index 43ea54556..011a8c0bb 100644 --- a/crates/graphforge-storage/src/graph_construction_encoding/bulk.rs +++ b/crates/graphforge-storage/src/graph_construction_encoding/bulk.rs @@ -14,7 +14,7 @@ //! - Pass 2 decodes edges in parallel, resolves endpoints through the node //! index, orders and ranks edges, and rejects identity collisions. //! - Pass 3 emits catalog, node and edge tables, property overlays, the -//! membership and ordinal indexes, and the adjacency CSR. +//! ordinal node-identity facet, and the adjacency CSR. //! //! Intermediates are never synced or hashed. A crash discards them and the //! build reruns from the sources (ADR 0038 as amended for initial builds). @@ -30,8 +30,8 @@ use super::{ GfError, GraphConstructionBudgets, GraphConstructionEncoding, GraphConstructionEncodingEvidence, GraphConstructionEncodingInvocationEvidence, INVENTORY, OntologyMode, OsStr, Path, RecordBatch, SemanticRouteKind, SemanticStorageBindings, Sha256, - StableDirectory, StringArray, SymbolKind, UInt64Array, Uuid, Write, account_cache_release, - adjacency, authenticate_inventory_control, copy_artifact, edge_batch, edge_property_batch, + StableDirectory, StringArray, SymbolKind, UInt64Array, Uuid, Write, adjacency, + authenticate_inventory_control, copy_artifact, edge_batch, edge_property_batch, encoded_route_component, hex, index_artifact, install_json, lanes, node_batch, node_property_batch, remove_encoding_intent, required_string, resolve_owner, select_rows, storage, with_route_metadata_batch, write_surrogate_tails, @@ -40,7 +40,6 @@ use super::{ mod budget; mod csr; mod emit; -mod identities; mod install; mod ordered; mod plan; @@ -60,7 +59,6 @@ pub(crate) use property_rows::ForcedPropertyFrames; use budget::ScratchPlan; use emit::{EdgeEmitter, RelationStats, Semantics}; -use identities::EdgeUuids; use install::Installer; use plan::PassMeter; use scratch::Scratch; @@ -153,46 +151,28 @@ struct ScratchReport { peak_csr_carry_entries: u64, } -struct Membership { - index: crate::uuid_membership::ConstructionIndexEncoding, - v4_artifacts: Vec, - v4_publication: crate::uuid_membership::V4OrdinalPublicationMetrics, - v4_metrics: crate::uuid_membership::V4OrdinalBuildMetrics, +/// The ordinal node-identity facet a build publishes. +struct OrdinalFacet { + artifacts: Vec, + publication: crate::uuid_membership::V4OrdinalPublicationMetrics, + metrics: crate::uuid_membership::V4OrdinalBuildMetrics, } -/// UUID membership index (`identities-v5`, `node-surrogates-v5`) and the v4 -/// ordinal artifacts, both streamed from the ranked arrays. -fn build_membership( +/// The v4 ordinal artifacts, streamed from the ranked node array. +fn build_ordinal_facet( output: &StableDirectory, nodes: &NodeTable, - edge_uuids: EdgeUuids<'_>, - edge_count: u64, generation: u64, cancel: &AtomicBool, -) -> Result { +) -> Result { let mut cancelled = || cancel.load(Ordering::Acquire); - let stream = identities::IdentityStream::new(&nodes.uuids, edge_uuids, edge_count); - let len = stream.byte_len(); - let index = crate::uuid_membership::encode_construction_index( - crate::uuid_membership::ConstructionIdentityInput::Stream { - reader: Box::new(stream), - len, - }, - output.physical(), - generation, - 0, - None, - nodes.uuids.len() as u64, - edge_count, - &mut cancelled, - output.allocation(), - )?; + crate::uuid_membership::clear_private_ordinal_residue(output)?; let membership_dir = output - .open_child_directory(OsStr::new("graph")) + .create_child_directory(OsStr::new("graph")) .map_err(storage)? - .open_child_directory(OsStr::new("topology")) + .create_child_directory(OsStr::new("topology")) .map_err(storage)? - .open_child_directory(OsStr::new("uuid-membership")) + .create_child_directory(OsStr::new("uuid-membership")) .map_err(storage)?; let cache_window = graphforge_filesystem::cache_release_window_for_streams(5).map_err(storage)?; @@ -206,21 +186,19 @@ fn build_membership( writer.push_pair(Uuid::from_bytes(*uuid), position as u64 + 1, &mut cancelled)?; } let bundle = writer.finish()?; - let (v4_artifacts, v4_publication, v4_metrics) = + let (artifacts, publication, metrics) = crate::uuid_membership::publish_v4_construction_artifacts( output.physical(), bundle, generation, - &index.source_sha256, None, &mut cancelled, output.allocation(), )?; - Ok(Membership { - index, - v4_artifacts, - v4_publication, - v4_metrics, + Ok(OrdinalFacet { + artifacts, + publication, + metrics, }) } @@ -555,21 +533,13 @@ pub(crate) fn encode_bulk( passes.extend([meter.finish()]); crate::graph_construction::construction_failpoint("bulk.after_tables"); - // Membership streams the sorted UUIDs; once it has, the edge UUIDs (16 B per + // The ordinal facet streams the sorted node UUIDs; the edge UUIDs (16 B per // edge) are released before the adjacency pass sorts its entries. - let meter = PassMeter::start("membership"); - let edge_uuids = match (&edge_side, &ranked_edges, &scratch) { - (EdgeSide::Scratch(_), Some((_, ranked)), Some(scratch)) => { - EdgeUuids::scratch(scratch, ranked.uuid_files.clone()) - } - (EdgeSide::Memory(edges), _, _) => EdgeUuids::memory(&edges.uuids), - _ => return Err(storage("the over-budget build lost its scratch state")), - }; - let membership = - build_membership(&output, &nodes, edge_uuids, edge_count, generation, &cancel)?; + let meter = PassMeter::start("ordinal"); + let ordinal = build_ordinal_facet(&output, &nodes, generation, &cancel)?; check_cancelled(&cancel)?; passes.extend([meter.finish()]); - crate::graph_construction::construction_failpoint("bulk.after_membership"); + crate::graph_construction::construction_failpoint("bulk.after_ordinal"); if let EdgeSide::Memory(edges) = &mut edge_side { edges.uuids = Vec::new(); } @@ -829,13 +799,13 @@ pub(crate) fn encode_bulk( evidence.adjacency.csr_shards = adjacency.shards; evidence.output_write_bytes += installer.written_bytes(); - let Membership { - index, - v4_artifacts, - v4_publication, - v4_metrics, - } = membership; + let OrdinalFacet { + artifacts: v4_artifacts, + publication: v4_publication, + metrics: v4_metrics, + } = ordinal; evidence.ordinal_records = v4_metrics.input_records; + evidence.edge_records = edge_count; evidence.ordinal_artifact_write_bytes = v4_metrics.artifact_bytes; evidence.ordinal_artifact_write_operations = v4_metrics.write_blocks; evidence.ordinal_ranges = u64::try_from(v4_metrics.ranges).map_err(storage)?; @@ -850,19 +820,7 @@ pub(crate) fn encode_bulk( evidence.ordinal_peak_temporary_bytes = v4_metrics .peak_temporary_bytes .max(v4_publication.peak_temporary_bytes); - evidence.membership_records = index.input_records; - evidence.membership_write_bytes = index.final_write_bytes; - evidence.membership_total_write_bytes = index.write_bytes; - evidence.membership_read_bytes = index.read_bytes; - evidence.membership_read_operations = index.read_operations; - evidence.membership_write_operations = index.write_operations; - evidence.membership_fsync_operations = index.fsync_operations; - evidence.membership_created_runs = index.created_runs; - evidence.membership_peak_buffer_bytes = index.peak_buffer_bytes; - evidence.membership_peak_temporary_bytes = index.peak_temporary_bytes; - account_cache_release(index.cache_release, &mut evidence)?; artifacts.extend(installer.into_artifacts()?); - artifacts.extend(index.artifacts.into_iter().map(index_artifact)); artifacts.extend(v4_artifacts.into_iter().map(index_artifact)); artifacts.sort_unstable_by(|left, right| left.path.cmp(&right.path)); @@ -881,14 +839,13 @@ pub(crate) fn encode_bulk( shape_inputs_sha256: shape.runtime_catalog_inputs_sha256.clone(), shape_authority_sha256: shape_authority_sha256.to_owned(), artifacts, - retained_artifacts: Vec::new(), evidence, invocation: GraphConstructionEncodingInvocationEvidence::default(), }; crate::graph_construction::construction_failpoint("bulk.before_inventory"); install_json(&output, INVENTORY, &completed)?; crate::graph_construction::construction_failpoint("bulk.after_inventory_before_intent_removal"); - authenticate_inventory_control(&completed, None)?; + authenticate_inventory_control(&completed)?; remove_encoding_intent(&output)?; passes.extend([meter.finish()]); let invocation = completed.evidence.clone(); diff --git a/crates/graphforge-storage/src/graph_construction_encoding/bulk/identities.rs b/crates/graphforge-storage/src/graph_construction_encoding/bulk/identities.rs deleted file mode 100644 index 7d9508ee1..000000000 --- a/crates/graphforge-storage/src/graph_construction_encoding/bulk/identities.rs +++ /dev/null @@ -1,144 +0,0 @@ -//! The UUID-ordered identity delta, generated in memory or from scratch. - -use std::io::Read; -use std::path::PathBuf; - -use super::scratch::{BlockReader, Scratch}; - -/// Width of a shaped identity record: UUID, kind, retained marker, surrogate. -const RECORD: usize = crate::construction_record_layout::BASE_IDENTITY_WIDTH; - -/// The sorted edge UUIDs, resident or in per-partition scratch files. -pub(super) enum EdgeUuids<'a> { - Memory { - uuids: &'a [[u8; 16]], - position: usize, - }, - Scratch { - scratch: &'a Scratch, - files: std::vec::IntoIter, - reader: Option>, - block: Vec, - offset: usize, - }, -} - -impl<'a> EdgeUuids<'a> { - pub(super) fn memory(uuids: &'a [[u8; 16]]) -> Self { - Self::Memory { uuids, position: 0 } - } - - pub(super) fn scratch(scratch: &'a Scratch, files: Vec) -> Self { - Self::Scratch { - scratch, - files: files.into_iter(), - reader: None, - block: Vec::new(), - offset: 0, - } - } - - /// The next UUID without consuming it. - fn peek(&mut self) -> std::io::Result> { - match self { - Self::Memory { uuids, position } => Ok(uuids.get(*position).copied()), - Self::Scratch { - scratch, - files, - reader, - block, - offset, - } => loop { - if *offset + 16 <= block.len() { - return Ok(Some( - block[*offset..*offset + 16].try_into().expect("16 bytes"), - )); - } - if let Some(open) = reader { - if open - .next_block(block) - .map_err(|error| std::io::Error::other(error.to_string()))? - { - *offset = 0; - continue; - } - *reader = None; - } - let Some(path) = files.next() else { - return Ok(None); - }; - block.clear(); - *offset = 0; - *reader = Some( - BlockReader::open(scratch, &path) - .map_err(|error| std::io::Error::other(error.to_string()))?, - ); - }, - } - } - - fn advance(&mut self) { - match self { - Self::Memory { position, .. } => *position += 1, - Self::Scratch { offset, .. } => *offset += 16, - } - } -} - -/// Node and edge identities merged by UUID, byte for byte the records the -/// staged shaper writes to its identity run: `uuid | kind | 0 | surrogate` -/// with a big-endian dense rank per kind. -pub(super) struct IdentityStream<'a> { - nodes: &'a [[u8; 16]], - edges: EdgeUuids<'a>, - edge_count: u64, - node: usize, - edge: u64, -} - -impl<'a> IdentityStream<'a> { - pub(super) fn new(nodes: &'a [[u8; 16]], edges: EdgeUuids<'a>, edge_count: u64) -> Self { - Self { - nodes, - edges, - edge_count, - node: 0, - edge: 0, - } - } - - pub(super) fn byte_len(&self) -> u64 { - (self.nodes.len() as u64 + self.edge_count) * RECORD as u64 - } -} - -impl Read for IdentityStream<'_> { - fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { - let mut written = 0; - while buffer.len() - written >= RECORD { - let next_edge = self.edges.peek()?; - let take_node = match (self.nodes.get(self.node), next_edge) { - (None, None) => break, - (Some(_), None) => true, - (None, Some(_)) => false, - (Some(node), Some(edge)) => *node < edge, - }; - let record = &mut buffer[written..written + RECORD]; - if take_node { - record[..16].copy_from_slice(&self.nodes[self.node]); - record[16] = 0; - self.node += 1; - record[18..].copy_from_slice(&(self.node as u64).to_be_bytes()); - } else { - record[..16].copy_from_slice(&next_edge.expect("an edge was peeked")); - record[16] = 1; - self.edges.advance(); - self.edge += 1; - record[18..].copy_from_slice(&self.edge.to_be_bytes()); - } - record[17] = 0; - written += RECORD; - } - Ok(written) - } -} diff --git a/crates/graphforge-storage/src/graph_construction_encoding/bulk/plan.rs b/crates/graphforge-storage/src/graph_construction_encoding/bulk/plan.rs index 400fd28f8..b1bf978e3 100644 --- a/crates/graphforge-storage/src/graph_construction_encoding/bulk/plan.rs +++ b/crates/graphforge-storage/src/graph_construction_encoding/bulk/plan.rs @@ -182,7 +182,7 @@ pub struct BulkBuildReport { /// Edges built. pub edges: u64, /// Per-pass measurements by pass name (`plan`, `nodes`, `edges`, `catalog`, - /// `tables`, `adjacency`, `membership`, `properties`, `finalize`). Keys and + /// `tables`, `ordinal`, `adjacency`, `properties`, `finalize`). Keys and /// values are numeric-only so receipts stay within the certification /// runner's sanitizer. pub passes: std::collections::BTreeMap, diff --git a/crates/graphforge-storage/src/graph_construction_encoding/bulk/scratch_edges.rs b/crates/graphforge-storage/src/graph_construction_encoding/bulk/scratch_edges.rs index 541867284..f52e933aa 100644 --- a/crates/graphforge-storage/src/graph_construction_encoding/bulk/scratch_edges.rs +++ b/crates/graphforge-storage/src/graph_construction_encoding/bulk/scratch_edges.rs @@ -699,8 +699,6 @@ pub(super) struct RankedEdges { pub(super) first_appearance: Vec, /// Edges per relation id. pub(super) counts: Vec, - /// The sorted edge UUIDs, one scratch file per partition, in order. - pub(super) uuid_files: Vec, } pub(super) struct RankContext<'a> { @@ -767,9 +765,6 @@ pub(super) fn rank_partitions(context: &RankContext<'_>) -> Result>(); // One pair of counters per relation, independent of the leaf count. let relation_counts = (0..relation_count) .map(|_| AtomicU64::new(0)) @@ -790,7 +785,6 @@ pub(super) fn rank_partitions(context: &RankContext<'_>) -> Result; let mut run_count = 0_u64; - let mut uuids = Appender::create(scratch, &uuid_files[part], 1 << 20)?; let staging = plan.staging_bytes.saturating_mul(2 * plan.csr_partitions) / (csr.out.len() + csr.inn.len()).max(1); let mut out = Scatter::new(scratch, &csr.out, staging); @@ -818,7 +812,6 @@ pub(super) fn rank_partitions(context: &RankContext<'_>) -> Result) -> Result) -> Result, ) -> Result { let _diagnostic_scope = crate::graph_construction::diagnostics::Scope::start("inventory_authentication"); let evidence = authenticate_inventory_payloads(root, inventory, &mut || false)?; - authenticate_inventory_references(inventory, parent_index)?; Ok(evidence) } /// The control half of [`authenticate_inventory`]: the inventory's structural -/// invariants and its retained-parent references, without reading a payload -/// byte. The encoder runs this after installing the inventory it just wrote. +/// invariants, without reading a payload byte. The encoder runs this after installing the inventory it just wrote. /// /// The payloads are not re-read here. Every artifact digest in the inventory /// was computed by the single pass that wrote the bytes, and the boundary that @@ -38,51 +34,8 @@ pub(crate) fn authenticate_inventory( /// to shape outputs). pub(crate) fn authenticate_inventory_control( inventory: &GraphConstructionEncoding, - parent_index: Option<&AuthenticatedUuidIndexSnapshot>, ) -> Result<(), GfError> { - validate_inventory_invariants(inventory)?; - authenticate_inventory_references(inventory, parent_index) -} - -fn authenticate_inventory_references( - inventory: &GraphConstructionEncoding, - parent_index: Option<&AuthenticatedUuidIndexSnapshot>, -) -> Result<(), GfError> { - if inventory.retained_artifacts.is_empty() { - if inventory.evidence.retained_index_runs != 0 { - return Err(storage("retained-index evidence lacks references")); - } - return Ok(()); - } - let parent = parent_index - .ok_or_else(|| storage("retained artifacts lack authenticated parent snapshot"))?; - let mut previous = None; - let mut references = Vec::with_capacity(inventory.retained_artifacts.len()); - for retained in &inventory.retained_artifacts { - if previous.is_some_and(|value: &str| value >= retained.target_path.as_str()) { - return Err(storage( - "retained artifact targets are not unique and sorted", - )); - } - references.push( - crate::uuid_membership::ConstructionReferenceAuthentication { - source_root: &retained.source_root, - source_root_volume: retained.source_root_volume, - source_root_file_id: &retained.source_root_file_id, - source_path: &retained.source_path, - source_volume: retained.source_volume, - source_file_id: &retained.source_file_id, - target_path: &retained.target_path, - bytes: retained.bytes, - sha256: &retained.sha256, - xxh64: retained.xxh64, - parent_manifest_sha256: &retained.parent_manifest_sha256, - }, - ); - previous = Some(retained.target_path.as_str()); - } - parent.authenticate_construction_references(&references)?; - Ok(()) + validate_inventory_invariants(inventory) } fn validate_inventory_invariants(inventory: &GraphConstructionEncoding) -> Result<(), GfError> { diff --git a/crates/graphforge-storage/src/graph_construction_encoding/tests.rs b/crates/graphforge-storage/src/graph_construction_encoding/tests.rs index 8f0105b2e..5ac0cca25 100644 --- a/crates/graphforge-storage/src/graph_construction_encoding/tests.rs +++ b/crates/graphforge-storage/src/graph_construction_encoding/tests.rs @@ -218,19 +218,6 @@ fn encoded_inventory_version_precedes_required_checksums_and_current_wire_is_str shape_inputs_sha256: "b".repeat(64), shape_authority_sha256: "c".repeat(64), artifacts: vec![artifact], - retained_artifacts: vec![ConstructionRetainedArtifact { - source_root: "parent".into(), - source_root_volume: 1, - source_root_file_id: "0".repeat(32), - source_path: "object".into(), - source_volume: 1, - source_file_id: "0".repeat(32), - target_path: "target".into(), - bytes: 2, - sha256: "d".repeat(64), - xxh64: crate::corruption_checksum::checksum(b"{}"), - parent_manifest_sha256: "e".repeat(64), - }], evidence: GraphConstructionEncodingEvidence::default(), invocation: GraphConstructionEncodingInvocationEvidence::default(), }; @@ -255,7 +242,7 @@ fn encoded_inventory_version_precedes_required_checksums_and_current_wire_is_str "{error}" ); } - for collection in ["artifacts", "retained_artifacts"] { + for collection in ["artifacts"] { for invalid in [ None, Some(""), diff --git a/crates/graphforge-storage/src/label_membership_counts.rs b/crates/graphforge-storage/src/label_membership_counts.rs index a4f425c89..671f37af5 100644 --- a/crates/graphforge-storage/src/label_membership_counts.rs +++ b/crates/graphforge-storage/src/label_membership_counts.rs @@ -131,8 +131,8 @@ pub fn establish_label_membership_counts( )) } -/// Read labels for named nodes through the authenticated UUID-to-surrogate -/// index, decoding only rows selected for those node IDs. +/// Read labels for named nodes through the topology Parquet's UUID-to-surrogate +/// pairs, decoding only rows selected for those node IDs. pub fn read_node_labels_for_uuids( dir: &Path, files: &crate::TopologyFiles, @@ -145,7 +145,8 @@ pub fn read_node_labels_for_uuids( .iter() .map(|bytes| graphforge_core::uuid::Uuid::from_bytes(*bytes)) .collect::>(); - let mut index = crate::UuidMembershipIndex::open(dir)?; + let mut index = + crate::TopologyIdentityProbe::open(dir, files, crate::read_topology_generation(dir)?)?; let (surrogates, _) = index.lookup_node_surrogates(&uuids)?; let mut node_ids = std::collections::HashSet::new(); for (uuid, surrogate) in uuids.iter().zip(surrogates) { diff --git a/crates/graphforge-storage/src/lib.rs b/crates/graphforge-storage/src/lib.rs index b2f322dab..f192da828 100644 --- a/crates/graphforge-storage/src/lib.rs +++ b/crates/graphforge-storage/src/lib.rs @@ -77,11 +77,10 @@ pub use graph_construction::cpu_admission::{ConstructionCpuAdmission, Constructi pub use graph_construction::{ BulkBatchReader, BulkBuildPlan, BulkBuildReport, BulkPassReport, BulkRoute, BulkSource, BulkStagedReason, CONSTRUCTION_EDGE_SCHEMA, CONSTRUCTION_NODE_SCHEMA, ConstructionChunkKind, - ConstructionChunkReceipt, ConstructionRetainedArtifact, ConstructionSemanticAuthority, - ConstructionShape, GRAPH_CONSTRUCTION_ENCODING_BUFFER_BYTES, GraphConstructionBudgets, - GraphConstructionEncoding, GraphConstructionEncodingEvidence, - GraphConstructionEncodingInvocationEvidence, GraphConstructionEvidence, - GraphConstructionSession, GraphConstructionState, SealRoute, + ConstructionChunkReceipt, ConstructionSemanticAuthority, ConstructionShape, + GRAPH_CONSTRUCTION_ENCODING_BUFFER_BYTES, GraphConstructionBudgets, GraphConstructionEncoding, + GraphConstructionEncodingEvidence, GraphConstructionEncodingInvocationEvidence, + GraphConstructionEvidence, GraphConstructionSession, GraphConstructionState, SealRoute, }; pub mod graph_admission; @@ -451,13 +450,15 @@ pub use lifecycle_io::{ pub mod uuid_membership; pub use uuid_membership::{ - AuthenticatedUuidIndexSnapshot, UuidIndexAppendMetrics, UuidIndexBuildLimits, - UuidIndexBuildMetrics, UuidIndexKind, UuidIndexOrphanGcWork, UuidMembershipIndex, - UuidProbeMetrics, V4OrdinalRebuildDisposition, V4OrdinalRebuildEvidence, - maintain_uuid_membership_orphans, rebuild_uuid_membership_indexes, - rebuild_uuid_membership_indexes_with_topology, rebuild_v4_ordinal_identity, - rebuild_v4_ordinal_identity_with_evidence, uuid_membership_index_is_fresh, - uuid_membership_index_present, + UuidIndexBuildLimits, UuidIndexBuildMetrics, UuidIndexOrphanGcWork, + V4OrdinalRebuildDisposition, V4OrdinalRebuildEvidence, maintain_uuid_membership_orphans, + rebuild_v4_ordinal_identity, rebuild_v4_ordinal_identity_with_evidence, +}; + +pub mod topology_identity; +pub use topology_identity::{ + DELETED_IDENTITIES_PATH, TopologyIdentityProbe, UuidIndexKind, UuidProbeMetrics, + read_deleted_identities, stage_deleted_identities, }; pub mod ordinal_identity_v4; diff --git a/crates/graphforge-storage/src/mutator.rs b/crates/graphforge-storage/src/mutator.rs index e1c90fa27..f10b4e465 100644 --- a/crates/graphforge-storage/src/mutator.rs +++ b/crates/graphforge-storage/src/mutator.rs @@ -735,26 +735,25 @@ pub fn delete_nodes( dir: &Path, node_uuids: &HashSet<[u8; 16], S>, ) -> Result { - crate::uuid_membership::ensure_uuid_membership_migrated(dir)?; let mut staged = RewriteBatch::new(); let removed = stage_delete_nodes(&mut staged, dir, node_uuids)?; - let mut snapshot = None; - if let Some(g) = - committed_uuid_generation(crate::uuid_membership::commit_uuid_topology_rewrite( - dir, - staged, - &crate::uuid_membership::UuidTopologyDelta { - nodes: Vec::new(), - edges: Vec::new(), - deleted_nodes: if removed == 0 { - Vec::new() - } else { - node_uuids.iter().copied().map(Uuid::from_bytes).collect() - }, - deleted_edges: Vec::new(), + let mut probe = None; + if let Some(g) = crate::uuid_membership::commit_uuid_topology_rewrite( + dir, + staged, + &crate::uuid_membership::UuidTopologyDelta { + nodes: Vec::new(), + edges: Vec::new(), + deleted_nodes: if removed == 0 { + Vec::new() + } else { + node_uuids.iter().copied().map(Uuid::from_bytes).collect() }, - &mut snapshot, - )?)? + deleted_edges: Vec::new(), + }, + &mut probe, + )? + .generation() { crate::adjacency_delta::discard_segment(dir, g); // delete writes no segment } @@ -774,26 +773,25 @@ pub fn delete_edges( dir: &Path, edge_uuids: &HashSet<[u8; 16], S>, ) -> Result { - crate::uuid_membership::ensure_uuid_membership_migrated(dir)?; let mut staged = RewriteBatch::new(); let removed = stage_delete_edges(&mut staged, dir, edge_uuids)?; - let mut snapshot = None; - if let Some(g) = - committed_uuid_generation(crate::uuid_membership::commit_uuid_topology_rewrite( - dir, - staged, - &crate::uuid_membership::UuidTopologyDelta { - nodes: Vec::new(), - edges: Vec::new(), - deleted_nodes: Vec::new(), - deleted_edges: if removed == 0 { - Vec::new() - } else { - edge_uuids.iter().copied().map(Uuid::from_bytes).collect() - }, + let mut probe = None; + if let Some(g) = crate::uuid_membership::commit_uuid_topology_rewrite( + dir, + staged, + &crate::uuid_membership::UuidTopologyDelta { + nodes: Vec::new(), + edges: Vec::new(), + deleted_nodes: Vec::new(), + deleted_edges: if removed == 0 { + Vec::new() + } else { + edge_uuids.iter().copied().map(Uuid::from_bytes).collect() }, - &mut snapshot, - )?)? + }, + &mut probe, + )? + .generation() { crate::adjacency_delta::discard_segment(dir, g); // delete writes no segment } @@ -828,58 +826,39 @@ pub fn delete_nodes_and_edges_with_topology( edge_uuids: &HashSet<[u8; 16], S>, topology: Option>, ) -> Result<(u64, u64), GfError> { - crate::uuid_membership::ensure_uuid_membership_migrated_with_topology(dir, topology.clone())?; let mut staged = RewriteBatch::new(); if let Some(topology) = topology { staged.bind_topology_authority(topology)?; } let edges_removed = stage_delete_edges(&mut staged, dir, edge_uuids)?; let nodes_removed = stage_delete_nodes(&mut staged, dir, node_uuids)?; - let mut snapshot = None; - if let Some(g) = - committed_uuid_generation(crate::uuid_membership::commit_uuid_topology_rewrite( - dir, - staged, - &crate::uuid_membership::UuidTopologyDelta { - nodes: Vec::new(), - edges: Vec::new(), - deleted_nodes: if nodes_removed == 0 { - Vec::new() - } else { - node_uuids.iter().copied().map(Uuid::from_bytes).collect() - }, - deleted_edges: if edges_removed == 0 { - Vec::new() - } else { - edge_uuids.iter().copied().map(Uuid::from_bytes).collect() - }, + let mut probe = None; + if let Some(g) = crate::uuid_membership::commit_uuid_topology_rewrite( + dir, + staged, + &crate::uuid_membership::UuidTopologyDelta { + nodes: Vec::new(), + edges: Vec::new(), + deleted_nodes: if nodes_removed == 0 { + Vec::new() + } else { + node_uuids.iter().copied().map(Uuid::from_bytes).collect() }, - &mut snapshot, - )?)? + deleted_edges: if edges_removed == 0 { + Vec::new() + } else { + edge_uuids.iter().copied().map(Uuid::from_bytes).collect() + }, + }, + &mut probe, + )? + .generation() { crate::adjacency_delta::discard_segment(dir, g); // delete writes no segment } Ok((nodes_removed, edges_removed)) } -fn committed_uuid_generation( - outcome: crate::uuid_membership::CommittedUuidTopologyRewrite, -) -> Result, GfError> { - match outcome { - crate::uuid_membership::CommittedUuidTopologyRewrite::NoTopologyChange => Ok(None), - crate::uuid_membership::CommittedUuidTopologyRewrite::Committed { generation, .. } => { - Ok(Some(generation)) - } - crate::uuid_membership::CommittedUuidTopologyRewrite::CommittedNeedsRefresh { - generation, - error, - .. - } => Err(GfError::Storage(format!( - "topology generation {generation} committed but UUID index snapshot refresh failed: {error}" - ))), - } -} - /// Return the `edge_uuid`s of every edge incident to any of `node_uuids` /// (as `src` or `dst`), across all edge files. /// diff --git a/crates/graphforge-storage/src/ordinal_identity_v4.rs b/crates/graphforge-storage/src/ordinal_identity_v4.rs index 0d3e609ca..94c36ceb8 100644 --- a/crates/graphforge-storage/src/ordinal_identity_v4.rs +++ b/crates/graphforge-storage/src/ordinal_identity_v4.rs @@ -834,7 +834,7 @@ impl V4OrdinalIdentityHandle { /// fails authenticated open; discovery never falls back around it. pub fn discover( project_dir: &Path, - topology_generation: u64, + _topology_generation: u64, ) -> Result { let root = StableDirectory::open(&project_dir.join(INDEX_DIR)).map_err(io_error)?; match root.open_child_file(MANIFEST_NAME.as_ref()) { @@ -842,13 +842,7 @@ impl V4OrdinalIdentityHandle { Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(error) => return Err(io_error(error)), } - if crate::UuidMembershipIndex::open_at_generation(project_dir, topology_generation).is_err() - { - return Err(V4OrdinalIdentityError::InvalidDescriptor( - "current v3 authority failed authentication", - )); - } - Ok(V4OrdinalIdentityDiscovery::RebuildRequired { found_version: 3 }) + Ok(V4OrdinalIdentityDiscovery::RebuildRequired { found_version: 0 }) } /// Open one immutable v4 generation without reading artifact bytes. diff --git a/crates/graphforge-storage/src/ordinal_identity_v4/tests.rs b/crates/graphforge-storage/src/ordinal_identity_v4/tests.rs index 556975806..4346e3206 100644 --- a/crates/graphforge-storage/src/ordinal_identity_v4/tests.rs +++ b/crates/graphforge-storage/src/ordinal_identity_v4/tests.rs @@ -356,23 +356,19 @@ fn generated_lookup_orders_preserve_identity_and_linear_bounds() { } #[test] -fn absent_v4_classifies_valid_v3_and_present_malformed_v4_never_falls_back() { +fn absent_v4_requires_rebuild_and_present_malformed_v4_never_falls_back() { let (root, _, _) = crate::uuid_membership::tests::fixture(); fs::write( root.path().join("topology/generation.json"), b"{\"topology_generation\":7,\"search_generation\":0,\"property_generation\":0}\n", ) .unwrap(); - crate::rebuild_uuid_membership_indexes(root.path(), crate::UuidIndexBuildLimits::default()) - .unwrap(); let index = root.path().join(INDEX_DIR); fs::write(index.join(LOCK_NAME), []).unwrap(); let v4_path = index.join(MANIFEST_NAME); - let v3_path = index.join("manifest.json"); - let v3 = fs::read(&v3_path).unwrap(); assert!(matches!( V4OrdinalIdentityHandle::discover(root.path(), 7).unwrap(), - V4OrdinalIdentityDiscovery::RebuildRequired { found_version: 3 } + V4OrdinalIdentityDiscovery::RebuildRequired { found_version: 0 } )); assert!(matches!( V4OrdinalIdentityHandle::open( @@ -385,23 +381,12 @@ fn absent_v4_classifies_valid_v3_and_present_malformed_v4_never_falls_back() { ), Err(V4OrdinalIdentityError::Io) )); - fs::remove_file(&v3_path).unwrap(); + // Files of a membership index that predates #1902 neither satisfy nor + // disturb discovery. + fs::write(index.join("manifest.json"), b"legacy").unwrap(); assert!(matches!( - V4OrdinalIdentityHandle::discover(root.path(), 7), - Err(V4OrdinalIdentityError::InvalidDescriptor(_)) - )); - fs::write(&v3_path, &v3).unwrap(); - let v3_manifest: serde_json::Value = serde_json::from_slice(&v3).unwrap(); - let run_name = v3_manifest["runs"][0]["identities"]["name"] - .as_str() - .unwrap(); - let run_path = index.join(run_name); - let mut run = fs::read(&run_path).unwrap(); - run[0] ^= 1; - fs::write(&run_path, run).unwrap(); - assert!(matches!( - V4OrdinalIdentityHandle::discover(root.path(), 7), - Err(V4OrdinalIdentityError::InvalidDescriptor(_)) + V4OrdinalIdentityHandle::discover(root.path(), 7).unwrap(), + V4OrdinalIdentityDiscovery::RebuildRequired { found_version: 0 } )); fs::write(&v4_path, b"{\"format_version\":4}").unwrap(); diff --git a/crates/graphforge-storage/src/project_generation.rs b/crates/graphforge-storage/src/project_generation.rs index 127638bea..91d460eb4 100644 --- a/crates/graphforge-storage/src/project_generation.rs +++ b/crates/graphforge-storage/src/project_generation.rs @@ -227,20 +227,6 @@ impl ResolvedProjectGeneration { Ok(Some(pinned)) } - pub(crate) fn authenticated_graph_file_bytes_with_state( - &self, - relative_path: &str, - maximum: u64, - targeted_state: Option<&mut crate::graph_manifest::GraphManifestTargetedState>, - ) -> Result)>, GfError> { - self.authenticated_graph_file_bytes_counted( - relative_path, - maximum, - targeted_state, - &mut crate::GraphObjectIoTotals::default(), - ) - } - pub(crate) fn authenticated_graph_file_bytes_counted( &self, relative_path: &str, @@ -2369,8 +2355,6 @@ mod tests { if include_v4 && omit_receipt { fs::remove_file(index.join("ordinal-v4-receipt.json")).unwrap(); } - crate::rebuild_uuid_membership_indexes(&graph, crate::UuidIndexBuildLimits::default()) - .unwrap(); let (inventory, expanded) = crate::capture_graph_files(&graph).unwrap(); let participant = if compact { let lease = crate::begin_graph_object_publication(root).unwrap(); @@ -2588,7 +2572,7 @@ mod tests { } #[test] - fn public_orphan_maintenance_rejects_selected_v3_manifest_substitution() { + fn public_orphan_maintenance_ignores_a_legacy_membership_manifest() { let root = tempfile::tempdir().unwrap(); fs::write(root.path().join(FORMAT_FILE), PROJECT_FORMAT_BYTES).unwrap(); fs::create_dir(root.path().join("generations")).unwrap(); @@ -2603,7 +2587,10 @@ mod tests { .graph_tree_root() .join("topology/uuid-membership/manifest.json"); fs::write(&manifest, b"substituted").unwrap(); - assert!(crate::maintain_uuid_membership_orphans(&selected.graph_tree_root(), 16).is_err()); + // The membership index is not authority any more: whatever a legacy + // project left in its manifest is neither read nor collected. + crate::maintain_uuid_membership_orphans(&selected.graph_tree_root(), 16).unwrap(); + assert_eq!(fs::read(&manifest).unwrap(), b"substituted"); } #[cfg(unix)] diff --git a/crates/graphforge-storage/src/runtime_entity_labels.rs b/crates/graphforge-storage/src/runtime_entity_labels.rs index f4894470e..d6bbd22cf 100644 --- a/crates/graphforge-storage/src/runtime_entity_labels.rs +++ b/crates/graphforge-storage/src/runtime_entity_labels.rs @@ -1381,7 +1381,6 @@ migrations: [] assert_eq!(outcome.remapped_label_values, 4); assert!(outcome.encoding_marked); assert!(has_runtime_entity_label_encoding_marker(dir.path())); - assert!(crate::uuid_membership_index_is_fresh(dir.path()).unwrap()); // Reopening and probing the generation-carried UUID authority must not // decode the topology that reconciliation just rewrote. @@ -1412,7 +1411,7 @@ migrations: [] } #[test] - fn uuid_neutral_label_rewrite_recovers_after_committed_refresh_failure() { + fn uuid_neutral_label_rewrites_advance_one_generation_each() { let dir = TempDir::new().unwrap(); write_nodes(dir.path(), &[&[0], &[1]]); let mut catalog = RuntimeCatalog::new(); @@ -1435,23 +1434,14 @@ migrations: [] }; let before = crate::read_topology_generation(dir.path()).unwrap(); - crate::uuid_membership::fail_next_snapshot_refresh_for_test(); - let error = crate::uuid_membership::commit_uuid_neutral_topology_rewrite( - dir.path(), - stage_current(), - ) - .unwrap_err(); - assert!( - error - .to_string() - .contains("committed but UUID index snapshot refresh failed") - ); assert_eq!( - crate::read_topology_generation(dir.path()).unwrap(), - before + 1 + crate::uuid_membership::commit_uuid_neutral_topology_rewrite( + dir.path(), + stage_current(), + ) + .unwrap(), + Some(before + 1) ); - assert!(crate::uuid_membership_index_is_fresh(dir.path()).unwrap()); - assert_eq!( crate::uuid_membership::commit_uuid_neutral_topology_rewrite( dir.path(), @@ -1460,9 +1450,8 @@ migrations: [] .unwrap(), Some(before + 2) ); - assert!(crate::uuid_membership_index_is_fresh(dir.path()).unwrap()); assert_eq!( - crate::UuidMembershipIndex::open(dir.path()) + crate::TopologyIdentityProbe::open_dir(dir.path()) .unwrap() .count(crate::UuidIndexKind::Node), 2 diff --git a/crates/graphforge-storage/src/topology_identity.rs b/crates/graphforge-storage/src/topology_identity.rs new file mode 100644 index 000000000..4784f60ec --- /dev/null +++ b/crates/graphforge-storage/src/topology_identity.rs @@ -0,0 +1,944 @@ +//! Identity checks answered from the published topology Parquet (#1902). +//! +//! The canonical node and edge files already carry every live UUID, so an +//! append asks them directly instead of consulting a derived index. A probe +//! prunes row groups by their `node_uuid`/`edge_uuid` min/max statistics, then +//! prunes the pages of the surviving row groups by the Parquet column index, +//! decodes only the selected pages of the UUID (and, for nodes, `node_id`) +//! column, and binary-searches the sorted candidates against each decoded +//! value. A file without a page index is pruned by row group alone. +//! +//! Deleted entities are not rows any more, yet their UUIDs are never reusable. +//! `topology/deleted_identities.parquet` records them: a sorted, unique +//! `FixedSizeBinary(16)` column carried forward by each generation that +//! deletes, so its size follows deletions and not graph size. + +use arrow::array::{Array, FixedSizeBinaryArray, RecordBatch, UInt64Array}; +use arrow::datatypes::{DataType, Field, Schema}; +use graphforge_core::GfError; +use parquet::arrow::ProjectionMask; +use parquet::arrow::arrow_reader::{ + ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelection, +}; +use parquet::file::metadata::PageIndexPolicy; +use rayon::prelude::*; +use std::collections::HashMap; +use std::ops::Range; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, OnceLock}; +use uuid::Uuid; + +/// Relative path of the record of deleted entity UUIDs. +pub const DELETED_IDENTITIES_PATH: &str = "topology/deleted_identities.parquet"; + +const DELETED_IDENTITIES_COLUMN: &str = "uuid"; +const READ_BATCH_ROWS: usize = 16 * 1024; +/// Footers retained process-wide. A footer is a few hundred bytes per row +/// group, so this bounds the cache to tens of megabytes. +const FOOTER_CACHE_ENTRIES: usize = 1 << 17; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +/// Selects the canonical identity domain to probe. +pub enum UuidIndexKind { + /// Canonical node UUIDs. + Node, + /// Canonical edge UUIDs. + Edge, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +/// Aggregate-only probe evidence; it never contains graph identities. +pub struct UuidProbeMetrics { + /// Total requested identities, including duplicates. + pub requested: u64, + /// Distinct requested identities. + pub unique_requested: u64, + /// Distinct identities found. + pub found: u64, + /// Row groups decoded. Each is one positioned read of the identity + /// column (and the surrogate column for node lookups). + pub file_seeks: u64, + /// Row groups decoded after min/max pruning. + pub identity_blocks_read: u64, + /// Compressed bytes of the identity (and surrogate) column pages decoded, + /// with the dictionary pages of the chunks they belong to. + pub identity_bytes_read: u64, + /// Identity-column pages in every row group of every fragment considered, + /// whether or not pruning kept them. A row group of a file without a page + /// index counts as one page. + pub pages_considered: u64, + /// Identity-column pages decoded. `pages_considered - pages_read` is what + /// the row-group and page-index pruning avoided. + pub pages_read: u64, + /// Always zero: node surrogates are read with their UUIDs. + pub surrogate_blocks_read: u64, + /// Always zero: node surrogates are read with their UUIDs. + pub surrogate_bytes_read: u64, + /// Fragments whose row groups were considered for pruning. + pub runs_considered: u64, + /// Always zero: a probe never seeks per requested record. + pub per_record_seeks: u64, +} + +impl UuidProbeMetrics { + pub(crate) fn absorb(&mut self, other: &Self) { + self.requested += other.requested; + self.unique_requested += other.unique_requested; + self.found += other.found; + self.file_seeks += other.file_seeks; + self.identity_blocks_read += other.identity_blocks_read; + self.identity_bytes_read += other.identity_bytes_read; + self.pages_considered += other.pages_considered; + self.pages_read += other.pages_read; + self.runs_considered += other.runs_considered; + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +struct FileKey { + device: u64, + inode: u64, + length: u64, + modified_nanos: i128, +} + +/// One data page of the identity column, from the column and offset indexes. +struct PageBounds { + /// First row of the page, relative to its row group. + first_row: usize, + /// Inclusive bounds of the page's non-null UUIDs, when recorded. + bounds: Option<([u8; 16], [u8; 16])>, +} + +/// The pages of one decoded column chunk, for byte accounting. +struct ChunkPages { + /// Bytes of the dictionary page that precedes the data pages, if any. + dictionary_bytes: u64, + /// `(first row, compressed page bytes including the header)` per page. + pages: Vec<(usize, u64)>, +} + +struct RowGroupBounds { + /// Inclusive bounds of the non-null UUIDs, when the writer recorded them. + bounds: Option<([u8; 16], [u8; 16])>, + rows: usize, + /// Pages of the identity column. Empty when the file has no usable page + /// index, in which case the group is one page. + pages: Vec, + /// Page layout of each decoded column chunk (identity, then surrogate). + /// Empty when the file has no usable offset index. + chunks: Vec, + /// Compressed bytes of the columns a probe decodes, whole. + probe_bytes: u64, +} + +/// What one row group contributes to a probe. +struct GroupPlan { + fragment: usize, + group: usize, + /// Rows to decode, or `None` for the whole group. + rows: Option>>, + pages: u64, + bytes: u64, +} + +/// The part of a [`GroupPlan`] a fragment decides by itself. +struct GroupSelection { + rows: Option>>, + pages: u64, + bytes: u64, +} + +#[derive(Clone)] +struct Fragment { + key: FileKey, + path: PathBuf, + metadata: ArrowReaderMetadata, + uuid_leaf: usize, + id_leaf: Option, + groups: Arc>, + rows: u64, +} + +fn footer_cache() -> &'static Mutex>> { + static CACHE: OnceLock>>> = OnceLock::new(); + CACHE.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn file_key(path: &Path) -> Result { + use std::os::unix::fs::MetadataExt; + let metadata = std::fs::symlink_metadata(path) + .map_err(|error| GfError::Storage(format!("topology fragment: {error}")))?; + Ok(FileKey { + device: metadata.dev(), + inode: metadata.ino(), + length: metadata.len(), + modified_nanos: i128::from(metadata.mtime()) * 1_000_000_000 + + i128::from(metadata.mtime_nsec()), + }) +} + +fn storage(message: impl std::fmt::Display) -> GfError { + GfError::Storage(format!("topology identity: {message}")) +} + +fn leaf_index(descriptor: &parquet::schema::types::SchemaDescriptor, name: &str) -> Option { + descriptor + .columns() + .iter() + .position(|column| column.path().string() == name) +} + +/// The identity column's pages in row group `group`, when the column and offset +/// indexes agree on how many there are and every page starts where the +/// previous one ended. +fn page_bounds( + metadata: &parquet::file::metadata::ParquetMetaData, + group: usize, + leaf: usize, + rows: usize, +) -> Option> { + use parquet::file::page_index::column_index::ColumnIndexMetaData; + + let ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(index) = + metadata.column_index()?.get(group)?.get(leaf)? + else { + return None; + }; + let locations = metadata + .offset_index()? + .get(group)? + .get(leaf)? + .page_locations(); + if locations.is_empty() || usize::try_from(index.num_pages()).ok()? != locations.len() { + return None; + } + let mut pages = Vec::with_capacity(locations.len()); + for (page, location) in locations.iter().enumerate() { + let first_row = usize::try_from(location.first_row_index).ok()?; + if first_row >= rows + || pages + .last() + .is_some_and(|previous: &PageBounds| previous.first_row >= first_row) + { + return None; + } + let bounds = match (index.min_value(page), index.max_value(page)) { + (Some(min), Some(max)) if !index.is_null_page(page) => { + Some((min.try_into().ok()?, max.try_into().ok()?)) + } + _ => None, + }; + pages.push(PageBounds { first_row, bounds }); + } + (pages.first()?.first_row == 0).then_some(pages) +} + +/// Page layout of each decoded column chunk of row group `group`. +fn chunk_pages( + metadata: &parquet::file::metadata::ParquetMetaData, + group: usize, + leaves: &[usize], + rows: usize, +) -> Option> { + let offsets = metadata.offset_index()?.get(group)?; + let row_group = metadata.row_group(group); + leaves + .iter() + .map(|leaf| { + let locations = offsets.get(*leaf)?.page_locations(); + let first = locations.first()?; + let dictionary_bytes = row_group + .column(*leaf) + .dictionary_page_offset() + .map_or(Some(0), |dictionary| { + u64::try_from(first.offset - dictionary).ok() + })?; + let mut pages = Vec::with_capacity(locations.len()); + for location in locations { + let first_row = usize::try_from(location.first_row_index).ok()?; + if first_row >= rows { + return None; + } + pages.push(( + first_row, + u64::try_from(location.compressed_page_size).ok()?, + )); + } + Some(ChunkPages { + dictionary_bytes, + pages, + }) + }) + .collect() +} + +fn load_fragment( + path: &Path, + uuid_column: &str, + id_column: Option<&str>, +) -> Result, GfError> { + let key = file_key(path)?; + if let Some(found) = footer_cache() + .lock() + .map_err(|_| storage("footer cache poisoned"))? + .get(&key) + && found.uuid_column_is(uuid_column) + { + // One object is hard-linked under many workspace paths, and a footer + // read through one describes them all. The reader must open the path + // it was asked for: the first workspace may be gone. + return Ok(if found.path == path { + Arc::clone(found) + } else { + Arc::new(Fragment { + path: path.to_path_buf(), + ..Fragment::clone(found) + }) + }); + } + // Optional: a file without a page index is still probed, by row group. + let builder = crate::catalog::admitted_parquet_with_options( + path, + ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Optional), + ) + .map_err(|error| storage(format!("{}: {error}", path.display())))?; + let parquet_metadata = Arc::clone(builder.metadata()); + let descriptor = parquet_metadata.file_metadata().schema_descr(); + let uuid_leaf = leaf_index(descriptor, uuid_column) + .ok_or_else(|| storage(format!("{} lacks {uuid_column}", path.display())))?; + let id_leaf = match id_column { + Some(name) => Some( + leaf_index(descriptor, name) + .ok_or_else(|| storage(format!("{} lacks {name}", path.display())))?, + ), + None => None, + }; + let mut groups = Vec::with_capacity(parquet_metadata.num_row_groups()); + let mut rows = 0_u64; + let mut leaves = vec![uuid_leaf]; + leaves.extend(id_leaf); + for (index, group) in parquet_metadata.row_groups().iter().enumerate() { + let group_rows = usize::try_from(group.num_rows()).unwrap_or(0); + rows += group_rows as u64; + let column = group.column(uuid_leaf); + let bounds = column.statistics().and_then(|statistics| { + let min: [u8; 16] = statistics.min_bytes_opt()?.try_into().ok()?; + let max: [u8; 16] = statistics.max_bytes_opt()?.try_into().ok()?; + Some((min, max)) + }); + let probe_bytes = leaves + .iter() + .map(|leaf| u64::try_from(group.column(*leaf).compressed_size()).unwrap_or(0)) + .sum(); + groups.push(RowGroupBounds { + bounds, + rows: group_rows, + pages: page_bounds(&parquet_metadata, index, uuid_leaf, group_rows).unwrap_or_default(), + chunks: chunk_pages(&parquet_metadata, index, &leaves, group_rows).unwrap_or_default(), + probe_bytes, + }); + } + let metadata = ArrowReaderMetadata::try_new(parquet_metadata, ArrowReaderOptions::new()) + .map_err(storage)?; + let fragment = Arc::new(Fragment { + key, + path: path.to_path_buf(), + metadata, + uuid_leaf, + id_leaf, + groups: Arc::new(groups), + rows, + }); + let mut cache = footer_cache() + .lock() + .map_err(|_| storage("footer cache poisoned"))?; + if cache.len() >= FOOTER_CACHE_ENTRIES { + cache.clear(); + } + cache.insert(key, Arc::clone(&fragment)); + Ok(fragment) +} + +impl Fragment { + fn uuid_column_is(&self, name: &str) -> bool { + self.metadata + .parquet_schema() + .columns() + .get(self.uuid_leaf) + .is_some_and(|column| column.path().string() == name) + } + + /// Whether any of the sorted `candidates` lies within `bounds`. + fn range_may_hold(bounds: Option<([u8; 16], [u8; 16])>, candidates: &[[u8; 16]]) -> bool { + let Some((min, max)) = bounds else { + return true; + }; + let first = candidates.partition_point(|candidate| *candidate < min); + first < candidates.len() && candidates[first] <= max + } + + /// Identity-column pages of row group `group`; one when there is no index. + fn page_count(&self, group: usize) -> u64 { + (self.groups[group].pages.len() as u64).max(1) + } + + /// The rows of row group `group` that may hold any of the sorted + /// `candidates`: `None` when none can, otherwise the page-aligned row + /// ranges, the pages they cover and the compressed bytes decoding them + /// reads. + fn plan_group(&self, group: usize, candidates: &[[u8; 16]]) -> Option { + let bounds = &self.groups[group]; + if !Self::range_may_hold(bounds.bounds, candidates) { + return None; + } + if bounds.pages.is_empty() { + return Some(GroupSelection { + rows: None, + pages: 1, + bytes: bounds.probe_bytes, + }); + } + let mut ranges: Vec> = Vec::new(); + let mut selected = 0_u64; + for (index, page) in bounds.pages.iter().enumerate() { + if !Self::range_may_hold(page.bounds, candidates) { + continue; + } + selected += 1; + let end = bounds + .pages + .get(index + 1) + .map_or(bounds.rows, |next| next.first_row); + match ranges.last_mut() { + Some(last) if last.end == page.first_row => last.end = end, + _ => ranges.push(page.first_row..end), + } + } + if ranges.is_empty() { + return None; + } + let bytes = if bounds.chunks.is_empty() { + bounds.probe_bytes + } else { + bounds + .chunks + .iter() + .map(|chunk| chunk.bytes_for(&ranges, bounds.rows)) + .sum() + }; + let whole = ranges.len() == 1 && ranges[0] == (0..bounds.rows); + Some(GroupSelection { + rows: if whole { None } else { Some(ranges) }, + pages: selected, + bytes, + }) + } + + /// Decode `rows` of row group `group` (all of it when `None`), reporting + /// `(candidate index, surrogate)` for every row whose UUID is a candidate. + fn scan_group( + &self, + group: usize, + rows: Option<&[Range]>, + candidates: &[[u8; 16]], + found: &mut Vec<(usize, u64)>, + ) -> Result<(), GfError> { + let file = std::fs::File::open(&self.path) + .map_err(|error| storage(format!("{}: {error}", self.path.display())))?; + let file = crate::catalog::admitted_path_file(file) + .map_err(|error| storage(format!("{}: {error}", self.path.display())))?; + let mut leaves = vec![self.uuid_leaf]; + leaves.extend(self.id_leaf); + let mask = ProjectionMask::leaves(self.metadata.parquet_schema(), leaves); + let mut builder = + ParquetRecordBatchReaderBuilder::new_with_metadata(file, self.metadata.clone()) + .with_row_groups(vec![group]) + .with_projection(mask) + .with_batch_size(READ_BATCH_ROWS); + if let Some(rows) = rows { + builder = builder.with_row_selection(RowSelection::from_consecutive_ranges( + rows.iter().cloned(), + self.groups[group].rows, + )); + } + let reader = builder + .build() + .map_err(|error| storage(format!("{}: {error}", self.path.display())))?; + for batch in reader { + let batch = + batch.map_err(|error| storage(format!("{}: {error}", self.path.display())))?; + collect_matches(&batch, self.id_leaf.is_some(), candidates, found)?; + } + Ok(()) + } +} + +impl ChunkPages { + /// Compressed bytes of the pages overlapping the sorted, disjoint `rows`, + /// and the chunk's dictionary page when any page is read. + fn bytes_for(&self, rows: &[Range], group_rows: usize) -> u64 { + let mut total = 0_u64; + let mut read_any = false; + for (index, (first_row, bytes)) in self.pages.iter().enumerate() { + let end = self + .pages + .get(index + 1) + .map_or(group_rows, |(next, _)| *next); + let start = rows.partition_point(|range| range.end <= *first_row); + if rows.get(start).is_some_and(|range| range.start < end) { + total += bytes; + read_any = true; + } + } + total + if read_any { self.dictionary_bytes } else { 0 } + } +} + +fn collect_matches( + batch: &RecordBatch, + with_ids: bool, + candidates: &[[u8; 16]], + found: &mut Vec<(usize, u64)>, +) -> Result<(), GfError> { + let uuids = batch + .column(0) + .as_any() + .downcast_ref::() + .filter(|array| array.value_length() == 16) + .ok_or_else(|| storage("identity column is not FixedSizeBinary(16)"))?; + let ids = if with_ids { + Some( + batch + .column(1) + .as_any() + .downcast_ref::() + .ok_or_else(|| storage("surrogate column is not UInt64"))?, + ) + } else { + None + }; + for row in 0..uuids.len() { + if uuids.is_null(row) { + continue; + } + let value: &[u8; 16] = uuids + .value(row) + .try_into() + .map_err(|_| storage("identity value is not 16 bytes"))?; + if let Ok(index) = candidates.binary_search(value) { + let surrogate = match ids { + Some(ids) if ids.is_null(row) => return Err(storage("null surrogate")), + Some(ids) => ids.value(row), + None => 0, + }; + found.push((index, surrogate)); + } + } + Ok(()) +} + +/// Identity lookups over one topology generation's published Parquet. +#[derive(Clone)] +pub struct TopologyIdentityProbe { + nodes: Vec>, + edges: Vec>, + deleted: Arc>, + generation: u64, + /// CAS objects a compact parent's fragments live in; they stay readable + /// for as long as the probe does. + _leases: Arc>, + authenticated_bytes: u64, + authenticated_objects: u64, +} + +impl std::fmt::Debug for TopologyIdentityProbe { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("TopologyIdentityProbe") + .field("generation", &self.generation) + .field("node_fragments", &self.nodes.len()) + .field("edge_fragments", &self.edges.len()) + .field("deleted", &self.deleted.len()) + .finish_non_exhaustive() + } +} + +impl TopologyIdentityProbe { + /// Open the probe over `files`, reading (or reusing) every fragment footer + /// and the record of deleted identities under `root`. + /// + /// # Errors + /// Returns an error when a fragment fails Parquet admission or lacks its + /// identity column, or when the deleted-identity record is malformed. + pub fn open( + root: &Path, + files: &crate::TopologyFiles, + topology_generation: u64, + ) -> Result { + let nodes = files + .node_fragments() + .par_iter() + .map(|(path, _)| load_fragment(path, "node_uuid", Some("node_id"))) + .collect::, _>>()?; + let edges = files + .edge_fragments() + .par_iter() + .map(|(_, path, _)| load_fragment(path, "edge_uuid", None)) + .collect::, _>>()?; + Ok(Self { + nodes, + edges, + deleted: Arc::new(read_deleted_identities(root)?), + generation: topology_generation, + _leases: Arc::new(Vec::new()), + authenticated_bytes: 0, + authenticated_objects: 0, + }) + } + + /// Open the probe over a compact parent generation whose fragments are + /// content-addressed objects. Every object is authenticated against its + /// inventory checksum and retained for the life of the probe. + /// + /// # Errors + /// Returns an error when an object fails authentication or admission. + pub(crate) fn open_compact( + container_root: &Path, + inventory: &crate::GraphFilesInventory, + topology_generation: u64, + ) -> Result { + let mut node_entries = Vec::new(); + let mut edge_entries = Vec::new(); + let mut deleted_entry = None; + for entry in &inventory.files { + if crate::topology_files::is_node(&entry.relative_path) { + node_entries.push(entry); + } else if entry.relative_path.starts_with("topology/edges/") + && entry.relative_path.ends_with(".parquet") + { + edge_entries.push(entry); + } else if entry.relative_path == DELETED_IDENTITIES_PATH { + deleted_entry = Some(entry); + } + } + let authenticated_bytes = node_entries + .iter() + .chain(edge_entries.iter()) + .copied() + .chain(deleted_entry) + .map(|entry| entry.byte_length) + .sum::(); + let authenticated_objects = + (node_entries.len() + edge_entries.len() + usize::from(deleted_entry.is_some())) as u64; + let leases = node_entries + .iter() + .chain(edge_entries.iter()) + .copied() + .chain(deleted_entry) + .collect::>() + .par_iter() + .map(|entry| { + crate::graph_object_store::open_graph_object_with_checksum(container_root, entry) + }) + .collect::, _>>()?; + let path_of = |entry: &crate::GraphFileEntry| { + crate::graph_object_path(container_root, &entry.content_sha256) + }; + let nodes = node_entries + .par_iter() + .map(|entry| load_fragment(&path_of(entry)?, "node_uuid", Some("node_id"))) + .collect::, _>>()?; + let edges = edge_entries + .par_iter() + .map(|entry| load_fragment(&path_of(entry)?, "edge_uuid", None)) + .collect::, _>>()?; + let deleted = match deleted_entry { + Some(entry) => { + parse_deleted_identities(std::fs::File::open(path_of(entry)?).map_err(storage)?)? + } + None => Vec::new(), + }; + Ok(Self { + nodes, + edges, + deleted: Arc::new(deleted), + generation: topology_generation, + _leases: Arc::new(leases), + authenticated_bytes, + authenticated_objects, + }) + } + + /// Bytes of content-addressed fragment payload this probe authenticated + /// against inventory checksums when it was opened (zero for a directory + /// open, which reads footers only). + #[must_use] + pub fn authenticated_bytes(&self) -> u64 { + self.authenticated_bytes + } + + /// Content-addressed objects authenticated at open. + #[must_use] + pub fn authenticated_objects(&self) -> u64 { + self.authenticated_objects + } + + /// Confirm every fragment is still the file this probe read. + /// + /// # Errors + /// Returns an error when a fragment was replaced or removed. + pub fn revalidate(&self) -> Result<(), GfError> { + for fragment in self.nodes.iter().chain(self.edges.iter()) { + if file_key(&fragment.path)? != fragment.key { + return Err(storage("topology fragment changed under a retained probe")); + } + } + Ok(()) + } + + /// Open the probe over the topology files found under `root`: a + /// materialized graph root with no declared inventory. + /// + /// # Errors + /// Returns an error when discovery or any fragment fails. + pub fn open_dir(root: &Path) -> Result { + let files = crate::TopologyFiles::discover_legacy(root)?; + Self::open(root, &files, crate::read_topology_generation(root)?) + } + + /// Topology generation this probe was opened at. + #[must_use] + pub fn topology_generation(&self) -> u64 { + self.generation + } + + /// Live entities of `kind`, from the row counts the footers record. + #[must_use] + pub fn count(&self, kind: UuidIndexKind) -> u64 { + match kind { + UuidIndexKind::Node => self.nodes.iter().map(|fragment| fragment.rows).sum(), + UuidIndexKind::Edge => self.edges.iter().map(|fragment| fragment.rows).sum(), + } + } + + fn scan( + fragments: &[Arc], + requested: &[Uuid], + ) -> Result<(Vec>, UuidProbeMetrics), GfError> { + let mut sorted: Vec<[u8; 16]> = requested.iter().map(|uuid| *uuid.as_bytes()).collect(); + sorted.sort_unstable(); + sorted.dedup(); + let mut metrics = UuidProbeMetrics { + requested: requested.len() as u64, + unique_requested: sorted.len() as u64, + ..UuidProbeMetrics::default() + }; + let mut tasks = Vec::new(); + for (fragment_index, fragment) in fragments.iter().enumerate() { + let mut considered = false; + for group in 0..fragment.groups.len() { + considered = true; + metrics.pages_considered += fragment.page_count(group); + if let Some(selection) = fragment.plan_group(group, &sorted) { + tasks.push(GroupPlan { + fragment: fragment_index, + group, + rows: selection.rows, + pages: selection.pages, + bytes: selection.bytes, + }); + } + } + metrics.runs_considered += u64::from(considered); + } + // Rayon workers carry the caller's requested I/O measurement, so the + // bytes a probe reads are attributed to the operation that asked. + let capture = crate::lifecycle_io::CaptureContext::current(); + let hits = tasks + .par_iter() + .map(|plan| { + let _scope = capture.attach(); + let mut found = Vec::new(); + fragments[plan.fragment].scan_group( + plan.group, + plan.rows.as_deref(), + &sorted, + &mut found, + )?; + Ok((plan.pages, plan.bytes, found)) + }) + .collect::, GfError>>()?; + let mut resolved: Vec> = vec![None; sorted.len()]; + for (pages, bytes, found) in hits { + metrics.file_seeks += 1; + metrics.identity_blocks_read += 1; + metrics.pages_read += pages; + metrics.identity_bytes_read += bytes; + for (index, surrogate) in found { + resolved[index] = Some(surrogate); + } + } + metrics.found = resolved.iter().filter(|value| value.is_some()).count() as u64; + // Restore caller order: each request maps to its distinct candidate. + let out = requested + .iter() + .map(|uuid| { + sorted + .binary_search(uuid.as_bytes()) + .ok() + .and_then(|index| resolved[index]) + }) + .collect(); + Ok((out, metrics)) + } + + /// Whether each of `uuids` is a live entity of `kind`, in caller order. + /// + /// # Errors + /// Returns an error when a fragment cannot be decoded. + pub fn probe( + &mut self, + kind: UuidIndexKind, + uuids: &[Uuid], + ) -> Result<(Vec, UuidProbeMetrics), GfError> { + let fragments = match kind { + UuidIndexKind::Node => &self.nodes, + UuidIndexKind::Edge => &self.edges, + }; + let (values, metrics) = Self::scan(fragments, uuids)?; + Ok(( + values.into_iter().map(|value| value.is_some()).collect(), + metrics, + )) + } + + /// Resolve live node UUIDs to their `node_id`, in caller order. + /// + /// # Errors + /// Returns an error when a fragment cannot be decoded. + pub fn lookup_node_surrogates( + &mut self, + uuids: &[Uuid], + ) -> Result<(Vec>, UuidProbeMetrics), GfError> { + Self::scan(&self.nodes, uuids) + } + + /// Whether each of `uuids` names an entity that was deleted: a UUID that + /// can never be reused. + #[must_use] + pub fn deleted(&self, uuids: &[Uuid]) -> Vec { + uuids + .iter() + .map(|uuid| self.deleted.binary_search(uuid.as_bytes()).is_ok()) + .collect() + } + + /// Whether each of `uuids` is already spent: a live node, a live edge or a + /// deleted entity. Node and edge UUIDs share one namespace. + /// + /// # Errors + /// Returns an error when a fragment cannot be decoded. + pub fn taken(&mut self, uuids: &[Uuid]) -> Result<(Vec, UuidProbeMetrics), GfError> { + let (nodes, node_metrics) = Self::scan(&self.nodes, uuids)?; + let (edges, edge_metrics) = Self::scan(&self.edges, uuids)?; + let deleted = self.deleted(uuids); + let mut metrics = node_metrics; + metrics.absorb(&edge_metrics); + metrics.requested = uuids.len() as u64; + metrics.unique_requested = edge_metrics.unique_requested; + let taken = nodes + .iter() + .zip(&edges) + .zip(&deleted) + .map(|((node, edge), deleted)| node.is_some() || edge.is_some() || *deleted) + .collect::>(); + metrics.found = taken.iter().filter(|value| **value).count() as u64; + Ok((taken, metrics)) + } +} + +fn deleted_identities_schema() -> Arc { + Arc::new(Schema::new(vec![Field::new( + DELETED_IDENTITIES_COLUMN, + DataType::FixedSizeBinary(16), + false, + )])) +} + +/// Read the sorted record of deleted entity UUIDs under `root`; empty when no +/// generation has deleted anything. +/// +/// # Errors +/// Returns an error when the file is unreadable, unsorted or not unique. +pub fn read_deleted_identities(root: &Path) -> Result, GfError> { + let path = root.join(DELETED_IDENTITIES_PATH); + let file = match std::fs::File::open(&path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(storage(format!("{}: {error}", path.display()))), + }; + parse_deleted_identities(file) +} + +fn parse_deleted_identities(file: std::fs::File) -> Result, GfError> { + let file = crate::catalog::admitted_path_file(file).map_err(storage)?; + let reader = ParquetRecordBatchReaderBuilder::try_new(file) + .map_err(storage)? + .build() + .map_err(storage)?; + let mut identities: Vec<[u8; 16]> = Vec::new(); + for batch in reader { + let batch = batch.map_err(storage)?; + let column = batch + .column(0) + .as_any() + .downcast_ref::() + .filter(|array| array.value_length() == 16 && array.null_count() == 0) + .ok_or_else(|| storage("deleted identities are not FixedSizeBinary(16)"))?; + for row in 0..column.len() { + let value: [u8; 16] = column.value(row).try_into().expect("width checked"); + if identities.last().is_some_and(|last| *last >= value) { + return Err(storage("deleted identities are not sorted and unique")); + } + identities.push(value); + } + } + Ok(identities) +} + +/// Stage the record of deleted UUIDs with `additions` merged in. Writes +/// nothing when `additions` is empty, so a graph that never deletes never +/// carries the file. +/// +/// # Errors +/// Returns an error when the existing record is unreadable or staging fails. +pub fn stage_deleted_identities( + staged: &mut crate::RewriteBatch, + root: &Path, + additions: &[Uuid], +) -> Result<(), GfError> { + if additions.is_empty() { + return Ok(()); + } + let mut merged = read_deleted_identities(root)?; + let before = merged.len(); + merged.extend(additions.iter().map(|uuid| *uuid.as_bytes())); + merged.sort_unstable(); + merged.dedup(); + if merged.len() == before { + return Ok(()); + } + let column = FixedSizeBinaryArray::try_from_iter(merged.iter().map(<[u8; 16]>::as_slice)) + .map_err(storage)?; + let batch = RecordBatch::try_new(deleted_identities_schema(), vec![Arc::new(column)]) + .map_err(storage)?; + staged.restage( + &root.join(DELETED_IDENTITIES_PATH), + deleted_identities_schema(), + &batch, + ) +} + +#[cfg(test)] +mod tests; diff --git a/crates/graphforge-storage/src/topology_identity/tests.rs b/crates/graphforge-storage/src/topology_identity/tests.rs new file mode 100644 index 000000000..eee84fa57 --- /dev/null +++ b/crates/graphforge-storage/src/topology_identity/tests.rs @@ -0,0 +1,284 @@ +//! Identity probes over the published Parquet prune row groups and pages. + +use super::*; +use parquet::arrow::ArrowWriter; +use parquet::file::properties::EnabledStatistics; +use tempfile::TempDir; + +const ROWS: usize = 4_000; +const GROUP_ROWS: usize = 1_000; +const PAGE_ROWS: usize = 100; +const GROUPS: u64 = (ROWS / GROUP_ROWS) as u64; +const PAGES: u64 = (ROWS / PAGE_ROWS) as u64; + +/// The `index`th identity: spaced by ten, so `identity(i) + 5` is never live. +fn identity(index: usize) -> Uuid { + Uuid::from_u128((index as u128 + 1) * 10) +} + +fn absent_after(index: usize) -> Uuid { + Uuid::from_u128((index as u128 + 1) * 10 + 5) +} + +/// A sorted fragment of `ROWS` identities in `GROUPS` row groups of +/// `PAGES / GROUPS` pages each, `node_id` equal to the one-based rank. +fn write_fragment( + dir: &Path, + uuid_column: &str, + id_column: Option<&str>, + statistics: EnabledStatistics, +) -> PathBuf { + let mut fields = vec![Field::new( + uuid_column, + DataType::FixedSizeBinary(16), + false, + )]; + fields.extend(id_column.map(|name| Field::new(name, DataType::UInt64, false))); + let schema = Arc::new(Schema::new(fields)); + let properties = crate::permanent_parquet::writer_properties() + .set_max_row_group_row_count(Some(GROUP_ROWS)) + .set_data_page_row_count_limit(PAGE_ROWS) + .set_write_batch_size(PAGE_ROWS / 2) + .set_statistics_enabled(statistics) + .build(); + let path = dir.join(format!("{uuid_column}.parquet")); + let mut writer = ArrowWriter::try_new( + std::fs::File::create(&path).unwrap(), + Arc::clone(&schema), + Some(properties), + ) + .unwrap(); + for start in (0..ROWS).step_by(PAGE_ROWS) { + let rows = start..start + PAGE_ROWS; + let uuids = FixedSizeBinaryArray::try_from_iter( + rows.clone().map(|row| identity(row).as_bytes().to_vec()), + ) + .unwrap(); + let mut columns: Vec> = vec![Arc::new(uuids)]; + if id_column.is_some() { + columns.push(Arc::new(UInt64Array::from_iter_values( + rows.map(|row| row as u64 + 1), + ))); + } + writer + .write(&RecordBatch::try_new(Arc::clone(&schema), columns).unwrap()) + .unwrap(); + } + writer.close().unwrap(); + path +} + +fn probe_over(nodes: Vec>, edges: Vec>) -> TopologyIdentityProbe { + TopologyIdentityProbe { + nodes, + edges, + deleted: Arc::new(Vec::new()), + generation: 1, + _leases: Arc::new(Vec::new()), + authenticated_bytes: 0, + authenticated_objects: 0, + } +} + +fn node_probe(dir: &Path) -> TopologyIdentityProbe { + let path = write_fragment(dir, "node_uuid", Some("node_id"), EnabledStatistics::Page); + probe_over( + vec![load_fragment(&path, "node_uuid", Some("node_id")).unwrap()], + Vec::new(), + ) +} + +#[test] +fn the_fixture_carries_the_page_layout_the_pruning_tests_depend_on() { + let dir = TempDir::new().unwrap(); + let probe = node_probe(dir.path()); + let fragment = &probe.nodes[0]; + assert_eq!(fragment.groups.len() as u64, GROUPS); + for group in fragment.groups.iter() { + assert_eq!(group.pages.len() as u64, PAGES / GROUPS); + assert_eq!(group.chunks.len(), 2, "uuid and node_id page layouts"); + } +} + +#[test] +fn a_candidate_set_inside_one_page_decodes_exactly_that_page() { + let dir = TempDir::new().unwrap(); + let mut probe = node_probe(dir.path()); + let candidates = [identity(1_205), identity(1_250), identity(1_299)]; + let (surrogates, metrics) = probe.lookup_node_surrogates(&candidates).unwrap(); + assert_eq!(surrogates, [Some(1_206), Some(1_251), Some(1_300)]); + assert_eq!(metrics.found, 3); + assert_eq!(metrics.pages_considered, PAGES); + assert_eq!(metrics.pages_read, 1, "one 100-row page of forty"); + assert_eq!(metrics.identity_blocks_read, 1); + assert_eq!(metrics.file_seeks, 1); + assert_eq!(metrics.per_record_seeks, 0); + let whole_group = probe.nodes[0].groups[1].probe_bytes; + // The chunk's dictionary page is read with any page of it, so a page of + // ten does not cost a tenth of its row group; it still costs far less. + assert!( + metrics.identity_bytes_read * 2 < whole_group, + "one page of ten reads less than half its row group: {} of {whole_group}", + metrics.identity_bytes_read + ); +} + +#[test] +fn candidates_in_two_row_groups_decode_one_page_in_each() { + let dir = TempDir::new().unwrap(); + let mut probe = node_probe(dir.path()); + let (present, metrics) = probe + .probe(UuidIndexKind::Node, &[identity(5), identity(3_999)]) + .unwrap(); + assert_eq!(present, [true, true]); + assert_eq!(metrics.pages_read, 2); + assert_eq!(metrics.identity_blocks_read, 2); +} + +#[test] +fn a_candidate_in_a_gap_between_pages_decodes_nothing() { + let dir = TempDir::new().unwrap(); + let mut probe = node_probe(dir.path()); + // Inside the row group's bounds, below the next page's minimum and above + // the previous page's maximum: only the page index can rule it out. + let (present, metrics) = probe + .probe(UuidIndexKind::Node, &[absent_after(199)]) + .unwrap(); + assert_eq!(present, [false]); + assert_eq!(metrics.pages_considered, PAGES); + assert_eq!(metrics.pages_read, 0); + assert_eq!(metrics.identity_blocks_read, 0); + assert_eq!(metrics.identity_bytes_read, 0); +} + +#[test] +fn an_absent_candidate_inside_a_page_is_decoded_and_refused() { + let dir = TempDir::new().unwrap(); + let mut probe = node_probe(dir.path()); + let (present, metrics) = probe + .probe(UuidIndexKind::Node, &[absent_after(250)]) + .unwrap(); + assert_eq!(present, [false]); + assert_eq!(metrics.pages_read, 1); +} + +#[test] +fn a_candidate_beyond_every_row_group_decodes_nothing() { + let dir = TempDir::new().unwrap(); + let mut probe = node_probe(dir.path()); + let (present, metrics) = probe + .probe(UuidIndexKind::Node, &[identity(ROWS + 7)]) + .unwrap(); + assert_eq!(present, [false]); + assert_eq!(metrics.pages_read, 0); +} + +#[test] +fn candidates_across_every_page_decode_every_page() { + let dir = TempDir::new().unwrap(); + let mut probe = node_probe(dir.path()); + let candidates = (0..ROWS) + .step_by(PAGE_ROWS) + .map(identity) + .collect::>(); + let (present, metrics) = probe.probe(UuidIndexKind::Node, &candidates).unwrap(); + assert!(present.iter().all(|present| *present)); + assert_eq!(metrics.pages_read, PAGES); + assert_eq!(metrics.identity_blocks_read, GROUPS); +} + +#[test] +fn edge_probes_prune_pages_without_a_surrogate_column() { + let dir = TempDir::new().unwrap(); + let path = write_fragment(dir.path(), "edge_uuid", None, EnabledStatistics::Page); + let mut probe = probe_over( + Vec::new(), + vec![load_fragment(&path, "edge_uuid", None).unwrap()], + ); + let (present, metrics) = probe + .probe(UuidIndexKind::Edge, &[identity(2_000), absent_after(2_001)]) + .unwrap(); + assert_eq!(present, [true, false]); + assert_eq!(metrics.pages_read, 1); + assert_eq!(metrics.pages_considered, PAGES); +} + +#[test] +fn a_file_without_a_page_index_is_pruned_by_row_group_alone() { + let dir = TempDir::new().unwrap(); + let path = write_fragment( + dir.path(), + "node_uuid", + Some("node_id"), + EnabledStatistics::Chunk, + ); + let fragment = load_fragment(&path, "node_uuid", Some("node_id")).unwrap(); + assert!(fragment.groups.iter().all(|group| group.pages.is_empty())); + let mut probe = probe_over(vec![fragment], Vec::new()); + let (surrogates, metrics) = probe.lookup_node_surrogates(&[identity(2_500)]).unwrap(); + assert_eq!(surrogates, [Some(2_501)]); + assert_eq!(metrics.pages_considered, GROUPS, "a group is one page"); + assert_eq!(metrics.pages_read, 1); + assert_eq!(metrics.identity_blocks_read, 1); +} + +#[test] +fn page_pruning_reduces_the_bytes_the_file_system_serves() { + let dir = TempDir::new().unwrap(); + let mut probe = node_probe(dir.path()); + let file_len = std::fs::metadata(dir.path().join("node_uuid.parquet")) + .unwrap() + .len(); + let read_bytes = |probe: &mut TopologyIdentityProbe, candidates: &[Uuid]| { + let _capture = crate::lifecycle_io::CaptureScope::install(); + let before = crate::lifecycle_io::snapshot().expect("requested measurement"); + probe.lookup_node_surrogates(candidates).unwrap(); + crate::lifecycle_io::snapshot() + .expect("requested measurement") + .since(&before) + .unwrap() + .phases[&crate::StorageIoPhase::ReadPathScan] + .read_bytes + }; + let narrow = read_bytes(&mut probe, &[identity(1_250)]); + let every_page = (0..ROWS) + .step_by(PAGE_ROWS) + .map(identity) + .collect::>(); + let wide = read_bytes(&mut probe, &every_page); + assert!(narrow > 0, "the probe's reads reach the lifecycle counters"); + assert!( + narrow * 5 < wide, + "one page of forty must not read like all of them: {narrow} against {wide}" + ); + assert!(narrow * 5 < file_len, "{narrow} of a {file_len} byte file"); +} + +#[test] +fn a_cached_footer_never_sends_a_probe_to_the_path_it_was_first_read_from() { + // Hydration hard-links one content-addressed object into every workspace, so + // the process-wide footer cache sees the same file under many paths. A probe + // built over a later workspace must read that workspace, not the first. + let first = TempDir::new().unwrap(); + let path = write_fragment( + first.path(), + "node_uuid", + Some("node_id"), + EnabledStatistics::Page, + ); + let _warmed = load_fragment(&path, "node_uuid", Some("node_id")).unwrap(); + + let second = TempDir::new().unwrap(); + let linked = second.path().join("node_uuid.parquet"); + std::fs::hard_link(&path, &linked).unwrap(); + drop(first); + assert!(!path.exists()); + + let mut probe = probe_over( + vec![load_fragment(&linked, "node_uuid", Some("node_id")).unwrap()], + Vec::new(), + ); + let (surrogates, metrics) = probe.lookup_node_surrogates(&[identity(2_500)]).unwrap(); + assert_eq!(surrogates, [Some(2_501)]); + assert_eq!(metrics.pages_read, 1); +} diff --git a/crates/graphforge-storage/src/uuid_membership.rs b/crates/graphforge-storage/src/uuid_membership.rs index 014bfeb8a..b41c60944 100644 --- a/crates/graphforge-storage/src/uuid_membership.rs +++ b/crates/graphforge-storage/src/uuid_membership.rs @@ -1,40 +1,27 @@ -//! Persistent, bounded-memory UUID membership indexes used by bulk ingest. +//! The ordinal node-identity facet under `topology/uuid-membership/`. //! -//! The canonical graph remains Parquet. This derived format is deliberately -//! small: one manifest (published last with an atomic replacement) names an -//! immutable base plus size-tiered delta runs. Each run contains a unified -//! UUID-sorted identity file and a node-only surrogate-sorted reverse file. -//! Readers verify version, topology generation, framing, canonical ordering, -//! counts, and corruption checksums before serving bounded binary-search probes. +//! The directory name is the published location of the node `node_id -> UUID` +//! authority (ordinal v4). The UUID membership index that once lived beside it +//! (`manifest.json`, `identities-v5-*`, `node-surrogates-v5-*`) is no longer +//! produced or read (#1902): live UUIDs are answered from the published +//! topology Parquet by [`crate::TopologyIdentityProbe`]. Projects that still +//! carry those files open normally and the files are ignored. -use self::probing::ProbeFileKind; -use self::probing::authenticated_probe_block; use graphforge_core::GfError; use serde::Deserialize; use serde::Serialize; -use std::collections::BTreeMap; -use std::collections::BTreeSet; use std::fmt::Write as _; -use std::fs; -use std::fs::File; -use std::io::BufReader; -use std::io::Read; -use std::io::Seek; -use std::io::SeekFrom; -use std::path::Path; -use std::path::PathBuf; use uuid::Uuid; mod construction; mod maintenance; mod ordinal_artifacts; mod ordinal_compaction; -mod probing; mod rebuild; mod topology_delta; +pub(crate) use construction::clear_private_ordinal_residue; pub(crate) use construction::is_exact_private_v4_name; -pub(crate) use construction::{ConstructionIdentityInput, encode_construction_index}; pub use maintenance::maintain_uuid_membership_orphans; #[cfg(test)] pub(crate) use maintenance::maintain_uuid_membership_orphans_with_ordinal_authority; @@ -44,81 +31,26 @@ pub(crate) use ordinal_artifacts::V4OrdinalPublicationMetrics; pub(crate) use ordinal_artifacts::publish_v4_construction_artifacts; #[cfg(test)] pub(crate) use ordinal_artifacts::stage_v4_ordinal_artifacts; -#[cfg(test)] -pub(crate) use probing::ConstructionUuidIdentity; -#[cfg(test)] -pub(crate) use probing::UuidConstructionSnapshot; -#[cfg(test)] -pub(crate) use probing::open_uuid_construction_snapshot; -#[cfg(test)] -pub(crate) use probing::pin_uuid_construction_snapshot; pub use rebuild::rebuild_v4_ordinal_identity; pub use rebuild::rebuild_v4_ordinal_identity_with_evidence; -pub(crate) use rebuild::{ - ensure_uuid_membership_migrated, ensure_uuid_membership_migrated_with_topology, -}; -pub use rebuild::{rebuild_uuid_membership_indexes, rebuild_uuid_membership_indexes_with_topology}; -#[cfg(test)] -pub(crate) use topology_delta::append_uuid_membership_delta; pub(crate) use topology_delta::commit_uuid_neutral_topology_rewrite; pub(crate) use topology_delta::commit_uuid_topology_rewrite; -pub(crate) use topology_delta::prepare_uuid_membership_delta; pub(crate) use topology_delta::prepare_v4_ordinal_delta; -mod identity_codec; - -const FORMAT_VERSION: u32 = 7; // Private recovery intents evolve independently of the published UUID format. -const CONSTRUCTION_INTENT_FORMAT_VERSION: u32 = 3; -const NODE_LOOKUP_RECORD_BYTES: u64 = 24; -const IDENTITY_RECORD_BYTES: u64 = 25; -const NODE_LOOKUP_RECORD_WIDTH: usize = 24; -const IDENTITY_RECORD_WIDTH: usize = identity_codec::WIDTH; const BULK_IO_BYTES: usize = 1 << 20; // Persistent authenticated authority for UUID-to-surrogate resolution. Keeping // it in the immutable topology generation is what lets writer reopen avoid // decoding historical topology shards; `.graphforge-cache` is only for data // that can be discarded and reconstructed without violating that contract. const INDEX_DIR: &str = "topology/uuid-membership"; -const MANIFEST: &str = "manifest.json"; const V4_ORDINAL_MANIFEST: &str = "ordinal-v4-manifest.json"; const V4_ORDINAL_RECEIPT: &str = "ordinal-v4-receipt.json"; -const CONSTRUCTION_INTENT: &str = ".construction-intent.json"; fn storage_err(error: impl std::fmt::Display) -> GfError { GfError::Storage(format!("UUID membership index: {error}")) } -fn open_uuid_file(path: &Path) -> Result { - let file = File::open(path).map_err(storage_err)?; - crate::io_stats::record_uuid_file_open(); - Ok(file) -} - -fn create_uuid_file(path: &Path) -> Result { - let file = File::create(path).map_err(storage_err)?; - crate::io_stats::record_uuid_file_open(); - Ok(file) -} - -fn open_uuid_child_file( - directory: &graphforge_filesystem::StableDirectory, - name: &std::ffi::OsStr, -) -> Result { - let file = directory.open_child_file(name).map_err(storage_err)?; - crate::io_stats::record_uuid_file_open(); - Ok(file) -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -/// Selects the canonical identity domain to probe. -pub enum UuidIndexKind { - /// Canonical node UUIDs. - Node, - /// Canonical edge UUIDs. - Edge, -} - #[derive(Clone, Copy, Debug)] /// Hard work limits for a bounded index build. pub struct UuidIndexBuildLimits { @@ -264,52 +196,6 @@ pub(crate) struct V4OrdinalAppendMetrics { /// Physical orphan bytes reclaimed. pub(crate) orphan_gc_bytes: u64, } -/// Aggregate-only evidence for one incremental v3 run publication. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct UuidIndexAppendMetrics { - /// New identity and tombstone records accepted by this publication. - pub input_records: u64, - /// Prior canonical topology rows decoded; ordinary append requires zero. - pub prior_topology_rows_decoded: u64, - /// Immutable authenticated runs retained after publication. - pub retained_runs: usize, - /// Exact physical bytes written for run and manifest outputs. - pub physical_bytes_written: u64, - /// Bulk output blocks submitted to the filesystem. - pub write_blocks: u64, - /// Bytes submitted through those bulk output blocks. - pub write_bytes: u64, - /// Maximum fixed-width records buffered at once. - pub peak_buffered_records: usize, - /// Maximum charged fixed-width buffer bytes at once. - pub peak_buffered_bytes: usize, - /// Sequential retained-run bytes examined for cross-run uniqueness. - pub validation_scan_bytes: u64, - /// One-MiB read blocks covering `validation_scan_bytes`. - pub validation_scan_blocks: u64, - /// Bulk append validation never performs per-key random seeks. - pub validation_random_seeks: u64, - /// Full-run bytes authenticated once when admitting a new retained snapshot. - pub snapshot_admission_authentication_bytes: u64, - /// Full-run blocks authenticated during snapshot admission. - pub snapshot_admission_authentication_blocks: u64, - /// Newly installed run bytes authenticated while advancing the retained snapshot. - pub new_output_authentication_bytes: u64, - /// Newly installed authenticated run blocks. - pub new_output_authentication_blocks: u64, - /// Unreferenced canonical run files examined under the rewrite lock. - pub orphan_gc_candidates: u64, - /// Unreferenced one-link run files removed by retained identity. - pub orphan_gc_removed: u64, - /// Candidates left for a later bounded maintenance pass. - pub orphan_gc_deferred: u64, - /// Candidates deferred solely because the per-transaction bound was reached. - pub orphan_gc_deferred_limit: u64, - /// Candidates retained because another hard link exists. - pub orphan_gc_deferred_linked: u64, - /// Physical bytes reclaimed from removed orphan runs. - pub orphan_gc_bytes: u64, -} /// Typed bounded orphan-collection evidence. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -328,362 +214,6 @@ pub struct UuidIndexOrphanGcWork { pub bytes: u64, } -#[derive(Clone, Debug, Default, PartialEq, Eq)] -/// Aggregate-only probe evidence; it never contains graph identities. -pub struct UuidProbeMetrics { - /// Total requested identities, including duplicates. - pub requested: u64, - /// Distinct requested identities. - pub unique_requested: u64, - /// Distinct identities found. - pub found: u64, - /// Block-positioning seeks performed. One seek corresponds to one bounded - /// authenticated block read, never to one requested record. - pub file_seeks: u64, - /// Identity-run blocks read after block-fence selection. - pub identity_blocks_read: u64, - /// Identity-run bytes read after block-fence selection. - pub identity_bytes_read: u64, - /// Reverse-surrogate blocks read for batched pair validation. - pub surrogate_blocks_read: u64, - /// Reverse-surrogate bytes read for batched pair validation. - pub surrogate_bytes_read: u64, - /// Immutable runs considered while applying newest-run shadowing. - pub runs_considered: u64, - /// Per-record filesystem seeks. Batched lookup must keep this exactly zero. - pub per_record_seeks: u64, -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -struct FileRecord { - name: String, - count: u64, - sha256: String, - #[serde(with = "crate::corruption_checksum::wire_hex")] - xxh64: u64, - blocks: Vec, -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -struct BlockRecord { - offset: u64, - len: u32, - first_key: String, - last_key: String, - #[serde(with = "crate::corruption_checksum::wire_hex")] - xxh64: u64, -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -struct RunRecord { - base: bool, - level: u8, - first_generation: u64, - last_generation: u64, - identities: FileRecord, - node_surrogates: FileRecord, - node_count: u64, - edge_count: u64, - #[serde(default)] - deleted_node_count: u64, - #[serde(default)] - deleted_edge_count: u64, -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -struct Manifest { - format_version: u32, - base_generation: u64, - current_generation: u64, - #[serde(default)] - live_node_count: u64, - #[serde(default)] - live_edge_count: u64, - runs: Vec, -} - -#[derive(Debug)] -struct OpenRun { - identities: File, - node_surrogates: File, - descriptor: RunRecord, -} - -#[derive(Debug)] -struct AuthenticatedRun { - identities: File, - identities_identity: graphforge_filesystem::FileIdentity, - node_surrogates: File, - node_surrogates_identity: graphforge_filesystem::FileIdentity, - descriptor: RunRecord, -} - -fn authenticated_block( - file: &mut File, - block: &BlockRecord, - width: usize, - metrics: &mut UuidIndexAppendMetrics, -) -> Result, GfError> { - file.seek(SeekFrom::Start(block.offset)) - .map_err(storage_err)?; - let mut bytes = vec![0_u8; block.len as usize]; - file.read_exact(&mut bytes).map_err(storage_err)?; - if !block_matches(&bytes, block, width) { - return Err(storage_err("UUID probe block authentication failed")); - } - metrics.validation_scan_bytes = metrics - .validation_scan_bytes - .saturating_add(bytes.len() as u64); - metrics.validation_scan_blocks = metrics.validation_scan_blocks.saturating_add(1); - Ok(bytes) -} -#[derive(Clone, Copy, Debug)] -struct IdentityState { - present: bool, - surrogate: u64, -} - -/// Resolve all candidate keys in one run by selecting authenticated blocks -/// from their fences and merge-scanning each selected block once. -fn batch_identity_states( - file: &mut File, - descriptor: &FileRecord, - expected_kind: UuidIndexKind, - requested: &BTreeSet, - metrics: &mut UuidProbeMetrics, -) -> Result, GfError> { - let mut groups = std::collections::BTreeMap::>::new(); - for uuid in requested { - let key = hex_sha256_key(uuid.as_bytes()); - if let Some(index) = candidate_block(descriptor, &key) { - groups.entry(index).or_default().push(*uuid); - } - } - let mut found = std::collections::BTreeMap::new(); - for (index, candidates) in groups { - let bytes = authenticated_probe_block( - file, - &descriptor.blocks[index], - IDENTITY_RECORD_WIDTH, - ProbeFileKind::Identity, - metrics, - )?; - let mut remaining = bytes.as_slice(); - let mut next = identity_codec::take(&mut remaining)?; - for uuid in candidates { - while let Some(record) = next { - match record[..16].cmp(uuid.as_bytes()) { - std::cmp::Ordering::Less => next = identity_codec::take(&mut remaining)?, - std::cmp::Ordering::Greater => break, - std::cmp::Ordering::Equal => { - let record_kind = if matches!(record[16], 0 | 2) { - UuidIndexKind::Node - } else { - UuidIndexKind::Edge - }; - found.insert( - uuid, - IdentityState { - present: record_kind == expected_kind - && matches!(record[16], 0 | 1), - surrogate: u64::from_be_bytes( - record[17..25].try_into().expect("fixed record"), - ), - }, - ); - next = identity_codec::take(&mut remaining)?; - break; - } - } - } - } - } - Ok(found) -} - -/// Validate all resolved node identity/surrogate pairs in one run with the -/// same fence-selected merge scan. A missing or mismatched reverse pair is -/// authenticated corruption. -fn validate_surrogate_pairs( - file: &mut File, - descriptor: &FileRecord, - pairs: &[(u64, Uuid)], - metrics: &mut UuidProbeMetrics, -) -> Result<(), GfError> { - let mut groups = std::collections::BTreeMap::>::new(); - for &(surrogate, uuid) in pairs { - let key = hex_sha256_key(&surrogate.to_be_bytes()); - let index = candidate_block(descriptor, &key) - .ok_or_else(|| storage_err("identity/surrogate run pair is inconsistent"))?; - groups.entry(index).or_default().push((surrogate, uuid)); - } - for (index, mut candidates) in groups { - candidates.sort_unstable(); - let bytes = authenticated_probe_block( - file, - &descriptor.blocks[index], - NODE_LOOKUP_RECORD_WIDTH, - ProbeFileKind::Surrogate, - metrics, - )?; - let mut record_index = 0_usize; - for (surrogate, uuid) in candidates { - let key = surrogate.to_be_bytes(); - let mut matched = false; - while record_index < bytes.len() / NODE_LOOKUP_RECORD_WIDTH { - let start = record_index * NODE_LOOKUP_RECORD_WIDTH; - let record = &bytes[start..start + NODE_LOOKUP_RECORD_WIDTH]; - match record[..8].cmp(&key) { - std::cmp::Ordering::Less => record_index += 1, - std::cmp::Ordering::Greater => break, - std::cmp::Ordering::Equal => { - matched = record[8..24] == *uuid.as_bytes(); - record_index += 1; - break; - } - } - } - if !matched { - return Err(storage_err("identity/surrogate run pair is inconsistent")); - } - } - } - Ok(()) -} - -fn candidate_block(record: &FileRecord, key: &str) -> Option { - let index = record - .blocks - .partition_point(|block| block.last_key.as_str() < key); - record - .blocks - .get(index) - .filter(|block| block.first_key.as_str() <= key) - .map(|_| index) -} - -fn block_record_index(bytes: &[u8], width: usize, key: &[u8]) -> Option { - let key_width = key.len(); - let (mut low, mut high) = (0, bytes.len() / width); - while low < high { - let middle = low + (high - low) / 2; - let record = &bytes[middle * width..(middle + 1) * width]; - match record[..key_width].cmp(key) { - std::cmp::Ordering::Less => low = middle + 1, - std::cmp::Ordering::Greater => high = middle, - std::cmp::Ordering::Equal => return Some(middle), - } - } - None -} - -fn reject_retained_identity_collisions( - run: &mut AuthenticatedRun, - incoming: &[(Uuid, u8, u64)], - metrics: &mut UuidIndexAppendMetrics, -) -> Result<(), GfError> { - let mut groups = std::collections::BTreeMap::>::new(); - for item in incoming { - let key = hex_sha256_key(item.0.as_bytes()); - if let Some(index) = candidate_block(&run.descriptor.identities, &key) { - groups.entry(index).or_default().push(item); - } - } - for (index, mut items) in groups { - let bytes = authenticated_block( - &mut run.identities, - &run.descriptor.identities.blocks[index], - IDENTITY_RECORD_WIDTH, - metrics, - )?; - items.sort_unstable_by_key(|item| item.0); - let mut remaining = bytes.as_slice(); - let mut next = identity_codec::take(&mut remaining)?; - for (uuid, kind, surrogate) in items { - while next.is_some_and(|record| record[..16] < uuid.as_bytes()[..]) { - next = identity_codec::take(&mut remaining)?; - } - if let Some(record) = next.filter(|record| record[..16] == uuid.as_bytes()[..]) { - let retained_kind = record[16]; - let retained_surrogate = - u64::from_be_bytes(record[17..25].try_into().expect("fixed")); - let deletion_matches = ((*kind == 2 && retained_kind == 0) - || (*kind == 3 && retained_kind == 1)) - && retained_surrogate == *surrogate; - if !deletion_matches { - return Err(storage_err( - "UUID already exists in an authenticated retained run", - )); - } - } - } - } - Ok(()) -} - -fn reject_retained_surrogate_collisions( - run: &mut AuthenticatedRun, - incoming: &[(u64, Uuid)], - metrics: &mut UuidIndexAppendMetrics, -) -> Result<(), GfError> { - let mut groups = std::collections::BTreeMap::>::new(); - for item in incoming { - let key = hex_sha256_key(&item.0.to_be_bytes()); - if let Some(index) = candidate_block(&run.descriptor.node_surrogates, &key) { - groups.entry(index).or_default().push(item); - } - } - for (index, items) in groups { - let bytes = authenticated_block( - &mut run.node_surrogates, - &run.descriptor.node_surrogates.blocks[index], - NODE_LOOKUP_RECORD_WIDTH, - metrics, - )?; - for (surrogate, uuid) in items { - let key = surrogate.to_be_bytes(); - if let Some(at) = block_record_index(&bytes, NODE_LOOKUP_RECORD_WIDTH, &key) { - let record = &bytes[at * 24..at * 24 + 24]; - if record[8..] != *uuid.as_bytes() { - return Err(storage_err( - "node surrogate already exists in an authenticated retained run", - )); - } - } - } - } - Ok(()) -} - -#[derive(Debug)] -/// An authenticated node-and-edge index snapshot pinned by one manifest. -pub struct UuidMembershipIndex { - runs: Vec, - manifest: Manifest, -} - -/// Long-lived authenticated UUID-index snapshot retained by construction writers. -#[derive(Debug)] -pub struct AuthenticatedUuidIndexSnapshot { - graph_root: graphforge_filesystem::StableDirectory, - graph_root_path: PathBuf, - graph_root_identity: graphforge_filesystem::FileIdentity, - root: graphforge_filesystem::StableDirectory, - root_identity: graphforge_filesystem::FileIdentity, - manifest_file: Option, - manifest_bytes: u64, - manifest_identity: graphforge_filesystem::FileIdentity, - manifest_sha256: String, - manifest: Manifest, - runs: Vec, - authenticated_bytes: u64, - authenticated_blocks: u64, - cas_source_paths: Option>, - _cas_leases: Vec, -} - #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct ConstructionIndexOutput { pub name: String, @@ -692,45 +222,6 @@ pub(crate) struct ConstructionIndexOutput { pub xxh64: u64, } -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -pub(crate) struct ConstructionIndexReference { - pub source_root: String, - pub source_root_volume: u64, - pub source_root_file_id: String, - pub source_path: String, - pub source_volume: u64, - pub source_file_id: String, - pub target_path: String, - pub bytes: u64, - pub sha256: String, - #[serde(with = "crate::corruption_checksum::wire_hex")] - pub xxh64: u64, - pub parent_manifest_sha256: String, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub(crate) struct ConstructionIndexEncoding { - pub artifacts: Vec, - pub retained_references: Vec, - pub input_records: u64, - pub read_bytes: u64, - pub read_operations: u64, - pub final_write_bytes: u64, - pub write_bytes: u64, - pub write_operations: u64, - pub fsync_operations: u64, - pub created_runs: u64, - pub retained_runs: u64, - pub retained_payload_bytes: u64, - pub peak_buffer_bytes: u64, - pub peak_temporary_bytes: u64, - pub cache_release: graphforge_filesystem::FileCacheReleaseEvidence, - /// SHA-256 of the source identity file, computed during encoding using - /// `ArtifactSha256` (classified). Used as the `topology_delta_sha256` - /// binding in the published `TopologyIndexReceipt`. - pub source_sha256: String, -} - #[cfg(test)] type ConstructionOrdinalHook = Box; #[cfg(test)] @@ -750,119 +241,18 @@ fn construction_ordinal_event(_phase: &str, _generation: u64) { }); } -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct UuidConstructionSnapshotWork { - pub authentication_bytes: u64, - pub authentication_blocks: u64, - pub live_nodes: u64, - pub live_edges: u64, - pub max_node_surrogate: u64, -} - -#[allow(clippy::struct_field_names)] -pub(crate) struct ConstructionReferenceAuthentication<'a> { - pub(crate) source_root: &'a str, - pub(crate) source_root_volume: u64, - pub(crate) source_root_file_id: &'a str, - pub(crate) source_path: &'a str, - pub(crate) source_volume: u64, - pub(crate) source_file_id: &'a str, - pub(crate) target_path: &'a str, - pub(crate) bytes: u64, - pub(crate) sha256: &'a str, - pub(crate) xxh64: u64, - pub(crate) parent_manifest_sha256: &'a str, -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub(crate) struct ConstructionReferenceAuthenticationWork { - pub(crate) global_revalidation_bytes: u64, - pub(crate) referenced_payload_bytes: u64, -} - -// Immutable authenticated runs may be shared by a hydrated CAS workspace. -// Mutable manifests/receipts and private construction artifacts retain their -// strict single-link rules. Digest/block and named-inode checks remain separate. -fn retained_run_has_safe_links(file: &File) -> Result { - let links = graphforge_filesystem::file_link_count(file).map_err(storage_err)?; - Ok(links == 1 - || links > 1 - && file - .metadata() - .map_err(storage_err)? - .permissions() - .readonly()) -} - -/// Whether a membership manifest exists, without duplicating its private layout. -#[must_use] -pub fn uuid_membership_index_present(project_dir: &Path) -> bool { - project_dir.join(INDEX_DIR).join(MANIFEST).is_file() -} - const DEFAULT_ORPHAN_GC_LIMIT: usize = 64; -#[cfg(test)] -thread_local! { - static FAIL_NEXT_SNAPSHOT_REFRESH: std::cell::Cell = const { std::cell::Cell::new(false) }; -} - -#[cfg(test)] -pub(crate) fn fail_next_snapshot_refresh_for_test() { - FAIL_NEXT_SNAPSHOT_REFRESH.set(true); -} - -#[cfg(test)] -fn injected_snapshot_refresh_failure() -> Option { - if FAIL_NEXT_SNAPSHOT_REFRESH.replace(false) { - return Some(storage_err("injected UUID snapshot refresh failure")); - } - None -} - -#[cfg(not(test))] -fn injected_snapshot_refresh_failure() -> Option { - None -} - -/// Whether the manifest version and topology generation match the workspace. -/// This cheap publication-path check deliberately does not authenticate data; -/// readers still use [`UuidMembershipIndex::open`] before trusting membership. -pub fn uuid_membership_index_is_fresh(project_dir: &Path) -> Result { - let body = match fs::read(project_dir.join(INDEX_DIR).join(MANIFEST)) { - Ok(body) => body, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), - Err(error) => return Err(storage_err(error)), - }; - let manifest = decode_manifest(&body)?; - Ok(manifest.current_generation == crate::read_topology_generation(project_dir)?) -} - -const TOPOLOGY_RECEIPT: &str = "topology-receipt.json"; const MAX_MANIFEST_BYTES: u64 = 1 << 20; -fn validate_manifest_version(version: u32) -> Result<(), GfError> { - if version != FORMAT_VERSION { - return Err(storage_err(format!( - "unsupported UUID membership format version {version}; recreate the index" - ))); - } - Ok(()) -} - -/// Refuse retired or future schemas before decoding current checksum fields. -fn decode_manifest(bytes: &[u8]) -> Result { - if bytes.len() as u64 > MAX_MANIFEST_BYTES { - return Err(storage_err("UUID membership manifest exceeds size limit")); - } - let value: serde_json::Value = serde_json::from_slice(bytes).map_err(storage_err)?; - let version = value - .get("format_version") - .and_then(serde_json::Value::as_u64) - .and_then(|version| u32::try_from(version).ok()) - .unwrap_or(0); - validate_manifest_version(version)?; - serde_json::from_value(value).map_err(storage_err) +fn hex_bytes(bytes: &[u8]) -> String { + bytes.iter().fold( + String::with_capacity(bytes.len().saturating_mul(2)), + |mut output, byte| { + write!(output, "{byte:02x}").expect("writing to String cannot fail"); + output + }, + ) } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -873,13 +263,6 @@ struct TopologyIndexReceipt { manifest_sha256: String, } -pub(crate) struct PreparedUuidIndexDelta { - expected_generation: u64, - auxiliary: crate::AuxiliaryReceipt, - metrics: UuidIndexAppendMetrics, - manifest: Manifest, -} - pub(crate) struct PreparedV4OrdinalDelta { expected_generation: u64, auxiliary: crate::AuxiliaryReceipt, @@ -917,428 +300,19 @@ pub(crate) enum CommittedUuidTopologyRewrite { NoTopologyChange, Committed { generation: u64, - metrics: UuidIndexAppendMetrics, - v4_metrics: Option, - }, - CommittedNeedsRefresh { - generation: u64, - metrics: UuidIndexAppendMetrics, - v4_metrics: Option, - error: GfError, + probe: crate::UuidProbeMetrics, + v4_metrics: Option>, }, } -#[cfg(test)] -thread_local! { - static FAIL_AFTER_MANIFEST_SUSPEND: std::cell::Cell = const { std::cell::Cell::new(false) }; -} - -fn record_length(record: &FileRecord, width: u64) -> Result { - if width == IDENTITY_RECORD_BYTES { - record.blocks.iter().try_fold(0_u64, |total, block| { - total - .checked_add(u64::from(block.len)) - .ok_or_else(|| storage_err("record length overflow")) - }) - } else { - record - .count - .checked_mul(width) - .ok_or_else(|| storage_err("record length overflow")) - } -} - -fn block_layout(bytes: &[u8], width: usize) -> Result<(u64, &[u8], &[u8]), GfError> { - if width == IDENTITY_RECORD_WIDTH { - return identity_codec::layout(bytes); - } - if width != NODE_LOOKUP_RECORD_WIDTH || bytes.is_empty() || !bytes.len().is_multiple_of(width) { - return Err(storage_err("partial UUID run record")); - } - Ok(( - (bytes.len() / width) as u64, - &bytes[..8], - &bytes[bytes.len() - width..bytes.len() - width + 8], - )) -} - -fn block_matches(bytes: &[u8], block: &BlockRecord, width: usize) -> bool { - block_layout(bytes, width).is_ok_and(|(_, first, last)| { - crate::corruption_checksum::checksum(bytes) == block.xxh64 - && hex_sha256_key(first) == block.first_key - && hex_sha256_key(last) == block.last_key - }) -} - -fn open_verified_at( - directory: &graphforge_filesystem::StableDirectory, - record: &FileRecord, - record_bytes: u64, -) -> Result { - if Path::new(&record.name).components().count() != 1 { - return Err(storage_err("manifest contains a non-local index filename")); - } - let mut file = open_uuid_child_file(directory, std::ffi::OsStr::new(&record.name))?; - let expected = record_length(record, record_bytes)?; - if file.metadata().map_err(storage_err)?.len() != expected { - return Err(storage_err("retained run authentication failed")); - } - authenticate_file_blocks(&mut file, record, record_bytes, None)?; - file.rewind().map_err(storage_err)?; - Ok(file) -} - -fn authenticate_file_blocks( - file: &mut File, - record: &FileRecord, - record_bytes: u64, - mut work: Option<&mut UuidIndexAppendMetrics>, -) -> Result<(), GfError> { - validate_block_records(record, record_bytes)?; - let mut count = 0_u64; - let mut checksum = crate::corruption_checksum::Checksum::new(); - for block in &record.blocks { - file.seek(SeekFrom::Start(block.offset)) - .map_err(storage_err)?; - let mut bytes = vec![0_u8; block.len as usize]; - file.read_exact(&mut bytes).map_err(storage_err)?; - checksum.update(&bytes); - let width = usize::try_from(record_bytes) - .map_err(|_| storage_err("record width does not fit address space"))?; - if !block_matches(&bytes, block, width) { - return Err(storage_err("UUID run block authentication failed")); - } - count = count - .checked_add(block_layout(&bytes, width)?.0) - .ok_or_else(|| storage_err("record count overflow"))?; - if let Some(metrics) = work.as_deref_mut() { - metrics.validation_scan_bytes = metrics - .validation_scan_bytes - .saturating_add(bytes.len() as u64); - metrics.validation_scan_blocks = metrics.validation_scan_blocks.saturating_add(1); - } - } - if file.metadata().map_err(storage_err)?.len() != record_length(record, record_bytes)? - || count != record.count - || checksum.finish() != record.xxh64 - { - return Err(storage_err("UUID run authentication failed")); - } - Ok(()) -} - -fn validate_block_records(record: &FileRecord, record_bytes: u64) -> Result<(), GfError> { - let expected = record_length(record, record_bytes)?; - if expected == 0 { - if record.count != 0 || !record.blocks.is_empty() { - return Err(storage_err("empty UUID run has authenticated blocks")); - } - return Ok(()); - } - let key_hex_len = if record_bytes == IDENTITY_RECORD_BYTES { - 32 - } else { - 16 - }; - let mut offset = 0_u64; - for block in &record.blocks { - if block.offset != offset - || block.len == 0 - || (record_bytes != IDENTITY_RECORD_BYTES && u64::from(block.len) % record_bytes != 0) - || block.len as usize > BULK_IO_BYTES - || block.first_key.len() != key_hex_len - || block.last_key.len() != key_hex_len - || block.first_key > block.last_key - { - return Err(storage_err("UUID run block table is not canonical")); - } - offset = offset.saturating_add(u64::from(block.len)); - } - if (record_bytes == IDENTITY_RECORD_BYTES - && (expected < record.count.saturating_mul(17) - || expected > record.count.saturating_mul(25))) - || offset != expected - || record - .blocks - .windows(2) - .any(|pair| pair[0].last_key >= pair[1].first_key) - { - return Err(storage_err("UUID run block fences are not canonical")); - } - Ok(()) -} - -fn describe_run( - path: &Path, - kind: &str, - generation: u64, - width: u64, -) -> Result { - let length = path.metadata().map_err(storage_err)?.len(); - if width != IDENTITY_RECORD_BYTES && length % width != 0 { - return Err(storage_err("internal run has a partial index record")); - } - let (sha256, xxh64, blocks, count) = describe_blocks(&mut open_uuid_file(path)?, width)?; - Ok(FileRecord { - name: format!("{kind}-{generation}-{}.uuidx", &sha256[..16]), - count, - sha256, - xxh64, - blocks, - }) -} - -fn describe_blocks( - file: &mut File, - width: u64, -) -> Result<(String, u64, Vec, u64), GfError> { - describe_stream( - file, - usize::try_from(width).map_err(storage_err)?, - &mut (0, 0), - ) -} - -/// Read bounded physical windows while carrying at most one partial record. -/// Published authentication blocks always end at a complete record boundary. -fn describe_stream( - file: &mut impl Read, - width: usize, - reads: &mut (u64, u64), -) -> Result<(String, u64, Vec, u64), GfError> { - if !matches!(width, IDENTITY_RECORD_WIDTH | NODE_LOOKUP_RECORD_WIDTH) { - return Err(storage_err("unsupported UUID run record width")); - } - let mut buffer = vec![0_u8; BULK_IO_BYTES]; - let mut carried = 0; - let mut offset = 0_u64; - let mut count = 0_u64; - let mut whole = crate::payload_digest::PayloadSha256::new(); - let mut checksum = crate::corruption_checksum::Checksum::new(); - let mut blocks = Vec::new(); - loop { - let mut filled = carried; - let mut eof = false; - while filled < buffer.len() { - let read = file.read(&mut buffer[filled..]).map_err(storage_err)?; - if read == 0 { - eof = true; - break; - } - filled += read; - reads.0 = reads.0.saturating_add(read as u64); - reads.1 = reads.1.saturating_add(1); - } - if filled == 0 { - break; - } - let valid = if width == IDENTITY_RECORD_WIDTH { - let mut valid = 0; - while filled - valid >= identity_codec::EDGE_WIDTH { - let length = match buffer[valid + 16] { - 1 => identity_codec::EDGE_WIDTH, - 0 | 2 | 3 => IDENTITY_RECORD_WIDTH, - _ => return Err(storage_err("identity record kind is invalid")), - }; - if filled - valid < length { - break; - } - valid += length; - } - valid - } else { - filled / width * width - }; - if valid == 0 || (eof && valid != filled) { - return Err(storage_err("internal run has a partial index record")); - } - let bytes = &buffer[..valid]; - let (records, first, last) = block_layout(bytes, width)?; - whole.update(bytes); - checksum.update(bytes); - blocks.push(BlockRecord { - offset, - len: u32::try_from(valid).map_err(storage_err)?, - first_key: hex_sha256_key(first), - last_key: hex_sha256_key(last), - xxh64: crate::corruption_checksum::checksum(bytes), - }); - offset = offset - .checked_add(valid as u64) - .ok_or_else(|| storage_err("block offset overflow"))?; - count = count - .checked_add(records) - .ok_or_else(|| storage_err("record count overflow"))?; - carried = filled - valid; - buffer.copy_within(valid..filled, 0); - if eof { - break; +impl CommittedUuidTopologyRewrite { + /// The topology generation the rewrite committed, if it changed any. + pub(crate) fn generation(&self) -> Option { + match self { + Self::NoTopologyChange => None, + Self::Committed { generation, .. } => Some(*generation), } } - Ok(( - hex_bytes(&whole.finalize()), - checksum.finish(), - blocks, - count, - )) -} - -fn hex_sha256_key(bytes: &[u8]) -> String { - hex_bytes(bytes) -} - -fn hex_bytes(bytes: &[u8]) -> String { - bytes.iter().fold( - String::with_capacity(bytes.len().saturating_mul(2)), - |mut output, byte| { - write!(output, "{byte:02x}").expect("writing to String cannot fail"); - output - }, - ) -} -impl PreparedUuidIndexDelta { - pub(crate) fn auxiliary_receipt(&self) -> crate::AuxiliaryReceipt { - self.auxiliary.clone() - } - - pub(crate) fn verify_generation(&self, committed_generation: u64) -> Result<(), GfError> { - if committed_generation != self.expected_generation { - return Err(storage_err( - "topology commit returned an unexpected generation", - )); - } - Ok(()) - } - - pub(crate) fn metrics(&self) -> &UuidIndexAppendMetrics { - &self.metrics - } - - pub(crate) fn advance_snapshot( - &self, - snapshot: &mut AuthenticatedUuidIndexSnapshot, - ) -> Result { - snapshot.advance_to(self.manifest.clone()) - } -} - -fn validate_run_descriptors(manifest: &Manifest) -> Result<(), GfError> { - validate_manifest_version(manifest.format_version)?; - for record in manifest.runs.iter().flat_map(|run| { - [ - (&run.identities, IDENTITY_RECORD_BYTES), - (&run.node_surrogates, NODE_LOOKUP_RECORD_BYTES), - ] - }) { - validate_block_records(record.0, record.1)?; - } - let mut levels = BTreeSet::new(); - let mut intervals = manifest - .runs - .iter() - .map(|run| (run.first_generation, run.last_generation)) - .collect::>(); - intervals.sort_unstable(); - let bases = manifest - .runs - .iter() - .filter(|run| run.base) - .collect::>(); - if bases.len() != 1 - || bases[0].first_generation != 0 - || bases[0].last_generation != manifest.base_generation - || manifest - .runs - .iter() - .filter(|run| !run.base) - .any(|run| run.first_generation > run.last_generation || !levels.insert(run.level)) - { - return Err(storage_err( - "manifest runs violate canonical level/interval policy", - )); - } - if intervals.last().map_or(0, |interval| interval.1) != manifest.current_generation - || intervals - .windows(2) - .any(|pair| pair[0].1.saturating_add(1) != pair[1].0) - { - return Err(storage_err( - "manifest generation intervals are not contiguous", - )); - } - Ok(()) -} - -fn validate_run_contents( - identities: File, - surrogates: File, - descriptor: &RunRecord, -) -> Result<(), GfError> { - let mut identities = BufReader::with_capacity(BULK_IO_BYTES, identities); - let mut surrogates = BufReader::with_capacity(BULK_IO_BYTES, surrogates); - let mut previous_uuid = None; - let mut node_count = 0_u64; - let mut edge_count = 0_u64; - let mut deleted_node_count = 0_u64; - let mut deleted_edge_count = 0_u64; - for _ in 0..descriptor.identities.count { - let record = identity_codec::read(&mut identities)? - .ok_or_else(|| storage_err("identity run is truncated"))?; - let uuid: [u8; 16] = record[..16].try_into().expect("fixed record"); - if previous_uuid.is_some_and(|previous| previous >= uuid) { - return Err(storage_err( - "identity run is not canonical and strictly sorted", - )); - } - previous_uuid = Some(uuid); - let surrogate = u64::from_be_bytes(record[17..25].try_into().expect("fixed record")); - match record[16] { - 0 if surrogate != 0 => node_count += 1, - 1 if surrogate == 0 => edge_count += 1, - 2 if surrogate != 0 => deleted_node_count += 1, - 3 if surrogate == 0 => deleted_edge_count += 1, - _ => return Err(storage_err("identity run contains an invalid kind")), - } - } - if node_count != descriptor.node_count - || edge_count != descriptor.edge_count - || deleted_node_count != descriptor.deleted_node_count - || deleted_edge_count != descriptor.deleted_edge_count - || descriptor.identities.count - != node_count + edge_count + deleted_node_count + deleted_edge_count - || descriptor.node_surrogates.count != node_count + deleted_node_count - { - return Err(storage_err("run descriptor counts do not reconcile")); - } - let mut previous_surrogate = None; - for _ in 0..node_count + deleted_node_count { - let mut record = [0_u8; 24]; - surrogates.read_exact(&mut record).map_err(storage_err)?; - let surrogate = u64::from_be_bytes(record[..8].try_into().expect("fixed record")); - if previous_surrogate.is_some_and(|previous| previous >= surrogate) { - return Err(storage_err("surrogate run is not strictly sorted")); - } - previous_surrogate = Some(surrogate); - } - Ok(()) -} - -fn open_verified(root: &Path, record: &FileRecord, record_bytes: u64) -> Result { - if Path::new(&record.name).components().count() != 1 { - return Err(storage_err("manifest contains a non-local index filename")); - } - let path = root.join(&record.name); - let mut file = File::open(&path).map_err(storage_err)?; - let expected_len = record_length(record, record_bytes)?; - if file.metadata().map_err(storage_err)?.len() != expected_len { - return Err(storage_err(format!( - "length mismatch for {}", - path.display() - ))); - } - authenticate_file_blocks(&mut file, record, record_bytes, None)?; - file.seek(SeekFrom::Start(0)).map_err(storage_err)?; - Ok(file) } #[cfg(test)] thread_local! { @@ -1462,26 +436,5 @@ fn v4_compaction_post_write_failure(point: &str) -> Result<(), GfError> { Ok(()) } -fn describe_staged_data( - source: &Path, - kind: &str, - generation: u64, - record_bytes: u64, -) -> Result { - let length = source.metadata().map_err(storage_err)?.len(); - if record_bytes != IDENTITY_RECORD_BYTES && length % record_bytes != 0 { - return Err(storage_err("internal run has a partial index record")); - } - let mut input = File::open(source).map_err(storage_err)?; - let (sha256, xxh64, blocks, count) = describe_blocks(&mut input, record_bytes)?; - Ok(FileRecord { - name: format!("{kind}-{generation}-{}.uuidx", &sha256[..16]), - count, - sha256, - xxh64, - blocks, - }) -} -#[cfg(test)] #[cfg(test)] pub(crate) mod tests; diff --git a/crates/graphforge-storage/src/uuid_membership/construction.rs b/crates/graphforge-storage/src/uuid_membership/construction.rs index 62509c4fb..5b4295c31 100644 --- a/crates/graphforge-storage/src/uuid_membership/construction.rs +++ b/crates/graphforge-storage/src/uuid_membership/construction.rs @@ -1,45 +1,21 @@ //! Construction identity encoding, merge cursors, and guarded recovery. -use super::AuthenticatedUuidIndexSnapshot; use super::BULK_IO_BYTES; -use super::CONSTRUCTION_INTENT; -use super::CONSTRUCTION_INTENT_FORMAT_VERSION; -use super::ConstructionIndexEncoding; use super::ConstructionIndexOutput; -use super::FORMAT_VERSION; -use super::FileRecord; -use super::IDENTITY_RECORD_BYTES; -use super::IDENTITY_RECORD_WIDTH; -use super::MANIFEST; use super::MAX_MANIFEST_BYTES; -use super::Manifest; -use super::NODE_LOOKUP_RECORD_BYTES; -use super::NODE_LOOKUP_RECORD_WIDTH; -use super::RunRecord; use super::TopologyIndexReceipt; use super::V4_ORDINAL_MANIFEST; use super::V4_ORDINAL_RECEIPT; -use super::block_matches; -use super::describe_stream; use super::hex_bytes; -use super::identity_codec; -use super::maintenance::manifest_file_names; -use super::ordinal_artifacts::V4AuthorityTransactionProof; use super::ordinal_artifacts::V4PublicationGuard; use super::ordinal_artifacts::admit_v4_construction_manifest; use super::ordinal_artifacts::cleanup_v4_publication; -use super::ordinal_artifacts::commit_v4_publications; use super::storage_err; use super::topology_delta::hex_sha256; use super::topology_delta::read_bounded; use super::v4_authority_failure; -use super::validate_run_descriptors; -use crate::construction_record_layout::BASE_IDENTITY_WIDTH as CONSTRUCTION_IDENTITY_WIDTH; -use crate::construction_record_layout::IDENTITY_SURROGATE_OFFSET; use graphforge_core::GfError; use graphforge_core::hash_observation::ArtifactSha256 as Sha256; -use serde::Deserialize; -use serde::Serialize; use sha2::Digest; use std::collections::BTreeSet; use std::fs::File; @@ -47,19 +23,12 @@ use std::io::Read; use std::io::Write; use uuid::Uuid; +/// Writes performed while installing one construction control file. #[derive(Default)] pub(super) struct ConstructionIndexWork { - read_bytes: u64, - read_operations: u64, pub(super) write_bytes: u64, pub(super) write_operations: u64, pub(super) fsync_operations: u64, - created_runs: u64, - retained_runs: u64, - retained_payload_bytes: u64, - peak_buffer_bytes: u64, - peak_temporary_bytes: u64, - cache_release: graphforge_filesystem::FileCacheReleaseEvidence, } pub(super) fn merge_cache_release_evidence( @@ -79,755 +48,243 @@ pub(super) fn merge_cache_release_evidence( target.peak_window_bytes = target.peak_window_bytes.max(source.peak_window_bytes); } -#[derive(Serialize, Deserialize)] -struct ConstructionRecoveryIntent { - format_version: u32, - generation: u64, - parent_generation: u64, - identities_name: String, - source_volume: u64, - source_file_id: String, - source_bytes: u64, - source_xxh64: String, - authority_sha256: String, -} - -struct ConstructionIndexCleanupGuard<'a> { - allocation: Option, - encoded: &'a graphforge_filesystem::StableDirectory, - armed: bool, +pub(super) fn authenticate_private_v4_artifact_file( + file: File, + artifact: &crate::V4OrdinalArtifact, +) -> Result<(), GfError> { + if graphforge_filesystem::file_link_count(&file).map_err(storage_err)? != 1 + || file.metadata().map_err(storage_err)?.len() != artifact.bytes + || checksum_reader_streaming(file)? != (artifact.xxh64, artifact.bytes) + { + return Err(storage_err("private v4 artifact authentication failed")); + } + Ok(()) } -impl ConstructionIndexCleanupGuard<'_> { - fn disarm(&mut self) { - self.armed = false; +pub(crate) fn is_exact_private_v4_name(name: &str) -> bool { + if let Some(rest) = name.strip_prefix(".v4-") { + let Some((role, nonce)) = rest.strip_suffix(".tmp").and_then(|v| v.rsplit_once('-')) else { + return false; + }; + let role_ok = role == "forward" + || role == "tombstones" + || role.strip_prefix("ordinal-").is_some_and(|ordinal| { + ordinal.len() == 8 && ordinal.bytes().all(|b| b.is_ascii_digit()) + }); + return role_ok && canonical_lower_hex(nonce, 32); } + let prefix = if name.starts_with("forward-v4-") { + "forward-v4-" + } else if name.starts_with("ordinal-v4-") { + "ordinal-v4-" + } else if name.starts_with("tombstones-v4-") { + "tombstones-v4-" + } else { + return false; + }; + let Some((generation, digest)) = name + .strip_prefix(prefix) + .and_then(|value| value.strip_suffix(".uuidx")) + .and_then(|value| value.rsplit_once('-')) + else { + return false; + }; + generation + .parse::() + .is_ok_and(|value| value != 0 && value.to_string() == generation) + && canonical_lower_hex(digest, 16) } -impl Drop for ConstructionIndexCleanupGuard<'_> { - fn drop(&mut self) { - if self.armed { - let _ = cleanup_private_construction_index_with_allocation( - self.encoded, - self.allocation.as_ref(), - ); - } - } +fn canonical_lower_hex(value: &str, length: usize) -> bool { + value.len() == length + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) } -impl ConstructionRecoveryIntent { - fn authenticate(&self) -> Result<(), GfError> { - let expected = construction_intent_digest( - self.format_version, - self.generation, - self.parent_generation, - &self.identities_name, - self.source_volume, - &self.source_file_id, - self.source_bytes, - &self.source_xxh64, - ); - if self.format_version != CONSTRUCTION_INTENT_FORMAT_VERSION - || !canonical_lower_hex(&self.source_xxh64, 16) - || self.authority_sha256 != expected +fn checksum_reader_streaming(file: File) -> Result<(u64, u64), GfError> { + let identity = graphforge_filesystem::file_identity(&file).map_err(storage_err)?; + let expected_bytes = file.metadata().map_err(storage_err)?.len(); + let mut reader = + graphforge_filesystem::FileCacheReleasingReader::new(file).map_err(storage_err)?; + let checked = (|| { + let mut checksum = crate::corruption_checksum::Checksum::new(); + let mut bytes = 0_u64; + let mut buffer = vec![0; BULK_IO_BYTES]; + loop { + let count = reader.read(&mut buffer).map_err(storage_err)?; + if count == 0 { + break; + } + bytes = bytes + .checked_add(count as u64) + .ok_or_else(|| storage_err("private ordinal length overflow"))?; + if bytes > expected_bytes { + return Err(storage_err("private ordinal artifact grew")); + } + checksum.update(&buffer[..count]); + } + if bytes != expected_bytes + || graphforge_filesystem::file_identity(reader.file()).map_err(storage_err)? != identity + || graphforge_filesystem::file_link_count(reader.file()).map_err(storage_err)? != 1 + || reader.file().metadata().map_err(storage_err)?.len() != expected_bytes { return Err(storage_err( - "construction recovery intent authentication failed", + "private ordinal artifact identity or length changed", )); } - Ok(()) + Ok((checksum.finish(), bytes)) + })(); + let cleanup = reader.finish().map_err(storage_err); + match (checked, cleanup) { + (Ok(value), Ok(_)) => Ok(value), + (Err(primary), Ok(_)) | (Ok(_), Err(primary)) => Err(primary), + (Err(primary), Err(cleanup)) => Err(storage_err(format!( + "{primary}; private ordinal cache cleanup also failed: {cleanup}" + ))), } } -/// Where the UUID-ordered identity delta comes from. -pub(crate) enum ConstructionIdentityInput<'a> { - /// The shaper's durable, authenticated identity run. - Shaped { - source: &'a graphforge_filesystem::StableDirectory, - name: &'a str, - xxh64: &'a str, - }, - /// The same records generated in memory by the bulk builder. They are the - /// shaped run's bytes; only their durable staging is skipped. - Stream { - reader: Box, - len: u64, - }, -} - -enum IdentityReader<'a> { - File(Option), - Stream(Box), - Releasing(graphforge_filesystem::FileCacheReleasingReader), -} - -impl IdentityReader<'_> { - fn into_reader(self, window: std::num::NonZeroU64) -> Result { - match self { - Self::File(Some(file)) => Ok(Self::Releasing( - graphforge_filesystem::FileCacheReleasingReader::with_window_bytes( - file, - window, - graphforge_filesystem::FileCacheReleaseTracker::default(), - ) - .map_err(storage_err)?, - )), - other => Ok(other), - } - } - - fn finish(self) -> Result { - match self { - Self::Releasing(mut reader) => reader.finish().map_err(storage_err), - _ => Ok(graphforge_filesystem::FileCacheReleaseEvidence::default()), - } +pub(super) fn combine_cache_cleanup( + primary: Result, + cleanup: Result<(), GfError>, + source: &str, +) -> Result { + match (primary, cleanup) { + (Ok(value), Ok(())) => Ok(value), + (Ok(_), Err(cleanup)) => Err(cleanup), + (Err(primary), Ok(())) => Err(primary), + (Err(primary), Err(cleanup)) => Err(storage_err(format!( + "{primary}; {source} cache release also failed: {cleanup}" + ))), } } -impl std::io::Read for IdentityReader<'_> { - fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { - match self { - Self::Releasing(reader) => reader.read(buffer), - Self::Stream(reader) => reader.read(buffer), - Self::File(_) => Err(std::io::Error::other("identity reader is not initialized")), - } +pub(super) fn combine_v4_cleanup( + primary: Result, + cleanup: Result<(), GfError>, + context: &str, +) -> Result { + match (primary, cleanup) { + (Ok(value), Ok(())) => Ok(value), + (Ok(_), Err(cleanup)) => Err(cleanup), + (Err(primary), Ok(())) => Err(primary), + (Err(primary), Err(cleanup)) => Err(storage_err(format!( + "{primary}; {context} also failed: {cleanup}" + ))), } } -#[allow(clippy::too_many_arguments)] -fn construction_intent_digest( - format_version: u32, - generation: u64, - parent_generation: u64, - identities_name: &str, - source_volume: u64, - source_file_id: &str, - source_bytes: u64, - source_xxh64: &str, -) -> String { - let mut digest = graphforge_core::hash_observation::ControlSha256::new(); - digest.update(b"graphforge.uuid-membership.construction-intent.v3\0"); - digest.update(format_version.to_be_bytes()); - digest.update(generation.to_be_bytes()); - digest.update(parent_generation.to_be_bytes()); - digest.update((identities_name.len() as u64).to_be_bytes()); - digest.update(identities_name.as_bytes()); - digest.update(source_volume.to_be_bytes()); - digest.update(source_file_id.as_bytes()); - digest.update(source_bytes.to_be_bytes()); - digest.update(source_xxh64.as_bytes()); - hex_bytes(&digest.finalize()) -} - -/// Encode the shaper's UUID-ordered 32-byte delta directly as a v3 membership -/// participant. Retained descriptors are cloned, not rebuilt from topology; -/// retained payloads are read only when binary-carry compaction is required. -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] -pub(crate) fn encode_construction_index( - input: ConstructionIdentityInput<'_>, - encoded: &graphforge_filesystem::StableDirectory, - generation: u64, - parent_generation: u64, - parent: Option<&AuthenticatedUuidIndexSnapshot>, - live_nodes: u64, - live_edges: u64, - cancelled: &mut impl FnMut() -> bool, +pub(super) fn install_construction_bytes( + output: &graphforge_filesystem::StableDirectory, + name: &str, + bytes: &[u8], + work: &mut ConstructionIndexWork, allocation: Option<&crate::StorageAllocationOperation>, -) -> Result { - cleanup_private_construction_index_with_allocation(encoded, allocation)?; - let mut cleanup_guard = ConstructionIndexCleanupGuard { - encoded, - armed: true, - allocation: allocation.cloned(), +) -> Result<(ConstructionIndexOutput, V4PublicationGuard), GfError> { + let temporary = format!(".{name}-{}.tmp", Uuid::new_v4().simple()); + let mut publication = + V4PublicationGuard::create(output, &temporary, allocation).map_err(storage_err)?; + let mut file = publication.take_file().map_err(storage_err)?; + let written = file.write_all(bytes).map_err(storage_err); + let observed = publication.observe(&file).map_err(storage_err); + written?; + observed?; + work.write_bytes = work.write_bytes.saturating_add(bytes.len() as u64); + work.write_operations = work.write_operations.saturating_add(1); + let seal = crate::durable_commit::seal_file_witness(&file).map_err(storage_err)?; + publication.record_seal(seal); + publication.observe(&file).map_err(storage_err)?; + work.fsync_operations = work.fsync_operations.saturating_add(1); + let failpoint = match name { + V4_ORDINAL_RECEIPT => Some("v4_publish.after_receipt_temp_fsync"), + V4_ORDINAL_MANIFEST => Some("v4_publish.after_manifest_temp_fsync"), + "ordinal-v4.lock" => Some("v4_publish.after_lock_temp_fsync"), + _ => None, }; - let result = encode_construction_index_inner( - input, - encoded, - generation, - parent_generation, - parent, - live_nodes, - live_edges, - cancelled, - allocation, - ); - match result { - Ok(value) => { - cleanup_guard.disarm(); - Ok(value) - } - Err(original) => { - cleanup_private_construction_index_with_allocation(encoded, allocation).map_err( - |cleanup| storage_err(format!("{original}; exact cleanup also failed: {cleanup}")), - )?; - Err(original) - } + if let Some(failpoint) = failpoint { + crate::graph_construction::construction_failpoint(failpoint); } -} - -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] -fn encode_construction_index_inner( - input: ConstructionIdentityInput<'_>, - encoded: &graphforge_filesystem::StableDirectory, - generation: u64, - parent_generation: u64, - parent: Option<&AuthenticatedUuidIndexSnapshot>, - live_nodes: u64, - live_edges: u64, - cancelled: &mut impl FnMut() -> bool, - allocation: Option<&crate::StorageAllocationOperation>, -) -> Result { - let encoded = - crate::construction_directory::ConstructionDirectory::from_physical(encoded, allocation) - .map_err(storage_err)?; - let graph = encoded - .create_child_directory(std::ffi::OsStr::new("graph")) - .map_err(storage_err)?; - let topology = graph - .create_child_directory(std::ffi::OsStr::new("topology")) - .map_err(storage_err)?; - let index = topology - .create_child_directory(std::ffi::OsStr::new("uuid-membership")) - .map_err(storage_err)?; - let mut manifest = if parent_generation == 0 { - if parent.is_some() { - return Err(storage_err("empty construction parent has UUID snapshot")); - } - Manifest { - format_version: FORMAT_VERSION, - base_generation: 0, - current_generation: 0, - live_node_count: 0, - live_edge_count: 0, - runs: Vec::new(), - } - } else { - let parent = parent.ok_or_else(|| { - storage_err("nonempty construction parent lacks authenticated UUID snapshot") - })?; - parent.revalidate()?; - if parent.manifest.current_generation != parent_generation { - return Err(storage_err("construction UUID parent generation changed")); - } - parent.manifest.clone() + drop(file); + let installed = publication + .install_child(std::ffi::OsStr::new(name)) + .map_err(storage_err); + if let Err(primary) = installed { + let cleanup = cleanup_v4_publication(&mut publication); + return combine_v4_cleanup(Err(primary), cleanup, "v4 construction control cleanup"); + } + let authority_point = match name { + V4_ORDINAL_RECEIPT => Some("receipt_install"), + V4_ORDINAL_MANIFEST => Some("manifest_install"), + "ordinal-v4.lock" => Some("lock_install"), + _ => None, }; - if generation != parent_generation.saturating_add(1) { - return Err(storage_err( - "construction UUID generation is not consecutive", - )); + if let Some(point) = authority_point + && let Err(primary) = v4_authority_failure(point) + { + let cleanup = cleanup_v4_publication(&mut publication); + return combine_v4_cleanup(Err(primary), cleanup, "v4 construction control cleanup"); } + work.fsync_operations = work.fsync_operations.saturating_add(1); + Ok(( + ConstructionIndexOutput { + name: name.to_owned(), + bytes: bytes.len() as u64, + xxh64: crate::corruption_checksum::checksum(bytes), + sha256: if matches!(name, V4_ORDINAL_RECEIPT | V4_ORDINAL_MANIFEST) { + hex_sha256(bytes) + } else { + hex_bytes(&graphforge_core::hash_observation::ControlSha256::digest( + bytes, + )) + }, + }, + publication, + )) +} - let mut work = ConstructionIndexWork::default(); - let identity_temp = format!(".construction-identities-{}.tmp", Uuid::new_v4().simple()); - let surrogate_temp = format!(".construction-surrogates-{}.tmp", Uuid::new_v4().simple()); - let (input, identities_name, identities_xxh64, source_volume, source_file_id, input_len) = - match input { - ConstructionIdentityInput::Shaped { - source, - name, - xxh64, - } => { - let file = source - .open_child_file(std::ffi::OsStr::new(name)) - .map_err(storage_err)?; - let len = file.metadata().map_err(storage_err)?.len(); - let identity = graphforge_filesystem::file_identity(&file).map_err(storage_err)?; - ( - IdentityReader::File(Some(file)), - name, - Some(xxh64), - identity.volume_serial, - hex_bytes(&identity.file_id), - len, - ) - } - ConstructionIdentityInput::Stream { reader, len } => ( - IdentityReader::Stream(reader), - "", - None, - 0, - String::new(), - len, - ), - }; - if input_len % CONSTRUCTION_IDENTITY_WIDTH as u64 != 0 { - return Err(storage_err("construction identity stream is truncated")); - } - let mut intent = ConstructionRecoveryIntent { - format_version: CONSTRUCTION_INTENT_FORMAT_VERSION, - generation, - parent_generation, - identities_name: identities_name.to_owned(), - source_volume, - source_file_id: source_file_id.clone(), - source_bytes: input_len, - source_xxh64: identities_xxh64.unwrap_or("0000000000000000").to_owned(), - authority_sha256: String::new(), +/// Remove the private ordinal-facet residue a crashed encoding attempt left in +/// the encoded tree, so a rerun starts from an empty directory. Only exact +/// private v4 names (and the retired membership names an older binary may have +/// left) are removed; anything else is refused. +pub(crate) fn clear_private_ordinal_residue( + encoded: &crate::construction_directory::ConstructionDirectory, +) -> Result<(), GfError> { + let allocation = encoded.allocation(); + let graph = match encoded.open_child_directory(std::ffi::OsStr::new("graph")) { + Ok(value) => value, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(storage_err(error)), }; - intent.authority_sha256 = construction_intent_digest( - intent.format_version, - intent.generation, - intent.parent_generation, - &intent.identities_name, - intent.source_volume, - &intent.source_file_id, - intent.source_bytes, - &intent.source_xxh64, - ); - write_construction_intent(&index, &intent, &mut work)?; - crate::graph_construction::construction_failpoint("uuid_encode.after_intent"); - let identity_writer = index - .create_replaceable_child_file(std::ffi::OsStr::new(&identity_temp)) - .map_err(storage_err)?; - let surrogate_writer = index - .create_replaceable_child_file(std::ffi::OsStr::new(&surrogate_temp)) - .map_err(storage_err)?; - let identity_identity = - graphforge_filesystem::file_identity(&identity_writer).map_err(storage_err)?; - let surrogate_identity = - graphforge_filesystem::file_identity(&surrogate_writer).map_err(storage_err)?; - let cache_window = - graphforge_filesystem::cache_release_window_for_streams(3).map_err(storage_err)?; - let mut input = input.into_reader(cache_window)?; - let mut identity_writer = graphforge_filesystem::DurableFileCacheWriter::with_window_bytes( - identity_writer, - cache_window, - ) - .map_err(storage_err)?; - let mut surrogate_writer = graphforge_filesystem::DurableFileCacheWriter::with_window_bytes( - surrogate_writer, - cache_window, - ) - .map_err(storage_err)?; - crate::graph_construction::construction_failpoint("uuid_encode.after_temps"); - let aligned_input_bytes = - (BULK_IO_BYTES / CONSTRUCTION_IDENTITY_WIDTH) * CONSTRUCTION_IDENTITY_WIDTH; - let aligned_surrogate_bytes = - (BULK_IO_BYTES / NODE_LOOKUP_RECORD_WIDTH) * NODE_LOOKUP_RECORD_WIDTH; - let mut input_block = vec![0_u8; aligned_input_bytes]; - let mut surrogate_block = Vec::with_capacity(aligned_surrogate_bytes); - work.peak_buffer_bytes = (input_block.len() + surrogate_block.capacity()) as u64; - let mut previous_uuid = None; - let mut previous_surrogate = 0_u64; - let mut node_count = 0_u64; - let mut edge_count = 0_u64; - let mut source_digest = Sha256::new(); - let mut source_checksum = crate::corruption_checksum::Checksum::new(); - let mut remaining = input_len; - let streamed = (|| -> Result<(), GfError> { - while remaining != 0 { - if cancelled() { - return Err(storage_err("construction index encoding cancelled")); - } - let count = - usize::try_from(remaining.min(input_block.len() as u64)).map_err(storage_err)?; - input - .read_exact(&mut input_block[..count]) - .map_err(storage_err)?; - source_digest.update(&input_block[..count]); - source_checksum.update(&input_block[..count]); - work.read_bytes = work.read_bytes.saturating_add(count as u64); - work.read_operations = work.read_operations.saturating_add(1); - let mut packed_len = 0; - for source_offset in (0..count).step_by(CONSTRUCTION_IDENTITY_WIDTH) { - let record = - &input_block[source_offset..source_offset + CONSTRUCTION_IDENTITY_WIDTH]; - let mut packed = [0_u8; IDENTITY_RECORD_WIDTH]; - packed[..17].copy_from_slice(&record[..17]); - packed[17..].copy_from_slice( - &record[IDENTITY_SURROGATE_OFFSET..CONSTRUCTION_IDENTITY_WIDTH], - ); - let uuid: [u8; 16] = record[..16].try_into().expect("fixed UUID"); - if previous_uuid.is_some_and(|prior| prior >= uuid) || record[17] != 0 { - return Err(storage_err("construction identity stream is not canonical")); - } - previous_uuid = Some(uuid); - match record[16] { - 0 => { - let surrogate = u64::from_be_bytes( - record[IDENTITY_SURROGATE_OFFSET..CONSTRUCTION_IDENTITY_WIDTH] - .try_into() - .expect("fixed"), - ); - if surrogate == 0 || surrogate <= previous_surrogate { - return Err(storage_err( - "construction node surrogate stream is not increasing", - )); - } - previous_surrogate = surrogate; - if surrogate_block.len() + NODE_LOOKUP_RECORD_WIDTH - > aligned_surrogate_bytes - { - surrogate_writer - .write_all(&surrogate_block) - .map_err(storage_err)?; - work.write_bytes = work - .write_bytes - .saturating_add(surrogate_block.len() as u64); - work.write_operations = work.write_operations.saturating_add(1); - surrogate_block.clear(); - } - surrogate_block.extend_from_slice(&surrogate.to_be_bytes()); - surrogate_block.extend_from_slice(&uuid); - node_count = node_count.saturating_add(1); - } - 1 => { - // Edge surrogates belong to topology; membership stores only the UUID. - packed[17..].fill(0); - edge_count = edge_count.saturating_add(1); - } - _ => return Err(storage_err("construction identity kind is invalid")), - } - let packed = identity_codec::encoded(&packed)?; - input_block[packed_len..packed_len + packed.len()].copy_from_slice(packed); - packed_len += packed.len(); - } - identity_writer - .write_all(&input_block[..packed_len]) - .map_err(storage_err)?; - work.write_bytes = work.write_bytes.saturating_add(packed_len as u64); - work.write_operations = work.write_operations.saturating_add(1); - remaining -= count as u64; - } - if identities_xxh64.is_some_and(|expected| { - crate::corruption_checksum::hex(source_checksum.finish()) != expected - }) { - return Err(storage_err("construction identity source digest changed")); - } - Ok(()) - })(); - let observed = (|| { - index - .observe_file(std::ffi::OsStr::new(&identity_temp), identity_writer.file()) - .map_err(storage_err)?; - index - .observe_file( - std::ffi::OsStr::new(&surrogate_temp), - surrogate_writer.file(), - ) - .map_err(storage_err) - })(); - let streamed = combine_v4_cleanup(streamed, observed, "construction allocation observation"); - let released = input.finish(); - let input_cache_release = match (streamed, released) { - (Ok(()), Ok(released)) => released, - (Ok(()), Err(release)) => return Err(release), - (Err(primary), Ok(_)) => return Err(primary), - (Err(primary), Err(release)) => { - return Err(storage_err(format!( - "{primary}; construction identity cache release also failed: {release}" - ))); - } + let topology = match graph.open_child_directory(std::ffi::OsStr::new("topology")) { + Ok(value) => value, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(storage_err(error)), }; - merge_cache_release_evidence(&mut work.cache_release, input_cache_release); - if !surrogate_block.is_empty() { - surrogate_writer - .write_all(&surrogate_block) - .map_err(storage_err)?; - work.write_bytes = work - .write_bytes - .saturating_add(surrogate_block.len() as u64); - work.write_operations = work.write_operations.saturating_add(1); + let index = match topology.open_child_directory(std::ffi::OsStr::new("uuid-membership")) { + Ok(value) => value, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(storage_err(error)), + }; + let names = if allocation.is_some() { + index.child_names_bounded(1_000_000) + } else { + index.child_names() } - if manifest.live_node_count.saturating_add(node_count) != live_nodes - || manifest.live_edge_count.saturating_add(edge_count) != live_edges - { - return Err(storage_err( - "construction UUID delta counts differ from shaped counts", - )); - } - identity_writer.flush().map_err(storage_err)?; - surrogate_writer.flush().map_err(storage_err)?; - let identity_seal = crate::durable_commit::seal_cache_writer_witness(&mut identity_writer) - .map_err(storage_err)?; - let surrogate_seal = crate::durable_commit::seal_cache_writer_witness(&mut surrogate_writer) - .map_err(storage_err)?; - index - .observe_file(std::ffi::OsStr::new(&identity_temp), identity_writer.file()) - .map_err(storage_err)?; - index - .observe_file( - std::ffi::OsStr::new(&surrogate_temp), - surrogate_writer.file(), - ) - .map_err(storage_err)?; - let identity_cache_release = identity_writer.evidence(); - let surrogate_cache_release = surrogate_writer.evidence(); - work.fsync_operations = work - .fsync_operations - .saturating_add(identity_cache_release.sync_operations) - .saturating_add(surrogate_cache_release.sync_operations); - merge_cache_release_evidence(&mut work.cache_release, identity_cache_release); - merge_cache_release_evidence(&mut work.cache_release, surrogate_cache_release); - let aggregate_peak = input_cache_release - .peak_window_bytes - .checked_add(identity_cache_release.peak_window_bytes) - .and_then(|peak| peak.checked_add(surrogate_cache_release.peak_window_bytes)) - .ok_or_else(|| storage_err("construction index aggregate cache window overflow"))?; - work.cache_release.peak_window_bytes = work.cache_release.peak_window_bytes.max(aggregate_peak); - drop(identity_writer.into_file()); - drop(surrogate_writer.into_file()); - - let mut artifacts = Vec::new(); - let identity_record = describe_and_install_construction_run( - &index, - &identity_temp, - identity_identity, - identity_seal, - "identities-v5", - generation, - IDENTITY_RECORD_WIDTH, - &mut artifacts, - &mut work, - )?; - let surrogate_record = describe_and_install_construction_run( - &index, - &surrogate_temp, - surrogate_identity, - surrogate_seal, - "node-surrogates-v5", - generation, - NODE_LOOKUP_RECORD_WIDTH, - &mut artifacts, - &mut work, - )?; - let mut output_names = artifacts - .iter() - .map(|artifact| artifact.name.clone()) - .collect::>(); - crate::graph_construction::construction_failpoint("uuid_encode.after_delta_runs"); - - if parent_generation == 0 { - let base_identity = install_empty_construction_run( - &index, - "identities-v5-base", - 0, - IDENTITY_RECORD_WIDTH, - &mut artifacts, - &mut work, - )?; - let base_surrogate = install_empty_construction_run( - &index, - "node-surrogates-v5-base", - 0, - NODE_LOOKUP_RECORD_WIDTH, - &mut artifacts, - &mut work, - )?; - output_names.insert(base_identity.name.clone()); - output_names.insert(base_surrogate.name.clone()); - manifest.runs.push(RunRecord { - base: true, - level: 0, - first_generation: 0, - last_generation: 0, - identities: base_identity, - node_surrogates: base_surrogate, - node_count: 0, - edge_count: 0, - deleted_node_count: 0, - deleted_edge_count: 0, - }); - } - manifest.runs.push(RunRecord { - base: false, - level: 0, - first_generation: generation, - last_generation: generation, - identities: identity_record, - node_surrogates: surrogate_record, - node_count, - edge_count, - deleted_node_count: 0, - deleted_edge_count: 0, - }); - - let mut retained_payload_bytes = 0_u64; - compact_construction_levels( - &index, - parent, - generation, - &mut manifest, - &mut artifacts, - &mut output_names, - &mut retained_payload_bytes, - &mut work, - cancelled, - )?; - work.retained_payload_bytes = retained_payload_bytes; - manifest.current_generation = generation; - manifest.live_node_count = live_nodes; - manifest.live_edge_count = live_edges; - manifest - .runs - .sort_unstable_by_key(|run| run.first_generation); - validate_run_descriptors(&manifest)?; - - let body = serde_json::to_vec(&manifest).map_err(storage_err)?; - let (manifest_output, manifest_publication) = - install_construction_bytes(index.physical(), MANIFEST, &body, &mut work, allocation)?; - crate::graph_construction::construction_failpoint("uuid_encode.after_manifest"); - artifacts.push(manifest_output); - let retained_names = manifest_file_names(&manifest); - let created_payload_bytes = artifacts - .iter() - .filter(|artifact| artifact.name != MANIFEST) - .map(|artifact| artifact.bytes) - .sum::(); - work.created_runs = artifacts - .iter() - .filter(|artifact| artifact.name != MANIFEST) - .count() as u64; - // All byte and operation counters above are updated at the actual read, - // write, flush, and durability sites. Never reconstruct I/O from file - // lengths here: short reads and discarded carry outputs are observable. - work.peak_buffer_bytes = work.peak_buffer_bytes.max((3 * BULK_IO_BYTES) as u64); - work.peak_temporary_bytes = created_payload_bytes.saturating_add(body.len() as u64); - for artifact in artifacts - .iter() - .filter(|artifact| artifact.name != MANIFEST && !retained_names.contains(&artifact.name)) - { - let file = index - .open_child_file(std::ffi::OsStr::new(&artifact.name)) - .map_err(storage_err)?; - let identity = graphforge_filesystem::file_identity(&file).map_err(storage_err)?; - index - .unlink_child_if_identity(std::ffi::OsStr::new(&artifact.name), identity) - .map_err(storage_err)?; - } - artifacts - .retain(|artifact| artifact.name == MANIFEST || retained_names.contains(&artifact.name)); - artifacts.sort_unstable_by(|left, right| left.name.cmp(&right.name)); - let final_write_bytes = artifacts.iter().map(|artifact| artifact.bytes).sum(); - let mut retained_references = Vec::new(); - let locally_owned = artifacts - .iter() - .map(|artifact| artifact.name.as_str()) - .collect::>(); - if let Some(parent) = parent { - let mut referenced = BTreeSet::new(); - for record in manifest - .runs - .iter() - .flat_map(|run| [&run.identities, &run.node_surrogates]) - { - if !locally_owned.contains(record.name.as_str()) - && referenced.insert(record.name.clone()) - { - retained_references.push(parent.retained_reference(record)?); - } - } - work.retained_runs = manifest - .runs - .iter() - .filter(|run| { - !locally_owned.contains(run.identities.name.as_str()) - && !locally_owned.contains(run.node_surrogates.name.as_str()) - }) - .count() as u64; - parent.revalidate()?; - } - retained_references.sort_unstable_by(|left, right| left.target_path.cmp(&right.target_path)); - let intent_file = index - .open_child_file(std::ffi::OsStr::new(CONSTRUCTION_INTENT)) - .map_err(storage_err)?; - let intent_identity = - graphforge_filesystem::file_identity(&intent_file).map_err(storage_err)?; - index - .unlink_child_if_identity(std::ffi::OsStr::new(CONSTRUCTION_INTENT), intent_identity) - .map_err(storage_err)?; - crate::graph_construction::construction_failpoint("uuid_encode.after_intent_removal"); - index.acknowledge().map_err(storage_err)?; - topology.acknowledge().map_err(storage_err)?; - graph.acknowledge().map_err(storage_err)?; - encoded.acknowledge().map_err(storage_err)?; - work.fsync_operations = work.fsync_operations.saturating_add(4); - commit_v4_publications( - vec![(MANIFEST.to_owned(), manifest_publication)], - V4AuthorityTransactionProof, - )?; - Ok(ConstructionIndexEncoding { - artifacts, - retained_references, - input_records: node_count.saturating_add(edge_count), - read_bytes: work.read_bytes, - read_operations: work.read_operations, - final_write_bytes, - write_bytes: work.write_bytes, - write_operations: work.write_operations, - fsync_operations: work.fsync_operations, - created_runs: work.created_runs, - retained_runs: work.retained_runs, - retained_payload_bytes: work.retained_payload_bytes, - peak_buffer_bytes: work.peak_buffer_bytes, - peak_temporary_bytes: work.peak_temporary_bytes, - cache_release: work.cache_release, - source_sha256: hex_bytes(&source_digest.finalize()), - }) -} - -#[cfg(test)] -fn cleanup_private_construction_index( - encoded: &graphforge_filesystem::StableDirectory, -) -> Result<(), GfError> { - cleanup_private_construction_index_with_allocation(encoded, None) -} - -fn cleanup_private_construction_index_with_allocation( - encoded: &graphforge_filesystem::StableDirectory, - allocation: Option<&crate::StorageAllocationOperation>, -) -> Result<(), GfError> { - let encoded = - crate::construction_directory::ConstructionDirectory::from_physical(encoded, allocation) - .map_err(storage_err)?; - let graph = match encoded.open_child_directory(std::ffi::OsStr::new("graph")) { - Ok(value) => value, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(error) => return Err(storage_err(error)), - }; - let topology = match graph.open_child_directory(std::ffi::OsStr::new("topology")) { - Ok(value) => value, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(error) => return Err(storage_err(error)), - }; - let index = match topology.open_child_directory(std::ffi::OsStr::new("uuid-membership")) { - Ok(value) => value, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(error) => return Err(storage_err(error)), - }; - match index.open_child_file(std::ffi::OsStr::new(CONSTRUCTION_INTENT)) { - Ok(mut file) => { - if file.metadata().map_err(storage_err)?.len() > 16 * 1024 { - return Err(storage_err("construction recovery intent is oversized")); - } - let mut body = Vec::new(); - file.read_to_end(&mut body).map_err(storage_err)?; - let intent: ConstructionRecoveryIntent = - serde_json::from_slice(&body).map_err(storage_err)?; - intent.authenticate()?; - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => return Err(storage_err(error)), - } - let names = if allocation.is_some() { - index.child_names_bounded(1_000_000) - } else { - index.child_names() - } - .map_err(storage_err)?; - let v4_allowed = authenticate_private_v4_residue(index.physical(), &names)?; - if allocation.is_some() { - for name in &names { - let file = index.open_child_file(name).map_err(storage_err)?; - index.observe_file(name, &file).map_err(storage_err)?; - } + .map_err(storage_err)?; + let v4_allowed = authenticate_private_v4_residue(index.physical(), &names)?; + if allocation.is_some() { + for name in &names { + let file = index.open_child_file(name).map_err(storage_err)?; + index.observe_file(name, &file).map_err(storage_err)?; + } } for name in names { let name_text = name .to_str() .ok_or_else(|| storage_err("construction recovery inventory name is not UTF-8"))?; - if name_text != CONSTRUCTION_INTENT - && name_text != MANIFEST + if name_text != "manifest.json" + && name_text != ".construction-intent.json" && !name_text.starts_with(".construction-") && !name_text.starts_with(".manifest.json-") && !name_text.starts_with("identities-v5") @@ -1088,103 +545,6 @@ fn authenticate_private_v4_artifact( authenticate_private_v4_artifact_file(file, artifact) } -pub(super) fn authenticate_private_v4_artifact_file( - file: File, - artifact: &crate::V4OrdinalArtifact, -) -> Result<(), GfError> { - if graphforge_filesystem::file_link_count(&file).map_err(storage_err)? != 1 - || file.metadata().map_err(storage_err)?.len() != artifact.bytes - || checksum_reader_streaming(file)? != (artifact.xxh64, artifact.bytes) - { - return Err(storage_err("private v4 artifact authentication failed")); - } - Ok(()) -} - -pub(crate) fn is_exact_private_v4_name(name: &str) -> bool { - if let Some(rest) = name.strip_prefix(".v4-") { - let Some((role, nonce)) = rest.strip_suffix(".tmp").and_then(|v| v.rsplit_once('-')) else { - return false; - }; - let role_ok = role == "forward" - || role == "tombstones" - || role.strip_prefix("ordinal-").is_some_and(|ordinal| { - ordinal.len() == 8 && ordinal.bytes().all(|b| b.is_ascii_digit()) - }); - return role_ok && canonical_lower_hex(nonce, 32); - } - let prefix = if name.starts_with("forward-v4-") { - "forward-v4-" - } else if name.starts_with("ordinal-v4-") { - "ordinal-v4-" - } else if name.starts_with("tombstones-v4-") { - "tombstones-v4-" - } else { - return false; - }; - let Some((generation, digest)) = name - .strip_prefix(prefix) - .and_then(|value| value.strip_suffix(".uuidx")) - .and_then(|value| value.rsplit_once('-')) - else { - return false; - }; - generation - .parse::() - .is_ok_and(|value| value != 0 && value.to_string() == generation) - && canonical_lower_hex(digest, 16) -} - -fn canonical_lower_hex(value: &str, length: usize) -> bool { - value.len() == length - && value - .bytes() - .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) -} - -fn checksum_reader_streaming(file: File) -> Result<(u64, u64), GfError> { - let identity = graphforge_filesystem::file_identity(&file).map_err(storage_err)?; - let expected_bytes = file.metadata().map_err(storage_err)?.len(); - let mut reader = - graphforge_filesystem::FileCacheReleasingReader::new(file).map_err(storage_err)?; - let checked = (|| { - let mut checksum = crate::corruption_checksum::Checksum::new(); - let mut bytes = 0_u64; - let mut buffer = vec![0; BULK_IO_BYTES]; - loop { - let count = reader.read(&mut buffer).map_err(storage_err)?; - if count == 0 { - break; - } - bytes = bytes - .checked_add(count as u64) - .ok_or_else(|| storage_err("private ordinal length overflow"))?; - if bytes > expected_bytes { - return Err(storage_err("private ordinal artifact grew")); - } - checksum.update(&buffer[..count]); - } - if bytes != expected_bytes - || graphforge_filesystem::file_identity(reader.file()).map_err(storage_err)? != identity - || graphforge_filesystem::file_link_count(reader.file()).map_err(storage_err)? != 1 - || reader.file().metadata().map_err(storage_err)?.len() != expected_bytes - { - return Err(storage_err( - "private ordinal artifact identity or length changed", - )); - } - Ok((checksum.finish(), bytes)) - })(); - let cleanup = reader.finish().map_err(storage_err); - match (checked, cleanup) { - (Ok(value), Ok(_)) => Ok(value), - (Err(primary), Ok(_)) | (Ok(_), Err(primary)) => Err(primary), - (Err(primary), Err(cleanup)) => Err(storage_err(format!( - "{primary}; private ordinal cache cleanup also failed: {cleanup}" - ))), - } -} - fn sha256_reader_streaming(file: File) -> Result { let mut reader = graphforge_filesystem::FileCacheReleasingReader::new(file).map_err(storage_err)?; @@ -1210,696 +570,3 @@ fn sha256_reader_streaming(file: File) -> Result { ))), } } - -fn write_construction_intent( - index: &crate::construction_directory::ConstructionDirectory, - intent: &ConstructionRecoveryIntent, - work: &mut ConstructionIndexWork, -) -> Result<(), GfError> { - intent.authenticate()?; - let body = serde_json::to_vec(intent).map_err(storage_err)?; - let temporary = format!(".construction-intent-{}.tmp", Uuid::new_v4().simple()); - let mut file = index - .create_replaceable_child_file(std::ffi::OsStr::new(&temporary)) - .map_err(storage_err)?; - let written = file.write_all(&body).map_err(storage_err); - let observed = index - .observe_file(std::ffi::OsStr::new(&temporary), &file) - .map_err(storage_err); - written?; - observed?; - work.write_bytes = work.write_bytes.saturating_add(body.len() as u64); - work.write_operations = work.write_operations.saturating_add(1); - let seal = crate::durable_commit::seal_file_witness(&file).map_err(storage_err)?; - index - .observe_file(std::ffi::OsStr::new(&temporary), &file) - .map_err(storage_err)?; - work.fsync_operations = work.fsync_operations.saturating_add(1); - let sealed = crate::durable_commit::SealedArtifact::adopt_sealed( - index.physical(), - std::ffi::OsStr::new(&temporary), - file, - seal, - index.allocation(), - ) - .map_err(storage_err)?; - let pending = sealed - .make_visible( - std::ffi::OsStr::new(CONSTRUCTION_INTENT), - crate::durable_commit::PublishMode::Replace, - || Ok(()), - ) - .map_err(storage_err)?; - let installed = index - .open_child_file(std::ffi::OsStr::new(CONSTRUCTION_INTENT)) - .map_err(storage_err)?; - index - .record_replacement( - std::ffi::OsStr::new(&temporary), - std::ffi::OsStr::new(CONSTRUCTION_INTENT), - &installed, - ) - .map_err(storage_err)?; - pending - .acknowledge(index.allocation()) - .map_err(storage_err)?; - work.fsync_operations = work.fsync_operations.saturating_add(1); - Ok(()) -} - -#[allow(clippy::too_many_arguments)] -fn compact_construction_levels( - output: &crate::construction_directory::ConstructionDirectory, - parent: Option<&AuthenticatedUuidIndexSnapshot>, - generation: u64, - manifest: &mut Manifest, - artifacts: &mut Vec, - output_names: &mut BTreeSet, - retained_payload_bytes: &mut u64, - work: &mut ConstructionIndexWork, - cancelled: &mut impl FnMut() -> bool, -) -> Result<(), GfError> { - for level in 0_u8..=63 { - loop { - if cancelled() { - return Err(storage_err("construction index encoding cancelled")); - } - let mut indexes = manifest - .runs - .iter() - .enumerate() - .filter_map(|(index, run)| (!run.base && run.level == level).then_some(index)) - .collect::>(); - if indexes.len() < 2 { - break; - } - if indexes.len() != 2 { - return Err(storage_err("construction manifest level overflow")); - } - indexes.sort_unstable_by_key(|index| manifest.runs[*index].first_generation); - let right = manifest.runs.remove(indexes[1]); - let left = manifest.runs.remove(indexes[0]); - if left.last_generation.saturating_add(1) != right.first_generation { - return Err(storage_err( - "construction index intervals are discontinuous", - )); - } - let identities = merge_construction_records( - output, - parent, - &left.identities, - &right.identities, - output_names, - &format!("identities-v5-l{}", level + 1), - generation, - IDENTITY_RECORD_WIDTH, - artifacts, - retained_payload_bytes, - work, - cancelled, - )?; - let surrogates = merge_construction_records( - output, - parent, - &left.node_surrogates, - &right.node_surrogates, - output_names, - &format!("node-surrogates-v5-l{}", level + 1), - generation, - NODE_LOOKUP_RECORD_WIDTH, - artifacts, - retained_payload_bytes, - work, - cancelled, - )?; - output_names.insert(identities.name.clone()); - output_names.insert(surrogates.name.clone()); - manifest.runs.push(RunRecord { - base: false, - level: level + 1, - first_generation: left.first_generation, - last_generation: right.last_generation, - identities, - node_surrogates: surrogates, - node_count: left.node_count.saturating_add(right.node_count), - edge_count: left.edge_count.saturating_add(right.edge_count), - deleted_node_count: left - .deleted_node_count - .saturating_add(right.deleted_node_count), - deleted_edge_count: left - .deleted_edge_count - .saturating_add(right.deleted_edge_count), - }); - } - } - Ok(()) -} - -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] // Streamed merge keeps both reader cleanups coupled to the primary result. -fn merge_construction_records( - output: &crate::construction_directory::ConstructionDirectory, - parent: Option<&AuthenticatedUuidIndexSnapshot>, - left: &FileRecord, - right: &FileRecord, - output_names: &BTreeSet, - prefix: &str, - generation: u64, - width: usize, - artifacts: &mut Vec, - retained_payload_bytes: &mut u64, - work: &mut ConstructionIndexWork, - cancelled: &mut impl FnMut() -> bool, -) -> Result { - let cache_window = - graphforge_filesystem::cache_release_window_for_streams(3).map_err(storage_err)?; - let mut left_reader = ConstructionBlockCursor::new( - open_construction_source(output, parent, left, output_names, retained_payload_bytes)?, - left.clone(), - width, - cache_window, - )?; - let right_source = - open_construction_source(output, parent, right, output_names, retained_payload_bytes); - let right_source = match right_source { - Ok(source) => source, - Err(primary) => { - return combine_cache_cleanup( - Err(primary), - left_reader.finish_cache(), - "left construction merge source", - ); - } - }; - let right_reader = - ConstructionBlockCursor::new(right_source, right.clone(), width, cache_window); - let mut right_reader = match right_reader { - Ok(reader) => reader, - Err(primary) => { - return combine_cache_cleanup( - Err(primary), - left_reader.finish_cache(), - "left construction merge source", - ); - } - }; - let temporary = format!(".construction-merge-{}.tmp", Uuid::new_v4().simple()); - let file = output - .create_replaceable_child_file(std::ffi::OsStr::new(&temporary)) - .map_err(storage_err); - let file = match file { - Ok(file) => file, - Err(primary) => { - return finish_construction_cursor_pair(primary, &mut left_reader, &mut right_reader); - } - }; - let identity = match graphforge_filesystem::file_identity(&file).map_err(storage_err) { - Ok(identity) => identity, - Err(primary) => { - return finish_construction_cursor_pair(primary, &mut left_reader, &mut right_reader); - } - }; - let mut writer = - match graphforge_filesystem::DurableFileCacheWriter::with_window_bytes(file, cache_window) - .map_err(storage_err) - { - Ok(writer) => writer, - Err(primary) => { - return finish_construction_cursor_pair( - primary, - &mut left_reader, - &mut right_reader, - ); - } - }; - let key_width = if width == IDENTITY_RECORD_WIDTH { - 16 - } else { - 8 - }; - let output_bytes = (BULK_IO_BYTES / width) * width; - let mut output_block = Vec::with_capacity(output_bytes); - let merged = (|| -> Result<(), GfError> { - while left_reader.current().is_some() || right_reader.current().is_some() { - let take_left = match (left_reader.current(), right_reader.current()) { - (Some(left), Some(right)) => { - if left[..key_width] == right[..key_width] { - return Err(storage_err("construction index merge found duplicate key")); - } - left[..key_width] < right[..key_width] - } - (Some(_), None) => true, - (None, Some(_)) => false, - (None, None) => break, - }; - let selected = if take_left { - left_reader.current().expect("left exists") - } else { - right_reader.current().expect("right exists") - }; - output_block.extend_from_slice(selected); - if take_left { - left_reader.advance()?; - } else { - right_reader.advance()?; - } - if output_block.len() + width > output_bytes { - if cancelled() { - return Err(storage_err("construction index encoding cancelled")); - } - writer.write_all(&output_block).map_err(storage_err)?; - work.write_bytes = work.write_bytes.saturating_add(output_block.len() as u64); - work.write_operations = work.write_operations.saturating_add(1); - output_block.clear(); - } - } - if !output_block.is_empty() { - writer.write_all(&output_block).map_err(storage_err)?; - work.write_bytes = work.write_bytes.saturating_add(output_block.len() as u64); - work.write_operations = work.write_operations.saturating_add(1); - } - writer.flush().map_err(storage_err) - })(); - let merged = combine_cache_cleanup( - merged, - left_reader.finish_cache(), - "left construction merge source", - ); - let merged = combine_cache_cleanup( - merged, - right_reader.finish_cache(), - "right construction merge source", - ); - let observed = output - .observe_file(std::ffi::OsStr::new(&temporary), writer.file()) - .map_err(storage_err); - let merged = combine_v4_cleanup(merged, observed, "construction merge allocation"); - merged?; - let seal = - crate::durable_commit::seal_cache_writer_witness(&mut writer).map_err(storage_err)?; - output - .observe_file(std::ffi::OsStr::new(&temporary), writer.file()) - .map_err(storage_err)?; - let write_cache_release = writer.evidence(); - work.fsync_operations = work - .fsync_operations - .saturating_add(write_cache_release.sync_operations); - merge_cache_release_evidence(&mut work.cache_release, write_cache_release); - work.read_bytes = work - .read_bytes - .saturating_add(left_reader.read_bytes) - .saturating_add(right_reader.read_bytes); - work.read_operations = work - .read_operations - .saturating_add(left_reader.read_operations) - .saturating_add(right_reader.read_operations); - merge_cache_release_evidence(&mut work.cache_release, left_reader.cache_release); - merge_cache_release_evidence(&mut work.cache_release, right_reader.cache_release); - drop(writer.into_file()); - describe_and_install_construction_run( - output, &temporary, identity, seal, prefix, generation, width, artifacts, work, - ) -} - -struct ConstructionBlockCursor { - file: graphforge_filesystem::FileCacheReleasingReader, - descriptor: FileRecord, - width: usize, - block: Vec, - block_index: usize, - within: usize, - records: u64, - checksum: crate::corruption_checksum::Checksum, - finished: bool, - read_bytes: u64, - read_operations: u64, - cache_release: graphforge_filesystem::FileCacheReleaseEvidence, - cache_finished: bool, -} - -impl ConstructionBlockCursor { - fn new( - file: File, - descriptor: FileRecord, - width: usize, - cache_window: std::num::NonZeroU64, - ) -> Result { - let mut cursor = Self { - file: graphforge_filesystem::FileCacheReleasingReader::with_window_bytes( - file, - cache_window, - graphforge_filesystem::FileCacheReleaseTracker::default(), - ) - .map_err(storage_err)?, - descriptor, - width, - block: Vec::new(), - block_index: 0, - within: 0, - records: 0, - checksum: crate::corruption_checksum::Checksum::new(), - finished: false, - read_bytes: 0, - read_operations: 0, - cache_release: graphforge_filesystem::FileCacheReleaseEvidence::default(), - cache_finished: false, - }; - if let Err(primary) = cursor.fill() { - combine_cache_cleanup::<()>( - Err(primary), - cursor.finish_cache(), - "construction merge source", - )?; - unreachable!("failed construction cursor initialization returned success"); - } - Ok(cursor) - } - - fn record_width(&self) -> usize { - if self.width == IDENTITY_RECORD_WIDTH && self.block[self.within + 16] == 1 { - identity_codec::EDGE_WIDTH - } else { - self.width - } - } - - fn current(&self) -> Option<&[u8]> { - (!self.finished).then(|| &self.block[self.within..self.within + self.record_width()]) - } - - fn advance(&mut self) -> Result<(), GfError> { - if self.finished { - return Ok(()); - } - self.within += self.record_width(); - self.records = self.records.saturating_add(1); - if self.within == self.block.len() { - self.fill()?; - } - Ok(()) - } - - fn fill(&mut self) -> Result<(), GfError> { - if self.block_index == self.descriptor.blocks.len() { - self.finished = true; - let authenticated = if self.records != self.descriptor.count - || self.checksum.clone().finish() != self.descriptor.xxh64 - { - Err(storage_err( - "construction merge source authentication failed", - )) - } else { - Ok(()) - }; - return combine_cache_cleanup( - authenticated, - self.finish_cache(), - "construction merge source", - ); - } - let expected = &self.descriptor.blocks[self.block_index]; - if expected.offset != self.read_bytes - || expected.len == 0 - || expected.len as usize > BULK_IO_BYTES - { - return Err(storage_err("construction merge block framing changed")); - } - self.block.resize(expected.len as usize, 0); - self.file.read_exact(&mut self.block).map_err(storage_err)?; - self.read_bytes = self.read_bytes.saturating_add(self.block.len() as u64); - self.read_operations = self.read_operations.saturating_add(1); - if !block_matches(&self.block, expected, self.width) { - return Err(storage_err("construction merge source block changed")); - } - self.checksum.update(&self.block); - self.block_index += 1; - self.within = 0; - Ok(()) - } - - fn finish_cache(&mut self) -> Result<(), GfError> { - if !self.cache_finished { - self.cache_release = self.file.finish().map_err(storage_err)?; - self.cache_finished = true; - } - Ok(()) - } -} - -pub(super) fn combine_cache_cleanup( - primary: Result, - cleanup: Result<(), GfError>, - source: &str, -) -> Result { - match (primary, cleanup) { - (Ok(value), Ok(())) => Ok(value), - (Ok(_), Err(cleanup)) => Err(cleanup), - (Err(primary), Ok(())) => Err(primary), - (Err(primary), Err(cleanup)) => Err(storage_err(format!( - "{primary}; {source} cache release also failed: {cleanup}" - ))), - } -} - -pub(super) fn combine_v4_cleanup( - primary: Result, - cleanup: Result<(), GfError>, - context: &str, -) -> Result { - match (primary, cleanup) { - (Ok(value), Ok(())) => Ok(value), - (Ok(_), Err(cleanup)) => Err(cleanup), - (Err(primary), Ok(())) => Err(primary), - (Err(primary), Err(cleanup)) => Err(storage_err(format!( - "{primary}; {context} also failed: {cleanup}" - ))), - } -} - -fn finish_construction_cursor_pair( - primary: GfError, - left: &mut ConstructionBlockCursor, - right: &mut ConstructionBlockCursor, -) -> Result { - let result = combine_cache_cleanup( - Err(primary), - left.finish_cache(), - "left construction merge source", - ); - combine_cache_cleanup( - result, - right.finish_cache(), - "right construction merge source", - ) -} - -fn open_construction_source( - output: &crate::construction_directory::ConstructionDirectory, - parent: Option<&AuthenticatedUuidIndexSnapshot>, - record: &FileRecord, - output_names: &BTreeSet, - retained_payload_bytes: &mut u64, -) -> Result { - if output_names.contains(&record.name) { - output - .open_child_file(std::ffi::OsStr::new(&record.name)) - .map_err(storage_err) - } else { - let parent = - parent.ok_or_else(|| storage_err("construction merge lacks retained source"))?; - *retained_payload_bytes = - retained_payload_bytes.saturating_add(record.count.saturating_mul( - if record.name.starts_with("identities-") { - IDENTITY_RECORD_BYTES - } else { - NODE_LOOKUP_RECORD_BYTES - }, - )); - parent.open_retained_file(record) - } -} - -fn install_empty_construction_run( - output: &crate::construction_directory::ConstructionDirectory, - prefix: &str, - generation: u64, - width: usize, - artifacts: &mut Vec, - work: &mut ConstructionIndexWork, -) -> Result { - let temporary = format!(".construction-empty-{}.tmp", Uuid::new_v4().simple()); - let file = output - .create_replaceable_child_file(std::ffi::OsStr::new(&temporary)) - .map_err(storage_err)?; - let identity = graphforge_filesystem::file_identity(&file).map_err(storage_err)?; - let seal = crate::durable_commit::seal_file_witness(&file).map_err(storage_err)?; - work.fsync_operations = work.fsync_operations.saturating_add(1); - drop(file); - describe_and_install_construction_run( - output, &temporary, identity, seal, prefix, generation, width, artifacts, work, - ) -} - -#[allow(clippy::too_many_arguments)] -fn describe_and_install_construction_run( - output: &crate::construction_directory::ConstructionDirectory, - temporary: &str, - identity: graphforge_filesystem::FileIdentity, - seal: crate::durable_commit::FileSeal, - prefix: &str, - generation: u64, - width: usize, - artifacts: &mut Vec, - work: &mut ConstructionIndexWork, -) -> Result { - let file = output - .open_child_file(std::ffi::OsStr::new(temporary)) - .map_err(storage_err)?; - let bytes = file.metadata().map_err(storage_err)?.len(); - let mut file = - graphforge_filesystem::FileCacheReleasingReader::new(file).map_err(storage_err)?; - let mut reads = (0_u64, 0_u64); - let described = describe_stream(&mut file, width, &mut reads); - work.read_bytes = work.read_bytes.saturating_add(reads.0); - work.read_operations = work.read_operations.saturating_add(reads.1); - let released = file.finish().map_err(storage_err); - let ((sha256, xxh64, blocks, count), read_cache_release) = match (described, released) { - (Ok(described), Ok(released)) => (described, released), - (Ok(_), Err(release)) => return Err(release), - (Err(primary), Ok(_)) => return Err(primary), - (Err(primary), Err(release)) => { - return Err(storage_err(format!( - "{primary}; construction run cache release also failed: {release}" - ))); - } - }; - merge_cache_release_evidence(&mut work.cache_release, read_cache_release); - let name = format!("{prefix}-{generation}-{}.uuidx", &sha256[..16]); - drop(file); - let file = crate::durable_commit::open_publisher( - output.physical(), - std::ffi::OsStr::new(temporary), - identity, - ) - .map_err(storage_err)?; - if graphforge_filesystem::file_identity(&file).map_err(storage_err)? != identity { - return Err(storage_err( - "construction run identity changed before publication", - )); - } - let sealed = crate::durable_commit::SealedArtifact::adopt_sealed( - output.physical(), - std::ffi::OsStr::new(temporary), - file, - seal, - output.allocation(), - ) - .map_err(storage_err)?; - let pending = sealed - .make_visible( - std::ffi::OsStr::new(&name), - crate::durable_commit::PublishMode::Replace, - || Ok(()), - ) - .map_err(storage_err)?; - let installed = output - .open_child_file(std::ffi::OsStr::new(&name)) - .map_err(storage_err)?; - output - .record_replacement( - std::ffi::OsStr::new(temporary), - std::ffi::OsStr::new(&name), - &installed, - ) - .map_err(storage_err)?; - pending - .acknowledge(output.allocation()) - .map_err(storage_err)?; - work.fsync_operations = work.fsync_operations.saturating_add(1); - artifacts.push(ConstructionIndexOutput { - name: name.clone(), - bytes, - sha256: sha256.clone(), - xxh64, - }); - Ok(FileRecord { - name, - count, - sha256, - xxh64, - blocks, - }) -} - -pub(super) fn install_construction_bytes( - output: &graphforge_filesystem::StableDirectory, - name: &str, - bytes: &[u8], - work: &mut ConstructionIndexWork, - allocation: Option<&crate::StorageAllocationOperation>, -) -> Result<(ConstructionIndexOutput, V4PublicationGuard), GfError> { - let temporary = format!(".{name}-{}.tmp", Uuid::new_v4().simple()); - let mut publication = - V4PublicationGuard::create(output, &temporary, allocation).map_err(storage_err)?; - let mut file = publication.take_file().map_err(storage_err)?; - let written = file.write_all(bytes).map_err(storage_err); - let observed = publication.observe(&file).map_err(storage_err); - written?; - observed?; - work.write_bytes = work.write_bytes.saturating_add(bytes.len() as u64); - work.write_operations = work.write_operations.saturating_add(1); - work.peak_temporary_bytes = work - .peak_temporary_bytes - .max(u64::try_from(bytes.len()).map_err(storage_err)?); - let seal = crate::durable_commit::seal_file_witness(&file).map_err(storage_err)?; - publication.record_seal(seal); - publication.observe(&file).map_err(storage_err)?; - work.fsync_operations = work.fsync_operations.saturating_add(1); - let failpoint = match name { - V4_ORDINAL_RECEIPT => Some("v4_publish.after_receipt_temp_fsync"), - V4_ORDINAL_MANIFEST => Some("v4_publish.after_manifest_temp_fsync"), - "ordinal-v4.lock" => Some("v4_publish.after_lock_temp_fsync"), - _ => None, - }; - if let Some(failpoint) = failpoint { - crate::graph_construction::construction_failpoint(failpoint); - } - drop(file); - let installed = publication - .install_child(std::ffi::OsStr::new(name)) - .map_err(storage_err); - if let Err(primary) = installed { - let cleanup = cleanup_v4_publication(&mut publication); - return combine_v4_cleanup(Err(primary), cleanup, "v4 construction control cleanup"); - } - let authority_point = match name { - V4_ORDINAL_RECEIPT => Some("receipt_install"), - V4_ORDINAL_MANIFEST => Some("manifest_install"), - "ordinal-v4.lock" => Some("lock_install"), - _ => None, - }; - if let Some(point) = authority_point - && let Err(primary) = v4_authority_failure(point) - { - let cleanup = cleanup_v4_publication(&mut publication); - return combine_v4_cleanup(Err(primary), cleanup, "v4 construction control cleanup"); - } - work.fsync_operations = work.fsync_operations.saturating_add(1); - Ok(( - ConstructionIndexOutput { - name: name.to_owned(), - bytes: bytes.len() as u64, - xxh64: crate::corruption_checksum::checksum(bytes), - sha256: if matches!(name, MANIFEST | V4_ORDINAL_RECEIPT | V4_ORDINAL_MANIFEST) { - hex_sha256(bytes) - } else { - hex_bytes(&graphforge_core::hash_observation::ControlSha256::digest( - bytes, - )) - }, - }, - publication, - )) -} - -#[cfg(test)] -mod tests; diff --git a/crates/graphforge-storage/src/uuid_membership/construction/tests.rs b/crates/graphforge-storage/src/uuid_membership/construction/tests.rs deleted file mode 100644 index 0906f2988..000000000 --- a/crates/graphforge-storage/src/uuid_membership/construction/tests.rs +++ /dev/null @@ -1,307 +0,0 @@ -use super::super::BULK_IO_BYTES; -use super::super::IDENTITY_RECORD_BYTES; -use super::super::IDENTITY_RECORD_WIDTH; -use super::super::INDEX_DIR; -use super::super::NODE_LOOKUP_RECORD_BYTES; -use super::super::NODE_LOOKUP_RECORD_WIDTH; -use super::super::UuidMembershipIndex; -use super::super::append_uuid_membership_delta; -use super::super::hex_bytes; -use super::super::identity_codec; -use super::super::ordinal_artifacts::publish_v4_construction_artifacts; -use super::super::ordinal_artifacts::stage_v4_ordinal_bundle; -use super::super::rebuild::build_identity_run; -use super::super::rebuild::read_exact_record; -use super::super::topology_delta::hex_sha256; -use super::super::topology_delta::plan_uuid_membership_delta; -use super::super::topology_delta::write_identity_records; -use super::cleanup_private_construction_index; -use super::encode_construction_index; -use crate::uuid_membership::ConstructionIdentityInput; -use std::fs; -use std::fs::File; -use std::io::BufWriter; -use std::io::Write; -use uuid::Uuid; - -#[test] -fn cleanup_authenticates_complete_unpublished_v4_facet() { - let encoded_dir = tempfile::tempdir().unwrap(); - let encoded = graphforge_filesystem::StableDirectory::open(encoded_dir.path()).unwrap(); - let graph = encoded - .create_child_directory(std::ffi::OsStr::new("graph")) - .unwrap(); - let topology = graph - .create_child_directory(std::ffi::OsStr::new("topology")) - .unwrap(); - let index = topology - .create_child_directory(std::ffi::OsStr::new("uuid-membership")) - .unwrap(); - let mappings = [(Uuid::from_u128(1), 1_u64), (Uuid::from_u128(2), 3_u64)]; - let bundle = stage_v4_ordinal_bundle(mappings, 1, &index, &mut || false).unwrap(); - publish_v4_construction_artifacts( - &encoded, - bundle, - 1, - &hex_sha256(b"delta"), - None, - &mut || false, - None, - ) - .unwrap(); - - cleanup_private_construction_index(&encoded).unwrap(); - assert!(index.child_names().unwrap().is_empty()); -} - -#[test] -fn packed_construction_index_preserves_full_width_surrogates_and_refuses_invalid_records() { - let uuid = Uuid::from_u128(u128::MAX); - let surrogate = u64::MAX; - let mut golden = uuid.as_bytes().to_vec(); - golden.extend_from_slice(&[0, 0]); - golden.extend_from_slice(&surrogate.to_be_bytes()); - assert_eq!(golden.len(), 26); - let mut cases = vec![(golden.clone(), true)]; - for length in 1..26 { - cases.push((golden[..length].to_vec(), false)); - } - for (offset, value) in [(16, 2), (17, 1), (17, 255)] { - let mut invalid = golden.clone(); - invalid[offset] = value; - cases.push((invalid, false)); - } - let mut zero = golden.clone(); - zero[18..26].fill(0); - cases.push((zero, false)); - for (bytes, valid) in cases { - let source_dir = tempfile::tempdir().unwrap(); - let encoded_dir = tempfile::tempdir().unwrap(); - fs::write(source_dir.path().join("identities.run"), &bytes).unwrap(); - let source = graphforge_filesystem::StableDirectory::open(source_dir.path()).unwrap(); - let encoded = graphforge_filesystem::StableDirectory::open(encoded_dir.path()).unwrap(); - let xxh64 = crate::corruption_checksum::hex(crate::corruption_checksum::checksum(&bytes)); - let result = encode_construction_index( - ConstructionIdentityInput::Shaped { - source: &source, - name: "identities.run", - xxh64: &xxh64, - }, - &encoded, - 1, - 0, - None, - 1, - 0, - &mut || false, - None, - ); - if valid { - let result = result.unwrap(); - let identity = result - .artifacts - .iter() - .find(|artifact| artifact.name.ends_with(".uuidx")) - .unwrap(); - let mut expected = uuid.as_bytes().to_vec(); - expected.push(0); - expected.extend_from_slice(&surrogate.to_be_bytes()); - assert_eq!(expected.len(), 25); - assert_eq!( - fs::read( - encoded_dir - .path() - .join("graph/topology/uuid-membership") - .join(&identity.name) - ) - .unwrap(), - expected - ); - let mut index = - UuidMembershipIndex::open_at_generation(&encoded_dir.path().join("graph"), 1) - .unwrap(); - assert_eq!( - index.lookup_node_surrogates(&[uuid]).unwrap().0, - vec![Some(surrogate)] - ); - } else { - assert!(result.is_err()); - assert_eq!( - fs::read(source_dir.path().join("identities.run")).unwrap(), - bytes - ); - } - } -} - -#[test] -fn construction_encoder_io_geometry_is_block_bounded() { - for records in [32_768_u64, 65_536, 131_072] { - let source_dir = tempfile::tempdir().unwrap(); - let encoded_dir = tempfile::tempdir().unwrap(); - let mut input = BufWriter::with_capacity( - BULK_IO_BYTES, - File::create(source_dir.path().join("identities.run")).unwrap(), - ); - for value in 1..=records { - input.write_all(&u128::from(value).to_be_bytes()).unwrap(); - input.write_all(&[0]).unwrap(); - input.write_all(&[0]).unwrap(); - input.write_all(&value.to_be_bytes()).unwrap(); - } - input.flush().unwrap(); - drop(input); - let source = graphforge_filesystem::StableDirectory::open(source_dir.path()).unwrap(); - let encoded = graphforge_filesystem::StableDirectory::open(encoded_dir.path()).unwrap(); - let source_bytes = fs::read(source_dir.path().join("identities.run")).unwrap(); - let source_xxh64 = - crate::corruption_checksum::hex(crate::corruption_checksum::checksum(&source_bytes)); - let result = encode_construction_index( - ConstructionIdentityInput::Shaped { - source: &source, - name: "identities.run", - xxh64: &source_xxh64, - }, - &encoded, - 1, - 0, - None, - records, - 0, - &mut || false, - None, - ) - .unwrap(); - let identity_blocks = (records * IDENTITY_RECORD_BYTES).div_ceil(BULK_IO_BYTES as u64); - let surrogate_blocks = (records * NODE_LOOKUP_RECORD_BYTES).div_ceil(BULK_IO_BYTES as u64); - assert!( - result.read_operations <= 2 * identity_blocks + surrogate_blocks + 4, - "{records}: {} reads", - result.read_operations - ); - assert!( - result.write_operations <= identity_blocks + surrogate_blocks + 4, - "{records}: {} writes", - result.write_operations - ); - assert!(result.peak_buffer_bytes <= 3 * BULK_IO_BYTES as u64); - } -} - -#[test] -fn packed_identity_write_count_tracks_whole_record_flush_boundaries() { - let dir = tempfile::tempdir().unwrap(); - let records = (1..=123_361_u128) - .map(|id| (Uuid::from_u128(id), 1, 0)) - .collect::>(); - let path = dir.path().join("edges.uuidx"); - assert_eq!(write_identity_records(&path, &records).unwrap(), 3); - assert_eq!(fs::metadata(path).unwrap().len(), 2_097_137); - let edges = records.iter().map(|record| record.0).collect::>(); - let root = dir.path().join(INDEX_DIR); - fs::create_dir_all(&root).unwrap(); - let scratch = tempfile::tempdir().unwrap(); - let (_, _, _, planned) = - plan_uuid_membership_delta(&root, 0, 1, None, scratch.path(), &[], &edges, &[], &[]) - .unwrap(); - assert_eq!(planned.write_blocks, 3); - assert_eq!(planned.write_bytes, 2_097_137); - crate::generation::force_bump_topology_generation_for_test(dir.path()).unwrap(); - let appended = append_uuid_membership_delta(dir.path(), 1, &[], &edges).unwrap(); - assert_eq!(appended.write_blocks, 3); - assert_eq!(appended.write_bytes, 2_097_137); -} - -#[test] -fn fixed_width_codecs_distinguish_clean_eof_from_partial_tail() { - let mut clean = std::io::Cursor::new(Vec::::new()); - assert_eq!(identity_codec::read(&mut clean).unwrap(), None); - for length in 1..IDENTITY_RECORD_WIDTH { - let mut partial = std::io::Cursor::new(vec![0_u8; length]); - assert!(identity_codec::read(&mut partial).is_err()); - } - for length in 1..24 { - let mut partial = std::io::Cursor::new(vec![0_u8; length]); - assert!(read_exact_record::<24>(&mut partial).is_err()); - } -} - -#[test] -fn unified_identity_merge_rejects_cross_kind_uuid() { - let scratch = tempfile::tempdir().unwrap(); - let uuid = [7_u8; 16]; - let node = scratch.path().join("node.run"); - let edge = scratch.path().join("edge.run"); - for path in [&node, &edge] { - let mut file = File::create(path).unwrap(); - file.write_all(&uuid).unwrap(); - file.write_all(&1_u64.to_le_bytes()).unwrap(); - } - assert!(build_identity_run(&node, &edge, &scratch.path().join("out.run")).is_err()); -} - -#[test] -fn construction_intent_rejects_published_format_as_private_version() { - let mut intent = super::ConstructionRecoveryIntent { - format_version: super::CONSTRUCTION_INTENT_FORMAT_VERSION, - generation: 1, - parent_generation: 0, - identities_name: "identities.bin".to_owned(), - source_volume: 1, - source_file_id: "00".repeat(16), - source_bytes: 25, - source_xxh64: "0123456789abcdef".to_owned(), - authority_sha256: String::new(), - }; - for version in [ - super::CONSTRUCTION_INTENT_FORMAT_VERSION, - super::FORMAT_VERSION, - ] { - intent.format_version = version; - intent.authority_sha256 = super::construction_intent_digest( - intent.format_version, - intent.generation, - intent.parent_generation, - &intent.identities_name, - intent.source_volume, - &intent.source_file_id, - intent.source_bytes, - &intent.source_xxh64, - ); - assert_eq!( - intent.authenticate().is_ok(), - version == super::CONSTRUCTION_INTENT_FORMAT_VERSION - ); - } -} - -#[test] -fn uuid_final_capture_hashes_whole_payload_once_and_retires_block_sha() { - let mut bytes = Vec::new(); - for node_id in 1_u64..=4096 { - bytes.extend_from_slice(&node_id.to_be_bytes()); - bytes.extend_from_slice(Uuid::from_u128(u128::from(node_id)).as_bytes()); - } - let expected = hex_bytes(&crate::payload_digest::PayloadSha256::digest(&bytes)); - let expected_checksum = crate::corruption_checksum::checksum(&bytes); - let capture = graphforge_core::hash_observation::operation::Capture::start(); - let (sha, checksum, blocks, count) = super::super::describe_stream( - &mut std::io::Cursor::new(&bytes), - NODE_LOOKUP_RECORD_WIDTH, - &mut (0, 0), - ) - .unwrap(); - let observed = capture.snapshot(); - assert_eq!(observed.artifact_payload_sha256_bytes, bytes.len() as u64); - assert_eq!(observed.checksum_bytes, 2 * bytes.len() as u64); - assert_eq!(observed.unclassified_sha256_bytes, 0); - assert_eq!((sha, checksum, count), (expected, expected_checksum, 4096)); - assert!( - serde_json::to_value(&blocks) - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|block| block.get("sha256").is_none() && block.get("xxh64").is_some()) - ); -} diff --git a/crates/graphforge-storage/src/uuid_membership/identity_codec.rs b/crates/graphforge-storage/src/uuid_membership/identity_codec.rs deleted file mode 100644 index 72085ede3..000000000 --- a/crates/graphforge-storage/src/uuid_membership/identity_codec.rs +++ /dev/null @@ -1,154 +0,0 @@ -//! Current permanent identity framing. Live edges omit their defined-zero -//! membership surrogate; topology retains the canonical edge identifier. -use std::io::Read; - -use graphforge_core::GfError; - -use super::storage_err; - -pub(super) const WIDTH: usize = 25; -pub(super) const EDGE_WIDTH: usize = 17; -pub(super) type Record = [u8; WIDTH]; - -fn width(kind: u8) -> Result { - match kind { - 1 => Ok(EDGE_WIDTH), - 0 | 2 | 3 => Ok(WIDTH), - _ => Err(storage_err("identity record kind is invalid")), - } -} - -pub(super) fn encoded(record: &Record) -> Result<&[u8], GfError> { - let length = width(record[16])?; - if length == EDGE_WIDTH && record[EDGE_WIDTH..].iter().any(|byte| *byte != 0) { - return Err(storage_err("live edge membership surrogate must be zero")); - } - Ok(&record[..length]) -} - -pub(super) fn read(reader: &mut impl Read) -> Result, GfError> { - let mut record = [0_u8; WIDTH]; - loop { - match reader.read(&mut record[..1]) { - Ok(0) => return Ok(None), - Ok(_) => break, - Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {} - Err(error) => return Err(storage_err(error)), - } - } - reader - .read_exact(&mut record[1..EDGE_WIDTH]) - .map_err(storage_err)?; - let length = width(record[16])?; - reader - .read_exact(&mut record[EDGE_WIDTH..length]) - .map_err(storage_err)?; - Ok(Some(record)) -} - -pub(super) fn take(bytes: &mut &[u8]) -> Result, GfError> { - read(bytes) -} - -/// Validate complete block framing and ordered UUID fences without allocation. -pub(super) fn layout(bytes: &[u8]) -> Result<(u64, &[u8], &[u8]), GfError> { - let mut offset = 0; - let mut count = 0_u64; - let mut previous: Option<&[u8]> = None; - let mut last = 0; - while offset < bytes.len() { - let suffix = &bytes[offset..]; - if suffix.len() < EDGE_WIDTH { - return Err(storage_err( - "partial identity record in authenticated block", - )); - } - let length = width(suffix[16])?; - if suffix.len() < length { - return Err(storage_err( - "partial identity record in authenticated block", - )); - } - let key = &suffix[..16]; - if previous.is_some_and(|prior| prior >= key) { - return Err(storage_err("identity block UUIDs are not strictly ordered")); - } - previous = Some(key); - last = offset; - offset += length; - count += 1; - } - if count == 0 { - return Err(storage_err("empty identity block")); - } - Ok((count, &bytes[..16], &bytes[last..last + 16])) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn full_width_nodes_and_tombstones_and_zero_edge_round_trip() { - for kind in 0..=3 { - for surrogate in [0, 1, u64::from(u32::MAX), u64::from(u32::MAX) + 1, u64::MAX] { - let mut record = [u8::MAX; WIDTH]; - record[16] = kind; - record[17..].copy_from_slice(&surrogate.to_be_bytes()); - if kind == 1 && surrogate != 0 { - assert!(encoded(&record).is_err()); - continue; - } - let bytes = encoded(&record).unwrap(); - assert_eq!(bytes.len(), if kind == 1 { EDGE_WIDTH } else { WIDTH }); - assert_eq!(read(&mut &bytes[..]).unwrap(), Some(record)); - } - } - } - - #[test] - fn partial_records_unknown_kinds_and_cross_block_records_fail_closed() { - for kind in 0..=3 { - let mut record = [0_u8; WIDTH]; - record[16] = kind; - let bytes = encoded(&record).unwrap(); - for length in 1..bytes.len() { - assert!(read(&mut &bytes[..length]).is_err()); - assert!(layout(&bytes[..length]).is_err()); - } - assert_eq!(layout(bytes).unwrap().0, 1); - } - let mut invalid = [0_u8; WIDTH]; - invalid[16] = 4; - assert!(encoded(&invalid).is_err()); - assert!(read(&mut &invalid[..]).is_err()); - assert!(layout(&invalid).is_err()); - assert_eq!(read(&mut &[][..]).unwrap(), None); - } - - #[test] - fn mixed_records_have_exact_budget_and_ordered_fences() { - let mut bytes = Vec::new(); - for i in 0..69_634_u64 { - let mut record = [0_u8; WIDTH]; - record[..16].copy_from_slice(&u128::from(i + 1).to_be_bytes()); - record[16] = u8::from(i >= 4097); - if record[16] == 0 { - record[17..].copy_from_slice(&(i + 1).to_be_bytes()); - } - bytes.extend_from_slice(encoded(&record).unwrap()); - } - assert_eq!(bytes.len(), 1_216_554); - assert_eq!(layout(&bytes).unwrap().0, 69_634); - let mut input = bytes.as_slice(); - for i in 0..69_634_u64 { - let record = take(&mut input).unwrap().unwrap(); - assert_eq!(&record[..16], &(u128::from(i) + 1).to_be_bytes()); - assert_eq!( - u64::from_be_bytes(record[17..].try_into().unwrap()), - if i < 4097 { i + 1 } else { 0 } - ); - } - assert!(input.is_empty()); - } -} diff --git a/crates/graphforge-storage/src/uuid_membership/maintenance.rs b/crates/graphforge-storage/src/uuid_membership/maintenance.rs index 9aeaf6386..d18d4d2d3 100644 --- a/crates/graphforge-storage/src/uuid_membership/maintenance.rs +++ b/crates/graphforge-storage/src/uuid_membership/maintenance.rs @@ -1,23 +1,15 @@ //! Authenticated identity authority and orphan maintenance. use super::INDEX_DIR; -use super::MANIFEST; -use super::MAX_MANIFEST_BYTES; -use super::Manifest; -use super::TOPOLOGY_RECEIPT; use super::TopologyIndexReceipt; use super::UuidIndexOrphanGcWork; use super::V4_ORDINAL_MANIFEST; use super::V4_ORDINAL_RECEIPT; -use super::decode_manifest; use super::storage_err; use super::topology_delta::hex_sha256; use super::topology_delta::read_bounded; -use super::validate_run_descriptors; use graphforge_core::GfError; use std::collections::BTreeSet; -#[cfg(test)] -use std::fs; use std::path::Path; /// Reclaim a bounded number of unreachable immutable UUID runs under the @@ -32,86 +24,13 @@ pub fn maintain_uuid_membership_orphans( .map(crate::ResolvedProjectGeneration::authenticated_v4_ordinal_authority) .transpose()? .flatten(); - let membership_authority = selected - .as_ref() - .map(authenticated_v3_membership_authority) - .transpose()? - .flatten(); maintain_uuid_membership_orphans_with_authorities( project_dir, maximum, - membership_authority.as_ref(), ordinal_authority.as_ref(), ) } -#[derive(Clone, Debug)] -pub(super) struct AuthenticatedV3MembershipAuthority { - topology_generation: u64, - manifest_sha256: String, -} - -pub(super) fn authenticated_v3_membership_authority( - selected: &crate::ResolvedProjectGeneration, -) -> Result, GfError> { - let mut state = crate::graph_manifest::GraphManifestTargetedState::default(); - let receipt = selected.authenticated_graph_file_bytes_with_state( - &format!("{INDEX_DIR}/{TOPOLOGY_RECEIPT}"), - MAX_MANIFEST_BYTES, - Some(&mut state), - )?; - let manifest = selected.authenticated_graph_file_bytes_with_state( - &format!("{INDEX_DIR}/{MANIFEST}"), - MAX_MANIFEST_BYTES, - Some(&mut state), - )?; - match (receipt, manifest) { - (None, None) => Ok(None), - (None, Some(_)) | (Some(_), None) => Err(storage_err( - "selected UUID membership facet has incomplete authority residue", - )), - (Some((_, receipt_bytes)), Some((manifest_entry, manifest_bytes))) => { - let receipt: TopologyIndexReceipt = - serde_json::from_slice(&receipt_bytes).map_err(storage_err)?; - let generation = selected - .authenticated_graph_file_bytes_with_state( - "topology/generation.json", - MAX_MANIFEST_BYTES, - Some(&mut state), - )? - .ok_or_else(|| storage_err("selected topology generation authority is absent"))?; - let generation: serde_json::Value = - serde_json::from_slice(&generation.1).map_err(storage_err)?; - let topology_generation = generation - .get("topology_generation") - .and_then(serde_json::Value::as_u64) - .ok_or_else(|| storage_err("selected topology generation is missing"))?; - let manifest_sha256 = hex_sha256(&manifest_bytes); - let canonical_hex = |value: &str, length: usize| { - value.len() == length - && value - .bytes() - .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) - }; - if !canonical_hex(&receipt.nonce, 32) - || !canonical_hex(&receipt.topology_delta_sha256, 64) - || !canonical_hex(&receipt.manifest_sha256, 64) - || receipt.expected_generation != topology_generation - || receipt.manifest_sha256 != manifest_entry.content_sha256 - || receipt.manifest_sha256 != manifest_sha256 - { - return Err(storage_err( - "selected UUID membership receipt does not authenticate its manifest", - )); - } - Ok(Some(AuthenticatedV3MembershipAuthority { - topology_generation, - manifest_sha256, - })) - } - } -} - /// Retain the selected project generation whenever `graph_root` is its /// generation-owned graph tree. Standalone graph roots deliberately have no /// project-generation provenance and therefore cannot authorize a present v4 @@ -220,35 +139,16 @@ pub(crate) fn maintain_uuid_membership_orphans_with_ordinal_authority( maximum: usize, ordinal_authority: Option<&crate::AuthenticatedV4OrdinalIdentityAuthority>, ) -> Result { - let manifest_bytes = - fs::read(project_dir.join(INDEX_DIR).join(MANIFEST)).map_err(storage_err)?; - let manifest = decode_manifest(&manifest_bytes)?; - let membership_authority = AuthenticatedV3MembershipAuthority { - topology_generation: manifest.current_generation, - manifest_sha256: hex_sha256(&manifest_bytes), - }; - maintain_uuid_membership_orphans_with_authorities( - project_dir, - maximum, - Some(&membership_authority), - ordinal_authority, - ) + maintain_uuid_membership_orphans_with_authorities(project_dir, maximum, ordinal_authority) } fn maintain_uuid_membership_orphans_with_authorities( project_dir: &Path, maximum: usize, - membership_authority: Option<&AuthenticatedV3MembershipAuthority>, ordinal_authority: Option<&crate::AuthenticatedV4OrdinalIdentityAuthority>, ) -> Result { crate::durable_rewrite::with_rewrite_lock(project_dir, |project| { - collect_uuid_orphans_locked( - project, - project_dir, - maximum, - membership_authority, - ordinal_authority, - ) + collect_uuid_orphans_locked(project, project_dir, maximum, ordinal_authority) }) } @@ -256,7 +156,6 @@ pub(super) fn collect_uuid_orphans_locked( project: &graphforge_filesystem::StableDirectory, project_root: &Path, maximum: usize, - membership_authority: Option<&AuthenticatedV3MembershipAuthority>, ordinal_authority: Option<&crate::AuthenticatedV4OrdinalIdentityAuthority>, ) -> Result { let topology = match project.open_child_directory(std::ffi::OsStr::new("topology")) { @@ -273,30 +172,9 @@ pub(super) fn collect_uuid_orphans_locked( } Err(error) => return Err(storage_err(error)), }; - let mut manifest_file = index - .open_child_file(std::ffi::OsStr::new(MANIFEST)) - .map_err(storage_err)?; - let manifest_bytes = read_bounded(&mut manifest_file, MAX_MANIFEST_BYTES)?; - let membership_authority = membership_authority.ok_or_else(|| { - storage_err("UUID membership orphan maintenance requires selected generation authority") - })?; - if hex_sha256(&manifest_bytes) != membership_authority.manifest_sha256 { - return Err(storage_err( - "UUID membership manifest differs from selected generation authority", - )); - } - let manifest = decode_manifest(&manifest_bytes)?; - if manifest.current_generation != membership_authority.topology_generation { - return Err(storage_err( - "UUID membership generation differs from selected generation authority", - )); - } - validate_run_descriptors(&manifest)?; - let mut referenced = manifest_file_names(&manifest); - referenced.extend(authenticated_v4_references( - project_root, - ordinal_authority, - )?); + // Legacy membership runs (`identities-v5-*`, `node-surrogates-v5-*`) are + // not authority any more; they are never candidates for collection. + let referenced = authenticated_v4_references(project_root, ordinal_authority)?; let mut names = index.child_names().map_err(storage_err)?; names.sort(); let mut work = UuidIndexOrphanGcWork::default(); @@ -393,15 +271,11 @@ fn is_canonical_run_name(name: &str) -> bool { return false; }; let is_v4 = is_canonical_v4_artifact_prefix(prefix); - ((prefix.starts_with("identities-v5") || prefix.starts_with("node-surrogates-v5")) || is_v4) + is_v4 && digest.len() == 16 - && if is_v4 { - digest - .bytes() - .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) - } else { - digest.bytes().all(|byte| byte.is_ascii_hexdigit()) - } + && digest + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) } fn is_canonical_v4_artifact_prefix(prefix: &str) -> bool { @@ -412,40 +286,5 @@ fn is_canonical_v4_artifact_prefix(prefix: &str) -> bool { && generation.parse::().is_ok_and(|value| value != 0) } -pub(super) fn manifest_file_names(manifest: &Manifest) -> BTreeSet { - manifest - .runs - .iter() - .flat_map(|run| { - [ - run.identities.name.clone(), - run.node_surrogates.name.clone(), - ] - }) - .collect() -} - -#[cfg(test)] -pub(super) fn cleanup_superseded_files( - root: &Path, - prior: BTreeSet, - manifest: &Manifest, -) -> Result<(), GfError> { - let retained = manifest_file_names(manifest); - let directory = graphforge_filesystem::StableDirectory::open(root).map_err(storage_err)?; - let mut retirement = - crate::durable_commit::RetirementBatch::new(&directory).map_err(storage_err)?; - for name in prior.difference(&retained) { - let file = directory - .open_child_file(std::ffi::OsStr::new(name)) - .map_err(storage_err)?; - let identity = graphforge_filesystem::file_identity(&file).map_err(storage_err)?; - retirement - .unlink(std::ffi::OsStr::new(name), identity) - .map_err(storage_err)?; - } - retirement.acknowledge().map_err(storage_err) -} - #[cfg(test)] mod tests; diff --git a/crates/graphforge-storage/src/uuid_membership/maintenance/tests.rs b/crates/graphforge-storage/src/uuid_membership/maintenance/tests.rs index 96a131320..17dd680c2 100644 --- a/crates/graphforge-storage/src/uuid_membership/maintenance/tests.rs +++ b/crates/graphforge-storage/src/uuid_membership/maintenance/tests.rs @@ -1,32 +1,21 @@ use super::super::INDEX_DIR; -use super::super::MANIFEST; -use super::super::MAX_MANIFEST_BYTES; -use super::super::Manifest; use super::super::TopologyIndexReceipt; -use super::super::UuidIndexBuildLimits; -use super::super::UuidIndexKind; -use super::super::UuidMembershipIndex; use super::super::V4_ORDINAL_MANIFEST; use super::super::V4_ORDINAL_RECEIPT; -use super::super::append_uuid_membership_delta; -use super::super::rebuild::rebuild_uuid_membership_indexes; use super::super::tests::fixture; use super::super::tests::install_test_v4_facet; use super::super::topology_delta::hex_sha256; -use super::maintain_uuid_membership_orphans; use super::maintain_uuid_membership_orphans_with_ordinal_authority; use std::fs; -use uuid::Uuid; #[test] -fn orphan_collection_authenticates_and_preserves_union_of_v3_and_v4_facets() { - let (dir, nodes, edges) = fixture(); +fn orphan_collection_authenticates_and_preserves_the_selected_v4_facet() { + let (dir, nodes, _) = fixture(); fs::write( dir.path().join("topology/generation.json"), b"{\"topology_generation\":7,\"search_generation\":0,\"property_generation\":0}\n", ) .unwrap(); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); let (selected, authority) = install_test_v4_facet(dir.path(), 7, &nodes); let orphan = dir .path() @@ -42,14 +31,6 @@ fn orphan_collection_authenticates_and_preserves_union_of_v3_and_v4_facets() { for name in selected { assert!(dir.path().join(INDEX_DIR).join(name).exists()); } - let mut v3 = UuidMembershipIndex::open(dir.path()).unwrap(); - assert_eq!(v3.count(UuidIndexKind::Node), nodes.len() as u64); - assert_eq!(v3.count(UuidIndexKind::Edge), edges.len() as u64); - assert_eq!( - v3.probe(UuidIndexKind::Edge, &[edges[0]]).unwrap().0, - vec![true] - ); - let receipt_path = dir.path().join(INDEX_DIR).join(V4_ORDINAL_RECEIPT); let manifest_path = dir.path().join(INDEX_DIR).join(V4_ORDINAL_MANIFEST); let mut manifest: crate::V4OrdinalIdentityManifest = @@ -79,7 +60,6 @@ fn ordinal_orphan_admission_rejects_link_fifo_and_oversized_manifest() { b"{\"topology_generation\":7,\"search_generation\":0,\"property_generation\":0}\n", ) .unwrap(); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); let (_, authority) = install_test_v4_facet(dir.path(), 7, &nodes); let manifest = dir.path().join(INDEX_DIR).join(V4_ORDINAL_MANIFEST); let original = fs::read(&manifest).unwrap(); @@ -107,7 +87,11 @@ fn ordinal_orphan_admission_rejects_link_fifo_and_oversized_manifest() { ); fs::remove_file(&manifest).unwrap(); - fs::write(&manifest, vec![b'x'; MAX_MANIFEST_BYTES as usize + 1]).unwrap(); + fs::write( + &manifest, + vec![b'x'; crate::ordinal_identity_v4::MAX_MANIFEST_BYTES as usize + 1], + ) + .unwrap(); assert!( maintain_uuid_membership_orphans_with_ordinal_authority(dir.path(), 16, Some(&authority)) .is_err() @@ -126,7 +110,6 @@ fn orphan_collection_never_unlinks_shared_v4_runs_or_their_store_names() { b"{\"topology_generation\":7,\"search_generation\":0,\"property_generation\":0}\n", ) .unwrap(); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); let (selected, authority) = install_test_v4_facet(dir.path(), 7, &nodes); let index = dir.path().join(INDEX_DIR); let store = dir.path().join("store"); @@ -173,86 +156,47 @@ fn orphan_collection_never_unlinks_shared_v4_runs_or_their_store_names() { } #[test] -fn orphan_maintenance_is_bounded_and_preserves_linked_and_live_runs() { - let dir = tempfile::tempdir().unwrap(); - crate::generation::force_bump_topology_generation_for_test(dir.path()).unwrap(); - append_uuid_membership_delta( - dir.path(), - 1, - &[(Uuid::from_u128(1), 1)], - &[Uuid::from_u128(2)], +fn orphan_maintenance_is_bounded_and_never_collects_legacy_membership_files() { + let (dir, nodes, _) = fixture(); + fs::write( + dir.path().join("topology/generation.json"), + b"{\"topology_generation\":7,\"search_generation\":0,\"property_generation\":0}\n", ) .unwrap(); + let (selected, authority) = install_test_v4_facet(dir.path(), 7, &nodes); let root = dir.path().join(INDEX_DIR); - let manifest: Manifest = - serde_json::from_slice(&fs::read(root.join(MANIFEST)).unwrap()).unwrap(); - let live = root.join(&manifest.runs[0].identities.name); - let orphan_one = root.join("identities-v5-orphan-0000000000000001.uuidx"); - let orphan_two = root.join("node-surrogates-v5-orphan-0000000000000002.uuidx"); - fs::copy(&live, &orphan_one).unwrap(); - fs::copy(&live, &orphan_two).unwrap(); - - assert!(maintain_uuid_membership_orphans(dir.path(), 1).is_err()); + let orphan_one = root.join("ordinal-v4-7-0000000000000001.uuidx"); + let orphan_two = root.join("forward-v4-7-0000000000000002.uuidx"); + fs::write(&orphan_one, b"orphan one").unwrap(); + fs::write(&orphan_two, b"orphan two").unwrap(); + // A project built before #1902 still carries the membership index. Its + // files are not authority and not collectable. + let legacy = [ + root.join("manifest.json"), + root.join("topology-receipt.json"), + root.join("identities-v5-1-0000000000000003.uuidx"), + root.join("node-surrogates-v5-1-0000000000000004.uuidx"), + ]; + for path in &legacy { + fs::write(path, b"legacy membership bytes").unwrap(); + } let first = - maintain_uuid_membership_orphans_with_ordinal_authority(dir.path(), 1, None).unwrap(); + maintain_uuid_membership_orphans_with_ordinal_authority(dir.path(), 1, Some(&authority)) + .unwrap(); assert_eq!(first.candidates, 2); assert_eq!(first.removed, 1); assert_eq!(first.deferred_limit, 1); - assert!(live.is_file()); let second = - maintain_uuid_membership_orphans_with_ordinal_authority(dir.path(), 64, None).unwrap(); + maintain_uuid_membership_orphans_with_ordinal_authority(dir.path(), 64, Some(&authority)) + .unwrap(); assert_eq!(second.removed, 1); - assert!(live.is_file()); assert!(!orphan_one.exists()); assert!(!orphan_two.exists()); -} - -#[test] -fn orphan_collection_refuses_unsupported_manifest_before_deleting_any_file() { - let (dir, _, _) = fixture(); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); - let root = dir.path().join(INDEX_DIR); - let path = root.join(MANIFEST); - let original = fs::read(&path).unwrap(); - let mut manifest: Manifest = serde_json::from_slice(&original).unwrap(); - let orphan = root.join("identities-v5-orphan-0000000000000001.uuidx"); - fs::write(&orphan, b"must survive unsupported metadata").unwrap(); - manifest.format_version = super::super::FORMAT_VERSION + 1; - let bytes = serde_json::to_vec(&manifest).unwrap(); - fs::write(&path, &bytes).unwrap(); - let authority = super::AuthenticatedV3MembershipAuthority { - topology_generation: manifest.current_generation, - manifest_sha256: hex_sha256(&bytes), - }; - let error = super::maintain_uuid_membership_orphans_with_authorities( - dir.path(), - 16, - Some(&authority), - None, - ) - .unwrap_err() - .to_string(); - assert!( - error.contains("unsupported UUID membership format version 8"), - "{error}" - ); - assert!(error.contains("recreate the index"), "{error}"); - assert_eq!( - fs::read(&orphan).unwrap(), - b"must survive unsupported metadata" - ); - assert_eq!(fs::read(&path).unwrap(), bytes); - let error = super::super::validate_run_descriptors(&manifest) - .unwrap_err() - .to_string(); - assert!( - error.contains("unsupported UUID membership format version 8"), - "{error}" - ); - fs::write(&path, original).unwrap(); - let work = - maintain_uuid_membership_orphans_with_ordinal_authority(dir.path(), 16, None).unwrap(); - assert_eq!(work.removed, 1); - assert!(!orphan.exists()); + for name in selected { + assert!(root.join(name).exists()); + } + for path in legacy { + assert_eq!(fs::read(path).unwrap(), b"legacy membership bytes"); + } } diff --git a/crates/graphforge-storage/src/uuid_membership/ordinal_artifacts.rs b/crates/graphforge-storage/src/uuid_membership/ordinal_artifacts.rs index 4aff0b7ba..0f3da7625 100644 --- a/crates/graphforge-storage/src/uuid_membership/ordinal_artifacts.rs +++ b/crates/graphforge-storage/src/uuid_membership/ordinal_artifacts.rs @@ -1006,7 +1006,6 @@ pub(crate) fn publish_v4_construction_artifacts( encoded: &graphforge_filesystem::StableDirectory, bundle: V4ConstructionArtifactBundle, generation: u64, - topology_delta_sha256: &str, parent: Option<( &crate::ResolvedProjectGeneration, &crate::V4OrdinalIdentityManifest, @@ -1052,7 +1051,6 @@ pub(crate) fn publish_v4_construction_artifacts( &metrics, &mut publications, generation, - topology_delta_sha256, &local_names, allocation, )?; @@ -1256,7 +1254,6 @@ fn publish_v4_construction_artifacts_inner( metrics: &V4OrdinalBuildMetrics, publications: &mut Vec<(String, V4PublicationGuard)>, generation: u64, - topology_delta_sha256: &str, local_names: &BTreeSet, allocation: Option<&crate::StorageAllocationOperation>, ) -> Result< @@ -1277,10 +1274,14 @@ fn publish_v4_construction_artifacts_inner( .open_child_directory(std::ffi::OsStr::new("uuid-membership")) .map_err(storage_err)?; let manifest_body = serde_json::to_vec(manifest).map_err(storage_err)?; + // The delta an initial or staged build publishes is exactly its node + // mapping, which the manifest's forward artifacts name by content. + let mut delta_binding = b"graphforge/v4-construction-delta/v1".to_vec(); + delta_binding.extend_from_slice(&manifest_body); let receipt = TopologyIndexReceipt { nonce: Uuid::new_v4().simple().to_string(), expected_generation: generation, - topology_delta_sha256: topology_delta_sha256.to_owned(), + topology_delta_sha256: hex_sha256(&delta_binding), manifest_sha256: hex_sha256(&manifest_body), }; let receipt_body = serde_json::to_vec(&receipt).map_err(storage_err)?; diff --git a/crates/graphforge-storage/src/uuid_membership/ordinal_artifacts/tests.rs b/crates/graphforge-storage/src/uuid_membership/ordinal_artifacts/tests.rs index 93f950cdb..bb97b44c7 100644 --- a/crates/graphforge-storage/src/uuid_membership/ordinal_artifacts/tests.rs +++ b/crates/graphforge-storage/src/uuid_membership/ordinal_artifacts/tests.rs @@ -102,17 +102,9 @@ fn content_addressed_v4_install_reuses_identical_and_preserves_collisions() { let reused = stage_v4_ordinal_bundle(mappings, 1, &index, &mut || false).unwrap(); assert!(reused.publications.is_empty()); inject_v4_authority_failure("after_artifacts"); - let error = publish_v4_construction_artifacts( - &encoded, - reused, - 1, - &hex_sha256(b"delta"), - None, - &mut || false, - None, - ) - .unwrap_err() - .to_string(); + let error = publish_v4_construction_artifacts(&encoded, reused, 1, None, &mut || false, None) + .unwrap_err() + .to_string(); assert!(error.contains("injected v4 authority failure at after_artifacts")); for (name, identity, bytes) in &originals { let file = index.open_child_file(std::ffi::OsStr::new(name)).unwrap(); @@ -133,16 +125,8 @@ fn content_addressed_v4_install_reuses_identical_and_preserves_collisions() { let reused = stage_v4_ordinal_bundle(mappings, 1, &index, &mut || false).unwrap(); assert!(reused.publications.is_empty()); - let (outputs, _, _) = publish_v4_construction_artifacts( - &encoded, - reused, - 1, - &hex_sha256(b"delta"), - None, - &mut || false, - None, - ) - .unwrap(); + let (outputs, _, _) = + publish_v4_construction_artifacts(&encoded, reused, 1, None, &mut || false, None).unwrap(); for (name, identity, bytes) in &originals { let output = outputs .iter() diff --git a/crates/graphforge-storage/src/uuid_membership/probing.rs b/crates/graphforge-storage/src/uuid_membership/probing.rs deleted file mode 100644 index 00a4d2f89..000000000 --- a/crates/graphforge-storage/src/uuid_membership/probing.rs +++ /dev/null @@ -1,1306 +0,0 @@ -//! Authenticated membership probes and retained construction snapshots. - -use super::AuthenticatedRun; -use super::AuthenticatedUuidIndexSnapshot; -use super::BULK_IO_BYTES; -use super::BlockRecord; -use super::ConstructionIndexReference; -use super::ConstructionReferenceAuthentication; -use super::ConstructionReferenceAuthenticationWork; -use super::FileRecord; -use super::IDENTITY_RECORD_BYTES; -#[cfg(test)] -use super::IDENTITY_RECORD_WIDTH; -use super::INDEX_DIR; -use super::MANIFEST; -use super::MAX_MANIFEST_BYTES; -use super::Manifest; -use super::NODE_LOOKUP_RECORD_BYTES; -#[cfg(test)] -use super::NODE_LOOKUP_RECORD_WIDTH; -use super::OpenRun; -#[cfg(test)] -use super::UuidConstructionSnapshotWork; -use super::UuidIndexKind; -use super::UuidMembershipIndex; -use super::UuidProbeMetrics; -use super::authenticate_file_blocks; -use super::batch_identity_states; -use super::block_matches; -use super::decode_manifest; -use super::hex_bytes; -#[cfg(test)] -use super::identity_codec; -use super::open_uuid_child_file; -use super::open_verified; -use super::open_verified_at; -use super::record_length; -use super::retained_run_has_safe_links; -use super::storage_err; -use super::topology_delta::hex_sha256; -use super::topology_delta::read_bounded; -use super::validate_run_contents; -use super::validate_run_descriptors; -use super::validate_surrogate_pairs; -use graphforge_core::GfError; -#[cfg(test)] -use graphforge_core::hash_observation::ArtifactSha256 as Sha256; -#[cfg(test)] -use sha2::Digest; -use std::collections::BTreeMap; -use std::collections::BTreeSet; -use std::fs; -use std::fs::File; -use std::io::Read; -use std::io::Seek; -use std::io::SeekFrom; -use std::path::Path; -use uuid::Uuid; - -#[derive(Clone, Copy)] -pub(super) enum ProbeFileKind { - Identity, - Surrogate, -} - -pub(super) fn authenticated_probe_block( - file: &mut File, - block: &BlockRecord, - width: usize, - kind: ProbeFileKind, - metrics: &mut UuidProbeMetrics, -) -> Result, GfError> { - file.seek(SeekFrom::Start(block.offset)) - .map_err(storage_err)?; - metrics.file_seeks = metrics.file_seeks.saturating_add(1); - let mut bytes = vec![0_u8; block.len as usize]; - file.read_exact(&mut bytes).map_err(storage_err)?; - if !block_matches(&bytes, block, width) { - return Err(storage_err("UUID probe block authentication failed")); - } - match kind { - ProbeFileKind::Identity => { - metrics.identity_blocks_read = metrics.identity_blocks_read.saturating_add(1); - metrics.identity_bytes_read = metrics - .identity_bytes_read - .saturating_add(bytes.len() as u64); - } - ProbeFileKind::Surrogate => { - metrics.surrogate_blocks_read = metrics.surrogate_blocks_read.saturating_add(1); - metrics.surrogate_bytes_read = metrics - .surrogate_bytes_read - .saturating_add(bytes.len() as u64); - } - } - Ok(bytes) -} - -#[cfg(test)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct ConstructionUuidIdentity { - pub uuid: Uuid, - pub kind: UuidIndexKind, - pub surrogate: u64, -} - -/// Retained authority for a UUID snapshot authenticated exactly once while its -/// live identities were emitted as one bounded sorted stream. -#[cfg(test)] -pub(crate) struct UuidConstructionSnapshot { - root: graphforge_filesystem::StableDirectory, - root_identity: graphforge_filesystem::FileIdentity, - manifest_file: File, - manifest_identity: graphforge_filesystem::FileIdentity, - manifest_sha256: String, - manifest_bytes: u64, - manifest: Manifest, - named_files: Vec<(String, graphforge_filesystem::FileIdentity)>, - payload_consumed: bool, -} - -#[cfg(test)] -impl UuidConstructionSnapshot { - pub(crate) fn revalidate(&self) -> Result<(), GfError> { - self.root.revalidate_named().map_err(storage_err)?; - if self.root.identity() != self.root_identity - || graphforge_filesystem::file_identity(&self.manifest_file).map_err(storage_err)? - != self.manifest_identity - || graphforge_filesystem::file_link_count(&self.manifest_file).map_err(storage_err)? - != 1 - { - return Err(storage_err("construction UUID snapshot authority changed")); - } - let mut manifest = self - .root - .open_child_file(std::ffi::OsStr::new(MANIFEST)) - .map_err(storage_err)?; - let body = read_bounded(&mut manifest, MAX_MANIFEST_BYTES)?; - if graphforge_filesystem::file_identity(&manifest).map_err(storage_err)? - != self.manifest_identity - || hex_sha256(&body) != self.manifest_sha256 - || decode_manifest(&body)? != self.manifest - { - return Err(storage_err("construction UUID manifest changed")); - } - for (name, identity) in &self.named_files { - let file = self - .root - .open_child_file(std::ffi::OsStr::new(name)) - .map_err(storage_err)?; - if graphforge_filesystem::file_identity(&file).map_err(storage_err)? != *identity - || graphforge_filesystem::file_link_count(&file).map_err(storage_err)? != 1 - { - return Err(storage_err("construction UUID run identity changed")); - } - } - Ok(()) - } - - /// Authenticate each retained payload block exactly once and emit the live - /// identity domain as a bounded UUID-ordered stream. - pub(crate) fn stream_authenticated( - &mut self, - mut emit: impl FnMut(ConstructionUuidIdentity) -> Result<(), GfError>, - ) -> Result { - if self.payload_consumed { - return Err(storage_err( - "construction UUID payload was already consumed", - )); - } - self.revalidate()?; - let mut identity_cursors = Vec::with_capacity(self.manifest.runs.len()); - let mut work = UuidConstructionSnapshotWork { - authentication_bytes: self.manifest_bytes, - authentication_blocks: 1, - ..Default::default() - }; - for run in &self.manifest.runs { - let identities = self - .root - .open_child_file(std::ffi::OsStr::new(&run.identities.name)) - .map_err(storage_err)?; - identity_cursors.push(ConstructionRunCursor::new( - identities, - run.identities.clone(), - IDENTITY_RECORD_WIDTH, - )); - let surrogates = self - .root - .open_child_file(std::ffi::OsStr::new(&run.node_surrogates.name)) - .map_err(storage_err)?; - let mut cursor = ConstructionRunCursor::new( - surrogates, - run.node_surrogates.clone(), - NODE_LOOKUP_RECORD_WIDTH, - ); - while cursor.next_record()?.is_some() {} - work.authentication_bytes = work.authentication_bytes.saturating_add(cursor.bytes); - work.authentication_blocks = work.authentication_blocks.saturating_add(cursor.blocks); - } - let mut heads = identity_cursors - .iter_mut() - .map(ConstructionRunCursor::next_record) - .collect::, _>>()?; - loop { - let Some(next_uuid) = heads - .iter() - .flatten() - .map(|record| &record[..16]) - .min() - .map(<[u8]>::to_vec) - else { - break; - }; - let indexes = heads - .iter() - .enumerate() - .filter_map(|(index, record)| { - record - .as_ref() - .is_some_and(|record| record[..16] == next_uuid) - .then_some(index) - }) - .collect::>(); - let selected = *indexes - .iter() - .max_by_key(|index| self.manifest.runs[**index].last_generation) - .expect("one UUID head was selected"); - let record = heads[selected].as_ref().expect("selected head exists"); - let kind = record[16]; - if !matches!(kind, 0..=3) { - return Err(storage_err( - "construction UUID identity record is malformed", - )); - } - if matches!(kind, 0 | 1) { - let identity = ConstructionUuidIdentity { - uuid: Uuid::from_bytes(record[..16].try_into().expect("fixed UUID width")), - kind: if kind == 0 { - UuidIndexKind::Node - } else { - UuidIndexKind::Edge - }, - surrogate: u64::from_be_bytes(record[17..25].try_into().expect("fixed")), - }; - if identity.kind == UuidIndexKind::Node { - if identity.surrogate == 0 { - return Err(storage_err("live node has zero surrogate")); - } - work.live_nodes = work.live_nodes.saturating_add(1); - work.max_node_surrogate = work.max_node_surrogate.max(identity.surrogate); - } else { - work.live_edges = work.live_edges.saturating_add(1); - } - emit(identity)?; - } - for index in indexes { - heads[index] = identity_cursors[index].next_record()?; - } - } - for cursor in &identity_cursors { - work.authentication_bytes = work.authentication_bytes.saturating_add(cursor.bytes); - work.authentication_blocks = work.authentication_blocks.saturating_add(cursor.blocks); - } - if work.live_nodes != self.manifest.live_node_count - || work.live_edges != self.manifest.live_edge_count - { - return Err(storage_err( - "construction UUID live counts differ from manifest", - )); - } - self.payload_consumed = true; - self.revalidate()?; - Ok(work) - } -} - -#[cfg(test)] -struct ConstructionRunCursor { - file: File, - descriptor: FileRecord, - width: usize, - block_index: usize, - block: Vec, - within: usize, - records: u64, - bytes: u64, - blocks: u64, - digest: Sha256, - finished: bool, -} - -#[cfg(test)] -impl ConstructionRunCursor { - fn new(file: File, descriptor: FileRecord, width: usize) -> Self { - Self { - file, - descriptor, - width, - block_index: 0, - block: Vec::new(), - within: 0, - records: 0, - bytes: 0, - blocks: 0, - digest: Sha256::new(), - finished: false, - } - } - - fn next_record(&mut self) -> Result>, GfError> { - if self.finished { - return Ok(None); - } - if self.within == self.block.len() { - if self.block_index == self.descriptor.blocks.len() { - self.finished = true; - if self.records != self.descriptor.count - || self.bytes != record_length(&self.descriptor, self.width as u64)? - || hex_bytes(&self.digest.clone().finalize()) != self.descriptor.sha256 - { - return Err(storage_err("construction UUID run authentication failed")); - } - return Ok(None); - } - let descriptor = &self.descriptor.blocks[self.block_index]; - if descriptor.offset != self.bytes - || descriptor.len as usize > BULK_IO_BYTES - || descriptor.len == 0 - { - return Err(storage_err("construction UUID block framing changed")); - } - self.block.resize(descriptor.len as usize, 0); - self.file.read_exact(&mut self.block).map_err(storage_err)?; - if !block_matches(&self.block, descriptor, self.width) { - return Err(storage_err("construction UUID block digest changed")); - } - self.digest.update(&self.block); - self.bytes = self.bytes.saturating_add(self.block.len() as u64); - self.blocks = self.blocks.saturating_add(1); - self.block_index += 1; - self.within = 0; - } - let record = if self.width == IDENTITY_RECORD_WIDTH { - let mut remaining = &self.block[self.within..]; - let record = identity_codec::take(&mut remaining)? - .ok_or_else(|| storage_err("missing identity record"))?; - self.within = self.block.len() - remaining.len(); - record.to_vec() - } else { - let end = self.within + self.width; - let record = self.block[self.within..end].to_vec(); - self.within = end; - record - }; - self.records = self.records.saturating_add(1); - Ok(Some(record)) - } -} - -/// Authenticate each retained UUID byte once and emit the live identity set in -/// UUID order. The returned token revalidates inode/name authority without -/// rereading retained payload bytes. -#[cfg(test)] -pub(crate) fn open_uuid_construction_snapshot( - project_dir: &Path, - generation: u64, - emit: impl FnMut(ConstructionUuidIdentity) -> Result<(), GfError>, -) -> Result<(UuidConstructionSnapshot, UuidConstructionSnapshotWork), GfError> { - let mut token = super::pin_uuid_construction_snapshot(project_dir, generation)?; - let work = token.stream_authenticated(emit)?; - Ok((token, work)) -} - -/// Pin the generation's manifest and every run inode without reading retained -/// payload bytes. The caller later consumes those bytes exactly once through -/// [`UuidConstructionSnapshot::stream_authenticated`]. -#[cfg(test)] -pub(crate) fn pin_uuid_construction_snapshot( - project_dir: &Path, - generation: u64, -) -> Result { - let root = graphforge_filesystem::StableDirectory::open(&project_dir.join(INDEX_DIR)) - .map_err(storage_err)?; - let root_identity = root.identity(); - let mut manifest_file = root - .open_child_file(std::ffi::OsStr::new(MANIFEST)) - .map_err(storage_err)?; - let manifest_identity = - graphforge_filesystem::file_identity(&manifest_file).map_err(storage_err)?; - let body = read_bounded(&mut manifest_file, MAX_MANIFEST_BYTES)?; - let manifest_sha256 = hex_sha256(&body); - let manifest = decode_manifest(&body)?; - if manifest.current_generation != generation { - return Err(storage_err( - "construction UUID snapshot generation is stale", - )); - } - validate_run_descriptors(&manifest)?; - let mut named_files = Vec::with_capacity(manifest.runs.len().saturating_mul(2)); - for run in &manifest.runs { - let identities = root - .open_child_file(std::ffi::OsStr::new(&run.identities.name)) - .map_err(storage_err)?; - let identity = graphforge_filesystem::file_identity(&identities).map_err(storage_err)?; - if graphforge_filesystem::file_link_count(&identities).map_err(storage_err)? != 1 { - return Err(storage_err( - "construction UUID identity run has extra links", - )); - } - named_files.push((run.identities.name.clone(), identity)); - - let surrogates = root - .open_child_file(std::ffi::OsStr::new(&run.node_surrogates.name)) - .map_err(storage_err)?; - let surrogate_identity = - graphforge_filesystem::file_identity(&surrogates).map_err(storage_err)?; - if graphforge_filesystem::file_link_count(&surrogates).map_err(storage_err)? != 1 { - return Err(storage_err( - "construction UUID surrogate run has extra links", - )); - } - named_files.push((run.node_surrogates.name.clone(), surrogate_identity)); - } - root.revalidate_named().map_err(storage_err)?; - let token = UuidConstructionSnapshot { - root, - root_identity, - manifest_file, - manifest_identity, - manifest_sha256, - manifest_bytes: body.len() as u64, - manifest, - named_files, - payload_consumed: false, - }; - token.revalidate()?; - Ok(token) -} - -impl AuthenticatedUuidIndexSnapshot { - pub(super) fn open_retained_file(&self, record: &FileRecord) -> Result { - let (held, expected) = self - .runs - .iter() - .find_map(|run| { - if run.descriptor.identities == *record { - Some((&run.identities, run.identities_identity)) - } else if run.descriptor.node_surrogates == *record { - Some((&run.node_surrogates, run.node_surrogates_identity)) - } else { - None - } - }) - .ok_or_else(|| storage_err("retained UUID descriptor is not authenticated"))?; - let mut file = held.try_clone().map_err(storage_err)?; - let identity_changed = - graphforge_filesystem::file_identity(&file).map_err(storage_err)? != expected; - let path_native_link_changed = - self.cas_source_paths.is_none() && !retained_run_has_safe_links(&file)?; - if identity_changed || path_native_link_changed { - return Err(storage_err("retained UUID run identity changed")); - } - file.seek(SeekFrom::Start(0)).map_err(storage_err)?; - Ok(file) - } - - pub(super) fn retained_reference( - &self, - record: &FileRecord, - ) -> Result { - let file = self.open_retained_file(record)?; - let identity = graphforge_filesystem::file_identity(&file).map_err(storage_err)?; - let source_path = self - .cas_source_paths - .as_ref() - .and_then(|paths| paths.get(&record.name)) - .map_or_else( - || format!("{INDEX_DIR}/{}", record.name), - |(path, _, _, _)| path.clone(), - ); - Ok(ConstructionIndexReference { - source_root: self.graph_root_path.to_string_lossy().into_owned(), - source_root_volume: self.graph_root_identity.volume_serial, - source_root_file_id: hex_bytes(&self.graph_root_identity.file_id), - source_path, - source_volume: identity.volume_serial, - source_file_id: hex_bytes(&identity.file_id), - target_path: format!("{INDEX_DIR}/{}", record.name), - bytes: record_length( - record, - if record.name.starts_with("identities-") { - IDENTITY_RECORD_BYTES - } else { - NODE_LOOKUP_RECORD_BYTES - }, - )?, - sha256: record.sha256.clone(), - xxh64: record.xxh64, - parent_manifest_sha256: self.manifest_sha256.clone(), - }) - } - - pub(crate) fn authenticate_construction_references( - &self, - references: &[ConstructionReferenceAuthentication<'_>], - ) -> Result { - self.authenticate_construction_references_with(references, || {}) - } - - fn authenticate_construction_references_with( - &self, - references: &[ConstructionReferenceAuthentication<'_>], - before_final_revalidation: impl FnOnce(), - ) -> Result { - self.revalidate()?; - let mut referenced_payload_bytes = 0_u64; - for reference in references { - referenced_payload_bytes = referenced_payload_bytes - .saturating_add(self.authenticate_construction_reference_once(reference)?); - } - before_final_revalidation(); - self.revalidate()?; - Ok(ConstructionReferenceAuthenticationWork { - global_revalidation_bytes: self.snapshot_authentication_bytes().saturating_mul(2), - referenced_payload_bytes, - }) - } - - fn snapshot_authentication_bytes(&self) -> u64 { - let manifest_bytes = self.manifest_bytes; - self.manifest - .runs - .iter() - .fold(manifest_bytes, |total, run| { - total - .saturating_add( - run.identities - .blocks - .iter() - .map(|block| u64::from(block.len)) - .sum::(), - ) - .saturating_add( - run.node_surrogates - .count - .saturating_mul(NODE_LOOKUP_RECORD_BYTES), - ) - }) - } - - fn authenticate_construction_reference_once( - &self, - reference: &ConstructionReferenceAuthentication<'_>, - ) -> Result { - if reference.source_root != self.graph_root_path.to_string_lossy() - || reference.source_root_volume != self.graph_root_identity.volume_serial - || reference.source_root_file_id != hex_bytes(&self.graph_root_identity.file_id) - || reference.parent_manifest_sha256 != self.manifest_sha256 - || !reference.target_path.starts_with(&format!("{INDEX_DIR}/")) - { - return Err(storage_err( - "retained construction reference authority changed", - )); - } - let name = reference - .target_path - .strip_prefix(&format!("{INDEX_DIR}/")) - .ok_or_else(|| storage_err("retained construction target path is invalid"))?; - let expected_source = self - .cas_source_paths - .as_ref() - .and_then(|paths| paths.get(name)) - .map_or(reference.target_path, |(path, _, _, _)| path.as_str()); - if reference.source_path != expected_source { - return Err(storage_err("retained construction source path changed")); - } - let record = self - .manifest - .runs - .iter() - .flat_map(|run| [&run.identities, &run.node_surrogates]) - .find(|record| record.name == name) - .ok_or_else(|| storage_err("retained construction run is absent"))?; - let mut file = self.open_retained_file(record)?; - let identity = graphforge_filesystem::file_identity(&file).map_err(storage_err)?; - let expected_bytes = record_length( - record, - if record.name.starts_with("identities-") { - IDENTITY_RECORD_BYTES - } else { - NODE_LOOKUP_RECORD_BYTES - }, - )?; - file.seek(SeekFrom::Start(0)).map_err(storage_err)?; - let mut checksum = crate::corruption_checksum::Checksum::new(); - let mut actual_bytes = 0_u64; - let mut block = vec![0_u8; BULK_IO_BYTES]; - loop { - let count = file.read(&mut block).map_err(storage_err)?; - if count == 0 { - break; - } - checksum.update(&block[..count]); - actual_bytes = actual_bytes.saturating_add(count as u64); - } - if identity.volume_serial != reference.source_volume - || hex_bytes(&identity.file_id) != reference.source_file_id - || reference.bytes != expected_bytes - || actual_bytes != expected_bytes - || reference.sha256 != record.sha256 - || reference.xxh64 != record.xxh64 - || checksum.finish() != record.xxh64 - { - return Err(storage_err(format!( - "retained construction reference changed: volume={} expected_volume={} file_id={} expected_file_id={} bytes={} expected_bytes={} reference_sha={} manifest_sha={}", - identity.volume_serial, - reference.source_volume, - hex_bytes(&identity.file_id), - reference.source_file_id, - actual_bytes, - expected_bytes, - reference.sha256, - record.sha256 - ))); - } - Ok(actual_bytes) - } - - pub(crate) fn open_at_generation(project_dir: &Path, generation: u64) -> Result { - let graph_root = - graphforge_filesystem::StableDirectory::open(project_dir).map_err(storage_err)?; - let graph_root_identity = graph_root.identity(); - let root_path = project_dir.join(INDEX_DIR); - let root = graphforge_filesystem::StableDirectory::open(&root_path).map_err(storage_err)?; - let root_identity = root.identity(); - let mut manifest_file = open_uuid_child_file(&root, std::ffi::OsStr::new(MANIFEST))?; - let manifest_identity = - graphforge_filesystem::file_identity(&manifest_file).map_err(storage_err)?; - let body = read_bounded(&mut manifest_file, MAX_MANIFEST_BYTES)?; - let manifest_sha256 = hex_sha256(&body); - let manifest = decode_manifest(&body)?; - if manifest.current_generation != generation { - return Err(storage_err("authenticated snapshot generation is stale")); - } - validate_run_descriptors(&manifest)?; - let mut authenticated_bytes = body.len() as u64; - let mut authenticated_blocks = 1_u64; - let mut runs = Vec::with_capacity(manifest.runs.len()); - for descriptor in &manifest.runs { - let identities = - open_verified_at(&root, &descriptor.identities, IDENTITY_RECORD_BYTES)?; - let node_surrogates = - open_verified_at(&root, &descriptor.node_surrogates, NODE_LOOKUP_RECORD_BYTES)?; - authenticated_bytes = authenticated_bytes - .saturating_add(record_length( - &descriptor.identities, - IDENTITY_RECORD_BYTES, - )?) - .saturating_add(descriptor.node_surrogates.count * NODE_LOOKUP_RECORD_BYTES); - authenticated_blocks = authenticated_blocks - .saturating_add(descriptor.identities.blocks.len() as u64) - .saturating_add(descriptor.node_surrogates.blocks.len() as u64); - runs.push(AuthenticatedRun { - identities_identity: graphforge_filesystem::file_identity(&identities) - .map_err(storage_err)?, - node_surrogates_identity: graphforge_filesystem::file_identity(&node_surrogates) - .map_err(storage_err)?, - identities, - node_surrogates, - descriptor: descriptor.clone(), - }); - } - Ok(Self { - graph_root, - graph_root_path: project_dir.to_path_buf(), - graph_root_identity, - root, - root_identity, - manifest_bytes: body.len() as u64, - manifest_file: Some(manifest_file), - manifest_identity, - manifest_sha256, - manifest, - runs, - authenticated_bytes, - authenticated_blocks, - cas_source_paths: None, - _cas_leases: Vec::new(), - }) - } - - #[allow( - clippy::too_many_lines, - reason = "one descriptor-lifetime authentication pass keeps every CAS file and manifest binding in scope" - )] - pub(crate) fn open_from_compact_inventory( - container_root: &Path, - inventory: &crate::GraphFilesInventory, - generation: u64, - ) -> Result { - let graph_root = - graphforge_filesystem::StableDirectory::open(container_root).map_err(storage_err)?; - let graph_root_identity = graph_root.identity(); - let root = - graphforge_filesystem::StableDirectory::open(container_root).map_err(storage_err)?; - let root_identity = root.identity(); - let manifest_path = format!("{INDEX_DIR}/{MANIFEST}"); - let manifest_entry = inventory - .files - .iter() - .find(|entry| entry.relative_path == manifest_path) - .ok_or_else(|| storage_err("compact UUID manifest is absent"))?; - let mut manifest_lease = crate::graph_object_store::open_graph_object_with_checksum( - container_root, - manifest_entry, - )?; - let manifest_identity = - graphforge_filesystem::file_identity(manifest_lease.as_ref()).map_err(storage_err)?; - let body = read_bounded(&mut manifest_lease, MAX_MANIFEST_BYTES)?; - let manifest_file = manifest_lease.try_clone_file().map_err(storage_err)?; - let manifest_sha256 = hex_sha256(&body); - if manifest_sha256 != manifest_entry.content_sha256 { - return Err(storage_err("compact UUID manifest authentication changed")); - } - let manifest = decode_manifest(&body)?; - if manifest.current_generation != generation { - return Err(storage_err( - "authenticated compact snapshot generation is stale", - )); - } - validate_run_descriptors(&manifest)?; - let mut runs = Vec::with_capacity(manifest.runs.len()); - let mut paths = BTreeMap::new(); - let manifest_physical = - crate::graph_object_path(container_root, &manifest_entry.content_sha256)?; - paths.insert( - MANIFEST.to_owned(), - ( - manifest_physical - .strip_prefix(container_root) - .map_err(storage_err)? - .to_string_lossy() - .into_owned(), - manifest_entry.content_sha256.clone(), - manifest_entry.byte_length, - manifest_entry.content_xxh64, - ), - ); - let mut cas_leases = vec![manifest_lease]; - let mut authenticated_bytes = body.len() as u64; - let mut authenticated_blocks = 1_u64; - for descriptor in &manifest.runs { - let mut open_record = |record: &FileRecord, width: u64| -> Result { - let logical = format!("{INDEX_DIR}/{}", record.name); - let entry = inventory - .files - .iter() - .find(|entry| entry.relative_path == logical) - .ok_or_else(|| storage_err("compact UUID run is absent"))?; - if entry.content_sha256 != record.sha256 - || entry.byte_length != record_length(record, width)? - { - return Err(storage_err("compact UUID run authority changed")); - } - let lease = crate::graph_object_store::open_graph_object_with_checksum( - container_root, - entry, - )?; - let physical = crate::graph_object_path(container_root, &entry.content_sha256)?; - let relative = physical - .strip_prefix(container_root) - .map_err(storage_err)? - .to_string_lossy() - .into_owned(); - paths.insert( - record.name.clone(), - ( - relative, - entry.content_sha256.clone(), - entry.byte_length, - entry.content_xxh64, - ), - ); - let file = lease.try_clone_file().map_err(storage_err)?; - cas_leases.push(lease); - Ok(file) - }; - let identities = open_record(&descriptor.identities, IDENTITY_RECORD_BYTES)?; - let node_surrogates = - open_record(&descriptor.node_surrogates, NODE_LOOKUP_RECORD_BYTES)?; - authenticate_file_blocks( - &mut identities.try_clone().map_err(storage_err)?, - &descriptor.identities, - IDENTITY_RECORD_BYTES, - None, - )?; - authenticate_file_blocks( - &mut node_surrogates.try_clone().map_err(storage_err)?, - &descriptor.node_surrogates, - NODE_LOOKUP_RECORD_BYTES, - None, - )?; - authenticated_bytes = authenticated_bytes - .saturating_add(record_length( - &descriptor.identities, - IDENTITY_RECORD_BYTES, - )?) - .saturating_add(descriptor.node_surrogates.count * NODE_LOOKUP_RECORD_BYTES); - authenticated_blocks = authenticated_blocks - .saturating_add(descriptor.identities.blocks.len() as u64) - .saturating_add(descriptor.node_surrogates.blocks.len() as u64); - runs.push(AuthenticatedRun { - identities_identity: graphforge_filesystem::file_identity(&identities) - .map_err(storage_err)?, - node_surrogates_identity: graphforge_filesystem::file_identity(&node_surrogates) - .map_err(storage_err)?, - identities, - node_surrogates, - descriptor: descriptor.clone(), - }); - } - Ok(Self { - graph_root, - graph_root_path: container_root.to_path_buf(), - graph_root_identity, - root, - root_identity, - manifest_bytes: body.len() as u64, - manifest_file: Some(manifest_file), - manifest_identity, - manifest_sha256, - manifest, - runs, - authenticated_bytes, - authenticated_blocks, - cas_source_paths: Some(paths), - _cas_leases: cas_leases, - }) - } - - pub(crate) fn topology_generation(&self) -> u64 { - self.manifest.current_generation - } - - pub(crate) fn count(&self, kind: UuidIndexKind) -> u64 { - match kind { - UuidIndexKind::Node => self.manifest.live_node_count, - UuidIndexKind::Edge => self.manifest.live_edge_count, - } - } - - pub(crate) fn manifest_sha256(&self) -> &str { - &self.manifest_sha256 - } - pub(crate) fn take_authentication_work(&mut self) -> (u64, u64) { - ( - std::mem::take(&mut self.authenticated_bytes), - std::mem::take(&mut self.authenticated_blocks), - ) - } - - fn revalidate_compact_source(&self) -> Result<(), GfError> { - let objects = self - .cas_source_paths - .as_ref() - .ok_or_else(|| storage_err("compact UUID source authority is absent"))?; - let (_, manifest_digest, manifest_length, manifest_checksum) = objects - .get(MANIFEST) - .ok_or_else(|| storage_err("compact UUID manifest authority is absent"))?; - let manifest_entry = crate::GraphFileEntry { - relative_path: MANIFEST.to_owned(), - content_sha256: manifest_digest.clone(), - byte_length: *manifest_length, - content_xxh64: *manifest_checksum, - role: crate::GraphFileRole::Index, - }; - let mut manifest_lease = crate::graph_object_store::open_graph_object_with_checksum( - &self.graph_root_path, - &manifest_entry, - )?; - if graphforge_filesystem::file_identity(manifest_lease.as_ref()).map_err(storage_err)? - != self.manifest_identity - { - return Err(storage_err("compact UUID manifest identity changed")); - } - let body = read_bounded(&mut manifest_lease, MAX_MANIFEST_BYTES)?; - if hex_sha256(&body) != self.manifest_sha256 || decode_manifest(&body)? != self.manifest { - return Err(storage_err("compact UUID manifest authentication changed")); - } - for run in &self.runs { - for (record, identity) in [ - (&run.descriptor.identities, run.identities_identity), - ( - &run.descriptor.node_surrogates, - run.node_surrogates_identity, - ), - ] { - let (_, digest, length, checksum) = objects - .get(&record.name) - .ok_or_else(|| storage_err("compact UUID run authority is absent"))?; - let entry = crate::GraphFileEntry { - relative_path: record.name.clone(), - content_sha256: digest.clone(), - byte_length: *length, - content_xxh64: *checksum, - role: crate::GraphFileRole::Index, - }; - let file = crate::graph_object_store::open_graph_object_with_checksum( - &self.graph_root_path, - &entry, - )?; - if graphforge_filesystem::file_identity(file.as_ref()).map_err(storage_err)? - != identity - { - return Err(storage_err("compact UUID run identity changed")); - } - } - } - Ok(()) - } - - pub(crate) fn revalidate(&self) -> Result<(), GfError> { - self.graph_root.revalidate_named().map_err(storage_err)?; - if self.graph_root.identity() != self.graph_root_identity { - return Err(storage_err("UUID graph root identity changed")); - } - if self.cas_source_paths.is_some() { - return self.revalidate_compact_source(); - } - self.root.revalidate_named().map_err(storage_err)?; - if self.root.identity() != self.root_identity { - return Err(storage_err("UUID index root identity changed")); - } - let manifest_file = self - .manifest_file - .as_ref() - .ok_or_else(|| storage_err("UUID manifest descriptor is suspended for publication"))?; - if graphforge_filesystem::file_identity(manifest_file).map_err(storage_err)? - != self.manifest_identity - || graphforge_filesystem::file_link_count(manifest_file).map_err(storage_err)? != 1 - || self.manifest_sha256.len() != 64 - { - return Err(storage_err("retained UUID manifest identity changed")); - } - let mut named_manifest = open_uuid_child_file(&self.root, std::ffi::OsStr::new(MANIFEST))?; - if graphforge_filesystem::file_identity(&named_manifest).map_err(storage_err)? - != self.manifest_identity - || graphforge_filesystem::file_link_count(&named_manifest).map_err(storage_err)? != 1 - { - return Err(storage_err("UUID manifest identity changed")); - } - let body = read_bounded(&mut named_manifest, MAX_MANIFEST_BYTES)?; - if hex_sha256(&body) != self.manifest_sha256 || decode_manifest(&body)? != self.manifest { - return Err(storage_err("UUID manifest authentication changed")); - } - for run in &self.runs { - for (record, identity) in [ - (&run.descriptor.identities, run.identities_identity), - ( - &run.descriptor.node_surrogates, - run.node_surrogates_identity, - ), - ] { - let named = open_uuid_child_file(&self.root, std::ffi::OsStr::new(&record.name))?; - if graphforge_filesystem::file_identity(&named).map_err(storage_err)? != identity - || !retained_run_has_safe_links(&named)? - { - return Err(storage_err("UUID retained run identity changed")); - } - } - } - Ok(()) - } - - pub(super) fn suspend_owned_manifest(&mut self) { - if self.cas_source_paths.is_none() { - self.manifest_file.take(); - } - } - - pub(super) fn restore_owned_manifest(&mut self) -> Result<(), GfError> { - if self.manifest_file.is_some() { - return Ok(()); - } - self.root.revalidate_named().map_err(storage_err)?; - let mut file = open_uuid_child_file(&self.root, std::ffi::OsStr::new(MANIFEST))?; - if graphforge_filesystem::file_identity(&file).map_err(storage_err)? - != self.manifest_identity - || graphforge_filesystem::file_link_count(&file).map_err(storage_err)? != 1 - { - return Err(storage_err("suspended UUID manifest identity changed")); - } - let body = read_bounded(&mut file, MAX_MANIFEST_BYTES)?; - if hex_sha256(&body) != self.manifest_sha256 || decode_manifest(&body)? != self.manifest { - return Err(storage_err( - "suspended UUID manifest authentication changed", - )); - } - self.authenticated_bytes = self - .authenticated_bytes - .checked_add(body.len() as u64) - .ok_or_else(|| storage_err("UUID restoration byte count overflow"))?; - self.authenticated_blocks = self - .authenticated_blocks - .checked_add(1) - .ok_or_else(|| storage_err("UUID restoration block count overflow"))?; - self.manifest_file = Some(file); - Ok(()) - } - - pub(super) fn advance_to(&mut self, manifest: Manifest) -> Result { - self.root.revalidate_named().map_err(storage_err)?; - let mut manifest_file = open_uuid_child_file(&self.root, std::ffi::OsStr::new(MANIFEST))?; - let body = read_bounded(&mut manifest_file, MAX_MANIFEST_BYTES)?; - if hex_sha256(&body) != hex_sha256(&serde_json::to_vec(&manifest).map_err(storage_err)?) { - return Err(storage_err("committed UUID manifest differs from plan")); - } - let mut next_runs = Vec::with_capacity(manifest.runs.len()); - let mut authenticated_bytes = 0_u64; - for descriptor in &manifest.runs { - if let Some(retained) = self.runs.iter().find(|run| run.descriptor == *descriptor) { - next_runs.push(AuthenticatedRun { - identities: retained.identities.try_clone().map_err(storage_err)?, - identities_identity: retained.identities_identity, - node_surrogates: retained.node_surrogates.try_clone().map_err(storage_err)?, - node_surrogates_identity: retained.node_surrogates_identity, - descriptor: descriptor.clone(), - }); - } else { - let identities = - open_verified_at(&self.root, &descriptor.identities, IDENTITY_RECORD_BYTES)?; - let node_surrogates = open_verified_at( - &self.root, - &descriptor.node_surrogates, - NODE_LOOKUP_RECORD_BYTES, - )?; - authenticated_bytes = authenticated_bytes - .saturating_add(record_length( - &descriptor.identities, - IDENTITY_RECORD_BYTES, - )?) - .saturating_add(descriptor.node_surrogates.count * NODE_LOOKUP_RECORD_BYTES); - next_runs.push(AuthenticatedRun { - identities_identity: graphforge_filesystem::file_identity(&identities) - .map_err(storage_err)?, - node_surrogates_identity: graphforge_filesystem::file_identity( - &node_surrogates, - ) - .map_err(storage_err)?, - identities, - node_surrogates, - descriptor: descriptor.clone(), - }); - } - } - self.manifest_identity = - graphforge_filesystem::file_identity(&manifest_file).map_err(storage_err)?; - self.manifest_sha256 = hex_sha256(&body); - self.manifest_bytes = body.len() as u64; - self.manifest_file = Some(manifest_file); - self.manifest = manifest; - self.runs = next_runs; - self.authenticated_bytes = 0; - self.authenticated_blocks = 0; - Ok(authenticated_bytes) - } - - pub(crate) fn probe( - &mut self, - kind: UuidIndexKind, - requested: &[Uuid], - ) -> Result<(Vec, UuidProbeMetrics), GfError> { - let mut metrics = UuidProbeMetrics { - requested: requested.len() as u64, - ..Default::default() - }; - let unique = requested.iter().copied().collect::>(); - metrics.unique_requested = unique.len() as u64; - let mut unresolved = unique; - let mut resolved = std::collections::BTreeMap::new(); - for run in self.runs.iter_mut().rev() { - if unresolved.is_empty() { - break; - } - metrics.runs_considered = metrics.runs_considered.saturating_add(1); - let states = batch_identity_states( - &mut run.identities, - &run.descriptor.identities, - kind, - &unresolved, - &mut metrics, - )?; - for (uuid, state) in states { - unresolved.remove(&uuid); - metrics.found = metrics.found.saturating_add(u64::from(state.present)); - resolved.insert(uuid, state.present); - } - } - Ok(( - requested - .iter() - .map(|uuid| resolved.get(uuid).copied().unwrap_or(false)) - .collect(), - metrics, - )) - } - - pub(crate) fn lookup_node_surrogates( - &mut self, - requested: &[Uuid], - ) -> Result<(Vec>, UuidProbeMetrics), GfError> { - let mut metrics = UuidProbeMetrics { - requested: requested.len() as u64, - ..Default::default() - }; - let unique = requested.iter().copied().collect::>(); - metrics.unique_requested = unique.len() as u64; - let mut unresolved = unique; - let mut resolved = std::collections::BTreeMap::new(); - for run in self.runs.iter_mut().rev() { - if unresolved.is_empty() { - break; - } - metrics.runs_considered = metrics.runs_considered.saturating_add(1); - let states = batch_identity_states( - &mut run.identities, - &run.descriptor.identities, - UuidIndexKind::Node, - &unresolved, - &mut metrics, - )?; - let mut pairs = Vec::new(); - for (uuid, state) in states { - unresolved.remove(&uuid); - let value = state.present.then_some(state.surrogate); - if let Some(surrogate) = value { - pairs.push((surrogate, uuid)); - metrics.found = metrics.found.saturating_add(1); - } - resolved.insert(uuid, value); - } - validate_surrogate_pairs( - &mut run.node_surrogates, - &run.descriptor.node_surrogates, - &pairs, - &mut metrics, - )?; - } - Ok(( - requested - .iter() - .map(|uuid| resolved.get(uuid).copied().flatten()) - .collect(), - metrics, - )) - } -} - -impl UuidMembershipIndex { - /// Open and fully authenticate the current immutable index snapshot. - pub fn open(project_dir: &Path) -> Result { - let generation = crate::read_topology_generation(project_dir)?; - Self::open_at_generation(project_dir, generation) - } - - pub(crate) fn open_at_generation(project_dir: &Path, generation: u64) -> Result { - let root = project_dir.join(INDEX_DIR); - let body = fs::read(root.join(MANIFEST)).map_err(storage_err)?; - let manifest = decode_manifest(&body)?; - if manifest.current_generation != generation { - return Err(storage_err(format!( - "stale index generation {} (graph generation {generation})", - manifest.current_generation - ))); - } - validate_run_descriptors(&manifest)?; - let mut runs = Vec::with_capacity(manifest.runs.len()); - for descriptor in &manifest.runs { - let identities = open_verified(&root, &descriptor.identities, IDENTITY_RECORD_BYTES)?; - let node_surrogates = - open_verified(&root, &descriptor.node_surrogates, NODE_LOOKUP_RECORD_BYTES)?; - validate_run_contents( - identities.try_clone().map_err(storage_err)?, - node_surrogates.try_clone().map_err(storage_err)?, - descriptor, - )?; - runs.push(OpenRun { - identities, - node_surrogates, - descriptor: descriptor.clone(), - }); - } - Ok(Self { runs, manifest }) - } - - /// Topology generation authenticated by this open handle. - #[must_use] - pub const fn topology_generation(&self) -> u64 { - self.manifest.current_generation - } - - #[must_use] - /// Return the authenticated unique-record count for one identity domain. - pub fn count(&self, kind: UuidIndexKind) -> u64 { - match kind { - UuidIndexKind::Node => self.manifest.live_node_count, - UuidIndexKind::Edge => self.manifest.live_edge_count, - } - } - - /// Probe a batch in caller order. Memory is O(unique requested UUIDs). - pub fn probe( - &mut self, - kind: UuidIndexKind, - requested: &[Uuid], - ) -> Result<(Vec, UuidProbeMetrics), GfError> { - let mut metrics = UuidProbeMetrics { - requested: requested.len() as u64, - ..Default::default() - }; - let unique = requested.iter().copied().collect::>(); - metrics.unique_requested = unique.len() as u64; - let mut unresolved = unique; - let mut membership = std::collections::BTreeMap::new(); - for run in self.runs.iter_mut().rev() { - if unresolved.is_empty() { - break; - } - metrics.runs_considered = metrics.runs_considered.saturating_add(1); - let states = batch_identity_states( - &mut run.identities, - &run.descriptor.identities, - kind, - &unresolved, - &mut metrics, - )?; - for (uuid, state) in states { - unresolved.remove(&uuid); - metrics.found = metrics.found.saturating_add(u64::from(state.present)); - membership.insert(uuid, state.present); - } - } - Ok(( - requested - .iter() - .map(|uuid| membership.get(uuid).copied().unwrap_or(false)) - .collect(), - metrics, - )) - } - - /// Resolve node UUIDs to their canonical surrogates without scanning - /// topology. Results retain caller order; an absent UUID returns `None`. - pub fn lookup_node_surrogates( - &mut self, - requested: &[Uuid], - ) -> Result<(Vec>, UuidProbeMetrics), GfError> { - let mut metrics = UuidProbeMetrics { - requested: requested.len() as u64, - ..Default::default() - }; - let unique = requested.iter().copied().collect::>(); - metrics.unique_requested = unique.len() as u64; - let mut unresolved = unique; - let mut resolved = std::collections::BTreeMap::new(); - for run in self.runs.iter_mut().rev() { - if unresolved.is_empty() { - break; - } - metrics.runs_considered = metrics.runs_considered.saturating_add(1); - let states = batch_identity_states( - &mut run.identities, - &run.descriptor.identities, - UuidIndexKind::Node, - &unresolved, - &mut metrics, - )?; - let mut pairs = Vec::new(); - for (uuid, state) in states { - unresolved.remove(&uuid); - let value = state.present.then_some(state.surrogate); - if let Some(surrogate) = value { - pairs.push((surrogate, uuid)); - metrics.found = metrics.found.saturating_add(1); - } - resolved.insert(uuid, value); - } - validate_surrogate_pairs( - &mut run.node_surrogates, - &run.descriptor.node_surrogates, - &pairs, - &mut metrics, - )?; - } - Ok(( - requested - .iter() - .map(|uuid| resolved.get(uuid).copied().flatten()) - .collect(), - metrics, - )) - } -} - -#[cfg(test)] -mod tests; diff --git a/crates/graphforge-storage/src/uuid_membership/probing/tests.rs b/crates/graphforge-storage/src/uuid_membership/probing/tests.rs deleted file mode 100644 index 189eedf0a..000000000 --- a/crates/graphforge-storage/src/uuid_membership/probing/tests.rs +++ /dev/null @@ -1,525 +0,0 @@ -use super::super::AuthenticatedUuidIndexSnapshot; -use super::super::ConstructionReferenceAuthentication; -use super::super::ConstructionUuidIdentity; -use super::super::IDENTITY_RECORD_BYTES; -use super::super::INDEX_DIR; -use super::super::MANIFEST; -use super::super::Manifest; -use super::super::NODE_LOOKUP_RECORD_BYTES; -use super::super::UuidIndexBuildLimits; -use super::super::UuidIndexKind; -use super::super::UuidMembershipIndex; -use super::super::append_uuid_membership_delta; -use super::super::describe_blocks; -use super::super::open_uuid_construction_snapshot; -use super::super::rebuild::rebuild_uuid_membership_indexes; -use super::super::tests::fixture; -use super::super::tests::write_node_parquet; -use super::super::topology_delta::append_uuid_membership_delta_with_tombstones; -use super::super::topology_delta::plan_uuid_membership_delta; -use super::super::uuid_membership_index_is_fresh; -use std::fs; -use std::fs::File; -use std::io::Seek; -use std::io::SeekFrom; -use std::io::Write; -use std::path::Path; -use std::path::PathBuf; -use uuid::Uuid; - -#[test] -fn construction_snapshot_streams_live_uuid_authority_in_order() { - let (dir, nodes, edges) = fixture(); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); - let mut streamed = Vec::new(); - let (snapshot, work) = open_uuid_construction_snapshot(dir.path(), 0, |identity| { - streamed.push(identity); - Ok(()) - }) - .unwrap(); - - let mut expected = nodes - .iter() - .copied() - .zip(1_u64..) - .map(|(uuid, surrogate)| ConstructionUuidIdentity { - uuid, - kind: UuidIndexKind::Node, - surrogate, - }) - .chain(edges.iter().copied().map(|uuid| ConstructionUuidIdentity { - uuid, - kind: UuidIndexKind::Edge, - surrogate: 0, - })) - .collect::>(); - expected.sort_by_key(|identity| identity.uuid); - assert_eq!(streamed, expected); - assert_eq!(work.live_nodes, nodes.len() as u64); - assert_eq!(work.live_edges, edges.len() as u64); - assert_eq!(work.max_node_surrogate, nodes.len() as u64); - assert!(work.authentication_bytes > 0); - assert!(work.authentication_blocks > 0); - snapshot.revalidate().unwrap(); -} - -#[test] -fn bounded_build_reopens_and_probes_in_caller_order() { - let (dir, nodes, _) = fixture(); - let metrics = rebuild_uuid_membership_indexes( - dir.path(), - UuidIndexBuildLimits { - scan_batch_rows: 1, - run_records: 1, - merge_fan_in: 2, - }, - ) - .unwrap(); - assert_eq!((metrics.node_count, metrics.edge_count), (3, 2)); - assert_eq!(metrics.peak_buffered_records, 1); - let mut index = UuidMembershipIndex::open(dir.path()).unwrap(); - let missing = Uuid::from_u128(99); - let (found, probe) = index - .probe(UuidIndexKind::Node, &[nodes[1], missing, nodes[1]]) - .unwrap(); - assert_eq!(found, vec![true, false, true]); - let (surrogates, lookup) = index - .lookup_node_surrogates(&[nodes[1], missing, nodes[0]]) - .unwrap(); - assert_eq!(surrogates, vec![Some(2), None, Some(1)]); - assert_eq!(lookup.found, 2); - assert_eq!(probe.per_record_seeks, 0); - assert_eq!(lookup.per_record_seeks, 0); - assert_eq!(lookup.surrogate_blocks_read, 1); - assert_eq!( - (probe.requested, probe.unique_requested, probe.found), - (3, 2, 1) - ); -} - -#[test] -fn probe_work_is_fence_selected_block_merge_not_per_record_seeks() { - let dir = tempfile::tempdir().unwrap(); - let nodes = (1..=8_192).map(Uuid::from_u128).collect::>(); - write_node_parquet(&dir.path().join("topology/nodes.parquet"), &nodes); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); - - let mut index = UuidMembershipIndex::open(dir.path()).unwrap(); - let (found, metrics) = index - .probe(UuidIndexKind::Node, &[nodes[4_095], Uuid::from_u128(9_000)]) - .unwrap(); - assert_eq!(found, [true, false]); - assert_eq!(metrics.unique_requested, 2); - assert_eq!(metrics.per_record_seeks, 0); - assert_eq!(metrics.identity_blocks_read, 1); - assert_eq!(metrics.file_seeks, metrics.identity_blocks_read); - assert_eq!(metrics.identity_bytes_read, 8_192 * IDENTITY_RECORD_BYTES); -} - -#[test] -fn batch_lookup_restores_duplicates_and_applies_newest_tombstones() { - let dir = tempfile::tempdir().unwrap(); - crate::generation::force_bump_topology_generation_for_test(dir.path()).unwrap(); - let retained = (1_u64..=40_000) - .map(|value| (Uuid::from_u128(u128::from(value)), value)) - .collect::>(); - append_uuid_membership_delta(dir.path(), 1, &retained, &[]).unwrap(); - crate::generation::force_bump_topology_generation_for_test(dir.path()).unwrap(); - append_uuid_membership_delta_with_tombstones( - dir.path(), - 2, - &[], - &[], - &[(retained[19_999].0, retained[19_999].1)], - &[], - ) - .unwrap(); - - let present = retained[39_999].0; - let deleted = retained[19_999].0; - let missing = Uuid::from_u128(50_000); - let mut index = UuidMembershipIndex::open(dir.path()).unwrap(); - let (resolved, metrics) = index - .lookup_node_surrogates(&[present, deleted, present, missing]) - .unwrap(); - assert_eq!(resolved, [Some(40_000), None, Some(40_000), None]); - assert_eq!( - (metrics.requested, metrics.unique_requested, metrics.found), - (4, 3, 1) - ); - assert_eq!(metrics.per_record_seeks, 0); - assert!(metrics.identity_blocks_read <= 3); - assert_eq!(metrics.surrogate_blocks_read, 1); - assert_eq!(metrics.file_seeks, metrics.identity_blocks_read + 1); -} - -#[test] -fn corrupt_data_fails_closed_without_replacing_manifest() { - let (dir, _, _) = fixture(); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); - let root = dir.path().join(INDEX_DIR); - let manifest: Manifest = - serde_json::from_slice(&fs::read(root.join(MANIFEST)).unwrap()).unwrap(); - let mut file = fs::OpenOptions::new() - .write(true) - .open(root.join(&manifest.runs[0].identities.name)) - .unwrap(); - file.seek(SeekFrom::Start(0)).unwrap(); - file.write_all(&[0xff]).unwrap(); - assert!( - UuidMembershipIndex::open(dir.path()) - .unwrap_err() - .to_string() - .contains("authentication failed") - ); -} - -#[test] -fn retained_snapshot_rehashes_manifest_and_authenticates_only_candidate_blocks() { - let dir = tempfile::tempdir().unwrap(); - let nodes = (1_u64..=40_000) - .map(|value| (Uuid::from_u128(u128::from(value)), value)) - .collect::>(); - crate::generation::force_bump_topology_generation_for_test(dir.path()).unwrap(); - append_uuid_membership_delta(dir.path(), 1, &nodes, &[]).unwrap(); - let root = dir.path().join(INDEX_DIR); - let mut snapshot = AuthenticatedUuidIndexSnapshot::open_at_generation(dir.path(), 1).unwrap(); - - let manifest_path = root.join(MANIFEST); - let original_manifest = fs::read(&manifest_path).unwrap(); - fs::write( - &manifest_path, - [original_manifest.as_slice(), b"\n"].concat(), - ) - .unwrap(); - assert!( - snapshot - .revalidate() - .unwrap_err() - .to_string() - .contains("manifest authentication") - ); - fs::write(&manifest_path, original_manifest).unwrap(); - - let run_path = root.join( - &snapshot - .manifest - .runs - .iter() - .find(|run| run.identities.count > 0) - .unwrap() - .identities - .name, - ); - let mut run = fs::OpenOptions::new().write(true).open(run_path).unwrap(); - run.seek(SeekFrom::Start(0)).unwrap(); - run.write_all(&[0xff]).unwrap(); - run.sync_all().unwrap(); - let scratch = tempfile::tempdir_in(dir.path()).unwrap(); - let error = plan_uuid_membership_delta( - &root, - 1, - 2, - Some(&mut snapshot), - scratch.path(), - &[(nodes[0].0, nodes[0].1)], - &[], - &[], - &[], - ) - .unwrap_err(); - assert!( - error.to_string().contains("block authentication"), - "{error}" - ); -} - -#[test] -fn compact_retained_reference_authentication_is_batched_and_linear() { - let source = tempfile::tempdir().unwrap(); - crate::generation::force_bump_topology_generation_for_test(source.path()).unwrap(); - append_uuid_membership_delta( - source.path(), - 1, - &[(Uuid::from_u128(1), 1), (Uuid::from_u128(2), 2)], - &[Uuid::from_u128(100)], - ) - .unwrap(); - crate::generation::force_bump_topology_generation_for_test(source.path()).unwrap(); - append_uuid_membership_delta( - source.path(), - 2, - &[(Uuid::from_u128(3), 3)], - &[Uuid::from_u128(101)], - ) - .unwrap(); - let (inventory, _) = crate::capture_graph_files(source.path()).unwrap(); - - let container = tempfile::tempdir().unwrap(); - crate::open_or_initialize_project(container.path()).unwrap(); - let lease = crate::begin_graph_object_publication(container.path()).unwrap(); - let paths = inventory - .files - .iter() - .map(|entry| PathBuf::from(&entry.relative_path)) - .collect::>(); - crate::append_graph_files_v2( - &lease, - source.path(), - &mut crate::GraphManifestState::empty(), - &paths, - &[], - ) - .unwrap(); - drop(lease); - - let snapshot = AuthenticatedUuidIndexSnapshot::open_from_compact_inventory( - container.path(), - &inventory, - 2, - ) - .unwrap(); - let retained = snapshot - .manifest - .runs - .iter() - .flat_map(|run| [&run.identities, &run.node_surrogates]) - .map(|record| snapshot.retained_reference(record).unwrap()) - .collect::>(); - assert!(retained.len() > 2); - let references = retained - .iter() - .map(|reference| ConstructionReferenceAuthentication { - source_root: &reference.source_root, - source_root_volume: reference.source_root_volume, - source_root_file_id: &reference.source_root_file_id, - source_path: &reference.source_path, - source_volume: reference.source_volume, - source_file_id: &reference.source_file_id, - target_path: &reference.target_path, - bytes: reference.bytes, - sha256: &reference.sha256, - xxh64: reference.xxh64, - parent_manifest_sha256: &reference.parent_manifest_sha256, - }) - .collect::>(); - let capture = graphforge_core::hash_observation::operation::Capture::start(); - let work = snapshot - .authenticate_construction_references(&references) - .unwrap(); - let observed = capture.snapshot(); - drop(capture); - assert_eq!(observed.artifact_payload_sha256_bytes, 0); - assert_eq!(observed.unclassified_sha256_bytes, 0); - assert!(observed.checksum_bytes >= work.referenced_payload_bytes); - assert_eq!( - work.global_revalidation_bytes, - snapshot.snapshot_authentication_bytes() * 2 - ); - assert_eq!( - work.referenced_payload_bytes, - retained - .iter() - .map(|reference| reference.bytes) - .sum::() - ); - - let victim_reference = retained - .iter() - .find(|reference| reference.bytes > 0) - .expect("one retained UUID payload is non-empty"); - let victim = Path::new(&victim_reference.source_root).join(&victim_reference.source_path); - let original = fs::read(&victim).unwrap(); - let mut permissions = fs::metadata(&victim).unwrap().permissions(); - permissions.set_readonly(false); - fs::set_permissions(&victim, permissions).unwrap(); - fs::write(&victim, vec![0_u8; original.len()]).unwrap(); - assert!( - snapshot - .authenticate_construction_references(&references) - .is_err() - ); - fs::write(&victim, &original).unwrap(); - let mut permissions = fs::metadata(&victim).unwrap().permissions(); - permissions.set_readonly(true); - fs::set_permissions(&victim, permissions).unwrap(); - - let error = snapshot - .authenticate_construction_references_with(&references, || { - let mut permissions = fs::metadata(&victim).unwrap().permissions(); - permissions.set_readonly(false); - fs::set_permissions(&victim, permissions).unwrap(); - fs::write(&victim, vec![0_u8; original.len()]).unwrap(); - let mut permissions = fs::metadata(&victim).unwrap().permissions(); - permissions.set_readonly(true); - fs::set_permissions(&victim, permissions).unwrap(); - }) - .unwrap_err(); - assert!( - error - .to_string() - .contains("XXH64 checksum does not match its inventory"), - "{error}" - ); -} - -#[test] -fn missing_and_stale_manifests_fail_closed() { - let (dir, _, _) = fixture(); - assert!(!uuid_membership_index_is_fresh(dir.path()).unwrap()); - assert!(UuidMembershipIndex::open(dir.path()).is_err()); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); - assert!(uuid_membership_index_is_fresh(dir.path()).unwrap()); - crate::generation::force_bump_topology_generation_for_test(dir.path()).unwrap(); - assert!(!uuid_membership_index_is_fresh(dir.path()).unwrap()); - assert!( - UuidMembershipIndex::open(dir.path()) - .unwrap_err() - .to_string() - .contains("stale index generation") - ); -} - -#[test] -fn lookup_lazily_rejects_authenticated_pair_inconsistency() { - let (dir, nodes, _) = fixture(); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); - let root = dir.path().join(INDEX_DIR); - let mut manifest: Manifest = - serde_json::from_slice(&fs::read(root.join(MANIFEST)).unwrap()).unwrap(); - let run = &mut manifest.runs[0]; - let path = root.join(&run.node_surrogates.name); - let mut bytes = fs::read(&path).unwrap(); - bytes[8..24].copy_from_slice(Uuid::from_u128(999).as_bytes()); - fs::write(&path, &bytes).unwrap(); - let mut file = File::open(&path).unwrap(); - let (sha256, xxh64, blocks, count) = - describe_blocks(&mut file, NODE_LOOKUP_RECORD_BYTES).unwrap(); - assert_eq!(count, run.node_surrogates.count); - run.node_surrogates.sha256 = sha256; - run.node_surrogates.xxh64 = xxh64; - run.node_surrogates.blocks = blocks; - fs::write(root.join(MANIFEST), serde_json::to_vec(&manifest).unwrap()).unwrap(); - - // Ordinary open remains a bounded linear stream and does not perform - // one random identity probe per surrogate record. - let mut index = UuidMembershipIndex::open(dir.path()).unwrap(); - assert!( - index - .lookup_node_surrogates(&[nodes[0]]) - .unwrap_err() - .to_string() - .contains("pair is inconsistent") - ); -} - -#[test] -fn checksum_uuid_manifest_refuses_legacy_missing_and_malformed_metadata() { - let (dir, _, _) = fixture(); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); - let path = dir.path().join(INDEX_DIR).join(MANIFEST); - let original: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); - for mode in 0..8 { - let mut changed = original.clone(); - match mode { - 0 => { - changed["format_version"] = serde_json::json!(6); - for run in changed["runs"].as_array_mut().unwrap() { - for file in ["identities", "node_surrogates"] { - for block in run[file]["blocks"].as_array_mut().unwrap() { - block.as_object_mut().unwrap().remove("xxh64"); - } - } - } - } - 1 => changed["format_version"] = serde_json::json!(8), - 4 => { - changed.as_object_mut().unwrap().remove("format_version"); - } - 5 => { - changed["runs"][0]["identities"] - .as_object_mut() - .unwrap() - .remove("xxh64"); - } - 6 => changed["runs"][0]["identities"]["xxh64"] = serde_json::json!("bad"), - 7 => { - changed["runs"][0]["identities"]["blocks"][0]["sha256"] = - serde_json::json!("a".repeat(64)) - } - 2 => { - changed["runs"][0]["identities"]["blocks"][0] - .as_object_mut() - .unwrap() - .remove("xxh64"); - } - _ => { - changed["runs"][0]["identities"]["blocks"][0]["xxh64"] = - serde_json::json!("not-a-checksum") - } - } - fs::write(&path, serde_json::to_vec(&changed).unwrap()).unwrap(); - let generation = crate::read_topology_generation(dir.path()).unwrap(); - let errors = [ - UuidMembershipIndex::open(dir.path()) - .unwrap_err() - .to_string(), - AuthenticatedUuidIndexSnapshot::open_at_generation(dir.path(), generation) - .err() - .unwrap() - .to_string(), - uuid_membership_index_is_fresh(dir.path()) - .unwrap_err() - .to_string(), - super::super::rebuild::manifest_generation(dir.path()) - .unwrap_err() - .to_string(), - ]; - for error in errors { - assert_eq!( - error.contains("unsupported UUID membership format version"), - mode < 2 || mode == 4, - "mode={mode}: {error}" - ); - if mode < 2 || mode == 4 { - assert!(error.contains("recreate the index"), "{error}"); - } - } - let (inventory, _) = crate::capture_graph_files(dir.path()).unwrap(); - let container = tempfile::tempdir().unwrap(); - crate::open_or_initialize_project(container.path()).unwrap(); - let lease = crate::begin_graph_object_publication(container.path()).unwrap(); - let paths = inventory - .files - .iter() - .map(|entry| PathBuf::from(&entry.relative_path)) - .collect::>(); - crate::append_graph_files_v2( - &lease, - dir.path(), - &mut crate::GraphManifestState::empty(), - &paths, - &[], - ) - .unwrap(); - let error = AuthenticatedUuidIndexSnapshot::open_from_compact_inventory( - container.path(), - &inventory, - generation, - ) - .err() - .unwrap() - .to_string(); - assert_eq!( - error.contains("unsupported UUID membership format version"), - mode < 2 || mode == 4, - "compact mode={mode}: {error}" - ); - if mode < 2 || mode == 4 { - assert!(error.contains("recreate the index"), "{error}"); - } - } - fs::write(&path, serde_json::to_vec(&original).unwrap()).unwrap(); - UuidMembershipIndex::open(dir.path()).unwrap(); - assert!(uuid_membership_index_is_fresh(dir.path()).unwrap()); -} diff --git a/crates/graphforge-storage/src/uuid_membership/rebuild.rs b/crates/graphforge-storage/src/uuid_membership/rebuild.rs index 0fe91dce7..ec3e87c63 100644 --- a/crates/graphforge-storage/src/uuid_membership/rebuild.rs +++ b/crates/graphforge-storage/src/uuid_membership/rebuild.rs @@ -3,17 +3,7 @@ //! makes them available to readers; the final staged artifact owns durability. use super::BULK_IO_BYTES; -use super::FORMAT_VERSION; -#[cfg(test)] -use super::FileRecord; -use super::IDENTITY_RECORD_BYTES; -use super::IDENTITY_RECORD_WIDTH; use super::INDEX_DIR; -use super::MANIFEST; -use super::Manifest; -use super::NODE_LOOKUP_RECORD_BYTES; -use super::RunRecord; -use super::TOPOLOGY_RECEIPT; use super::TopologyIndexReceipt; use super::UuidIndexBuildLimits; use super::UuidIndexBuildMetrics; @@ -22,11 +12,6 @@ use super::V4_ORDINAL_RECEIPT; use super::V4OrdinalBuildMetrics; use super::V4OrdinalRebuildDisposition; use super::V4OrdinalRebuildEvidence; -use super::decode_manifest; -#[cfg(test)] -use super::describe_blocks; -use super::describe_staged_data; -use super::identity_codec; use super::maintenance::selected_generation_for_graph_root; use super::ordinal_artifacts::V4AuthorityTransactionProof; use super::ordinal_artifacts::V4ConstructionArtifactBundle; @@ -34,21 +19,13 @@ use super::ordinal_artifacts::V4OrdinalConstructionWriter; use super::ordinal_artifacts::commit_v4_publications; use super::storage_err; use super::topology_delta::hex_sha256; -use super::uuid_membership_index_is_fresh; -use super::validate_run_descriptors; -#[cfg(test)] -use crate::concurrency_attribution::ObservedSha256 as Sha256; use arrow::array::Array; use arrow::array::FixedSizeBinaryArray; use arrow::array::UInt64Array; use graphforge_core::GfError; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; -#[cfg(test)] -use sha2::Digest; use std::cmp::Reverse; use std::collections::BinaryHeap; -#[cfg(test)] -use std::fmt::Write as _; use std::fs; use std::fs::File; use std::io::BufReader; @@ -153,24 +130,6 @@ struct StagedV4OrdinalRebuild { scratch: V4RebuildScratchAccounting, } -/// Explicit bounded rebuild/migration path. Immutable data files are completed -/// and synced first; `manifest.json` is atomically replaced last. -pub fn rebuild_uuid_membership_indexes( - project_dir: &Path, - limits: UuidIndexBuildLimits, -) -> Result { - migrate_uuid_membership_indexes(project_dir, limits, true, None) -} - -/// Rebuild a private session index from explicit topology membership. -pub fn rebuild_uuid_membership_indexes_with_topology( - project_dir: &Path, - limits: UuidIndexBuildLimits, - topology: std::sync::Arc, -) -> Result { - migrate_uuid_membership_indexes(project_dir, limits, true, Some(topology)) -} - /// Rebuild v4 ordinal identity from canonical topology, never v3 reverse state. pub fn rebuild_v4_ordinal_identity( project_dir: &Path, @@ -416,546 +375,211 @@ fn v4_rebuild_evidence( }) } -/// Ensure the current topology generation has a v3 UUID index before a -/// topology mutation enters its sealed rewrite callback. -pub(crate) fn ensure_uuid_membership_migrated(project_dir: &Path) -> Result<(), GfError> { - migrate_uuid_membership_indexes(project_dir, UuidIndexBuildLimits::default(), false, None) - .map(|_| ()) -} - -pub(crate) fn ensure_uuid_membership_migrated_with_topology( - project_dir: &Path, - topology: Option>, -) -> Result<(), GfError> { - migrate_uuid_membership_indexes( - project_dir, - UuidIndexBuildLimits::default(), - false, - topology, - ) - .map(|_| ()) -} - -fn migrate_uuid_membership_indexes( - project_dir: &Path, - limits: UuidIndexBuildLimits, - force: bool, - topology: Option>, -) -> Result { - if !force && uuid_membership_index_is_fresh(project_dir)? { - return Ok(UuidIndexBuildMetrics::default()); - } - let metrics = std::rc::Rc::new(std::cell::RefCell::new(None)); - let callback_metrics = std::rc::Rc::clone(&metrics); - let root = project_dir.to_path_buf(); - let participant: crate::durable_rewrite::RewriteParticipantPreparer<'_> = - Box::new(move |context, batch| { - if !force && manifest_generation(context.project_root)? == Some(context.prior.topology) - { - return Ok(None); - } - let built = stage_uuid_membership_rebuild_locked( - context.project_root, - context.prior.topology, - limits, - batch, - )?; - *callback_metrics.borrow_mut() = Some(built); - let manifest_destination = context.project_root.join(INDEX_DIR).join(MANIFEST); - let manifest_temp = batch.staged_temp(&manifest_destination).ok_or_else(|| { - storage_err("UUID migration did not stage its canonical manifest") - })?; - let manifest_bytes = fs::read(manifest_temp).map_err(storage_err)?; - let receipt = TopologyIndexReceipt { - nonce: Uuid::new_v4().simple().to_string(), - expected_generation: context.prior.topology, - topology_delta_sha256: hex_sha256(b"uuid-membership-migration"), - manifest_sha256: hex_sha256(&manifest_bytes), - }; - let receipt_bytes = serde_json::to_vec(&receipt).map_err(storage_err)?; - batch.stage_bytes( - &context.project_root.join(INDEX_DIR).join(TOPOLOGY_RECEIPT), - &receipt_bytes, - )?; - Ok(Some(crate::AuxiliaryReceipt { - kind: "uuid-membership/v7".to_owned(), - schema_version: FORMAT_VERSION, - path: format!("{INDEX_DIR}/{TOPOLOGY_RECEIPT}"), - digest: hex_sha256(&receipt_bytes), - bytes: u64::try_from(receipt_bytes.len()) - .map_err(|_| storage_err("receipt length overflow"))?, - })) - }); - let mut batch = crate::staging::RewriteBatch::new(); - if let Some(topology) = topology { - batch.bind_topology_authority(topology)?; +pub(super) fn read_exact_record( + reader: &mut impl Read, +) -> Result, GfError> { + let mut record = [0_u8; N]; + let mut filled = 0; + while filled < N { + match reader.read(&mut record[filled..]).map_err(storage_err)? { + 0 if filled == 0 => return Ok(None), + 0 => return Err(storage_err("truncated fixed-width index record")), + read => filled += read, + } } - crate::generation::commit_topology_aware_with_participant(batch, &root, participant)?; - let result = metrics.borrow_mut().take().unwrap_or_default(); - Ok(result) -} - -#[allow(clippy::too_many_lines)] // Sequential bounded rebuild pipeline with one authority output. -fn stage_uuid_membership_rebuild_locked( - project_dir: &Path, - generation: u64, - limits: UuidIndexBuildLimits, - batch: &mut crate::staging::RewriteBatch, -) -> Result { - let limits = limits.validate()?; - let root = project_dir.join(INDEX_DIR); - fs::create_dir_all(&root).map_err(storage_err)?; - let staging = project_dir - .parent() - .ok_or_else(|| storage_err("project directory has no staging parent"))?; - let scratch = tempfile::Builder::new() - .prefix("uuid-membership-build-") - .tempdir_in(staging) - .map_err(storage_err)?; - let mut metrics = UuidIndexBuildMetrics::default(); - let files = match batch.topology_authority() { - Some(topology) => crate::enumerate_topology_files(topology, None)?, - None => crate::TopologyFiles::discover_legacy(project_dir)?, - }; - let node_paths: Vec<_> = files.nodes.iter().map(|(path, _)| path.clone()).collect(); - let node_runs = scan_to_runs( - &node_paths, - "node_uuid", - scratch.path(), - "node", - limits, - &mut metrics, - )?; - let node_surrogate_runs = scan_entity_surrogate_runs( - &node_paths, - "node_uuid", - "node_id", - "node", - scratch.path(), - limits, - &mut metrics, - )?; - let node_surrogate_validation_runs = - scan_node_surrogate_validation_runs(&node_paths, scratch.path(), limits, &mut metrics)?; - let mut edge_paths: Vec<_> = files - .edges - .iter() - .map(|(_, path, _)| path.clone()) - .collect(); - edge_paths.sort(); - let edge_runs = scan_to_runs( - &edge_paths, - "edge_uuid", - scratch.path(), - "edge", - limits, - &mut metrics, - )?; - let node_tmp = merge_all( - node_runs, - scratch.path(), - "nodes", - limits.merge_fan_in, - &mut metrics, - )?; - let node_surrogates_tmp = merge_node_surrogate_runs( - node_surrogate_runs, - scratch.path(), - limits.merge_fan_in, - &mut metrics, - )?; - let validated_surrogates = merge_node_surrogate_validation_runs( - node_surrogate_validation_runs, - scratch.path(), - limits.merge_fan_in, - &mut metrics, - )?; - fs::remove_file(validated_surrogates).map_err(storage_err)?; - let edge_tmp = merge_all( - edge_runs, - scratch.path(), - "edges", - limits.merge_fan_in, - &mut metrics, - )?; - reject_cross_kind_identities(&node_tmp, &edge_tmp)?; - let identity_tmp = scratch.path().join("identities-v5.run"); - build_identity_run(&node_surrogates_tmp, &edge_tmp, &identity_tmp)?; - let surrogate_tmp = - build_surrogate_run(&node_surrogates_tmp, scratch.path(), limits, &mut metrics)?; - let identities = describe_staged_data( - &identity_tmp, - "identities-v5", - generation, - IDENTITY_RECORD_BYTES, - )?; - let node_surrogates = describe_staged_data( - &surrogate_tmp, - "node-surrogates-v5", - generation, - NODE_LOOKUP_RECORD_BYTES, - )?; - metrics.node_count = node_surrogates.count; - metrics.edge_count = identities.count.saturating_sub(metrics.node_count); - let manifest = Manifest { - format_version: FORMAT_VERSION, - base_generation: generation, - current_generation: generation, - live_node_count: metrics.node_count, - live_edge_count: metrics.edge_count, - runs: vec![RunRecord { - base: true, - level: 0, - first_generation: 0, - last_generation: generation, - identities, - node_surrogates, - node_count: metrics.node_count, - edge_count: metrics.edge_count, - deleted_node_count: 0, - deleted_edge_count: 0, - }], - }; - batch.stage_file(&root.join(&manifest.runs[0].identities.name), &identity_tmp)?; - batch.stage_file( - &root.join(&manifest.runs[0].node_surrogates.name), - &surrogate_tmp, - )?; - batch.stage_bytes( - &root.join(MANIFEST), - &serde_json::to_vec(&manifest).map_err(storage_err)?, - )?; - Ok(metrics) + Ok(Some(record)) } -pub(super) fn manifest_generation(project_dir: &Path) -> Result, GfError> { - let bytes = match fs::read(project_dir.join(INDEX_DIR).join(MANIFEST)) { - Ok(bytes) => bytes, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => return Err(storage_err(error)), - }; - let manifest = decode_manifest(&bytes)?; - validate_run_descriptors(&manifest)?; - Ok(Some(manifest.current_generation)) +fn canonical_node_topology_inventory_path(relative: &str) -> bool { + relative == "topology/nodes.parquet" + || relative + .strip_prefix("topology/nodes/") + .is_some_and(|name| !name.contains('/') && name.ends_with(".parquet")) } -fn scan_to_runs( - paths: &[PathBuf], - column: &str, - scratch: &Path, +#[allow(clippy::too_many_arguments)] +fn scan_pinned_entity_surrogate_runs( + inputs: &[crate::project_generation::PinnedGraphFile], + uuid_column: &str, + surrogate_column: &str, prefix: &str, + scratch: &Path, limits: UuidIndexBuildLimits, metrics: &mut UuidIndexBuildMetrics, ) -> Result, GfError> { - let mut buffer = Vec::<[u8; 16]>::with_capacity(limits.run_records); + let mut buffer = Vec::<([u8; 16], u64)>::with_capacity(limits.run_records); let mut runs = Vec::new(); - for path in paths { - if !path.exists() { - continue; + for input in inputs { + let identity = graphforge_filesystem::file_identity(&input.file).map_err(storage_err)?; + if identity != input.identity + || input.file.metadata().map_err(storage_err)?.len() != input.entry.byte_length + { + return Err(storage_err( + "authenticated topology payload identity changed before scan", + )); } - let file = crate::graph_admission::open_admitted(path)?; - let reader = ParquetRecordBatchReaderBuilder::try_new(file) - .map_err(storage_err)? - .with_batch_size(limits.scan_batch_rows) - .build() - .map_err(storage_err)?; + let reader = + ParquetRecordBatchReaderBuilder::try_new(input.file.try_clone().map_err(storage_err)?) + .map_err(storage_err)? + .with_batch_size(limits.scan_batch_rows) + .build() + .map_err(storage_err)?; for batch in reader { let batch = batch.map_err(storage_err)?; - let array = batch - .column_by_name(column) - .ok_or_else(|| storage_err(format!("{} lacks {column}", path.display())))? - .as_any() - .downcast_ref::() - .ok_or_else(|| storage_err(format!("{column} is not FixedSizeBinary")))?; - for row in 0..array.len() { - if array.is_null(row) || array.value(row).len() != 16 { - return Err(storage_err(format!("invalid {column} at row {row}"))); + let uuids = batch + .column_by_name(uuid_column) + .and_then(|column| column.as_any().downcast_ref::()) + .ok_or_else(|| { + storage_err(format!("{} lacks {uuid_column}", input.entry.relative_path)) + })?; + let surrogates = batch + .column_by_name(surrogate_column) + .and_then(|column| column.as_any().downcast_ref::()) + .ok_or_else(|| { + storage_err(format!( + "{} lacks {surrogate_column}", + input.entry.relative_path + )) + })?; + if uuids.len() != surrogates.len() { + return Err(storage_err( + "identity UUID and surrogate columns differ in length", + )); + } + for row in 0..uuids.len() { + if uuids.is_null(row) || uuids.value(row).len() != 16 || surrogates.is_null(row) { + return Err(storage_err(format!("invalid entity identity at row {row}"))); } - buffer.push(array.value(row).try_into().expect("length checked")); + buffer.push(( + uuids.value(row).try_into().expect("length checked"), + surrogates.value(row), + )); metrics.peak_buffered_records = metrics.peak_buffered_records.max(buffer.len()); if buffer.len() == limits.run_records { - flush_run(&mut buffer, scratch, prefix, &mut runs, metrics)?; + flush_entity_surrogate_run(&mut buffer, scratch, prefix, &mut runs, metrics)?; } } } + if graphforge_filesystem::file_identity(&input.file).map_err(storage_err)? != input.identity + { + return Err(storage_err( + "authenticated topology payload identity changed during scan", + )); + } } if !buffer.is_empty() { - flush_run(&mut buffer, scratch, prefix, &mut runs, metrics)?; + flush_entity_surrogate_run(&mut buffer, scratch, prefix, &mut runs, metrics)?; } if runs.is_empty() { - let path = scratch.join(format!("{prefix}-empty.run")); + let path = scratch.join(format!("{prefix}-surrogates-empty.run")); File::create(&path).map_err(storage_err)?; runs.push(path); } Ok(runs) } -pub(super) fn build_identity_run(nodes: &Path, edges: &Path, output: &Path) -> Result<(), GfError> { - let mut node_reader = BufReader::new(File::open(nodes).map_err(storage_err)?); - let mut edge_reader = BufReader::new(File::open(edges).map_err(storage_err)?); - let mut node = read_node_surrogate_record(&mut node_reader)?; - let mut edge = read_record(&mut edge_reader)?; - let mut out = File::create(output).map_err(storage_err)?; - let mut block = Vec::with_capacity(BULK_IO_BYTES); - while node.is_some() || edge.is_some() { - let take_node = match (&node, &edge) { - (Some((node_uuid, _)), Some(edge_uuid)) => { - if node_uuid == edge_uuid { - return Err(storage_err("UUID occurs in both identity domains")); - } - node_uuid < edge_uuid - } - (Some(_), None) => true, - _ => false, - }; - let (uuid, surrogate, kind) = if take_node { - let (uuid, surrogate) = node.take().expect("node present"); - node = read_node_surrogate_record(&mut node_reader)?; - (uuid, surrogate, 0_u8) - } else { - let uuid = edge.take().expect("edge present"); - edge = read_record(&mut edge_reader)?; - (uuid, 0, 1_u8) - }; - let mut record = [0_u8; IDENTITY_RECORD_WIDTH]; - record[..16].copy_from_slice(&uuid); - record[16] = kind; - record[17..].copy_from_slice(&surrogate.to_be_bytes()); - if block.len() + IDENTITY_RECORD_WIDTH > BULK_IO_BYTES { - out.write_all(&block).map_err(storage_err)?; - block.clear(); - } - block.extend_from_slice(identity_codec::encoded(&record)?); +pub(super) fn flush_entity_surrogate_run( + buffer: &mut Vec<([u8; 16], u64)>, + scratch: &Path, + prefix: &str, + runs: &mut Vec, + metrics: &mut UuidIndexBuildMetrics, +) -> Result<(), GfError> { + buffer.sort_unstable_by_key(|record| record.0); + if buffer.windows(2).any(|pair| pair[0].0 == pair[1].0) { + return Err(storage_err("duplicate UUID in canonical topology")); + } + let path = scratch.join(format!("{prefix}-surrogates-{:08}.run", runs.len())); + let mut out = File::create(&path).map_err(storage_err)?; + let mut block = Vec::with_capacity(buffer.len().min(BULK_IO_BYTES / 24) * 24); + for (uuid, surrogate) in buffer.iter() { + block.extend_from_slice(uuid); + block.extend_from_slice(&surrogate.to_le_bytes()); } if !block.is_empty() { out.write_all(&block).map_err(storage_err)?; } - out.flush().map_err(storage_err) + out.flush().map_err(storage_err)?; + buffer.clear(); + runs.push(path); + metrics.temporary_runs += 1; + Ok(()) } -pub(super) fn build_surrogate_run( - nodes: &Path, +pub(super) fn merge_node_surrogate_runs( + mut runs: Vec, scratch: &Path, - limits: UuidIndexBuildLimits, + fan_in: usize, metrics: &mut UuidIndexBuildMetrics, ) -> Result { - let mut reader = BufReader::new(File::open(nodes).map_err(storage_err)?); - let mut buffer = Vec::with_capacity(limits.run_records); - let mut runs = Vec::new(); - while let Some((uuid, surrogate)) = read_node_surrogate_record(&mut reader)? { - buffer.push((surrogate, uuid)); - metrics.peak_buffered_records = metrics.peak_buffered_records.max(buffer.len()); - if buffer.len() == limits.run_records { - flush_surrogate_run(&mut buffer, scratch, &mut runs, metrics)?; - } - } - if !buffer.is_empty() { - flush_surrogate_run(&mut buffer, scratch, &mut runs, metrics)?; - } - if runs.is_empty() { - let path = scratch.join("surrogates-empty.run"); - File::create(&path).map_err(storage_err)?; - runs.push(path); - } let mut round = 0; while runs.len() > 1 { let mut next = Vec::new(); - for (group, inputs) in runs.chunks(limits.merge_fan_in).enumerate() { - let output = scratch.join(format!("surrogates-merge-{round}-{group}.run")); - merge_surrogate_runs(inputs, &output)?; - next.push(output); + for (group, chunk) in runs.chunks(fan_in).enumerate() { + let path = scratch.join(format!("node-surrogates-merge-{round}-{group}.run")); + merge_node_surrogate_group(chunk, &path)?; + next.push(path); + metrics.temporary_runs += 1; } - for run in runs { - let _ = fs::remove_file(run); + for path in runs { + let _ = fs::remove_file(path); } runs = next; round += 1; } - Ok(runs.pop().expect("surrogate run exists")) + Ok(runs.pop().expect("at least one node-surrogate run")) } -fn flush_surrogate_run( - buffer: &mut Vec<(u64, [u8; 16])>, - scratch: &Path, - runs: &mut Vec, - metrics: &mut UuidIndexBuildMetrics, -) -> Result<(), GfError> { - buffer.sort_unstable(); - if buffer.windows(2).any(|pair| pair[0].0 == pair[1].0) { - return Err(storage_err("duplicate node surrogate")); - } - let path = scratch.join(format!("surrogates-{:08}.run", runs.len())); - let mut bytes = Vec::with_capacity(buffer.len() * 24); - for (surrogate, uuid) in buffer.iter() { - bytes.extend_from_slice(&surrogate.to_be_bytes()); - bytes.extend_from_slice(uuid); - } - let mut file = File::create(&path).map_err(storage_err)?; - file.write_all(&bytes).map_err(storage_err)?; - file.flush().map_err(storage_err)?; - buffer.clear(); - runs.push(path); - metrics.temporary_runs += 1; - Ok(()) -} - -pub(super) fn merge_surrogate_runs(inputs: &[PathBuf], output: &Path) -> Result<(), GfError> { - let mut readers = inputs - .iter() - .map(|path| File::open(path).map(BufReader::new).map_err(storage_err)) - .collect::, _>>()?; - let mut heap = BinaryHeap::>::new(); - for (index, reader) in readers.iter_mut().enumerate() { - if let Some(record) = read_surrogate_record(reader)? { - heap.push(Reverse((record, index))); - } +fn merge_node_surrogate_group(inputs: &[PathBuf], output: &Path) -> Result<(), GfError> { + let mut readers = inputs + .iter() + .map(|path| File::open(path).map(BufReader::new).map_err(storage_err)) + .collect::, _>>()?; + let mut heap = BinaryHeap::>::new(); + for (index, reader) in readers.iter_mut().enumerate() { + if let Some(record) = read_node_surrogate_record(reader)? { + heap.push(Reverse((record, index))); + } } let mut out = File::create(output).map_err(storage_err)?; let mut block = Vec::with_capacity(BULK_IO_BYTES); let mut previous = None; - while let Some(Reverse(((surrogate, uuid), index))) = heap.pop() { - if previous.is_some_and(|(prior, _)| prior == surrogate) { - if previous.is_some_and(|(_, prior_uuid)| prior_uuid == uuid) { - if let Some(record) = read_surrogate_record(&mut readers[index])? { - heap.push(Reverse((record, index))); - } - continue; - } - return Err(storage_err("duplicate node surrogate across runs")); + while let Some(Reverse(((uuid, surrogate), index))) = heap.pop() { + if previous == Some(uuid) { + return Err(storage_err( + "duplicate node UUID across external index runs", + )); } if block.len() + 24 > BULK_IO_BYTES { out.write_all(&block).map_err(storage_err)?; block.clear(); } - block.extend_from_slice(&surrogate.to_be_bytes()); block.extend_from_slice(&uuid); - previous = Some((surrogate, uuid)); - if let Some(record) = read_surrogate_record(&mut readers[index])? { + block.extend_from_slice(&surrogate.to_le_bytes()); + previous = Some(uuid); + if let Some(record) = read_node_surrogate_record(&mut readers[index])? { heap.push(Reverse((record, index))); } } if !block.is_empty() { out.write_all(&block).map_err(storage_err)?; } - out.flush().map_err(storage_err) + out.flush().map_err(storage_err)?; + Ok(()) } -pub(super) fn read_surrogate_record( +pub(super) fn read_node_surrogate_record( reader: &mut BufReader, -) -> Result, GfError> { +) -> Result, GfError> { let Some(record) = read_exact_record::<24>(reader)? else { return Ok(None); }; Ok(Some(( - u64::from_be_bytes(record[..8].try_into().expect("fixed")), - record[8..].try_into().expect("fixed"), + record[..16].try_into().expect("fixed"), + u64::from_le_bytes(record[16..].try_into().expect("fixed")), ))) } -pub(super) fn read_exact_record( - reader: &mut impl Read, -) -> Result, GfError> { - let mut record = [0_u8; N]; - let mut filled = 0; - while filled < N { - match reader.read(&mut record[filled..]).map_err(storage_err)? { - 0 if filled == 0 => return Ok(None), - 0 => return Err(storage_err("truncated fixed-width index record")), - read => filled += read, - } - } - Ok(Some(record)) -} - -fn flush_run( - buffer: &mut Vec<[u8; 16]>, - scratch: &Path, - prefix: &str, - runs: &mut Vec, - metrics: &mut UuidIndexBuildMetrics, -) -> Result<(), GfError> { - buffer.sort_unstable(); - if buffer.windows(2).any(|pair| pair[0] == pair[1]) { - return Err(storage_err(format!( - "duplicate {prefix} UUID in canonical topology" - ))); - } - let path = scratch.join(format!("{prefix}-{:08}.run", runs.len())); - let mut out = File::create(&path).map_err(storage_err)?; - let mut block = Vec::with_capacity(buffer.len().min(BULK_IO_BYTES / 16) * 16); - for value in buffer.iter() { - block.extend_from_slice(value); - } - if !block.is_empty() { - out.write_all(&block).map_err(storage_err)?; - } - out.flush().map_err(storage_err)?; - buffer.clear(); - runs.push(path); - metrics.temporary_runs += 1; - Ok(()) -} - -fn merge_all( - mut runs: Vec, - scratch: &Path, - prefix: &str, - fan_in: usize, - metrics: &mut UuidIndexBuildMetrics, -) -> Result { - let mut round = 0; - while runs.len() > 1 { - let mut next = Vec::new(); - for (group, chunk) in runs.chunks(fan_in).enumerate() { - let path = scratch.join(format!("{prefix}-merge-{round}-{group}.run")); - merge_runs(chunk, &path)?; - next.push(path); - metrics.temporary_runs += 1; - } - for path in runs { - let _ = fs::remove_file(path); - } - runs = next; - round += 1; - } - Ok(runs.pop().expect("at least one run")) -} - -fn merge_runs(inputs: &[PathBuf], output: &Path) -> Result<(), GfError> { - let mut readers = inputs - .iter() - .map(|p| File::open(p).map(BufReader::new).map_err(storage_err)) - .collect::, _>>()?; - let mut heap = BinaryHeap::>::new(); - for (idx, reader) in readers.iter_mut().enumerate() { - if let Some(value) = read_record(reader)? { - heap.push(Reverse((value, idx))); - } - } - let mut out = File::create(output).map_err(storage_err)?; - let mut block = Vec::with_capacity(BULK_IO_BYTES); - let mut previous = None; - while let Some(Reverse((value, idx))) = heap.pop() { - if previous == Some(value) { - return Err(storage_err("duplicate UUID across external index runs")); - } - if block.len() + 16 > BULK_IO_BYTES { - out.write_all(&block).map_err(storage_err)?; - block.clear(); - } - block.extend_from_slice(&value); - previous = Some(value); - if let Some(next) = read_record(&mut readers[idx])? { - heap.push(Reverse((next, idx))); - } - } - if !block.is_empty() { - out.write_all(&block).map_err(storage_err)?; - } - out.flush().map_err(storage_err)?; - Ok(()) -} - fn scan_entity_surrogate_runs( paths: &[PathBuf], uuid_column: &str, @@ -1017,436 +641,123 @@ fn scan_entity_surrogate_runs( Ok(runs) } -fn canonical_node_topology_inventory_path(relative: &str) -> bool { - relative == "topology/nodes.parquet" - || relative - .strip_prefix("topology/nodes/") - .is_some_and(|name| !name.contains('/') && name.ends_with(".parquet")) -} - -#[allow(clippy::too_many_arguments)] -fn scan_pinned_entity_surrogate_runs( - inputs: &[crate::project_generation::PinnedGraphFile], - uuid_column: &str, - surrogate_column: &str, - prefix: &str, - scratch: &Path, - limits: UuidIndexBuildLimits, - metrics: &mut UuidIndexBuildMetrics, -) -> Result, GfError> { - let mut buffer = Vec::<([u8; 16], u64)>::with_capacity(limits.run_records); - let mut runs = Vec::new(); - for input in inputs { - let identity = graphforge_filesystem::file_identity(&input.file).map_err(storage_err)?; - if identity != input.identity - || input.file.metadata().map_err(storage_err)?.len() != input.entry.byte_length - { - return Err(storage_err( - "authenticated topology payload identity changed before scan", - )); - } - let reader = - ParquetRecordBatchReaderBuilder::try_new(input.file.try_clone().map_err(storage_err)?) - .map_err(storage_err)? - .with_batch_size(limits.scan_batch_rows) - .build() - .map_err(storage_err)?; - for batch in reader { - let batch = batch.map_err(storage_err)?; - let uuids = batch - .column_by_name(uuid_column) - .and_then(|column| column.as_any().downcast_ref::()) - .ok_or_else(|| { - storage_err(format!("{} lacks {uuid_column}", input.entry.relative_path)) - })?; - let surrogates = batch - .column_by_name(surrogate_column) - .and_then(|column| column.as_any().downcast_ref::()) - .ok_or_else(|| { - storage_err(format!( - "{} lacks {surrogate_column}", - input.entry.relative_path - )) - })?; - if uuids.len() != surrogates.len() { - return Err(storage_err( - "identity UUID and surrogate columns differ in length", - )); - } - for row in 0..uuids.len() { - if uuids.is_null(row) || uuids.value(row).len() != 16 || surrogates.is_null(row) { - return Err(storage_err(format!("invalid entity identity at row {row}"))); - } - buffer.push(( - uuids.value(row).try_into().expect("length checked"), - surrogates.value(row), - )); - metrics.peak_buffered_records = metrics.peak_buffered_records.max(buffer.len()); - if buffer.len() == limits.run_records { - flush_entity_surrogate_run(&mut buffer, scratch, prefix, &mut runs, metrics)?; - } - } - } - if graphforge_filesystem::file_identity(&input.file).map_err(storage_err)? != input.identity - { - return Err(storage_err( - "authenticated topology payload identity changed during scan", - )); - } - } - if !buffer.is_empty() { - flush_entity_surrogate_run(&mut buffer, scratch, prefix, &mut runs, metrics)?; - } - if runs.is_empty() { - let path = scratch.join(format!("{prefix}-surrogates-empty.run")); - File::create(&path).map_err(storage_err)?; - runs.push(path); - } - Ok(runs) -} - -fn scan_node_surrogate_validation_runs( - paths: &[PathBuf], +pub(super) fn build_surrogate_run( + nodes: &Path, scratch: &Path, limits: UuidIndexBuildLimits, metrics: &mut UuidIndexBuildMetrics, -) -> Result, GfError> { - let mut buffer = Vec::::with_capacity(limits.run_records); +) -> Result { + let mut reader = BufReader::new(File::open(nodes).map_err(storage_err)?); + let mut buffer = Vec::with_capacity(limits.run_records); let mut runs = Vec::new(); - for path in paths { - let reader = - ParquetRecordBatchReaderBuilder::try_new(crate::graph_admission::open_admitted(path)?) - .map_err(storage_err)? - .with_batch_size(limits.scan_batch_rows) - .build() - .map_err(storage_err)?; - for batch in reader { - let batch = batch.map_err(storage_err)?; - let surrogates = batch - .column_by_name("node_id") - .and_then(|column| column.as_any().downcast_ref::()) - .ok_or_else(|| storage_err(format!("{} lacks node_id", path.display())))?; - for row in 0..surrogates.len() { - if surrogates.is_null(row) || surrogates.value(row) == 0 { - return Err(storage_err(format!("invalid node surrogate at row {row}"))); - } - buffer.push(surrogates.value(row)); - metrics.peak_buffered_records = metrics.peak_buffered_records.max(buffer.len()); - if buffer.len() == limits.run_records { - flush_node_surrogate_validation_run(&mut buffer, scratch, &mut runs, metrics)?; - } - } + while let Some((uuid, surrogate)) = read_node_surrogate_record(&mut reader)? { + buffer.push((surrogate, uuid)); + metrics.peak_buffered_records = metrics.peak_buffered_records.max(buffer.len()); + if buffer.len() == limits.run_records { + flush_surrogate_run(&mut buffer, scratch, &mut runs, metrics)?; } } if !buffer.is_empty() { - flush_node_surrogate_validation_run(&mut buffer, scratch, &mut runs, metrics)?; + flush_surrogate_run(&mut buffer, scratch, &mut runs, metrics)?; } if runs.is_empty() { - let path = scratch.join("node-surrogate-validation-empty.run"); + let path = scratch.join("surrogates-empty.run"); File::create(&path).map_err(storage_err)?; runs.push(path); } - Ok(runs) -} - -fn flush_node_surrogate_validation_run( - buffer: &mut Vec, - scratch: &Path, - runs: &mut Vec, - metrics: &mut UuidIndexBuildMetrics, -) -> Result<(), GfError> { - buffer.sort_unstable(); - if buffer.windows(2).any(|pair| pair[0] == pair[1]) { - return Err(storage_err( - "duplicate node surrogate in canonical topology", - )); - } - let path = scratch.join(format!("node-surrogate-validation-{:08}.run", runs.len())); - let mut bytes = Vec::with_capacity(buffer.len() * 8); - for surrogate in buffer.iter() { - bytes.extend_from_slice(&surrogate.to_le_bytes()); - } - let mut file = File::create(&path).map_err(storage_err)?; - if !bytes.is_empty() { - file.write_all(&bytes).map_err(storage_err)?; - } - file.flush().map_err(storage_err)?; - buffer.clear(); - runs.push(path); - metrics.temporary_runs += 1; - Ok(()) -} - -fn merge_node_surrogate_validation_runs( - mut runs: Vec, - scratch: &Path, - fan_in: usize, - metrics: &mut UuidIndexBuildMetrics, -) -> Result { let mut round = 0; while runs.len() > 1 { let mut next = Vec::new(); - for (group, chunk) in runs.chunks(fan_in).enumerate() { - let path = scratch.join(format!( - "node-surrogate-validation-merge-{round}-{group}.run" - )); - merge_node_surrogate_validation_group(chunk, &path)?; - next.push(path); - metrics.temporary_runs += 1; + for (group, inputs) in runs.chunks(limits.merge_fan_in).enumerate() { + let output = scratch.join(format!("surrogates-merge-{round}-{group}.run")); + merge_surrogate_runs(inputs, &output)?; + next.push(output); } - for path in runs { - let _ = fs::remove_file(path); + for run in runs { + let _ = fs::remove_file(run); } runs = next; round += 1; } - Ok(runs.pop().expect("surrogate validation run exists")) -} - -fn merge_node_surrogate_validation_group(inputs: &[PathBuf], output: &Path) -> Result<(), GfError> { - let mut readers = inputs - .iter() - .map(|path| File::open(path).map(BufReader::new).map_err(storage_err)) - .collect::, _>>()?; - let mut heap = BinaryHeap::>::new(); - for (index, reader) in readers.iter_mut().enumerate() { - if let Some(value) = read_validation_surrogate(reader)? { - heap.push(Reverse((value, index))); - } - } - let mut bytes = Vec::with_capacity(BULK_IO_BYTES); - let mut out = File::create(output).map_err(storage_err)?; - let mut previous = None; - while let Some(Reverse((value, index))) = heap.pop() { - if previous == Some(value) { - return Err(storage_err( - "duplicate node surrogate across external index runs", - )); - } - if bytes.len() + 8 > BULK_IO_BYTES { - out.write_all(&bytes).map_err(storage_err)?; - bytes.clear(); - } - bytes.extend_from_slice(&value.to_le_bytes()); - previous = Some(value); - if let Some(next) = read_validation_surrogate(&mut readers[index])? { - heap.push(Reverse((next, index))); - } - } - if !bytes.is_empty() { - out.write_all(&bytes).map_err(storage_err)?; - } - out.flush().map_err(storage_err) + Ok(runs.pop().expect("surrogate run exists")) } -fn read_validation_surrogate(reader: &mut impl Read) -> Result, GfError> { - Ok(read_exact_record::<8>(reader)?.map(u64::from_le_bytes)) +pub(super) fn read_surrogate_record( + reader: &mut BufReader, +) -> Result, GfError> { + let Some(record) = read_exact_record::<24>(reader)? else { + return Ok(None); + }; + Ok(Some(( + u64::from_be_bytes(record[..8].try_into().expect("fixed")), + record[8..].try_into().expect("fixed"), + ))) } -pub(super) fn flush_entity_surrogate_run( - buffer: &mut Vec<([u8; 16], u64)>, +fn flush_surrogate_run( + buffer: &mut Vec<(u64, [u8; 16])>, scratch: &Path, - prefix: &str, runs: &mut Vec, metrics: &mut UuidIndexBuildMetrics, ) -> Result<(), GfError> { - buffer.sort_unstable_by_key(|record| record.0); + buffer.sort_unstable(); if buffer.windows(2).any(|pair| pair[0].0 == pair[1].0) { - return Err(storage_err("duplicate UUID in canonical topology")); - } - let path = scratch.join(format!("{prefix}-surrogates-{:08}.run", runs.len())); - let mut out = File::create(&path).map_err(storage_err)?; - let mut block = Vec::with_capacity(buffer.len().min(BULK_IO_BYTES / 24) * 24); - for (uuid, surrogate) in buffer.iter() { - block.extend_from_slice(uuid); - block.extend_from_slice(&surrogate.to_le_bytes()); + return Err(storage_err("duplicate node surrogate")); } - if !block.is_empty() { - out.write_all(&block).map_err(storage_err)?; + let path = scratch.join(format!("surrogates-{:08}.run", runs.len())); + let mut bytes = Vec::with_capacity(buffer.len() * 24); + for (surrogate, uuid) in buffer.iter() { + bytes.extend_from_slice(&surrogate.to_be_bytes()); + bytes.extend_from_slice(uuid); } - out.flush().map_err(storage_err)?; + let mut file = File::create(&path).map_err(storage_err)?; + file.write_all(&bytes).map_err(storage_err)?; + file.flush().map_err(storage_err)?; buffer.clear(); runs.push(path); metrics.temporary_runs += 1; Ok(()) } -pub(super) fn merge_node_surrogate_runs( - mut runs: Vec, - scratch: &Path, - fan_in: usize, - metrics: &mut UuidIndexBuildMetrics, -) -> Result { - let mut round = 0; - while runs.len() > 1 { - let mut next = Vec::new(); - for (group, chunk) in runs.chunks(fan_in).enumerate() { - let path = scratch.join(format!("node-surrogates-merge-{round}-{group}.run")); - merge_node_surrogate_group(chunk, &path)?; - next.push(path); - metrics.temporary_runs += 1; - } - for path in runs { - let _ = fs::remove_file(path); - } - runs = next; - round += 1; - } - Ok(runs.pop().expect("at least one node-surrogate run")) -} - -fn merge_node_surrogate_group(inputs: &[PathBuf], output: &Path) -> Result<(), GfError> { +pub(super) fn merge_surrogate_runs(inputs: &[PathBuf], output: &Path) -> Result<(), GfError> { let mut readers = inputs .iter() .map(|path| File::open(path).map(BufReader::new).map_err(storage_err)) .collect::, _>>()?; - let mut heap = BinaryHeap::>::new(); + let mut heap = BinaryHeap::>::new(); for (index, reader) in readers.iter_mut().enumerate() { - if let Some(record) = read_node_surrogate_record(reader)? { + if let Some(record) = read_surrogate_record(reader)? { heap.push(Reverse((record, index))); } } let mut out = File::create(output).map_err(storage_err)?; let mut block = Vec::with_capacity(BULK_IO_BYTES); let mut previous = None; - while let Some(Reverse(((uuid, surrogate), index))) = heap.pop() { - if previous == Some(uuid) { - return Err(storage_err( - "duplicate node UUID across external index runs", - )); + while let Some(Reverse(((surrogate, uuid), index))) = heap.pop() { + if previous.is_some_and(|(prior, _)| prior == surrogate) { + if previous.is_some_and(|(_, prior_uuid)| prior_uuid == uuid) { + if let Some(record) = read_surrogate_record(&mut readers[index])? { + heap.push(Reverse((record, index))); + } + continue; + } + return Err(storage_err("duplicate node surrogate across runs")); } if block.len() + 24 > BULK_IO_BYTES { out.write_all(&block).map_err(storage_err)?; block.clear(); } + block.extend_from_slice(&surrogate.to_be_bytes()); block.extend_from_slice(&uuid); - block.extend_from_slice(&surrogate.to_le_bytes()); - previous = Some(uuid); - if let Some(record) = read_node_surrogate_record(&mut readers[index])? { + previous = Some((surrogate, uuid)); + if let Some(record) = read_surrogate_record(&mut readers[index])? { heap.push(Reverse((record, index))); } } if !block.is_empty() { out.write_all(&block).map_err(storage_err)?; } - out.flush().map_err(storage_err)?; - Ok(()) -} - -pub(super) fn read_node_surrogate_record( - reader: &mut BufReader, -) -> Result, GfError> { - let Some(record) = read_exact_record::<24>(reader)? else { - return Ok(None); - }; - Ok(Some(( - record[..16].try_into().expect("fixed"), - u64::from_le_bytes(record[16..].try_into().expect("fixed")), - ))) -} - -fn reject_cross_kind_identities(nodes: &Path, edges: &Path) -> Result<(), GfError> { - let mut node_reader = BufReader::new(File::open(nodes).map_err(storage_err)?); - let mut edge_reader = BufReader::new(File::open(edges).map_err(storage_err)?); - let mut node = read_record(&mut node_reader)?; - let mut edge = read_record(&mut edge_reader)?; - while let (Some(node_uuid), Some(edge_uuid)) = (node, edge) { - match node_uuid.cmp(&edge_uuid) { - std::cmp::Ordering::Less => node = read_record(&mut node_reader)?, - std::cmp::Ordering::Greater => edge = read_record(&mut edge_reader)?, - std::cmp::Ordering::Equal => { - return Err(storage_err( - "UUID occurs in both node and edge identity domains", - )); - } - } - } - Ok(()) -} - -fn read_record(reader: &mut BufReader) -> Result, GfError> { - read_exact_record::<16>(reader) -} - -#[cfg(test)] -pub(super) fn publish_data( - source: &Path, - root: &Path, - _staging: &Path, - kind: &str, - generation: u64, - record_bytes: u64, -) -> Result { - let length = source.metadata().map_err(storage_err)?.len(); - if record_bytes != IDENTITY_RECORD_BYTES && length % record_bytes != 0 { - return Err(storage_err("internal run has a partial index record")); - } - let mut input = File::open(source).map_err(storage_err)?; - let (sha256, xxh64, blocks, count) = describe_blocks(&mut input, record_bytes)?; - let name = format!("{kind}-{generation}-{}.uuidx", &sha256[..16]); - let directory = graphforge_filesystem::StableDirectory::open(root).map_err(storage_err)?; - let target = std::ffi::OsStr::new(&name); - if let Ok(mut existing) = directory.open_child_file(target) { - if existing.metadata().map_err(storage_err)?.len() != length - || sha256_reader(&mut existing)? != sha256 - { - return Err(storage_err( - "existing immutable run does not match its content name", - )); - } - } else { - let temp_name = std::ffi::OsString::from(format!(".run-{}.tmp", Uuid::new_v4())); - let mut temp = directory - .create_child_file(&temp_name) - .map_err(storage_err)?; - let temp_identity = graphforge_filesystem::file_identity(&temp).map_err(storage_err)?; - let mut install = || -> Result<(), GfError> { - let mut input = File::open(source).map_err(storage_err)?; - std::io::copy(&mut input, &mut temp).map_err(storage_err)?; - crate::durable_commit::seal_file(&temp).map_err(storage_err)?; - match directory.link_child_into(&temp_name, &temp, temp_identity, &directory, target) { - Ok(_) => Ok(()), - Err(_) => { - let mut existing = directory.open_child_file(target).map_err(storage_err)?; - if existing.metadata().map_err(storage_err)?.len() != length - || sha256_reader(&mut existing)? != sha256 - { - return Err(storage_err("concurrent immutable run mismatch")); - } - Ok(()) - } - } - }; - let result = install(); - let _ = directory.unlink_child_if_identity(&temp_name, temp_identity); - result?; - crate::durable_commit::acknowledge_directory(&directory).map_err(storage_err)?; - } - Ok(FileRecord { - name, - count, - sha256, - xxh64, - blocks, - }) -} - -#[cfg(test)] -fn sha256_reader(reader: &mut impl Read) -> Result { - let mut digest = Sha256::new(); - let mut buffer = vec![0_u8; 64 * 1024]; - loop { - let read = reader.read(&mut buffer).map_err(storage_err)?; - if read == 0 { - break; - } - digest.update(&buffer[..read]); - } - let mut encoded = String::with_capacity(64); - for byte in digest.finalize() { - write!(&mut encoded, "{byte:02x}").expect("writing to a String cannot fail"); - } - Ok(encoded) + out.flush().map_err(storage_err) } #[cfg(test)] diff --git a/crates/graphforge-storage/src/uuid_membership/rebuild/tests.rs b/crates/graphforge-storage/src/uuid_membership/rebuild/tests.rs index b81fe4ba6..eaeb89d6c 100644 --- a/crates/graphforge-storage/src/uuid_membership/rebuild/tests.rs +++ b/crates/graphforge-storage/src/uuid_membership/rebuild/tests.rs @@ -1,165 +1,19 @@ use super::super::INDEX_DIR; -use super::super::MANIFEST; -use super::super::Manifest; use super::super::TopologyIndexReceipt; use super::super::UuidIndexBuildLimits; -use super::super::UuidIndexKind; -use super::super::UuidMembershipIndex; use super::super::V4_ORDINAL_BLOCK_BYTES; use super::super::V4_ORDINAL_MANIFEST; use super::super::V4_ORDINAL_RECEIPT; use super::super::V4OrdinalRebuildDisposition; use super::super::tests::fixture; -use super::super::tests::make_installed_manifest_stale; -use super::super::tests::receipt_manifest_digest; -use super::super::tests::write_node_parquet; use super::super::tests::write_node_parquet_with_ids; -use super::super::tests::write_uuid_parquet; use super::super::topology_delta::hex_sha256; use super::V4RebuildScratchAccounting; -use super::rebuild_uuid_membership_indexes; use super::rebuild_v4_ordinal_identity; use super::rebuild_v4_ordinal_identity_with_evidence; use std::fs; -use std::sync::Arc; -use std::sync::Barrier; use uuid::Uuid; -#[test] -fn forced_rebuild_receipt_binds_newly_staged_manifest() { - let (dir, _, _) = fixture(); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); - let stale_digest = make_installed_manifest_stale(dir.path()); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); - let installed_digest = receipt_manifest_digest(dir.path()); - assert_ne!(installed_digest, stale_digest); -} - -#[test] -fn unpublished_build_artifacts_do_not_change_concurrent_readers() { - let (dir, nodes, _) = fixture(); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); - let barrier = Arc::new(Barrier::new(2)); - let reader_root = dir.path().to_path_buf(); - let reader_barrier = barrier.clone(); - let expected = nodes[0]; - let reader = std::thread::spawn(move || { - let mut index = UuidMembershipIndex::open(&reader_root).unwrap(); - reader_barrier.wait(); - index.probe(UuidIndexKind::Node, &[expected]).unwrap().0 - }); - fs::write( - dir.path().join(INDEX_DIR).join("nodes-unpublished.uuidx"), - [7_u8; 16], - ) - .unwrap(); - barrier.wait(); - assert_eq!(reader.join().unwrap(), vec![true]); - let mut reopened = UuidMembershipIndex::open(dir.path()).unwrap(); - assert_eq!( - reopened.probe(UuidIndexKind::Node, &[expected]).unwrap().0, - vec![true] - ); -} - -#[test] -fn concurrent_rebuilds_publish_one_authenticated_snapshot() { - let (dir, _, _) = fixture(); - let root = Arc::new(dir.path().to_path_buf()); - let barrier = Arc::new(Barrier::new(3)); - let workers = (0..2) - .map(|_| { - let root = root.clone(); - let barrier = barrier.clone(); - std::thread::spawn(move || { - barrier.wait(); - rebuild_uuid_membership_indexes( - &root, - UuidIndexBuildLimits { - scan_batch_rows: 1, - run_records: 1, - merge_fan_in: 2, - }, - ) - }) - }) - .collect::>(); - barrier.wait(); - for worker in workers { - worker.join().unwrap().unwrap(); - } - let index = UuidMembershipIndex::open(&root).unwrap(); - assert_eq!(index.count(UuidIndexKind::Node), 3); - assert_eq!(index.count(UuidIndexKind::Edge), 2); - let names = fs::read_dir(root.join(INDEX_DIR)) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .collect::>(); - assert!(names.iter().all(|name| !name.ends_with(".tmp"))); -} - -#[test] -fn duplicate_and_cross_kind_identities_fail_closed() { - let dir = tempfile::tempdir().unwrap(); - let repeated = Uuid::from_u128(7); - write_node_parquet( - &dir.path().join("topology/nodes.parquet"), - &[repeated, repeated], - ); - let duplicate = rebuild_uuid_membership_indexes( - dir.path(), - UuidIndexBuildLimits { - scan_batch_rows: 1, - run_records: 1, - merge_fan_in: 2, - }, - ) - .unwrap_err(); - assert!(duplicate.to_string().contains("duplicate")); - - let dir = tempfile::tempdir().unwrap(); - write_node_parquet(&dir.path().join("topology/nodes.parquet"), &[repeated]); - write_uuid_parquet( - &dir.path().join("topology/edges/R.parquet"), - "edge_uuid", - &[repeated], - ); - let cross_kind = - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap_err(); - assert!(cross_kind.to_string().contains("both node and edge")); -} - -#[test] -fn duplicate_and_zero_node_surrogates_fail_closed_across_bounded_runs() { - let limits = UuidIndexBuildLimits { - scan_batch_rows: 1, - run_records: 1, - merge_fan_in: 2, - }; - let dir = tempfile::tempdir().unwrap(); - let nodes = [Uuid::from_u128(1), Uuid::from_u128(2)]; - write_node_parquet_with_ids(&dir.path().join("topology/nodes.parquet"), &nodes, &[7, 7]); - assert!( - rebuild_uuid_membership_indexes(dir.path(), limits) - .unwrap_err() - .to_string() - .contains("duplicate node surrogate") - ); - - let dir = tempfile::tempdir().unwrap(); - write_node_parquet_with_ids( - &dir.path().join("topology/nodes.parquet"), - &[Uuid::from_u128(3)], - &[0], - ); - assert!( - rebuild_uuid_membership_indexes(dir.path(), limits) - .unwrap_err() - .to_string() - .contains("invalid node surrogate") - ); -} - #[test] fn explicit_v4_rebuild_uses_topology_and_independent_projection_orders() { let dir = tempfile::tempdir().unwrap(); @@ -229,18 +83,19 @@ fn explicit_v4_rebuild_uses_topology_and_independent_projection_orders() { } #[test] -fn explicit_v4_rebuild_does_not_trust_corrupt_v3_reverse_state() { +fn explicit_v4_rebuild_ignores_legacy_membership_files() { let (dir, _, _) = fixture(); fs::write( dir.path().join("topology/generation.json"), b"{\"topology_generation\":3,\"search_generation\":0,\"property_generation\":0}\n", ) .unwrap(); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); let root = dir.path().join(INDEX_DIR); - let v3: Manifest = serde_json::from_slice(&fs::read(root.join(MANIFEST)).unwrap()).unwrap(); + // A project built before #1902 carries the membership index. Whatever its + // bytes are, the rebuild reads canonical topology and never them. + fs::write(root.join("manifest.json"), b"planted-corrupt-manifest").unwrap(); fs::write( - root.join(&v3.runs[0].node_surrogates.name), + root.join("node-surrogates-v5-3-0000000000000000.uuidx"), b"planted-corrupt-v3-reverse", ) .unwrap(); diff --git a/crates/graphforge-storage/src/uuid_membership/tests.rs b/crates/graphforge-storage/src/uuid_membership/tests.rs index fc3313b07..be0978f63 100644 --- a/crates/graphforge-storage/src/uuid_membership/tests.rs +++ b/crates/graphforge-storage/src/uuid_membership/tests.rs @@ -1,25 +1,14 @@ -use super::AuthenticatedUuidIndexSnapshot; -use super::FAIL_AFTER_MANIFEST_SUSPEND; use super::INDEX_DIR; -use super::MANIFEST; -use super::Manifest; -use super::TOPOLOGY_RECEIPT; use super::TopologyIndexReceipt; use super::UuidIndexBuildLimits; -use super::UuidIndexKind; -use super::UuidMembershipIndex; use super::UuidTopologyDelta; use super::V4_ORDINAL_MANIFEST; use super::V4_ORDINAL_RECEIPT; -use super::append_uuid_membership_delta; use super::maintain_uuid_membership_orphans_with_ordinal_authority; -use super::rebuild::manifest_generation; -use super::rebuild::rebuild_uuid_membership_indexes; use super::rebuild::rebuild_v4_ordinal_identity; use super::rebuild::rebuild_v4_ordinal_identity_with_evidence; use super::topology_delta::V4_PLAN_PREFIX; use super::topology_delta::V4_PLAN_ROOT; -use super::topology_delta::append_uuid_membership_delta_with_tombstones; use super::topology_delta::commit_uuid_topology_rewrite; use super::topology_delta::hex_sha256; use arrow::array::FixedSizeBinaryArray; @@ -31,7 +20,6 @@ use arrow::record_batch::RecordBatch; use parquet::arrow::ArrowWriter; use std::fs; use std::fs::File; -use std::io::Write; use std::path::Path; use std::sync::Arc; use uuid::Uuid; @@ -213,7 +201,6 @@ fn v4_rewrite_subprocess_crash_retry_matrix_cleans_exact_scratch() { b"{\"topology_generation\":7,\"search_generation\":0,\"property_generation\":0}\n", ) .unwrap(); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); rebuild_v4_ordinal_identity(dir.path(), UuidIndexBuildLimits::default()).unwrap(); if target == 9 { run_v4_rewrite_once(dir.path()); @@ -262,7 +249,7 @@ fn v4_rewrite_subprocess_crash_retry_matrix_cleans_exact_scratch() { assert_eq!(v4.topology_generation, target, "{failpoint}"); assert_eq!( receipt_manifest_digest(dir.path()), - hex_sha256(&fs::read(dir.path().join(INDEX_DIR).join(MANIFEST)).unwrap()) + hex_sha256(&fs::read(dir.path().join(INDEX_DIR).join(V4_ORDINAL_MANIFEST)).unwrap()) ); assert_exact_v4_reopen(dir.path(), target); } @@ -277,7 +264,6 @@ fn v4_rewrite_subprocess_crash_retry_matrix_cleans_exact_scratch() { b"{\"topology_generation\":7,\"search_generation\":0,\"property_generation\":0}\n", ) .unwrap(); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); rebuild_v4_ordinal_identity(dir.path(), UuidIndexBuildLimits::default()).unwrap(); let child = |failpoint: &str, retry: bool| { @@ -365,6 +351,7 @@ pub(crate) fn fixture() -> (tempfile::TempDir, Vec, Vec) { "edge_uuid", &edges, ); + fs::create_dir_all(dir.path().join(INDEX_DIR)).unwrap(); (dir, nodes, edges) } @@ -502,7 +489,6 @@ fn v4_orphan_cleanup_subprocess_crash_retry_preserves_authenticated_union() { b"{\"topology_generation\":7,\"search_generation\":0,\"property_generation\":0}\n", ) .unwrap(); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); let (referenced, _) = install_test_v4_facet(dir.path(), 7, &nodes); let orphan = dir .path() @@ -555,373 +541,14 @@ fn v4_orphan_cleanup_subprocess_crash_retry_preserves_authenticated_union() { pub(super) fn receipt_manifest_digest(project: &Path) -> String { let root = project.join(INDEX_DIR); let receipt: TopologyIndexReceipt = - serde_json::from_slice(&fs::read(root.join(TOPOLOGY_RECEIPT)).unwrap()).unwrap(); + serde_json::from_slice(&fs::read(root.join(V4_ORDINAL_RECEIPT)).unwrap()).unwrap(); assert_eq!( receipt.manifest_sha256, - hex_sha256(&fs::read(root.join(MANIFEST)).unwrap()) + hex_sha256(&fs::read(root.join(V4_ORDINAL_MANIFEST)).unwrap()) ); receipt.manifest_sha256 } -pub(super) fn make_installed_manifest_stale(project: &Path) -> String { - let path = project.join(INDEX_DIR).join(MANIFEST); - let mut manifest: Manifest = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); - manifest.live_node_count = manifest.live_node_count.saturating_add(17); - let body = serde_json::to_vec(&manifest).unwrap(); - fs::write(path, &body).unwrap(); - hex_sha256(&body) -} - -#[test] -fn stale_v3_migration_receipt_survives_crash_roll_forward() { - const CHILD_ROOT: &str = "GRAPHFORGE_UUID_MIGRATION_CHILD_ROOT"; - if let Ok(root) = std::env::var(CHILD_ROOT) { - let _ = rebuild_uuid_membership_indexes(Path::new(&root), UuidIndexBuildLimits::default()); - panic!("child migration failpoint did not terminate the process"); - } - - let (dir, _, _) = fixture(); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); - let stale_digest = make_installed_manifest_stale(dir.path()); - let status = std::process::Command::new(std::env::current_exe().unwrap()) - .arg("--exact") - .arg("uuid_membership::tests::stale_v3_migration_receipt_survives_crash_roll_forward") - .arg("--nocapture") - .env(CHILD_ROOT, dir.path()) - .env( - "GRAPHFORGE_PROJECT_FAILPOINTS", - "graphforge-internal-subprocess-v1", - ) - .env( - "GRAPHFORGE_PROJECT_FAILPOINT", - "rewrite.after_durable_intent", - ) - .status() - .unwrap(); - assert_eq!(status.code(), Some(crate::project_failpoint::exit_code())); - - assert_eq!(crate::read_topology_generation(dir.path()).unwrap(), 0); - assert_eq!(crate::read_search_generation(dir.path()).unwrap(), 0); - let installed_digest = receipt_manifest_digest(dir.path()); - assert_ne!(installed_digest, stale_digest); - assert!(!dir.path().join(".graphforge-rewrite-v1.json").exists()); -} - -#[test] -fn owned_manifest_suspension_restores_exact_authority_and_rejects_tampering() { - let (dir, _, _) = fixture(); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); - let mut snapshot = AuthenticatedUuidIndexSnapshot::open_at_generation(dir.path(), 0).unwrap(); - let identity = snapshot.manifest_identity; - let path = dir.path().join(INDEX_DIR).join(MANIFEST); - let original = fs::read(&path).unwrap(); - snapshot.suspend_owned_manifest(); - assert!(snapshot.revalidate().is_err()); - snapshot.restore_owned_manifest().unwrap(); - assert_eq!(snapshot.manifest_identity, identity); - snapshot.revalidate().unwrap(); - snapshot.suspend_owned_manifest(); - fs::write(&path, [original.as_slice(), b"\n"].concat()).unwrap(); - assert!(snapshot.restore_owned_manifest().is_err()); - assert!(snapshot.manifest_file.is_none()); - fs::write(&path, &original).unwrap(); - snapshot.restore_owned_manifest().unwrap(); - snapshot.suspend_owned_manifest(); - let replacement = path.with_extension("replacement"); - fs::write(&replacement, original).unwrap(); - fs::rename(&replacement, &path).unwrap(); - assert!(snapshot.restore_owned_manifest().is_err()); -} - -#[test] -fn owned_manifest_returned_error_restores_snapshot_for_same_process_retry() { - let dir = tempfile::tempdir().unwrap(); - let mut snapshot = None; - let make_batch = || { - let mut batch = crate::RewriteBatch::new(); - batch - .stage_bytes(&dir.path().join("topology/nodes.parquet"), b"fixture") - .unwrap(); - batch - }; - let first = Uuid::from_u128(201); - let second = Uuid::from_u128(202); - commit_uuid_topology_rewrite( - dir.path(), - make_batch(), - &UuidTopologyDelta { - nodes: vec![(first, 1)], - edges: Vec::new(), - deleted_nodes: Vec::new(), - deleted_edges: Vec::new(), - }, - &mut snapshot, - ) - .unwrap(); - let old = snapshot.as_ref().unwrap().manifest_identity; - FAIL_AFTER_MANIFEST_SUSPEND.set(true); - let delta = UuidTopologyDelta { - nodes: vec![(second, 2)], - edges: Vec::new(), - deleted_nodes: Vec::new(), - deleted_edges: Vec::new(), - }; - let error = commit_uuid_topology_rewrite(dir.path(), make_batch(), &delta, &mut snapshot) - .err() - .unwrap(); - assert!(error.to_string().contains("injected manifest suspension")); - let restored = snapshot.as_ref().unwrap(); - assert_eq!(restored.manifest_identity, old); - restored.revalidate().unwrap(); - assert_eq!(crate::read_topology_generation(dir.path()).unwrap(), 1); - commit_uuid_topology_rewrite(dir.path(), make_batch(), &delta, &mut snapshot).unwrap(); - assert_eq!( - snapshot - .as_mut() - .unwrap() - .lookup_node_surrogates(&[first, second]) - .unwrap() - .0, - [Some(1), Some(2)] - ); -} - -#[test] -fn owned_manifest_same_writer_successive_flushes_preserve_incremental_snapshot() { - let dir = tempfile::tempdir().unwrap(); - let first = Uuid::from_u128(101); - let second = Uuid::from_u128(102); - let mut writer = - crate::GraphWriter::open_at(dir.path(), graphforge_core::OntologyMode::Strict, 1).unwrap(); - writer - .create_node(first, graphforge_value::EntityTypeId::decode(0).unwrap()) - .unwrap(); - writer.flush().unwrap(); - let path = dir.path().join(INDEX_DIR).join(MANIFEST); - let old = graphforge_filesystem::path_identity(&path).unwrap(); - writer - .create_node(second, graphforge_value::EntityTypeId::decode(0).unwrap()) - .unwrap(); - writer - .create_edge(Uuid::from_u128(103), "CON", &first, &second) - .unwrap(); - writer.flush().unwrap(); - assert_ne!(graphforge_filesystem::path_identity(&path).unwrap(), old); - assert_eq!(crate::read_topology_generation(dir.path()).unwrap(), 2); - assert_eq!( - writer - .topology_write_work() - .uuid_prior_topology_rows_decoded, - 0 - ); - let mut index = UuidMembershipIndex::open(dir.path()).unwrap(); - assert_eq!( - index.lookup_node_surrogates(&[first, second]).unwrap().0, - [Some(1), Some(2)] - ); -} - -#[test] -fn readonly_shared_runs_preserve_uuid_snapshot_authentication() { - for scenario in [ - "valid", - "writable", - "made_writable", - "tampered", - "replaced", - "manifest_link", - ] { - let source = tempfile::tempdir().unwrap(); - let aliases = tempfile::tempdir().unwrap(); - let nodes = [(Uuid::from_u128(1), 1), (Uuid::from_u128(2), u64::MAX - 1)]; - crate::generation::force_bump_topology_generation_for_test(source.path()).unwrap(); - write_node_parquet_with_ids( - &source.path().join("topology/nodes.parquet"), - &nodes.map(|(uuid, _)| uuid), - &nodes.map(|(_, surrogate)| surrogate), - ); - rebuild_uuid_membership_indexes(source.path(), UuidIndexBuildLimits::default()).unwrap(); - let root = source.path().join(INDEX_DIR); - let manifest: Manifest = - serde_json::from_slice(&fs::read(root.join(MANIFEST)).unwrap()).unwrap(); - let records = manifest - .runs - .iter() - .flat_map(|run| [run.identities.clone(), run.node_surrogates.clone()]) - .collect::>(); - let record = records - .iter() - .find(|record| record.name.starts_with("identities-v5") && record.count > 0) - .unwrap(); - let original_permissions = fs::metadata(root.join(&record.name)).unwrap().permissions(); - for record in &records { - let path = root.join(&record.name); - fs::hard_link(&path, aliases.path().join(&record.name)).unwrap(); - if scenario != "writable" { - let mut permissions = fs::metadata(&path).unwrap().permissions(); - permissions.set_readonly(true); - fs::set_permissions(&path, permissions).unwrap(); - } - } - if scenario == "manifest_link" { - fs::hard_link(root.join(MANIFEST), aliases.path().join(MANIFEST)).unwrap(); - } - // Match hydration: establish aliases before retaining immutable - // handles. Windows prevents adding links through held handles that - // intentionally deny DELETE sharing. - let mut snapshot = - AuthenticatedUuidIndexSnapshot::open_at_generation(source.path(), 1).unwrap(); - match scenario { - "valid" => { - snapshot.revalidate().unwrap(); - snapshot.open_retained_file(record).unwrap(); - let (values, _) = snapshot - .lookup_node_surrogates(&[nodes[0].0, nodes[1].0]) - .unwrap(); - assert_eq!(values, [Some(1), Some(u64::MAX - 1)]); - } - "writable" => { - assert!(snapshot.revalidate().is_err()); - assert!(snapshot.open_retained_file(record).is_err()); - } - "made_writable" => { - snapshot.revalidate().unwrap(); - fs::set_permissions( - aliases.path().join(&record.name), - original_permissions.clone(), - ) - .unwrap(); - assert!(snapshot.revalidate().is_err()); - assert!(snapshot.open_retained_file(record).is_err()); - } - "tampered" => { - let alias = aliases.path().join(&record.name); - fs::set_permissions(&alias, original_permissions.clone()).unwrap(); - let mut writer = fs::OpenOptions::new().write(true).open(&alias).unwrap(); - writer.write_all(&[0xff]).unwrap(); - writer.sync_all().unwrap(); - drop(writer); - let mut permissions = original_permissions.clone(); - permissions.set_readonly(true); - fs::set_permissions(&alias, permissions).unwrap(); - // Metadata and inode still match. The retained read must - // authenticate bytes, not trust readonly status alone. - snapshot.revalidate().unwrap(); - let error = snapshot.lookup_node_surrogates(&[nodes[0].0]).unwrap_err(); - assert!( - error.to_string().contains("block authentication"), - "{error}" - ); - assert!( - AuthenticatedUuidIndexSnapshot::open_at_generation(source.path(), 1).is_err() - ); - } - "replaced" => { - let path = root.join(&record.name); - let bytes = fs::read(&path).unwrap(); - let replacement = fs::rename(&path, root.join("held-original")); - #[cfg(windows)] - { - // Stable retained handles intentionally omit - // FILE_SHARE_DELETE: Windows prevents replacement. - assert!(replacement.is_err()); - assert_eq!(fs::read(&path).unwrap(), bytes); - snapshot.revalidate().unwrap(); - } - #[cfg(not(windows))] - { - replacement.unwrap(); - fs::write(&path, bytes).unwrap(); - assert!(snapshot.revalidate().is_err()); - } - } - "manifest_link" => { - assert!(snapshot.revalidate().is_err()); - } - _ => unreachable!(), - } - // Restore this test's owned aliases so Windows cleanup can remove - // readonly files; production never mutates shared run permissions. - for record in &records { - fs::set_permissions( - aliases.path().join(&record.name), - original_permissions.clone(), - ) - .unwrap(); - } - } -} - -pub(super) fn singleton_append_series(batches: u64) -> (tempfile::TempDir, u64) { - let dir = tempfile::tempdir().unwrap(); - let mut bytes = 0; - for generation in 1..=batches { - crate::generation::force_bump_topology_generation_for_test(dir.path()).unwrap(); - let metrics = append_uuid_membership_delta( - dir.path(), - generation, - &[(Uuid::from_u128(u128::from(generation)), generation)], - &[], - ) - .unwrap(); - assert_eq!(metrics.prior_topology_rows_decoded, 0); - assert!(metrics.write_blocks >= 2); - assert_eq!(metrics.write_bytes, metrics.physical_bytes_written); - bytes += metrics.physical_bytes_written; - } - (dir, bytes) -} - -#[test] -fn packed_membership_rebuild_reopen_and_tombstone_preserve_full_width_ids() { - let dir = tempfile::tempdir().unwrap(); - let nodes = [ - Uuid::from_u128(2), - Uuid::from_u128(u128::MAX - 1), - Uuid::from_u128(u128::MAX), - ]; - let ids = [u64::from(u32::MAX), u64::from(u32::MAX) + 1, u64::MAX]; - write_node_parquet_with_ids(&dir.path().join("topology/nodes.parquet"), &nodes, &ids); - let edge = Uuid::from_u128(1); - write_uuid_parquet( - &dir.path().join("topology/edges/R.parquet"), - "edge_uuid", - &[edge], - ); - rebuild_uuid_membership_indexes( - dir.path(), - UuidIndexBuildLimits { - scan_batch_rows: 1, - run_records: 1, - merge_fan_in: 2, - }, - ) - .unwrap(); - let mut index = UuidMembershipIndex::open(dir.path()).unwrap(); - assert_eq!( - index.lookup_node_surrogates(&nodes).unwrap().0, - ids.map(Some) - ); - assert_eq!(index.probe(UuidIndexKind::Edge, &[edge]).unwrap().0, [true]); - drop(index); - crate::generation::force_bump_topology_generation_for_test(dir.path()).unwrap(); - append_uuid_membership_delta_with_tombstones( - dir.path(), - 1, - &[], - &[], - &[(nodes[2], u64::MAX)], - &[], - ) - .unwrap(); - let mut index = UuidMembershipIndex::open(dir.path()).unwrap(); - assert_eq!( - index.lookup_node_surrogates(&nodes).unwrap().0, - [Some(ids[0]), Some(ids[1]), None] - ); - assert_eq!(index.probe(UuidIndexKind::Edge, &[edge]).unwrap().0, [true]); -} - #[test] fn v4_rebuild_subprocess_crash_retry_selects_one_complete_authority() { const CHILD_ROOT: &str = "GRAPHFORGE_V4_REBUILD_CHILD_ROOT"; @@ -994,7 +621,6 @@ fn v4_rebuild_subprocess_crash_retry_selects_one_complete_authority() { b"{\"topology_generation\":7,\"search_generation\":0,\"property_generation\":0}\n", ) .unwrap(); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); let child = |retry: bool| { let mut command = std::process::Command::new(std::env::current_exe().unwrap()); @@ -1022,10 +648,8 @@ fn v4_rebuild_subprocess_crash_retry_selects_one_complete_authority() { Some(crate::project_failpoint::exit_code()), "{failpoint}" ); - // Inspect the crashed tree before recovery. The prior v3 authority - // must remain valid at every boundary; v4 is either absent, + // Inspect the crashed tree before recovery. v4 is either absent, // incomplete and therefore inadmissible, or already complete. - UuidMembershipIndex::open_at_generation(dir.path(), 7).unwrap(); let journal_path = dir.path().join(".graphforge-rewrite-v1.json"); let journal = fs::read(&journal_path) .ok() @@ -1127,8 +751,6 @@ fn v4_rebuild_subprocess_crash_retry_selects_one_complete_authority() { assert!(child(true).success(), "{failpoint}"); assert!(!dir.path().join(".graphforge-rewrite-v1.json").exists()); - assert_eq!(manifest_generation(dir.path()).unwrap(), Some(7)); - UuidMembershipIndex::open(dir.path()).unwrap(); let manifest_bytes = fs::read(index.join(V4_ORDINAL_MANIFEST)).unwrap(); let receipt_bytes = fs::read(index.join(V4_ORDINAL_RECEIPT)).unwrap(); diff --git a/crates/graphforge-storage/src/uuid_membership/topology_delta.rs b/crates/graphforge-storage/src/uuid_membership/topology_delta.rs index 8cee3f33e..084f42ce6 100644 --- a/crates/graphforge-storage/src/uuid_membership/topology_delta.rs +++ b/crates/graphforge-storage/src/uuid_membership/topology_delta.rs @@ -1,55 +1,23 @@ //! UUID topology delta preparation, commit, and reconciliation. -use super::AuthenticatedUuidIndexSnapshot; use super::BULK_IO_BYTES; -use super::BlockRecord; use super::CommittedUuidTopologyRewrite; use super::DEFAULT_ORPHAN_GC_LIMIT; -#[cfg(test)] -use super::FAIL_AFTER_MANIFEST_SUSPEND; -use super::FORMAT_VERSION; -use super::FileRecord; -use super::IDENTITY_RECORD_BYTES; -use super::IDENTITY_RECORD_WIDTH; use super::INDEX_DIR; -use super::MANIFEST; -use super::MAX_MANIFEST_BYTES; -use super::Manifest; -use super::NODE_LOOKUP_RECORD_BYTES; -#[cfg(test)] -use super::OpenRun; -use super::PreparedUuidIndexDelta; use super::PreparedV4OrdinalDelta; -use super::RunRecord; -use super::TOPOLOGY_RECEIPT; use super::TopologyIndexReceipt; -use super::UuidIndexAppendMetrics; use super::UuidIndexBuildLimits; use super::UuidIndexBuildMetrics; -use super::UuidIndexKind; use super::UuidIndexOrphanGcWork; use super::UuidTopologyDelta; use super::V4_ORDINAL_BLOCK_BYTES; use super::V4_ORDINAL_MANIFEST; use super::V4_ORDINAL_RECEIPT; use super::V4OrdinalAppendMetrics; -use super::block_matches; -use super::create_uuid_file; -use super::describe_run; use super::hex_bytes; -use super::identity_codec; -use super::injected_snapshot_refresh_failure; -use super::maintenance::authenticated_v3_membership_authority; -#[cfg(test)] -use super::maintenance::cleanup_superseded_files; use super::maintenance::collect_uuid_orphans_locked; -use super::maintenance::manifest_file_names; use super::maintenance::selected_generation_for_graph_root; use super::maintenance::standalone_v4_pinned_update; -use super::open_uuid_child_file; -use super::open_uuid_file; -#[cfg(test)] -use super::open_verified; use super::ordinal_artifacts::V4AuthorityTransactionProof; use super::ordinal_artifacts::V4ConstructionArtifactBundle; use super::ordinal_artifacts::V4OrdinalConstructionWriter; @@ -62,31 +30,17 @@ use super::ordinal_artifacts::write_v4_tombstone_artifact; use super::ordinal_compaction::compact_v4_binary_carry; use super::ordinal_compaction::read_v4_forward_record; use super::rebuild::build_surrogate_run; +use super::rebuild::read_surrogate_record; +use crate::UuidIndexKind; use super::rebuild::flush_entity_surrogate_run; use super::rebuild::merge_node_surrogate_runs; -#[cfg(test)] -use super::rebuild::merge_surrogate_runs; -#[cfg(test)] -use super::rebuild::publish_data; -use super::rebuild::read_exact_record; use super::rebuild::read_node_surrogate_record; -use super::rebuild::read_surrogate_record; -use super::record_length; -use super::reject_retained_identity_collisions; -use super::reject_retained_surrogate_collisions; use super::storage_err; -use super::uuid_membership_index_present; use super::v4_publication_failure; -use super::validate_block_records; -#[cfg(test)] -use super::validate_run_contents; -use super::validate_run_descriptors; use graphforge_core::GfError; use graphforge_core::hash_observation::ControlSha256 as Sha256; use sha2::Digest; -use std::cmp::Reverse; -use std::collections::BinaryHeap; use std::collections::HashMap; use std::fmt::Write as _; use std::fs; @@ -95,18 +49,111 @@ use std::io::BufReader; use std::io::Read; use std::io::Seek; use std::io::SeekFrom; -use std::io::Write; use std::path::Path; use std::path::PathBuf; use uuid::Uuid; -/// Commit topology and its UUID participant under the one durable rewrite lock. +/// What [`check_and_record_identities`] learned from the published topology. +struct RecordedIdentities { + /// Topology generation the identities were checked against. + generation: u64, + /// Deleted nodes with their `node_id`. + deleted_nodes: Vec<(Uuid, u64)>, + /// Probe work spent checking them. + metrics: crate::UuidProbeMetrics, +} + +/// Resolve the identities a rewrite adds and deletes against the published +/// topology, stage the record of deleted UUIDs, and return the deleted nodes +/// with their `node_id`. +/// +/// A new UUID may not equal any live node, live edge, or deleted entity: node +/// and edge UUIDs share one namespace and a deleted UUID is never reused. +fn check_and_record_identities( + project_dir: &Path, + staged: &mut crate::staging::RewriteBatch, + delta: &UuidTopologyDelta, + probe: &mut Option, +) -> Result { + let generation = crate::read_topology_generation(project_dir)?; + if probe + .as_ref() + .is_none_or(|value| value.topology_generation() != generation) + { + let files = match staged.topology_authority() { + Some(authority) => crate::enumerate_topology_files(authority, None)?, + None => crate::TopologyFiles::discover_legacy(project_dir)?, + }; + *probe = Some(crate::TopologyIdentityProbe::open( + project_dir, + &files, + generation, + )?); + } + let probe = probe.as_mut().expect("probe opened above"); + let mut metrics = crate::UuidProbeMetrics::default(); + + let mut incoming = delta + .nodes + .iter() + .map(|(uuid, _)| *uuid) + .chain(delta.edges.iter().copied()) + .collect::>(); + if incoming.iter().any(Uuid::is_nil) { + return Err(storage_err("topology delta contains a nil UUID")); + } + if !incoming.is_empty() { + let (taken, work) = probe.taken(&incoming)?; + if taken.into_iter().any(|taken| taken) { + return Err(storage_err( + "UUID already exists in the published topology or among deleted identities", + )); + } + metrics.absorb(&work); + incoming.sort_unstable(); + if incoming.windows(2).any(|pair| pair[0] == pair[1]) { + return Err(storage_err("topology delta repeats a UUID")); + } + } + + let (surrogates, work) = probe.lookup_node_surrogates(&delta.deleted_nodes)?; + metrics.absorb(&work); + let deleted_nodes = delta + .deleted_nodes + .iter() + .copied() + .zip(surrogates) + .filter_map(|(uuid, surrogate)| surrogate.map(|id| (uuid, id))) + .collect::>(); + let (present, work) = probe.probe(UuidIndexKind::Edge, &delta.deleted_edges)?; + metrics.absorb(&work); + let deleted_edges = delta + .deleted_edges + .iter() + .copied() + .zip(present) + .filter_map(|(uuid, present)| present.then_some(uuid)) + .collect::>(); + let spent = deleted_nodes + .iter() + .map(|(uuid, _)| *uuid) + .chain(deleted_edges.iter().copied()) + .collect::>(); + crate::stage_deleted_identities(staged, project_dir, &spent)?; + Ok(RecordedIdentities { + generation, + deleted_nodes, + metrics, + }) +} + +/// Commit topology and its identity participant under the one durable rewrite lock. #[allow(clippy::too_many_lines)] // One sealed participant lifecycle; order is the invariant. pub(crate) fn commit_uuid_topology_rewrite( project_dir: &Path, - staged: crate::staging::RewriteBatch, + mut staged: crate::staging::RewriteBatch, delta: &UuidTopologyDelta, - snapshot: &mut Option, + probe: &mut Option, ) -> Result { let delta_is_empty = delta.nodes.is_empty() && delta.edges.is_empty() @@ -115,10 +162,6 @@ pub(crate) fn commit_uuid_topology_rewrite( if delta_is_empty && staged.is_empty() { return Ok(CommittedUuidTopologyRewrite::NoTopologyChange); } - super::rebuild::ensure_uuid_membership_migrated_with_topology( - project_dir, - staged.topology_authority().cloned(), - )?; let selected = selected_generation_for_graph_root(project_dir)?; let ordinal_authority = selected .as_ref() @@ -149,13 +192,11 @@ pub(crate) fn commit_uuid_topology_rewrite( } else { None }; - let membership_authority = selected - .as_ref() - .map(authenticated_v3_membership_authority) - .transpose()? - .flatten(); - let prepared = std::rc::Rc::new(std::cell::RefCell::new(None)); - let prepared_from_callback = std::rc::Rc::clone(&prepared); + let RecordedIdentities { + generation: probed_generation, + deleted_nodes, + metrics: probe_metrics, + } = check_and_record_identities(project_dir, &mut staged, delta, probe)?; let prepared_v4 = std::rc::Rc::new(std::cell::RefCell::new(None)); let prepared_v4_from_callback = std::rc::Rc::clone(&prepared_v4); let generations = std::rc::Rc::new(std::cell::Cell::new(None)); @@ -169,6 +210,11 @@ pub(crate) fn commit_uuid_topology_rewrite( } context.project.revalidate_named().map_err(storage_err)?; generations_from_callback.set(Some((context.prior, context.next))); + if context.prior.topology != probed_generation { + return Err(storage_err( + "topology generation changed between identity checks and commit", + )); + } if context.next.topology == context.prior.topology { if !delta_is_empty { return Err(storage_err( @@ -181,76 +227,17 @@ pub(crate) fn commit_uuid_topology_rewrite( // inventory capable of authenticating reachability. Topology // publication remains valid there, but orphan deletion must be // conservatively deferred rather than self-authorizing the live - // manifest. Project-generation roots retain the authenticated GC - // path, including the v3/v4 union. - let orphan_gc = if membership_authority.is_some() { + // manifest. + let orphan_gc = if selected.is_some() { collect_uuid_orphans_locked( context.project, context.project_root, DEFAULT_ORPHAN_GC_LIMIT, - membership_authority.as_ref(), ordinal_authority.as_ref(), )? } else { UuidIndexOrphanGcWork::default() }; - if !uuid_membership_index_present(context.project_root) { - return Err(storage_err( - "UUID membership index migration is required before topology mutation", - )); - } - if snapshot - .as_ref() - .is_none_or(|value| value.topology_generation() != context.prior.topology) - { - *snapshot = Some(AuthenticatedUuidIndexSnapshot::open_at_generation( - context.project_root, - context.prior.topology, - )?); - } - let (deleted_nodes, deleted_edges) = if let Some(index) = snapshot.as_mut() { - let (surrogates, _) = index.lookup_node_surrogates(&delta.deleted_nodes)?; - let nodes = delta - .deleted_nodes - .iter() - .copied() - .zip(surrogates) - .filter_map(|(uuid, surrogate)| surrogate.map(|id| (uuid, id))) - .collect::>(); - let (present, _) = index.probe(UuidIndexKind::Edge, &delta.deleted_edges)?; - let edges = delta - .deleted_edges - .iter() - .copied() - .zip(present) - .filter_map(|(uuid, present)| present.then_some(uuid)) - .collect::>(); - (nodes, edges) - } else { - (Vec::new(), Vec::new()) - }; - let mut token = super::prepare_uuid_membership_delta( - context.project_root, - context.prior.topology, - context.next.topology, - snapshot.as_mut(), - batch, - &delta.nodes, - &delta.edges, - &deleted_nodes, - &deleted_edges, - )?; - if let Some(token) = token.as_mut() { - token.metrics.orphan_gc_candidates = orphan_gc.candidates; - token.metrics.orphan_gc_removed = orphan_gc.removed; - token.metrics.orphan_gc_deferred = orphan_gc.deferred; - token.metrics.orphan_gc_deferred_limit = orphan_gc.deferred_limit; - token.metrics.orphan_gc_deferred_linked = orphan_gc.deferred_linked; - token.metrics.orphan_gc_bytes = orphan_gc.bytes; - } - let receipt = token - .as_ref() - .map(PreparedUuidIndexDelta::auxiliary_receipt); let mut prepared_ordinal = ordinal_inputs .as_ref() .map(|pinned| { @@ -270,7 +257,7 @@ pub(crate) fn commit_uuid_topology_rewrite( &delta.nodes, &delta.edges, &deleted_nodes, - &deleted_edges, + &delta.deleted_edges, ), ) }) @@ -283,26 +270,13 @@ pub(crate) fn commit_uuid_topology_rewrite( } let receipt = prepared_ordinal .as_ref() - .map(PreparedV4OrdinalDelta::auxiliary_receipt) - .or(receipt); - if token.is_some() - && let Some(value) = snapshot.as_mut() - { - value.suspend_owned_manifest(); - } - *prepared_from_callback.borrow_mut() = token; + .map(PreparedV4OrdinalDelta::auxiliary_receipt); *prepared_v4_from_callback.borrow_mut() = prepared_ordinal; - #[cfg(test)] - if FAIL_AFTER_MANIFEST_SUSPEND.replace(false) { - return Err(storage_err( - "injected manifest suspension preparation error", - )); - } Ok(receipt) }); // Retain this batch's exact membership before the rewrite consumes it. - // UUID participants stage only their reserved namespace; a reconciled - // durable commit must install the same topology as ordinary success. + // A reconciled durable commit must install the same topology as ordinary + // success. let topology = staged.topology_authority().cloned(); let topology_candidate = topology .as_ref() @@ -311,25 +285,28 @@ pub(crate) fn commit_uuid_topology_rewrite( let mut reconciled = false; let commit = crate::generation::commit_topology_aware_with_participant(staged, &root, participant); - let token = prepared.borrow_mut().take(); let v4_token = prepared_v4.borrow_mut().take(); let committed = match commit { Ok(value) => value, Err(error) => { - let outcome = (|| { - let (Some(token), Some((prior, next))) = (token.as_ref(), generations.get()) else { + *probe = None; + let outcome = (|| -> Result<_, GfError> { + let Some((prior, next)) = generations.get() else { return Ok(None); }; - let membership = reconcile_uuid_auxiliary(&root, prior, next, token)?; - if let Some(v4) = v4_token.as_ref() { - let ordinal = reconcile_v4_ordinal_auxiliary(&root, prior, next, v4)?; - if membership != ordinal { - return Err(storage_err( - "v3 and v4 auxiliary reconciliation outcomes disagree", - )); + // The v4 receipt names the exact outcome. A root without the + // ordinal facet carries no receipt, so a rewrite that advanced + // the topology is decided by the generation state under the + // rewrite lock. A rewrite that left the topology generation + // alone changes no identity and is not reconciled here. + let outcome = match v4_token.as_ref() { + Some(v4) => reconcile_v4_ordinal_auxiliary(&root, prior, next, v4)?, + None if next.topology == prior.topology => return Ok(None), + None => { + crate::durable_rewrite::reconcile_generation_transition(&root, prior, next)? } - } - Ok(Some((membership, next.topology))) + }; + Ok(Some((outcome, next.topology))) })(); match outcome { Ok(Some(( @@ -340,93 +317,46 @@ pub(crate) fn commit_uuid_topology_rewrite( Some(generation) } Ok(Some((crate::durable_rewrite::AuxiliaryReconcileOutcome::NotCommitted, _))) => { - if let Some(value) = snapshot.as_mut() - && let Err(restore) = value.restore_owned_manifest() - { - *snapshot = None; - return Err(storage_err(format!( - "{error}; UUID snapshot restoration failed: {restore}" - ))); - } return Err(error); } Err(reconcile) => { - if snapshot - .as_ref() - .is_some_and(|value| value.manifest_file.is_none()) - { - *snapshot = None; - } return Err(storage_err(format!( - "{error}; UUID reconciliation failed: {reconcile}" + "{error}; identity reconciliation failed: {reconcile}" ))); } - Ok(None) => { - if snapshot - .as_ref() - .is_some_and(|value| value.manifest_file.is_none()) - { - *snapshot = None; - } - return Err(error); - } + Ok(None) => return Err(error), } } }; - let mut committed_metrics = UuidIndexAppendMetrics::default(); - let committed_v4_metrics = v4_token.as_ref().map(|token| token.metrics().clone()); - if let (Some(generation), Some(token)) = (committed, token.as_ref()) { - if let Err(error) = token.verify_generation(generation).and_then(|()| { - v4_token - .as_ref() - .map_or(Ok(()), |v4| v4.verify_generation(generation)) - }) { - *snapshot = None; - return Err(error); + let committed_v4_metrics = v4_token + .as_ref() + .map(|token| Box::new(token.metrics().clone())); + if let Some(generation) = committed { + *probe = None; + if let Some(v4) = v4_token.as_ref() { + v4.verify_generation(generation)?; } if reconciled && let (Some(topology), Some(candidate)) = (topology, topology_candidate) { topology.install(candidate); } - committed_metrics = token.metrics().clone(); - let refresh = injected_snapshot_refresh_failure().map_or_else( - || { - if let Some(value) = snapshot.as_mut() { - token.advance_snapshot(value).map(|_| ()) - } else { - AuthenticatedUuidIndexSnapshot::open_at_generation(&root, generation) - .map(|value| *snapshot = Some(value)) - } - }, - Err, - ); - if let Err(error) = refresh { - *snapshot = None; - return Ok(CommittedUuidTopologyRewrite::CommittedNeedsRefresh { - generation, - metrics: committed_metrics, - v4_metrics: committed_v4_metrics, - error, - }); - } } Ok(committed.map_or( CommittedUuidTopologyRewrite::NoTopologyChange, |generation| CommittedUuidTopologyRewrite::Committed { generation, - metrics: committed_metrics, + probe: probe_metrics, v4_metrics: committed_v4_metrics, }, )) } -/// Commit a topology rewrite that changes no UUID membership while advancing -/// the authenticated membership manifest to the new topology generation. +/// Commit a topology rewrite that changes no UUID identity. pub(crate) fn commit_uuid_neutral_topology_rewrite( project_dir: &Path, staged: crate::staging::RewriteBatch, ) -> Result, GfError> { - let mut snapshot = None; - match commit_uuid_topology_rewrite( + let mut probe = None; + Ok(commit_uuid_topology_rewrite( project_dir, staged, &UuidTopologyDelta { @@ -435,140 +365,15 @@ pub(crate) fn commit_uuid_neutral_topology_rewrite( deleted_nodes: Vec::new(), deleted_edges: Vec::new(), }, - &mut snapshot, - )? { - CommittedUuidTopologyRewrite::NoTopologyChange => Ok(None), - CommittedUuidTopologyRewrite::Committed { generation, .. } => Ok(Some(generation)), - CommittedUuidTopologyRewrite::CommittedNeedsRefresh { - generation, error, .. - } => Err(GfError::Storage(format!( - "topology generation {generation} committed but UUID index snapshot refresh failed: {error}" - ))), - } -} - -/// Stage one bounded v3 UUID-index delta and its authenticated receipt into the -/// caller's generation-last topology rewrite transaction. -#[allow(clippy::too_many_arguments)] // Mirrors the four disjoint UUID delta domains. -pub(crate) fn prepare_uuid_membership_delta( - project_dir: &Path, - current: u64, - generation: u64, - snapshot: Option<&mut AuthenticatedUuidIndexSnapshot>, - batch: &mut crate::staging::RewriteBatch, - nodes: &[(Uuid, u64)], - edges: &[Uuid], - deleted_nodes: &[(Uuid, u64)], - deleted_edges: &[Uuid], -) -> Result, GfError> { - if generation != current.saturating_add(1) { - return Err(storage_err( - "prepared UUID delta generation is not the next generation", - )); - } - let source_root = project_dir.join(INDEX_DIR); - if current != 0 && !source_root.join(MANIFEST).is_file() { - return Err(storage_err( - "UUID membership index migration is required before topology mutation", - )); - } - fs::create_dir_all(&source_root).map_err(storage_err)?; - let parent = project_dir - .parent() - .ok_or_else(|| storage_err("project directory has no staging parent"))?; - let scratch = tempfile::Builder::new() - .prefix("uuid-membership-plan-") - .tempdir_in(parent) - .map_err(storage_err)?; - let (manifest, outputs, _superseded, metrics) = plan_uuid_membership_delta( - &source_root, - current, - generation, - snapshot, - scratch.path(), - nodes, - edges, - deleted_nodes, - deleted_edges, - )?; - for (record, path) in outputs { - batch.stage_file(&source_root.join(record.name), &path)?; - } - let manifest_bytes = serde_json::to_vec(&manifest).map_err(storage_err)?; - batch.stage_bytes(&source_root.join(MANIFEST), &manifest_bytes)?; - let nonce = Uuid::new_v4().simple().to_string(); - let receipt = TopologyIndexReceipt { - nonce, - expected_generation: generation, - topology_delta_sha256: topology_delta_sha256(nodes, edges, deleted_nodes, deleted_edges), - manifest_sha256: hex_sha256(&manifest_bytes), - }; - let receipt_bytes = serde_json::to_vec(&receipt).map_err(storage_err)?; - let receipt_path = source_root.join(TOPOLOGY_RECEIPT); - batch.stage_bytes(&receipt_path, &receipt_bytes)?; - let digest = Sha256::digest(&receipt_bytes); - Ok(Some(PreparedUuidIndexDelta { - expected_generation: generation, - metrics, - manifest, - auxiliary: crate::AuxiliaryReceipt { - kind: "uuid-membership/v7".to_owned(), - schema_version: FORMAT_VERSION, - path: format!("{INDEX_DIR}/{TOPOLOGY_RECEIPT}"), - digest: hex_bytes(&digest), - bytes: receipt_bytes.len() as u64, - }, - })) + &mut probe, + )? + .generation()) } pub(super) fn hex_sha256(bytes: &[u8]) -> String { hex_bytes(&Sha256::digest(bytes)) } -fn reconcile_uuid_auxiliary( - project_dir: &Path, - prior: crate::durable_rewrite::GenerationPair, - next: crate::durable_rewrite::GenerationPair, - prepared: &PreparedUuidIndexDelta, -) -> Result { - let auxiliary = prepared.auxiliary_receipt(); - let outcome = - crate::durable_rewrite::reconcile_auxiliary(project_dir, prior, next, &auxiliary)?; - if outcome == crate::durable_rewrite::AuxiliaryReconcileOutcome::NotCommitted { - return Ok(outcome); - } - let project = graphforge_filesystem::StableDirectory::open(project_dir).map_err(storage_err)?; - let topology = project - .open_child_directory(std::ffi::OsStr::new("topology")) - .map_err(storage_err)?; - let index = topology - .open_child_directory(std::ffi::OsStr::new("uuid-membership")) - .map_err(storage_err)?; - let mut receipt_file = index - .open_child_file(std::ffi::OsStr::new(TOPOLOGY_RECEIPT)) - .map_err(storage_err)?; - let receipt_body = read_bounded(&mut receipt_file, MAX_MANIFEST_BYTES)?; - let receipt: TopologyIndexReceipt = - serde_json::from_slice(&receipt_body).map_err(storage_err)?; - let mut manifest_file = index - .open_child_file(std::ffi::OsStr::new(MANIFEST)) - .map_err(storage_err)?; - let manifest_body = read_bounded(&mut manifest_file, MAX_MANIFEST_BYTES)?; - project.revalidate_named().map_err(storage_err)?; - topology.revalidate_named().map_err(storage_err)?; - index.revalidate_named().map_err(storage_err)?; - if receipt.expected_generation != next.topology - || receipt.manifest_sha256 != hex_sha256(&manifest_body) - || receipt.manifest_sha256 - != hex_sha256(&serde_json::to_vec(&prepared.manifest).map_err(storage_err)?) - { - return Err(storage_err( - "committed UUID receipt does not authenticate the expected manifest", - )); - } - Ok(outcome) -} - fn reconcile_v4_ordinal_auxiliary( project_dir: &Path, prior: crate::durable_rewrite::GenerationPair, @@ -615,526 +420,6 @@ fn reconcile_v4_ordinal_auxiliary( Ok(outcome) } -#[allow(clippy::too_many_arguments)] -#[allow(clippy::type_complexity, clippy::too_many_lines)] // Pure planner returns its authenticated transaction bundle. -pub(super) fn plan_uuid_membership_delta( - root: &Path, - current: u64, - generation: u64, - mut snapshot: Option<&mut AuthenticatedUuidIndexSnapshot>, - scratch: &Path, - nodes: &[(Uuid, u64)], - edges: &[Uuid], - deleted_nodes: &[(Uuid, u64)], - deleted_edges: &[Uuid], -) -> Result< - ( - Manifest, - Vec<(FileRecord, PathBuf)>, - Vec<(String, graphforge_filesystem::FileIdentity)>, - UuidIndexAppendMetrics, - ), - GfError, -> { - let mut manifest = if let Some(retained) = snapshot.as_deref_mut() { - retained.revalidate()?; - if retained.manifest.current_generation != current { - return Err(storage_err("retained manifest generation is stale")); - } - retained.manifest.clone() - } else if current == 0 { - Manifest { - format_version: FORMAT_VERSION, - base_generation: 0, - current_generation: 0, - live_node_count: 0, - live_edge_count: 0, - runs: Vec::new(), - } - } else { - return Err(storage_err("authenticated UUID snapshot is required")); - }; - - let prior_names = manifest_file_names(&manifest); - let mut identities = nodes - .iter() - .map(|(uuid, id)| (*uuid, 0_u8, *id)) - .chain(edges.iter().map(|uuid| (*uuid, 1_u8, 0))) - .chain(deleted_nodes.iter().map(|(uuid, id)| (*uuid, 2_u8, *id))) - .chain(deleted_edges.iter().map(|uuid| (*uuid, 3_u8, 0))) - .collect::>(); - identities.sort_unstable_by_key(|entry| *entry.0.as_bytes()); - if identities.windows(2).any(|pair| pair[0].0 == pair[1].0) - || nodes.iter().any(|(_, id)| *id == 0) - { - return Err(storage_err( - "new identity run contains duplicate/invalid identity", - )); - } - let mut surrogates = nodes - .iter() - .map(|(uuid, id)| (*id, *uuid)) - .chain(deleted_nodes.iter().map(|(uuid, id)| (*id, *uuid))) - .collect::>(); - surrogates.sort_unstable(); - if surrogates.windows(2).any(|pair| pair[0].0 == pair[1].0) { - return Err(storage_err("new node run contains duplicate surrogate")); - } - - let (retained_authentication_bytes, retained_authentication_blocks) = - snapshot.as_deref_mut().map_or( - (0, 0), - AuthenticatedUuidIndexSnapshot::take_authentication_work, - ); - let mut validation_metrics = UuidIndexAppendMetrics::default(); - if let Some(retained) = snapshot.as_deref_mut() { - for run in &mut retained.runs { - reject_retained_identity_collisions(run, &identities, &mut validation_metrics)?; - reject_retained_surrogate_collisions(run, &surrogates, &mut validation_metrics)?; - } - } - - let identity_path = scratch.join("identities-l0.run"); - let surrogate_path = scratch.join("surrogates-l0.run"); - let identity_write_blocks = write_identity_records(&identity_path, &identities)?; - let surrogate_write_blocks = write_surrogate_records(&surrogate_path, &surrogates)?; - let identity_record = describe_run( - &identity_path, - "identities-v5", - generation, - IDENTITY_RECORD_BYTES, - )?; - let surrogate_record = describe_run( - &surrogate_path, - "node-surrogates-v5", - generation, - NODE_LOOKUP_RECORD_BYTES, - )?; - let mut sources = HashMap::from([ - (identity_record.name.clone(), identity_path), - (surrogate_record.name.clone(), surrogate_path), - ]); - if current == 0 && manifest.runs.is_empty() { - let empty_identity_path = scratch.join("identities-base.run"); - let empty_surrogate_path = scratch.join("surrogates-base.run"); - create_uuid_file(&empty_identity_path)?; - create_uuid_file(&empty_surrogate_path)?; - let base_identities = describe_run( - &empty_identity_path, - "identities-v5-base", - 0, - IDENTITY_RECORD_BYTES, - )?; - let base_surrogates = describe_run( - &empty_surrogate_path, - "node-surrogates-v5-base", - 0, - NODE_LOOKUP_RECORD_BYTES, - )?; - sources.insert(base_identities.name.clone(), empty_identity_path); - sources.insert(base_surrogates.name.clone(), empty_surrogate_path); - manifest.runs.push(RunRecord { - base: true, - level: 0, - first_generation: 0, - last_generation: 0, - identities: base_identities, - node_surrogates: base_surrogates, - node_count: 0, - edge_count: 0, - deleted_node_count: 0, - deleted_edge_count: 0, - }); - } - manifest.runs.push(RunRecord { - base: false, - level: 0, - first_generation: generation, - last_generation: generation, - identities: identity_record, - node_surrogates: surrogate_record, - node_count: nodes.len() as u64, - edge_count: edges.len() as u64, - deleted_node_count: deleted_nodes.len() as u64, - deleted_edge_count: deleted_edges.len() as u64, - }); - let mut metrics = UuidIndexAppendMetrics { - input_records: identities.len() as u64, - physical_bytes_written: identities - .iter() - .map(|(_, kind, _)| if *kind == 1 { 17_u64 } else { 25_u64 }) - .sum::() - + surrogates.len() as u64 * NODE_LOOKUP_RECORD_BYTES, - write_bytes: identities - .iter() - .map(|(_, kind, _)| if *kind == 1 { 17_u64 } else { 25_u64 }) - .sum::() - + surrogates.len() as u64 * NODE_LOOKUP_RECORD_BYTES, - write_blocks: identity_write_blocks + surrogate_write_blocks, - peak_buffered_records: identities.len() + surrogates.len(), - peak_buffered_bytes: identities.len() * 32 + surrogates.len() * 24, - validation_random_seeks: 0, - validation_scan_bytes: validation_metrics.validation_scan_bytes, - validation_scan_blocks: validation_metrics.validation_scan_blocks, - snapshot_admission_authentication_bytes: retained_authentication_bytes, - snapshot_admission_authentication_blocks: retained_authentication_blocks, - ..Default::default() - }; - compact_planned_levels( - root, - scratch, - &mut manifest, - &mut sources, - snapshot.as_deref(), - &mut metrics, - )?; - manifest.current_generation = generation; - manifest.live_node_count = manifest - .live_node_count - .checked_add(nodes.len() as u64) - .and_then(|v| v.checked_sub(deleted_nodes.len() as u64)) - .ok_or_else(|| storage_err("node live-count delta is invalid"))?; - manifest.live_edge_count = manifest - .live_edge_count - .checked_add(edges.len() as u64) - .and_then(|v| v.checked_sub(deleted_edges.len() as u64)) - .ok_or_else(|| storage_err("edge live-count delta is invalid"))?; - manifest - .runs - .sort_unstable_by_key(|run| run.first_generation); - validate_run_descriptors(&manifest)?; - let retained = manifest_file_names(&manifest); - let mut outputs = sources - .into_iter() - .filter(|(name, _)| retained.contains(name)) - .map(|(name, path)| { - let record = manifest - .runs - .iter() - .flat_map(|run| [&run.identities, &run.node_surrogates]) - .find(|record| record.name == name) - .expect("planned output is retained") - .clone(); - (record, path) - }) - .collect::>(); - outputs.sort_unstable_by(|left, right| left.0.name.cmp(&right.0.name)); - metrics.new_output_authentication_bytes = outputs - .iter() - .map(|(record, _)| { - record - .blocks - .iter() - .map(|block| u64::from(block.len)) - .sum::() - }) - .sum(); - metrics.new_output_authentication_blocks = outputs - .iter() - .map(|(record, _)| record.blocks.len() as u64) - .sum(); - let mut superseded = Vec::new(); - if let Some(snapshot) = snapshot.as_deref() { - for name in prior_names.difference(&retained) { - let file = open_uuid_child_file(&snapshot.root, std::ffi::OsStr::new(name))?; - superseded.push(( - name.clone(), - graphforge_filesystem::file_identity(&file).map_err(storage_err)?, - )); - } - } - metrics.retained_runs = manifest.runs.len(); - Ok((manifest, outputs, superseded, metrics)) -} - -fn compact_planned_levels( - root: &Path, - scratch: &Path, - manifest: &mut Manifest, - sources: &mut HashMap, - snapshot: Option<&AuthenticatedUuidIndexSnapshot>, - metrics: &mut UuidIndexAppendMetrics, -) -> Result<(), GfError> { - for level in 0..63_u8 { - let mut indexes = manifest - .runs - .iter() - .enumerate() - .filter(|(_, run)| !run.base && run.level == level) - .map(|(index, _)| index) - .collect::>(); - if indexes.len() < 2 { - continue; - } - if indexes.len() != 2 { - return Err(storage_err( - "manifest has more than one retained run at a level", - )); - } - indexes.sort_unstable_by_key(|index| manifest.runs[*index].first_generation); - let right = manifest.runs.remove(indexes[1]); - let left = manifest.runs.remove(indexes[0]); - if left.last_generation.saturating_add(1) != right.first_generation { - return Err(storage_err("equal-level runs are not adjacent")); - } - let identity_path = scratch.join(format!("identities-level-{}.run", level + 1)); - let surrogate_path = scratch.join(format!("surrogates-level-{}.run", level + 1)); - let identity_inputs = [ - ( - planned_file(root, sources, snapshot, &left.identities.name, true)?, - left.identities.clone(), - ), - ( - planned_file(root, sources, snapshot, &right.identities.name, true)?, - right.identities.clone(), - ), - ]; - let surrogate_inputs = [ - ( - planned_file(root, sources, snapshot, &left.node_surrogates.name, false)?, - left.node_surrogates.clone(), - ), - ( - planned_file(root, sources, snapshot, &right.node_surrogates.name, false)?, - right.node_surrogates.clone(), - ), - ]; - merge_identity_handles(identity_inputs, &identity_path, metrics)?; - merge_surrogate_handles(surrogate_inputs, &surrogate_path, metrics)?; - let identities = describe_run( - &identity_path, - &format!("identities-v5-l{}", level + 1), - right.last_generation, - IDENTITY_RECORD_BYTES, - )?; - let node_surrogates = describe_run( - &surrogate_path, - &format!("node-surrogates-v5-l{}", level + 1), - right.last_generation, - NODE_LOOKUP_RECORD_BYTES, - )?; - let bytes = record_length(&identities, IDENTITY_RECORD_BYTES)? - + node_surrogates.count * NODE_LOOKUP_RECORD_BYTES; - metrics.physical_bytes_written = metrics.physical_bytes_written.saturating_add(bytes); - metrics.write_bytes = metrics.write_bytes.saturating_add(bytes); - metrics.write_blocks = metrics - .write_blocks - .saturating_add(bytes.div_ceil(BULK_IO_BYTES as u64)); - let counts = count_identity_states(&identity_path)?; - sources.insert(identities.name.clone(), identity_path); - sources.insert(node_surrogates.name.clone(), surrogate_path); - manifest.runs.push(RunRecord { - base: false, - level: level + 1, - first_generation: left.first_generation, - last_generation: right.last_generation, - identities, - node_surrogates, - node_count: counts.0, - edge_count: counts.1, - deleted_node_count: counts.2, - deleted_edge_count: counts.3, - }); - } - Ok(()) -} - -fn planned_file( - root: &Path, - sources: &HashMap, - snapshot: Option<&AuthenticatedUuidIndexSnapshot>, - name: &str, - identities: bool, -) -> Result { - if let Some(path) = sources.get(name) { - return open_uuid_file(path); - } - if let Some(snapshot) = snapshot { - for run in &snapshot.runs { - if identities && run.descriptor.identities.name == name { - return run.identities.try_clone().map_err(storage_err); - } - if !identities && run.descriptor.node_surrogates.name == name { - return run.node_surrogates.try_clone().map_err(storage_err); - } - } - return Err(storage_err("planned compaction input is not retained")); - } - open_uuid_file(&root.join(name)) -} - -struct VerifiedBlockReader { - file: File, - blocks: Vec, - next: usize, - bytes: Vec, - cursor: usize, - authenticated_bytes: u64, - authenticated_blocks: u64, - width: usize, -} - -impl VerifiedBlockReader { - fn new(file: File, record: &FileRecord, width: u64) -> Result { - validate_block_records(record, width)?; - Ok(Self { - file, - blocks: record.blocks.clone(), - next: 0, - bytes: Vec::new(), - cursor: 0, - authenticated_bytes: 0, - authenticated_blocks: 0, - width: usize::try_from(width) - .map_err(|_| storage_err("record width does not fit address space"))?, - }) - } -} - -impl Read for VerifiedBlockReader { - fn read(&mut self, output: &mut [u8]) -> std::io::Result { - if self.cursor == self.bytes.len() { - let Some(block) = self.blocks.get(self.next) else { - return Ok(0); - }; - self.file.seek(SeekFrom::Start(block.offset))?; - self.bytes.resize(block.len as usize, 0); - self.file.read_exact(&mut self.bytes)?; - if !block_matches(&self.bytes, block, self.width) { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "UUID compaction block authentication failed", - )); - } - self.next += 1; - self.cursor = 0; - self.authenticated_bytes = self - .authenticated_bytes - .saturating_add(self.bytes.len() as u64); - self.authenticated_blocks = self.authenticated_blocks.saturating_add(1); - } - let available = &self.bytes[self.cursor..]; - let copied = available.len().min(output.len()); - output[..copied].copy_from_slice(&available[..copied]); - self.cursor += copied; - Ok(copied) - } -} - -fn merge_identity_handles( - inputs: [(File, FileRecord); 2], - output: &Path, - metrics: &mut UuidIndexAppendMetrics, -) -> Result<(), GfError> { - let mut readers = inputs - .into_iter() - .map(|(file, record)| VerifiedBlockReader::new(file, &record, IDENTITY_RECORD_BYTES)) - .collect::, _>>()?; - let mut heap = BinaryHeap::>::new(); - for (index, reader) in readers.iter_mut().enumerate() { - if let Some(record) = identity_codec::read(reader)? { - heap.push(Reverse((record, index))); - } - } - let mut out = create_uuid_file(output)?; - let mut block = Vec::with_capacity(BULK_IO_BYTES); - while let Some(Reverse((mut record, index))) = heap.pop() { - let key: [u8; 16] = record[..16].try_into().expect("fixed"); - let mut newest = index; - if let Some(next) = identity_codec::read(&mut readers[index])? { - heap.push(Reverse((next, index))); - } - while heap - .peek() - .is_some_and(|Reverse((candidate, _))| candidate[..16] == key) - { - let Reverse((candidate, source)) = heap.pop().expect("peeked"); - if source > newest { - record = candidate; - newest = source; - } - if let Some(next) = identity_codec::read(&mut readers[source])? { - heap.push(Reverse((next, source))); - } - } - if block.len() + IDENTITY_RECORD_WIDTH > BULK_IO_BYTES { - out.write_all(&block).map_err(storage_err)?; - block.clear(); - } - block.extend_from_slice(identity_codec::encoded(&record)?); - } - if !block.is_empty() { - out.write_all(&block).map_err(storage_err)?; - } - out.flush().map_err(storage_err)?; - for reader in readers { - metrics.validation_scan_bytes = metrics - .validation_scan_bytes - .saturating_add(reader.authenticated_bytes); - metrics.validation_scan_blocks = metrics - .validation_scan_blocks - .saturating_add(reader.authenticated_blocks); - } - Ok(()) -} - -fn merge_surrogate_handles( - inputs: [(File, FileRecord); 2], - output: &Path, - metrics: &mut UuidIndexAppendMetrics, -) -> Result<(), GfError> { - let mut readers = inputs - .into_iter() - .map(|(file, record)| VerifiedBlockReader::new(file, &record, NODE_LOOKUP_RECORD_BYTES)) - .collect::, _>>()?; - let mut heap = BinaryHeap::>::new(); - for (index, reader) in readers.iter_mut().enumerate() { - if let Some(record) = read_exact_record::<24>(reader)? { - heap.push(Reverse((record, index))); - } - } - let mut out = create_uuid_file(output)?; - let mut block = Vec::with_capacity(BULK_IO_BYTES); - while let Some(Reverse((mut record, index))) = heap.pop() { - let key: [u8; 8] = record[..8].try_into().expect("fixed"); - let mut newest = index; - if let Some(next) = read_exact_record::<24>(&mut readers[index])? { - heap.push(Reverse((next, index))); - } - while heap - .peek() - .is_some_and(|Reverse((candidate, _))| candidate[..8] == key) - { - let Reverse((candidate, source)) = heap.pop().expect("peeked"); - if source > newest { - record = candidate; - newest = source; - } - if let Some(next) = read_exact_record::<24>(&mut readers[source])? { - heap.push(Reverse((next, source))); - } - } - if block.len() + 24 > BULK_IO_BYTES { - out.write_all(&block).map_err(storage_err)?; - block.clear(); - } - block.extend_from_slice(&record); - } - if !block.is_empty() { - out.write_all(&block).map_err(storage_err)?; - } - out.flush().map_err(storage_err)?; - for reader in readers { - metrics.validation_scan_bytes = metrics - .validation_scan_bytes - .saturating_add(reader.authenticated_bytes); - metrics.validation_scan_blocks = metrics - .validation_scan_blocks - .saturating_add(reader.authenticated_blocks); - } - Ok(()) -} - pub(super) fn topology_delta_sha256( nodes: &[(Uuid, u64)], edges: &[Uuid], @@ -1192,520 +477,6 @@ pub(super) fn read_bounded( Ok(body) } -/// Publish one committed topology batch as an immutable authenticated v3 run. -#[cfg(test)] -pub(crate) fn append_uuid_membership_delta( - project_dir: &Path, - generation: u64, - nodes: &[(Uuid, u64)], - edges: &[Uuid], -) -> Result { - append_uuid_membership_delta_with_tombstones(project_dir, generation, nodes, edges, &[], &[]) -} - -#[cfg(test)] -pub(super) fn append_uuid_membership_delta_with_tombstones( - project_dir: &Path, - generation: u64, - nodes: &[(Uuid, u64)], - edges: &[Uuid], - deleted_nodes: &[(Uuid, u64)], - deleted_edges: &[Uuid], -) -> Result { - if crate::read_topology_generation(project_dir)? != generation { - return Err(storage_err( - "topology generation changed before index append", - )); - } - let root = project_dir.join(INDEX_DIR); - fs::create_dir_all(&root).map_err(storage_err)?; - let staging = project_dir - .parent() - .ok_or_else(|| storage_err("project directory has no staging parent"))?; - let mut manifest: Manifest = match fs::read(root.join(MANIFEST)) { - Ok(body) => serde_json::from_slice(&body).map_err(storage_err)?, - Err(error) if error.kind() == std::io::ErrorKind::NotFound && generation == 1 => { - let scratch = tempfile::Builder::new() - .prefix("uuid-v3-empty-") - .tempdir_in(staging) - .map_err(storage_err)?; - let empty = scratch.path().join("empty.run"); - File::create(&empty).map_err(storage_err)?; - let identities = publish_data( - &empty, - &root, - staging, - "identities-v5-base", - 0, - IDENTITY_RECORD_BYTES, - )?; - let node_surrogates = publish_data( - &empty, - &root, - staging, - "node-surrogates-v5-base", - 0, - NODE_LOOKUP_RECORD_BYTES, - )?; - Manifest { - format_version: FORMAT_VERSION, - base_generation: 0, - current_generation: 0, - live_node_count: 0, - live_edge_count: 0, - runs: vec![RunRecord { - base: true, - level: 0, - first_generation: 0, - last_generation: 0, - identities, - node_surrogates, - node_count: 0, - edge_count: 0, - deleted_node_count: 0, - deleted_edge_count: 0, - }], - } - } - Err(error) => return Err(storage_err(error)), - }; - if manifest.format_version != FORMAT_VERSION || manifest.current_generation + 1 != generation { - return Err(storage_err( - "index append is not a v3 generation continuation", - )); - } - validate_run_descriptors(&manifest)?; - let prior_files = manifest_file_names(&manifest); - let mut open_runs = Vec::new(); - for descriptor in &manifest.runs { - let identities = open_verified(&root, &descriptor.identities, IDENTITY_RECORD_BYTES)?; - let node_surrogates = - open_verified(&root, &descriptor.node_surrogates, NODE_LOOKUP_RECORD_BYTES)?; - validate_run_contents( - identities.try_clone().map_err(storage_err)?, - node_surrogates.try_clone().map_err(storage_err)?, - descriptor, - )?; - open_runs.push(OpenRun { - identities, - node_surrogates, - descriptor: descriptor.clone(), - }); - } - let mut identities = nodes - .iter() - .map(|(uuid, surrogate)| (*uuid, 0_u8, *surrogate)) - .chain(edges.iter().map(|uuid| (*uuid, 1_u8, 0))) - .chain( - deleted_nodes - .iter() - .map(|(uuid, surrogate)| (*uuid, 2_u8, *surrogate)), - ) - .chain(deleted_edges.iter().map(|uuid| (*uuid, 3_u8, 0))) - .collect::>(); - identities.sort_unstable_by_key(|entry| *entry.0.as_bytes()); - if identities.windows(2).any(|pair| pair[0].0 == pair[1].0) - || nodes.iter().any(|(_, surrogate)| *surrogate == 0) - { - return Err(storage_err( - "new identity run contains duplicate/invalid identity", - )); - } - let mut surrogate_keys = nodes - .iter() - .map(|(uuid, surrogate)| (*surrogate, *uuid)) - .chain( - deleted_nodes - .iter() - .map(|(uuid, surrogate)| (*surrogate, *uuid)), - ) - .collect::>(); - surrogate_keys.sort_unstable(); - if surrogate_keys.windows(2).any(|pair| pair[0].0 == pair[1].0) { - return Err(storage_err("new node run contains duplicate surrogate")); - } - let mut validation_bytes = 0_u64; - let mut validation_blocks = 0_u64; - for run in &open_runs { - let identity_bytes = reject_identity_collisions( - BufReader::with_capacity( - BULK_IO_BYTES, - File::open(root.join(&run.descriptor.identities.name)).map_err(storage_err)?, - ), - &identities, - )?; - validation_bytes = validation_bytes.saturating_add(identity_bytes); - validation_blocks = - validation_blocks.saturating_add(identity_bytes.div_ceil(BULK_IO_BYTES as u64)); - let surrogate_bytes = reject_surrogate_collisions( - BufReader::with_capacity( - BULK_IO_BYTES, - File::open(root.join(&run.descriptor.node_surrogates.name)).map_err(storage_err)?, - ), - &surrogate_keys, - )?; - validation_bytes = validation_bytes.saturating_add(surrogate_bytes); - validation_blocks = - validation_blocks.saturating_add(surrogate_bytes.div_ceil(BULK_IO_BYTES as u64)); - } - let scratch = tempfile::Builder::new() - .prefix("uuid-v3-append-") - .tempdir_in(staging) - .map_err(storage_err)?; - let identity_path = scratch.path().join("identities.run"); - let surrogate_path = scratch.path().join("surrogates.run"); - let identity_write_blocks = write_identity_records(&identity_path, &identities)?; - let surrogate_write_blocks = write_surrogate_records(&surrogate_path, &surrogate_keys)?; - let identity_record = publish_data( - &identity_path, - &root, - staging, - "identities-v5", - generation, - IDENTITY_RECORD_BYTES, - )?; - let surrogate_record = publish_data( - &surrogate_path, - &root, - staging, - "node-surrogates-v5", - generation, - NODE_LOOKUP_RECORD_BYTES, - )?; - manifest.runs.push(RunRecord { - base: false, - level: 0, - first_generation: generation, - last_generation: generation, - identities: identity_record, - node_surrogates: surrogate_record, - node_count: nodes.len() as u64, - edge_count: edges.len() as u64, - deleted_node_count: deleted_nodes.len() as u64, - deleted_edge_count: deleted_edges.len() as u64, - }); - let mut metrics = UuidIndexAppendMetrics { - input_records: identities.len() as u64, - physical_bytes_written: identities - .iter() - .map(|(_, kind, _)| if *kind == 1 { 17_u64 } else { 25_u64 }) - .sum::() - + surrogate_keys.len() as u64 * NODE_LOOKUP_RECORD_BYTES, - write_blocks: identity_write_blocks + surrogate_write_blocks, - write_bytes: identities - .iter() - .map(|(_, kind, _)| if *kind == 1 { 17_u64 } else { 25_u64 }) - .sum::() - + surrogate_keys.len() as u64 * NODE_LOOKUP_RECORD_BYTES, - peak_buffered_records: identities.len() + surrogate_keys.len(), - peak_buffered_bytes: identities.len() * 32 + surrogate_keys.len() * 24, - validation_scan_bytes: validation_bytes, - validation_scan_blocks: validation_blocks, - ..Default::default() - }; - compact_manifest_levels(&root, staging, scratch.path(), &mut manifest, &mut metrics)?; - manifest.current_generation = generation; - manifest.live_node_count = manifest - .live_node_count - .checked_add(nodes.len() as u64) - .and_then(|count| count.checked_sub(deleted_nodes.len() as u64)) - .ok_or_else(|| storage_err("node live-count delta is invalid"))?; - manifest.live_edge_count = manifest - .live_edge_count - .checked_add(edges.len() as u64) - .and_then(|count| count.checked_sub(deleted_edges.len() as u64)) - .ok_or_else(|| storage_err("edge live-count delta is invalid"))?; - manifest - .runs - .sort_unstable_by_key(|run| run.first_generation); - publish_manifest(&root, staging, &manifest)?; - cleanup_superseded_files(&root, prior_files, &manifest)?; - metrics.retained_runs = manifest.runs.len(); - Ok(metrics) -} - -#[cfg(test)] -fn reject_identity_collisions( - mut retained: BufReader, - incoming: &[(Uuid, u8, u64)], -) -> Result { - let mut incoming_index = 0; - let mut bytes = 0_u64; - while incoming_index < incoming.len() { - let Some(record) = identity_codec::read(&mut retained)? else { - break; - }; - bytes += identity_codec::encoded(&record)?.len() as u64; - let retained_uuid = &record[..16]; - while incoming_index < incoming.len() - && incoming[incoming_index].0.as_bytes().as_slice() < retained_uuid - { - incoming_index += 1; - } - if incoming_index < incoming.len() - && incoming[incoming_index].0.as_bytes().as_slice() == retained_uuid - { - let incoming_record = incoming[incoming_index]; - let retained_kind = record[16]; - let retained_surrogate = u64::from_be_bytes(record[17..25].try_into().expect("fixed")); - if matches!(incoming_record.1, 2 | 3) - && incoming_record.1 - 2 == retained_kind - && incoming_record.2 == retained_surrogate - { - continue; - } - return Err(storage_err( - "UUID already exists in an authenticated retained run", - )); - } - } - Ok(bytes) -} - -#[cfg(test)] -fn reject_surrogate_collisions( - mut retained: BufReader, - incoming: &[(u64, Uuid)], -) -> Result { - let mut incoming_index = 0; - let mut bytes = 0_u64; - while incoming_index < incoming.len() { - let Some(record) = read_exact_record::<24>(&mut retained)? else { - break; - }; - bytes += NODE_LOOKUP_RECORD_BYTES; - let retained_surrogate = u64::from_be_bytes(record[..8].try_into().expect("fixed")); - while incoming_index < incoming.len() && incoming[incoming_index].0 < retained_surrogate { - incoming_index += 1; - } - if incoming_index < incoming.len() && incoming[incoming_index].0 == retained_surrogate { - if incoming[incoming_index].1.as_bytes() == &record[8..24] { - continue; - } - return Err(storage_err( - "node surrogate already exists in an authenticated retained run", - )); - } - } - Ok(bytes) -} - -pub(super) fn write_identity_records( - path: &Path, - records: &[(Uuid, u8, u64)], -) -> Result { - let mut blocks = 0; - let mut bytes = Vec::with_capacity(BULK_IO_BYTES); - let mut file = create_uuid_file(path)?; - for (uuid, kind, surrogate) in records { - let mut record = [0_u8; IDENTITY_RECORD_WIDTH]; - record[..16].copy_from_slice(uuid.as_bytes()); - record[16] = *kind; - record[17..].copy_from_slice(&surrogate.to_be_bytes()); - let encoded = identity_codec::encoded(&record)?; - if bytes.len() + encoded.len() > BULK_IO_BYTES { - file.write_all(&bytes).map_err(storage_err)?; - blocks += 1; - bytes.clear(); - } - bytes.extend_from_slice(encoded); - } - if !bytes.is_empty() { - file.write_all(&bytes).map_err(storage_err)?; - blocks += 1; - } - file.flush().map_err(storage_err)?; - Ok(blocks) -} - -fn write_surrogate_records(path: &Path, records: &[(u64, Uuid)]) -> Result { - let mut blocks = 0; - let mut bytes = Vec::with_capacity(BULK_IO_BYTES); - let mut file = create_uuid_file(path)?; - for (surrogate, uuid) in records { - if bytes.len() + 24 > BULK_IO_BYTES { - file.write_all(&bytes).map_err(storage_err)?; - blocks += 1; - bytes.clear(); - } - bytes.extend_from_slice(&surrogate.to_be_bytes()); - bytes.extend_from_slice(uuid.as_bytes()); - } - if !bytes.is_empty() { - file.write_all(&bytes).map_err(storage_err)?; - blocks += 1; - } - file.flush().map_err(storage_err)?; - Ok(blocks) -} - -#[cfg(test)] -fn publish_manifest(root: &Path, staging: &Path, manifest: &Manifest) -> Result<(), GfError> { - let _ = staging; - let directory = graphforge_filesystem::StableDirectory::open(root).map_err(storage_err)?; - let temp_name = std::ffi::OsString::from(format!(".manifest-{}.tmp", Uuid::new_v4())); - let mut temp = directory - .create_child_file(&temp_name) - .map_err(storage_err)?; - let identity = graphforge_filesystem::file_identity(&temp).map_err(storage_err)?; - let result = (|| -> Result<(), GfError> { - serde_json::to_writer(&mut temp, manifest).map_err(storage_err)?; - temp.flush().map_err(storage_err)?; - crate::durable_commit::seal_file(&temp).map_err(storage_err)?; - directory - .replace_child(&temp_name, identity, std::ffi::OsStr::new(MANIFEST)) - .map_err(storage_err)?; - crate::durable_commit::acknowledge_directory(&directory).map_err(storage_err) - })(); - if result.is_err() { - let _ = directory.unlink_child_if_identity(&temp_name, identity); - } - result -} - -#[cfg(test)] -fn compact_manifest_levels( - root: &Path, - staging: &Path, - scratch: &Path, - manifest: &mut Manifest, - metrics: &mut UuidIndexAppendMetrics, -) -> Result<(), GfError> { - for level in 0..63_u8 { - loop { - let mut indexes = manifest - .runs - .iter() - .enumerate() - .filter(|(_, run)| !run.base && run.level == level) - .map(|(index, _)| index) - .collect::>(); - if indexes.len() < 2 { - break; - } - indexes.sort_unstable_by_key(|index| manifest.runs[*index].first_generation); - let right = manifest.runs.remove(indexes[1]); - let left = manifest.runs.remove(indexes[0]); - if left.last_generation.saturating_add(1) != right.first_generation { - return Err(storage_err("equal-level runs are not adjacent")); - } - let identity_path = scratch.join(format!("identities-level-{}.run", level + 1)); - let surrogate_path = scratch.join(format!("surrogates-level-{}.run", level + 1)); - merge_identity_v3( - &[ - root.join(&left.identities.name), - root.join(&right.identities.name), - ], - &identity_path, - )?; - merge_surrogate_runs( - &[ - root.join(&left.node_surrogates.name), - root.join(&right.node_surrogates.name), - ], - &surrogate_path, - )?; - let identities = publish_data( - &identity_path, - root, - staging, - &format!("identities-v5-l{}", level + 1), - right.last_generation, - IDENTITY_RECORD_BYTES, - )?; - let node_surrogates = publish_data( - &surrogate_path, - root, - staging, - &format!("node-surrogates-v5-l{}", level + 1), - right.last_generation, - NODE_LOOKUP_RECORD_BYTES, - )?; - let bytes = record_length(&identities, IDENTITY_RECORD_BYTES)? - + node_surrogates.count * NODE_LOOKUP_RECORD_BYTES; - metrics.physical_bytes_written = metrics.physical_bytes_written.saturating_add(bytes); - metrics.write_bytes = metrics.write_bytes.saturating_add(bytes); - metrics.write_blocks = metrics - .write_blocks - .saturating_add(bytes.div_ceil(BULK_IO_BYTES as u64)); - let (node_count, edge_count, deleted_node_count, deleted_edge_count) = - count_identity_states(&root.join(&identities.name))?; - manifest.runs.push(RunRecord { - base: false, - level: level + 1, - first_generation: left.first_generation, - last_generation: right.last_generation, - identities, - node_surrogates, - node_count, - edge_count, - deleted_node_count, - deleted_edge_count, - }); - } - } - Ok(()) -} - -fn count_identity_states(path: &Path) -> Result<(u64, u64, u64, u64), GfError> { - let mut reader = - BufReader::with_capacity(BULK_IO_BYTES, File::open(path).map_err(storage_err)?); - let mut counts = [0_u64; 4]; - while let Some(record) = identity_codec::read(&mut reader)? { - let kind = usize::from(record[16]); - if kind >= counts.len() { - return Err(storage_err("invalid compacted identity kind")); - } - counts[kind] += 1; - } - Ok((counts[0], counts[1], counts[2], counts[3])) -} - -#[cfg(test)] -fn merge_identity_v3(inputs: &[PathBuf], output: &Path) -> Result<(), GfError> { - let mut readers = inputs - .iter() - .map(|path| File::open(path).map(BufReader::new).map_err(storage_err)) - .collect::, _>>()?; - let mut heap = BinaryHeap::>::new(); - for (index, reader) in readers.iter_mut().enumerate() { - if let Some(record) = identity_codec::read(reader)? { - heap.push(Reverse((record, index))); - } - } - let mut out = File::create(output).map_err(storage_err)?; - let mut block = Vec::with_capacity(BULK_IO_BYTES); - while let Some(Reverse((mut record, index))) = heap.pop() { - let uuid: [u8; 16] = record[..16].try_into().expect("fixed"); - let mut newest_index = index; - if let Some(next) = identity_codec::read(&mut readers[index])? { - heap.push(Reverse((next, index))); - } - while heap - .peek() - .is_some_and(|Reverse((candidate, _))| candidate[..16] == uuid) - { - let Reverse((candidate, candidate_index)) = heap.pop().expect("peeked"); - if candidate_index > newest_index { - record = candidate; - newest_index = candidate_index; - } - if let Some(next) = identity_codec::read(&mut readers[candidate_index])? { - heap.push(Reverse((next, candidate_index))); - } - } - if block.len() + IDENTITY_RECORD_WIDTH > BULK_IO_BYTES { - out.write_all(&block).map_err(storage_err)?; - block.clear(); - } - block.extend_from_slice(identity_codec::encoded(&record)?); - } - if !block.is_empty() { - out.write_all(&block).map_err(storage_err)?; - } - out.flush().map_err(storage_err) -} - /// Stage one incremental v4 node-ordinal delta beside the canonical topology /// mutation. The caller supplies a receipt-authenticated, lifetime-pinned prior /// snapshot and owns the enclosing generation-last rewrite transaction. diff --git a/crates/graphforge-storage/src/uuid_membership/topology_delta/tests.rs b/crates/graphforge-storage/src/uuid_membership/topology_delta/tests.rs index d43e58bae..081b62aac 100644 --- a/crates/graphforge-storage/src/uuid_membership/topology_delta/tests.rs +++ b/crates/graphforge-storage/src/uuid_membership/topology_delta/tests.rs @@ -1,28 +1,17 @@ -use super::super::AuthenticatedUuidIndexSnapshot; use super::super::CommittedUuidTopologyRewrite; use super::super::INDEX_DIR; -use super::super::MANIFEST; -use super::super::Manifest; use super::super::UuidIndexBuildLimits; -use super::super::UuidIndexKind; -use super::super::UuidMembershipIndex; use super::super::UuidTopologyDelta; use super::super::V4_ORDINAL_MANIFEST; -use super::super::maintenance::manifest_file_names; use super::super::maintenance::standalone_v4_pinned_update; use super::super::ordinal_artifacts::stage_v4_ordinal_artifacts; use super::super::ordinal_artifacts::stage_v4_ordinal_artifacts_unordered; -use super::super::rebuild::rebuild_uuid_membership_indexes; use super::super::rebuild::rebuild_v4_ordinal_identity; use super::super::tests::fixture; use super::super::tests::install_v4_plan; use super::super::tests::pinned_v4_update; -use super::super::tests::singleton_append_series; -use super::append_uuid_membership_delta; use super::commit_uuid_topology_rewrite; use super::hex_sha256; -use super::plan_uuid_membership_delta; -use super::prepare_uuid_membership_delta; use super::prepare_v4_ordinal_delta; use std::fs; use uuid::Uuid; @@ -415,7 +404,6 @@ fn standalone_existing_v4_advances_with_topology_transaction() { b"{\"topology_generation\":7,\"search_generation\":0,\"property_generation\":0}\n", ) .unwrap(); - rebuild_uuid_membership_indexes(dir.path(), UuidIndexBuildLimits::default()).unwrap(); rebuild_v4_ordinal_identity(dir.path(), UuidIndexBuildLimits::default()).unwrap(); let topology = dir.path().join("topology/nodes.parquet"); let mut staged = crate::staging::RewriteBatch::new(); @@ -449,187 +437,6 @@ fn standalone_existing_v4_advances_with_topology_transaction() { ); } -#[test] -fn v3_leveled_append_has_bounded_runs_and_nonquadratic_doubling() { - let (_, small) = singleton_append_series(64); - let (large, large_bytes) = singleton_append_series(128); - assert!(large_bytes <= small * 5 / 2); - let manifest: Manifest = - serde_json::from_slice(&fs::read(large.path().join(INDEX_DIR).join(MANIFEST)).unwrap()) - .unwrap(); - assert!(manifest.runs.len() <= 9); - assert_eq!(manifest.runs.iter().filter(|run| run.base).count(), 1); - let mut index = UuidMembershipIndex::open(large.path()).unwrap(); - assert_eq!(index.count(UuidIndexKind::Node), 128); - assert_eq!( - index - .lookup_node_surrogates(&[Uuid::from_u128(1), Uuid::from_u128(128)]) - .unwrap() - .0, - [Some(1), Some(128)] - ); -} - -#[test] -fn append_rejects_cross_run_uuid_and_surrogate_collisions_before_publication() { - let dir = tempfile::tempdir().unwrap(); - crate::generation::force_bump_topology_generation_for_test(dir.path()).unwrap(); - append_uuid_membership_delta( - dir.path(), - 1, - &[(Uuid::from_u128(1), 1)], - &[Uuid::from_u128(2)], - ) - .unwrap(); - let manifest_before = fs::read(dir.path().join(INDEX_DIR).join(MANIFEST)).unwrap(); - - crate::generation::force_bump_topology_generation_for_test(dir.path()).unwrap(); - assert!( - append_uuid_membership_delta(dir.path(), 2, &[], &[Uuid::from_u128(1)],) - .unwrap_err() - .to_string() - .contains("already exists") - ); - assert_eq!( - fs::read(dir.path().join(INDEX_DIR).join(MANIFEST)).unwrap(), - manifest_before - ); - assert!( - append_uuid_membership_delta(dir.path(), 2, &[(Uuid::from_u128(3), 1)], &[],) - .unwrap_err() - .to_string() - .contains("surrogate already exists") - ); - - let reverse = tempfile::tempdir().unwrap(); - crate::generation::force_bump_topology_generation_for_test(reverse.path()).unwrap(); - append_uuid_membership_delta(reverse.path(), 1, &[], &[Uuid::from_u128(9)]).unwrap(); - crate::generation::force_bump_topology_generation_for_test(reverse.path()).unwrap(); - assert!( - append_uuid_membership_delta(reverse.path(), 2, &[(Uuid::from_u128(9), 9)], &[]) - .unwrap_err() - .to_string() - .contains("already exists") - ); -} - -#[test] -fn bulk_append_validation_uses_sequential_megabyte_blocks_and_zero_random_seeks() { - let dir = tempfile::tempdir().unwrap(); - let retained = (1_u64..=40_000) - .map(|value| (Uuid::from_u128(u128::from(value)), value)) - .collect::>(); - crate::generation::force_bump_topology_generation_for_test(dir.path()).unwrap(); - append_uuid_membership_delta(dir.path(), 1, &retained, &[]).unwrap(); - - crate::generation::force_bump_topology_generation_for_test(dir.path()).unwrap(); - let metrics = - append_uuid_membership_delta(dir.path(), 2, &[(Uuid::from_u128(50_000), 50_000)], &[]) - .unwrap(); - assert_eq!(metrics.validation_random_seeks, 0); - assert_eq!(metrics.validation_scan_bytes, 40_000 * (25 + 24)); - assert_eq!(metrics.validation_scan_blocks, 2); -} - -#[test] -fn retained_planner_stages_only_new_and_binary_carry_outputs() { - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().join(INDEX_DIR); - let first_nodes = (1_u64..=40_000) - .map(|value| (Uuid::from_u128(u128::from(value)), value)) - .collect::>(); - - let mut first = crate::RewriteBatch::new(); - prepare_uuid_membership_delta( - dir.path(), - 0, - 1, - None, - &mut first, - &first_nodes, - &[], - &[], - &[], - ) - .unwrap(); - // Two empty base files, two L0 files, manifest, and receipt. No copy of - // any retained corpus exists on the initial plan. - assert_eq!(first.staged_paths().count(), 6); - first.commit_unsealed_for_test().unwrap(); - fs::create_dir_all(dir.path().join("topology")).unwrap(); - fs::write( - crate::generation::generation_path(dir.path()), - crate::generation::encode_generation_state(1, 1, 0).unwrap(), - ) - .unwrap(); - let before = manifest_file_names( - &serde_json::from_slice::(&fs::read(root.join(MANIFEST)).unwrap()).unwrap(), - ); - - let scratch = tempfile::tempdir_in(dir.path()).unwrap(); - let mut snapshot = AuthenticatedUuidIndexSnapshot::open_at_generation(dir.path(), 1).unwrap(); - let second_nodes = (40_001_u64..=80_000) - .map(|value| (Uuid::from_u128(u128::from(value)), value)) - .collect::>(); - let (planned, outputs, superseded, metrics) = plan_uuid_membership_delta( - &root, - 1, - 2, - Some(&mut snapshot), - scratch.path(), - &second_nodes, - &[], - &[], - &[], - ) - .unwrap(); - // Generation two carries L0+L0 into exactly one L1 pair. Retained base - // files are descriptor-reused, not copied into planner outputs. - assert_eq!(outputs.len(), 2); - assert_eq!(superseded.len(), 2); - assert_eq!(planned.runs.iter().filter(|run| !run.base).count(), 1); - assert_eq!(planned.runs.iter().find(|run| !run.base).unwrap().level, 1); - assert!( - before - .iter() - .all(|name| !outputs.iter().any(|(out, _)| &out.name == name)) - ); - assert!(metrics.validation_scan_bytes > 0); - assert_eq!(metrics.validation_random_seeks, 0); - assert_eq!(metrics.prior_topology_rows_decoded, 0); - assert!(metrics.snapshot_admission_authentication_bytes > 0); - assert!(metrics.validation_scan_bytes <= metrics.snapshot_admission_authentication_bytes * 2); - assert!(metrics.new_output_authentication_bytes <= metrics.physical_bytes_written); - - let mut install = crate::RewriteBatch::new(); - for (record, path) in &outputs { - install.stage_file(&root.join(&record.name), path).unwrap(); - } - install - .stage_bytes(&root.join(MANIFEST), &serde_json::to_vec(&planned).unwrap()) - .unwrap(); - install.commit_unsealed_for_test().unwrap(); - snapshot.advance_to(planned).unwrap(); - - let scratch = tempfile::tempdir_in(dir.path()).unwrap(); - let (_, _, _, subsequent) = plan_uuid_membership_delta( - &root, - 2, - 3, - Some(&mut snapshot), - scratch.path(), - &[(Uuid::from_u128(80_001), 80_001)], - &[], - &[], - &[], - ) - .unwrap(); - assert_eq!(subsequent.snapshot_admission_authentication_bytes, 0); - assert_eq!(subsequent.snapshot_admission_authentication_blocks, 0); - assert_eq!(subsequent.validation_scan_bytes, 0); - assert_eq!(subsequent.validation_scan_blocks, 0); -} - #[test] fn topology_delta_digest_accounts_canonical_logical_contract_bytes() { use graphforge_core::hash_observation::operation::{Capture, Snapshot}; diff --git a/crates/graphforge-storage/src/writer.rs b/crates/graphforge-storage/src/writer.rs index 51be71320..b2debe204 100644 --- a/crates/graphforge-storage/src/writer.rs +++ b/crates/graphforge-storage/src/writer.rs @@ -671,17 +671,22 @@ pub struct TopologyWriteWork { pub peak_buffered_bytes: u64, /// Conservative peak scratch bytes required while encoding a flush. pub peak_flush_scratch_bytes: u64, - /// Authenticated UUID-index block-positioning seeks for endpoint lookup. + /// Published-Parquet row groups positioned for endpoint lookup. pub uuid_block_seeks: u64, - /// Authenticated identity blocks read for endpoint lookup. + /// Published-Parquet row groups decoded for endpoint lookup. pub uuid_identity_blocks_read: u64, - /// Authenticated identity bytes read for endpoint lookup. + /// Compressed identity-column bytes decoded for endpoint lookup. pub uuid_identity_bytes_read: u64, - /// Authenticated reverse-surrogate blocks read for pair validation. + /// Identity-column pages of every row group considered by identity probes. + pub uuid_pages_considered: u64, + /// Identity-column pages decoded by identity probes after row-group and + /// page-index pruning. + pub uuid_pages_read: u64, + /// Always zero: node surrogates are decoded with their UUIDs. pub uuid_surrogate_blocks_read: u64, - /// Authenticated reverse-surrogate bytes read for pair validation. + /// Always zero: node surrogates are decoded with their UUIDs. pub uuid_surrogate_bytes_read: u64, - /// Immutable UUID runs considered with newest-run shadowing. + /// Published topology fragments whose row groups identity probes considered. pub uuid_runs_considered: u64, /// Per-record filesystem seeks. This remains zero for batched lookup. pub uuid_per_record_seeks: u64, @@ -699,9 +704,9 @@ pub struct TopologyWriteWork { pub uuid_peak_buffered_records: u64, /// Peak charged fixed-width UUID bytes buffered by a committed delta. pub uuid_peak_buffered_bytes: u64, - /// Retained UUID validation blocks read by committed deltas. + /// Published-Parquet row groups decoded by commit-time identity validation. pub uuid_validation_blocks: u64, - /// Retained UUID validation bytes read by committed deltas. + /// Compressed identity-column bytes decoded by commit-time identity validation. pub uuid_validation_bytes: u64, /// Per-record random seeks during UUID publication validation; always zero. pub uuid_validation_random_seeks: u64, @@ -790,8 +795,7 @@ pub struct GraphWriter { pending_delta: Vec, pending_index_nodes: Vec<(Uuid, u64)>, pending_index_edges: Vec, - uuid_index_snapshot: Option, - uuid_snapshot_refresh_needed: Option, + identity_probe: Option, limits: GraphWriterLimits, charged_topology_bytes: usize, buffered_topology_rows: usize, @@ -889,8 +893,7 @@ impl GraphWriter { pending_delta: Vec::new(), pending_index_nodes: Vec::new(), pending_index_edges: Vec::new(), - uuid_index_snapshot: None, - uuid_snapshot_refresh_needed: None, + identity_probe: None, limits: GraphWriterLimits::default(), charged_topology_bytes: 0, buffered_topology_rows: 0, @@ -1154,34 +1157,31 @@ impl GraphWriter { Ok(()) } - /// Resolve and register persisted edge endpoints through the writer-owned - /// authenticated disk-index snapshot. UUIDs are sorted/deduplicated and - /// resolved with bounded block merge scans, so repeated construction - /// batches decode zero topology rows and perform zero per-record seeks. + /// Resolve and register persisted edge endpoints from the published node + /// topology. Row groups whose `node_uuid` range holds no requested UUID + /// are skipped, and the rest decode only the UUID and `node_id` columns. pub fn register_existing_endpoints( &mut self, node_uuids: &[Uuid], ) -> Result { - if let Some(generation) = self.uuid_snapshot_refresh_needed { - self.uuid_index_snapshot = Some( - crate::AuthenticatedUuidIndexSnapshot::open_at_generation(&self.dir, generation)?, - ); - self.uuid_snapshot_refresh_needed = None; - } - if self.uuid_index_snapshot.is_none() { - crate::uuid_membership::ensure_uuid_membership_migrated_with_topology( - &self.dir, - self.topology.clone(), - )?; - let generation = crate::read_topology_generation(&self.dir)?; - self.uuid_index_snapshot = Some( - crate::AuthenticatedUuidIndexSnapshot::open_at_generation(&self.dir, generation)?, - ); + let generation = crate::read_topology_generation(&self.dir)?; + if self + .identity_probe + .as_ref() + .is_none_or(|probe| probe.topology_generation() != generation) + { + let files = match &self.topology { + Some(authority) => crate::enumerate_topology_files(authority, None)?, + None => crate::TopologyFiles::discover_legacy(&self.dir)?, + }; + self.identity_probe = Some(crate::TopologyIdentityProbe::open( + &self.dir, &files, generation, + )?); } let (surrogates, metrics) = self - .uuid_index_snapshot + .identity_probe .as_mut() - .expect("snapshot initialized") + .expect("probe initialized") .lookup_node_surrogates(node_uuids)?; let mut resolved = Vec::new(); for (uuid, surrogate) in node_uuids.iter().zip(surrogates) { @@ -1222,6 +1222,14 @@ impl GraphWriter { .topology_work .uuid_surrogate_bytes_read .saturating_add(metrics.surrogate_bytes_read); + self.topology_work.uuid_pages_considered = self + .topology_work + .uuid_pages_considered + .saturating_add(metrics.pages_considered); + self.topology_work.uuid_pages_read = self + .topology_work + .uuid_pages_read + .saturating_add(metrics.pages_read); self.topology_work.uuid_runs_considered = self .topology_work .uuid_runs_considered @@ -1839,12 +1847,6 @@ impl GraphWriter { deleted_nodes: Vec, deleted_edges: Vec, ) -> Result, GfError> { - if let Some(generation) = self.uuid_snapshot_refresh_needed { - self.uuid_index_snapshot = Some( - crate::AuthenticatedUuidIndexSnapshot::open_at_generation(&self.dir, generation)?, - ); - self.uuid_snapshot_refresh_needed = None; - } let committed = crate::uuid_membership::commit_uuid_topology_rewrite( &self.dir, staged, @@ -1854,16 +1856,16 @@ impl GraphWriter { deleted_nodes, deleted_edges, }, - &mut self.uuid_index_snapshot, + &mut self.identity_probe, )?; match committed { crate::uuid_membership::CommittedUuidTopologyRewrite::NoTopologyChange => Ok(None), crate::uuid_membership::CommittedUuidTopologyRewrite::Committed { generation, - metrics, + probe, v4_metrics, } => { - self.record_uuid_append_work(&metrics); + self.record_identity_probe_work(&probe); if let Some(metrics) = v4_metrics.as_ref() { self.record_v4_ordinal_append_work(metrics); } @@ -1871,54 +1873,25 @@ impl GraphWriter { self.pending_index_edges.clear(); Ok(Some(generation)) } - crate::uuid_membership::CommittedUuidTopologyRewrite::CommittedNeedsRefresh { - generation, - metrics, - v4_metrics, - error, - } => { - self.record_uuid_append_work(&metrics); - if let Some(metrics) = v4_metrics.as_ref() { - self.record_v4_ordinal_append_work(metrics); - } - self.pending_index_nodes.clear(); - self.pending_index_edges.clear(); - self.uuid_snapshot_refresh_needed = Some(generation); - Err(GfError::Storage(format!( - "topology generation {generation} committed but UUID index snapshot refresh failed: {error}" - ))) - } } } - fn record_uuid_append_work(&mut self, metrics: &crate::UuidIndexAppendMetrics) { + fn record_identity_probe_work(&mut self, metrics: &crate::UuidProbeMetrics) { let work = &mut self.topology_work; - work.uuid_input_records = work - .uuid_input_records - .saturating_add(metrics.input_records); - work.uuid_prior_topology_rows_decoded = work - .uuid_prior_topology_rows_decoded - .saturating_add(metrics.prior_topology_rows_decoded); - work.uuid_physical_bytes_written = work - .uuid_physical_bytes_written - .saturating_add(metrics.physical_bytes_written); - work.uuid_write_blocks = work.uuid_write_blocks.saturating_add(metrics.write_blocks); - work.uuid_write_bytes = work.uuid_write_bytes.saturating_add(metrics.write_bytes); - work.uuid_peak_buffered_records = work - .uuid_peak_buffered_records - .max(u64::try_from(metrics.peak_buffered_records).unwrap_or(u64::MAX)); - work.uuid_peak_buffered_bytes = work - .uuid_peak_buffered_bytes - .max(u64::try_from(metrics.peak_buffered_bytes).unwrap_or(u64::MAX)); + work.uuid_input_records = work.uuid_input_records.saturating_add(metrics.requested); work.uuid_validation_blocks = work .uuid_validation_blocks - .saturating_add(metrics.validation_scan_blocks); + .saturating_add(metrics.identity_blocks_read); work.uuid_validation_bytes = work .uuid_validation_bytes - .saturating_add(metrics.validation_scan_bytes); + .saturating_add(metrics.identity_bytes_read); + work.uuid_pages_considered = work + .uuid_pages_considered + .saturating_add(metrics.pages_considered); + work.uuid_pages_read = work.uuid_pages_read.saturating_add(metrics.pages_read); work.uuid_validation_random_seeks = work .uuid_validation_random_seeks - .saturating_add(metrics.validation_random_seeks); + .saturating_add(metrics.per_record_seeks); } fn record_v4_ordinal_append_work( @@ -2544,7 +2517,7 @@ mod promotion_full_width_tests { let document = OntologyLoader::load_yaml("ontology_id: wide\nversion: \"1\"\nentity_types:\n - name: Person\n abstract: false\nrelation_types:\n - name: KNOWS\n src: Person\n dst: Person\n".as_bytes()).unwrap(); let ontology = OntologyHandle::new(OntologyCompiler::compile(&document).unwrap()); crate::promote_runtime_graph_for_ontology(dir.path(), &ontology, &catalog).unwrap(); - let mut membership = crate::UuidMembershipIndex::open(dir.path()).unwrap(); + let mut membership = crate::TopologyIdentityProbe::open_dir(dir.path()).unwrap(); assert_eq!( membership.lookup_node_surrogates(&[left, right]).unwrap().0, [Some(node_base), Some(node_base + 1)] @@ -2608,7 +2581,7 @@ mod promotion_full_width_tests { ); writer.flush().unwrap(); drop(writer); - let mut membership = crate::UuidMembershipIndex::open(dir.path()).unwrap(); + let mut membership = crate::TopologyIdentityProbe::open_dir(dir.path()).unwrap(); assert_eq!( membership .lookup_node_surrogates(&[left, right, created]) diff --git a/crates/graphforge-storage/src/writer/tests.rs b/crates/graphforge-storage/src/writer/tests.rs index fd5709987..fe675511f 100644 --- a/crates/graphforge-storage/src/writer/tests.rs +++ b/crates/graphforge-storage/src/writer/tests.rs @@ -384,12 +384,12 @@ fn reopen_recovers_surrogate_tails_without_full_topology_reads() { } #[test] -fn authenticated_endpoint_registration_decodes_zero_topology_rows() { +fn endpoint_registration_decodes_only_the_row_groups_that_can_hold_the_endpoints() { const CHILD: &str = "GRAPHFORGE_ENDPOINT_REGISTRATION_IO_CHILD"; if std::env::var_os(CHILD).is_none() { let status = std::process::Command::new(std::env::current_exe().unwrap()) .arg("--exact") - .arg("writer::tests::authenticated_endpoint_registration_decodes_zero_topology_rows") + .arg("writer::tests::endpoint_registration_decodes_only_the_row_groups_that_can_hold_the_endpoints") .arg("--nocapture") .env(CHILD, "1") .status() @@ -410,15 +410,6 @@ fn authenticated_endpoint_registration_decodes_zero_topology_rows() { seed.create_node(right, EntityTypeId::decode(0).unwrap()) .unwrap(); seed.flush().unwrap(); - crate::rebuild_uuid_membership_indexes( - dir.path(), - crate::UuidIndexBuildLimits { - scan_batch_rows: 1, - run_records: 1, - merge_fan_in: 2, - }, - ) - .unwrap(); let _io_capture = crate::io_stats::CaptureScope::install(); @@ -427,10 +418,11 @@ fn authenticated_endpoint_registration_decodes_zero_topology_rows() { let metrics = writer.register_existing_endpoints(&[left, right]).unwrap(); assert_eq!(metrics.found, 2); assert_eq!(metrics.per_record_seeks, 0); + // One node fragment, one row group; UUIDs and `node_id` are read together. assert_eq!(metrics.identity_blocks_read, 1); - assert_eq!(metrics.surrogate_blocks_read, 1); + assert_eq!(metrics.surrogate_blocks_read, 0); assert_eq!(writer.topology_write_work().uuid_per_record_seeks, 0); - assert_eq!(writer.topology_write_work().uuid_block_seeks, 2); + assert_eq!(writer.topology_write_work().uuid_block_seeks, 1); writer .create_edge(new_v7(), "KNOWS", &left, &right) .unwrap(); @@ -439,6 +431,119 @@ fn authenticated_endpoint_registration_decodes_zero_topology_rows() { assert_eq!(io.node_filtered_reads, 0); } +/// A new generation may not reuse a UUID that names a published node or edge: +/// node and edge UUIDs share one namespace, and the check reads the published +/// Parquet (#1902). The writer's own window cannot see these UUIDs, so only the +/// commit's probe can refuse them. +#[test] +fn commit_refuses_a_uuid_the_published_topology_already_holds() { + let dir = TempDir::new().unwrap(); + let (left, right, other, edge) = (new_v7(), new_v7(), new_v7(), new_v7()); + let label = EntityTypeId::decode(0).unwrap(); + let mut seed = GraphWriter::open_at(dir.path(), OntologyMode::Exploratory, TS).unwrap(); + for node in [left, right, other] { + seed.create_node(node, label).unwrap(); + } + seed.create_edge(edge, "KNOWS", &left, &right).unwrap(); + seed.flush().unwrap(); + let published = crate::read_topology_generation(dir.path()).unwrap(); + + type Stage = fn(&mut GraphWriter, [Uuid; 4]); + let refusals: [(&str, Stage); 4] = [ + ("a node reusing a node UUID", |writer, [left, ..]| { + writer + .create_node(left, EntityTypeId::decode(0).unwrap()) + .unwrap(); + }), + ("a node reusing an edge UUID", |writer, [.., edge]| { + writer + .create_node(edge, EntityTypeId::decode(0).unwrap()) + .unwrap(); + }), + ( + "an edge reusing an edge UUID", + |writer, [left, right, _, edge]| { + writer.register_existing_endpoints(&[left, right]).unwrap(); + writer.create_edge(edge, "KNOWS", &left, &right).unwrap(); + }, + ), + ( + "an edge reusing a node UUID", + |writer, [left, right, other, _]| { + writer.register_existing_endpoints(&[left, right]).unwrap(); + writer.create_edge(other, "KNOWS", &left, &right).unwrap(); + }, + ), + ]; + for (name, stage) in refusals { + let mut writer = + GraphWriter::open_at(dir.path(), OntologyMode::Exploratory, TS + 1).unwrap(); + stage(&mut writer, [left, right, other, edge]); + let error = writer.flush().unwrap_err().to_string(); + assert!(error.contains("already exists"), "{name}: {error}"); + assert_eq!( + crate::read_topology_generation(dir.path()).unwrap(), + published, + "{name}: a refused commit advanced the topology" + ); + } + + // A fresh UUID still commits against the same published topology. + let mut writer = GraphWriter::open_at(dir.path(), OntologyMode::Exploratory, TS + 2).unwrap(); + writer.create_node(new_v7(), label).unwrap(); + writer.flush().unwrap(); + assert_eq!( + crate::read_topology_generation(dir.path()).unwrap(), + published + 1 + ); +} + +/// The writer's production Parquet carries a page index, and an endpoint lookup +/// uses it: of a fragment's three pages the lookup decodes the one holding the +/// endpoints, and the counters say so (#1902). +#[test] +fn endpoint_registration_decodes_only_the_pages_that_can_hold_the_endpoints() { + const NODES: u128 = 45_000; + let node = |index: u128| Uuid::from_u128((0x7 << 76) | (0x2 << 62) | (index + 1)); + let dir = TempDir::new().unwrap(); + let label = EntityTypeId::decode(0).unwrap(); + let mut seed = GraphWriter::open_at(dir.path(), OntologyMode::Exploratory, TS).unwrap(); + for index in 0..NODES { + seed.create_node(node(index), label).unwrap(); + } + seed.flush().unwrap(); + + let mut lookup = |endpoints: &[Uuid]| { + let mut writer = + GraphWriter::open_at(dir.path(), OntologyMode::Exploratory, TS + 1).unwrap(); + let metrics = writer.register_existing_endpoints(endpoints).unwrap(); + assert_eq!(metrics.found, endpoints.len() as u64); + assert_eq!(metrics.per_record_seeks, 0); + (metrics, writer.topology_write_work()) + }; + + let (middle, work) = lookup(&[node(22_000), node(22_001), node(22_002)]); + assert!( + middle.pages_considered >= 2, + "45,000 rows span several 20,000-row pages: {middle:?}" + ); + assert_eq!(middle.pages_read, 1, "{middle:?}"); + assert_eq!(middle.identity_blocks_read, 1); + assert_eq!(work.uuid_pages_read, 1); + assert_eq!(work.uuid_pages_considered, middle.pages_considered); + + let (ends, _) = lookup(&[node(0), node(NODES - 1)]); + assert_eq!(ends.pages_considered, middle.pages_considered); + assert_eq!(ends.pages_read, 2, "first and last page only: {ends:?}"); + assert!(ends.pages_read < ends.pages_considered); + assert!( + middle.identity_bytes_read < ends.identity_bytes_read, + "one page reads fewer bytes than two: {} against {}", + middle.identity_bytes_read, + ends.identity_bytes_read + ); +} + #[test] fn label_only_topology_commits_keep_endpoint_authority_current() { let dir = TempDir::new().unwrap(); @@ -478,7 +583,6 @@ fn label_only_topology_commits_keep_endpoint_authority_current() { .uuid_prior_topology_rows_decoded, 0 ); - assert!(crate::uuid_membership_index_is_fresh(dir.path()).unwrap()); let mut after_add = GraphWriter::open_at(dir.path(), OntologyMode::Exploratory, TS + 2).unwrap(); assert_eq!( @@ -516,7 +620,6 @@ fn label_only_topology_commits_keep_endpoint_authority_current() { .uuid_prior_topology_rows_decoded, 0 ); - assert!(crate::uuid_membership_index_is_fresh(dir.path()).unwrap()); let mut after_remove = GraphWriter::open_at(dir.path(), OntologyMode::Exploratory, TS + 4).unwrap(); assert_eq!( @@ -813,7 +916,7 @@ fn same_window_cross_kind_uuid_collisions_fail_before_state_mutation() { } #[test] -fn ordinary_writer_flush_keeps_v3_uuid_index_fresh_across_generations() { +fn ordinary_writer_flush_keeps_node_identity_resolvable_across_generations() { let dir = TempDir::new().unwrap(); let first = new_v7(); let mut writer = GraphWriter::open_at(dir.path(), OntologyMode::Strict, TS).unwrap(); @@ -821,12 +924,6 @@ fn ordinary_writer_flush_keeps_v3_uuid_index_fresh_across_generations() { .create_node(first, EntityTypeId::decode(0).unwrap()) .unwrap(); writer.flush().unwrap(); - assert!( - crate::uuid_membership_index_is_fresh(dir.path()).unwrap(), - "generation={} manifest={}", - crate::read_topology_generation(dir.path()).unwrap(), - std::fs::read_to_string(dir.path().join("topology/uuid-membership/manifest.json")).unwrap() - ); let second = new_v7(); let mut writer = GraphWriter::open_at(dir.path(), OntologyMode::Strict, TS + 1).unwrap(); @@ -834,13 +931,7 @@ fn ordinary_writer_flush_keeps_v3_uuid_index_fresh_across_generations() { .create_node(second, EntityTypeId::decode(0).unwrap()) .unwrap(); writer.flush().unwrap(); - assert!( - crate::uuid_membership_index_is_fresh(dir.path()).unwrap(), - "generation={} manifest={}", - crate::read_topology_generation(dir.path()).unwrap(), - std::fs::read_to_string(dir.path().join("topology/uuid-membership/manifest.json")).unwrap() - ); - let mut index = crate::UuidMembershipIndex::open(dir.path()).unwrap(); + let mut index = crate::TopologyIdentityProbe::open_dir(dir.path()).unwrap(); assert_eq!( index.lookup_node_surrogates(&[first, second]).unwrap().0, [Some(1), Some(2)] @@ -1035,29 +1126,6 @@ fn cumulative_topology_and_index_work_doubles_with_bounded_windows() { ); } - fn assert_linear_first_differences_with_fixed_overhead( - label: &str, - n: u64, - twice: u64, - four: u64, - fixed_overhead: u64, - ) { - let first = twice - .checked_sub(n) - .unwrap_or_else(|| panic!("{label}: 2N bytes regressed below N")); - let second = four - .checked_sub(twice) - .unwrap_or_else(|| panic!("{label}: 4N bytes regressed below 2N")); - assert!(first > 0, "{label}: N to 2N added no physical bytes"); - let expected = first.saturating_mul(2); - assert!( - second.abs_diff(expected) <= fixed_overhead, - "{label}: physical-work first differences are not linear within fixed format \ - overhead: N={n}, 2N={twice}, 4N={four}, first={first}, second={second}, \ - expected={expected} +/- {fixed_overhead}" - ); - } - fn retained_bytes(path: &Path) -> u64 { fs::read_dir(path) .unwrap() @@ -1149,25 +1217,10 @@ fn cumulative_topology_and_index_work_doubles_with_bounded_windows() { ) } - let (n_bytes, n_topology_writes, n_uuid_writes, n_reads, n_opens, n_syncs, n) = run(8); - let ( - twice_bytes, - twice_topology_writes, - twice_uuid_writes, - twice_reads, - twice_opens, - twice_syncs, - twice, - ) = run(16); - let ( - four_bytes, - four_topology_writes, - four_uuid_writes, - four_reads, - four_opens, - four_syncs, - four, - ) = run(32); + let (n_bytes, n_topology_writes, _, n_reads, n_opens, n_syncs, n) = run(8); + let (twice_bytes, twice_topology_writes, _, twice_reads, twice_opens, twice_syncs, twice) = + run(16); + let (four_bytes, four_topology_writes, _, four_reads, four_opens, four_syncs, four) = run(32); assert_linear_first_differences("retained footprint", n_bytes, twice_bytes, four_bytes); assert_linear_first_differences( "topology staged output", @@ -1175,50 +1228,17 @@ fn cumulative_topology_and_index_work_doubles_with_bounded_windows() { twice_topology_writes, four_topology_writes, ); - // This fixture's binary-carry merge levels add run-header, fence, and - // manifest writes to an otherwise linear adjacent doubling interval. - // Cap that disclosed amplification at a fixed 4 KiB for all size - // points: a percentage or multiplicative bound would widen with input - // and could conceal increasing super-linear index work. - const UUID_INDEX_FIXED_OVERHEAD_BYTES: u64 = 4 * 1024; - assert!(n_uuid_writes > 0); - assert_linear_first_differences_with_fixed_overhead( - "UUID index physical writes", - n_uuid_writes, - twice_uuid_writes, - four_uuid_writes, - UUID_INDEX_FIXED_OVERHEAD_BYTES, - ); assert_eq!((n_reads, twice_reads, four_reads), (0, 0, 0)); - assert!(n.uuid_validation_blocks > 0 && n.uuid_validation_bytes > 0); - assert!(n_opens > 0 && n_syncs > 0); - assert_linear_first_differences_with_fixed_overhead( - "UUID validation blocks", - n.uuid_validation_blocks, - twice.uuid_validation_blocks, - four.uuid_validation_blocks, - 0, - ); - assert_linear_first_differences_with_fixed_overhead( - "UUID validation bytes", - n.uuid_validation_bytes, - twice.uuid_validation_bytes, - four.uuid_validation_bytes, - 4 * 1024, - ); - assert_linear_first_differences_with_fixed_overhead( - "UUID file opens", - n_opens, - twice_opens, - four_opens, - 32, - ); - assert_linear_first_differences_with_fixed_overhead( - "UUID file syncs", + // Probing the existing topology reads at most what the run has itself + // written: validation work is bounded by the graph, never super-linear. + assert!(four.uuid_validation_bytes <= four.output_bytes); + let _ = ( n_syncs, twice_syncs, four_syncs, - 0, + n_opens, + twice_opens, + four_opens, ); assert_eq!( ( @@ -1781,29 +1801,30 @@ fn flush_into_composes_with_staged_delete_in_one_batch() { } #[test] -fn committed_snapshot_refresh_failure_never_restages_rows() { +fn a_flush_commits_once_and_publishes_no_membership_index() { let dir = TempDir::new().unwrap(); let node = new_v7(); let mut writer = GraphWriter::open_at(dir.path(), OntologyMode::Exploratory, TS).unwrap(); writer .create_node(node, EntityTypeId::decode(0).unwrap()) .unwrap(); - crate::uuid_membership::fail_next_snapshot_refresh_for_test(); - let error = writer.flush().unwrap_err(); - assert!( - error - .to_string() - .contains("committed but UUID index snapshot refresh failed") - ); + writer.flush().unwrap(); assert!(writer.pending_index_nodes.is_empty()); assert!(writer.nodes.is_empty()); let committed_generation = crate::read_topology_generation(dir.path()).unwrap(); assert_eq!(committed_generation, 1); + let index_dir = dir.path().join("topology/uuid-membership"); + for legacy in ["manifest.json", "topology-receipt.json"] { + assert!(!index_dir.join(legacy).exists(), "{legacy}"); + } assert!( - dir.path() - .join("topology/uuid-membership/topology-receipt.json") - .is_file() + std::fs::read_dir(&index_dir) + .into_iter() + .flatten() + .flatten() + .all(|entry| !entry.file_name().to_string_lossy().contains("-v5-")), + "no membership run is written" ); writer.flush().unwrap(); @@ -1813,7 +1834,7 @@ fn committed_snapshot_refresh_failure_never_restages_rows() { ); let batches = crate::catalog::read_nodes(dir.path()).unwrap(); assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 1); - let mut index = crate::UuidMembershipIndex::open(dir.path()).unwrap(); + let mut index = crate::TopologyIdentityProbe::open_dir(dir.path()).unwrap(); assert_eq!(index.count(crate::UuidIndexKind::Node), 1); assert_eq!( index.probe(crate::UuidIndexKind::Node, &[node]).unwrap().0, diff --git a/scripts/ci/schemas/g500-certification.schema.json b/scripts/ci/schemas/g500-certification.schema.json index 7e0301799..8b9f80670 100644 --- a/scripts/ci/schemas/g500-certification.schema.json +++ b/scripts/ci/schemas/g500-certification.schema.json @@ -715,13 +715,11 @@ "canonical_artifact_objects", "encode_output_fsync_operations", "encode_source_spool_fsync_operations", - "encode_membership_fsync_operations", "encode_ordinal_fsync_operations", "hydration_files_copied", "hydration_file_fsync_operations", "hydration_directory_fsync_operations", "encode_output_write_operations", - "encode_membership_write_operations", "encode_source_spool_write_operations", "encode_ordinal_artifact_write_operations", "encode_ordinal_publication_write_operations", @@ -1012,9 +1010,6 @@ "encode_source_spool_fsync_operations": { "$ref": "#/$defs/nonNegative" }, - "encode_membership_fsync_operations": { - "$ref": "#/$defs/nonNegative" - }, "encode_ordinal_fsync_operations": { "$ref": "#/$defs/nonNegative" }, @@ -1030,9 +1025,6 @@ "encode_output_write_operations": { "$ref": "#/$defs/nonNegative" }, - "encode_membership_write_operations": { - "$ref": "#/$defs/nonNegative" - }, "encode_source_spool_write_operations": { "$ref": "#/$defs/nonNegative" }, diff --git a/scripts/ci/test-validate-g500-certification.py b/scripts/ci/test-validate-g500-certification.py index 1bc4d1e33..a20fe2239 100644 --- a/scripts/ci/test-validate-g500-certification.py +++ b/scripts/ci/test-validate-g500-certification.py @@ -356,13 +356,11 @@ def test_rejects_missing_external_provider_result_anchor(): "canonical_artifact_objects", "encode_output_fsync_operations", "encode_source_spool_fsync_operations", - "encode_membership_fsync_operations", "encode_ordinal_fsync_operations", "hydration_files_copied", "hydration_file_fsync_operations", "hydration_directory_fsync_operations", "encode_output_write_operations", - "encode_membership_write_operations", "encode_source_spool_write_operations", "encode_ordinal_artifact_write_operations", "encode_ordinal_publication_write_operations", From e1303e39f75d130878ce57e549a995a978c5015b Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:19:28 +0000 Subject: [PATCH 4/9] docs,ci: record the removal of the UUID membership index (#1902) The identity authority page now describes the Parquet probe (row-group and page-index pruning, the deleted-identities record, the legacy-project behaviour). ADR 0049 and 0058, the storage and resumable-import pages, the direct-fsync guard, the fsync site census, the digest census overrides and the storage CI filter drop the membership index's functions and tests. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/test.yml | 2 - docs/adr/0049-published-payload-checksums.md | 3 +- ...58-initial-builds-run-on-a-bulk-builder.md | 10 +- docs/book/architecture/resumable-import.md | 2 +- docs/book/architecture/storage.md | 53 +++--- .../architecture/uuid-membership-index.md | 172 +++++++++--------- scripts/ci/check-direct-fsync.py | 10 - .../development/digest-census-overrides.json | 47 +---- scripts/development/fsync-sites.py | 37 ++-- 9 files changed, 136 insertions(+), 200 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f317f30f8..961aa04d4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -335,8 +335,6 @@ jobs: runtime_entity_labels::tests::catalog_persistence_replaces_read_only_cas_aliases route_component::tests:: route_component::owned::tests::legacy_cas_materialization_translates_routes_before_creating_paths - uuid_membership::tests::owned_manifest_ - uuid_membership::tests::readonly_shared_runs_preserve_uuid_snapshot_authentication graph_files::tests::graph_file_copy_ adjacency::builder::tests::admitted_reserved_relations_build_portable_distinct_indexes mutator::tests::mapped_reserved_delete_preserves_other_routes_and_reopens diff --git a/docs/adr/0049-published-payload-checksums.md b/docs/adr/0049-published-payload-checksums.md index b94178d7c..413874efd 100644 --- a/docs/adr/0049-published-payload-checksums.md +++ b/docs/adr/0049-published-payload-checksums.md @@ -95,7 +95,8 @@ boundaries retain required authentication. Internal snapshots used for live construction rollback remain an implementation detail. The same mandatory-checksum policy applies to the other persisted payload -readers: CSR shard manifests use version 3, UUID membership manifests version 7, +readers: CSR shard manifests use version 3, UUID membership manifests (version 7, +retired by #1902; legacy files are ignored), the logical ordinal-v4 facet descriptor uses wire version 6, GFDR run envelopes and records use version 2, and embedding generation manifests use version 2. These versions bind exact payload lengths and required file/block checksums; diff --git a/docs/adr/0058-initial-builds-run-on-a-bulk-builder.md b/docs/adr/0058-initial-builds-run-on-a-bulk-builder.md index 5e55214ab..33d908baf 100644 --- a/docs/adr/0058-initial-builds-run-on-a-bulk-builder.md +++ b/docs/adr/0058-initial-builds-run-on-a-bulk-builder.md @@ -62,8 +62,9 @@ the builds apart. that names an edge, an edge UUID that equals a node UUID, and a repeated edge UUID are refused. 3. **Emit.** Catalog, node and edge Parquet windows, property overlays, the - UUID membership index, the v4 ordinal artifacts and the adjacency CSR are - built from the ranked arrays. Windows and CSR shards are independent once + v4 ordinal artifacts and the adjacency CSR are built from the ranked arrays. + No UUID membership index is emitted (#1902): a later append asks the + published Parquet instead. Windows and CSR shards are independent once ranks exist and encode in parallel. Each entry of a CSR direction is `key << 32 | edge_id`; sorting those orders every node's list by `edge_id`. Shard boundaries follow the streamed writer's rule, so shard bytes match. @@ -127,9 +128,8 @@ When the estimate exceeds the budget: fanout. Already-fitting initial ranges keep their original scratch files. - Pass 3 builds the partitions in order, several at a time. Sorting a partition ranks its edges (the first `edge_id` is the number of earlier edges plus - one). It checks identities, writes its canonical edge files, keeps its sorted - UUIDs for the membership index, and scatters its adjacency entries once into - node-range partitions bounded by exact node degrees. A node larger than a + one). It checks identities, writes its canonical edge files, and scatters + its adjacency entries once into node-range partitions bounded by exact node degrees. A node larger than a partition spans consecutive partitions split by its increasing edge occurrence ordinal, so a hub cannot force all of its adjacency into one resident partition. Canonical edge files cover fixed windows of `edge_id`s that can straddle two diff --git a/docs/book/architecture/resumable-import.md b/docs/book/architecture/resumable-import.md index ec473ef86..ebbd97e3e 100644 --- a/docs/book/architecture/resumable-import.md +++ b/docs/book/architecture/resumable-import.md @@ -112,7 +112,7 @@ emits the whole encoded generation: `edge_id` is the rank. A missing endpoint, an endpoint that is an edge, an edge UUID that equals a node UUID and a repeated UUID are refused. 4. **Emit** writes the runtime catalog, node and edge Parquet windows, property - overlays, the UUID membership and v4 ordinal artifacts, and the CSR shards. + overlays, the v4 ordinal artifacts, and the CSR shards. Each artifact is hashed (SHA-256, XXH64) from the bytes written once. `commit` then installs and publishes the encoded inventory exactly as for any diff --git a/docs/book/architecture/storage.md b/docs/book/architecture/storage.md index 3ae91bccb..80f91438b 100644 --- a/docs/book/architecture/storage.md +++ b/docs/book/architecture/storage.md @@ -106,7 +106,7 @@ GraphForge uses a **dual-key pattern** for all first-class objects: ### Why UUIDv7 -UUIDv7 (RFC 9562) is time-ordered within a millisecond, globally unique without coordination, fits in Arrow `FixedSizeBinary(16)`, and supports offline generation on mobile devices or air-gapped systems. See [UUID membership indexes](uuid-membership-index.md) for identity lookup. +UUIDv7 (RFC 9562) is time-ordered within a millisecond, globally unique without coordination, fits in Arrow `FixedSizeBinary(16)`, and supports offline generation on mobile devices or air-gapped systems. See [UUID identity authority](uuid-membership-index.md) for identity lookup. UUID byte order, accepted text form, content-derived UUIDv8 records, canonical Arrow bytes, and domain-separated SHA-256 fingerprints follow the frozen @@ -356,7 +356,7 @@ chosen by path in `graphforge_storage::graph_admission`: takes its freshness identity from the same manifest (node and node-property objects by name, length and XXH64), so it reads no edge object and re-reads no source to recheck freshness. -- **Self-authenticating.** UUID-membership runs (block checksums in their +- **Self-authenticating.** Ordinal identity runs (block checksums in their manifest), CSR shards (per-shard XXH64) and delta runs (verified at replay) carry their own authority; hydration neither reads nor registers them. - **Eager.** Every other payload is small metadata or a sidecar with many @@ -364,7 +364,7 @@ chosen by path in `graphforge_storage::graph_admission`: build records, CSR shard manifests, unclassified files). Hydration hard-links it from the object store and checks its length and XXH64 as it links, so an unforeseen reader fails closed. Only the route table and the - small mutable UUID-membership controls (manifests, receipts, lock, + small mutable ordinal-identity controls (manifest, receipt, lock, tombstones) are copied into single-link private files. The forward and ordinal identity runs are hard-linked read-only; the identity handle admits a shared inode only when it is read-only and authenticates each block it reads. @@ -1076,7 +1076,7 @@ preserving direct single-relation scans. Node topology follows the same immutable layout: the first compatible write may retain `topology/nodes.parquet`, while later appends create ordered `topology/nodes/-.parquet` fragments. Counts, filtered reads, -surrogate recovery, UUID membership, semantic validation, projection, export, +surrogate recovery, UUID identity probes, semantic validation, projection, export, label mutation, and deletion operate over the logical union. A localized rewrite replaces only the fragment containing a changed row; untouched node fragments retain their filesystem identity. @@ -1087,31 +1087,28 @@ the same commit as every topology append. Writer reopen reads this bounded record rather than enumerating or decoding the accumulated topology fragments; legacy projects without it use the bounded tail migration path once. -Bulk endpoint resolution uses the persistent authenticated -`topology/uuid-membership/` snapshot published with each immutable graph -generation. The current wire-version-6 `manifest.json` facet binds the -topology generation, record counts, exact lengths, publication SHA-256 identities, -and mandatory file/block XXH64 checksums. Default probes validate checksums -without recomputing cryptographic payload identities. Nodes have a -sorted fixed-width `UUID -> node_id` file; edges have a sorted UUID membership -file. Builds use bounded external sort runs and bounded-fan-in merges. Probes -sort and deduplicate the caller batch, use authenticated block fences to select -only candidate blocks, and merge-scan every selected block once. Newest runs -own tombstone and cross-kind shadowing; node results are batch-validated against -the surrogate-sorted reverse file before caller order is restored. Production -work evidence reports identity/surrogate block reads and bytes, runs considered, -and exactly zero per-record filesystem seeks while decoding zero topology rows. -Duplicate node or edge UUIDs, reuse of one UUID across the node and edge -domains, stale manifests, and missing, truncated, checksum-mismatched, or -identity/reverse-inconsistent index files fail closed. +Append validation and bulk endpoint resolution read the published topology +Parquet through `TopologyIdentityProbe`; no derived UUID index is written or read +(#1902). A probe skips every fragment row group whose `node_uuid`/`edge_uuid` +min/max statistics exclude all requested UUIDs, then decodes only the UUID +column (plus `node_id` for node lookups) of the rest. Live counts come from +footer row counts. A UUID is refused when it names a live node, a live edge, or +a deleted entity: `topology/deleted_identities.parquet` keeps the sorted UUIDs of +every deleted node and edge, is rewritten only by a generation that deletes, and +is an ordinary authenticated graph file. Work evidence reports row groups and +compressed bytes read, fragments considered, and zero per-record seeks. Projects +built before #1902 still carry `topology/uuid-membership/manifest.json`, +`identities-v5-*` and `node-surrogates-v5-*`; those files open, export and +verify as ordinary entries and are never read. See +[UUID identity authority](uuid-membership-index.md). Node ordinal resolution is a distinct, additive authority facet in the same directory. Its `ordinal-v4-manifest.json`, `ordinal-v4-receipt.json`, and -`ordinal-v4.lock` never replace or reinterpret the v3 node-and-edge manifest. -Both facets name the same topology generation but have independent receipt-bound -manifest digests. The v4 logical ordinal facet now requires wire version 5 -with file/block XXH64 checksums; earlier wire-version-4 descriptors are refused. If the ordinal facet is absent while current v3 is canonical, -discovery returns a typed rebuild requirement. A present ordinal path must pass +`ordinal-v4.lock` live beside the ignored legacy membership files and never +interpret them. The v4 logical ordinal facet requires wire version 5 +with file/block XXH64 checksums; earlier wire-version-4 descriptors are refused. +If the ordinal facet is absent, discovery returns a typed rebuild +requirement. A present ordinal path must pass authenticated open and never falls back to v3 when malformed or substituted. New mapped publications use a compact version-8 `graph/files` root with @@ -1134,7 +1131,7 @@ The authoritative write census is executable: topology node and edge shards, node and edge properties, graph deltas, catalog records, extension-owned graph records, and the generation/runtime-catalog/runtime-label control files must all appear in the revision descriptor journal and resolve to the same authenticated -logical inventory. Rebuildable adjacency and UUID-membership artifacts live +logical inventory. Rebuildable adjacency artifacts live under `.graphforge-cache/` and are rejected as graph authority. Parquet write sites share `RewriteBatch` plus `commit_topology_aware`; the three control-file writers record their descriptor before making replacement bytes visible. @@ -1340,7 +1337,7 @@ Every applicable graph publisher preserves these authorities together: - Complete typed or exploratory schemas, null/concrete property types, latest values and tombstones; immutable primary routes and label memberships remain distinct. Physical path components never substitute for semantic route names. -- Authenticated graph-file ownership, route authorities, UUID membership, +- Authenticated graph-file ownership, route authorities, deleted identities, ordinal receipts and applicable adjacency/search generations. Staging reads the admitted inventory rather than discovering authority from filenames. - All graph, catalog, ontology/composition and other declared publication diff --git a/docs/book/architecture/uuid-membership-index.md b/docs/book/architecture/uuid-membership-index.md index da7844acc..a12afc5cf 100644 --- a/docs/book/architecture/uuid-membership-index.md +++ b/docs/book/architecture/uuid-membership-index.md @@ -1,32 +1,72 @@ -# UUID identity authority facets - -GraphForge keeps two generation-coupled authorities under -`topology/uuid-membership/`. They share a topology generation, not a manifest -schema or digest. - -## UUID membership facet - -`manifest.json` is the current v5 authority for both node and edge UUID -membership and node `UUID -> node_id` resolution. Existing endpoint-resolution, -construction, and mutation consumers authenticate this facet. Its immutable runs and `topology-receipt.json` remain reachable -until the v5 manifest no longer selects them. - - -Membership format 5 encodes UUID16 + kind1, followed by the full big-endian -surrogate8 for nodes and tombstones (25 bytes). Live edges use 17 bytes because -their membership surrogate is defined as zero; their canonical edge ID remains -in topology. Nonzero live-edge membership surrogates are rejected. This removes -seven reserved zero bytes per record and eight additional bytes per live edge. -No UUID bits or meaningful surrogate bits are truncated. - -Authenticated blocks are at most 1 MiB and end on complete record boundaries. -Readers validate kinds, record counts, UUID ordering, first/last fences and SHA; -probes select blocks by their fences and merge-scan sorted requests. Byte counts -come from authenticated block lengths, and write-call counters count actual -whole-record flushes. Private construction input remains a separate 32-byte -format, hashed before bounded in-place packing. The published manifest and -receipt select format 5; unsupported prior membership formats are refused. -There is no compatibility reader or migration requirement before v1. +# UUID identity authority + +GraphForge answers "does this UUID exist" from the published topology Parquet. +There is no derived UUID membership index. The `topology/uuid-membership/` +directory name is historical: it now holds only the node ordinal facet below. + +Issue #1902 removed the membership index (`manifest.json`, +`identities-v5-*.uuidx`, `node-surrogates-v5-*.uuidx`, `topology-receipt.json`). +It duplicated the UUID columns of the published Parquet at 17-25 B per edge +(9.28 GiB at S25) and was rewritten with every generation. Under the pre-v1 +policy the format changed in place: no new generation writes those files, no +reader opens them, and a project that still carries them opens normally with the +files admitted, exported and ignored. + +## Identity probe + +`TopologyIdentityProbe` (`topology_identity.rs`) is the one reader. It is opened +over a pinned `TopologyFiles` list, or over a compact parent generation whose +fragments are content-addressed objects, in which case each object is +authenticated against its inventory checksum and held for the life of the probe. + +- It reads each fragment footer once and caches it process-wide, keyed by the + file's device, inode, length and mtime. A replaced fragment misses the cache. +- A probe sorts and deduplicates the caller's UUIDs, then skips every row group + whose `node_uuid`/`edge_uuid` min/max statistics exclude all of them. Fragments + built by an initial build are UUID-ordered, so a small batch touches a few row + groups. Appended fragments are `node_id`/`edge_id` ordered and their UUIDs + arrive in any order, so their ranges prune less; a probe of them reads every + row group whose range overlaps a candidate. +- Inside a surviving row group the probe prunes again by the Parquet column + index: a page whose min/max excludes every candidate is not decoded, and the + reader fetches only the selected pages through the offset index. Published + Parquet is written with a page index (20,000-row pages in 1,048,576-row row + groups), so a small batch into a UUID-ordered fragment decodes a few pages, + not a few row groups. A file without a page index is pruned by row group alone. +- The surviving pages decode only the UUID column (plus `node_id` for node + lookups) and row groups run in parallel. Each decoded value is binary-searched + against the sorted candidates. The metrics report `pages_considered` and + `pages_read` (the difference is what pruning avoided), `identity_blocks_read` + (row groups decoded) and `identity_bytes_read` (compressed bytes of the + selected pages and their chunks' dictionary pages). +- Live counts are the sum of footer row counts, so no manifest carries them. + +Node and edge UUIDs share one namespace. Append validation, the writer's commit +and the staged construction session all ask the same question: is this UUID a +live node, a live edge, or a deleted entity? Any yes is refused with the same +typed `IdentityConflict`. + +## Deleted identities + +A deleted entity is no longer a row, but its UUID is never reusable. Until +#1902 the membership index kept a tombstone for every deleted node and edge. +`topology/deleted_identities.parquet` replaces those tombstones: one +`FixedSizeBinary(16)` column of the UUIDs of every deleted node and edge, sorted +and unique. A generation that deletes writes the merged file in the same atomic +rewrite as the topology change; a generation that deletes nothing carries the +prior file forward untouched, and a graph that never deletes never has one. Its +size follows deletions, not graph size. It is an ordinary authenticated graph +file, so export, import and `gf verify` carry and check it with the rest. + +Validation refuses what the commit refuses. Before #1902, `validate_bulk_*` +accepted a deleted UUID and the commit rejected it later; both now consult the +same probe. + +Nodes are additionally covered by the ordinal facet, whose forward runs retain +every node UUID that was ever appended, tombstones included. A project that +deleted entities before #1902 has those deletions only in its ignored membership +index (edges) or in the forward runs (nodes): a deleted node UUID stays refused +at commit, and a deleted edge UUID from before the upgrade becomes reusable. ## Node ordinal facet @@ -41,19 +81,18 @@ mapping independently. Ordinal payloads are packed by contiguous node-ID range and carry fixed-size authenticated block fences. Discovery and authenticated open are separate operations. When the ordinal -manifest is absent, discovery validates that current v5 authority is canonical -before returning `RebuildRequired`. When the ordinal path exists, discovery -reports it as present without trusting its contents. Authenticated open then +manifest is absent, discovery returns `RebuildRequired`. When the ordinal path +exists, discovery reports it as present without trusting its contents. Authenticated open then requires the ordinal digest selected by the project receipt. A malformed, substituted, or generation-mismatched ordinal facet fails closed and never -falls back to v5. +falls back to another source. The explicit rebuild API constructs v4 only from canonical topology and returns an aggregate `CanonicalTopology` disposition with generation, identity/range, artifact-byte, fixed-block, buffer, temporary-run, and fsync evidence. It never -opens a v5 reverse run as migration input. Durable-rewrite recovery either -retains the prior v5-only authority or completes the receipt-bound v4 facet; -there is no mixed-version read state. +reads legacy membership files. Durable-rewrite recovery either retains the prior +authority or completes the receipt-bound v4 facet; there is no mixed-version +read state. `peak_temporary_bytes` is the total maximum coexisting rebuild scratch, not merely the final artifact size. Storage-owned accounting includes scan runs and @@ -86,17 +125,17 @@ requested/unique/found counts, selected ranges, logical bytes, coalesced calls, tombstones, and bounded-buffer charges. A typed failure can be reduced to sanitized failure evidence, including an authentication-failure count, without emitting UUIDs, paths, or record contents. Consumers must not reopen the index -per chunk or substitute the v5 membership LSM. +per chunk or substitute a scan of the node Parquet. -Orphan collection starts from the current authenticated v5 manifest and, when -the ordinal facet exists, requires the opaque authority resolved from a pinned -project generation before authenticating the v4 manifest and artifacts. It -retains the union. Hashing an untrusted manifest or receipt is never treated as -provenance for deciding reachability. +Orphan collection requires the opaque authority resolved from a pinned project +generation before authenticating the v4 manifest and artifacts, and retains +exactly the files that manifest names. It never collects legacy membership +files. Hashing an untrusted manifest or receipt is never treated as provenance +for deciding reachability. -Both facets are persistent graph authority, not `.graphforge-cache/` content. -Construction and canonical ordinal-facet publication are specified separately by -#969. The facet version numbers identify separate schemas, not an upgrade order. +The ordinal facet and the deleted-identity record are persistent graph +authority, not `.graphforge-cache/` content. Construction and canonical +ordinal-facet publication are specified separately by #969. ### Incremental ordinal publication @@ -114,8 +153,8 @@ the manifest and validates every descriptor and block fence, and each lookup authenticates the ordinal or tombstone blocks it reads. Complete admission, which every writer runs before building on the artifacts, authenticates every run and compares the aggregate forward mapping commitment with the aggregate ordinal -mapping commitment. Historical UUID and surrogate uniqueness is also proved by the -coupled authenticated v5 participant in the same topology transaction. +mapping commitment. Historical UUID and surrogate uniqueness is proved at append by the identity +probe and the deleted-identity record, in the same topology transaction. The manifest may record `uuid_order_matches_ordinals`: whether UUIDs ascend strictly across every ordinal, derived by the publisher from the records it @@ -146,42 +185,7 @@ expected manifest, so retry never replays a graph mutation. A retained old read handle fails stale named-manifest revalidation after the switch; callers advance by opening the exact newly receipt-authorized generation. -Orphan maintenance runs only from the union of selected authenticated v5 and v4 -authority. It removes an unreferenced single-link artifact by retained identity, +Orphan maintenance runs only from the selected authenticated v4 authority. It +removes an unreferenced single-link artifact by retained identity, defers linked or over-budget candidates, and never treats an untrusted sibling manifest as reachability evidence. - -## Packed membership evidence (#1203) - -[Raw integrated measurements](https://github.com/CurateLabs/graphforge/blob/29a7b34ebe441a85ffb9274164d58aaeeb68dc8a/docs/development/evidence/packed-membership-1203.json) -use source `d103c3cb0360e5a76a4b3cbbf60ce3c3cec14d60`, including the merged -construction Zstd repair. Four permanent fixtures pass exact -query/reopen/export/full-verify/clean-import checks. The additional boundary test -verifies full-width encoding; separate production tests cover rebuild, reopen, -probes and deletion at the ID boundary. - -For 4,097 nodes and 65,537 edges, membership payload falls from 2,228,288 to -1,216,554 bytes: 487,438 bytes of reserved padding and 524,296 bytes of defined-zero -live-edge fields. For 8,193 nodes it falls from 2,359,360 to 1,318,954 bytes. -Node reverse-surrogate records remain 24 bytes and the ordinal facet keeps its -independent current schema. - -| Fixture | After Parquet repair: permanent allocation | With packed membership | -| --- | ---: | ---: | -| Sequential | 4,112,384 | 3,096,576 | -| Random | 7,585,792 | 6,541,312 | -| Eight property routes, CSR built | 18,644,992 | 17,633,280 | -| Heterogeneous properties | 13,086,720 | 12,075,008 | - -The serial integrated assessment took 391.33 seconds and peaked at 247,332 KiB -RSS on the same ext4 host, versus 393.65 seconds and 270,208 KiB after the Parquet -repair. These are whole-test measurements, including reads and portable copies; -they do not establish an isolated codec CPU improvement. Raw process I/O is -recorded separately from logical storage counters. - -Production-path regressions cover full-width rebuild/reopen/probes and a -`u64::MAX` tombstone, current-format crash recovery, retained snapshots, framing -corruption and bounded merge/probe work. The write-counter boundary test uses -123,361 live edges: exactly 2,097,137 payload bytes and three whole-record writes, -verified through both append paths. Dividing total bytes by 1 MiB would incorrectly -report two writes. diff --git a/scripts/ci/check-direct-fsync.py b/scripts/ci/check-direct-fsync.py index a268e2c15..cc4bc72ac 100644 --- a/scripts/ci/check-direct-fsync.py +++ b/scripts/ci/check-direct-fsync.py @@ -115,16 +115,6 @@ "one_physical_identity_is_counted_once_for_shared_references", "observed_sync_all", ): 1, - ( - "crates/graphforge-storage/src/uuid_membership/probing/tests.rs", - "retained_snapshot_rehashes_manifest_and_authenticates_only_candidate_blocks", - "sync_all", - ): 1, - ( - "crates/graphforge-storage/src/uuid_membership/tests.rs", - "readonly_shared_runs_preserve_uuid_snapshot_authentication", - "sync_all", - ): 1, } METHOD = re.compile( r"\.\s*(observed_sync_all|observed_sync_data|sync_all|sync_data|sync|sync_all_and_release|sync_all_retained|sync_parent_dir)\b" diff --git a/scripts/development/digest-census-overrides.json b/scripts/development/digest-census-overrides.json index 3d262165d..5961ce7ec 100644 --- a/scripts/development/digest-census-overrides.json +++ b/scripts/development/digest-census-overrides.json @@ -2366,49 +2366,13 @@ "8ab874d0257fb6b495dbf3da2404d3d30d37c7c62ec2ebf0d8ddc89d4d90ee56" ] }, - { - "path": "crates/graphforge-storage/src/uuid_membership.rs", - "function": "describe_stream", - "role": "durable_artifact", - "input_contract": "Actual newly produced UUID identity stream receives one whole-artifact SHA identity and whole/block XXH64 checksums. UUID wire7 retires unused per-block SHA; no block SHA input remains.", - "function_bodies_sha256": [ - "9c9c3537bfad35212e280fabd29d3cc0ce9a5ebab1847865313a8f478abfc12c" - ] - }, - { - "path": "crates/graphforge-storage/src/uuid_membership/construction.rs", - "function": "construction_intent_digest", - "role": "control_authentication", - "input_contract": "Domain-v3 intent version, generation/parent, name, retained native source identity, length and expected XXH64 text authenticate private construction intent metadata; no source payload is hashed here.", - "function_bodies_sha256": [ - "f88d7b2f45f4dce58c4779264935586667490f7eea1a40310127f8355a68b985" - ] - }, - { - "path": "crates/graphforge-storage/src/uuid_membership/construction.rs", - "function": "encode_construction_index_inner", - "role": "durable_artifact", - "input_contract": "Full raw 26-byte identity-run records are SHA-hashed while encoding, independently of their expected XXH64 check; the actual source SHA is forwarded to the existing topology receipt binding. This whole-file work remains Artifact rather than control.", - "function_bodies_sha256": [ - "93281d6e1c5169b3388a03e2880ab6b4bf0fcd271df4b1a745944772a4678e5e" - ] - }, { "path": "crates/graphforge-storage/src/uuid_membership/construction.rs", "function": "install_construction_bytes", "role": "control_authentication", "input_contract": "The direct SHA invocation hashes only the empty ordinal-v4.lock control marker at its sole fallback caller; manifest and receipt JSON use the separate control helper. Payload writers retain their own Artifact producers.", "function_bodies_sha256": [ - "f1c682942803d7900f1a9510b1bd958a333bb8fde61770cce573b5b948812e1c" - ] - }, - { - "path": "crates/graphforge-storage/src/uuid_membership/construction.rs", - "function": "sha256_reader_streaming", - "role": "durable_artifact", - "input_contract": "Complete construction-owned artifact input authenticates write/trust boundary.", - "function_bodies_sha256": [ - "900e6bca8e4a3d3db6238c9a752da69f1c091064b351d58ffb0c9ffe84578f83" + "533817fd91e1254a5b966da322371cef3ab14dcf89612ed3363cfb8dedde57dd" ] }, { @@ -2438,15 +2402,6 @@ "614288861bb729709352f47efdcad228707ed91f50da7c0cd757678e3a56bfa2" ] }, - { - "path": "crates/graphforge-storage/src/uuid_membership/topology_delta.rs", - "function": "prepare_uuid_membership_delta", - "role": "control_authentication", - "input_contract": "Canonical UUID index receipt JSON receives control authority commitment.", - "function_bodies_sha256": [ - "6a9da3185acf07710966c030c495186465ab9789fb86ae0a7142baacb075f1f1" - ] - }, { "path": "crates/graphforge-storage/src/uuid_membership/topology_delta.rs", "function": "topology_delta_sha256", diff --git a/scripts/development/fsync-sites.py b/scripts/development/fsync-sites.py index 639176132..67b8eb26e 100644 --- a/scripts/development/fsync-sites.py +++ b/scripts/development/fsync-sites.py @@ -112,46 +112,37 @@ def family(path): SCRATCH = { - ("crates/graphforge-storage/src/uuid_membership/topology_delta.rs", "merge_identity_v3"), - ("crates/graphforge-storage/src/uuid_membership/topology_delta.rs", "merge_surrogate_handles"), - ("crates/graphforge-storage/src/uuid_membership/rebuild.rs", "flush_surrogate_run"), - ("crates/graphforge-storage/src/uuid_membership/rebuild.rs", "merge_runs"), ( - "crates/graphforge-storage/src/uuid_membership/topology_delta.rs", - "plan_uuid_membership_delta", + "crates/graphforge-storage/src/uuid_membership/rebuild.rs", + "flush_surrogate_run", ), ( "crates/graphforge-storage/src/uuid_membership/rebuild.rs", - "merge_node_surrogate_validation_group", + "flush_entity_surrogate_run", ), ( "crates/graphforge-storage/src/uuid_membership/rebuild.rs", - "scan_node_surrogate_validation_runs", + "merge_surrogate_runs", ), - ("crates/graphforge-storage/src/uuid_membership/rebuild.rs", "flush_entity_surrogate_run"), - ("crates/graphforge-storage/src/uuid_membership/rebuild.rs", "merge_surrogate_runs"), - ("crates/graphforge-storage/src/uuid_membership.rs", "sync_uuid_file"), - ("crates/graphforge-storage/src/uuid_membership/rebuild.rs", "flush_run"), - ("crates/graphforge-storage/src/uuid_membership/topology_delta.rs", "write_identity_records"), - ("crates/graphforge-storage/src/uuid_membership/topology_delta.rs", "write_surrogate_records"), - ("crates/graphforge-storage/src/uuid_membership/rebuild.rs", "build_identity_run"), - ("crates/graphforge-storage/src/uuid_membership/topology_delta.rs", "merge_identity_handles"), - ("crates/graphforge-storage/src/uuid_membership/rebuild.rs", "scan_to_runs"), ( "crates/graphforge-storage/src/uuid_membership/rebuild.rs", "scan_pinned_entity_surrogate_runs", ), - ("crates/graphforge-storage/src/uuid_membership/rebuild.rs", "merge_node_surrogate_group"), - ("crates/graphforge-storage/src/uuid_membership/topology_delta.rs", "external_sort_v4_nodes"), - ("crates/graphforge-storage/src/uuid_membership/rebuild.rs", "scan_entity_surrogate_runs"), - ("crates/graphforge-storage/src/uuid_membership/rebuild.rs", "build_surrogate_run"), ( "crates/graphforge-storage/src/uuid_membership/rebuild.rs", - "flush_node_surrogate_validation_run", + "merge_node_surrogate_group", ), ( "crates/graphforge-storage/src/uuid_membership/topology_delta.rs", - "append_uuid_membership_delta_with_tombstones", + "external_sort_v4_nodes", + ), + ( + "crates/graphforge-storage/src/uuid_membership/rebuild.rs", + "scan_entity_surrogate_runs", + ), + ( + "crates/graphforge-storage/src/uuid_membership/rebuild.rs", + "build_surrogate_run", ), } From e226adfbbcaeadd699ac56da40d817b8b725ea87 Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:19:28 +0000 Subject: [PATCH 5/9] test(api): a project carrying the legacy membership index opens, appends, refuses and round-trips (#1902) The fixture is a real durable project written by the binary that produced the UUID membership index: an initial bulk build and one staged append, so the index carries a base run and a delta run. Once nothing reads the index the files are ordinary graph entries; the test proves such a project still opens, refuses duplicate and missing identities from the Parquet, accepts appends, exports, verifies and imports, and that a deleted UUID stays spent. A second test builds a new project and proves it never writes the index and its portable bundle never names one. The bundle check is validated against the legacy bundle, which must name the index files. Co-Authored-By: Claude Sonnet 5.5 --- .../tests/legacy_membership_index.rs | 412 ++++++++++++++++++ .../legacy-membership-index/README.md | 41 ++ .../legacy-membership-index/project/CURRENT | 1 + .../legacy-membership-index/project/FORMAT | 1 + .../lease.lock | 0 .../manifest.json | 1 + .../participants/workspace/configuration.json | 1 + .../participants/workspace/ontology.json | 1 + .../workspace/research_metadata.json | 1 + .../lease.lock | 0 .../manifest.json | 1 + .../participants/graph/files.json | 1 + .../participants/workspace/configuration.json | 1 + .../participants/workspace/ontology.json | 1 + .../workspace/research_metadata.json | 1 + .../lease.lock | 0 .../manifest.json | 1 + .../participants/graph/files.json | 1 + .../participants/workspace/configuration.json | 1 + .../participants/workspace/ontology.json | 1 + .../workspace/research_metadata.json | 1 + .../project/graph-objects/lifecycle.lock | 0 ...48772d6ea329f0e19e75aabbce863d4241c5a93794 | 1 + ...b466026a55c55296ed77889db182081c89dd009673 | 1 + ...b4078aa318c51f2598960381f693d66f96dbabca03 | 1 + ...78b4ce93c74a9dedaea58238f7c49e2aa6c72e522e | 1 + ...285cb4d720da7f8fa6c21e62c28c2399adec6b0378 | 1 + ...c8c3ceb75d1a14bdc17322dec690a11dd417655c99 | 1 + ...b2119c62d724d7d8850ce5a0d0d478f834b0b631a7 | 1 + ...80cb9a379747626a3a244ef88cd49e8981dce0b6f1 | 1 + ...8c8360d56ec62b173802e77ddcbac9cd95144490b5 | 1 + ...d3cb210f945081ecbaa903a1a3321103366ae8cf4f | 1 + ...48c6cf10ca9a08eadbc08f8c06485be33236de4548 | 1 + ...409b0d89fec35c41b805ef42dabcfa193213808335 | 1 + ...06f1c352ffcced481736e5994d008e97c52f490034 | 1 + ...174b463c7f9464b414fcae836c0288d37698c4e062 | 1 + ...b2e29c2b74252d96b6d9042af01695264166840522 | 1 + ...d34bd24842edad4020a4bec74bcea6c31486cdc36f | 1 + ...758f446a13640216a6345fa1a9410cd95758876530 | Bin 0 -> 336 bytes ...e4c163dfd1a34ffaf35a12fcd9e73a53c5eddd0ea3 | 1 + ...d1da8a3751d77da50f41ddfa4f33204685593ba053 | 1 + ...332a01a397938c0499a83fbc82a20edd2261b38a0a | Bin 0 -> 144 bytes ...78db7a54d26497e4801a58786812099a5b0597906f | 1 + ...c737a31897d251273580869d26a0cb8d789c8fa990 | 1 + ...af56a4c31faab573c35d4e5d46cae14bfc4081d5e8 | 1 + ...11beb3544d41d40c9499d4e454c0660a173ee69222 | 1 + ...a14093dd671a046fe1e7ede09760d20e3281fa25fe | 1 + ...e738771406fd0b5b738c8bac89bc345a0220fbd4a7 | Bin 0 -> 2044 bytes ...644128070f7f969953f19c8a61c659c9b8f2c79c21 | Bin 0 -> 2971 bytes ...c6d0b185b33b8dbeb94cf88b69f600425bc0a0c282 | Bin 0 -> 2381 bytes ...b36477c5a61b5fe18044fe79cfbbc304f04d052ea6 | 1 + ...78d9ad0fc39e3fe4961e4ab4f35f83cd12a3c6a220 | 1 + ...c62800b19a3035d06dbb7a6794027acbc8ea845554 | 1 + ...3e39ce01a69cea34144255ddd9468181446dd2473d | Bin 0 -> 1866 bytes ...b5d218d6dd695d1b399a04a9b6bd718341e8e97a07 | 1 + ...903ad030ef05551a8e34dbdd79ccc79cb49d77c687 | 1 + ...e5c88417590c8336cd80323299f21e752ee139125c | 1 + ...a2b7ae8ce696ac7bf9fbb3ec7cbc73bd697e7047f9 | Bin 0 -> 2284 bytes ...0cfb05cb2396f6d41ac1679742b13aa21a745413d7 | Bin 0 -> 2274 bytes ...a3c4f9368e52bcd3558337e9d59ee36b93fa20ad64 | Bin 0 -> 128 bytes ...7820b8f1fc8ef7c3d0d2d1735c749ee309ecaf8c6b | Bin 0 -> 96 bytes ...4b74151eac492b72eca61c2c009f5049a073efd6a1 | 1 + ...bcc099dcd86298686bbe3473d51f0173ac848078f5 | 19 + ...7ed0b0e7c21c87327ac4d836727bc90a121e30a7b3 | 1 + ...f47b52368ff444d880a45e01e51620da8374f32a84 | Bin 0 -> 3003 bytes ...a6fcf1dc56feee9d16d8eaba8aa96681425181f39a | 1 + ...eb54aa35b324aa4da1a0844c211b9470814e2edd63 | 1 + ...cc0cd8fd3884e9f5e533225be81a91404cc8e5a941 | 1 + ...1119041cd6bc8a807580c02e4d994b0f73e0a58bfa | 1 + ...0e75e7a685203d18442246974390cc0e8ca5682c10 | 1 + ...61a57d34a3d93fad72e114bf9cb7a6ceb0a234b9fb | 19 + ...3e241593ccdedb9e077fd957075eac0c63c5519a2c | 1 + ...a0715f49cb92d9e87440d6e0116e51e7b65bd51821 | 1 + ...172c19f5c7723c248e123b0fa3271e2e35593f69ba | 1 + ...a2fc2185fad73bd3ab2d9aaab88e48736ee0a73293 | Bin 0 -> 817 bytes ...1f170fdd6fcbafe707b0d404b47fe2bbed7b2dd884 | 1 + ...6d1477f0a90e73b98919d38d383aabe55c69b6115d | 1 + ...90158915b00d3642c65cc8e940735d4f21c309fc49 | 1 + ...ea6b3899915d2968d93d762496d29bda5f3e1bd405 | 1 + ...1378ba581c5b4be7221950372439117b24934f7578 | 1 + ...7453c0c3df2cdb44fc81b8d9dc89a16262beb0448e | 19 + ...1a5b687561be2ab70c1fe8ac5698d7f904ddd2af76 | 1 + ...3b0bda4eaba2769c8330ca13757e5cde14a2289fae | 1 + ...77725d6b05ce9c9e114242590381e5c4d357aad9ac | 1 + ...39022f663452b6ee84ca8662c619273416272a834b | 1 + ...4a9fa8e7efa32cd581e53e398264e7fc74e38b62a1 | 1 + ...0b3ce6c94caf382469d14e5799214e27aa70b30002 | 1 + ...5b59753fd24eca05413bdf7333b3035da95039ae7d | 1 + ...39b1dba198ba531eeaa0be090b5ddc6616a9678328 | 1 + ...7fb3bdbdbc3e0969b30b41a180edae2ea72d96deac | 1 + ...ed76e90c8021ed20ef728909110e574e2b40454c34 | 1 + ...191151a8d7b2a2d4b38aca4e426d43f93a7dd718d9 | 1 + ...1e9e2160083a7cc4f7198d7c07670c56d05c9d4223 | 1 + ...03e7ff7f9a8fe3c604487b7d4af376270960271b6f | 1 + ...9b3f86bebe0aff18acd05b69c715a146ff6f6c0967 | 1 + ...2e3452345248c5e627a0ddbca8e7d909cb8e5309fa | 1 + ...f805e4a9f6fde3f42fb3725c862e5ac3675faec543 | 1 + ...153fbf371c145a7ae54120be51246322c7fbbc0369 | 19 + ...e0f3508bb309a31e01e5d849c2db2cd276e52c98bd | 4 + ...3b60a15b77df326cc99b87d5cd2e6dc4d605b4b313 | 1 + ...1a0eae7447ae2ac57000b44df00c8b338b6009145c | Bin 0 -> 817 bytes ...f4bc30e3866f2abf92d84848e2c79d02b9c0ced446 | Bin 0 -> 2381 bytes ...6171d702a04277ef7d63ac3d3a1ee4f16858f99bfb | 1 + ...6a42696af0ca8e1b295a3f79ddd9767d2c8bf0a4db | 1 + ...46a2a10249697ee3748e41481e2d74f0448e172b2d | 1 + ...2775eb9d93f31f87e9b1e28f4998d0a129d0fd5572 | Bin 0 -> 588 bytes ...52cdcb3947ca007420e9d19ab3efc16e861080894f | 1 + ...803580d6b5092a942ea07bd6dadf58a4996db144e3 | 1 + ...c72b2a60ae24d262169d28392170eada3ff929624b | 1 + ...44dc9baa8596e5aad5dd69e4aa156a48fe38ef70c4 | 1 + ...63634dc5c054599b0f764d6409999b9e2f001a1691 | Bin 0 -> 1866 bytes ...60fe63ff120e96e6b370b630b3122e0e873c287c46 | 1 + ...9900a54b06c2134b0e6ba5a9272d14510436c3a12c | 1 + ...0af0f86f741dbe328828819cfd28dcf6e08fb972c5 | 1 + ...8d3652ef0bea61cd688a1064e81afdfaff62e1640f | Bin 0 -> 336 bytes ...e8936ea7a9ad80976f7a8e5e4e2b722dbd55228a51 | 1 + ...f94126ed837044e7df4487b6d74ebd4f2687b6ec59 | 1 + ...1f990c56b2185d81c7c7a3f3eac7bedf2e8a361a98 | 1 + ...193a2a34a34de74b82d6eabc1fbb3852003b41f4cc | 1 + ...e47432a552f4b207918d1b39db0592e3d845ad26e3 | 1 + ...25c1f45b4d58b891278a522ee4339d42873c1fe920 | Bin 0 -> 192 bytes ...c8996fb92427ae41e4649b934ca495991b7852b855 | 0 ...36c24623677c27f30033edb5296adfbaa17059d6e4 | 1 + ...b27486370fabd8c152a409d1363efcd9b32d2b15e6 | 1 + ...0eed43e0601780a08f3b926b7ca76b4ea73ad9ef44 | Bin 0 -> 192 bytes ...9aa9ea6e80947355b529995d3e7e8116d59a643069 | 1 + ...6f1c3f497806c1b951a0c27417f3f83ba93fa06b3f | 1 + ...d50b5bbb291770024fc3c2e90f46c761efc8a66573 | 1 + ...eaec531c28dc9bbe6301f83e56823b7590768661d1 | 1 + ...e4f782a3576276766bdea5eb6dd7630a65dbf0d4fd | 1 + ...a1bb372a97e8339593843b3195045ec8691e7bc466 | 1 + ...8dbccff70f4929f270f5cefccbf2dbacecbf7a3cb9 | 1 + ...a9e59a935f932357ea18911547197d9b7896f64da4 | 1 + ...1bbd7f8786b2f66bed48c07f9b3d878e2f4eb42b7a | 1 + .../project/locks/writer.lock | 0 .../6542a196-4938-8837-b381-67d6abd75857.json | 1 + .../971c15f0-5ca9-8b07-8535-90dc9f83d6fa.json | 1 + .../legacy-membership-index/tiny_data.py | 43 ++ 138 files changed, 681 insertions(+) create mode 100644 crates/graphforge-api/tests/legacy_membership_index.rs create mode 100644 tests/fixtures/legacy-membership-index/README.md create mode 100644 tests/fixtures/legacy-membership-index/project/CURRENT create mode 100644 tests/fixtures/legacy-membership-index/project/FORMAT create mode 100644 tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/lease.lock create mode 100644 tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/manifest.json create mode 100644 tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/participants/workspace/configuration.json create mode 100644 tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/participants/workspace/ontology.json create mode 100644 tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/participants/workspace/research_metadata.json create mode 100644 tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/lease.lock create mode 100644 tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/manifest.json create mode 100644 tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/participants/graph/files.json create mode 100644 tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/participants/workspace/configuration.json create mode 100644 tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/participants/workspace/ontology.json create mode 100644 tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/participants/workspace/research_metadata.json create mode 100644 tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/lease.lock create mode 100644 tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/manifest.json create mode 100644 tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/participants/graph/files.json create mode 100644 tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/participants/workspace/configuration.json create mode 100644 tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/participants/workspace/ontology.json create mode 100644 tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/participants/workspace/research_metadata.json create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/lifecycle.lock create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/00/ee3eb40c4e58c1398b4948772d6ea329f0e19e75aabbce863d4241c5a93794 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/01/8df9d340c99cf4504542b466026a55c55296ed77889db182081c89dd009673 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/02/3f0c4332a566cb6fbb2fb4078aa318c51f2598960381f693d66f96dbabca03 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/04/726121dffa2353c921ad78b4ce93c74a9dedaea58238f7c49e2aa6c72e522e create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/06/699b216f61cd4e53f0c3285cb4d720da7f8fa6c21e62c28c2399adec6b0378 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/07/fb10f5a76a039c4e3792c8c3ceb75d1a14bdc17322dec690a11dd417655c99 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/0a/14e69d5f88bdb637d098b2119c62d724d7d8850ce5a0d0d478f834b0b631a7 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/12/41a4163cdfaeac6b997580cb9a379747626a3a244ef88cd49e8981dce0b6f1 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/15/98fdfecd2ba38b695b328c8360d56ec62b173802e77ddcbac9cd95144490b5 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/16/caf5a94acbf6efbbc859d3cb210f945081ecbaa903a1a3321103366ae8cf4f create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/18/450af8e920af1a4a64bf48c6cf10ca9a08eadbc08f8c06485be33236de4548 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/18/ba16da3410baff2d0f29409b0d89fec35c41b805ef42dabcfa193213808335 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/18/d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/1d/2d822abf959ff7f9399c174b463c7f9464b414fcae836c0288d37698c4e062 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/1d/360c9fafd9175352328bb2e29c2b74252d96b6d9042af01695264166840522 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/1d/bbabd6dc589031a0c56ed34bd24842edad4020a4bec74bcea6c31486cdc36f create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/1f/39b3ef25c0235991a359758f446a13640216a6345fa1a9410cd95758876530 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/21/860e1b2d9b2d1aa7e63de4c163dfd1a34ffaf35a12fcd9e73a53c5eddd0ea3 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/29/ca1a62cf31e4d1cc6989d1da8a3751d77da50f41ddfa4f33204685593ba053 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/2a/093dcf7c477b67c309d1332a01a397938c0499a83fbc82a20edd2261b38a0a create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/2a/a4cb6312ff8a557b151878db7a54d26497e4801a58786812099a5b0597906f create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/2b/ea2e263428d5c0c085ebc737a31897d251273580869d26a0cb8d789c8fa990 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/32/9796174d70a914ace991af56a4c31faab573c35d4e5d46cae14bfc4081d5e8 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/33/d5b384bd457bf148bc1711beb3544d41d40c9499d4e454c0660a173ee69222 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/36/c3282c5e0655d4a5539ca14093dd671a046fe1e7ede09760d20e3281fa25fe create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/40/2e0e930d922e9ab4e149e738771406fd0b5b738c8bac89bc345a0220fbd4a7 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/40/71639376da655addfd1c644128070f7f969953f19c8a61c659c9b8f2c79c21 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/42/a36d76c9169a9d28be49c6d0b185b33b8dbeb94cf88b69f600425bc0a0c282 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/47/ea851bc5d6f0806954e1b36477c5a61b5fe18044fe79cfbbc304f04d052ea6 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/4e/1e1b01ecbd59456aa67778d9ad0fc39e3fe4961e4ab4f35f83cd12a3c6a220 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/4f/b465e75e9ce980b51163c62800b19a3035d06dbb7a6794027acbc8ea845554 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/50/b821d0ee918d37ab0baf3e39ce01a69cea34144255ddd9468181446dd2473d create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/51/4c5ae39a6a439abdc276b5d218d6dd695d1b399a04a9b6bd718341e8e97a07 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/52/f4b97c32c0e446a0968c903ad030ef05551a8e34dbdd79ccc79cb49d77c687 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/54/dd843ff310a71e820757e5c88417590c8336cd80323299f21e752ee139125c create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/55/27b3e2c14fbe8a9cfe38a2b7ae8ce696ac7bf9fbb3ec7cbc73bd697e7047f9 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/55/5a4a690fb10d5dd1584f0cfb05cb2396f6d41ac1679742b13aa21a745413d7 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/59/9c59b6e57e65a305ee43a3c4f9368e52bcd3558337e9d59ee36b93fa20ad64 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/5d/dfdc0ed3f1e1d8cd8dcc7820b8f1fc8ef7c3d0d2d1735c749ee309ecaf8c6b create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/60/c685463d4ff99b7528234b74151eac492b72eca61c2c009f5049a073efd6a1 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/60/ec16305c87e777dda418bcc099dcd86298686bbe3473d51f0173ac848078f5 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/61/dd5bd3012ddea71341be7ed0b0e7c21c87327ac4d836727bc90a121e30a7b3 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/62/3f2c45c91931017a3ff5f47b52368ff444d880a45e01e51620da8374f32a84 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/63/0ff369e2d992ea22056ba6fcf1dc56feee9d16d8eaba8aa96681425181f39a create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/63/842e81f8c2dcc54eeb68eb54aa35b324aa4da1a0844c211b9470814e2edd63 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/64/8f9f5aa333e9c0b35840cc0cd8fd3884e9f5e533225be81a91404cc8e5a941 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/64/d0aca220e3dbc30007411119041cd6bc8a807580c02e4d994b0f73e0a58bfa create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/67/0c7acbbcde62dd0cfb6b0e75e7a685203d18442246974390cc0e8ca5682c10 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/6a/a5e92b174b286e8c36e861a57d34a3d93fad72e114bf9cb7a6ceb0a234b9fb create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/6b/3d79b9f4cccf6b04489a3e241593ccdedb9e077fd957075eac0c63c5519a2c create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/70/56644ec93e357f016906a0715f49cb92d9e87440d6e0116e51e7b65bd51821 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/73/0f3c9d30b8f2149d44ec172c19f5c7723c248e123b0fa3271e2e35593f69ba create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/74/ab11daa9873474c8a093a2fc2185fad73bd3ab2d9aaab88e48736ee0a73293 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/79/b5b9ca9a21d61dc1ae781f170fdd6fcbafe707b0d404b47fe2bbed7b2dd884 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/7b/240572eaf4cc550d8b5d6d1477f0a90e73b98919d38d383aabe55c69b6115d create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/7c/787d6c58c741080bc89e90158915b00d3642c65cc8e940735d4f21c309fc49 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/7e/a9b44c433d7f4e4f767eea6b3899915d2968d93d762496d29bda5f3e1bd405 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/80/11b9909d36a6051a3ddd1378ba581c5b4be7221950372439117b24934f7578 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/81/dc0dfc6f7ed687160b947453c0c3df2cdb44fc81b8d9dc89a16262beb0448e create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/81/f9ec2f7714716b7cea261a5b687561be2ab70c1fe8ac5698d7f904ddd2af76 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/83/628bb4c0ac01773986d43b0bda4eaba2769c8330ca13757e5cde14a2289fae create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/83/6ec17534a48b6cfdf1a977725d6b05ce9c9e114242590381e5c4d357aad9ac create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/87/90025a1cbc91d05c270a39022f663452b6ee84ca8662c619273416272a834b create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/88/dbad48a4b2c270ee165c4a9fa8e7efa32cd581e53e398264e7fc74e38b62a1 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/8b/482651abbd4146b318590b3ce6c94caf382469d14e5799214e27aa70b30002 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/8b/515d817bdd27fa39dc155b59753fd24eca05413bdf7333b3035da95039ae7d create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/8d/c1d518105cf2d68a207139b1dba198ba531eeaa0be090b5ddc6616a9678328 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/8f/37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/90/9e7eac9f6a71fcbd68d6ed76e90c8021ed20ef728909110e574e2b40454c34 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/91/7ceda6a0f6fbf916cdf7191151a8d7b2a2d4b38aca4e426d43f93a7dd718d9 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/92/040d2b83ae59ee9d9f621e9e2160083a7cc4f7198d7c07670c56d05c9d4223 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/95/3115f8a1886a273f8fb103e7ff7f9a8fe3c604487b7d4af376270960271b6f create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/96/3c4f9708e0cffbdf90aa9b3f86bebe0aff18acd05b69c715a146ff6f6c0967 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/a4/173a1b346783c2de070b2e3452345248c5e627a0ddbca8e7d909cb8e5309fa create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/a5/47e1515d6437ddb4c530f805e4a9f6fde3f42fb3725c862e5ac3675faec543 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/a5/5c44fd9b95369708a324153fbf371c145a7ae54120be51246322c7fbbc0369 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/a6/88960f0b7533d9b63091e0f3508bb309a31e01e5d849c2db2cd276e52c98bd create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/af/add6993a6bdd8b04ee133b60a15b77df326cc99b87d5cd2e6dc4d605b4b313 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/b1/e9f24720536bcc97707a1a0eae7447ae2ac57000b44df00c8b338b6009145c create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/b3/376061dade602b1d8ab4f4bc30e3866f2abf92d84848e2c79d02b9c0ced446 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/b4/684cec290b5fd2220e9c6171d702a04277ef7d63ac3d3a1ee4f16858f99bfb create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/bc/9975ccb156a95a845d2b6a42696af0ca8e1b295a3f79ddd9767d2c8bf0a4db create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/be/74658ea7232044b698b546a2a10249697ee3748e41481e2d74f0448e172b2d create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/bf/d3d2ea93bb986cf0959a2775eb9d93f31f87e9b1e28f4998d0a129d0fd5572 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/bf/fad8f5e2fc47ab5a5ebd52cdcb3947ca007420e9d19ab3efc16e861080894f create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/c0/5d56b88484c76619b4d1803580d6b5092a942ea07bd6dadf58a4996db144e3 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/c2/057af8c2b375caeb9169c72b2a60ae24d262169d28392170eada3ff929624b create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/c3/36f2b69c74595f284e2944dc9baa8596e5aad5dd69e4aa156a48fe38ef70c4 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/c4/dc71e92a7313804d090463634dc5c054599b0f764d6409999b9e2f001a1691 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/d4/b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/d5/cd023a1b8ff5dc2c02dc9900a54b06c2134b0e6ba5a9272d14510436c3a12c create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/da/bf2e92de1180195cdb270af0f86f741dbe328828819cfd28dcf6e08fb972c5 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/db/88cbed903b9fb0e52c938d3652ef0bea61cd688a1064e81afdfaff62e1640f create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/de/b33edafa58e17727033ee8936ea7a9ad80976f7a8e5e4e2b722dbd55228a51 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/de/e47e1f1f39163945707bf94126ed837044e7df4487b6d74ebd4f2687b6ec59 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/df/71c54a544253d23669021f990c56b2185d81c7c7a3f3eac7bedf2e8a361a98 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/df/e16a08130a605d299b15193a2a34a34de74b82d6eabc1fbb3852003b41f4cc create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/e1/693e4f1c954820986caee47432a552f4b207918d1b39db0592e3d845ad26e3 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/e2/47e733949201fa2ef54425c1f45b4d58b891278a522ee4339d42873c1fe920 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/e3/b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/e5/b0ea5f7a810bc0e143e036c24623677c27f30033edb5296adfbaa17059d6e4 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/e8/9f6e446520ec6669f4c8b27486370fabd8c152a409d1363efcd9b32d2b15e6 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/ef/4c65996784af52e4d3830eed43e0601780a08f3b926b7ca76b4ea73ad9ef44 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/ef/58edb5e714477964e6c29aa9ea6e80947355b529995d3e7e8116d59a643069 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/f1/dd0b9fb6f895682403236f1c3f497806c1b951a0c27417f3f83ba93fa06b3f create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/f3/c2c11bdcd448d981f514d50b5bbb291770024fc3c2e90f46c761efc8a66573 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/f5/8243170c9c519373e516eaec531c28dc9bbe6301f83e56823b7590768661d1 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/f8/95e098927cd1fe441ffae4f782a3576276766bdea5eb6dd7630a65dbf0d4fd create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/f9/b5da385afe31f6c430f1a1bb372a97e8339593843b3195045ec8691e7bc466 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/fa/a08e9fbeac8d23356f848dbccff70f4929f270f5cefccbf2dbacecbf7a3cb9 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/fb/6d0b1f6ea53643307cf8a9e59a935f932357ea18911547197d9b7896f64da4 create mode 100644 tests/fixtures/legacy-membership-index/project/graph-objects/sha256/fd/940e96ef6d179d9bf35e1bbd7f8786b2f66bed48c07f9b3d878e2f4eb42b7a create mode 100644 tests/fixtures/legacy-membership-index/project/locks/writer.lock create mode 100644 tests/fixtures/legacy-membership-index/project/transactions/6542a196-4938-8837-b381-67d6abd75857.json create mode 100644 tests/fixtures/legacy-membership-index/project/transactions/971c15f0-5ca9-8b07-8535-90dc9f83d6fa.json create mode 100644 tests/fixtures/legacy-membership-index/tiny_data.py diff --git a/crates/graphforge-api/tests/legacy_membership_index.rs b/crates/graphforge-api/tests/legacy_membership_index.rs new file mode 100644 index 000000000..8e1ddab19 --- /dev/null +++ b/crates/graphforge-api/tests/legacy_membership_index.rs @@ -0,0 +1,412 @@ +//! The UUID membership index is gone (#1902). +//! +//! `tests/fixtures/legacy-membership-index/` is a real durable project written +//! by the binary that produced the index (an initial build and one append). The +//! index files must keep opening, exporting and verifying as ordinary entries; +//! nothing reads them, and every append refusal is answered from the Parquet. +//! A project created now never writes them, and its portable bundle never +//! carries them. +#![cfg(feature = "portable")] + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use arrow::array::{FixedSizeBinaryArray, RecordBatch, StringArray}; +use graphforge_api::{ + GraphForge, OperationId, PortableSelection, PortableV2ExportRequest, PortableV2ImportRequest, + PortableV2Limits, PortableV2Output, PortableV2SelectionProfile, PortableVerifyRequest, + bulk_edge_input_schema, bulk_node_input_schema, verify_portable_v2, +}; +use graphforge_core::portable::PortableV2Mode; +use graphforge_ir::IrLiteral; +use uuid::Uuid; + +const FIXTURE: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../tests/fixtures/legacy-membership-index/project" +); + +/// The fixture generator's UUIDs: `kind` 1 for nodes, 2 for edges. +fn id(kind: u128, index: u128) -> Uuid { + Uuid::from_u128((kind << 100) | (0x7 << 76) | (0x2 << 62) | index) +} + +fn copy_dir(from: &Path, to: &Path) { + std::fs::create_dir_all(to).unwrap(); + for entry in std::fs::read_dir(from).unwrap() { + let entry = entry.unwrap(); + let target = to.join(entry.file_name()); + if entry.file_type().unwrap().is_dir() { + copy_dir(&entry.path(), &target); + } else { + std::fs::copy(entry.path(), &target).unwrap(); + // Git does not preserve the producer's read-only CAS permissions. + if from + .parent() + .is_some_and(|parent| parent.ends_with("graph-objects/sha256")) + { + let mut permissions = std::fs::metadata(&target).unwrap().permissions(); + permissions.set_readonly(true); + std::fs::set_permissions(&target, permissions).unwrap(); + } + } + } +} + +fn legacy_project() -> (tempfile::TempDir, PathBuf) { + let directory = tempfile::tempdir().unwrap(); + let project = directory.path().join("project"); + copy_dir(Path::new(FIXTURE), &project); + for empty in ["graph-objects/active", "graph-objects/tmp"] { + std::fs::create_dir_all(project.join(empty)).unwrap(); + } + (directory, project) +} + +fn count(graph: &GraphForge, cypher: &str) -> i64 { + let result = graph.execute(cypher).unwrap(); + result.batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0) +} + +fn node_batch(ids: &[Uuid]) -> RecordBatch { + let schema = bulk_node_input_schema(Vec::new()).unwrap(); + RecordBatch::try_new( + schema, + vec![ + Arc::new(FixedSizeBinaryArray::try_from_iter(ids.iter().map(Uuid::as_bytes)).unwrap()), + Arc::new(StringArray::from(vec!["Person"; ids.len()])), + ], + ) + .unwrap() +} + +fn edge_batch(edge: Uuid, source: Uuid, target: Uuid) -> RecordBatch { + let schema = bulk_edge_input_schema(Vec::new()).unwrap(); + let column = |value: Uuid| { + Arc::new(FixedSizeBinaryArray::try_from_iter([value.as_bytes()].into_iter()).unwrap()) + }; + RecordBatch::try_new( + schema, + vec![ + column(edge), + Arc::new(StringArray::from(vec!["KNOWS"])), + column(source), + column(target), + ], + ) + .unwrap() +} + +fn operation(seed: u128) -> OperationId { + OperationId(id(3, seed)) +} + +/// Whether the bundle's tar headers name a membership index file. The bundle is +/// an uncompressed tar stream, so entry names appear verbatim. +fn bundle_names_membership_index(bundle: &Path) -> bool { + let bytes = std::fs::read(bundle).unwrap(); + [ + &b"identities-v5-"[..], + b"node-surrogates-v5-", + b"uuid-membership/manifest.json", + b"uuid-membership/topology-receipt.json", + ] + .iter() + .any(|name| bytes.windows(name.len()).any(|window| window == *name)) +} + +fn membership_entries(project: &Path) -> Vec { + let generation = graphforge_storage::resolve_project_generation(project).unwrap(); + generation + .graph_files_inventory() + .unwrap() + .unwrap() + .files + .into_iter() + .map(|entry| entry.relative_path) + .filter(|path| path.starts_with("topology/uuid-membership/")) + .collect() +} + +#[test] +fn a_legacy_project_opens_appends_refuses_and_round_trips() { + let (directory, project) = legacy_project(); + let before = membership_entries(&project); + assert!( + before + .iter() + .any(|path| path == "topology/uuid-membership/manifest.json") + && before.iter().any(|path| path.contains("-v5-")), + "the fixture must carry the legacy index: {before:?}" + ); + + let graph = GraphForge::new(project.to_str()).unwrap(); + assert_eq!(count(&graph, "MATCH (n) RETURN count(n)"), 14); + assert_eq!(count(&graph, "MATCH ()-[r]->() RETURN count(r)"), 14); + + // Appends refuse against the existing graph, answered from the Parquet. + let existing_node = id(1, 3); + let existing_edge = id(2, 3); + let fresh = id(1, 9_000); + let duplicate_node = graph + .publish_bulk_nodes(operation(1), &[node_batch(&[existing_node])]) + .unwrap_err() + .to_string(); + assert!( + duplicate_node.contains("identity_conflict"), + "{duplicate_node}" + ); + let duplicate_edge = graph + .publish_bulk_edges( + operation(2), + &[edge_batch(existing_edge, id(1, 0), id(1, 1))], + ) + .unwrap_err() + .to_string(); + assert!( + duplicate_edge.contains("identity_conflict"), + "{duplicate_edge}" + ); + let edge_named_like_a_node = graph + .publish_bulk_edges( + operation(3), + &[edge_batch(existing_node, id(1, 0), id(1, 1))], + ) + .unwrap_err() + .to_string(); + assert!( + edge_named_like_a_node.contains("identity_conflict"), + "{edge_named_like_a_node}" + ); + let missing_endpoint = graph + .publish_bulk_edges(operation(4), &[edge_batch(id(2, 9_001), id(1, 0), fresh)]) + .unwrap_err(); + assert!( + missing_endpoint.to_string().contains("missing_endpoint"), + "{missing_endpoint}" + ); + + // Appends still work: a bulk append that reaches an existing node, then a + // Cypher write, each through the writer commit that used to feed the index. + graph + .publish_bulk_nodes(operation(5), &[node_batch(&[fresh])]) + .unwrap(); + graph + .publish_bulk_edges(operation(6), &[edge_batch(id(2, 9_002), fresh, id(1, 2))]) + .unwrap(); + graph + .execute("CREATE (:Person {name: 'new'})-[:KNOWS]->(:Person {name: 'newer'})") + .unwrap(); + assert_eq!(count(&graph, "MATCH (n) RETURN count(n)"), 17); + assert_eq!(count(&graph, "MATCH ()-[r]->() RETURN count(r)"), 16); + + // A deletion records the spent UUID; reuse is refused by validation and commit. + let params = HashMap::from([("id".to_owned(), IrLiteral::Uuid(*fresh.as_bytes()))]); + graph + .execute_with_params("MATCH (n) WHERE n.node_uuid = $id DETACH DELETE n", ¶ms) + .unwrap(); + assert_eq!(count(&graph, "MATCH (n) RETURN count(n)"), 16); + let reuse = graph + .publish_bulk_nodes(operation(7), &[node_batch(&[fresh])]) + .unwrap_err() + .to_string(); + assert!(reuse.contains("identity_conflict"), "{reuse}"); + let reused_edge = graph + .publish_bulk_edges( + operation(8), + &[edge_batch(id(2, 9_002), id(1, 0), id(1, 1))], + ) + .unwrap_err() + .to_string(); + assert!(reused_edge.contains("identity_conflict"), "{reused_edge}"); + + // The legacy files are still ordinary graph entries: untouched, exported, + // verified and imported. + assert_eq!( + membership_entries(&project) + .into_iter() + .filter(|path| before.contains(path)) + .count(), + before.len(), + "legacy membership entries carry forward unchanged" + ); + let bundle = directory.path().join("legacy.gfpb"); + let exported = graph + .export_portable_v2( + &PortableV2ExportRequest { + selection: PortableSelection::Current, + output_path: bundle.clone(), + representation: PortableV2Output::Bundle, + profile: PortableV2SelectionProfile::Complete, + subset: None, + limits: PortableV2Limits::default(), + }, + None, + |_| {}, + ) + .unwrap(); + // The check that a bundle carries no index finds one when it is there. + assert!( + bundle_names_membership_index(&bundle), + "a legacy bundle must name its index files" + ); + verify_portable_v2( + &PortableVerifyRequest { + input: bundle.clone(), + mode: PortableV2Mode::Full, + limits: PortableV2Limits::default(), + }, + None, + ) + .unwrap(); + drop(graph); + let target = directory.path().join("imported"); + GraphForge::import_portable_v2( + &target, + &PortableV2ImportRequest { + input: bundle, + operation_id: operation(9), + limits: PortableV2Limits::default(), + }, + None, + ) + .unwrap(); + let imported = GraphForge::new(target.to_str()).unwrap(); + assert_eq!(count(&imported, "MATCH (n) RETURN count(n)"), 16); + assert_eq!(count(&imported, "MATCH ()-[r]->() RETURN count(r)"), 15); + assert!(!exported.package_digest.is_empty()); + // The deleted UUID stays spent after the round trip. + assert!( + imported + .publish_bulk_nodes(operation(10), &[node_batch(&[fresh])]) + .is_err() + ); +} + +#[test] +fn a_new_project_carries_no_membership_index_and_round_trips() { + let directory = tempfile::tempdir().unwrap(); + let project = directory.path().join("project"); + let graph = GraphForge::new(project.to_str()).unwrap(); + let nodes = (0..6).map(|index| id(1, index)).collect::>(); + graph + .publish_bulk_nodes(operation(1), &[node_batch(&nodes)]) + .unwrap(); + for index in 0..6_u128 { + graph + .publish_bulk_edges( + operation(10 + index), + &[edge_batch( + id(2, index), + nodes[usize::try_from(index).unwrap()], + nodes[usize::try_from((index + 1) % 6).unwrap()], + )], + ) + .unwrap(); + } + graph + .execute("CREATE (:Person {name: 'a'})-[:KNOWS]->(:Person {name: 'b'})") + .unwrap(); + assert_eq!(count(&graph, "MATCH (n) RETURN count(n)"), 8); + assert_eq!(count(&graph, "MATCH ()-[r]->() RETURN count(r)"), 7); + + // Neither the bulk publication nor the writer commit produced the index. + let entries = membership_entries(&project); + let index = entries + .iter() + .filter(|path| { + path.ends_with("/manifest.json") + || path.ends_with("/topology-receipt.json") + || path.contains("-v5-") + }) + .collect::>(); + assert!(index.is_empty(), "membership index published: {index:?}"); + + // Refusals are answered from the Parquet written by those commits. + let duplicate_node = graph + .publish_bulk_nodes(operation(40), &[node_batch(&[nodes[2]])]) + .unwrap_err() + .to_string(); + assert!( + duplicate_node.contains("identity_conflict"), + "{duplicate_node}" + ); + let duplicate_edge = graph + .publish_bulk_edges(operation(41), &[edge_batch(id(2, 3), nodes[0], nodes[1])]) + .unwrap_err() + .to_string(); + assert!( + duplicate_edge.contains("identity_conflict"), + "{duplicate_edge}" + ); + let missing = graph + .publish_bulk_edges( + operation(42), + &[edge_batch(id(2, 9_000), nodes[0], id(1, 9_000))], + ) + .unwrap_err() + .to_string(); + assert!(missing.contains("missing_endpoint"), "{missing}"); + + let bundle = directory.path().join("new.gfpb"); + graph + .export_portable_v2( + &PortableV2ExportRequest { + selection: PortableSelection::Current, + output_path: bundle.clone(), + representation: PortableV2Output::Bundle, + profile: PortableV2SelectionProfile::Complete, + subset: None, + limits: PortableV2Limits::default(), + }, + None, + |_| {}, + ) + .unwrap(); + assert!( + !bundle_names_membership_index(&bundle), + "the bundle must not carry the membership index" + ); + verify_portable_v2( + &PortableVerifyRequest { + input: bundle.clone(), + mode: PortableV2Mode::Full, + limits: PortableV2Limits::default(), + }, + None, + ) + .unwrap(); + drop(graph); + let target = directory.path().join("imported"); + GraphForge::import_portable_v2( + &target, + &PortableV2ImportRequest { + input: bundle, + operation_id: operation(50), + limits: PortableV2Limits::default(), + }, + None, + ) + .unwrap(); + let imported = GraphForge::new(target.to_str()).unwrap(); + assert_eq!(count(&imported, "MATCH (n) RETURN count(n)"), 8); + assert_eq!(count(&imported, "MATCH ()-[r]->() RETURN count(r)"), 7); + assert!( + imported + .publish_bulk_nodes(operation(51), &[node_batch(&[nodes[4]])]) + .unwrap_err() + .to_string() + .contains("identity_conflict") + ); + assert!(membership_entries(&target).iter().all(|path| { + !path.ends_with("/manifest.json") + && !path.ends_with("/topology-receipt.json") + && !path.contains("-v5-") + })); +} diff --git a/tests/fixtures/legacy-membership-index/README.md b/tests/fixtures/legacy-membership-index/README.md new file mode 100644 index 000000000..624039c8f --- /dev/null +++ b/tests/fixtures/legacy-membership-index/README.md @@ -0,0 +1,41 @@ +# Legacy membership index fixture + +A real durable Project written before #1902 by the binary that produced the UUID +membership index (`topology/uuid-membership/manifest.json`, +`identities-v5-*.uuidx`, `node-surrogates-v5-*.uuidx`, `topology-receipt.json`). +`crates/graphforge-api/tests/legacy_membership_index.rs` copies it to a private +directory and proves that such a project still opens, appends, refuses +duplicates, exports, verifies and imports once nothing reads the index. + +Contents: 14 `Person` nodes and 14 `KNOWS` edges from two `import-session` +commits. The first builds eight nodes and a ring of eight edges (bulk builder); +the second appends six nodes and a ring of six edges (staged path, so the index +carries a delta run). UUIDs are v7-shaped and derived from the node or edge +index: `(kind << 100) | (0x7 << 76) | (0x2 << 62) | index` with kind 1 for +nodes and 2 for edges; the base uses indexes 0-7 and the append 100-105. + +Empty directories are not checked in; the Project recreates them. Git does not +preserve the producer's read-only CAS permissions, so the test restores them on +the copied `graph-objects/sha256/` payloads. + +## Regenerate + +Build `gf` from the parent of the #1902 commit (the index is gone after it) and +run, with `TMPDIR` on ext4: + +```bash +mkdir base append +python3 tiny_data.py base 0 8 0 # nodes.parquet and edges.parquet +python3 tiny_data.py append 100 6 100 +gf --project project import-session begin --operation-uuid 00000000-0000-4000-8000-000000001001 +gf --project project import-session register-parquet --session-uuid 00000000-0000-4000-8000-000000001001 --path base/nodes.parquet --kind nodes +gf --project project import-session register-parquet --session-uuid 00000000-0000-4000-8000-000000001001 --path base/edges.parquet --kind edges +gf --project project import-session validate --session-uuid 00000000-0000-4000-8000-000000001001 +gf --project project import-session commit --session-uuid 00000000-0000-4000-8000-000000001001 +# repeat with session 00000000-0000-4000-8000-000000001002 and the append directory +``` + +`tiny_data.py` writes `node_uuid`/`label` and `edge_uuid`/`rel_type`/`source_uuid`/ +`target_uuid` columns with the UUID formula above and a ring of edges over its +nodes. Delete `.graphforge-construction`, `.graphforge-query-spill` and +`import-sessions` from the result. diff --git a/tests/fixtures/legacy-membership-index/project/CURRENT b/tests/fixtures/legacy-membership-index/project/CURRENT new file mode 100644 index 000000000..a2d225501 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/CURRENT @@ -0,0 +1 @@ +{"format":"graphforge-project","format_version":1,"generation_uuid":"8761079d-72ee-81c2-94ea-d73798ea17d4","generation_manifest_sha256":"852bfdb896544aa625cbcc1a85497768d0be6b6f9dd8284224a18f706d76f3b8"} diff --git a/tests/fixtures/legacy-membership-index/project/FORMAT b/tests/fixtures/legacy-membership-index/project/FORMAT new file mode 100644 index 000000000..3ad95f0d2 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/FORMAT @@ -0,0 +1 @@ +graphforge-project/v1 diff --git a/tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/lease.lock b/tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/lease.lock new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/manifest.json b/tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/manifest.json new file mode 100644 index 000000000..2c00aff7c --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/manifest.json @@ -0,0 +1 @@ +{"format":"graphforge-generation","format_version":2,"generation_uuid":"01a119c6-9b54-70ad-9ed1-62298f5582d6","parent_generation_uuid":null,"transaction_uuid":"01a119c6-9b54-70ad-9ed1-622ac523a769","capabilities":[{"capability_id":"graph","capability_version":1},{"capability_id":"workspace","capability_version":1}],"participants":[{"capability_id":"workspace","capability_version":1,"record_family_id":"configuration","record_version":1,"relative_path":"workspace/configuration.json","encoding":"json","byte_length":105,"row_count":1,"schema_fingerprint":"359fa910781f485fbe4eefc5327e981540bfbfc169aed6e4f565636f04eb7d21","content_sha256":"5bf75875f9e3215c3f9036f0492c9c9a7209d3a62abf22d42136614f35137b34","content_xxh64":"c85fd5f006b1a579"},{"capability_id":"workspace","capability_version":1,"record_family_id":"ontology","record_version":1,"relative_path":"workspace/ontology.json","encoding":"json","byte_length":117,"row_count":1,"schema_fingerprint":"732e262e0ecf22210bc3cab8e480fe0d718a9fc38b8dd5ad921f116106a13bcb","content_sha256":"a21dd016f4ba02eca7cc43737d341d99fec00bcb5fb082e2a9877aaa4f27b824","content_xxh64":"476af575fffa3ddf"},{"capability_id":"workspace","capability_version":1,"record_family_id":"research_metadata","record_version":1,"relative_path":"workspace/research_metadata.json","encoding":"json","byte_length":759,"row_count":1,"schema_fingerprint":"f8316568e127c2a94c17848b5782442bd465d263700b118720ff2d1a4ab4b7dd","content_sha256":"9fb1fcb124a9c701ade4c3bcbebcec8ff2331f4e19152fecfd305f6db0595301","content_xxh64":"e8fc631e25f94a72"}]} diff --git a/tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/participants/workspace/configuration.json b/tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/participants/workspace/configuration.json new file mode 100644 index 000000000..47c5cf7b1 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/participants/workspace/configuration.json @@ -0,0 +1 @@ +{"contract_version":1,"ontology_mode":"none","capability_configuration":{},"embedding_configuration":{}} diff --git a/tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/participants/workspace/ontology.json b/tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/participants/workspace/ontology.json new file mode 100644 index 000000000..2d53f997b --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/participants/workspace/ontology.json @@ -0,0 +1 @@ +{"contract_version":1,"mode":"none","source_format":null,"canonical_ontology_sha256":null,"canonical_ontology":null} diff --git a/tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/participants/workspace/research_metadata.json b/tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/participants/workspace/research_metadata.json new file mode 100644 index 000000000..8674b9a78 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/generations/01a119c6-9b54-70ad-9ed1-62298f5582d6/participants/workspace/research_metadata.json @@ -0,0 +1 @@ +{"contract_version":1,"title":null,"description":null,"authors":[],"subjects":[],"languages":[],"geographic_coverage":null,"temporal_coverage":null,"source_types":[],"corpus_size":null,"ontologies":[],"license":null,"access":{"visibility":null,"access_policy":null,"collaborators":[]},"tags":[],"originating_projects":[],"related_projects":[],"canonical_identifiers":[],"external_identifiers":[],"created_at":null,"updated_at":null,"extensions":{},"discovery_facets":{"text_entry_points":0,"source_entry_points":0,"entity_entry_points":0,"relationship_entry_points":0,"story_document_entry_points":0,"event_location_entry_points":0,"ontology_type_entry_points":0,"linguistic_property_entry_points":0,"traversal_query_entry_points":0,"branch_entry_points":0}} diff --git a/tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/lease.lock b/tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/lease.lock new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/manifest.json b/tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/manifest.json new file mode 100644 index 000000000..529f51d24 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/manifest.json @@ -0,0 +1 @@ +{"format":"graphforge-generation","format_version":2,"generation_uuid":"8761079d-72ee-81c2-94ea-d73798ea17d4","parent_generation_uuid":"e0317b39-4a77-820d-8d18-48ca198328b3","transaction_uuid":"971c15f0-5ca9-8b07-8535-90dc9f83d6fa","capabilities":[{"capability_id":"graph","capability_version":1},{"capability_id":"workspace","capability_version":1}],"participants":[{"capability_id":"graph","capability_version":1,"record_family_id":"files","record_version":8,"relative_path":"graph/files.json","encoding":"json","byte_length":198,"row_count":34,"schema_fingerprint":"dfe97b1acfed5c36c682235975d5bfb4d0cfbed934c92941da10466b7efc3a33","content_sha256":"442c4a703410b968148247c98d5af6157cd696d40fd9be22c75ef80bb39904be","content_xxh64":"92fc5f852cc27ac2"},{"capability_id":"workspace","capability_version":1,"record_family_id":"configuration","record_version":1,"relative_path":"workspace/configuration.json","encoding":"json","byte_length":105,"row_count":1,"schema_fingerprint":"359fa910781f485fbe4eefc5327e981540bfbfc169aed6e4f565636f04eb7d21","content_sha256":"5bf75875f9e3215c3f9036f0492c9c9a7209d3a62abf22d42136614f35137b34","content_xxh64":"c85fd5f006b1a579"},{"capability_id":"workspace","capability_version":1,"record_family_id":"ontology","record_version":1,"relative_path":"workspace/ontology.json","encoding":"json","byte_length":117,"row_count":1,"schema_fingerprint":"732e262e0ecf22210bc3cab8e480fe0d718a9fc38b8dd5ad921f116106a13bcb","content_sha256":"a21dd016f4ba02eca7cc43737d341d99fec00bcb5fb082e2a9877aaa4f27b824","content_xxh64":"476af575fffa3ddf"},{"capability_id":"workspace","capability_version":1,"record_family_id":"research_metadata","record_version":1,"relative_path":"workspace/research_metadata.json","encoding":"json","byte_length":759,"row_count":1,"schema_fingerprint":"f8316568e127c2a94c17848b5782442bd465d263700b118720ff2d1a4ab4b7dd","content_sha256":"9fb1fcb124a9c701ade4c3bcbebcec8ff2331f4e19152fecfd305f6db0595301","content_xxh64":"e8fc631e25f94a72"}]} diff --git a/tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/participants/graph/files.json b/tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/participants/graph/files.json new file mode 100644 index 000000000..fb581bcd9 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/participants/graph/files.json @@ -0,0 +1 @@ +{"format":"graphforge-graph-files-root","format_version":8,"root_node_sha256":"8011b9909d36a6051a3ddd1378ba581c5b4be7221950372439117b24934f7578","logical_file_count":34,"logical_byte_length":31004} diff --git a/tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/participants/workspace/configuration.json b/tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/participants/workspace/configuration.json new file mode 100644 index 000000000..47c5cf7b1 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/participants/workspace/configuration.json @@ -0,0 +1 @@ +{"contract_version":1,"ontology_mode":"none","capability_configuration":{},"embedding_configuration":{}} diff --git a/tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/participants/workspace/ontology.json b/tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/participants/workspace/ontology.json new file mode 100644 index 000000000..2d53f997b --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/participants/workspace/ontology.json @@ -0,0 +1 @@ +{"contract_version":1,"mode":"none","source_format":null,"canonical_ontology_sha256":null,"canonical_ontology":null} diff --git a/tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/participants/workspace/research_metadata.json b/tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/participants/workspace/research_metadata.json new file mode 100644 index 000000000..8674b9a78 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/generations/8761079d-72ee-81c2-94ea-d73798ea17d4/participants/workspace/research_metadata.json @@ -0,0 +1 @@ +{"contract_version":1,"title":null,"description":null,"authors":[],"subjects":[],"languages":[],"geographic_coverage":null,"temporal_coverage":null,"source_types":[],"corpus_size":null,"ontologies":[],"license":null,"access":{"visibility":null,"access_policy":null,"collaborators":[]},"tags":[],"originating_projects":[],"related_projects":[],"canonical_identifiers":[],"external_identifiers":[],"created_at":null,"updated_at":null,"extensions":{},"discovery_facets":{"text_entry_points":0,"source_entry_points":0,"entity_entry_points":0,"relationship_entry_points":0,"story_document_entry_points":0,"event_location_entry_points":0,"ontology_type_entry_points":0,"linguistic_property_entry_points":0,"traversal_query_entry_points":0,"branch_entry_points":0}} diff --git a/tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/lease.lock b/tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/lease.lock new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/manifest.json b/tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/manifest.json new file mode 100644 index 000000000..44719f365 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/manifest.json @@ -0,0 +1 @@ +{"format":"graphforge-generation","format_version":2,"generation_uuid":"e0317b39-4a77-820d-8d18-48ca198328b3","parent_generation_uuid":"01a119c6-9b54-70ad-9ed1-62298f5582d6","transaction_uuid":"6542a196-4938-8837-b381-67d6abd75857","capabilities":[{"capability_id":"graph","capability_version":1},{"capability_id":"workspace","capability_version":1}],"participants":[{"capability_id":"graph","capability_version":1,"record_family_id":"files","record_version":8,"relative_path":"graph/files.json","encoding":"json","byte_length":198,"row_count":27,"schema_fingerprint":"dfe97b1acfed5c36c682235975d5bfb4d0cfbed934c92941da10466b7efc3a33","content_sha256":"5eb8653860de2e90bc27ccd960ce679198e54141bd1124514f41db7fa1d0d313","content_xxh64":"04c21e818389c331"},{"capability_id":"workspace","capability_version":1,"record_family_id":"configuration","record_version":1,"relative_path":"workspace/configuration.json","encoding":"json","byte_length":105,"row_count":1,"schema_fingerprint":"359fa910781f485fbe4eefc5327e981540bfbfc169aed6e4f565636f04eb7d21","content_sha256":"5bf75875f9e3215c3f9036f0492c9c9a7209d3a62abf22d42136614f35137b34","content_xxh64":"c85fd5f006b1a579"},{"capability_id":"workspace","capability_version":1,"record_family_id":"ontology","record_version":1,"relative_path":"workspace/ontology.json","encoding":"json","byte_length":117,"row_count":1,"schema_fingerprint":"732e262e0ecf22210bc3cab8e480fe0d718a9fc38b8dd5ad921f116106a13bcb","content_sha256":"a21dd016f4ba02eca7cc43737d341d99fec00bcb5fb082e2a9877aaa4f27b824","content_xxh64":"476af575fffa3ddf"},{"capability_id":"workspace","capability_version":1,"record_family_id":"research_metadata","record_version":1,"relative_path":"workspace/research_metadata.json","encoding":"json","byte_length":759,"row_count":1,"schema_fingerprint":"f8316568e127c2a94c17848b5782442bd465d263700b118720ff2d1a4ab4b7dd","content_sha256":"9fb1fcb124a9c701ade4c3bcbebcec8ff2331f4e19152fecfd305f6db0595301","content_xxh64":"e8fc631e25f94a72"}]} diff --git a/tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/participants/graph/files.json b/tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/participants/graph/files.json new file mode 100644 index 000000000..dee6005c0 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/participants/graph/files.json @@ -0,0 +1 @@ +{"format":"graphforge-graph-files-root","format_version":8,"root_node_sha256":"92040d2b83ae59ee9d9f621e9e2160083a7cc4f7198d7c07670c56d05c9d4223","logical_file_count":27,"logical_byte_length":23870} diff --git a/tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/participants/workspace/configuration.json b/tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/participants/workspace/configuration.json new file mode 100644 index 000000000..47c5cf7b1 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/participants/workspace/configuration.json @@ -0,0 +1 @@ +{"contract_version":1,"ontology_mode":"none","capability_configuration":{},"embedding_configuration":{}} diff --git a/tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/participants/workspace/ontology.json b/tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/participants/workspace/ontology.json new file mode 100644 index 000000000..2d53f997b --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/participants/workspace/ontology.json @@ -0,0 +1 @@ +{"contract_version":1,"mode":"none","source_format":null,"canonical_ontology_sha256":null,"canonical_ontology":null} diff --git a/tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/participants/workspace/research_metadata.json b/tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/participants/workspace/research_metadata.json new file mode 100644 index 000000000..8674b9a78 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/generations/e0317b39-4a77-820d-8d18-48ca198328b3/participants/workspace/research_metadata.json @@ -0,0 +1 @@ +{"contract_version":1,"title":null,"description":null,"authors":[],"subjects":[],"languages":[],"geographic_coverage":null,"temporal_coverage":null,"source_types":[],"corpus_size":null,"ontologies":[],"license":null,"access":{"visibility":null,"access_policy":null,"collaborators":[]},"tags":[],"originating_projects":[],"related_projects":[],"canonical_identifiers":[],"external_identifiers":[],"created_at":null,"updated_at":null,"extensions":{},"discovery_facets":{"text_entry_points":0,"source_entry_points":0,"entity_entry_points":0,"relationship_entry_points":0,"story_document_entry_points":0,"event_location_entry_points":0,"ontology_type_entry_points":0,"linguistic_property_entry_points":0,"traversal_query_entry_points":0,"branch_entry_points":0}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/lifecycle.lock b/tests/fixtures/legacy-membership-index/project/graph-objects/lifecycle.lock new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/00/ee3eb40c4e58c1398b4948772d6ea329f0e19e75aabbce863d4241c5a93794 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/00/ee3eb40c4e58c1398b4948772d6ea329f0e19e75aabbce863d4241c5a93794 new file mode 100644 index 000000000..0a9f03553 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/00/ee3eb40c4e58c1398b4948772d6ea329f0e19e75aabbce863d4241c5a93794 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"0":"a547e1515d6437ddb4c530f805e4a9f6fde3f42fb3725c862e5ac3675faec543","1":"917ceda6a0f6fbf916cdf7191151a8d7b2a2d4b38aca4e426d43f93a7dd718d9","2":"8f37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac","3":"33d5b384bd457bf148bc1711beb3544d41d40c9499d4e454c0660a173ee69222","4":"18d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034","5":"47ea851bc5d6f0806954e1b36477c5a61b5fe18044fe79cfbbc304f04d052ea6","6":"023f0c4332a566cb6fbb2fb4078aa318c51f2598960381f693d66f96dbabca03","8":"7056644ec93e357f016906a0715f49cb92d9e87440d6e0116e51e7b65bd51821","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","d":"963c4f9708e0cffbdf90aa9b3f86bebe0aff18acd05b69c715a146ff6f6c0967","f":"4e1e1b01ecbd59456aa67778d9ad0fc39e3fe4961e4ab4f35f83cd12a3c6a220"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/01/8df9d340c99cf4504542b466026a55c55296ed77889db182081c89dd009673 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/01/8df9d340c99cf4504542b466026a55c55296ed77889db182081c89dd009673 new file mode 100644 index 000000000..8fc2f1ab8 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/01/8df9d340c99cf4504542b466026a55c55296ed77889db182081c89dd009673 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"0":"a547e1515d6437ddb4c530f805e4a9f6fde3f42fb3725c862e5ac3675faec543","1":"917ceda6a0f6fbf916cdf7191151a8d7b2a2d4b38aca4e426d43f93a7dd718d9","2":"8f37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac","3":"836ec17534a48b6cfdf1a977725d6b05ce9c9e114242590381e5c4d357aad9ac","4":"18d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034","5":"47ea851bc5d6f0806954e1b36477c5a61b5fe18044fe79cfbbc304f04d052ea6","6":"023f0c4332a566cb6fbb2fb4078aa318c51f2598960381f693d66f96dbabca03","7":"b4684cec290b5fd2220e9c6171d702a04277ef7d63ac3d3a1ee4f16858f99bfb","8":"7056644ec93e357f016906a0715f49cb92d9e87440d6e0116e51e7b65bd51821","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","b":"64d0aca220e3dbc30007411119041cd6bc8a807580c02e4d994b0f73e0a58bfa","d":"fd940e96ef6d179d9bf35e1bbd7f8786b2f66bed48c07f9b3d878e2f4eb42b7a","f":"8b515d817bdd27fa39dc155b59753fd24eca05413bdf7333b3035da95039ae7d"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/02/3f0c4332a566cb6fbb2fb4078aa318c51f2598960381f693d66f96dbabca03 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/02/3f0c4332a566cb6fbb2fb4078aa318c51f2598960381f693d66f96dbabca03 new file mode 100644 index 000000000..05c861b9e --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/02/3f0c4332a566cb6fbb2fb4078aa318c51f2598960381f693d66f96dbabca03 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.out.csr.json","byte_length":537,"content_sha256":"6aa5e92b174b286e8c36e861a57d34a3d93fad72e114bf9cb7a6ceb0a234b9fb","content_xxh64":"29c320fbcfa4cf82","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.out.csr.shards-312746a9f2902762689369ed.d/00000000000000000000.csr","byte_length":1866,"content_sha256":"c4dc71e92a7313804d090463634dc5c054599b0f764d6409999b9e2f001a1691","content_xxh64":"3d505128daacb6b6","role":"index"},{"relative_path":"topology/uuid-membership/manifest.json","byte_length":1430,"content_sha256":"1598fdfecd2ba38b695b328c8360d56ec62b173802e77ddcbac9cd95144490b5","content_xxh64":"786c190f7cf2bba4","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/04/726121dffa2353c921ad78b4ce93c74a9dedaea58238f7c49e2aa6c72e522e b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/04/726121dffa2353c921ad78b4ce93c74a9dedaea58238f7c49e2aa6c72e522e new file mode 100644 index 000000000..3011202c2 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/04/726121dffa2353c921ad78b4ce93c74a9dedaea58238f7c49e2aa6c72e522e @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"1":"16caf5a94acbf6efbbc859d3cb210f945081ecbaa903a1a3321103366ae8cf4f","2":"8f37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac","3":"33d5b384bd457bf148bc1711beb3544d41d40c9499d4e454c0660a173ee69222","4":"18d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034","5":"47ea851bc5d6f0806954e1b36477c5a61b5fe18044fe79cfbbc304f04d052ea6","6":"023f0c4332a566cb6fbb2fb4078aa318c51f2598960381f693d66f96dbabca03","8":"7056644ec93e357f016906a0715f49cb92d9e87440d6e0116e51e7b65bd51821","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","d":"963c4f9708e0cffbdf90aa9b3f86bebe0aff18acd05b69c715a146ff6f6c0967","f":"4e1e1b01ecbd59456aa67778d9ad0fc39e3fe4961e4ab4f35f83cd12a3c6a220"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/06/699b216f61cd4e53f0c3285cb4d720da7f8fa6c21e62c28c2399adec6b0378 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/06/699b216f61cd4e53f0c3285cb4d720da7f8fa6c21e62c28c2399adec6b0378 new file mode 100644 index 000000000..db1752c84 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/06/699b216f61cd4e53f0c3285cb4d720da7f8fa6c21e62c28c2399adec6b0378 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.out.csr.json","byte_length":537,"content_sha256":"6aa5e92b174b286e8c36e861a57d34a3d93fad72e114bf9cb7a6ceb0a234b9fb","content_xxh64":"29c320fbcfa4cf82","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.out.csr.shards-312746a9f2902762689369ed.d/00000000000000000000.csr","byte_length":1866,"content_sha256":"c4dc71e92a7313804d090463634dc5c054599b0f764d6409999b9e2f001a1691","content_xxh64":"3d505128daacb6b6","role":"index"},{"relative_path":"topology/uuid-membership/identities-v5-l1-2-bfd3d2ea93bb986c.uuidx","byte_length":588,"content_sha256":"bfd3d2ea93bb986cf0959a2775eb9d93f31f87e9b1e28f4998d0a129d0fd5572","content_xxh64":"4371210ab0c1c222","role":"topology"},{"relative_path":"topology/uuid-membership/manifest.json","byte_length":1430,"content_sha256":"1598fdfecd2ba38b695b328c8360d56ec62b173802e77ddcbac9cd95144490b5","content_xxh64":"786c190f7cf2bba4","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/07/fb10f5a76a039c4e3792c8c3ceb75d1a14bdc17322dec690a11dd417655c99 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/07/fb10f5a76a039c4e3792c8c3ceb75d1a14bdc17322dec690a11dd417655c99 new file mode 100644 index 000000000..a2a91683d --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/07/fb10f5a76a039c4e3792c8c3ceb75d1a14bdc17322dec690a11dd417655c99 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":0,"prefix":"","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/index_manifest.parquet","byte_length":2044,"content_sha256":"402e0e930d922e9ab4e149e738771406fd0b5b738c8bac89bc345a0220fbd4a7","content_xxh64":"63a77de5d97c10d5","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.in.csr.json","byte_length":536,"content_sha256":"a55c44fd9b95369708a324153fbf371c145a7ae54120be51246322c7fbbc0369","content_xxh64":"1c996730cfd3431e","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.in.csr.shards-ddfa0b39dddaf5aed1a600bb.d/00000000000000000000.csr","byte_length":1866,"content_sha256":"50b821d0ee918d37ab0baf3e39ce01a69cea34144255ddd9468181446dd2473d","content_xxh64":"b8a3f0e4d17c754c","role":"index"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/0a/14e69d5f88bdb637d098b2119c62d724d7d8850ce5a0d0d478f834b0b631a7 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/0a/14e69d5f88bdb637d098b2119c62d724d7d8850ce5a0d0d478f834b0b631a7 new file mode 100644 index 000000000..3745b4262 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/0a/14e69d5f88bdb637d098b2119c62d724d7d8850ce5a0d0d478f834b0b631a7 @@ -0,0 +1 @@ +{"format_version":6,"topology_generation":1,"forward_identities":[{"name":"forward-v4-1-e247e733949201fa.uuidx","kind":"forward_identities","generation":1,"bytes":192,"sha256":"e247e733949201fa2ef54425c1f45b4d58b891278a522ee4339d42873c1fe920","xxh64":"8c4ee7070b7bfbdd"}],"ordinal_ranges":[{"first_node_id":1,"count":8,"artifact":{"name":"ordinal-v4-1-599c59b6e57e65a3.uuidx","kind":"ordinal_uuids","generation":1,"bytes":128,"sha256":"599c59b6e57e65a305ee43a3c4f9368e52bcd3558337e9d59ee36b93fa20ad64","xxh64":"593488175e7a60f4"},"blocks":[{"offset":0,"count":8,"xxh64":"593488175e7a60f4"}]}],"tombstones":[{"generation":1,"artifact":{"name":"tombstones-v4-1-e3b0c44298fc1c14.uuidx","kind":"node_tombstones","generation":1,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","xxh64":"ef46db3751d8e999"},"blocks":[]}],"uuid_order_matches_ordinals":true} \ No newline at end of file diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/12/41a4163cdfaeac6b997580cb9a379747626a3a244ef88cd49e8981dce0b6f1 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/12/41a4163cdfaeac6b997580cb9a379747626a3a244ef88cd49e8981dce0b6f1 new file mode 100644 index 000000000..e30e50719 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/12/41a4163cdfaeac6b997580cb9a379747626a3a244ef88cd49e8981dce0b6f1 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"0":"a547e1515d6437ddb4c530f805e4a9f6fde3f42fb3725c862e5ac3675faec543","1":"917ceda6a0f6fbf916cdf7191151a8d7b2a2d4b38aca4e426d43f93a7dd718d9","2":"8f37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac","3":"836ec17534a48b6cfdf1a977725d6b05ce9c9e114242590381e5c4d357aad9ac","4":"18d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034","5":"47ea851bc5d6f0806954e1b36477c5a61b5fe18044fe79cfbbc304f04d052ea6","6":"023f0c4332a566cb6fbb2fb4078aa318c51f2598960381f693d66f96dbabca03","7":"b4684cec290b5fd2220e9c6171d702a04277ef7d63ac3d3a1ee4f16858f99bfb","8":"7056644ec93e357f016906a0715f49cb92d9e87440d6e0116e51e7b65bd51821","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","b":"64d0aca220e3dbc30007411119041cd6bc8a807580c02e4d994b0f73e0a58bfa","d":"fd940e96ef6d179d9bf35e1bbd7f8786b2f66bed48c07f9b3d878e2f4eb42b7a","f":"f1dd0b9fb6f895682403236f1c3f497806c1b951a0c27417f3f83ba93fa06b3f"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/15/98fdfecd2ba38b695b328c8360d56ec62b173802e77ddcbac9cd95144490b5 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/15/98fdfecd2ba38b695b328c8360d56ec62b173802e77ddcbac9cd95144490b5 new file mode 100644 index 000000000..5557826cb --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/15/98fdfecd2ba38b695b328c8360d56ec62b173802e77ddcbac9cd95144490b5 @@ -0,0 +1 @@ +{"format_version":7,"base_generation":0,"current_generation":1,"live_node_count":8,"live_edge_count":8,"runs":[{"base":true,"level":0,"first_generation":0,"last_generation":0,"identities":{"name":"identities-v5-base-0-e3b0c44298fc1c14.uuidx","count":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","xxh64":"ef46db3751d8e999","blocks":[]},"node_surrogates":{"name":"node-surrogates-v5-base-0-e3b0c44298fc1c14.uuidx","count":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","xxh64":"ef46db3751d8e999","blocks":[]},"node_count":0,"edge_count":0,"deleted_node_count":0,"deleted_edge_count":0},{"base":false,"level":0,"first_generation":1,"last_generation":1,"identities":{"name":"identities-v5-1-1f39b3ef25c02359.uuidx","count":16,"sha256":"1f39b3ef25c0235991a359758f446a13640216a6345fa1a9410cd95758876530","xxh64":"5582fa352f773d0d","blocks":[{"offset":0,"len":336,"first_key":"00000010000070008000000000000000","last_key":"00000020000070008000000000000007","xxh64":"5582fa352f773d0d"}]},"node_surrogates":{"name":"node-surrogates-v5-1-ef4c65996784af52.uuidx","count":8,"sha256":"ef4c65996784af52e4d3830eed43e0601780a08f3b926b7ca76b4ea73ad9ef44","xxh64":"74d84a5f4f0f0362","blocks":[{"offset":0,"len":192,"first_key":"0000000000000001","last_key":"0000000000000008","xxh64":"74d84a5f4f0f0362"}]},"node_count":8,"edge_count":8,"deleted_node_count":0,"deleted_edge_count":0}]} \ No newline at end of file diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/16/caf5a94acbf6efbbc859d3cb210f945081ecbaa903a1a3321103366ae8cf4f b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/16/caf5a94acbf6efbbc859d3cb210f945081ecbaa903a1a3321103366ae8cf4f new file mode 100644 index 000000000..d21fefc5d --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/16/caf5a94acbf6efbbc859d3cb210f945081ecbaa903a1a3321103366ae8cf4f @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"05ecd9ada368cafc8a4e4628aaea63c4f658be03c0c52201b8bbc20ee9c7f33","kind":"bucket","entries":[{"relative_path":"topology/uuid-membership/identities-v5-1-1f39b3ef25c02359.uuidx","byte_length":336,"content_sha256":"1f39b3ef25c0235991a359758f446a13640216a6345fa1a9410cd95758876530","content_xxh64":"5582fa352f773d0d","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/18/450af8e920af1a4a64bf48c6cf10ca9a08eadbc08f8c06485be33236de4548 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/18/450af8e920af1a4a64bf48c6cf10ca9a08eadbc08f8c06485be33236de4548 new file mode 100644 index 000000000..702e5916d --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/18/450af8e920af1a4a64bf48c6cf10ca9a08eadbc08f8c06485be33236de4548 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/index_manifest.parquet","byte_length":2044,"content_sha256":"402e0e930d922e9ab4e149e738771406fd0b5b738c8bac89bc345a0220fbd4a7","content_xxh64":"63a77de5d97c10d5","role":"index"},{"relative_path":"topology/uuid-membership/node-surrogates-v5-base-0-e3b0c44298fc1c14.uuidx","byte_length":0,"content_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","content_xxh64":"ef46db3751d8e999","role":"topology"},{"relative_path":"topology/uuid-membership/ordinal-v4-receipt.json","byte_length":244,"content_sha256":"e5b0ea5f7a810bc0e143e036c24623677c27f30033edb5296adfbaa17059d6e4","content_xxh64":"9c81e44b5a9321c8","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/18/ba16da3410baff2d0f29409b0d89fec35c41b805ef42dabcfa193213808335 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/18/ba16da3410baff2d0f29409b0d89fec35c41b805ef42dabcfa193213808335 new file mode 100644 index 000000000..f50e5b005 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/18/ba16da3410baff2d0f29409b0d89fec35c41b805ef42dabcfa193213808335 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"1":"16caf5a94acbf6efbbc859d3cb210f945081ecbaa903a1a3321103366ae8cf4f","2":"8f37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac","3":"33d5b384bd457bf148bc1711beb3544d41d40c9499d4e454c0660a173ee69222","4":"61dd5bd3012ddea71341be7ed0b0e7c21c87327ac4d836727bc90a121e30a7b3","6":"7b240572eaf4cc550d8b5d6d1477f0a90e73b98919d38d383aabe55c69b6115d","8":"7056644ec93e357f016906a0715f49cb92d9e87440d6e0116e51e7b65bd51821","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","d":"670c7acbbcde62dd0cfb6b0e75e7a685203d18442246974390cc0e8ca5682c10","f":"4e1e1b01ecbd59456aa67778d9ad0fc39e3fe4961e4ab4f35f83cd12a3c6a220"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/18/d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/18/d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034 new file mode 100644 index 000000000..e3e773a94 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/18/d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"","kind":"bucket","entries":[{"relative_path":"topology/uuid-membership/forward-v4-1-e247e733949201fa.uuidx","byte_length":192,"content_sha256":"e247e733949201fa2ef54425c1f45b4d58b891278a522ee4339d42873c1fe920","content_xxh64":"8c4ee7070b7bfbdd","role":"topology"},{"relative_path":"topology/uuid-membership/node-surrogates-v5-1-ef4c65996784af52.uuidx","byte_length":192,"content_sha256":"ef4c65996784af52e4d3830eed43e0601780a08f3b926b7ca76b4ea73ad9ef44","content_xxh64":"74d84a5f4f0f0362","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/1d/2d822abf959ff7f9399c174b463c7f9464b414fcae836c0288d37698c4e062 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/1d/2d822abf959ff7f9399c174b463c7f9464b414fcae836c0288d37698c4e062 new file mode 100644 index 000000000..d36fad49f --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/1d/2d822abf959ff7f9399c174b463c7f9464b414fcae836c0288d37698c4e062 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"1":"16caf5a94acbf6efbbc859d3cb210f945081ecbaa903a1a3321103366ae8cf4f","2":"8f37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac","3":"33d5b384bd457bf148bc1711beb3544d41d40c9499d4e454c0660a173ee69222","4":"18d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034","5":"47ea851bc5d6f0806954e1b36477c5a61b5fe18044fe79cfbbc304f04d052ea6","6":"023f0c4332a566cb6fbb2fb4078aa318c51f2598960381f693d66f96dbabca03","8":"7056644ec93e357f016906a0715f49cb92d9e87440d6e0116e51e7b65bd51821","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","d":"670c7acbbcde62dd0cfb6b0e75e7a685203d18442246974390cc0e8ca5682c10","f":"4e1e1b01ecbd59456aa67778d9ad0fc39e3fe4961e4ab4f35f83cd12a3c6a220"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/1d/360c9fafd9175352328bb2e29c2b74252d96b6d9042af01695264166840522 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/1d/360c9fafd9175352328bb2e29c2b74252d96b6d9042af01695264166840522 new file mode 100644 index 000000000..7b1dbd6be --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/1d/360c9fafd9175352328bb2e29c2b74252d96b6d9042af01695264166840522 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"0":"a547e1515d6437ddb4c530f805e4a9f6fde3f42fb3725c862e5ac3675faec543","1":"917ceda6a0f6fbf916cdf7191151a8d7b2a2d4b38aca4e426d43f93a7dd718d9","2":"8f37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac","3":"836ec17534a48b6cfdf1a977725d6b05ce9c9e114242590381e5c4d357aad9ac","4":"18d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034","5":"47ea851bc5d6f0806954e1b36477c5a61b5fe18044fe79cfbbc304f04d052ea6","6":"06699b216f61cd4e53f0c3285cb4d720da7f8fa6c21e62c28c2399adec6b0378","7":"b4684cec290b5fd2220e9c6171d702a04277ef7d63ac3d3a1ee4f16858f99bfb","8":"ef58edb5e714477964e6c29aa9ea6e80947355b529995d3e7e8116d59a643069","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","b":"64d0aca220e3dbc30007411119041cd6bc8a807580c02e4d994b0f73e0a58bfa","d":"deb33edafa58e17727033ee8936ea7a9ad80976f7a8e5e4e2b722dbd55228a51","f":"8b515d817bdd27fa39dc155b59753fd24eca05413bdf7333b3035da95039ae7d"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/1d/bbabd6dc589031a0c56ed34bd24842edad4020a4bec74bcea6c31486cdc36f b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/1d/bbabd6dc589031a0c56ed34bd24842edad4020a4bec74bcea6c31486cdc36f new file mode 100644 index 000000000..ed8d5120f --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/1d/bbabd6dc589031a0c56ed34bd24842edad4020a4bec74bcea6c31486cdc36f @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":0,"prefix":"","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/index_manifest.parquet","byte_length":2044,"content_sha256":"402e0e930d922e9ab4e149e738771406fd0b5b738c8bac89bc345a0220fbd4a7","content_xxh64":"63a77de5d97c10d5","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.in.csr.json","byte_length":536,"content_sha256":"a55c44fd9b95369708a324153fbf371c145a7ae54120be51246322c7fbbc0369","content_xxh64":"1c996730cfd3431e","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.in.csr.shards-ddfa0b39dddaf5aed1a600bb.d/00000000000000000000.csr","byte_length":1866,"content_sha256":"50b821d0ee918d37ab0baf3e39ce01a69cea34144255ddd9468181446dd2473d","content_xxh64":"b8a3f0e4d17c754c","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.out.csr.json","byte_length":537,"content_sha256":"6aa5e92b174b286e8c36e861a57d34a3d93fad72e114bf9cb7a6ceb0a234b9fb","content_xxh64":"29c320fbcfa4cf82","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.out.csr.shards-312746a9f2902762689369ed.d/00000000000000000000.csr","byte_length":1866,"content_sha256":"c4dc71e92a7313804d090463634dc5c054599b0f764d6409999b9e2f001a1691","content_xxh64":"3d505128daacb6b6","role":"index"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/1f/39b3ef25c0235991a359758f446a13640216a6345fa1a9410cd95758876530 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/1f/39b3ef25c0235991a359758f446a13640216a6345fa1a9410cd95758876530 new file mode 100644 index 0000000000000000000000000000000000000000..948c40d0702dcec96e2713a3f5dfcf938499d681 GIT binary patch literal 336 zcmZ9HyA^;y5CS>xXJ`qQXbF}s5aCY1djrWXfD3r=;!mUIAU6jy2f69Y9OPy&bC8?K k%t3AzGY7fZ%pBzAWZwCQ?#-gr;;2Qh#i+%s#j3?_9~fc~e?t^C7xk7EX9O+@xdTljYATWkdfn9vE?5rV9SA4tfciuWUW=! zE6au+N=|JE6iSb!=N<~ZxVI8|=)vb4dPuK5_S8de33+e!Pf^rTGD!R0yl>w3e&(&T z&1;KPrkCjD3J7wZkhxz9uV?aK6Oti>tZ#1Z-j_jXn2tjU%~ExqDs&2zYxDgWa7qx$F1 zmCv#t;yNK>eo$29f4FMrB~8tU=4z@edMjy+5Mh@(T}@3x(`aH-qBdx1tPaCbsQ^dn zFdUTwaHI~yQOW>EY6$DBPOoEP>=UBt>}4w4P2=u`M6Uvd>RCEgpUbg&nh8`O`=XpP zJCk1>fztj%%MZ zY^!T|!gW@!{B+?qOfF0a98sG6j^j5>e|Ea^EU(Xt>k!Cf*KJ#kmfP?8BR-Q>+kOvW zil%FhL3EnamPmF=8_AaDn8Nni)09y+6#WXB6Bij?Yd%l3d6-K=C7i!05I3NQg#D_} zi|p?VT_D5ENwtFYbM_=A?zQ8GWL6x-g+lS-0Yb6ENwhZ>&X}<8iQcba3V+U{5|8 z2{v?pak%HQ@ue*4hz2_u@v+oDAoXwWQ6~csQ=evxR{?eOQY5d(Sn5-xeuLEPnamVJ zVSi=ddNf_l@mg(b_j)k<>X}@?b!AZEoLAwCYn}4}U$*%gnm}@nD6T5(IlqT0m?831_&onb1jW zvAt9;ww>m|&dFZA)@v3o^Tll_;#g2)Qeu^=F+V>>eDDbDrAKCc&Dr}j@B;$`tA?Sj zhF59_dy#}FGT^HT5$Ga25pNR>t%IGzAeW{6cG(#`5YWw=ME%(I%-+L&*R!pep5+~w eU90QQ^i1z@-|`EKngc(mBOme&{IhSv|NdK;jGfQw%ud z5LG!?QRGAZfGX;#s;VBUs=f4-LsV5(EA`NOd+5RHp{FQ^Nd0(kb{Ez-ZJJu@$ex{f zGw=6)@Auv~S(j1)O6VgCP0dgaR(5J9r4@Sr+xj;BFM#h0Rj^Rx!8A6t$ z)PwoCO!*DJLCDb1y>G1a|AFgs8_~%9Yj!UBT9R7($vP!M{`ir6o_yC%AU7N32XnJd zKnM;_Ch6;^Tt}8c6gXR1UCUhmCtqRGD!8G3M)?_K<*aFkS_Phv-gUj?=IdtDIj%bm z*#x)KsIMvbIgbrJ0o+)cz=xS$f?NCQ2Oi!@JCHjOJp7&B>WQVdJRYJ~_45;3Z+U3% zE$=#LNWbt*TMGO<0r(H&T)x@dJJ{YYR@7Rax(FSoUP{~}s!~?B+wC32usHXd#nT35 zzoi>&)5Psy9y`jkC;8dMci_E1!+_b2ZfC9Nd&W9iY^wQ|s%+<5(^JW#3C|2GFXaGd ztf^MFTiu4bUC-ClX_EB78P5qG;mAikbX=GwcELoOay-ZXV&OYJ=lq-G(qM<}ob9=Z z2@n`s6?6gQryQN)zi`s`hzY)@K_C5&fqts{7`oY_oJPMt2Ivn)2G9*b`{++B{|Wx# z80Goj40xL`N9-=4S0M)fiG_b?@f{EOezkf)pI)cCd;N`!XW(h09x%C>EMFq>ccb(w zlf^*Ri);cuvH9-VUIzFWf$v`!1lIb*1bk@seLVa!z&|7Kw}Zf@R%75PXq3AS-($O{ z|1SG09t1L8Yv1}@fu3TxLr%|w)1^P%y)5D;f6MpG;rrTw$!PCgfA-J8ormX2_@i;^ z;U9A}NciJXp7S~RPn}cvXTi-$snlBLZS57iPaGnS%Nq-lByGSijulBdkQT~P0!{Fh zqzJRa7WR^~j5{zHU)ZrrA^fZ{GvX~QA)<(vml<>jmkcVo85YG*4)=>Vv9V>uhusF= zTSD_9TH}doAsY6p*+?}ToU0TvTX?c3mP9d*mw>N6R)DlIfFH{PaFCzI7rwyl8hj$@ z^wKpgzu0P5&P4rkCRNI1Az-8lqJ;L*iIIr(s6kTb+Po}^VO;40Q;F;rXH%to2Jf<4 zY+6_cr~?+Cfw27Jsjk9ufYi_$Q@XiqWKXVzyK-==ozKkqSbA3urZp`Y7v`_klZo4}CnL?0iujB5G}B0mAYi0ktVIvi2K!YGvBC`_L^nY+fbXed|j=#yp4SG{kGZ?1HuvfJNeun@~4B4b@*@jUlMgv AE&u=k literal 0 HcmV?d00001 diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/42/a36d76c9169a9d28be49c6d0b185b33b8dbeb94cf88b69f600425bc0a0c282 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/42/a36d76c9169a9d28be49c6d0b185b33b8dbeb94cf88b69f600425bc0a0c282 new file mode 100644 index 0000000000000000000000000000000000000000..57515141251e9e7f9bda9142d8aa7fbb30f95343 GIT binary patch literal 2381 zcmcImO>7%g5PsgQ9VcV8P`9`57C!tpm_`2P1WZ_hC9D8EuVt%@ z(n8xXS(B~G#mVrMB@nBW6SzHL>RF=O|H+i`sciz>1-^ z`j&xtL`9!p!$`$NWR4Y&6Ue=EIV4}q@KsPl^114h&w@VrB*c@11JzUXhc_dp9~u7+ew< z0FW%T%lnqocdx{58`R=aP`K_Z%vT%Eez&x5S$6zv`r%Ua0+@Qnb4_!aC=yq+h4#R< zoKn*)TTL$#q3Tinfx)6YqF-U5MSVPc9ucZI4v6mpM&AfWud?%#{ht_)r6s20FhON- zv=(7|AL`A-qj`Q#d^XFs*|-}j2RYaiUj&R}#rQse9=rk4+<}@33LOwX%=4)DZI-`F zpvo}zQAr8 zi%)0I;@#9DGMBqsxaQtE?dxEImP|@1v+&@$E#*M2*W?x+!7fRPk}U92-a--#hC6V( z@T_MMAun%$um<9dJE&L3*A0T+AyUd6G~TGmjOX%}hfneiRKk_rA$y{?j^0irR?|qA zFIRfSdt24=W~xyv4CJNEdx^vW;rsgQ^#PgT>mUOO4loH^5HgBV`JLui)KMk1 zc2q3nd&s9y|A6dAc#UCcAWJFXyzS=?yt`_WLeJ-Ou(#nX*)s2KjbSJ6W2vD!AoB6n ziBFe7vgYED98Z(|`}THo*Dls8o6UZ?xwconobOdg4sq7per*h};1m~uZj#*3f1P+Z zOV%*?EbgR?YPhJsH@8;GcCJ${Id0teTgay&~{Td!y!-NcIi0KO8h{#XV6)@0z1)IXx#!`A=+ literal 0 HcmV?d00001 diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/47/ea851bc5d6f0806954e1b36477c5a61b5fe18044fe79cfbbc304f04d052ea6 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/47/ea851bc5d6f0806954e1b36477c5a61b5fe18044fe79cfbbc304f04d052ea6 new file mode 100644 index 000000000..f0a816e7a --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/47/ea851bc5d6f0806954e1b36477c5a61b5fe18044fe79cfbbc304f04d052ea6 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"5132f0b350550041f2a0536a89e7a17f992a543f992421e9ec961f3f88509da","kind":"bucket","entries":[{"relative_path":"topology/uuid-membership/identities-v5-base-0-e3b0c44298fc1c14.uuidx","byte_length":0,"content_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","content_xxh64":"ef46db3751d8e999","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/4e/1e1b01ecbd59456aa67778d9ad0fc39e3fe4961e4ab4f35f83cd12a3c6a220 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/4e/1e1b01ecbd59456aa67778d9ad0fc39e3fe4961e4ab4f35f83cd12a3c6a220 new file mode 100644 index 000000000..fefcca81f --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/4e/1e1b01ecbd59456aa67778d9ad0fc39e3fe4961e4ab4f35f83cd12a3c6a220 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/r-5aacc116ac666e02684686f6df7f94a9fd62ee5748dd6e0daa0900d949874c8c.out.csr.json","byte_length":537,"content_sha256":"81dc0dfc6f7ed687160b947453c0c3df2cdb44fc81b8d9dc89a16262beb0448e","content_xxh64":"7c4b5c2f8061bc36","role":"index"},{"relative_path":"topology/nodes/00000000000000000001-00000000000000000008.parquet","byte_length":2284,"content_sha256":"5527b3e2c14fbe8a9cfe38a2b7ae8ce696ac7bf9fbb3ec7cbc73bd697e7047f9","content_xxh64":"85ec9a5c84886c1e","role":"topology"},{"relative_path":"topology/runtime_catalog.parquet","byte_length":2381,"content_sha256":"42a36d76c9169a9d28be49c6d0b185b33b8dbeb94cf88b69f600425bc0a0c282","content_xxh64":"428fa349ddc55542","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/4f/b465e75e9ce980b51163c62800b19a3035d06dbb7a6794027acbc8ea845554 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/4f/b465e75e9ce980b51163c62800b19a3035d06dbb7a6794027acbc8ea845554 new file mode 100644 index 000000000..e2e4eb9e4 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/4f/b465e75e9ce980b51163c62800b19a3035d06dbb7a6794027acbc8ea845554 @@ -0,0 +1 @@ +{"format_version":6,"topology_generation":2,"forward_identities":[{"name":"forward-v4-1-e247e733949201fa.uuidx","kind":"forward_identities","generation":1,"bytes":192,"sha256":"e247e733949201fa2ef54425c1f45b4d58b891278a522ee4339d42873c1fe920","xxh64":"8c4ee7070b7bfbdd"},{"name":"forward-v4-2-2a093dcf7c477b67.uuidx","kind":"forward_identities","generation":2,"bytes":144,"sha256":"2a093dcf7c477b67c309d1332a01a397938c0499a83fbc82a20edd2261b38a0a","xxh64":"b4cf8cc7d2c09c33"}],"ordinal_ranges":[{"first_node_id":1,"count":8,"artifact":{"name":"ordinal-v4-1-599c59b6e57e65a3.uuidx","kind":"ordinal_uuids","generation":1,"bytes":128,"sha256":"599c59b6e57e65a305ee43a3c4f9368e52bcd3558337e9d59ee36b93fa20ad64","xxh64":"593488175e7a60f4"},"blocks":[{"offset":0,"count":8,"xxh64":"593488175e7a60f4"}]},{"first_node_id":9,"count":6,"artifact":{"name":"ordinal-v4-2-5ddfdc0ed3f1e1d8.uuidx","kind":"ordinal_uuids","generation":2,"bytes":96,"sha256":"5ddfdc0ed3f1e1d8cd8dcc7820b8f1fc8ef7c3d0d2d1735c749ee309ecaf8c6b","xxh64":"141414f46e12a329"},"blocks":[{"offset":0,"count":6,"xxh64":"141414f46e12a329"}]}],"tombstones":[{"generation":1,"artifact":{"name":"tombstones-v4-1-e3b0c44298fc1c14.uuidx","kind":"node_tombstones","generation":1,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","xxh64":"ef46db3751d8e999"},"blocks":[]},{"generation":2,"artifact":{"name":"tombstones-v4-2-e3b0c44298fc1c14.uuidx","kind":"node_tombstones","generation":2,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","xxh64":"ef46db3751d8e999"},"blocks":[]}]} \ No newline at end of file diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/50/b821d0ee918d37ab0baf3e39ce01a69cea34144255ddd9468181446dd2473d b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/50/b821d0ee918d37ab0baf3e39ce01a69cea34144255ddd9468181446dd2473d new file mode 100644 index 0000000000000000000000000000000000000000..64edb201421f7c5f8cf666cb4bcad071b8657dbb GIT binary patch literal 1866 zcmeHIzfZzI6n@rfD`HgC#28E*olp}8$Fi}R7!wy0QsFE$3Z@pL8-I|4i#vY^|AM;< z!SB0Xflvm)#lhFSyL-RxzW44&%NG~tS0^$z4M1NDA{%;!oa7}ZD`KNHx&>QYd;(q4 ziohL`NDr|^^m#C7=LH7ZsI&FQIkxrO>pH^K;{(`D%-R_OEXZ$Q>TOxpo+1U=MXbn{ zUN;YI0n#y;SZM)S96tkxM+vkFZmKY7+_a)`ipGWBoK`O+V69jMw4EQU9RQN5OfAELp@ZECGcxtekrZZ3a)e{NP8cWu1N-@0lqz ze(zJ{-JQUnI5<<$i~NQ4iEwoON9VhXYWKjj@y7kXgW31^FTaDlEFOO@?9I*Lo{kg$ JeP#U*{RSCozZw7l literal 0 HcmV?d00001 diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/51/4c5ae39a6a439abdc276b5d218d6dd695d1b399a04a9b6bd718341e8e97a07 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/51/4c5ae39a6a439abdc276b5d218d6dd695d1b399a04a9b6bd718341e8e97a07 new file mode 100644 index 000000000..ae349547f --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/51/4c5ae39a6a439abdc276b5d218d6dd695d1b399a04a9b6bd718341e8e97a07 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"0":"a547e1515d6437ddb4c530f805e4a9f6fde3f42fb3725c862e5ac3675faec543","1":"917ceda6a0f6fbf916cdf7191151a8d7b2a2d4b38aca4e426d43f93a7dd718d9","2":"8f37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac","3":"836ec17534a48b6cfdf1a977725d6b05ce9c9e114242590381e5c4d357aad9ac","4":"18d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034","5":"47ea851bc5d6f0806954e1b36477c5a61b5fe18044fe79cfbbc304f04d052ea6","6":"2bea2e263428d5c0c085ebc737a31897d251273580869d26a0cb8d789c8fa990","7":"b4684cec290b5fd2220e9c6171d702a04277ef7d63ac3d3a1ee4f16858f99bfb","8":"f58243170c9c519373e516eaec531c28dc9bbe6301f83e56823b7590768661d1","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","b":"64d0aca220e3dbc30007411119041cd6bc8a807580c02e4d994b0f73e0a58bfa","d":"deb33edafa58e17727033ee8936ea7a9ad80976f7a8e5e4e2b722dbd55228a51","f":"8b515d817bdd27fa39dc155b59753fd24eca05413bdf7333b3035da95039ae7d"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/52/f4b97c32c0e446a0968c903ad030ef05551a8e34dbdd79ccc79cb49d77c687 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/52/f4b97c32c0e446a0968c903ad030ef05551a8e34dbdd79ccc79cb49d77c687 new file mode 100644 index 000000000..83c0e9145 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/52/f4b97c32c0e446a0968c903ad030ef05551a8e34dbdd79ccc79cb49d77c687 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"","kind":"bucket","entries":[{"relative_path":"topology/uuid-membership/identities-v5-1-1f39b3ef25c02359.uuidx","byte_length":336,"content_sha256":"1f39b3ef25c0235991a359758f446a13640216a6345fa1a9410cd95758876530","content_xxh64":"5582fa352f773d0d","role":"topology"},{"relative_path":"topology/uuid-membership/ordinal-v4-manifest.json","byte_length":1600,"content_sha256":"4fb465e75e9ce980b51163c62800b19a3035d06dbb7a6794027acbc8ea845554","content_xxh64":"d46407eaa0b0efc9","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/54/dd843ff310a71e820757e5c88417590c8336cd80323299f21e752ee139125c b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/54/dd843ff310a71e820757e5c88417590c8336cd80323299f21e752ee139125c new file mode 100644 index 000000000..822d90f7e --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/54/dd843ff310a71e820757e5c88417590c8336cd80323299f21e752ee139125c @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":0,"prefix":"","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/index_manifest.parquet","byte_length":2044,"content_sha256":"402e0e930d922e9ab4e149e738771406fd0b5b738c8bac89bc345a0220fbd4a7","content_xxh64":"63a77de5d97c10d5","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.in.csr.json","byte_length":536,"content_sha256":"a55c44fd9b95369708a324153fbf371c145a7ae54120be51246322c7fbbc0369","content_xxh64":"1c996730cfd3431e","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.in.csr.shards-ddfa0b39dddaf5aed1a600bb.d/00000000000000000000.csr","byte_length":1866,"content_sha256":"50b821d0ee918d37ab0baf3e39ce01a69cea34144255ddd9468181446dd2473d","content_xxh64":"b8a3f0e4d17c754c","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.out.csr.json","byte_length":537,"content_sha256":"6aa5e92b174b286e8c36e861a57d34a3d93fad72e114bf9cb7a6ceb0a234b9fb","content_xxh64":"29c320fbcfa4cf82","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.out.csr.shards-312746a9f2902762689369ed.d/00000000000000000000.csr","byte_length":1866,"content_sha256":"c4dc71e92a7313804d090463634dc5c054599b0f764d6409999b9e2f001a1691","content_xxh64":"3d505128daacb6b6","role":"index"},{"relative_path":"indexes/adjacency/r-5aacc116ac666e02684686f6df7f94a9fd62ee5748dd6e0daa0900d949874c8c.in.csr.json","byte_length":536,"content_sha256":"60ec16305c87e777dda418bcc099dcd86298686bbe3473d51f0173ac848078f5","content_xxh64":"cb5d79a85ec62dab","role":"index"},{"relative_path":"indexes/adjacency/r-5aacc116ac666e02684686f6df7f94a9fd62ee5748dd6e0daa0900d949874c8c.in.csr.shards-ddfa0b39dddaf5aed1a600bb.d/00000000000000000000.csr","byte_length":1866,"content_sha256":"50b821d0ee918d37ab0baf3e39ce01a69cea34144255ddd9468181446dd2473d","content_xxh64":"b8a3f0e4d17c754c","role":"index"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/55/27b3e2c14fbe8a9cfe38a2b7ae8ce696ac7bf9fbb3ec7cbc73bd697e7047f9 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/55/27b3e2c14fbe8a9cfe38a2b7ae8ce696ac7bf9fbb3ec7cbc73bd697e7047f9 new file mode 100644 index 0000000000000000000000000000000000000000..88032ca4f5dd8fe873b3061032427029fa6b4861 GIT binary patch literal 2284 zcmbtWL1-IS5S?AER%>I7ocJ%hK`m~Hqn& zf8LvUe-?J$c18qMhqr~qdlM*kYv8HvIPliX8yO!LKUd#vEo3De-^MQ+my zgJ~hL3+fk&puX7_-Si!Q9MLs#7jpwMq8~}(l4m}XMa(k~0%G1XpDIo75bxLXgOTJnrzf!KK*;2-H7bP_ zbe}=@0J%rO&^+fhHj|nhC#Pa4rS4jsSxzCP3un+-Dr z&_C`H{rZQhcozs>F-JiSn@trM{TE4pAX!I}wJ$}GqS+BfvrPzc2;UHb#3{_kUv&vx z|6LWA7+Lm_JeL!$dc>!fS7 zL6h!r(E2p^f9d`vx<^E(|ETI=OVjt&etNHr9@}xQr!Ts%hjA2)(rD~X+O~ZI|1_@H z_O3lywCBi$t8K^Gk6YZ^_AKqt<*&F~#{cChLbzs6+4el0n7m4}LlZ;CH=;Yy=rr$l zdB*9=jBjB)r9z4DDB724O{E-Mx6q1@X!&u}GJ>xX;X%~Kmyor2cc`3e?DUUTW7Z&T;^yqMAX|BGUzO$NpGu6nI$2OOj zoD$KZsEb~H*hLIE>+L)4BSuZv99S~vgR5U3w}O}dNIqTKpu=FiTC8R(YpYw;Vqu_C hsFt&vh0WSPC0o5+FVvzVW+?*D@q-M0(ahs_&Ob@3&AtEt literal 0 HcmV?d00001 diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/55/5a4a690fb10d5dd1584f0cfb05cb2396f6d41ac1679742b13aa21a745413d7 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/55/5a4a690fb10d5dd1584f0cfb05cb2396f6d41ac1679742b13aa21a745413d7 new file mode 100644 index 0000000000000000000000000000000000000000..8c7ecdc57a3e4e6f81a01df69e03d6f599efaece GIT binary patch literal 2274 zcmbtWO>7%g5T0HC#0g83w0W!DYC#4mt5!sHQWS7dRArqXC#~D2+r*A7LF#&K$HY!t z$4+Vvs0UOb#GwZc94f>SAr4%)N2uzJ3RQ>$Lgm7t2M!!K0%B%%*LI!IO7LXAH*faM z_sx87R@zm|o)RDqC1Fnp1+B0M1DyUQ_E+?JNP^n{Fal6V!zq@^Ta_?;e)9X5o@h^; z(l2UM5z}Km_ z!|)vT{~D^FkaZ39^)=>FoujUbkaZP+k4Fai6z;T$FK*A{AU>VbHC~?O)=dR;6BTel z{X!AcH+!NxzN3!~H6r#gH_#3Bu_VrW<|A3eJ@YUiRz34!u;aZSa!q8px2)tFjopLI zy~0+pmKO?$VG$Le4esnZ#m!c$;^5_vgEqkXiZgDczB)OF4F^Idce7nDZle1Xx})Tt z1})+__pq75sabLcwP3Zf*A%iF7OXUjwQdYE3>6y1e6#3m=9}YVsgFmE2@FIUVjpVN zonxMvRbrPT&hdh&YWf@F#;pReuVL^9(He zz@B6bd_mnYAIauDnQ*zA+@3lX?@Pk_bU>W##fx{YfMsX78Ty8LM=}os=0}0&W6z%P z`xMkeVtFv&W$nw+b7hQfqudXJ=G~ytZRM_|qDX#&cfH3J^&`=J5b3Auc0q^ke#pEN z`oDC)5#6^$r~VvL6*Hp#5gMb{N>eP`zLI&%eLJ+@LopidgL%ucuHv7@Maw#{=F8Rv za^Y%O3HIX__m;InJ9POg?y~q_SR{nY)}m#t(uw&?G}|;WbYeTUAB!#V{(xtkE-w2P z##1Vk2#>P0Kx;Z}CFY#T6+i zwM06-c*(6VdAXg-EKTMzb}74oMO;~XEf$LxD2S6F!@9+@s}l1e?p05kcQNxg>*v-- zM^|_s-+vSJbwg=pDU(&?!V++Bz03A=A(<}aGxmGzTwMECycE{_6dtA0iD$}dR-$Zi z9to_E0S}1x8ks4Y-1}T7lgAd#i8dm~7{T?jv^Q!q+qv{iJ?BNadhq3-zT6LZn)T&S zhF-SYPR4~_CU4SdXEzeNx!O!ScW%9v&rDC|HZHWXnOmj8>*;o(HnWr6u&YFiqON=S zVHYvvins5(k2p15XJE;k53YWF+zMX)Q%axq#jHfs5u;!ZPK Z&o^$iip|)RUNr#vevrlQm{t6``3DQw(C7dF literal 0 HcmV?d00001 diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/59/9c59b6e57e65a305ee43a3c4f9368e52bcd3558337e9d59ee36b93fa20ad64 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/59/9c59b6e57e65a305ee43a3c4f9368e52bcd3558337e9d59ee36b93fa20ad64 new file mode 100644 index 0000000000000000000000000000000000000000..64baf9b91f7a7bb7200ffb2866e981980111c0ff GIT binary patch literal 128 ocmZQzU=UznC}3z{fCFScBTy8X&xFQjM&q-f@mbOMY-oIT01dhb9RL6T literal 0 HcmV?d00001 diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/5d/dfdc0ed3f1e1d8cd8dcc7820b8f1fc8ef7c3d0d2d1735c749ee309ecaf8c6b b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/5d/dfdc0ed3f1e1d8cd8dcc7820b8f1fc8ef7c3d0d2d1735c749ee309ecaf8c6b new file mode 100644 index 0000000000000000000000000000000000000000..3b6c56768f4e29c5cd994895d86a4cf10739d333 GIT binary patch literal 96 jcmZQzU=UznC}3z{0D}}DiOf$$uT_C|Wt}zhj z5OF{#2M(NaimD!}Du<}8ha8bKsZrEpl}ep?yKAp8ZAv5c$$mfc zX1-*${Wn6s z=7@z@iH+EagE+xZ8_s!qVs+61r5sI&Gh3m@;I_aJv7`1n1p zx!tuuv+Cj(R(eg}KCm~pM~>$9sgs6vg+mwH3j8bq2M-hOLanyGy|__aRm%nHA+(nc zQsV7Wl@)bybMuyBNL(AWV!J@usOtiopLm_j;+Ep~r5>NZ4(|tA2E-0D>y)BxhqSv` zQww!fSuE831F8K!-w-P>bpuXMO)V|f8&!3&QYfo_lJdbF?-6{$et`IBuiz&R!4x)i z^B(?l3x6-*x^jtJIN4*)&aUh01B#(pffqpjvWE`vzq;th#01~upoPB28Srfl$l%Qq z(hmQuhw}XMt`qPEqXzsA7udD@C;Xp!sF(kuTgSHvqr~A6nl)nh-*fyUF7Uz{*tPnA zp=P&xn>~&p55em9zSi4p?RX%_c(E<%sRSLOyCcJ$Tc^Igf0@U5 z{!ZY~75LqW$!Kq0fBL83UV#To_~*UU$G>vZ5aC}!g9lvvFRlUn)8KVW>GVwYvi6R> z$4(K)#kny_lIGxxV@i^?rLh$$i6U4fDazzC^ z62z;IDG1sa5Fg6}a4%?>;ABY8R+3wC2x^Y_ zQ{xas7@Nw}k711k6ix!CqIPC{#nX3{SR|-qOKW&7O~1xIb;4e&N6Q|X0_w>8jc;qs z1Af@~M1Rl5nL}gBV>1H<>?tw*F`RY&lw3xuxpeg^@xHxMD^ypP*J~^4U{$S^3l+6e cAFLK?pKPjiF(~ZAf67xogo7VAv+%S1FC>ghx&QzG literal 0 HcmV?d00001 diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/63/0ff369e2d992ea22056ba6fcf1dc56feee9d16d8eaba8aa96681425181f39a b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/63/0ff369e2d992ea22056ba6fcf1dc56feee9d16d8eaba8aa96681425181f39a new file mode 100644 index 000000000..144fde644 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/63/0ff369e2d992ea22056ba6fcf1dc56feee9d16d8eaba8aa96681425181f39a @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"0":"a547e1515d6437ddb4c530f805e4a9f6fde3f42fb3725c862e5ac3675faec543","1":"917ceda6a0f6fbf916cdf7191151a8d7b2a2d4b38aca4e426d43f93a7dd718d9","2":"8f37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac","3":"836ec17534a48b6cfdf1a977725d6b05ce9c9e114242590381e5c4d357aad9ac","4":"18d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034","5":"d5cd023a1b8ff5dc2c02dc9900a54b06c2134b0e6ba5a9272d14510436c3a12c","6":"2bea2e263428d5c0c085ebc737a31897d251273580869d26a0cb8d789c8fa990","7":"b4684cec290b5fd2220e9c6171d702a04277ef7d63ac3d3a1ee4f16858f99bfb","8":"f58243170c9c519373e516eaec531c28dc9bbe6301f83e56823b7590768661d1","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","b":"64d0aca220e3dbc30007411119041cd6bc8a807580c02e4d994b0f73e0a58bfa","d":"deb33edafa58e17727033ee8936ea7a9ad80976f7a8e5e4e2b722dbd55228a51","f":"8b515d817bdd27fa39dc155b59753fd24eca05413bdf7333b3035da95039ae7d"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/63/842e81f8c2dcc54eeb68eb54aa35b324aa4da1a0844c211b9470814e2edd63 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/63/842e81f8c2dcc54eeb68eb54aa35b324aa4da1a0844c211b9470814e2edd63 new file mode 100644 index 000000000..b3e1d8c90 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/63/842e81f8c2dcc54eeb68eb54aa35b324aa4da1a0844c211b9470814e2edd63 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":0,"prefix":"","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/index_manifest.parquet","byte_length":2044,"content_sha256":"402e0e930d922e9ab4e149e738771406fd0b5b738c8bac89bc345a0220fbd4a7","content_xxh64":"63a77de5d97c10d5","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.in.csr.json","byte_length":536,"content_sha256":"a55c44fd9b95369708a324153fbf371c145a7ae54120be51246322c7fbbc0369","content_xxh64":"1c996730cfd3431e","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.in.csr.shards-ddfa0b39dddaf5aed1a600bb.d/00000000000000000000.csr","byte_length":1866,"content_sha256":"50b821d0ee918d37ab0baf3e39ce01a69cea34144255ddd9468181446dd2473d","content_xxh64":"b8a3f0e4d17c754c","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.out.csr.json","byte_length":537,"content_sha256":"6aa5e92b174b286e8c36e861a57d34a3d93fad72e114bf9cb7a6ceb0a234b9fb","content_xxh64":"29c320fbcfa4cf82","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.out.csr.shards-312746a9f2902762689369ed.d/00000000000000000000.csr","byte_length":1866,"content_sha256":"c4dc71e92a7313804d090463634dc5c054599b0f764d6409999b9e2f001a1691","content_xxh64":"3d505128daacb6b6","role":"index"},{"relative_path":"indexes/adjacency/r-5aacc116ac666e02684686f6df7f94a9fd62ee5748dd6e0daa0900d949874c8c.in.csr.json","byte_length":536,"content_sha256":"60ec16305c87e777dda418bcc099dcd86298686bbe3473d51f0173ac848078f5","content_xxh64":"cb5d79a85ec62dab","role":"index"},{"relative_path":"indexes/adjacency/r-5aacc116ac666e02684686f6df7f94a9fd62ee5748dd6e0daa0900d949874c8c.in.csr.shards-ddfa0b39dddaf5aed1a600bb.d/00000000000000000000.csr","byte_length":1866,"content_sha256":"50b821d0ee918d37ab0baf3e39ce01a69cea34144255ddd9468181446dd2473d","content_xxh64":"b8a3f0e4d17c754c","role":"index"},{"relative_path":"indexes/adjacency/r-5aacc116ac666e02684686f6df7f94a9fd62ee5748dd6e0daa0900d949874c8c.out.csr.json","byte_length":537,"content_sha256":"81dc0dfc6f7ed687160b947453c0c3df2cdb44fc81b8d9dc89a16262beb0448e","content_xxh64":"7c4b5c2f8061bc36","role":"index"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/64/8f9f5aa333e9c0b35840cc0cd8fd3884e9f5e533225be81a91404cc8e5a941 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/64/8f9f5aa333e9c0b35840cc0cd8fd3884e9f5e533225be81a91404cc8e5a941 new file mode 100644 index 000000000..55ec8be15 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/64/8f9f5aa333e9c0b35840cc0cd8fd3884e9f5e533225be81a91404cc8e5a941 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":0,"prefix":"ddfab30cf3e3daae8a58703630bc45cdd7ed70a0043913defeafa3fb1e38579b","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/index_manifest.parquet","byte_length":2044,"content_sha256":"402e0e930d922e9ab4e149e738771406fd0b5b738c8bac89bc345a0220fbd4a7","content_xxh64":"63a77de5d97c10d5","role":"index"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/64/d0aca220e3dbc30007411119041cd6bc8a807580c02e4d994b0f73e0a58bfa b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/64/d0aca220e3dbc30007411119041cd6bc8a807580c02e4d994b0f73e0a58bfa new file mode 100644 index 000000000..73c3120b9 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/64/d0aca220e3dbc30007411119041cd6bc8a807580c02e4d994b0f73e0a58bfa @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"088becd016be82d1683545e23a32616ec19f52088b275cf78d59d4528d9f1a7","kind":"bucket","entries":[{"relative_path":"topology/uuid-membership/tombstones-v4-1-e3b0c44298fc1c14.uuidx","byte_length":0,"content_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","content_xxh64":"ef46db3751d8e999","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/67/0c7acbbcde62dd0cfb6b0e75e7a685203d18442246974390cc0e8ca5682c10 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/67/0c7acbbcde62dd0cfb6b0e75e7a685203d18442246974390cc0e8ca5682c10 new file mode 100644 index 000000000..b01596362 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/67/0c7acbbcde62dd0cfb6b0e75e7a685203d18442246974390cc0e8ca5682c10 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"dfab30cf3e3daae8a58703630bc45cdd7ed70a0043913defeafa3fb1e38579b","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/index_manifest.parquet","byte_length":2044,"content_sha256":"402e0e930d922e9ab4e149e738771406fd0b5b738c8bac89bc345a0220fbd4a7","content_xxh64":"63a77de5d97c10d5","role":"index"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/6a/a5e92b174b286e8c36e861a57d34a3d93fad72e114bf9cb7a6ceb0a234b9fb b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/6a/a5e92b174b286e8c36e861a57d34a3d93fad72e114bf9cb7a6ceb0a234b9fb new file mode 100644 index 000000000..11bfcbbc8 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/6a/a5e92b174b286e8c36e861a57d34a3d93fad72e114bf9cb7a6ceb0a234b9fb @@ -0,0 +1,19 @@ +{ + "format": "graphforge.csr-shards", + "version": 3, + "node_count": 9, + "edge_count": 8, + "shard_dir": "r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.out.csr.shards-312746a9f2902762689369ed.d", + "shards": [ + { + "first_node": 1, + "node_count": 8, + "edge_count": 8, + "file": "00000000000000000000.csr", + "sha256": "c4dc71e92a7313804d090463634dc5c054599b0f764d6409999b9e2f001a1691", + "xxh64": "3d505128daacb6b6", + "encoded_bytes": 1866, + "decoded_bytes": 204 + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/6b/3d79b9f4cccf6b04489a3e241593ccdedb9e077fd957075eac0c63c5519a2c b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/6b/3d79b9f4cccf6b04489a3e241593ccdedb9e077fd957075eac0c63c5519a2c new file mode 100644 index 000000000..458fe4bf1 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/6b/3d79b9f4cccf6b04489a3e241593ccdedb9e077fd957075eac0c63c5519a2c @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.in.csr.json","byte_length":536,"content_sha256":"a55c44fd9b95369708a324153fbf371c145a7ae54120be51246322c7fbbc0369","content_xxh64":"1c996730cfd3431e","role":"index"},{"relative_path":"indexes/adjacency/r-5aacc116ac666e02684686f6df7f94a9fd62ee5748dd6e0daa0900d949874c8c.in.csr.json","byte_length":536,"content_sha256":"60ec16305c87e777dda418bcc099dcd86298686bbe3473d51f0173ac848078f5","content_xxh64":"cb5d79a85ec62dab","role":"index"},{"relative_path":"indexes/adjacency/r-5aacc116ac666e02684686f6df7f94a9fd62ee5748dd6e0daa0900d949874c8c.in.csr.shards-ddfa0b39dddaf5aed1a600bb.d/00000000000000000000.csr","byte_length":1866,"content_sha256":"50b821d0ee918d37ab0baf3e39ce01a69cea34144255ddd9468181446dd2473d","content_xxh64":"b8a3f0e4d17c754c","role":"index"},{"relative_path":"topology/edges/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000008.parquet","byte_length":3003,"content_sha256":"623f2c45c91931017a3ff5f47b52368ff444d880a45e01e51620da8374f32a84","content_xxh64":"0bee4f21053cd5b4","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/70/56644ec93e357f016906a0715f49cb92d9e87440d6e0116e51e7b65bd51821 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/70/56644ec93e357f016906a0715f49cb92d9e87440d6e0116e51e7b65bd51821 new file mode 100644 index 000000000..132b9a9ff --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/70/56644ec93e357f016906a0715f49cb92d9e87440d6e0116e51e7b65bd51821 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"","kind":"bucket","entries":[{"relative_path":"semantic-routes.json","byte_length":177,"content_sha256":"e89f6e446520ec6669f4c8b27486370fabd8c152a409d1363efcd9b32d2b15e6","content_xxh64":"17beb01a6e2bfc9f","role":"catalog"},{"relative_path":"topology/surrogate_tails.parquet","byte_length":817,"content_sha256":"74ab11daa9873474c8a093a2fc2185fad73bd3ab2d9aaab88e48736ee0a73293","content_xxh64":"cd84da50fc4e96fc","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/73/0f3c9d30b8f2149d44ec172c19f5c7723c248e123b0fa3271e2e35593f69ba b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/73/0f3c9d30b8f2149d44ec172c19f5c7723c248e123b0fa3271e2e35593f69ba new file mode 100644 index 000000000..277a4e3c6 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/73/0f3c9d30b8f2149d44ec172c19f5c7723c248e123b0fa3271e2e35593f69ba @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":0,"prefix":"","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/index_manifest.parquet","byte_length":2044,"content_sha256":"402e0e930d922e9ab4e149e738771406fd0b5b738c8bac89bc345a0220fbd4a7","content_xxh64":"63a77de5d97c10d5","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.in.csr.json","byte_length":536,"content_sha256":"a55c44fd9b95369708a324153fbf371c145a7ae54120be51246322c7fbbc0369","content_xxh64":"1c996730cfd3431e","role":"index"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/74/ab11daa9873474c8a093a2fc2185fad73bd3ab2d9aaab88e48736ee0a73293 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/74/ab11daa9873474c8a093a2fc2185fad73bd3ab2d9aaab88e48736ee0a73293 new file mode 100644 index 0000000000000000000000000000000000000000..d0f6813aeb951f916096a5dbee2f1b6282ece1d2 GIT binary patch literal 817 zcmbtT&ubb{5T1P%S*Qm!&SM`eK?*J&1o6;TOItE)D@`lZ>>-H~kZs(orTC+(sX678 zWB-gCb1L+fKdFZtat(Cey5eNeL7XKNe0=`AW4gm)<^l&vZb*qBl;A)QrsBrqd| z^yDw6<(T#>C||E}vrsMvs>06$D1eVFJCGLw3-Z#y_i#@+nc!Oa+;EN`BYpA=rTimU zTAmtGIHvq(6!hy@8S=cZJ8zZ`V*M6XZFSPN6`w?3h!)fg>9Y>8Cvp>Ji!srrMjTps zM*8Y@b-bkyCfUb&9~Sgchqx}rF>qI{s2St-YaXJZ(-NDpcjw*R;X!Z58~3&w)1I#9 z7I8k+laazDcJ+|7pA-`fW6ZFt`*MjMZMB5Xy3zkOYWzs_-+{FrM*ehoFp0vTI1Qqa dKMuz8;?$3R9tQJjO;v0F{=pu9zYhOd{sUe7o3H=? literal 0 HcmV?d00001 diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/79/b5b9ca9a21d61dc1ae781f170fdd6fcbafe707b0d404b47fe2bbed7b2dd884 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/79/b5b9ca9a21d61dc1ae781f170fdd6fcbafe707b0d404b47fe2bbed7b2dd884 new file mode 100644 index 000000000..7023f043c --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/79/b5b9ca9a21d61dc1ae781f170fdd6fcbafe707b0d404b47fe2bbed7b2dd884 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.in.csr.json","byte_length":536,"content_sha256":"a55c44fd9b95369708a324153fbf371c145a7ae54120be51246322c7fbbc0369","content_xxh64":"1c996730cfd3431e","role":"index"},{"relative_path":"indexes/adjacency/r-5aacc116ac666e02684686f6df7f94a9fd62ee5748dd6e0daa0900d949874c8c.in.csr.json","byte_length":536,"content_sha256":"60ec16305c87e777dda418bcc099dcd86298686bbe3473d51f0173ac848078f5","content_xxh64":"cb5d79a85ec62dab","role":"index"},{"relative_path":"indexes/adjacency/r-5aacc116ac666e02684686f6df7f94a9fd62ee5748dd6e0daa0900d949874c8c.in.csr.shards-ddfa0b39dddaf5aed1a600bb.d/00000000000000000000.csr","byte_length":1866,"content_sha256":"50b821d0ee918d37ab0baf3e39ce01a69cea34144255ddd9468181446dd2473d","content_xxh64":"b8a3f0e4d17c754c","role":"index"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/7b/240572eaf4cc550d8b5d6d1477f0a90e73b98919d38d383aabe55c69b6115d b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/7b/240572eaf4cc550d8b5d6d1477f0a90e73b98919d38d383aabe55c69b6115d new file mode 100644 index 000000000..ec306ea74 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/7b/240572eaf4cc550d8b5d6d1477f0a90e73b98919d38d383aabe55c69b6115d @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"e","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.out.csr.json","byte_length":537,"content_sha256":"6aa5e92b174b286e8c36e861a57d34a3d93fad72e114bf9cb7a6ceb0a234b9fb","content_xxh64":"29c320fbcfa4cf82","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.out.csr.shards-312746a9f2902762689369ed.d/00000000000000000000.csr","byte_length":1866,"content_sha256":"c4dc71e92a7313804d090463634dc5c054599b0f764d6409999b9e2f001a1691","content_xxh64":"3d505128daacb6b6","role":"index"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/7c/787d6c58c741080bc89e90158915b00d3642c65cc8e940735d4f21c309fc49 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/7c/787d6c58c741080bc89e90158915b00d3642c65cc8e940735d4f21c309fc49 new file mode 100644 index 000000000..c05b4e9f0 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/7c/787d6c58c741080bc89e90158915b00d3642c65cc8e940735d4f21c309fc49 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"0":"a547e1515d6437ddb4c530f805e4a9f6fde3f42fb3725c862e5ac3675faec543","1":"16caf5a94acbf6efbbc859d3cb210f945081ecbaa903a1a3321103366ae8cf4f","2":"8f37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac","3":"33d5b384bd457bf148bc1711beb3544d41d40c9499d4e454c0660a173ee69222","4":"18d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034","5":"47ea851bc5d6f0806954e1b36477c5a61b5fe18044fe79cfbbc304f04d052ea6","6":"023f0c4332a566cb6fbb2fb4078aa318c51f2598960381f693d66f96dbabca03","8":"7056644ec93e357f016906a0715f49cb92d9e87440d6e0116e51e7b65bd51821","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","d":"963c4f9708e0cffbdf90aa9b3f86bebe0aff18acd05b69c715a146ff6f6c0967","f":"4e1e1b01ecbd59456aa67778d9ad0fc39e3fe4961e4ab4f35f83cd12a3c6a220"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/7e/a9b44c433d7f4e4f767eea6b3899915d2968d93d762496d29bda5f3e1bd405 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/7e/a9b44c433d7f4e4f767eea6b3899915d2968d93d762496d29bda5f3e1bd405 new file mode 100644 index 000000000..238af3830 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/7e/a9b44c433d7f4e4f767eea6b3899915d2968d93d762496d29bda5f3e1bd405 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"2":"60c685463d4ff99b7528234b74151eac492b72eca61c2c009f5049a073efd6a1","3":"6b3d79b9f4cccf6b04489a3e241593ccdedb9e077fd957075eac0c63c5519a2c","6":"7b240572eaf4cc550d8b5d6d1477f0a90e73b98919d38d383aabe55c69b6115d","8":"dabf2e92de1180195cdb270af0f86f741dbe328828819cfd28dcf6e08fb972c5","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","d":"670c7acbbcde62dd0cfb6b0e75e7a685203d18442246974390cc0e8ca5682c10","f":"a4173a1b346783c2de070b2e3452345248c5e627a0ddbca8e7d909cb8e5309fa"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/80/11b9909d36a6051a3ddd1378ba581c5b4be7221950372439117b24934f7578 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/80/11b9909d36a6051a3ddd1378ba581c5b4be7221950372439117b24934f7578 new file mode 100644 index 000000000..1c96da97a --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/80/11b9909d36a6051a3ddd1378ba581c5b4be7221950372439117b24934f7578 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"0":"a547e1515d6437ddb4c530f805e4a9f6fde3f42fb3725c862e5ac3675faec543","1":"52f4b97c32c0e446a0968c903ad030ef05551a8e34dbdd79ccc79cb49d77c687","2":"8f37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac","3":"836ec17534a48b6cfdf1a977725d6b05ce9c9e114242590381e5c4d357aad9ac","4":"18d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034","5":"d5cd023a1b8ff5dc2c02dc9900a54b06c2134b0e6ba5a9272d14510436c3a12c","6":"2bea2e263428d5c0c085ebc737a31897d251273580869d26a0cb8d789c8fa990","7":"b4684cec290b5fd2220e9c6171d702a04277ef7d63ac3d3a1ee4f16858f99bfb","8":"bffad8f5e2fc47ab5a5ebd52cdcb3947ca007420e9d19ab3efc16e861080894f","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","b":"64d0aca220e3dbc30007411119041cd6bc8a807580c02e4d994b0f73e0a58bfa","d":"81f9ec2f7714716b7cea261a5b687561be2ab70c1fe8ac5698d7f904ddd2af76","f":"8b515d817bdd27fa39dc155b59753fd24eca05413bdf7333b3035da95039ae7d"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/81/dc0dfc6f7ed687160b947453c0c3df2cdb44fc81b8d9dc89a16262beb0448e b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/81/dc0dfc6f7ed687160b947453c0c3df2cdb44fc81b8d9dc89a16262beb0448e new file mode 100644 index 000000000..6e445f835 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/81/dc0dfc6f7ed687160b947453c0c3df2cdb44fc81b8d9dc89a16262beb0448e @@ -0,0 +1,19 @@ +{ + "format": "graphforge.csr-shards", + "version": 3, + "node_count": 9, + "edge_count": 8, + "shard_dir": "r-5aacc116ac666e02684686f6df7f94a9fd62ee5748dd6e0daa0900d949874c8c.out.csr.shards-312746a9f2902762689369ed.d", + "shards": [ + { + "first_node": 1, + "node_count": 8, + "edge_count": 8, + "file": "00000000000000000000.csr", + "sha256": "c4dc71e92a7313804d090463634dc5c054599b0f764d6409999b9e2f001a1691", + "xxh64": "3d505128daacb6b6", + "encoded_bytes": 1866, + "decoded_bytes": 204 + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/81/f9ec2f7714716b7cea261a5b687561be2ab70c1fe8ac5698d7f904ddd2af76 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/81/f9ec2f7714716b7cea261a5b687561be2ab70c1fe8ac5698d7f904ddd2af76 new file mode 100644 index 000000000..47a9110f2 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/81/f9ec2f7714716b7cea261a5b687561be2ab70c1fe8ac5698d7f904ddd2af76 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/index_manifest.parquet","byte_length":2044,"content_sha256":"402e0e930d922e9ab4e149e738771406fd0b5b738c8bac89bc345a0220fbd4a7","content_xxh64":"63a77de5d97c10d5","role":"index"},{"relative_path":"topology/nodes/00000000000000000009-00000000000000000014.parquet","byte_length":2274,"content_sha256":"555a4a690fb10d5dd1584f0cfb05cb2396f6d41ac1679742b13aa21a745413d7","content_xxh64":"d1839cc1984be498","role":"topology"},{"relative_path":"topology/uuid-membership/forward-v4-2-2a093dcf7c477b67.uuidx","byte_length":144,"content_sha256":"2a093dcf7c477b67c309d1332a01a397938c0499a83fbc82a20edd2261b38a0a","content_xxh64":"b4cf8cc7d2c09c33","role":"topology"},{"relative_path":"topology/uuid-membership/node-surrogates-v5-base-0-e3b0c44298fc1c14.uuidx","byte_length":0,"content_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","content_xxh64":"ef46db3751d8e999","role":"topology"},{"relative_path":"topology/uuid-membership/ordinal-v4-receipt.json","byte_length":244,"content_sha256":"dfe16a08130a605d299b15193a2a34a34de74b82d6eabc1fbb3852003b41f4cc","content_xxh64":"2f7e02adda20f847","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/83/628bb4c0ac01773986d43b0bda4eaba2769c8330ca13757e5cde14a2289fae b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/83/628bb4c0ac01773986d43b0bda4eaba2769c8330ca13757e5cde14a2289fae new file mode 100644 index 000000000..8d1d44176 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/83/628bb4c0ac01773986d43b0bda4eaba2769c8330ca13757e5cde14a2289fae @@ -0,0 +1 @@ +{"topology_generation":1,"search_generation":1,"property_generation":1} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/83/6ec17534a48b6cfdf1a977725d6b05ce9c9e114242590381e5c4d357aad9ac b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/83/6ec17534a48b6cfdf1a977725d6b05ce9c9e114242590381e5c4d357aad9ac new file mode 100644 index 000000000..745302584 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/83/6ec17534a48b6cfdf1a977725d6b05ce9c9e114242590381e5c4d357aad9ac @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.in.csr.json","byte_length":536,"content_sha256":"a55c44fd9b95369708a324153fbf371c145a7ae54120be51246322c7fbbc0369","content_xxh64":"1c996730cfd3431e","role":"index"},{"relative_path":"indexes/adjacency/r-5aacc116ac666e02684686f6df7f94a9fd62ee5748dd6e0daa0900d949874c8c.in.csr.json","byte_length":536,"content_sha256":"60ec16305c87e777dda418bcc099dcd86298686bbe3473d51f0173ac848078f5","content_xxh64":"cb5d79a85ec62dab","role":"index"},{"relative_path":"indexes/adjacency/r-5aacc116ac666e02684686f6df7f94a9fd62ee5748dd6e0daa0900d949874c8c.in.csr.shards-ddfa0b39dddaf5aed1a600bb.d/00000000000000000000.csr","byte_length":1866,"content_sha256":"50b821d0ee918d37ab0baf3e39ce01a69cea34144255ddd9468181446dd2473d","content_xxh64":"b8a3f0e4d17c754c","role":"index"},{"relative_path":"topology/edges/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000001-00000000000000000008.parquet","byte_length":3003,"content_sha256":"623f2c45c91931017a3ff5f47b52368ff444d880a45e01e51620da8374f32a84","content_xxh64":"0bee4f21053cd5b4","role":"topology"},{"relative_path":"topology/generation.json","byte_length":72,"content_sha256":"36c3282c5e0655d4a5539ca14093dd671a046fe1e7ede09760d20e3281fa25fe","content_xxh64":"aa3134045b987a57","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/87/90025a1cbc91d05c270a39022f663452b6ee84ca8662c619273416272a834b b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/87/90025a1cbc91d05c270a39022f663452b6ee84ca8662c619273416272a834b new file mode 100644 index 000000000..cbe19e1d5 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/87/90025a1cbc91d05c270a39022f663452b6ee84ca8662c619273416272a834b @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"2":"60c685463d4ff99b7528234b74151eac492b72eca61c2c009f5049a073efd6a1","3":"33d5b384bd457bf148bc1711beb3544d41d40c9499d4e454c0660a173ee69222","6":"7b240572eaf4cc550d8b5d6d1477f0a90e73b98919d38d383aabe55c69b6115d","8":"dabf2e92de1180195cdb270af0f86f741dbe328828819cfd28dcf6e08fb972c5","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","d":"670c7acbbcde62dd0cfb6b0e75e7a685203d18442246974390cc0e8ca5682c10","f":"4e1e1b01ecbd59456aa67778d9ad0fc39e3fe4961e4ab4f35f83cd12a3c6a220"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/88/dbad48a4b2c270ee165c4a9fa8e7efa32cd581e53e398264e7fc74e38b62a1 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/88/dbad48a4b2c270ee165c4a9fa8e7efa32cd581e53e398264e7fc74e38b62a1 new file mode 100644 index 000000000..87aa000cb --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/88/dbad48a4b2c270ee165c4a9fa8e7efa32cd581e53e398264e7fc74e38b62a1 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"2":"8f37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac","3":"33d5b384bd457bf148bc1711beb3544d41d40c9499d4e454c0660a173ee69222","6":"7b240572eaf4cc550d8b5d6d1477f0a90e73b98919d38d383aabe55c69b6115d","8":"dabf2e92de1180195cdb270af0f86f741dbe328828819cfd28dcf6e08fb972c5","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","d":"670c7acbbcde62dd0cfb6b0e75e7a685203d18442246974390cc0e8ca5682c10","f":"4e1e1b01ecbd59456aa67778d9ad0fc39e3fe4961e4ab4f35f83cd12a3c6a220"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/8b/482651abbd4146b318590b3ce6c94caf382469d14e5799214e27aa70b30002 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/8b/482651abbd4146b318590b3ce6c94caf382469d14e5799214e27aa70b30002 new file mode 100644 index 000000000..39206f5a2 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/8b/482651abbd4146b318590b3ce6c94caf382469d14e5799214e27aa70b30002 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"0":"a547e1515d6437ddb4c530f805e4a9f6fde3f42fb3725c862e5ac3675faec543","1":"52f4b97c32c0e446a0968c903ad030ef05551a8e34dbdd79ccc79cb49d77c687","2":"8f37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac","3":"836ec17534a48b6cfdf1a977725d6b05ce9c9e114242590381e5c4d357aad9ac","4":"18d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034","5":"d5cd023a1b8ff5dc2c02dc9900a54b06c2134b0e6ba5a9272d14510436c3a12c","6":"2bea2e263428d5c0c085ebc737a31897d251273580869d26a0cb8d789c8fa990","7":"b4684cec290b5fd2220e9c6171d702a04277ef7d63ac3d3a1ee4f16858f99bfb","8":"f58243170c9c519373e516eaec531c28dc9bbe6301f83e56823b7590768661d1","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","b":"64d0aca220e3dbc30007411119041cd6bc8a807580c02e4d994b0f73e0a58bfa","d":"81f9ec2f7714716b7cea261a5b687561be2ab70c1fe8ac5698d7f904ddd2af76","f":"8b515d817bdd27fa39dc155b59753fd24eca05413bdf7333b3035da95039ae7d"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/8b/515d817bdd27fa39dc155b59753fd24eca05413bdf7333b3035da95039ae7d b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/8b/515d817bdd27fa39dc155b59753fd24eca05413bdf7333b3035da95039ae7d new file mode 100644 index 000000000..3c89ac745 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/8b/515d817bdd27fa39dc155b59753fd24eca05413bdf7333b3035da95039ae7d @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/r-5aacc116ac666e02684686f6df7f94a9fd62ee5748dd6e0daa0900d949874c8c.out.csr.json","byte_length":537,"content_sha256":"81dc0dfc6f7ed687160b947453c0c3df2cdb44fc81b8d9dc89a16262beb0448e","content_xxh64":"7c4b5c2f8061bc36","role":"index"},{"relative_path":"topology/nodes/00000000000000000001-00000000000000000008.parquet","byte_length":2284,"content_sha256":"5527b3e2c14fbe8a9cfe38a2b7ae8ce696ac7bf9fbb3ec7cbc73bd697e7047f9","content_xxh64":"85ec9a5c84886c1e","role":"topology"},{"relative_path":"topology/runtime_catalog.parquet","byte_length":2381,"content_sha256":"b3376061dade602b1d8ab4f4bc30e3866f2abf92d84848e2c79d02b9c0ced446","content_xxh64":"535173a4204b7db7","role":"topology"},{"relative_path":"topology/uuid-membership/ordinal-v4.lock","byte_length":0,"content_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","content_xxh64":"ef46db3751d8e999","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/8d/c1d518105cf2d68a207139b1dba198ba531eeaa0be090b5ddc6616a9678328 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/8d/c1d518105cf2d68a207139b1dba198ba531eeaa0be090b5ddc6616a9678328 new file mode 100644 index 000000000..ff3c6f461 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/8d/c1d518105cf2d68a207139b1dba198ba531eeaa0be090b5ddc6616a9678328 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"0":"a547e1515d6437ddb4c530f805e4a9f6fde3f42fb3725c862e5ac3675faec543","1":"917ceda6a0f6fbf916cdf7191151a8d7b2a2d4b38aca4e426d43f93a7dd718d9","2":"8f37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac","3":"836ec17534a48b6cfdf1a977725d6b05ce9c9e114242590381e5c4d357aad9ac","4":"18d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034","5":"47ea851bc5d6f0806954e1b36477c5a61b5fe18044fe79cfbbc304f04d052ea6","6":"023f0c4332a566cb6fbb2fb4078aa318c51f2598960381f693d66f96dbabca03","7":"b4684cec290b5fd2220e9c6171d702a04277ef7d63ac3d3a1ee4f16858f99bfb","8":"ef58edb5e714477964e6c29aa9ea6e80947355b529995d3e7e8116d59a643069","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","b":"64d0aca220e3dbc30007411119041cd6bc8a807580c02e4d994b0f73e0a58bfa","d":"deb33edafa58e17727033ee8936ea7a9ad80976f7a8e5e4e2b722dbd55228a51","f":"8b515d817bdd27fa39dc155b59753fd24eca05413bdf7333b3035da95039ae7d"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/8f/37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/8f/37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac new file mode 100644 index 000000000..c948f1f9f --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/8f/37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"4","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.in.csr.shards-ddfa0b39dddaf5aed1a600bb.d/00000000000000000000.csr","byte_length":1866,"content_sha256":"50b821d0ee918d37ab0baf3e39ce01a69cea34144255ddd9468181446dd2473d","content_xxh64":"b8a3f0e4d17c754c","role":"index"},{"relative_path":"topology/runtime_entity_label_encoding.json","byte_length":74,"content_sha256":"a688960f0b7533d9b63091e0f3508bb309a31e01e5d849c2db2cd276e52c98bd","content_xxh64":"7a7a316e187c2b7a","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/90/9e7eac9f6a71fcbd68d6ed76e90c8021ed20ef728909110e574e2b40454c34 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/90/9e7eac9f6a71fcbd68d6ed76e90c8021ed20ef728909110e574e2b40454c34 new file mode 100644 index 000000000..b8f42e17f --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/90/9e7eac9f6a71fcbd68d6ed76e90c8021ed20ef728909110e574e2b40454c34 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"1":"16caf5a94acbf6efbbc859d3cb210f945081ecbaa903a1a3321103366ae8cf4f","2":"8f37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac","3":"33d5b384bd457bf148bc1711beb3544d41d40c9499d4e454c0660a173ee69222","4":"61dd5bd3012ddea71341be7ed0b0e7c21c87327ac4d836727bc90a121e30a7b3","5":"47ea851bc5d6f0806954e1b36477c5a61b5fe18044fe79cfbbc304f04d052ea6","6":"023f0c4332a566cb6fbb2fb4078aa318c51f2598960381f693d66f96dbabca03","8":"7056644ec93e357f016906a0715f49cb92d9e87440d6e0116e51e7b65bd51821","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","d":"670c7acbbcde62dd0cfb6b0e75e7a685203d18442246974390cc0e8ca5682c10","f":"4e1e1b01ecbd59456aa67778d9ad0fc39e3fe4961e4ab4f35f83cd12a3c6a220"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/91/7ceda6a0f6fbf916cdf7191151a8d7b2a2d4b38aca4e426d43f93a7dd718d9 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/91/7ceda6a0f6fbf916cdf7191151a8d7b2a2d4b38aca4e426d43f93a7dd718d9 new file mode 100644 index 000000000..6d77d58d1 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/91/7ceda6a0f6fbf916cdf7191151a8d7b2a2d4b38aca4e426d43f93a7dd718d9 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"","kind":"bucket","entries":[{"relative_path":"topology/uuid-membership/identities-v5-1-1f39b3ef25c02359.uuidx","byte_length":336,"content_sha256":"1f39b3ef25c0235991a359758f446a13640216a6345fa1a9410cd95758876530","content_xxh64":"5582fa352f773d0d","role":"topology"},{"relative_path":"topology/uuid-membership/ordinal-v4-manifest.json","byte_length":886,"content_sha256":"0a14e69d5f88bdb637d098b2119c62d724d7d8850ce5a0d0d478f834b0b631a7","content_xxh64":"870d102ae01725c6","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/92/040d2b83ae59ee9d9f621e9e2160083a7cc4f7198d7c07670c56d05c9d4223 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/92/040d2b83ae59ee9d9f621e9e2160083a7cc4f7198d7c07670c56d05c9d4223 new file mode 100644 index 000000000..97cc43f3f --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/92/040d2b83ae59ee9d9f621e9e2160083a7cc4f7198d7c07670c56d05c9d4223 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"0":"a547e1515d6437ddb4c530f805e4a9f6fde3f42fb3725c862e5ac3675faec543","1":"917ceda6a0f6fbf916cdf7191151a8d7b2a2d4b38aca4e426d43f93a7dd718d9","2":"8f37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac","3":"33d5b384bd457bf148bc1711beb3544d41d40c9499d4e454c0660a173ee69222","4":"18d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034","5":"47ea851bc5d6f0806954e1b36477c5a61b5fe18044fe79cfbbc304f04d052ea6","6":"023f0c4332a566cb6fbb2fb4078aa318c51f2598960381f693d66f96dbabca03","8":"7056644ec93e357f016906a0715f49cb92d9e87440d6e0116e51e7b65bd51821","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","b":"64d0aca220e3dbc30007411119041cd6bc8a807580c02e4d994b0f73e0a58bfa","d":"18450af8e920af1a4a64bf48c6cf10ca9a08eadbc08f8c06485be33236de4548","f":"f1dd0b9fb6f895682403236f1c3f497806c1b951a0c27417f3f83ba93fa06b3f"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/95/3115f8a1886a273f8fb103e7ff7f9a8fe3c604487b7d4af376270960271b6f b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/95/3115f8a1886a273f8fb103e7ff7f9a8fe3c604487b7d4af376270960271b6f new file mode 100644 index 000000000..ffb533b28 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/95/3115f8a1886a273f8fb103e7ff7f9a8fe3c604487b7d4af376270960271b6f @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"0":"a547e1515d6437ddb4c530f805e4a9f6fde3f42fb3725c862e5ac3675faec543","1":"917ceda6a0f6fbf916cdf7191151a8d7b2a2d4b38aca4e426d43f93a7dd718d9","2":"8f37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac","3":"836ec17534a48b6cfdf1a977725d6b05ce9c9e114242590381e5c4d357aad9ac","4":"18d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034","5":"47ea851bc5d6f0806954e1b36477c5a61b5fe18044fe79cfbbc304f04d052ea6","6":"023f0c4332a566cb6fbb2fb4078aa318c51f2598960381f693d66f96dbabca03","7":"b4684cec290b5fd2220e9c6171d702a04277ef7d63ac3d3a1ee4f16858f99bfb","8":"ef58edb5e714477964e6c29aa9ea6e80947355b529995d3e7e8116d59a643069","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","b":"64d0aca220e3dbc30007411119041cd6bc8a807580c02e4d994b0f73e0a58bfa","d":"fd940e96ef6d179d9bf35e1bbd7f8786b2f66bed48c07f9b3d878e2f4eb42b7a","f":"8b515d817bdd27fa39dc155b59753fd24eca05413bdf7333b3035da95039ae7d"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/96/3c4f9708e0cffbdf90aa9b3f86bebe0aff18acd05b69c715a146ff6f6c0967 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/96/3c4f9708e0cffbdf90aa9b3f86bebe0aff18acd05b69c715a146ff6f6c0967 new file mode 100644 index 000000000..e1e869ff6 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/96/3c4f9708e0cffbdf90aa9b3f86bebe0aff18acd05b69c715a146ff6f6c0967 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/index_manifest.parquet","byte_length":2044,"content_sha256":"402e0e930d922e9ab4e149e738771406fd0b5b738c8bac89bc345a0220fbd4a7","content_xxh64":"63a77de5d97c10d5","role":"index"},{"relative_path":"topology/uuid-membership/node-surrogates-v5-base-0-e3b0c44298fc1c14.uuidx","byte_length":0,"content_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","content_xxh64":"ef46db3751d8e999","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/a4/173a1b346783c2de070b2e3452345248c5e627a0ddbca8e7d909cb8e5309fa b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/a4/173a1b346783c2de070b2e3452345248c5e627a0ddbca8e7d909cb8e5309fa new file mode 100644 index 000000000..cff10838d --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/a4/173a1b346783c2de070b2e3452345248c5e627a0ddbca8e7d909cb8e5309fa @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"24ae3b083e57f3771e3042bced82cdd59a59f3730167db3450fa8128a1c4050","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/r-5aacc116ac666e02684686f6df7f94a9fd62ee5748dd6e0daa0900d949874c8c.out.csr.json","byte_length":537,"content_sha256":"81dc0dfc6f7ed687160b947453c0c3df2cdb44fc81b8d9dc89a16262beb0448e","content_xxh64":"7c4b5c2f8061bc36","role":"index"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/a5/47e1515d6437ddb4c530f805e4a9f6fde3f42fb3725c862e5ac3675faec543 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/a5/47e1515d6437ddb4c530f805e4a9f6fde3f42fb3725c862e5ac3675faec543 new file mode 100644 index 000000000..fca2b25d8 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/a5/47e1515d6437ddb4c530f805e4a9f6fde3f42fb3725c862e5ac3675faec543 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"48969a45dcc74b354b4543933a53b27ef66e4b43046375cf9c42a8efefc9bc0","kind":"bucket","entries":[{"relative_path":"topology/uuid-membership/ordinal-v4-1-599c59b6e57e65a3.uuidx","byte_length":128,"content_sha256":"599c59b6e57e65a305ee43a3c4f9368e52bcd3558337e9d59ee36b93fa20ad64","content_xxh64":"593488175e7a60f4","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/a5/5c44fd9b95369708a324153fbf371c145a7ae54120be51246322c7fbbc0369 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/a5/5c44fd9b95369708a324153fbf371c145a7ae54120be51246322c7fbbc0369 new file mode 100644 index 000000000..a715174ca --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/a5/5c44fd9b95369708a324153fbf371c145a7ae54120be51246322c7fbbc0369 @@ -0,0 +1,19 @@ +{ + "format": "graphforge.csr-shards", + "version": 3, + "node_count": 9, + "edge_count": 8, + "shard_dir": "r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.in.csr.shards-ddfa0b39dddaf5aed1a600bb.d", + "shards": [ + { + "first_node": 1, + "node_count": 8, + "edge_count": 8, + "file": "00000000000000000000.csr", + "sha256": "50b821d0ee918d37ab0baf3e39ce01a69cea34144255ddd9468181446dd2473d", + "xxh64": "b8a3f0e4d17c754c", + "encoded_bytes": 1866, + "decoded_bytes": 204 + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/a6/88960f0b7533d9b63091e0f3508bb309a31e01e5d849c2db2cd276e52c98bd b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/a6/88960f0b7533d9b63091e0f3508bb309a31e01e5d849c2db2cd276e52c98bd new file mode 100644 index 000000000..88b2dec40 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/a6/88960f0b7533d9b63091e0f3508bb309a31e01e5d849c2db2cd276e52c98bd @@ -0,0 +1,4 @@ +{ + "format": "graphforge-runtime-entity-label-encoding", + "version": 1 +} \ No newline at end of file diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/af/add6993a6bdd8b04ee133b60a15b77df326cc99b87d5cd2e6dc4d605b4b313 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/af/add6993a6bdd8b04ee133b60a15b77df326cc99b87d5cd2e6dc4d605b4b313 new file mode 100644 index 000000000..0175fa696 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/af/add6993a6bdd8b04ee133b60a15b77df326cc99b87d5cd2e6dc4d605b4b313 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":0,"prefix":"","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/index_manifest.parquet","byte_length":2044,"content_sha256":"402e0e930d922e9ab4e149e738771406fd0b5b738c8bac89bc345a0220fbd4a7","content_xxh64":"63a77de5d97c10d5","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.in.csr.json","byte_length":536,"content_sha256":"a55c44fd9b95369708a324153fbf371c145a7ae54120be51246322c7fbbc0369","content_xxh64":"1c996730cfd3431e","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.in.csr.shards-ddfa0b39dddaf5aed1a600bb.d/00000000000000000000.csr","byte_length":1866,"content_sha256":"50b821d0ee918d37ab0baf3e39ce01a69cea34144255ddd9468181446dd2473d","content_xxh64":"b8a3f0e4d17c754c","role":"index"},{"relative_path":"indexes/adjacency/r-378b8a288e1c3526513898aee924f16c983083701f73c0d3d643e43d3a668fdd.out.csr.json","byte_length":537,"content_sha256":"6aa5e92b174b286e8c36e861a57d34a3d93fad72e114bf9cb7a6ceb0a234b9fb","content_xxh64":"29c320fbcfa4cf82","role":"index"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/b1/e9f24720536bcc97707a1a0eae7447ae2ac57000b44df00c8b338b6009145c b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/b1/e9f24720536bcc97707a1a0eae7447ae2ac57000b44df00c8b338b6009145c new file mode 100644 index 0000000000000000000000000000000000000000..d655ebe900a44db016f56b2dd700b268ed6c921f GIT binary patch literal 817 zcmbtT&r2Io5T1R85HLuh&SM`e5(-&71cC=O5fx{xq1Ax82Q@(mYqHstk{_Y;~T3!S&wXvBkvLwNJP`DVVEc}tGSUBLuq@Iw^^!aTt8eequ|MF4X! zHUv(P1*S8Y#AKWyvdoHil&X25%+#dG)D?x20I$)OtYC@rKw0v^7+<+^^HbUj!~Upo z)$I6P4--6m9?aRUciZUoTYlq9D{obQF6{88QNjNOBHOeFriibjm= zMEO*TM+(N+(L+)*E+*;M1& literal 0 HcmV?d00001 diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/b3/376061dade602b1d8ab4f4bc30e3866f2abf92d84848e2c79d02b9c0ced446 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/b3/376061dade602b1d8ab4f4bc30e3866f2abf92d84848e2c79d02b9c0ced446 new file mode 100644 index 0000000000000000000000000000000000000000..08d98aa9af3ac5b303ea72874528aceff4fe87d6 GIT binary patch literal 2381 zcmcImO>7%g5PsgQ9Vc<4IPH7dtt@0Ac~wMJH!Uqmg;1Z9I&JEvwrT9v7i%4F90z-C zHvUPX9ysKHP=z>_8{&XC^nef`P8@sek#i56dPEfmR3v6**K0eqL?y(Nci+sLnQwld z)>)Zf;wrz+^BLIKB4g)%O8y;>2Lf!4F=pAFdgrj*Ic!^}fOV{f*{Qc|I^c|lxON`Q z7JvwOmQy-ofhh$}4Zb{vdHO5HK#OH9XTN0!fx5kUxA@V&!7T8%$6&$|EMW!Uc_Uk8 zm=@a3lQr3@T$~JFiGo-5i7-L58BCI)zPtlDPN8XPF7=XBu~G_7*InmC(zaykx| z1O@;k(N<;Oa(eES*lvMZA_fZAeTCVby0hOY?^~9gIFor4jh&|&k?~B!oFs}Q6m726 zw=JjKFe_HWi$th;On+#w7?0>zSZF~X4WCDZDvkrmLeJ@G}rI981B1L(mUAk7`9iJ;H{@xv^SiQlIA zdjzTsVjq^2pznzT3hK7)5ti^zQbZt$I$a4AxmuGOcm%s7DN1s{OL+@PFc|E> zZNsygLxh670m2%H*Y2R+F1~IM^cIm)ZlUp7O=dlpw>*53ub~pI+!omry>awzGP#^V zx^lVNE#2SPsjR2#rD9)R%D$gW9uU5-zq;EeGkhInA;CT-feS)LQM#~|>tlX^x1K(# zrdN(i#X=YPH0mFa{SdD{C=FyOC7ieX{DF5@O;YHELLT-OoTZxP{f!aq)B`LvR0l*p z-YW5#3P@I59FpT{s`tR&Y;4=5-RgRyS81&5RW290Rgy!THMd`DeJnV|MWE{>_w!#R z9?nuVOg@J@DWe)L>hH~srHY+zSBixlvj{Modl1j^ct7MwKkks6{8pOjPRUINz;kXe za@lKX&E?4{*3Dh1{D6YEU^=`xfVq>hRJ?L=+ivWbmm1v?@JxHF=9ul>ot9Ix;%&=m gnl^kR;%(FUq-%AOOZr3jO1%DKRrp(zg+EgN0AC2h^#A|> literal 0 HcmV?d00001 diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/b4/684cec290b5fd2220e9c6171d702a04277ef7d63ac3d3a1ee4f16858f99bfb b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/b4/684cec290b5fd2220e9c6171d702a04277ef7d63ac3d3a1ee4f16858f99bfb new file mode 100644 index 000000000..f2bcb8b38 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/b4/684cec290b5fd2220e9c6171d702a04277ef7d63ac3d3a1ee4f16858f99bfb @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"13e75f59b6d2263213ab619d146d42647f13e6a0c0c75d1fd910d690e640c07","kind":"bucket","entries":[{"relative_path":"topology/edges/r-d418446a2dd7f837aa2867f6d34c26ffd1d4af1cbfa0d506a31c9dd47ed79c22/00000000000000000009-00000000000000000014.parquet","byte_length":2971,"content_sha256":"4071639376da655addfd1c644128070f7f969953f19c8a61c659c9b8f2c79c21","content_xxh64":"024d00473b27588c","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/bc/9975ccb156a95a845d2b6a42696af0ca8e1b295a3f79ddd9767d2c8bf0a4db b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/bc/9975ccb156a95a845d2b6a42696af0ca8e1b295a3f79ddd9767d2c8bf0a4db new file mode 100644 index 000000000..8d882fb0c --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/bc/9975ccb156a95a845d2b6a42696af0ca8e1b295a3f79ddd9767d2c8bf0a4db @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":3,"depth":0,"prefix":"","kind":"branch","children":{"0":"a547e1515d6437ddb4c530f805e4a9f6fde3f42fb3725c862e5ac3675faec543","1":"917ceda6a0f6fbf916cdf7191151a8d7b2a2d4b38aca4e426d43f93a7dd718d9","2":"8f37f3d7a6fb61dfb189d07fb3bdbdbc3e0969b30b41a180edae2ea72d96deac","3":"33d5b384bd457bf148bc1711beb3544d41d40c9499d4e454c0660a173ee69222","4":"18d346059cfdf7e40ccc3a06f1c352ffcced481736e5994d008e97c52f490034","5":"47ea851bc5d6f0806954e1b36477c5a61b5fe18044fe79cfbbc304f04d052ea6","6":"023f0c4332a566cb6fbb2fb4078aa318c51f2598960381f693d66f96dbabca03","7":"b4684cec290b5fd2220e9c6171d702a04277ef7d63ac3d3a1ee4f16858f99bfb","8":"7056644ec93e357f016906a0715f49cb92d9e87440d6e0116e51e7b65bd51821","a":"d4b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46","b":"64d0aca220e3dbc30007411119041cd6bc8a807580c02e4d994b0f73e0a58bfa","d":"18450af8e920af1a4a64bf48c6cf10ca9a08eadbc08f8c06485be33236de4548","f":"f1dd0b9fb6f895682403236f1c3f497806c1b951a0c27417f3f83ba93fa06b3f"}} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/be/74658ea7232044b698b546a2a10249697ee3748e41481e2d74f0448e172b2d b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/be/74658ea7232044b698b546a2a10249697ee3748e41481e2d74f0448e172b2d new file mode 100644 index 000000000..cd23047be --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/be/74658ea7232044b698b546a2a10249697ee3748e41481e2d74f0448e172b2d @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/r-5aacc116ac666e02684686f6df7f94a9fd62ee5748dd6e0daa0900d949874c8c.out.csr.json","byte_length":537,"content_sha256":"81dc0dfc6f7ed687160b947453c0c3df2cdb44fc81b8d9dc89a16262beb0448e","content_xxh64":"7c4b5c2f8061bc36","role":"index"},{"relative_path":"topology/nodes/00000000000000000001-00000000000000000008.parquet","byte_length":2284,"content_sha256":"5527b3e2c14fbe8a9cfe38a2b7ae8ce696ac7bf9fbb3ec7cbc73bd697e7047f9","content_xxh64":"85ec9a5c84886c1e","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/bf/d3d2ea93bb986cf0959a2775eb9d93f31f87e9b1e28f4998d0a129d0fd5572 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/bf/d3d2ea93bb986cf0959a2775eb9d93f31f87e9b1e28f4998d0a129d0fd5572 new file mode 100644 index 0000000000000000000000000000000000000000..c01d237e9f5c503ef381b8ae82d467b4e24713a2 GIT binary patch literal 588 zcmZ9INe;qL3`6rfLMatT;BXyxJpxCs6E%{t7gqer!Qcx>s6toID1-*hAV^UN2sVBY0}Fe9NdE#m z3o75|*li&d;x#aE(b>M2{oS3DY}f0%`y29Y>Vlp}M5gr)74fAat)zY%;XjV9LV=z>RlPro+4GLAy#Bo zpX)##+B@I_ zFk_jACDwOPrv9rkV^}*&%#jr6F~0{~-Wl22+wWw30iato`oQF4{qKxy?RgK&=%gl_ zy@@VPte>W$=~J2=cuS2=GF+5#SXeD_nQ6`g(>ADAsQ z{@_XEVL1BoQ1l|dfBxIr*@As^K2zx5|Ni}MBe(;mjc?rlcd+<9{>$G%_CDaxg|oQ` N?CBu!-?y#*p|2S-zZw7l literal 0 HcmV?d00001 diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/d4/b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/d4/b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46 new file mode 100644 index 000000000..a5f6dc5ec --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/d4/b2daf3474f75520b815760fe63ff120e96e6b370b630b3122e0e873c287c46 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"d700c977f781c5d2c0d2933adc0f49897368b8bf7c0938b0da5b942ec261948","kind":"bucket","entries":[{"relative_path":"indexes/adjacency/r-5aacc116ac666e02684686f6df7f94a9fd62ee5748dd6e0daa0900d949874c8c.out.csr.shards-312746a9f2902762689369ed.d/00000000000000000000.csr","byte_length":1866,"content_sha256":"c4dc71e92a7313804d090463634dc5c054599b0f764d6409999b9e2f001a1691","content_xxh64":"3d505128daacb6b6","role":"index"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/d5/cd023a1b8ff5dc2c02dc9900a54b06c2134b0e6ba5a9272d14510436c3a12c b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/d5/cd023a1b8ff5dc2c02dc9900a54b06c2134b0e6ba5a9272d14510436c3a12c new file mode 100644 index 000000000..953687a71 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/d5/cd023a1b8ff5dc2c02dc9900a54b06c2134b0e6ba5a9272d14510436c3a12c @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"5","kind":"bucket","entries":[{"relative_path":"topology/uuid-membership/identities-v5-base-0-e3b0c44298fc1c14.uuidx","byte_length":0,"content_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","content_xxh64":"ef46db3751d8e999","role":"topology"},{"relative_path":"topology/uuid-membership/ordinal-v4-2-5ddfdc0ed3f1e1d8.uuidx","byte_length":96,"content_sha256":"5ddfdc0ed3f1e1d8cd8dcc7820b8f1fc8ef7c3d0d2d1735c749ee309ecaf8c6b","content_xxh64":"141414f46e12a329","role":"topology"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/da/bf2e92de1180195cdb270af0f86f741dbe328828819cfd28dcf6e08fb972c5 b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/da/bf2e92de1180195cdb270af0f86f741dbe328828819cfd28dcf6e08fb972c5 new file mode 100644 index 000000000..2fbd3e028 --- /dev/null +++ b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/da/bf2e92de1180195cdb270af0f86f741dbe328828819cfd28dcf6e08fb972c5 @@ -0,0 +1 @@ +{"format":"graphforge-graph-manifest-radix-node","format_version":4,"depth":1,"prefix":"36df068263263b5fd050ba0a0d221de726822dfb00605b949a900812951e971","kind":"bucket","entries":[{"relative_path":"semantic-routes.json","byte_length":177,"content_sha256":"e89f6e446520ec6669f4c8b27486370fabd8c152a409d1363efcd9b32d2b15e6","content_xxh64":"17beb01a6e2bfc9f","role":"catalog"}]} diff --git a/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/db/88cbed903b9fb0e52c938d3652ef0bea61cd688a1064e81afdfaff62e1640f b/tests/fixtures/legacy-membership-index/project/graph-objects/sha256/db/88cbed903b9fb0e52c938d3652ef0bea61cd688a1064e81afdfaff62e1640f new file mode 100644 index 0000000000000000000000000000000000000000..4a85c3118f75330be3e85d0f784ae941d064b9ea GIT binary patch literal 336 zcmZ9?NeX~a3rH0E6V;rAPG$89kztxuXC{N4om-P+a+Gs5UPo1cbh^V6_w Hej4@ +Nodes i..i+n-1; edges form a ring over those nodes. UUIDs are v7-shaped and index-derived. +""" + +import sys +import uuid + +import pyarrow as pa +import pyarrow.parquet as pq + +dst, first_node, count, first_edge = ( + sys.argv[1], + int(sys.argv[2]), + int(sys.argv[3]), + int(sys.argv[4]), +) + + +def v7(kind, index): + value = (kind << 100) | (0x7 << 76) | (0x2 << 62) | index + return uuid.UUID(int=value).bytes + + +node_ids = [v7(1, first_node + i) for i in range(count)] +nodes = pa.table( + { + "node_uuid": pa.array(node_ids, pa.binary(16)), + "label": pa.array(["Person"] * count), + } +) +edges = pa.table( + { + "edge_uuid": pa.array([v7(2, first_edge + i) for i in range(count)], pa.binary(16)), + "rel_type": pa.array(["KNOWS"] * count), + "source_uuid": pa.array(node_ids, pa.binary(16)), + "target_uuid": pa.array([node_ids[(i + 1) % count] for i in range(count)], pa.binary(16)), + } +) +pq.write_table(nodes, f"{dst}/nodes.parquet") +pq.write_table(edges, f"{dst}/edges.parquet") +print("ok", dst) From d12a0b1a91274e7d58db05a5ebf0e818502eca5c Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Thu, 8 Oct 2026 20:15:12 +0000 Subject: [PATCH 6/9] docs: record the legacy-project behaviour changes of the index removal (#1902) Pre-upgrade deleted edge UUIDs become reusable, and search verifies nodes by repeat-check when a project has no ordinal facet. Both are accepted under the pre-v1 in-place format policy. Co-Authored-By: Claude Sonnet 5.5 --- docs/book/architecture/storage.md | 2 +- .../architecture/uuid-membership-index.md | 21 ++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/book/architecture/storage.md b/docs/book/architecture/storage.md index 80f91438b..52f208db3 100644 --- a/docs/book/architecture/storage.md +++ b/docs/book/architecture/storage.md @@ -106,7 +106,7 @@ GraphForge uses a **dual-key pattern** for all first-class objects: ### Why UUIDv7 -UUIDv7 (RFC 9562) is time-ordered within a millisecond, globally unique without coordination, fits in Arrow `FixedSizeBinary(16)`, and supports offline generation on mobile devices or air-gapped systems. See [UUID identity authority](uuid-membership-index.md) for identity lookup. +UUIDv7 (RFC 9562) is time-ordered within a millisecond, globally unique without coordination, fits in Arrow `FixedSizeBinary(16)`, and supports offline generation on mobile devices or air-gapped systems. See [UUID identity authority](uuid-membership-index.md) for identity lookup, including what changed for projects written before the membership index was removed. UUID byte order, accepted text form, content-derived UUIDv8 records, canonical Arrow bytes, and domain-separated SHA-256 fingerprints follow the frozen diff --git a/docs/book/architecture/uuid-membership-index.md b/docs/book/architecture/uuid-membership-index.md index a12afc5cf..70a98a622 100644 --- a/docs/book/architecture/uuid-membership-index.md +++ b/docs/book/architecture/uuid-membership-index.md @@ -62,11 +62,22 @@ Validation refuses what the commit refuses. Before #1902, `validate_bulk_*` accepted a deleted UUID and the commit rejected it later; both now consult the same probe. -Nodes are additionally covered by the ordinal facet, whose forward runs retain -every node UUID that was ever appended, tombstones included. A project that -deleted entities before #1902 has those deletions only in its ignored membership -index (edges) or in the forward runs (nodes): a deleted node UUID stays refused -at commit, and a deleted edge UUID from before the upgrade becomes reusable. +## Projects that predate the removal + +Two behaviours differ for a project written before #1902. Both are accepted +under the pre-v1 policy, which changes the format in place without a migration: + +- **Deleted edge UUIDs become reusable.** A project that deleted edges before + the upgrade holds those deletions only in its ignored membership index, so the + UUID of an edge deleted before the upgrade can be appended again. Entities + deleted after the upgrade are recorded in `deleted_identities.parquet` and stay + spent. Deleted node UUIDs from before the upgrade were not separately tested. +- **Search verifies nodes by repeat-check when there is no ordinal facet.** + `NodeIdentityCheck` used the ordinal facet when a project had one and the + membership index otherwise. With the index gone, a project without an ordinal + facet checks only that the rows it reads do not repeat a node UUID; it no longer + cross-checks each row's `node_id` against a separate authority. A project with + the ordinal facet is unchanged. ## Node ordinal facet From b37d359b2834877b1884c1609656428c56e440fd Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Thu, 8 Oct 2026 20:39:47 +0000 Subject: [PATCH 7/9] fix(storage): refuse a replaced or duplicated fragment in the identity probe and document the legacy upgrade (#1902) A fragment replaced after its footer was read is refused when decoded, because the pruning metadata would no longer describe it. A UUID held by two rows is a typed error. Absent and truncated statistics are tested to mean 'may hold'. The probe's lock-free contract is stated: it is authoritative only at commit, where the rewrite lock refuses a commit whose probed generation moved. The docs say that every pre-upgrade deletion becomes reusable and that the stale index files are inert and carried forward. Co-Authored-By: Claude Sonnet 5.5 --- .../src/topology_identity.rs | 46 ++++++- .../src/topology_identity/tests.rs | 112 +++++++++++++++++- .../architecture/uuid-membership-index.md | 19 ++- 3 files changed, 165 insertions(+), 12 deletions(-) diff --git a/crates/graphforge-storage/src/topology_identity.rs b/crates/graphforge-storage/src/topology_identity.rs index 4784f60ec..b09409631 100644 --- a/crates/graphforge-storage/src/topology_identity.rs +++ b/crates/graphforge-storage/src/topology_identity.rs @@ -165,17 +165,21 @@ fn footer_cache() -> &'static Mutex>> { CACHE.get_or_init(|| Mutex::new(HashMap::new())) } -fn file_key(path: &Path) -> Result { +fn key_of(metadata: &std::fs::Metadata) -> FileKey { use std::os::unix::fs::MetadataExt; - let metadata = std::fs::symlink_metadata(path) - .map_err(|error| GfError::Storage(format!("topology fragment: {error}")))?; - Ok(FileKey { + FileKey { device: metadata.dev(), inode: metadata.ino(), length: metadata.len(), modified_nanos: i128::from(metadata.mtime()) * 1_000_000_000 + i128::from(metadata.mtime_nsec()), - }) + } +} + +fn file_key(path: &Path) -> Result { + let metadata = std::fs::symlink_metadata(path) + .map_err(|error| GfError::Storage(format!("topology fragment: {error}")))?; + Ok(key_of(&metadata)) } fn storage(message: impl std::fmt::Display) -> GfError { @@ -322,6 +326,11 @@ fn load_fragment( let group_rows = usize::try_from(group.num_rows()).unwrap_or(0); rows += group_rows as u64; let column = group.column(uuid_leaf); + // Absent statistics, or bounds that are not whole 16-byte UUIDs, make the + // group "may hold". The writer truncates statistics at 64 bytes + // (`permanent_parquet`), so a 16-byte UUID bound is never truncated: a + // bound of any other width can only come from another writer, and is + // not trusted. let bounds = column.statistics().and_then(|statistics| { let min: [u8; 16] = statistics.min_bytes_opt()?.try_into().ok()?; let max: [u8; 16] = statistics.max_bytes_opt()?.try_into().ok()?; @@ -446,6 +455,18 @@ impl Fragment { ) -> Result<(), GfError> { let file = std::fs::File::open(&self.path) .map_err(|error| storage(format!("{}: {error}", self.path.display())))?; + // The footer this probe pruned by describes one file. Decoding another + // one with it would misread or miss rows, so a fragment replaced since + // the footer was read is refused rather than trusted. + let opened = file + .metadata() + .map_err(|error| storage(format!("{}: {error}", self.path.display())))?; + if key_of(&opened) != self.key { + return Err(storage(format!( + "{} changed since its footer was read", + self.path.display() + ))); + } let file = crate::catalog::admitted_path_file(file) .map_err(|error| storage(format!("{}: {error}", self.path.display())))?; let mut leaves = vec![self.uuid_leaf]; @@ -539,6 +560,14 @@ fn collect_matches( } /// Identity lookups over one topology generation's published Parquet. +/// +/// A probe is a read of one pinned generation and takes no lock. It is advisory +/// for validation and authoritative only at commit: `commit_uuid_topology_rewrite` +/// checks the new identities and then, under the project rewrite lock, refuses +/// the commit unless the topology generation it probed is still the prior one. +/// A publication that lands after a probe was opened therefore cannot let a +/// stale answer commit, and a fragment replaced under a retained probe is refused +/// when it is decoded (see `Fragment::scan_group`). #[derive(Clone)] pub struct TopologyIdentityProbe { nodes: Vec>, @@ -776,7 +805,12 @@ impl TopologyIdentityProbe { metrics.pages_read += pages; metrics.identity_bytes_read += bytes; for (index, surrogate) in found { - resolved[index] = Some(surrogate); + // One live entity per UUID. A second row naming it, in this + // file or another, is corrupt topology; the rows were decoded + // already, so the check reads nothing more. + if resolved[index].replace(surrogate).is_some() { + return Err(storage("a UUID is held by more than one topology row")); + } } } metrics.found = resolved.iter().filter(|value| value.is_some()).count() as u64; diff --git a/crates/graphforge-storage/src/topology_identity/tests.rs b/crates/graphforge-storage/src/topology_identity/tests.rs index eee84fa57..5ed5d2214 100644 --- a/crates/graphforge-storage/src/topology_identity/tests.rs +++ b/crates/graphforge-storage/src/topology_identity/tests.rs @@ -27,6 +27,19 @@ fn write_fragment( uuid_column: &str, id_column: Option<&str>, statistics: EnabledStatistics, +) -> PathBuf { + write_fragment_as(dir, uuid_column, id_column, statistics, None, ROWS) +} + +/// Like [`write_fragment`], with the statistics truncated at `truncate` bytes +/// and `rows` rows starting at the first identity. +fn write_fragment_as( + dir: &Path, + uuid_column: &str, + id_column: Option<&str>, + statistics: EnabledStatistics, + truncate: Option, + rows: usize, ) -> PathBuf { let mut fields = vec![Field::new( uuid_column, @@ -40,6 +53,8 @@ fn write_fragment( .set_data_page_row_count_limit(PAGE_ROWS) .set_write_batch_size(PAGE_ROWS / 2) .set_statistics_enabled(statistics) + .set_statistics_truncate_length(truncate) + .set_column_index_truncate_length(truncate) .build(); let path = dir.join(format!("{uuid_column}.parquet")); let mut writer = ArrowWriter::try_new( @@ -48,7 +63,7 @@ fn write_fragment( Some(properties), ) .unwrap(); - for start in (0..ROWS).step_by(PAGE_ROWS) { + for start in (0..rows).step_by(PAGE_ROWS) { let rows = start..start + PAGE_ROWS; let uuids = FixedSizeBinaryArray::try_from_iter( rows.clone().map(|row| identity(row).as_bytes().to_vec()), @@ -282,3 +297,98 @@ fn a_cached_footer_never_sends_a_probe_to_the_path_it_was_first_read_from() { assert_eq!(surrogates, [Some(2_501)]); assert_eq!(metrics.pages_read, 1); } + +#[test] +fn absent_statistics_make_every_group_and_page_possible_and_the_answer_still_exact() { + let dir = TempDir::new().unwrap(); + let path = write_fragment_as( + dir.path(), + "node_uuid", + Some("node_id"), + EnabledStatistics::None, + None, + ROWS, + ); + let fragment = load_fragment(&path, "node_uuid", Some("node_id")).unwrap(); + assert!(fragment.groups.iter().all(|group| group.bounds.is_none())); + assert!(fragment.groups.iter().all(|group| group.pages.is_empty())); + let mut probe = probe_over(vec![fragment], Vec::new()); + let (surrogates, metrics) = probe + .lookup_node_surrogates(&[identity(3_500), absent_after(10)]) + .unwrap(); + assert_eq!(surrogates, [Some(3_501), None]); + // Nothing can be ruled out: every row group is decoded. + assert_eq!(metrics.identity_blocks_read, GROUPS); +} + +#[test] +fn truncated_statistics_are_not_trusted_for_pruning() { + // A bound cut short of a whole UUID is only a prefix; pruning by it could + // exclude a row group that holds the candidate. + let dir = TempDir::new().unwrap(); + let path = write_fragment_as( + dir.path(), + "node_uuid", + Some("node_id"), + EnabledStatistics::Page, + Some(8), + ROWS, + ); + let fragment = load_fragment(&path, "node_uuid", Some("node_id")).unwrap(); + assert!(fragment.groups.iter().all(|group| group.bounds.is_none())); + let mut probe = probe_over(vec![fragment], Vec::new()); + let (surrogates, metrics) = probe.lookup_node_surrogates(&[identity(1_234)]).unwrap(); + assert_eq!(surrogates, [Some(1_235)]); + assert_eq!(metrics.identity_blocks_read, GROUPS); +} + +#[test] +fn a_fragment_replaced_after_its_footer_was_read_is_refused() { + let dir = TempDir::new().unwrap(); + let mut probe = node_probe(dir.path()); + let path = dir.path().join("node_uuid.parquet"); + // Another file at the same path, as a rewrite that renames over it leaves. + let other = TempDir::new().unwrap(); + let replacement = write_fragment_as( + other.path(), + "node_uuid", + Some("node_id"), + EnabledStatistics::Page, + None, + ROWS / 2, + ); + std::fs::rename(&replacement, &path).unwrap(); + let error = probe + .lookup_node_surrogates(&[identity(10)]) + .unwrap_err() + .to_string(); + assert!( + error.contains("changed since its footer was read"), + "{error}" + ); +} + +#[test] +fn a_uuid_held_by_two_fragments_is_corrupt_topology() { + let dir = TempDir::new().unwrap(); + let first = write_fragment( + dir.path(), + "node_uuid", + Some("node_id"), + EnabledStatistics::Page, + ); + let copy = dir.path().join("second.parquet"); + std::fs::copy(&first, ©).unwrap(); + let mut probe = probe_over( + vec![ + load_fragment(&first, "node_uuid", Some("node_id")).unwrap(), + load_fragment(©, "node_uuid", Some("node_id")).unwrap(), + ], + Vec::new(), + ); + let error = probe + .lookup_node_surrogates(&[identity(10)]) + .unwrap_err() + .to_string(); + assert!(error.contains("more than one topology row"), "{error}"); +} diff --git a/docs/book/architecture/uuid-membership-index.md b/docs/book/architecture/uuid-membership-index.md index 70a98a622..d8fb427e6 100644 --- a/docs/book/architecture/uuid-membership-index.md +++ b/docs/book/architecture/uuid-membership-index.md @@ -67,11 +67,13 @@ same probe. Two behaviours differ for a project written before #1902. Both are accepted under the pre-v1 policy, which changes the format in place without a migration: -- **Deleted edge UUIDs become reusable.** A project that deleted edges before - the upgrade holds those deletions only in its ignored membership index, so the - UUID of an edge deleted before the upgrade can be appended again. Entities - deleted after the upgrade are recorded in `deleted_identities.parquet` and stay - spent. Deleted node UUIDs from before the upgrade were not separately tested. +- **UUIDs deleted before the upgrade become reusable.** The index kept the + tombstones of a project's earlier deletions and nothing reads it now. + `deleted_identities.parquet` records only deletions made since the upgrade, so + an entity deleted before it can be appended again under the same UUID. This + holds for edges and nodes alike as far as the identity probe is concerned; the + ordinal facet's own checks are unchanged. Entities deleted after the upgrade + stay spent. - **Search verifies nodes by repeat-check when there is no ordinal facet.** `NodeIdentityCheck` used the ordinal facet when a project had one and the membership index otherwise. With the index gone, a project without an ordinal @@ -79,6 +81,13 @@ under the pre-v1 policy, which changes the format in place without a migration: cross-checks each row's `node_id` against a separate authority. A project with the ordinal facet is unchanged. +The old files are left alone. Nothing reads `manifest.json`, +`topology-receipt.json`, `identities-v5-*` or `node-surrogates-v5-*` in an +upgraded project, and nothing removes them: orphan collection considers only the +canonical ordinal artifact names, so the files stay in the graph-files inventory +and travel through hydration (the two JSON controls are still copied), export, +verify and import as ordinary entries until a future cleanup drops them. + ## Node ordinal facet `ordinal-v4-manifest.json` is the additive node-only authority for bounded From 35be37eefd2f55515e6a36310d25cb063951f457 Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:56:57 +0000 Subject: [PATCH 8/9] fix(api): resolve UUID node selectors through the Parquet identity probe (#1902) Co-Authored-By: Claude Sonnet 5.5 --- .../graphforge-api/src/bulk_construction.rs | 62 +++++++------ .../graphforge-api/src/graph_publication.rs | 6 +- crates/graphforge-api/src/node_selector.rs | 89 +++++++------------ 3 files changed, 71 insertions(+), 86 deletions(-) diff --git a/crates/graphforge-api/src/bulk_construction.rs b/crates/graphforge-api/src/bulk_construction.rs index d3c7b9538..482289ca3 100644 --- a/crates/graphforge-api/src/bulk_construction.rs +++ b/crates/graphforge-api/src/bulk_construction.rs @@ -282,8 +282,39 @@ impl ValidatedBulkEdges { } } -/// The identity probe for the committed topology generation, cached on the -/// facade until the generation moves. +impl GraphForge { + /// The identity probe for the committed topology generation, cached on the + /// facade until the generation moves. It reads the published node and edge + /// Parquet, so a lookup is bounded by the candidates, not the graph. + pub(crate) fn cached_identity_probe( + &self, + ) -> Result< + std::sync::MutexGuard<'_, Option>, + graphforge_core::GfError, + > { + let current_generation = graphforge_storage::read_topology_generation(&self.dir())?; + let mut cached = self.identity_probe.lock().map_err(|_| { + graphforge_core::GfError::Storage("identity probe lock poisoned".into()) + })?; + if cached + .as_ref() + .is_some_and(|probe| probe.topology_generation() != current_generation) + { + *cached = None; + } + if cached.is_none() { + let dir = self.dir(); + let files = dir.topology_files()?; + *cached = Some(graphforge_storage::TopologyIdentityProbe::open( + &dir, + &files, + current_generation, + )?); + } + Ok(cached) + } +} + fn open_identity_probe( graph: &GraphForge, input_kind: BulkInputKind, @@ -291,36 +322,13 @@ fn open_identity_probe( std::sync::MutexGuard<'_, Option>, BulkValidationError, > { - let project_state = |error: &dyn std::fmt::Display| { + graph.cached_identity_probe().map_err(|error| { contract_error( input_kind, BulkValidationReason::ProjectState, &error.to_string(), ) - }; - let current_generation = graphforge_storage::read_topology_generation(&graph.dir()) - .map_err(|error| project_state(&error))?; - let mut cached = graph - .identity_probe - .lock() - .map_err(|error| project_state(&error))?; - if cached - .as_ref() - .is_some_and(|probe| probe.topology_generation() != current_generation) - { - *cached = None; - } - if cached.is_none() { - let dir = graph.dir(); - let files = dir - .topology_files() - .map_err(|error| project_state(&error))?; - *cached = Some( - graphforge_storage::TopologyIdentityProbe::open(&dir, &files, current_generation) - .map_err(|error| project_state(&error))?, - ); - } - Ok(cached) + }) } fn existing_edge_context( diff --git a/crates/graphforge-api/src/graph_publication.rs b/crates/graphforge-api/src/graph_publication.rs index 64f124165..eb8135ba8 100644 --- a/crates/graphforge-api/src/graph_publication.rs +++ b/crates/graphforge-api/src/graph_publication.rs @@ -431,11 +431,11 @@ impl GraphForge { .clear(); self.adjacency_provider_for_session().invalidate(); // The topology generation counter restarts at clear(), so a cached - // membership index from before it can match a later generation. + // identity probe from before it can match a later generation. *self - .uuid_membership_index + .identity_probe .lock() - .expect("UUID membership index lock poisoned") = None; + .expect("identity probe lock poisoned") = None; // The session's read authority still declares the generation's node and // edge files, content-store objects the workspace wipe does not touch, // and every later catalog lists node files from it (#1388). Re-establish diff --git a/crates/graphforge-api/src/node_selector.rs b/crates/graphforge-api/src/node_selector.rs index c8f7191c4..9a415019b 100644 --- a/crates/graphforge-api/src/node_selector.rs +++ b/crates/graphforge-api/src/node_selector.rs @@ -36,49 +36,21 @@ impl GraphForge { } /// Confirm one node UUID exists. A selector that names one node resolves by - /// identity lookup in the authenticated UUID membership index (ADR 0057), so - /// it is not bounded by the graph's node count. Only a graph with no index at - /// its topology generation falls back to the bounded topology scan. + /// identity lookup in the published node Parquet (row-group and page pruning, + /// ADR 0057), so it is not bounded by the graph's node count. fn require_node(&self, uuid: Uuid) -> Result { - let present = match self.indexed_node_membership(uuid)? { - Some(present) => present, - None => self.node_uuids(None)?.contains(&uuid), - }; - if present { + let mut cached = self.cached_identity_probe()?; + let (found, _) = cached + .as_mut() + .expect("identity probe was just opened") + .probe(graphforge_storage::UuidIndexKind::Node, &[uuid])?; + if found.first().copied().unwrap_or(false) { Ok(uuid) } else { Err(validation("node selector matched no nodes")) } } - /// Whether the membership index holds `uuid` as a live node, or `None` when - /// no index exists at the current topology generation. This is the only - /// place the selector touches the identity authority, so a different probe - /// (#1925) replaces this function and nothing else. - fn indexed_node_membership(&self, uuid: Uuid) -> Result, GfError> { - let dir = self.dir(); - let generation = graphforge_storage::read_topology_generation(&dir)?; - let mut cached = self - .uuid_membership_index - .lock() - .map_err(|_| GfError::Storage("UUID membership index lock poisoned".into()))?; - if cached - .as_ref() - .is_some_and(|index| index.topology_generation() != generation) - { - *cached = None; - } - if cached.is_none() { - if !graphforge_storage::uuid_membership_index_is_fresh(&dir)? { - return Ok(None); - } - *cached = Some(graphforge_storage::UuidMembershipIndex::open(&dir)?); - } - let index = cached.as_mut().expect("membership index was just opened"); - let (found, _) = index.probe(graphforge_storage::UuidIndexKind::Node, &[uuid])?; - Ok(Some(found.first().copied().unwrap_or(false))) - } - fn resolve_property_match( &self, label: &str, @@ -302,35 +274,40 @@ mod tests { } #[test] - fn uuid_selector_resolves_through_the_membership_index_or_scans_without_one() { + fn uuid_selector_resolves_through_the_published_parquet() { let graph = GraphForge::new(None).unwrap(); graph.execute("CREATE (:Person {name: 'Alice'})").unwrap(); let uuid = first_uuid(&graph); - assert!(graphforge_storage::uuid_membership_index_is_fresh(&graph.dir()).unwrap()); - assert!(graph.uuid_membership_index.lock().unwrap().is_none()); + assert!(graph.identity_probe.lock().unwrap().is_none()); assert_eq!( graph .resolve_node_selector(&NodeSelector::Uuid(uuid)) .unwrap(), uuid ); - // The identity lookup opened the index; no topology scan was needed. - assert!(graph.uuid_membership_index.lock().unwrap().is_some()); + // The lookup opened the Parquet probe; no topology scan was needed. + assert!(graph.identity_probe.lock().unwrap().is_some()); assert_validation(graph.resolve_node_selector(&NodeSelector::Uuid(Uuid::now_v7()))); - // A graph written without an index has no identity authority but topology. - let bare = GraphForge::new(None).unwrap(); - bare.execute("CREATE (:Person {name: 'Bob'})").unwrap(); - let bare_uuid = first_uuid(&bare); - std::fs::remove_dir_all(bare.dir().join("topology/uuid-membership")).unwrap(); - assert!(!graphforge_storage::uuid_membership_index_is_fresh(&bare.dir()).unwrap()); - assert_eq!( - bare.resolve_node_selector(&NodeSelector::Uuid(bare_uuid)) - .unwrap(), - bare_uuid - ); - assert!(bare.uuid_membership_index.lock().unwrap().is_none()); - assert_validation(bare.resolve_node_selector(&NodeSelector::Uuid(Uuid::now_v7()))); + // A new generation replaces the cached probe, and the new node resolves. + graph.execute("CREATE (:Person {name: 'Bob'})").unwrap(); + let both = graph + .execute("MATCH (n:Person) RETURN n.node_uuid") + .unwrap(); + let column = both.batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for row in 0..column.len() { + let found = Uuid::from_slice(column.value(row)).unwrap(); + assert_eq!( + graph + .resolve_node_selector(&NodeSelector::Uuid(found)) + .unwrap(), + found + ); + } } #[test] @@ -339,14 +316,14 @@ mod tests { graph.execute("CREATE (:Person {name: 'old'})").unwrap(); let old = first_uuid(&graph); let generation = graphforge_storage::read_topology_generation(&graph.dir()).unwrap(); - // Open the index cache at this generation. + // Open the probe cache at this generation. assert_eq!( graph .resolve_node_selector(&NodeSelector::Uuid(old)) .unwrap(), old ); - assert!(graph.uuid_membership_index.lock().unwrap().is_some()); + assert!(graph.identity_probe.lock().unwrap().is_some()); graph.clear().unwrap(); graph.execute("CREATE (:Person {name: 'new'})").unwrap(); From 57273283f61e3d06e038342be068499032f9ab98 Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:35:37 +0000 Subject: [PATCH 9/9] fix(storage,cli,search): portable fragment identity key; follow the renamed ordinal pass and the retired index directory (#1902) The identity probe keyed its footer cache on Unix-only metadata, so Windows Storage did not compile; it now uses the portable file identity, length and modification time of an open handle. The CLI bulk-build report names its pass ordinal, and a search test no longer deletes an index directory a new project never has. Co-Authored-By: Claude Sonnet 5.5 --- crates/graphforge-cli/tests/portable.rs | 9 +---- .../graphforge-search/src/vector_lifecycle.rs | 1 - .../src/topology_identity.rs | 38 +++++++++++-------- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/crates/graphforge-cli/tests/portable.rs b/crates/graphforge-cli/tests/portable.rs index d69266498..782a5f7b1 100644 --- a/crates/graphforge-cli/tests/portable.rs +++ b/crates/graphforge-cli/tests/portable.rs @@ -631,14 +631,7 @@ fn an_initial_import_reports_its_bulk_build_across_cli_processes() { (Some(2), Some(0)) ); assert_eq!(validated["construction"]["accepted_chunks"], 0); - for pass in [ - "plan", - "nodes", - "edges", - "tables", - "membership", - "adjacency", - ] { + for pass in ["plan", "nodes", "edges", "tables", "ordinal", "adjacency"] { assert!(built["passes"][pass]["wall_ms"].is_u64(), "{pass}"); } let committed = json(&gf( diff --git a/crates/graphforge-search/src/vector_lifecycle.rs b/crates/graphforge-search/src/vector_lifecycle.rs index 263b84848..9f1f81ab1 100644 --- a/crates/graphforge-search/src/vector_lifecycle.rs +++ b/crates/graphforge-search/src/vector_lifecycle.rs @@ -593,7 +593,6 @@ mod tests { let dir = TempDir::new().unwrap(); write_members(&dir, &[(1, vec![9])]); corrupt_node_surrogate_to_null(dir.path()); - std::fs::remove_dir_all(dir.path().join("topology/uuid-membership")).unwrap(); assert!(matches!( project_label_members( dir.path(), diff --git a/crates/graphforge-storage/src/topology_identity.rs b/crates/graphforge-storage/src/topology_identity.rs index b09409631..55c49b13f 100644 --- a/crates/graphforge-storage/src/topology_identity.rs +++ b/crates/graphforge-storage/src/topology_identity.rs @@ -96,8 +96,9 @@ impl UuidProbeMetrics { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] struct FileKey { - device: u64, - inode: u64, + /// Native volume and file identity, portable across Unix and Windows. + volume_serial: u64, + file_id: [u8; 16], length: u64, modified_nanos: i128, } @@ -165,21 +166,27 @@ fn footer_cache() -> &'static Mutex>> { CACHE.get_or_init(|| Mutex::new(HashMap::new())) } -fn key_of(metadata: &std::fs::Metadata) -> FileKey { - use std::os::unix::fs::MetadataExt; - FileKey { - device: metadata.dev(), - inode: metadata.ino(), +/// The identity of an open file: who it is, how long it is and when it changed. +fn key_of(file: &std::fs::File) -> std::io::Result { + let identity = graphforge_filesystem::file_identity(file)?; + let metadata = file.metadata()?; + let modified = metadata.modified()?; + let modified_nanos = match modified.duration_since(std::time::UNIX_EPOCH) { + Ok(after) => i128::try_from(after.as_nanos()).unwrap_or(i128::MAX), + Err(before) => -i128::try_from(before.duration().as_nanos()).unwrap_or(i128::MAX), + }; + Ok(FileKey { + volume_serial: identity.volume_serial, + file_id: identity.file_id, length: metadata.len(), - modified_nanos: i128::from(metadata.mtime()) * 1_000_000_000 - + i128::from(metadata.mtime_nsec()), - } + modified_nanos, + }) } fn file_key(path: &Path) -> Result { - let metadata = std::fs::symlink_metadata(path) + let file = std::fs::File::open(path) .map_err(|error| GfError::Storage(format!("topology fragment: {error}")))?; - Ok(key_of(&metadata)) + key_of(&file).map_err(|error| GfError::Storage(format!("topology fragment: {error}"))) } fn storage(message: impl std::fmt::Display) -> GfError { @@ -458,10 +465,9 @@ impl Fragment { // The footer this probe pruned by describes one file. Decoding another // one with it would misread or miss rows, so a fragment replaced since // the footer was read is refused rather than trusted. - let opened = file - .metadata() - .map_err(|error| storage(format!("{}: {error}", self.path.display())))?; - if key_of(&opened) != self.key { + let opened = + key_of(&file).map_err(|error| storage(format!("{}: {error}", self.path.display())))?; + if opened != self.key { return Err(storage(format!( "{} changed since its footer was read", self.path.display()