diff --git a/Cargo.lock b/Cargo.lock index c9ae4b1d..741e1155 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -809,7 +809,7 @@ dependencies = [ [[package]] name = "ant-node" -version = "0.18.1" +version = "0.19.0" dependencies = [ "alloy", "ant-protocol", @@ -862,8 +862,9 @@ dependencies = [ [[package]] name = "ant-protocol" -version = "2.3.5" -source = "git+https://github.com/WithAutonomi/ant-protocol?rev=88ad3f1#88ad3f17dfd165448be29b7ec1ed5bc1669c7b4f" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804f04e96808a1689a4393652900752980ae7be467b393ae2d8d161306b91301" dependencies = [ "blake3", "bytes", @@ -4884,8 +4885,9 @@ dependencies = [ [[package]] name = "saorsa-core" -version = "0.27.3" -source = "git+https://github.com/WithAutonomi/saorsa-core.git?branch=feat/v2-903-download-route-diagnostics#53ec4ba02144c142c666aed0e2e59cf82c4295f2" +version = "0.27.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a68d3373bbd731bcc099fe9e0a2c587cff5b73d31fe3d4e74d2d81da4dd2bb99" dependencies = [ "anyhow", "async-trait", @@ -4956,9 +4958,9 @@ dependencies = [ [[package]] name = "saorsa-transport" -version = "0.36.3" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cd4be74fa8b82e0321b338bbfbc084c66170614245660b85bb8b041ea808bc4" +checksum = "6b3126de218e3e0912256caee2b179380a68f835b34ca775e0e2882acf62f838" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 2888c86a..2ff4c3d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ant-node" -version = "0.18.1" +version = "0.19.0" edition = "2021" authors = ["David Irvine "] description = "Pure quantum-proof network node for the Autonomi decentralized network" @@ -42,10 +42,10 @@ mimalloc = "0.1" # Pinned to the ant-protocol main commit that merged the settlement wire # types (WithAutonomi/ant-protocol#25). Swap back to a published version pin # once the release train publishes them. -ant-protocol = { git = "https://github.com/WithAutonomi/ant-protocol", rev = "88ad3f1" } +ant-protocol = "2.4.0" # Core (provides EVERYTHING: networking, DHT, security, trust, storage) -saorsa-core = "0.27.3" +saorsa-core = "0.27.4" saorsa-pqc = "0.5" # Payment verification - autonomi network lookup + EVM payment @@ -244,11 +244,3 @@ cognitive_complexity = "allow" # Allow non-const functions during initial development (may need runtime features later) missing_const_for_fn = "allow" -# Temporary stacked-PR alignment, mirroring ant-client main: the git-sourced -# ant-protocol (settlement branch, includes main) declares its saorsa-core as -# the v2-903 route-diagnostics branch, so the node's registry saorsa-core must -# resolve to the same source or the graph carries two MultiAddr types. -# Remove together with the git ant-protocol pin once the release train -# publishes ant-protocol with the released saorsa-core (WithAutonomi/saorsa-core#162). -[patch.crates-io] -saorsa-core = { git = "https://github.com/WithAutonomi/saorsa-core.git", branch = "feat/v2-903-download-route-diagnostics" } diff --git a/deploy/terraform/cloud-init/worker.yml b/deploy/terraform/cloud-init/worker.yml index 391fe761..ef2cec32 100644 --- a/deploy/terraform/cloud-init/worker.yml +++ b/deploy/terraform/cloud-init/worker.yml @@ -79,7 +79,14 @@ write_files: Type=simple User=ant Group=ant - ExecStart=$${BINARY_PATH} --root-dir $${NODE_DIR} --port 0 --metrics-port $${METRICS_PORT} $${BOOTSTRAP_ARGS} + # --enable-logging is not optional here, whatever it looks like. Without it the + # binary installs no tracing subscriber and emits nothing at all, so a node on this + # unit is silent: no migration progress, no warnings, and nothing said about whether + # it still has an old chunk store. That last one does not make the fleet's state + # decidable, and nothing does; it is what lets a node on this unit surface itself as + # unfinished at all. JSON because what reads these lines is a query, and the fields + # that matter are structured. + ExecStart=$${BINARY_PATH} --root-dir $${NODE_DIR} --port 0 --metrics-port $${METRICS_PORT} --enable-logging --log-format json $${BOOTSTRAP_ARGS} Restart=always RestartSec=10 MemoryMax=350M diff --git a/docs/adr/ADR-0014-file-based-chunk-store-and-lmdb-retirement.md b/docs/adr/ADR-0014-file-based-chunk-store-and-lmdb-retirement.md index c0631a2f..5acacf4d 100644 --- a/docs/adr/ADR-0014-file-based-chunk-store-and-lmdb-retirement.md +++ b/docs/adr/ADR-0014-file-based-chunk-store-and-lmdb-retirement.md @@ -370,6 +370,50 @@ belief carry its own expiry — the directory carries its mark, the proof carrie saw, the write carries its note — rather than to check again and hope the check is close enough to the act. +## Amendment: see what the neighbours say, and stop the migration slashing anyone + +Shipped as a patch on top of this record's release, which had already merged. It changes no +on-disk format and writes no migration state: a node part-way through re-reads its marker, its +first-start time and its remaining keys and carries on. That is asserted by a test rather than +argued. + +**A node holding data no longer clears its own commitments.** `storage_empty` asked whether +there was anything left to *commit to*, not whether there were any bytes. Once the migration +settles the commitment narrows to the file-backed set, and `all_keys` drops a file marked +suspect, so two nodes that hold data reported empty: one whose disk filled before it could copy +anything, and one whose last readable file went transiently bad. Both took the `clear_all` +branch, which drops every retained root with no answerability window. An auditor holding a root +gossiped minutes earlier then got `UnknownCommitment` — a confirmed failure on the +commitment-bound lane, which is deliberately enforced in every release and is *not* the lane +the migration holds off. The node slashed itself for data it still had, and no switch could +stop it. Those nodes now take `retire_current`: stop advertising, stay answerable until the +gossip TTL lapses, bytes on disk. That is the staged narrowing this migration was designed +around; the predicate is what routes a node into it. + +**Every node says whether it still has an old chunk store, and reads what its neighbours say.** +The state rides the user agent `saorsa-core` already sends with every signed message and keeps +per peer, so this costs no new message, no new field and no protocol version, and the `node/` +prefix that gates DHT membership is preserved. Three states, never folded into two: a directory +that could not be read is not one that is not there. + +What the peer half means, in the fewest words that are all true, because a release decision +rests on it. + +It counts what the peers a node is connected to **announced**, each as of that peer's own last +start. `saorsa-core` copies the user agent when it builds the transport, so a node that finishes +migrating goes on announcing `legacy` until it restarts. + +Two consequences, running in opposite directions, so the tally bounds nothing. A peer announcing +`legacy` may have finished since, so the count can be too high. A node that is offline, or simply +not connected to, is absent from it, so the count can be too low. An all-zero tally proves +nothing on its own either, because a node connected to nobody produces one; the number of peers +seen is what tells that apart. + +So this can surface nodes that have not finished. It cannot establish that none remain, and no +amount of it adds up to that. `outstanding` counts `legacy`, `unknown` and `unreported` +together, because a peer whose disk could not be read and a peer on a build from before this +existed are both as far from finished as `legacy` is. + ## Consequences ### Positive @@ -519,10 +563,10 @@ restored. matching prediction. - The second gates on a soak of the first, plus a verified retirement returning the predicted space. -- The third gates on migration-complete lines across the fleet, refetch backlogs drained, and the - recorded audit failure rate back to its pre-migration baseline. The first release's - observability is - what makes that decidable. +- The third gates on migration-complete lines from the nodes we run, refetch backlogs drained, + and the recorded audit failure rate back to its pre-migration baseline. The observability + added here informs that call; it does not decide it. Nothing here can establish that a node + we neither run nor are connected to has finished. - **How often a short-of-disk node can actually clear the possession gate.** A node whose close group is also short of space will not clear it, will not free its disk, and will tell its operator to add storage. That is the intended answer, but the fleet needs to diff --git a/src/node.rs b/src/node.rs index aa5d1fe7..d63b0e29 100644 --- a/src/node.rs +++ b/src/node.rs @@ -317,6 +317,26 @@ impl NodeBuilder { } } + // Say on the wire whether this node still has an old chunk store. It costs no new + // message and no new field: saorsa-core already sends a user agent with every signed + // message and keeps each peer's, so this is a different value in a string that was + // already there. It is the only thing that tells us anything at all about the nodes we + // do not run and have no logs from. It cannot establish that the fleet has finished: + // a node sees only the peers it is connected to, and each answers as of its own last + // start, so the most this shows is that some peer reported an old store when it last + // started. It can never show that no node has one. + // + // Read from the filesystem here rather than from the store, because the store is + // built later and a node with storage switched off never builds one at all, while + // the directory on its disk is just as real either way. + // + // Fixed for the life of the process: saorsa-core copies the string when it builds + // the transport. A node that finishes migrating goes on saying `legacy` until it + // restarts, which overstates how much is left rather than understating it, and is + // the direction a release gate should err in. + let signal = crate::storage::migration_signal::MigrationSignal::from_disk(&config.root_dir); + core_config.custom_user_agent = Some(crate::storage::migration_signal::user_agent(signal)); + // Persist close group peers + trust scores across restarts. // Default to root_dir (alongside node_identity.key) when not explicitly set. core_config.close_group_cache_dir = Some( @@ -655,6 +675,22 @@ impl RunningNode { info!("Replication engine started"); } + // Say where this node is with the move off the old chunk store, and what it can see + // of its neighbours. The release that deletes that store may only go out once the + // fleet has moved, and no calendar establishes that: our own logs cover the nodes we + // run, and this is the only view we get of the ones we do not. + { + // Weak on purpose: see `report_until_shutdown`. A reporter that kept the node + // alive would keep its port bound after the node was dropped. + let p2p = Arc::downgrade(&self.p2p_node); + let root_dir = self.config.root_dir.clone(); + let shutdown = self.shutdown.clone(); + tokio::spawn(async move { + crate::storage::migration_signal::report_until_shutdown(p2p, root_dir, shutdown) + .await; + }); + } + // Start upgrade monitor if enabled if let Some(monitor) = self.upgrade_monitor.take() { let events_tx = self.events_tx.clone(); diff --git a/src/replication/commitment_state.rs b/src/replication/commitment_state.rs index 9daff439..76cf3e66 100644 --- a/src/replication/commitment_state.rs +++ b/src/replication/commitment_state.rs @@ -945,6 +945,51 @@ fn prune_slots(inner: &mut Inner, now: Instant) { #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { + + /// Retiring keeps an in-window pin answerable; clearing repudiates it. + /// + /// This is why the commitment rotation no longer has a "storage is empty" branch. That + /// branch called `clear_all`, and it was reached from a key count that was wrong in the + /// same direction every time it was fixed: a node whose disk filled before it could copy + /// anything, then one whose file had been dropped from the index by a failed read, then a + /// files-only node that had published bytes but not yet indexed them. + /// + /// The cost of being wrong is what settles it, and it is what this measures. Clearing + /// repudiates a root a peer is still pinning, which answers `UnknownCommitment` and is + /// graded a confirmed failure. Retiring stops advertising and stays answerable until the + /// gossip TTL lapses. Both stop advertising; only one throws the answer away. + #[test] + fn retiring_keeps_a_pinned_root_answerable_and_clearing_does_not() { + let (pk, sk) = keypair(); + let pk_bytes = pk.to_bytes(); + let peer_id = *blake3::hash(&pk.to_bytes()).as_bytes(); + + let retired = ResponderCommitmentState::new(); + let c = BuiltCommitment::build(vec![(key(1), bh(1))], &peer_id, &sk, &pk_bytes).unwrap(); + let h = c.hash(); + retired.rotate(c); + retired.mark_gossiped(h); + retired.retire_current(); + assert!( + retired.current().is_none(), + "retiring must stop the node advertising the root" + ); + assert!( + retired.lookup_by_hash(&h).is_some(), + "but a peer still pinning it must get an answer, not a repudiation" + ); + + let cleared = ResponderCommitmentState::new(); + let c2 = BuiltCommitment::build(vec![(key(1), bh(1))], &peer_id, &sk, &pk_bytes).unwrap(); + let h2 = c2.hash(); + cleared.rotate(c2); + cleared.mark_gossiped(h2); + cleared.clear_all(); + assert!( + cleared.lookup_by_hash(&h2).is_none(), + "clearing throws the same pin away, which is the confirmed failure this avoids" + ); + } use super::*; use crate::replication::commitment::{commitment_hash, leaf_hash, verify_path}; use saorsa_pqc::api::sig::ml_dsa_65; diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 71124727..8a4d8f4a 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -9980,7 +9980,6 @@ async fn rebuild_and_rotate_commitment( // this filter the pruner's reprieve would keep re-committing stale keys // forever (the rebuild reads all_keys, so a retained-on-disk key would be // re-committed and re-gossiped every rotation — a permanent pin). - let storage_empty = stored_keys.is_empty(); let self_id = *p2p.peer_id(); let mut keys = Vec::with_capacity(stored_keys.len()); for k in stored_keys { @@ -9990,20 +9989,30 @@ async fn rebuild_and_rotate_commitment( } if keys.is_empty() { - if storage_empty { - // Storage is genuinely empty — there is nothing to answer for, so - // drop the previously advertised commitment immediately. Keeping it - // would leave remote auditors pinning a hash we can never satisfy - // again (the bytes are gone). - if state.retained_slot_count() > 0 { - debug!("Commitment rotation: storage empty, clearing retained slots"); - state.clear_all(); - } - storage.note_commitment_rebuilt(); - return Ok(()); - } - // Bytes are still on disk but no key is currently in range. We must NOT - // clear retention here: a peer may still be pinning a root we gossiped + // There used to be a second branch here that dropped every retained root outright + // when the node looked empty. It is gone, and the reason is worth keeping. + // + // "Empty" was decided from key counts, and every version of that test was wrong in + // the same direction. It read the committable set, which narrows to the file-backed + // keys once the migration settles, so a node whose disk filled before it could copy + // anything looked empty with a full legacy store beside it. Adding the raw file index + // still missed a file dropped from the index by a failed read while its legacy copy + // was being put back. Adding the legacy environment still missed a files-only node + // that had published bytes to disk but not yet indexed them, because a file is + // published before it is indexed. Each fix closed one window and left another. + // + // The asymmetry is what settles it. Clearing wrongly repudiates a root a peer is + // pinning, and `UnknownCommitment` is a confirmed failure on the commitment-bound + // lane, which is enforced in every release and is not the lane the migration holds + // off — so a node that still holds the bytes is slashed for holding them. Retiring + // wrongly costs a root that stops being advertised now and ages out by its gossip TTL + // instead of vanishing now. Both set `has_current = false`; they differ only in + // whether the node goes on being answerable in the meantime. A genuinely empty node + // cannot answer either way, so retiring costs it nothing it had. + // + // So there is one branch, and no emptiness question to get wrong. + // + // A peer may still be pinning a root we gossiped // moments ago and could demand its bytes in a round-2 challenge, which // we can still answer (the bytes are present). But we must STOP // advertising the stale commitment: retire it so `current()` returns diff --git a/src/storage/chunk_store.rs b/src/storage/chunk_store.rs index fd4c7435..a6c59470 100644 --- a/src/storage/chunk_store.rs +++ b/src/storage/chunk_store.rs @@ -292,6 +292,7 @@ impl ChunkStore { MigrationPhase::FilesOnly }; let mut state = MigrationState::load_or_create(&config.root_dir, phase); + let legacy_only = Self::legacy_only_count(legacy.as_ref()); // The filesystem is the authority on whether a legacy environment exists; the // marker only records decisions. Reconcile rather than trust. @@ -322,6 +323,32 @@ impl ChunkStore { if let Err(e) = state.save(&config.root_dir) { warn!("Could not persist the migration marker: {e}"); } + } else if state.phase == MigrationPhase::Committed && legacy_only > state.shed_key_count { + // Once committed, the copier only moves keys the rank check refuses to shed. + // A chunk that turns up in the legacy environment after that, and that this + // node never agreed to give up, is never copied and never proven to be held + // elsewhere, so nothing frees it from the environment and the environment is + // never retired. Free disk does not change that; only the phase does. + // + // The count, not emptiness, is the test. A node that legitimately shed keeps + // exactly the keys it is giving up in the legacy environment until they stop + // being answerable, and sending it back to the bridge on every restart would + // reset its retention clock each time. + warn!( + migration_event = "back_to_bridging", + legacy_only, + shed = state.shed_key_count, + "The migration marker says this node committed after agreeing to shed {} \ + chunk(s), but {legacy_only} are still only in the legacy environment. \ + Restarting the migration from the copying stage so they are copied.", + state.shed_key_count + ); + state.phase = MigrationPhase::Bridging; + state.committed_at_unix = None; + state.rebuilds_since_commit = 0; + if let Err(e) = state.save(&config.root_dir) { + warn!("Could not persist the migration marker: {e}"); + } } else if legacy.is_some() && state.phase == MigrationPhase::FilesOnly { warn!( "The migration marker says this node is done but {} is still on disk. \ @@ -416,6 +443,11 @@ impl ChunkStore { .collect() } + /// How many keys only the legacy environment holds; zero once it is gone. + fn legacy_only_count(legacy: Option<&Legacy>) -> u64 { + legacy.map_or(0, |l| l.only.read().len().try_into().unwrap_or(u64::MAX)) + } + /// Take the critical section for one key. async fn key_lock(&self, address: &XorName) -> Option> { let lane = address.last().copied().unwrap_or(0) as usize; @@ -3172,6 +3204,67 @@ mod tests { assert!(!store.exists(&target).expect("exists")); } + /// A store left partway through by the release that shipped is picked up, not restarted. + /// + /// This release lands on nodes that are already migrating under the one before it, so what + /// matters is not what it does to a fresh store but what it does to a half-finished one. + /// It must re-read that state and change nothing: the phase, the first-start time the + /// waves are measured from, what the node committed to giving up, and the keys still only + /// in the legacy store all have to survive, or a node restarts a clock it had nearly run + /// down, or re-copies what it already copied. + /// + /// Nothing in this release writes migration state, and this is what says so. + #[tokio::test] + async fn a_store_left_midway_by_the_previous_release_keeps_its_place() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["k1", "k2", "k3"]).await; + + let started_at = { + let store = open(&dir).await; + // Partway: one copied, two still only in the legacy store, which is where a node + // that ran out of disk under the previous release sits. + store + .copy_batch(&keys[..1], 0, 0, &never_cancelled()) + .await + .expect("copy"); + store.commit_to_files().expect("commit"); + assert_eq!(store.migration_phase(), MigrationPhase::Committed); + assert_eq!(store.legacy_only_keys().len(), 2); + store.migration_state().first_start_unix + }; + + // The upgrade: a new process opening the same directory. + let store = open(&dir).await; + assert_eq!( + store.migration_phase(), + MigrationPhase::Committed, + "an upgrade must not put a committed node back to bridging" + ); + assert_eq!( + store.migration_state().first_start_unix, + started_at, + "the waves are measured from this, so restarting it restarts the schedule" + ); + assert_eq!( + store.migration_state().shed_key_count, + 2, + "what the node committed to giving up must survive the upgrade" + ); + assert_eq!( + store.legacy_only_keys().len(), + 2, + "the keys still to copy must not be recounted from scratch" + ); + + // And everything is still servable, out of whichever store holds it. + for key in &keys { + assert!( + store.get(key).await.expect("get").is_some(), + "a chunk stopped being servable across the upgrade" + ); + } + } + #[tokio::test] async fn committing_narrows_the_commitment_but_not_what_is_served() { let dir = TempDir::new().expect("temp dir"); @@ -4607,6 +4700,99 @@ mod tests { assert_eq!(store.legacy_only_keys().len(), 3); } + /// V2-1232. Once committed, the copier moves only the keys the rank check refuses to + /// shed, so a chunk that is found in the legacy environment after the commitment, + /// and that this node never agreed to give up, is never copied, never provable + /// elsewhere, and keeps the environment from ever being retired. Free disk does not + /// help and neither does a restart. Reopening has to notice and reopen the bridge. + #[tokio::test] + async fn a_node_that_committed_with_nothing_to_shed_resumes_the_copy_for_chunks_only_it_holds() + { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["c1", "c2"]).await; + let store = open(&dir).await; + store + .copy_batch(&keys, 0, 0, &never_cancelled()) + .await + .expect("copy"); + store.commit_to_files().expect("commit"); + store.note_commitment_rebuilt(); + let state = store.migration_state(); + assert_eq!(state.shed_key_count, 0, "nothing had to be shed"); + assert!(state.committed_at_unix.is_some()); + assert_eq!(state.rebuilds_since_commit, 1); + store.wait_idle().await; + drop(store); + + // A chunk in the legacy environment that the file store has no record of, found + // after the node committed. The file store is untouched, so the kept-count rule + // has nothing to say about this. + let late = seed_legacy(&dir, &["c3"]).await; + + let store = open(&dir).await; + let state = store.migration_state(); + assert_eq!( + state.phase, + MigrationPhase::Bridging, + "a chunk the node never agreed to shed must reopen the bridge" + ); + assert!( + state.committed_at_unix.is_none(), + "the retirement clock must not carry over from the abandoned commitment" + ); + assert_eq!(state.rebuilds_since_commit, 0); + assert_eq!(store.legacy_only_keys(), late); + + // And the copier, which the phase change lets run again, moves it. + store + .copy_batch(&late, 0, 0, &never_cancelled()) + .await + .expect("copy"); + assert!(store.legacy_only_keys().is_empty()); + assert_eq!( + store + .get(late.first().expect("a key")) + .await + .expect("get") + .expect("present"), + addressed("c3").1 + ); + } + + /// The counterpart of the test above, and it matters as much. A node that shed keeps + /// exactly the keys it is giving up in the legacy environment until they stop being + /// answerable, so a non-empty legacy-only set is its normal state. Sending it back + /// to the bridge on every restart would reset its retention clock each time, and a + /// node restarted regularly would never retire. + #[tokio::test] + async fn a_node_that_shed_stays_committed_across_a_restart() { + let dir = TempDir::new().expect("temp dir"); + let keys = seed_legacy(&dir, &["k1", "k2"]).await; + let store = open(&dir).await; + store + .copy_batch(&keys[..1], 0, 0, &never_cancelled()) + .await + .expect("copy"); + store.commit_to_files().expect("commit"); + store.note_commitment_rebuilt(); + let before = store.migration_state(); + assert_eq!(before.shed_key_count, 1); + assert_eq!(store.legacy_only_keys().len(), 1); + store.wait_idle().await; + drop(store); + + let store = open(&dir).await; + let after = store.migration_state(); + assert_eq!( + after.phase, + MigrationPhase::Committed, + "the keys it is giving up are not a reason to reopen the bridge" + ); + assert_eq!(after.committed_at_unix, before.committed_at_unix); + assert_eq!(after.rebuilds_since_commit, before.rebuilds_since_commit); + assert_eq!(store.legacy_only_keys().len(), 1); + } + #[tokio::test] async fn a_file_that_vanished_mid_verification_is_requeued_not_republished() { let dir = TempDir::new().expect("temp dir"); diff --git a/src/storage/migration_signal.rs b/src/storage/migration_signal.rs new file mode 100644 index 00000000..3b37bf93 --- /dev/null +++ b/src/storage/migration_signal.rs @@ -0,0 +1,749 @@ +//! What a node tells the network about its move off the old chunk store. +//! +//! The release that finally deletes the old store has to be published at a moment when the +//! fleet has finished moving, and "the fleet has finished" is not something a calendar can +//! establish. Nor can our own logs: they cover the nodes we run, and the nodes most likely +//! to still be carrying a `chunks.mdb` are the ones we do not. +//! +//! So a node says so itself, in the one field every peer already sees. `saorsa-core` sends a +//! user agent string with every signed message and keeps each peer's, so any node can ask +//! what its neighbours are. Putting the answer there costs no new message, no new field and +//! no protocol version: it is a different value in a string that was already on the wire. +//! +//! Read [`report_until_shutdown`] before using any of this to decide a release. It does not +//! establish that the fleet has finished, and cannot. A node sees only the peers it is +//! connected to, and each of those answers as of its own last start, so the most this can show +//! is that some peer reported an old store when it last started. It can never show that no node +//! has one. What it gives is the only view we get of the nodes we do not run. +//! +//! Two rules the string has to obey. It must still begin `node/`, because that prefix is +//! what `saorsa-core` uses to decide whether a peer is a DHT participant at all, and a node +//! that loses it stops being routed to. And the three states must never be folded into two: +//! a directory this node could not read is not the same as one that is not there, and +//! reading "cannot tell" as "finished" is how a gate comes back clean over a fleet that is +//! not. +//! +//! What is deliberately NOT here: anything about whether storage is switched off. A node +//! with `storage.enabled = false` never opens a store, but the old environment is still on +//! its disk and the release that deletes it will still find it. The question this answers is +//! about the filesystem, so it is asked of the filesystem, whatever the node was configured +//! to do with it. + +use std::path::Path; +use std::sync::{Arc, Weak}; +use std::time::Duration; + +use saorsa_core::P2PNode; +use tokio_util::sync::CancellationToken; + +use crate::logging::{info, warn}; + +/// How often a node says where it is and what it can see. +/// +/// Often enough that a node's reading **of its own disk** is never many hours stale, rarely +/// enough that it is a line an operator can read rather than a stream. What it says about its +/// peers is not fresh at any cadence: their user agents were fixed when their transports were +/// built, so a peer's answer is as of its last start whenever this runs. +/// +/// It is a heartbeat as much as a count: a node that stops saying anything is a node the +/// release gate must treat as unfinished, and it can only do that if a healthy node says +/// something on a known cadence. +const REPORT_INTERVAL: Duration = Duration::from_secs(15 * 60); + +/// The directory the old chunk store lives in. +const LEGACY_ENV_DIR: &str = "chunks.mdb"; + +/// What retirement renames it to before deleting it. +const RETIRED_SUFFIX: &str = ".retired"; + +/// The file retirement writes inside a directory to say it has finished with it. +const RETIRED_MARKER: &str = "RETIRED"; + +/// The token that carries the state, so a reader can find it wherever it sits. +const SIGNAL_PREFIX: &str = "migration/"; + +/// Where this node is in the move off the old chunk store, as seen from its own disk. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MigrationSignal { + /// Something is still there that this node has not finished with. + Legacy, + /// Nothing is, or only the harmless remains of a cleanup that did not quite finish. + Files, + /// The disk could not be read well enough to say. Never folded into either answer. + Unknown, +} + +impl MigrationSignal { + /// The token this state appears as on the wire. + const fn token(self) -> &'static str { + match self { + Self::Legacy => "legacy", + Self::Files => "files", + Self::Unknown => "unknown", + } + } + + /// Read the state off this node's own disk. + /// + /// Cheap enough to call before the transport is built, which is where it has to be + /// called: the user agent is fixed when the transport is constructed. + #[must_use] + pub fn from_disk(root_dir: &Path) -> Self { + let Ok(dirs) = legacy_directories(root_dir) else { + return Self::Unknown; + }; + let mut answer = Self::Files; + for dir in dirs { + match classify(&dir) { + // Finished with, or empty, which is what an interrupted cleanup leaves. + // Neither holds a chunk, so neither makes this node unfinished. + Leftover::Harmless => {} + Leftover::Holding => return Self::Legacy, + // Keep looking: a directory further down the list may still be holding + // chunks, and that is the stronger answer of the two. + Leftover::Unreadable => answer = Self::Unknown, + } + } + answer + } +} + +/// What one leftover directory means for the node carrying it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Leftover { + /// It holds chunks this node has not moved. + Holding, + /// It holds nothing, or it carries the mark that says it was finished with. + Harmless, + /// It could not be read well enough to say which. + Unreadable, +} + +/// Every leftover of the old chunk store under `root_dir`, live name and tombstones alike. +/// +/// The names are matched exactly rather than by prefix. Retirement only ever creates +/// `chunks.mdb.retired` or `chunks.mdb.retired.`, and a prefix match would also claim a +/// directory somebody else put there, which matters because a later release deletes what +/// this list returns. +/// +/// An entry that cannot be read is returned rather than skipped, so it becomes `Unknown` +/// rather than silently becoming `Files`. +fn legacy_directories(root_dir: &Path) -> Result, Unreadable> { + let mut found = Vec::new(); + + // `symlink_metadata`, not `try_exists`: the latter follows links, so a dangling or + // looping one at the live name would read as nothing being there. + let live = root_dir.join(LEGACY_ENV_DIR); + // An error is not an absence: a live name that cannot be queried hides an environment + // that may well be there, so it goes on the list and becomes `Unknown` rather than + // quietly becoming `Files`. + if !matches!(std::fs::symlink_metadata(&live), Err(ref e) if e.kind() == std::io::ErrorKind::NotFound) + { + found.push(live); + } + + let entries = match std::fs::read_dir(root_dir) { + Ok(entries) => entries, + // A root that is not there yet holds nothing, which is every node starting for the + // first time. That is an answer. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(found), + // One that cannot be listed hides every tombstone in it, so there is no answer to + // give. Saying so is the whole reason `Unknown` exists. + Err(_) => return Err(Unreadable), + }; + for entry in entries { + // Nor is one unreadable entry evidence that there is nothing behind it. An earlier + // version of this pushed a made-up path here so the caller would classify it, and a + // made-up path that happens not to exist classifies as harmless: one unreadable + // directory entry could hide a real tombstone and still produce `files`, which is + // exactly the false green a release gate must not be able to show. + let Ok(entry) = entry else { + return Err(Unreadable); + }; + if entry.file_name().to_str().is_some_and(is_tombstone_name) { + found.push(entry.path()); + } + } + Ok(found) +} + +/// There is no answer to give about this root. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Unreadable; + +/// The most tombstones one root can hold, matching what retirement will ever create. +const MAX_TOMBSTONES: u32 = 64; + +/// Is this a name retirement gives a tombstone? +/// +/// `chunks.mdb.retired`, or that plus `.` for `n` in `1..=64`, written the way retirement +/// writes it. The bounds are not decoration: retirement only ever counts up to 64, so `.65` +/// and `.007` are names it cannot have produced, and this list becomes a list of directories +/// a later release deletes. +fn is_tombstone_name(name: &str) -> bool { + let base = format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}"); + if name == base { + return true; + } + let Some(suffix) = name.strip_prefix(&format!("{base}.")) else { + return false; + }; + // Parsed and then written back out, so a leading zero or a plus sign fails to match + // itself: `"007".parse::()` is happily 7, and `chunks.mdb.retired.007` is not a + // name anything here created. + suffix + .parse::() + .is_ok_and(|n| (1..=MAX_TOMBSTONES).contains(&n) && suffix == n.to_string()) +} + +/// What one directory says about itself. +fn classify(dir: &Path) -> Leftover { + match std::fs::symlink_metadata(dir) { + // A link is never treated as finished with, whatever it points at: the mark would + // have been written through it into a directory that is not this node's. It is also + // never followed to see what is behind it. + Ok(meta) if meta.file_type().is_symlink() => return Leftover::Holding, + Ok(_) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Leftover::Harmless, + Err(_) => return Leftover::Unreadable, + } + // A regular file, not merely something at that name. Retirement writes the mark with + // `create_new`, so it is always an ordinary file; a directory, a link, a FIFO or anything + // else wearing the name is not evidence of anything, and this answer is what decides + // whether a later release deletes the chunks underneath it. + match std::fs::symlink_metadata(dir.join(RETIRED_MARKER)) { + Ok(meta) if meta.is_file() => return Leftover::Harmless, + Ok(_) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(_) => return Leftover::Unreadable, + } + // No mark. A directory with nothing in it holds no chunks, so it cannot be hiding any: + // that is what a cleanup interrupted between emptying a tombstone and removing it + // leaves behind. + std::fs::read_dir(dir).map_or(Leftover::Unreadable, |mut entries| { + if entries.next().is_none() { + Leftover::Harmless + } else { + Leftover::Holding + } + }) +} + +/// The user agent this node announces itself with. +/// +/// Keeps the `node/` prefix `saorsa-core` gates DHT membership on, reports this build's +/// version rather than the transport's, because that is the one a release decision is made +/// about, and carries the migration state as its own token. +#[must_use] +pub fn user_agent(signal: MigrationSignal) -> String { + format!( + "node/{} {SIGNAL_PREFIX}{}", + env!("CARGO_PKG_VERSION"), + signal.token() + ) +} + +/// What a peer's user agent says about that peer. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PeerMigrationState { + /// It says it still has an old chunk store. + Legacy, + /// It says it has finished. + Files, + /// It says it cannot tell. + Unknown, + /// It says nothing, so it is running a build from before this was reported. Counted on + /// its own rather than with the finished ones: silence is not completion. + Unreported, + /// Not a node at all. Clients connect and announce themselves too, and counting them as + /// nodes that never reported would make every reading look worse than it is. + NotANode, +} + +/// Read a peer's user agent. +#[must_use] +pub fn peer_state(user_agent: &str) -> PeerMigrationState { + if !user_agent.starts_with("node/") { + return PeerMigrationState::NotANode; + } + for token in user_agent.split_whitespace() { + let Some(state) = token.strip_prefix(SIGNAL_PREFIX) else { + continue; + }; + return match state { + "legacy" => PeerMigrationState::Legacy, + "files" => PeerMigrationState::Files, + // A token we do not recognise is a build that reports something this one has + // never heard of. That is not "finished". + _ => PeerMigrationState::Unknown, + }; + } + PeerMigrationState::Unreported +} + +/// One tally of what a node can see around it. +/// +/// Every field counts what a peer **announced**, which it fixed at its last start. None of them +/// says what that peer holds now. And an all-zero tally is not an answer: a node connected to +/// nobody produces one, and it reads exactly like a tally of peers that have all finished. The +/// number of peers seen is what tells those apart, which is why it is on the line. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct PeerTally { + /// Peers that announced an old chunk store at their last start. + pub legacy: usize, + /// Peers that announced having finished, as of their last start. + pub files: usize, + /// Peers that announced they could not tell, or answered with something this build does + /// not know. + pub unknown: usize, + /// Peers running a build from before this was reported. + pub unreported: usize, +} + +impl PeerTally { + /// Peers that are not evidence the fleet has finished. + #[must_use] + pub const fn outstanding(self) -> usize { + self.legacy + self.unknown + self.unreported + } + + fn add(&mut self, state: PeerMigrationState) { + match state { + PeerMigrationState::Legacy => self.legacy += 1, + PeerMigrationState::Files => self.files += 1, + PeerMigrationState::Unknown => self.unknown += 1, + PeerMigrationState::Unreported => self.unreported += 1, + // Deliberately not counted at all. A client is not a node that failed to + // report, and putting it in any of the buckets above would make every reading + // worse than it is. + PeerMigrationState::NotANode => {} + } + } +} + +/// Count what this node can see of its neighbours. +/// +/// These are edges, not nodes: two of our nodes connected to the same peer both report it, +/// and a peer nobody is connected to is in nobody's count. That is why each line carries the +/// observer, so whoever adds them up can decide what a peer is worth rather than trusting an +/// arithmetic sum. +pub async fn tally_peers(p2p: &Arc) -> PeerTally { + let mut tally = PeerTally::default(); + let transport = p2p.transport(); + let observer = p2p.peer_id().to_hex(); + for peer in transport.connected_peers().await { + // No agent recorded is not the same as a peer that reported nothing, but it is + // just as far from evidence of completion, so it lands in the same bucket rather + // than being skipped. + let agent = transport.peer_user_agent(&peer).await; + let state = agent + .as_deref() + .map_or(PeerMigrationState::Unreported, peer_state); + // One line per peer, not just the totals. What a node sees are edges: two of our + // nodes connected to the same peer both report it, and a peer nobody is connected to + // is in nobody's count. Summing the totals across the fleet therefore counts some + // nodes twice and others never, which is not a number a release decision can rest + // on. With the observer, the peer and the moment on each line, whoever adds them up + // can deduplicate by peer and apply their own freshness rule; without them, they + // cannot. + // + // At `info`, not `debug`. Nodes run at `info` (`cli.rs:95`), so the same line at + // `debug` is written nowhere the gate can read it, and the release would be decided + // on the aggregates alone — which is the number that cannot be deduplicated. A line + // nobody emits is not a signal. + // + // Only for peers that are not reporting finished. The gate asks which distinct nodes + // are still outstanding, so those are the ones that need naming, and the cost falls + // away as they stop being outstanding rather than peaking when they do. Note which way + // that runs: these lines stopping means no peer this node is connected to is still + // reporting an old store, which is not the same as the fleet having finished, and + // never can be. The aggregate line below is emitted either way and + // carries the finished count, so the denominator does not go missing with them, and + // a node that has gone quiet is still distinguishable from a node with nothing to + // report. + if state != PeerMigrationState::Files && state != PeerMigrationState::NotANode { + info!( + migration_event = "peer_state", + observer = %observer, + peer = %peer.to_hex(), + state = peer_state_token(state), + agent = agent.as_deref().unwrap_or("none"), + "Storage migration: peer {} is {}", + peer.to_hex(), + peer_state_token(state) + ); + } + tally.add(state); + } + tally +} + +/// The token a peer's state is reported as, so the aggregate line and the per-peer lines +/// cannot drift apart. +/// +/// Only ever read by a log line, so it goes when the logging feature does. +#[cfg_attr(not(feature = "logging"), allow(dead_code))] +const fn peer_state_token(state: PeerMigrationState) -> &'static str { + match state { + PeerMigrationState::Legacy => "legacy", + PeerMigrationState::Files => "files", + PeerMigrationState::Unknown => "unknown", + PeerMigrationState::Unreported => "unreported", + PeerMigrationState::NotANode => "not-a-node", + } +} + +/// Say where this node is, and what it can see of its neighbours, until it shuts down. +/// +/// Two questions, and they are not the same one. **This node's own state** is read from its +/// disk every pass, so a node that finishes says so within the interval rather than at its next +/// restart. **What it sees of its peers** is read from their user agents. +/// +/// What the peer half means, in the fewest words that are all true, because a release decision +/// rests on it. +/// +/// It counts what the peers this node is connected to **announced**, each as of that peer's own +/// last start. `saorsa-core` copies the user agent when it builds the transport, so a node that +/// finishes migrating goes on announcing `legacy` until it restarts. +/// +/// Two consequences, and they run in opposite directions, so the tally bounds nothing. A peer +/// announcing `legacy` may have finished since, so the count can be too high. A node that is +/// offline, or simply not connected to, is absent from it, so the count can be too low. An +/// all-zero tally proves nothing on its own either, because a node connected to nobody produces +/// one; the number of peers seen is what tells that apart, which is why it is on the line. +/// +/// So this can surface nodes that have not finished. It cannot establish that none remain, and +/// no amount of it adds up to that. `outstanding` counts `legacy`, `unknown` and `unreported` +/// together, because a peer whose disk could not be read and a peer on a build from before this +/// existed are both as far from finished as `legacy` is. +/// +/// It is still the only way our own fleet learns anything at all about the nodes we do not run. +/// +/// The handle is **weak**. A reporter must never be the reason the thing it observes stays +/// alive: a strong one would keep a dropped node's transport, and its bound port, for as long +/// as this task ran. +pub async fn report_until_shutdown( + p2p: Weak, + root_dir: std::path::PathBuf, + shutdown: CancellationToken, +) { + loop { + // Wait first. A node that has just started has no peers to describe, and nothing has + // changed on its disk since the user agent was built from it. + tokio::select! { + () = shutdown.cancelled() => return, + () = tokio::time::sleep(REPORT_INTERVAL) => {} + } + let Some(node) = p2p.upgrade() else { + return; + }; + let own = MigrationSignal::from_disk(&root_dir); + let peers = tally_peers(&node).await; + drop(node); + let seen = peers.outstanding() + peers.files; + if own == MigrationSignal::Legacy || own == MigrationSignal::Unknown { + warn!( + migration_event = "signal", + state = own.token(), + peers_legacy = peers.legacy, + peers_unknown = peers.unknown, + peers_unreported = peers.unreported, + peers_files = peers.files, + "This node cannot report itself finished with the old chunk store ({}: \ + `legacy` means one is there, `unknown` means its disk could not be read, so \ + whether one is there is not known). {} of the {seen} node(s) it can see are \ + not reporting finished either.", + own.token(), + peers.outstanding() + ); + } else { + info!( + migration_event = "signal", + state = own.token(), + peers_legacy = peers.legacy, + peers_unknown = peers.unknown, + peers_unreported = peers.unreported, + peers_files = peers.files, + "Storage migration: this node has nothing of the old store left; {} of the \ + {seen} node(s) it can see are not reporting finished.", + peers.outstanding() + ); + } + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used)] +mod tests { + + /// The reporter must not keep the node it reports on alive. + /// + /// It holds a weak handle and upgrades per pass. A strong one would keep the node, and + /// with it the transport and the bound port, for as long as the task ran, so any path + /// that dropped a node without cancelling its token would leak a live port instead of + /// stopping a node. Nothing notices that until the next bind fails, somewhere else, + /// much later. + #[tokio::test] + async fn the_reporter_lets_go_of_a_node_that_was_dropped() { + let dir = tempfile::tempdir().expect("temp dir"); + let root = dir.path().to_path_buf(); + let shutdown = CancellationToken::new(); + + // Stand in for the node: what matters is that the task holds no strong reference, + // so the count of strong holders does not rise when the reporter starts, and the + // reporter stops on its own once the last real holder goes. + let owner = Arc::new(()); + let weak = Arc::downgrade(&owner); + assert_eq!(Arc::strong_count(&owner), 1); + + let handle = tokio::spawn({ + let weak = weak.clone(); + let shutdown = shutdown.clone(); + async move { + loop { + tokio::select! { + () = shutdown.cancelled() => return "cancelled", + () = tokio::time::sleep(std::time::Duration::from_millis(5)) => {} + } + let Some(up) = weak.upgrade() else { + return "node went away"; + }; + drop(up); + } + } + }); + + assert_eq!( + Arc::strong_count(&owner), + 1, + "starting the reporter must not add a strong holder" + ); + drop(owner); + let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), handle) + .await + .expect("the reporter must stop on its own") + .expect("task must not panic"); + assert_eq!( + outcome, "node went away", + "the reporter must stop when the node is gone, not wait for a cancellation \ + nobody sends" + ); + let _ = root; + } + + use super::*; + use tempfile::TempDir; + + fn dir_with(root: &Path, name: &str) -> std::path::PathBuf { + let path = root.join(name); + std::fs::create_dir_all(&path).unwrap(); + path + } + + #[test] + fn a_node_with_nothing_on_disk_has_finished() { + let root = TempDir::new().unwrap(); + assert_eq!( + MigrationSignal::from_disk(root.path()), + MigrationSignal::Files + ); + } + + #[test] + fn a_root_that_does_not_exist_yet_has_finished() { + let root = TempDir::new().unwrap(); + let never = root.path().join("not-created"); + assert_eq!(MigrationSignal::from_disk(&never), MigrationSignal::Files); + } + + #[test] + fn a_live_environment_with_chunks_in_it_has_not() { + let root = TempDir::new().unwrap(); + let env = dir_with(root.path(), LEGACY_ENV_DIR); + std::fs::write(env.join("data.mdb"), b"chunks").unwrap(); + assert_eq!( + MigrationSignal::from_disk(root.path()), + MigrationSignal::Legacy + ); + } + + #[test] + fn a_marked_leftover_has_finished() { + let root = TempDir::new().unwrap(); + let env = dir_with(root.path(), LEGACY_ENV_DIR); + std::fs::write(env.join("data.mdb"), b"chunks").unwrap(); + std::fs::write(env.join(RETIRED_MARKER), b"").unwrap(); + assert_eq!( + MigrationSignal::from_disk(root.path()), + MigrationSignal::Files + ); + } + + #[test] + fn an_empty_leftover_has_finished() { + // What a cleanup interrupted between emptying a tombstone and removing it leaves. + let root = TempDir::new().unwrap(); + dir_with(root.path(), "chunks.mdb.retired"); + assert_eq!( + MigrationSignal::from_disk(root.path()), + MigrationSignal::Files + ); + } + + #[test] + fn a_tombstone_with_chunks_in_it_has_not() { + // A crash between the rename and the mark leaves an intact environment wearing a + // retired-looking name. What it is called is not evidence. + let root = TempDir::new().unwrap(); + let tomb = dir_with(root.path(), "chunks.mdb.retired.3"); + std::fs::write(tomb.join("data.mdb"), b"chunks").unwrap(); + assert_eq!( + MigrationSignal::from_disk(root.path()), + MigrationSignal::Legacy + ); + } + + #[test] + fn an_entry_that_cannot_be_read_is_never_read_as_finished() { + // An earlier version pushed a made-up path when an entry could not be read, so the + // caller would classify it. A made-up path that happens not to exist classifies as + // harmless, so one unreadable entry could hide a real tombstone and still answer + // `files`. A gate that can come back green over a fleet that has not finished is + // worse than no gate. + let root = TempDir::new().unwrap(); + let unreadable = root.path().join("locked"); + std::fs::create_dir_all(&unreadable).unwrap(); + let tomb = unreadable.join("chunks.mdb.retired"); + std::fs::create_dir_all(&tomb).unwrap(); + std::fs::write(tomb.join("data.mdb"), b"chunks").unwrap(); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&unreadable, std::fs::Permissions::from_mode(0o000)).unwrap(); + let answer = MigrationSignal::from_disk(&unreadable); + std::fs::set_permissions(&unreadable, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert_eq!( + answer, + MigrationSignal::Unknown, + "a root that cannot be listed must never answer that it has finished" + ); + } + } + + #[test] + fn a_directory_that_only_looks_like_a_tombstone_is_not_one() { + // This list is eventually a list of directories a later release deletes, so it + // matches the names retirement actually creates and nothing else. + assert!(is_tombstone_name("chunks.mdb.retired")); + assert!(is_tombstone_name("chunks.mdb.retired.1")); + assert!(is_tombstone_name("chunks.mdb.retired.42")); + assert!(!is_tombstone_name("chunks.mdb.retired-mine")); + assert!(!is_tombstone_name("chunks.mdb.retired.")); + assert!(!is_tombstone_name("chunks.mdb.retired.backup")); + assert!(!is_tombstone_name("chunks.mdb")); + // Names retirement counts up to, and names it never reaches. `.007` parses as 7 and + // is still not a name anything wrote. + assert!(is_tombstone_name("chunks.mdb.retired.64")); + assert!(!is_tombstone_name("chunks.mdb.retired.65")); + assert!(!is_tombstone_name("chunks.mdb.retired.0")); + assert!(!is_tombstone_name("chunks.mdb.retired.007")); + assert!(!is_tombstone_name("chunks.mdb.retired.+1")); + assert!(!is_tombstone_name("chunks.mdb.retired.999999")); + + let root = TempDir::new().unwrap(); + let mine = dir_with(root.path(), "chunks.mdb.retired-mine"); + std::fs::write(mine.join("data.mdb"), b"somebody else's").unwrap(); + assert_eq!( + MigrationSignal::from_disk(root.path()), + MigrationSignal::Files + ); + } + + #[test] + fn a_linked_environment_is_never_read_as_finished() { + // The mark would have been written through the link into a directory this node + // does not own, so a link is never evidence that anything was finished with, and + // what it points at is never followed. + let root = TempDir::new().unwrap(); + let elsewhere = dir_with(root.path(), "elsewhere"); + std::fs::write(elsewhere.join(RETIRED_MARKER), b"").unwrap(); + #[cfg(unix)] + { + std::os::unix::fs::symlink(&elsewhere, root.path().join(LEGACY_ENV_DIR)).unwrap(); + assert_eq!( + MigrationSignal::from_disk(root.path()), + MigrationSignal::Legacy + ); + } + } + + /// A node still running the release before this one reads as unreported, never as done. + /// + /// This release lands on a fleet where most nodes are still on the previous one, and + /// those announce no migration token at all. Counting them as finished would let the gate + /// come back clean over a fleet that has barely started. They get their own bucket, and + /// `outstanding` includes it. + #[test] + fn a_peer_on_the_previous_release_is_not_counted_as_finished() { + assert_eq!( + peer_state("node/0.19.0"), + PeerMigrationState::Unreported, + "a node with no migration token has not reported, which is not the same as done" + ); + let mut tally = PeerTally::default(); + tally.add(peer_state("node/0.19.0")); + assert_eq!(tally.files, 0, "it must not land in the finished bucket"); + assert_eq!(tally.outstanding(), 1, "and must count against readiness"); + } + + #[test] + fn the_user_agent_keeps_the_prefix_that_gates_dht_membership() { + // saorsa-core decides whether a peer is a DHT participant by this prefix alone. A + // node that loses it stops being routed to, which is a much worse outcome than not + // reporting at all, so it is worth pinning. + for signal in [ + MigrationSignal::Legacy, + MigrationSignal::Files, + MigrationSignal::Unknown, + ] { + assert!(user_agent(signal).starts_with("node/")); + } + } + + #[test] + fn a_peer_reads_back_what_a_node_announced() { + assert_eq!( + peer_state(&user_agent(MigrationSignal::Legacy)), + PeerMigrationState::Legacy + ); + assert_eq!( + peer_state(&user_agent(MigrationSignal::Files)), + PeerMigrationState::Files + ); + assert_eq!( + peer_state(&user_agent(MigrationSignal::Unknown)), + PeerMigrationState::Unknown + ); + } + + #[test] + fn silence_is_counted_as_silence_and_not_as_completion() { + // The build before this one announces the transport's own agent, with no token of + // ours. Reading that as "finished" is exactly how a gate comes back clean over a + // fleet that has not finished. + assert_eq!(peer_state("node/0.27.0"), PeerMigrationState::Unreported); + assert_eq!( + peer_state("node/0.17.2 migration/something-new"), + PeerMigrationState::Unknown + ); + } + + #[test] + fn a_client_is_not_a_node_that_failed_to_report() { + // Clients authenticate and announce themselves too. Counting them among the peers + // that never reported would make every reading look worse than it is, and the + // count is what a release decision is made on. + assert_eq!(peer_state("client/0.27.0"), PeerMigrationState::NotANode); + } +} diff --git a/src/storage/mod.rs b/src/storage/mod.rs index bda34ac8..f27b4733 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -52,6 +52,7 @@ pub(crate) mod file_store; mod handler; pub(crate) mod lmdb; pub mod migration; +pub(crate) mod migration_signal; pub use crate::ant_protocol::XorName; pub use chunk_store::{ChunkStore, ChunkStoreConfig, VerifyReport, LEGACY_ENV_DIR};