From f1691587a4b848312b333dc17d26b779ddb84763 Mon Sep 17 00:00:00 2001 From: Enigbe Date: Tue, 7 Apr 2026 11:27:40 +0100 Subject: [PATCH 01/14] Implement tiered storage This commit adds `TierStore`, a tiered `KVStore` implementation that routes node persistence across three storage roles: - a primary store for durable, authoritative data - an optional backup store for a second durable copy of primary-backed data - an optional ephemeral store for rebuildable cached data such as the network graph and scorer TierStore routes ephemeral cache data to the ephemeral store when configured, while durable data remains primary and backup. Reads do not consult the backup store during normal operation. Unpaginated listings expose the logical contents of the primary and ephemeral tiers without consulting the backup store; paginated listings currently expose only the primary tier. For primary+backup writes and removals, this implementation treats the backup store as part of the persistence success path rather than as a best-effort background mirror. Earlier designs used asynchronous backup queueing to avoid blocking the primary path, but that weakens the durability contract by allowing primary success to be reported before backup persistence has completed. TierStore now issues primary and backup operations together and only returns success once both complete. This gives callers a clearer persistence guarantee when a backup store is configured: acknowledged primary+backup mutations have been attempted against both durable stores. The tradeoff is that dual-store operations are not atomic across stores, so an error may still be returned after one store has already been updated. Additionally, adds unit coverage for the current contract, including: - basic read/write/remove/list persistence - routing of ephemeral data away from the primary store - backup participation in the foreground success path for writes and removals Assisted-by: Amp (AI coding agent) --- src/io/mod.rs | 1 + src/io/tier_store.rs | 1454 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1455 insertions(+) create mode 100644 src/io/tier_store.rs diff --git a/src/io/mod.rs b/src/io/mod.rs index c11475c43..6d616f409 100644 --- a/src/io/mod.rs +++ b/src/io/mod.rs @@ -15,6 +15,7 @@ pub mod postgres_store; pub mod sqlite_store; #[cfg(test)] pub(crate) mod test_utils; +pub(crate) mod tier_store; pub(crate) mod utils; #[cfg(feature = "storage-vss")] pub mod vss_store; diff --git a/src/io/tier_store.rs b/src/io/tier_store.rs new file mode 100644 index 000000000..c24ea4d0d --- /dev/null +++ b/src/io/tier_store.rs @@ -0,0 +1,1454 @@ +// This file is Copyright its original authors, visible in version control history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license , at your option. You may not use this file except in +// accordance with one or both of these licenses. +#![allow(dead_code)] // TODO: Temporal warning silencer. Will be removed in later commit. + +use std::collections::HashMap; +use std::future::Future; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +use lightning::util::persist::{ + KVStore, PageToken, PaginatedKVStore, PaginatedListResponse, NETWORK_GRAPH_PERSISTENCE_KEY, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, SCORER_PERSISTENCE_KEY, + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, +}; +use lightning::{io, log_error}; +use tokio::sync::Mutex as TokioMutex; + +use crate::io::utils::{check_namespace_key_validity, EXTERNAL_PATHFINDING_SCORES_CACHE_KEY}; +use crate::logger::{LdkLogger, Logger}; +use crate::types::DynStore; + +/// A 3-tiered [`KVStore`] implementation that routes data across +/// storage backends that may be local or remote: +/// - a primary store for durable, authoritative persistence, +/// - an optional backup store that maintains an additional durable copy of +/// primary-backed data, and +/// - an optional ephemeral store for non-critical, rebuildable cached data. +/// +/// When a backup store is configured, writes and removals for primary-backed data +/// are issued to the primary and backup stores concurrently and only succeed once +/// both stores complete successfully. +/// +/// Reads and lists do not consult the backup store during normal operation. +/// Ephemeral data is read from and written to the ephemeral store when configured. +/// Unpaginated listings expose the logical contents of the primary and ephemeral +/// stores, while paginated listings expose only the primary store until cross-tier +/// pagination semantics are defined. +/// +/// Note that dual-store writes and removals are not atomic across the primary and +/// backup stores. If one store succeeds and the other fails, the operation +/// returns an error even though one store may already reflect the change. +pub(crate) struct TierStore { + inner: Arc, +} + +impl TierStore { + pub fn new(primary_store: Arc, logger: Arc) -> Self { + let inner = Arc::new(TierStoreInner::new(primary_store, Arc::clone(&logger))); + + Self { inner } + } + + /// Configures a backup store for primary-backed data. + /// + /// Once set, writes and removals targeting the primary tier succeed only if both + /// the primary and backup stores succeed. The two operations are issued + /// concurrently, and any failure is returned to the caller. + /// + /// Note: dual-store writes/removals are not atomic. An error may be returned + /// after the primary store has already been updated if the backup store fails. + /// + /// The backup store is not consulted for normal reads or lists. + pub fn set_backup_store(&mut self, backup: Arc) { + debug_assert_eq!(Arc::strong_count(&self.inner), 1); + + let inner = Arc::get_mut(&mut self.inner).expect( + "TierStore should not be shared during configuration. No other references should exist", + ); + + inner.backup_store = Some(backup); + } + + /// Configures the ephemeral store for non-critical, rebuildable data. + /// + /// When configured, selected cache-like data is routed to this store instead of + /// the primary store. + pub fn set_ephemeral_store(&mut self, ephemeral: Arc) { + debug_assert_eq!(Arc::strong_count(&self.inner), 1); + + let inner = Arc::get_mut(&mut self.inner).expect( + "TierStore should not be shared during configuration. No other references should exist", + ); + + inner.ephemeral_store = Some(ephemeral); + } +} + +impl KVStore for TierStore { + fn read( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> impl Future, io::Error>> + 'static + Send { + let inner = Arc::clone(&self.inner); + + let primary_namespace = primary_namespace.to_string(); + let secondary_namespace = secondary_namespace.to_string(); + let key = key.to_string(); + + async move { inner.read_internal(primary_namespace, secondary_namespace, key).await } + } + + fn write( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, + ) -> impl Future> + 'static + Send { + let inner = Arc::clone(&self.inner); + let locking_key = inner.build_locking_key(primary_namespace, secondary_namespace, key); + let (lock_ref, version) = inner.get_new_version_and_lock_ref(locking_key.clone()); + + let primary_namespace = primary_namespace.to_string(); + let secondary_namespace = secondary_namespace.to_string(); + let key = key.to_string(); + + async move { + inner + .write_internal( + primary_namespace, + secondary_namespace, + key, + buf, + lock_ref, + locking_key, + version, + ) + .await + } + } + + fn remove( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, + ) -> impl Future> + 'static + Send { + let inner = Arc::clone(&self.inner); + let locking_key = inner.build_locking_key(primary_namespace, secondary_namespace, key); + let (lock_ref, version) = inner.get_new_version_and_lock_ref(locking_key.clone()); + + let primary_namespace = primary_namespace.to_string(); + let secondary_namespace = secondary_namespace.to_string(); + let key = key.to_string(); + + async move { + inner + .remove_internal( + primary_namespace, + secondary_namespace, + key, + lazy, + lock_ref, + locking_key, + version, + ) + .await + } + } + + fn list( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> impl Future, io::Error>> + 'static + Send { + let inner = Arc::clone(&self.inner); + + let primary_namespace = primary_namespace.to_string(); + let secondary_namespace = secondary_namespace.to_string(); + + async move { inner.list_internal(primary_namespace, secondary_namespace).await } + } +} + +impl PaginatedKVStore for TierStore { + fn list_paginated( + &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, + ) -> impl Future> + 'static + Send { + let inner = Arc::clone(&self.inner); + + let primary_namespace = primary_namespace.to_string(); + let secondary_namespace = secondary_namespace.to_string(); + + async move { + inner.list_paginated_internal(primary_namespace, secondary_namespace, page_token).await + } + } +} + +struct TierStoreInner { + /// The authoritative store for durable data. + primary_store: Arc, + /// The store used for non-critical, rebuildable cached data. + ephemeral_store: Option>, + /// An optional second durable store for primary-backed data. + backup_store: Option>, + /// Per-key locks for serializing primary+backup operations and skipping stale writes. + locks: Mutex>>>, + next_write_version: AtomicU64, + logger: Arc, +} + +impl TierStoreInner { + /// Creates a tier store with the primary data store. + pub fn new(primary_store: Arc, logger: Arc) -> Self { + Self { + primary_store, + ephemeral_store: None, + backup_store: None, + locks: Mutex::new(HashMap::new()), + next_write_version: AtomicU64::new(1), + logger, + } + } + + fn get_new_version_and_lock_ref(&self, locking_key: String) -> (Arc>, u64) { + let version = self.next_write_version.fetch_add(1, Ordering::Relaxed); + if version == u64::MAX { + panic!("TierStore version counter overflowed"); + } + + let mut locks = self.locks.lock().expect("lock"); + let lock_ref = + Arc::clone(locks.entry(locking_key).or_insert_with(|| Arc::new(TokioMutex::new(0)))); + + (lock_ref, version) + } + + fn clean_locks(&self, lock_ref: &Arc>, locking_key: String) { + let mut locks = self.locks.lock().expect("lock"); + let strong_count = Arc::strong_count(lock_ref); + debug_assert!(strong_count >= 2, "Unexpected TierStore lock strong count"); + if strong_count == 2 { + locks.remove(&locking_key); + } + } + + fn build_locking_key( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> String { + if primary_namespace.is_empty() { + key.to_owned() + } else { + format!("{}#{}#{}", primary_namespace, secondary_namespace, key) + } + } + + /// Reads from the primary data store. + async fn read_primary( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result> { + match KVStore::read( + self.primary_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + ) + .await + { + Ok(data) => Ok(data), + Err(e) => Err(e), + } + } + + /// Lists keys from the primary data store. + async fn list_primary( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result> { + match KVStore::list(self.primary_store.as_ref(), primary_namespace, secondary_namespace) + .await + { + Ok(keys) => Ok(keys), + Err(e) => { + log_error!( + self.logger, + "Failed to list from primary store for namespace {}/{}: {}.", + primary_namespace, + secondary_namespace, + e + ); + Err(e) + }, + } + } + + async fn write_primary_backup_async( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, + ) -> io::Result<()> { + if let Some(backup_store) = self.backup_store.as_ref() { + let primary_fut = KVStore::write( + self.primary_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + buf.clone(), + ); + + let backup_fut = KVStore::write( + backup_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + buf, + ); + + let (primary_res, backup_res) = tokio::join!(primary_fut, backup_fut); + + self.handle_primary_backup_results( + "write", + primary_namespace, + secondary_namespace, + key, + primary_res, + backup_res, + ) + } else { + KVStore::write( + self.primary_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + buf, + ) + .await + } + } + + async fn remove_primary_backup_async( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, + ) -> io::Result<()> { + let primary_fut = KVStore::remove( + self.primary_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + lazy, + ); + + if let Some(backup_store) = self.backup_store.as_ref() { + let backup_fut = KVStore::remove( + backup_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + lazy, + ); + + let (primary_res, backup_res) = tokio::join!(primary_fut, backup_fut); + + self.handle_primary_backup_results( + "removal", + primary_namespace, + secondary_namespace, + key, + primary_res, + backup_res, + ) + } else { + primary_fut.await + } + } + + async fn execute_locked_write( + &self, lock_ref: Arc>, locking_key: String, version: u64, callback: F, + ) -> io::Result<()> + where + F: FnOnce() -> Fut, + Fut: Future>, + { + let res = { + let mut last_written_version = lock_ref.lock().await; + + if version <= *last_written_version { + Ok(()) + } else { + let res = callback().await; + // A failed multi-store operation may still have updated one of its stores. We record + // the attempted version regardless so an older operation cannot overwrite newer state. + *last_written_version = version; + res + } + }; + + self.clean_locks(&lock_ref, locking_key); + res + } + + async fn read_internal( + &self, primary_namespace: String, secondary_namespace: String, key: String, + ) -> io::Result> { + check_namespace_key_validity( + primary_namespace.as_str(), + secondary_namespace.as_str(), + Some(key.as_str()), + "read", + )?; + + if is_ephemeral_cached_key(&primary_namespace, &secondary_namespace, &key) { + if let Some(eph_store) = self.ephemeral_store.as_ref() { + // We don't retry ephemeral-store reads here. Local failures are treated as + // terminal for this access path rather than falling back to another store. + return KVStore::read( + eph_store.as_ref(), + &primary_namespace, + &secondary_namespace, + &key, + ) + .await; + } + } + + self.read_primary(&primary_namespace, &secondary_namespace, &key).await + } + + async fn write_internal( + &self, primary_namespace: String, secondary_namespace: String, key: String, buf: Vec, + lock_ref: Arc>, locking_key: String, version: u64, + ) -> io::Result<()> { + check_namespace_key_validity( + primary_namespace.as_str(), + secondary_namespace.as_str(), + Some(key.as_str()), + "write", + )?; + + if is_ephemeral_cached_key(&primary_namespace, &secondary_namespace, &key) { + if let Some(eph_store) = self.ephemeral_store.as_ref() { + let eph_store = Arc::clone(eph_store); + return self + .execute_locked_write(lock_ref, locking_key, version, || async move { + KVStore::write( + eph_store.as_ref(), + primary_namespace.as_str(), + secondary_namespace.as_str(), + key.as_str(), + buf, + ) + .await + }) + .await; + } + } + + self.execute_locked_write(lock_ref, locking_key, version, || async move { + self.write_primary_backup_async( + primary_namespace.as_str(), + secondary_namespace.as_str(), + key.as_str(), + buf, + ) + .await + }) + .await + } + + async fn remove_internal( + &self, primary_namespace: String, secondary_namespace: String, key: String, lazy: bool, + lock_ref: Arc>, locking_key: String, version: u64, + ) -> io::Result<()> { + check_namespace_key_validity( + primary_namespace.as_str(), + secondary_namespace.as_str(), + Some(key.as_str()), + "remove", + )?; + + if is_ephemeral_cached_key(&primary_namespace, &secondary_namespace, &key) { + if let Some(eph_store) = self.ephemeral_store.as_ref() { + let eph_store = Arc::clone(eph_store); + return self + .execute_locked_write(lock_ref, locking_key, version, || async move { + KVStore::remove( + eph_store.as_ref(), + primary_namespace.as_str(), + secondary_namespace.as_str(), + key.as_str(), + lazy, + ) + .await + }) + .await; + } + } + + self.execute_locked_write(lock_ref, locking_key, version, || async move { + self.remove_primary_backup_async( + primary_namespace.as_str(), + secondary_namespace.as_str(), + key.as_str(), + lazy, + ) + .await + }) + .await + } + + async fn list_internal( + &self, primary_namespace: String, secondary_namespace: String, + ) -> io::Result> { + check_namespace_key_validity( + primary_namespace.as_str(), + secondary_namespace.as_str(), + None, + "list", + )?; + + let mut keys = self.list_primary(&primary_namespace, &secondary_namespace).await?; + + let Some(ephemeral_store) = self.ephemeral_store.as_ref() else { + return Ok(keys); + }; + + if primary_namespace != NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE + && primary_namespace != SCORER_PERSISTENCE_PRIMARY_NAMESPACE + { + return Ok(keys); + } + + // The ephemeral store is authoritative for keys routed there. Exclude stale + // primary copies, then add only routed keys from the ephemeral store. + keys.retain(|key| !is_ephemeral_cached_key(&primary_namespace, &secondary_namespace, key)); + + let ephemeral_keys = + KVStore::list(ephemeral_store.as_ref(), &primary_namespace, &secondary_namespace) + .await?; + + for key in ephemeral_keys { + if is_ephemeral_cached_key(&primary_namespace, &secondary_namespace, &key) + && !keys.contains(&key) + { + keys.push(key); + } + } + + Ok(keys) + } + + async fn list_paginated_internal( + &self, primary_namespace: String, secondary_namespace: String, + page_token: Option, + ) -> io::Result { + check_namespace_key_validity( + primary_namespace.as_str(), + secondary_namespace.as_str(), + None, + "list_paginated", + )?; + + // TODO(@enigbe): Merge listings across tiers once `PaginatedKVStore` provides a + // shared ordering and cursor contract that permits a correct cross-store merge. + // Until then, listings intentionally expose only the primary tier. + PaginatedKVStore::list_paginated( + self.primary_store.as_ref(), + &primary_namespace, + &secondary_namespace, + page_token, + ) + .await + } + + fn handle_primary_backup_results( + &self, op: &str, primary_namespace: &str, secondary_namespace: &str, key: &str, + primary_res: io::Result<()>, backup_res: io::Result<()>, + ) -> io::Result<()> { + match (primary_res, backup_res) { + (Ok(()), Ok(())) => Ok(()), + (Err(primary_err), Ok(())) => { + log_error!( + self.logger, + "Primary {} failed after backup {} succeeded for key {}/{}/{}; primary and backup may have diverged: {}", + op, + op, + primary_namespace, + secondary_namespace, + key, + primary_err + ); + Err(primary_err) + }, + (Ok(()), Err(backup_err)) => { + log_error!( + self.logger, + "Backup {} failed after primary {} succeeded for key {}/{}/{}; primary and backup may have diverged: {}", + op, + op, + primary_namespace, + secondary_namespace, + key, + backup_err + ); + Err(backup_err) + }, + (Err(primary_err), Err(backup_err)) => { + log_error!( + self.logger, + "Primary and backup {}s both failed for key {}/{}/{}: primary={}, backup={}", + op, + primary_namespace, + secondary_namespace, + key, + primary_err, + backup_err + ); + Err(primary_err) + }, + } + } +} + +fn is_ephemeral_cached_key(pn: &str, sn: &str, key: &str) -> bool { + matches!( + (pn, sn, key), + (NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, _, NETWORK_GRAPH_PERSISTENCE_KEY) + | (SCORER_PERSISTENCE_PRIMARY_NAMESPACE, _, SCORER_PERSISTENCE_KEY) + | (SCORER_PERSISTENCE_PRIMARY_NAMESPACE, _, EXTERNAL_PATHFINDING_SCORES_CACHE_KEY) + ) +} + +#[cfg(test)] +mod tests { + use std::future::Future; + use std::panic::RefUnwindSafe; + use std::path::PathBuf; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + use lightning::util::logger::Level; + use lightning::util::persist::{ + CHANNEL_MANAGER_PERSISTENCE_KEY, CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + }; + use lightning_persister::fs_store::v2::FilesystemStoreV2; + + use super::*; + use crate::io::test_utils::{ + do_read_write_remove_list_persist, random_storage_path, InMemoryStore, + }; + use crate::io::tier_store::TierStore; + use crate::logger::Logger; + use crate::types::{DynStore, DynStoreWrapper}; + + impl RefUnwindSafe for TierStore {} + + struct CleanupDir(PathBuf); + impl Drop for CleanupDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + fn setup_tier_store(primary_store: Arc, logger: Arc) -> TierStore { + TierStore::new(primary_store, logger) + } + + enum FailureMode { + List, + Write { attempts: Arc }, + } + + /// A store that injects a selected failure while delegating other operations to an inner + /// [`InMemoryStore`]. + struct FailingStore { + inner: InMemoryStore, + failure_mode: FailureMode, + } + + impl FailingStore { + fn new(failure_mode: FailureMode) -> Self { + Self { inner: InMemoryStore::new(), failure_mode } + } + } + + impl KVStore for FailingStore { + fn read( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> impl Future, io::Error>> + 'static + Send { + KVStore::read(&self.inner, primary_namespace, secondary_namespace, key) + } + fn write( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, + ) -> impl Future> + 'static + Send { + let write = if let FailureMode::Write { attempts } = &self.failure_mode { + attempts.fetch_add(1, Ordering::Relaxed); + None + } else { + Some(KVStore::write(&self.inner, primary_namespace, secondary_namespace, key, buf)) + }; + async move { + match write { + Some(write) => write.await, + None => Err(io::Error::new(io::ErrorKind::Other, "write failed")), + } + } + } + fn remove( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, + ) -> impl Future> + 'static + Send { + KVStore::remove(&self.inner, primary_namespace, secondary_namespace, key, lazy) + } + fn list( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> impl Future, io::Error>> + 'static + Send { + let list = match &self.failure_mode { + FailureMode::List => None, + FailureMode::Write { .. } => { + Some(KVStore::list(&self.inner, primary_namespace, secondary_namespace)) + }, + }; + async move { + match list { + Some(list) => list.await, + None => Err(io::Error::new(io::ErrorKind::Other, "list failed")), + } + } + } + } + + impl PaginatedKVStore for FailingStore { + fn list_paginated( + &self, primary_namespace: &str, secondary_namespace: &str, + page_token: Option, + ) -> impl Future> + 'static + Send { + let list = match &self.failure_mode { + FailureMode::List => None, + FailureMode::Write { .. } => Some(PaginatedKVStore::list_paginated( + &self.inner, + primary_namespace, + secondary_namespace, + page_token, + )), + }; + async move { + match list { + Some(list) => list.await, + None => Err(io::Error::new(io::ErrorKind::Other, "list_paginated failed")), + } + } + } + } + + #[tokio::test] + async fn write_read_list_remove() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + + let _cleanup = CleanupDir(base_dir.clone()); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let tier = setup_tier_store(primary_store, logger); + + do_read_write_remove_list_persist(&tier).await; + } + + #[tokio::test] + async fn ephemeral_routing() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + + let _cleanup = CleanupDir(base_dir.clone()); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + + let ephemeral_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("ephemeral")).unwrap())); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + let data = vec![42u8; 32]; + + tier.write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + data.clone(), + ) + .await + .unwrap(); + + tier.write( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + data.clone(), + ) + .await + .unwrap(); + + let primary_read_ng = primary_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await; + let ephemeral_read_ng = ephemeral_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await; + + let primary_read_cm = primary_store + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + ) + .await; + let ephemeral_read_cm = ephemeral_store + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + ) + .await; + + assert!(primary_read_ng.is_err()); + assert_eq!(ephemeral_read_ng.unwrap(), data); + + assert!(ephemeral_read_cm.is_err()); + assert_eq!(primary_read_cm.unwrap(), data); + } + + #[tokio::test] + async fn external_pathfinding_scores_cache_routes_to_ephemeral_store() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + + let _cleanup = CleanupDir(base_dir.clone()); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + + let ephemeral_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("ephemeral")).unwrap())); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + let data = vec![42u8; 32]; + tier.write( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + data.clone(), + ) + .await + .unwrap(); + + assert!(primary_store + .read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + ) + .await + .is_err()); + assert_eq!( + tier.read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + ) + .await + .unwrap(), + data + ); + + tier.remove( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + false, + ) + .await + .unwrap(); + assert!(ephemeral_store + .read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + ) + .await + .is_err()); + } + + #[tokio::test] + async fn list_exposes_primary_and_routed_ephemeral_keys() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + + let _cleanup = CleanupDir(base_dir.clone()); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + + let ephemeral_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("ephemeral")).unwrap())); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + // A durable root-namespace key, routed to primary since it isn't ephemeral-cached. + tier.write( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + vec![1u8; 32], + ) + .await + .unwrap(); + + // The ephemeral-cached key, routed to the ephemeral store. + tier.write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![2u8; 32], + ) + .await + .unwrap(); + + // A decoy sitting in the ephemeral store under an unrelated namespace. This must + // never leak into a listing for that namespace just because an ephemeral + // store happens to be configured. + ephemeral_store + .write( + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + "ephemeral-decoy", + vec![3u8; 32], + ) + .await + .unwrap(); + ephemeral_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + "ephemeral-root-decoy", + vec![4u8; 32], + ) + .await + .unwrap(); + + // This is `list("", "")`: CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE and + // NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE are the same empty string, so both + // keys live in the exact namespace. + let root_keys = KVStore::list( + &tier, + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await + .unwrap(); + + // Unpaginated listing exposes the logical view across both tiers without leaking + // unrelated keys from the ephemeral store. + assert!(root_keys.contains(&CHANNEL_MANAGER_PERSISTENCE_KEY.to_string())); + assert!(root_keys.contains(&NETWORK_GRAPH_PERSISTENCE_KEY.to_string())); + assert!(!root_keys.contains(&"ephemeral-root-decoy".to_string())); + + let monitor_keys = KVStore::list( + &tier, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await + .unwrap(); + + // The unrelated-namespace decoy sitting in the ephemeral store must not leak + // into a listing for a namespace it was never routed to. + assert!(!monitor_keys.contains(&"ephemeral-decoy".to_string())); + } + + #[tokio::test] + async fn list_paginated_only_exposes_primary_keys() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + + let _cleanup = CleanupDir(base_dir.clone()); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + + let ephemeral_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("ephemeral")).unwrap())); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + tier.write( + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + "monitor-key", + vec![1u8; 32], + ) + .await + .unwrap(); + + // This decoy uses the same namespace but the opposite physical store, so it + // would show up if paginated listing routed to the wrong tier. + ephemeral_store + .write( + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + "ephemeral-decoy", + vec![2u8; 32], + ) + .await + .unwrap(); + + // This key shares the network graph's namespace tuple ("", "") but is not + // itself an ephemeral-cached key, standing in for durable root-namespace data + // such as `manager`/`output_sweeper`/`peers`. It must still be listed even + // though the ephemeral store is configured and authoritative for + // `network_graph`/`scorer` specifically. + primary_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + "other-root-namespace-key", + vec![3u8; 32], + ) + .await + .unwrap(); + + tier.write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![4u8; 32], + ) + .await + .unwrap(); + + let primary_response = PaginatedKVStore::list_paginated( + &tier, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + None, + ) + .await + .unwrap(); + assert_eq!(primary_response.keys, vec!["monitor-key".to_string()]); + + let root_response = PaginatedKVStore::list_paginated( + &tier, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + None, + ) + .await + .unwrap(); + + assert_eq!(root_response.keys, vec!["other-root-namespace-key".to_string()]); + } + + #[tokio::test] + async fn listings_only_consult_ephemeral_store_for_routed_namespaces() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + + let _cleanup = CleanupDir(base_dir.clone()); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + + // An ephemeral store whose `list`/`list_paginated` always fail. + let ephemeral_store: Arc = + Arc::new(DynStoreWrapper(FailingStore::new(FailureMode::List))); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + // A durable key in a namespace that can never hold an ephemeral-cached key. + tier.write( + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + "monitor-key", + vec![1u8; 32], + ) + .await + .unwrap(); + + // Listing that namespace must not consult (or depend on) the ephemeral store, so it + // succeeds even though the ephemeral list would fail. + let monitor_keys = KVStore::list( + &tier, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await + .unwrap(); + assert_eq!(monitor_keys, vec!["monitor-key".to_string()]); + + // The paginated path always exposes only the primary store. + let monitor_page = PaginatedKVStore::list_paginated( + &tier, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + None, + ) + .await + .unwrap(); + assert_eq!(monitor_page.keys, vec!["monitor-key".to_string()]); + + // An unpaginated root listing must consult the authoritative ephemeral store. + assert!(KVStore::list( + &tier, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await + .is_err()); + } + + #[tokio::test] + async fn list_hides_stale_primary_copy_when_ephemeral_key_is_missing() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + + let _cleanup = CleanupDir(base_dir.clone()); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + // A durable root-namespace key that must always be discoverable. + tier.write( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + vec![1u8; 32], + ) + .await + .unwrap(); + + // A stale copy of `network_graph` sitting in primary as if it had been persisted there + // before the ephemeral store was configured. The ephemeral store holds no copy, so + // `read` routes to ephemeral and would fail for this key. + primary_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![2u8; 32], + ) + .await + .unwrap(); + + let root_keys = KVStore::list( + &tier, + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await + .unwrap(); + + // The ephemeral store is authoritative for routed keys, so its missing entry hides + // the stale primary copy. + assert!(root_keys.contains(&CHANNEL_MANAGER_PERSISTENCE_KEY.to_string())); + assert!(!root_keys.contains(&NETWORK_GRAPH_PERSISTENCE_KEY.to_string())); + } + + #[tokio::test] + async fn primary_backed_writes_preserve_latest_call_order() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + + let _cleanup = CleanupDir(base_dir.clone()); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let tier = setup_tier_store(primary_store, logger); + + let old_data = vec![1u8; 32]; + let new_data = vec![2u8; 32]; + + let old_write = tier.write( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + old_data, + ); + let new_write = tier.write( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + new_data.clone(), + ); + + new_write.await.unwrap(); + old_write.await.unwrap(); + + // Stale data doesn't overwrite latest + let persisted = tier + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + ) + .await + .unwrap(); + assert_eq!(persisted, new_data); + } + + #[tokio::test] + async fn failed_newer_backup_write_still_supersedes_older_write() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + + let backup_write_attempts = Arc::new(AtomicUsize::new(0)); + let backup_store: Arc = + Arc::new(DynStoreWrapper(FailingStore::new(FailureMode::Write { + attempts: Arc::clone(&backup_write_attempts), + }))); + tier.set_backup_store(backup_store); + + let old_data = vec![1u8; 32]; + let new_data = vec![2u8; 32]; + let old_write = tier.write( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + old_data, + ); + let new_write = tier.write( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + new_data.clone(), + ); + + // The primary write succeeds, but the same newer write fails on the backup. + assert!(new_write.await.is_err()); + // The older operation must be treated as stale even though the newer operation failed. + old_write.await.unwrap(); + + let persisted = primary_store + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + ) + .await + .unwrap(); + assert_eq!(persisted, new_data); + assert_eq!(backup_write_attempts.load(Ordering::Relaxed), 1); + } + + #[tokio::test] + async fn ephemeral_writes_preserve_latest_call_order() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + + let _cleanup = CleanupDir(base_dir.clone()); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let mut tier = setup_tier_store(primary_store, logger); + + let ephemeral_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("ephemeral")).unwrap())); + tier.set_ephemeral_store(ephemeral_store); + + let old_data = vec![1u8; 32]; + let new_data = vec![2u8; 32]; + + let old_write = tier.write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + old_data, + ); + let new_write = tier.write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + new_data.clone(), + ); + + new_write.await.unwrap(); + old_write.await.unwrap(); + + let persisted = tier + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .unwrap(); + assert_eq!(persisted, new_data); + } + + #[tokio::test] + async fn ephemeral_removes_preserve_latest_call_order() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + + let _cleanup = CleanupDir(base_dir.clone()); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let mut tier = setup_tier_store(primary_store, logger); + + let ephemeral_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("ephemeral")).unwrap())); + tier.set_ephemeral_store(ephemeral_store); + + let data = vec![2u8; 32]; + + let stale_remove = tier.remove( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + true, + ); + let new_write = tier.write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + data.clone(), + ); + + new_write.await.unwrap(); + stale_remove.await.unwrap(); + + let persisted = tier + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .unwrap(); + assert_eq!(persisted, data); + } + + #[tokio::test] + async fn backup_write_is_part_of_success_path() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + + let _cleanup = CleanupDir(base_dir.clone()); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + + let backup_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("backup")).unwrap())); + tier.set_backup_store(Arc::clone(&backup_store)); + + let data = vec![42u8; 32]; + + tier.write( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + data.clone(), + ) + .await + .unwrap(); + + let primary_read = primary_store + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + ) + .await; + let backup_read = backup_store + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + ) + .await; + + assert_eq!(primary_read.unwrap(), data); + assert_eq!(backup_read.unwrap(), data); + } + + #[tokio::test] + async fn backup_remove_is_part_of_success_path() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + + let _cleanup = CleanupDir(base_dir.clone()); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + + let backup_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("backup")).unwrap())); + tier.set_backup_store(Arc::clone(&backup_store)); + + let data = vec![42u8; 32]; + let key = CHANNEL_MANAGER_PERSISTENCE_KEY; + + tier.write( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + key, + data, + ) + .await + .unwrap(); + + tier.remove( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + key, + true, + ) + .await + .unwrap(); + + let primary_read = primary_store + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + key, + ) + .await; + let backup_read = backup_store + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + key, + ) + .await; + + assert!(primary_read.is_err()); + assert!(backup_read.is_err()); + } +} From 2a9ecd61b427da220ed3583eba92714c1fc2b928 Mon Sep 17 00:00:00 2001 From: Enigbe Date: Mon, 17 Aug 2026 05:33:46 +0100 Subject: [PATCH 02/14] Prepare an internal TierStore index database TierStore needs a single persistent ordering domain before it can provide correct pagination across primary and ephemeral stores because the wrapped stores' native ordering and pagination tokens are not comparable. In this commit, we create an internal SQLite index database automatically whenever ephemeral storage is configured and give it a persistent identity as well as an exclusive SQLite lock for TierStore's lifetime. This is a prefactor and only establishes the internal storage plumbing; listing behavior remains unchanged. Assisted-by: Amp (AI coding agent) --- src/io/sqlite_store/mod.rs | 75 ++++++++++++++++++++++++++- src/io/tier_store.rs | 103 ++++++++++++++++++++++++++++++++++++- 2 files changed, 176 insertions(+), 2 deletions(-) diff --git a/src/io/sqlite_store/mod.rs b/src/io/sqlite_store/mod.rs index 258722059..22bfd5710 100644 --- a/src/io/sqlite_store/mod.rs +++ b/src/io/sqlite_store/mod.rs @@ -12,12 +12,14 @@ use std::future::Future; use std::path::PathBuf; use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; +use std::time::Duration; use lightning::io; use lightning::util::persist::{ KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse, }; use lightning_types::string::PrintableString; +use rusqlite::ffi::ErrorCode; use rusqlite::{named_params, Connection}; use crate::io::utils::check_namespace_key_validity; @@ -26,6 +28,8 @@ mod migrations; /// LDK Node's database file name. pub const SQLITE_DB_FILE_NAME: &str = "ldk_node_data.sqlite"; +/// LDK Node's internal tier-store index database file name. +pub(crate) const SQLITE_TIER_INDEX_DB_FILE_NAME: &str = "ldk_node_tier_index.sqlite"; /// LDK Node's table in which we store all data. pub const KV_TABLE_NAME: &str = "ldk_node_data"; @@ -41,6 +45,17 @@ const SCHEMA_USER_VERSION: u16 = 3; // The number of entries returned per page in paginated list operations. const PAGE_SIZE: usize = 50; +fn exclusive_lock_error_kind(error: &rusqlite::Error) -> io::ErrorKind { + match error { + rusqlite::Error::SqliteFailure(error, _) + if matches!(error.code, ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) => + { + io::ErrorKind::AlreadyExists + }, + _ => io::ErrorKind::Other, + } +} + /// A [`KVStore`] implementation that writes to and reads from an [SQLite] database. /// /// [SQLite]: https://sqlite.org @@ -62,7 +77,22 @@ impl SqliteStore { pub fn new( data_dir: PathBuf, db_file_name: Option, kv_table_name: Option, ) -> io::Result { - let inner = Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name)?); + Self::new_internal(data_dir, db_file_name, kv_table_name, false) + } + + /// Constructs a new [`SqliteStore`] which exclusively owns its database for its lifetime. + pub(crate) fn new_exclusive( + data_dir: PathBuf, db_file_name: Option, kv_table_name: Option, + ) -> io::Result { + Self::new_internal(data_dir, db_file_name, kv_table_name, true) + } + + fn new_internal( + data_dir: PathBuf, db_file_name: Option, kv_table_name: Option, + exclusive: bool, + ) -> io::Result { + let inner = + Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name, exclusive)?); let next_write_version = AtomicU64::new(1); Ok(Self { inner, next_write_version }) @@ -230,6 +260,7 @@ struct SqliteStoreInner { impl SqliteStoreInner { fn new( data_dir: PathBuf, db_file_name: Option, kv_table_name: Option, + exclusive: bool, ) -> io::Result { let db_file_name = db_file_name.unwrap_or(DEFAULT_SQLITE_DB_FILE_NAME.to_string()); let kv_table_name = kv_table_name.unwrap_or(DEFAULT_KV_TABLE_NAME.to_string()); @@ -251,6 +282,33 @@ impl SqliteStoreInner { io::Error::new(io::ErrorKind::Other, msg) })?; + if exclusive { + connection.busy_timeout(Duration::ZERO).map_err(|e| { + let msg = format!( + "Failed to configure exclusive database lock timeout for {}: {}", + db_file_path.display(), + e + ); + io::Error::new(io::ErrorKind::Other, msg) + })?; + connection.pragma_update(None, "locking_mode", "EXCLUSIVE").map_err(|e| { + let msg = format!( + "Failed to exclusively lock database file {}: {}", + db_file_path.display(), + e + ); + io::Error::new(io::ErrorKind::Other, msg) + })?; + connection.execute_batch("BEGIN EXCLUSIVE; COMMIT;").map_err(|e| { + let msg = format!( + "Failed to exclusively lock database file {}: {}", + db_file_path.display(), + e + ); + io::Error::new(exclusive_lock_error_kind(&e), msg) + })?; + } + let sql = format!("SELECT user_version FROM pragma_user_version"); let version_res: u16 = connection.query_row(&sql, [], |row| row.get(0)).map_err(|e| { let msg = format!("Failed to read PRAGMA user_version: {}", e); @@ -700,6 +758,21 @@ mod tests { } } + #[test] + fn exclusive_lock_error_kind_distinguishes_contention_from_other_failures() { + for result_code in [rusqlite::ffi::SQLITE_BUSY, rusqlite::ffi::SQLITE_LOCKED] { + let error = + rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(result_code), None); + assert_eq!(exclusive_lock_error_kind(&error), io::ErrorKind::AlreadyExists); + } + + let io_error = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_IOERR), + None, + ); + assert_eq!(exclusive_lock_error_kind(&io_error), io::ErrorKind::Other); + } + #[tokio::test] async fn read_write_remove_list_persist() { let mut temp_path = random_storage_path(); diff --git a/src/io/tier_store.rs b/src/io/tier_store.rs index c24ea4d0d..3bce3f09a 100644 --- a/src/io/tier_store.rs +++ b/src/io/tier_store.rs @@ -8,6 +8,7 @@ use std::collections::HashMap; use std::future::Future; +use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; @@ -19,9 +20,63 @@ use lightning::util::persist::{ use lightning::{io, log_error}; use tokio::sync::Mutex as TokioMutex; +use crate::io::sqlite_store::{SqliteStore, KV_TABLE_NAME, SQLITE_TIER_INDEX_DB_FILE_NAME}; use crate::io::utils::{check_namespace_key_validity, EXTERNAL_PATHFINDING_SCORES_CACHE_KEY}; use crate::logger::{LdkLogger, Logger}; -use crate::types::DynStore; +use crate::types::{DynStore, DynStoreWrapper}; + +const INDEX_DATABASE_ID_LEN: usize = 16; +const INDEX_METADATA_PRIMARY_NAMESPACE: &str = "_tier_store_metadata"; +const INDEX_DATABASE_ID_KEY: &str = "index_database_id"; + +pub(crate) struct TierStoreIndex { + // Holding the store keeps its exclusive SQLite lock for the lifetime of the tier store. + _store: Arc, +} + +impl TierStoreIndex { + async fn new(data_dir: PathBuf) -> io::Result { + let store = SqliteStore::new_exclusive( + data_dir, + Some(SQLITE_TIER_INDEX_DB_FILE_NAME.to_string()), + Some(KV_TABLE_NAME.to_string()), + )?; + let store: Arc = Arc::new(DynStoreWrapper(store)); + Self::read_or_create_database_id(store.as_ref()).await?; + Ok(Self { _store: store }) + } + + async fn read_or_create_database_id( + store: &DynStore, + ) -> io::Result<[u8; INDEX_DATABASE_ID_LEN]> { + match KVStore::read(store, INDEX_METADATA_PRIMARY_NAMESPACE, "", INDEX_DATABASE_ID_KEY) + .await + { + Ok(bytes) => bytes.try_into().map_err(|_| { + io::Error::new(io::ErrorKind::InvalidData, "Invalid tier-store index database ID") + }), + Err(e) if e.kind() == io::ErrorKind::NotFound => { + let mut database_id = [0; INDEX_DATABASE_ID_LEN]; + getrandom::fill(&mut database_id).map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("Failed to generate tier-store index database ID: {e}"), + ) + })?; + KVStore::write( + store, + INDEX_METADATA_PRIMARY_NAMESPACE, + "", + INDEX_DATABASE_ID_KEY, + database_id.to_vec(), + ) + .await?; + Ok(database_id) + }, + Err(e) => Err(e), + } + } +} /// A 3-tiered [`KVStore`] implementation that routes data across /// storage backends that may be local or remote: @@ -87,6 +142,20 @@ impl TierStore { inner.ephemeral_store = Some(ephemeral); } + + pub(crate) fn set_index_store(&mut self, index: TierStoreIndex) { + debug_assert_eq!(Arc::strong_count(&self.inner), 1); + + let inner = Arc::get_mut(&mut self.inner).expect( + "TierStore should not be shared during configuration. No other references should exist", + ); + + inner.index = Some(index); + } +} + +pub(crate) async fn setup_index_store(data_dir: PathBuf) -> io::Result { + TierStoreIndex::new(data_dir).await } impl KVStore for TierStore { @@ -188,6 +257,8 @@ struct TierStoreInner { ephemeral_store: Option>, /// An optional second durable store for primary-backed data. backup_store: Option>, + /// The local store used to index the logical contents across tiers. + index: Option, /// Per-key locks for serializing primary+backup operations and skipping stale writes. locks: Mutex>>>, next_write_version: AtomicU64, @@ -201,6 +272,7 @@ impl TierStoreInner { primary_store, ephemeral_store: None, backup_store: None, + index: None, locks: Mutex::new(HashMap::new()), next_write_version: AtomicU64::new(1), logger, @@ -649,6 +721,35 @@ mod tests { TierStore::new(primary_store, logger) } + #[tokio::test] + async fn index_store_is_internal_persistent_sqlite_store() { + let base_dir = random_storage_path(); + let _cleanup = CleanupDir(base_dir.clone()); + + let index = setup_index_store(base_dir.clone()).await.unwrap(); + assert!(base_dir.join(SQLITE_TIER_INDEX_DB_FILE_NAME).exists()); + + let database_id = + TierStoreIndex::read_or_create_database_id(index._store.as_ref()).await.unwrap(); + let persisted_database_id = + TierStoreIndex::read_or_create_database_id(index._store.as_ref()).await.unwrap(); + assert_ne!(database_id, [0; INDEX_DATABASE_ID_LEN]); + assert_eq!(persisted_database_id, database_id); + } + + #[tokio::test] + async fn index_store_rejects_second_owner() { + let base_dir = random_storage_path(); + let _cleanup = CleanupDir(base_dir.clone()); + + let _index = setup_index_store(base_dir.clone()).await.unwrap(); + let error = match setup_index_store(base_dir).await { + Ok(_) => panic!("a second index-store owner must be rejected"), + Err(e) => e, + }; + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + } + enum FailureMode { List, Write { attempts: Arc }, From f1605cb5ea868f7e24bbda5b55879b136c41458d Mon Sep 17 00:00:00 2001 From: Enigbe Date: Mon, 17 Aug 2026 09:15:21 +0100 Subject: [PATCH 03/14] Index TierStore listings across storage tiers Primary and ephemeral stores maintain independent creation orders, so their paginated listings cannot be merged while preserving the PaginatedKVStore ordering contract. In this commit, the local TierStore index records logical key membership in one SQLite ordering domain. We initialize each namespace from the primary store on first use, preserving its existing paginated order, then maintain the index after writes and removals. We also route both list methods through the index so keys from either tier share a consistent creation order. Assisted-by: Amp (AI coding agent) --- src/io/tier_store.rs | 419 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 400 insertions(+), 19 deletions(-) diff --git a/src/io/tier_store.rs b/src/io/tier_store.rs index 3bce3f09a..622bacc4b 100644 --- a/src/io/tier_store.rs +++ b/src/io/tier_store.rs @@ -12,6 +12,7 @@ use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; +use bitcoin::hashes::{sha256, Hash, HashEngine}; use lightning::util::persist::{ KVStore, PageToken, PaginatedKVStore, PaginatedListResponse, NETWORK_GRAPH_PERSISTENCE_KEY, NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, SCORER_PERSISTENCE_KEY, @@ -26,15 +27,19 @@ use crate::logger::{LdkLogger, Logger}; use crate::types::{DynStore, DynStoreWrapper}; const INDEX_DATABASE_ID_LEN: usize = 16; +const INDEX_ENTRIES_PRIMARY_NAMESPACE: &str = "_tier_store_entries"; const INDEX_METADATA_PRIMARY_NAMESPACE: &str = "_tier_store_metadata"; const INDEX_DATABASE_ID_KEY: &str = "index_database_id"; +const INDEX_NAMESPACE_READY_KEY_PREFIX: &str = "ready_"; +const INDEX_ENTRY_VALUE: &[u8] = &[1]; pub(crate) struct TierStoreIndex { // Holding the store keeps its exclusive SQLite lock for the lifetime of the tier store. - _store: Arc, + store: Arc, } impl TierStoreIndex { + /// Opens the internal SQLite index and ensures that it has a persistent database identity. async fn new(data_dir: PathBuf) -> io::Result { let store = SqliteStore::new_exclusive( data_dir, @@ -43,9 +48,150 @@ impl TierStoreIndex { )?; let store: Arc = Arc::new(DynStoreWrapper(store)); Self::read_or_create_database_id(store.as_ref()).await?; - Ok(Self { _store: store }) + Ok(Self { store }) } + /// Constructs an index over the supplied store for tests that do not need SQLite persistence. + #[cfg(test)] + fn from_store(store: Arc) -> Self { + Self { store } + } + + /// Derives the internal secondary namespace for a logical namespace pair. + /// + /// Length-prefixing keeps distinct pairs from producing the same hash input. The original pair + /// is also stored as namespace metadata so that a hash collision can be detected. + fn namespace_id(primary_namespace: &str, secondary_namespace: &str) -> String { + let mut engine = sha256::Hash::engine(); + engine.input(&(primary_namespace.len() as u64).to_be_bytes()); + engine.input(primary_namespace.as_bytes()); + engine.input(&(secondary_namespace.len() as u64).to_be_bytes()); + engine.input(secondary_namespace.as_bytes()); + sha256::Hash::from_engine(engine).to_string() + } + + /// Derives the metadata key that records whether a logical namespace is index-backed. + fn namespace_ready_key(primary_namespace: &str, secondary_namespace: &str) -> String { + format!( + "{}{}", + INDEX_NAMESPACE_READY_KEY_PREFIX, + Self::namespace_id(primary_namespace, secondary_namespace) + ) + } + + /// Encodes the original logical namespace pair for collision detection. + fn namespace_metadata(primary_namespace: &str, secondary_namespace: &str) -> Vec { + // The fixed five-byte overhead is one format-version byte plus two big-endian u16 + // namespace-length prefixes: 1 + 2 + 2 = 5. + let mut metadata = + Vec::with_capacity(5 + primary_namespace.len() + secondary_namespace.len()); + metadata.push(1); + metadata.extend_from_slice(&(primary_namespace.len() as u16).to_be_bytes()); + metadata.extend_from_slice(primary_namespace.as_bytes()); + metadata.extend_from_slice(&(secondary_namespace.len() as u16).to_be_bytes()); + metadata.extend_from_slice(secondary_namespace.as_bytes()); + metadata + } + + /// Returns whether the namespace's index is authoritative for listing. + /// + /// Returns an error if the stored namespace metadata does not match the requested namespace, + /// which indicates either corrupt metadata or a namespace-ID collision. + async fn is_namespace_ready( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result { + match KVStore::read( + self.store.as_ref(), + INDEX_METADATA_PRIMARY_NAMESPACE, + "", + &Self::namespace_ready_key(primary_namespace, secondary_namespace), + ) + .await + { + Ok(metadata) + if metadata == Self::namespace_metadata(primary_namespace, secondary_namespace) => + { + Ok(true) + }, + Ok(_) => Err(io::Error::new( + io::ErrorKind::InvalidData, + "Tier-store index namespace collision", + )), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e), + } + } + + /// Marks the namespace's index as authoritative by persisting its original namespace pair. + async fn mark_namespace_ready( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result<()> { + KVStore::write( + self.store.as_ref(), + INDEX_METADATA_PRIMARY_NAMESPACE, + "", + &Self::namespace_ready_key(primary_namespace, secondary_namespace), + Self::namespace_metadata(primary_namespace, secondary_namespace), + ) + .await + } + + /// Adds a logical key to the namespace's ordered listing index. + /// + /// Rewriting an existing key preserves its original index-store creation order. + async fn write_entry( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result<()> { + KVStore::write( + self.store.as_ref(), + INDEX_ENTRIES_PRIMARY_NAMESPACE, + &Self::namespace_id(primary_namespace, secondary_namespace), + key, + INDEX_ENTRY_VALUE.to_vec(), + ) + .await + } + + /// Removes a logical key from the namespace's listing index. + async fn remove_entry( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, + ) -> io::Result<()> { + KVStore::remove( + self.store.as_ref(), + INDEX_ENTRIES_PRIMARY_NAMESPACE, + &Self::namespace_id(primary_namespace, secondary_namespace), + key, + lazy, + ) + .await + } + + /// Lists all logical keys recorded in the namespace's index. + async fn list( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result> { + KVStore::list( + self.store.as_ref(), + INDEX_ENTRIES_PRIMARY_NAMESPACE, + &Self::namespace_id(primary_namespace, secondary_namespace), + ) + .await + } + + /// Lists logical keys in the index store's creation order. + async fn list_paginated( + &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, + ) -> io::Result { + PaginatedKVStore::list_paginated( + self.store.as_ref(), + INDEX_ENTRIES_PRIMARY_NAMESPACE, + &Self::namespace_id(primary_namespace, secondary_namespace), + page_token, + ) + .await + } + + /// Reads the index database identity, creating and persisting one when it is absent. async fn read_or_create_database_id( store: &DynStore, ) -> io::Result<[u8; INDEX_DATABASE_ID_LEN]> { @@ -91,9 +237,9 @@ impl TierStoreIndex { /// /// Reads and lists do not consult the backup store during normal operation. /// Ephemeral data is read from and written to the ephemeral store when configured. -/// Unpaginated listings expose the logical contents of the primary and ephemeral -/// stores, while paginated listings expose only the primary store until cross-tier -/// pagination semantics are defined. +/// Namespaces are indexed locally so unpaginated and paginated listings expose the +/// same logical contents in cross-tier creation order. Existing primary-store keys +/// are imported into the index before a namespace is first modified or listed. /// /// Note that dual-store writes and removals are not atomic across the primary and /// backup stores. If one store succeeds and the other fails, the operation @@ -259,6 +405,8 @@ struct TierStoreInner { backup_store: Option>, /// The local store used to index the logical contents across tiers. index: Option, + /// Per-namespace locks for serializing first-use index initialization. + index_initialization_locks: Mutex>>>, /// Per-key locks for serializing primary+backup operations and skipping stale writes. locks: Mutex>>>, next_write_version: AtomicU64, @@ -273,6 +421,7 @@ impl TierStoreInner { ephemeral_store: None, backup_store: None, index: None, + index_initialization_locks: Mutex::new(HashMap::new()), locks: Mutex::new(HashMap::new()), next_write_version: AtomicU64::new(1), logger, @@ -301,6 +450,26 @@ impl TierStoreInner { } } + /// Returns the lock that serializes initialization of the given logical namespace. + fn get_index_initialization_lock( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> Arc> { + let namespace_id = TierStoreIndex::namespace_id(primary_namespace, secondary_namespace); + let mut locks = self.index_initialization_locks.lock().expect("lock"); + Arc::clone(locks.entry(namespace_id).or_insert_with(|| Arc::new(TokioMutex::new(())))) + } + + /// Removes an initialization lock after its final active user releases it. + fn clean_index_initialization_locks( + &self, lock_ref: &Arc>, primary_namespace: &str, secondary_namespace: &str, + ) { + let namespace_id = TierStoreIndex::namespace_id(primary_namespace, secondary_namespace); + let mut locks = self.index_initialization_locks.lock().expect("lock"); + if Arc::strong_count(lock_ref) == 2 { + locks.remove(&namespace_id); + } + } + fn build_locking_key( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, ) -> String { @@ -488,6 +657,7 @@ impl TierStoreInner { Some(key.as_str()), "write", )?; + self.ensure_namespace_indexed(&primary_namespace, &secondary_namespace).await?; if is_ephemeral_cached_key(&primary_namespace, &secondary_namespace, &key) { if let Some(eph_store) = self.ephemeral_store.as_ref() { @@ -501,7 +671,13 @@ impl TierStoreInner { key.as_str(), buf, ) - .await + .await?; + if let Some(index) = self.index.as_ref() { + index + .write_entry(&primary_namespace, &secondary_namespace, &key) + .await?; + } + Ok(()) }) .await; } @@ -514,7 +690,11 @@ impl TierStoreInner { key.as_str(), buf, ) - .await + .await?; + if let Some(index) = self.index.as_ref() { + index.write_entry(&primary_namespace, &secondary_namespace, &key).await?; + } + Ok(()) }) .await } @@ -529,6 +709,7 @@ impl TierStoreInner { Some(key.as_str()), "remove", )?; + self.ensure_namespace_indexed(&primary_namespace, &secondary_namespace).await?; if is_ephemeral_cached_key(&primary_namespace, &secondary_namespace, &key) { if let Some(eph_store) = self.ephemeral_store.as_ref() { @@ -542,7 +723,13 @@ impl TierStoreInner { key.as_str(), lazy, ) - .await + .await?; + if let Some(index) = self.index.as_ref() { + index + .remove_entry(&primary_namespace, &secondary_namespace, &key, lazy) + .await?; + } + Ok(()) }) .await; } @@ -555,7 +742,11 @@ impl TierStoreInner { key.as_str(), lazy, ) - .await + .await?; + if let Some(index) = self.index.as_ref() { + index.remove_entry(&primary_namespace, &secondary_namespace, &key, lazy).await?; + } + Ok(()) }) .await } @@ -570,7 +761,71 @@ impl TierStoreInner { "list", )?; - let mut keys = self.list_primary(&primary_namespace, &secondary_namespace).await?; + self.ensure_namespace_indexed(&primary_namespace, &secondary_namespace).await?; + if let Some(index) = self.index.as_ref() { + return index.list(&primary_namespace, &secondary_namespace).await; + } + + self.list_value_stores(&primary_namespace, &secondary_namespace).await + } + + /// Imports a namespace's existing primary-store keys and makes its index authoritative. + /// + /// Primary pagination returns keys from newest to oldest, so the complete result is reversed + /// before insertion to preserve that order in the local index. The readiness marker is written + /// last so a failed import is safely retried before the index can be observed. + async fn ensure_namespace_indexed( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result<()> { + let Some(index) = self.index.as_ref() else { + return Ok(()); + }; + if index.is_namespace_ready(primary_namespace, secondary_namespace).await? { + return Ok(()); + } + + let lock_ref = self.get_index_initialization_lock(primary_namespace, secondary_namespace); + let result: io::Result<()> = async { + let _guard = lock_ref.lock().await; + if index.is_namespace_ready(primary_namespace, secondary_namespace).await? { + Ok(()) + } else { + let mut keys = Vec::new(); + let mut page_token = None; + loop { + let page = PaginatedKVStore::list_paginated( + self.primary_store.as_ref(), + primary_namespace, + secondary_namespace, + page_token, + ) + .await?; + keys.extend(page.keys); + match page.next_page_token { + Some(next_page_token) => page_token = Some(next_page_token), + None => break, + } + } + + for key in keys.into_iter().rev() { + index.write_entry(primary_namespace, secondary_namespace, &key).await?; + } + index.mark_namespace_ready(primary_namespace, secondary_namespace).await + } + } + .await; + self.clean_index_initialization_locks(&lock_ref, primary_namespace, secondary_namespace); + result + } + + /// Lists the authoritative logical keys directly from the primary and ephemeral value stores. + /// + /// Ephemeral-routed keys are taken only from the ephemeral store, excluding stale primary + /// copies. This path is used when no local index store is configured. + async fn list_value_stores( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result> { + let mut keys = self.list_primary(primary_namespace, secondary_namespace).await?; let Some(ephemeral_store) = self.ephemeral_store.as_ref() else { return Ok(keys); @@ -584,14 +839,13 @@ impl TierStoreInner { // The ephemeral store is authoritative for keys routed there. Exclude stale // primary copies, then add only routed keys from the ephemeral store. - keys.retain(|key| !is_ephemeral_cached_key(&primary_namespace, &secondary_namespace, key)); + keys.retain(|key| !is_ephemeral_cached_key(primary_namespace, secondary_namespace, key)); let ephemeral_keys = - KVStore::list(ephemeral_store.as_ref(), &primary_namespace, &secondary_namespace) - .await?; + KVStore::list(ephemeral_store.as_ref(), primary_namespace, secondary_namespace).await?; for key in ephemeral_keys { - if is_ephemeral_cached_key(&primary_namespace, &secondary_namespace, &key) + if is_ephemeral_cached_key(primary_namespace, secondary_namespace, &key) && !keys.contains(&key) { keys.push(key); @@ -612,9 +866,13 @@ impl TierStoreInner { "list_paginated", )?; - // TODO(@enigbe): Merge listings across tiers once `PaginatedKVStore` provides a - // shared ordering and cursor contract that permits a correct cross-store merge. - // Until then, listings intentionally expose only the primary tier. + self.ensure_namespace_indexed(&primary_namespace, &secondary_namespace).await?; + if let Some(index) = self.index.as_ref() { + return index + .list_paginated(&primary_namespace, &secondary_namespace, page_token) + .await; + } + PaginatedKVStore::list_paginated( self.primary_store.as_ref(), &primary_namespace, @@ -721,6 +979,11 @@ mod tests { TierStore::new(primary_store, logger) } + fn set_test_index_store(tier: &mut TierStore) { + let store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + tier.set_index_store(TierStoreIndex::from_store(store)); + } + #[tokio::test] async fn index_store_is_internal_persistent_sqlite_store() { let base_dir = random_storage_path(); @@ -730,9 +993,9 @@ mod tests { assert!(base_dir.join(SQLITE_TIER_INDEX_DB_FILE_NAME).exists()); let database_id = - TierStoreIndex::read_or_create_database_id(index._store.as_ref()).await.unwrap(); + TierStoreIndex::read_or_create_database_id(index.store.as_ref()).await.unwrap(); let persisted_database_id = - TierStoreIndex::read_or_create_database_id(index._store.as_ref()).await.unwrap(); + TierStoreIndex::read_or_create_database_id(index.store.as_ref()).await.unwrap(); assert_ne!(database_id, [0; INDEX_DATABASE_ID_LEN]); assert_eq!(persisted_database_id, database_id); } @@ -750,6 +1013,124 @@ mod tests { assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); } + #[tokio::test] + async fn indexed_listing_orders_keys_across_primary_and_ephemeral_stores() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(primary_store, logger); + set_test_index_store(&mut tier); + tier.set_ephemeral_store(ephemeral_store); + + for key in ["primary-a", NETWORK_GRAPH_PERSISTENCE_KEY, "primary-b"] { + tier.write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + key, + vec![1], + ) + .await + .unwrap(); + } + + let page = PaginatedKVStore::list_paginated( + &tier, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + None, + ) + .await + .unwrap(); + assert_eq!( + page.keys, + vec![ + "primary-b".to_string(), + NETWORK_GRAPH_PERSISTENCE_KEY.to_string(), + "primary-a".to_string(), + ] + ); + + let mut listed = KVStore::list( + &tier, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await + .unwrap(); + listed.sort(); + assert_eq!( + listed, + vec![ + NETWORK_GRAPH_PERSISTENCE_KEY.to_string(), + "primary-a".to_string(), + "primary-b".to_string(), + ] + ); + } + + #[tokio::test] + async fn indexed_listing_preserves_updates_and_reorders_recreated_keys() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(primary_store, logger); + set_test_index_store(&mut tier); + + for key in ["a", "b"] { + tier.write("namespace", "", key, vec![1]).await.unwrap(); + } + tier.write("namespace", "", "a", vec![2]).await.unwrap(); + + let page = PaginatedKVStore::list_paginated(&tier, "namespace", "", None).await.unwrap(); + assert_eq!(page.keys, vec!["b".to_string(), "a".to_string()]); + + tier.remove("namespace", "", "a", false).await.unwrap(); + tier.write("namespace", "", "a", vec![3]).await.unwrap(); + let page = PaginatedKVStore::list_paginated(&tier, "namespace", "", None).await.unwrap(); + assert_eq!(page.keys, vec!["a".to_string(), "b".to_string()]); + } + + #[tokio::test] + async fn namespace_initialization_preserves_existing_primary_order_across_pages() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + for i in 0..55 { + primary_store + .write("namespace", "", &format!("existing-{i:02}"), vec![1]) + .await + .unwrap(); + } + let mut tier = setup_tier_store(primary_store, logger); + set_test_index_store(&mut tier); + + let mut actual = Vec::new(); + let mut page_token = None; + loop { + let page = + PaginatedKVStore::list_paginated(&tier, "namespace", "", page_token).await.unwrap(); + actual.extend(page.keys); + match page.next_page_token { + Some(next_page_token) => page_token = Some(next_page_token), + None => break, + } + } + + let expected = (0..55).rev().map(|i| format!("existing-{i:02}")).collect::>(); + assert_eq!(actual, expected); + assert_eq!(KVStore::list(&tier, "namespace", "").await.unwrap().len(), 55); + } + enum FailureMode { List, Write { attempts: Arc }, From ab843f39b936c6f276aab0ee244b616476c81a16 Mon Sep 17 00:00:00 2001 From: Enigbe Date: Mon, 17 Aug 2026 20:34:11 +0100 Subject: [PATCH 04/14] Recover interrupted TierStore membership changes Value-store and index updates cannot be committed atomically, so failures can leave keys partially created or removed across storage tiers. In this commit we: - Persist creation and removal intent in the local index database before applying external changes. We recover pending operations before listing or modifying their namespace, including rechecking recovery under the per-key lock to prevent queued operations from racing with newly recorded journal entries. - Roll "creates" forward to all required stores, remove keys from the listing index before deleting their value copies, and retain failed operations for retry. Keep ordinary updates unjournaled and reject unindexed existing values as corruption. - Add deterministic failure and synchronization instrumentation to test recovery across interrupted writes, removals, primary/backup divergence, and queued operations. Assisted-by: Amp (AI coding agent) --- src/io/tier_store.rs | 843 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 735 insertions(+), 108 deletions(-) diff --git a/src/io/tier_store.rs b/src/io/tier_store.rs index 622bacc4b..419e6bcc5 100644 --- a/src/io/tier_store.rs +++ b/src/io/tier_store.rs @@ -18,7 +18,8 @@ use lightning::util::persist::{ NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, SCORER_PERSISTENCE_KEY, SCORER_PERSISTENCE_PRIMARY_NAMESPACE, }; -use lightning::{io, log_error}; +use lightning::util::ser::{Readable, Writeable}; +use lightning::{impl_writeable_tlv_based, impl_writeable_tlv_based_enum, io, log_error}; use tokio::sync::Mutex as TokioMutex; use crate::io::sqlite_store::{SqliteStore, KV_TABLE_NAME, SQLITE_TIER_INDEX_DB_FILE_NAME}; @@ -28,11 +29,76 @@ use crate::types::{DynStore, DynStoreWrapper}; const INDEX_DATABASE_ID_LEN: usize = 16; const INDEX_ENTRIES_PRIMARY_NAMESPACE: &str = "_tier_store_entries"; +const INDEX_JOURNAL_PRIMARY_NAMESPACE: &str = "_tier_store_journal"; const INDEX_METADATA_PRIMARY_NAMESPACE: &str = "_tier_store_metadata"; const INDEX_DATABASE_ID_KEY: &str = "index_database_id"; const INDEX_NAMESPACE_READY_KEY_PREFIX: &str = "ready_"; const INDEX_ENTRY_VALUE: &[u8] = &[1]; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ValueTier { + Primary, + Ephemeral, +} + +impl_writeable_tlv_based_enum!(ValueTier, + (0, Primary) => {}, + (2, Ephemeral) => {}, +); + +#[derive(Debug, PartialEq, Eq)] +enum JournalOperation { + Create { + /// The complete intended value, retained locally so recovery does not depend on which remote + /// or local value-store write completed before interruption. + value: Vec, + }, + Remove { + lazy: bool, + }, +} + +impl_writeable_tlv_based_enum!(JournalOperation, + (0, Create) => { + (0, value, required), + }, + (2, Remove) => { + (0, lazy, required), + }, +); + +#[derive(Debug, PartialEq, Eq)] +struct JournalEntry { + primary_namespace: String, + secondary_namespace: String, + key: String, + tier: ValueTier, + requires_backup: bool, + operation: JournalOperation, +} + +impl_writeable_tlv_based!(JournalEntry, { + (0, primary_namespace, required), + (2, secondary_namespace, required), + (4, key, required), + (6, tier, required), + (8, requires_backup, required), + (10, operation, required), +}); + +impl JournalEntry { + /// Encodes a pending operation for persistence in the local index database. + fn serialize(&self) -> Vec { + Writeable::encode(self) + } + + /// Decodes and validates a pending operation from the local index database. + fn deserialize(encoded: &[u8]) -> io::Result { + Readable::read(&mut &*encoded) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "Invalid journal entry")) + } +} + pub(crate) struct TierStoreIndex { // Holding the store keeps its exclusive SQLite lock for the lifetime of the tier store. store: Arc, @@ -152,6 +218,25 @@ impl TierStoreIndex { .await } + /// Returns whether the namespace's listing index contains the logical key. + async fn contains_entry( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result { + match KVStore::read( + self.store.as_ref(), + INDEX_ENTRIES_PRIMARY_NAMESPACE, + &Self::namespace_id(primary_namespace, secondary_namespace), + key, + ) + .await + { + Ok(value) if value == INDEX_ENTRY_VALUE => Ok(true), + Ok(_) => Err(io::Error::new(io::ErrorKind::InvalidData, "Invalid index entry")), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e), + } + } + /// Removes a logical key from the namespace's listing index. async fn remove_entry( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, @@ -191,6 +276,81 @@ impl TierStoreIndex { .await } + /// Persists a pending operation before its value-store effects begin. + async fn write_journal_entry(&self, entry: &JournalEntry) -> io::Result<()> { + KVStore::write( + self.store.as_ref(), + INDEX_JOURNAL_PRIMARY_NAMESPACE, + &Self::namespace_id(&entry.primary_namespace, &entry.secondary_namespace), + &entry.key, + entry.serialize(), + ) + .await + } + + /// Reads a pending operation for a logical key. + async fn read_journal_entry( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result { + let encoded = KVStore::read( + self.store.as_ref(), + INDEX_JOURNAL_PRIMARY_NAMESPACE, + &Self::namespace_id(primary_namespace, secondary_namespace), + key, + ) + .await?; + let entry = JournalEntry::deserialize(&encoded)?; + if entry.primary_namespace != primary_namespace + || entry.secondary_namespace != secondary_namespace + || entry.key != key + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Journal entry identity does not match its location", + )); + } + Ok(entry) + } + + /// Lists pending-operation keys from oldest to newest journal insertion. + async fn list_journal_entries( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result> { + let secondary_namespace = Self::namespace_id(primary_namespace, secondary_namespace); + let mut keys = Vec::new(); + let mut page_token = None; + loop { + let page = PaginatedKVStore::list_paginated( + self.store.as_ref(), + INDEX_JOURNAL_PRIMARY_NAMESPACE, + &secondary_namespace, + page_token, + ) + .await?; + keys.extend(page.keys); + match page.next_page_token { + Some(next_page_token) => page_token = Some(next_page_token), + None => break, + } + } + keys.reverse(); + Ok(keys) + } + + /// Clears a pending operation after all of its intended effects are durable. + async fn remove_journal_entry( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result<()> { + KVStore::remove( + self.store.as_ref(), + INDEX_JOURNAL_PRIMARY_NAMESPACE, + &Self::namespace_id(primary_namespace, secondary_namespace), + key, + false, + ) + .await + } + /// Reads the index database identity, creating and persisting one when it is absent. async fn read_or_create_database_id( store: &DynStore, @@ -434,11 +594,13 @@ impl TierStoreInner { panic!("TierStore version counter overflowed"); } - let mut locks = self.locks.lock().expect("lock"); - let lock_ref = - Arc::clone(locks.entry(locking_key).or_insert_with(|| Arc::new(TokioMutex::new(0)))); + (self.get_lock_ref(locking_key), version) + } - (lock_ref, version) + /// Returns the lock that serializes operations for one logical key. + fn get_lock_ref(&self, locking_key: String) -> Arc> { + let mut locks = self.locks.lock().expect("lock"); + Arc::clone(locks.entry(locking_key).or_insert_with(|| Arc::new(TokioMutex::new(0)))) } fn clean_locks(&self, lock_ref: &Arc>, locking_key: String) { @@ -657,48 +819,51 @@ impl TierStoreInner { Some(key.as_str()), "write", )?; - self.ensure_namespace_indexed(&primary_namespace, &secondary_namespace).await?; - - if is_ephemeral_cached_key(&primary_namespace, &secondary_namespace, &key) { - if let Some(eph_store) = self.ephemeral_store.as_ref() { - let eph_store = Arc::clone(eph_store); - return self - .execute_locked_write(lock_ref, locking_key, version, || async move { - KVStore::write( - eph_store.as_ref(), - primary_namespace.as_str(), - secondary_namespace.as_str(), - key.as_str(), - buf, - ) - .await?; - if let Some(index) = self.index.as_ref() { - index - .write_entry(&primary_namespace, &secondary_namespace, &key) - .await?; - } - Ok(()) - }) - .await; - } - } + self.prepare_namespace(&primary_namespace, &secondary_namespace).await?; self.execute_locked_write(lock_ref, locking_key, version, || async move { - self.write_primary_backup_async( - primary_namespace.as_str(), - secondary_namespace.as_str(), - key.as_str(), - buf, - ) - .await?; - if let Some(index) = self.index.as_ref() { - index.write_entry(&primary_namespace, &secondary_namespace, &key).await?; - } - Ok(()) + self.write_locked(&primary_namespace, &secondary_namespace, &key, buf).await }) .await } + /// Writes one key after recovering any pending operation while its per-key lock is held. + async fn write_locked( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, value: Vec, + ) -> io::Result<()> { + self.recover_key_locked(primary_namespace, secondary_namespace, key).await?; + let tier = self.value_tier(primary_namespace, secondary_namespace, key); + let Some(index) = self.index.as_ref() else { + return self + .write_value(tier, primary_namespace, secondary_namespace, key, value) + .await; + }; + + if index.contains_entry(primary_namespace, secondary_namespace, key).await? { + return self + .write_value(tier, primary_namespace, secondary_namespace, key, value) + .await; + } + if self.value_exists(tier, primary_namespace, secondary_namespace, key).await? { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Value exists without a tier-store index entry", + )); + } + + let journal_entry = JournalEntry { + primary_namespace: primary_namespace.to_string(), + secondary_namespace: secondary_namespace.to_string(), + key: key.to_string(), + tier, + requires_backup: tier == ValueTier::Primary && self.backup_store.is_some(), + operation: JournalOperation::Create { value }, + }; + index.write_journal_entry(&journal_entry).await?; + self.apply_pending_create(&journal_entry).await?; + index.remove_journal_entry(primary_namespace, secondary_namespace, key).await + } + async fn remove_internal( &self, primary_namespace: String, secondary_namespace: String, key: String, lazy: bool, lock_ref: Arc>, locking_key: String, version: u64, @@ -709,45 +874,179 @@ impl TierStoreInner { Some(key.as_str()), "remove", )?; - self.ensure_namespace_indexed(&primary_namespace, &secondary_namespace).await?; + self.prepare_namespace(&primary_namespace, &secondary_namespace).await?; - if is_ephemeral_cached_key(&primary_namespace, &secondary_namespace, &key) { - if let Some(eph_store) = self.ephemeral_store.as_ref() { - let eph_store = Arc::clone(eph_store); - return self - .execute_locked_write(lock_ref, locking_key, version, || async move { - KVStore::remove( - eph_store.as_ref(), - primary_namespace.as_str(), - secondary_namespace.as_str(), - key.as_str(), - lazy, + self.execute_locked_write(lock_ref, locking_key, version, || async move { + self.remove_locked(&primary_namespace, &secondary_namespace, &key, lazy).await + }) + .await + } + + /// Removes one key after recovering any pending operation while its per-key lock is held. + async fn remove_locked( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, + ) -> io::Result<()> { + self.recover_key_locked(primary_namespace, secondary_namespace, key).await?; + let tier = self.value_tier(primary_namespace, secondary_namespace, key); + let Some(index) = self.index.as_ref() else { + return self + .remove_value(tier, primary_namespace, secondary_namespace, key, lazy) + .await; + }; + + let journal_entry = JournalEntry { + primary_namespace: primary_namespace.to_string(), + secondary_namespace: secondary_namespace.to_string(), + key: key.to_string(), + tier, + requires_backup: tier == ValueTier::Primary && self.backup_store.is_some(), + operation: JournalOperation::Remove { lazy }, + }; + index.write_journal_entry(&journal_entry).await?; + self.apply_pending_remove(&journal_entry).await?; + index.remove_journal_entry(primary_namespace, secondary_namespace, key).await + } + + /// Selects the authoritative value tier for a logical key. + fn value_tier( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> ValueTier { + if self.ephemeral_store.is_some() + && is_ephemeral_cached_key(primary_namespace, secondary_namespace, key) + { + ValueTier::Ephemeral + } else { + ValueTier::Primary + } + } + + /// Checks for an existing value before classifying an unindexed key as a new creation. + async fn value_exists( + &self, tier: ValueTier, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result { + let store = match tier { + ValueTier::Primary => &self.primary_store, + ValueTier::Ephemeral => self.ephemeral_store.as_ref().ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "Ephemeral store is unavailable") + })?, + }; + match KVStore::read(store.as_ref(), primary_namespace, secondary_namespace, key).await { + Ok(_) => Ok(true), + Err(e) if e.kind() == io::ErrorKind::NotFound => { + if tier == ValueTier::Primary { + if let Some(backup_store) = self.backup_store.as_ref() { + return match KVStore::read( + backup_store.as_ref(), + primary_namespace, + secondary_namespace, + key, ) - .await?; - if let Some(index) = self.index.as_ref() { - index - .remove_entry(&primary_namespace, &secondary_namespace, &key, lazy) - .await?; - } - Ok(()) - }) - .await; - } + .await + { + Ok(_) => Err(io::Error::new( + io::ErrorKind::InvalidData, + "Backup value exists without a primary or index entry", + )), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e), + }; + } + } + Ok(false) + }, + Err(e) => Err(e), } + } - self.execute_locked_write(lock_ref, locking_key, version, || async move { - self.remove_primary_backup_async( - primary_namespace.as_str(), - secondary_namespace.as_str(), - key.as_str(), - lazy, - ) + /// Writes a value to its authoritative tier and any required backup. + async fn write_value( + &self, tier: ValueTier, primary_namespace: &str, secondary_namespace: &str, key: &str, + value: Vec, + ) -> io::Result<()> { + match tier { + ValueTier::Primary => { + self.write_primary_backup_async(primary_namespace, secondary_namespace, key, value) + .await + }, + ValueTier::Ephemeral => { + let store = self.ephemeral_store.as_ref().ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "Ephemeral store is unavailable") + })?; + KVStore::write(store.as_ref(), primary_namespace, secondary_namespace, key, value) + .await + }, + } + } + + /// Removes a value from its authoritative tier and any required backup. + async fn remove_value( + &self, tier: ValueTier, primary_namespace: &str, secondary_namespace: &str, key: &str, + lazy: bool, + ) -> io::Result<()> { + match tier { + ValueTier::Primary => { + self.remove_primary_backup_async(primary_namespace, secondary_namespace, key, lazy) + .await + }, + ValueTier::Ephemeral => { + let store = self.ephemeral_store.as_ref().ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "Ephemeral store is unavailable") + })?; + KVStore::remove(store.as_ref(), primary_namespace, secondary_namespace, key, lazy) + .await + }, + } + } + + /// Completes a journaled create and makes it visible in the listing index. + async fn apply_pending_create(&self, entry: &JournalEntry) -> io::Result<()> { + let JournalOperation::Create { value } = &entry.operation else { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Expected pending create")); + }; + if entry.requires_backup && self.backup_store.is_none() { + return Err(io::Error::new( + io::ErrorKind::NotFound, + "Required backup store is unavailable", + )); + } + self.write_value( + entry.tier, + &entry.primary_namespace, + &entry.secondary_namespace, + &entry.key, + value.clone(), + ) + .await?; + self.index + .as_ref() + .expect("pending operations require an index") + .write_entry(&entry.primary_namespace, &entry.secondary_namespace, &entry.key) + .await + } + + /// Completes a journaled removal, hiding the key before deleting its value copies. + async fn apply_pending_remove(&self, entry: &JournalEntry) -> io::Result<()> { + let JournalOperation::Remove { lazy } = &entry.operation else { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Expected pending removal")); + }; + if entry.requires_backup && self.backup_store.is_none() { + return Err(io::Error::new( + io::ErrorKind::NotFound, + "Required backup store is unavailable", + )); + } + self.index + .as_ref() + .expect("pending operations require an index") + .remove_entry(&entry.primary_namespace, &entry.secondary_namespace, &entry.key, *lazy) .await?; - if let Some(index) = self.index.as_ref() { - index.remove_entry(&primary_namespace, &secondary_namespace, &key, lazy).await?; - } - Ok(()) - }) + self.remove_value( + entry.tier, + &entry.primary_namespace, + &entry.secondary_namespace, + &entry.key, + *lazy, + ) .await } @@ -761,7 +1060,7 @@ impl TierStoreInner { "list", )?; - self.ensure_namespace_indexed(&primary_namespace, &secondary_namespace).await?; + self.prepare_namespace(&primary_namespace, &secondary_namespace).await?; if let Some(index) = self.index.as_ref() { return index.list(&primary_namespace, &secondary_namespace).await; } @@ -818,6 +1117,55 @@ impl TierStoreInner { result } + /// Initializes a namespace and recovers all pending membership changes before use. + async fn prepare_namespace( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result<()> { + self.ensure_namespace_indexed(primary_namespace, secondary_namespace).await?; + self.recover_namespace(primary_namespace, secondary_namespace).await + } + + /// Completes journaled creates and removals before exposing a namespace. + async fn recover_namespace( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result<()> { + let Some(index) = self.index.as_ref() else { + return Ok(()); + }; + for key in index.list_journal_entries(primary_namespace, secondary_namespace).await? { + let locking_key = self.build_locking_key(primary_namespace, secondary_namespace, &key); + let lock_ref = self.get_lock_ref(locking_key.clone()); + let result: io::Result<()> = async { + let _guard = lock_ref.lock().await; + self.recover_key_locked(primary_namespace, secondary_namespace, &key).await + } + .await; + self.clean_locks(&lock_ref, locking_key); + result?; + } + Ok(()) + } + + /// Recovers one key while its per-key operation lock is held by the caller. + async fn recover_key_locked( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result<()> { + let Some(index) = self.index.as_ref() else { + return Ok(()); + }; + let entry = + match index.read_journal_entry(primary_namespace, secondary_namespace, key).await { + Ok(entry) => entry, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(e), + }; + match &entry.operation { + JournalOperation::Create { .. } => self.apply_pending_create(&entry).await?, + JournalOperation::Remove { .. } => self.apply_pending_remove(&entry).await?, + } + index.remove_journal_entry(primary_namespace, secondary_namespace, key).await + } + /// Lists the authoritative logical keys directly from the primary and ephemeral value stores. /// /// Ephemeral-routed keys are taken only from the ephemeral store, excluding stale primary @@ -866,7 +1214,7 @@ impl TierStoreInner { "list_paginated", )?; - self.ensure_namespace_indexed(&primary_namespace, &secondary_namespace).await?; + self.prepare_namespace(&primary_namespace, &secondary_namespace).await?; if let Some(index) = self.index.as_ref() { return index .list_paginated(&primary_namespace, &secondary_namespace, page_token) @@ -946,7 +1294,7 @@ mod tests { use std::panic::RefUnwindSafe; use std::path::PathBuf; use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::Arc; + use std::sync::{Arc, Mutex}; use lightning::util::logger::Level; use lightning::util::persist::{ @@ -957,6 +1305,7 @@ mod tests { NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, SCORER_PERSISTENCE_SECONDARY_NAMESPACE, }; use lightning_persister::fs_store::v2::FilesystemStoreV2; + use tokio::sync::oneshot; use super::*; use crate::io::test_utils::{ @@ -984,6 +1333,24 @@ mod tests { tier.set_index_store(TierStoreIndex::from_store(store)); } + #[test] + fn journal_entry_roundtrips() { + for operation in [ + JournalOperation::Create { value: vec![0, 1, 2, 255] }, + JournalOperation::Remove { lazy: true }, + ] { + let entry = JournalEntry { + primary_namespace: "primary".to_string(), + secondary_namespace: "secondary".to_string(), + key: "key".to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation, + }; + assert_eq!(JournalEntry::deserialize(&entry.serialize()).unwrap(), entry); + } + } + #[tokio::test] async fn index_store_is_internal_persistent_sqlite_store() { let base_dir = random_storage_path(); @@ -1131,25 +1498,246 @@ mod tests { assert_eq!(KVStore::list(&tier, "namespace", "").await.unwrap().len(), 55); } - enum FailureMode { - List, - Write { attempts: Arc }, + #[tokio::test] + async fn pending_creates_roll_forward_to_primary_and_backup_in_journal_order() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let index_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), Arc::clone(&logger)); + tier.set_backup_store(Arc::clone(&backup_store)); + tier.set_index_store(TierStoreIndex::from_store(Arc::clone(&index_store))); + tier.inner.ensure_namespace_indexed("namespace", "").await.unwrap(); + + for (key, value) in [("first", vec![1]), ("second", vec![2])] { + let entry = JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: key.to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation: JournalOperation::Create { value: value.clone() }, + }; + tier.inner.index.as_ref().unwrap().write_journal_entry(&entry).await.unwrap(); + // Simulate the backup write winning the race before interruption. + backup_store.write("namespace", "", key, value).await.unwrap(); + } + + drop(tier); + let mut recovered_tier = setup_tier_store(Arc::clone(&primary_store), logger); + recovered_tier.set_backup_store(Arc::clone(&backup_store)); + recovered_tier.set_index_store(TierStoreIndex::from_store(index_store)); + + let page = + PaginatedKVStore::list_paginated(&recovered_tier, "namespace", "", None).await.unwrap(); + assert_eq!(page.keys, vec!["second".to_string(), "first".to_string()]); + for (key, value) in [("first", vec![1]), ("second", vec![2])] { + assert_eq!(primary_store.read("namespace", "", key).await.unwrap(), value); + assert_eq!(backup_store.read("namespace", "", key).await.unwrap(), value); + assert!(recovered_tier + .inner + .index + .as_ref() + .unwrap() + .read_journal_entry("namespace", "", key) + .await + .is_err()); + } + } + + #[tokio::test] + async fn pending_removal_hides_index_entry_before_removing_value_copies() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + tier.set_backup_store(Arc::clone(&backup_store)); + set_test_index_store(&mut tier); + tier.write("namespace", "", "key", vec![1]).await.unwrap(); + + let entry = JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: "key".to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation: JournalOperation::Remove { lazy: false }, + }; + let index = tier.inner.index.as_ref().unwrap(); + index.write_journal_entry(&entry).await.unwrap(); + index.remove_entry("namespace", "", "key", false).await.unwrap(); + assert!(!index.contains_entry("namespace", "", "key").await.unwrap()); + assert!(primary_store.read("namespace", "", "key").await.is_ok()); + assert!(backup_store.read("namespace", "", "key").await.is_ok()); + assert!(index.read_journal_entry("namespace", "", "key").await.is_ok()); + + assert!(KVStore::list(&tier, "namespace", "").await.unwrap().is_empty()); + assert!(primary_store.read("namespace", "", "key").await.is_err()); + assert!(backup_store.read("namespace", "", "key").await.is_err()); + assert!(index.read_journal_entry("namespace", "", "key").await.is_err()); + } + + #[tokio::test] + async fn failed_create_retains_journal_without_exposing_index_entry() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + let attempts = Arc::new(AtomicUsize::new(0)); + let backup_store: Arc = + Arc::new(DynStoreWrapper(InstrumentedStore::new(StoreBehavior::FailWrite { + attempts: Arc::clone(&attempts), + }))); + tier.set_backup_store(backup_store); + set_test_index_store(&mut tier); + + assert!(tier.write("namespace", "", "key", vec![1]).await.is_err()); + let index = tier.inner.index.as_ref().unwrap(); + assert!(primary_store.read("namespace", "", "key").await.is_ok()); + assert!(!index.contains_entry("namespace", "", "key").await.unwrap()); + assert!(matches!( + index.read_journal_entry("namespace", "", "key").await.unwrap().operation, + JournalOperation::Create { .. } + )); + assert_eq!(attempts.load(Ordering::Relaxed), 1); + } + + #[tokio::test] + async fn queued_write_recovers_pending_create_under_key_lock_before_classifying() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let (snapshot_taken_tx, snapshot_taken_rx) = oneshot::channel(); + let (allow_return_tx, allow_return_rx) = oneshot::channel(); + let index_store: Arc = + Arc::new(DynStoreWrapper(InstrumentedStore::new(StoreBehavior::GateJournalList { + snapshot_taken: Mutex::new(Some(snapshot_taken_tx)), + allow_return: Mutex::new(Some(allow_return_rx)), + }))); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + tier.set_index_store(TierStoreIndex::from_store(index_store)); + tier.inner.ensure_namespace_indexed("namespace", "").await.unwrap(); + + let tier = Arc::new(tier); + let locking_key = tier.inner.build_locking_key("namespace", "", "key"); + let lock_ref = tier.inner.get_lock_ref(locking_key); + let guard = lock_ref.lock().await; + let write_task = { + let tier = Arc::clone(&tier); + tokio::spawn(async move { tier.write("namespace", "", "key", vec![2]).await }) + }; + snapshot_taken_rx.await.unwrap(); + + let pending = JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: "key".to_string(), + tier: ValueTier::Primary, + requires_backup: false, + operation: JournalOperation::Create { value: vec![1] }, + }; + let index = tier.inner.index.as_ref().unwrap(); + index.write_journal_entry(&pending).await.unwrap(); + primary_store.write("namespace", "", "key", vec![1]).await.unwrap(); + allow_return_tx.send(()).unwrap(); + drop(guard); + drop(lock_ref); + + write_task.await.unwrap().unwrap(); + assert_eq!(primary_store.read("namespace", "", "key").await.unwrap(), vec![2]); + assert!(index.contains_entry("namespace", "", "key").await.unwrap()); + assert!(index.read_journal_entry("namespace", "", "key").await.is_err()); } - /// A store that injects a selected failure while delegating other operations to an inner - /// [`InMemoryStore`]. - struct FailingStore { + #[tokio::test] + async fn update_rejects_value_without_index_entry() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.write("namespace", "", "key", vec![1]).await.unwrap(); + tier.inner + .index + .as_ref() + .unwrap() + .remove_entry("namespace", "", "key", false) + .await + .unwrap(); + + let error = tier.write("namespace", "", "key", vec![2]).await.unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!(primary_store.read("namespace", "", "key").await.unwrap(), vec![1]); + } + + #[tokio::test] + async fn failed_removal_retains_journal_and_hides_index_entry() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = + Arc::new(DynStoreWrapper(InstrumentedStore::new(StoreBehavior::FailRemove))); + let mut tier = setup_tier_store(primary_store, logger); + tier.set_backup_store(backup_store); + set_test_index_store(&mut tier); + tier.write("namespace", "", "key", vec![1]).await.unwrap(); + + assert!(tier.remove("namespace", "", "key", false).await.is_err()); + let index = tier.inner.index.as_ref().unwrap(); + assert!(!index.contains_entry("namespace", "", "key").await.unwrap()); + assert!(matches!( + index.read_journal_entry("namespace", "", "key").await.unwrap().operation, + JournalOperation::Remove { .. } + )); + assert!(KVStore::list(&tier, "namespace", "").await.is_err()); + } + + enum StoreBehavior { + FailList, + FailWrite { + attempts: Arc, + }, + FailRemove, + GateJournalList { + snapshot_taken: Mutex>>, + allow_return: Mutex>>, + }, + } + + /// A store that injects selected failures or synchronization points while delegating other + /// operations to an inner [`InMemoryStore`]. + struct InstrumentedStore { inner: InMemoryStore, - failure_mode: FailureMode, + behavior: StoreBehavior, } - impl FailingStore { - fn new(failure_mode: FailureMode) -> Self { - Self { inner: InMemoryStore::new(), failure_mode } + impl InstrumentedStore { + fn new(behavior: StoreBehavior) -> Self { + Self { inner: InMemoryStore::new(), behavior } } } - impl KVStore for FailingStore { + impl KVStore for InstrumentedStore { fn read( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, ) -> impl Future, io::Error>> + 'static + Send { @@ -1158,7 +1746,7 @@ mod tests { fn write( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, ) -> impl Future> + 'static + Send { - let write = if let FailureMode::Write { attempts } = &self.failure_mode { + let write = if let StoreBehavior::FailWrite { attempts } = &self.behavior { attempts.fetch_add(1, Ordering::Relaxed); None } else { @@ -1174,14 +1762,31 @@ mod tests { fn remove( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, ) -> impl Future> + 'static + Send { - KVStore::remove(&self.inner, primary_namespace, secondary_namespace, key, lazy) + let remove = match &self.behavior { + StoreBehavior::FailRemove => None, + _ => Some(KVStore::remove( + &self.inner, + primary_namespace, + secondary_namespace, + key, + lazy, + )), + }; + async move { + match remove { + Some(remove) => remove.await, + None => Err(io::Error::new(io::ErrorKind::Other, "remove failed")), + } + } } fn list( &self, primary_namespace: &str, secondary_namespace: &str, ) -> impl Future, io::Error>> + 'static + Send { - let list = match &self.failure_mode { - FailureMode::List => None, - FailureMode::Write { .. } => { + let list = match &self.behavior { + StoreBehavior::FailList => None, + StoreBehavior::FailWrite { .. } + | StoreBehavior::FailRemove + | StoreBehavior::GateJournalList { .. } => { Some(KVStore::list(&self.inner, primary_namespace, secondary_namespace)) }, }; @@ -1194,25 +1799,47 @@ mod tests { } } - impl PaginatedKVStore for FailingStore { + impl PaginatedKVStore for InstrumentedStore { fn list_paginated( &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, ) -> impl Future> + 'static + Send { - let list = match &self.failure_mode { - FailureMode::List => None, - FailureMode::Write { .. } => Some(PaginatedKVStore::list_paginated( - &self.inner, - primary_namespace, - secondary_namespace, - page_token, - )), + let fails = matches!(&self.behavior, StoreBehavior::FailList); + let gate = match &self.behavior { + StoreBehavior::GateJournalList { snapshot_taken, allow_return } => { + if primary_namespace == INDEX_JOURNAL_PRIMARY_NAMESPACE { + let snapshot_taken = snapshot_taken.lock().unwrap().take(); + let allow_return = allow_return.lock().unwrap().take(); + match (snapshot_taken, allow_return) { + (Some(snapshot_taken), Some(allow_return)) => { + Some((snapshot_taken, allow_return)) + }, + _ => None, + } + } else { + None + } + }, + _ => None, }; + let list = PaginatedKVStore::list_paginated( + &self.inner, + primary_namespace, + secondary_namespace, + page_token, + ); async move { - match list { - Some(list) => list.await, - None => Err(io::Error::new(io::ErrorKind::Other, "list_paginated failed")), + if fails { + return Err(io::Error::new(io::ErrorKind::Other, "list_paginated failed")); } + let response = list.await?; + if let Some((snapshot_taken, allow_return)) = gate { + let _ = snapshot_taken.send(()); + allow_return.await.map_err(|_| { + io::Error::new(io::ErrorKind::Other, "journal-list gate was dropped") + })?; + }; + Ok(response) } } } @@ -1552,7 +2179,7 @@ mod tests { // An ephemeral store whose `list`/`list_paginated` always fail. let ephemeral_store: Arc = - Arc::new(DynStoreWrapper(FailingStore::new(FailureMode::List))); + Arc::new(DynStoreWrapper(InstrumentedStore::new(StoreBehavior::FailList))); tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); // A durable key in a namespace that can never hold an ephemeral-cached key. @@ -1704,7 +2331,7 @@ mod tests { let backup_write_attempts = Arc::new(AtomicUsize::new(0)); let backup_store: Arc = - Arc::new(DynStoreWrapper(FailingStore::new(FailureMode::Write { + Arc::new(DynStoreWrapper(InstrumentedStore::new(StoreBehavior::FailWrite { attempts: Arc::clone(&backup_write_attempts), }))); tier.set_backup_store(backup_store); From eb801bb91f1dc725c2a16752cba311696d704bd3 Mon Sep 17 00:00:00 2001 From: Enigbe Date: Tue, 18 Aug 2026 07:37:16 +0100 Subject: [PATCH 05/14] Bind TierStore pagination tokens to their index context The local index store supplies an opaque pagination token, but returning that token directly would allow callers to reuse it with another logical namespace or a different index database. In this commit we wrap the index token in a versioned TierStore token containing the logical namespace identity and persistent index database ID and we validate this context before passing the opaque token back to the index store, rejecting any malformed, unsupported, or mismatched tokens. Assisted-by: Amp (AI coding agent) --- src/io/tier_store.rs | 182 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 174 insertions(+), 8 deletions(-) diff --git a/src/io/tier_store.rs b/src/io/tier_store.rs index 419e6bcc5..84654ec18 100644 --- a/src/io/tier_store.rs +++ b/src/io/tier_store.rs @@ -13,6 +13,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use bitcoin::hashes::{sha256, Hash, HashEngine}; +use bitcoin::hex::{DisplayHex, FromHex}; use lightning::util::persist::{ KVStore, PageToken, PaginatedKVStore, PaginatedListResponse, NETWORK_GRAPH_PERSISTENCE_KEY, NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, SCORER_PERSISTENCE_KEY, @@ -28,6 +29,7 @@ use crate::logger::{LdkLogger, Logger}; use crate::types::{DynStore, DynStoreWrapper}; const INDEX_DATABASE_ID_LEN: usize = 16; +const PAGE_TOKEN_FORMAT_VERSION: u8 = 1; const INDEX_ENTRIES_PRIMARY_NAMESPACE: &str = "_tier_store_entries"; const INDEX_JOURNAL_PRIMARY_NAMESPACE: &str = "_tier_store_journal"; const INDEX_METADATA_PRIMARY_NAMESPACE: &str = "_tier_store_metadata"; @@ -99,9 +101,65 @@ impl JournalEntry { } } +struct TierStorePageToken { + format_version: u8, + index_database_id: Vec, + namespace_id: String, + index_page_token: String, +} + +impl_writeable_tlv_based!(TierStorePageToken, { + (0, format_version, required), + (2, index_database_id, required), + (4, namespace_id, required), + (6, index_page_token, required), +}); + +impl TierStorePageToken { + /// Wraps an index-store page token with the context required to validate its later use. + fn encode( + index_database_id: &[u8; INDEX_DATABASE_ID_LEN], namespace_id: String, + index_page_token: PageToken, + ) -> PageToken { + let token = Self { + format_version: PAGE_TOKEN_FORMAT_VERSION, + index_database_id: index_database_id.to_vec(), + namespace_id, + index_page_token: index_page_token.to_string(), + }; + PageToken::new(Writeable::encode(&token).to_lower_hex_string()) + } + + /// Decodes a TierStore token and rejects tokens issued for another index or namespace. + fn decode( + token: PageToken, expected_database_id: &[u8; INDEX_DATABASE_ID_LEN], + expected_namespace_id: &str, + ) -> io::Result { + let encoded = Vec::from_hex(token.as_str()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "Invalid TierStore page token") + })?; + let mut reader = &*encoded; + let token: Self = Readable::read(&mut reader).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "Invalid TierStore page token") + })?; + if !reader.is_empty() + || token.format_version != PAGE_TOKEN_FORMAT_VERSION + || token.index_database_id != expected_database_id + || token.namespace_id != expected_namespace_id + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "TierStore page token does not belong to this index namespace", + )); + } + Ok(PageToken::new(token.index_page_token)) + } +} + pub(crate) struct TierStoreIndex { // Holding the store keeps its exclusive SQLite lock for the lifetime of the tier store. store: Arc, + database_id: [u8; INDEX_DATABASE_ID_LEN], } impl TierStoreIndex { @@ -113,14 +171,22 @@ impl TierStoreIndex { Some(KV_TABLE_NAME.to_string()), )?; let store: Arc = Arc::new(DynStoreWrapper(store)); - Self::read_or_create_database_id(store.as_ref()).await?; - Ok(Self { store }) + let database_id = Self::read_or_create_database_id(store.as_ref()).await?; + Ok(Self { store, database_id }) } /// Constructs an index over the supplied store for tests that do not need SQLite persistence. #[cfg(test)] fn from_store(store: Arc) -> Self { - Self { store } + Self { store, database_id: [1; INDEX_DATABASE_ID_LEN] } + } + + /// Constructs a test index with a specified database identity. + #[cfg(test)] + fn from_store_with_database_id( + store: Arc, database_id: [u8; INDEX_DATABASE_ID_LEN], + ) -> Self { + Self { store, database_id } } /// Derives the internal secondary namespace for a logical namespace pair. @@ -263,17 +329,25 @@ impl TierStoreIndex { .await } - /// Lists logical keys in the index store's creation order. + /// Lists logical keys in the index store's creation order using a namespace-bound token. async fn list_paginated( &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, ) -> io::Result { - PaginatedKVStore::list_paginated( + let namespace_id = Self::namespace_id(primary_namespace, secondary_namespace); + let index_page_token = page_token + .map(|token| TierStorePageToken::decode(token, &self.database_id, &namespace_id)) + .transpose()?; + let response = PaginatedKVStore::list_paginated( self.store.as_ref(), INDEX_ENTRIES_PRIMARY_NAMESPACE, - &Self::namespace_id(primary_namespace, secondary_namespace), - page_token, + &namespace_id, + index_page_token, ) - .await + .await?; + let next_page_token = response + .next_page_token + .map(|token| TierStorePageToken::encode(&self.database_id, namespace_id, token)); + Ok(PaginatedListResponse { keys: response.keys, next_page_token }) } /// Persists a pending operation before its value-store effects begin. @@ -1351,6 +1425,57 @@ mod tests { } } + #[test] + fn page_token_roundtrips_and_validates_context() { + let database_id = [2; INDEX_DATABASE_ID_LEN]; + let namespace_id = TierStoreIndex::namespace_id("primary", "secondary"); + let token = TierStorePageToken::encode( + &database_id, + namespace_id.clone(), + PageToken::new("opaque:index-token".to_string()), + ); + + let decoded = + TierStorePageToken::decode(token.clone(), &database_id, &namespace_id).unwrap(); + assert_eq!(decoded.as_str(), "opaque:index-token"); + assert_eq!( + TierStorePageToken::decode(token.clone(), &[3; INDEX_DATABASE_ID_LEN], &namespace_id) + .unwrap_err() + .kind(), + io::ErrorKind::InvalidInput + ); + assert_eq!( + TierStorePageToken::decode(token, &database_id, "another-namespace") + .unwrap_err() + .kind(), + io::ErrorKind::InvalidInput + ); + assert_eq!( + TierStorePageToken::decode( + PageToken::new("not-a-tier-store-token".to_string()), + &database_id, + &namespace_id, + ) + .unwrap_err() + .kind(), + io::ErrorKind::InvalidInput + ); + let unsupported_version = TierStorePageToken { + format_version: PAGE_TOKEN_FORMAT_VERSION + 1, + index_database_id: database_id.to_vec(), + namespace_id: namespace_id.clone(), + index_page_token: "opaque:index-token".to_string(), + }; + let unsupported_version = + PageToken::new(Writeable::encode(&unsupported_version).to_lower_hex_string()); + assert_eq!( + TierStorePageToken::decode(unsupported_version, &database_id, &namespace_id) + .unwrap_err() + .kind(), + io::ErrorKind::InvalidInput + ); + } + #[tokio::test] async fn index_store_is_internal_persistent_sqlite_store() { let base_dir = random_storage_path(); @@ -1498,6 +1623,47 @@ mod tests { assert_eq!(KVStore::list(&tier, "namespace", "").await.unwrap().len(), 55); } + #[tokio::test] + async fn paginated_listing_rejects_tokens_from_another_namespace_or_index() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), Arc::clone(&logger)); + let index_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + tier.set_index_store(TierStoreIndex::from_store_with_database_id( + index_store, + [1; INDEX_DATABASE_ID_LEN], + )); + for i in 0..51 { + tier.write("namespace", "", &format!("key-{i:02}"), vec![1]).await.unwrap(); + } + let token = PaginatedKVStore::list_paginated(&tier, "namespace", "", None) + .await + .unwrap() + .next_page_token + .unwrap(); + + let namespace_error = + PaginatedKVStore::list_paginated(&tier, "other-namespace", "", Some(token.clone())) + .await + .unwrap_err(); + assert_eq!(namespace_error.kind(), io::ErrorKind::InvalidInput); + + let replacement_index_store: Arc = + Arc::new(DynStoreWrapper(InMemoryStore::new())); + tier.set_index_store(TierStoreIndex::from_store_with_database_id( + replacement_index_store, + [2; INDEX_DATABASE_ID_LEN], + )); + let index_error = PaginatedKVStore::list_paginated(&tier, "namespace", "", Some(token)) + .await + .unwrap_err(); + assert_eq!(index_error.kind(), io::ErrorKind::InvalidInput); + } + #[tokio::test] async fn pending_creates_roll_forward_to_primary_and_backup_in_journal_order() { let base_dir = random_storage_path(); From c5fb8e95ff50413eb15a5b8dc81c648604d08d1f Mon Sep 17 00:00:00 2001 From: Enigbe Date: Tue, 18 Aug 2026 18:39:03 +0100 Subject: [PATCH 06/14] Migrate existing cache values into ephemeral storage Nodes adopting tiered storage may already hold cache values in primary storage, while a missing index leaves existing ephemeral values without a recoverable position in the cross-store ordering. For this commit, we: - Prepare namespaces on reads as well as writes and listings so first access cannot bypass migration or journal recovery. - Rebuild missing indexes from primary ordering, discard ephemeral values whose ordering cannot be recovered, and move indexed cache values to ephemeral storage without changing their index positions. - Copy values before removing primary and backup copies, and persist a completion marker so interrupted reconciliation can safely resume without repeating it on every access. - Preserve namespace-specific cache routing for future namespace changes. Assisted-by: Amp (AI coding agent) --- src/io/tier_store.rs | 483 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 466 insertions(+), 17 deletions(-) diff --git a/src/io/tier_store.rs b/src/io/tier_store.rs index 84654ec18..86a562c73 100644 --- a/src/io/tier_store.rs +++ b/src/io/tier_store.rs @@ -35,6 +35,7 @@ const INDEX_JOURNAL_PRIMARY_NAMESPACE: &str = "_tier_store_journal"; const INDEX_METADATA_PRIMARY_NAMESPACE: &str = "_tier_store_metadata"; const INDEX_DATABASE_ID_KEY: &str = "index_database_id"; const INDEX_NAMESPACE_READY_KEY_PREFIX: &str = "ready_"; +const INDEX_CACHE_READY_KEY_PREFIX: &str = "cache_ready_"; const INDEX_ENTRY_VALUE: &[u8] = &[1]; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -211,6 +212,15 @@ impl TierStoreIndex { ) } + /// Derives the metadata key recording that indexed cache values occupy the ephemeral tier. + fn cache_ready_key(primary_namespace: &str, secondary_namespace: &str) -> String { + format!( + "{}{}", + INDEX_CACHE_READY_KEY_PREFIX, + Self::namespace_id(primary_namespace, secondary_namespace) + ) + } + /// Encodes the original logical namespace pair for collision detection. fn namespace_metadata(primary_namespace: &str, secondary_namespace: &str) -> Vec { // The fixed five-byte overhead is one format-version byte plus two big-endian u16 @@ -232,13 +242,31 @@ impl TierStoreIndex { async fn is_namespace_ready( &self, primary_namespace: &str, secondary_namespace: &str, ) -> io::Result { - match KVStore::read( - self.store.as_ref(), - INDEX_METADATA_PRIMARY_NAMESPACE, - "", + self.is_namespace_marker_set( &Self::namespace_ready_key(primary_namespace, secondary_namespace), + primary_namespace, + secondary_namespace, + ) + .await + } + + /// Returns whether indexed cache values have been reconciled into ephemeral storage. + async fn is_cache_ready( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result { + self.is_namespace_marker_set( + &Self::cache_ready_key(primary_namespace, secondary_namespace), + primary_namespace, + secondary_namespace, ) .await + } + + async fn is_namespace_marker_set( + &self, marker_key: &str, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result { + match KVStore::read(self.store.as_ref(), INDEX_METADATA_PRIMARY_NAMESPACE, "", marker_key) + .await { Ok(metadata) if metadata == Self::namespace_metadata(primary_namespace, secondary_namespace) => @@ -257,12 +285,35 @@ impl TierStoreIndex { /// Marks the namespace's index as authoritative by persisting its original namespace pair. async fn mark_namespace_ready( &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result<()> { + self.set_namespace_marker( + &Self::namespace_ready_key(primary_namespace, secondary_namespace), + primary_namespace, + secondary_namespace, + ) + .await + } + + /// Marks indexed cache values as reconciled into ephemeral storage. + async fn mark_cache_ready( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result<()> { + self.set_namespace_marker( + &Self::cache_ready_key(primary_namespace, secondary_namespace), + primary_namespace, + secondary_namespace, + ) + .await + } + + async fn set_namespace_marker( + &self, marker_key: &str, primary_namespace: &str, secondary_namespace: &str, ) -> io::Result<()> { KVStore::write( self.store.as_ref(), INDEX_METADATA_PRIMARY_NAMESPACE, "", - &Self::namespace_ready_key(primary_namespace, secondary_namespace), + marker_key, Self::namespace_metadata(primary_namespace, secondary_namespace), ) .await @@ -473,7 +524,9 @@ impl TierStoreIndex { /// Ephemeral data is read from and written to the ephemeral store when configured. /// Namespaces are indexed locally so unpaginated and paginated listings expose the /// same logical contents in cross-tier creation order. Existing primary-store keys -/// are imported into the index before a namespace is first modified or listed. +/// are imported into the index before a namespace is first accessed. Cache values +/// imported from primary storage are then moved to ephemeral storage without changing +/// their index positions. /// /// Note that dual-store writes and removals are not atomic across the primary and /// backup stores. If one store succeeds and the other fails, the operation @@ -865,6 +918,7 @@ impl TierStoreInner { Some(key.as_str()), "read", )?; + self.prepare_namespace(&primary_namespace, &secondary_namespace).await?; if is_ephemeral_cached_key(&primary_namespace, &secondary_namespace, &key) { if let Some(eph_store) = self.ephemeral_store.as_ref() { @@ -1145,8 +1199,10 @@ impl TierStoreInner { /// Imports a namespace's existing primary-store keys and makes its index authoritative. /// /// Primary pagination returns keys from newest to oldest, so the complete result is reversed - /// before insertion to preserve that order in the local index. The readiness marker is written - /// last so a failed import is safely retried before the index can be observed. + /// before insertion to preserve that order in the local index. Values already present in the + /// ephemeral store are discarded because their positions relative to primary keys cannot be + /// reconstructed without the index. Cache values still in the primary store are retained and + /// subsequently moved to ephemeral storage without changing their imported index positions. async fn ensure_namespace_indexed( &self, primary_namespace: &str, secondary_namespace: &str, ) -> io::Result<()> { @@ -1163,6 +1219,8 @@ impl TierStoreInner { if index.is_namespace_ready(primary_namespace, secondary_namespace).await? { Ok(()) } else { + self.discard_unindexed_ephemeral_cache(primary_namespace, secondary_namespace) + .await?; let mut keys = Vec::new(); let mut page_token = None; loop { @@ -1191,12 +1249,138 @@ impl TierStoreInner { result } - /// Initializes a namespace and recovers all pending membership changes before use. + /// Initializes a namespace, recovers pending membership changes, and reconciles cache placement. async fn prepare_namespace( &self, primary_namespace: &str, secondary_namespace: &str, ) -> io::Result<()> { self.ensure_namespace_indexed(primary_namespace, secondary_namespace).await?; - self.recover_namespace(primary_namespace, secondary_namespace).await + self.recover_namespace(primary_namespace, secondary_namespace).await?; + self.ensure_ephemeral_cache_reconciled(primary_namespace, secondary_namespace).await + } + + /// Discards cache values whose ordering cannot be recovered from a missing index. + async fn discard_unindexed_ephemeral_cache( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result<()> { + let Some(ephemeral_store) = self.ephemeral_store.as_ref() else { + return Ok(()); + }; + for key in ephemeral_cache_keys(primary_namespace) { + KVStore::remove( + ephemeral_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + false, + ) + .await?; + } + Ok(()) + } + + /// Reconciles cache placement once and records completion for subsequent namespace accesses. + async fn ensure_ephemeral_cache_reconciled( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result<()> { + let (Some(index), Some(_)) = (self.index.as_ref(), self.ephemeral_store.as_ref()) else { + return Ok(()); + }; + if ephemeral_cache_keys(primary_namespace).next().is_none() + || index.is_cache_ready(primary_namespace, secondary_namespace).await? + { + return Ok(()); + } + + let lock_ref = self.get_index_initialization_lock(primary_namespace, secondary_namespace); + let result: io::Result<()> = async { + let _guard = lock_ref.lock().await; + if index.is_cache_ready(primary_namespace, secondary_namespace).await? { + Ok(()) + } else { + self.reconcile_ephemeral_cache(primary_namespace, secondary_namespace).await?; + index.mark_cache_ready(primary_namespace, secondary_namespace).await + } + } + .await; + self.clean_index_initialization_locks(&lock_ref, primary_namespace, secondary_namespace); + result + } + + /// Moves indexed cache values to ephemeral storage without changing their index positions. + /// + /// Destination writes precede source removals. Repeating this after interruption either copies + /// the primary value again or removes a stale primary/backup copy after finding the destination. + async fn reconcile_ephemeral_cache( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result<()> { + let (Some(index), Some(ephemeral_store)) = + (self.index.as_ref(), self.ephemeral_store.as_ref()) + else { + return Ok(()); + }; + + for key in ephemeral_cache_keys(primary_namespace) { + let locking_key = self.build_locking_key(primary_namespace, secondary_namespace, key); + let lock_ref = self.get_lock_ref(locking_key.clone()); + let result: io::Result<()> = async { + let _guard = lock_ref.lock().await; + self.recover_key_locked(primary_namespace, secondary_namespace, key).await?; + if !index.contains_entry(primary_namespace, secondary_namespace, key).await? { + return Ok(()); + } + + match KVStore::read( + ephemeral_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + ) + .await + { + Ok(_) => {}, + Err(e) if e.kind() == io::ErrorKind::NotFound => { + match self.read_primary(primary_namespace, secondary_namespace, key).await { + Ok(value) => { + KVStore::write( + ephemeral_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + value, + ) + .await?; + }, + Err(e) if e.kind() == io::ErrorKind::NotFound => { + self.remove_primary_backup_async( + primary_namespace, + secondary_namespace, + key, + false, + ) + .await?; + return index + .remove_entry( + primary_namespace, + secondary_namespace, + key, + false, + ) + .await; + }, + Err(e) => return Err(e), + } + }, + Err(e) => return Err(e), + } + + self.remove_primary_backup_async(primary_namespace, secondary_namespace, key, false) + .await + } + .await; + self.clean_locks(&lock_ref, locking_key); + result?; + } + Ok(()) } /// Completes journaled creates and removals before exposing a namespace. @@ -1353,13 +1537,20 @@ impl TierStoreInner { } } -fn is_ephemeral_cached_key(pn: &str, sn: &str, key: &str) -> bool { - matches!( - (pn, sn, key), - (NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, _, NETWORK_GRAPH_PERSISTENCE_KEY) - | (SCORER_PERSISTENCE_PRIMARY_NAMESPACE, _, SCORER_PERSISTENCE_KEY) - | (SCORER_PERSISTENCE_PRIMARY_NAMESPACE, _, EXTERNAL_PATHFINDING_SCORES_CACHE_KEY) - ) +fn is_ephemeral_cached_key(pn: &str, _sn: &str, key: &str) -> bool { + ephemeral_cache_keys(pn).any(|cache_key| cache_key == key) +} + +fn ephemeral_cache_keys(primary_namespace: &str) -> impl Iterator + '_ { + [ + (NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_KEY), + (SCORER_PERSISTENCE_PRIMARY_NAMESPACE, SCORER_PERSISTENCE_KEY), + (SCORER_PERSISTENCE_PRIMARY_NAMESPACE, EXTERNAL_PATHFINDING_SCORES_CACHE_KEY), + ] + .into_iter() + .filter_map(move |(cache_namespace, cache_key)| { + (primary_namespace == cache_namespace).then_some(cache_key) + }) } #[cfg(test)] @@ -1589,6 +1780,264 @@ mod tests { assert_eq!(page.keys, vec!["a".to_string(), "b".to_string()]); } + #[tokio::test] + async fn adopting_ephemeral_storage_migrates_cache_keys_without_reordering_them() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + for key in ["old", NETWORK_GRAPH_PERSISTENCE_KEY, "new"] { + primary_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + key, + key.as_bytes().to_vec(), + ) + .await + .unwrap(); + backup_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + key, + key.as_bytes().to_vec(), + ) + .await + .unwrap(); + } + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.set_backup_store(Arc::clone(&backup_store)); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + let page = PaginatedKVStore::list_paginated( + &tier, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + None, + ) + .await + .unwrap(); + assert_eq!( + page.keys, + vec!["new".to_string(), NETWORK_GRAPH_PERSISTENCE_KEY.to_string(), "old".to_string(),] + ); + assert_eq!( + ephemeral_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .unwrap(), + NETWORK_GRAPH_PERSISTENCE_KEY.as_bytes() + ); + assert!(primary_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err()); + assert!(backup_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err()); + } + + #[tokio::test] + async fn missing_index_discards_ephemeral_only_cache_data_before_reading() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + ephemeral_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![1], + ) + .await + .unwrap(); + let mut tier = setup_tier_store(primary_store, logger); + set_test_index_store(&mut tier); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + assert!(tier + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err()); + assert!(ephemeral_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err()); + assert!(KVStore::list( + &tier, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await + .unwrap() + .is_empty()); + } + + #[tokio::test] + async fn missing_index_rebuilds_cache_from_primary_instead_of_ephemeral() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + primary_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![2], + ) + .await + .unwrap(); + ephemeral_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![1], + ) + .await + .unwrap(); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + assert_eq!( + tier.read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .unwrap(), + vec![2] + ); + assert!(primary_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err()); + assert!(tier + .inner + .index + .as_ref() + .unwrap() + .contains_entry( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .unwrap()); + } + + #[tokio::test] + async fn namespace_preparation_finishes_interrupted_cache_migration() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + let index = tier.inner.index.as_ref().unwrap(); + for key in ["old", NETWORK_GRAPH_PERSISTENCE_KEY, "new"] { + index + .write_entry( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + key, + ) + .await + .unwrap(); + } + index + .mark_namespace_ready( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await + .unwrap(); + primary_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![1], + ) + .await + .unwrap(); + ephemeral_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![1], + ) + .await + .unwrap(); + + let page = PaginatedKVStore::list_paginated( + &tier, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + None, + ) + .await + .unwrap(); + assert_eq!( + page.keys, + vec!["new".to_string(), NETWORK_GRAPH_PERSISTENCE_KEY.to_string(), "old".to_string(),] + ); + assert!(primary_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err()); + } + #[tokio::test] async fn namespace_initialization_preserves_existing_primary_order_across_pages() { let base_dir = random_storage_path(); From 3a99fce8112c049463ed3373a24185253cd690e5 Mon Sep 17 00:00:00 2001 From: Enigbe Date: Mon, 24 Aug 2026 11:13:03 +0100 Subject: [PATCH 07/14] Isolate TierStore read recovery by key Preparing an entire namespace before every read allows one unrecoverable journal entry to block reads of unrelated keys. Cache migration has similar coupling because readiness is recorded for the whole namespace and all cache keys are reconciled together. In this commit, we: - Initialize the ordering index before reads, then hold the requested key's operation lock continuously while recovering its journal entry, reconciling its cache placement, and reading its value. - Track cache readiness per logical key and retain the original key identity as metadata to detect hash collisions. - Keep writes, removals, and listings on the existing namespace-wide preparation path for now (addressed in follow up). - Add coverage for unrelated pending operations, pending removals, independent cache migration, and lock re-entry deadlocks. Assisted-by: Amp (AI coding agent) --- src/io/tier_store.rs | 508 ++++++++++++++++++++++++++++++++----------- 1 file changed, 379 insertions(+), 129 deletions(-) diff --git a/src/io/tier_store.rs b/src/io/tier_store.rs index 86a562c73..90417309e 100644 --- a/src/io/tier_store.rs +++ b/src/io/tier_store.rs @@ -212,12 +212,24 @@ impl TierStoreIndex { ) } - /// Derives the metadata key recording that indexed cache values occupy the ephemeral tier. - fn cache_ready_key(primary_namespace: &str, secondary_namespace: &str) -> String { + /// Derives the internal identity for one logical cache key. + fn cache_id(primary_namespace: &str, secondary_namespace: &str, key: &str) -> String { + let mut engine = sha256::Hash::engine(); + engine.input(&(primary_namespace.len() as u64).to_be_bytes()); + engine.input(primary_namespace.as_bytes()); + engine.input(&(secondary_namespace.len() as u64).to_be_bytes()); + engine.input(secondary_namespace.as_bytes()); + engine.input(&(key.len() as u64).to_be_bytes()); + engine.input(key.as_bytes()); + sha256::Hash::from_engine(engine).to_string() + } + + /// Derives the metadata key recording that one indexed cache value occupies the ephemeral tier. + fn cache_ready_key(primary_namespace: &str, secondary_namespace: &str, key: &str) -> String { format!( "{}{}", INDEX_CACHE_READY_KEY_PREFIX, - Self::namespace_id(primary_namespace, secondary_namespace) + Self::cache_id(primary_namespace, secondary_namespace, key) ) } @@ -235,6 +247,22 @@ impl TierStoreIndex { metadata } + /// Encodes the original logical cache-key identity for collision detection. + fn cache_metadata(primary_namespace: &str, secondary_namespace: &str, key: &str) -> Vec { + // The fixed seven-byte overhead is one format-version byte plus three big-endian u16 + // component-length prefixes: 1 + 2 + 2 + 2 = 7. + let mut metadata = + Vec::with_capacity(7 + primary_namespace.len() + secondary_namespace.len() + key.len()); + metadata.push(1); + metadata.extend_from_slice(&(primary_namespace.len() as u16).to_be_bytes()); + metadata.extend_from_slice(primary_namespace.as_bytes()); + metadata.extend_from_slice(&(secondary_namespace.len() as u16).to_be_bytes()); + metadata.extend_from_slice(secondary_namespace.as_bytes()); + metadata.extend_from_slice(&(key.len() as u16).to_be_bytes()); + metadata.extend_from_slice(key.as_bytes()); + metadata + } + /// Returns whether the namespace's index is authoritative for listing. /// /// Returns an error if the stored namespace metadata does not match the requested namespace, @@ -242,41 +270,34 @@ impl TierStoreIndex { async fn is_namespace_ready( &self, primary_namespace: &str, secondary_namespace: &str, ) -> io::Result { - self.is_namespace_marker_set( + self.is_marker_set( &Self::namespace_ready_key(primary_namespace, secondary_namespace), - primary_namespace, - secondary_namespace, + Self::namespace_metadata(primary_namespace, secondary_namespace), ) .await } - /// Returns whether indexed cache values have been reconciled into ephemeral storage. + /// Returns whether one indexed cache value has been reconciled into ephemeral storage. async fn is_cache_ready( - &self, primary_namespace: &str, secondary_namespace: &str, + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, ) -> io::Result { - self.is_namespace_marker_set( - &Self::cache_ready_key(primary_namespace, secondary_namespace), - primary_namespace, - secondary_namespace, + self.is_marker_set( + &Self::cache_ready_key(primary_namespace, secondary_namespace, key), + Self::cache_metadata(primary_namespace, secondary_namespace, key), ) .await } - async fn is_namespace_marker_set( - &self, marker_key: &str, primary_namespace: &str, secondary_namespace: &str, + async fn is_marker_set( + &self, marker_key: &str, expected_metadata: Vec, ) -> io::Result { match KVStore::read(self.store.as_ref(), INDEX_METADATA_PRIMARY_NAMESPACE, "", marker_key) .await { - Ok(metadata) - if metadata == Self::namespace_metadata(primary_namespace, secondary_namespace) => - { - Ok(true) + Ok(metadata) if metadata == expected_metadata => Ok(true), + Ok(_) => { + Err(io::Error::new(io::ErrorKind::InvalidData, "Tier-store index marker collision")) }, - Ok(_) => Err(io::Error::new( - io::ErrorKind::InvalidData, - "Tier-store index namespace collision", - )), Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), Err(e) => Err(e), } @@ -286,35 +307,31 @@ impl TierStoreIndex { async fn mark_namespace_ready( &self, primary_namespace: &str, secondary_namespace: &str, ) -> io::Result<()> { - self.set_namespace_marker( + self.set_marker( &Self::namespace_ready_key(primary_namespace, secondary_namespace), - primary_namespace, - secondary_namespace, + Self::namespace_metadata(primary_namespace, secondary_namespace), ) .await } - /// Marks indexed cache values as reconciled into ephemeral storage. + /// Marks one indexed cache value as reconciled into ephemeral storage. async fn mark_cache_ready( - &self, primary_namespace: &str, secondary_namespace: &str, + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, ) -> io::Result<()> { - self.set_namespace_marker( - &Self::cache_ready_key(primary_namespace, secondary_namespace), - primary_namespace, - secondary_namespace, + self.set_marker( + &Self::cache_ready_key(primary_namespace, secondary_namespace, key), + Self::cache_metadata(primary_namespace, secondary_namespace, key), ) .await } - async fn set_namespace_marker( - &self, marker_key: &str, primary_namespace: &str, secondary_namespace: &str, - ) -> io::Result<()> { + async fn set_marker(&self, marker_key: &str, metadata: Vec) -> io::Result<()> { KVStore::write( self.store.as_ref(), INDEX_METADATA_PRIMARY_NAMESPACE, "", marker_key, - Self::namespace_metadata(primary_namespace, secondary_namespace), + metadata, ) .await } @@ -918,23 +935,39 @@ impl TierStoreInner { Some(key.as_str()), "read", )?; - self.prepare_namespace(&primary_namespace, &secondary_namespace).await?; + self.ensure_namespace_indexed(&primary_namespace, &secondary_namespace).await?; - if is_ephemeral_cached_key(&primary_namespace, &secondary_namespace, &key) { - if let Some(eph_store) = self.ephemeral_store.as_ref() { - // We don't retry ephemeral-store reads here. Local failures are treated as - // terminal for this access path rather than falling back to another store. - return KVStore::read( - eph_store.as_ref(), - &primary_namespace, - &secondary_namespace, - &key, - ) - .await; + let locking_key = self.build_locking_key(&primary_namespace, &secondary_namespace, &key); + let lock_ref = self.get_lock_ref(locking_key.clone()); + let result: io::Result> = async { + let _guard = lock_ref.lock().await; + self.recover_key_locked(&primary_namespace, &secondary_namespace, &key).await?; + self.reconcile_ephemeral_cache_key_locked( + &primary_namespace, + &secondary_namespace, + &key, + ) + .await?; + + if is_ephemeral_cached_key(&primary_namespace, &secondary_namespace, &key) { + if let Some(eph_store) = self.ephemeral_store.as_ref() { + // We don't retry ephemeral-store reads here. Local failures are treated as + // terminal for this access path rather than falling back to another store. + return KVStore::read( + eph_store.as_ref(), + &primary_namespace, + &secondary_namespace, + &key, + ) + .await; + } } - } - self.read_primary(&primary_namespace, &secondary_namespace, &key).await + self.read_primary(&primary_namespace, &secondary_namespace, &key).await + } + .await; + self.clean_locks(&lock_ref, locking_key); + result } async fn write_internal( @@ -1278,44 +1311,11 @@ impl TierStoreInner { Ok(()) } - /// Reconciles cache placement once and records completion for subsequent namespace accesses. + /// Reconciles every cache key before an operation that prepares the complete namespace. async fn ensure_ephemeral_cache_reconciled( &self, primary_namespace: &str, secondary_namespace: &str, ) -> io::Result<()> { - let (Some(index), Some(_)) = (self.index.as_ref(), self.ephemeral_store.as_ref()) else { - return Ok(()); - }; - if ephemeral_cache_keys(primary_namespace).next().is_none() - || index.is_cache_ready(primary_namespace, secondary_namespace).await? - { - return Ok(()); - } - - let lock_ref = self.get_index_initialization_lock(primary_namespace, secondary_namespace); - let result: io::Result<()> = async { - let _guard = lock_ref.lock().await; - if index.is_cache_ready(primary_namespace, secondary_namespace).await? { - Ok(()) - } else { - self.reconcile_ephemeral_cache(primary_namespace, secondary_namespace).await?; - index.mark_cache_ready(primary_namespace, secondary_namespace).await - } - } - .await; - self.clean_index_initialization_locks(&lock_ref, primary_namespace, secondary_namespace); - result - } - - /// Moves indexed cache values to ephemeral storage without changing their index positions. - /// - /// Destination writes precede source removals. Repeating this after interruption either copies - /// the primary value again or removes a stale primary/backup copy after finding the destination. - async fn reconcile_ephemeral_cache( - &self, primary_namespace: &str, secondary_namespace: &str, - ) -> io::Result<()> { - let (Some(index), Some(ephemeral_store)) = - (self.index.as_ref(), self.ephemeral_store.as_ref()) - else { + let (Some(_), Some(_)) = (self.index.as_ref(), self.ephemeral_store.as_ref()) else { return Ok(()); }; @@ -1325,56 +1325,12 @@ impl TierStoreInner { let result: io::Result<()> = async { let _guard = lock_ref.lock().await; self.recover_key_locked(primary_namespace, secondary_namespace, key).await?; - if !index.contains_entry(primary_namespace, secondary_namespace, key).await? { - return Ok(()); - } - - match KVStore::read( - ephemeral_store.as_ref(), + self.reconcile_ephemeral_cache_key_locked( primary_namespace, secondary_namespace, key, ) .await - { - Ok(_) => {}, - Err(e) if e.kind() == io::ErrorKind::NotFound => { - match self.read_primary(primary_namespace, secondary_namespace, key).await { - Ok(value) => { - KVStore::write( - ephemeral_store.as_ref(), - primary_namespace, - secondary_namespace, - key, - value, - ) - .await?; - }, - Err(e) if e.kind() == io::ErrorKind::NotFound => { - self.remove_primary_backup_async( - primary_namespace, - secondary_namespace, - key, - false, - ) - .await?; - return index - .remove_entry( - primary_namespace, - secondary_namespace, - key, - false, - ) - .await; - }, - Err(e) => return Err(e), - } - }, - Err(e) => return Err(e), - } - - self.remove_primary_backup_async(primary_namespace, secondary_namespace, key, false) - .await } .await; self.clean_locks(&lock_ref, locking_key); @@ -1383,6 +1339,74 @@ impl TierStoreInner { Ok(()) } + /// Moves one indexed cache value to ephemeral storage while its per-key lock is held. + /// + /// Destination writes precede source removals. Repeating this after interruption either copies + /// the primary value again or removes a stale primary/backup copy after finding the destination. + async fn reconcile_ephemeral_cache_key_locked( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result<()> { + let (Some(index), Some(ephemeral_store)) = + (self.index.as_ref(), self.ephemeral_store.as_ref()) + else { + return Ok(()); + }; + if !is_ephemeral_cached_key(primary_namespace, secondary_namespace, key) + || index.is_cache_ready(primary_namespace, secondary_namespace, key).await? + { + return Ok(()); + } + + if index.contains_entry(primary_namespace, secondary_namespace, key).await? { + match KVStore::read( + ephemeral_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + ) + .await + { + Ok(_) => {}, + Err(e) if e.kind() == io::ErrorKind::NotFound => { + match self.read_primary(primary_namespace, secondary_namespace, key).await { + Ok(value) => { + KVStore::write( + ephemeral_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + value, + ) + .await?; + }, + Err(e) if e.kind() == io::ErrorKind::NotFound => { + self.remove_primary_backup_async( + primary_namespace, + secondary_namespace, + key, + false, + ) + .await?; + index + .remove_entry(primary_namespace, secondary_namespace, key, false) + .await?; + return index + .mark_cache_ready(primary_namespace, secondary_namespace, key) + .await; + }, + Err(e) => return Err(e), + } + }, + Err(e) => return Err(e), + } + + self.remove_primary_backup_async(primary_namespace, secondary_namespace, key, false) + .await?; + } + + index.mark_cache_ready(primary_namespace, secondary_namespace, key).await + } + /// Completes journaled creates and removals before exposing a namespace. async fn recover_namespace( &self, primary_namespace: &str, secondary_namespace: &str, @@ -2038,6 +2062,89 @@ mod tests { .is_err()); } + #[tokio::test] + async fn reading_one_cache_key_only_reconciles_that_key() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + for (key, value) in + [(SCORER_PERSISTENCE_KEY, vec![1]), (EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, vec![2])] + { + primary_store + .write( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + key, + value, + ) + .await + .unwrap(); + } + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + assert_eq!( + tier.read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + SCORER_PERSISTENCE_KEY, + ) + .await + .unwrap(), + vec![1] + ); + assert!(primary_store + .read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + SCORER_PERSISTENCE_KEY, + ) + .await + .is_err()); + assert!(ephemeral_store + .read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + ) + .await + .is_err()); + assert_eq!( + primary_store + .read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + ) + .await + .unwrap(), + vec![2] + ); + + let index = tier.inner.index.as_ref().unwrap(); + assert!(index + .is_cache_ready( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + SCORER_PERSISTENCE_KEY, + ) + .await + .unwrap()); + assert!(!index + .is_cache_ready( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + ) + .await + .unwrap()); + } + #[tokio::test] async fn namespace_initialization_preserves_existing_primary_order_across_pages() { let base_dir = random_storage_path(); @@ -2200,6 +2307,149 @@ mod tests { assert!(index.read_journal_entry("namespace", "", "key").await.is_err()); } + #[tokio::test] + async fn read_recovers_only_the_requested_key() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + for (key, value) in [("stuck", vec![1]), ("readable", vec![2])] { + primary_store.write("namespace", "", key, value).await.unwrap(); + } + let mut tier = setup_tier_store(primary_store, logger); + set_test_index_store(&mut tier); + tier.inner.ensure_namespace_indexed("namespace", "").await.unwrap(); + let pending = JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: "stuck".to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation: JournalOperation::Remove { lazy: false }, + }; + let index = tier.inner.index.as_ref().unwrap(); + index.write_journal_entry(&pending).await.unwrap(); + + assert_eq!(tier.read("namespace", "", "readable").await.unwrap(), vec![2]); + assert!(index.read_journal_entry("namespace", "", "stuck").await.is_ok()); + assert!(tier.read("namespace", "", "stuck").await.is_err()); + } + + #[tokio::test] + async fn read_recovers_pending_removal_before_returning_value() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + tier.set_backup_store(Arc::clone(&backup_store)); + set_test_index_store(&mut tier); + tier.write("namespace", "", "key", vec![1]).await.unwrap(); + + let pending = JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: "key".to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation: JournalOperation::Remove { lazy: false }, + }; + let index = tier.inner.index.as_ref().unwrap(); + index.write_journal_entry(&pending).await.unwrap(); + index.remove_entry("namespace", "", "key", false).await.unwrap(); + + let error = tier.read("namespace", "", "key").await.unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::NotFound); + assert!(primary_store.read("namespace", "", "key").await.is_err()); + assert!(backup_store.read("namespace", "", "key").await.is_err()); + assert!(index.read_journal_entry("namespace", "", "key").await.is_err()); + } + + #[tokio::test] + async fn read_recovers_pending_cache_operation_without_relocking_the_key() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + primary_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![1], + ) + .await + .unwrap(); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + tier.inner + .ensure_namespace_indexed( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await + .unwrap(); + let pending = JournalEntry { + primary_namespace: NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE.to_string(), + secondary_namespace: NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE.to_string(), + key: NETWORK_GRAPH_PERSISTENCE_KEY.to_string(), + tier: ValueTier::Primary, + requires_backup: false, + operation: JournalOperation::Create { value: vec![2] }, + }; + let index = tier.inner.index.as_ref().unwrap(); + index.write_journal_entry(&pending).await.unwrap(); + + let value = tokio::time::timeout( + std::time::Duration::from_secs(5), + tier.read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ), + ) + .await + .expect("read deadlocked while preparing its cache key") + .unwrap(); + assert_eq!(value, vec![2]); + assert!(primary_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err()); + assert_eq!( + ephemeral_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .unwrap(), + vec![2] + ); + assert!(index + .read_journal_entry( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err()); + } + #[tokio::test] async fn failed_create_retains_journal_without_exposing_index_entry() { let base_dir = random_storage_path(); From 2df10c0d867fe33d361b6641d0e13c56077637b1 Mon Sep 17 00:00:00 2001 From: Enigbe Date: Mon, 24 Aug 2026 11:39:06 +0100 Subject: [PATCH 08/14] Isolate TierStore mutations by key Preparing an entire namespace before every write or removal allows one unrecoverable journal entry to block mutations of unrelated keys sharing that namespace. In this commit, we: - Initialize the ordering index before mutations, then recover journal state and reconcile cache placement for only the requested key while holding its existing operation lock. - Reserve complete namespace recovery and cache reconciliation for listing operations. - Remove the obsolete journal-list snapshot gate and add coverage showing that a stuck key does not block unrelated writes or removals, while listings retain their deliberate fail-fast behavior. - Verify that touching one cache key no longer migrates another cache key in the same namespace. Assisted-by: Amp (AI coding agent) --- src/io/tier_store.rs | 218 +++++++++++++++++++++++++++---------------- 1 file changed, 135 insertions(+), 83 deletions(-) diff --git a/src/io/tier_store.rs b/src/io/tier_store.rs index 90417309e..65d1e6546 100644 --- a/src/io/tier_store.rs +++ b/src/io/tier_store.rs @@ -941,13 +941,7 @@ impl TierStoreInner { let lock_ref = self.get_lock_ref(locking_key.clone()); let result: io::Result> = async { let _guard = lock_ref.lock().await; - self.recover_key_locked(&primary_namespace, &secondary_namespace, &key).await?; - self.reconcile_ephemeral_cache_key_locked( - &primary_namespace, - &secondary_namespace, - &key, - ) - .await?; + self.prepare_key_locked(&primary_namespace, &secondary_namespace, &key).await?; if is_ephemeral_cached_key(&primary_namespace, &secondary_namespace, &key) { if let Some(eph_store) = self.ephemeral_store.as_ref() { @@ -980,7 +974,7 @@ impl TierStoreInner { Some(key.as_str()), "write", )?; - self.prepare_namespace(&primary_namespace, &secondary_namespace).await?; + self.ensure_namespace_indexed(&primary_namespace, &secondary_namespace).await?; self.execute_locked_write(lock_ref, locking_key, version, || async move { self.write_locked(&primary_namespace, &secondary_namespace, &key, buf).await @@ -988,11 +982,11 @@ impl TierStoreInner { .await } - /// Writes one key after recovering any pending operation while its per-key lock is held. + /// Prepares and writes one key while its per-key operation lock is held. async fn write_locked( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, value: Vec, ) -> io::Result<()> { - self.recover_key_locked(primary_namespace, secondary_namespace, key).await?; + self.prepare_key_locked(primary_namespace, secondary_namespace, key).await?; let tier = self.value_tier(primary_namespace, secondary_namespace, key); let Some(index) = self.index.as_ref() else { return self @@ -1035,7 +1029,7 @@ impl TierStoreInner { Some(key.as_str()), "remove", )?; - self.prepare_namespace(&primary_namespace, &secondary_namespace).await?; + self.ensure_namespace_indexed(&primary_namespace, &secondary_namespace).await?; self.execute_locked_write(lock_ref, locking_key, version, || async move { self.remove_locked(&primary_namespace, &secondary_namespace, &key, lazy).await @@ -1043,11 +1037,11 @@ impl TierStoreInner { .await } - /// Removes one key after recovering any pending operation while its per-key lock is held. + /// Prepares and removes one key while its per-key operation lock is held. async fn remove_locked( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, ) -> io::Result<()> { - self.recover_key_locked(primary_namespace, secondary_namespace, key).await?; + self.prepare_key_locked(primary_namespace, secondary_namespace, key).await?; let tier = self.value_tier(primary_namespace, secondary_namespace, key); let Some(index) = self.index.as_ref() else { return self @@ -1221,7 +1215,7 @@ impl TierStoreInner { "list", )?; - self.prepare_namespace(&primary_namespace, &secondary_namespace).await?; + self.prepare_namespace_for_listing(&primary_namespace, &secondary_namespace).await?; if let Some(index) = self.index.as_ref() { return index.list(&primary_namespace, &secondary_namespace).await; } @@ -1282,8 +1276,8 @@ impl TierStoreInner { result } - /// Initializes a namespace, recovers pending membership changes, and reconciles cache placement. - async fn prepare_namespace( + /// Prepares a complete namespace before exposing its index through a listing. + async fn prepare_namespace_for_listing( &self, primary_namespace: &str, secondary_namespace: &str, ) -> io::Result<()> { self.ensure_namespace_indexed(primary_namespace, secondary_namespace).await?; @@ -1324,13 +1318,7 @@ impl TierStoreInner { let lock_ref = self.get_lock_ref(locking_key.clone()); let result: io::Result<()> = async { let _guard = lock_ref.lock().await; - self.recover_key_locked(primary_namespace, secondary_namespace, key).await?; - self.reconcile_ephemeral_cache_key_locked( - primary_namespace, - secondary_namespace, - key, - ) - .await + self.prepare_key_locked(primary_namespace, secondary_namespace, key).await } .await; self.clean_locks(&lock_ref, locking_key); @@ -1407,6 +1395,14 @@ impl TierStoreInner { index.mark_cache_ready(primary_namespace, secondary_namespace, key).await } + /// Recovers and reconciles one key while its per-key operation lock is held by the caller. + async fn prepare_key_locked( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result<()> { + self.recover_key_locked(primary_namespace, secondary_namespace, key).await?; + self.reconcile_ephemeral_cache_key_locked(primary_namespace, secondary_namespace, key).await + } + /// Completes journaled creates and removals before exposing a namespace. async fn recover_namespace( &self, primary_namespace: &str, secondary_namespace: &str, @@ -1496,7 +1492,7 @@ impl TierStoreInner { "list_paginated", )?; - self.prepare_namespace(&primary_namespace, &secondary_namespace).await?; + self.prepare_namespace_for_listing(&primary_namespace, &secondary_namespace).await?; if let Some(index) = self.index.as_ref() { return index .list_paginated(&primary_namespace, &secondary_namespace, page_token) @@ -1583,7 +1579,7 @@ mod tests { use std::panic::RefUnwindSafe; use std::path::PathBuf; use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::{Arc, Mutex}; + use std::sync::Arc; use lightning::util::logger::Level; use lightning::util::persist::{ @@ -1594,7 +1590,6 @@ mod tests { NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, SCORER_PERSISTENCE_SECONDARY_NAMESPACE, }; use lightning_persister::fs_store::v2::FilesystemStoreV2; - use tokio::sync::oneshot; use super::*; use crate::io::test_utils::{ @@ -2145,6 +2140,80 @@ mod tests { .unwrap()); } + #[tokio::test] + async fn writing_one_cache_key_only_reconciles_that_key() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + for (key, value) in + [(SCORER_PERSISTENCE_KEY, vec![1]), (EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, vec![2])] + { + primary_store + .write( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + key, + value, + ) + .await + .unwrap(); + } + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + tier.write( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + SCORER_PERSISTENCE_KEY, + vec![3], + ) + .await + .unwrap(); + assert!(primary_store + .read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + SCORER_PERSISTENCE_KEY, + ) + .await + .is_err()); + assert_eq!( + ephemeral_store + .read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + SCORER_PERSISTENCE_KEY, + ) + .await + .unwrap(), + vec![3] + ); + assert!(ephemeral_store + .read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + ) + .await + .is_err()); + assert_eq!( + primary_store + .read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + ) + .await + .unwrap(), + vec![2] + ); + } + #[tokio::test] async fn namespace_initialization_preserves_existing_primary_order_across_pages() { let base_dir = random_storage_path(); @@ -2337,6 +2406,40 @@ mod tests { assert!(tier.read("namespace", "", "stuck").await.is_err()); } + #[tokio::test] + async fn writes_and_removals_recover_only_the_requested_key() { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let _cleanup = CleanupDir(base_dir); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + for (key, value) in [("stuck", vec![1]), ("writable", vec![2]), ("removable", vec![3])] { + primary_store.write("namespace", "", key, value).await.unwrap(); + } + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.inner.ensure_namespace_indexed("namespace", "").await.unwrap(); + let pending = JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: "stuck".to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation: JournalOperation::Remove { lazy: false }, + }; + let index = tier.inner.index.as_ref().unwrap(); + index.write_journal_entry(&pending).await.unwrap(); + + tier.write("namespace", "", "writable", vec![4]).await.unwrap(); + tier.remove("namespace", "", "removable", false).await.unwrap(); + + assert_eq!(primary_store.read("namespace", "", "writable").await.unwrap(), vec![4]); + assert!(primary_store.read("namespace", "", "removable").await.is_err()); + assert!(index.read_journal_entry("namespace", "", "stuck").await.is_ok()); + assert!(KVStore::list(&tier, "namespace", "").await.is_err()); + } + #[tokio::test] async fn read_recovers_pending_removal_before_returning_value() { let base_dir = random_storage_path(); @@ -2479,34 +2582,18 @@ mod tests { } #[tokio::test] - async fn queued_write_recovers_pending_create_under_key_lock_before_classifying() { + async fn write_recovers_pending_create_under_key_lock_before_classifying() { let base_dir = random_storage_path(); let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); let _cleanup = CleanupDir(base_dir); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); - let (snapshot_taken_tx, snapshot_taken_rx) = oneshot::channel(); - let (allow_return_tx, allow_return_rx) = oneshot::channel(); - let index_store: Arc = - Arc::new(DynStoreWrapper(InstrumentedStore::new(StoreBehavior::GateJournalList { - snapshot_taken: Mutex::new(Some(snapshot_taken_tx)), - allow_return: Mutex::new(Some(allow_return_rx)), - }))); + let index_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); tier.set_index_store(TierStoreIndex::from_store(index_store)); tier.inner.ensure_namespace_indexed("namespace", "").await.unwrap(); - let tier = Arc::new(tier); - let locking_key = tier.inner.build_locking_key("namespace", "", "key"); - let lock_ref = tier.inner.get_lock_ref(locking_key); - let guard = lock_ref.lock().await; - let write_task = { - let tier = Arc::clone(&tier); - tokio::spawn(async move { tier.write("namespace", "", "key", vec![2]).await }) - }; - snapshot_taken_rx.await.unwrap(); - let pending = JournalEntry { primary_namespace: "namespace".to_string(), secondary_namespace: String::new(), @@ -2518,11 +2605,8 @@ mod tests { let index = tier.inner.index.as_ref().unwrap(); index.write_journal_entry(&pending).await.unwrap(); primary_store.write("namespace", "", "key", vec![1]).await.unwrap(); - allow_return_tx.send(()).unwrap(); - drop(guard); - drop(lock_ref); - write_task.await.unwrap().unwrap(); + tier.write("namespace", "", "key", vec![2]).await.unwrap(); assert_eq!(primary_store.read("namespace", "", "key").await.unwrap(), vec![2]); assert!(index.contains_entry("namespace", "", "key").await.unwrap()); assert!(index.read_journal_entry("namespace", "", "key").await.is_err()); @@ -2579,14 +2663,8 @@ mod tests { enum StoreBehavior { FailList, - FailWrite { - attempts: Arc, - }, + FailWrite { attempts: Arc }, FailRemove, - GateJournalList { - snapshot_taken: Mutex>>, - allow_return: Mutex>>, - }, } /// A store that injects selected failures or synchronization points while delegating other @@ -2649,9 +2727,7 @@ mod tests { ) -> impl Future, io::Error>> + 'static + Send { let list = match &self.behavior { StoreBehavior::FailList => None, - StoreBehavior::FailWrite { .. } - | StoreBehavior::FailRemove - | StoreBehavior::GateJournalList { .. } => { + StoreBehavior::FailWrite { .. } | StoreBehavior::FailRemove => { Some(KVStore::list(&self.inner, primary_namespace, secondary_namespace)) }, }; @@ -2670,23 +2746,6 @@ mod tests { page_token: Option, ) -> impl Future> + 'static + Send { let fails = matches!(&self.behavior, StoreBehavior::FailList); - let gate = match &self.behavior { - StoreBehavior::GateJournalList { snapshot_taken, allow_return } => { - if primary_namespace == INDEX_JOURNAL_PRIMARY_NAMESPACE { - let snapshot_taken = snapshot_taken.lock().unwrap().take(); - let allow_return = allow_return.lock().unwrap().take(); - match (snapshot_taken, allow_return) { - (Some(snapshot_taken), Some(allow_return)) => { - Some((snapshot_taken, allow_return)) - }, - _ => None, - } - } else { - None - } - }, - _ => None, - }; let list = PaginatedKVStore::list_paginated( &self.inner, primary_namespace, @@ -2697,14 +2756,7 @@ mod tests { if fails { return Err(io::Error::new(io::ErrorKind::Other, "list_paginated failed")); } - let response = list.await?; - if let Some((snapshot_taken, allow_return)) = gate { - let _ = snapshot_taken.send(()); - allow_return.await.map_err(|_| { - io::Error::new(io::ErrorKind::Other, "journal-list gate was dropped") - })?; - }; - Ok(response) + list.await } } } From ed08cd26c2dfdd2c2c249eccdae79e5aacebbfe2 Mon Sep 17 00:00:00 2001 From: Enigbe Date: Tue, 7 Apr 2026 19:17:22 +0100 Subject: [PATCH 09/14] Integrate TierStore into NodeBuilder TierStore remains internal until NodeBuilder provisions and installs its storage backends. Add builder options for local ephemeral and backup SQLite stores, wrap the configured primary store in TierStore, and pass the resulting store into node construction. When ephemeral storage is enabled, automatically create the persistent ordering index in the node's storage directory and require TierStore to own it exclusively. Update filesystem-backed tests and add integration coverage confirming that configured backup storage receives durable primary-backed data. Assisted-by: Amp (AI coding agent) --- src/builder.rs | 96 ++++++++++++++++++++++++++++++++- src/io/sqlite_store/mod.rs | 4 ++ src/io/tier_store.rs | 1 - tests/common/mod.rs | 6 +-- tests/integration_tests_rust.rs | 73 +++++++++++++++++++++++++ 5 files changed, 174 insertions(+), 6 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index ab641cf86..7077a9622 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -72,6 +72,7 @@ use crate::gossip::GossipSource; use crate::io::fs_store::open_or_migrate_fs_store; #[cfg(feature = "storage-sqlite")] use crate::io::sqlite_store::SqliteStore; +use crate::io::tier_store::{setup_index_store, TierStore}; use crate::io::utils::{ read_all_objects, read_event_queue, read_external_pathfinding_scores_from_cache, read_n_objects, read_network_graph, read_node_metrics, read_output_sweeper, read_peer_info, @@ -173,6 +174,12 @@ impl std::fmt::Debug for LogWriterConfig { } } +#[derive(Default, Debug)] +struct TierStoreConfig { + ephemeral_storage_dir_path: Option, + backup_storage_dir_path: Option, +} + /// An error encountered during building a [`Node`]. /// /// [`Node`]: crate::Node @@ -326,6 +333,7 @@ pub struct NodeBuilder { liquidity_source_config: Option, log_writer_config: Option, async_payments_role: Option, + tier_store_config: Option, runtime_handle: Option, pathfinding_scores_sync_config: Option, probing_config: Option, @@ -347,6 +355,7 @@ impl NodeBuilder { let gossip_source_config = None; let liquidity_source_config = None; let log_writer_config = None; + let tier_store_config = None; let runtime_handle = None; let pathfinding_scores_sync_config = None; let probing_config = None; @@ -356,6 +365,7 @@ impl NodeBuilder { gossip_source_config, liquidity_source_config, log_writer_config, + tier_store_config, runtime_handle, async_payments_role: None, pathfinding_scores_sync_config, @@ -686,6 +696,41 @@ impl NodeBuilder { self } + /// Configures a local SQLite backup store for disaster recovery. + /// + /// When building with tiered storage, a SQLite store will be created at the + /// given directory path using [`SQLITE_BACKUP_DB_FILE_NAME`] as its database + /// file name. It receives a second durable copy of data written to the + /// primary store. + /// + /// Writes and removals for primary-backed data only succeed once both the + /// primary and backup SQLite stores complete successfully. + /// + /// If not set, durable data will be stored only in the primary store. + /// + /// [`SQLITE_BACKUP_DB_FILE_NAME`]: crate::io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME + #[cfg(not(feature = "uniffi"))] + pub fn set_backup_storage_dir_path(&mut self, backup_storage_dir_path: String) -> &mut Self { + let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default()); + tier_store_config.backup_storage_dir_path = Some(backup_storage_dir_path.into()); + self + } + + /// Configures the ephemeral storage directory path for non-critical, frequently-accessed data. + /// + /// When set, a local SQLite store is created at this path for ephemeral data like + /// the network graph and scorer. Data stored here can be rebuilt if lost. + /// + /// If not set, non-critical data will be stored in the primary store. + #[cfg(not(feature = "uniffi"))] + pub fn set_ephemeral_storage_dir_path( + &mut self, ephemeral_storage_dir_path: String, + ) -> &mut Self { + let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default()); + tier_store_config.ephemeral_storage_dir_path = Some(ephemeral_storage_dir_path.into()); + self + } + /// Builds a [`Node`] instance with a [`SqliteStore`] backend and according to the options /// previously configured. #[cfg(feature = "storage-sqlite")] @@ -901,11 +946,18 @@ impl NodeBuilder { } /// Builds a [`Node`] instance according to the options previously configured. + /// + /// The provided `kv_store` will be used as the primary storage backend. Optionally, + /// an ephemeral store for frequently-accessed non-critical data (e.g., network graph, scorer) + /// and a local SQLite backup store for disaster recovery can be configured via + /// [`set_ephemeral_storage_dir_path`] and [`set_backup_storage_dir_path`]. + /// + /// [`set_ephemeral_storage_dir_path`]: Self::set_ephemeral_storage_dir_path + /// [`set_backup_storage_dir_path`]: Self::set_backup_storage_dir_path pub fn build_with_store( &self, node_entropy: NodeEntropy, kv_store: S, ) -> Result { let logger = setup_logger(&self.log_writer_config, &self.config)?; - self.build_with_store_and_logger(node_entropy, kv_store, logger) } @@ -930,6 +982,46 @@ impl NodeBuilder { fn build_with_store_runtime_and_logger( &self, node_entropy: NodeEntropy, kv_store: S, runtime: Arc, logger: Arc, ) -> Result { + let ts_config = self.tier_store_config.as_ref(); + let primary_store = Arc::new(DynStoreWrapper(kv_store)); + let mut tier_store = TierStore::new(primary_store, Arc::clone(&logger)); + if let Some(config) = ts_config { + if let Some(ephemeral_storage_dir_path) = config.ephemeral_storage_dir_path.as_ref() { + let index_store = runtime + .block_on(setup_index_store(self.config.storage_dir_path.clone().into())) + .map_err(|e| { + log_error!(logger, "Failed to setup tier-store index: {}", e); + BuildError::KVStoreSetupFailed + })?; + let ephemeral_store = SqliteStore::new( + ephemeral_storage_dir_path.clone(), + Some(io::sqlite_store::SQLITE_EPHEMERAL_DB_FILE_NAME.to_string()), + Some(io::sqlite_store::KV_TABLE_NAME.to_string()), + ) + .map_err(|e| { + log_error!(logger, "Failed to setup ephemeral SQLite store: {}", e); + BuildError::KVStoreSetupFailed + })?; + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(ephemeral_store)); + tier_store.set_index_store(index_store); + tier_store.set_ephemeral_store(ephemeral_store); + } + + if let Some(backup_storage_dir_path) = config.backup_storage_dir_path.as_ref() { + let backup_store = SqliteStore::new( + backup_storage_dir_path.clone(), + Some(io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME.to_string()), + Some(io::sqlite_store::KV_TABLE_NAME.to_string()), + ) + .map_err(|e| { + log_error!(logger, "Failed to setup backup SQLite store: {}", e); + BuildError::KVStoreSetupFailed + })?; + let backup_store: Arc = Arc::new(DynStoreWrapper(backup_store)); + tier_store.set_backup_store(backup_store); + } + } + let seed_bytes = node_entropy.to_seed_bytes(); let config = Arc::new(self.config.clone()); @@ -944,7 +1036,7 @@ impl NodeBuilder { seed_bytes, runtime, logger, - Arc::new(DynStoreWrapper(kv_store)), + Arc::new(DynStoreWrapper(tier_store)), ) } } diff --git a/src/io/sqlite_store/mod.rs b/src/io/sqlite_store/mod.rs index 22bfd5710..6702f842d 100644 --- a/src/io/sqlite_store/mod.rs +++ b/src/io/sqlite_store/mod.rs @@ -30,6 +30,10 @@ mod migrations; pub const SQLITE_DB_FILE_NAME: &str = "ldk_node_data.sqlite"; /// LDK Node's internal tier-store index database file name. pub(crate) const SQLITE_TIER_INDEX_DB_FILE_NAME: &str = "ldk_node_tier_index.sqlite"; +/// LDK Node's backup database file name. +pub const SQLITE_BACKUP_DB_FILE_NAME: &str = "ldk_node_data_backup.sqlite"; +/// LDK Node's ephemeral database file name. +pub const SQLITE_EPHEMERAL_DB_FILE_NAME: &str = "ldk_node_data_ephemeral.sqlite"; /// LDK Node's table in which we store all data. pub const KV_TABLE_NAME: &str = "ldk_node_data"; diff --git a/src/io/tier_store.rs b/src/io/tier_store.rs index 65d1e6546..b1a862ed3 100644 --- a/src/io/tier_store.rs +++ b/src/io/tier_store.rs @@ -4,7 +4,6 @@ // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license , at your option. You may not use this file except in // accordance with one or both of these licenses. -#![allow(dead_code)] // TODO: Temporal warning silencer. Will be removed in later commit. use std::collections::HashMap; use std::future::Future; diff --git a/tests/common/mod.rs b/tests/common/mod.rs index af6a49c69..607edaed8 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -62,7 +62,7 @@ use lightning::ln::msgs::SocketAddress; use lightning::routing::gossip::NodeAlias; use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; use lightning_invoice::{Bolt11InvoiceDescription, Description}; -use lightning_persister::fs_store::v1::FilesystemStore; +use lightning_persister::fs_store::v2::FilesystemStoreV2; use lightning_types::payment::{PaymentHash, PaymentPreimage}; use logging::TestLogWriter; use rand::distr::Alphanumeric; @@ -1957,7 +1957,7 @@ impl PaginatedKVStore for TestSyncStore { struct TestSyncStoreInner { serializer: tokio::sync::RwLock<()>, test_store: InMemoryStore, - fs_store: FilesystemStore, + fs_store: FilesystemStoreV2, #[cfg(feature = "storage-sqlite")] sqlite_store: SqliteStore, } @@ -1967,7 +1967,7 @@ impl TestSyncStoreInner { let serializer = tokio::sync::RwLock::new(()); let mut fs_dir = dest_dir.clone(); fs_dir.push("fs_store"); - let fs_store = FilesystemStore::new(fs_dir); + let fs_store = FilesystemStoreV2::new(fs_dir).unwrap(); #[cfg(feature = "storage-sqlite")] let mut sql_dir = dest_dir.clone(); #[cfg(feature = "storage-sqlite")] diff --git a/tests/integration_tests_rust.rs b/tests/integration_tests_rust.rs index 0dad32d6a..e7753693e 100644 --- a/tests/integration_tests_rust.rs +++ b/tests/integration_tests_rust.rs @@ -38,6 +38,8 @@ use ldk_node::config::{ AsyncPaymentsRole, EsploraSyncConfig, ADDRESS_POOL_SIZE, DEFAULT_FULL_SCAN_STOP_GAP, }; use ldk_node::entropy::NodeEntropy; +#[cfg(not(feature = "uniffi"))] +use ldk_node::io::sqlite_store::SqliteStore; use ldk_node::liquidity::LSPS2ServiceConfig; use ldk_node::payment::{ ConfirmationStatus, PayerProofOptions, PaymentDetails, PaymentDirection, PaymentKind, @@ -4938,3 +4940,74 @@ async fn do_lsps2_multi_lsp_picks_cheapest(reverse_order: bool) { cheap.stop().unwrap(); expensive.stop().unwrap(); } + +// Builder backup-store configuration is not yet exposed via FFI (see #871) +#[cfg(not(feature = "uniffi"))] +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn builder_configures_sqlite_backup_store() { + let (bitcoind, electrsd) = setup_bitcoind_and_electrsd(); + let chain_source = random_chain_source(&bitcoind, &electrsd); + + let mut config_a = random_config(); + config_a.store_type = TestStoreType::Sqlite; + let primary_dir = config_a.node_config.storage_dir_path.clone(); + let backup_dir = common::random_storage_path(); + + // Build node_a with backup storage configured + setup_builder!(builder_a, config_a.node_config.clone()); + builder_a.set_chain_source_esplora( + format!("http://{}", electrsd.esplora_url.as_ref().unwrap()), + None, + ); + builder_a.set_filesystem_logger(None, None); + builder_a.set_backup_storage_dir_path(backup_dir.to_str().unwrap().to_owned()); + + let node_a = builder_a.build(config_a.node_entropy.into()).unwrap(); + node_a.start().unwrap(); + assert!(node_a.status().is_running); + assert!(node_a.status().latest_fee_rate_cache_update_timestamp.is_some()); + + let mut config_b = random_config(); + config_b.node_config.manually_handle_unknown_bolt11_payments = true; + let node_b = setup_node(&chain_source, config_b); + + do_channel_full_cycle( + node_a, + node_b, + &bitcoind.client, + &electrsd.client, + false, + true, + true, + false, + ) + .await; + + let primary_store = SqliteStore::new( + primary_dir.into(), + Some(ldk_node::io::sqlite_store::SQLITE_DB_FILE_NAME.to_string()), + Some(ldk_node::io::sqlite_store::KV_TABLE_NAME.to_string()), + ) + .unwrap(); + + let backup_store = SqliteStore::new( + backup_dir, + Some(ldk_node::io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME.to_string()), + Some(ldk_node::io::sqlite_store::KV_TABLE_NAME.to_string()), + ) + .unwrap(); + + for (pn, sn, key) in [ + ("bdk_wallet", "", "descriptor"), + ("bdk_wallet", "", "change_descriptor"), + ("bdk_wallet", "", "network"), + ("", "", "node_metrics"), + ("", "", "events"), + ("", "", "peers"), + ] { + let primary = primary_store.read(pn, sn, key).await.unwrap(); + let backup = backup_store.read(pn, sn, key).await.unwrap(); + + assert_eq!(backup, primary, "backup mismatch for {pn}/{sn}/{key}"); + } +} From 13633ed46009a6a5f3597c46e0d41a3f296368e0 Mon Sep 17 00:00:00 2001 From: Enigbe Date: Mon, 24 Aug 2026 14:06:48 +0100 Subject: [PATCH 10/14] Gate tiered storage behind a Cargo feature Add the opt-in storage-tier feature, gate tier-specific APIs and implementation, and preserve direct store usage when disabled. Document the feature, expose it on docs.rs, and run targeted tier storage tests in CI. AI-assisted: Developed with Amp. --- .github/workflows/rust.yml | 8 +++ Cargo.toml | 3 +- README.md | 4 +- src/builder.rs | 88 +++++++++++++++++++-------------- src/io/mod.rs | 1 + src/io/sqlite_store/mod.rs | 4 ++ tests/integration_tests_rust.rs | 4 +- 7 files changed, 70 insertions(+), 42 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ea3df44b2..79d30998c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -86,6 +86,14 @@ jobs: if: "matrix.platform != 'windows-latest'" run: | RUSTFLAGS="--cfg no_download --cfg cycle_tests --cfg tokio_unstable" cargo test + - name: Test tiered storage + if: matrix.check-fmt + run: | + cargo test --lib --features storage-tier io::tier_store + RUSTFLAGS="--cfg no_download" cargo test \ + --features storage-tier \ + --test integration_tests_rust \ + builder_configures_sqlite_backup_store - name: Test with UniFFI support on Rust ${{ matrix.toolchain }} if: "matrix.platform != 'windows-latest' && matrix.build-uniffi" run: | diff --git a/Cargo.toml b/Cargo.toml index 0110c027b..11bec90d1 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ keywords = ["bitcoin", "lightning", "ldk", "bdk"] categories = ["cryptography::cryptocurrencies"] [package.metadata.docs.rs] -features = ["storage-postgres-vendored-tls"] +features = ["storage-postgres-vendored-tls", "storage-tier"] rustdoc-args = ["--cfg", "docsrs"] [lib] @@ -53,6 +53,7 @@ chain-electrum = [ ] chain-bitcoind = ["dep:lightning-block-sync"] storage-sqlite = ["dep:rusqlite"] +storage-tier = ["storage-sqlite"] storage-filesystem = ["dep:lightning-persister"] storage-vss = ["dep:vss-client", "dep:prost"] storage-postgres = ["dep:tokio-postgres", "dep:native-tls", "dep:postgres-native-tls"] diff --git a/README.md b/README.md index ec9de516f..f42a8f3ff 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,7 @@ LDK Node's optional dependencies are grouped by the functionality they provide: | `chain-electrum` | Electrum chain source | | `chain-bitcoind` | Bitcoin Core RPC and REST chain source | | `storage-sqlite` | SQLite storage | +| `storage-tier` | Tiered storage with optional ephemeral and backup SQLite stores | | `storage-filesystem` | Filesystem storage | | `storage-vss` | Versioned Storage Service storage | | `storage-postgres` | PostgreSQL storage | @@ -88,7 +89,8 @@ LDK Node's optional dependencies are grouped by the functionality they provide: The `default` feature set preserves the native Rust API's previous behavior. It enables all three chain sources, SQLite, filesystem and VSS storage, and unified payments. PostgreSQL and UniFFI -remain opt-in. Every build must enable at least one chain source feature. +remain opt-in. Tiered storage is also opt-in; enabling `storage-tier` automatically enables +`storage-sqlite`. Every build must enable at least one chain source feature. On Linux, `storage-postgres` uses the system OpenSSL installation and requires the OpenSSL development headers and `pkg-config`. Enable `storage-postgres-vendored-tls` instead to build diff --git a/src/builder.rs b/src/builder.rs index 7077a9622..4f1b242be 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -11,7 +11,7 @@ use std::convert::TryInto; use std::default::Default; #[cfg(feature = "unified-payments")] use std::net::ToSocketAddrs; -#[cfg(feature = "storage-filesystem")] +#[cfg(any(feature = "storage-filesystem", feature = "storage-tier"))] use std::path::PathBuf; use std::sync::{Arc, Mutex, Once, RwLock}; use std::time::SystemTime; @@ -72,6 +72,7 @@ use crate::gossip::GossipSource; use crate::io::fs_store::open_or_migrate_fs_store; #[cfg(feature = "storage-sqlite")] use crate::io::sqlite_store::SqliteStore; +#[cfg(feature = "storage-tier")] use crate::io::tier_store::{setup_index_store, TierStore}; use crate::io::utils::{ read_all_objects, read_event_queue, read_external_pathfinding_scores_from_cache, @@ -174,6 +175,7 @@ impl std::fmt::Debug for LogWriterConfig { } } +#[cfg(feature = "storage-tier")] #[derive(Default, Debug)] struct TierStoreConfig { ephemeral_storage_dir_path: Option, @@ -333,6 +335,7 @@ pub struct NodeBuilder { liquidity_source_config: Option, log_writer_config: Option, async_payments_role: Option, + #[cfg(feature = "storage-tier")] tier_store_config: Option, runtime_handle: Option, pathfinding_scores_sync_config: Option, @@ -355,6 +358,7 @@ impl NodeBuilder { let gossip_source_config = None; let liquidity_source_config = None; let log_writer_config = None; + #[cfg(feature = "storage-tier")] let tier_store_config = None; let runtime_handle = None; let pathfinding_scores_sync_config = None; @@ -365,6 +369,7 @@ impl NodeBuilder { gossip_source_config, liquidity_source_config, log_writer_config, + #[cfg(feature = "storage-tier")] tier_store_config, runtime_handle, async_payments_role: None, @@ -709,7 +714,7 @@ impl NodeBuilder { /// If not set, durable data will be stored only in the primary store. /// /// [`SQLITE_BACKUP_DB_FILE_NAME`]: crate::io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME - #[cfg(not(feature = "uniffi"))] + #[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))] pub fn set_backup_storage_dir_path(&mut self, backup_storage_dir_path: String) -> &mut Self { let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default()); tier_store_config.backup_storage_dir_path = Some(backup_storage_dir_path.into()); @@ -722,7 +727,7 @@ impl NodeBuilder { /// the network graph and scorer. Data stored here can be rebuilt if lost. /// /// If not set, non-critical data will be stored in the primary store. - #[cfg(not(feature = "uniffi"))] + #[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))] pub fn set_ephemeral_storage_dir_path( &mut self, ephemeral_storage_dir_path: String, ) -> &mut Self { @@ -982,45 +987,52 @@ impl NodeBuilder { fn build_with_store_runtime_and_logger( &self, node_entropy: NodeEntropy, kv_store: S, runtime: Arc, logger: Arc, ) -> Result { - let ts_config = self.tier_store_config.as_ref(); - let primary_store = Arc::new(DynStoreWrapper(kv_store)); - let mut tier_store = TierStore::new(primary_store, Arc::clone(&logger)); - if let Some(config) = ts_config { - if let Some(ephemeral_storage_dir_path) = config.ephemeral_storage_dir_path.as_ref() { - let index_store = runtime - .block_on(setup_index_store(self.config.storage_dir_path.clone().into())) + #[cfg(feature = "storage-tier")] + let store: Arc = { + let ts_config = self.tier_store_config.as_ref(); + let primary_store = Arc::new(DynStoreWrapper(kv_store)); + let mut tier_store = TierStore::new(primary_store, Arc::clone(&logger)); + if let Some(config) = ts_config { + if let Some(ephemeral_storage_dir_path) = config.ephemeral_storage_dir_path.as_ref() + { + let index_store = runtime + .block_on(setup_index_store(self.config.storage_dir_path.clone().into())) + .map_err(|e| { + log_error!(logger, "Failed to setup tier-store index: {}", e); + BuildError::KVStoreSetupFailed + })?; + let ephemeral_store = SqliteStore::new( + ephemeral_storage_dir_path.clone(), + Some(io::sqlite_store::SQLITE_EPHEMERAL_DB_FILE_NAME.to_string()), + Some(io::sqlite_store::KV_TABLE_NAME.to_string()), + ) .map_err(|e| { - log_error!(logger, "Failed to setup tier-store index: {}", e); + log_error!(logger, "Failed to setup ephemeral SQLite store: {}", e); BuildError::KVStoreSetupFailed })?; - let ephemeral_store = SqliteStore::new( - ephemeral_storage_dir_path.clone(), - Some(io::sqlite_store::SQLITE_EPHEMERAL_DB_FILE_NAME.to_string()), - Some(io::sqlite_store::KV_TABLE_NAME.to_string()), - ) - .map_err(|e| { - log_error!(logger, "Failed to setup ephemeral SQLite store: {}", e); - BuildError::KVStoreSetupFailed - })?; - let ephemeral_store: Arc = Arc::new(DynStoreWrapper(ephemeral_store)); - tier_store.set_index_store(index_store); - tier_store.set_ephemeral_store(ephemeral_store); - } + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(ephemeral_store)); + tier_store.set_index_store(index_store); + tier_store.set_ephemeral_store(ephemeral_store); + } - if let Some(backup_storage_dir_path) = config.backup_storage_dir_path.as_ref() { - let backup_store = SqliteStore::new( - backup_storage_dir_path.clone(), - Some(io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME.to_string()), - Some(io::sqlite_store::KV_TABLE_NAME.to_string()), - ) - .map_err(|e| { - log_error!(logger, "Failed to setup backup SQLite store: {}", e); - BuildError::KVStoreSetupFailed - })?; - let backup_store: Arc = Arc::new(DynStoreWrapper(backup_store)); - tier_store.set_backup_store(backup_store); + if let Some(backup_storage_dir_path) = config.backup_storage_dir_path.as_ref() { + let backup_store = SqliteStore::new( + backup_storage_dir_path.clone(), + Some(io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME.to_string()), + Some(io::sqlite_store::KV_TABLE_NAME.to_string()), + ) + .map_err(|e| { + log_error!(logger, "Failed to setup backup SQLite store: {}", e); + BuildError::KVStoreSetupFailed + })?; + let backup_store: Arc = Arc::new(DynStoreWrapper(backup_store)); + tier_store.set_backup_store(backup_store); + } } - } + Arc::new(DynStoreWrapper(tier_store)) + }; + #[cfg(not(feature = "storage-tier"))] + let store: Arc = Arc::new(DynStoreWrapper(kv_store)); let seed_bytes = node_entropy.to_seed_bytes(); let config = Arc::new(self.config.clone()); @@ -1036,7 +1048,7 @@ impl NodeBuilder { seed_bytes, runtime, logger, - Arc::new(DynStoreWrapper(tier_store)), + store, ) } } diff --git a/src/io/mod.rs b/src/io/mod.rs index 6d616f409..58e2efc85 100644 --- a/src/io/mod.rs +++ b/src/io/mod.rs @@ -15,6 +15,7 @@ pub mod postgres_store; pub mod sqlite_store; #[cfg(test)] pub(crate) mod test_utils; +#[cfg(feature = "storage-tier")] pub(crate) mod tier_store; pub(crate) mod utils; #[cfg(feature = "storage-vss")] diff --git a/src/io/sqlite_store/mod.rs b/src/io/sqlite_store/mod.rs index 6702f842d..f469f593a 100644 --- a/src/io/sqlite_store/mod.rs +++ b/src/io/sqlite_store/mod.rs @@ -29,10 +29,13 @@ mod migrations; /// LDK Node's database file name. pub const SQLITE_DB_FILE_NAME: &str = "ldk_node_data.sqlite"; /// LDK Node's internal tier-store index database file name. +#[cfg(feature = "storage-tier")] pub(crate) const SQLITE_TIER_INDEX_DB_FILE_NAME: &str = "ldk_node_tier_index.sqlite"; /// LDK Node's backup database file name. +#[cfg(feature = "storage-tier")] pub const SQLITE_BACKUP_DB_FILE_NAME: &str = "ldk_node_data_backup.sqlite"; /// LDK Node's ephemeral database file name. +#[cfg(feature = "storage-tier")] pub const SQLITE_EPHEMERAL_DB_FILE_NAME: &str = "ldk_node_data_ephemeral.sqlite"; /// LDK Node's table in which we store all data. pub const KV_TABLE_NAME: &str = "ldk_node_data"; @@ -85,6 +88,7 @@ impl SqliteStore { } /// Constructs a new [`SqliteStore`] which exclusively owns its database for its lifetime. + #[cfg(feature = "storage-tier")] pub(crate) fn new_exclusive( data_dir: PathBuf, db_file_name: Option, kv_table_name: Option, ) -> io::Result { diff --git a/tests/integration_tests_rust.rs b/tests/integration_tests_rust.rs index e7753693e..cf9fbbd07 100644 --- a/tests/integration_tests_rust.rs +++ b/tests/integration_tests_rust.rs @@ -38,7 +38,7 @@ use ldk_node::config::{ AsyncPaymentsRole, EsploraSyncConfig, ADDRESS_POOL_SIZE, DEFAULT_FULL_SCAN_STOP_GAP, }; use ldk_node::entropy::NodeEntropy; -#[cfg(not(feature = "uniffi"))] +#[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))] use ldk_node::io::sqlite_store::SqliteStore; use ldk_node::liquidity::LSPS2ServiceConfig; use ldk_node::payment::{ @@ -4942,7 +4942,7 @@ async fn do_lsps2_multi_lsp_picks_cheapest(reverse_order: bool) { } // Builder backup-store configuration is not yet exposed via FFI (see #871) -#[cfg(not(feature = "uniffi"))] +#[cfg(all(not(feature = "uniffi"), feature = "storage-tier"))] #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn builder_configures_sqlite_backup_store() { let (bitcoind, electrsd) = setup_bitcoind_and_electrsd(); From 616c7d90d69c6e6317ad53bbf95ef5a572f2702b Mon Sep 17 00:00:00 2001 From: Enigbe Date: Fri, 28 Aug 2026 18:13:40 +0100 Subject: [PATCH 11/14] fix(test): Deduplicate TierStore test environment setup Add a shared helper for creating temporary storage paths, loggers, and cleanup guards across TierStore tests. --- src/io/tier_store.rs | 186 ++++++++++--------------------------------- 1 file changed, 41 insertions(+), 145 deletions(-) diff --git a/src/io/tier_store.rs b/src/io/tier_store.rs index b1a862ed3..616b87cf9 100644 --- a/src/io/tier_store.rs +++ b/src/io/tier_store.rs @@ -1607,6 +1607,14 @@ mod tests { } } + fn setup_test_environment() -> (PathBuf, Arc, CleanupDir) { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let cleanup = CleanupDir(base_dir.clone()); + (base_dir, logger, cleanup) + } + fn setup_tier_store(primary_store: Arc, logger: Arc) -> TierStore { TierStore::new(primary_store, logger) } @@ -1716,10 +1724,7 @@ mod tests { #[tokio::test] async fn indexed_listing_orders_keys_across_primary_and_ephemeral_stores() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); @@ -1775,10 +1780,7 @@ mod tests { #[tokio::test] async fn indexed_listing_preserves_updates_and_reorders_recreated_keys() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let mut tier = setup_tier_store(primary_store, logger); @@ -1800,10 +1802,7 @@ mod tests { #[tokio::test] async fn adopting_ephemeral_storage_migrates_cache_keys_without_reordering_them() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); @@ -1876,10 +1875,7 @@ mod tests { #[tokio::test] async fn missing_index_discards_ephemeral_only_cache_data_before_reading() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); @@ -1924,10 +1920,7 @@ mod tests { #[tokio::test] async fn missing_index_rebuilds_cache_from_primary_instead_of_ephemeral() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); @@ -1987,10 +1980,7 @@ mod tests { #[tokio::test] async fn namespace_preparation_finishes_interrupted_cache_migration() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); @@ -2058,10 +2048,7 @@ mod tests { #[tokio::test] async fn reading_one_cache_key_only_reconciles_that_key() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); @@ -2141,10 +2128,7 @@ mod tests { #[tokio::test] async fn writing_one_cache_key_only_reconciles_that_key() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); @@ -2215,10 +2199,7 @@ mod tests { #[tokio::test] async fn namespace_initialization_preserves_existing_primary_order_across_pages() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); for i in 0..55 { @@ -2249,10 +2230,7 @@ mod tests { #[tokio::test] async fn paginated_listing_rejects_tokens_from_another_namespace_or_index() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let mut tier = setup_tier_store(Arc::clone(&primary_store), Arc::clone(&logger)); @@ -2290,10 +2268,7 @@ mod tests { #[tokio::test] async fn pending_creates_roll_forward_to_primary_and_backup_in_journal_order() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); @@ -2341,10 +2316,7 @@ mod tests { #[tokio::test] async fn pending_removal_hides_index_entry_before_removing_value_copies() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); @@ -2377,10 +2349,7 @@ mod tests { #[tokio::test] async fn read_recovers_only_the_requested_key() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); for (key, value) in [("stuck", vec![1]), ("readable", vec![2])] { @@ -2407,10 +2376,7 @@ mod tests { #[tokio::test] async fn writes_and_removals_recover_only_the_requested_key() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); for (key, value) in [("stuck", vec![1]), ("writable", vec![2]), ("removable", vec![3])] { @@ -2441,10 +2407,7 @@ mod tests { #[tokio::test] async fn read_recovers_pending_removal_before_returning_value() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); @@ -2474,10 +2437,7 @@ mod tests { #[tokio::test] async fn read_recovers_pending_cache_operation_without_relocking_the_key() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); @@ -2554,10 +2514,7 @@ mod tests { #[tokio::test] async fn failed_create_retains_journal_without_exposing_index_entry() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); @@ -2582,10 +2539,7 @@ mod tests { #[tokio::test] async fn write_recovers_pending_create_under_key_lock_before_classifying() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let index_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); @@ -2613,10 +2567,7 @@ mod tests { #[tokio::test] async fn update_rejects_value_without_index_entry() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); @@ -2637,10 +2588,7 @@ mod tests { #[tokio::test] async fn failed_removal_retains_journal_and_hides_index_entry() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let backup_store: Arc = @@ -2762,11 +2710,7 @@ mod tests { #[tokio::test] async fn write_read_list_remove() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - - let _cleanup = CleanupDir(base_dir.clone()); + let (base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); @@ -2777,11 +2721,7 @@ mod tests { #[tokio::test] async fn ephemeral_routing() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - - let _cleanup = CleanupDir(base_dir.clone()); + let (base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); @@ -2850,11 +2790,7 @@ mod tests { #[tokio::test] async fn external_pathfinding_scores_cache_routes_to_ephemeral_store() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - - let _cleanup = CleanupDir(base_dir.clone()); + let (base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); @@ -2913,11 +2849,7 @@ mod tests { #[tokio::test] async fn list_exposes_primary_and_routed_ephemeral_keys() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - - let _cleanup = CleanupDir(base_dir.clone()); + let (base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); @@ -3001,11 +2933,7 @@ mod tests { #[tokio::test] async fn list_paginated_only_exposes_primary_keys() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - - let _cleanup = CleanupDir(base_dir.clone()); + let (base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); @@ -3084,11 +3012,7 @@ mod tests { #[tokio::test] async fn listings_only_consult_ephemeral_store_for_routed_namespaces() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - - let _cleanup = CleanupDir(base_dir.clone()); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); @@ -3142,11 +3066,7 @@ mod tests { #[tokio::test] async fn list_hides_stale_primary_copy_when_ephemeral_key_is_missing() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - - let _cleanup = CleanupDir(base_dir.clone()); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); @@ -3193,11 +3113,7 @@ mod tests { #[tokio::test] async fn primary_backed_writes_preserve_latest_call_order() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - - let _cleanup = CleanupDir(base_dir.clone()); + let (base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); @@ -3236,11 +3152,7 @@ mod tests { #[tokio::test] async fn failed_newer_backup_write_still_supersedes_older_write() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - - let _cleanup = CleanupDir(base_dir); + let (_base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); @@ -3286,11 +3198,7 @@ mod tests { #[tokio::test] async fn ephemeral_writes_preserve_latest_call_order() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - - let _cleanup = CleanupDir(base_dir.clone()); + let (base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); @@ -3332,11 +3240,7 @@ mod tests { #[tokio::test] async fn ephemeral_removes_preserve_latest_call_order() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - - let _cleanup = CleanupDir(base_dir.clone()); + let (base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); @@ -3377,11 +3281,7 @@ mod tests { #[tokio::test] async fn backup_write_is_part_of_success_path() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - - let _cleanup = CleanupDir(base_dir.clone()); + let (base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); @@ -3423,11 +3323,7 @@ mod tests { #[tokio::test] async fn backup_remove_is_part_of_success_path() { - let base_dir = random_storage_path(); - let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); - let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); - - let _cleanup = CleanupDir(base_dir.clone()); + let (base_dir, logger, _cleanup) = setup_test_environment(); let primary_store: Arc = Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); From 568b8f6ee04bab4a4dd6540c7baf8c3f3bfb27d9 Mon Sep 17 00:00:00 2001 From: Enigbe Date: Wed, 26 Aug 2026 08:09:18 +0100 Subject: [PATCH 12/14] Track TierStore backup synchronization generations A newly configured backup and a previously configured backup that missed primary-only operations are both potentially incomplete. Without durable synchronization metadata, TierStore cannot distinguish either case from a backup containing the current primary state. In this commit, we: - Persist an opaque synchronization generation in the authoritative primary store and compare it with the backup's last completed generation. - Classify missing or mismatched backup completion records as requiring synchronization, while preserving matching generations across restarts. - Rotate an existing primary generation when restarting without the backup so later primary-only operations invalidate its previous completion record. - Avoid creating synchronization metadata for stores that have never configured a backup, preventing an unnecessary write on every startup. - Initialize backup synchronization metadata after NodeBuilder finishes configuring TierStore and test new, stale, synchronized, and removed-backup cases. This commit only establishes synchronization detection. Copying primary data into a new or stale backup follows separately. Assisted-by: Amp (AI coding agent) --- src/builder.rs | 4 + src/io/tier_store.rs | 265 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 268 insertions(+), 1 deletion(-) diff --git a/src/builder.rs b/src/builder.rs index 4f1b242be..65610eccf 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -1029,6 +1029,10 @@ impl NodeBuilder { tier_store.set_backup_store(backup_store); } } + runtime.block_on(tier_store.initialize_backup_synchronization()).map_err(|e| { + log_error!(logger, "Failed to initialize tier-store backup synchronization: {}", e); + BuildError::KVStoreSetupFailed + })?; Arc::new(DynStoreWrapper(tier_store)) }; #[cfg(not(feature = "storage-tier"))] diff --git a/src/io/tier_store.rs b/src/io/tier_store.rs index 616b87cf9..590834d3c 100644 --- a/src/io/tier_store.rs +++ b/src/io/tier_store.rs @@ -28,7 +28,6 @@ use crate::logger::{LdkLogger, Logger}; use crate::types::{DynStore, DynStoreWrapper}; const INDEX_DATABASE_ID_LEN: usize = 16; -const PAGE_TOKEN_FORMAT_VERSION: u8 = 1; const INDEX_ENTRIES_PRIMARY_NAMESPACE: &str = "_tier_store_entries"; const INDEX_JOURNAL_PRIMARY_NAMESPACE: &str = "_tier_store_journal"; const INDEX_METADATA_PRIMARY_NAMESPACE: &str = "_tier_store_metadata"; @@ -37,6 +36,22 @@ const INDEX_NAMESPACE_READY_KEY_PREFIX: &str = "ready_"; const INDEX_CACHE_READY_KEY_PREFIX: &str = "cache_ready_"; const INDEX_ENTRY_VALUE: &[u8] = &[1]; +const PAGE_TOKEN_FORMAT_VERSION: u8 = 1; + +const GENERATION_ID_LEN: usize = 16; + +const PRIMARY_SYNC_GENERATION_KEY: &str = "primary_generation"; + +const BACKUP_SYNC_PRIMARY_NAMESPACE: &str = "_tier_store_backup_sync"; +const BACKUP_SYNC_COMPLETED_GENERATION_KEY: &str = "completed_generation"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum BackupSyncStatus { + NotConfigured, + Synchronized, + Required, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ValueTier { Primary, @@ -601,6 +616,26 @@ impl TierStore { inner.index = Some(index); } + + /// Initializes the durable metadata used to determine whether the backup matches the primary. + /// + /// This must be called once after the optional backup store has been configured and before the + /// `TierStore` is used. If no backup is configured but a primary generation already exists, it + /// writes a new generation so any backup completed against the earlier generation will be + /// recognized as stale if it returns. If no generation exists, no backup has yet been tracked and + /// no metadata is created. If a backup is configured, it reads or creates the primary generation + /// and compares it with the backup's completion record without modifying the completion record. + /// + /// Returns [`BackupSyncStatus::NotConfigured`] when no backup is present, whether or not an + /// existing primary generation was rotated, [`BackupSyncStatus::Synchronized`] when both records match, or + /// [`BackupSyncStatus::Required`] when the backup has no completion record or records another + /// generation. A `Required` result only classifies the backup; it does not perform synchronization. + /// + /// Returns an error if generation metadata cannot be read, generated, or persisted, or if a + /// stored generation has an invalid length. + pub(crate) async fn initialize_backup_synchronization(&self) -> io::Result { + self.inner.initialize_backup_synchronization().await + } } pub(crate) async fn setup_index_store(data_dir: PathBuf) -> io::Result { @@ -731,6 +766,139 @@ impl TierStoreInner { } } + /// Rotates or compares the stores' backup-synchronization generations. + /// + /// Without a configured backup, this rotates an existing primary generation so primary-only + /// operation invalidates any earlier backup completion. It leaves primary metadata absent when no + /// backup has ever established a generation. With a configured backup, this preserves the primary + /// generation (creating it if absent) and compares it with the completion generation stored in the + /// backup. A missing or different backup completion is classified as requiring synchronization; + /// this method does not perform that synchronization. + async fn initialize_backup_synchronization(&self) -> io::Result { + if self.backup_store.is_none() { + match Self::read_generation_id(self.primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY) + .await + { + Ok(_) => { + let generation_id = Self::generate_primary_generation_id()?; + self.write_primary_generation_id(generation_id).await?; + }, + Err(e) if e.kind() == io::ErrorKind::NotFound => {}, + Err(e) => return Err(e), + } + return Ok(BackupSyncStatus::NotConfigured); + } + + let primary_generation_id = self.read_or_create_primary_sync_generation_id().await?; + match self.read_backup_completed_generation_id().await { + Ok(completed_generation_id) if completed_generation_id == primary_generation_id => { + Ok(BackupSyncStatus::Synchronized) + }, + Ok(_) => Ok(BackupSyncStatus::Required), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(BackupSyncStatus::Required), + Err(e) => Err(e), + } + } + + /// Generates an opaque random identity for one synchronization generation. + fn generate_primary_generation_id() -> io::Result<[u8; GENERATION_ID_LEN]> { + let mut generation_id = [0; GENERATION_ID_LEN]; + getrandom::fill(&mut generation_id).map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("Failed to generate tier-store backup synchronization generation: {e}"), + ) + })?; + Ok(generation_id) + } + + /// Reads the primary's current synchronization generation, creating and persisting one if absent. + /// + /// An existing generation is preserved so a matching backup completion remains valid across + /// restarts where the backup stays configured. + async fn read_or_create_primary_sync_generation_id(&self) -> io::Result<[u8; GENERATION_ID_LEN]> { + match Self::read_generation_id(self.primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY).await { + Ok(generation_id) => Ok(generation_id), + Err(e) if e.kind() == io::ErrorKind::NotFound => { + let generation_id = Self::generate_primary_generation_id()?; + self.write_primary_generation_id(generation_id).await?; + Ok(generation_id) + }, + Err(e) => Err(e), + } + } + + /// Persists the current synchronization generation directly in the authoritative primary store. + /// + /// The write intentionally bypasses backup replication: changing this record invalidates an old + /// backup, whose completion record must remain unchanged until synchronization actually finishes. + async fn write_primary_generation_id( + &self, generation_id: [u8; GENERATION_ID_LEN], + ) -> io::Result<()> { + KVStore::write( + self.primary_store.as_ref(), + BACKUP_SYNC_PRIMARY_NAMESPACE, + "", + PRIMARY_SYNC_GENERATION_KEY, + generation_id.to_vec(), + ) + .await + } + + /// Reads the generation for which the configured backup last completed synchronization. + /// + /// The completion record is proof that all durable primary data was copied and stale backup data + /// was removed for that generation. Its absence therefore means synchronization is required; it + /// must not be inferred from individual values already present in the backup. + /// + /// Returns [`io::ErrorKind::NotFound`] when no backup is configured or the configured backup has + /// no completion record. Other storage errors and malformed completion records are propagated so + /// callers cannot mistake an unreadable record for proof that the backup is current. + async fn read_backup_completed_generation_id(&self) -> io::Result<[u8; GENERATION_ID_LEN]> { + let backup_store = self.backup_store.as_ref().ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "Backup store is not configured") + })?; + Self::read_generation_id(backup_store.as_ref(), BACKUP_SYNC_COMPLETED_GENERATION_KEY).await + } + + /// Reads and validates a synchronization generation from the given store metadata key. + /// + /// Primary generations and backup completion generations use the same opaque, fixed-width value + /// representation but live under different keys. This helper centralizes the shared storage + /// location and length validation without assigning ordering semantics to the random bytes. + /// + /// Returns [`io::ErrorKind::NotFound`] when the key is absent and + /// [`io::ErrorKind::InvalidData`] when its value is not exactly + /// [`GENERATION_ID_LEN`] bytes. All other underlying storage errors are propagated. + async fn read_generation_id( + store: &DynStore, key: &str, + ) -> io::Result<[u8; GENERATION_ID_LEN]> { + let generation_id = KVStore::read(store, BACKUP_SYNC_PRIMARY_NAMESPACE, "", key).await?; + generation_id.try_into().map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "Invalid tier-store backup synchronization generation", + ) + }) + } + + #[cfg(test)] + async fn write_backup_completed_generation_id( + &self, generation_id: [u8; GENERATION_ID_LEN], + ) -> io::Result<()> { + let backup_store = self.backup_store.as_ref().ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "Backup store is not configured") + })?; + KVStore::write( + backup_store.as_ref(), + BACKUP_SYNC_PRIMARY_NAMESPACE, + "", + BACKUP_SYNC_COMPLETED_GENERATION_KEY, + generation_id.to_vec(), + ) + .await + } + fn get_new_version_and_lock_ref(&self, locking_key: String) -> (Arc>, u64) { let version = self.next_write_version.fetch_add(1, Ordering::Relaxed); if version == u64::MAX { @@ -1624,6 +1792,101 @@ mod tests { tier.set_index_store(TierStoreIndex::from_store(store)); } + #[tokio::test] + async fn backup_sync_metadata_remains_absent_when_a_backup_has_never_been_configured() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + + let tier = setup_tier_store(Arc::clone(&primary_store), Arc::clone(&logger)); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::NotConfigured + ); + drop(tier); + let restarted_tier = setup_tier_store(Arc::clone(&primary_store), logger); + assert_eq!( + restarted_tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::NotConfigured + ); + let error = + TierStoreInner::read_generation_id(primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY) + .await + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::NotFound); + } + + #[tokio::test] + async fn backup_sync_generation_rotates_after_backup_is_removed() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + + let mut tier = setup_tier_store(Arc::clone(&primary_store), Arc::clone(&logger)); + tier.set_backup_store(backup_store); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + let configured_generation_id = + TierStoreInner::read_generation_id(primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY) + .await + .unwrap(); + drop(tier); + + let restarted_tier = setup_tier_store(Arc::clone(&primary_store), logger); + assert_eq!( + restarted_tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::NotConfigured + ); + let rotated_generation_id = + TierStoreInner::read_generation_id(primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY) + .await + .unwrap(); + + assert_ne!(rotated_generation_id, configured_generation_id); + } + + #[tokio::test] + async fn backup_sync_status_tracks_persisted_completion() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + + let mut tier = setup_tier_store(Arc::clone(&primary_store), Arc::clone(&logger)); + tier.set_backup_store(Arc::clone(&backup_store)); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + let generation_id = + TierStoreInner::read_generation_id(primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY) + .await + .unwrap(); + + let mut stale_generation_id = generation_id; + stale_generation_id[0] ^= 1; + tier.inner.write_backup_completed_generation_id(stale_generation_id).await.unwrap(); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + + tier.inner.write_backup_completed_generation_id(generation_id).await.unwrap(); + drop(tier); + let mut restarted_tier = setup_tier_store(Arc::clone(&primary_store), logger); + restarted_tier.set_backup_store(backup_store); + assert_eq!( + restarted_tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Synchronized + ); + assert_eq!( + TierStoreInner::read_generation_id(primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY) + .await + .unwrap(), + generation_id + ); + } + #[test] fn journal_entry_roundtrips() { for operation in [ From b6e35aef653cac5d3cd064ceea4488cee80f9e1a Mon Sep 17 00:00:00 2001 From: Enigbe Date: Wed, 26 Aug 2026 17:13:31 +0100 Subject: [PATCH 13/14] Recover TierStore writes after a backup is removed A write can be interrupted while a backup is configured, leaving unfinished work in the journal. If the node then restarts without that backup, recovery must not wait forever for a store that was deliberately removed. Updates to existing keys were also not journaled. If the primary write succeeded but the backup write failed, TierStore did not retain the value needed to finish that update later. Separately, a matching primary generation alone does not prove a backup is current. If the local TierStore index has been replaced, the old index may have contained unfinished operations that were lost with it, and generation comparison alone would not detect this. In this commit, we: - Journal updates that must be written to both primary and backup storage. - Allow unfinished creates, updates, and removals to finish using only the primary store after restarting without the backup. - Rotate the primary synchronization generation first, ensuring that the missing backup is recognized as out of date when it is configured again. - Reopen an existing TierStore index even when the backup is no longer configured, so unfinished journal entries can still be recovered. - Store the index database identity alongside the primary generation in the backup completion record, and treat a completion from another index as requiring synchronization. - Add tests for failed updates, recovery without a backup, attempts to recover before backup synchronization has been initialized, completion encoding, and index replacement. A later commit will copy the current primary data into a new or outdated backup. Assisted-by: Amp (AI coding agent) --- src/builder.rs | 25 ++- src/io/tier_store.rs | 471 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 411 insertions(+), 85 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 65610eccf..73e429393 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -992,15 +992,27 @@ impl NodeBuilder { let ts_config = self.tier_store_config.as_ref(); let primary_store = Arc::new(DynStoreWrapper(kv_store)); let mut tier_store = TierStore::new(primary_store, Arc::clone(&logger)); + let tier_index_exists = PathBuf::from(&self.config.storage_dir_path) + .join(io::sqlite_store::SQLITE_TIER_INDEX_DB_FILE_NAME) + .exists(); + let tier_index_required = ts_config + .map(|config| { + config.ephemeral_storage_dir_path.is_some() + || config.backup_storage_dir_path.is_some() + }) + .unwrap_or(false); + if tier_index_required || tier_index_exists { + let index_store = runtime + .block_on(setup_index_store(self.config.storage_dir_path.clone().into())) + .map_err(|e| { + log_error!(logger, "Failed to setup tier-store index: {}", e); + BuildError::KVStoreSetupFailed + })?; + tier_store.set_index_store(index_store); + } if let Some(config) = ts_config { if let Some(ephemeral_storage_dir_path) = config.ephemeral_storage_dir_path.as_ref() { - let index_store = runtime - .block_on(setup_index_store(self.config.storage_dir_path.clone().into())) - .map_err(|e| { - log_error!(logger, "Failed to setup tier-store index: {}", e); - BuildError::KVStoreSetupFailed - })?; let ephemeral_store = SqliteStore::new( ephemeral_storage_dir_path.clone(), Some(io::sqlite_store::SQLITE_EPHEMERAL_DB_FILE_NAME.to_string()), @@ -1011,7 +1023,6 @@ impl NodeBuilder { BuildError::KVStoreSetupFailed })?; let ephemeral_store: Arc = Arc::new(DynStoreWrapper(ephemeral_store)); - tier_store.set_index_store(index_store); tier_store.set_ephemeral_store(ephemeral_store); } diff --git a/src/io/tier_store.rs b/src/io/tier_store.rs index 590834d3c..2cb6b5b09 100644 --- a/src/io/tier_store.rs +++ b/src/io/tier_store.rs @@ -45,6 +45,48 @@ const PRIMARY_SYNC_GENERATION_KEY: &str = "primary_generation"; const BACKUP_SYNC_PRIMARY_NAMESPACE: &str = "_tier_store_backup_sync"; const BACKUP_SYNC_COMPLETED_GENERATION_KEY: &str = "completed_generation"; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct BackupSyncCompletion { + /// The primary generation whose durable values were copied. + primary_generation_id: [u8; GENERATION_ID_LEN], + /// The index database whose journal had been fully recovered before copying. + index_database_id: [u8; INDEX_DATABASE_ID_LEN], +} + +impl BackupSyncCompletion { + /// Constructs the completion expected for the currently configured primary and index stores. + fn new( + primary_generation_id: [u8; GENERATION_ID_LEN], + index_database_id: [u8; INDEX_DATABASE_ID_LEN], + ) -> Self { + Self { primary_generation_id, index_database_id } + } + + /// Encodes both fixed-width identities into the value persisted in the backup store. + #[cfg(test)] + fn encode(self) -> Vec { + let mut encoded = Vec::with_capacity(GENERATION_ID_LEN + INDEX_DATABASE_ID_LEN); + encoded.extend_from_slice(&self.primary_generation_id); + encoded.extend_from_slice(&self.index_database_id); + encoded + } + + /// Decodes and validates the completion value read from the backup store. + fn decode(encoded: Vec) -> io::Result { + if encoded.len() != GENERATION_ID_LEN + INDEX_DATABASE_ID_LEN { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Invalid tier-store backup synchronization completion", + )); + } + let mut primary_generation_id = [0; GENERATION_ID_LEN]; + primary_generation_id.copy_from_slice(&encoded[..GENERATION_ID_LEN]); + let mut index_database_id = [0; INDEX_DATABASE_ID_LEN]; + index_database_id.copy_from_slice(&encoded[GENERATION_ID_LEN..]); + Ok(Self { primary_generation_id, index_database_id }) + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum BackupSyncStatus { NotConfigured, @@ -70,6 +112,11 @@ enum JournalOperation { /// or local value-store write completed before interruption. value: Vec, }, + Update { + /// The complete intended value, retained locally so an interrupted dual-store update can be + /// retried against the configured stores or completed against primary-only storage. + value: Vec, + }, Remove { lazy: bool, }, @@ -82,6 +129,9 @@ impl_writeable_tlv_based_enum!(JournalOperation, (2, Remove) => { (0, lazy, required), }, + (4, Update) => { + (0, value, required), + }, ); #[derive(Debug, PartialEq, Eq)] @@ -204,6 +254,11 @@ impl TierStoreIndex { Self { store, database_id } } + /// Returns the persistent identity of this index database. + fn database_id(&self) -> [u8; INDEX_DATABASE_ID_LEN] { + self.database_id + } + /// Derives the internal secondary namespace for a logical namespace pair. /// /// Length-prefixing keeps distinct pairs from producing the same hash input. The original pair @@ -624,15 +679,17 @@ impl TierStore { /// writes a new generation so any backup completed against the earlier generation will be /// recognized as stale if it returns. If no generation exists, no backup has yet been tracked and /// no metadata is created. If a backup is configured, it reads or creates the primary generation - /// and compares it with the backup's completion record without modifying the completion record. + /// and compares it and the current index database identity with the backup's completion record + /// without modifying that record. /// /// Returns [`BackupSyncStatus::NotConfigured`] when no backup is present, whether or not an - /// existing primary generation was rotated, [`BackupSyncStatus::Synchronized`] when both records match, or - /// [`BackupSyncStatus::Required`] when the backup has no completion record or records another - /// generation. A `Required` result only classifies the backup; it does not perform synchronization. + /// existing primary generation was rotated, [`BackupSyncStatus::Synchronized`] when the completion + /// matches both identities, or [`BackupSyncStatus::Required`] when the backup has no completion + /// record or records another generation or index. A `Required` result only classifies the backup; + /// it does not perform synchronization. /// - /// Returns an error if generation metadata cannot be read, generated, or persisted, or if a - /// stored generation has an invalid length. + /// Returns an error if a configured backup has no index, synchronization metadata cannot be read, + /// generated, or persisted, or stored metadata has an invalid length. pub(crate) async fn initialize_backup_synchronization(&self) -> io::Result { self.inner.initialize_backup_synchronization().await } @@ -743,6 +800,8 @@ struct TierStoreInner { backup_store: Option>, /// The local store used to index the logical contents across tiers. index: Option, + /// The result of durable backup-synchronization initialization for this run. + backup_sync_status: Mutex>, /// Per-namespace locks for serializing first-use index initialization. index_initialization_locks: Mutex>>>, /// Per-key locks for serializing primary+backup operations and skipping stale writes. @@ -759,6 +818,7 @@ impl TierStoreInner { ephemeral_store: None, backup_store: None, index: None, + backup_sync_status: Mutex::new(None), index_initialization_locks: Mutex::new(HashMap::new()), locks: Mutex::new(HashMap::new()), next_write_version: AtomicU64::new(1), @@ -766,16 +826,16 @@ impl TierStoreInner { } } - /// Rotates or compares the stores' backup-synchronization generations. + /// Rotates the primary generation or compares the backup's synchronization completion. /// /// Without a configured backup, this rotates an existing primary generation so primary-only /// operation invalidates any earlier backup completion. It leaves primary metadata absent when no - /// backup has ever established a generation. With a configured backup, this preserves the primary - /// generation (creating it if absent) and compares it with the completion generation stored in the - /// backup. A missing or different backup completion is classified as requiring synchronization; - /// this method does not perform that synchronization. + /// backup has ever established a generation. With a configured backup, this compares the stored + /// completion with both the primary generation and the current index database identity. A missing + /// or different backup completion is classified as requiring synchronization; this method does not + /// perform that synchronization. async fn initialize_backup_synchronization(&self) -> io::Result { - if self.backup_store.is_none() { + let status = if self.backup_store.is_none() { match Self::read_generation_id(self.primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY) .await { @@ -786,18 +846,20 @@ impl TierStoreInner { Err(e) if e.kind() == io::ErrorKind::NotFound => {}, Err(e) => return Err(e), } - return Ok(BackupSyncStatus::NotConfigured); - } - - let primary_generation_id = self.read_or_create_primary_sync_generation_id().await?; - match self.read_backup_completed_generation_id().await { - Ok(completed_generation_id) if completed_generation_id == primary_generation_id => { - Ok(BackupSyncStatus::Synchronized) - }, - Ok(_) => Ok(BackupSyncStatus::Required), - Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(BackupSyncStatus::Required), - Err(e) => Err(e), - } + BackupSyncStatus::NotConfigured + } else { + let expected_completion = self.current_backup_sync_completion().await?; + match self.read_backup_sync_completion().await { + Ok(completion) if completion == expected_completion => { + BackupSyncStatus::Synchronized + }, + Ok(_) => BackupSyncStatus::Required, + Err(e) if e.kind() == io::ErrorKind::NotFound => BackupSyncStatus::Required, + Err(e) => return Err(e), + } + }; + *self.backup_sync_status.lock().expect("lock") = Some(status); + Ok(status) } /// Generates an opaque random identity for one synchronization generation. @@ -816,8 +878,12 @@ impl TierStoreInner { /// /// An existing generation is preserved so a matching backup completion remains valid across /// restarts where the backup stays configured. - async fn read_or_create_primary_sync_generation_id(&self) -> io::Result<[u8; GENERATION_ID_LEN]> { - match Self::read_generation_id(self.primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY).await { + async fn read_or_create_primary_sync_generation_id( + &self, + ) -> io::Result<[u8; GENERATION_ID_LEN]> { + match Self::read_generation_id(self.primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY) + .await + { Ok(generation_id) => Ok(generation_id), Err(e) if e.kind() == io::ErrorKind::NotFound => { let generation_id = Self::generate_primary_generation_id()?; @@ -828,6 +894,22 @@ impl TierStoreInner { } } + /// Returns the completion record a synchronized backup must contain for the current stores. + async fn current_backup_sync_completion(&self) -> io::Result { + let index_database_id = self + .index + .as_ref() + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::Other, + "TierStore index is required for backup synchronization", + ) + })? + .database_id(); + let primary_generation_id = self.read_or_create_primary_sync_generation_id().await?; + Ok(BackupSyncCompletion::new(primary_generation_id, index_database_id)) + } + /// Persists the current synchronization generation directly in the authoritative primary store. /// /// The write intentionally bypasses backup replication: changing this record invalidates an old @@ -845,29 +927,32 @@ impl TierStoreInner { .await } - /// Reads the generation for which the configured backup last completed synchronization. + /// Reads the configured backup's synchronization completion record. /// - /// The completion record is proof that all durable primary data was copied and stale backup data - /// was removed for that generation. Its absence therefore means synchronization is required; it - /// must not be inferred from individual values already present in the backup. + /// The record identifies both the primary generation and index database for which all durable + /// primary data was copied and stale backup data was removed. Its absence therefore means + /// synchronization is required; it must not be inferred from individual backup values. /// /// Returns [`io::ErrorKind::NotFound`] when no backup is configured or the configured backup has /// no completion record. Other storage errors and malformed completion records are propagated so /// callers cannot mistake an unreadable record for proof that the backup is current. - async fn read_backup_completed_generation_id(&self) -> io::Result<[u8; GENERATION_ID_LEN]> { + async fn read_backup_sync_completion(&self) -> io::Result { let backup_store = self.backup_store.as_ref().ok_or_else(|| { io::Error::new(io::ErrorKind::NotFound, "Backup store is not configured") })?; - Self::read_generation_id(backup_store.as_ref(), BACKUP_SYNC_COMPLETED_GENERATION_KEY).await + let encoded = KVStore::read( + backup_store.as_ref(), + BACKUP_SYNC_PRIMARY_NAMESPACE, + "", + BACKUP_SYNC_COMPLETED_GENERATION_KEY, + ) + .await?; + BackupSyncCompletion::decode(encoded) } - /// Reads and validates a synchronization generation from the given store metadata key. - /// - /// Primary generations and backup completion generations use the same opaque, fixed-width value - /// representation but live under different keys. This helper centralizes the shared storage - /// location and length validation without assigning ordering semantics to the random bytes. + /// Reads and validates the primary store's synchronization generation. /// - /// Returns [`io::ErrorKind::NotFound`] when the key is absent and + /// Returns [`io::ErrorKind::NotFound`] when the generation is absent and /// [`io::ErrorKind::InvalidData`] when its value is not exactly /// [`GENERATION_ID_LEN`] bytes. All other underlying storage errors are propagated. async fn read_generation_id( @@ -882,9 +967,16 @@ impl TierStoreInner { }) } + /// Records that the configured backup matches the current primary generation and index database. + /// + /// This must be written only after journal recovery, primary-value copying, and stale-value + /// removal have all succeeded. Writing it last ensures a failed synchronization leaves the + /// previous completion generation unchanged and will be retried on the next startup. + /// + /// Returns an error when no backup is configured or the completion record cannot be persisted. #[cfg(test)] - async fn write_backup_completed_generation_id( - &self, generation_id: [u8; GENERATION_ID_LEN], + async fn write_backup_sync_completion( + &self, completion: BackupSyncCompletion, ) -> io::Result<()> { let backup_store = self.backup_store.as_ref().ok_or_else(|| { io::Error::new(io::ErrorKind::NotFound, "Backup store is not configured") @@ -894,7 +986,7 @@ impl TierStoreInner { BACKUP_SYNC_PRIMARY_NAMESPACE, "", BACKUP_SYNC_COMPLETED_GENERATION_KEY, - generation_id.to_vec(), + completion.encode(), ) .await } @@ -1161,28 +1253,40 @@ impl TierStoreInner { .await; }; - if index.contains_entry(primary_namespace, secondary_namespace, key).await? { - return self - .write_value(tier, primary_namespace, secondary_namespace, key, value) - .await; - } - if self.value_exists(tier, primary_namespace, secondary_namespace, key).await? { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Value exists without a tier-store index entry", - )); - } - + let requires_backup = tier == ValueTier::Primary && self.backup_store.is_some(); + let operation = if index.contains_entry(primary_namespace, secondary_namespace, key).await? + { + if !requires_backup { + return self + .write_value(tier, primary_namespace, secondary_namespace, key, value) + .await; + } + JournalOperation::Update { value } + } else { + if self.value_exists(tier, primary_namespace, secondary_namespace, key).await? { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Value exists without a tier-store index entry", + )); + } + JournalOperation::Create { value } + }; let journal_entry = JournalEntry { primary_namespace: primary_namespace.to_string(), secondary_namespace: secondary_namespace.to_string(), key: key.to_string(), tier, - requires_backup: tier == ValueTier::Primary && self.backup_store.is_some(), - operation: JournalOperation::Create { value }, + requires_backup, + operation, }; index.write_journal_entry(&journal_entry).await?; - self.apply_pending_create(&journal_entry).await?; + match &journal_entry.operation { + JournalOperation::Create { .. } => self.apply_pending_create(&journal_entry).await?, + JournalOperation::Update { .. } => self.apply_pending_update(&journal_entry).await?, + JournalOperation::Remove { .. } => { + unreachable!("write created a removal journal entry") + }, + } index.remove_journal_entry(primary_namespace, secondary_namespace, key).await } @@ -1325,12 +1429,7 @@ impl TierStoreInner { let JournalOperation::Create { value } = &entry.operation else { return Err(io::Error::new(io::ErrorKind::InvalidData, "Expected pending create")); }; - if entry.requires_backup && self.backup_store.is_none() { - return Err(io::Error::new( - io::ErrorKind::NotFound, - "Required backup store is unavailable", - )); - } + self.prepare_pending_backup_recovery(entry).await?; self.write_value( entry.tier, &entry.primary_namespace, @@ -1346,17 +1445,28 @@ impl TierStoreInner { .await } + /// Completes a journaled update without changing the key's existing index membership or order. + async fn apply_pending_update(&self, entry: &JournalEntry) -> io::Result<()> { + let JournalOperation::Update { value } = &entry.operation else { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Expected pending update")); + }; + self.prepare_pending_backup_recovery(entry).await?; + self.write_value( + entry.tier, + &entry.primary_namespace, + &entry.secondary_namespace, + &entry.key, + value.clone(), + ) + .await + } + /// Completes a journaled removal, hiding the key before deleting its value copies. async fn apply_pending_remove(&self, entry: &JournalEntry) -> io::Result<()> { let JournalOperation::Remove { lazy } = &entry.operation else { return Err(io::Error::new(io::ErrorKind::InvalidData, "Expected pending removal")); }; - if entry.requires_backup && self.backup_store.is_none() { - return Err(io::Error::new( - io::ErrorKind::NotFound, - "Required backup store is unavailable", - )); - } + self.prepare_pending_backup_recovery(entry).await?; self.index .as_ref() .expect("pending operations require an index") @@ -1372,6 +1482,25 @@ impl TierStoreInner { .await } + /// Confirms that primary-only recovery has durably invalidated the absent backup. + /// + /// Operations journaled with `requires_backup` normally finish against both durable stores. If + /// this run intentionally omitted the backup, initialization must first rotate the primary + /// generation. Reading or creating that generation here confirms the invalidation is durable + /// before recovery changes primary state; future resilvering then repairs the absent backup. + async fn prepare_pending_backup_recovery(&self, entry: &JournalEntry) -> io::Result<()> { + if !entry.requires_backup || self.backup_store.is_some() { + return Ok(()); + } + if *self.backup_sync_status.lock().expect("lock") != Some(BackupSyncStatus::NotConfigured) { + return Err(io::Error::new( + io::ErrorKind::NotFound, + "Required backup store is unavailable and backup synchronization is not initialized", + )); + } + self.read_or_create_primary_sync_generation_id().await.map(|_| ()) + } + async fn list_internal( &self, primary_namespace: String, secondary_namespace: String, ) -> io::Result> { @@ -1570,7 +1699,7 @@ impl TierStoreInner { self.reconcile_ephemeral_cache_key_locked(primary_namespace, secondary_namespace, key).await } - /// Completes journaled creates and removals before exposing a namespace. + /// Completes journaled creates, updates, and removals before exposing a namespace. async fn recover_namespace( &self, primary_namespace: &str, secondary_namespace: &str, ) -> io::Result<()> { @@ -1606,6 +1735,7 @@ impl TierStoreInner { }; match &entry.operation { JournalOperation::Create { .. } => self.apply_pending_create(&entry).await?, + JournalOperation::Update { .. } => self.apply_pending_update(&entry).await?, JournalOperation::Remove { .. } => self.apply_pending_remove(&entry).await?, } index.remove_journal_entry(primary_namespace, secondary_namespace, key).await @@ -1792,6 +1922,17 @@ mod tests { tier.set_index_store(TierStoreIndex::from_store(store)); } + #[test] + fn backup_sync_completion_roundtrips_and_rejects_invalid_length() { + let completion = + BackupSyncCompletion::new([2; GENERATION_ID_LEN], [3; INDEX_DATABASE_ID_LEN]); + assert_eq!(BackupSyncCompletion::decode(completion.encode()).unwrap(), completion); + assert_eq!( + BackupSyncCompletion::decode(vec![0; GENERATION_ID_LEN]).unwrap_err().kind(), + io::ErrorKind::InvalidData + ); + } + #[tokio::test] async fn backup_sync_metadata_remains_absent_when_a_backup_has_never_been_configured() { let (_base_dir, logger, _cleanup) = setup_test_environment(); @@ -1823,6 +1964,7 @@ mod tests { let mut tier = setup_tier_store(Arc::clone(&primary_store), Arc::clone(&logger)); tier.set_backup_store(backup_store); + set_test_index_store(&mut tier); assert_eq!( tier.initialize_backup_synchronization().await.unwrap(), BackupSyncStatus::Required @@ -1854,27 +1996,27 @@ mod tests { let mut tier = setup_tier_store(Arc::clone(&primary_store), Arc::clone(&logger)); tier.set_backup_store(Arc::clone(&backup_store)); + set_test_index_store(&mut tier); assert_eq!( tier.initialize_backup_synchronization().await.unwrap(), BackupSyncStatus::Required ); - let generation_id = - TierStoreInner::read_generation_id(primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY) - .await - .unwrap(); + let expected_completion = tier.inner.current_backup_sync_completion().await.unwrap(); - let mut stale_generation_id = generation_id; - stale_generation_id[0] ^= 1; - tier.inner.write_backup_completed_generation_id(stale_generation_id).await.unwrap(); + let mut stale_completion = tier.inner.current_backup_sync_completion().await.unwrap(); + stale_completion.primary_generation_id[0] ^= 1; + tier.inner.write_backup_sync_completion(stale_completion).await.unwrap(); assert_eq!( tier.initialize_backup_synchronization().await.unwrap(), BackupSyncStatus::Required ); - tier.inner.write_backup_completed_generation_id(generation_id).await.unwrap(); + let completion = tier.inner.current_backup_sync_completion().await.unwrap(); + tier.inner.write_backup_sync_completion(completion).await.unwrap(); drop(tier); let mut restarted_tier = setup_tier_store(Arc::clone(&primary_store), logger); restarted_tier.set_backup_store(backup_store); + set_test_index_store(&mut restarted_tier); assert_eq!( restarted_tier.initialize_backup_synchronization().await.unwrap(), BackupSyncStatus::Synchronized @@ -1883,7 +2025,43 @@ mod tests { TierStoreInner::read_generation_id(primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY) .await .unwrap(), - generation_id + expected_completion.primary_generation_id + ); + } + + #[tokio::test] + async fn replacing_the_index_requires_backup_synchronization() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + + let mut tier = setup_tier_store(Arc::clone(&primary_store), Arc::clone(&logger)); + tier.set_backup_store(Arc::clone(&backup_store)); + let first_index_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + tier.set_index_store(TierStoreIndex::from_store_with_database_id( + first_index_store, + [1; INDEX_DATABASE_ID_LEN], + )); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + let completion = tier.inner.current_backup_sync_completion().await.unwrap(); + tier.inner.write_backup_sync_completion(completion).await.unwrap(); + drop(tier); + + let mut restarted_tier = setup_tier_store(primary_store, logger); + restarted_tier.set_backup_store(backup_store); + let replacement_index_store: Arc = + Arc::new(DynStoreWrapper(InMemoryStore::new())); + restarted_tier.set_index_store(TierStoreIndex::from_store_with_database_id( + replacement_index_store, + [2; INDEX_DATABASE_ID_LEN], + )); + + assert_eq!( + restarted_tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required ); } @@ -1891,6 +2069,7 @@ mod tests { fn journal_entry_roundtrips() { for operation in [ JournalOperation::Create { value: vec![0, 1, 2, 255] }, + JournalOperation::Update { value: vec![255, 2, 1, 0] }, JournalOperation::Remove { lazy: true }, ] { let entry = JournalEntry { @@ -2610,6 +2789,116 @@ mod tests { assert!(index.read_journal_entry("namespace", "", "key").await.is_err()); } + #[tokio::test] + async fn pending_backup_operations_roll_forward_without_the_backup_after_restart() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let index_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + for store in [&primary_store, &backup_store] { + store.write("namespace", "", "update", vec![1]).await.unwrap(); + store.write("namespace", "", "remove", vec![2]).await.unwrap(); + } + + let mut configured_tier = setup_tier_store(Arc::clone(&primary_store), Arc::clone(&logger)); + configured_tier.set_backup_store(Arc::clone(&backup_store)); + configured_tier.set_index_store(TierStoreIndex::from_store(Arc::clone(&index_store))); + assert_eq!( + configured_tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + configured_tier.inner.ensure_namespace_indexed("namespace", "").await.unwrap(); + let completion = configured_tier.inner.current_backup_sync_completion().await.unwrap(); + configured_tier.inner.write_backup_sync_completion(completion).await.unwrap(); + let entries = [ + ("create", JournalOperation::Create { value: vec![3] }), + ("update", JournalOperation::Update { value: vec![4] }), + ("remove", JournalOperation::Remove { lazy: false }), + ]; + for (key, operation) in entries { + configured_tier + .inner + .index + .as_ref() + .unwrap() + .write_journal_entry(&JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: key.to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation, + }) + .await + .unwrap(); + } + drop(configured_tier); + + let mut primary_only_tier = setup_tier_store(Arc::clone(&primary_store), logger); + primary_only_tier.set_index_store(TierStoreIndex::from_store(index_store)); + assert_eq!( + primary_only_tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::NotConfigured + ); + let keys = KVStore::list(&primary_only_tier, "namespace", "").await.unwrap(); + + assert!(keys.contains(&"create".to_string())); + assert!(keys.contains(&"update".to_string())); + assert!(!keys.contains(&"remove".to_string())); + assert_eq!(primary_store.read("namespace", "", "create").await.unwrap(), vec![3]); + assert_eq!(primary_store.read("namespace", "", "update").await.unwrap(), vec![4]); + assert!(primary_store.read("namespace", "", "remove").await.is_err()); + assert!(backup_store.read("namespace", "", "create").await.is_err()); + assert_eq!(backup_store.read("namespace", "", "update").await.unwrap(), vec![1]); + assert_eq!(backup_store.read("namespace", "", "remove").await.unwrap(), vec![2]); + let index = primary_only_tier.inner.index.as_ref().unwrap(); + for key in ["create", "update", "remove"] { + assert!(index.read_journal_entry("namespace", "", key).await.is_err()); + } + let rotated_generation_id = + TierStoreInner::read_generation_id(primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY) + .await + .unwrap(); + assert_ne!(rotated_generation_id, completion.primary_generation_id); + let encoded_completion = backup_store + .read(BACKUP_SYNC_PRIMARY_NAMESPACE, "", BACKUP_SYNC_COMPLETED_GENERATION_KEY) + .await + .unwrap(); + assert_eq!(BackupSyncCompletion::decode(encoded_completion).unwrap(), completion); + } + + #[tokio::test] + async fn pending_backup_recovery_requires_synchronization_initialization() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + primary_store.write("namespace", "", "key", vec![1]).await.unwrap(); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.inner.ensure_namespace_indexed("namespace", "").await.unwrap(); + let pending = JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: "key".to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation: JournalOperation::Update { value: vec![2] }, + }; + let index = tier.inner.index.as_ref().unwrap(); + index.write_journal_entry(&pending).await.unwrap(); + assert_eq!(*tier.inner.backup_sync_status.lock().unwrap(), None); + + let error = tier.read("namespace", "", "key").await.unwrap_err(); + + assert_eq!(error.kind(), io::ErrorKind::NotFound); + assert_eq!(primary_store.read("namespace", "", "key").await.unwrap(), vec![1]); + assert!(matches!( + index.read_journal_entry("namespace", "", "key").await.unwrap().operation, + JournalOperation::Update { value } if value == vec![2] + )); + } + #[tokio::test] async fn read_recovers_only_the_requested_key() { let (_base_dir, logger, _cleanup) = setup_test_environment(); @@ -2800,6 +3089,32 @@ mod tests { assert_eq!(attempts.load(Ordering::Relaxed), 1); } + #[tokio::test] + async fn failed_update_retains_journal_without_changing_index_membership() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.write("namespace", "", "key", vec![1]).await.unwrap(); + let attempts = Arc::new(AtomicUsize::new(0)); + let backup_store: Arc = + Arc::new(DynStoreWrapper(InstrumentedStore::new(StoreBehavior::FailWrite { + attempts: Arc::clone(&attempts), + }))); + tier.set_backup_store(backup_store); + + assert!(tier.write("namespace", "", "key", vec![2]).await.is_err()); + let index = tier.inner.index.as_ref().unwrap(); + assert_eq!(primary_store.read("namespace", "", "key").await.unwrap(), vec![2]); + assert!(index.contains_entry("namespace", "", "key").await.unwrap()); + assert!(matches!( + index.read_journal_entry("namespace", "", "key").await.unwrap().operation, + JournalOperation::Update { value } if value == vec![2] + )); + assert_eq!(attempts.load(Ordering::Relaxed), 1); + } + #[tokio::test] async fn write_recovers_pending_create_under_key_lock_before_classifying() { let (_base_dir, logger, _cleanup) = setup_test_environment(); From e56a9f688bf3ee40d29493c49ea288600c8b6ba2 Mon Sep 17 00:00:00 2001 From: Enigbe Date: Fri, 28 Aug 2026 01:12:10 +0100 Subject: [PATCH 14/14] Synchronize TierStore backups before node startup A newly configured backup starts empty, while a backup restored after primary-only operation may contain missing or stale values. Treating either backup as current could leave it unusable for recovery. In this commit, we: - Add exhaustive key enumeration to the dynamic store interface when tiered storage is enabled, while preserving the existing requirements for builds without tiered storage. - Recover pending journal operations before taking the primary-store snapshot. - Copy all durable primary values into the backup and remove values that no longer exist in primary. - Exclude ephemeral cache values from the backup and preserve TierStore's synchronization metadata during stale-value cleanup. - Write the backup completion record only after every synchronization step succeeds, ensuring interrupted attempts are retried safely. - Run required backup synchronization during node construction and fail the build if synchronization cannot complete. - Implement `MigratableKVStore` for existing test stores so the tiered-storage feature continues compiling and exercising their original behavior. - Add coverage for successful synchronization, backup-only configuration, journal recovery, stale-value removal, metadata preservation, and retries after failures at each stage. Assisted-by: Amp (AI coding agent) --- src/builder.rs | 225 ++++++++++-- src/data_store.rs | 55 ++- src/io/tier_store.rs | 586 +++++++++++++++++++++++++++++++- src/io/utils.rs | 16 +- src/peer_store.rs | 14 +- src/types.rs | 62 +++- src/wallet/mod.rs | 59 +++- src/wallet/persist.rs | 13 +- tests/common/mod.rs | 15 + tests/integration_tests_rust.rs | 22 ++ 10 files changed, 1024 insertions(+), 43 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 73e429393..55493616a 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -41,6 +41,8 @@ use lightning::routing::scoring::{ }; use lightning::sign::{EntropySource, NodeSigner}; use lightning::util::config::HTLCInterceptionFlags; +#[cfg(feature = "storage-tier")] +use lightning::util::persist::MigratableKVStore; use lightning::util::persist::{ KVStore, PaginatedKVStore, CHANNEL_MANAGER_PERSISTENCE_KEY, CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, @@ -73,7 +75,7 @@ use crate::io::fs_store::open_or_migrate_fs_store; #[cfg(feature = "storage-sqlite")] use crate::io::sqlite_store::SqliteStore; #[cfg(feature = "storage-tier")] -use crate::io::tier_store::{setup_index_store, TierStore}; +use crate::io::tier_store::{setup_index_store, BackupSyncStatus, TierStore}; use crate::io::utils::{ read_all_objects, read_event_queue, read_external_pathfinding_scores_from_cache, read_n_objects, read_network_graph, read_node_metrics, read_output_sweeper, read_peer_info, @@ -959,6 +961,7 @@ impl NodeBuilder { /// /// [`set_ephemeral_storage_dir_path`]: Self::set_ephemeral_storage_dir_path /// [`set_backup_storage_dir_path`]: Self::set_backup_storage_dir_path + #[cfg(not(feature = "storage-tier"))] pub fn build_with_store( &self, node_entropy: NodeEntropy, kv_store: S, ) -> Result { @@ -966,6 +969,26 @@ impl NodeBuilder { self.build_with_store_and_logger(node_entropy, kv_store, logger) } + /// Builds a [`Node`] instance according to the options previously configured. + /// + /// The provided `kv_store` will be used as the primary storage backend. Optionally, + /// an ephemeral store for frequently-accessed non-critical data (e.g., network graph, scorer) + /// and a local SQLite backup store for disaster recovery can be configured via + /// [`set_ephemeral_storage_dir_path`] and [`set_backup_storage_dir_path`]. + /// + /// The store must implement [`MigratableKVStore`] so a configured backup can be backfilled or + /// resilvered from the complete set of primary-store keys before the node starts. + /// + /// [`set_ephemeral_storage_dir_path`]: Self::set_ephemeral_storage_dir_path + /// [`set_backup_storage_dir_path`]: Self::set_backup_storage_dir_path + #[cfg(feature = "storage-tier")] + pub fn build_with_store( + &self, node_entropy: NodeEntropy, kv_store: S, + ) -> Result { + let logger = setup_logger(&self.log_writer_config, &self.config)?; + self.build_with_store_and_logger(node_entropy, kv_store, logger) + } + fn setup_runtime(&self, logger: &Arc) -> Result, BuildError> { if let Some(handle) = self.runtime_handle.as_ref() { Ok(Arc::new(Runtime::with_handle(handle.clone(), Arc::clone(logger)))) @@ -977,6 +1000,7 @@ impl NodeBuilder { } } + #[cfg(not(feature = "storage-tier"))] fn build_with_store_and_logger( &self, node_entropy: NodeEntropy, kv_store: S, logger: Arc, ) -> Result { @@ -984,10 +1008,30 @@ impl NodeBuilder { self.build_with_store_runtime_and_logger(node_entropy, kv_store, runtime, logger) } + #[cfg(feature = "storage-tier")] + fn build_with_store_and_logger< + S: PaginatedKVStore + MigratableKVStore + Send + Sync + 'static, + >( + &self, node_entropy: NodeEntropy, kv_store: S, logger: Arc, + ) -> Result { + let runtime = self.setup_runtime(&logger)?; + self.build_with_store_runtime_and_logger(node_entropy, kv_store, runtime, logger) + } + + #[cfg(not(feature = "storage-tier"))] fn build_with_store_runtime_and_logger( &self, node_entropy: NodeEntropy, kv_store: S, runtime: Arc, logger: Arc, ) -> Result { - #[cfg(feature = "storage-tier")] + let store: Arc = Arc::new(DynStoreWrapper(kv_store)); + self.build_with_dyn_store(node_entropy, store, runtime, logger) + } + + #[cfg(feature = "storage-tier")] + fn build_with_store_runtime_and_logger< + S: PaginatedKVStore + MigratableKVStore + Send + Sync + 'static, + >( + &self, node_entropy: NodeEntropy, kv_store: S, runtime: Arc, logger: Arc, + ) -> Result { let store: Arc = { let ts_config = self.tier_store_config.as_ref(); let primary_store = Arc::new(DynStoreWrapper(kv_store)); @@ -1040,15 +1084,27 @@ impl NodeBuilder { tier_store.set_backup_store(backup_store); } } - runtime.block_on(tier_store.initialize_backup_synchronization()).map_err(|e| { - log_error!(logger, "Failed to initialize tier-store backup synchronization: {}", e); - BuildError::KVStoreSetupFailed - })?; + runtime + .block_on(async { + let status = tier_store.initialize_backup_synchronization().await?; + if status == BackupSyncStatus::Required { + tier_store.synchronize_backup().await?; + } + Ok::<(), bitcoin::io::Error>(()) + }) + .map_err(|e| { + log_error!(logger, "Failed to prepare or synchronize tier-store backup: {}", e); + BuildError::KVStoreSetupFailed + })?; Arc::new(DynStoreWrapper(tier_store)) }; - #[cfg(not(feature = "storage-tier"))] - let store: Arc = Arc::new(DynStoreWrapper(kv_store)); + self.build_with_dyn_store(node_entropy, store, runtime, logger) + } + fn build_with_dyn_store( + &self, node_entropy: NodeEntropy, store: Arc, runtime: Arc, + logger: Arc, + ) -> Result { let seed_bytes = node_entropy.to_seed_bytes(); let config = Arc::new(self.config.clone()); @@ -1594,11 +1650,19 @@ impl ArcedNodeBuilder { /// Builds a [`Node`] instance according to the options previously configured. // Note that the generics here don't actually work for Uniffi, but we don't currently expose // this so its not needed. + #[cfg(not(feature = "storage-tier"))] pub fn build_with_store( &self, node_entropy: Arc, kv_store: S, ) -> Result, BuildError> { self.inner.read().expect("lock").build_with_store(*node_entropy, kv_store).map(Arc::new) } + + #[cfg(feature = "storage-tier")] + pub fn build_with_store( + &self, node_entropy: Arc, kv_store: S, + ) -> Result, BuildError> { + self.inner.read().expect("lock").build_with_store(*node_entropy, kv_store).map(Arc::new) + } } /// Builds a [`Node`] instance according to the options previously configured. @@ -2698,11 +2762,13 @@ pub(crate) fn sanitize_alias(alias_str: &str) -> Result { #[cfg(test)] mod tests { use std::future::Future; + #[cfg(all(feature = "storage-tier", not(feature = "uniffi")))] + use std::path::PathBuf; use std::sync::Arc; use lightning::io; use lightning::util::persist::{ - KVStore, PageToken, PaginatedKVStore, PaginatedListResponse, + KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse, CHANNEL_MANAGER_PERSISTENCE_KEY, CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, }; @@ -2710,18 +2776,52 @@ mod tests { use super::{sanitize_alias, BuildError, NodeAlias, NodeBuilder}; use crate::entropy::NodeEntropy; use crate::io::test_utils::InMemoryStore; + #[cfg(all(feature = "storage-tier", not(feature = "uniffi")))] + use crate::io::{ + sqlite_store::{SqliteStore, KV_TABLE_NAME, SQLITE_BACKUP_DB_FILE_NAME}, + test_utils::random_storage_path, + }; use crate::logger::Logger; - struct ChannelManagerReadFailingStore(InMemoryStore); + #[cfg(all(feature = "storage-tier", not(feature = "uniffi")))] + struct CleanupDir(PathBuf); + + #[cfg(all(feature = "storage-tier", not(feature = "uniffi")))] + impl Drop for CleanupDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + #[derive(Clone, Copy)] + enum BuilderStoreBehavior { + #[cfg(all(feature = "storage-tier", not(feature = "uniffi")))] + Normal, + FailChannelManagerRead, + #[cfg(all(feature = "storage-tier", not(feature = "uniffi")))] + FailListAllKeys, + } + + struct BuilderTestStore { + inner: InMemoryStore, + behavior: BuilderStoreBehavior, + } + + impl BuilderTestStore { + fn new(behavior: BuilderStoreBehavior) -> Self { + Self { inner: InMemoryStore::new(), behavior } + } + } - impl KVStore for ChannelManagerReadFailingStore { + impl KVStore for BuilderTestStore { fn read( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, ) -> impl Future, io::Error>> + 'static + Send { - let fail_read = primary_namespace == CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE + let fail_read = matches!(self.behavior, BuilderStoreBehavior::FailChannelManagerRead) + && primary_namespace == CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE && secondary_namespace == CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE && key == CHANNEL_MANAGER_PERSISTENCE_KEY; - let read = KVStore::read(&self.0, primary_namespace, secondary_namespace, key); + let read = KVStore::read(&self.inner, primary_namespace, secondary_namespace, key); async move { if fail_read { Err(io::Error::new(io::ErrorKind::Other, "channel manager read failed")) @@ -2734,29 +2834,29 @@ mod tests { fn write( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, ) -> impl Future> + 'static + Send { - KVStore::write(&self.0, primary_namespace, secondary_namespace, key, buf) + KVStore::write(&self.inner, primary_namespace, secondary_namespace, key, buf) } fn remove( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, ) -> impl Future> + 'static + Send { - KVStore::remove(&self.0, primary_namespace, secondary_namespace, key, lazy) + KVStore::remove(&self.inner, primary_namespace, secondary_namespace, key, lazy) } fn list( &self, primary_namespace: &str, secondary_namespace: &str, ) -> impl Future, io::Error>> + 'static + Send { - KVStore::list(&self.0, primary_namespace, secondary_namespace) + KVStore::list(&self.inner, primary_namespace, secondary_namespace) } } - impl PaginatedKVStore for ChannelManagerReadFailingStore { + impl PaginatedKVStore for BuilderTestStore { fn list_paginated( &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, ) -> impl Future> + 'static + Send { PaginatedKVStore::list_paginated( - &self.0, + &self.inner, primary_namespace, secondary_namespace, page_token, @@ -2764,6 +2864,26 @@ mod tests { } } + impl MigratableKVStore for BuilderTestStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send + { + #[cfg(all(feature = "storage-tier", not(feature = "uniffi")))] + let fail = matches!(self.behavior, BuilderStoreBehavior::FailListAllKeys); + #[cfg(any(not(feature = "storage-tier"), feature = "uniffi"))] + let fail = false; + let list = MigratableKVStore::list_all_keys(&self.inner); + async move { + if fail { + Err(io::Error::new(io::ErrorKind::Other, "list_all_keys failed")) + } else { + list.await + } + } + } + } + #[test] fn channel_manager_read_failure_fails_build() { let builder = NodeBuilder::new(); @@ -2775,13 +2895,80 @@ mod tests { let result = builder.build_with_store_and_logger( node_entropy, - ChannelManagerReadFailingStore(InMemoryStore::new()), + BuilderTestStore::new(BuilderStoreBehavior::FailChannelManagerRead), logger, ); assert!(matches!(result, Err(BuildError::ReadFailed))); } + #[cfg(all(feature = "storage-tier", not(feature = "uniffi")))] + #[test] + fn builder_synchronizes_a_configured_backup_before_returning() { + let base_dir = random_storage_path(); + let _cleanup = CleanupDir(base_dir.clone()); + let storage_dir = base_dir.join("node"); + let backup_dir = base_dir.join("backup"); + let runtime = tokio::runtime::Runtime::new().unwrap(); + let primary_store = BuilderTestStore::new(BuilderStoreBehavior::Normal); + runtime.block_on(primary_store.write("namespace", "", "current", vec![1])).unwrap(); + let backup_store = SqliteStore::new( + backup_dir.clone(), + Some(SQLITE_BACKUP_DB_FILE_NAME.to_string()), + Some(KV_TABLE_NAME.to_string()), + ) + .unwrap(); + runtime + .block_on(async { backup_store.write("namespace", "", "stale", vec![2]).await }) + .unwrap(); + drop(backup_store); + drop(runtime); + + let mut builder = NodeBuilder::new(); + builder + .set_storage_dir_path(storage_dir.to_string_lossy().into_owned()) + .set_backup_storage_dir_path(backup_dir.to_string_lossy().into_owned()); + let node = builder + .build_with_store(NodeEntropy::from_seed_bytes([42; 64]), primary_store) + .unwrap(); + + let backup_store = SqliteStore::new( + backup_dir, + Some(SQLITE_BACKUP_DB_FILE_NAME.to_string()), + Some(KV_TABLE_NAME.to_string()), + ) + .unwrap(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + assert_eq!( + runtime + .block_on(async { backup_store.read("namespace", "", "current").await }) + .unwrap(), + vec![1] + ); + assert!(runtime + .block_on(async { backup_store.read("namespace", "", "stale").await }) + .is_err()); + drop(node); + } + + #[cfg(all(feature = "storage-tier", not(feature = "uniffi")))] + #[test] + fn builder_fails_when_backup_synchronization_cannot_list_primary_keys() { + let base_dir = random_storage_path(); + let _cleanup = CleanupDir(base_dir.clone()); + let mut builder = NodeBuilder::new(); + builder + .set_storage_dir_path(base_dir.join("node").to_string_lossy().into_owned()) + .set_backup_storage_dir_path(base_dir.join("backup").to_string_lossy().into_owned()); + + let result = builder.build_with_store( + NodeEntropy::from_seed_bytes([42; 64]), + BuilderTestStore::new(BuilderStoreBehavior::FailListAllKeys), + ); + + assert!(matches!(result, Err(BuildError::KVStoreSetupFailed))); + } + #[test] fn sanitize_empty_node_alias() { // Empty node alias diff --git a/src/data_store.rs b/src/data_store.rs index a9fe0d0f5..b424615d5 100644 --- a/src/data_store.rs +++ b/src/data_store.rs @@ -750,7 +750,9 @@ mod tests { use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Duration; - use lightning::util::persist::{PageToken, PaginatedKVStore, PaginatedListResponse}; + use lightning::util::persist::{ + MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse, + }; use lightning::util::test_utils::TestLogger; use lightning::{impl_writeable_tlv_based, io}; use tokio::sync::Notify; @@ -902,6 +904,16 @@ mod tests { } } + impl MigratableKVStore for FailingStore { + fn list_all_keys( + &self, + ) -> impl std::future::Future, io::Error>> + + 'static + + Send { + async { Err(io::Error::new(io::ErrorKind::Other, "list_all_keys failed")) } + } + } + fn new_failing_data_store(objects: Vec) -> DataStore> { let store: Arc = Arc::new(DynStoreWrapper(FailingStore)); let logger = Arc::new(TestLogger::new()); @@ -968,6 +980,16 @@ mod tests { } } + impl MigratableKVStore for GatedStore { + fn list_all_keys( + &self, + ) -> impl std::future::Future, io::Error>> + + 'static + + Send { + MigratableKVStore::list_all_keys(&self.inner) + } + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn readers_wait_for_in_flight_writes() { let write_parked = Arc::new(Notify::new()); @@ -1345,6 +1367,17 @@ mod tests { } } + impl MigratableKVStore for CountingStore { + fn list_all_keys( + &self, + ) -> impl std::future::Future, io::Error>> + + 'static + + Send { + self.lists.fetch_add(1, Ordering::Relaxed); + MigratableKVStore::list_all_keys(&self.inner) + } + } + /// A store whose writes and removals can be made to fail on demand, while reads keep working. /// /// Note a store that fails *reads* would be useless for testing the write paths of a bounded @@ -1404,6 +1437,16 @@ mod tests { } } + impl MigratableKVStore for WriteFailingStore { + fn list_all_keys( + &self, + ) -> impl std::future::Future, io::Error>> + + 'static + + Send { + MigratableKVStore::list_all_keys(&self.inner) + } + } + /// Returns a bounded store of the given capacity, together with a handle on the underlying /// `KVStore`, and the ids of `num_objects` objects inserted through it. /// @@ -1802,6 +1845,16 @@ mod tests { } } + impl MigratableKVStore for PhantomKeyStore { + fn list_all_keys( + &self, + ) -> impl std::future::Future, io::Error>> + + 'static + + Send { + MigratableKVStore::list_all_keys(&self.inner) + } + } + /// Sweeps every page and returns the objects in the order they were listed. async fn list_all_pages( data_store: &DataStore, P>, diff --git a/src/io/tier_store.rs b/src/io/tier_store.rs index 2cb6b5b09..22bdea94b 100644 --- a/src/io/tier_store.rs +++ b/src/io/tier_store.rs @@ -5,7 +5,7 @@ // http://opensource.org/licenses/MIT>, at your option. You may not use this file except in // accordance with one or both of these licenses. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::future::Future; use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; @@ -14,9 +14,9 @@ use std::sync::{Arc, Mutex}; use bitcoin::hashes::{sha256, Hash, HashEngine}; use bitcoin::hex::{DisplayHex, FromHex}; use lightning::util::persist::{ - KVStore, PageToken, PaginatedKVStore, PaginatedListResponse, NETWORK_GRAPH_PERSISTENCE_KEY, - NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, SCORER_PERSISTENCE_KEY, - SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse, + NETWORK_GRAPH_PERSISTENCE_KEY, NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_KEY, SCORER_PERSISTENCE_PRIMARY_NAMESPACE, }; use lightning::util::ser::{Readable, Writeable}; use lightning::{impl_writeable_tlv_based, impl_writeable_tlv_based_enum, io, log_error}; @@ -63,7 +63,6 @@ impl BackupSyncCompletion { } /// Encodes both fixed-width identities into the value persisted in the backup store. - #[cfg(test)] fn encode(self) -> Vec { let mut encoded = Vec::with_capacity(GENERATION_ID_LEN + INDEX_DATABASE_ID_LEN); encoded.extend_from_slice(&self.primary_generation_id); @@ -548,6 +547,33 @@ impl TierStoreIndex { Ok(keys) } + /// Returns every pending operation stored in the index, validating its encoded location. + async fn list_all_journal_entries(&self) -> io::Result> { + let mut entries = Vec::new(); + for (primary_namespace, secondary_namespace, key) in + MigratableKVStore::list_all_keys(self.store.as_ref()).await? + { + if primary_namespace != INDEX_JOURNAL_PRIMARY_NAMESPACE { + continue; + } + let encoded = + KVStore::read(self.store.as_ref(), &primary_namespace, &secondary_namespace, &key) + .await?; + let entry = JournalEntry::deserialize(&encoded)?; + if TierStoreIndex::namespace_id(&entry.primary_namespace, &entry.secondary_namespace) + != secondary_namespace + || entry.key != key + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Journal entry identity does not match its location", + )); + } + entries.push(entry); + } + Ok(entries) + } + /// Clears a pending operation after all of its intended effects are durable. async fn remove_journal_entry( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, @@ -693,6 +719,21 @@ impl TierStore { pub(crate) async fn initialize_backup_synchronization(&self) -> io::Result { self.inner.initialize_backup_synchronization().await } + + /// Makes a configured backup match the primary store before normal node operation begins. + /// + /// Pending journal entries are recovered first. Current primary values are then copied, stale + /// backup values are removed, and the backup's completion generation is updated last. The old + /// completion generation remains unchanged if any earlier step fails. + pub(crate) async fn synchronize_backup(&mut self) -> io::Result<()> { + let inner = Arc::get_mut(&mut self.inner).ok_or_else(|| { + io::Error::new( + io::ErrorKind::Other, + "TierStore must be exclusively owned during backup synchronization", + ) + })?; + inner.synchronize_backup().await + } } pub(crate) async fn setup_index_store(data_dir: PathBuf) -> io::Result { @@ -791,6 +832,15 @@ impl PaginatedKVStore for TierStore { } } +impl MigratableKVStore for TierStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send { + let inner = Arc::clone(&self.inner); + async move { inner.list_all_keys_internal().await } + } +} + struct TierStoreInner { /// The authoritative store for durable data. primary_store: Arc, @@ -974,7 +1024,6 @@ impl TierStoreInner { /// previous completion generation unchanged and will be retried on the next startup. /// /// Returns an error when no backup is configured or the completion record cannot be persisted. - #[cfg(test)] async fn write_backup_sync_completion( &self, completion: BackupSyncCompletion, ) -> io::Result<()> { @@ -991,6 +1040,81 @@ impl TierStoreInner { .await } + /// Copies all durable primary data into a stale backup and removes backup-only data. + async fn synchronize_backup(&self) -> io::Result<()> { + match *self.backup_sync_status.lock().expect("lock") { + Some(BackupSyncStatus::Synchronized) => return Ok(()), + Some(BackupSyncStatus::Required) => {}, + Some(BackupSyncStatus::NotConfigured) | None => { + return Err(io::Error::new( + io::ErrorKind::Other, + "Backup synchronization is not required or has not been initialized", + )); + }, + } + let backup_store = self.backup_store.as_ref().ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "Backup store is not configured") + })?; + self.recover_all_journal_entries().await?; + let completion = self.current_backup_sync_completion().await?; + let primary_keys: HashSet<_> = + MigratableKVStore::list_all_keys(self.primary_store.as_ref()) + .await? + .into_iter() + .filter(|(primary_namespace, _, _)| { + primary_namespace != BACKUP_SYNC_PRIMARY_NAMESPACE + }) + .filter(|(primary_namespace, secondary_namespace, key)| { + self.ephemeral_store.is_none() + || !is_ephemeral_cached_key(primary_namespace, secondary_namespace, key) + }) + .collect(); + + for (primary_namespace, secondary_namespace, key) in &primary_keys { + let value = KVStore::read( + self.primary_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + ) + .await?; + KVStore::write( + backup_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + value, + ) + .await?; + } + + for (primary_namespace, secondary_namespace, key) in + MigratableKVStore::list_all_keys(backup_store.as_ref()).await? + { + if primary_namespace == BACKUP_SYNC_PRIMARY_NAMESPACE { + continue; + } + if !primary_keys.contains(&( + primary_namespace.clone(), + secondary_namespace.clone(), + key.clone(), + )) { + KVStore::remove( + backup_store.as_ref(), + &primary_namespace, + &secondary_namespace, + &key, + false, + ) + .await?; + } + } + + self.write_backup_sync_completion(completion).await?; + *self.backup_sync_status.lock().expect("lock") = Some(BackupSyncStatus::Synchronized); + Ok(()) + } + fn get_new_version_and_lock_ref(&self, locking_key: String) -> (Arc>, u64) { let version = self.next_write_version.fetch_add(1, Ordering::Relaxed); if version == u64::MAX { @@ -1519,6 +1643,25 @@ impl TierStoreInner { self.list_value_stores(&primary_namespace, &secondary_namespace).await } + /// Returns every logical TierStore key for data migration. + async fn list_all_keys_internal(&self) -> io::Result> { + self.recover_all_journal_entries().await?; + let mut keys: HashSet<_> = MigratableKVStore::list_all_keys(self.primary_store.as_ref()) + .await? + .into_iter() + .filter(|(primary_namespace, _, _)| primary_namespace != BACKUP_SYNC_PRIMARY_NAMESPACE) + .collect(); + + if let Some(ephemeral_store) = self.ephemeral_store.as_ref() { + for key in MigratableKVStore::list_all_keys(ephemeral_store.as_ref()).await? { + if is_ephemeral_cached_key(&key.0, &key.1, &key.2) { + keys.insert(key); + } + } + } + Ok(keys.into_iter().collect()) + } + /// Imports a namespace's existing primary-store keys and makes its index authoritative. /// /// Primary pagination returns keys from newest to oldest, so the complete result is reversed @@ -1720,6 +1863,34 @@ impl TierStoreInner { Ok(()) } + /// Recovers every journal entry before taking a complete migration snapshot. + async fn recover_all_journal_entries(&self) -> io::Result<()> { + let Some(index) = self.index.as_ref() else { + return Ok(()); + }; + for entry in index.list_all_journal_entries().await? { + let locking_key = self.build_locking_key( + &entry.primary_namespace, + &entry.secondary_namespace, + &entry.key, + ); + let lock_ref = self.get_lock_ref(locking_key.clone()); + let result: io::Result<()> = async { + let _guard = lock_ref.lock().await; + self.recover_key_locked( + &entry.primary_namespace, + &entry.secondary_namespace, + &entry.key, + ) + .await + } + .await; + self.clean_locks(&lock_ref, locking_key); + result?; + } + Ok(()) + } + /// Recovers one key while its per-key operation lock is held by the caller. async fn recover_key_locked( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, @@ -1922,6 +2093,20 @@ mod tests { tier.set_index_store(TierStoreIndex::from_store(store)); } + fn test_backup_sync_completion( + primary_generation_id: [u8; GENERATION_ID_LEN], + ) -> BackupSyncCompletion { + BackupSyncCompletion::new(primary_generation_id, [1; INDEX_DATABASE_ID_LEN]) + } + + async fn read_test_backup_sync_completion(store: &DynStore) -> BackupSyncCompletion { + let encoded = store + .read(BACKUP_SYNC_PRIMARY_NAMESPACE, "", BACKUP_SYNC_COMPLETED_GENERATION_KEY) + .await + .unwrap(); + BackupSyncCompletion::decode(encoded).unwrap() + } + #[test] fn backup_sync_completion_roundtrips_and_rejects_invalid_length() { let completion = @@ -2065,6 +2250,331 @@ mod tests { ); } + #[tokio::test] + async fn backup_synchronization_copies_current_data_removes_stale_data_and_recovers_journal() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + primary_store.write("namespace", "", "current", vec![2]).await.unwrap(); + primary_store.write("namespace", "", "new", vec![3]).await.unwrap(); + backup_store.write("namespace", "", "current", vec![1]).await.unwrap(); + backup_store.write("namespace", "", "stale", vec![4]).await.unwrap(); + backup_store + .write(BACKUP_SYNC_PRIMARY_NAMESPACE, "", "other_metadata", vec![6]) + .await + .unwrap(); + + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + tier.set_backup_store(Arc::clone(&backup_store)); + set_test_index_store(&mut tier); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + let expected_completion = tier.inner.current_backup_sync_completion().await.unwrap(); + let pending = JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: "pending".to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation: JournalOperation::Create { value: vec![5] }, + }; + tier.inner.index.as_ref().unwrap().write_journal_entry(&pending).await.unwrap(); + + tier.synchronize_backup().await.unwrap(); + + assert_eq!(backup_store.read("namespace", "", "current").await.unwrap(), vec![2]); + assert_eq!(backup_store.read("namespace", "", "new").await.unwrap(), vec![3]); + assert_eq!(backup_store.read("namespace", "", "pending").await.unwrap(), vec![5]); + assert!(backup_store.read("namespace", "", "stale").await.is_err()); + assert_eq!( + backup_store.read(BACKUP_SYNC_PRIMARY_NAMESPACE, "", "other_metadata").await.unwrap(), + vec![6] + ); + assert!(tier + .inner + .index + .as_ref() + .unwrap() + .read_journal_entry("namespace", "", "pending") + .await + .is_err()); + assert_eq!( + read_test_backup_sync_completion(backup_store.as_ref()).await, + expected_completion + ); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Synchronized + ); + } + + #[tokio::test] + async fn backup_synchronization_excludes_ephemeral_cache_values() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + for store in [&primary_store, &backup_store] { + store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![1], + ) + .await + .unwrap(); + } + + let mut tier = setup_tier_store(primary_store, logger); + tier.set_backup_store(Arc::clone(&backup_store)); + tier.set_ephemeral_store(ephemeral_store); + set_test_index_store(&mut tier); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + + tier.synchronize_backup().await.unwrap(); + + assert!(backup_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err()); + } + + #[tokio::test] + async fn failed_stale_cleanup_preserves_backup_completion_generation() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let remove_attempts = Arc::new(AtomicUsize::new(0)); + let backup_store: Arc = + Arc::new(DynStoreWrapper(InstrumentedStore::new(StoreBehavior::FailRemoveOnce { + attempts: Arc::clone(&remove_attempts), + }))); + primary_store.write("namespace", "", "current", vec![2]).await.unwrap(); + backup_store.write("namespace", "", "stale", vec![1]).await.unwrap(); + let old_completion = test_backup_sync_completion([7; GENERATION_ID_LEN]); + backup_store + .write( + BACKUP_SYNC_PRIMARY_NAMESPACE, + "", + BACKUP_SYNC_COMPLETED_GENERATION_KEY, + old_completion.encode(), + ) + .await + .unwrap(); + + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + tier.set_backup_store(Arc::clone(&backup_store)); + set_test_index_store(&mut tier); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + let expected_completion = tier.inner.current_backup_sync_completion().await.unwrap(); + + assert!(tier.synchronize_backup().await.is_err()); + + assert_eq!(read_test_backup_sync_completion(backup_store.as_ref()).await, old_completion); + assert_eq!(remove_attempts.load(Ordering::Relaxed), 1); + + tier.synchronize_backup().await.unwrap(); + + assert!(backup_store.read("namespace", "", "stale").await.is_err()); + assert_eq!( + read_test_backup_sync_completion(backup_store.as_ref()).await, + expected_completion + ); + } + + #[tokio::test] + async fn failed_journal_recovery_preserves_backup_completion_generation_and_can_retry() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let write_attempts = Arc::new(AtomicUsize::new(0)); + let backup_store: Arc = + Arc::new(DynStoreWrapper(InstrumentedStore::new(StoreBehavior::FailWriteOnce { + primary_namespace: "namespace", + key: "pending", + attempts: Arc::clone(&write_attempts), + }))); + let old_completion = test_backup_sync_completion([7; GENERATION_ID_LEN]); + backup_store + .write( + BACKUP_SYNC_PRIMARY_NAMESPACE, + "", + BACKUP_SYNC_COMPLETED_GENERATION_KEY, + old_completion.encode(), + ) + .await + .unwrap(); + + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + tier.set_backup_store(Arc::clone(&backup_store)); + set_test_index_store(&mut tier); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + let expected_completion = tier.inner.current_backup_sync_completion().await.unwrap(); + let pending = JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: "pending".to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation: JournalOperation::Create { value: vec![1] }, + }; + tier.inner.index.as_ref().unwrap().write_journal_entry(&pending).await.unwrap(); + + assert!(tier.synchronize_backup().await.is_err()); + + assert_eq!(write_attempts.load(Ordering::Relaxed), 1); + assert_eq!(primary_store.read("namespace", "", "pending").await.unwrap(), vec![1]); + assert_eq!( + tier.inner + .index + .as_ref() + .unwrap() + .read_journal_entry("namespace", "", "pending") + .await + .unwrap(), + pending + ); + assert_eq!(read_test_backup_sync_completion(backup_store.as_ref()).await, old_completion); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + + tier.synchronize_backup().await.unwrap(); + + assert_eq!(backup_store.read("namespace", "", "pending").await.unwrap(), vec![1]); + assert!(tier + .inner + .index + .as_ref() + .unwrap() + .read_journal_entry("namespace", "", "pending") + .await + .is_err()); + assert_eq!( + read_test_backup_sync_completion(backup_store.as_ref()).await, + expected_completion + ); + } + + #[tokio::test] + async fn failed_primary_copy_preserves_backup_completion_generation_and_can_retry() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + primary_store.write("namespace", "", "current", vec![1]).await.unwrap(); + let write_attempts = Arc::new(AtomicUsize::new(0)); + let backup_store: Arc = + Arc::new(DynStoreWrapper(InstrumentedStore::new(StoreBehavior::FailWriteOnce { + primary_namespace: "namespace", + key: "current", + attempts: Arc::clone(&write_attempts), + }))); + let old_completion = test_backup_sync_completion([7; GENERATION_ID_LEN]); + backup_store + .write( + BACKUP_SYNC_PRIMARY_NAMESPACE, + "", + BACKUP_SYNC_COMPLETED_GENERATION_KEY, + old_completion.encode(), + ) + .await + .unwrap(); + + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + tier.set_backup_store(Arc::clone(&backup_store)); + set_test_index_store(&mut tier); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + let expected_completion = tier.inner.current_backup_sync_completion().await.unwrap(); + + assert!(tier.synchronize_backup().await.is_err()); + + assert_eq!(write_attempts.load(Ordering::Relaxed), 1); + assert!(backup_store.read("namespace", "", "current").await.is_err()); + assert_eq!(read_test_backup_sync_completion(backup_store.as_ref()).await, old_completion); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + + tier.synchronize_backup().await.unwrap(); + + assert_eq!(backup_store.read("namespace", "", "current").await.unwrap(), vec![1]); + assert_eq!( + read_test_backup_sync_completion(backup_store.as_ref()).await, + expected_completion + ); + } + + #[tokio::test] + async fn failed_completion_write_preserves_previous_generation_and_can_retry() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + primary_store.write("namespace", "", "current", vec![1]).await.unwrap(); + let write_attempts = Arc::new(AtomicUsize::new(0)); + let backup_store = InstrumentedStore::new(StoreBehavior::FailWriteOnce { + primary_namespace: BACKUP_SYNC_PRIMARY_NAMESPACE, + key: BACKUP_SYNC_COMPLETED_GENERATION_KEY, + attempts: Arc::clone(&write_attempts), + }); + let old_completion = test_backup_sync_completion([7; GENERATION_ID_LEN]); + backup_store + .inner + .write( + BACKUP_SYNC_PRIMARY_NAMESPACE, + "", + BACKUP_SYNC_COMPLETED_GENERATION_KEY, + old_completion.encode(), + ) + .await + .unwrap(); + backup_store.inner.write("namespace", "", "stale", vec![2]).await.unwrap(); + let backup_store: Arc = Arc::new(DynStoreWrapper(backup_store)); + + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + tier.set_backup_store(Arc::clone(&backup_store)); + set_test_index_store(&mut tier); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + let expected_completion = tier.inner.current_backup_sync_completion().await.unwrap(); + + assert!(tier.synchronize_backup().await.is_err()); + + assert_eq!(write_attempts.load(Ordering::Relaxed), 1); + assert_eq!(backup_store.read("namespace", "", "current").await.unwrap(), vec![1]); + assert!(backup_store.read("namespace", "", "stale").await.is_err()); + assert_eq!(read_test_backup_sync_completion(backup_store.as_ref()).await, old_completion); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + + tier.synchronize_backup().await.unwrap(); + + assert_eq!( + read_test_backup_sync_completion(backup_store.as_ref()).await, + expected_completion + ); + } + #[test] fn journal_entry_roundtrips() { for operation in [ @@ -2861,11 +3371,7 @@ mod tests { .await .unwrap(); assert_ne!(rotated_generation_id, completion.primary_generation_id); - let encoded_completion = backup_store - .read(BACKUP_SYNC_PRIMARY_NAMESPACE, "", BACKUP_SYNC_COMPLETED_GENERATION_KEY) - .await - .unwrap(); - assert_eq!(BackupSyncCompletion::decode(encoded_completion).unwrap(), completion); + assert_eq!(read_test_backup_sync_completion(backup_store.as_ref()).await, completion); } #[tokio::test] @@ -3188,8 +3694,18 @@ mod tests { enum StoreBehavior { FailList, - FailWrite { attempts: Arc }, + FailWrite { + attempts: Arc, + }, + FailWriteOnce { + primary_namespace: &'static str, + key: &'static str, + attempts: Arc, + }, FailRemove, + FailRemoveOnce { + attempts: Arc, + }, } /// A store that injects selected failures or synchronization points while delegating other @@ -3214,11 +3730,28 @@ mod tests { fn write( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, ) -> impl Future> + 'static + Send { - let write = if let StoreBehavior::FailWrite { attempts } = &self.behavior { - attempts.fetch_add(1, Ordering::Relaxed); - None - } else { - Some(KVStore::write(&self.inner, primary_namespace, secondary_namespace, key, buf)) + let write = match &self.behavior { + StoreBehavior::FailWrite { attempts } => { + attempts.fetch_add(1, Ordering::Relaxed); + None + }, + StoreBehavior::FailWriteOnce { + primary_namespace: failed_namespace, + key: failed_key, + attempts, + } if primary_namespace == *failed_namespace + && key == *failed_key + && attempts.fetch_add(1, Ordering::Relaxed) == 0 => + { + None + }, + _ => Some(KVStore::write( + &self.inner, + primary_namespace, + secondary_namespace, + key, + buf, + )), }; async move { match write { @@ -3232,6 +3765,11 @@ mod tests { ) -> impl Future> + 'static + Send { let remove = match &self.behavior { StoreBehavior::FailRemove => None, + StoreBehavior::FailRemoveOnce { attempts } + if attempts.fetch_add(1, Ordering::Relaxed) == 0 => + { + None + }, _ => Some(KVStore::remove( &self.inner, primary_namespace, @@ -3252,7 +3790,10 @@ mod tests { ) -> impl Future, io::Error>> + 'static + Send { let list = match &self.behavior { StoreBehavior::FailList => None, - StoreBehavior::FailWrite { .. } | StoreBehavior::FailRemove => { + StoreBehavior::FailWrite { .. } + | StoreBehavior::FailWriteOnce { .. } + | StoreBehavior::FailRemove + | StoreBehavior::FailRemoveOnce { .. } => { Some(KVStore::list(&self.inner, primary_namespace, secondary_namespace)) }, }; @@ -3286,6 +3827,15 @@ mod tests { } } + impl MigratableKVStore for InstrumentedStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send + { + MigratableKVStore::list_all_keys(&self.inner) + } + } + #[tokio::test] async fn write_read_list_remove() { let (base_dir, logger, _cleanup) = setup_test_environment(); diff --git a/src/io/utils.rs b/src/io/utils.rs index 30fc0c62d..004590284 100644 --- a/src/io/utils.rs +++ b/src/io/utils.rs @@ -780,7 +780,9 @@ mod read_objects_tests { use std::sync::Arc; use lightning::impl_writeable_tlv_based; - use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; + use lightning::util::persist::{ + KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse, + }; use lightning::util::ser::Writeable; use lightning::util::test_utils::TestLogger; @@ -980,6 +982,18 @@ mod read_objects_tests { } } + impl MigratableKVStore for StuckTokenStore { + fn list_all_keys( + &self, + ) -> impl std::future::Future< + Output = Result, lightning::io::Error>, + > + + 'static + + Send { + MigratableKVStore::list_all_keys(&self.inner) + } + } + #[tokio::test] async fn a_page_token_that_does_not_advance_is_an_error() { let stuck = StuckTokenStore { diff --git a/src/peer_store.rs b/src/peer_store.rs index 8345bf711..716e6d600 100644 --- a/src/peer_store.rs +++ b/src/peer_store.rs @@ -174,7 +174,9 @@ mod tests { use std::sync::Arc; use bitcoin::io; - use lightning::util::persist::{PageToken, PaginatedKVStore, PaginatedListResponse}; + use lightning::util::persist::{ + MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse, + }; use lightning::util::test_utils::TestLogger; use super::*; @@ -219,6 +221,16 @@ mod tests { } } + impl MigratableKVStore for FailingStore { + fn list_all_keys( + &self, + ) -> impl std::future::Future, io::Error>> + + 'static + + Send { + async { Err(io::Error::new(io::ErrorKind::Other, "list_all_keys failed")) } + } + } + #[tokio::test] async fn peer_info_persistence() { let store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); diff --git a/src/types.rs b/src/types.rs index 1a61daa10..df292b369 100644 --- a/src/types.rs +++ b/src/types.rs @@ -25,6 +25,8 @@ use lightning::routing::gossip; use lightning::routing::router::DefaultRouter; use lightning::routing::scoring::{CombinedScorer, ProbabilisticScoringFeeParameters}; use lightning::sign::InMemorySigner; +#[cfg(feature = "storage-tier")] +use lightning::util::persist::MigratableKVStore; use lightning::util::persist::{ KVStore, MonitorUpdatingPersisterAsync, PageToken, PaginatedKVStore, PaginatedListResponse, }; @@ -72,6 +74,16 @@ pub(crate) trait DynStoreTrait: Send + Sync { dyn Future> + Send + 'static, >, >; + #[cfg(feature = "storage-tier")] + fn list_all_keys_async( + &self, + ) -> Pin< + Box< + dyn Future, bitcoin::io::Error>> + + Send + + 'static, + >, + >; } impl<'a> KVStore for dyn DynStoreTrait + 'a { @@ -113,6 +125,16 @@ impl<'a> PaginatedKVStore for dyn DynStoreTrait + 'a { } } +#[cfg(feature = "storage-tier")] +impl<'a> MigratableKVStore for dyn DynStoreTrait + 'a { + fn list_all_keys( + &self, + ) -> impl Future, bitcoin::io::Error>> + Send + 'static + { + DynStoreTrait::list_all_keys_async(self) + } +} + pub(crate) type DynStore = dyn DynStoreTrait; // Newtype wrapper that implements `KVStore` for `Arc`. This is needed because `KVStore` @@ -161,9 +183,34 @@ impl PaginatedKVStore for DynStoreRef { } } +#[cfg(feature = "storage-tier")] +impl MigratableKVStore for DynStoreRef { + fn list_all_keys( + &self, + ) -> impl Future, bitcoin::io::Error>> + Send + 'static + { + DynStoreTrait::list_all_keys_async(&*self.0) + } +} + pub(crate) struct DynStoreWrapper(pub(crate) T); -impl DynStoreTrait for DynStoreWrapper { +// With tiered storage enabled, dynamic stores must support exhaustive key listing so backup +// synchronization can copy every primary value. Without tiered storage, stores only need to +// implement `PaginatedKVStore`, preserving compatibility for existing custom stores. +#[cfg(not(feature = "storage-tier"))] +trait DynStoreSource: PaginatedKVStore + Send + Sync {} + +#[cfg(feature = "storage-tier")] +trait DynStoreSource: PaginatedKVStore + MigratableKVStore + Send + Sync {} + +#[cfg(not(feature = "storage-tier"))] +impl DynStoreSource for T {} + +#[cfg(feature = "storage-tier")] +impl DynStoreSource for T {} + +impl DynStoreTrait for DynStoreWrapper { fn read_async( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, ) -> Pin, bitcoin::io::Error>> + Send + 'static>> { @@ -202,6 +249,19 @@ impl DynStoreTrait for DynStoreWrapper { page_token, )) } + + #[cfg(feature = "storage-tier")] + fn list_all_keys_async( + &self, + ) -> Pin< + Box< + dyn Future, bitcoin::io::Error>> + + Send + + 'static, + >, + > { + Box::pin(MigratableKVStore::list_all_keys(&self.0)) + } } pub(crate) type AsyncPersister = MonitorUpdatingPersisterAsync< diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index b9c12b4a7..d6d340aed 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -2695,7 +2695,9 @@ mod tests { use bitcoin::hashes::Hash; use bitcoin::Network; use lightning::io; - use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; + use lightning::util::persist::{ + KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse, + }; use super::*; #[cfg(all(not(feature = "chain-esplora"), feature = "chain-electrum"))] @@ -2783,6 +2785,15 @@ mod tests { } } + impl MigratableKVStore for FailSwitchStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send + { + MigratableKVStore::list_all_keys(&*self.inner) + } + } + /// Constructs a `Wallet` around the given store, either creating a fresh BDK wallet or /// loading the one the store already holds. async fn new_test_wallet(store: Arc, load_existing: bool) -> Arc { @@ -3076,6 +3087,16 @@ mod tests { } } + impl MigratableKVStore for SnapshotStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send + { + let keys = self.data.lock().unwrap().keys().cloned().collect(); + async move { Ok(keys) } + } + } + #[tokio::test] async fn pool_survives_a_crash_at_any_point_during_refill() { let snapshot_store = SnapshotStore::new(); @@ -3190,6 +3211,15 @@ mod tests { } } + impl MigratableKVStore for GatedStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send + { + MigratableKVStore::list_all_keys(&*self.inner) + } + } + #[tokio::test] async fn aborting_a_refill_mid_persist_loses_no_reveals() { let gated_store = GatedStore::new(); @@ -3398,6 +3428,15 @@ mod tests { } } + impl MigratableKVStore for RecordOnlyStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send + { + MigratableKVStore::list_all_keys(&*self.inner) + } + } + #[tokio::test] async fn failed_get_new_address_leaves_the_pool_record_covering_the_pool() { let record_store = RecordOnlyStore::new(); @@ -3514,6 +3553,15 @@ mod tests { } } + impl MigratableKVStore for RecordFailStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send + { + MigratableKVStore::list_all_keys(&*self.inner) + } + } + #[tokio::test] async fn crash_after_a_failed_record_write_re_derives_the_same_indices() { let record_store = RecordFailStore::new(); @@ -3660,6 +3708,15 @@ mod tests { } } + impl MigratableKVStore for NamespaceGatedStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send + { + MigratableKVStore::list_all_keys(&*self.inner) + } + } + fn dummy_tx() -> Transaction { Transaction { version: bitcoin::transaction::Version::TWO, diff --git a/src/wallet/persist.rs b/src/wallet/persist.rs index 6384d0fce..80c4c973a 100644 --- a/src/wallet/persist.rs +++ b/src/wallet/persist.rs @@ -314,7 +314,9 @@ mod tests { use bdk_wallet::{AsyncWalletPersister, ChangeSet, Wallet as BdkWallet}; use bitcoin::Network; use lightning::io; - use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; + use lightning::util::persist::{ + KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse, + }; use super::KVStoreWalletPersister; use crate::io::test_utils::InMemoryStore; @@ -378,6 +380,15 @@ mod tests { } } + impl MigratableKVStore for GatedStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send + { + MigratableKVStore::list_all_keys(&*self.inner) + } + } + #[tokio::test] async fn retains_pending_changes_when_persist_is_cancelled() { let gated_store = GatedStore { diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 607edaed8..c0a770baa 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -60,6 +60,8 @@ use ldk_node::{ use lightning::io; use lightning::ln::msgs::SocketAddress; use lightning::routing::gossip::NodeAlias; +#[cfg(feature = "storage-tier")] +use lightning::util::persist::MigratableKVStore; use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; use lightning_invoice::{Bolt11InvoiceDescription, Description}; use lightning_persister::fs_store::v2::FilesystemStoreV2; @@ -1954,6 +1956,19 @@ impl PaginatedKVStore for TestSyncStore { } } +#[cfg(feature = "storage-tier")] +impl MigratableKVStore for TestSyncStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send { + let inner = Arc::clone(&self.inner); + async move { + let _guard = inner.serializer.read().await; + MigratableKVStore::list_all_keys(&inner.test_store).await + } + } +} + struct TestSyncStoreInner { serializer: tokio::sync::RwLock<()>, test_store: InMemoryStore, diff --git a/tests/integration_tests_rust.rs b/tests/integration_tests_rust.rs index cf9fbbd07..9249cddcc 100644 --- a/tests/integration_tests_rust.rs +++ b/tests/integration_tests_rust.rs @@ -49,6 +49,8 @@ use ldk_node::{BuildError, Builder, Event, Node, NodeError, ReserveType}; use lightning::ln::channelmanager::PaymentId; use lightning::routing::gossip::{NodeAlias, NodeId}; use lightning::routing::router::RouteParametersConfig; +#[cfg(feature = "storage-tier")] +use lightning::util::persist::MigratableKVStore; use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; use lightning_invoice::{Bolt11InvoiceDescription, Description}; use lightning_types::payment::{PaymentHash, PaymentPreimage}; @@ -144,6 +146,16 @@ impl PaginatedKVStore for ContendedStore { } } +#[cfg(feature = "storage-tier")] +impl MigratableKVStore for ContendedStore { + fn list_all_keys( + &self, + ) -> impl Future, lightning::io::Error>> + 'static + Send + { + MigratableKVStore::list_all_keys(&*self.inner) + } +} + #[test] fn wallet_store_contention_does_not_stall_runtime() { let (ready_sender, ready_receiver) = mpsc::sync_channel(1); @@ -299,6 +311,16 @@ impl PaginatedKVStore for WalletPersistGatedStore { } } +#[cfg(feature = "storage-tier")] +impl MigratableKVStore for WalletPersistGatedStore { + fn list_all_keys( + &self, + ) -> impl Future, lightning::io::Error>> + 'static + Send + { + MigratableKVStore::list_all_keys(&*self.inner) + } +} + // LDK invokes the sync `SignerProvider::get_shutdown_scriptpubkey` callback on a runtime worker // thread while holding channel locks when a node accepts (or opens) a channel. If deriving the // shutdown script waits on wallet persistence, a contended wallet store wedges the event handler