From c89bd9fb61a86e59dba1ed6640dee70d13dc43c9 Mon Sep 17 00:00:00 2001 From: Jakub Zajkowski Date: Mon, 20 Jul 2026 15:02:03 +0200 Subject: [PATCH 1/5] CORE-285 reworking storage and protocol upgrade MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - **Disk-backed indexed.** `block_height_index` / `switch_block_era_id_index` / `transaction_hash_index` are now LMDB tables. If a node boots against a store where they're missing or empty, they're rebuilt from a one-off full header scan - **`IndexedLmdbBlockStore` merged into `LmdbBlockStore`.** Eliminated the separate wrapper type and the largely pass-through `DataReader`/`DataWriter` duplication; index-aware logic now lives directly on `LmdbBlockStore`. Also removed the now-unused in-memory `temp_map`. - **Protocol upgrade commit restructured.** Dropped `ReactorState::Upgrading` / `upgrading_instruction.rs`. The commit now happens synchronously in `MainReactor::new`, before the event loop starts, for a node restarting with its tip already at the pre-activation switch block; `CatchUp` just finishes (signs + gossips) the resulting immediate switch block. Also removed a redundant in-`CatchUp` commit path for nodes syncing through a _historical_ activation point — verified empirically against `dev` that this case never needs it; those nodes just fetch the post-upgrade chain normally, like any other historical data. - **New test coverage**: `emergency_upgrade.rs` (hard-reset upgrade with block peeling) and `legacy_storage_reindex_and_upgrade.rs`, which boots a 4-node network from a committed pre-refactor block store (real blocks/transactions across 3 eras, no persisted indexes), confirms the indexes rebuild and the network stays live, then drives it through an ordinary protocol upgrade. - **Test harness additions** in `fixture.rs`: `new_with_keys_and_storage_dirs` (boot nodes from pre-existing storage) and a generalized `schedule_upgrade` (arbitrary era/version, not just era 2). --- Cargo.lock | 1 + executor/evm/src/block_hash.rs | 7 +- .../src/components/block_accumulator/tests.rs | 17 +- node/src/components/contract_runtime.rs | 144 +- node/src/components/contract_runtime/tests.rs | 14 +- node/src/components/contract_runtime/utils.rs | 81 +- node/src/components/fetcher/tests.rs | 15 +- node/src/components/gossiper/tests.rs | 19 +- node/src/components/storage.rs | 352 +++-- node/src/components/storage/event.rs | 3 + node/src/components/storage/tests.rs | 74 +- .../components/transaction_acceptor/tests.rs | 17 +- node/src/effect.rs | 39 +- node/src/effect/requests.rs | 31 +- node/src/reactor/main_reactor.rs | 82 +- node/src/reactor/main_reactor/catch_up.rs | 44 +- node/src/reactor/main_reactor/control.rs | 268 ++-- node/src/reactor/main_reactor/error.rs | 4 + .../src/reactor/main_reactor/reactor_state.rs | 4 - node/src/reactor/main_reactor/tests.rs | 2 + .../main_reactor/tests/emergency_upgrade.rs | 108 ++ .../src/reactor/main_reactor/tests/fixture.rs | 135 +- .../legacy_storage_reindex_and_upgrade.rs | 279 ++++ .../lmdb/casper-example/data.lmdb | Bin 0 -> 1048576 bytes .../lmdb/casper-example/storage.lmdb | Bin 0 -> 3145728 bytes .../main_reactor/upgrading_instruction.rs | 27 - resources/test/rest_schema_status.json | 7 - storage/Cargo.toml | 1 + .../lmdb/indexed_lmdb_block_store.rs | 1256 ----------------- .../src/block_store/lmdb/lmdb_block_store.rs | 1199 +++++++++++++++- storage/src/block_store/lmdb/lmdb_ext.rs | 125 +- storage/src/block_store/lmdb/mod.rs | 3 - storage/src/block_store/lmdb/temp_map.rs | 70 - .../types/block_hash_height_and_era.rs | 37 +- .../src/data_access_layer/protocol_upgrade.rs | 2 +- 35 files changed, 2615 insertions(+), 1852 deletions(-) create mode 100644 node/src/reactor/main_reactor/tests/emergency_upgrade.rs create mode 100644 node/src/reactor/main_reactor/tests/legacy_storage_reindex_and_upgrade.rs create mode 100644 node/src/reactor/main_reactor/tests/resources/legacy_storage_no_index/lmdb/casper-example/data.lmdb create mode 100644 node/src/reactor/main_reactor/tests/resources/legacy_storage_no_index/lmdb/casper-example/storage.lmdb delete mode 100644 node/src/reactor/main_reactor/upgrading_instruction.rs delete mode 100644 storage/src/block_store/lmdb/indexed_lmdb_block_store.rs delete mode 100644 storage/src/block_store/lmdb/temp_map.rs diff --git a/Cargo.lock b/Cargo.lock index a9a57a88d2..813cfc97b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1700,6 +1700,7 @@ dependencies = [ "itertools 0.10.5", "linked-hash-map", "lmdb-rkv", + "lmdb-rkv-sys", "num", "num-derive", "num-rational", diff --git a/executor/evm/src/block_hash.rs b/executor/evm/src/block_hash.rs index e565e190e0..975cbade01 100644 --- a/executor/evm/src/block_hash.rs +++ b/executor/evm/src/block_hash.rs @@ -3,8 +3,7 @@ use std::sync::Arc; use casper_storage::block_store::{ - lmdb::IndexedLmdbBlockStore, types::BlockHeight, BlockStoreError, BlockStoreProvider, - DataReader, + lmdb::LmdbBlockStore, types::BlockHeight, BlockStoreError, BlockStoreProvider, DataReader, }; use casper_types::{BlockHash, BlockHeader}; @@ -42,12 +41,12 @@ impl BlockHashProvider for NoBlockHashProvider { /// Block hash provider backed by Casper's indexed LMDB block store. #[derive(Clone, Debug)] pub struct IndexedLmdbBlockHashProvider { - block_store: Arc, + block_store: Arc, } impl IndexedLmdbBlockHashProvider { /// Creates a block hash provider backed by `block_store`. - pub fn new(block_store: Arc) -> Self { + pub fn new(block_store: Arc) -> Self { Self { block_store } } } diff --git a/node/src/components/block_accumulator/tests.rs b/node/src/components/block_accumulator/tests.rs index 21fd8d98be..b13baf6b6b 100644 --- a/node/src/components/block_accumulator/tests.rs +++ b/node/src/components/block_accumulator/tests.rs @@ -187,10 +187,20 @@ impl Reactor for MockReactor { ) .unwrap(); - let storage = Storage::new( + let protocol_version = ProtocolVersion::from_parts(1, 0, 0); + let (storage_root, mut storage_block_store) = + storage::open_block_store(&storage_withdir, "test").unwrap(); + storage::prune_block_store( + &mut storage_block_store, + chainspec.hard_reset_to_start_of_era(), + protocol_version, + ) + .unwrap(); + let mut storage = Storage::new( &storage_withdir, - None, - ProtocolVersion::from_parts(1, 0, 0), + storage_root, + storage_block_store, + protocol_version, EraId::default(), "test", chainspec.transaction_config.max_ttl.into(), @@ -200,6 +210,7 @@ impl Reactor for MockReactor { TransactionConfig::default(), ) .unwrap(); + storage.initialize_for_test(); let reactor = MockReactor { storage, diff --git a/node/src/components/contract_runtime.rs b/node/src/components/contract_runtime.rs index ecc1c4bd3c..10e898c1ca 100644 --- a/node/src/components/contract_runtime.rs +++ b/node/src/components/contract_runtime.rs @@ -14,7 +14,6 @@ mod utils; use std::{ cmp::Ordering, - collections::BTreeMap, convert::TryInto, fmt::{self, Debug, Formatter}, path::Path, @@ -33,7 +32,7 @@ use casper_storage::{ data_access_layer::{ AddressableEntityRequest, AddressableEntityResult, BlockStore, DataAccessLayer, EntryPointExistsRequest, ExecutionResultsChecksumRequest, FlushRequest, FlushResult, - GenesisRequest, GenesisResult, TrieRequest, + GenesisRequest, GenesisResult, ProtocolUpgradeRequest, ProtocolUpgradeResult, TrieRequest, }, global_state::{ state::{lmdb::LmdbGlobalState, CommitProvider, StateProvider}, @@ -44,13 +43,13 @@ use casper_storage::{ tracking_copy::TrackingCopyError, }; use casper_types::{ - account::AccountHash, ActivationPoint, Chainspec, ChainspecRawBytes, ChainspecRegistry, - EntityAddr, EraId, Key, PublicKey, + account::AccountHash, ActivationPoint, Chainspec, ChainspecRawBytes, ChainspecRegistry, Digest, + EntityAddr, EraId, Key, ProtocolUpgradeConfig, }; use crate::{ components::{fetcher::FetchResponse, Component, ComponentState}, - contract_runtime::{types::EraPrice, utils::handle_protocol_upgrade}, + contract_runtime::types::EraPrice, effect::{ announcements::{ ContractRuntimeAnnouncement, FatalAnnouncement, MetaBlockAnnouncement, @@ -62,10 +61,7 @@ use crate::{ }, fatal, protocol::Message, - types::{ - BlockPayload, ExecutableBlock, FinalizedBlock, InternalEraReport, MetaBlockState, - TrieOrChunk, TrieOrChunkId, - }, + types::{TrieOrChunk, TrieOrChunkId}, NodeRng, }; pub(crate) use config::Config; @@ -307,6 +303,47 @@ impl ContractRuntime { result } + /// Commits a protocol upgrade against global state and flushes it to disk. + /// + /// The commit itself runs on the blocking thread-pool (via `run_intensive_task`), since it + /// can take a long time; this lets the caller bound the wait with a timeout instead of + /// stalling its task indefinitely. + pub(crate) async fn commit_protocol_upgrade( + &self, + upgrade_config: ProtocolUpgradeConfig, + ) -> Result { + debug!(?upgrade_config, "upgrade"); + let start = Instant::now(); + let upgrade_request = ProtocolUpgradeRequest::new(upgrade_config); + + let data_access_layer = Arc::clone(&self.data_access_layer); + let metrics = Arc::clone(&self.metrics); + run_intensive_task(move || { + let result = data_access_layer.protocol_upgrade(upgrade_request); + if result.is_success() { + info!("committed upgrade"); + metrics + .commit_upgrade + .observe(start.elapsed().as_secs_f64()); + let flush_req = FlushRequest::new(); + if let FlushResult::Failure(err) = data_access_layer.flush(flush_req) { + return Err(format!("{:?}", err)); + } + } + + match result { + ProtocolUpgradeResult::RootNotFound => { + Err("Root not found for protocol upgrade".to_string()) + } + ProtocolUpgradeResult::Failure(err) => Err(format!("{:?}", err)), + ProtocolUpgradeResult::Success { + post_state_hash, .. + } => Ok(post_state_hash), + } + }) + .await + } + /// Handles a contract runtime request. fn handle_contract_runtime_request( &mut self, @@ -546,91 +583,6 @@ impl ContractRuntime { } .ignore() } - ContractRuntimeRequest::UpdatePreState { new_pre_state } => { - let next_block_height = new_pre_state.next_block_height(); - self.set_execution_pre_state(new_pre_state); - let current_price = self.current_gas_price.gas_price(); - async move { - let block_header = match effect_builder - .get_highest_complete_block_header_from_storage() - .await - { - Some(header) - if header.is_switch_block() - && (header.height() + 1 == next_block_height) => - { - header - } - Some(_) => { - return fatal!( - effect_builder, - "Latest complete block is not a switch block to update state" - ) - .await; - } - None => { - return fatal!( - effect_builder, - "No complete block header found to update post upgrade state" - ) - .await; - } - }; - - let payload = BlockPayload::new( - BTreeMap::new(), - vec![], - Default::default(), - false, - current_price, - ); - - let finalized_block = FinalizedBlock::new( - payload, - Some(InternalEraReport::default()), - block_header.timestamp(), - block_header.next_block_era_id(), - next_block_height, - PublicKey::System, - ); - - info!("Enqueuing block for execution post state refresh"); - - effect_builder - .enqueue_block_for_execution( - ExecutableBlock::from_finalized_block_and_transactions( - finalized_block, - vec![], - ), - MetaBlockState::new_not_to_be_gossiped(), - ) - .await; - } - .ignore() - } - ContractRuntimeRequest::DoProtocolUpgrade { - protocol_upgrade_config, - next_block_height, - parent_hash, - parent_seed, - } => { - let mut effects = Effects::new(); - let data_access_layer = Arc::clone(&self.data_access_layer); - let metrics = Arc::clone(&self.metrics); - effects.extend( - handle_protocol_upgrade( - effect_builder, - data_access_layer, - metrics, - *protocol_upgrade_config, - next_block_height, - parent_hash, - parent_seed, - ) - .ignore(), - ); - effects - } ContractRuntimeRequest::EnqueueBlockForExecution { executable_block, key_block_height_for_activation_point, @@ -847,6 +799,10 @@ impl ContractRuntime { pub(crate) fn current_era_price(&self) -> EraPrice { self.current_gas_price } + + pub(crate) fn current_gas_price(&self) -> u8 { + self.current_gas_price.gas_price() + } } impl Component for ContractRuntime diff --git a/node/src/components/contract_runtime/tests.rs b/node/src/components/contract_runtime/tests.rs index cfcfcd0987..714aaebf0d 100644 --- a/node/src/components/contract_runtime/tests.rs +++ b/node/src/components/contract_runtime/tests.rs @@ -124,9 +124,18 @@ impl reactor::Reactor for Reactor { } let storage_withdir = WithDir::new(storage_tempdir.path(), storage_config); - let storage = Storage::new( + let (storage_root, mut storage_block_store) = + storage::open_block_store(&storage_withdir, "test").unwrap(); + storage::prune_block_store( + &mut storage_block_store, + chainspec.hard_reset_to_start_of_era(), + chainspec.protocol_version(), + ) + .unwrap(); + let mut storage = Storage::new( &storage_withdir, - None, + storage_root, + storage_block_store, chainspec.protocol_version(), EraId::default(), "test", @@ -137,6 +146,7 @@ impl reactor::Reactor for Reactor { TransactionConfig::default(), ) .unwrap(); + storage.initialize_for_test(); let contract_runtime = ContractRuntime::new(storage.root_path(), &config.config, chainspec, registry)?; diff --git a/node/src/components/contract_runtime/utils.rs b/node/src/components/contract_runtime/utils.rs index a4789cb33b..280a046340 100644 --- a/node/src/components/contract_runtime/utils.rs +++ b/node/src/components/contract_runtime/utils.rs @@ -10,7 +10,6 @@ use std::{ fmt::Debug, ops::Range, sync::{Arc, Mutex}, - time::Instant, }; use tracing::{debug, error, info}; @@ -36,15 +35,10 @@ use crate::{ use casper_binary_port::SpeculativeExecutionResult; use casper_execution_engine::engine_state::{ExecutionEngineV1, WasmV1Result}; use casper_storage::{ - data_access_layer::{ - DataAccessLayer, FlushRequest, FlushResult, ProtocolUpgradeRequest, ProtocolUpgradeResult, - TransferResult, - }, - global_state::state::{lmdb::LmdbGlobalState, CommitProvider, StateProvider}, -}; -use casper_types::{ - BlockHash, Chainspec, Digest, EraId, Gas, Key, ProtocolUpgradeConfig, Transaction, + data_access_layer::{DataAccessLayer, TransferResult}, + global_state::state::lmdb::LmdbGlobalState, }; +use casper_types::{BlockHash, Chainspec, EraId, Gas, Key, Transaction}; /// Maximum number of resource intensive tasks that can be run in parallel. /// @@ -493,75 +487,6 @@ pub(super) async fn exec_and_check_next( } } -pub(super) async fn handle_protocol_upgrade( - effect_builder: EffectBuilder, - data_access_layer: Arc>, - metrics: Arc, - upgrade_config: ProtocolUpgradeConfig, - next_block_height: u64, - parent_hash: BlockHash, - parent_seed: Digest, -) where - REv: From - + From - + From - + From - + From - + Send, -{ - debug!(?upgrade_config, "upgrade"); - let start = Instant::now(); - let upgrade_request = ProtocolUpgradeRequest::new(upgrade_config); - - let result = run_intensive_task(move || { - let result = data_access_layer.protocol_upgrade(upgrade_request); - if result.is_success() { - info!("committed upgrade"); - metrics - .commit_upgrade - .observe(start.elapsed().as_secs_f64()); - let flush_req = FlushRequest::new(); - if let FlushResult::Failure(err) = data_access_layer.flush(flush_req) { - return Err(format!("{:?}", err)); - } - } - - Ok(result) - }) - .await; - - match result { - Err(error_msg) => { - // The only way this happens is if there is a problem in the flushing. - error!(%error_msg, ":Error in post upgrade flush"); - fatal!(effect_builder, "{}", error_msg).await; - } - Ok(result) => match result { - ProtocolUpgradeResult::RootNotFound => { - let error_msg = "Root not found for protocol upgrade"; - fatal!(effect_builder, "{}", error_msg).await; - } - ProtocolUpgradeResult::Failure(err) => { - fatal!(effect_builder, "{:?}", err).await; - } - ProtocolUpgradeResult::Success { - post_state_hash, .. - } => { - let post_upgrade_state = ExecutionPreState::new( - next_block_height, - post_state_hash, - parent_hash, - parent_seed, - ); - - effect_builder - .update_contract_runtime_state(post_upgrade_state) - .await - } - }, - } -} - fn generate_range_by_index( highest_era: u64, batch_size: u64, diff --git a/node/src/components/fetcher/tests.rs b/node/src/components/fetcher/tests.rs index 4807d80a9a..9b2a230e2f 100644 --- a/node/src/components/fetcher/tests.rs +++ b/node/src/components/fetcher/tests.rs @@ -283,10 +283,20 @@ impl ReactorTrait for Reactor { ) -> Result<(Self, Effects), Self::Error> { let network = InMemoryNetwork::::new(event_queue, rng); - let storage = Storage::new( - &WithDir::new(cfg.temp_dir.path(), cfg.storage_config), + let storage_with_dir = WithDir::new(cfg.temp_dir.path(), cfg.storage_config); + let (storage_root, mut storage_block_store) = + storage::open_block_store(&storage_with_dir, &chainspec.network_config.name).unwrap(); + storage::prune_block_store( + &mut storage_block_store, chainspec.hard_reset_to_start_of_era(), chainspec.protocol_config.version, + ) + .unwrap(); + let mut storage = Storage::new( + &storage_with_dir, + storage_root, + storage_block_store, + chainspec.protocol_config.version, chainspec.protocol_config.activation_point.era_id(), &chainspec.network_config.name, chainspec.transaction_config.max_ttl.into(), @@ -296,6 +306,7 @@ impl ReactorTrait for Reactor { TransactionConfig::default(), ) .unwrap(); + storage.initialize_for_test(); let fake_transaction_acceptor = FakeTransactionAcceptor::new(); let transaction_fetcher = diff --git a/node/src/components/gossiper/tests.rs b/node/src/components/gossiper/tests.rs index 55cfbffa19..54744fd29c 100644 --- a/node/src/components/gossiper/tests.rs +++ b/node/src/components/gossiper/tests.rs @@ -150,7 +150,7 @@ impl reactor::Reactor for Reactor { fn new( config: Self::Config, - _chainspec: Arc, + chainspec: Arc, _chainspec_raw_bytes: Arc, _network_identity: NetworkIdentity, registry: &Registry, @@ -159,10 +159,20 @@ impl reactor::Reactor for Reactor { ) -> Result<(Self, Effects), Self::Error> { let (storage_config, storage_tempdir) = storage::Config::new_for_tests(1); let storage_withdir = WithDir::new(storage_tempdir.path(), storage_config); - let storage = Storage::new( + let protocol_version = ProtocolVersion::from_parts(1, 0, 0); + let (storage_root, mut storage_block_store) = + storage::open_block_store(&storage_withdir, "test").unwrap(); + storage::prune_block_store( + &mut storage_block_store, + chainspec.hard_reset_to_start_of_era(), + protocol_version, + ) + .unwrap(); + let mut storage = Storage::new( &storage_withdir, - None, - ProtocolVersion::from_parts(1, 0, 0), + storage_root, + storage_block_store, + protocol_version, EraId::default(), "test", MAX_TTL.into(), @@ -172,6 +182,7 @@ impl reactor::Reactor for Reactor { TransactionConfig::default(), ) .unwrap(); + storage.initialize_for_test(); let fake_transaction_acceptor = FakeTransactionAcceptor::new(); let transaction_gossiper = Gossiper::<{ Transaction::ID_IS_COMPLETE_ITEM }, _>::new( diff --git a/node/src/components/storage.rs b/node/src/components/storage.rs index 318bcb16be..885a686715 100644 --- a/node/src/components/storage.rs +++ b/node/src/components/storage.rs @@ -41,7 +41,7 @@ mod tests; mod utils; use casper_storage::block_store::{ - lmdb::{IndexedLmdbBlockStore, LmdbBlockStore}, + lmdb::LmdbBlockStore, types::{ ApprovalsHashes, BlockExecutionResults, BlockHashHeightAndEra, BlockHeight, BlockTransfers, LatestSwitchBlock, StateStore, StateStoreKey, Tip, TransactionFinalizedApprovals, @@ -68,7 +68,7 @@ use casper_types::{ execution::{execution_result_v1, ExecutionResult, ExecutionResultV1}, Approval, ApprovalsHash, AvailableBlockRange, Block, BlockBody, BlockHash, BlockHeader, BlockHeaderWithSignatures, BlockSignatures, BlockSignaturesV1, BlockSignaturesV2, BlockV2, - ChainNameDigest, DeployHash, EraId, ExecutionInfo, FinalitySignature, ProtocolVersion, + ChainNameDigest, DeployHash, Digest, EraId, ExecutionInfo, FinalitySignature, ProtocolVersion, Timestamp, Transaction, TransactionConfig, TransactionHash, TransactionId, Transfer, U512, }; use datasize::DataSize; @@ -80,7 +80,7 @@ use tracing::{debug, error, info, warn}; use crate::{ components::{ fetcher::{FetchItem, FetchResponse}, - Component, + Component, ComponentState, InitializedComponent, }, effect::{ announcements::FatalAnnouncement, @@ -126,8 +126,8 @@ const STORAGE_FILES: [&str; 5] = [ pub struct Storage { /// Storage location. root: PathBuf, - /// Block store - pub(crate) block_store: IndexedLmdbBlockStore, + /// Block store. + block_store: LmdbBlockStore, /// Runs of completed blocks known in storage. completed_blocks: DisjointSequences, /// The activation point era of the current protocol version. @@ -153,6 +153,13 @@ pub struct Storage { transaction_config: TransactionConfig, /// The utilization of blocks. utilization_tracker: BTreeMap>, + /// Component initialization state. + state: ComponentState, + /// The protocol version this node is running. + #[data_size(skip)] + protocol_version: ProtocolVersion, + /// Whether a force resync was requested. + force_resync: bool, } #[allow(clippy::large_enum_variant)] @@ -193,41 +200,82 @@ where _rng: &mut NodeRng, event: Self::Event, ) -> Effects { - let result = match event { - Event::StorageRequest(req) => self.handle_storage_request(*req), - Event::NetRequestIncoming(ref incoming) => { - match self.handle_net_request_incoming::(effect_builder, incoming) { - Ok(effects) => Ok(effects), - Err(GetRequestError::Fatal(fatal_error)) => Err(fatal_error), - Err(ref other_err) => { - warn!( - sender=%incoming.sender, - err=display_error(other_err), - "error handling net request" + match &self.state { + ComponentState::Fatal(msg) => { + error!( + msg, + "should not handle this event when this component has fatal error" + ); + Effects::new() + } + ComponentState::Uninitialized => { + warn!(?event, "uninitialized component received event"); + Effects::new() + } + ComponentState::Initializing => match event { + Event::Initialize => match self.do_initialize() { + Ok(()) => { + >::set_state( + self, + ComponentState::Initialized, ); - // We could still send the requester a "not found" message, and could do - // so even in the fatal case, but it is safer to not do so at the - // moment, giving less surface area for possible amplification attacks. - Ok(Effects::new()) + Effects::new() } + Err(err) => fatal!(effect_builder, "storage error: {}", err).ignore(), + }, + _ => { + warn!( + ?event, + "initializing component received non-Initialize event" + ); + Effects::new() } - } - Event::MarkBlockCompletedRequest(req) => self.handle_mark_block_completed_request(req), - Event::MakeBlockExecutableRequest(req) => { - let ret = self.make_executable_block(&req.block_hash); - match ret { - Ok(maybe) => Ok(req.responder.respond(maybe).ignore()), - Err(err) => Err(err), + }, + ComponentState::Initialized => { + let result = match event { + Event::Initialize => { + info!("Storage: skipping initialization, already initialized"); + Ok(Effects::new()) + } + Event::StorageRequest(req) => self.handle_storage_request(*req), + Event::NetRequestIncoming(ref incoming) => { + match self.handle_net_request_incoming::(effect_builder, incoming) { + Ok(effects) => Ok(effects), + Err(GetRequestError::Fatal(fatal_error)) => Err(fatal_error), + Err(ref other_err) => { + warn!( + sender=%incoming.sender, + err=display_error(other_err), + "error handling net request" + ); + // We could still send the requester a "not found" message, and + // could do so even in the fatal case, but it is safer to not do + // so at the moment, giving less surface area for possible + // amplification attacks. + Ok(Effects::new()) + } + } + } + Event::MarkBlockCompletedRequest(req) => { + self.handle_mark_block_completed_request(req) + } + Event::MakeBlockExecutableRequest(req) => { + let ret = self.make_executable_block(&req.block_hash); + match ret { + Ok(maybe) => Ok(req.responder.respond(maybe).ignore()), + Err(err) => Err(err), + } + } + }; + + // Any error is turned into a fatal effect, the component itself does not panic. + // Note that we are dropping a lot of responders this way, but since we are + // crashing with fatal anyway, it should not matter. + match result { + Ok(effects) => effects, + Err(err) => fatal!(effect_builder, "storage error: {}", err).ignore(), } } - }; - - // Any error is turned into a fatal effect, the component itself does not panic. Note that - // we are dropping a lot of responders this way, but since we are crashing with fatal - // anyway, it should not matter. - match result { - Ok(effects) => effects, - Err(err) => fatal!(effect_builder, "storage error: {}", err).ignore(), } } @@ -236,12 +284,160 @@ where } } +impl InitializedComponent for Storage +where + REv: From + From> + Send, +{ + fn state(&self) -> &ComponentState { + &self.state + } + + fn set_state(&mut self, new_state: ComponentState) { + info!( + ?new_state, + name = >::name(self), + "component state changed" + ); + self.state = new_state; + } +} + +/// Opens (and, if necessary, builds the disk-backed indexes of) a node's block store. +pub fn open_block_store( + cfg: &WithDir, + network_name: &str, +) -> Result<(PathBuf, LmdbBlockStore), FatalStorageError> { + let config = cfg.value(); + + // Create the database directory. + let mut root = cfg.with_dir(config.path.clone()); + let network_subdir = root.join(network_name); + + if !network_subdir.exists() { + fs::create_dir_all(&network_subdir).map_err(|err| { + FatalStorageError::CreateDatabaseDirectory(network_subdir.clone(), err) + })?; + } + + if should_move_storage_files_to_network_subdir(&root, &STORAGE_FILES)? { + move_storage_files_to_network_subdir(&root, &network_subdir, &STORAGE_FILES)?; + } + + root = network_subdir; + + // Calculate the upper bound for the memory map that is potentially used. + let total_size = config + .max_block_store_size + .saturating_add(config.max_deploy_store_size) + .saturating_add(config.max_deploy_metadata_store_size); + + let mut block_store = LmdbBlockStore::new(root.as_path(), total_size)?; + block_store.init()?; + + Ok((root, block_store)) +} + +/// Performs the chainspec-driven hard-reset prune: deletes every stored block (and its body, +/// execution results, and index entries) at or after `hard_reset_to_start_of_era` that isn't +/// from the current protocol version. No-op if `hard_reset_to_start_of_era` is `None`. +pub fn prune_block_store( + block_store: &mut LmdbBlockStore, + hard_reset_to_start_of_era: Option, + protocol_version: ProtocolVersion, +) -> Result<(), FatalStorageError> { + let Some(invalid_era) = hard_reset_to_start_of_era else { + return Ok(()); + }; + + info!("pruning block store"); + + let tip_height = { + let ro_txn = block_store.checkout_ro()?; + match DataReader::::read(&ro_txn, Tip)? { + Some(header) => header.height(), + None => { + info!("block store is empty, nothing to prune"); + return Ok(()); + } + } + }; + let total_headers = tip_height + 1; + let progress_step = (total_headers / 20).max(1); + + // First pass (read-only): scan every header to decide which blocks to delete, and whether + // each body hash referenced along the way is still needed by at least one retained block + // (bodies are only ever deleted once no retained header references them any more). + let mut blocks_to_delete = Vec::new(); + let mut body_hash_retained: HashMap = HashMap::new(); + { + let ro_txn = block_store.checkout_ro()?; + for height in 0..=tip_height { + if height % progress_step == 0 { + info!( + percent_complete = (height * 100 / total_headers.max(1)), + height, total_headers, "pruning block store: scanning" + ); + } + let header: BlockHeader = match ro_txn.read(height)? { + Some(header) => header, + None => continue, + }; + + // Retain blocks from eras before the hard reset era, and blocks after this era if + // they are from the current protocol version (as otherwise a node restart would + // purge them again, despite them being valid). + let should_retain = + header.era_id() < invalid_era || header.protocol_version() == protocol_version; + + let retained = body_hash_retained + .entry(*header.body_hash()) + .or_insert(false); + *retained = *retained || should_retain; + + if !should_retain { + blocks_to_delete.push((header.block_hash(), header.height(), header.era_id())); + } + } + } + + if blocks_to_delete.is_empty() { + info!("block store pruning complete: nothing to prune"); + return Ok(()); + } + let blocks_to_delete_count = blocks_to_delete.len(); + + // Second pass (read-write): delete the execution results and the block itself for each + // block being pruned -- in that order, since deleting the execution results needs to read + // the block (header + body) to find its transaction hashes -- then purge any block body no + // longer referenced by a retained block. + { + let mut rw_txn = block_store.checkout_rw()?; + for (block_hash, block_height, era_id) in blocks_to_delete { + DataWriter::::delete( + &mut rw_txn, + BlockHashHeightAndEra::new(block_hash, block_height, era_id), + )?; + DataWriter::::delete(&mut rw_txn, block_hash)?; + } + for (body_hash, retained) in body_hash_retained { + if !retained { + DataWriter::::delete(&mut rw_txn, body_hash)?; + } + } + rw_txn.commit()?; + } + + info!(blocks_to_delete_count, "block store pruning complete"); + Ok(()) +} + impl Storage { - /// Creates a new storage component. + /// Ctor and init #[allow(clippy::too_many_arguments)] pub fn new( cfg: &WithDir, - hard_reset_to_start_of_era: Option, + root: PathBuf, + block_store: LmdbBlockStore, protocol_version: ProtocolVersion, activation_era: EraId, network_name: &str, @@ -252,38 +448,11 @@ impl Storage { transaction_config: TransactionConfig, ) -> Result { let config = cfg.value(); - - // Create the database directory. - let mut root = cfg.with_dir(config.path.clone()); - let network_subdir = root.join(network_name); - - if !network_subdir.exists() { - fs::create_dir_all(&network_subdir).map_err(|err| { - FatalStorageError::CreateDatabaseDirectory(network_subdir.clone(), err) - })?; - } - - if should_move_storage_files_to_network_subdir(&root, &STORAGE_FILES)? { - move_storage_files_to_network_subdir(&root, &network_subdir, &STORAGE_FILES)?; - } - - root = network_subdir; - - // Calculate the upper bound for the memory map that is potentially used. - let total_size = config - .max_block_store_size - .saturating_add(config.max_deploy_store_size) - .saturating_add(config.max_deploy_metadata_store_size); - - let block_store = LmdbBlockStore::new(root.as_path(), total_size)?; - let indexed_block_store = - IndexedLmdbBlockStore::new(block_store, hard_reset_to_start_of_era, protocol_version)?; - let metrics = registry.map(Metrics::new).transpose()?; - let mut component = Self { + Ok(Self { root, - block_store: indexed_block_store, + block_store, completed_blocks: Default::default(), activation_era, key_block_height_for_activation_point: None, @@ -295,10 +464,16 @@ impl Storage { metrics, chain_name_hash: ChainNameDigest::from_chain_name(network_name), transaction_config, - }; + state: ComponentState::Uninitialized, + protocol_version, + force_resync, + }) + } - if force_resync { - let force_resync_file_path = component.root_path().join(FORCE_RESYNC_FILE_NAME); + /// Performs completed-blocks bookkeeping (and force-resync marker handling, if configured). + fn do_initialize(&mut self) -> Result<(), FatalStorageError> { + if self.force_resync { + let force_resync_file_path = self.root_path().join(FORCE_RESYNC_FILE_NAME); // Check if resync is already in progress. Force resync will kick // in only when the marker file didn't exist before. // Use `OpenOptions::create_new` to atomically check for the file @@ -313,10 +488,10 @@ impl Storage { // is now created, initialize force resync. info!("initializing force resync"); // Default `storage.completed_blocks`. - component.completed_blocks = Default::default(); - component.persist_completed_blocks()?; + self.completed_blocks = Default::default(); + self.persist_completed_blocks()?; // Exit the initialization function early. - return Ok(component); + return Ok(()); } Err(io_err) if io_err.kind() == ErrorKind::AlreadyExists => { info!("skipping force resync as marker file exists"); @@ -332,7 +507,7 @@ impl Storage { } { - let ro_txn = component.block_store.checkout_ro()?; + let ro_txn = self.block_store.checkout_ro()?; let maybe_state_store: Option> = ro_txn.read(StateStoreKey::new( Cow::Borrowed(COMPLETED_BLOCKS_STORAGE_KEY), ))?; @@ -349,7 +524,7 @@ impl Storage { sequences.clear(); } - component.completed_blocks = sequences; + self.completed_blocks = sequences; } None => { // No state so far. We can make the following observations: @@ -375,8 +550,10 @@ impl Storage { for height in (0..=highest_block_header.height()).rev() { let maybe_header: Option = ro_txn.read(height)?; match maybe_header { - Some(header) if header.protocol_version() < protocol_version => { - component.completed_blocks = + Some(header) + if header.protocol_version() < self.protocol_version => + { + self.completed_blocks = DisjointSequences::new(Sequence::new(0, header.height())); break; } @@ -387,9 +564,9 @@ impl Storage { } } } - component.persist_completed_blocks()?; - component.warm_up_utilization_tracker()?; - Ok(component) + self.persist_completed_blocks()?; + self.warm_up_utilization_tracker()?; + Ok(()) } /// Assume: @@ -671,9 +848,10 @@ impl Storage { StorageRequest::GetApprovalsHashes { block_hash, responder, - } => responder - .respond(self.block_store.checkout_ro()?.read(block_hash)?) - .ignore(), + } => { + let maybe_item = self.block_store.checkout_ro()?.read(block_hash)?; + responder.respond(maybe_item).ignore() + } StorageRequest::GetHighestCompleteBlock { responder } => responder .respond(self.get_highest_complete_block()?) .ignore(), @@ -2342,6 +2520,16 @@ fn successful_transfers(execution_result: &ExecutionResult) -> Vec { // only ever be used when writing tests. #[cfg(test)] impl Storage { + /// Drives this component through its `InitializedComponent` initialization step, mirroring + /// what the reactor's `initialize_next_component` does at startup. Test harnesses across the + /// crate that construct a `Storage` directly (rather than via the full reactor) must call + /// this before issuing any `StorageRequest`s against it. + pub(crate) fn initialize_for_test(&mut self) { + self.do_initialize() + .expect("storage initialization should succeed"); + self.state = ComponentState::Initialized; + } + /// Directly returns a transaction with finalized approvals from internal store. /// /// # Panics diff --git a/node/src/components/storage/event.rs b/node/src/components/storage/event.rs index 8f02790c9a..2b54c45e9d 100644 --- a/node/src/components/storage/event.rs +++ b/node/src/components/storage/event.rs @@ -16,6 +16,8 @@ const_assert!(_STORAGE_EVENT_SIZE <= 32); #[derive(Debug, From, Serialize)] #[repr(u8)] pub(crate) enum Event { + /// Initializing event + Initialize, /// Storage request. #[from] StorageRequest(Box), @@ -32,6 +34,7 @@ pub(crate) enum Event { impl Display for Event { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { match self { + Event::Initialize => write!(f, "initialize"), Event::StorageRequest(req) => req.fmt(f), Event::NetRequestIncoming(incoming) => incoming.fmt(f), Event::MarkBlockCompletedRequest(req) => req.fmt(f), diff --git a/node/src/components/storage/tests.rs b/node/src/components/storage/tests.rs index 3c0b482337..6623309070 100644 --- a/node/src/components/storage/tests.rs +++ b/node/src/components/storage/tests.rs @@ -33,8 +33,8 @@ use casper_types::{ use tempfile::tempdir; use super::{ - move_storage_files_to_network_subdir, should_move_storage_files_to_network_subdir, Config, - Storage, FORCE_RESYNC_FILE_NAME, + move_storage_files_to_network_subdir, open_block_store, prune_block_store, + should_move_storage_files_to_network_subdir, Config, Storage, FORCE_RESYNC_FILE_NAME, }; use crate::{ components::fetcher::{FetchItem, FetchResponse}, @@ -187,10 +187,17 @@ fn create_sync_leap_test_chain( /// Panics if setting up the storage fixture fails. fn storage_fixture(harness: &ComponentHarness) -> Storage { let cfg = new_config(harness); - Storage::new( - &WithDir::new(harness.tmp.path(), cfg), - None, - ProtocolVersion::from_parts(1, 0, 0), + let cfg = WithDir::new(harness.tmp.path(), cfg); + let protocol_version = ProtocolVersion::from_parts(1, 0, 0); + let (root, mut block_store) = + open_block_store(&cfg, "test").expect("could not open block store fixture"); + prune_block_store(&mut block_store, None, protocol_version) + .expect("could not prune block store fixture"); + let mut storage = Storage::new( + &cfg, + root, + block_store, + protocol_version, EraId::default(), "test", MAX_TTL.into(), @@ -199,7 +206,9 @@ fn storage_fixture(harness: &ComponentHarness) -> Storage { false, TransactionConfig::default(), ) - .expect("could not create storage component fixture") + .expect("could not create storage component fixture"); + storage.initialize_for_test(); + storage } /// Storage component test fixture. @@ -218,19 +227,33 @@ fn storage_fixture_from_parts( recent_era_count: Option, ) -> Storage { let cfg = new_config(harness); - Storage::new( - &WithDir::new(harness.tmp.path(), cfg), + let cfg = WithDir::new(harness.tmp.path(), cfg); + let network_name = network_name.unwrap_or("test"); + let protocol_version = protocol_version.unwrap_or(ProtocolVersion::V1_0_0); + let (root, mut block_store) = + open_block_store(&cfg, network_name).expect("could not open block store fixture"); + prune_block_store( + &mut block_store, hard_reset_to_start_of_era, - protocol_version.unwrap_or(ProtocolVersion::V1_0_0), + protocol_version, + ) + .expect("could not prune block store fixture"); + let mut storage = Storage::new( + &cfg, + root, + block_store, + protocol_version, EraId::default(), - network_name.unwrap_or("test"), + network_name, max_ttl.unwrap_or(MAX_TTL).into(), recent_era_count.unwrap_or(RECENT_ERA_COUNT), None, false, TransactionConfig::default(), ) - .expect("could not create storage component fixture from parts") + .expect("could not create storage component fixture from parts"); + storage.initialize_for_test(); + storage } /// Storage component test fixture with force resync enabled. @@ -241,10 +264,16 @@ fn storage_fixture_from_parts( /// /// Panics if setting up the storage fixture fails. fn storage_fixture_with_force_resync(cfg: &WithDir) -> Storage { - Storage::new( + let protocol_version = ProtocolVersion::from_parts(1, 0, 0); + let (root, mut block_store) = + open_block_store(cfg, "test").expect("could not open block store fixture"); + prune_block_store(&mut block_store, None, protocol_version) + .expect("could not prune block store fixture"); + let mut storage = Storage::new( cfg, - None, - ProtocolVersion::from_parts(1, 0, 0), + root, + block_store, + protocol_version, EraId::default(), "test", MAX_TTL.into(), @@ -253,7 +282,9 @@ fn storage_fixture_with_force_resync(cfg: &WithDir) -> Storage { true, TransactionConfig::default(), ) - .expect("could not create storage component fixture") + .expect("could not create storage component fixture"); + storage.initialize_for_test(); + storage } /// Storage component test fixture. @@ -1777,10 +1808,15 @@ fn should_create_subdir_named_after_network() { let cfg = new_config(&harness); let network_name = "test"; + let with_dir = WithDir::new(harness.tmp.path(), cfg.clone()); + let protocol_version = ProtocolVersion::from_parts(1, 0, 0); + let (root, mut block_store) = open_block_store(&with_dir, network_name).unwrap(); + prune_block_store(&mut block_store, None, protocol_version).unwrap(); let storage = Storage::new( - &WithDir::new(harness.tmp.path(), cfg.clone()), - None, - ProtocolVersion::from_parts(1, 0, 0), + &with_dir, + root, + block_store, + protocol_version, EraId::default(), network_name, MAX_TTL.into(), diff --git a/node/src/components/transaction_acceptor/tests.rs b/node/src/components/transaction_acceptor/tests.rs index 0d7705244c..d2f068fc82 100644 --- a/node/src/components/transaction_acceptor/tests.rs +++ b/node/src/components/transaction_acceptor/tests.rs @@ -1418,10 +1418,20 @@ impl reactor::Reactor for Reactor { let transaction_acceptor = TransactionAcceptor::new(Config::default(), Arc::clone(&chainspec), registry)?; - let storage = Storage::new( + let protocol_version = ProtocolVersion::from_parts(1, 0, 0); + let (storage_root, mut storage_block_store) = + storage::open_block_store(&storage_with_dir, "test").unwrap(); + storage::prune_block_store( + &mut storage_block_store, + chainspec.hard_reset_to_start_of_era(), + protocol_version, + ) + .unwrap(); + let mut storage = Storage::new( &storage_with_dir, - None, - ProtocolVersion::from_parts(1, 0, 0), + storage_root, + storage_block_store, + protocol_version, EraId::default(), "test", chainspec.transaction_config.max_ttl.into(), @@ -1431,6 +1441,7 @@ impl reactor::Reactor for Reactor { TransactionConfig::default(), ) .unwrap(); + storage.initialize_for_test(); let reactor = Reactor { storage, diff --git a/node/src/effect.rs b/node/src/effect.rs index fd33526dcc..ba09f3b1b8 100644 --- a/node/src/effect.rs +++ b/node/src/effect.rs @@ -134,8 +134,8 @@ use casper_types::{ Approval, AvailableBlockRange, Block, BlockHash, BlockHeader, BlockSignatures, BlockSynchronizerStatus, BlockV2, ChainspecRawBytes, DeployHash, Digest, EntityAddr, EraId, ExecutionInfo, FinalitySignature, FinalitySignatureId, FinalitySignatureV2, HashAddr, Key, - NextUpgrade, Package, PackageAddr, ProtocolUpgradeConfig, PublicKey, TimeDiff, Timestamp, - Transaction, TransactionHash, TransactionId, Transfer, U512, + NextUpgrade, Package, PackageAddr, PublicKey, TimeDiff, Timestamp, Transaction, + TransactionHash, TransactionId, Transfer, U512, }; use crate::{ @@ -152,7 +152,6 @@ use crate::{ network::{blocklist::BlocklistJustification, FromIncoming, NetworkInsights}, transaction_acceptor, }, - contract_runtime::ExecutionPreState, effect::announcements::NonExecutableBlockAnnouncement, failpoints::FailpointActivation, reactor::{main_reactor::ReactorState, EventQueueHandle, QueueKind}, @@ -961,18 +960,6 @@ impl EffectBuilder { .await; } - pub(crate) async fn update_contract_runtime_state(self, new_pre_state: ExecutionPreState) - where - REv: From, - { - self.event_queue - .schedule( - ContractRuntimeRequest::UpdatePreState { new_pre_state }, - QueueKind::ContractRuntime, - ) - .await; - } - /// Announces validators for upcoming era. pub(crate) async fn announce_upcoming_era_validators( self, @@ -1815,28 +1802,6 @@ impl EffectBuilder { .await; } - pub(crate) async fn enqueue_protocol_upgrade( - self, - upgrade_config: ProtocolUpgradeConfig, - next_block_height: u64, - parent_hash: BlockHash, - parent_seed: Digest, - ) where - REv: From, - { - self.event_queue - .schedule( - ContractRuntimeRequest::DoProtocolUpgrade { - protocol_upgrade_config: Box::new(upgrade_config), - next_block_height, - parent_hash, - parent_seed, - }, - QueueKind::Control, - ) - .await; - } - /// Checks whether the transactions included in the block exist on the network and that /// the block is valid. pub(crate) async fn validate_block( diff --git a/node/src/effect/requests.rs b/node/src/effect/requests.rs index c04a27d98e..4fea7eda52 100644 --- a/node/src/effect/requests.rs +++ b/node/src/effect/requests.rs @@ -34,8 +34,8 @@ use casper_types::{ execution::ExecutionResult, Approval, AvailableBlockRange, Block, BlockHash, BlockHeader, BlockSignatures, BlockSynchronizerStatus, BlockV2, ChainspecRawBytes, DeployHash, Digest, DisplayIter, EntityAddr, EraId, ExecutionInfo, FinalitySignature, FinalitySignatureId, - HashAddr, NextUpgrade, ProtocolUpgradeConfig, PublicKey, TimeDiff, Timestamp, Transaction, - TransactionHash, TransactionId, Transfer, + HashAddr, NextUpgrade, PublicKey, TimeDiff, Timestamp, Transaction, TransactionHash, + TransactionId, Transfer, }; use super::{AutoClosingResponder, GossipTarget, Responder}; @@ -53,7 +53,6 @@ use crate::{ network::NetworkInsights, transaction_acceptor, }, - contract_runtime::ExecutionPreState, reactor::main_reactor::ReactorState, types::{ appendable_block::AppendableBlock, BlockExecutionResultsOrChunk, @@ -890,15 +889,6 @@ pub(crate) enum ContractRuntimeRequest { era_id: EraId, responder: Responder>, }, - DoProtocolUpgrade { - protocol_upgrade_config: Box, - next_block_height: u64, - parent_hash: BlockHash, - parent_seed: Digest, - }, - UpdatePreState { - new_pre_state: ExecutionPreState, - }, } impl Display for ContractRuntimeRequest { @@ -993,23 +983,6 @@ impl Display for ContractRuntimeRequest { formatted_contract_hash, entry_point_name, state_root_hash ) } - ContractRuntimeRequest::DoProtocolUpgrade { - protocol_upgrade_config, - .. - } => { - write!( - formatter, - "execute protocol upgrade against config: {:?}", - protocol_upgrade_config - ) - } - ContractRuntimeRequest::UpdatePreState { new_pre_state } => { - write!( - formatter, - "Updating contract runtimes execution prestate: {:?}", - new_pre_state - ) - } } } } diff --git a/node/src/reactor/main_reactor.rs b/node/src/reactor/main_reactor.rs index 9375fde4dd..4f00583e0d 100644 --- a/node/src/reactor/main_reactor.rs +++ b/node/src/reactor/main_reactor.rs @@ -15,7 +15,6 @@ mod reactor_state; #[cfg(test)] mod tests; mod upgrade_shutdown; -mod upgrading_instruction; mod validate; use std::{collections::BTreeMap, convert::TryInto, sync::Arc, time::Instant}; @@ -26,9 +25,10 @@ use prometheus::Registry; use tracing::{debug, error, info, warn}; use casper_binary_port::{LastProgress, NetworkName, Uptime}; +use casper_storage::block_store::{types::Tip, BlockStoreProvider, DataReader}; use casper_types::{ - bytesrepr, Block, BlockHash, BlockV2, Chainspec, ChainspecRawBytes, EraId, FinalitySignature, - FinalitySignatureV2, PublicKey, TimeDiff, Timestamp, Transaction, U512, + bytesrepr, Block, BlockHash, BlockHeader, BlockV2, Chainspec, ChainspecRawBytes, Digest, EraId, + FinalitySignature, FinalitySignatureV2, PublicKey, TimeDiff, Timestamp, Transaction, U512, }; #[cfg(test)] @@ -48,7 +48,7 @@ use crate::{ network::{self, GossipedAddress, Identity as NetworkIdentity, Network}, rest_server::RestServer, shutdown_trigger::{self, CompletedBlockInfo, ShutdownTrigger}, - storage::Storage, + storage::{self, Storage}, sync_leaper::SyncLeaper, transaction_acceptor::{self, TransactionAcceptor}, transaction_buffer::{self, TransactionBuffer}, @@ -205,6 +205,31 @@ pub(crate) struct MainReactor { prevent_validator_shutdown: bool, force_catchup: bool, + + /// Set when a protocol upgrade has been committed against global state but its immediate + /// switch block hasn't yet been produced, signed, and gossiped -- deferred until the node + /// can actually reach peers (see `MainReactor::maybe_finish_pending_upgrade`). + pending_immediate_switch_block: Option, + + /// Set to the current time when the immediate switch block for a protocol upgrade is + /// enqueued for execution, and cleared once the upgrade is observed to be complete (i.e. + /// `should_commit_upgrade` becomes false again, meaning the local tip has advanced past the + /// switch block). If it takes longer than `upgrade_timeout` for that to happen, the reactor + /// bails out fatally rather than waiting forever. + upgrade_started_at: Option, +} + +/// The information needed to produce the deterministic "immediate switch block" following a +/// protocol upgrade, once the node is ready to sign and gossip it. +#[derive(Clone, DataSize, Debug)] +pub(super) struct PendingImmediateSwitchBlock { + next_block_height: u64, + #[data_size(skip)] + post_state_hash: Digest, + parent_hash: BlockHash, + parent_seed: Digest, + era_id: EraId, + timestamp: Timestamp, } impl reactor::Reactor for MainReactor { @@ -1115,10 +1140,46 @@ impl reactor::Reactor for MainReactor { let storage_config = WithDir::new(&root_dir, config.storage.clone()); - let hard_reset_to_start_of_era = chainspec.hard_reset_to_start_of_era(); + // Open (and, if necessary, build the disk-backed indexes of) the block store as early + // as possible, before anything else touches disk-backed state. + let (storage_root, mut block_store) = + storage::open_block_store(&storage_config, &chainspec.network_config.name)?; + storage::prune_block_store( + &mut block_store, + chainspec.hard_reset_to_start_of_era(), + protocol_version, + )?; + + let contract_runtime = ContractRuntime::new( + &storage_root, + &config.contract_runtime, + chainspec.clone(), + registry, + )?; + + // If our local tip (post-prune) is the activation point, commit the protocol upgrade + // synchronously now. The resulting immediate switch block is *not* produced yet: that's + // deferred until the node can actually sign and gossip it + // (see `MainReactor::maybe_finish_pending_upgrade`). + let local_tip: Option = { + let ro_txn = block_store + .checkout_ro() + .map_err(storage::FatalStorageError::from)?; + DataReader::::read(&ro_txn, Tip) + .map_err(storage::FatalStorageError::from)? + }; + let pending_immediate_switch_block = Self::commit_upgrade_if_needed( + &contract_runtime, + &chainspec, + &chainspec_raw_bytes, + local_tip.as_ref(), + config.node.upgrade_timeout, + )?; + let storage = Storage::new( &storage_config, - hard_reset_to_start_of_era, + storage_root, + block_store, protocol_version, chainspec.protocol_config.activation_point.era_id(), &chainspec.network_config.name, @@ -1129,13 +1190,6 @@ impl reactor::Reactor for MainReactor { chainspec.transaction_config.clone(), )?; - let contract_runtime = ContractRuntime::new( - storage.root_path(), - &config.contract_runtime, - chainspec.clone(), - registry, - )?; - let allow_handshake = config.node.sync_handling != SyncHandling::Isolated; let network = Network::new( @@ -1287,6 +1341,8 @@ impl reactor::Reactor for MainReactor { finality_signature_creation: true, prevent_validator_shutdown, force_catchup: false, + pending_immediate_switch_block, + upgrade_started_at: None, }; info!("MainReactor: instantiated"); diff --git a/node/src/reactor/main_reactor/catch_up.rs b/node/src/reactor/main_reactor/catch_up.rs index f98134a566..dd4615a393 100644 --- a/node/src/reactor/main_reactor/catch_up.rs +++ b/node/src/reactor/main_reactor/catch_up.rs @@ -29,7 +29,6 @@ pub(super) enum CatchUpInstruction { ShutdownForUpgrade, CaughtUp, CommitGenesis, - CommitUpgrade, } impl MainReactor { @@ -265,7 +264,7 @@ impl MainReactor { SyncInstruction::BlockSync { block_hash } => { Some(self.catch_up_block_sync(effect_builder, block_hash)) } - SyncInstruction::CaughtUp { .. } => self.catch_up_check_transition(), + SyncInstruction::CaughtUp { .. } => self.catch_up_check_transition(effect_builder), } } @@ -413,11 +412,42 @@ impl MainReactor { } } - fn catch_up_check_transition(&mut self) -> Option { - // we may be starting back up after a shutdown for upgrade; if so we need to - // commit upgrade now before proceeding further - if self.should_commit_upgrade() { - return Some(CatchUpInstruction::CommitUpgrade); + fn catch_up_check_transition( + &mut self, + effect_builder: EffectBuilder, + ) -> Option { + // We may be starting back up after a shutdown for upgrade -- i.e. `MainReactor::new` + // found the local tip already sitting at the pre-activation switch block and committed + // the upgrade synchronously before the reactor even existed. If so, `CatchUp` just needs + // to finish (sign + gossip) the resulting immediate switch block below. + // + // A node catching up through a *historical* activation point (a new node joining an + // already-upgraded network) does NOT commit the upgrade itself here: it acquires the + // post-upgrade chain the same way it acquires everything else during catch-up, via the + // block synchronizer fetching from peers. Verified empirically against `dev`: a joining + // node given a pre-upgrade trusted hash and left to sync forward through a live upgrade + // never invokes the local commit path (no "committing protocol upgrade" / "switch to + // Upgrading" log line), while the already-running nodes that restart into it do. + if self.pending_immediate_switch_block.is_some() { + // `CatchUp` is only reachable after the `Initialize` peer-gate has passed, so it's + // always safe to finish (sign + gossip) the pending upgrade block right away. + if let Some(effects) = self.maybe_finish_pending_upgrade(effect_builder) { + return Some(CatchUpInstruction::Do(Duration::ZERO, effects)); + } + } + // If we're still waiting for the upgrade's immediate switch block to land (i.e. to be + // executed, stored, and marked complete), bail out fatally rather than wait forever if + // that's taking longer than `upgrade_timeout`. + if let Some(started_at) = self.upgrade_started_at { + if !self.should_commit_upgrade() { + // the local tip has advanced past the switch block: the upgrade is complete. + self.upgrade_started_at = None; + } else if started_at.elapsed() > self.upgrade_timeout { + return Some(CatchUpInstruction::Fatal(format!( + "protocol upgrade did not complete within {}", + self.upgrade_timeout + ))); + } } // we may need to shutdown to go thru an upgrade if self.should_shutdown_for_upgrade() { diff --git a/node/src/reactor/main_reactor/control.rs b/node/src/reactor/main_reactor/control.rs index d62e8a034e..d751945155 100644 --- a/node/src/reactor/main_reactor/control.rs +++ b/node/src/reactor/main_reactor/control.rs @@ -1,23 +1,27 @@ -use std::time::Duration; +use std::{collections::BTreeMap, sync::Arc, time::Duration}; +use tokio::runtime::Handle; use tracing::{debug, error, info, trace}; use casper_storage::data_access_layer::GenesisResult; -use casper_types::{BlockHash, BlockHeader, Digest, EraId, PublicKey, Timestamp}; +use casper_types::{ + BlockHash, BlockHeader, Chainspec, ChainspecRawBytes, Digest, EraId, PublicKey, TimeDiff, + Timestamp, +}; use crate::{ components::{ binary_port, block_synchronizer::{self, BlockSynchronizerProgress}, - contract_runtime::ExecutionPreState, - diagnostics_port, event_stream_server, network, rest_server, upgrade_watcher, + contract_runtime::{ContractRuntime, ExecutionPreState}, + diagnostics_port, event_stream_server, network, rest_server, storage, upgrade_watcher, }, effect::{announcements::ControlAnnouncement, EffectBuilder, EffectExt, Effects}, fatal, reactor::main_reactor::{ catch_up::CatchUpInstruction, genesis_instruction::GenesisInstruction, - keep_up::KeepUpInstruction, upgrade_shutdown::UpgradeShutdownInstruction, - upgrading_instruction::UpgradingInstruction, utils, validate::ValidateInstruction, - MainEvent, MainReactor, ReactorState, + keep_up::KeepUpInstruction, upgrade_shutdown::UpgradeShutdownInstruction, utils, + validate::ValidateInstruction, Error, MainEvent, MainReactor, PendingImmediateSwitchBlock, + ReactorState, }, types::{BlockPayload, ExecutableBlock, FinalizedBlock, InternalEraReport, MetaBlockState}, NodeRng, @@ -63,39 +67,38 @@ impl MainReactor { None => { if self.sync_handling.is_isolated() { // If node is "isolated" it doesn't care about peers - if let Err(msg) = self.refresh_contract_runtime() { - return ( - Duration::ZERO, - fatal!(effect_builder, "{}", msg).ignore(), - ); - } + let effects = match self + .refresh_contract_runtime_or_finish_upgrade(effect_builder) + { + Ok(effects) => effects, + Err(msg) => { + return ( + Duration::ZERO, + fatal!(effect_builder, "{}", msg).ignore(), + ) + } + }; self.state = ReactorState::KeepUp; - return (Duration::ZERO, Effects::new()); + return (Duration::ZERO, effects); } if false == self.net.has_sufficient_fully_connected_peers() { info!("Initialize: awaiting sufficient fully-connected peers"); return (initialization_logic_default_delay.into(), Effects::new()); } - if let Err(msg) = self.refresh_contract_runtime() { - return (Duration::ZERO, fatal!(effect_builder, "{}", msg).ignore()); - } + let effects = match self + .refresh_contract_runtime_or_finish_upgrade(effect_builder) + { + Ok(effects) => effects, + Err(msg) => { + return (Duration::ZERO, fatal!(effect_builder, "{}", msg).ignore()) + } + }; info!("Initialize: switch to CatchUp"); self.state = ReactorState::CatchUp; - (Duration::ZERO, Effects::new()) + (Duration::ZERO, effects) } } } - ReactorState::Upgrading => match self.upgrading_instruction() { - UpgradingInstruction::CheckLater(msg, wait) => { - debug!("Upgrading: {}", msg); - (wait, Effects::new()) - } - UpgradingInstruction::CatchUp => { - info!("Upgrading: switch to CatchUp"); - self.state = ReactorState::CatchUp; - (Duration::ZERO, Effects::new()) - } - }, ReactorState::CatchUp => match self.catch_up_instruction(effect_builder, rng) { CatchUpInstruction::Fatal(msg) => { (Duration::ZERO, fatal!(effect_builder, "{}", msg).ignore()) @@ -122,20 +125,6 @@ impl MainReactor { fatal!(effect_builder, "failed to commit genesis: {}", msg).ignore(), ), }, - CatchUpInstruction::CommitUpgrade => match self.commit_upgrade(effect_builder) { - Ok(effects) => { - info!("CatchUp: switch to Upgrading"); - self.block_synchronizer.purge(); - self.state = ReactorState::Upgrading; - self.last_progress = Timestamp::now(); - self.attempts = 0; - (Duration::ZERO, effects) - } - Err(msg) => ( - Duration::ZERO, - fatal!(effect_builder, "failed to commit upgrade: {}", msg).ignore(), - ), - }, CatchUpInstruction::CheckLater(msg, wait) => { debug!("CatchUp: {}", msg); (wait, Effects::new()) @@ -263,6 +252,15 @@ impl MainReactor { &mut self, effect_builder: EffectBuilder, ) -> Option> { + // storage must be ready before anything else touches disk-backed state (other + // components, e.g. transaction_buffer, read from it during their own init). + if let Some(effects) = utils::initialize_component( + effect_builder, + &mut self.storage, + MainEvent::Storage(storage::Event::Initialize), + ) { + return Some(effects); + } // open the diagnostic port first to make sure it can bind & to be responsive during init. if let Some(effects) = utils::initialize_component( effect_builder, @@ -434,52 +432,158 @@ impl MainReactor { } } - fn upgrading_instruction(&self) -> UpgradingInstruction { - UpgradingInstruction::should_commit_upgrade( - self.should_commit_upgrade(), - self.control_logic_default_delay.into(), - self.last_progress, - self.upgrade_timeout, + /// If `tip_header` is a switch block that is the last block before the chainspec's + /// activation point, synchronously commits the protocol upgrade against `contract_runtime`'s + /// global state. Returns the info needed to later produce, sign, and gossip the resulting + /// immediate switch block, once the reactor is ready to do so (see + /// [`Self::maybe_finish_pending_upgrade`]). Returns `Ok(None)` if no upgrade is due. + /// + /// This is an associated function (rather than a `&self` method) so it can be called from + /// `MainReactor::new`, before the reactor itself has been constructed -- that's the only + /// call site: a fresh restart whose local tip already sits at the pre-activation switch + /// block (e.g. after a live node shuts itself down for the upgrade). A node still *catching + /// up* through a historical activation point does not go through here; it just receives the + /// post-upgrade chain via the ordinary block-synchronizer fetch path, like any other + /// historical data. + pub(super) fn commit_upgrade_if_needed( + contract_runtime: &ContractRuntime, + chainspec: &Arc, + chainspec_raw_bytes: &Arc, + tip_header: Option<&BlockHeader>, + upgrade_timeout: TimeDiff, + ) -> Result, Error> { + let Some(tip_header) = tip_header else { + return Ok(None); + }; + if !(tip_header.is_switch_block() + && tip_header.is_last_block_before_activation(&chainspec.protocol_config)) + { + return Ok(None); + } + + info!( + era_id = %tip_header.era_id(), + height = tip_header.height(), + "committing protocol upgrade" + ); + + let upgrade_config = chainspec + .upgrade_config_from_parts( + *tip_header.state_root_hash(), + tip_header.protocol_version(), + chainspec.protocol_config.activation_point.era_id(), + chainspec_raw_bytes.clone(), + ) + .map_err(Error::ProtocolUpgrade)?; + + // Executing protocol upgrade can be time consuming. It's executed in the background so the + // upgrade_timeout can be enforced. This function stays synchronous -- it's called + // from `MainReactor::new`, before the reactor's async event loop exists -- so the + // wait for that bounded future to resolve is bridged onto a dedicated scoped + // thread, which calls `Handle::block_on` directly. + let handle = Handle::current(); + let post_state_hash = std::thread::scope(|scope| { + scope + .spawn(|| { + handle.block_on(async { + match tokio::time::timeout( + Duration::from(upgrade_timeout), + contract_runtime.commit_protocol_upgrade(upgrade_config), + ) + .await + { + Ok(result) => result, + Err(_) => Err(format!( + "protocol upgrade did not complete within {}", + upgrade_timeout + )), + } + }) + }) + .join() + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) + }) + .map_err(Error::ProtocolUpgrade)?; + + Ok(Some(PendingImmediateSwitchBlock { + next_block_height: tip_header.height() + 1, + post_state_hash, + parent_hash: tip_header.block_hash(), + parent_seed: *tip_header.accumulated_seed(), + era_id: tip_header.next_block_era_id(), + // Adding one second here to make sure the timestamp is monotonically growing - + // it's important for EVM smart contracts + timestamp: tip_header + .timestamp() + .saturating_add(TimeDiff::from_seconds(1)), + })) + } + + /// If a protocol upgrade has been committed and its immediate switch block hasn't yet been + /// produced, builds the effects to enqueue it for execution, which will get it signed (by + /// this validator, if applicable) and gossiped through the normal block-execution pipeline + /// (see `main_reactor::handle_meta_block`). + /// + /// The caller MUST NOT invoke this before the node can actually reach peers -- broadcasting a + /// finality signature to zero connected peers silently drops it with no retry (see + /// `network::broadcast_message_to_validators`). This is why callers only invoke it once the + /// existing `Initialize` peer-gate (`has_sufficient_fully_connected_peers`, or isolated mode) + /// has passed, or from within `CatchUp`, which is only reachable after that same gate. + pub(super) fn maybe_finish_pending_upgrade( + &mut self, + effect_builder: EffectBuilder, + ) -> Option> { + let pending = self.pending_immediate_switch_block.take()?; + self.upgrade_started_at = Some(Timestamp::now()); + self.contract_runtime + .set_execution_pre_state(ExecutionPreState::new( + pending.next_block_height, + pending.post_state_hash, + pending.parent_hash, + pending.parent_seed, + )); + + let current_price = self.contract_runtime.current_gas_price(); + let payload = BlockPayload::new( + BTreeMap::new(), + vec![], + Default::default(), + false, + current_price, + ); + let finalized_block = FinalizedBlock::new( + payload, + Some(InternalEraReport::default()), + pending.timestamp, + pending.era_id, + pending.next_block_height, + PublicKey::System, + ); + + info!("producing immediate switch block after protocol upgrade"); + + Some( + effect_builder + .enqueue_block_for_execution( + ExecutableBlock::from_finalized_block_and_transactions(finalized_block, vec![]), + MetaBlockState::new_not_to_be_gossiped(), + ) + .ignore(), ) } - fn commit_upgrade( + /// Either finishes a pending protocol upgrade (producing its immediate switch block) or, if + /// none is pending, refreshes contract runtime's execution pre-state from the local tip as + /// usual. Used at the two points the reactor exits `ReactorState::Initialize`. + fn refresh_contract_runtime_or_finish_upgrade( &mut self, effect_builder: EffectBuilder, ) -> Result, String> { - let header = match self.get_local_tip_header()? { - Some(header) if header.is_switch_block() => header, - Some(_) => { - return Err("Latest complete block is not a switch block".to_string()); - } - None => { - return Err("No complete block found in storage".to_string()); - } - }; - - match self.chainspec.upgrade_config_from_parts( - *header.state_root_hash(), - header.protocol_version(), - self.chainspec.protocol_config.activation_point.era_id(), - self.chainspec_raw_bytes.clone(), - ) { - Ok(cfg) => { - let mut effects = Effects::new(); - let next_block_height = header.height() + 1; - effects.extend( - effect_builder - .enqueue_protocol_upgrade( - cfg, - next_block_height, - header.block_hash(), - *header.accumulated_seed(), - ) - .ignore(), - ); - Ok(effects) - } - Err(msg) => Err(msg), + if let Some(effects) = self.maybe_finish_pending_upgrade(effect_builder) { + return Ok(effects); } + self.refresh_contract_runtime()?; + Ok(Effects::new()) } pub(super) fn should_shutdown_for_upgrade(&self) -> bool { diff --git a/node/src/reactor/main_reactor/error.rs b/node/src/reactor/main_reactor/error.rs index 86a58a1a5b..4e0cb96d52 100644 --- a/node/src/reactor/main_reactor/error.rs +++ b/node/src/reactor/main_reactor/error.rs @@ -66,6 +66,10 @@ pub(crate) enum Error { /// `BinaryPort` component error. #[error("binary port: {0}")] BinaryPort(#[from] BinaryPortInitializationError), + + /// Failed to commit a protocol upgrade. + #[error("failed to commit protocol upgrade: {0}")] + ProtocolUpgrade(String), } impl From for Error { diff --git a/node/src/reactor/main_reactor/reactor_state.rs b/node/src/reactor/main_reactor/reactor_state.rs index 5cbdff2011..6ea1c22d12 100644 --- a/node/src/reactor/main_reactor/reactor_state.rs +++ b/node/src/reactor/main_reactor/reactor_state.rs @@ -21,9 +21,7 @@ use serde::{Deserialize, Serialize}; /// CatchUp --> ShutdownAfterCatchingUp /// KeepUp --> ShutdownForUpgrade /// Validate --> ShutdownForUpgrade -/// CatchUp --> Upgrading /// CatchUp -->|at genesis| Validate -/// Upgrading --> CatchUp /// ShutdownForUpgrade --> End /// ``` /// ```mermaid @@ -63,8 +61,6 @@ pub enum ReactorState { Initialize, /// Orient to the network and attempt to catch up to tip. CatchUp, - /// Running commit upgrade and creating immediate switch block. - Upgrading, /// Stay caught up with tip. KeepUp, /// Node is currently caught up and is an active validator. diff --git a/node/src/reactor/main_reactor/tests.rs b/node/src/reactor/main_reactor/tests.rs index c9944e25dc..f7b9298199 100644 --- a/node/src/reactor/main_reactor/tests.rs +++ b/node/src/reactor/main_reactor/tests.rs @@ -2,9 +2,11 @@ mod auction; mod binary_port; mod configs_override; mod consensus_rules; +mod emergency_upgrade; mod fixture; mod gas_price; mod initial_stakes; +mod legacy_storage_reindex_and_upgrade; mod network_general; mod rejoining_node; mod rewards; diff --git a/node/src/reactor/main_reactor/tests/emergency_upgrade.rs b/node/src/reactor/main_reactor/tests/emergency_upgrade.rs new file mode 100644 index 0000000000..b218c5634b --- /dev/null +++ b/node/src/reactor/main_reactor/tests/emergency_upgrade.rs @@ -0,0 +1,108 @@ +use std::{collections::BTreeMap, sync::Arc}; + +use casper_types::{ + ActivationPoint, ChainspecRawBytes, GlobalStateUpdate, ProtocolVersion, PublicKey, U512, +}; + +use crate::reactor::main_reactor::tests::{ + fixture::TestFixture, initial_stakes::InitialStakes, ERA_ONE, ERA_THREE, ERA_TWO, ONE_MIN, +}; + +/// Exercises an emergency protocol upgrade that requires "peeling" (hard-resetting) blocks +/// already stored under the old protocol version -- as would happen if a chain kept producing +/// blocks past the point an emergency fix needed to roll back to -- combined with a +/// `global_state_update` (an emergency validator-set confirmation). +/// +/// This also verifies that the resulting immediate switch block still gets signed and enough +/// finality signatures gossiped around the (freshly restarted) network to be marked complete, +/// even though the protocol upgrade itself is committed before the reactor has any peers. +#[tokio::test] +async fn emergency_upgrade_requiring_block_peeling() { + let initial_stakes = InitialStakes::AllEqual { + count: 4, + stake: 100_000_000_000, + }; + let mut fixture = TestFixture::new(initial_stakes, None).await; + + // Run the network well past era 2, as if the chain had kept producing blocks after the point + // an emergency upgrade should have activated. + fixture.run_until_consensus_in_era(ERA_THREE, ONE_MIN).await; + + // Build the "new" chainspec: a bumped protocol version, activating at era 2 (rolling back to + // the era 1 switch block -- everything from era 2 onward, produced under the old protocol + // version, must be peeled), with `hard_reset` set so storage actually performs that peel on + // restart, plus an emergency `global_state_update` re-confirming the validator set. + let mut new_chainspec = (*fixture.chainspec).clone(); + let old_version = new_chainspec.protocol_config.version; + let old_version_parts = old_version.value(); + let new_version = ProtocolVersion::from_parts( + old_version_parts.major, + old_version_parts.minor, + old_version_parts.patch + 1, + ); + new_chainspec.protocol_config.version = new_version; + new_chainspec.protocol_config.activation_point = ActivationPoint::EraId(ERA_TWO); + new_chainspec.protocol_config.hard_reset = true; + + let validators: BTreeMap<_, _> = fixture + .node_contexts + .iter() + .map(|node_context| { + ( + PublicKey::from(node_context.secret_key.as_ref()), + U512::from(100_000_000_000u64), + ) + }) + .collect(); + new_chainspec.protocol_config.global_state_update = Some(GlobalStateUpdate { + validators: Some(validators), + entries: BTreeMap::new(), + }); + let new_chainspec = Arc::new(new_chainspec); + let new_chainspec_raw_bytes: Arc = Arc::clone(&fixture.chainspec_raw_bytes); + + // Restart every node with the new chainspec, reusing its storage directory -- so the + // already-stored, now-stale, era-2+ blocks are still on disk to be peeled. + let node_count = fixture.node_contexts.len(); + let node_contexts: Vec<_> = (0..node_count) + .map(|_| fixture.remove_and_stop_node(0)) + .collect(); + for node_context in node_contexts { + fixture + .add_node_with_chainspec( + node_context.secret_key, + node_context.config, + node_context.storage_dir, + Arc::clone(&new_chainspec), + Arc::clone(&new_chainspec_raw_bytes), + ) + .await; + } + + // The network should come back up, apply the upgrade, and continue producing (and + // completing!) blocks -- proving the deferred sign+gossip mechanism for the immediate switch + // block worked across the restart. + fixture.run_until_block_height(3, ONE_MIN).await; + + for runner in fixture.network.nodes().values() { + let storage = runner.main_reactor().storage(); + + // The era 1 switch block (height 2) predates the hard-reset era and must be untouched. + let era_one_switch_header = storage + .read_block_header_by_height(2, false) + .expect("should not error reading storage") + .expect("era 1 switch block should still be present"); + assert_eq!(era_one_switch_header.era_id(), ERA_ONE); + assert_eq!(era_one_switch_header.protocol_version(), old_version); + + // Any era-2+ blocks stored before the restart, under the OLD protocol version, must have + // been peeled: the immediate switch block at height 3 must be the first block of era 2, + // carrying the NEW protocol version. + let post_upgrade_header = storage + .read_block_header_by_height(3, false) + .expect("should not error reading storage") + .expect("post-upgrade immediate switch block should be present"); + assert_eq!(post_upgrade_header.era_id(), ERA_TWO); + assert_eq!(post_upgrade_header.protocol_version(), new_version); + } +} diff --git a/node/src/reactor/main_reactor/tests/fixture.rs b/node/src/reactor/main_reactor/tests/fixture.rs index 106aaa5cee..dbebfc4fbb 100644 --- a/node/src/reactor/main_reactor/tests/fixture.rs +++ b/node/src/reactor/main_reactor/tests/fixture.rs @@ -122,6 +122,31 @@ impl TestFixture { stakes: BTreeMap, spec_override: Option, ) -> Self { + Self::new_with_keys_and_storage_dirs(rng, secret_keys, stakes, spec_override, None).await + } + + /// As [`Self::new_with_keys`], but if `existing_storage_dirs` is given, each node's storage + /// is pointed at the corresponding entry (by index, matching `secret_keys`) instead of a + /// fresh, empty temp dir -- used to boot the network from a pre-existing block store, e.g. + /// one produced by a different build of the node, to exercise on-disk backward-compatibility + /// behavior (such as rebuilding indexes that an older node version never persisted). + /// + /// As with [`Self::new`], runs the network until all nodes leave `ReactorState::Initialize`; + /// when resuming from non-empty storage this does not re-run genesis. + pub(crate) async fn new_with_keys_and_storage_dirs( + rng: TestRng, + secret_keys: Vec>, + stakes: BTreeMap, + spec_override: Option, + existing_storage_dirs: Option>>, + ) -> Self { + if let Some(dirs) = &existing_storage_dirs { + assert_eq!( + dirs.len(), + secret_keys.len(), + "existing_storage_dirs must have one entry per secret key" + ); + } testing::init_logging(); // Load the `local` chainspec. @@ -250,12 +275,16 @@ impl TestFixture { chainspec_raw_bytes: Arc::new(chainspec_raw_bytes), }; - for secret_key in secret_keys { - let (config, storage_dir) = fixture.create_node_config( + for (idx, secret_key) in secret_keys.into_iter().enumerate() { + let existing_storage_dir = existing_storage_dirs + .as_ref() + .map(|dirs| Arc::clone(&dirs[idx])); + let (config, storage_dir) = fixture.create_node_config_with_storage_dir( secret_key.as_ref(), None, storage_multiplier, node_config_override.clone(), + existing_storage_dir, ); fixture.add_node(secret_key, config, storage_dir).await; } @@ -382,6 +411,28 @@ impl TestFixture { maybe_trusted_hash: Option, storage_multiplier: u8, node_config_override: NodeConfigOverride, + ) -> (Config, Arc) { + self.create_node_config_with_storage_dir( + secret_key, + maybe_trusted_hash, + storage_multiplier, + node_config_override, + None, + ) + } + + /// As [`Self::create_node_config`], but if `existing_storage_dir` is given, points the node's + /// storage config at that directory instead of allocating a fresh, empty one -- used to boot + /// a node from a pre-existing block store (e.g. one produced by a different build of the + /// node, to exercise on-disk migration/backward-compatibility behavior). + #[track_caller] + pub(crate) fn create_node_config_with_storage_dir( + &mut self, + secret_key: &SecretKey, + maybe_trusted_hash: Option, + storage_multiplier: u8, + node_config_override: NodeConfigOverride, + existing_storage_dir: Option>, ) -> (Config, Arc) { // Set the network configuration. let network_cfg = match self.node_contexts.first() { @@ -417,8 +468,25 @@ impl TestFixture { cfg.node.idle_tolerance = idle } - // Additionally set up storage in a temporary directory. - let (storage_cfg, temp_dir) = storage::Config::new_for_tests(storage_multiplier); + // Additionally set up storage, either in a fresh temporary directory or, if given, an + // existing one already populated with a block store to resume from. + let (storage_cfg, temp_dir) = match existing_storage_dir { + Some(temp_dir) => { + let storage_cfg = storage::Config { + path: temp_dir.path().join("lmdb"), + max_block_store_size: 1024 * 1024 * storage_multiplier as usize, + max_deploy_store_size: 1024 * 1024 * storage_multiplier as usize, + max_deploy_metadata_store_size: 1024 * 1024 * storage_multiplier as usize, + max_state_store_size: 12 * 1024 * storage_multiplier as usize, + ..Default::default() + }; + (storage_cfg, temp_dir) + } + None => { + let (storage_cfg, temp_dir) = storage::Config::new_for_tests(storage_multiplier); + (storage_cfg, Arc::new(temp_dir)) + } + }; // ...and the secret key for our validator. { let secret_key_path = temp_dir.path().join("secret_key"); @@ -432,7 +500,7 @@ impl TestFixture { cfg.contract_runtime.max_global_state_size = Some(1024 * 1024 * storage_multiplier as usize); - (cfg, Arc::new(temp_dir)) + (cfg, temp_dir) } /// Adds a node to the network. @@ -467,6 +535,42 @@ impl TestFixture { id } + /// Adds a node to the network, running under the given chainspec rather than the fixture's + /// own -- used to simulate a node restarting with a new binary/chainspec version, e.g. after + /// a protocol upgrade. + /// + /// As with [`Self::add_node`], if re-adding a previously-removed node, the `secret_key`, + /// `config` and `storage_dir` returned in the `NodeContext` during removal should be used + /// here so the same storage dir is reused across both executions. + pub(crate) async fn add_node_with_chainspec( + &mut self, + secret_key: Arc, + config: Config, + storage_dir: Arc, + chainspec: Arc, + chainspec_raw_bytes: Arc, + ) -> NodeId { + let (id, _) = self + .network + .add_node_with_config_and_chainspec( + WithDir::new(RESOURCES_PATH.join("local"), config.clone()), + chainspec, + chainspec_raw_bytes, + &mut self.rng, + ) + .await + .expect("could not add node to reactor"); + let node_context = NodeContext { + id, + secret_key, + config, + storage_dir, + }; + self.node_contexts.push(node_context); + info!("added node {} with id {}", self.node_contexts.len() - 1, id); + id + } + pub(crate) async fn add_node_from_context_idx(&mut self, idx: usize) -> NodeId { let node_context = self.node_contexts.get(idx).unwrap(); let secret_key = node_context.secret_key.clone(); @@ -684,12 +788,29 @@ impl TestFixture { } pub(crate) async fn schedule_upgrade_for_era_two(&mut self) { + self.schedule_upgrade(ERA_TWO, ProtocolVersion::from_parts(999, 0, 0)) + .await; + } + + /// Announces an upcoming upgrade to every node's upgrade watcher, without touching its + /// chainspec -- as a real node would learn of one by picking up a new chainspec file dropped + /// alongside its current binary, ahead of the new binary itself being deployed. Each node + /// keeps running under its current protocol version until it reaches the switch block just + /// before `activation_era`, at which point it transitions to `ReactorState::ShutdownForUpgrade` + /// (see [`Self::run_until`] with a `ShutdownForUpgrade` condition, then restart the nodes with + /// a chainspec whose `activation_point` is `activation_era`, e.g. via + /// [`Self::add_node_with_chainspec`]). + pub(crate) async fn schedule_upgrade( + &mut self, + activation_era: EraId, + new_protocol_version: ProtocolVersion, + ) { for runner in self.network.runners_mut() { runner .process_injected_effects(|effect_builder| { let upgrade = NextUpgrade::new( - ActivationPoint::EraId(ERA_TWO), - ProtocolVersion::from_parts(999, 0, 0), + ActivationPoint::EraId(activation_era), + new_protocol_version, ); effect_builder .upgrade_watcher_announcement(Some(upgrade)) diff --git a/node/src/reactor/main_reactor/tests/legacy_storage_reindex_and_upgrade.rs b/node/src/reactor/main_reactor/tests/legacy_storage_reindex_and_upgrade.rs new file mode 100644 index 0000000000..ab37ca8c28 --- /dev/null +++ b/node/src/reactor/main_reactor/tests/legacy_storage_reindex_and_upgrade.rs @@ -0,0 +1,279 @@ +use std::{process::Command, sync::Arc}; + +use casper_types::{ + ActivationPoint, BlockV2, ChainspecRawBytes, EraId, PricingMode, ProtocolVersion, PublicKey, + SecretKey, Transaction, U512, +}; + +use crate::{ + reactor::main_reactor::{ + tests::{fixture::TestFixture, Nodes, ONE_MIN}, + ReactorState, + }, + types::transaction::transaction_v1_builder::TransactionV1Builder, +}; + +/// Path to a block store produced by an older node build that never persisted the +/// `block_height_index` / `switch_block_era_id_index` / `transaction_hash_index` databases (they +/// were kept purely in memory, rebuilt via a full header scan on every startup). Real blocks and +/// transactions across 3 eras, for 4 validators; captured once from `dev` and checked in as a +/// fixture -- see the module doc on [`boots_from_legacy_storage_reindexes_and_survives_upgrade`]. +/// +/// A single copy is committed, not one per node: the four validators that originally produced +/// this chain agree on identical blocks/transactions/global state by construction (that's what +/// consensus means), so any one of their block stores is a valid, fully interchangeable starting +/// point for any node in this test. (Their `storage.lmdb` files aren't literally byte-identical -- +/// LMDB's on-disk page layout isn't deterministic even for equivalent logical content -- but +/// `data.lmdb`, the global state, was confirmed byte-identical across all four when this fixture +/// was captured.) Per-validator consensus vote-history (`unit_files`) is deliberately not +/// included: it's specific to a validator's own identity, so reusing one node's copy for another +/// would be wrong, and going without it is fine here since era_supervisor just creates the +/// directory fresh and starts with no prior voting history, same as any other new process. +const LEGACY_FIXTURE_DIR: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/reactor/main_reactor/tests/resources/legacy_storage_no_index" +); + +fn fixed_secret_key(byte: u8) -> Arc { + Arc::new(SecretKey::ed25519_from_bytes([byte; SecretKey::ED25519_LENGTH]).unwrap()) +} + +fn transfer(from: &SecretKey, to: PublicKey, transfer_id: u64, chain_name: &str) -> Transaction { + let mut txn = Transaction::from( + TransactionV1Builder::new_transfer(30_000_000_000u64, None, to, Some(transfer_id)) + .unwrap() + .with_initiator_addr(PublicKey::from(from)) + .with_pricing_mode(PricingMode::Fixed { + gas_price_tolerance: 5, + additional_computation_factor: 0, + }) + .with_chain_name(chain_name.to_string()) + .build() + .unwrap(), + ); + txn.sign(from); + txn +} + +/// Boots a 4-validator network directly from a block store that predates the on-disk +/// `block_height_index` / `switch_block_era_id_index` / `transaction_hash_index` databases (real +/// blocks and transactions spanning 3 eras, produced by an older node build -- see +/// `LEGACY_FIXTURE_DIR`), and checks that: +/// +/// 1. The reactor detects the missing indexes and rebuilds them from a header scan on startup +/// (rather than erroring, or silently behaving as if the chain were empty) -- exercised here by +/// reading a switch block for an era predating the fixture, and by looking up the execution +/// results of transactions baked into the fixture, both of which are served via the rebuilt +/// indexes. +/// 2. The resumed network is fully live: it accepts and executes a brand new transaction. +/// 3. It survives a protocol upgrade: restarting with a bumped protocol version and a future +/// activation point, the network continues past the activation era under the new protocol +/// version, produces the resulting immediate switch block, and keeps processing transactions +/// afterwards. +#[tokio::test] +async fn boots_from_legacy_storage_reindexes_and_survives_upgrade() { + let secret_keys: Vec> = (1..=4u8).map(fixed_secret_key).collect(); + let public_keys: Vec = secret_keys + .iter() + .map(|key| PublicKey::from(key.as_ref())) + .collect(); + let stakes = public_keys + .iter() + .cloned() + .map(|public_key| { + ( + public_key, + ( + U512::from(700_000_000_000_000_000u64), + U512::from(100_000_000_000_000u64), + ), + ) + }) + .collect(); + + // Copy the single committed fixture into a fresh temp dir per node (so the test doesn't + // mutate checked-in data, and each node gets its own independent, isolated copy to run + // against -- see the doc on `LEGACY_FIXTURE_DIR` for why one copy is enough). + let mut storage_dirs = Vec::new(); + for _ in 0..4 { + let temp_dir = tempfile::tempdir().expect("should create temp dir"); + let status = Command::new("cp") + .arg("-r") + .arg(format!("{LEGACY_FIXTURE_DIR}/.")) + .arg(temp_dir.path()) + .status() + .expect("failed to spawn cp"); + assert!(status.success(), "cp failed"); + storage_dirs.push(Arc::new(temp_dir)); + } + + let rng = casper_types::testing::TestRng::new(); + let mut fixture = TestFixture::new_with_keys_and_storage_dirs( + rng, + secret_keys.clone(), + stakes, + None, + Some(storage_dirs), + ) + .await; + + // The fixture was captured at height 6, era 3 -- confirm we resumed it rather than starting a + // fresh genesis (which would be height 0, era 0). + let resumed_at = fixture.highest_complete_block(); + assert!( + resumed_at.height() >= 6, + "should have resumed the pre-existing chain from the legacy fixture, not restarted genesis" + ); + + // The switch-block-era-id index was empty on disk; this only succeeds if it was rebuilt. + let _ = fixture.switch_block(EraId::new(1)); + + // The transaction-hash index was empty on disk too; look up the execution results of the + // transactions actually baked into the fixture's blocks (rather than recomputing their + // hashes, which aren't reproducible -- the builder stamps each with `Timestamp::now()`). + let node_0 = fixture.node_contexts[0].id; + let mut legacy_txn_count = 0; + for height in 1..=resumed_at.height() { + let Ok(block_v2) = BlockV2::try_from(fixture.get_block_by_height(height)) else { + continue; + }; + for txn_hash in block_v2.all_transactions() { + let result = fixture + .network + .nodes() + .get(&node_0) + .expect("should have node 0") + .main_reactor() + .storage() + .read_execution_result(txn_hash); + assert!( + result.is_some(), + "transaction {txn_hash} from the legacy fixture should be readable via the \ + rebuilt transaction_hash_index" + ); + legacy_txn_count += 1; + } + } + assert_eq!( + legacy_txn_count, 3, + "expected to find the 3 transfers baked into the legacy fixture" + ); + + // The resumed network should be fully live: it can accept and execute a new transaction. + let chain_name = fixture.chainspec.network_config.name.clone(); + let pre_upgrade_txn = transfer( + secret_keys[0].as_ref(), + public_keys[1].clone(), + 100, + &chain_name, + ); + let pre_upgrade_txn_hash = pre_upgrade_txn.hash(); + fixture.inject_transaction(pre_upgrade_txn).await; + fixture + .run_until_executed_transaction(&pre_upgrade_txn_hash, ONE_MIN) + .await; + + // Now drive it through a protocol upgrade, the same two-phase way a real deployment would: + // + // 1. Announce the upcoming upgrade (a new chainspec dropped next to the still-running old + // binary) via the upgrade watcher, without touching the running chainspec. The network keeps + // validating under the old protocol version until it reaches the switch block just before + // the activation era, at which point every node shuts down for upgrade. + // 2. Only then restart every node -- reusing its (now-fully-indexed) storage dir -- with a + // chainspec whose `activation_point` is that same era. `activation_point` must never be a + // not-yet-reached era relative to a *live* chainspec: `ChainspecConsensusExt` treats it as + // "the era immediately after the most recent upgrade or restart", so setting it to a future + // era on a running chainspec (rather than one being restarted right at that point) trips + // `earliest_relevant_era`'s invariant in `EraSupervisor::create_required_eras`. + // + // No hard reset / global state update: this is an ordinary forward upgrade, not an emergency + // rollback (contrast with `emergency_upgrade.rs`, which upgrades at an already-passed era). + let activation_era = fixture + .highest_complete_block() + .era_id() + .successor() + .successor(); + let old_version = fixture.chainspec.protocol_config.version; + let old_version_parts = old_version.value(); + let new_version = ProtocolVersion::from_parts( + old_version_parts.major, + old_version_parts.minor, + old_version_parts.patch + 1, + ); + + fixture.schedule_upgrade(activation_era, new_version).await; + // Wait not just for every node to report `ShutdownForUpgrade`, but for their local tips to + // have actually converged on the same block first: a node can flip its reactor state to + // `ShutdownForUpgrade` while a peer is still finishing executing/storing the last block or + // two under the old protocol version, and stopping nodes non-atomically while they're still + // staggered like that risks storing conflicting blocks at the same height across them. + fixture + .run_until( + |nodes: &Nodes| { + if !nodes + .values() + .all(|runner| runner.main_reactor().state == ReactorState::ShutdownForUpgrade) + { + return false; + } + let mut tip_hashes = nodes.values().map(|runner| { + runner + .main_reactor() + .storage() + .get_highest_complete_block() + .ok() + .flatten() + .map(|block| *block.hash()) + }); + let Some(first) = tip_hashes.next() else { + return false; + }; + tip_hashes.all(|hash| hash == first) + }, + ONE_MIN, + ) + .await; + + let mut new_chainspec = (*fixture.chainspec).clone(); + new_chainspec.protocol_config.version = new_version; + new_chainspec.protocol_config.activation_point = ActivationPoint::EraId(activation_era); + let new_chainspec = Arc::new(new_chainspec); + let new_chainspec_raw_bytes: Arc = Arc::clone(&fixture.chainspec_raw_bytes); + + let node_count = fixture.node_contexts.len(); + let node_contexts: Vec<_> = (0..node_count) + .map(|_| fixture.remove_and_stop_node(0)) + .collect(); + for node_context in node_contexts { + fixture + .add_node_with_chainspec( + node_context.secret_key, + node_context.config, + node_context.storage_dir, + Arc::clone(&new_chainspec), + Arc::clone(&new_chainspec_raw_bytes), + ) + .await; + } + + // The network should come back up, commit the upgrade, and produce the resulting immediate + // switch block under the new protocol version. + fixture + .run_until_stored_switch_block_header(activation_era, ONE_MIN) + .await; + let post_upgrade_header = fixture.switch_block(activation_era); + assert_eq!(post_upgrade_header.protocol_version(), new_version); + + // And the upgraded network is still fully live. + let post_upgrade_txn = transfer( + secret_keys[1].as_ref(), + public_keys[2].clone(), + 101, + &chain_name, + ); + let post_upgrade_txn_hash = post_upgrade_txn.hash(); + fixture.inject_transaction(post_upgrade_txn).await; + fixture + .run_until_executed_transaction(&post_upgrade_txn_hash, ONE_MIN) + .await; +} diff --git a/node/src/reactor/main_reactor/tests/resources/legacy_storage_no_index/lmdb/casper-example/data.lmdb b/node/src/reactor/main_reactor/tests/resources/legacy_storage_no_index/lmdb/casper-example/data.lmdb new file mode 100644 index 0000000000000000000000000000000000000000..ff13f3c949bac4f143df9257c628854d52a99eea GIT binary patch literal 1048576 zcmeEP2Rv5mA3x_cUVBxP5D^+EWMzf2XIdhA?=171h5ET5x^pVMF5Kc z76B{*SOl;LU=hF~fJFd{z!wNW@^=70093#o@;l^T+{XUEB7j8zivSh@ECN^run1rg zz#@P}0E++?0W1Po1paXZwxe2r0Ehr}HQ7Dd`_z@y_RKd40YLJ0mOr#{&BBd&X+Hqq zEZ#=|TYzo2++-K=Pvf=`QxQGK-G-|{$cHC~48;3{-;K9Oh)rM-z#{N}h`^4yJ~IMM zZ1!ij^pIaR5VK_aEOA3$!p_W)Q>W`xx^Q}QVz*Z}fL|eP>}*SN#P;1C{xVs;8(zA_ z6Sgh=R~)a1m*bT(oyGysYCZ(O^&UUf&H_RLfNi1VhSovuPp_Z6D!iP=^V)K_olB?X zY(dCg$LL)RK8%uKBnb4|3j|#wQ&f!}h>6^OT9ci6j_6FoWSc^{edpTr$*WSrGPsWc z{)O@ygIhWb=~D^<5r1N*FiBw)=P;rITb zU&L`k4({=&3h*F=TYR#U_HiW{G;cEqqOB!3TbVT=PT5!m6mB5*u`JM}01*BxyA2ZB z1br^uqhD3wY~cBkoV9Up?}6mmtWh6}(75Ahc$ePpctGG>8yJBOh90hg2brQ-Mb}Ow zB8pr`YuP%gHK_TX8J2>zJErmgaxi&x_n?R>3$?K0H6DXj#{=0;JgfFA3YEwYR5MzJ zm*o(H;7LJeva2U(v=1by8{JiKuXz~Y7m_!~e@SbM{7BIK+;Cm~2{+jYpxRCORE`0O4~V-`jA{TN)b%BZW)}|h6$p>Vv$A_BMmG^CcobBu zUH3s&Q+PK+`L>Z=8EKa`X`BbedyyhJbF;ISRfE6a@%| z(6>id=*^yQoDH9yP1*nK!`SP?Ao$7VuHs;@vm1F5LhDu?a<`o2sv#hO$`qcPrf4PTqdGaVzI5N^T9S zz+A<Ya!rq``A|JX>g;04rmwD2P9HMo5R6GCn37R8j1YQ+UIlg#5&NiLuBh(5huu0uVibr+UjG)?iOU34^gyxY-V)I?r6n!g@qU7ZwK2| z*rHFR-?|pwTy&Rb#6H9g1j*}Xg}2p(-3-*I5eT#k6+`^NQzfVJ=)f1>hkjb8oP#z0jO4kVE$=VFq<~V_9>Up>nZ=2 z-+|?G^GtrfcK?aM~gXe3XZym>dC3q@*fF$ zgd+O?JGRVy;242tr^0Bo2@BhOYeM8dQ3v$BqJBgF z5(wNJ-brd5PibR19!%!T82F5{#`0`F?(=?BUY zan+6YwpvDn$`*aPM;z-)|7gM^9;Nc_9-t|iX`Ld#U2{H_GOsAu zqep9}uEhD0$>CZaJb15a6aL``;>47nu4P%K!BW;KxB};B8uhV@| zzC;=)L^DGDHeZdvXq9K)L)mWsy4^KaFqO$FTNBw^(C+^vy*9Kq#5_6vCGTwzjI*90 ze9S;MLP1{6t0Xinp? zS$xdzcfA`_KXJCvM-&7kpG56W?#hkmt=qNXR`HP%_8?c6Ku-q^&aGJ^G(zW;QOoJf zf*5?}MpaG(iK7K$LQ`(s5tqJ*9zu(!j2Qy{pR?yC0SM~Gr}f7id2IgP{K!3q4Lw@z zh`#MWS?X!WBw-oZ?Zz%EI9Go6qM+TpA=l*kjB_J*q=TfxX%CS)Yrm`QkBPLvt8q$| zyw<2O0^pxfAp@xbskkKWU;7~a?DQI6t&p;SY@^bK<9ZvTUET8+_(yMUh0D$HkDxV) zQP!vqb+}issgk-*5#{llw_jb-oo)2$~tB5r$F5ArK(;JuD> zv#6W;_==&MX5OG?F^$VP_nLOJ8-8eDwn)CIsrm2_ zdZL0Z5wCenRqm;$-u#4@x2}EtRNBd~*vV9Gi)@kimtwe7&_vih%h%@Ysf8V=p@+&Q zg3tFmnp_xmQ>ZITUvSuXYrR4Wv!{2s2x;2>GK^N3U#s8;u7SYV5JSW!gW`P~$~sna zHw%~->FnZcYu%!z$?Yu_xH_q5ZXCNjjX%Y{#`I<6B}Zk-oP)F~G_lLz&uGRFvmK1KpyMOzzg>ASN- zHNz(kcpJ>9AS_eb=o`(i%L#_@c625-;!L!(UERhhyycAN<|uMe+;>xs6Sm%2<&5A4 z7k@vYkf;~WRnP2c(A}HHlsNFNbuW%=&Ojf3$kj`}Et}O?>63lXi&;w4pRumJwu!Zc zfwqaRm4St+l`*1Qf|Vli-GGr$wAsB;&ykV(*B8w;3x;GiYzcL0yQ`;x{34sFm4l(3 zldc8g;?7-c)*YYM7q=ezC_zj1eC+NS5sJaPk%w>xwuC;)jzLnRYO}L;v@+1PH#9Z2 zGPSnTH8#|?({(UJn5|>K*SYSLnz7lA^#k?oF(0GEm95O}cLXP-<37c?^%9BZEIU{` z=vrvoJKES-978xM8oA>?EZchQvuf&C?)8MjN}L<>GNhjq$T4r0&b`NjM6+~}s6Rbj z3tcOH!(~bIqkxvs**MzS8Kw zhNyNgrBIgulKFxxQHY=mLZ8u9EzeGZ%D#5to;K>fwYH&)sl5XVb?v#(Sg*w?88aV13YA=WQ8<+psB{U*v ztR3uht?Z2q?Utv}ZA3TQ!J3qu5p_m4-`?8MPT!COopjK(Gd6UXOD^DxCN?!7#hdFA z`Uko;^xq;+9`=zV2Z#2a8CTqUh?O}u4e#EY;)abPFU!H3)_fbBRX26A4B4P~@_xSU z%?^aa5z}g~IE0I|>>d~sme7Y&tsiyp1@p+QHR1T!nDXSIZ>=8f`V>Ep)i)FfM+-+T z?n!EIIB6kpLAikIUhzI!Usg@j3Q0-kD88V`&6_-m7sSjGpVMm(yb03UZaICiFN4V` zVvRTUTZ^6s|LB39&hrNdr~#jdK9QFaza(KKRw1Pz2_`#F%ubp>N6jShc474^paaM1qq?>`#QTnx@g zH(Mc2!C3j2Z-wWy^4Kntb=vtwZiBpg!U><;zRbP$yu``X@0wB2qBe0(T-I0lrp*IN zb{H4DrFJEK5>KsmPABZNH9Li`C~pdy2$oZTrh#V!B+*9ZyBvQ|{TMfYgRgrJZ(-6< zJ5RWG!C;PGsj~WoM<94=SXW?%c}Ew;sYuVLY3d}Ajs6i=HrYRkWJ%pA@~C*@+++Yj z#yh;;<#xuF?;q!`ODWixz|i#OSRr9zwD1eH&Q~k}^8ckd^ZmXMqE|rxBcM~;IAG7? zoEPo(S?cL);7*d%(e(G96jY>zOgR~+b;afuN#89B1Q3OYNJ9tXcZ?Q?cLvqgykl*! z9CtF=5fo<|EBp2e{_wKwAW(9eXlheUqNP&n<=HCXVorUB#w>}Smz>9NYId7e}tk!?gJMk*QtuBb=;oM?vE1oXdAz6*Mlh<1c)BjZjALAi*yH+ zPjXO=sWl|u%2LyllK|Dfhr z$d=O8IJM^sbLx+{vmMZlK@)?3{KeyaygrBVA|B`(Z!H<;o9y4zWO_J2wp?FHb;+IhzNDw&wfY9^6;pM zqi>#z%t5AoCh?c{#ojll+aw7fYHA3QHW!&{jZ6rRMux;V5t5&*YGW2N2|GNNw5DG_ ze;!prz+hqYG>`FbKaXo_MD<$bu7}lrq6=~cVaUUE6dRkhDSTH2WP_-0#n)93=KV~; zk3ZD*ua~l<$bjBiqTgxpR(kjpTVGN?nINRIy`I1OX4s3oI2va8qV%V^IMEj#&@Ry3 zJ2pTvR4r+EgN-8JD~6tjh1j&DVy`mQ%PZb^FHgt}GA__@Tf4P9=iPDgarGYQ*iQdeZ)-YJy@& z#0zvbW5i{=MM$+t?t`)KRT8(f^kJ9uYiu#^5k;>BVF4)H0TjiL3l zg@rWxU+RkW8MTQn&?$F%Hko-19!;bC$K}u^-hQxO-q3f{O2Z>ioLq7LyHfsu$bosqJ4S$rU48F#rQUgkXm+7d6N1|7 z)`BX$ya8jcjiT#67LXdDxfTx@i`=AH*+25vw7~Rr-w63lO9Ma z_9OFRq{(eL7U)(_#eC=)(BsQij|d-Cd{fobc2FdhK(n^{z2_~w^eu|e8!f6UPgZU9 zfV9*dc5BJQ<$LYgd zsuxRoK-RJ9L;hv@1v?FUh4y+f?PqqW$B#jP*E& z4k&PctjbP$t7}^3E3!ZbCu)b}UF_>muGib*a?*C=_$r(5o_zv^l6&g*nMF}IcibEc zZPkJ_JmItu$QSAj*cHdloBYCrIgoiurFN}q3iq=Xp$RwO`34zgFWkrcyBFw0@FTX2 zfae{Cg%n(N>7`4J3m=U(kbmLjiFBbMuHrb%ut1l*=kN_HfU!T1GHu{nyQ=U{CeJ-KzRBN{XF1I!S1mLv5L4X|TSjJs5NOiSn zT=Zat!M;!ITuqxf0|J_qyr+C=>*hapzoh5CUnSW6|3A0?hgP2}wsPJN>3tk!3|ET@ z8#S`gVhy`l-fR)~bi37_)ybI>TT+~I@&`|?5I1KB?OhHzxd4}RV+q;!)^7zG4UUJr z&JJ9q;(tWU)?2v_HQhnb!+B$4U0ovWnq(b|+Hu=1nvk`3+D>Yw)&>_fb}_o3msIfF z91vjfH;KCRZ0s!;<)dB74=JBnNEFo!IqF#q3g^^}h@5dk6#~!hf!F(hvr(CQfP}bbl5HtbN}HY2*7QK|B&IY{R9CAiVT4msqllR4pQmUvq7jIN@ohSXF8k!;&YAi#zL+Pb=UyjP6!Glm zJ@&Dl#c_2p?g!s=#JeY7FZ+wDAS|Ei3-Ukw00A0Nv!1(tPyN?NsWoZ~h1rHibT++B z>aiuy$78L1b!c9e0Rp*0_HQogZL={R<0r?dqb}%q!PEBQUGLM`P~1uu#p8SbP%gT2 zaQpAMEYut4CKLQ_7)6G%Qc}5`R3TsB)Nb?L8ONZ5K!PV0y4<+NFqXaq!EG$r+kU{r$Yp3nd~m4 zqqK|i{`^#r|AnvT>KplY^65lAJCrE+1OE2;l!kB`o$_;x6GnJB5y-UPuKC1>J_lP+ zUAJyxrFTwvB7SoUE{FW-XTh|++)6kcFLEQ@E{9)vQ=~^9iJh%Z+j?W#C9i|!NbJ4UK4*g11YOc@k)m17Xj2khM|}rVYb$L7 zLkrzwh*cT-DgCSJr#$XOU)J$DO%y+_WjQkx+@6@N>s2#;2Z2QEPZ`?jY8#@I%p6QD z4ecFtEp4})brv{f=V6fgoK(dpxW|X!{`Pr&aLp?-m z)%81+xa$Rbl}((H8&H|QP@Xf?H?=V}v~sZ5wztx?u{W`HK)jUn3muV>G$H6Ab8BiQ zxGY6C<90qwG(5fDV{LDau^EyLz7TCELpysE?g%E*x23=GXreX}%`xztbab=es8QJRV<#X&t&1C-L7o`tEt zwz=W4IMf}~J;0ThwZ^p94>4J$d29QlM>qJ+;geg7Y3XwvE+VE|i*!NRZ2t=VFeK1> z)!lk^){@cow?661O}I@VZC*H|{cjaE2f_;dL#d=Ws1KTTt=?1?kGpP2guf$$Hln6N zs3-Ut;9l{{n)fWKA`qzK+=6V~r~sZ5e%=UX$(K;S}mtC}|8$s}3F>%Uk2Qf?4_>MRk@rHrfQhJg`q zv)ht)c1xyjNPHpkK>U^Ux(rj2TXtWY5=AI()YoY}gAsoKPG|;#k$tNlv1S|sjWf5L z>lt9oS~d3g%%gJ>-IZI7{Rr2_Hli8@f^fF0Xd;X>=|PaEzbbN~sBF4;N7ZYCbkpS0 zq4D$e3?cPbpARehmCJ zNnP19$GJH;Uyu9Mk+m`Cx0ze_&_THW;C4^2CcltJ*7NNtk>?(n*;7Mjw%3|@<)-zN z((7*W_2`&|iiOU<$94|{r0g>5j^W>RP`tJMs*=Y+8cJ1N*{&>I>!eFWLgd^rORm14 zhA*D~<@>=!eGoXCD-7Raa6<1z-2RP;A95hDyn`6}2Y}Xla9lNv#$tK>e?9$!&p)h| z8D=f{B_E9Wru+p~%X~%xv}w!y0Ro2rPgwnbPa?<@z$2o0UjS$ufU87ke*myn0RPhO z0nYfuq&qx%$}a*5=Td??~O$=6iG*J*K1 zyXRKLZLWh9v-iY~TE-low|_-ZgCLyx(n67cIAJoQFy{~ciVp?|z=|%H`UnDWT5OB7 zE4xktfyi|hSw`C{a@Pu3lO8VpK5MRG5PmFXz zSrzncebMFeqBGVs3Ovt^9J?w>m?xc``lo8@hu#&@i*2O@nIey$Pnw$p7&sThY}5K{ z0mknCe=XksaBbNAKgwtTvjdZlFJ>iuhiYZNYs)Lb@T>Hz_y6UNJ5nSGm%xHRv#E-zz0`% zu{;GW@{NLQ#$X(X)Kw0c6ULy+gR}oYbLY`?aI8!;Cm&4*r~i+Z=Ah|*SN(t9h4%ff z`u{`=?Sor2Q7q8G>HnieSLpHh)%yQT3-!XOJN#4n|Em|;2geH8xIp)-_5acSI?&^` zQu_b%doxB@{eyX)A|n8&|1Y%AzF)2Xziy#kIPMFi|Bn_*&mRb2E#N~GP9{M{LE=O7 zXX}5g?LUMR_KZaUivSh@ECN^run1rgz#{PX5P-D#vDSWn566|rjrGm)JruFNS^ngk z1r;?8ePDgFd~@FX&c0btu>+w4ZU4U=FPuz|#ExhNc^SDKkDb(&n314~upHkA`R5n_ zKkEArvJ--{_5`5@eeVpyGJ?OC8}1$m%y!&j$`-8~;X?**yXM1~dV3=7oDh?N+IGAV zrku-FAE6RZTyR-4&GRnpz3mCp-4;V^i_ur{Xr#fiM&ujQJ%wnCvwU_-erR7OOhTa;E}kRj43 zE>oX=GeRN3O@0?Rcs|u+D09ayDHwJ-=E&)DkW(mB8=nXv3A}~2LO3H z@t~$7q%59savSnvdGRnb^1ySgjR@sEMs2I}=nqz2ki*1p6q-{;0C+}l49_O95kR{6 zaB>EeZoYBXercV~E4C^R4jgY4sM$L}bMB~#uf#xCp?Y`k9W76k@74iX_Dc50<(0iX zI0C_kBXl<(W)srXpWx#w+q?Su8UDv`evyz!z}NfMr3WNGrkf8yUwik+`>aAXwFKFO zLgJIS4*Z|E3N_gVZp0;RsQPr?JSr9=ZEyG}0O=3zbte@$SL9f7Q&^N7jA&9Od#UQK znmO_)ldv^(stbiM2wr$KJ)0=g{ibL4o+b*Zowg~8ZjnlbHUmmK-PtY*=WRn1t%z?w zR9zUUgM-V%{9)f>%O1n|B=#+~1j)bbEe6eI_o5S<+3;@fu+SUkI`U5RlJP3uv9xoK zLxxPZjA=aTf4V}?Kgp*T-PhMo3s+A>wMiyB+m|pRPLXT2nm01&ul-cG&kp@)gXFd8 z=|^4bY?NzEE1p&F*ulY2LrK)9M)fIBn1nSOXhRdhKK~$)9*Uc=E4RmxTbFyQM}~yT z-Pdm37tW;GCidS7yP?A2iYf%&8^m#b_UFyl2#!chJSO!gE(&vcsuW6m7ot|8hVYM($C5geiyLvkj=Elsh$QHfP1$-MP(g z?AnaVo1v`B^j#Ruxz+WiX{)>UK5}t7PsyfW;hWU4jj5p}D*gj;7w4u!Sr4!lM*sA* zlEkhtFfcj$VvX_jP{RN8K>o7&w_r76e;L-mykAgseF%USXd>h$Ga&gylt9!(XaOb@ zDi9(F%y8t8v^f61ph=L;e@%o|{JQl1_G-GvZ8w}v?!QwIyXLX=MGwfRsD3K>+-{7% z3~<AgJbg=t=V>lQvS#0zWrWHWNDw96Gz(gKPIbzDL9cm-`wQKdmz$oVW? zbb91`%BKA)RIRyqqop0fw1?OpZ@%?bJ&`?Go*p9($jXk#-}YMl7n<{9PO0ZA_=P^F z&&;pM7~$P{gK&-?=1eL}p>@v<-iIlh?i01%@P9GPdq3siXWU%ckS>|9Oxy28%y802p~92FFa%*ahGY_z zNsidQyTe~5t9Qdow|K&~rT>cK74dSsQl`^5Nc2|(wk8lPg?uR$82QFNH?oXy?v zuS9j68*=M?mNVL-|As62QjU@lsyvKcrTtq5ZDrdD>5vvMGzq%owRw#kR@fVKOyooN zsSqk}=Q2-Pkwdg@k81e={7G854c6Ds)_770R3vjYD&8E;ikdANcyO~_9yv2~f_@>s zB~TeS7vEAB+C$}^W(Bipb8Me-3B8{3e>whM6<>kkY^EzwdDQI1_`xrtxI-LjSZ5>T zVDjkhK@n9JYGKD~JO-_f2eO@bR_#|5Dv=+kX0!}1%RzCdPYODdT|GggeIQBQ=&pi$ z&BFk{ki0?uOIl;(M}qF>hU@ZAxXDHU)o#is_l7<1;*<4k*hoJvaGUt>n(o}AattUA z^$2qkyT-a7<525y%?%MFyuUWsczXbV_Vtc>+ZwDwS-nDg411@2>xjawISytOl*)VT z7ZX?`g>UWoGb00S!zKYf!6hQ9XncP_C$LZ5OEyIt&F_X z;IwB$wO)Yyo#G>m?@BIAuTaD|IF;+USpcWMjV8j#rwr7||C=*3nsAXMR%l%V=ZIu( zG^ZWpO~R{=YZ0&B^El`>VMPEVbM-qy;P{}*{tV~6p=#LC{MPiGx0gcSMeqMLkV52j zq#8&@l1hSbf=ZP0II1NBfo~Ooz5Y!EAkW9JqS0@b0Nubwx`L4S{`tqgbiX3f6$nfb z9n2=a9deS9W$L(AXPAkll2h*0-S&-Cmz>!&5M1`?9$^{^%WZB!!U!Ls3v5V=^g0l! zH+ztf<1(HWZ>0Q*9EoDtkMU$YvvH#`kmS#h^uUd4B=x2x)BQUa$Y5&X&DFagUOd{srFm=Y140ecS{nD0C>`8 z4evJIB!__fPsduEeJnPX;P$Di*AH^(i1bLAF@?^-)F0i`NB`llIET}W;4!ZJ17b%J z+5zGoqL~#Q_R2bsT|uBUA$AMVEa~t-Z^jMI$)M-D9K!k&#!4M@Rm~c76$2a?N&evd zzyp~DG5E}ls+_)}`IR;BUdOpv)J=VS z#nJdmRKL}4*fV|_;_ldjs#xV191ZPA3+v4y`8V2!K|J@qE zXovLk2Ljj#L?Y(n|82xIAo@grjsFK>gFXLR1h5u?zZUQB)dumfw1|=t_U({nZWBoB z8y)_9D(a2-;h_EJ-+wfqxfq<0Zni?4g0b>3-wMxZ<*{8P>$LNW+y;5~gcClweVKdh zd5M#&-!-F}irU0EaamvGn>G(9*aavalez%6@ zieiTT@5dFLYd5A$PY0}xDa<(k&Nim6H_tVm5#ALr0?y8D9tFZhfA84%e=zgn7u<#< z3E235i(tchSY9OkB7upEQOx~&e^K<8KmO(M9t)L}x-j?79X9=xBt?l6U|c>Nps`3Yq865^mDItjra8k}_2`udVpu-~naN~& zmQ1{TJ!)G70%Ozl>J5)$Qq>tYSjkY}$HdfK)ZaSFbEuGIjkRgea3gjs{^lHl-Z59? zwvbqw)M*DBk!%^TG=6@ArTG3JO^1Fb!-K;rwKX^AUP*9uq4w<}8!M{`)y~s?;=~y( z;LgWQ3zU7~(Es2}DM1Y908Rs334i}SYDaO>N9_oHTKB1khj#`(@^=pLXP1B0p*d8# z(d3Mym;aKC=#Q1HkQXAc2}Pxws;{1gvRLN=83*c>Dt)XS))F?_S*U;hWh6A zj+UIkFQapSySWcVCX8o2Y>V;47}ofEtUqjipm&6tG$nu*mCMk@P~UMrxU!+WqXjBP zGL*Zi(Eqq*5h7H z6swRJt#@GA@!-n&Z&EOJWqGpOeP^cklUA!EZ>7>!t+LQ-(h8pJm3Q=~s+iLh#M+ep zX~3ZsGhCDr00CznIA1g&K)zT9djEe2hnuX8*ny~l(1kD+c?fw3=L7LmLRO+syhSo> z;vYi*(w~J5Gl+0+C}d#TI@ES$CPpGcs$(Z#a{uE=*$Ks&myE+qH5n~vPj*WO-r`OS zHVhtOaPqKX;i(e6vb0CzbQ14X8hWcA@Ma{$OjF{@w_LQkJzpx}qJ3QmeS36;-t76t z+3?xfl>N^>jJ*!LYDMrPhHrCHO(K4n`@^Nx59No#eHk~I zshnW-!YTI;(?4m08nUGW1ial8jkBjOs+57IWkS-Xt3sIgQ*n*r-_KYnI+#VfC@j&U zQYd!xKvl|-b4i3G*TwH>CLfC49YA{XXzi(s_uhFoHt)yRLDRrRqWPl8AN=vtLN)(G zp*caI#V0#yA6Jq=^EQJZ+FF9Mm01Jgl#NwD;RbRa%L2{$@I%W{T3O!ykH+o<0cI(! zn5r(XB!1Ep+MC2LYgC=PGq#oWJZHNXQOd{zk5$Xsim7w}8{Bi3Ub@t{@X=@k`4?WE zNEaI7DvrbRKtf;Ym3yN?EBDaz3Ag&z2ny}bVU5^5ReC0R*DV?0hRbAwHT}3;4OW<0 z?7;K3R&Q)E5RKKl)TjAAjxq0{TK@Iv9Zv3yaUS7VJf&MqDieasmEn*OOQrm%HxjFW6uBtw zyD7&BTkouLMsS0Rzn@S@6z&bYOZoqGZLo;M z&rb+I+Is=-_%@#?==JlzU~n&x@%8Eh)rp@C%yM>ntUn+|V{Y1p(K_tsE)o{e_{rhg zf2MSPu@B!#VETFedJ+G4giz=^6tF9foj3V~33DLxluGSd)fDb$EkYA+!1E0<%wD*U z`B6NFjYAKO8@u|;Nz>RO)###PXeRjvBg+$5r8AOb_<&wTjHxo3|0@KZyB)zjnw8*TEQdKQEx z1j9j!~F;hwcYhfdK4YX}&WS_YJ$eTJ}no2tDGC5GTF-+`x#=l8JSj`{zhMo`$XMbM3=0 z+_e;>y$I)Z9K-byTzKk5?EXRm1egxe;g!9NT&sLdCSboLeSD|z>`i9o8>`2CyVZgp zZsvgs{ku^~;7rz`dj51?A$j2J>p>Q|#w~8Alk_vl3d^lxnF)p}UF z`tG|+kz!nIy|un9v;|C9o85Bh!gXd^4w(nb%EmM*$$a}6;ngPAEGzu$Ni-Xmox;&Q z9yLGlAT0s%t$U_Yx88B-4ay)EY}$TryOYFQ%Hq`4I_J&~@dL*dQCb4cplw;15wGgR zPNuJWo5pHvd>wF1uR{DESgn>dbByBe6}#~2fplKU6!$Bs)7mif$n)-MKkjqIq2*xo z!?QKyDvk(xvY=cU6gdb^cRY3S3Js1d{NOb@d@AIgib4!q{wEhsfV7xj8_{p`z54$1 z_n>z^>8xkQ>p`==9k=#8{9t$eknZ|8mdEl$qp6_J71oJ8BN$0d;eFripuNi>Cl}z7 zZY&}D-ukUTqrvfz*V%!qRQ!*K*?KG2q3{MlxTsiYB3O$y5Mc2)iMsS`>@64Nqg~1m zDW6$L6x9ql>RAj5=hTddoN+=G0&nt`Vd%fnY*Pjzws|#c;B}JsZ|T}L?x6OJ;vVUf z%wz~`0>OsXLGDklpS&u(oW}Foa=4vKr{!!x$X>_jT@5~rl40{s#XxXGf8X=>X={-k za>@86Cgh(DB=yGD4NvVdRWq=c0B=tH=V#hd2cdU9@m?OT^5p7Oo2(R`#9uLp;Z&H;NI3?<4`7V3O_jKrCJ-(C?H~2 z-#cBYcU~cyU1-#Vp!T}8pb9TYbm7 ziB*iMeeI>?9@p;Z9T&?&OC_*gs+VTb%BLt7i$nn1fY&Jb|3&=MxNXF*aj1x%<8H&% zAmqc7Lk8k~!teeEbpU?W(*tp$DMPQ($HbnASE5X&%-p%*?%gU{+oPn@LdsVZ*Q!X1 z-p6S5zlbCT36w!5*W~()b0c@8gQUZ050N@+zpL$!iL}70aY~iE)~GQ80wu%Yc9*IS zvc&Fw+G2A5NT^)7dd&W8(RBp(vlCg{GP*IS=651l`4tg%z5D&~g}x1NF{&2y_4}L+ zq)B8C;vhU@PvXY5n&>NTrslL2A>oqDF9ozcf&fTn1sgOHA6EN_5Y=)5;ChdrYG(l< z!J=gW+`3|+rLK`FszwjQL~cK=$xb~-bf#glO`+Vrb8Y(MRViT^+{Xa_959TqTIax; zxv$>$U_`HFAQ2b(HLw~N7k2=9AHi;;Fx$b!4LQv3;2?xywF?mbEV~U7+5~+r-J@Ss z;cVdfk({-0Z|{NR*{o3?i_o~^^BO+T-U1H@TtK9Qp@(bWL8fR{(X|tah$7d~TDFdA z4Qjq;hNWQbj;TCUyvkqU4ICd3ccmEBSaJrwGz7B?2l@(x$KzSqy%eLH2oyXDD%P(1 zAgd|7o1uJLsYp~c-tK~8s}TV4p&?o>Pe91*QUPf0Yah7#gh0 zy)4mNW$0MacxsQ***ivGS)6>&^Ql@_db@x>!CL;YFpBn=89jhG)*L{bQN?}ORM>R! z-Ki_A=Gn^Z(zttCE_drAg2fA1>1cz6=gTBy>Yn>ZFaphjFFfwxriUk<{ycTPo-#}B zPD=c}`so2zi8Hfa__h}@`j-4$y>Jj9WC6VOXmi?vXi7uYAoJI`pL7)8Ic;&{+p=NW-lMO2P2-jB0Qr}_vrh$-J?x9+QwjkVh| zO8@TF<+6m6C{=KWdQR)00h%@UlZxstsFZZr4L)?*MfPM&Y9A~09!3HXjsRU#BL3J8 z((c|_SBGS399xpyU_RYfVr&KoB*ha;DIQU36w?QTnn2JUlJC9nVR`xL%p=~`Sg(CK zZ!Kha11oGkwVc*gA1ijZAj^EXLOXyv-2{x3?()<)G@;v%U*ph7idRZK+SX_4^78%~ zyM~Jas8)j@T-*aR5mwIxg~Fc>NdEuqR2YpmVPU&(O^Ey_>VUpi)NkmY9L~O8kQ`Ds z`84f|$Ps@k1iFBID?vzn=L`avy_HZQw}y9;TE|n`n2raN`7#DRCBdaqRQKWp#i6Mg}p^ z`{7Y4-|hjLl9|>i0^Bv{Qz`R`f<1b)cIt}EBwT#Cc8upY_|7fs+;@`j8pGeo41WvL z`SA6UerULt7lGkuqD_(J548V(HQY*)a3VF_aQqg$m&8P96JzXwMF5Kc76FVAfVeUr zU2pDD&CTFMTE=>M42scC%elUi0%%@7xm15bB1QZ%YC(GAJ@Q>vTSdIoC1!GwueSX}#YL;x~Btduop6v&c2ip ztx;Pj%r-Qlv*~S8k1csV9&7EZL-GsN=MLGwxv00z#(0dM9H)-Dpyvfo+lzO-PiI4M zD_Ine?_D6nP$#QcF3Yz1k)y!j>-ZC)oznu3v~=xxM^z+(*9-9;mFjxEP?`vS#Fi28 zyu+}Ng3GsUXCVKeaH_u;?l69)VcRm?9VjvcVx+xd%PYoEiw~e& zu(lk{?tOW!hDhdW=$0q0fhHk05Z4-2z*hI z0RPDYlDDn2_6%13|LaB-EC2s>SpWK(AiXiFAORr`Sr(fs-${-Q=mw`7I%4j27Wt*$psTiw0)k&DxLN;U-x-=vOh zObsnj@gIo0I5!>2dH}zI`=OWsU-ap({y<4`E)s z1O7%r8(cF2zpoooF6R8F6431LpsEMz{Kv$u9ckdl7Pn0xsK$g9d{c(9^Zx4<@@ZkvE&4<~9H1#L=_{#RKzJ7-P zFx&R84uO_p}l9u755%uWsXh5yZ5HJVWY^)a`2`#-v(#ZO`R-5HmKhF`L;JZ z5DrI7tG(h7F4D4lU`$v-A5OJ?)WH|bBe&Ls(|Se{yfC|Tt9A=HDwHgwzW%Kh?{bZ9 zD4sB!vB9^>TFJ==0uPy$CIjVF#2=&>p8t4foi(qsmUg*(&;v6(m3_y|HFZ(7fq*{2 zjUGy=rsP_UyS|$RZnadueH+5(M)Kn3E#cw@1|F2E{ogr2fa>^7=dC5w?;=;pCo#o`{G{hh69#ifW61|#tpz`3o%(&;36p6ZGG>z6otoUeQVjuMjym`hzkEJ_lP+UAJyxrFTwvB7SoUE{FW-XTh|++)899QWIS(0}De& z8{K1;hE@&@Ns(R$BK2ku5^`L|)8dVkKanF*Ec-E@jAu4(RHk4E7nQ@()XD+Q@7sdo z@kU4=zF$I|g?waVAf-g3y_^O5K|3tJOPNeQy+=#$!GEyqVlb+uRF zo#aGAPKL$7*dxuHpL0yJBuFyY65*eLAL2}*?Eg*^O>L@4v{Y)nJXp(KK3<>0co7eDjklJJ^G$cK z)4tUwwd)*?I~bF8A`!L~m^VDJG=b5i(MnhxiQf*&b_n#K*R>6 z>!+;)-OEB(aUAhEbjT|g*Q%je8TEowk~xYmD01^AkKzR}v&850+5>Nbw6 zdg{$jczNsE*H5LL42zvi<+jKcd4DN}O9f4Ywc`2OG=z2)u)$*RSss)W_yuj_h*Tbm zXzeDGmLEQ*|0;E_&vvB3rsG^Ycg#imLYYEV2Fjr=ZwZzRR1p9JP=lQJ{|EX0>m~Z9 zE&soU2KE{jfxnIb^d&3{$~;Ulej?>G)4t8Ioc+pgoQymrhVGo(9^~6QX7*YJBd!tb zq7WN1<*$#_U(d-umlbROkG22*f3g3UTj#|BX$fHc|NZqh$4cddBsEz7e=CLP%9Vrl z|Cf6Y=j_qu+`KyR_%fkrRu&)g`(5t_)lZyl^brLCtpC52n`=->KTCsoh4t{T{{Nr> z`#(9OV~o!q=>2~Jo&y;RSs_URUNsRffe}6~p#y1dIS&h(LIWI0}T{Gp3keO@1MdtmoTPBF{ZC zv!{m6Y_B!*%1!GjrPtl$>(Md$$I<(lF~xo_TiHMMeE#>!n!+xx14BbrXw8AOKU)so ze>V+jreQsI|MW3tg?J27?-%*h^|Qj;>cVaY>TgDT!rL!> zGw$KUn)KGw_8bagpFao9Khi(Fdq2B^-}_rmv)Fy~{=X2Po6LfAHEAKfFi|y<3K>pN ziSLD24ay;DAyLvI{eL$B@#DOK4sH>6{^-; zywTDQVcJ7%k2l|XtDeZ7EKiToy6Iar{>f67@THvqkoWdb=^pir?3gMm#psgCeK5KZl@g_M0`kI zRlRq<*v3F*!Nv=6N zVWyglmifp4i$U~2NWa&RIY`4TWO7{g2HQC4iR5nYR+geik{S^kfleJRwwjV(WM_k+ z&qVC={C$}w_IY0Mw}wgP0v`xAU@YRLq+jTWjHC%c51Ct2Gr?skx*50gVWQ#b^&V?` zbBxUZzzbCtdWi`wGtt&1;7LJeva2U(v=1by8{JiKuXz~Y7m_!~e@SbM{7BIK+;Aug z+G5vW&DxNFz^&p^lY>h+;GCNNIR~8K;GbHB?hE?Buumh*N$eWy=Ch#wW6-DGtM1mT zvzBte_9^u?ulH0>=28y$KK-FpP%tpe`UG)}p$cL9v}9elm<6s=1mx`&jb2+(28qxC zYvQ@MCWCTN;!#qsA^%xxuK&b}bdi^@N4q}7&tvrs1;Ww7k&Anh+8a(<2wYGu;JR15 zkJgt}bA>DmTweoT_npJ+3z&I>n#D9O=iF=B(Qf#mf!QMYrl#h@L#Uy{S{6X__s7p` z7qcJ$MxYQ+jf|aCgSZgSpU@ruIsOIY>E&UQ7HP2wECN^ren$i#ISAGx%e`)dcL*5{ul~(Qh2Ha~ypA{JsB9rW6Je)7+rOM4JL<}H2JnCgWIM)OC zaIppEd4CJ*+C|=99p_GDW>DFEgzB!n>E(N_a=m`KiKboc+PX56#(2xAm%qqU2ohLW zJ^v6m0Q4-^lj^Nnol->cnV53NiBN-cEY$WUwz|W18&BV?EhtBSHq1L|gcWQQ49q905E2t;=*C6>9=YmGF%IW1TLg%UjtDjnR zeKNB(vuPqsPH0b{1Ds5+C)?IIphk1!&_m6ZnD8{$BY@xlx+6MuY2)KCxhaf0paHG>%;k!{LM3%DQ}KaX9lHQe?G+EzvHd zmiUb5{eK-ojjWR70uBX{9xgyohoHcp!goOWK+L{B<^DHaz|PA*jMncwFI6G=9&BKt zFCu?KzIu=Ivp;XXMsP%8;xVZ|aamXq@@y!na}_A+V6DP?EbtM4c6ast11*?E%Qw&g zGIeci?5v%1E$p@RO$_zT?Hw(TynRvka8STi^o|V9t|uRcscs+MP~adOuWXQao#kBd z7%eK7p^KruBii?tww*>imSt;mx?-VJNT&#hs9NyOY{ z%RRQ>G7I7tyYjNunD+W1Cd)K$ZGZIW2H!b+a%(XyeXher#B^(sF26_;i3Aqp3_rAs zF@>Hx*T@uAqX%Llx1ZKzr=BA^(=gelP;TG3HhuD{l&}o$<6rYVP;yWoH0xTusVp9M z-H-@>M+R*~O@&ZT@H4=@;*~Wo+H)ERz-}8+RDb;d0msaWeX3i5%Y^m2UMiF1*!k8B zGZ47Y-KwSycrr=WF&{1KuUD{`69izV&LZu6!Wi0fy4h{XJG&**Hzd9gc_9AEdR>Mo z$t}CDO^G5DH|p!Oo>`$i2-a5X4+HRve;PsF?cl^L7(Jc8r!N1G##;Nv5MUD4>Kj=x zzxg1tjH|U-UAUYtOm{QVqwj^|Y&-XP5;Bqf$5L}qUgb@%?rPPZ*{S;xC zdv>$TXKHtY$W27vRL!w_rN!LeVE;6UCC>Q7q&qtFN{^#cS?gof5nvptW?p-a5|V86Vf@96*7yApV+wl03o zHC*#hGKG*RWe6b>p=8QXQX-kjJd4ajl6j^iLzE z?_8z#>b-k@Kkh#F?qTh<_t|IfHT;*2j(4~WwW`*u8)9Mc{a==ty~jnR@oT}=fx!O# zR#FmOcQ3vo<+``u+&TEN+jeb^o#v_OJCENrzP}0sJ@WOnxGGH$h-b3OHx;ZZV3e@B zv2FAHvHh*3BSJ*z^v3LjM0A_-tnW4mJeX13Ml-UCM8&(XijiqlQFqf8w#s!kq!R8B zZ7D3Z8O9QdBO>1XDrYx+prTCgcy~-};T1vfr8-Yi;Q4E3W7|q@3J*I*9meUo_?klt z_yz=8N|rc39UAAE$_bEYlvcaB+V*X=;&F$qBdo7lGAmB`LMthH1>hYYv5(J#}vbh;;kiFk|hQN@| zO-WDgZt2e-&3VfH8LjGk^FN&UZlx(NwrO@i-r>UiL&P_oRIdr1+2O6tK&xS(*qv`+ zn~_c?K`oR_0HAUTjh9=A%XLxKjxblNJzg4ZVu1OeN6`jv#_BqV)mudtu@@-eEA&>79g%o221+`SDHr zlZ=(0`uU*U7|5&n_5!F0W~CLjxkqb#mWWqn@i(a!kK`xJl=RmWzf?v~4|=lB5={!Q z%u|oC|2IX)LiYbo6t~H&$g)UxlbWJK!F7a}QC(mo49qg0AD>uk4#4x~+<{9@n)zeZ zJmlJ!EH#UFIW(xQ3~t`%)~01!S+V-)Cji$dk1HC2u~8lY_*9D&+2w~%wSq_E%XK|# z1N|o-+Kt|)T)N?fZh zSqsrvm21Z}t5H4E@Y1+2+;D-kBYLuXZV!UT`JY=LK#+d#BNJn~De5pzyUUM!3ZqYj zE6Ekgbd!68KQB$aSFQTlN&*5n9_BzGT0rVD#b}ABuV{0LXqVM7{oZR;*9Ej`W^PIE zw@&t$od{6XxSjSupgrJX#!i6@mX+EQJHu=vU z2SH$^yk2s3w_k=BrKkQXnM|GPV`XD&xfA(c`jKT1*LW}ej9?H*wJ{{7S^JFs;YU&y z``kaXa9z-O5d_dXS^UjI(K=Oi5j$^NIi_jEU4LY#5PCy%qcmD=kA~D71+Z5|&b_z{ zes2C&Yu7biHunP1$+^y1ve)+Up2()X${txRfRn@tF|5iWdh)B7xpO_-;h@kZG>bV(IP)l8UL)_0_G$=Pj~s42~z(vqjaPc{0%C zxMpblvvK%OBsE~f>>P@>$BOLAvc_stcKMWFrBi=+tSY2Y%zOw;w@eXpE7sT|0mxE~JnPW1L{h-PLC+&fP=D6ZG~hwN^9{vH3mAj%?c9 zvzVRmTq4>A>l~5JEI6HxLExXa|AvHUz5$y1h|l!|-rKt&ydl&zDs;8-#3y|NEJWcE1VjGtV?aIKdNa8RM899T!D21Rj7q#=@Aga$!%wL{XT3fVD zYi+O9PCdq@JR-FwxBB-~*_rv6NQ;6@kT4uTywCo<(9T7EM4Bq`nU+dR$R?B(Ko`P~oe zT=R2+0O$_u|4))n-9Y6s(|MJ;E9&0nY8Ua&wICQZKjdm zV4tmeIDSWwUH^^^UOXo^U0#RDgctf%X*@rXm_2H*nYmZ#hIh zZF=h5ut@BVJ00}EP8%TO7aZB2MG}0H^-2gK&E7l2)32Ug>C&XzmY;lc!@7{Mv(xI& z2lF#oyK$N^boCbJ=z6v_xVa?K@~Bt^XS`m=(b%4lI!fNnziLl@4L+9Z&l*OY>lK=l z`5_sx;J6n{=)SC|N!PC+U$)v-Zg1r_M==8G#6-t*Q}25r8?V#;kdgtiu7JI<;wkc*sWtLE`dlHFfDP_loy6XtmBJU8;tAd zp;AzOcgUzEXIp?ET4j}oz=ri`%(IMC5j6Ma{gd!cZ;cR_bWw_%sTO1hD4e}R{o;A zeYf|g>0`EM(20`(mISG!&*WdekUu8#P@IqZj?~%{D=0@E1z*xMXfM7k1n%zxP)F3M zz$M{J1Z3}?9aVT7v@BRJsv_*7>5WDYL(U|3uPd|6Ck3uz6YKz@;F=6^xQ;?OFfSyG z?Yn`q5eCVNL(8GfbJ&QFC*ppOLE3HqOR;IhZP|o;%0Z7KncoQQz`%-`8`v0InXua# z9I`gCae}`1%L^kbvbL~ss_N_{yz%PjVC_54-TubY>L}~%PSz&NT&0aMBCosBn&woW z04!NnKI!pRBw#O5T#bR*+VXMHsn@HSUl>t3biGT_PCnU1#4P38ZdH7MXV(pw06$Nc2u29o|_ zj$*w2f4Y85H2I;$vDUPz5@9WoL+|UkoasslM}yC!UOj}z6+||ugtKBle=Hb8g z2HuY>y1ssX7rR*Y5qSOYziAS}r@}?E4*>A?A&WRazFXxC0)=eN8{IB<`VH?@*%om~ zYP2UXYKgyVg%7jT9_GUhbz-wr;`RTykJfh&##|FC=9@x6z5qcaS=7W-#AJD-og_gd z+X&+^;)w-a@H=DOpjQsFO2SGPR_?drW^rK_bpbU~lPAN>0EPN~x{*YH7|}z46l$0R zaLf&r!&)CZ&1oWd)utb#SIHMkt_ot|5Y%r}*9t$?%O6-I*&(S#l0IaFtcm!o&JKVR zO`dqNe5`h$Hmw-j8vT&7X+d)D%kqz9Ysjy)dY%Z$(&n#!%yxL5g82mAG#E$I;D5!@ z0G3K3sdnbP(>I~=T8^S?wnhngeJt=sh3GGEk9d6{%{9hoH1b*^J|Y0`b3~`d|9Z})_>3X z1Z!InW9DnjP`C7j}j5t-OGzSuc4>#PegJs^^5Maq>cA{ z+c***mFz-F?OWZsQrbLb?^wpNKBFtM@iJl0H@s&TG&kq31;BLRJ-hId-QQobJM4;G zydlrdXu?yH1-&Lh210AS^JLX;wjTCBaU{n+t?ydQRdqghfPDTT`I7`1YcY=a8bl9D zxmP}b#CFIzf!oWM$T?qUO-wEW5u=^`6A%CD8LDYtlAgY; zg4%kn+(%-BTbj!bqfHL5$Up0a>O{;rUuVDK>f5UyRNj8mOCTP(H_l+qULFZuqmQDZ z72D`?kBB|^qwk10;X2{>BU;y=VQcfY; z!|hK**865g>YtZ(e|bT2?~Uqx;r8{!pFB^CT)&*~BT~-A^K-)0@bIT759g;IZn#48 z+2<4g@+W7$pZ|=}-)!?Zg2X}~Y4C;d|7Rh}qt>MkrcgB(d6ZUFy@=K!7qcn+C%3se6Ltje zyh)R7R%*Mpz9PCEJjHmiJ^e)PN@l+8H30Fz#9q~+z;eTia}DDxvP0^PZ|>BHt0=(HL=5N z-4Km?KN~I(@IS0-c5&*YdIe}vA)#RLpY_r3re`6gEuD5bJQ%|D^j>uQ?k9SjD7CGo zophI3wQrwLnqwLWRB}~VoHDZ#WoI%kTbffQmQI&*wZn6Gx$HagGgP_Z(J*&$+e_gw z8ZrD@AV#C8PKL?N?gs6(mIF~)4E<$M7g{73;wI~PdiE7mRzBDX55YgPT>}A5ORH8{ zG~4bX!DmYy20IHQ(nN#q#PGG5H)O6L9tT!q>BDtD2U}|(p{U?^j`^8LcpyN^1^62> zU+RotxDmAseTB*E*dux??qhMHsthD76RrvH$UvP(3jP9T{l+H{lQ#tt;*5eQ^JQn-yi!;(3j#1U7kGl0~OC zP#mQSd%w%-)t?sq;|CBx1Omrq27%&t8z^m4Y3(e=BdLPf!yoh4T0crUY5!l%IN56_ za$+YFJCO8JijcpXq%rXH_E^oc3cQJe=Co;NSY0fg)>T;y^KwDi79{NXjFkUK682+B zY)!kcYT&wQ-5s`Gy27P?yWT!{;hC)}X&LmwIeUw13r@%%$T)^07vDTPTT8b6Of>I9 z_i%Id-V6G#<*8qlrQQ&}|2)&o65={OGST->J=l*W@xIR6!-{~y==Fz87Sr6dZxV>l zkr1i7XmQ1b?ADN&0Z@pMtFT0n&g1_h`n95vKt5)BMGA1vcMk<{0o=qwRK^se=x4;c z$a+ZPNd(c{#6HCHS@4NP<^b%M)PG&9ZVTTLp{|ThU+*5bcUBJulHAG}iE8-BnF5!p zE8z6D{z2M6U|ABi(Cgwy%1K*M9{1Pj+;)$ixOcdJRn^IDb;r7volJ)s0D(=4^v|X( zE!$455B%6qY|Wk4LDD24{WL$ur*1mzdc(>eqXK&2Ix{L1h`sM^-;i84DVhQdmh>w% zPB78;)DXo)*iT^K-xlS9_7q4#@xE+_cTPq;7_oj;DT#D^l+4g(Qik@7-Pvkaxikf+ zX%$p9d~gv~z-e_x#$_6-91E`UnNk~*q+rs&QgV1eWNR$x!|RzsjESpe@bmxbNEaT4uM$=VtIrugWa=dD`XA29AGIh1qcl}YzA8!VO>W?Ew2V2v5^e!JrhXS7NA zGKus=;*~uw$(Aa5`Q}(=>ad*n1C$Og|h!izw zk>)W*p=lmk%L{L&$2JU24cMo2>h&WRzv#&LqEWw*cY#s2)K!vsu6%Ig2PjL$^dU6w zzvE3i*Utr=g+SoI+N>!en@dalBFJtt&u}QNde6~qDo1TVUdOJ>xrxy85AOZbsU9p7 z*?}o1W%Jl-4?S|atq&rulU9YXue|e$6z~{O*UEF<7OjaBuYWEYKkhP!zhvqU!1e#F zB8G2wVnGsw&#S=Ep&LHcer$~q6R6lSzOy!z>_w}ICR?0So5w?qb0UqE%1(?olX2QN zA)jiI9F?FRfZ%B*UJ7gT`}&cl6ukY`W>2nimEIj-cx}|hRVuC^GH3EBPdb7uTk z((q`G`#f0xj9Q+uQ$vX=DKw?q%#eR@w~EmF>WdjK3@j>wq0tPe2(-Pv(VNo5<~ak~ z<>l!9bv$b4pZU~BHw1Ijgu15V601-rSx@>gXdJonB3+lef=8S6HreVsU9NIKgkuxi zlH8B@(fpx__s4k+1uOx|sLQBltGkYpAb&%)op3ivF#i1iLmb8zp5*|%r=vzYR|wS( zEICs$zFE7esszpR_2Z5$d zE!L+v;&mfW*uG9F_|zOJxK@pThT}B{`%+t=NWlXH5GnsFn*Zg$!I;%cBE((Pm($$1 znqQM5yyc#EV!fIRhvlIMV($%aKRrm^j?+6HSvInGkpvKdVxfVO z39sw$@TpNZ$7*ARY5=^eKnw}+6(wJO0RpHZRJ@6k*(-Ldy&EFxYF}|TT93P!ua8Kv zKd$)VH1WGHOZ@0Pq_{gt78Nn*&=;qwQc&Zz!+o-9l1a$t@!b*9UVXC{FAaaRg|Ewl zfXOzUpmWZCZlZ>FQo0D0nEbdB0^L~5Yw80pjFCpXkN>I+2q1z6e^us-FF+v5%IZ^{ z)#?&TQ-St=rLfR*xfxu}a>+uQ1e~xMBKR-Julr%lG7- z)81coWk>lM=Aa#o(mQTz;CKQ=RCjzq=6BsL@HoD)_DYekqR)#!sVCkSE=~>}5v)7t zS8%TP2CKoUVDHy6U(x`94k3YDe^~^3|N9xsezP0UX=*V6Fh6etN7D#+#iAuTx3pF= z585NGidq)us)&N^(0T)cNFu=Ba(otx zXm}q-wtCmLsoRE$DoLv)M$Ad-a&0%Ni-?4cwVEc?zb~XT#c3ahEUQ5`UbMGSVv*6m zmVB6g!zXfbb<;b+jtWW->ejy*m(IsT@Lfa;gn!*W4x{Bj0c5}~O~tMHJ9nvRZk?S6 zs9CW$bi+RT#r*$?xfItIpAeq`b%34x1N9B^XB6z@>XbATkyMG~t0_-Wl2G5Fg#7?| z$p7#J*iJs{3t&f%_Xe0}Fg}6j0G55o)8}81O9B0c$%HwoVE*U` z2?_gQ?J}ZR2{-L>%HW)K8P>D62VBh9DUe~@w$3<$xsK%M?fiZj+NNrtXa#kEb)hZ| zfC}Z>VtlG|V?g>X)}>Xh#y;<WS}F3b(gj|Ve~+h|5s zk*IhVRxvV-D(Y_9!dAKNhE&2GqAi7`Hp2jFv@uEPvbe;-^g_@w*fD&|`)hYc*!qSP zulBkxZ@xaBF*Jj}aRh)_S&WeCl0Bcs@2Q=+k6w zJ#D^HS$68J`l)_**&{Q4#P%mKlL+zpJ+z*`iDC7(x&5=Xo^VvRmXamTPlv{Nrg8!# z8l}~4uC{$!t$5sF>j>+smduJ%zEE%}r+R+dh<=7;cQZ=vpHMG!1b)p0}Ypm4E zit56>MLGeL^^maBdMVQrFhmP_oaNKWNZpaMzw@ny+q1jN92!rCK~#g_l^|~LBJ(Bt zM*N1u z)BZ%yJhu<9&*ek%M_)Ua4+(8}7PyTaup1+<5Gg@xP6%lqA5u@_ZST7{a`x(z<4?tX zHybdhYOk8BkG)L9ztX|Xdgws`Q+6TeDYoY4hl)tkk~Tfn>}uf(Q(iKc56Ogl)?Skx z=qDS~E}I_cV%?o~vJ^!?9w^KiUC&%p#Gv)eK>Dp|=jOTk)Vh3{5Bd!pa-<}W+f3or zQZ?f?bU-d$_X?YOFw3%%6Wnw8JfoxQr+A*ETw+PPUZbUK5G*I>*ieoU;Rl%*(Swyy>p0kd71$pWK@9veDN{{DfMez^&4q%)#8cvUBZ< zL!w*PGvsrP=sd3~97_*Lv^nej!w!h=e;?v~=kmFOJ6BS4S|7CUL{$hTbk|E~)KI-% zI(j9Ob*^0`5wFl(J|tHY(Ybs`4nP8P`H-vuq~`MZ`UYt3BRf|%VF=6VtcEco7R*PgkHD4LNa#FpH zQ6pREal6FF!@!dj+gJJ#J`h_zmrsf~j&~SLbb2SD;wEW$UVePj{v>1Nr+z+YHwN-* zzP+>Y^4Ni_t$VN9kc2x-$kd;y58UX&+d;77=0v9aL(5MFpCqMQgy-rzXm~|A>D0yz zH{)DSyf{3z+R3(Zr|k*7^E9GC73W$|XEw~`%TGNJY03XoG4(Kaxo2I@*s|U$OV$vZ z1>ZS%mLY;n^8m)I6SKbT$r2pj$gC`io}!GG%K50+=fTYsc82Xm+Nh)7lAFd!=(&6@ z7CN~bPn;AsspW26s;Dp~7(QNUxGPFQDapYuv*+weer!Gfn0?+cHz>##Ae7jNDu|?= zB#VNX{1#akaVW7ksm<&oGM69!f#(3819%R69|z#`TxfxmczM;TTDRNA0V+E>hS$2E zJCGdi6`#AjT>Z+X{#*d%)mX&Fv39WY^mMXpoc!wzqFH?pCKNxaPCsLR$5ETtK6{>< zKR5$va~j7ZE2gt`2vtl@m>gaeCr>YHwBsCwsaITSlEf=}mSI180QF2RBzkzeoHBEUJn`6({do#YdWOfh$oS?eklQX8Dm<13Krxv!vh8cqhSNl}~B!tsM%2aOb=IuUcu(}^_g(DQvq&N)XXyzApgy-!BW zp;ZxP-RhhBA}QUOVc@QgxqvByzjH60}$&9)DagffL$HfWHYg?>hS#*5^xFay9?w@^uz}O&L+$!VJohvF{ z(+RbSTb3M<wIPU)G| z%h)dZW?#tPGs1ncGF;TdoZ+;xH_qy7IR?v%Ff}R^ z%We0`y|?qG#aq)ad9v_(5ZJMOKk-JX;$mUd6VeCMp0MioKabE8w4OToHjl$4ZkhkW z>-=^J5V&ULB{;I0MdgIM6!nWl7B_#&*6FC*f)Y0;j_9I&JZA^=v%;`Uit|c;>Y%W2 z5^GLb(u~c(%Uk-1)@N@iONyZ6)SjO#xV=a+DZI`3XS`qhBy2f@Bx8a^mWb|qc)Q&n z8=qNu8m11+`TrmgOEpa(NZgJ(2#6D`rgSG~Cut_FBsQHDEBsJXFfxs|D`(UF`)SO} zvJP0)joWuKL@}0i`s(J?MV2&mv%B%(w4QuW(877(Z#Y9u^6_(_%hRD6bGvbXCA!yKgOwR|38iuq{uB|1>SW{x5H~jJ2(VV_BY8XH|cWU;EzsoIQ zS4VP@R6=Hg&`IorK#?v_|J9U?71bXSEfbF8^j`ftt@itpdf&2+?v>$lncjFv;L2|4 z{V4q~8Sl*(s=OW54m@zj%-Mmyyw?S{2#)S=E4X}LCbdE$iJL3H@^1I5fnAXIHv3>9*x3-`ub+WbEv;`t!m3OxA9kW(=aS zKN$a(u7Nvm4`_CQ_gHT~d;Zd~2S*x?$@bh{YZgMvc)SVXHr_t)XHGdGtPq7A@Rd_2 zTq^m7D>C&u#R}E@q@i{J>Cza3I|4fgbkDxyU5(>g8%f>uBVAcQ*tVa7iwy$N0#cVL zMoUC}MVm`RyR44s_g<^IE}%^_b4z-^b+X5-Gy*$ik<^#p=ELvj3$Q?_MkFovJX;pm zn*apjnXK|n1*-}eC9H02+kAg)e{1Q85D_}PF*_j<-KLz6Z5RX)?T|3}aP&ujv7dYf zcR4FBA!|3z$T_PkdqY9|EdjF)JB`q56{dS94=jZwNVvF3AF*Xn5q@Io+9w>VN4!U329_?O#5J`i&&rBiqdLIt9ov!C7jjR7Zt|^EwbwIeKsc-t zn@L1;`Z7~Uf4;7Qu_t&}p<|?!pP+ERSG1v$W}Vo+hz3j-|9cmVAL3nORl$A7ruJxn z=BnX%^+}$p;(V6&klo78`-jpbMaOU(EhO#ie~-VP_Za3i(jtk|JLmI?kKbm05R| zuCh^in0necUU+%X6S-%f6ziYwU8cKiPbkhPQL{0YN#L1A8-12F`S_949kf}r7dEW8 zd9#|?em~`ugxYCUMs397JJ{RKs|9{sJTo~Q+_LX@5e`v%1Yxw`edKEk#%EIRI&&a*_Z#%q?^J0w?tcjwW z@yTl)IQVw`wXOF?9dh>=EKlKjpiDNB0|uPt{1Ft_t&vWi2t zFT|USJ)&AKhA&kQ-6w7DuXZ2m76>9~#W0xwB&jzDAR0I!i^V}8&m@%d;Ho^-@isc{ z@LY*;+GgofYBU0@4u2EoRT0ih4A4_k^d&k46$zk4Zgp#5lJ)HdfN$;JS7 zxfqPk9HvvlMmwJjonu75R>~qaH>dt&EN3{z^lnnpLetn$7A&5`X8Uja`&lQLMSc(K z!}~ZSzX_b44ZQ#VFF(0w{Yx-pV$T19Xd$X*N*~HgnME1Pg zvO#C7C(`)`b4FS$s~;TZ!oMX8tcjMq(9zLK|J=aT ztjTy%tXjMLMLd_rR{N+BrCMeTez2t~&b^OvWU))sBk zTH9;2Q;%_|(+KNJ_7CDS{O-dETc{$!iQ;Ub3NT9YCbe*rz@GVt0Uwd07h}$Wb8i}w zd8A$>M1n<{e~QM1brnU^DA&+Hvo)1e1v+}iA74s2dWgAlvq|q|y-dtX0X`0lVbCWn zdKp|BrTl?D>}?DEbJmA@FXk3zMctoxbnYkOgZ%FEZ&pr?=>-a40ZbwPe=e#biaTT? zB&NiTWQ3${=y;+&qTjujzZ(7UE)p@~9yMs>+9$E8N%0-}CH+ZinImdsbt|?y9AmCA zQL;9@c2{mcWIc}Lx``B>9Q|FpNZ59Qe(xg_W4bBoFiyM6k9-QFPlYSV70PszdxSqP zO}$sG`e!>*%p2Z`CQm$BK2|$Wn^uf%jef}4v>>_nW%u{d-Jj4ETIcB#rFAu4>o!b&ivvPGG(V8+R52uOTMOt%t)utb#SIHMk zt_ot|5Y%r}*9t$?%O6-I*&(S#l0IYvQM8C_?!OL|^~Yz(&-G~i`+b(-Ju9NYKO*b! z=>B+gf5b`hjSLHV*MI=xcKnMbkS#~QC@^>X`&sn=SwH?^bbt5_iMK!d_5l6czJ+(6 zc+1_NKC;}fjlm*C&KR1vQ$xVi{PUB;Pw(?0Ht=gl&X~oa#B>*<|92tEqt>Mkrl`e_uJmZRu0m20;OU&dc8cvZ=>s;FVH)a6ksT6K+nxNiv(oLCXjz5m zv#xt}ZFc*F5y71$w)54LO?@Xcnof65c`!DZebqsAPi{Yg%Fq}iUpokB@sA4x{12;| zU7R|pUIAKENGMn=jp7i?Av8@Lnzm7OI+*09@<(1M1s#g|tF!kV%OIu5l_}HB+LN>; zjIwxtUEs-Eul$?Zw1^L2^86w$;UJP6^Rupj03x8sY;F*!~h;sB*)j;aWiu$=eZ=69f<=L%?^3Oe6Vw#fPcuT91)xlLm-Y$@$D?Dd;b&~9o@=k%Kzri0jrjZw_%g9bC`-~GRd*C?v^o%Zm63Lvy%vW zrX; z)xdSrx;t#WbcIX(cD;S@!ZTY{(lY3UbM_Y3mPOL@AO&xL*uIE*p8v`q&i51c0Yg+N zq7ELTQED%Am$w5dDEHOL2u1iRv%1$G)|yx=BN$cp0^=eCU8M6-ixBrrD6X3F!jOn{ zeDP|7aUDHW3d-*e8MWkW3lK!Btnv`pFb73L3e3h2!PJfd7=c7mVyf+AL!^nMF9?TG zG~}|RPUrx<_8-F@{PnkT07h;uRm!)&)8H(=H<$Qhbk~%4gPwt-$cVaZBKXbN!Pf0Kt+8+Ax7sR+ zk}f&(M&*^<$Hq%}R@%q+wef$tWRWjRk;jt;f{SPUijP$PT{G|ywiG*19z-^|sy5Af zM@QyN^`=sOBd4aL49XX>L{GD+>UK%G03hGfg80KHAE^pI^&Tk8tldH+r+GEbG{})* zD!zkiyC@GK(;g{7&4oLUHr56jOkBWeMqc6SW-+7|RkP?*U(!)PG;-thCgwdn57u0Ju`6wL zmNF}d$JG0S^Z!f=-sRQC>7_EA?|XTNt0gJ=+BE6lJuQVNOUSm!b_+8%j&jffht6iR5BXzCb zP4~#A`@=!|gLciNS*Ko}VgXz%aoCn*1(R|C{)Wt#IwKfvL@h&KVe&fmh~A3(Se&RT0}0E7YXS~PN}pQ=$HvXc9gzZh z!|&nG`TcJeD=x&u5d;XvKTdd&bfO$K9?-ZtncBCjcJKJ++JiIlOPcxwIwf-Fxb|;H zn*0@{hW{lfO5g$Zv1A7jJB>s1q)ezTI8F*?zW%DwhLA9~VWVhxrGi(ke@lO8iD>DU z0qfxIVfFuK2!*HuC~lCSA+#Yqg{C8xC7%BopTKhf&jCCK{vi&)_I^kKGvHNv(a40% zw-Av!Q5|Lk!%H49o?-W!ZkS0I-kN)-a>9afS~4}mQ>yIwokb(&@2Lp3&qIQ5g5vWS zWkfMb5C(=ZSC6MVg&e0e^g2b&?|L==&3e>8xDazPBnfHS)M9;#BVIT1gzf8;f=|tn zf@{?XXgFSTurIX*iWEFRfEo!A^zZ5RFWCxwx_nm@A~?!1nkc$UQ*KFw#0$f_s^rU> zyV(Yp3gp(;bfFeWRvct#akJSSih>2L^?i7W@s}hLwm;VY_+RgngZ0c3Mv~uxCkrYG@8iB3uBdI!Bsv} zYGaZVO!`+!4iAWIjU|0}JyVD=QP!94RU39Rkt_u;nGliZFnN%W7eGMnVZxTI?t-{C z_a#?cE8SPVI>Oy8+{a0We{KFSgTyg45Da{0AU?fveK$>DyidXueTEcgXx!;ljt}Fx zayCdclyYKO51|mO-8z}Gw#@BKL>~FZX2Dy6F0wCZOLIEzyLP>n+3lhF{Uh|dl{}6m zm>rS%03~A2tf3s3`H9JCWNu(#<7j7Mr0-zh+W+XyE)Pzr%Uj*uEYQm%MV((bR36zI zoyl{zX3%2Gc0-@wSo*4f6%5mQdGcFoz|Ea?&TdP-r9l?m)h9vRi! z^1S;J*erx1g&x__l2NlmMPI`B6aCl4>bCG55$ek5^!4s>duR1vAjz$qk*J1`oGEar zy27Hr+o8t4yg&faCl@o<5#e9Htam~8klBedqe6k$``-2q$#s*WDZpS!zf$7_6Mat& zQA~vW#2=@&1=ag?X2`NegVr!L=n=}K;OLs|pXj}e<5!XSS3X_E@K$UrA|of~%Q7F} zH&Q>@n0DFpKo{%ow3DSM0`fp%&ggpPq9O*ZX9m)5O*?T~!{+6kMr%+Xh_n=nQ|^6S zcD^IXkxxbX)2E1P5Oh};*nY!O-qJfmP-JkoOjGtV))4+<#S*LaKCU^qFRB8f4+Psb zKJ_jmJRh4D^l37;o;F{pEIaj9{ZzlZ?2#EiV*8VrOaMNx#g}6V-Sf?Vhq3>^M65{_ zPRU8pK;B6jOnQkJ@Bcr~o<&XI{r@rS`~{7{@c#dpH5c#yzo<|9!Yi9ymD!Pvr@t$r z_22LR|1}4`KW2H}+!+c9mLFY&2*^8JxPOTFrjzP5!81F&wHatN3>3Tb4Qw;g$t0+SlCe5uIQRd5;_4gl|E1u& zf5ftXws9N{_hGcI69Pn8(0T3V2AiURH z)O_+);Aw8li)yPC2)DLp_86feWeT}jm?Jj< z($n$=$D54zI#1{=9Uy%0VRX%fGd#f~sd44Z^r_2cf$0EJ2Uzlsn(nthwGgC zH=GM#X%5|F@@Q|rq3Fe3aj_$+&#Rj|wWV@MbnAMCe6A6l=Y?>=a6uErAyOhv4lwxR z|F`EL|NPgO^Zz4)b%ddmjOa%MWyJRh4->A#`~T1L2cN)m;P>DFY&(jmmPX!7l0ZF1 z18>fF+0x~cz&J3#g~DXwl}#G_C<onGnMg6C>_bu$vDO9eB4(SWU#f zaYN`J1+n^gp>O5&CF5;9x9XP$%ExgZp^n-|y-2d@zgJrdf=B>7zsK}kJSz2I^khW4 zS=`Z+LC-P5Hf|DStmPX!fGR@8n>d-hVz=76A)>DK6?dccxQqGvh!p$diZ4zRzXL!d zKMw*Vw-Se(kg05ikp}6bch0T7I^}lxHP^o5x99?nMDT2MJ9mwe6k-@gCuiVnlD=P~}lviUB8^_wg&ePM$vT^dSH;88SJ(y7Zs5-IurDXX!yNh4Nb28d;hc>zg{; z7(43QnK)S38l#8{DdfT!Czx_~_1TJZ_t5bKy**2<6-`8Je$TQan|3#v0V-&1VPj$K zY>nxaiJ1YmgAN8xCa9c3Kk9L;T=5Uo|wm?aIN5`7mF2S`hl6xMg}%UCRYDf$De$S z;pzXg0@K^M?%6sxVyncwN9VpW`82T<_U1N(VtuOumt>Zi|OrOEpS9o$1xMn z(aONld_Dnv0YE=s2!+BHXxYk<7(k#ZVXvFz>mr2_uqKl{(nR<88Cgp z`~|?+9YEKHj*T6RMI5?EZrAPEgR&s#xFEu>M;FEVUgPmVcm__p7i8-oB#0mkS)gyg zGLdy68BSqWJ{)Ry4Y1-YCw!~1tA0S>fK;#iL5^rF?MQK1K;XuyT zh!wvovZ|Ta z*ElY2zCSFAj$C&(fIa8>$CP6d9LAc0M8h1HGOOR?;Ey~jtD5t3hU=Xy(9+3I;_pnJ zkU2qkbYMiNvt^3fN&JBEp&OeW4?Yb8mLYN;Q?JaZlUFLV>=3~k8bkw4YA|bL%Yc*y z7yTyZv%MNBOdIO?v2c#h6>~a49&m#j$Bgx+{+kO282SN6`nOd1d4Dr2HttIC(x)-7D2{ zdMcq9SdGyx;D4!x?Q-zDy!Lk$7d#OR}Yk?^CISXHrJC7Y_b067aiXT36c9`{4cqH!I4d z#q%0f32gGXB#TaOpg2ku_I{VuD}XqScx*jo&zl){r!0B`dy0ZcQG*s~9%B@m=ApH` z@Me1K&Kh!ohqahfuOGSiMMuULjrxte3yiv@u9D1i<%1hPKv^oL4@Eeed$s&Q%FjzJ zNG%Ei2i9gy5!qZ?;uk@7n|X#qan*Z{Zc{mG1M)g{UCvE}o`UXnTF5H>HWqa|X7{ z%hCPoc+}26^Qn(+2)v9J%r$U6;FpN1OFF+3Gu8u5v(xV-wqw z+>iLt{MI1h{N3z636mY8|KEfXqKYTuB)Nm$M7#-wLdO$^6MZ1=!3fas7oG!OZ~(>{ zU(bA#Pf01v?Yx<+;*jkN@g`%BsMd?&OVva7N!$CY-Jg5U=6OH?KtRZqem5U*Q=cSD{%h9peYr3f!!FvtH$B-R|?e8awZOXjT~tdn*GP zy*$!!`M{Ig^6owG@Zqt>F)#O@f;C<;8gs~2ka_6k&EWV=gh^F>^bI>2M%hpBuDoUM zb?E{pJOohth!&cdydY38==l7k;W|6BF)?a_`}BpqPlY?5zIyX$CYtazm#W7$Y>glg z&t#QvDp*y(C}DMD+vfXY`&&y#gox1TjoAr_=r-m2xoq5?YwbD|kM5&0F;1@zeaU*@ z)cG#n{sjJJ7R`$FBD}NPR2)r~ffQ_|?0_Kg$`ieM6#EDab>0wsl(0F>%V{ApKKf?q z$GrG$=5d#RzbA?*8W6u$dc#~B08<8MPwRhnz?hl4F!b}ly;z@R#?q6&+@tv|BQ;^c z(~|yY^`s6gV<6e^w2>!q_+zLSf5(X)4FvmNORu3=STBNLywr7rUOCJv2`gP#x!;PL z#f4ea1=LJUo(wYs6zcowMiK#BBT6{thGHX17{gjAq;2(|F&jHc;D0~Y-vVw0dTbUD zS@w60_BY?#uM@bgZeN$ge6zS~)8}pAy0SQ7RRHV&k|P@Oocpa}kSuK0A{sr-avl#B z_}L7^^}$$Rh!I5GJeY*VeY9|X?LU}N+(t99ibTb`u!@msR8e=+7PiWDH>48o5N#*j0EIlj~`ws)s^k=)k%I}aw%x4kYNi|)`{r1O8+E73Kc zg>OPaPZ-|{XK$t3eJAac@xe=5yqE8mX0WvAgh&U$h8)o?{S4*P9g`%4%Mx>FFPB7m z_v&phkeWVq@)_fpFeVdTnu8tXFaFI10)W0n0jq&RR33FYVLQPt>L5xQ0ujnH0(yE3Q3$`{{6t?EQ^ZCPO_G_nUF> zmc-bokvu(^eyxs|8YBU-u-#%ugen3sc9Va7xj{c;R9_fVI|DmA2V2O#%~9XT+{DPz z(b<}|*?NTwYsc{xl@yzu1aBX$FbE7I+pUn6UEX=LJbHOAGgQjN&BVxg)+E-%(b)>J zY{j+o9JCIstwaQQp#}%H9T`{MwugJ=@pDACo|iUqN~>trRdg_aU-Vo`N4y2 zC%P+7_g-EBj1F>lYcrCFCefNk0K|w;Euq~0o}9jcvnLM zjmK1jnDCv~5at;J*a1c3!n%s0X_RYdpxK&AssbIo09ll4^))t9q*2bExaNKzEtN)3Os-9Y;0S}P2pk3 zsKYot7k_Z(J5nIfQnJMP>CiaOR8D|IqqN%1)wXY|6^}b?9btXdl38)e7aGn!Ss5SKiH!iE%`TWip{CM3f$Ib+d$dd z=I6y!@>FV?pEcn-i2JMv2<+ItpLnBGaj~%K3F(7rPgwQ)pGW8kT2GyPo5$f2x6J>u z%s-R_fooP?f+MS0R8F`{QNKuJar38aosPOKC~X{i83bJ}U z=dT0-XT5B0MjK@Bcb*|I~>b76h0#4kDTf zq7xHl9fK|cct^6Oh=tZ6!T-}EL$Rm9K6F>FlwDzw`s7ru5(LvQ_9x=vBRwiCmkWGoHL{c+?%Wl;vrM@$r7`FR0r~(Y6aGvM zap9{AKAj+j{UokZFmtu1*k}K$OsqoD@T#9d=|}tazSGX9Wh#koa0C%xJ!l~N!`tCX z*yzsldtu@@-eEA&>79g%o221+`SDHrlZ=(0`uU*U7|5&n_5vuxxY1auNdcjG>QMkU z&_Hm8nwM~hDxRv7d^7oO^hwe@^dxZuqCxh2CHMrM1HTyu;1hoabNm%IkD5CV2(PRN z_UaWW${2hp9P3{=a4F=5+O87~KYZfnk9?MN3AkBsPwJ!DjJJI$ku=+~Q19h?EqA{e zrl-sf`|A^bt&?E-QM)CyMpQZ(lkGdB`tGTI(5c{J={0`j&iBA=_1uXcUV9L20T5k) zA+ZQFlF5Y5vxd))&d!~)5LMiWUbg^n7^(}is9>vvJ~7UMh{n)g1!O^zM|@SFCMK7G zh|$jeiHCpn4ArzRNl)KaL2W%(?jte6EzM^t%^RJq~N0FKX!#fLd30RZ+sD1R60 zkY26Lb$rXCcJsUYqLnIlC2L*W%tUhcVj6d6UJnlPNz6kCk0J@|WrL4hIzRaT=EN~q z?D2pd*?kt{9{raW8Z(cO*H|==8Swb281X`KA}zycVAu7h?;oqqK597Mv6Ir1^DY%ZAOpwaa=)*lE3#=?NI31wGF4>13qt$l2fd*23-C-DM7qC&O@( z8~moyQ9N4el`X|FPcNr1tW+*J|EPc<=|s(6Ef$L_{QSHn98C;5;5YN4NB?Guz>>p` zgZvFiA%>_i!?c8S49EJ%A+`iLFbmAlUHonXrEMy$oyB-0RWN(_WByv}N2!eGGZ~zS zTt4PHp9YZQ!D6sk=61Ry0LC<8_Cf(HfD$1MRT;SwX*+s=NQv+X*+J4J82u>z`c4kO z@qQ4&I1uXxk&&qWr~^a}cK%1^?@ob0@kz3%h(U+GI8~K`8n+$plU0*ULOzf0j*#~1 zo4t5xh)^{MAbJlY)bRDsP`I{jIzi{0{oF(i@1%4QDlz$SB?P*$nAg+?UKk^dcpv|H z84$30M^n79;Q5rRSb>z-fbLin?+rSFy2PSO_4~Ydb{iyPa)W^KNsqT80egw!Y7ETQ zmXC{0ys^X=@;T46&+Gg$4FZUMw%9hJtgJrOS*B<4SjxV7$ ztskmZ>gZIDAZvQFQnbJ*uCdoLafdHXmJeEofE+d@Y0X@#)aXA3#1*258;e!9e4x4>1g2H{(@gGYnrb^dWw;G?f$ht8& zo>e3a|JDV2y(21(eh;?%$_m;M*iXXRMZorq=#*tx`FgX$7xeCw z4^!zij~$2!vRzFRyjdh0^Bf{xkfDVj={Wjfq{5MqBg(mh;|D!2EaeInA|7jfzWTz2 z^@*K`Rs2N;5Fq6O{0*5ebw)7Uh+2le!sK=A5xo`nu{cpx1`?JD*Lg)cv|&dwyZWnk zz@9O+HxirAT;6?K`&L#SQQheh+x(S;i91xux|4NHm&N^Q;lI@_9@@zuZL7aUC{DoR za6u3dcPPIBPSvQzrQB({wbnW=TCwEAE%M{;tPLN%^A3H|hI2xnAb{w{gUyZKUw+24 zuafxvI{yn zSVH$@MNPVX1^KeowsLzbw>gRtP$wojrki@-3)y&`7S@-1HFOL5;-4`@!NnGCLEZRP z=AS2unQKjCzEu>!1&9zkQHc--k+hR!Q81I=BI_a!B^Jj^0OomwPvAL#=K!7q|7{My z+HDSP7NUx0@r-I`w})xTv8Hxynkinn@+#eUaF1r>y*0wPV)hNbFLph%hl2!``XhoA zAVSj5+OEL63!(^EkIEHlo{;`{?1cbrgQVI$+Q(M1CAEVPZu-EcOSQvNM_jN7+J8B? zIO|bwFJF?Uy~ui0DvP&I95okJkDAjtI%uQ_)w!5@RJiXrYh34NRO)>)Vh*i}FzZ&| z+!smd&K#HJJ6Q`zgC{#KhDsjp#1XAg5STMsf@c(`rB$menr(NH;IpL;gPnyDX`(@Q zV))w38!}fAk3)`+IL;`nG8LL1MDy6QzjrLnzntp-_ag+u$JQ&*Uvpt|e#u$SG*pTE zNFC4XJ392Dk4FA2*I@g2&MWt=1g?ktf8h9h zD;<>N2)UBfx~*H}j1WrhGo|E6ky7hSq?UvzH=8K;Ex9VWiE`h`x{sW@A=m#q-@enf z?C$sLzu5iR*E}A~e80`icjh_so$q`;^ZC5rbu?PqUthB2pj!3Tsv`&zbMC{F275E6P=qIGY%}ZSysW&diJI}XCJOV;bz$V z`HY~MqxCv=D4^!uNN%%GDKq9a^E;(o4C~nse&|s6+LnCJlWsLO>0x=Y(ypJoM12=y zSLSQ{qI_`d{`Rx{57wv7%TIe$uUO;XdlIi)g?bgC!BT{Cm5m%^suFl9?i!P%n$_)f zj>T3ylKMo6qzcW`8^2(vZMMKa4ae5Ya?` z{N)4h{h#!2xvbB&vJN4oi|rI61AN9_=RAA;=!}<+_mkxM+bc#m9PKvd zT7#68|8x+za^Gow*t+uU)Wd^VX(jdaZa(&O_LHcPtqQ5h6n-FqLY z->0*l;wQvZ*fzyNw#j7kh%+^;uDl7o@Zw1)?e>L!+%P(S^4#}JmOctq>(1eSBci@+ zR+$^~54gOack%t4_wQr7p8X~3O4v%zg4erOsB5(7pl5?WJ)mE;>A)Xv#g}Xupz`ce zY?9PRU1j)Kjk_%@3Ps(_9B4Iv$o}U6&z_}TSu&(j!&w_nRSqAOvCpNK*cw4Tvvh{ z*X;J7wmxnPZH?0_hg_N_Xe_DQ$kOZ9>r#8J_4u`gNx^ej`+}_tJ=z!9Lo2g#c-kg; z3t2_iqCOUiDDnB6rBcqurTu3Pt$x1k(yA%};oDn28>V}F_a z_^GanQPmDXaT>(5>bc!hH_SV`S?bEvpoXI=U(@WkFMGMo>7lQkUpH!cS)7Sb1%G&Y z*UNoZLogM}YR4E}Ft}TwR)MnGW_m+(r;2?A7!W`J0R;YY0#f^NRn2(SDtCN}p1f|b zMs}(7)1D>R&ajd>E`i=3nD%8rHoqLa@ECN&tyWmu8WJO#)M{6OAKmrKQS_F z>^Y-%0na+gEXvfH-9Mmk_NRQ_KR@rF?TT$M6;vftRwPsZ>+Stf{%5haUi|e|=FcwQ z)%)tWX9KEP&Ny+`IqKk~&2yr(uf}Rsvzu~urpf(U9mQ~_@?7$+9&V<}E-b404~g+0 zV3Lbc{fO#7Zv*@u zS~b4d_NmSd<0Xd8g6;DkFW1U@n#obOcEbklI?}SAOw|nXEfazr3d#zH6|T|Z$f{p# z&rT^8(s$P0>6->7of_j}Hh=W^ZHoF4RV$L0Wdtdk-2tJPm`9=x%I{(8$O)!za7@QDXT_3-b0Fc7BL0aM|QMdj`5%lfWLoiS|dlTGcq zIUc$dxb(uY1vYC3ZfWN7V2nIph^p#0c}!hZO=MD1bfYWn9NhQWtA@`bXDt||XE^at zMunz>X7|jBF7|N1wvS`#{^pxE&52YSeY?7?<8>oT{LgpRJhg zA91DjV2Ax(R_}P3Z)?dLrQWQWID29C>RKL~te-4fP{TnyDJZ6YbSUL-NVPS$tj!J{ z)^nU^^0K<-^SD~uwAO^RJ@51IXmFD5g6f%?rvz>Dqls}-XH=~gY5zFl+TDV-c29p8 zaCvp>$u)J_R~%EL?)M)=6x6Ms`*9jVWudB8t%6Imr)XsqD4hSCR#mNDdiCl&|{A9XiS9@R|M!%5{; zXzjhJofL6RrM4EQu05*t^2+JU@yplLzB26TjT-&ae~g>e+IN1VQA|npP7ri zZY|#O`qIprRqQnjmA_Tqw1}4wZ#zaPCbj7O(}Sun|BE=2;*;g$qFP#7nNka(tFjS$ z_YSPn=~dnG!$X??P~hFbpVmC9GpFO>YsNxnPa#^(7D7_Kx_pvw`s7W~ooht9Uajgn z!{oT`j|VqqHZ!_-NQf_IIC)rHpF)C0sCjeOpE4RwUbeqXkpq6N0lykrTx@;6Tg>~7 z&nJ6?FP2A(5NqY7CR_Og>8@2}E8ee@MX0cFsGy-Lt3}Q_^~y`-#H$U@ zPe{;OHz#V!>o;ptjW)JRh+Vor^=;5RYWE<{M9D#}@X;Ve z2sj%wySS5H{Ed*5S#~dtr$4xCe7(%+o}1Rj#mqX9eP#_UM?|@=+H&My7<59drX|w# zK~>JZf5GqEN8g4&d_yKooH*IBxAvO5LC1R54ND94n-%(<-Pv1x9w(U%3ib^Q&B==V zhs&z*=D+(_>bGW88iKKqR4}RFPlg%thXJMYkIG+Fx2o1D=@wFrJ!KXI5csDBq%xDr z08;AK73P-G&nvD4rh=-lZptgaOjxBoe4=BErB7@9vaIMayT@}vOe+p?7-tcvnH)NM zz{KI{d8uT=q_G{`H@#iruva#Cua)iKBD2cZU#&H8>63S!9ejq&_ivwfZuw7j`^-bJ zy{AR%>91{;&?lzH{N~dQcJwS&F&N!6Lx*i(qo4@Zupk^|+&z;vNp(Wt$bVf= z-hP*wRd2!R!PBlp9oSjBYn>l{O*Qu|x<~7mkaZe2PfKf+sZOXQ&8qzL3O0Qg9J&)2 zUEg4wQ^jV>dmWmVkX6NEUiBM+`D33Qolqpt>|Z2boJ{yHC4SD09=ZBhM09nF>Q{#O zRT}SkY_iURwZ1VecJ;2jDAs7Z>CD6$YLyICM~{qrP&CbAeGi%YhRW0KWtwg*l65jT zZCZ=F2dlVG)hn^^qWtKp`3#Zhk$Jue>Q73)Bq}fC8rpICEvJ{rF1fbLL$3xFcZ*t1 zd9cc`Yj{5GLC1z(i#6(|Y$L4v=zM}y-@97>!L<0{2{BnIi@O9QZZGO^dfI`hJD#NO z-*<22@g?&&DeHSx1z8ZJGD*y_`(;xSEixw6G!ODl>%1-bQO$`$f&GnE73(DC)pS^?OyL{*DCisI`=K{Bx8UgQ;JMgiCa8< zsk8W_vr(#>RhcYR!@2&ev27!S!(;6yww`_NPQA7Pr_9Q{Xt29L&tiAC^lMi(XYG1| z)Fxe3PCLr1e{?)Pc<+-sx8z+`Ca6`xAV{qa|EgJk&J!42p^)MObmalOF1as*Htv-6DOr$d@U943qo_{xRG)!J78Vi=6AYCrVuSz!2q1s}0tg_000IagfWS8-;IG&hT}8?y zaZ>*1TQO8sj&tgw1Z9n-AdiusGgMHHrFwx<%GXjoz=!32sSUx0<$bA~KS;5>t|H5_ zI4R9%s(4=zKCE}hdAp{3q#>_6=;GM6l~d;~_Ksr58pV53J&3e@IqwU?TjhjQ{+H@K z#PyUFo@k2s|DI?{832b!)dteyj1l-B5|CmZqN(Jl+ndn?^ z-*)|@#WEdd`{@Q>sTW{w*7n>6)zSsvgQF!q)A2asb|)~sb<0Gnv|16(43?jB9awr; zd5fy!Uq?=@eX7b0bt7Vn&pN~*ANl{t|4076oG-(_=6~r)3i&vSF(z!~`un!GRH zw&Jhl?+k2d=v?5qX&~kv$|6D|@S`@4I3d{dF4~9DmF3W$Heeb_e|tVx%s;d@bww&BZb8Q;PZj@ISF}RZV?m&Mhgs%dew%wCc2FeRZ8-3j-!R zYHISf-RR3ULh%cym!*zt5!-CJyhEzW1RG>``9p8D-;{d&XFV<(QPFiz=W=^*6|Hr( z!O)%k8&%J^R@t#ntw}avMWc7hYpGO)LX)?hRh1&{QFhM`@j;0$d)K7BDBa?T!>Rj+ zkJefL(}K4~T6XD6Ja?-bt4iu;l9iklavAxUseB|0vi4NNuXReh_ zN11F~czEht%X5o2myX(fw5y$4$aBwL6T+GqcIbLDss8lHOmR;7ndB(+>*rx@b^FMs zn|kKQ`VI08+3Me>d-=|TmM-hO@QCY)ic$m5oSB2my^HOo)!uE9+t`W`HpVRmbX=o9 zaD2p``BpbalztXCLYzx6eBK*InGjjhW8=V@2ey>3@T%BQpJRP5Y& zI#=yBDnE2jyNx3br};MMxmNpC`0ck1()v1&tn=KV#mrJxb^6)dxuTrST05fhGub?! zr&jI#8@a5rdC|Dvd{?{kfdM-H#S0v&)-z`xq-+Pj>d)<3HnYtF1N#Bl4l|@49@7WwVeg03XxN}w_{r_b?R&tuUDr!p0 zUlTS?i|zCo${_OYZp+?p+`3_7R-R8sOMW-; zKVC?Y3F@6OIf>*=h3A{@Us#orzs<_<%BM$%4RW}*!ReNb;mw4lNwrU0U+zBiUvS2J zLbt(BJEi`iZiBx&-4rj>Ip@AyufV+9^F=!S{q_O>(6v@&FeGsc)&1s3{U5}liit2! zH^$IIKOz4U-FZ@niQjgYDFhHe009IL_zD72%jn<>4QY|^!&RZj)dNlLZmpTBBWSCN zkgn)bsq^^P9eyi&l-HGdiVxiQyub0Bz?p+PcIm&Pc z>euZy!J(q=>m#==`Il{8sEfVrA|JC|rP{|<2$(d-+F(uBy=s;F{=XZcTuvsatYsB% z%T=Zg_*l# zLxmZ=#?H!q(RRD#3xjbncPCvl-~xyMGPKl*BwU=S)habhp;MrmXP|^snb~ zc=2tmb|nMN=LY$g9&|J{XhW9Xj9()KnW~&3ny=C`R#vUvxN>knrMZSpv~IrfeKlgz z#$;1jIsee9R@e;;Oupehx`p&KWis*g0B<+>cK^X9vn|I!6aT~?W_VF?>G?kCV%^&II z>pj3tU!3%D_I7vkQA{d*#hC}Y8st;#lK4XWH1S_<{C`zRkpyj3X>XK~x3BM-(@qh2 z+ge<`V$cvQglt)iVYaM`VPoB&b*30xC=jl5SSx>i#hD?^#`qf%uoKTP!Bm(yxmca5 zb)tQyZ+rPey5D3EvqY^2ow}qxs?lFw%RZ!hxZ(s%d_F%PY!opt56|RxBOHDb~^1|!3 zsn>fJ>r<>psI;`a+N-Vq#ptBjDc6S#xw`01@``&~%L%_cwz$*1gl@h0Mgw!AGE1|6 zFqX-T!dI19urPRON23^{4Ry=y+jq!hL_dSKR`x48lyI@ zQ6~;sYj*jy&(0qoSq;=acK7Aeu}&d9waR1)<$v3-JX4<(xJ4#(JKJr6&GJWOL!3Mg z2ltN*j~kjCvElr%f_)dIMYp@LK^|g0caRBHACF&c(td7*QDrP1G;oiYKKS_Lqi^3V zneHywre+p>J}+lhsrP}(CyE;5PsNulU*&PeigycIMA+P_TQBC}$?UeTI=s7J`m}t? zdU@zT@lu-`IA4fTp3N#BlxbXOIW_!Qo0fxg?-d+->8kE>o6_&b&YRt2&brhICGKx_ zA0odp%h#m{b*S7w@%KzNv7ukVmE(##b&KvYa9{iJDaWQyowfB*{bhZ#413QwvSp4G z(O$WXAe$O;@9z`}iVT?h);QYQEHq+8#gV5Y%~#j8 zPTOOqjE$DLyZ-U!)q-o?j?S1k;jaHvkCTt)`zMyjceGl8l0iirY;*4&REF2oD6)P) z?873dZzk-Yzr26ol>AFy^d0u@@yr`ho16nvUL?r5#qXT-dyp?T>RRCJ7^}W-2Cf>@ zc83tN+;+~>tx5Kl##I_8dtH57KKuR=wdyBSr-D{1QoO3{3yK|3r+z~DUHR{fq?(an zuj1`%pBw0V=b(OtY`;+t-=w5IzPYnZ(<(-?@{wcaMN7y3E69A|hw6W_2kJb({F6o1 zfBW-3JN)=_rJ9k+dxN}oH=JPL=6St%*{!8?GOJqJue%VC8lDznQ7F`J+1EIlGT~v1 zX6A;^%)*RXB&~AlIx@tjPRUcYJ6g60SbMDVzQGR$%C}!8#D~8ep>t|z^}3DctUk0| zIOtTiuED*lf#XZ=Ja7H3dYQQ&@8qYvqdL#sr>yO-=Kb`(L%CPd$pb1E9QS6S|BCJR z<~hGeUNwD1U;S6{{YxDDxn`Jg?EJa%E2+u@sz$Bk^qci6jf=ZKHFS~d{xH)kMY|o2 zdhI%7bMpzMx-~66bZ~}z?_{zS?JDaJ8+5{H=0N?*nZw-AZ8zU@~RnnH6wd>{nhO10eplb=|C;19#2wL%96$T z+s0LQ^;)pXz2Vc{L;WU?w4Art?52N{&UzjuYTeU)rWntQF28*r)#mQ?d(B&2Dr8f4 zL~Mt#ksafQXLhVR-f~IftySf2^7e7=qs>n@A76LFq|?_td;e^+rbrPFmrH$TX5Y8- z4J~{?QSwyr%&F3o7oTznzKMCIgsodLdF}tU%k*vHSbg`Et%jBJYs*YAQ$LXuf24xuZiJLmsz;ckc zhh}N--SK@pC*O4R_Pn)1tsn9)+fw?AwR>%tvMs)L+kl^gtj|r@u<6a?;F`yW1a4V# zFR6@k#gGZN-hT+EUN40Yml99Gjptkz?nrrNIc(+JNt5qqkg+r@yv(w`$9*HxbdA;ap>n6g`a|fCn z>9wKdl*C=t(|4uIZL!x(Ufi$ty0J_) zYH+W$4Hqq`?UrJ3rAS-bteRmjl3lvYvTZltd-(FZtE5n0c@M#VXgW?&6>cin-wwDm zB=FG>BMk~fG|?Y_`M`VsC;eM4>$9z_LrCdjI~OUhG4#e#o*SQ7c}4Smg_}$usgJ78 zR-dy2K-1hSnTAkQcqz~R_p+dv|1V81Enk}EsQen5Q^ZPt1_ZuQff&VpnF`S+k=y*o z96fweYg@UH@%QS)M?bz%d(Nb|N1LZ6*>_o#_P2ZdSD$%d{$^p{mSfhh+}}?au_s^C zvc*Ctlusy2pQtPt9dN4lisppkS})H|d>cJv`RoWv{*FrSG|(8C~t9 z|98qB|5ABA%Dm4nCoOwSvzs{Z;G$bcI{NQzQT*VN8jpYSdJ%Z3P!S!q>TmiVznD3) zw%^wIV_(~xxgsP)+w@!a z3bs<;`NdxE^iGy7{j88-a-EJ=Z`=4x>K?OfMB%O;Q=Q6Gy}2vr23jnzs7&saXWPlq zn1dk`R0b54W=uBUJNPGek9wvh2PG6)olt*S(be0pjeAzA#nS>S3$6_dmvq%DmC|=! z^pXjm)OWsKuV}Y@chb678Tx2eO!3TwSsN}}6_0q6YI(Q!mgMAYA%&jnFM?vxA5A9Os$5tgiVyuGTiKHDPVf`#d}voTR&;dZy;7 zT)Y3rgiO#g7bdxwY|fZbJZV-jooyw@Pdj7mVKFVDUWekkrL+BVpZNaWR(-R|+?apB z<^8;i@8`UKAKUfpFIiW@R(ck^-n~LyqeTZj8~o|Uf8|zvJD%1x@LFm#eDI5z1w%~3 z&Q?zuc4ldbb<2zUgEQ|+)P{OB|%IE6sCW=H|z% zuY6rkIj*f7*HDftE5|waV}iUbea=u(Iaa2kl=rdE89vOHl9#Z@poH!IJ`J1!_r9xqoK@ z0R#|0009IL_^%2`HLR-I$NmdS*yNbSr#Y&j+LuW|Ui4kx@^rYy-PKJ(w8( zCgRzR5xdi$%sU-@wO4YxABPT?moUFt{EL;(M=XknoH=Ii&{8#CUKudd=2=FUkj@Eq zrxu@@lBB8AUvZTGwI$5F=hWApmaauS?fKgB|IFt}PkWYnx@Z@wa>4lYE4|Vq)^2OQ zWA^#2i;GvV+qv${7R~vKjtUwFY~B4v==UCRGok+8wl3QPo%g&Nk^XRn?}}#0`I5>{ zQ5OI81gV}wEdN^wg*4Y2F4Ica>|!{@ppa%AgOvq{|NqY%X+HIz5s-fEzy0|ClakL@ zEdHQ=){N-YtL5?EAGZ#E(V=w@iwX9v_kFD<3`&I7xo(TKHjX>jEQz-MyHPLnq8$?%~{8PJ|VwXln=U*(mhbCZcqEf z+S!NL&)%G|FEF4}UDtvSf9agRQuie7n6wfbN`IzW_K7WRG*|IbDDmVOH z|Gf!)7v7mYCvIC!*$oHF`9bS;t(tRn=f2v{M$Q_m*7qX!4EX;{$b`!~3;Ujb^1|}% z>d988n%eLC&g;RUwhO&mJuh?f^76euO_X1m=_3Aa*1?)q`?7=k=md)%@G_ZFywRmp_`hTlPuM@;_VJwcL;Y+Y@TG z&{!SkTFRqA z%i9Ji&rJ^5pDhyZ=Av=!(Aoqetx^r|+Yj;+tFPpPru1EH9$(tXd+wyyXI$!^4+$u` zeOtn|($=qi65EFgp(D2$xqYTu8C8ijDauvTR~*(ypA>+8Ex(N-sC?AALl-L%5tn|i{SM-E-r`?U^t6lWryTJFh`XF`hh zP;W<5zXqu_O)ec-R5&B#-D!^#yZf3icGF+u&@^rKg&OYiXb*zjk}(hJOzxR4;;6HG zmBbhI-roAo!%wQvnp9?RpN`&9jisSYQ>n`8uQ$Eo)Gt0)k+sd|h5SE~gxqR*p+5$0d~G;>xj@*H9da z=8P1{EXBzj4LnN(jZJx2WF3t!61Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** T5I_I{1Q0*~0R#~EpAq;!&pZCT literal 0 HcmV?d00001 diff --git a/node/src/reactor/main_reactor/tests/resources/legacy_storage_no_index/lmdb/casper-example/storage.lmdb b/node/src/reactor/main_reactor/tests/resources/legacy_storage_no_index/lmdb/casper-example/storage.lmdb new file mode 100644 index 0000000000000000000000000000000000000000..6ce199db52f1148a241bdc3039c2f5631e218a54 GIT binary patch literal 3145728 zcmeF)3A|0!;yCbgOxJwPk&2{AlPM{b2$eFVlpWAhWdY3jkOZqYe(x+*YL!l16OPiBFJzU|Vrj^q_n-CyC zfB*pk1PBlyK!5-N0)L*sp>p}3Q~pnjB}m_)*blu+n-`S-3#A7Q?gLyLENV)C009C7 z2oNAZfB*pk1PJ_T0+}*m$`mOQSs8a_oXDr|i0RjXF5Fqd$Ch%D55BsT+=!iV)4s05~drP(vc|X0TRF6BKowEGN z+e?=3)qBk=EsOt<_+<3Z6+ko+yGvgxwk6SRSDm%stfli8e77ZY*Qv3kT5r0c&TSiC?A+kP#=X{7UQ~PA z2?K6z)33(1P1CR6{N3zk_eNWG-+D>aFRt1Zx1q>0@zo!$pL21ET3rU5_+jGJ?Qehk zgPOO#U#E5V&be>t{LSo(=3Uid+WiT$e(C(x))%|JSbcuhxu-sqIqu;Gd7}j$D{w)L z4<`InXT?K>`!yRp@6p$9pZjUA7Nwr*dC$}=StGBUwl7=i)yCB4cb7cqLHjA;ZtMEe*8;CvW3MTjpRs)H>GxE!DCl%d@D!xY+>7!Ep3~Q zKMgsKKj?9%w#1Tt$C=txu@t?GeB1E1`USqe|FPftymHdz-_`mu`%|AhHTdk?dcJkv zubGBk{B!)R{~+JqcX9W;FWr4Y^TuPZzx~~7?pgHFqmK-JbNfYOn(dCiR7<%k{ z8@94z=G$>UUcS9r!_q~ncAZe6)!MF&U+-3-;0@PaGN ze(rh0>+cx0c5~i3b$8v>a`nX8eapsOb9%1}E-V)pmMM~@diEa)Mn({Ur)TcM8{Y9FS@rt<@d*(|HZX4etd1j z!I|4WoUx&9+<_f;pPY8ih(tGMaUGmH*E6UQq^uK9eu6) zFA2+6e0KLg$hVL5+wf&_j$Om{U$^?asxzB)`LK14IVTS-Q2fW_e$hAMJ2pLbz71Ph zkSAj2rP%2#^?bXd_QQLwt$ouOr;U0!>wD{FeOjv8%K6b59V?aS-thE|rFV3ShGpz9 zl(?|{aYVjNKg}L>z71PP^`DZDalSp?r6SqFLI(Nv`Y}(`Og-O*ZBs_R4U0KQc}l)L z-k`^w+N4(orlmIY*C9CH2ETTZNT$dsk&Ed9RjjdOCYZGLvuTMv{XT-ZPss0rwt3boZhe7&@wZdU6B0owu#Tgx`9N)euJYmf2lY8g_0Wv z7a#QNT@MUiG~v?$8}>BmIQo-97mum>^VG{Xw_CEVT;~;UUbpq>)pzXrc~aa}{jTm^ zbk=z%6`s6$^WZfX#6`o-meE>MrZzdVPyG&K8uS|e)V1dhDK`1Hew{Avo;>fnvQ?g| zoqK-#i)Umj8acmvv2)s%>@a9)qs#jJHhbWGkFGvAyyx{NESjD(`r(jQPRLrcQ|Hfm zJ@B`Kc@Nx|ee>2@Wp8ZO{fvi)R9L-gcJJtUBbE<8?djD`ww*k)Z{ON4oZsWz96h#8 zJ!RO?Rdts&?bQ3DiT7Xh%@g$sewDcXq^+Ajua~rSa_<9$H|^;2-m=dwpAlV>E%XXI z+N#aVFHg=?YDTgA1M1{2+vJw#2A7}x=CU0t->Ell$b)SY>g;a#^osG1ef!(}6WSiU zq4Jy#*Hs;Q-f|G0PJ{q3S*0b-Z5vDcrQ3+~@nb;IhJ<7U$`Sc_8zZtgg{nD=&ec|pa##b3M zrDvJOn+H_6y!2h~R=FutcrF;RaDD!f3tt=F`lZGz&mQtwp(zKtuO9v7uxGBw)_hdm z32Qo5D7c{L!aW1t{k7MiI;}o^<%_)SzpT@}-!nb0eD9$g^WNMV7oH2|y*>Bxxmka% zvv=O?llRwseZiLEzr9pr`qxD~MyU?O%Zup_vxLHL$xM0wxGVky2eCFDP zHD4@IasA}EvxndR%=n$r)!!t%HaUC03N;$_Y*1}dp7307-L%}b_wB#2)8tKe@7}TL zhf#mK>6)^u@;q93%(h#Ke|})r)L$mwy17BO!Ba~1nX`XM)|iji4x90P&&IdS%@Pd@5S#=4 z++1)~_nyVO7u#}mgN73x>)v%s&r2^kuh{r23l-Z^asJoszSuYYnG??aWLmSA`(1Ww z>Dr}Ezh&{7&kJ9=c-F+5yS;OIkM*^)M#6K!7r8UvI`7wW9_ZC$UY5@DtB*cdrPuj0 zPJZtC317YP$S3)l6w9)*%BrW9pW3iQ@}|1KWPNnNCyPdZ{nC56dfqgw;fmzPc4rCC z1>@daJ^kD!RZdUp*RpT6DNW{_Gp6{K`>vnx`Okf7-&X&drB!boI&I4heV2WbyG+g6 z4~#0dV9$^DG~4^?x=I(Gc=xqeY#tpA&jqWRZ^*i&=I&owfBozD3$E!{sbZlgd)~CU zW2v_LO5R=OhZk?mH@WTGA5TB+qE@G_98mYJ>E%8e_2sm=mAb7Q^x}C}U%c+wxQtxz z#I$DTe(~mw@gw^Te!1hc55Ftbszm18@7_3N(Pzu%$8E`fK{PBtu=j(D_dn-caOGS3 z8*F;Fe*ga0Z7tHaX!(~;s#5Qq?zeAy_>!^nJ|A-X9pH z*4o@*_zMdn{cn8u)@D1Wj?WpM3#Q)m-ZKksZ{4Wx$=RAzojvvEHMutKzU7`bH$1ufS6;q+;GE4#U#-3I@u$AK{ITEa z^5z@Qzr9Aj=4A%nS*!H>XEtt;uU+r;bLxFne%H!^CC>b4`TCZY^A6tF;FApvws!cs zNv{XTZ_8QvoqIa&T6oIrVJ}xp{C4Rx=T{yvr1YJI7B_9N;K7;WdmcI$^sYN)ZJp=N z-#GV^^5>q@6Ht!|5&@sxcip9*Et#*AnmdswP+c8`{(3>x6eE0;l^9@ z-~33KZ{uIu`((#z6EA9W#^-I6b3S@M_psuv7A1W0 zQR5Cj+&`;spOHVl(lbYRF8HKbmDZc9@4EBmffF9Q=gw}k)>Y4P!T49!bsxUE+Ptru zmFPJyzV)~-r&PPJMw2EpXI^vpS@lnvvZ~laTW+4%wM3tKWisap&jp3fetzee-$q9| z9qe~Tr6+Ittii;CcO^|-v2)jqMs4EywQ6yHiLnn1*?7&oX#+-&&N=+fyGoC%@Xqi% zO0BQ6@2Yp-+|xH#__^S!Uz$F6{kX5o%r0HD@@vB;x7hJU=3+k$y8p#~d)BX-m-pGn zODrDIe%gIKM=pPLccX{KR-5qal-gAfCg!|qXOUl$n^tO&DdSx5=v@a#Y~68E=}X_O zcx%#%o1eey+a@2adG*paaxZ+M?}})L{?V`i>6sv7^5^7&QN7RaeQ(mJ)mMyfKX=?$ zb0)qW_xSan7I|mQ*b&Xo+WcLaLf?M-OzU?Z9DRM$67Al7U~$=gCBErYb5Oxe1xGzL zJ?Z_Cm9vKDf}ICW*mA|5CHYtM$*Z3kICpaKTGwG*C~INgTX~zto&a-a!Ia%xyr|H&HGNg`u~jl z><+`|XvQ^vSn>Y_y9yP2={vXt% zhSfbByB;-cWx?%^!|PFF<6>osoEYhr*d?iL#}0|DnkRJX)TLv$gmy{I+ax5lNlZ#R zRDxS1v1N~Z#afocUIo)8k+c}|MUEX$dZ6E#987IG;mA2bg3`7pPkJmZI<~s& zpGZs2AkMT{avl~-Fehw!WIE#0V`!6@&?>RZKarBW!FtkClKY5wf_Y)nAT1e(Kx+Me zi`-4~+@H5!o>h65$4|=JBDebgFh}zV5FkK+0D-@{z_ip2C>i;;-@}Mq0f*OE{n0iI ze}VG9`v0(=H_^XX{r{)w$5dET)6_%Yzp4jOTHR{NNLcgA7*L=T)(9_6>Hl)SDzsVu z|EK;cfl}BxH>MPX4ZDANxajdTVu?oon|e|NO2^Z1Be>TSyO&uc@?MUuc~<6nFNga7 zSPt?cK!5-N0t5*BrwV+RT7PytV?6}bjj<}xk|~>$DbhA&*-ujb!iI@Bbe$NDR7zd) z(v<(1BehfhXNlaD@;_^&Ys&wVQ{ph>Vo0zdM7n3 zeM7f5M(@0^M&5E;FPPA9+N$@rmAbCumZ??pR~UEt=XDxYSb9#jUDq|eV|?a0i3Ji0 zZQ1u}wKk^=dVS)fImXwhv!V0^;lPAI! z{o{EetZ(nmFVcGc|4YsjVLKr-PXyk1ww!z-) z2Q!*jq#XCBL`G&hy#N2mOzQuGGo0zaTcB9#q4fV@41?HdJ-jaFKYxUQjM|>C`_KCS z>;JF+zyAOF|Lgy+|G)nK`v2?yumAu5M*sgq&+E$+DH>@I-ynWT{B!Z0;$!3g?Nk2$ zcNcoC@82j8wDt^bnHClxxak>GP=t+u61%hUufM~f{BQg}`-0e}8mbVB`N8|8Hp2_yj z;neRVkpk({w0I(sAbjQY?}PC9(?91)p9b~#LHLV~3Lgxc5o^p8d!_Xeh@?JMCheP8 z`~NoaZPLsCz4LyWG2%hmeE$g$AVA>1Tj0CY%?t0f82Y?`TT2eTZsd?3Z|~IXj3G1T zf7x#I^0FOj6&g3WMw#*5rcB-bV6Ib^e7m*ys8PLtJacj3=>ylz>%U?{?~SDgocnmA z4VN~4vUl|IybrbSKD$_s+~q&~EqA?L^)I@6Pt&gRSJs@G|Gj>j^Q}5DQZ~KGaPXrT z3}%zsf+uA&j_*EO=!PVvHuNkR3GJI1)`c<8!FwmI*?!0O<=(FK(NA4oxhbLe=+aB; z=KOJDwC{^&{8Hm$EkGo$UynbDid zJk#akOHcY?P06u!U;b);!u=Hro%m?s^0`M(c=*FC1#;{St~OGC-b2U4i;MJo;O#LV z<_^6T85gPi_N=*WZVW90X9NsuTN|u8b{P_y&KQ4OWI^!)lWxow7kQ!Yiq9{~7MkX? zh4Qxj{iu)h>enh{VH{X5fq$eIZ`=wz|7AQXC z!x2+84C=Gq0Uo@%(0m2A}+M!xh^f8{2MnraljjzxC{m4PH3$$}`{k z#UMgGM%4^jnVMQ|9F#*!T4c3u--f<*V23@A1T0 zuf6=_-i80%(V8t%y=T53Zw}2gLAWrVJD>l& zqCv(&VW~gnLSdVdw$M?@Yr8jil3YqUQeLUqCw=-)9I2t5t%D;qv_E=SOwDqReXdct zI;VYI`Q`6_-1XaoBi7y76P_6~>D4>FeyHT-Bc46E(So;zwyjhy|2ZXE zKUZh>6;tER-d3mOthJ^m{{|iTBlZNJ9g+j z+d4&pN7MxwG?>O{GDyPqP&D+x0k@>aTQU;1!NKZdd*_(19gmkgeOC6%h7Yd3^U@qA zd^NW5`1;dIUw1)#{yNuOxVKw{I`@6kv)INdqjDzBXmUb}j~mw9R(8o5r!*@)w&V%d zHMq0nuBMTdg9a99w!KG_(wl0W7{BE5M~bajwf)^;`y1T+#B=-0o$$?B87VlfqaZl4 z4$t^!r6xRkq+qE$BvOA# z5C4rg)^mTOf3yulMk|oe(rZ|N;3q4%KniQ+a$LWaQP2PL{J)`5qw{^yR~D0j4e=+%-u{#lEyV03{<~w`mUBLt%q1#8#72T-O2 z2oNCf-zgBZ0tssb5!zYcpKT?Pdhb58704eQ!yuy-NN79pumC|r!#}qbi28r^|LXs* zo}~U?{r?}T|9`RHp7pEd<$d%`G zrj;6G%J@YHZG9gW;OGm2Kc`*bU%dbS1?eh5O?%MJzhvt7*(2rC=LesUx4nM#^yPzo z1VO+5ApU>Uk09vxAA}DsQ-Xd5vGRZU$R)W3<|?1w0zmmceHWSh7YSrdy>1Qf-5k~$ zJVp|GLecNPdZf&btuWT>CZSWOE*-liv`cEcLmvo-p*wneSHkA!%^$X!Oo4`8Oo3 z+0b*;#S>5M@Osm@@7~%laM`PipIAKj&T-GKtoTl*Xt@*LJ@4wPAMNpK>l#<>?6|)2 z7p-~?Ui1261se>!<+9EX)c-hJv`_a^Uwr)W^CuP_cU!x~dpd7AVQ|seU*Fes;^ILw zzt~i=$9Xv;Tc6tU{mDNyseWF!0~gz8l5`Em7U z=UwvD8KwK)`N5`dwqJjD|1bJp8Xx_1?e_9}*DPB-{e_n+tZBb%!v(i})&H%*wYxr5 z<I6C(BB2}+kT=TZ93r~3XjK3{eT`kkDmgD0lpW3#6-s&5QM_NTrPtPr3 zrNXfJv7IpRnp_Q^N}6`8FqLoYuW?pZF~$Ha>d7(6g%7@BQ?&-_K2}_tyJqaLezD-uJ?!uGP;e zSupd?ggdTn)9}QBmDb!;Hs6+Aa#Lt-`k`f`&vw3CH16B0$7O5NCjPfpHCya1wrgLN zw%4^vsu7*BFA|)4!iv~o^HVa@|K_=8bjJfZmY#Cb5BcKfSFCr*{wtFoxN+Q=&IN90 z_ek@T&;91@n%ScF^(|cS+=^q9ADz1QuCKcHZC7eemhT&0|H<|XZmYKCiaSny;&{$I zbDtYkCg;ICyS9BdY1;X}KK@z4c~x8f(z@QaC;jcqn@@Uw+j+-x?ukr(Yh4UO@%^wZIod5RIjQo%-(D{qO?dX@ts`$)dgsnNZ+Y*O5?h*W$n<^L z$h?UUbsbyv%B#w@?vasO!uqj?&6ivL6}csnGMqu|xBbLOx5O?Sd)p2P?b`M{ zJdM8-&k?hNNYl5sVC2t>D_0~bIUzZ*c~Wx6E{SO?4T?*LvmH4n=pdbuoUdXtHe(2I^Na1 zePVJ#tAyl)-$}y}vw{Sr$CEp4ixOJKt{_8Mg2V*FS*LAPTzXp47e8Wk$NX?q8uCX{ zYos$arFpmXvhP3M*TMb?rs-Rg_o$eTx%Ojx86+red-9~m(xPLlyZ(u^WOS5IizVk_ zu^eNS$NcKZbi}8}&?YgVRbrQaA|)9YifJjyeMCIRT=6l!4APPjS<1QWlt{PC1+vY} zQ9AC9JhAcrx&;~H7|Z*L0D=FjK+vK%?7;$|Pv*O4=G|wXnf0A=?ON=*;FfPL9n-pC z#oaFtSUPw|+cLXvUb1z0SU0f9#$4-f{jJZK%_lFsBGcBFk~Wt8Th_B@%&u{1`PcWA z==9z5b23iS$JLuVF7m@^W6!#|Sm+={agpy{x~F5I*TP2>oA^$}2lj259Qw%QVAaQ! z&RDTNZb!AA^Ouf`22FQE)111{=QF=7^V;m2G8PJJyOp|7;%z5gv2x3yg^tyb>+f0= zADz6j@}&)*6`YiVU+LHyVtXm|FDyCH(E96)5yispfQ80n{J+=U&mUK7%*Sho&G^1& z?BROG{|nm88~-nM#D+He2p#M$<7f@LXA~Nf@&B^4ukrBGD;H)D9)D;2ztrw6#{WC6 zBQ>;^-1vWYecR-tHLo6evf%&H_<#SZ_Wy@pHl`n`DZea{=;$43qguv39R51|YB6@n zm~lHTrTl+*|NrIjleGUo9L{?J1PBlyK!Cu1h(J)PQ2t-^*|Pa@Tk>D(8$LPitcx-qtj>g`cKw z=5ah@CM8@nbl|&KDKG66O|2I=HRVHe+>e)Uuhy`1k*ZxMRA{xfYvb3uRVaAFwU^AP zyKP~~_Cx1QiMHN!L!H|;zSz0JhmCu!t-PrAwi5>2+NNKPZJVZFzxlh_&F(Gz;heV~ zdHme-hS%RQZ0+W}b?WZAtL5s6wfmNhyXN#>7hG5_?)^HgyLZlgOXqK9Uo`Kk7Srxe znDtBNueQF}^~LJ*v(7#Bq0CvbMqW8>U$!ifbw6i~25AhG!p>l^?FqsjPn)MeH1v53 zF`H=E5PLytQ^Fn+-BJ7DJ=fO0>5S7xy`1&Eb+bM#Rc+<`=!}k)N_1~{`o_{bI)#$| zdg9e3I=<3>(Y*yKzd!E$FRq>O<7*=h&fNClj16_;4(z!5y?OAO3*y47W@=5D+T_eW^*fAd&};Zp*Pc71 z*yP{(b-K8F^1ScLR(Y;=?)mXAo{_C+Eu)85XiaWdfZ+G?c=yq9)xk&a-&b|R z>Y3wa)%?8g*t|n#{Wv39dg%O}^PWsx8rQFE*PEv_>bB?lJJx?wV&YlvJ(jC}<<^zf zmi@Hqw#)N>Gjaa+kCv*LJ$xVi(XsQx_tA)j>+_FX_}cK+FEw6y_K?pCO*zng_2@5$ zJ#$62=A-IPSktjW!39MZ?iujzue}D+GF3`{ey~Uth4L_-`-Om_9c5L#Iu6t?bO4)tXHhQ)yd++~ZIB>9I#Q zjB3`cNZ}3R=RDv3m!@um8XP{~x@e|G)nKv0U&M?f+l)#%A5mcz8&K)vIRrj$So#&w(FnkG@6W!nT-qAH*y=wan=e*LPZPw|NHq=jicWwU` ztJaKab$90QtHS}OHeB%18I7JUvF(IS_fNlh_0zS!z2=5*^WJ-J@0*riy6?d5H$Hf= z!zXX7ex%~K%Eg-&t6yzQr&^_owpyHbU9Dc##&yb{B>!eNJ zZ+x|Pp+(Q!@I$q6vxT@Rg1VAB@`{G+_zi3Bn)mc!CAe<2}0kpB!)ezYH302oNAZfB*pk z1PG+J8UG{g|AXPpf+p?S|ED&sPRmGiftLq%hU3}|M>WSL7U)=#_4G)e}(b?(vqA#5|qD#9iG-sKN2aKy63Y;DyQdw zAOi&T@WFWeK^6_d|Eum71m~W#_WxrdJ2SP5J0KfEh?-?q!vcDStJ^f#tfzM}EcHlLO&9DVlPZwl=ku)cc!_fF3A=wp{( zTd@A;Gph|)nJ}+a`)|9gz4XkCQI*4*yob$?WsE-P>pTWhd^x zy65UEA}6d|Hs+JNYmBe{LL_r)39@7)thaO6{Mc3Ug*Bd>cGDwo)jIWrCAoUcpWgny z<(qCPbZzcSwm!Xl;q7-Wo%~00)A@r|J~bus?i(Lf`=#~4KAmS}z3i021N#(ja%bc3 zekhV>{G-2rmAojq?3n(Y3gmB@_vW$tzG^gh>qpn_y?I{gleV;4-t)YrJ=^8 z`lmeiX!rcx{ocQKfBoH6au;hpb!PAF6>AO6vFx1}$`^gI#(jmp`7&FyeYXobzEo-d zosZ5b*ZQsXMSA90_H2>s2HZB}p^j(e>vw6z@!?m=$8!z*;I4$q2Tm)uDc^&`?;Wwf z$k=M71`d1i^rU6OOHaJ8+LBqN3P(%MuK2+GF0I?|T>3__@o$xS?vz#Q?wX$C`KPLW ze%;2^r5~*u*^pW&Eg1i-YVFG1yZpp%|o{tET~hi}Hw?}O~0cINocc>Z6|FG2bL=)L7L0RjXF5FkK+0D;2= zl>ZYtb?VZwoAUp`gK2poen~Qf2=1=hL@OOipN(keu*O zJn24PaJ;4YD*vZ?WHi4?Q^}p~+a@ugRbrPwENmLwiwFy6J^=y*2oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK;ZwiL&pLD0001k{H;f^uMjd|z<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 RV8DO@0|pEjFkryIHxO5YQ=R|- literal 0 HcmV?d00001 diff --git a/node/src/reactor/main_reactor/upgrading_instruction.rs b/node/src/reactor/main_reactor/upgrading_instruction.rs deleted file mode 100644 index 63e44af633..0000000000 --- a/node/src/reactor/main_reactor/upgrading_instruction.rs +++ /dev/null @@ -1,27 +0,0 @@ -use std::time::Duration; - -use casper_types::{TimeDiff, Timestamp}; - -pub(super) enum UpgradingInstruction { - CheckLater(String, Duration), - CatchUp, -} - -impl UpgradingInstruction { - pub(super) fn should_commit_upgrade( - should_commit_upgrade: bool, - wait: Duration, - last_progress: Timestamp, - upgrade_timeout: TimeDiff, - ) -> UpgradingInstruction { - if should_commit_upgrade { - if last_progress.elapsed() > upgrade_timeout { - UpgradingInstruction::CatchUp - } else { - UpgradingInstruction::CheckLater("awaiting upgrade".to_string(), wait) - } - } else { - UpgradingInstruction::CatchUp - } - } -} diff --git a/resources/test/rest_schema_status.json b/resources/test/rest_schema_status.json index f1a156d42d..29d816c307 100644 --- a/resources/test/rest_schema_status.json +++ b/resources/test/rest_schema_status.json @@ -308,13 +308,6 @@ "CatchUp" ] }, - { - "description": "Running commit upgrade and creating immediate switch block.", - "type": "string", - "enum": [ - "Upgrading" - ] - }, { "description": "Stay caught up with tip.", "type": "string", diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 8e1de74053..a5e486f769 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -16,6 +16,7 @@ casper-types = { version = "7.0.0", path = "../types", features = ["datasize", " datasize = "0.2.4" either = "1.8.1" lmdb-rkv = "0.14" +lmdb-rkv-sys = "0.11" num = { version = "0.4.0", default-features = false } num-derive = { workspace = true } num-rational = { version = "0.4.0", features = ["serde"] } diff --git a/storage/src/block_store/lmdb/indexed_lmdb_block_store.rs b/storage/src/block_store/lmdb/indexed_lmdb_block_store.rs deleted file mode 100644 index 2db3030e60..0000000000 --- a/storage/src/block_store/lmdb/indexed_lmdb_block_store.rs +++ /dev/null @@ -1,1256 +0,0 @@ -use std::{ - borrow::Cow, - collections::{btree_map, hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet}, -}; - -use super::{ - lmdb_block_store::LmdbBlockStore, lmdb_ext::LmdbExtError, temp_map::TempMap, DbTableId, -}; -use datasize::DataSize; -use lmdb::{ - Environment, RoTransaction, RwCursor, RwTransaction, Transaction as LmdbTransaction, WriteFlags, -}; - -use tracing::info; - -use super::versioned_databases::VersionedDatabases; -use crate::block_store::{ - block_provider::{BlockStoreTransaction, DataReader, DataWriter}, - types::{ - ApprovalsHashes, BlockExecutionResults, BlockHashHeightAndEra, BlockHeight, BlockTransfers, - LatestSwitchBlock, StateStore, StateStoreKey, Tip, TransactionFinalizedApprovals, - }, - BlockStoreError, BlockStoreProvider, DbRawBytesSpec, -}; -use casper_types::{ - execution::ExecutionResult, Approval, Block, BlockBody, BlockHash, BlockHeader, - BlockSignatures, Digest, EraId, ProtocolVersion, Transaction, TransactionHash, Transfer, -}; - -/// Indexed lmdb block store. -#[derive(DataSize, Debug)] -pub struct IndexedLmdbBlockStore { - /// Block store - block_store: LmdbBlockStore, - /// A map of block height to block ID. - block_height_index: BTreeMap, - /// A map of era ID to switch block ID. - switch_block_era_id_index: BTreeMap, - /// A map of transaction hashes to hashes, heights and era IDs of blocks containing them. - transaction_hash_index: BTreeMap, -} - -impl IndexedLmdbBlockStore { - fn get_reader(&self) -> Result, BlockStoreError> { - let txn = self - .block_store - .env - .begin_ro_txn() - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; - Ok(IndexedLmdbBlockStoreReadTransaction { - txn, - block_store: self, - }) - } - - /// Inserts the relevant entries to the index. - /// - /// If a duplicate entry is encountered, index is not updated and an error is returned. - fn insert_to_transaction_index( - transaction_hash_index: &mut BTreeMap, - block_hash: BlockHash, - block_height: u64, - era_id: EraId, - transaction_hashes: Vec, - ) -> Result<(), BlockStoreError> { - if let Some(hash) = transaction_hashes.iter().find(|hash| { - transaction_hash_index - .get(hash) - .is_some_and(|old_details| old_details.block_hash != block_hash) - }) { - return Err(BlockStoreError::DuplicateTransaction { - transaction_hash: *hash, - first: transaction_hash_index[hash].block_hash, - second: block_hash, - }); - } - - for hash in transaction_hashes { - transaction_hash_index.insert( - hash, - BlockHashHeightAndEra::new(block_hash, block_height, era_id), - ); - } - - Ok(()) - } - - /// Inserts the relevant entries to the two indices. - /// - /// If a duplicate entry is encountered, neither index is updated and an error is returned. - pub(super) fn insert_to_block_header_indices( - block_height_index: &mut BTreeMap, - switch_block_era_id_index: &mut BTreeMap, - block_header: &BlockHeader, - ) -> Result<(), BlockStoreError> { - let block_hash = block_header.block_hash(); - if let Some(first) = block_height_index.get(&block_header.height()) { - if *first != block_hash { - return Err(BlockStoreError::DuplicateBlock { - height: block_header.height(), - first: *first, - second: block_hash, - }); - } - } - - if block_header.is_switch_block() { - match switch_block_era_id_index.entry(block_header.era_id()) { - btree_map::Entry::Vacant(entry) => { - let _ = entry.insert(block_hash); - } - btree_map::Entry::Occupied(entry) => { - if *entry.get() != block_hash { - return Err(BlockStoreError::DuplicateEraId { - era_id: block_header.era_id(), - first: *entry.get(), - second: block_hash, - }); - } - } - } - } - - let _ = block_height_index.insert(block_header.height(), block_hash); - Ok(()) - } - - /// Ctor. - pub fn new( - block_store: LmdbBlockStore, - hard_reset_to_start_of_era: Option, - protocol_version: ProtocolVersion, - ) -> Result { - // We now need to restore the block-height index. Log messages allow timing here. - info!("indexing block store"); - let mut block_height_index = BTreeMap::new(); - let mut switch_block_era_id_index = BTreeMap::new(); - let mut transaction_hash_index = BTreeMap::new(); - let mut block_txn = block_store - .env - .begin_rw_txn() - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; - - let mut deleted_block_hashes = HashSet::new(); - // Map of all block body hashes, with their values representing whether to retain the - // corresponding block bodies or not. - let mut block_body_hashes = HashMap::new(); - let mut deleted_transaction_hashes = HashSet::::new(); - - let mut init_fn = - |cursor: &mut RwCursor, block_header: BlockHeader| -> Result<(), BlockStoreError> { - let should_retain_block = match hard_reset_to_start_of_era { - Some(invalid_era) => { - // Retain blocks from eras before the hard reset era, and blocks after this - // era if they are from the current protocol version (as otherwise a node - // restart would purge them again, despite them being valid). - block_header.era_id() < invalid_era - || block_header.protocol_version() == protocol_version - } - None => true, - }; - - // If we don't already have the block body hash in the collection, insert it with - // the value `should_retain_block`. - // - // If there is an existing value, the updated value should be `false` iff the - // existing value and `should_retain_block` are both `false`. - // Otherwise the updated value should be `true`. - match block_body_hashes.entry(*block_header.body_hash()) { - Entry::Vacant(entry) => { - entry.insert(should_retain_block); - } - Entry::Occupied(entry) => { - let value = entry.into_mut(); - *value = *value || should_retain_block; - } - } - - let body_txn = block_store - .env - .begin_ro_txn() - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; - let maybe_block_body = block_store - .block_body_dbs - .get(&body_txn, block_header.body_hash()) - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; - if !should_retain_block { - let _ = deleted_block_hashes.insert(block_header.block_hash()); - - match &maybe_block_body { - Some(BlockBody::V1(v1_body)) => deleted_transaction_hashes.extend( - v1_body - .deploy_and_transfer_hashes() - .map(TransactionHash::from), - ), - Some(BlockBody::V2(v2_body)) => { - let transactions = v2_body.all_transactions(); - deleted_transaction_hashes.extend(transactions) - } - None => (), - } - - cursor - .del(WriteFlags::empty()) - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; - return Ok(()); - } - - Self::insert_to_block_header_indices( - &mut block_height_index, - &mut switch_block_era_id_index, - &block_header, - )?; - - if let Some(block_body) = maybe_block_body { - let transaction_hashes = match block_body { - BlockBody::V1(v1) => v1 - .deploy_and_transfer_hashes() - .map(TransactionHash::from) - .collect(), - BlockBody::V2(v2) => v2.all_transactions().copied().collect(), - }; - Self::insert_to_transaction_index( - &mut transaction_hash_index, - block_header.block_hash(), - block_header.height(), - block_header.era_id(), - transaction_hashes, - )?; - } - - Ok(()) - }; - - block_store - .block_header_dbs - .for_each_value_in_current(&mut block_txn, &mut init_fn)?; - block_store - .block_header_dbs - .for_each_value_in_legacy(&mut block_txn, &mut init_fn)?; - - info!("block store reindexing complete"); - block_txn - .commit() - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; - - let deleted_block_body_hashes = block_body_hashes - .into_iter() - .filter_map(|(body_hash, retain)| (!retain).then_some(body_hash)) - .collect(); - initialize_block_body_dbs( - &block_store.env, - block_store.block_body_dbs, - deleted_block_body_hashes, - )?; - initialize_block_metadata_dbs( - &block_store.env, - block_store.block_metadata_dbs, - deleted_block_hashes, - )?; - initialize_execution_result_dbs( - &block_store.env, - block_store.execution_result_dbs, - deleted_transaction_hashes, - ) - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; - - Ok(Self { - block_store, - block_height_index, - switch_block_era_id_index, - transaction_hash_index, - }) - } -} - -/// Purges stale entries from the block body databases. -fn initialize_block_body_dbs( - env: &Environment, - block_body_dbs: VersionedDatabases, - deleted_block_body_hashes: HashSet, -) -> Result<(), BlockStoreError> { - info!("initializing block body databases"); - let mut txn = env - .begin_rw_txn() - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; - for body_hash in deleted_block_body_hashes { - block_body_dbs - .delete(&mut txn, &body_hash) - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; - } - txn.commit() - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; - info!("block body database initialized"); - Ok(()) -} - -/// Purges stale entries from the block metadata database. -fn initialize_block_metadata_dbs( - env: &Environment, - block_metadata_dbs: VersionedDatabases, - deleted_block_hashes: HashSet, -) -> Result<(), BlockStoreError> { - let block_count_to_be_deleted = deleted_block_hashes.len(); - info!( - block_count_to_be_deleted, - "initializing block metadata database" - ); - let mut txn = env - .begin_rw_txn() - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; - for block_hash in deleted_block_hashes { - block_metadata_dbs - .delete(&mut txn, &block_hash) - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))? - } - txn.commit() - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; - info!("block metadata database initialized"); - Ok(()) -} - -/// Purges stale entries from the execution result databases. -fn initialize_execution_result_dbs( - env: &Environment, - execution_result_dbs: VersionedDatabases, - deleted_transaction_hashes: HashSet, -) -> Result<(), LmdbExtError> { - let exec_results_count_to_be_deleted = deleted_transaction_hashes.len(); - info!( - exec_results_count_to_be_deleted, - "initializing execution result databases" - ); - let mut txn = env.begin_rw_txn()?; - for hash in deleted_transaction_hashes { - execution_result_dbs.delete(&mut txn, &hash)?; - } - txn.commit()?; - info!("execution result databases initialized"); - Ok(()) -} - -pub struct IndexedLmdbBlockStoreRWTransaction<'t> { - txn: RwTransaction<'t>, - block_store: &'t LmdbBlockStore, - block_height_index: TempMap<'t, u64, BlockHash>, - switch_block_era_id_index: TempMap<'t, EraId, BlockHash>, - transaction_hash_index: TempMap<'t, TransactionHash, BlockHashHeightAndEra>, -} - -impl IndexedLmdbBlockStoreRWTransaction<'_> { - /// Check if the block height index can be updated. - fn should_update_block_height_index( - &self, - block_height: u64, - block_hash: &BlockHash, - ) -> Result { - if let Some(first) = self.block_height_index.get(&block_height) { - // There is a block in the index at this height - if first != *block_hash { - Err(BlockStoreError::DuplicateBlock { - height: block_height, - first, - second: *block_hash, - }) - } else { - // Same value already in index, no need to update it. - Ok(false) - } - } else { - // Value not in index, update. - Ok(true) - } - } - - /// Check if the switch block index can be updated. - fn should_update_switch_block_index( - &self, - block_header: &BlockHeader, - ) -> Result { - if block_header.is_switch_block() { - let era_id = block_header.era_id(); - if let Some(entry) = self.switch_block_era_id_index.get(&era_id) { - let block_hash = block_header.block_hash(); - if entry != block_hash { - Err(BlockStoreError::DuplicateEraId { - era_id, - first: entry, - second: block_hash, - }) - } else { - // already in index, no need to update. - Ok(false) - } - } else { - // not in the index, update. - Ok(true) - } - } else { - // not a switch block. - Ok(false) - } - } - - // Check if the transaction hash index can be updated. - fn should_update_transaction_hash_index( - &self, - transaction_hashes: &[TransactionHash], - block_hash: &BlockHash, - ) -> Result { - if let Some(hash) = transaction_hashes.iter().find(|hash| { - self.transaction_hash_index - .get(hash) - .is_some_and(|old_details| old_details.block_hash != *block_hash) - }) { - return Err(BlockStoreError::DuplicateTransaction { - transaction_hash: *hash, - first: self.transaction_hash_index.get(hash).unwrap().block_hash, - second: *block_hash, - }); - } - Ok(true) - } -} - -pub struct IndexedLmdbBlockStoreReadTransaction<'t> { - txn: RoTransaction<'t>, - block_store: &'t IndexedLmdbBlockStore, -} - -enum LmdbBlockStoreIndex { - BlockHeight(IndexPosition), - SwitchBlockEraId(IndexPosition), -} - -enum IndexPosition { - Tip, - Key(K), -} - -enum DataType { - Block, - BlockHeader, - ApprovalsHashes, - BlockSignatures, -} - -impl IndexedLmdbBlockStoreReadTransaction<'_> { - fn block_hash_from_index(&self, index: LmdbBlockStoreIndex) -> Option<&BlockHash> { - match index { - LmdbBlockStoreIndex::BlockHeight(position) => match position { - IndexPosition::Tip => self.block_store.block_height_index.values().last(), - IndexPosition::Key(height) => self.block_store.block_height_index.get(&height), - }, - LmdbBlockStoreIndex::SwitchBlockEraId(position) => match position { - IndexPosition::Tip => self.block_store.switch_block_era_id_index.values().last(), - IndexPosition::Key(era_id) => { - self.block_store.switch_block_era_id_index.get(&era_id) - } - }, - } - } - - fn read_block_indexed( - &self, - index: LmdbBlockStoreIndex, - ) -> Result, BlockStoreError> { - self.block_hash_from_index(index) - .and_then(|block_hash| { - self.block_store - .block_store - .get_single_block(&self.txn, block_hash) - .transpose() - }) - .transpose() - } - - fn read_block_header_indexed( - &self, - index: LmdbBlockStoreIndex, - ) -> Result, BlockStoreError> { - self.block_hash_from_index(index) - .and_then(|block_hash| { - self.block_store - .block_store - .get_single_block_header(&self.txn, block_hash) - .transpose() - }) - .transpose() - } - - fn read_block_signatures_indexed( - &self, - index: LmdbBlockStoreIndex, - ) -> Result, BlockStoreError> { - self.block_hash_from_index(index) - .and_then(|block_hash| { - self.block_store - .block_store - .get_block_signatures(&self.txn, block_hash) - .transpose() - }) - .transpose() - } - - fn read_approvals_hashes_indexed( - &self, - index: LmdbBlockStoreIndex, - ) -> Result, BlockStoreError> { - self.block_hash_from_index(index) - .and_then(|block_hash| { - self.block_store - .block_store - .read_approvals_hashes(&self.txn, block_hash) - .transpose() - }) - .transpose() - } - - fn contains_data_indexed( - &self, - index: LmdbBlockStoreIndex, - data_type: DataType, - ) -> Result { - self.block_hash_from_index(index) - .map_or(Ok(false), |block_hash| match data_type { - DataType::Block => self - .block_store - .block_store - .block_exists(&self.txn, block_hash), - DataType::BlockHeader => self - .block_store - .block_store - .block_header_exists(&self.txn, block_hash), - DataType::ApprovalsHashes => self - .block_store - .block_store - .approvals_hashes_exist(&self.txn, block_hash), - DataType::BlockSignatures => self - .block_store - .block_store - .block_signatures_exist(&self.txn, block_hash), - }) - } - - pub fn get_switch_block_height(&self, era_id: EraId) -> Result, BlockStoreError> { - let index = LmdbBlockStoreIndex::SwitchBlockEraId(IndexPosition::Key(era_id)); - match self.block_hash_from_index(index) { - Some(block_hash) => { - let maybe_header: Option = self.read(*block_hash)?; - Ok(maybe_header.map(|header| header.height())) - } - None => Ok(None), - } - } -} - -impl BlockStoreTransaction for IndexedLmdbBlockStoreReadTransaction<'_> { - fn commit(self) -> Result<(), BlockStoreError> { - Ok(()) - } - - fn rollback(self) { - self.txn.abort(); - } -} - -impl BlockStoreTransaction for IndexedLmdbBlockStoreRWTransaction<'_> { - fn commit(self) -> Result<(), BlockStoreError> { - self.txn - .commit() - .map_err(|e| BlockStoreError::InternalStorage(Box::new(LmdbExtError::from(e))))?; - - self.block_height_index.commit(); - self.switch_block_era_id_index.commit(); - self.transaction_hash_index.commit(); - Ok(()) - } - - fn rollback(self) { - self.txn.abort(); - } -} - -impl BlockStoreProvider for IndexedLmdbBlockStore { - type Reader<'t> = IndexedLmdbBlockStoreReadTransaction<'t>; - type ReaderWriter<'t> = IndexedLmdbBlockStoreRWTransaction<'t>; - - fn checkout_ro(&self) -> Result, BlockStoreError> { - self.get_reader() - } - - fn checkout_rw(&mut self) -> Result, BlockStoreError> { - let txn = self - .block_store - .env - .begin_rw_txn() - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; - - Ok(IndexedLmdbBlockStoreRWTransaction { - txn, - block_store: &self.block_store, - block_height_index: TempMap::new(&mut self.block_height_index), - switch_block_era_id_index: TempMap::new(&mut self.switch_block_era_id_index), - transaction_hash_index: TempMap::new(&mut self.transaction_hash_index), - }) - } -} - -impl DataReader for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, key: BlockHash) -> Result, BlockStoreError> { - self.block_store - .block_store - .get_single_block(&self.txn, &key) - } - - fn exists(&self, key: BlockHash) -> Result { - self.block_store.block_store.block_exists(&self.txn, &key) - } -} - -impl DataReader for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, key: BlockHash) -> Result, BlockStoreError> { - self.block_store - .block_store - .get_single_block_header(&self.txn, &key) - } - - fn exists(&self, key: BlockHash) -> Result { - self.block_store - .block_store - .block_header_exists(&self.txn, &key) - } -} - -impl DataReader for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, key: BlockHash) -> Result, BlockStoreError> { - self.block_store - .block_store - .read_approvals_hashes(&self.txn, &key) - } - - fn exists(&self, key: BlockHash) -> Result { - self.block_store - .block_store - .block_header_exists(&self.txn, &key) - } -} - -impl DataReader for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, key: BlockHash) -> Result, BlockStoreError> { - self.block_store - .block_store - .get_block_signatures(&self.txn, &key) - } - - fn exists(&self, key: BlockHash) -> Result { - self.block_store - .block_store - .block_signatures_exist(&self.txn, &key) - } -} - -impl DataReader for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, key: BlockHeight) -> Result, BlockStoreError> { - self.read_block_indexed(LmdbBlockStoreIndex::BlockHeight(IndexPosition::Key(key))) - } - - fn exists(&self, key: BlockHeight) -> Result { - self.contains_data_indexed( - LmdbBlockStoreIndex::BlockHeight(IndexPosition::Key(key)), - DataType::Block, - ) - } -} - -impl DataReader for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, key: BlockHeight) -> Result, BlockStoreError> { - self.read_block_header_indexed(LmdbBlockStoreIndex::BlockHeight(IndexPosition::Key(key))) - } - - fn exists(&self, key: BlockHeight) -> Result { - self.contains_data_indexed( - LmdbBlockStoreIndex::BlockHeight(IndexPosition::Key(key)), - DataType::BlockHeader, - ) - } -} - -impl DataReader for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, key: BlockHeight) -> Result, BlockStoreError> { - self.read_approvals_hashes_indexed(LmdbBlockStoreIndex::BlockHeight(IndexPosition::Key( - key, - ))) - } - - fn exists(&self, key: BlockHeight) -> Result { - self.contains_data_indexed( - LmdbBlockStoreIndex::BlockHeight(IndexPosition::Key(key)), - DataType::ApprovalsHashes, - ) - } -} - -impl DataReader for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, key: BlockHeight) -> Result, BlockStoreError> { - self.read_block_signatures_indexed(LmdbBlockStoreIndex::BlockHeight(IndexPosition::Key( - key, - ))) - } - - fn exists(&self, key: BlockHeight) -> Result { - self.contains_data_indexed( - LmdbBlockStoreIndex::BlockHeight(IndexPosition::Key(key)), - DataType::BlockSignatures, - ) - } -} - -/// Retrieves single switch block by era ID by looking it up in the index and returning it. -impl DataReader for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, key: EraId) -> Result, BlockStoreError> { - self.read_block_indexed(LmdbBlockStoreIndex::SwitchBlockEraId(IndexPosition::Key( - key, - ))) - } - - fn exists(&self, key: EraId) -> Result { - self.contains_data_indexed( - LmdbBlockStoreIndex::SwitchBlockEraId(IndexPosition::Key(key)), - DataType::Block, - ) - } -} - -/// Retrieves single switch block header by era ID by looking it up in the index and returning -/// it. -impl DataReader for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, key: EraId) -> Result, BlockStoreError> { - self.read_block_header_indexed(LmdbBlockStoreIndex::SwitchBlockEraId(IndexPosition::Key( - key, - ))) - } - - fn exists(&self, key: EraId) -> Result { - self.contains_data_indexed( - LmdbBlockStoreIndex::SwitchBlockEraId(IndexPosition::Key(key)), - DataType::BlockHeader, - ) - } -} - -impl DataReader for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, key: EraId) -> Result, BlockStoreError> { - self.read_approvals_hashes_indexed(LmdbBlockStoreIndex::SwitchBlockEraId( - IndexPosition::Key(key), - )) - } - - fn exists(&self, key: EraId) -> Result { - self.contains_data_indexed( - LmdbBlockStoreIndex::SwitchBlockEraId(IndexPosition::Key(key)), - DataType::ApprovalsHashes, - ) - } -} - -impl DataReader for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, key: EraId) -> Result, BlockStoreError> { - self.read_block_signatures_indexed(LmdbBlockStoreIndex::SwitchBlockEraId( - IndexPosition::Key(key), - )) - } - - fn exists(&self, key: EraId) -> Result { - self.contains_data_indexed( - LmdbBlockStoreIndex::SwitchBlockEraId(IndexPosition::Key(key)), - DataType::BlockSignatures, - ) - } -} - -impl DataReader for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, _key: Tip) -> Result, BlockStoreError> { - self.read_block_header_indexed(LmdbBlockStoreIndex::BlockHeight(IndexPosition::Tip)) - } - - fn exists(&self, _key: Tip) -> Result { - self.contains_data_indexed( - LmdbBlockStoreIndex::BlockHeight(IndexPosition::Tip), - DataType::BlockHeader, - ) - } -} - -impl DataReader for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, _key: Tip) -> Result, BlockStoreError> { - self.read_block_indexed(LmdbBlockStoreIndex::BlockHeight(IndexPosition::Tip)) - } - - fn exists(&self, _key: Tip) -> Result { - self.contains_data_indexed( - LmdbBlockStoreIndex::BlockHeight(IndexPosition::Tip), - DataType::Block, - ) - } -} - -impl DataReader for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, _key: LatestSwitchBlock) -> Result, BlockStoreError> { - self.read_block_header_indexed(LmdbBlockStoreIndex::SwitchBlockEraId(IndexPosition::Tip)) - } - - fn exists(&self, _key: LatestSwitchBlock) -> Result { - self.contains_data_indexed( - LmdbBlockStoreIndex::SwitchBlockEraId(IndexPosition::Tip), - DataType::BlockHeader, - ) - } -} - -impl DataReader - for IndexedLmdbBlockStoreReadTransaction<'_> -{ - fn read(&self, key: TransactionHash) -> Result, BlockStoreError> { - Ok(self.block_store.transaction_hash_index.get(&key).copied()) - } - - fn exists(&self, key: TransactionHash) -> Result { - Ok(self.block_store.transaction_hash_index.contains_key(&key)) - } -} - -impl DataReader for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, key: TransactionHash) -> Result, BlockStoreError> { - self.block_store - .block_store - .transaction_dbs - .get(&self.txn, &key) - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err))) - } - - fn exists(&self, key: TransactionHash) -> Result { - self.block_store - .block_store - .transaction_exists(&self.txn, &key) - } -} - -impl DataReader> for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, key: TransactionHash) -> Result>, BlockStoreError> { - self.block_store - .block_store - .finalized_transaction_approvals_dbs - .get(&self.txn, &key) - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err))) - } - - fn exists(&self, key: TransactionHash) -> Result { - self.block_store - .block_store - .finalized_transaction_approvals_dbs - .exists(&self.txn, &key) - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err))) - } -} - -impl DataReader for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, key: TransactionHash) -> Result, BlockStoreError> { - self.block_store - .block_store - .execution_result_dbs - .get(&self.txn, &key) - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err))) - } - - fn exists(&self, key: TransactionHash) -> Result { - self.block_store - .block_store - .execution_result_dbs - .exists(&self.txn, &key) - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err))) - } -} - -impl DataReader> for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read(&self, StateStoreKey(key): StateStoreKey) -> Result>, BlockStoreError> { - self.block_store - .block_store - .read_state_store(&self.txn, &key) - } - - fn exists(&self, StateStoreKey(key): StateStoreKey) -> Result { - self.block_store - .block_store - .state_store_key_exists(&self.txn, &key) - } -} - -impl DataReader<(DbTableId, Vec), DbRawBytesSpec> for IndexedLmdbBlockStoreReadTransaction<'_> { - fn read( - &self, - (id, key): (DbTableId, Vec), - ) -> Result, BlockStoreError> { - if key.is_empty() { - return Ok(None); - } - let store = &self.block_store.block_store; - let res = match id { - DbTableId::BlockHeader => store.block_header_dbs.get_raw(&self.txn, &key), - DbTableId::BlockBody => store.block_body_dbs.get_raw(&self.txn, &key), - DbTableId::ApprovalsHashes => store.approvals_hashes_dbs.get_raw(&self.txn, &key), - DbTableId::BlockMetadata => store.block_metadata_dbs.get_raw(&self.txn, &key), - DbTableId::Transaction => store.transaction_dbs.get_raw(&self.txn, &key), - DbTableId::ExecutionResult => store.execution_result_dbs.get_raw(&self.txn, &key), - DbTableId::Transfer => store.transfer_dbs.get_raw(&self.txn, &key), - DbTableId::FinalizedTransactionApprovals => store - .finalized_transaction_approvals_dbs - .get_raw(&self.txn, &key), - }; - res.map_err(|err| BlockStoreError::InternalStorage(Box::new(err))) - } - - fn exists(&self, key: (DbTableId, Vec)) -> Result { - self.read(key).map(|res| res.is_some()) - } -} - -impl DataWriter for IndexedLmdbBlockStoreRWTransaction<'_> { - /// Writes a block to storage. - /// - /// Returns `Ok(true)` if the block has been successfully written, `Ok(false)` if a part of it - /// couldn't be written because it already existed, and `Err(_)` if there was an error. - fn write(&mut self, data: &Block) -> Result { - let block_header = data.clone_header(); - let block_hash = data.hash(); - let block_height = data.height(); - let era_id = data.era_id(); - let transaction_hashes: Vec = match &data { - Block::V1(v1) => v1 - .deploy_and_transfer_hashes() - .map(TransactionHash::from) - .collect(), - Block::V2(v2) => v2.all_transactions().copied().collect(), - }; - - let update_height_index = - self.should_update_block_height_index(block_height, block_hash)?; - let update_switch_block_index = self.should_update_switch_block_index(&block_header)?; - let update_transaction_hash_index = - self.should_update_transaction_hash_index(&transaction_hashes, block_hash)?; - - let key = self.block_store.write_block(&mut self.txn, data)?; - - if update_height_index { - self.block_height_index.insert(block_height, *block_hash); - } - - if update_switch_block_index { - self.switch_block_era_id_index.insert(era_id, *block_hash); - } - - if update_transaction_hash_index { - for hash in transaction_hashes { - self.transaction_hash_index.insert( - hash, - BlockHashHeightAndEra::new(*block_hash, block_height, era_id), - ); - } - } - - Ok(key) - } - - fn delete(&mut self, key: BlockHash) -> Result<(), BlockStoreError> { - let maybe_block = self.block_store.get_single_block(&self.txn, &key)?; - - if let Some(block) = maybe_block { - let transaction_hashes: Vec = match &block { - Block::V1(v1) => v1 - .deploy_and_transfer_hashes() - .map(TransactionHash::from) - .collect(), - Block::V2(v2) => v2.all_transactions().copied().collect(), - }; - - self.block_store.delete_block_header(&mut self.txn, &key)?; - - /* - TODO: currently we don't delete the block body since other blocks may reference it. - self.block_store - .delete_block_body(&mut self.txn, block.body_hash())?; - */ - - self.block_height_index.remove(block.height()); - - if block.is_switch_block() { - self.switch_block_era_id_index.remove(block.era_id()); - } - - for hash in transaction_hashes { - self.transaction_hash_index.remove(hash); - } - - self.block_store - .delete_finality_signatures(&mut self.txn, &key)?; - } - Ok(()) - } -} - -impl DataWriter for IndexedLmdbBlockStoreRWTransaction<'_> { - fn write(&mut self, data: &ApprovalsHashes) -> Result { - self.block_store.write_approvals_hashes(&mut self.txn, data) - } - - fn delete(&mut self, key: BlockHash) -> Result<(), BlockStoreError> { - self.block_store - .delete_approvals_hashes(&mut self.txn, &key) - } -} - -impl DataWriter for IndexedLmdbBlockStoreRWTransaction<'_> { - fn write(&mut self, data: &BlockSignatures) -> Result { - self.block_store - .write_finality_signatures(&mut self.txn, data) - } - - fn delete(&mut self, key: BlockHash) -> Result<(), BlockStoreError> { - self.block_store - .delete_finality_signatures(&mut self.txn, &key) - } -} - -impl DataWriter for IndexedLmdbBlockStoreRWTransaction<'_> { - fn write(&mut self, data: &BlockHeader) -> Result { - let block_hash = data.block_hash(); - let block_height = data.height(); - let era_id = data.era_id(); - - let update_height_index = - self.should_update_block_height_index(block_height, &block_hash)?; - let update_switch_block_index = self.should_update_switch_block_index(data)?; - - let key = self.block_store.write_block_header(&mut self.txn, data)?; - - if update_height_index { - self.block_height_index.insert(block_height, block_hash); - } - - if update_switch_block_index { - self.switch_block_era_id_index.insert(era_id, block_hash); - } - - Ok(key) - } - - fn delete(&mut self, key: BlockHash) -> Result<(), BlockStoreError> { - let maybe_block_header = self.block_store.get_single_block_header(&self.txn, &key)?; - - if let Some(block_header) = maybe_block_header { - self.block_store.delete_block_header(&mut self.txn, &key)?; - - if block_header.is_switch_block() { - self.switch_block_era_id_index.remove(block_header.era_id()); - } - - self.block_height_index.remove(block_header.height()); - } - Ok(()) - } -} - -impl DataWriter for IndexedLmdbBlockStoreRWTransaction<'_> { - fn write(&mut self, data: &Transaction) -> Result { - self.block_store.write_transaction(&mut self.txn, data) - } - - fn delete(&mut self, key: TransactionHash) -> Result<(), BlockStoreError> { - self.block_store.delete_transaction(&mut self.txn, &key) - } -} - -impl DataWriter - for IndexedLmdbBlockStoreRWTransaction<'_> -{ - fn write( - &mut self, - data: &TransactionFinalizedApprovals, - ) -> Result { - self.block_store - .finalized_transaction_approvals_dbs - .put( - &mut self.txn, - &data.transaction_hash, - &data.finalized_approvals, - true, - ) - .map(|_| data.transaction_hash) - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err))) - } - - fn delete(&mut self, key: TransactionHash) -> Result<(), BlockStoreError> { - self.block_store - .finalized_transaction_approvals_dbs - .delete(&mut self.txn, &key) - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err))) - } -} - -impl DataWriter - for IndexedLmdbBlockStoreRWTransaction<'_> -{ - fn write( - &mut self, - data: &BlockExecutionResults, - ) -> Result { - let transaction_hashes: Vec = data.exec_results.keys().copied().collect(); - let block_hash = data.block_info.block_hash; - let block_height = data.block_info.block_height; - let era_id = data.block_info.era_id; - - let update_transaction_hash_index = - self.should_update_transaction_hash_index(&transaction_hashes, &block_hash)?; - - let _ = self.block_store.write_execution_results( - &mut self.txn, - &block_hash, - data.exec_results.clone(), - )?; - - if update_transaction_hash_index { - for hash in transaction_hashes { - self.transaction_hash_index.insert( - hash, - BlockHashHeightAndEra::new(block_hash, block_height, era_id), - ); - } - } - - Ok(data.block_info) - } - - fn delete(&mut self, _key: BlockHashHeightAndEra) -> Result<(), BlockStoreError> { - Err(BlockStoreError::UnsupportedOperation) - } -} - -impl DataWriter for IndexedLmdbBlockStoreRWTransaction<'_> { - fn write(&mut self, data: &BlockTransfers) -> Result { - self.block_store - .write_transfers(&mut self.txn, &data.block_hash, &data.transfers) - .map(|_| data.block_hash) - } - - fn delete(&mut self, key: BlockHash) -> Result<(), BlockStoreError> { - self.block_store.delete_transfers(&mut self.txn, &key) - } -} - -impl DataWriter, StateStore> for IndexedLmdbBlockStoreRWTransaction<'_> { - fn write(&mut self, data: &StateStore) -> Result, BlockStoreError> { - self.block_store - .write_state_store(&mut self.txn, data.key.clone(), &data.value)?; - Ok(data.key.clone()) - } - - fn delete(&mut self, key: Cow<'static, [u8]>) -> Result<(), BlockStoreError> { - self.block_store.delete_state_store(&mut self.txn, key) - } -} - -impl DataReader for IndexedLmdbBlockStoreRWTransaction<'_> { - fn read(&self, query: TransactionHash) -> Result, BlockStoreError> { - self.block_store - .transaction_dbs - .get(&self.txn, &query) - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err))) - } - - fn exists(&self, query: TransactionHash) -> Result { - self.block_store.transaction_exists(&self.txn, &query) - } -} - -impl DataReader for IndexedLmdbBlockStoreRWTransaction<'_> { - fn read(&self, key: BlockHash) -> Result, BlockStoreError> { - self.block_store.get_block_signatures(&self.txn, &key) - } - - fn exists(&self, key: BlockHash) -> Result { - self.block_store.block_signatures_exist(&self.txn, &key) - } -} - -impl DataReader> for IndexedLmdbBlockStoreRWTransaction<'_> { - fn read(&self, query: TransactionHash) -> Result>, BlockStoreError> { - self.block_store - .finalized_transaction_approvals_dbs - .get(&self.txn, &query) - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err))) - } - - fn exists(&self, query: TransactionHash) -> Result { - self.block_store - .finalized_transaction_approvals_dbs - .exists(&self.txn, &query) - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err))) - } -} - -impl DataReader for IndexedLmdbBlockStoreRWTransaction<'_> { - fn read(&self, key: BlockHash) -> Result, BlockStoreError> { - self.block_store.get_single_block(&self.txn, &key) - } - - fn exists(&self, key: BlockHash) -> Result { - self.block_store.block_exists(&self.txn, &key) - } -} - -impl DataReader for IndexedLmdbBlockStoreRWTransaction<'_> { - fn read(&self, key: BlockHash) -> Result, BlockStoreError> { - self.block_store.get_single_block_header(&self.txn, &key) - } - - fn exists(&self, key: BlockHash) -> Result { - self.block_store.block_header_exists(&self.txn, &key) - } -} - -impl DataReader for IndexedLmdbBlockStoreRWTransaction<'_> { - fn read(&self, query: TransactionHash) -> Result, BlockStoreError> { - self.block_store - .execution_result_dbs - .get(&self.txn, &query) - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err))) - } - - fn exists(&self, query: TransactionHash) -> Result { - self.block_store - .execution_result_dbs - .exists(&self.txn, &query) - .map_err(|err| BlockStoreError::InternalStorage(Box::new(err))) - } -} - -impl DataReader> for IndexedLmdbBlockStoreRWTransaction<'_> { - fn read(&self, key: BlockHash) -> Result>, BlockStoreError> { - self.block_store.get_transfers(&self.txn, &key) - } - - fn exists(&self, key: BlockHash) -> Result { - self.block_store.has_transfers(&self.txn, &key) - } -} diff --git a/storage/src/block_store/lmdb/lmdb_block_store.rs b/storage/src/block_store/lmdb/lmdb_block_store.rs index 4e975fb2e1..84bae007de 100644 --- a/storage/src/block_store/lmdb/lmdb_block_store.rs +++ b/storage/src/block_store/lmdb/lmdb_block_store.rs @@ -1,33 +1,39 @@ use std::{ borrow::Cow, - collections::{BTreeSet, HashMap}, + collections::{btree_map, BTreeMap, BTreeSet, HashMap}, path::{Path, PathBuf}, sync::Arc, }; use datasize::DataSize; -use tracing::{debug, error}; +use tracing::{debug, error, info}; use casper_types::{ execution::{execution_result_v1, ExecutionResult, ExecutionResultV1}, - Approval, Block, BlockBody, BlockHash, BlockHeader, BlockSignatures, Digest, Transaction, - TransactionHash, Transfer, + Approval, Block, BlockBody, BlockHash, BlockHeader, BlockSignatures, Digest, EraId, + Transaction, TransactionHash, Transfer, }; use super::{ - lmdb_ext::{LmdbExtError, TransactionExt}, + lmdb_ext::{ + append_by_be_u64_key, append_value_bytesrepr, delete_by_be_u64_key, delete_value_bytesrepr, + get_by_be_u64_key, get_last_by_be_u64_key, put_by_be_u64_key, LmdbExtError, TransactionExt, + WriteTransactionExt, + }, versioned_databases::VersionedDatabases, + DbTableId, }; use crate::block_store::{ error::BlockStoreError, types::{ - ApprovalsHashes, BlockExecutionResults, BlockHashHeightAndEra, BlockTransfers, StateStore, - TransactionFinalizedApprovals, Transfers, + ApprovalsHashes, BlockExecutionResults, BlockHashHeightAndEra, BlockHeight, BlockTransfers, + LatestSwitchBlock, StateStore, StateStoreKey, Tip, TransactionFinalizedApprovals, + Transfers, }, - BlockStoreProvider, BlockStoreTransaction, DataReader, DataWriter, + BlockStoreProvider, BlockStoreTransaction, DataReader, DataWriter, DbRawBytesSpec, }; use lmdb::{ - Database, DatabaseFlags, Environment, EnvironmentFlags, RoTransaction, RwTransaction, + Database, DatabaseFlags, Environment, EnvironmentFlags, RoTransaction, RwCursor, RwTransaction, Transaction as LmdbTransaction, WriteFlags, }; @@ -39,7 +45,7 @@ const STORAGE_DB_FILENAME: &str = "storage.lmdb"; const MAX_TRANSACTIONS: u32 = 5; /// Maximum number of allowed dbs. -const MAX_DB_COUNT: u32 = 17; +const MAX_DB_COUNT: u32 = 20; /// OS-specific lmdb flags. #[cfg(not(target_os = "macos"))] @@ -80,6 +86,16 @@ pub struct LmdbBlockStore { /// The finalized transaction approvals databases. pub(super) finalized_transaction_approvals_dbs: VersionedDatabases>, + /// Disk-backed index of block height to block hash. + #[data_size(skip)] + pub(super) block_height_index_db: Database, + /// Disk-backed index of era ID to switch block hash. + #[data_size(skip)] + pub(super) switch_block_era_id_index_db: Database, + /// Disk-backed index of transaction hash to the hash, height and era of the block containing + /// it. + #[data_size(skip)] + pub(super) transaction_hash_index_db: Database, } impl LmdbBlockStore { @@ -114,6 +130,16 @@ impl LmdbBlockStore { VersionedDatabases::new(&env, "approvals_hashes", "versioned_approvals_hashes") .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + let block_height_index_db = env + .create_db(Some("block_height_index"), DatabaseFlags::empty()) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + let switch_block_era_id_index_db = env + .create_db(Some("switch_block_era_id_index"), DatabaseFlags::empty()) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + let transaction_hash_index_db = env + .create_db(Some("transaction_hash_index"), DatabaseFlags::empty()) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + Ok(Self { root: root_path.to_path_buf(), env: Arc::new(env), @@ -126,9 +152,255 @@ impl LmdbBlockStore { transfer_dbs, state_store_db, finalized_transaction_approvals_dbs, + block_height_index_db, + switch_block_era_id_index_db, + transaction_hash_index_db, }) } + /// Initializes the disk-backed indexes. This operation can be time + /// consuming because it needs to go through all entries in block + /// headers db. If the index has data it assumes that no reindexing + /// is needed. + pub fn init(&mut self) -> Result<(), BlockStoreError> { + let ro_txn = self + .env + .begin_ro_txn() + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + let index_is_empty = ro_txn + .stat(self.block_height_index_db) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))? + .entries() + == 0; + let headers_exist = header_count(&ro_txn, self)? > 0; + ro_txn + .commit() + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + + if headers_exist && index_is_empty { + info!("block store indexes appear to be missing; building them from a full scan"); + self.rebuild_indexes()?; + } + + Ok(()) + } + + /// Performs an unconditional one-off full rebuild of the disk-backed block-height/ + /// switch-block-era-id/transaction-hash indexes, by scanning every block header currently in + /// storage. Exposed for tests; startup code should use [`Self::init`], which only rebuilds + /// when necessary. + #[cfg(test)] + pub fn reindex(&mut self) -> Result<(), BlockStoreError> { + self.rebuild_indexes() + } + + fn rebuild_indexes(&mut self) -> Result<(), BlockStoreError> { + info!("reindexing block store"); + + let mut block_height_index = BTreeMap::new(); + let mut switch_block_era_id_index = BTreeMap::new(); + let mut transaction_hash_index = BTreeMap::new(); + + let mut block_txn = self + .env + .begin_rw_txn() + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + + let total_headers = header_count(&block_txn, self)?; + let progress_step = (total_headers / 20).max(1); + let mut processed: usize = 0; + + // First pass: scan every header through the cursor to build the height/switch-block + // indexes and collect the headers for the second pass below. + let mut headers = Vec::new(); + let mut collect_fn = + |_cursor: &mut RwCursor, block_header: BlockHeader| -> Result<(), BlockStoreError> { + processed += 1; + if processed.is_multiple_of(progress_step) { + info!( + percent_complete = (processed * 100 / total_headers.max(1)), + processed, total_headers, "reindexing block store: scanning headers" + ); + } + + Self::insert_to_block_header_indices( + &mut block_height_index, + &mut switch_block_era_id_index, + &block_header, + )?; + headers.push(block_header); + + Ok(()) + }; + + self.block_header_dbs + .for_each_value_in_current(&mut block_txn, &mut collect_fn)?; + self.block_header_dbs + .for_each_value_in_legacy(&mut block_txn, &mut collect_fn)?; + + // Second pass: the cursor's borrow of `block_txn` has ended, so each block body can be + // read through the transaction we already have open, instead of a fresh one per header. + let total_bodies = headers.len(); + let body_progress_step = (total_bodies / 20).max(1); + for (processed, block_header) in headers.iter().enumerate() { + if processed.is_multiple_of(body_progress_step) { + info!( + percent_complete = (processed * 100 / total_bodies.max(1)), + processed, total_bodies, "reindexing block store: scanning bodies" + ); + } + + let maybe_block_body = self + .block_body_dbs + .get(&block_txn, block_header.body_hash()) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + if let Some(block_body) = &maybe_block_body { + let transaction_hashes = block_transaction_hashes(block_body); + Self::insert_to_transaction_index( + &mut transaction_hash_index, + block_header.block_hash(), + block_header.height(), + block_header.era_id(), + transaction_hashes, + )?; + } + } + + // The scan above makes no changes to the header dbs (unlike `prune`), so this can just be + // rolled back rather than committed. + block_txn.abort(); + + let mut index_txn = self + .env + .begin_rw_txn() + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + + index_txn + .clear_db(self.block_height_index_db) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + index_txn + .clear_db(self.switch_block_era_id_index_db) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + index_txn + .clear_db(self.transaction_hash_index_db) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + + // `block_height_index`/`switch_block_era_id_index`/`transaction_hash_index` are + // `BTreeMap`s, so iterating them yields ascending key order; combined with the `clear_db` + // calls above, this lets us use LMDB's `APPEND` flag to skip the usual B-tree + // search/rebalance per insert (a significant speedup for a full rebuild). This is safe + // because: the two `u64`/`EraId`-keyed indexes use `append_by_be_u64_key`, whose + // big-endian key encoding is specifically chosen so ascending numeric order is ascending + // byte order; and `TransactionHash`'s derived `Ord` (variant tag, then digest bytes) + // matches its `bytesrepr` encoding (tag byte, then raw digest bytes) byte-for-byte. + for (height, block_hash) in block_height_index { + append_by_be_u64_key( + &mut index_txn, + self.block_height_index_db, + height, + &block_hash, + ) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + } + for (era_id, block_hash) in switch_block_era_id_index { + append_by_be_u64_key( + &mut index_txn, + self.switch_block_era_id_index_db, + era_id.value(), + &block_hash, + ) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + } + for (transaction_hash, block_info) in transaction_hash_index { + append_value_bytesrepr( + &mut index_txn, + self.transaction_hash_index_db, + &transaction_hash, + &block_info, + ) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + } + + index_txn + .commit() + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + + info!("block store reindexing complete"); + Ok(()) + } + + /// Inserts the relevant entries to the index. + /// + /// If a duplicate entry is encountered, index is not updated and an error is returned. + fn insert_to_transaction_index( + transaction_hash_index: &mut BTreeMap, + block_hash: BlockHash, + block_height: u64, + era_id: EraId, + transaction_hashes: Vec, + ) -> Result<(), BlockStoreError> { + if let Some(hash) = transaction_hashes.iter().find(|hash| { + transaction_hash_index + .get(hash) + .is_some_and(|old_details| old_details.block_hash != block_hash) + }) { + return Err(BlockStoreError::DuplicateTransaction { + transaction_hash: *hash, + first: transaction_hash_index[hash].block_hash, + second: block_hash, + }); + } + + for hash in transaction_hashes { + transaction_hash_index.insert( + hash, + BlockHashHeightAndEra::new(block_hash, block_height, era_id), + ); + } + + Ok(()) + } + + /// Inserts the relevant entries to the two indices. + /// + /// If a duplicate entry is encountered, neither index is updated and an error is returned. + fn insert_to_block_header_indices( + block_height_index: &mut BTreeMap, + switch_block_era_id_index: &mut BTreeMap, + block_header: &BlockHeader, + ) -> Result<(), BlockStoreError> { + let block_hash = block_header.block_hash(); + if let Some(first) = block_height_index.get(&block_header.height()) { + if *first != block_hash { + return Err(BlockStoreError::DuplicateBlock { + height: block_header.height(), + first: *first, + second: block_hash, + }); + } + } + + if block_header.is_switch_block() { + match switch_block_era_id_index.entry(block_header.era_id()) { + btree_map::Entry::Vacant(entry) => { + let _ = entry.insert(block_hash); + } + btree_map::Entry::Occupied(entry) => { + if *entry.get() != block_hash { + return Err(BlockStoreError::DuplicateEraId { + era_id: block_header.era_id(), + first: *entry.get(), + second: block_hash, + }); + } + } + } + } + + let _ = block_height_index.insert(block_header.height(), block_hash); + Ok(()) + } + /// Write finality signatures. pub fn write_finality_signatures( &self, @@ -555,6 +827,32 @@ impl LmdbBlockStore { } } +fn header_count( + txn: &Tx, + block_store: &LmdbBlockStore, +) -> Result { + let current = txn + .stat(block_store.block_header_dbs.current) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))? + .entries(); + let legacy = txn + .stat(block_store.block_header_dbs.legacy) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))? + .entries(); + Ok(current + legacy) +} + +/// Returns the transaction hashes referenced by a block body. +fn block_transaction_hashes(block_body: &BlockBody) -> Vec { + match block_body { + BlockBody::V1(v1) => v1 + .deploy_and_transfer_hashes() + .map(TransactionHash::from) + .collect(), + BlockBody::V2(v2) => v2.all_transactions().copied().collect(), + } +} + pub(crate) fn new_environment( total_size: usize, root: &Path, @@ -793,24 +1091,605 @@ where } } +enum LmdbBlockStoreIndex { + BlockHeight(IndexPosition), + SwitchBlockEraId(IndexPosition), +} + +enum IndexPosition { + Tip, + Key(K), +} + +enum DataType { + Block, + BlockHeader, + ApprovalsHashes, + BlockSignatures, +} + +impl LmdbBlockStoreTransaction<'_, T> +where + T: LmdbTransaction, +{ + fn block_hash_from_index( + &self, + index: LmdbBlockStoreIndex, + ) -> Result, BlockStoreError> { + let result = match index { + LmdbBlockStoreIndex::BlockHeight(position) => match position { + IndexPosition::Tip => get_last_by_be_u64_key::<_, BlockHash>( + &self.txn, + self.block_store.block_height_index_db, + ), + IndexPosition::Key(height) => get_by_be_u64_key::<_, BlockHash>( + &self.txn, + self.block_store.block_height_index_db, + height, + ), + }, + LmdbBlockStoreIndex::SwitchBlockEraId(position) => match position { + IndexPosition::Tip => get_last_by_be_u64_key::<_, BlockHash>( + &self.txn, + self.block_store.switch_block_era_id_index_db, + ), + IndexPosition::Key(era_id) => get_by_be_u64_key::<_, BlockHash>( + &self.txn, + self.block_store.switch_block_era_id_index_db, + era_id.value(), + ), + }, + }; + result.map_err(|err| BlockStoreError::InternalStorage(Box::new(err))) + } + + fn read_block_indexed( + &self, + index: LmdbBlockStoreIndex, + ) -> Result, BlockStoreError> { + match self.block_hash_from_index(index)? { + Some(block_hash) => self.block_store.get_single_block(&self.txn, &block_hash), + None => Ok(None), + } + } + + fn read_block_header_indexed( + &self, + index: LmdbBlockStoreIndex, + ) -> Result, BlockStoreError> { + match self.block_hash_from_index(index)? { + Some(block_hash) => self + .block_store + .get_single_block_header(&self.txn, &block_hash), + None => Ok(None), + } + } + + fn read_block_signatures_indexed( + &self, + index: LmdbBlockStoreIndex, + ) -> Result, BlockStoreError> { + match self.block_hash_from_index(index)? { + Some(block_hash) => self + .block_store + .get_block_signatures(&self.txn, &block_hash), + None => Ok(None), + } + } + + fn read_approvals_hashes_indexed( + &self, + index: LmdbBlockStoreIndex, + ) -> Result, BlockStoreError> { + match self.block_hash_from_index(index)? { + Some(block_hash) => self + .block_store + .read_approvals_hashes(&self.txn, &block_hash), + None => Ok(None), + } + } + + fn contains_data_indexed( + &self, + index: LmdbBlockStoreIndex, + data_type: DataType, + ) -> Result { + match self.block_hash_from_index(index)? { + Some(block_hash) => match data_type { + DataType::Block => self.block_store.block_exists(&self.txn, &block_hash), + DataType::BlockHeader => { + self.block_store.block_header_exists(&self.txn, &block_hash) + } + DataType::ApprovalsHashes => self + .block_store + .approvals_hashes_exist(&self.txn, &block_hash), + DataType::BlockSignatures => self + .block_store + .block_signatures_exist(&self.txn, &block_hash), + }, + None => Ok(false), + } + } + + /// Returns the height of the switch block for the given era, if known. + pub fn get_switch_block_height(&self, era_id: EraId) -> Result, BlockStoreError> { + let index = LmdbBlockStoreIndex::SwitchBlockEraId(IndexPosition::Key(era_id)); + match self.block_hash_from_index(index)? { + Some(block_hash) => { + let maybe_header: Option = self.read(block_hash)?; + Ok(maybe_header.map(|header| header.height())) + } + None => Ok(None), + } + } +} + +impl DataReader for LmdbBlockStoreTransaction<'_, T> +where + T: LmdbTransaction, +{ + fn read(&self, key: BlockHeight) -> Result, BlockStoreError> { + self.read_block_indexed(LmdbBlockStoreIndex::BlockHeight(IndexPosition::Key(key))) + } + + fn exists(&self, key: BlockHeight) -> Result { + self.contains_data_indexed( + LmdbBlockStoreIndex::BlockHeight(IndexPosition::Key(key)), + DataType::Block, + ) + } +} + +impl DataReader for LmdbBlockStoreTransaction<'_, T> +where + T: LmdbTransaction, +{ + fn read(&self, key: BlockHeight) -> Result, BlockStoreError> { + self.read_block_header_indexed(LmdbBlockStoreIndex::BlockHeight(IndexPosition::Key(key))) + } + + fn exists(&self, key: BlockHeight) -> Result { + self.contains_data_indexed( + LmdbBlockStoreIndex::BlockHeight(IndexPosition::Key(key)), + DataType::BlockHeader, + ) + } +} + +impl DataReader for LmdbBlockStoreTransaction<'_, T> +where + T: LmdbTransaction, +{ + fn read(&self, key: BlockHeight) -> Result, BlockStoreError> { + self.read_approvals_hashes_indexed(LmdbBlockStoreIndex::BlockHeight(IndexPosition::Key( + key, + ))) + } + + fn exists(&self, key: BlockHeight) -> Result { + self.contains_data_indexed( + LmdbBlockStoreIndex::BlockHeight(IndexPosition::Key(key)), + DataType::ApprovalsHashes, + ) + } +} + +impl DataReader for LmdbBlockStoreTransaction<'_, T> +where + T: LmdbTransaction, +{ + fn read(&self, key: BlockHeight) -> Result, BlockStoreError> { + self.read_block_signatures_indexed(LmdbBlockStoreIndex::BlockHeight(IndexPosition::Key( + key, + ))) + } + + fn exists(&self, key: BlockHeight) -> Result { + self.contains_data_indexed( + LmdbBlockStoreIndex::BlockHeight(IndexPosition::Key(key)), + DataType::BlockSignatures, + ) + } +} + +/// Retrieves single switch block by era ID by looking it up in the index and returning it. +impl DataReader for LmdbBlockStoreTransaction<'_, T> +where + T: LmdbTransaction, +{ + fn read(&self, key: EraId) -> Result, BlockStoreError> { + self.read_block_indexed(LmdbBlockStoreIndex::SwitchBlockEraId(IndexPosition::Key( + key, + ))) + } + + fn exists(&self, key: EraId) -> Result { + self.contains_data_indexed( + LmdbBlockStoreIndex::SwitchBlockEraId(IndexPosition::Key(key)), + DataType::Block, + ) + } +} + +/// Retrieves single switch block header by era ID by looking it up in the index and returning +/// it. +impl DataReader for LmdbBlockStoreTransaction<'_, T> +where + T: LmdbTransaction, +{ + fn read(&self, key: EraId) -> Result, BlockStoreError> { + self.read_block_header_indexed(LmdbBlockStoreIndex::SwitchBlockEraId(IndexPosition::Key( + key, + ))) + } + + fn exists(&self, key: EraId) -> Result { + self.contains_data_indexed( + LmdbBlockStoreIndex::SwitchBlockEraId(IndexPosition::Key(key)), + DataType::BlockHeader, + ) + } +} + +impl DataReader for LmdbBlockStoreTransaction<'_, T> +where + T: LmdbTransaction, +{ + fn read(&self, key: EraId) -> Result, BlockStoreError> { + self.read_approvals_hashes_indexed(LmdbBlockStoreIndex::SwitchBlockEraId( + IndexPosition::Key(key), + )) + } + + fn exists(&self, key: EraId) -> Result { + self.contains_data_indexed( + LmdbBlockStoreIndex::SwitchBlockEraId(IndexPosition::Key(key)), + DataType::ApprovalsHashes, + ) + } +} + +impl DataReader for LmdbBlockStoreTransaction<'_, T> +where + T: LmdbTransaction, +{ + fn read(&self, key: EraId) -> Result, BlockStoreError> { + self.read_block_signatures_indexed(LmdbBlockStoreIndex::SwitchBlockEraId( + IndexPosition::Key(key), + )) + } + + fn exists(&self, key: EraId) -> Result { + self.contains_data_indexed( + LmdbBlockStoreIndex::SwitchBlockEraId(IndexPosition::Key(key)), + DataType::BlockSignatures, + ) + } +} + +impl DataReader for LmdbBlockStoreTransaction<'_, T> +where + T: LmdbTransaction, +{ + fn read(&self, _key: Tip) -> Result, BlockStoreError> { + self.read_block_header_indexed(LmdbBlockStoreIndex::BlockHeight(IndexPosition::Tip)) + } + + fn exists(&self, _key: Tip) -> Result { + self.contains_data_indexed( + LmdbBlockStoreIndex::BlockHeight(IndexPosition::Tip), + DataType::BlockHeader, + ) + } +} + +impl DataReader for LmdbBlockStoreTransaction<'_, T> +where + T: LmdbTransaction, +{ + fn read(&self, _key: Tip) -> Result, BlockStoreError> { + self.read_block_indexed(LmdbBlockStoreIndex::BlockHeight(IndexPosition::Tip)) + } + + fn exists(&self, _key: Tip) -> Result { + self.contains_data_indexed( + LmdbBlockStoreIndex::BlockHeight(IndexPosition::Tip), + DataType::Block, + ) + } +} + +impl DataReader for LmdbBlockStoreTransaction<'_, T> +where + T: LmdbTransaction, +{ + fn read(&self, _key: LatestSwitchBlock) -> Result, BlockStoreError> { + self.read_block_header_indexed(LmdbBlockStoreIndex::SwitchBlockEraId(IndexPosition::Tip)) + } + + fn exists(&self, _key: LatestSwitchBlock) -> Result { + self.contains_data_indexed( + LmdbBlockStoreIndex::SwitchBlockEraId(IndexPosition::Tip), + DataType::BlockHeader, + ) + } +} + +impl DataReader for LmdbBlockStoreTransaction<'_, T> +where + T: LmdbTransaction, +{ + fn read(&self, key: TransactionHash) -> Result, BlockStoreError> { + self.txn + .get_value_bytesrepr(self.block_store.transaction_hash_index_db, &key) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err))) + } + + fn exists(&self, key: TransactionHash) -> Result { + self.txn + .value_exists_bytesrepr(self.block_store.transaction_hash_index_db, &key) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err))) + } +} + +impl DataReader> for LmdbBlockStoreTransaction<'_, T> +where + T: LmdbTransaction, +{ + fn read(&self, StateStoreKey(key): StateStoreKey) -> Result>, BlockStoreError> { + self.block_store.read_state_store(&self.txn, &key) + } + + fn exists(&self, StateStoreKey(key): StateStoreKey) -> Result { + self.block_store.state_store_key_exists(&self.txn, &key) + } +} + +impl DataReader<(DbTableId, Vec), DbRawBytesSpec> for LmdbBlockStoreTransaction<'_, T> +where + T: LmdbTransaction, +{ + fn read( + &self, + (id, key): (DbTableId, Vec), + ) -> Result, BlockStoreError> { + if key.is_empty() { + return Ok(None); + } + let store = self.block_store; + let res = match id { + DbTableId::BlockHeader => store.block_header_dbs.get_raw(&self.txn, &key), + DbTableId::BlockBody => store.block_body_dbs.get_raw(&self.txn, &key), + DbTableId::ApprovalsHashes => store.approvals_hashes_dbs.get_raw(&self.txn, &key), + DbTableId::BlockMetadata => store.block_metadata_dbs.get_raw(&self.txn, &key), + DbTableId::Transaction => store.transaction_dbs.get_raw(&self.txn, &key), + DbTableId::ExecutionResult => store.execution_result_dbs.get_raw(&self.txn, &key), + DbTableId::Transfer => store.transfer_dbs.get_raw(&self.txn, &key), + DbTableId::FinalizedTransactionApprovals => store + .finalized_transaction_approvals_dbs + .get_raw(&self.txn, &key), + }; + res.map_err(|err| BlockStoreError::InternalStorage(Box::new(err))) + } + + fn exists(&self, key: (DbTableId, Vec)) -> Result { + self.read(key).map(|res| res.is_some()) + } +} + +impl<'t> LmdbBlockStoreTransaction<'t, RwTransaction<'t>> { + /// Check if the block height index can be updated. + fn should_update_block_height_index( + &self, + block_height: u64, + block_hash: &BlockHash, + ) -> Result { + match get_by_be_u64_key::<_, BlockHash>( + &self.txn, + self.block_store.block_height_index_db, + block_height, + ) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))? + { + // There is a block in the index at this height + Some(first) if first != *block_hash => Err(BlockStoreError::DuplicateBlock { + height: block_height, + first, + second: *block_hash, + }), + // Same value already in index, no need to update it. + Some(_) => Ok(false), + // Value not in index, update. + None => Ok(true), + } + } + + /// Check if the switch block index can be updated. + fn should_update_switch_block_index( + &self, + block_header: &BlockHeader, + ) -> Result { + if !block_header.is_switch_block() { + return Ok(false); + } + let era_id = block_header.era_id(); + match get_by_be_u64_key::<_, BlockHash>( + &self.txn, + self.block_store.switch_block_era_id_index_db, + era_id.value(), + ) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))? + { + Some(entry) if entry != block_header.block_hash() => { + Err(BlockStoreError::DuplicateEraId { + era_id, + first: entry, + second: block_header.block_hash(), + }) + } + // already in index, no need to update. + Some(_) => Ok(false), + // not in the index, update. + None => Ok(true), + } + } + + // Check if the transaction hash index can be updated. + fn should_update_transaction_hash_index( + &self, + transaction_hashes: &[TransactionHash], + block_hash: &BlockHash, + ) -> Result { + for hash in transaction_hashes { + if let Some(old_details) = self + .txn + .get_value_bytesrepr::<_, BlockHashHeightAndEra>( + self.block_store.transaction_hash_index_db, + hash, + ) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))? + { + if old_details.block_hash != *block_hash { + return Err(BlockStoreError::DuplicateTransaction { + transaction_hash: *hash, + first: old_details.block_hash, + second: *block_hash, + }); + } + } + } + Ok(true) + } +} + impl<'t> DataWriter for LmdbBlockStoreTransaction<'t, RwTransaction<'t>> { /// Writes a block to storage. + /// + /// Returns `Ok(true)` if the block has been successfully written, `Ok(false)` if a part of it + /// couldn't be written because it already existed, and `Err(_)` if there was an error. fn write(&mut self, data: &Block) -> Result { - self.block_store.write_block(&mut self.txn, data) + let block_header = data.clone_header(); + let block_hash = data.hash(); + let block_height = data.height(); + let era_id = data.era_id(); + let transaction_hashes: Vec = match &data { + Block::V1(v1) => v1 + .deploy_and_transfer_hashes() + .map(TransactionHash::from) + .collect(), + Block::V2(v2) => v2.all_transactions().copied().collect(), + }; + + let update_height_index = + self.should_update_block_height_index(block_height, block_hash)?; + let update_switch_block_index = self.should_update_switch_block_index(&block_header)?; + let update_transaction_hash_index = + self.should_update_transaction_hash_index(&transaction_hashes, block_hash)?; + + let key = self.block_store.write_block(&mut self.txn, data)?; + + if update_height_index { + put_by_be_u64_key( + &mut self.txn, + self.block_store.block_height_index_db, + block_height, + block_hash, + ) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + } + + if update_switch_block_index { + put_by_be_u64_key( + &mut self.txn, + self.block_store.switch_block_era_id_index_db, + era_id.value(), + block_hash, + ) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + } + + if update_transaction_hash_index { + for hash in transaction_hashes { + self.txn + .put_value_bytesrepr( + self.block_store.transaction_hash_index_db, + &hash, + &BlockHashHeightAndEra::new(*block_hash, block_height, era_id), + true, + ) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + } + } + + Ok(key) } fn delete(&mut self, key: BlockHash) -> Result<(), BlockStoreError> { - let maybe_block = self.block_store.get_single_block_header(&self.txn, &key)?; + let maybe_block = self.block_store.get_single_block(&self.txn, &key)?; + + if let Some(block) = maybe_block { + let transaction_hashes: Vec = match &block { + Block::V1(v1) => v1 + .deploy_and_transfer_hashes() + .map(TransactionHash::from) + .collect(), + Block::V2(v2) => v2.all_transactions().copied().collect(), + }; - if let Some(block_header) = maybe_block { self.block_store.delete_block_header(&mut self.txn, &key)?; + + /* + TODO: currently we don't delete the block body since other blocks may reference it. self.block_store - .delete_block_body(&mut self.txn, block_header.body_hash())?; + .delete_block_body(&mut self.txn, block.body_hash())?; + */ + + delete_by_be_u64_key( + &mut self.txn, + self.block_store.block_height_index_db, + block.height(), + ) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + + if block.is_switch_block() { + delete_by_be_u64_key( + &mut self.txn, + self.block_store.switch_block_era_id_index_db, + block.era_id().value(), + ) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + } + + for hash in transaction_hashes { + delete_value_bytesrepr( + &mut self.txn, + self.block_store.transaction_hash_index_db, + &hash, + ) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + } + + self.block_store + .delete_finality_signatures(&mut self.txn, &key)?; } Ok(()) } } +impl<'t> DataWriter for LmdbBlockStoreTransaction<'t, RwTransaction<'t>> { + /// Not supported: a block body is always written together with its header, as part of + /// writing a whole `Block` (see the `DataWriter` impl above). + fn write(&mut self, _data: &BlockBody) -> Result { + Err(BlockStoreError::UnsupportedOperation) + } + + /// Deletes a block body by its hash. Callers are responsible for only doing so once no + /// retained block header still references this body hash. + fn delete(&mut self, key: Digest) -> Result<(), BlockStoreError> { + self.block_store.delete_block_body(&mut self.txn, &key) + } +} + impl<'t> DataWriter for LmdbBlockStoreTransaction<'t, RwTransaction<'t>> { @@ -840,11 +1719,62 @@ impl<'t> DataWriter impl<'t> DataWriter for LmdbBlockStoreTransaction<'t, RwTransaction<'t>> { fn write(&mut self, data: &BlockHeader) -> Result { - self.block_store.write_block_header(&mut self.txn, data) + let block_hash = data.block_hash(); + let block_height = data.height(); + let era_id = data.era_id(); + + let update_height_index = + self.should_update_block_height_index(block_height, &block_hash)?; + let update_switch_block_index = self.should_update_switch_block_index(data)?; + + let key = self.block_store.write_block_header(&mut self.txn, data)?; + + if update_height_index { + put_by_be_u64_key( + &mut self.txn, + self.block_store.block_height_index_db, + block_height, + &block_hash, + ) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + } + + if update_switch_block_index { + put_by_be_u64_key( + &mut self.txn, + self.block_store.switch_block_era_id_index_db, + era_id.value(), + &block_hash, + ) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + } + + Ok(key) } fn delete(&mut self, key: BlockHash) -> Result<(), BlockStoreError> { - self.block_store.delete_block_header(&mut self.txn, &key) + let maybe_block_header = self.block_store.get_single_block_header(&self.txn, &key)?; + + if let Some(block_header) = maybe_block_header { + self.block_store.delete_block_header(&mut self.txn, &key)?; + + if block_header.is_switch_block() { + delete_by_be_u64_key( + &mut self.txn, + self.block_store.switch_block_era_id_index_db, + block_header.era_id().value(), + ) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + } + + delete_by_be_u64_key( + &mut self.txn, + self.block_store.block_height_index_db, + block_header.height(), + ) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + } + Ok(()) } } @@ -922,7 +1852,13 @@ impl<'t> DataWriter &mut self, data: &BlockExecutionResults, ) -> Result { + let transaction_hashes: Vec = data.exec_results.keys().copied().collect(); let block_hash = data.block_info.block_hash; + let block_height = data.block_info.block_height; + let era_id = data.block_info.era_id; + + let update_transaction_hash_index = + self.should_update_transaction_hash_index(&transaction_hashes, &block_hash)?; let _ = self.block_store.write_execution_results( &mut self.txn, @@ -930,6 +1866,19 @@ impl<'t> DataWriter data.exec_results.clone(), )?; + if update_transaction_hash_index { + for hash in transaction_hashes { + self.txn + .put_value_bytesrepr( + self.block_store.transaction_hash_index_db, + &hash, + &BlockHashHeightAndEra::new(block_hash, block_height, era_id), + true, + ) + .map_err(|err| BlockStoreError::InternalStorage(Box::new(err)))?; + } + } + Ok(data.block_info) } @@ -942,3 +1891,221 @@ impl<'t> DataWriter Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use casper_types::{ + testing::TestRng, BlockHeaderV2, EraEndV2, ProtocolVersion, PublicKey, SecretKey, Timestamp, + }; + use once_cell::sync::OnceCell; + use rand::Rng; + use tempfile::TempDir; + + /// Number of headers to write: deliberately > 256 so that, were the disk-backed indexes ever + /// keyed by plain little-endian `bytesrepr` bytes instead of the big-endian encoding, the + /// `APPEND`-based bulk write in `reindex` would violate LMDB's required key order (since + /// little-endian byte-lexicographic order diverges from numeric order once values exceed a + /// single byte) and fail loudly rather than silently produce a wrong index. + const HEADER_COUNT: u64 = 300; + + fn header_at_height(rng: &mut TestRng, height: u64, proposer: &PublicKey) -> BlockHeader { + let is_switch_block = height % 10 == 9; + let era_id = EraId::new(height / 10); + let era_end = is_switch_block.then(|| EraEndV2::random(rng)); + BlockHeader::V2(BlockHeaderV2::new( + BlockHash::random(rng), + Digest::random(rng), + Digest::hash(height.to_le_bytes()), + rng.gen(), + Digest::random(rng), + era_end, + Timestamp::now(), + era_id, + height, + ProtocolVersion::V1_0_0, + proposer.clone(), + 1, + None, + OnceCell::new(), + )) + } + + #[test] + fn reindex_rebuilds_disk_backed_indexes_via_append() { + let rng = &mut TestRng::new(); + let tempdir = TempDir::new().expect("should create tempdir"); + let mut store = + LmdbBlockStore::new(tempdir.path(), 64 * 1024 * 1024).expect("should create store"); + + let secret_key = SecretKey::random(rng); + let proposer = PublicKey::from(&secret_key); + + let mut headers = Vec::new(); + { + let mut rw_txn = store.checkout_rw().expect("should checkout rw"); + for height in 0..HEADER_COUNT { + let header = header_at_height(rng, height, &proposer); + let _ = DataWriter::::write(&mut rw_txn, &header) + .expect("should write header"); + headers.push(header); + } + rw_txn.commit().expect("should commit"); + } + + store.reindex().expect("reindex should succeed"); + + let ro_txn = store.checkout_ro().expect("should checkout ro"); + + // Spot-check a handful of heights, including ones that require crossing the + // little-endian single-byte boundary (e.g. 255 -> 256) to catch ordering bugs. + for &height in &[0u64, 1, 254, 255, 256, 257, HEADER_COUNT - 1] { + let expected_hash = headers[height as usize].block_hash(); + let actual: Option = ro_txn.read(height).expect("read by height"); + assert_eq!( + actual.expect("header should exist").block_hash(), + expected_hash, + "wrong header at height {height}" + ); + } + + // Tip should be the highest height. + let tip: Option = ro_txn.read(Tip).expect("read tip"); + assert_eq!( + tip.expect("tip should exist").height(), + HEADER_COUNT - 1, + "tip should be the highest height" + ); + + // Switch blocks (height % 10 == 9) should be resolvable by era ID. + for &height in &[9u64, 99, 259, HEADER_COUNT - 1] { + assert_eq!( + height % 10, + 9, + "test bug: {height} is not a switch block height" + ); + let era_id = EraId::new(height / 10); + let expected_hash = headers[height as usize].block_hash(); + let actual: Option = ro_txn.read(era_id).expect("read by era id"); + assert_eq!( + actual + .expect("switch block header should exist") + .block_hash(), + expected_hash, + "wrong switch block header for era {era_id}" + ); + } + + // Latest switch block should be the highest-height header with `is_switch_block()` set + // (derived from `headers` directly, rather than hardcoded, to avoid off-by-one mistakes). + let latest_switch_block_height = headers + .iter() + .filter(|header| header.is_switch_block()) + .map(|header| header.height()) + .max() + .expect("should have at least one switch block"); + let latest_switch: Option = + DataReader::::read(&ro_txn, LatestSwitchBlock) + .expect("read latest switch block"); + assert_eq!( + latest_switch + .expect("latest switch block should exist") + .height(), + latest_switch_block_height, + "wrong latest switch block" + ); + } + + #[test] + fn init_builds_index_when_headers_exist_but_index_is_empty() { + let rng = &mut TestRng::new(); + let tempdir = TempDir::new().expect("should create tempdir"); + let mut store = + LmdbBlockStore::new(tempdir.path(), 64 * 1024 * 1024).expect("should create store"); + + let secret_key = SecretKey::random(rng); + let proposer = PublicKey::from(&secret_key); + + // Write a header directly via a raw transaction, bypassing the index-maintaining + // `DataWriter` impl -- simulating a migration from a binary version that didn't yet + // maintain these disk-backed indexes. + let header = header_at_height(rng, 0, &proposer); + { + let mut txn = store.env.begin_rw_txn().expect("should begin rw txn"); + let _ = store + .write_block_header(&mut txn, &header) + .expect("should write header"); + txn.commit().expect("should commit"); + } + + // The index hasn't been told about this header yet. + { + let ro_txn = store.checkout_ro().expect("should checkout ro"); + let by_height: Option = ro_txn.read(0u64).expect("read by height"); + assert!(by_height.is_none(), "index should not exist yet"); + } + + store.init().expect("init should succeed"); + + let ro_txn = store.checkout_ro().expect("should checkout ro"); + let by_height: Option = ro_txn.read(0u64).expect("read by height"); + assert_eq!( + by_height + .expect("header should be indexed after init") + .block_hash(), + header.block_hash(), + "init should have built the height index from the existing headers" + ); + } + + #[test] + fn init_does_not_rebuild_an_already_populated_index() { + let rng = &mut TestRng::new(); + let tempdir = TempDir::new().expect("should create tempdir"); + let mut store = + LmdbBlockStore::new(tempdir.path(), 64 * 1024 * 1024).expect("should create store"); + + let secret_key = SecretKey::random(rng); + let proposer = PublicKey::from(&secret_key); + + // Write two headers through the normal, index-maintaining path. + let headers: Vec = (0..2) + .map(|height| header_at_height(rng, height, &proposer)) + .collect(); + { + let mut rw_txn = store.checkout_rw().expect("should checkout rw"); + for header in &headers { + let _ = DataWriter::::write(&mut rw_txn, header) + .expect("should write header"); + } + rw_txn.commit().expect("should commit"); + } + + // Directly corrupt the height index by deleting the entry for height 0, without touching + // the header itself -- an inconsistency that only a full rebuild would fix. + { + let mut index_txn = store.env.begin_rw_txn().expect("should begin rw txn"); + delete_by_be_u64_key(&mut index_txn, store.block_height_index_db, 0) + .expect("should delete index entry"); + index_txn.commit().expect("should commit"); + } + + store.init().expect("init should succeed"); + + // Since the index wasn't empty (height 1's entry is still present), `init` must have + // skipped rebuilding it -- so the deleted entry for height 0 stays missing. + let ro_txn = store.checkout_ro().expect("should checkout ro"); + let by_height_0: Option = ro_txn.read(0u64).expect("read by height"); + assert!( + by_height_0.is_none(), + "init should not have rebuilt an already-populated index" + ); + let by_height_1: Option = ro_txn.read(1u64).expect("read by height"); + assert_eq!( + by_height_1 + .expect("height 1 should still be indexed") + .block_hash(), + headers[1].block_hash() + ); + } +} diff --git a/storage/src/block_store/lmdb/lmdb_ext.rs b/storage/src/block_store/lmdb/lmdb_ext.rs index 3f965f1270..6da614a3c8 100644 --- a/storage/src/block_store/lmdb/lmdb_ext.rs +++ b/storage/src/block_store/lmdb/lmdb_ext.rs @@ -12,7 +12,8 @@ use std::{any::TypeId, collections::BTreeSet}; -use lmdb::{Database, RwTransaction, Transaction, WriteFlags}; +use lmdb::{Cursor, Database, RwTransaction, Transaction, WriteFlags}; +use lmdb_sys::MDB_LAST; use serde::de::DeserializeOwned; #[cfg(test)] use serde::Serialize; @@ -410,6 +411,128 @@ pub(super) fn deserialize_bytesrepr(raw: &[u8]) -> Resul } } +/// Converts a `u64` into big-endian bytes suitable for use as an LMDB key where the natural +/// byte-lexicographic ordering of keys must match the numeric ordering of the values. +/// +/// `bytesrepr` encodes integers little-endian, which does not have this property, so keys for +/// indexes that need ordered ("get last") lookups (e.g. the tip of the block-height index) are +/// stored using this encoding instead of `serialize_bytesrepr`. +#[inline(always)] +fn be_key_bytes(key: u64) -> [u8; 8] { + key.to_be_bytes() +} + +/// Helper function to load a `bytesrepr`-serialized value keyed by a big-endian-encoded `u64`. +pub(super) fn get_by_be_u64_key( + txn: &Tx, + db: Database, + key: u64, +) -> Result, LmdbExtError> { + match txn.get(db, &be_key_bytes(key)) { + Ok(raw) => deserialize_bytesrepr(raw).map(Some), + Err(lmdb::Error::NotFound) => Ok(None), + Err(err) => Err(err.into()), + } +} + +/// Helper function to write a `bytesrepr`-serialized value keyed by a big-endian-encoded `u64`. +pub(super) fn put_by_be_u64_key( + txn: &mut RwTransaction, + db: Database, + key: u64, + value: &V, +) -> Result<(), LmdbExtError> { + let serialized_value = serialize_bytesrepr(value)?; + txn.put( + db, + &be_key_bytes(key), + &serialized_value, + WriteFlags::empty(), + )?; + Ok(()) +} + +/// Deletes the value keyed by a big-endian-encoded `u64`, tolerating a missing entry. +pub(super) fn delete_by_be_u64_key( + txn: &mut RwTransaction, + db: Database, + key: u64, +) -> Result<(), LmdbExtError> { + match txn.del(db, &be_key_bytes(key), None) { + Ok(()) | Err(lmdb::Error::NotFound) => Ok(()), + Err(err) => Err(err.into()), + } +} + +/// Deletes the value keyed by a `bytesrepr`-serialized key, tolerating a missing entry. +pub(super) fn delete_value_bytesrepr( + txn: &mut RwTransaction, + db: Database, + key: &K, +) -> Result<(), LmdbExtError> { + let serialized_key = serialize_bytesrepr(key)?; + match txn.del(db, &serialized_key, None) { + Ok(()) | Err(lmdb::Error::NotFound) => Ok(()), + Err(err) => Err(err.into()), + } +} + +/// Helper function to write a `bytesrepr`-serialized value keyed by a big-endian-encoded `u64`, +/// using LMDB's `APPEND` flag for fast bulk loading. +/// +/// Callers MUST insert keys in strictly increasing order (e.g. by iterating a `BTreeMap` in its +/// natural order) and the key must be greater than any key already in the database (e.g. the +/// database was just cleared) — otherwise LMDB returns an error rather than corrupting data. +pub(super) fn append_by_be_u64_key( + txn: &mut RwTransaction, + db: Database, + key: u64, + value: &V, +) -> Result<(), LmdbExtError> { + let serialized_value = serialize_bytesrepr(value)?; + txn.put( + db, + &be_key_bytes(key), + &serialized_value, + WriteFlags::APPEND, + )?; + Ok(()) +} + +/// Helper function to write a `bytesrepr`-serialized value keyed by a `bytesrepr`-serialized key, +/// using LMDB's `APPEND` flag for fast bulk loading. +/// +/// Callers MUST insert keys in strictly increasing order (by the key type's `Ord` impl, which +/// must agree with the byte-lexicographic order of its `bytesrepr` encoding — e.g. by iterating a +/// `BTreeMap` in its natural order) and the key must be greater than any key already in the +/// database (e.g. the database was just cleared) — otherwise LMDB returns an error rather than +/// corrupting data. +pub(super) fn append_value_bytesrepr( + txn: &mut RwTransaction, + db: Database, + key: &K, + value: &V, +) -> Result<(), LmdbExtError> { + let serialized_key = serialize_bytesrepr(key)?; + let serialized_value = serialize_bytesrepr(value)?; + txn.put(db, &serialized_key, &serialized_value, WriteFlags::APPEND)?; + Ok(()) +} + +/// Returns the `bytesrepr`-deserialized value associated with the highest big-endian-encoded +/// `u64` key in the database, if any (i.e. the equivalent of `BTreeMap::values().last()`). +pub(super) fn get_last_by_be_u64_key( + txn: &Tx, + db: Database, +) -> Result, LmdbExtError> { + let cursor = txn.open_ro_cursor(db)?; + match cursor.get(None, None, MDB_LAST) { + Ok((_, raw)) => deserialize_bytesrepr(raw).map(Some), + Err(lmdb::Error::NotFound) => Ok(None), + Err(err) => Err(err.into()), + } +} + /// Serializes into a buffer. #[inline(always)] pub(super) fn serialize_bytesrepr(value: &T) -> Result, LmdbExtError> { diff --git a/storage/src/block_store/lmdb/mod.rs b/storage/src/block_store/lmdb/mod.rs index 8c43d7b446..98892c5556 100644 --- a/storage/src/block_store/lmdb/mod.rs +++ b/storage/src/block_store/lmdb/mod.rs @@ -1,12 +1,9 @@ mod lmdb_ext; -mod temp_map; mod versioned_databases; -mod indexed_lmdb_block_store; mod lmdb_block_store; use core::convert::TryFrom; -pub use indexed_lmdb_block_store::IndexedLmdbBlockStore; pub use lmdb_block_store::LmdbBlockStore; #[cfg(test)] diff --git a/storage/src/block_store/lmdb/temp_map.rs b/storage/src/block_store/lmdb/temp_map.rs deleted file mode 100644 index 9e26afef9e..0000000000 --- a/storage/src/block_store/lmdb/temp_map.rs +++ /dev/null @@ -1,70 +0,0 @@ -use std::collections::BTreeMap; - -enum EntryState { - Deleted, - Occupied(V), -} - -/// A wrapper over a BTreeMap that stores changes to the backing map only temporarily. -/// The backing map will not be altered until the temporary changes are committed. -pub(crate) struct TempMap<'a, K, V: 'a> { - base_index: &'a mut BTreeMap, - new_index: BTreeMap>, -} - -impl<'a, K, V> TempMap<'a, K, V> -where - K: Ord, - V: 'a + Copy, -{ - /// Creates a new temporary map that is backed by a BTreeMap - pub(crate) fn new(base_index: &'a mut BTreeMap) -> Self { - Self { - base_index, - new_index: BTreeMap::>::new(), - } - } - - /// Reads the value contained in the map at the specified key. - pub(crate) fn get(&self, key: &K) -> Option { - if let Some(state) = self.new_index.get(key) { - match state { - EntryState::Occupied(val) => Some(*val), - EntryState::Deleted => None, - } - } else { - self.base_index.get(key).copied() - } - } - - /// Checks if a key exists in this map. - pub(crate) fn contains_key(&self, key: &K) -> bool { - if self.new_index.contains_key(key) { - true - } else { - self.base_index.contains_key(key) - } - } - - /// Sets the value at the specified key index. - pub(crate) fn insert(&mut self, key: K, val: V) { - self.new_index.insert(key, EntryState::Occupied(val)); - } - - /// Removes the value from the map. - pub(crate) fn remove(&mut self, key: K) { - if self.contains_key(&key) { - self.new_index.insert(key, EntryState::Deleted); - } - } - - /// Saves temporary changes to the backing map. - pub(crate) fn commit(self) { - for (key, val) in self.new_index { - match val { - EntryState::Occupied(val) => self.base_index.insert(key, val), - EntryState::Deleted => self.base_index.remove(&key), - }; - } - } -} diff --git a/storage/src/block_store/types/block_hash_height_and_era.rs b/storage/src/block_store/types/block_hash_height_and_era.rs index 8aa5e64431..2b6bd2b385 100644 --- a/storage/src/block_store/types/block_hash_height_and_era.rs +++ b/storage/src/block_store/types/block_hash_height_and_era.rs @@ -4,7 +4,10 @@ use rand::Rng; #[cfg(test)] use casper_types::testing::TestRng; -use casper_types::{BlockHash, BlockHashAndHeight, EraId}; +use casper_types::{ + bytesrepr::{self, FromBytes, ToBytes}, + BlockHash, BlockHashAndHeight, EraId, +}; /// Aggregates block identifying information. #[derive(Clone, Copy, Debug, DataSize)] @@ -43,3 +46,35 @@ impl From for BlockHashAndHeight { BlockHashAndHeight::new(bhhe.block_hash, bhhe.block_height) } } + +impl ToBytes for BlockHashHeightAndEra { + fn to_bytes(&self) -> Result, bytesrepr::Error> { + let mut buffer = bytesrepr::allocate_buffer(self)?; + buffer.extend(self.block_hash.to_bytes()?); + buffer.extend(self.block_height.to_bytes()?); + buffer.extend(self.era_id.to_bytes()?); + Ok(buffer) + } + + fn serialized_length(&self) -> usize { + self.block_hash.serialized_length() + + self.block_height.serialized_length() + + self.era_id.serialized_length() + } +} + +impl FromBytes for BlockHashHeightAndEra { + fn from_bytes(bytes: &[u8]) -> Result<(Self, &[u8]), bytesrepr::Error> { + let (block_hash, remainder) = BlockHash::from_bytes(bytes)?; + let (block_height, remainder) = u64::from_bytes(remainder)?; + let (era_id, remainder) = EraId::from_bytes(remainder)?; + Ok(( + BlockHashHeightAndEra { + block_hash, + block_height, + era_id, + }, + remainder, + )) + } +} diff --git a/storage/src/data_access_layer/protocol_upgrade.rs b/storage/src/data_access_layer/protocol_upgrade.rs index 691d06f720..f74c8c9530 100644 --- a/storage/src/data_access_layer/protocol_upgrade.rs +++ b/storage/src/data_access_layer/protocol_upgrade.rs @@ -3,7 +3,7 @@ use casper_types::{execution::Effects, Digest, ProtocolUpgradeConfig}; use crate::system::protocol_upgrade::ProtocolUpgradeError; /// Request to upgrade the protocol. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] pub struct ProtocolUpgradeRequest { config: ProtocolUpgradeConfig, } From 44e6dd0b248af429a8e69db4e2e6157221a08a34 Mon Sep 17 00:00:00 2001 From: Jakub Zajkowski Date: Tue, 21 Jul 2026 13:52:06 +0200 Subject: [PATCH 2/5] CORE-285 --- storage/src/block_store/lmdb/lmdb_block_store.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/storage/src/block_store/lmdb/lmdb_block_store.rs b/storage/src/block_store/lmdb/lmdb_block_store.rs index 84bae007de..7601ef7b22 100644 --- a/storage/src/block_store/lmdb/lmdb_block_store.rs +++ b/storage/src/block_store/lmdb/lmdb_block_store.rs @@ -1638,11 +1638,8 @@ impl<'t> DataWriter for LmdbBlockStoreTransaction<'t, RwTransa self.block_store.delete_block_header(&mut self.txn, &key)?; - /* - TODO: currently we don't delete the block body since other blocks may reference it. self.block_store .delete_block_body(&mut self.txn, block.body_hash())?; - */ delete_by_be_u64_key( &mut self.txn, From c7814d259bd8ca8fab5ecbf1e7c0bca601e3b699 Mon Sep 17 00:00:00 2001 From: Jakub Zajkowski Date: Tue, 21 Jul 2026 14:12:26 +0200 Subject: [PATCH 3/5] CORE-285 Moving commit_upgrade_if_needed to a separate file for more clarity --- node/src/reactor/main_reactor.rs | 22 ++-- node/src/reactor/main_reactor/control.rs | 100 +--------------- .../reactor/main_reactor/protocol_upgrade.rs | 110 ++++++++++++++++++ 3 files changed, 121 insertions(+), 111 deletions(-) create mode 100644 node/src/reactor/main_reactor/protocol_upgrade.rs diff --git a/node/src/reactor/main_reactor.rs b/node/src/reactor/main_reactor.rs index 4f00583e0d..69edae05e8 100644 --- a/node/src/reactor/main_reactor.rs +++ b/node/src/reactor/main_reactor.rs @@ -6,6 +6,7 @@ mod error; mod event; mod fetchers; mod memory_metrics; +mod protocol_upgrade; mod utils; mod catch_up; @@ -77,7 +78,11 @@ use crate::{ reactor::{ self, event_queue_metrics::EventQueueMetrics, - main_reactor::{fetchers::Fetchers, upgrade_shutdown::SignatureGossipTracker}, + main_reactor::{ + fetchers::Fetchers, + protocol_upgrade::{commit_upgrade_if_needed, PendingImmediateSwitchBlock}, + upgrade_shutdown::SignatureGossipTracker, + }, EventQueueHandle, QueueKind, }, types::{ @@ -219,19 +224,6 @@ pub(crate) struct MainReactor { upgrade_started_at: Option, } -/// The information needed to produce the deterministic "immediate switch block" following a -/// protocol upgrade, once the node is ready to sign and gossip it. -#[derive(Clone, DataSize, Debug)] -pub(super) struct PendingImmediateSwitchBlock { - next_block_height: u64, - #[data_size(skip)] - post_state_hash: Digest, - parent_hash: BlockHash, - parent_seed: Digest, - era_id: EraId, - timestamp: Timestamp, -} - impl reactor::Reactor for MainReactor { type Event = MainEvent; type Config = WithDir; @@ -1168,7 +1160,7 @@ impl reactor::Reactor for MainReactor { DataReader::::read(&ro_txn, Tip) .map_err(storage::FatalStorageError::from)? }; - let pending_immediate_switch_block = Self::commit_upgrade_if_needed( + let pending_immediate_switch_block = commit_upgrade_if_needed( &contract_runtime, &chainspec, &chainspec_raw_bytes, diff --git a/node/src/reactor/main_reactor/control.rs b/node/src/reactor/main_reactor/control.rs index d751945155..cd6a616318 100644 --- a/node/src/reactor/main_reactor/control.rs +++ b/node/src/reactor/main_reactor/control.rs @@ -1,18 +1,14 @@ -use std::{collections::BTreeMap, sync::Arc, time::Duration}; -use tokio::runtime::Handle; +use std::{collections::BTreeMap, time::Duration}; use tracing::{debug, error, info, trace}; use casper_storage::data_access_layer::GenesisResult; -use casper_types::{ - BlockHash, BlockHeader, Chainspec, ChainspecRawBytes, Digest, EraId, PublicKey, TimeDiff, - Timestamp, -}; +use casper_types::{BlockHash, BlockHeader, Digest, EraId, PublicKey, Timestamp}; use crate::{ components::{ binary_port, block_synchronizer::{self, BlockSynchronizerProgress}, - contract_runtime::{ContractRuntime, ExecutionPreState}, + contract_runtime::ExecutionPreState, diagnostics_port, event_stream_server, network, rest_server, storage, upgrade_watcher, }, effect::{announcements::ControlAnnouncement, EffectBuilder, EffectExt, Effects}, @@ -20,8 +16,7 @@ use crate::{ reactor::main_reactor::{ catch_up::CatchUpInstruction, genesis_instruction::GenesisInstruction, keep_up::KeepUpInstruction, upgrade_shutdown::UpgradeShutdownInstruction, utils, - validate::ValidateInstruction, Error, MainEvent, MainReactor, PendingImmediateSwitchBlock, - ReactorState, + validate::ValidateInstruction, MainEvent, MainReactor, ReactorState, }, types::{BlockPayload, ExecutableBlock, FinalizedBlock, InternalEraReport, MetaBlockState}, NodeRng, @@ -432,93 +427,6 @@ impl MainReactor { } } - /// If `tip_header` is a switch block that is the last block before the chainspec's - /// activation point, synchronously commits the protocol upgrade against `contract_runtime`'s - /// global state. Returns the info needed to later produce, sign, and gossip the resulting - /// immediate switch block, once the reactor is ready to do so (see - /// [`Self::maybe_finish_pending_upgrade`]). Returns `Ok(None)` if no upgrade is due. - /// - /// This is an associated function (rather than a `&self` method) so it can be called from - /// `MainReactor::new`, before the reactor itself has been constructed -- that's the only - /// call site: a fresh restart whose local tip already sits at the pre-activation switch - /// block (e.g. after a live node shuts itself down for the upgrade). A node still *catching - /// up* through a historical activation point does not go through here; it just receives the - /// post-upgrade chain via the ordinary block-synchronizer fetch path, like any other - /// historical data. - pub(super) fn commit_upgrade_if_needed( - contract_runtime: &ContractRuntime, - chainspec: &Arc, - chainspec_raw_bytes: &Arc, - tip_header: Option<&BlockHeader>, - upgrade_timeout: TimeDiff, - ) -> Result, Error> { - let Some(tip_header) = tip_header else { - return Ok(None); - }; - if !(tip_header.is_switch_block() - && tip_header.is_last_block_before_activation(&chainspec.protocol_config)) - { - return Ok(None); - } - - info!( - era_id = %tip_header.era_id(), - height = tip_header.height(), - "committing protocol upgrade" - ); - - let upgrade_config = chainspec - .upgrade_config_from_parts( - *tip_header.state_root_hash(), - tip_header.protocol_version(), - chainspec.protocol_config.activation_point.era_id(), - chainspec_raw_bytes.clone(), - ) - .map_err(Error::ProtocolUpgrade)?; - - // Executing protocol upgrade can be time consuming. It's executed in the background so the - // upgrade_timeout can be enforced. This function stays synchronous -- it's called - // from `MainReactor::new`, before the reactor's async event loop exists -- so the - // wait for that bounded future to resolve is bridged onto a dedicated scoped - // thread, which calls `Handle::block_on` directly. - let handle = Handle::current(); - let post_state_hash = std::thread::scope(|scope| { - scope - .spawn(|| { - handle.block_on(async { - match tokio::time::timeout( - Duration::from(upgrade_timeout), - contract_runtime.commit_protocol_upgrade(upgrade_config), - ) - .await - { - Ok(result) => result, - Err(_) => Err(format!( - "protocol upgrade did not complete within {}", - upgrade_timeout - )), - } - }) - }) - .join() - .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) - }) - .map_err(Error::ProtocolUpgrade)?; - - Ok(Some(PendingImmediateSwitchBlock { - next_block_height: tip_header.height() + 1, - post_state_hash, - parent_hash: tip_header.block_hash(), - parent_seed: *tip_header.accumulated_seed(), - era_id: tip_header.next_block_era_id(), - // Adding one second here to make sure the timestamp is monotonically growing - - // it's important for EVM smart contracts - timestamp: tip_header - .timestamp() - .saturating_add(TimeDiff::from_seconds(1)), - })) - } - /// If a protocol upgrade has been committed and its immediate switch block hasn't yet been /// produced, builds the effects to enqueue it for execution, which will get it signed (by /// this validator, if applicable) and gossiped through the normal block-execution pipeline diff --git a/node/src/reactor/main_reactor/protocol_upgrade.rs b/node/src/reactor/main_reactor/protocol_upgrade.rs new file mode 100644 index 0000000000..9381b85705 --- /dev/null +++ b/node/src/reactor/main_reactor/protocol_upgrade.rs @@ -0,0 +1,110 @@ +use std::{sync::Arc, time::Duration}; +use tokio::runtime::Handle; +use tracing::info; + +use casper_types::{ + BlockHash, BlockHeader, Chainspec, ChainspecRawBytes, Digest, EraId, TimeDiff, Timestamp, +}; +use datasize::DataSize; + +use crate::{components::contract_runtime::ContractRuntime, reactor::main_reactor::Error}; + +/// The information needed to produce the deterministic "immediate switch block" following a +/// protocol upgrade, once the node is ready to sign and gossip it. +#[derive(Clone, DataSize, Debug)] +pub(super) struct PendingImmediateSwitchBlock { + next_block_height: u64, + #[data_size(skip)] + post_state_hash: Digest, + parent_hash: BlockHash, + parent_seed: Digest, + era_id: EraId, + timestamp: Timestamp, +} + +/// If `tip_header` is a switch block that is the last block before the chainspec's +/// activation point, synchronously commits the protocol upgrade against `contract_runtime`'s +/// global state. Returns the info needed to later produce, sign, and gossip the resulting +/// immediate switch block, once the reactor is ready to do so (see +/// [`Self::maybe_finish_pending_upgrade`]). Returns `Ok(None)` if no upgrade is due. +/// +/// This is an associated function (rather than a `&self` method) so it can be called from +/// `MainReactor::new`, before the reactor itself has been constructed -- that's the only +/// call site: a fresh restart whose local tip already sits at the pre-activation switch +/// block (e.g. after a live node shuts itself down for the upgrade). A node still *catching +/// up* through a historical activation point does not go through here; it just receives the +/// post-upgrade chain via the ordinary block-synchronizer fetch path, like any other +/// historical data. +pub(super) fn commit_upgrade_if_needed( + contract_runtime: &ContractRuntime, + chainspec: &Arc, + chainspec_raw_bytes: &Arc, + tip_header: Option<&BlockHeader>, + upgrade_timeout: TimeDiff, +) -> Result, Error> { + let Some(tip_header) = tip_header else { + return Ok(None); + }; + if !(tip_header.is_switch_block() + && tip_header.is_last_block_before_activation(&chainspec.protocol_config)) + { + return Ok(None); + } + + info!( + era_id = %tip_header.era_id(), + height = tip_header.height(), + "committing protocol upgrade" + ); + + let upgrade_config = chainspec + .upgrade_config_from_parts( + *tip_header.state_root_hash(), + tip_header.protocol_version(), + chainspec.protocol_config.activation_point.era_id(), + chainspec_raw_bytes.clone(), + ) + .map_err(Error::ProtocolUpgrade)?; + + // Executing protocol upgrade can be time consuming. It's executed in the background so the + // upgrade_timeout can be enforced. This function stays synchronous -- it's called + // from `MainReactor::new`, before the reactor's async event loop exists -- so the + // wait for that bounded future to resolve is bridged onto a dedicated scoped + // thread, which calls `Handle::block_on` directly. + let handle = Handle::current(); + let post_state_hash = std::thread::scope(|scope| { + scope + .spawn(|| { + handle.block_on(async { + match tokio::time::timeout( + Duration::from(upgrade_timeout), + contract_runtime.commit_protocol_upgrade(upgrade_config), + ) + .await + { + Ok(result) => result, + Err(_) => Err(format!( + "protocol upgrade did not complete within {}", + upgrade_timeout + )), + } + }) + }) + .join() + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) + }) + .map_err(Error::ProtocolUpgrade)?; + + Ok(Some(PendingImmediateSwitchBlock { + next_block_height: tip_header.height() + 1, + post_state_hash, + parent_hash: tip_header.block_hash(), + parent_seed: *tip_header.accumulated_seed(), + era_id: tip_header.next_block_era_id(), + // Adding one second here to make sure the timestamp is monotonically growing - + // it's important for EVM smart contracts + timestamp: tip_header + .timestamp() + .saturating_add(TimeDiff::from_seconds(1)), + })) +} From ac3d374c4736b426ac0e9d71203a38161cfcee2d Mon Sep 17 00:00:00 2001 From: Jakub Zajkowski Date: Tue, 21 Jul 2026 22:23:41 +0200 Subject: [PATCH 4/5] CORE-285 Fixing lint errors --- node/src/reactor/main_reactor.rs | 2 +- node/src/reactor/main_reactor/control.rs | 14 +++++----- .../reactor/main_reactor/protocol_upgrade.rs | 26 +++++++++++++++++++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/node/src/reactor/main_reactor.rs b/node/src/reactor/main_reactor.rs index 69edae05e8..4515052454 100644 --- a/node/src/reactor/main_reactor.rs +++ b/node/src/reactor/main_reactor.rs @@ -28,7 +28,7 @@ use tracing::{debug, error, info, warn}; use casper_binary_port::{LastProgress, NetworkName, Uptime}; use casper_storage::block_store::{types::Tip, BlockStoreProvider, DataReader}; use casper_types::{ - bytesrepr, Block, BlockHash, BlockHeader, BlockV2, Chainspec, ChainspecRawBytes, Digest, EraId, + bytesrepr, Block, BlockHash, BlockHeader, BlockV2, Chainspec, ChainspecRawBytes, EraId, FinalitySignature, FinalitySignatureV2, PublicKey, TimeDiff, Timestamp, Transaction, U512, }; diff --git a/node/src/reactor/main_reactor/control.rs b/node/src/reactor/main_reactor/control.rs index cd6a616318..f2b63a5734 100644 --- a/node/src/reactor/main_reactor/control.rs +++ b/node/src/reactor/main_reactor/control.rs @@ -445,10 +445,10 @@ impl MainReactor { self.upgrade_started_at = Some(Timestamp::now()); self.contract_runtime .set_execution_pre_state(ExecutionPreState::new( - pending.next_block_height, - pending.post_state_hash, - pending.parent_hash, - pending.parent_seed, + pending.next_block_height(), + pending.post_state_hash(), + pending.parent_hash(), + pending.parent_seed(), )); let current_price = self.contract_runtime.current_gas_price(); @@ -462,9 +462,9 @@ impl MainReactor { let finalized_block = FinalizedBlock::new( payload, Some(InternalEraReport::default()), - pending.timestamp, - pending.era_id, - pending.next_block_height, + pending.timestamp(), + pending.era_id(), + pending.next_block_height(), PublicKey::System, ); diff --git a/node/src/reactor/main_reactor/protocol_upgrade.rs b/node/src/reactor/main_reactor/protocol_upgrade.rs index 9381b85705..c5e70df6ee 100644 --- a/node/src/reactor/main_reactor/protocol_upgrade.rs +++ b/node/src/reactor/main_reactor/protocol_upgrade.rs @@ -22,6 +22,32 @@ pub(super) struct PendingImmediateSwitchBlock { timestamp: Timestamp, } +impl PendingImmediateSwitchBlock { + pub(super) fn next_block_height(&self) -> u64 { + self.next_block_height + } + + pub(super) fn post_state_hash(&self) -> Digest { + self.post_state_hash + } + + pub(super) fn parent_hash(&self) -> BlockHash { + self.parent_hash + } + + pub(super) fn parent_seed(&self) -> Digest { + self.parent_seed + } + + pub(super) fn era_id(&self) -> EraId { + self.era_id + } + + pub(super) fn timestamp(&self) -> Timestamp { + self.timestamp + } +} + /// If `tip_header` is a switch block that is the last block before the chainspec's /// activation point, synchronously commits the protocol upgrade against `contract_runtime`'s /// global state. Returns the info needed to later produce, sign, and gossip the resulting From ada747f17602fb8b5c6a40c09e28e8e3928e4e83 Mon Sep 17 00:00:00 2001 From: Jakub Zajkowski Date: Fri, 24 Jul 2026 13:05:11 +0200 Subject: [PATCH 5/5] CORE-285 Added skip_protocol_upgrade flag to node config --- node/src/reactor/main_reactor.rs | 21 ++++++++++++------- .../reactor/main_reactor/protocol_upgrade.rs | 2 +- .../main_reactor/tests/emergency_upgrade.rs | 11 ++++------ node/src/types/node_config.rs | 6 ++++++ .../integration-test/config-example.toml | 5 +++++ resources/local/config.toml | 5 +++++ resources/mainnet/config-example.toml | 5 +++++ resources/production/config-example.toml | 5 +++++ resources/testnet/config-example.toml | 5 +++++ 9 files changed, 50 insertions(+), 15 deletions(-) diff --git a/node/src/reactor/main_reactor.rs b/node/src/reactor/main_reactor.rs index 4515052454..872e4c3459 100644 --- a/node/src/reactor/main_reactor.rs +++ b/node/src/reactor/main_reactor.rs @@ -1160,13 +1160,20 @@ impl reactor::Reactor for MainReactor { DataReader::::read(&ro_txn, Tip) .map_err(storage::FatalStorageError::from)? }; - let pending_immediate_switch_block = commit_upgrade_if_needed( - &contract_runtime, - &chainspec, - &chainspec_raw_bytes, - local_tip.as_ref(), - config.node.upgrade_timeout, - )?; + + // config.node.skip_protocol_upgrade allows to skip the protocol upgrade. In this flow we + // rely that the node will sync_leap the immediate switch blocks from peers. + let pending_immediate_switch_block = if config.node.skip_protocol_upgrade { + None + } else { + commit_upgrade_if_needed( + &contract_runtime, + &chainspec, + &chainspec_raw_bytes, + local_tip.as_ref(), + config.node.upgrade_timeout, + )? + }; let storage = Storage::new( &storage_config, diff --git a/node/src/reactor/main_reactor/protocol_upgrade.rs b/node/src/reactor/main_reactor/protocol_upgrade.rs index c5e70df6ee..5dad94a76e 100644 --- a/node/src/reactor/main_reactor/protocol_upgrade.rs +++ b/node/src/reactor/main_reactor/protocol_upgrade.rs @@ -52,7 +52,7 @@ impl PendingImmediateSwitchBlock { /// activation point, synchronously commits the protocol upgrade against `contract_runtime`'s /// global state. Returns the info needed to later produce, sign, and gossip the resulting /// immediate switch block, once the reactor is ready to do so (see -/// [`Self::maybe_finish_pending_upgrade`]). Returns `Ok(None)` if no upgrade is due. +/// [`Self::maybe_finish_pending_upgrade`]). Returns `Ok(None)` if no upgrade is due /// /// This is an associated function (rather than a `&self` method) so it can be called from /// `MainReactor::new`, before the reactor itself has been constructed -- that's the only diff --git a/node/src/reactor/main_reactor/tests/emergency_upgrade.rs b/node/src/reactor/main_reactor/tests/emergency_upgrade.rs index b218c5634b..347dff2469 100644 --- a/node/src/reactor/main_reactor/tests/emergency_upgrade.rs +++ b/node/src/reactor/main_reactor/tests/emergency_upgrade.rs @@ -8,10 +8,8 @@ use crate::reactor::main_reactor::tests::{ fixture::TestFixture, initial_stakes::InitialStakes, ERA_ONE, ERA_THREE, ERA_TWO, ONE_MIN, }; -/// Exercises an emergency protocol upgrade that requires "peeling" (hard-resetting) blocks -/// already stored under the old protocol version -- as would happen if a chain kept producing -/// blocks past the point an emergency fix needed to roll back to -- combined with a -/// `global_state_update` (an emergency validator-set confirmation). +/// Exercises an protocol upgrade that requires "peeling" blocks already stored under the old +/// protocol version. /// /// This also verifies that the resulting immediate switch block still gets signed and enough /// finality signatures gossiped around the (freshly restarted) network to be marked complete, @@ -79,9 +77,8 @@ async fn emergency_upgrade_requiring_block_peeling() { .await; } - // The network should come back up, apply the upgrade, and continue producing (and - // completing!) blocks -- proving the deferred sign+gossip mechanism for the immediate switch - // block worked across the restart. + // The network should come back up, apply the upgrade, and continue producing + // blocks fixture.run_until_block_height(3, ONE_MIN).await; for runner in fixture.network.nodes().values() { diff --git a/node/src/types/node_config.rs b/node/src/types/node_config.rs index f0f2081a3d..bb04cfed6a 100644 --- a/node/src/types/node_config.rs +++ b/node/src/types/node_config.rs @@ -91,6 +91,11 @@ pub struct NodeConfig { /// If true, prevents a node from shutting down if it is supposed to be a validator in the era. pub prevent_validator_shutdown: bool, + + /// If true, skips committing a protocol upgrade locally when the node's tip is the last + /// block before the activation point, and instead lets the node acquire the post-upgrade + /// chain via the ordinary block-synchronizer fetch path, as if it were catching up. + pub skip_protocol_upgrade: bool, } impl Default for NodeConfig { @@ -105,6 +110,7 @@ impl Default for NodeConfig { shutdown_for_upgrade_timeout: DEFAULT_SHUTDOWN_FOR_UPGRADE_TIMEOUT.parse().unwrap(), upgrade_timeout: DEFAULT_UPGRADE_TIMEOUT.parse().unwrap(), prevent_validator_shutdown: false, + skip_protocol_upgrade: false, } } } diff --git a/resources/integration-test/config-example.toml b/resources/integration-test/config-example.toml index 72a782ef34..2e14f11abc 100644 --- a/resources/integration-test/config-example.toml +++ b/resources/integration-test/config-example.toml @@ -68,6 +68,11 @@ upgrade_timeout = '30 seconds' # other restarting nodes. This config is inert on non-validating nodes. prevent_validator_shutdown = false +# If true, skips committing a protocol upgrade locally when this node's tip is the last +# block before the upgrade's activation point. Instead, the node will acquire the +# post-upgrade chain via the ordinary syncing process, as if it were catching up. +skip_protocol_upgrade = false + # ================================= # Configuration options for logging # ================================= diff --git a/resources/local/config.toml b/resources/local/config.toml index d7692ec4ed..17cfd20872 100644 --- a/resources/local/config.toml +++ b/resources/local/config.toml @@ -68,6 +68,11 @@ upgrade_timeout = '3 hours' # other restarting nodes. This config is inert on non-validating nodes. prevent_validator_shutdown = false +# If true, skips committing a protocol upgrade locally when this node's tip is the last +# block before the upgrade's activation point. Instead, the node will acquire the +# post-upgrade chain via the ordinary syncing process, as if it were catching up. +skip_protocol_upgrade = false + # ================================= # Configuration options for logging # ================================= diff --git a/resources/mainnet/config-example.toml b/resources/mainnet/config-example.toml index dba2fa6f17..3b60eec6f7 100644 --- a/resources/mainnet/config-example.toml +++ b/resources/mainnet/config-example.toml @@ -68,6 +68,11 @@ upgrade_timeout = '30 seconds' # other restarting nodes. This config is inert on non-validating nodes. prevent_validator_shutdown = false +# If true, skips committing a protocol upgrade locally when this node's tip is the last +# block before the upgrade's activation point. Instead, the node will acquire the +# post-upgrade chain via the ordinary syncing process, as if it were catching up. +skip_protocol_upgrade = false + # ================================= # Configuration options for logging # ================================= diff --git a/resources/production/config-example.toml b/resources/production/config-example.toml index 861697c3f7..f4579254b7 100644 --- a/resources/production/config-example.toml +++ b/resources/production/config-example.toml @@ -68,6 +68,11 @@ upgrade_timeout = '30 seconds' # other restarting nodes. This config is inert on non-validating nodes. prevent_validator_shutdown = false +# If true, skips committing a protocol upgrade locally when this node's tip is the last +# block before the upgrade's activation point. Instead, the node will acquire the +# post-upgrade chain via the ordinary syncing process, as if it were catching up. +skip_protocol_upgrade = false + # ================================= # Configuration options for logging # ================================= diff --git a/resources/testnet/config-example.toml b/resources/testnet/config-example.toml index b69e0900f8..aee450f57c 100644 --- a/resources/testnet/config-example.toml +++ b/resources/testnet/config-example.toml @@ -68,6 +68,11 @@ upgrade_timeout = '30 seconds' # other restarting nodes. This config is inert on non-validating nodes. prevent_validator_shutdown = false +# If true, skips committing a protocol upgrade locally when this node's tip is the last +# block before the upgrade's activation point. Instead, the node will acquire the +# post-upgrade chain via the ordinary syncing process, as if it were catching up. +skip_protocol_upgrade = false + # ================================= # Configuration options for logging # =================================