Feature/core 288 optimisations of indexing process

executor/evm/src/block_hash.rs

@@ -3,8 +3,7 @@
 use std::sync::Arc;
 
 use casper_storage::block_store::{
-    lmdb::IndexedLmdbBlockStore, types::BlockHeight, BlockStoreError, BlockStoreProvider,
-    DataReader,
+    lmdb::LmdbBlockStore, types::BlockHeight, BlockStoreError, BlockStoreProvider, DataReader,
 };
 use casper_types::{BlockHash, BlockHeader};
 
@@ -42,12 +41,12 @@ impl BlockHashProvider for NoBlockHashProvider {
 /// Block hash provider backed by Casper's indexed LMDB block store.
 #[derive(Clone, Debug)]
 pub struct IndexedLmdbBlockHashProvider {
-    block_store: Arc<IndexedLmdbBlockStore>,
+    block_store: Arc<LmdbBlockStore>,
 }
 
 impl IndexedLmdbBlockHashProvider {
     /// Creates a block hash provider backed by `block_store`.
-    pub fn new(block_store: Arc<IndexedLmdbBlockStore>) -> Self {
+    pub fn new(block_store: Arc<LmdbBlockStore>) -> Self {
         Self { block_store }
     }
 }

node/src/cli.rs

@@ -6,10 +6,13 @@ pub mod arglang;
 
 use std::{
     alloc::System,
+    borrow::Cow,
     fs,
     path::{Path, PathBuf},
+    println,
     str::FromStr,
     sync::Arc,
+    time::{Duration, Instant},
 };
 
 use anyhow::{self, bail, Context};
@@ -19,14 +22,20 @@ use stats_alloc::{StatsAlloc, INSTRUMENTED_SYSTEM};
 use structopt::StructOpt;
 use toml::{value::Table, Value};
 use tracing::{error, info};
+use tracing_subscriber::EnvFilter;
 
-use casper_types::{Chainspec, ChainspecRawBytes};
+use casper_storage::block_store::{
+    lmdb::LmdbBlockStore,
+    types::{BlockHashHeightAndEra, StateStoreKey},
+    BlockStoreProvider, DataReader,
+};
+use casper_types::{Chainspec, ChainspecRawBytes, TransactionHash};
 
 use crate::{
     components::network::Identity as NetworkIdentity,
     logging,
     reactor::{main_reactor, Runner},
-    setup_signal_hooks,
+    setup_signal_hooks, storage,
     types::ExitCode,
     utils::{
         chain_specification::validate_chainspec, config_specification::validate_config, Loadable,
@@ -88,6 +97,91 @@ pub enum Cli {
         /// Path to configuration file.
         config: PathBuf,
     },
+    /// Rebuild the disk-backed block-store indexes from scratch and report timing statistics.
+    ///
+    /// Clears the block-height, switch-block-era-id, and transaction-hash index databases, then
+    /// repopulates them by scanning every block header currently in storage, printing progress
+    /// as it goes. Does not touch any other stored data.
+    BuildIndexes {
+        /// Path to the LMDB storage directory (the directory containing `storage.lmdb`), i.e.
+        /// the configured `storage.path` joined with the network name subdirectory.
+        lmdb_path: PathBuf,
+
+        /// Upper bound (in bytes) for the LMDB memory map. Must be at least as large as the
+        /// store's configured `storage.max_block_store_size` + `max_deploy_store_size` +
+        /// `max_deploy_metadata_store_size` (summed), or opening the environment will fail.
+        /// Defaults to the sum of those three settings' default values.
+        #[structopt(long)]
+        max_size: Option<usize>,
+    },
+    /// Look up a single raw entry in one of the disk-backed block-store indexes.
+    ///
+    /// Prints the value stored under `key` (or `None` if there is no entry) using `Debug`
+    /// formatting. This reads the index itself and does not resolve the value any further: e.g.
+    /// for `block_height_index_db` this prints the indexed block hash, not the block it
+    /// identifies.
+    ReadIndex {
+        /// Path to the LMDB storage directory (the directory containing `storage.lmdb`), i.e.
+        /// the configured `storage.path` joined with the network name subdirectory.
+        lmdb_path: PathBuf,
+
+        /// Which index database to read from.
+        index: IndexName,
+
+        /// The key to look up in `index`: a `u64` block height for `block_height_index_db`, a
+        /// `u64` era id for `switch_block_era_id_index_db`, or a JSON-encoded `TransactionHash`
+        /// (e.g. `{"Version1":"0101..."}`) for `transaction_hash_index_db`.
+        key: String,
+
+        /// Upper bound (in bytes) for the LMDB memory map. See `build-indexes --max-size` for
+        /// details on the default.
+        #[structopt(long)]
+        max_size: Option<usize>,
+    },
+    /// Show the `completed_blocks` disjoint sequences in human-readable form.
+    ///
+    /// Reads the state-store entry storage uses to track which block heights it has complete
+    /// data for, and prints it as a comma-separated list of inclusive `[high, low]` ranges (e.g.
+    /// `[20, 15], [8, 4]` means heights 15-20 and 4-8 are complete but 9-14 and 0-3 are not).
+    ReadCompletedBlocks {
+        /// Path to the LMDB storage directory (the directory containing `storage.lmdb`), i.e.
+        /// the configured `storage.path` joined with the network name subdirectory.
+        lmdb_path: PathBuf,
+
+        /// Upper bound (in bytes) for the LMDB memory map. See `build-indexes --max-size` for
+        /// details on the default.
+        #[structopt(long)]
+        max_size: Option<usize>,
+    },
+}
+
+/// One of the disk-backed indexes maintained by [`LmdbBlockStore::rebuild_indexes`], as named on
+/// the `read-index` CLI command line by its database name.
+#[derive(Debug, Clone, Copy)]
+pub enum IndexName {
+    /// `block_height_index_db`: keyed by `u64` block height.
+    BlockHeight,
+    /// `switch_block_era_id_index_db`: keyed by `u64` era id.
+    SwitchBlockEraId,
+    /// `transaction_hash_index_db`: keyed by `TransactionHash`.
+    TransactionHash,
+}
+
+impl FromStr for IndexName {
+    type Err = anyhow::Error;
+
+    fn from_str(input: &str) -> Result<Self, Self::Err> {
+        match input {
+            "block_height_index_db" => Ok(IndexName::BlockHeight),
+            "switch_block_era_id_index_db" => Ok(IndexName::SwitchBlockEraId),
+            "transaction_hash_index_db" => Ok(IndexName::TransactionHash),
+            other => bail!(
+                "unknown index {:?}: expected one of `block_height_index_db`, \
+                 `switch_block_era_id_index_db`, `transaction_hash_index_db`",
+                other
+            ),
+        }
+    }
 }
 
 #[derive(Debug)]
@@ -264,6 +358,145 @@ impl Cli {
                     }
                 }
             }
+            Cli::BuildIndexes {
+                lmdb_path,
+                max_size,
+            } => {
+                logging::init_with_config(&Default::default())?;
+                // The default filter only enables `info` level for the `casper_node` crate
+                // itself; `LmdbBlockStore::rebuild_indexes` (in `casper_storage`) logs its
+                // progress at `info` too, so widen the default here unless the operator set
+                // their own `RUST_LOG`, otherwise this command runs silent.
+                if std::env::var("RUST_LOG").is_err() {
+                    logging::reload_global_env_filter(EnvFilter::new(
+                        "warn,casper_node=info,casper_storage=info",
+                    ))?;
+                }
+
+                // The default mirrors `storage::Config::default()`'s
+                // `max_block_store_size + max_deploy_store_size + max_deploy_metadata_store_size`.
+                let default_storage_config = storage::Config::default();
+                let max_size = max_size.unwrap_or_else(|| {
+                    default_storage_config.max_block_store_size
+                        + default_storage_config.max_deploy_store_size
+                        + default_storage_config.max_deploy_metadata_store_size
+                });
+
+                info!(
+                    build_version = %crate::VERSION_STRING.as_str(),
+                    path = %lmdb_path.display(),
+                    "build-indexes: opening block store"
+                );
+
+                let mut block_store = LmdbBlockStore::new(&lmdb_path, max_size)?;
+
+                info!("build-indexes: clearing and rebuilding disk-backed indexes");
+                let start = Instant::now();
+                let stats = block_store.rebuild_indexes()?;
+                let elapsed = start.elapsed();
+
+                let per_block = if stats.headers_processed > 0 {
+                    elapsed / stats.headers_processed as u32
+                } else {
+                    Duration::ZERO
+                };
+
+                info!(
+                    headers_processed = stats.headers_processed,
+                    transactions_indexed = stats.transactions_indexed,
+                    elapsed_secs = elapsed.as_secs_f64(),
+                    per_block_micros = per_block.as_micros() as u64,
+                    "build-indexes: summary"
+                );
+                println!(
+                    "build-indexes: rebuilt indexes for {} block header(s) ({} transaction(s) \
+                     indexed) in {:.3}s ({:.3} ms/block)",
+                    stats.headers_processed,
+                    stats.transactions_indexed,
+                    elapsed.as_secs_f64(),
+                    per_block.as_secs_f64() * 1000.0,
+                );
+
+                Ok(ExitCode::Success as i32)
+            }
+            Cli::ReadIndex {
+                lmdb_path,
+                index,
+                key,
+                max_size,
+            } => {
+                logging::init_with_config(&Default::default())?;
+
+                // Mirrors `storage::Config::default()`'s summed store sizes; see the equivalent
+                // comment on `Cli::BuildIndexes`.
+                let default_storage_config = storage::Config::default();
+                let max_size = max_size.unwrap_or_else(|| {
+                    default_storage_config.max_block_store_size
+                        + default_storage_config.max_deploy_store_size
+                        + default_storage_config.max_deploy_metadata_store_size
+                });
+
+                let block_store = LmdbBlockStore::new(&lmdb_path, max_size)?;
+
+                match index {
+                    IndexName::BlockHeight => {
+                        let height: u64 = key
+                            .parse()
+                            .with_context(|| format!("{:?} is not a valid u64 block height", key))?;
+                        let value = block_store.read_block_height_index_entry(height)?;
+                        println!("{:?}", value);
+                    }
+                    IndexName::SwitchBlockEraId => {
+                        let era_id: u64 = key
+                            .parse()
+                            .with_context(|| format!("{:?} is not a valid u64 era id", key))?;
+                        let value = block_store.read_switch_block_era_id_index_entry(era_id)?;
+                        println!("{:?}", value);
+                    }
+                    IndexName::TransactionHash => {
+                        let transaction_hash: TransactionHash =
+                            serde_json::from_str(&key).with_context(|| {
+                                format!("{:?} is not a valid JSON-encoded TransactionHash", key)
+                            })?;
+                        let value: Option<BlockHashHeightAndEra> =
+                            block_store.checkout_ro()?.read(transaction_hash)?;
+                        println!("{:?}", value);
+                    }
+                }
+
+                Ok(ExitCode::Success as i32)
+            }
+            Cli::ReadCompletedBlocks {
+                lmdb_path,
+                max_size,
+            } => {
+                logging::init_with_config(&Default::default())?;
+
+                // Mirrors `storage::Config::default()`'s summed store sizes; see the equivalent
+                // comment on `Cli::BuildIndexes`.
+                let default_storage_config = storage::Config::default();
+                let max_size = max_size.unwrap_or_else(|| {
+                    default_storage_config.max_block_store_size
+                        + default_storage_config.max_deploy_store_size
+                        + default_storage_config.max_deploy_metadata_store_size
+                });
+
+                let block_store = LmdbBlockStore::new(&lmdb_path, max_size)?;
+                let maybe_raw: Option<Vec<u8>> = block_store.checkout_ro()?.read(
+                    StateStoreKey::new(Cow::Borrowed(storage::COMPLETED_BLOCKS_STORAGE_KEY)),
+                )?;
+
+                match maybe_raw {
+                    Some(raw) => {
+                        let rendered = storage::disjoint_sequences::render(raw)
+                            .context("failed to parse completed_blocks state-store entry")?;
+                        println!("{}", rendered);
+                    }
+                    None => println!("no completed_blocks entry found"),
+                }
+
+                Ok(ExitCode::Success as i32)
+            }
         }
     }
 

node/src/components/block_accumulator/tests.rs

@@ -187,10 +187,20 @@ impl Reactor for MockReactor {
         )
         .unwrap();
 
-        let storage = Storage::new(
+        let protocol_version = ProtocolVersion::from_parts(1, 0, 0);
+        let (storage_root, mut storage_block_store) =
+            storage::open_block_store(&storage_withdir, "test").unwrap();
+        storage::prune_block_store(
+            &mut storage_block_store,
+            chainspec.hard_reset_to_start_of_era(),
+            protocol_version,
+        )
+        .unwrap();
+        let mut storage = Storage::new(
             &storage_withdir,
-            None,
-            ProtocolVersion::from_parts(1, 0, 0),
+            storage_root,
+            storage_block_store,
+            protocol_version,
             EraId::default(),
             "test",
             chainspec.transaction_config.max_ttl.into(),
@@ -200,6 +210,7 @@ impl Reactor for MockReactor {
             TransactionConfig::default(),
         )
         .unwrap();
+        storage.initialize_for_test();
 
         let reactor = MockReactor {
             storage,