diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d6bcbb0..2b0a6701 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,23 +34,18 @@ jobs: components: rustfmt - name: Run tests run: | - cargo check --all-targets --no-default-features --features tokio - cargo check --all-targets --no-default-features --features tokio,sparse - cargo check --all-targets --no-default-features --features tokio,sparse,cache - cargo check --all-targets --no-default-features --features async-std - cargo check --all-targets --no-default-features --features async-std,sparse - cargo check --all-targets --no-default-features --features async-std,sparse,cache - cargo test --no-default-features --features js_interop_tests,tokio - cargo test --no-default-features --features js_interop_tests,tokio,shared-core - cargo test --no-default-features --features js_interop_tests,tokio,sparse - cargo test --no-default-features --features js_interop_tests,tokio,sparse,cache - cargo test --no-default-features --features js_interop_tests,async-std - cargo test --no-default-features --features js_interop_tests,async-std,shared-core - cargo test --no-default-features --features js_interop_tests,async-std,sparse - cargo test --no-default-features --features js_interop_tests,async-std,sparse,cache - cargo test --benches --no-default-features --features tokio - cargo test --benches --no-default-features --features async-std - + cargo check --all-targets + cargo check --all-targets --all-features + cargo check --all-targets --no-default-features + cargo check --all-targets --no-default-features --features sparse + cargo check --all-targets --no-default-features --features sparse,cache + cargo test + cargo test --all-features + cargo test --no-default-features + cargo test --no-default-features --features js_interop_tests + cargo test --no-default-features --features js_interop_tests,sparse + cargo test --no-default-features --features js_interop_tests,sparse,cache + cargo test --benches --no-default-features test-windows: runs-on: windows-latest @@ -61,22 +56,17 @@ jobs: components: rustfmt - name: Run tests run: | - cargo check --all-targets --no-default-features --features tokio - cargo check --all-targets --no-default-features --features tokio,sparse - cargo check --all-targets --no-default-features --features tokio,sparse,cache - cargo check --all-targets --no-default-features --features async-std - cargo check --all-targets --no-default-features --features async-std,sparse - cargo check --all-targets --no-default-features --features async-std,sparse,cache - cargo test --no-default-features --features tokio - cargo test --no-default-features --features tokio,shared-core - cargo test --no-default-features --features tokio,sparse - cargo test --no-default-features --features tokio,sparse,cache - cargo test --no-default-features --features async-std - cargo test --no-default-features --features async-std,shared-core - cargo test --no-default-features --features async-std,sparse - cargo test --no-default-features --features async-std,sparse,cache - cargo test --benches --no-default-features --features tokio - cargo test --benches --no-default-features --features async-std + cargo check --all-targets + cargo check --all-targets --all-features + cargo check --all-targets --no-default-features + cargo check --all-targets --no-default-features --features sparse + cargo check --all-targets --no-default-features --features sparse,cache + cargo test + cargo test --all-features + cargo test --no-default-features + cargo test --no-default-features --features sparse + cargo test --no-default-features --features sparse,cache + cargo test --benches --no-default-features test-macos: runs-on: macos-latest @@ -88,22 +78,18 @@ jobs: components: rustfmt - name: Run tests run: | - cargo check --all-targets --no-default-features --features tokio - cargo check --all-targets --no-default-features --features tokio,sparse - cargo check --all-targets --no-default-features --features tokio,sparse,cache - cargo check --all-targets --no-default-features --features async-std - cargo check --all-targets --no-default-features --features async-std,sparse - cargo check --all-targets --no-default-features --features async-std,sparse,cache - cargo test --no-default-features --features js_interop_tests,tokio - cargo test --no-default-features --features js_interop_tests,tokio,shared-core - cargo test --no-default-features --features js_interop_tests,tokio,sparse - cargo test --no-default-features --features js_interop_tests,tokio,sparse,cache - cargo test --no-default-features --features js_interop_tests,async-std - cargo test --no-default-features --features js_interop_tests,async-std,shared-core - cargo test --no-default-features --features js_interop_tests,async-std,sparse - cargo test --no-default-features --features js_interop_tests,async-std,sparse,cache - cargo test --benches --no-default-features --features tokio - cargo test --benches --no-default-features --features async-std + cargo check --all-targets + cargo check --all-targets --all-features + cargo check --all-targets --no-default-features + cargo check --all-targets --no-default-features --features sparse + cargo check --all-targets --no-default-features --features sparse,cache + cargo test + cargo test --all-features + cargo test --no-default-features + cargo test --no-default-features --features js_interop_tests + cargo test --no-default-features --features js_interop_tests,sparse + cargo test --no-default-features --features js_interop_tests,sparse,cache + cargo test --benches --no-default-features build-extra: runs-on: ubuntu-latest @@ -115,24 +101,17 @@ jobs: targets: wasm32-unknown-unknown - name: Build WASM run: | - cargo build --target=wasm32-unknown-unknown --no-default-features --features tokio - cargo build --target=wasm32-unknown-unknown --no-default-features --features async-std + cargo build --target=wasm32-unknown-unknown --no-default-features - name: Build release run: | - cargo build --release --no-default-features --features tokio - cargo build --release --no-default-features --features tokio,sparse - cargo build --release --no-default-features --features tokio,sparse,cache - cargo build --release --no-default-features --features async-std - cargo build --release --no-default-features --features async-std,sparse - cargo build --release --no-default-features --features async-std,sparse,cache + cargo build --release --no-default-features + cargo build --release --no-default-features --features sparse + cargo build --release --no-default-features --features sparse,cache - name: Run examples run: | - cargo run --no-default-features --features tokio --example disk - cargo run --no-default-features --features async-std --example disk - cargo run --no-default-features --features tokio --example memory - cargo run --no-default-features --features async-std --example memory - cargo run --no-default-features --features tokio --example replication - cargo run --no-default-features --features async-std --example replication + cargo run --no-default-features --example disk + cargo run --no-default-features --example memory + cargo run --no-default-features --example replication lint: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 292f6504..ce5feef4 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ my-first-dataset/ feed.db/ .vscode tests/js/work +.claude +docs diff --git a/CHANGELOG.md b/CHANGELOG.md index 83cadd1a..07a6ba48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,10 +11,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +* Replication over the Hypercore wire protocol, behind the (default) `replication` feature: + * `Hypercore::replicate` builds a `Replicator` driving a Noise-encrypted connection, and + `Hypercore::replicator` builds an unconnected one to attach a connection to later. + * `Replicator::with_connection` and `Replicator::with_connection_stream` attach a transport. + * `Hypercore::attach_replicator` attaches a `Replicator` to the core. + * `Hypercore::attach_channel` joins a channel on an already-multiplexed `hypercore-protocol` + connection, for hosting several cores over one stream. + * Polling `Hypercore::get` drives replication, so a request for a block the core does not have + is served from peers. +* `HypercoreError::Protocol`, wrapping `hypercore_protocol::Error`. +* `From for HypercoreError`. + ### Changed +* `Hypercore` is now `Clone`, sharing one underlying core between clones. +* `append`, `append_batch`, `get`, `create_proof`, `verify_and_apply_proof`, `missing_nodes` and + `missing_nodes_from_merkle_tree_index` take `&self` rather than `&mut self`. +* `key_pair` returns `PartialKeypair` by value instead of `&PartialKeypair`. +* `verify_and_apply_proof` takes `Proof` by value instead of `&Proof`. +* `StorageTraits` now requires `Send + Sync`. +* Internals were reworked to produce owned (`'static`) futures rather than futures borrowing the + core, which is what lets replication hold work in flight across calls. +* `random-access-storage` bumped to `6.0.0-alpha`, `random-access-memory` and `random-access-disk` + to `4.0.0-alpha`. +* The `replication` feature now also pulls in `hypercore-protocol` and `hypercore_handshake`. +* `src/core.rs` was split into `src/core/mod.rs` and `src/core/inner.rs`. + ### Removed +* The `tokio` and `async-std` features. `async-std` is no longer supported and tokio is used + unconditionally, so there is no longer a runtime to select. +* The `shared-core` feature and its `SharedCore` type. `Hypercore` is now `Clone` and its methods + take `&self`, so wrapping it in an `Arc>` is no longer needed. + ## [0.16.0] - 2026-02-18 diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..b8ebfef1 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,123 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +This is the **hypercore** crate, a Rust implementation of the Hypercore protocol - a secure, distributed append-only log. It aims for binary compatibility with the Javascript LTS version for disk storage while maintaining interoperability with the wire protocol. + +Key characteristics: +- **100% Safe Rust**: Uses `#![forbid(unsafe_code)]` +- **Structured concurrency**: Avoids `tokio::spawn`; work only happens when polled + +## JavaScript implementation + +The original JavaScript implementation of hypercore is located in `/home/blake/git/hyper/js/hypercore/.`. + +## Building and Testing + +### Basic commands +```bash +# Run tests +cargo test + +# Run linting (note: strict lints enabled) +cargo clippy +``` + +### Running specific tests +```bash +# Run a single test +cargo test test_name + +# Run tests with specific runtime +cargo test --no-default-features --features tokio +cargo test --no-default-features --features async-std +``` + +### JavaScript interoperability tests +To verify compatibility with the original Javascript implementation: +```bash +cargo test --features js_interop_tests +``` + +These tests use the `rusty_nodejs_repl` crate to run Javascript code side-by-side with Rust, verifying that both implementations produce identical on-disk results. + +### Running examples +```bash +cargo run --example memory +cargo run --example disk +cargo run --example replication +``` + +## Architecture + +## Style guide + +Favor reduced nesting wherever possible. + +### Core Data Structures + +The `Hypercore` struct is the main entry point, built using `HypercoreBuilder`. Internally, it consists of: + +- **Oplog** (`oplog/`): Append-only operation log with dual-header structure for crash recovery. Headers alternate between two memory areas for atomic updates. Stores entries with CRC checksums. + +- **MerkleTree** (`tree/`): Sparse merkle tree storing cryptographic hashes. Uses root indices to track tree state. Supports optional Moka cache for node lookups (with `cache` feature). + +- **BlockStore** (`data/`): Stores actual data blocks indexed by position. + +- **Bitfield** (`bitfield/`): Tracks which blocks are available locally (for sparse replication). Has `fixed` and `dynamic` implementations. + +- **Storage** (`storage/`): Abstraction over storage backends (memory/disk). Splits data into four stores: `tree`, `data`, `bitfield`, and `oplog`. Non-WASM targets use `random-access-disk`; WASM uses `random-access-memory`. + +### Replication + +The `replication/` module (enabled with `replication` feature, on by default) handles: + +- **Events** (`events.rs`): Broadcast events to peers using `async-broadcast` +- **Peer**: Represents remote peers with protocol state + +Replication works by creating proofs in a source hypercore and verifying/applying them to destination hypercores. + +### Key Modules + +- **crypto/**: Cryptographic operations (signing, verification, hashing) using ed25519-dalek and blake2 +- **encoding.rs**: Compact binary encoding utilities +- **builder.rs**: Fluent API for constructing Hypercore instances +- **common/**: Shared types including error types, store definitions, and internal utilities + +## Features + +Default features: `sparse`, `replication`, `cache` + +- `tokio` / `async-std`: Choose async runtime (mutually exclusive, at least one required) +- `sparse`: Enables sparse file support for disk storage +- `replication`: Enables replication support and event broadcasting +- `cache`: Moka-based cache for merkle tree nodes +- `js_interop_tests`: Enables Javascript interoperability tests (dev-only) + +## Code Style and Lints + +This crate has strict linting enabled (see `.clippy.toml` and `lib.rs`): +- `#![forbid(unsafe_code)]` - no unsafe code allowed +- `#![cfg_attr(test, deny(warnings))]` - warnings become errors in tests +- Additional lints: `unreachable_pub`, `redundant_lifetimes`, `clippy::needless_pass_by_value`, etc. + +All public APIs must have documentation (`missing_docs` warning). + +## Testing Strategy + +Tests are organized in: +- `tests/core.rs`: Core functionality tests +- `tests/model.rs`: Property-based tests using `proptest` +- `tests/js_interop.rs`: Javascript interoperability verification +- `tests/common/`: Shared test utilities +- `tests/js/`: Javascript test harness code + +## Important Implementation Details + +- **Header structure**: Uses dual-header approach where headers alternate between two 4KB regions for atomic updates +- **Oplog entries**: Each entry has an 8-byte "leader" (4-byte CRC + 4-byte length/flags) +- **Storage flush**: Uses `skip_flush_count` for batching writes (mirrors JS `autoFlush`) +- **Node indexing**: Uses flat-tree indices for merkle tree navigation +- **Crash recovery**: Oplog design ensures consistency even if process crashes during write diff --git a/Cargo.toml b/Cargo.toml index e831b78e..e85c80ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hypercore" -version = "0.16.0" +version = "0.17.0-alpha.1" license = "MIT OR Apache-2.0" description = "Secure, distributed, append-only log" documentation = "https://docs.rs/hypercore" @@ -32,21 +32,39 @@ flat-tree = "6" merkle-tree-stream = "0.12" pretty-hash = "0.4" rand = "0.8" -random-access-memory = "3" -random-access-storage = "5" sha2 = "0.10" futures = "0.3" crc32fast = "1" intmap = "2" moka = { version = "0.12", optional = true, features = ["sync"] } async-broadcast = { version = "0.7.1", optional = true } -async-lock = {version = "3.4.0", optional = true } +futures-lite = "2.6.1" [dependencies.hypercore_schema] version = "0.2.0" -[target.'cfg(not(target_arch = "wasm32"))'.dependencies] -random-access-disk = { version = "3", default-features = false } +[dependencies.hypercore-protocol] +version = "0.7.2" +optional = true + +[dependencies.hypercore_handshake] +version = "0.6.0" +optional = true + +[dependencies.tokio] +version = "1" +default-features = false +features = ["time"] + +[dependencies.random-access-storage] +version = "6.0.0-alpha" + +[dependencies.random-access-memory] +version = "4.0.0-alpha" + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.random-access-disk] +version = "4.0.0-alpha" +default-features = false [dev-dependencies] anyhow = "1.0.70" @@ -55,8 +73,9 @@ proptest-derive = "0.5.1" data-encoding = "2.2.0" remove_dir_all = "0.7.0" tempfile = "3.1.0" -async-std = { version = "1.12.0", features = ["attributes"] } -tokio = { version = "1.27.0", default-features = false, features = ["macros", "rt", "rt-multi-thread"] } +tokio = { version = "1.27.0", default-features = false, features = ["macros", "rt", "rt-multi-thread", "io-util", "time"] } +tokio-util = { version = "0.7", features = ["compat"] } +uint24le_framing = { version = "0.2.0" } tokio-test = "0.4" sha2 = "0.10" criterion = { version = "0.4", features = ["async_std", "async_tokio"] } @@ -64,12 +83,9 @@ test-log = { version = "0.2.11", default-features = false, features = ["trace"] tracing-subscriber = { version = "0.3.16", features = ["env-filter", "fmt"] } [features] -default = ["tokio", "sparse", "replication", "cache"] -replication = ["dep:async-broadcast"] -shared-core = ["replication", "dep:async-lock"] +default = ["sparse", "replication", "cache"] +replication = ["dep:async-broadcast", "dep:hypercore-protocol", "dep:hypercore_handshake"] sparse = ["random-access-disk/sparse"] -tokio = ["random-access-disk/tokio"] -async-std = ["random-access-disk/async-std"] cache = ["moka"] # Used only in interoperability tests under tests/js-interop which use the javascript version of hypercore # to verify that this crate works. To run them, use: diff --git a/README.md b/README.md index a95afaeb..bee98820 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ corresponding wire protocol implementation. - [x] Get entries from hypercore - [x] Clear range from hypercore, with optional support for sparse files - [x] Support basic replication by creating proofs in a source hypercore and verifying and applying them to a destination hypercore -- [x] Support `tokio` or `async-std` runtimes +- [x] ~~Support `tokio` or `async-std` runtimes~~ `async-std` support dropped `tokio` used unconditionally - [x] Support WASM for in-memory storage - [x] Test Javascript interoperability for supported features - [x] Add optional read cache diff --git a/benches/disk.rs b/benches/disk.rs index 3292df03..b0c9be08 100644 --- a/benches/disk.rs +++ b/benches/disk.rs @@ -1,7 +1,5 @@ use std::time::{Duration, Instant}; -#[cfg(feature = "async-std")] -use criterion::async_executor::AsyncStdExecutor; use criterion::{Criterion, black_box, criterion_group, criterion_main}; use hypercore::{Hypercore, HypercoreBuilder, HypercoreError, Storage}; use tempfile::Builder as TempfileBuilder; @@ -10,12 +8,6 @@ fn bench_create_disk(c: &mut Criterion) { let mut group = c.benchmark_group("slow_call"); group.measurement_time(Duration::from_secs(20)); - #[cfg(feature = "async-std")] - group.bench_function("create_disk", move |b| { - b.to_async(AsyncStdExecutor) - .iter(|| create_hypercore("create")); - }); - #[cfg(feature = "tokio")] group.bench_function("create_disk", move |b| { let rt = tokio::runtime::Runtime::new().unwrap(); b.to_async(&rt).iter(|| create_hypercore("create")); @@ -51,11 +43,6 @@ fn bench_write_disk(c: &mut Criterion) { let mut group = c.benchmark_group("slow_call"); group.measurement_time(Duration::from_secs(20)); - #[cfg(feature = "async-std")] - group.bench_function("write disk", |b| { - b.to_async(AsyncStdExecutor).iter_custom(write_disk); - }); - #[cfg(feature = "tokio")] group.bench_function("write disk", |b| { let rt = tokio::runtime::Runtime::new().unwrap(); b.to_async(&rt).iter_custom(write_disk); @@ -63,7 +50,7 @@ fn bench_write_disk(c: &mut Criterion) { } async fn write_disk(iters: u64) -> Duration { - let mut hypercore = create_hypercore("write").await.unwrap(); + let hypercore = create_hypercore("write").await.unwrap(); let data = Vec::from("hello"); let start = Instant::now(); for _ in 0..iters { @@ -76,11 +63,6 @@ fn bench_read_disk(c: &mut Criterion) { let mut group = c.benchmark_group("slow_call"); group.measurement_time(Duration::from_secs(20)); - #[cfg(feature = "async-std")] - group.bench_function("read disk", |b| { - b.to_async(AsyncStdExecutor).iter_custom(read_disk); - }); - #[cfg(feature = "tokio")] group.bench_function("read disk", |b| { let rt = tokio::runtime::Runtime::new().unwrap(); b.to_async(&rt).iter_custom(read_disk); @@ -88,7 +70,7 @@ fn bench_read_disk(c: &mut Criterion) { } async fn read_disk(iters: u64) -> Duration { - let mut hypercore = create_hypercore("read").await.unwrap(); + let hypercore = create_hypercore("read").await.unwrap(); let data = Vec::from("hello"); for _ in 0..iters { hypercore.append(&data).await.unwrap(); @@ -104,11 +86,6 @@ fn bench_clear_disk(c: &mut Criterion) { let mut group = c.benchmark_group("slow_call"); group.measurement_time(Duration::from_secs(20)); - #[cfg(feature = "async-std")] - group.bench_function("clear disk", |b| { - b.to_async(AsyncStdExecutor).iter_custom(clear_disk); - }); - #[cfg(feature = "tokio")] group.bench_function("clear disk", |b| { let rt = tokio::runtime::Runtime::new().unwrap(); b.to_async(&rt).iter_custom(clear_disk); diff --git a/benches/memory.rs b/benches/memory.rs index ac8015a5..6312a0a0 100644 --- a/benches/memory.rs +++ b/benches/memory.rs @@ -1,17 +1,13 @@ -use std::time::{Duration, Instant}; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; -#[cfg(feature = "async-std")] -use criterion::async_executor::AsyncStdExecutor; use criterion::{Criterion, black_box, criterion_group, criterion_main}; use hypercore::{Hypercore, HypercoreBuilder, HypercoreError, Storage}; use random_access_memory::RandomAccessMemory; fn bench_create_memory(c: &mut Criterion) { - #[cfg(feature = "async-std")] - c.bench_function("create memory", |b| { - b.to_async(AsyncStdExecutor).iter(|| create_hypercore(1024)); - }); - #[cfg(feature = "tokio")] c.bench_function("create memory", |b| { let rt = tokio::runtime::Runtime::new().unwrap(); b.to_async(&rt).iter(|| create_hypercore(1024)); @@ -25,7 +21,7 @@ async fn create_hypercore(page_size: usize) -> Result let storage = Storage::open( |_| { Box::pin(async move { - Ok(Box::new(RandomAccessMemory::new(page_size)) as Box) + Ok(Arc::new(RandomAccessMemory::new(page_size)) as Arc) }) }, false, @@ -44,7 +40,7 @@ async fn create_hypercore(page_size: usize) -> Result let storage = Storage::open( |_| { Box::pin(async move { - Ok(Box::new(RandomAccessMemory::new(page_size)) as Box) + Ok(Arc::new(RandomAccessMemory::new(page_size)) as Arc) }) }, false, @@ -54,11 +50,6 @@ async fn create_hypercore(page_size: usize) -> Result } fn bench_write_memory(c: &mut Criterion) { - #[cfg(feature = "async-std")] - c.bench_function("write memory", |b| { - b.to_async(AsyncStdExecutor).iter_custom(write_memory); - }); - #[cfg(feature = "tokio")] c.bench_function("write memory", |b| { let rt = tokio::runtime::Runtime::new().unwrap(); b.to_async(&rt).iter_custom(write_memory); @@ -66,7 +57,7 @@ fn bench_write_memory(c: &mut Criterion) { } async fn write_memory(iters: u64) -> Duration { - let mut hypercore = create_hypercore(1024).await.unwrap(); + let hypercore = create_hypercore(1024).await.unwrap(); let data = Vec::from("hello"); let start = Instant::now(); for _ in 0..iters { @@ -76,11 +67,6 @@ async fn write_memory(iters: u64) -> Duration { } fn bench_read_memory(c: &mut Criterion) { - #[cfg(feature = "async-std")] - c.bench_function("read memory", |b| { - b.to_async(AsyncStdExecutor).iter_custom(read_memory); - }); - #[cfg(feature = "tokio")] c.bench_function("read memory", |b| { let rt = tokio::runtime::Runtime::new().unwrap(); b.to_async(&rt).iter_custom(read_memory); @@ -88,7 +74,7 @@ fn bench_read_memory(c: &mut Criterion) { } async fn read_memory(iters: u64) -> Duration { - let mut hypercore = create_hypercore(1024).await.unwrap(); + let hypercore = create_hypercore(1024).await.unwrap(); let data = Vec::from("hello"); for _ in 0..iters { hypercore.append(&data).await.unwrap(); @@ -101,11 +87,6 @@ async fn read_memory(iters: u64) -> Duration { } fn bench_clear_memory(c: &mut Criterion) { - #[cfg(feature = "async-std")] - c.bench_function("clear memory", |b| { - b.to_async(AsyncStdExecutor).iter_custom(clear_memory); - }); - #[cfg(feature = "tokio")] c.bench_function("clear memory", |b| { let rt = tokio::runtime::Runtime::new().unwrap(); b.to_async(&rt).iter_custom(clear_memory); diff --git a/examples/disk.rs b/examples/disk.rs index d99b7a10..040574f4 100644 --- a/examples/disk.rs +++ b/examples/disk.rs @@ -1,12 +1,8 @@ -#[cfg(feature = "async-std")] -use async_std::main as async_main; use hypercore::{HypercoreBuilder, HypercoreError, Storage}; use tempfile::Builder; -#[cfg(feature = "tokio")] -use tokio::main as async_main; /// Example about using an in-memory hypercore. -#[async_main] +#[tokio::main] async fn main() { // For the purposes of this example, first create a // temporary directory to hold hypercore. @@ -23,7 +19,7 @@ async fn main() { .expect("Could not create disk storage"); // Build a new disk hypercore - let mut hypercore = HypercoreBuilder::new(storage) + let hypercore = HypercoreBuilder::new(storage) .build() .await .expect("Could not create disk hypercore"); diff --git a/examples/memory.rs b/examples/memory.rs index a510ed6d..5ffe4569 100644 --- a/examples/memory.rs +++ b/examples/memory.rs @@ -1,11 +1,7 @@ -#[cfg(feature = "async-std")] -use async_std::main as async_main; use hypercore::{HypercoreBuilder, HypercoreError, Storage}; -#[cfg(feature = "tokio")] -use tokio::main as async_main; /// Example about using an in-memory hypercore. -#[async_main] +#[tokio::main] async fn main() { // Create a memory storage let storage = Storage::new_memory() diff --git a/examples/replication.rs b/examples/replication.rs index f2943796..c735342b 100644 --- a/examples/replication.rs +++ b/examples/replication.rs @@ -1,15 +1,11 @@ -#[cfg(feature = "async-std")] -use async_std::main as async_main; use hypercore::{Hypercore, HypercoreBuilder, HypercoreError, PartialKeypair, Storage}; use hypercore_schema::{RequestBlock, RequestUpgrade}; use tempfile::Builder; -#[cfg(feature = "tokio")] -use tokio::main as async_main; /// Example on how to replicate a (disk) hypercore to another (memory) hypercore. /// NB: The replication functions used here are low-level, built for use in the wire /// protocol. -#[async_main] +#[tokio::main] async fn main() { // For the purposes of this example, first create a // temporary directory to hold hypercore. @@ -99,7 +95,7 @@ async fn replicate_index( // Then the proof is verified and applied to the replicated party. assert!( replicated_hypercore - .verify_and_apply_proof(&proof) + .verify_and_apply_proof(proof) .await .expect("Verifying and applying proof failed") ); diff --git a/src/common/error.rs b/src/common/error.rs index 89ec0b37..c67685a9 100644 --- a/src/common/error.rs +++ b/src/common/error.rs @@ -1,4 +1,5 @@ use compact_encoding::EncodingError; +use random_access_storage::RandomAccessError; use thiserror::Error; use crate::Store; @@ -48,6 +49,12 @@ pub enum HypercoreError { /// Context for the error context: String, }, + /// An operation gave up waiting + #[error("Operation timed out. {context}")] + Timeout { + /// Context for the error + context: String, + }, /// Unexpected IO error occured #[error("Unrecoverable input/output error occured.{}", .context.as_ref().map_or_else(String::new, |ctx| format!(" {ctx}.")))] @@ -58,6 +65,10 @@ pub enum HypercoreError { #[source] source: std::io::Error, }, + + #[cfg(feature = "replication")] + #[error("hypercore_protocol Error")] + Protocol(#[from] hypercore_protocol::Error), } impl From for HypercoreError { @@ -76,3 +87,33 @@ impl From for HypercoreError { } } } + +impl From for HypercoreError { + fn from(value: RandomAccessError) -> Self { + map_random_access_err(value) + } +} + +pub(crate) fn map_random_access_err(err: RandomAccessError) -> HypercoreError { + match err { + RandomAccessError::IO { + return_code, + context, + source, + } => HypercoreError::IO { + context: Some(format!( + "RandomAccess IO error. Context: {context:?}, return_code: {return_code:?}", + )), + source, + }, + RandomAccessError::OutOfBounds { + offset, + end, + length, + } => HypercoreError::InvalidOperation { + context: format!( + "RandomAccess out of bounds. Offset: {offset}, end: {end:?}, length: {length}", + ), + }, + } +} diff --git a/src/core.rs b/src/core.rs deleted file mode 100644 index 3adc9a50..00000000 --- a/src/core.rs +++ /dev/null @@ -1,1181 +0,0 @@ -//! Hypercore's main abstraction. Exposes an append-only, secure log structure. -use ed25519_dalek::Signature; -use futures::future::Either; -use std::convert::TryFrom; -use std::fmt::Debug; -use tracing::instrument; - -#[cfg(feature = "cache")] -use crate::common::cache::CacheOptions; -use crate::{ - bitfield::Bitfield, - common::{BitfieldUpdate, HypercoreError, NodeByteRange, StoreInfo, ValuelessProof}, - crypto::{PartialKeypair, generate_signing_key}, - data::BlockStore, - oplog::{Header, MAX_OPLOG_ENTRIES_BYTE_SIZE, Oplog}, - storage::Storage, - tree::{MerkleTree, MerkleTreeChangeset}, -}; - -use hypercore_schema::{Proof, RequestBlock, RequestSeek, RequestUpgrade}; - -#[derive(Debug)] -pub(crate) struct HypercoreOptions { - pub(crate) key_pair: Option, - pub(crate) open: bool, - #[cfg(feature = "cache")] - pub(crate) node_cache_options: Option, -} - -impl HypercoreOptions { - pub(crate) fn new() -> Self { - Self { - key_pair: None, - open: false, - #[cfg(feature = "cache")] - node_cache_options: None, - } - } -} - -/// Hypercore is an append-only log structure. -#[derive(Debug)] -pub struct Hypercore { - pub(crate) key_pair: PartialKeypair, - pub(crate) storage: Storage, - pub(crate) oplog: Oplog, - pub(crate) tree: MerkleTree, - pub(crate) block_store: BlockStore, - pub(crate) bitfield: Bitfield, - skip_flush_count: u8, // autoFlush in Javascript - header: Header, - #[cfg(feature = "replication")] - events: crate::replication::events::Events, -} - -/// Response from append, matches that of the Javascript result -#[derive(Debug, PartialEq)] -pub struct AppendOutcome { - /// Length of the hypercore after append - pub length: u64, - /// Byte length of the hypercore after append - pub byte_length: u64, -} - -/// Info about the hypercore -#[derive(Debug, PartialEq)] -pub struct Info { - /// Length of the hypercore - pub length: u64, - /// Byte length of the hypercore - pub byte_length: u64, - /// Continuous length of entries in the hypercore with data - /// starting from index 0 - pub contiguous_length: u64, - /// Fork index. 0 if hypercore not forked. - pub fork: u64, - /// True if hypercore is writeable, false if read-only - pub writeable: bool, -} - -impl Hypercore { - /// Creates/opens new hypercore using given storage and options - pub(crate) async fn new( - mut storage: Storage, - mut options: HypercoreOptions, - ) -> Result { - let key_pair: Option = if options.open { - if options.key_pair.is_some() { - return Err(HypercoreError::BadArgument { - context: "Key pair can not be used when building an openable hypercore" - .to_string(), - }); - } - None - } else { - Some(options.key_pair.take().unwrap_or_else(|| { - let signing_key = generate_signing_key(); - PartialKeypair { - public: signing_key.verifying_key(), - secret: Some(signing_key), - } - })) - }; - - // Open/create oplog - let mut oplog_open_outcome = match Oplog::open(&key_pair, None)? { - Either::Right(value) => value, - Either::Left(instruction) => { - let info = storage.read_info(instruction).await?; - match Oplog::open(&key_pair, Some(info))? { - Either::Right(value) => value, - Either::Left(_) => { - return Err(HypercoreError::InvalidOperation { - context: "Could not open oplog".to_string(), - }); - } - } - } - }; - storage - .flush_infos(&oplog_open_outcome.infos_to_flush) - .await?; - - // Open/create tree - let mut tree = match MerkleTree::open( - &oplog_open_outcome.header.tree, - None, - #[cfg(feature = "cache")] - &options.node_cache_options, - )? { - Either::Right(value) => value, - Either::Left(instructions) => { - let infos = storage.read_infos(&instructions).await?; - match MerkleTree::open( - &oplog_open_outcome.header.tree, - Some(&infos), - #[cfg(feature = "cache")] - &options.node_cache_options, - )? { - Either::Right(value) => value, - Either::Left(_) => { - return Err(HypercoreError::InvalidOperation { - context: "Could not open tree".to_string(), - }); - } - } - } - }; - - // Create block store instance - let block_store = BlockStore::default(); - - // Open bitfield - let mut bitfield = match Bitfield::open(None) { - Either::Right(value) => value, - Either::Left(instruction) => { - let info = storage.read_info(instruction).await?; - match Bitfield::open(Some(info)) { - Either::Right(value) => value, - Either::Left(instruction) => { - let info = storage.read_info(instruction).await?; - match Bitfield::open(Some(info)) { - Either::Right(value) => value, - Either::Left(_) => { - return Err(HypercoreError::InvalidOperation { - context: "Could not open bitfield".to_string(), - }); - } - } - } - } - } - }; - - // Process entries stored only to the oplog and not yet flushed into bitfield or tree - if let Some(entries) = oplog_open_outcome.entries { - for entry in entries.iter() { - for node in &entry.tree_nodes { - tree.add_node(node.clone()); - } - - if let Some(bitfield_update) = &entry.bitfield { - bitfield.update(bitfield_update); - update_contiguous_length( - &mut oplog_open_outcome.header, - &bitfield, - bitfield_update, - ); - } - if let Some(tree_upgrade) = &entry.tree_upgrade { - // TODO: Generalize Either response stack - let mut changeset = - match tree.truncate(tree_upgrade.length, tree_upgrade.fork, None)? { - Either::Right(value) => value, - Either::Left(instructions) => { - let infos = storage.read_infos(&instructions).await?; - match tree.truncate( - tree_upgrade.length, - tree_upgrade.fork, - Some(&infos), - )? { - Either::Right(value) => value, - Either::Left(_) => { - return Err(HypercoreError::InvalidOperation { - context: format!( - "Could not truncate tree to length {}", - tree_upgrade.length - ), - }); - } - } - } - }; - changeset.ancestors = tree_upgrade.ancestors; - changeset.hash = Some(changeset.hash()); - changeset.signature = - Some(Signature::try_from(&*tree_upgrade.signature).map_err(|_| { - HypercoreError::InvalidSignature { - context: "Could not parse changeset signature".to_string(), - } - })?); - - // Update the header with this changeset to make in-memory value match that - // of the stored value. - oplog_open_outcome.oplog.update_header_with_changeset( - &changeset, - None, - &mut oplog_open_outcome.header, - )?; - - // TODO: Skip reorg hints for now, seems to only have to do with replication - // addReorgHint(header.hints.reorgs, tree, batch) - - // Commit changeset to in-memory tree - tree.commit(changeset)?; - } - } - } - - let oplog = oplog_open_outcome.oplog; - let header = oplog_open_outcome.header; - let key_pair = header.key_pair.clone(); - - Ok(Hypercore { - key_pair, - storage, - oplog, - tree, - block_store, - bitfield, - header, - skip_flush_count: 0, - #[cfg(feature = "replication")] - events: crate::replication::events::Events::new(), - }) - } - - /// Gets basic info about the Hypercore - pub fn info(&self) -> Info { - Info { - length: self.tree.length, - byte_length: self.tree.byte_length, - contiguous_length: self.header.hints.contiguous_length, - fork: self.tree.fork, - writeable: self.key_pair.secret.is_some(), - } - } - - /// Appends a data slice to the hypercore. - #[instrument(err, skip_all, fields(data_len = data.len()))] - pub async fn append(&mut self, data: &[u8]) -> Result { - self.append_batch(&[data]).await - } - - /// Appends a given batch of data slices to the hypercore. - #[instrument(err, skip_all, fields(batch_len = batch.as_ref().len()))] - pub async fn append_batch, B: AsRef<[A]>>( - &mut self, - batch: B, - ) -> Result { - let secret_key = match &self.key_pair.secret { - Some(key) => key, - None => return Err(HypercoreError::NotWritable), - }; - - if !batch.as_ref().is_empty() { - // Create a changeset for the tree - let mut changeset = self.tree.changeset(); - let mut batch_length: usize = 0; - for data in batch.as_ref().iter() { - batch_length += changeset.append(data.as_ref()); - } - changeset.hash_and_sign(secret_key); - - // Write the received data to the block store - let info = - self.block_store - .append_batch(batch.as_ref(), batch_length, self.tree.byte_length); - self.storage.flush_info(info).await?; - - // Append the changeset to the Oplog - let bitfield_update = BitfieldUpdate { - drop: false, - start: changeset.ancestors, - length: changeset.batch_length, - }; - let outcome = self.oplog.append_changeset( - &changeset, - Some(bitfield_update.clone()), - false, - &self.header, - )?; - self.storage.flush_infos(&outcome.infos_to_flush).await?; - self.header = outcome.header; - - // Write to bitfield - self.bitfield.update(&bitfield_update); - - // Contiguous length is known only now - update_contiguous_length(&mut self.header, &self.bitfield, &bitfield_update); - - // Commit changeset to in-memory tree - self.tree.commit(changeset)?; - - // Now ready to flush - if self.should_flush_bitfield_and_tree_and_oplog() { - self.flush_bitfield_and_tree_and_oplog(false).await?; - } - - #[cfg(feature = "replication")] - { - use tracing::trace; - - trace!(bitfield_update = ?bitfield_update, "Hppercore.append_batch emit DataUpgrade & Have"); - let _ = self.events.send(crate::replication::events::DataUpgrade {}); - let _ = self - .events - .send(crate::replication::events::Have::from(&bitfield_update)); - } - } - - // Return the new value - Ok(AppendOutcome { - length: self.tree.length, - byte_length: self.tree.byte_length, - }) - } - - #[cfg(feature = "replication")] - /// Subscribe to core events relevant to replication - pub fn event_subscribe(&self) -> async_broadcast::Receiver { - self.events.channel.new_receiver() - } - - /// Check if core has the block at the given `index` locally - #[instrument(ret, skip(self))] - pub fn has(&self, index: u64) -> bool { - self.bitfield.get(index) - } - - /// Read value at given index, if any. - #[instrument(err, skip(self))] - pub async fn get(&mut self, index: u64) -> Result>, HypercoreError> { - if !self.bitfield.get(index) { - #[cfg(feature = "replication")] - // if not in this core, emit Event::Get(index) - { - use tracing::trace; - - trace!(index = index, "Hppercore emit 'get' event"); - self.events.send_on_get(index); - } - return Ok(None); - } - - let byte_range = self.byte_range(index, None).await?; - - // TODO: Generalize Either response stack - let data = match self.block_store.read(&byte_range, None) { - Either::Right(value) => value, - Either::Left(instruction) => { - let info = self.storage.read_info(instruction).await?; - match self.block_store.read(&byte_range, Some(info)) { - Either::Right(value) => value, - Either::Left(_) => { - return Err(HypercoreError::InvalidOperation { - context: "Could not read block storage range".to_string(), - }); - } - } - } - }; - - Ok(Some(data.to_vec())) - } - - /// Clear data for entries between start and end (exclusive) indexes. - #[instrument(err, skip(self))] - pub async fn clear(&mut self, start: u64, end: u64) -> Result<(), HypercoreError> { - if start >= end { - // NB: This is what javascript does, so we mimic that here - return Ok(()); - } - // Write to oplog - let infos_to_flush = self.oplog.clear(start, end)?; - self.storage.flush_infos(&infos_to_flush).await?; - - // Set bitfield - self.bitfield.set_range(start, end - start, false); - - // Set contiguous length - if start < self.header.hints.contiguous_length { - self.header.hints.contiguous_length = start; - } - - // Find the biggest hole that can be punched into the data - let start = if let Some(index) = self.bitfield.last_index_of(true, start) { - index + 1 - } else { - 0 - }; - let end = if let Some(index) = self.bitfield.index_of(true, end) { - index - } else { - self.tree.length - }; - - // Find byte offset for first value - let mut infos: Vec = Vec::new(); - let clear_offset = match self.tree.byte_offset(start, None)? { - Either::Right(value) => value, - Either::Left(instructions) => { - let new_infos = self.storage.read_infos_to_vec(&instructions).await?; - infos.extend(new_infos); - match self.tree.byte_offset(start, Some(&infos))? { - Either::Right(value) => value, - Either::Left(_) => { - return Err(HypercoreError::InvalidOperation { - context: format!("Could not read offset for index {start} from tree"), - }); - } - } - } - }; - - // Find byte range for last value - let last_byte_range = self.byte_range(end - 1, Some(&infos)).await?; - - let clear_length = (last_byte_range.index + last_byte_range.length) - clear_offset; - - // Clear blocks - let info_to_flush = self.block_store.clear(clear_offset, clear_length); - self.storage.flush_info(info_to_flush).await?; - - // Now ready to flush - if self.should_flush_bitfield_and_tree_and_oplog() { - self.flush_bitfield_and_tree_and_oplog(false).await?; - } - - Ok(()) - } - - /// Access the key pair. - pub fn key_pair(&self) -> &PartialKeypair { - &self.key_pair - } - - /// Create a proof for given request - #[instrument(err, skip_all)] - pub async fn create_proof( - &mut self, - block: Option, - hash: Option, - seek: Option, - upgrade: Option, - ) -> Result, HypercoreError> { - let valueless_proof = self - .create_valueless_proof(block, hash, seek, upgrade) - .await?; - let value: Option> = if let Some(block) = valueless_proof.block.as_ref() { - let value = self.get(block.index).await?; - if value.is_none() { - // The data value requested in the proof can not be read, we return None here - // and let the party requesting figure out what to do. - return Ok(None); - } - value - } else { - None - }; - Ok(Some(valueless_proof.into_proof(value))) - } - - /// Verify and apply proof received from peer, returns true if changed, false if not - /// possible to apply. - #[instrument(skip_all)] - pub async fn verify_and_apply_proof(&mut self, proof: &Proof) -> Result { - if proof.fork != self.tree.fork { - return Ok(false); - } - let changeset = self.verify_proof(proof).await?; - if !self.tree.commitable(&changeset) { - return Ok(false); - } - - // In javascript there's _verifyExclusive and _verifyShared based on changeset.upgraded, but - // here we do only one. _verifyShared groups together many subsequent changesets into a single - // oplog push, and then flushes in the end only for the whole group. - let bitfield_update: Option = if let Some(block) = &proof.block.as_ref() { - let byte_offset = - match self - .tree - .byte_offset_in_changeset(block.index, &changeset, None)? - { - Either::Right(value) => value, - Either::Left(instructions) => { - let infos = self.storage.read_infos_to_vec(&instructions).await?; - match self.tree.byte_offset_in_changeset( - block.index, - &changeset, - Some(&infos), - )? { - Either::Right(value) => value, - Either::Left(_) => { - return Err(HypercoreError::InvalidOperation { - context: format!( - "Could not read offset for index {} from tree", - block.index - ), - }); - } - } - } - }; - - // Write the value to the block store - let info_to_flush = self.block_store.put(&block.value, byte_offset); - self.storage.flush_info(info_to_flush).await?; - - // Return a bitfield update for the given value - Some(BitfieldUpdate { - drop: false, - start: block.index, - length: 1, - }) - } else { - // Only from DataBlock can there be changes to the bitfield - None - }; - - // Append the changeset to the Oplog - let outcome = self.oplog.append_changeset( - &changeset, - bitfield_update.clone(), - false, - &self.header, - )?; - self.storage.flush_infos(&outcome.infos_to_flush).await?; - self.header = outcome.header; - - if let Some(bitfield_update) = &bitfield_update { - // Write to bitfield - self.bitfield.update(bitfield_update); - - // Contiguous length is known only now - update_contiguous_length(&mut self.header, &self.bitfield, bitfield_update); - } - - // Commit changeset to in-memory tree - self.tree.commit(changeset)?; - - // Now ready to flush - if self.should_flush_bitfield_and_tree_and_oplog() { - self.flush_bitfield_and_tree_and_oplog(false).await?; - } - - #[cfg(feature = "replication")] - { - if proof.upgrade.is_some() { - // Notify replicator if we receieved an upgrade - let _ = self.events.send(crate::replication::events::DataUpgrade {}); - } - - // Notify replicator if we receieved a bitfield update - if let Some(ref bitfield) = bitfield_update { - let _ = self - .events - .send(crate::replication::events::Have::from(bitfield)); - } - } - Ok(true) - } - - /// Used to fill the nodes field of a `RequestBlock` during - /// synchronization. - #[instrument(err, skip(self))] - pub async fn missing_nodes(&mut self, index: u64) -> Result { - self.missing_nodes_from_merkle_tree_index(index * 2).await - } - - /// Get missing nodes using a merkle tree index. Advanced variant of missing_nodex - /// that allow for special cases of searching directly from the merkle tree. - #[instrument(err, skip(self))] - pub async fn missing_nodes_from_merkle_tree_index( - &mut self, - merkle_tree_index: u64, - ) -> Result { - match self.tree.missing_nodes(merkle_tree_index, None)? { - Either::Right(value) => Ok(value), - Either::Left(instructions) => { - let mut instructions = instructions; - let mut infos: Vec = vec![]; - loop { - infos.extend(self.storage.read_infos_to_vec(&instructions).await?); - match self.tree.missing_nodes(merkle_tree_index, Some(&infos))? { - Either::Right(value) => { - return Ok(value); - } - Either::Left(new_instructions) => { - instructions = new_instructions; - } - } - } - } - } - } - - /// Makes the hypercore read-only by deleting the secret key. Returns true if the - /// hypercore was changed, false if the hypercore was already read-only. This is useful - /// in scenarios where a hypercore should be made immutable after initial values have - /// been stored. - #[instrument(err, skip_all)] - pub async fn make_read_only(&mut self) -> Result { - if self.key_pair.secret.is_some() { - self.key_pair.secret = None; - self.header.key_pair.secret = None; - // Need to flush clearing traces to make sure both oplog slots are cleared - self.flush_bitfield_and_tree_and_oplog(true).await?; - Ok(true) - } else { - Ok(false) - } - } - - async fn byte_range( - &mut self, - index: u64, - initial_infos: Option<&[StoreInfo]>, - ) -> Result { - match self.tree.byte_range(index, initial_infos)? { - Either::Right(value) => Ok(value), - Either::Left(instructions) => { - let mut instructions = instructions; - let mut infos: Vec = vec![]; - loop { - infos.extend(self.storage.read_infos_to_vec(&instructions).await?); - match self.tree.byte_range(index, Some(&infos))? { - Either::Right(value) => { - return Ok(value); - } - Either::Left(new_instructions) => { - instructions = new_instructions; - } - } - } - } - } - } - - async fn create_valueless_proof( - &mut self, - block: Option, - hash: Option, - seek: Option, - upgrade: Option, - ) -> Result { - match self.tree.create_valueless_proof( - block.as_ref(), - hash.as_ref(), - seek.as_ref(), - upgrade.as_ref(), - None, - )? { - Either::Right(value) => Ok(value), - Either::Left(instructions) => { - let mut instructions = instructions; - let mut infos: Vec = vec![]; - loop { - infos.extend(self.storage.read_infos_to_vec(&instructions).await?); - match self.tree.create_valueless_proof( - block.as_ref(), - hash.as_ref(), - seek.as_ref(), - upgrade.as_ref(), - Some(&infos), - )? { - Either::Right(value) => { - return Ok(value); - } - Either::Left(new_instructions) => { - instructions = new_instructions; - } - } - } - } - } - } - - /// Verify a proof received from a peer. Returns a changeset that should be - /// applied. - async fn verify_proof(&mut self, proof: &Proof) -> Result { - match self.tree.verify_proof(proof, &self.key_pair.public, None)? { - Either::Right(value) => Ok(value), - Either::Left(instructions) => { - let infos = self.storage.read_infos_to_vec(&instructions).await?; - match self - .tree - .verify_proof(proof, &self.key_pair.public, Some(&infos))? - { - Either::Right(value) => Ok(value), - Either::Left(_) => Err(HypercoreError::InvalidOperation { - context: "Could not verify proof from tree".to_string(), - }), - } - } - } - } - - fn should_flush_bitfield_and_tree_and_oplog(&mut self) -> bool { - if self.skip_flush_count == 0 - || self.oplog.entries_byte_length >= MAX_OPLOG_ENTRIES_BYTE_SIZE - { - self.skip_flush_count = 3; - true - } else { - self.skip_flush_count -= 1; - false - } - } - - async fn flush_bitfield_and_tree_and_oplog( - &mut self, - clear_traces: bool, - ) -> Result<(), HypercoreError> { - let infos = self.bitfield.flush(); - self.storage.flush_infos(&infos).await?; - let infos = self.tree.flush(); - self.storage.flush_infos(&infos).await?; - let infos = self.oplog.flush(&self.header, clear_traces)?; - self.storage.flush_infos(&infos).await?; - Ok(()) - } -} - -fn update_contiguous_length( - header: &mut Header, - bitfield: &Bitfield, - bitfield_update: &BitfieldUpdate, -) { - let end = bitfield_update.start + bitfield_update.length; - let mut c = header.hints.contiguous_length; - if bitfield_update.drop { - if c <= end && c > bitfield_update.start { - c = bitfield_update.start; - } - } else if c <= end && c >= bitfield_update.start { - c = end; - while bitfield.get(c) { - c += 1; - } - } - - if c != header.hints.contiguous_length { - header.hints.contiguous_length = c; - } -} - -#[cfg(test)] -pub(crate) mod tests { - use super::*; - - #[async_std::test] - async fn core_create_proof_block_only() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - - let proof = hypercore - .create_proof(Some(RequestBlock { index: 4, nodes: 2 }), None, None, None) - .await? - .unwrap(); - let block = proof.block.unwrap(); - assert_eq!(proof.upgrade, None); - assert_eq!(proof.seek, None); - assert_eq!(block.index, 4); - assert_eq!(block.nodes.len(), 2); - assert_eq!(block.nodes[0].index, 10); - assert_eq!(block.nodes[1].index, 13); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_block_and_upgrade() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - let proof = hypercore - .create_proof( - Some(RequestBlock { index: 4, nodes: 0 }), - None, - None, - Some(RequestUpgrade { - start: 0, - length: 10, - }), - ) - .await? - .unwrap(); - let block = proof.block.unwrap(); - let upgrade = proof.upgrade.unwrap(); - assert_eq!(proof.seek, None); - assert_eq!(block.index, 4); - assert_eq!(block.nodes.len(), 3); - assert_eq!(block.nodes[0].index, 10); - assert_eq!(block.nodes[1].index, 13); - assert_eq!(block.nodes[2].index, 3); - assert_eq!(upgrade.start, 0); - assert_eq!(upgrade.length, 10); - assert_eq!(upgrade.nodes.len(), 1); - assert_eq!(upgrade.nodes[0].index, 17); - assert_eq!(upgrade.additional_nodes.len(), 0); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_block_and_upgrade_and_additional() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - let proof = hypercore - .create_proof( - Some(RequestBlock { index: 4, nodes: 0 }), - None, - None, - Some(RequestUpgrade { - start: 0, - length: 8, - }), - ) - .await? - .unwrap(); - let block = proof.block.unwrap(); - let upgrade = proof.upgrade.unwrap(); - assert_eq!(proof.seek, None); - assert_eq!(block.index, 4); - assert_eq!(block.nodes.len(), 3); - assert_eq!(block.nodes[0].index, 10); - assert_eq!(block.nodes[1].index, 13); - assert_eq!(block.nodes[2].index, 3); - assert_eq!(upgrade.start, 0); - assert_eq!(upgrade.length, 8); - assert_eq!(upgrade.nodes.len(), 0); - assert_eq!(upgrade.additional_nodes.len(), 1); - assert_eq!(upgrade.additional_nodes[0].index, 17); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_block_and_upgrade_from_existing_state() -> Result<(), HypercoreError> - { - let mut hypercore = create_hypercore_with_data(10).await?; - let proof = hypercore - .create_proof( - Some(RequestBlock { index: 1, nodes: 0 }), - None, - None, - Some(RequestUpgrade { - start: 1, - length: 9, - }), - ) - .await? - .unwrap(); - let block = proof.block.unwrap(); - let upgrade = proof.upgrade.unwrap(); - assert_eq!(proof.seek, None); - assert_eq!(block.index, 1); - assert_eq!(block.nodes.len(), 0); - assert_eq!(upgrade.start, 1); - assert_eq!(upgrade.length, 9); - assert_eq!(upgrade.nodes.len(), 3); - assert_eq!(upgrade.nodes[0].index, 5); - assert_eq!(upgrade.nodes[1].index, 11); - assert_eq!(upgrade.nodes[2].index, 17); - assert_eq!(upgrade.additional_nodes.len(), 0); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_block_and_upgrade_from_existing_state_with_additional() - -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - let proof = hypercore - .create_proof( - Some(RequestBlock { index: 1, nodes: 0 }), - None, - None, - Some(RequestUpgrade { - start: 1, - length: 5, - }), - ) - .await? - .unwrap(); - let block = proof.block.unwrap(); - let upgrade = proof.upgrade.unwrap(); - assert_eq!(proof.seek, None); - assert_eq!(block.index, 1); - assert_eq!(block.nodes.len(), 0); - assert_eq!(upgrade.start, 1); - assert_eq!(upgrade.length, 5); - assert_eq!(upgrade.nodes.len(), 2); - assert_eq!(upgrade.nodes[0].index, 5); - assert_eq!(upgrade.nodes[1].index, 9); - assert_eq!(upgrade.additional_nodes.len(), 2); - assert_eq!(upgrade.additional_nodes[0].index, 13); - assert_eq!(upgrade.additional_nodes[1].index, 17); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_block_and_seek_1_no_upgrade() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - let proof = hypercore - .create_proof( - Some(RequestBlock { index: 4, nodes: 2 }), - None, - Some(RequestSeek { bytes: 8 }), - None, - ) - .await? - .unwrap(); - let block = proof.block.unwrap(); - assert_eq!(proof.seek, None); // seek included in block - assert_eq!(proof.upgrade, None); - assert_eq!(block.index, 4); - assert_eq!(block.nodes.len(), 2); - assert_eq!(block.nodes[0].index, 10); - assert_eq!(block.nodes[1].index, 13); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_block_and_seek_2_no_upgrade() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - let proof = hypercore - .create_proof( - Some(RequestBlock { index: 4, nodes: 2 }), - None, - Some(RequestSeek { bytes: 10 }), - None, - ) - .await? - .unwrap(); - let block = proof.block.unwrap(); - assert_eq!(proof.seek, None); // seek included in block - assert_eq!(proof.upgrade, None); - assert_eq!(block.index, 4); - assert_eq!(block.nodes.len(), 2); - assert_eq!(block.nodes[0].index, 10); - assert_eq!(block.nodes[1].index, 13); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_block_and_seek_3_no_upgrade() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - let proof = hypercore - .create_proof( - Some(RequestBlock { index: 4, nodes: 2 }), - None, - Some(RequestSeek { bytes: 13 }), - None, - ) - .await? - .unwrap(); - let block = proof.block.unwrap(); - let seek = proof.seek.unwrap(); - assert_eq!(proof.upgrade, None); - assert_eq!(block.index, 4); - assert_eq!(block.nodes.len(), 1); - assert_eq!(block.nodes[0].index, 10); - assert_eq!(seek.nodes.len(), 2); - assert_eq!(seek.nodes[0].index, 12); - assert_eq!(seek.nodes[1].index, 14); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_block_and_seek_to_tree_no_upgrade() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(16).await?; - let proof = hypercore - .create_proof( - Some(RequestBlock { index: 0, nodes: 4 }), - None, - Some(RequestSeek { bytes: 26 }), - None, - ) - .await? - .unwrap(); - let block = proof.block.unwrap(); - let seek = proof.seek.unwrap(); - assert_eq!(proof.upgrade, None); - assert_eq!(block.nodes.len(), 3); - assert_eq!(block.nodes[0].index, 2); - assert_eq!(block.nodes[1].index, 5); - assert_eq!(block.nodes[2].index, 11); - assert_eq!(seek.nodes.len(), 2); - assert_eq!(seek.nodes[0].index, 19); - assert_eq!(seek.nodes[1].index, 27); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_block_and_seek_with_upgrade() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - let proof = hypercore - .create_proof( - Some(RequestBlock { index: 4, nodes: 2 }), - None, - Some(RequestSeek { bytes: 13 }), - Some(RequestUpgrade { - start: 8, - length: 2, - }), - ) - .await? - .unwrap(); - let block = proof.block.unwrap(); - let seek = proof.seek.unwrap(); - let upgrade = proof.upgrade.unwrap(); - assert_eq!(block.index, 4); - assert_eq!(block.nodes.len(), 1); - assert_eq!(block.nodes[0].index, 10); - assert_eq!(seek.nodes.len(), 2); - assert_eq!(seek.nodes[0].index, 12); - assert_eq!(seek.nodes[1].index, 14); - assert_eq!(upgrade.nodes.len(), 1); - assert_eq!(upgrade.nodes[0].index, 17); - assert_eq!(upgrade.additional_nodes.len(), 0); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_seek_with_upgrade() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - let proof = hypercore - .create_proof( - None, - None, - Some(RequestSeek { bytes: 13 }), - Some(RequestUpgrade { - start: 0, - length: 10, - }), - ) - .await? - .unwrap(); - let seek = proof.seek.unwrap(); - let upgrade = proof.upgrade.unwrap(); - assert_eq!(proof.block, None); - assert_eq!(seek.nodes.len(), 4); - assert_eq!(seek.nodes[0].index, 12); - assert_eq!(seek.nodes[1].index, 14); - assert_eq!(seek.nodes[2].index, 9); - assert_eq!(seek.nodes[3].index, 3); - assert_eq!(upgrade.nodes.len(), 1); - assert_eq!(upgrade.nodes[0].index, 17); - assert_eq!(upgrade.additional_nodes.len(), 0); - Ok(()) - } - - #[async_std::test] - async fn core_verify_proof_invalid_signature() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - // Invalid clone hypercore with a different public key - let mut hypercore_clone = create_hypercore_with_data(0).await?; - let proof = hypercore - .create_proof( - None, - Some(RequestBlock { index: 6, nodes: 0 }), - None, - Some(RequestUpgrade { - start: 0, - length: 10, - }), - ) - .await? - .unwrap(); - assert!( - hypercore_clone - .verify_and_apply_proof(&proof) - .await - .is_err() - ); - Ok(()) - } - - #[async_std::test] - async fn core_verify_and_apply_proof() -> Result<(), HypercoreError> { - let mut main = create_hypercore_with_data(10).await?; - let mut clone = create_hypercore_with_data_and_key_pair( - 0, - PartialKeypair { - public: main.key_pair.public, - secret: None, - }, - ) - .await?; - let index = 6; - let nodes = clone.missing_nodes(index).await?; - let proof = main - .create_proof( - None, - Some(RequestBlock { index, nodes }), - None, - Some(RequestUpgrade { - start: 0, - length: 10, - }), - ) - .await? - .unwrap(); - assert!(clone.verify_and_apply_proof(&proof).await?); - let main_info = main.info(); - let clone_info = clone.info(); - assert_eq!(main_info.byte_length, clone_info.byte_length); - assert_eq!(main_info.length, clone_info.length); - assert!(main.get(6).await?.is_some()); - assert!(clone.get(6).await?.is_none()); - - // Fetch data for index 6 and verify it is found - let index = 6; - let nodes = clone.missing_nodes(index).await?; - let proof = main - .create_proof(Some(RequestBlock { index, nodes }), None, None, None) - .await? - .unwrap(); - assert!(clone.verify_and_apply_proof(&proof).await?); - Ok(()) - } - - pub(crate) async fn create_hypercore_with_data( - length: u64, - ) -> Result { - let signing_key = generate_signing_key(); - create_hypercore_with_data_and_key_pair( - length, - PartialKeypair { - public: signing_key.verifying_key(), - secret: Some(signing_key), - }, - ) - .await - } - - pub(crate) async fn create_hypercore_with_data_and_key_pair( - length: u64, - key_pair: PartialKeypair, - ) -> Result { - let storage = Storage::new_memory().await?; - let mut hypercore = Hypercore::new( - storage, - HypercoreOptions { - key_pair: Some(key_pair), - open: false, - #[cfg(feature = "cache")] - node_cache_options: None, - }, - ) - .await?; - for i in 0..length { - hypercore.append(format!("#{}", i).as_bytes()).await?; - } - Ok(hypercore) - } -} diff --git a/src/core/inner.rs b/src/core/inner.rs new file mode 100644 index 00000000..4758b6eb --- /dev/null +++ b/src/core/inner.rs @@ -0,0 +1,1242 @@ +use std::{ + future::Future, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; +#[cfg(feature = "replication")] +use std::sync::Weak; + +use ed25519_dalek::Signature; +use futures::future::Either; +use random_access_storage::BoxFuture; +use std::sync::Mutex; +use tracing::instrument; + +use crate::{ + bitfield::Bitfield, + common::{BitfieldUpdate, HypercoreError, NodeByteRange, StoreInfo, ValuelessProof}, + crypto::{PartialKeypair, generate_signing_key}, + data::BlockStore, + oplog::{Header, MAX_OPLOG_ENTRIES_BYTE_SIZE, Oplog, OplogCreateHeaderOutcome}, + storage::Storage, + tree::{MerkleTree, MerkleTreeChangeset}, +}; +use hypercore_schema::{Proof, RequestBlock, RequestSeek, RequestUpgrade}; + +use super::{AppendOutcome, HypercoreOptions, Info}; + +#[derive(Debug)] +pub(crate) struct HypercoreInnerInner { + pub(crate) key_pair: PartialKeypair, + pub(crate) storage: Storage, + pub(crate) oplog: Oplog, + pub(crate) tree: MerkleTree, + pub(crate) block_store: BlockStore, + pub(crate) bitfield: Bitfield, + pub(crate) skip_flush_count: u8, + pub(crate) header: Header, + #[cfg(feature = "replication")] + pub(crate) events: crate::replication::events::Events, +} + +impl HypercoreInnerInner { + pub(crate) async fn new( + storage: Storage, + mut options: HypercoreOptions, + ) -> Result { + let key_pair: Option = if options.open { + if options.key_pair.is_some() { + return Err(HypercoreError::BadArgument { + context: "Key pair can not be used when building an openable hypercore" + .to_string(), + }); + } + None + } else { + Some(options.key_pair.take().unwrap_or_else(|| { + let signing_key = generate_signing_key(); + PartialKeypair { + public: signing_key.verifying_key(), + secret: Some(signing_key), + } + })) + }; + + // Open/create oplog + let mut oplog_open_outcome = match Oplog::open(&key_pair, None)? { + Either::Right(value) => value, + Either::Left(instruction) => { + let info = storage.read_info(instruction).await?; + match Oplog::open(&key_pair, Some(info))? { + Either::Right(value) => value, + Either::Left(_) => { + return Err(HypercoreError::InvalidOperation { + context: "Could not open oplog".to_string(), + }); + } + } + } + }; + storage + .flush_infos(Vec::from(oplog_open_outcome.infos_to_flush)) + .await?; + + // Open/create tree + let mut tree = match MerkleTree::open( + &oplog_open_outcome.header.tree, + None, + #[cfg(feature = "cache")] + &options.node_cache_options, + )? { + Either::Right(value) => value, + Either::Left(instructions) => { + let infos = storage.read_infos(Vec::from(instructions)).await?; + match MerkleTree::open( + &oplog_open_outcome.header.tree, + Some(&infos), + #[cfg(feature = "cache")] + &options.node_cache_options, + )? { + Either::Right(value) => value, + Either::Left(_) => { + return Err(HypercoreError::InvalidOperation { + context: "Could not open tree".to_string(), + }); + } + } + } + }; + + // Create block store instance + let block_store = BlockStore::default(); + + // Open bitfield + let mut bitfield = match Bitfield::open(None) { + Either::Right(value) => value, + Either::Left(instruction) => { + let info = storage.read_info(instruction).await?; + match Bitfield::open(Some(info)) { + Either::Right(value) => value, + Either::Left(instruction) => { + let info = storage.read_info(instruction).await?; + match Bitfield::open(Some(info)) { + Either::Right(value) => value, + Either::Left(_) => { + return Err(HypercoreError::InvalidOperation { + context: "Could not open bitfield".to_string(), + }); + } + } + } + } + } + }; + + // Process entries stored only to the oplog and not yet flushed into bitfield or tree + if let Some(entries) = oplog_open_outcome.entries { + for entry in entries.iter() { + for node in &entry.tree_nodes { + tree.add_node(node.clone()); + } + + if let Some(bitfield_update) = &entry.bitfield { + bitfield.update(bitfield_update); + update_contiguous_length( + &mut oplog_open_outcome.header, + &bitfield, + bitfield_update, + ); + } + if let Some(tree_upgrade) = &entry.tree_upgrade { + let mut changeset = + match tree.truncate(tree_upgrade.length, tree_upgrade.fork, None)? { + Either::Right(value) => value, + Either::Left(instructions) => { + let infos = storage.read_infos(Vec::from(instructions)).await?; + match tree.truncate( + tree_upgrade.length, + tree_upgrade.fork, + Some(&infos), + )? { + Either::Right(value) => value, + Either::Left(_) => { + return Err(HypercoreError::InvalidOperation { + context: format!( + "Could not truncate tree to length {}", + tree_upgrade.length + ), + }); + } + } + } + }; + changeset.ancestors = tree_upgrade.ancestors; + changeset.hash = Some(changeset.hash()); + changeset.signature = + Some(Signature::try_from(&*tree_upgrade.signature).map_err(|_| { + HypercoreError::InvalidSignature { + context: "Could not parse changeset signature".to_string(), + } + })?); + + oplog_open_outcome.oplog.update_header_with_changeset( + &changeset, + None, + &mut oplog_open_outcome.header, + )?; + + tree.commit(changeset)?; + } + } + } + + let oplog = oplog_open_outcome.oplog; + let header = oplog_open_outcome.header; + let key_pair = header.key_pair.clone(); + + Ok(Self { + key_pair, + storage, + oplog, + tree, + block_store, + bitfield, + header, + skip_flush_count: 0, + #[cfg(feature = "replication")] + events: crate::replication::events::Events::new(), + }) + } + + pub(crate) fn info(&self) -> Info { + Info { + length: self.tree.length, + byte_length: self.tree.byte_length, + contiguous_length: self.header.hints.contiguous_length, + fork: self.tree.fork, + writeable: self.key_pair.secret.is_some(), + } + } + + pub(crate) fn key_pair(&self) -> &PartialKeypair { + &self.key_pair + } + + #[instrument(ret, skip(self))] + pub(crate) fn has(&self, index: u64) -> bool { + self.bitfield.get(index) + } + + #[cfg(feature = "replication")] + pub(crate) fn event_subscribe( + &self, + ) -> async_broadcast::Receiver { + self.events.channel.new_receiver() + } + + pub(crate) fn append_outcome(&self) -> AppendOutcome { + AppendOutcome { + length: self.tree.length, + byte_length: self.tree.byte_length, + } + } + + pub(crate) fn should_flush_bitfield_and_tree_and_oplog(&mut self) -> bool { + if self.skip_flush_count == 0 + || self.oplog.entries_byte_length >= MAX_OPLOG_ENTRIES_BYTE_SIZE + { + self.skip_flush_count = 3; + true + } else { + self.skip_flush_count -= 1; + false + } + } + + pub(crate) fn flush_bitfield_and_tree_and_oplog( + &mut self, + clear_traces: bool, + ) -> BoxFuture> { + let mut infos = vec![]; + infos.extend(self.bitfield.flush()); + infos.extend(self.tree.flush()); + match self.oplog.flush(&self.header, clear_traces) { + Ok(opinfo) => infos.extend(opinfo), + Err(e) => return Box::pin(async { Err(e) }), + } + + self.storage.flush_infos(infos) + } +} + +/// Shared slot for a background replicator, driven whenever `Hypercore::get` polls. +#[cfg(feature = "replication")] +pub(crate) type BackgroundFuture = Arc< + Mutex> + Send>>>>, +>; + +/// Directory of the peers currently replicating this core. +/// +/// Each entry is owned by the `ChannelState` that writes it, so a `Weak` that no longer +/// upgrades means that channel is gone. Dead entries are pruned whenever the list is read. +#[cfg(feature = "replication")] +pub(crate) type PeerRegistry = Arc>>>; + +pub(crate) struct HypercoreInner { + pub(crate) inner: Arc>, + /// Replicator driven in-band by any `Hypercore::get` that must wait for a block. + #[cfg(feature = "replication")] + pub(crate) background: BackgroundFuture, + /// Observers of every connected peer's sync state. Read by `Hypercore::update`. + #[cfg(feature = "replication")] + pub(crate) peers: PeerRegistry, +} + +impl std::fmt::Debug for HypercoreInner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HypercoreInner").finish_non_exhaustive() + } +} + +impl Clone for HypercoreInner { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + #[cfg(feature = "replication")] + background: self.background.clone(), + #[cfg(feature = "replication")] + peers: self.peers.clone(), + } + } +} + +impl HypercoreInner { + pub(crate) async fn new( + storage: Storage, + options: HypercoreOptions, + ) -> Result { + Ok(Self { + inner: Arc::new(Mutex::new( + HypercoreInnerInner::new(storage, options).await?, + )), + #[cfg(feature = "replication")] + background: Arc::new(Mutex::new(None)), + #[cfg(feature = "replication")] + peers: Arc::new(Mutex::new(Vec::new())), + }) + } + pub(crate) fn info(&self) -> Info { + self.inner.lock().unwrap().info() + } + pub(crate) fn key_pair(&self) -> PartialKeypair { + self.inner.lock().unwrap().key_pair().clone() + } + + pub(crate) fn has(&self, index: u64) -> bool { + self.inner.lock().unwrap().has(index) + } + + #[cfg(feature = "replication")] + pub(crate) fn event_subscribe( + &self, + ) -> async_broadcast::Receiver { + self.inner.lock().unwrap().event_subscribe() + } + + /// Best-effort broadcast of a core event. See [`crate::replication::events::Events::send`]. + #[cfg(feature = "replication")] + pub(crate) fn send_event>(&self, event: T) { + let _ = self.inner.lock().unwrap().events.send(event); + } + + /// Start observing `peer`'s sync state. The caller keeps the only strong reference. + #[cfg(feature = "replication")] + pub(crate) fn register_peer(&self, peer: &Arc) { + let mut peers = self.peers.lock().unwrap(); + peers.retain(|p| p.strong_count() > 0); + peers.push(Arc::downgrade(peer)); + } + + /// The peers still replicating this core, pruning any whose channel has gone away. + #[cfg(feature = "replication")] + pub(crate) fn peers(&self) -> Vec> { + let mut peers = self.peers.lock().unwrap(); + peers.retain(|p| p.strong_count() > 0); + peers.iter().filter_map(Weak::upgrade).collect() + } + pub(crate) fn append_outcome(&self) -> AppendOutcome { + self.inner.lock().unwrap().append_outcome() + } + pub(crate) fn should_flush_bitfield_and_tree_and_oplog(&self) -> bool { + self.inner + .lock() + .unwrap() + .should_flush_bitfield_and_tree_and_oplog() + } + pub(crate) fn flush_bitfield_and_tree_and_oplog( + &self, + clear_traces: bool, + ) -> BoxFuture> { + self.inner + .lock() + .unwrap() + .flush_bitfield_and_tree_and_oplog(clear_traces) + } + + pub(crate) fn verify_proof(&self, proof: Proof) -> VerifyProofFuture { + VerifyProofFuture { + inner: self.inner.clone(), + proof, + infos: None, + pending_read: None, + } + } + pub(crate) fn missing_nodes_from_merkle_tree_index( + &self, + merkle_tree_index: u64, + ) -> MissingNodesFuture { + MissingNodesFuture { + inner: self.inner.clone(), + merkle_tree_index, + infos: Vec::new(), + pending_read: None, + } + } + pub(crate) fn verify_and_apply_proof(&self, proof: Proof) -> VerifyAndApplyProofFuture { + VerifyAndApplyProofFuture { + inner: self.inner.clone(), + proof, + changeset: None, + bitfield_update: None, + pending_header: None, + verify_fut: None, + byte_offset_infos: Vec::new(), + byte_offset_read_fut: None, + flush_block_fut: None, + flush_oplog_fut: None, + flush_all_fut: None, + } + } + + pub(crate) fn byte_range(&self, index: u64, initial_infos: Vec) -> ByteRangeFuture { + ByteRangeFuture { + inner: self.inner.clone(), + index, + infos: initial_infos, + pending_read: None, + } + } + + pub(crate) fn get(&self, index: u64) -> GetFuture { + GetFuture { + inner: self.inner.clone(), + index, + byte_range_fut: None, + byte_range: None, + block_read_fut: None, + #[cfg(feature = "replication")] + background: self.background.clone(), + #[cfg(feature = "replication")] + waiting: None, + } + } + + pub(crate) fn create_proof( + &self, + block: Option, + hash: Option, + seek: Option, + upgrade: Option, + ) -> CreateProofFuture { + CreateProofFuture { + inner: self.inner.clone(), + block, + hash, + seek, + upgrade, + valueless_proof_fut: None, + valueless_proof: None, + get_fut: None, + } + } +} + +pub(crate) struct CreateProofFuture { + inner: Arc>, + block: Option, + hash: Option, + seek: Option, + upgrade: Option, + // Phase 1: build the proof structure (without block data) + valueless_proof_fut: Option, + valueless_proof: Option, + // Phase 2: fetch the block value (only when proof.block is Some) + get_fut: Option, +} + +impl Future for CreateProofFuture { + type Output = Result, HypercoreError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + loop { + // Phase 2: fetch block value. + if let Some(fut) = this.get_fut.as_mut() { + match Pin::new(fut).poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(data)) => { + this.get_fut = None; + let vp = this.valueless_proof.take().unwrap(); + return match data { + // Block not present locally — can't serve the proof. + None => Poll::Ready(Ok(None)), + Some(bytes) => { + Poll::Ready(Ok(Some(vp.into_proof(Some(bytes.into_vec()))))) + } + }; + } + } + } + + // Phase 1: build the valueless proof. + if let Some(fut) = this.valueless_proof_fut.as_mut() { + match Pin::new(fut).poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(vp)) => { + this.valueless_proof_fut = None; + if let Some(block) = vp.block.as_ref() { + let index = block.index; + this.valueless_proof = Some(vp); + this.get_fut = Some(GetFuture { + inner: this.inner.clone(), + index, + byte_range_fut: None, + byte_range: None, + block_read_fut: None, + // CreateProofFuture serves local data only; no waiting. + #[cfg(feature = "replication")] + background: Arc::new(Mutex::new(None)), + #[cfg(feature = "replication")] + waiting: None, + }); + continue; + } + return Poll::Ready(Ok(Some(vp.into_proof(None)))); + } + } + } + + // Initial: start the valueless proof future. + this.valueless_proof_fut = Some(ValuelessProofFuture::new( + this.inner.clone(), + this.block.take(), + this.hash.take(), + this.seek.take(), + this.upgrade.take(), + Vec::new(), + None, + )); + } + } +} + +pub(crate) struct GetFuture { + inner: Arc>, + index: u64, + // Phase 1: resolve byte range + byte_range_fut: Option, + // Phase 2: read block from storage (only needed if block_store has no cached value) + byte_range: Option, + block_read_fut: Option>>, + // Replication: drive the background replicator while waiting for this block. + #[cfg(feature = "replication")] + background: BackgroundFuture, + #[cfg(feature = "replication")] + waiting: Option>, +} + +impl GetFuture { + /// Drive the background replicator and wait for a `Have` or `DataUpgrade` event. + /// Returns `Poll::Pending` while waiting, `Poll::Ready(Ok(None))` if no replicator + /// is attached or replication has finished without delivering the block. + #[cfg(feature = "replication")] + fn poll_background_and_wait( + &mut self, + cx: &mut Context<'_>, + ) -> Poll>, HypercoreError>> { + use crate::replication::events::Event; + + // Drive the background replicator. + let bg_done = { + let mut bg = self.background.lock().unwrap(); + match bg.as_mut() { + None => true, + Some(fut) => match fut.as_mut().poll(cx) { + Poll::Ready(Ok(())) => { + *bg = None; + true + } + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Pending => false, + }, + } + }; + + // Drain events; any Have/DataUpgrade means new data may be available. + if let Some(ref mut rx) = self.waiting { + use futures::Stream as _; + loop { + match Pin::new(&mut *rx).poll_next(cx) { + Poll::Ready(Some(Event::Have(_) | Event::DataUpgrade(_))) => { + self.waiting = None; + cx.waker().wake_by_ref(); + return Poll::Pending; + } + Poll::Ready(Some( + Event::Get(_) | Event::PeerSync(_) | Event::Upgrade(_), + )) => {} + Poll::Ready(None) => return Poll::Ready(Ok(None)), + Poll::Pending => break, + } + } + } + + // Background finished without delivering the block. + if bg_done { + return Poll::Ready(Ok(None)); + } + + Poll::Pending + } +} + +impl Future for GetFuture { + type Output = Result>, HypercoreError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + // TODO: we really need to generalize the Either response stack + loop { + // Phase 2: storage read for block data (highest priority when active). + if let Some(fut) = this.block_read_fut.as_mut() { + match fut.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(info)) => { + this.block_read_fut = None; + let inner = this.inner.lock().unwrap(); + let byte_range = this.byte_range.as_ref().unwrap(); + return match inner.block_store.read(byte_range, Some(info)) { + Either::Right(data) => Poll::Ready(Ok(Some(data))), + Either::Left(_) => Poll::Ready(Err(HypercoreError::InvalidOperation { + context: "Could not read block storage range".to_string(), + })), + }; + } + } + } + + // Phase 1: resolve the byte range. + if let Some(fut) = this.byte_range_fut.as_mut() { + match Pin::new(fut).poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(byte_range)) => { + this.byte_range_fut = None; + let inner = this.inner.lock().unwrap(); + match inner.block_store.read(&byte_range, None) { + Either::Right(data) => return Poll::Ready(Ok(Some(data))), + Either::Left(instruction) => { + let storage = inner.storage.clone(); + this.block_read_fut = Some(storage.read_info(instruction)); + this.byte_range = Some(byte_range); + // Loop to poll block_read_fut immediately. + } + } + } + } + continue; + } + + // Initial: check bitfield; if block is missing, wait for replication. + { + let inner = this.inner.lock().unwrap(); + if !inner.bitfield.get(this.index) { + #[cfg(not(feature = "replication"))] + return Poll::Ready(Ok(None)); + + #[cfg(feature = "replication")] + { + // First miss: check whether a background replicator is attached. + if this.waiting.is_none() { + inner.events.send_on_get(this.index); + if this.background.lock().unwrap().is_none() { + // No replicator — return None immediately (original behaviour). + return Poll::Ready(Ok(None)); + } + // Subscribe before emitting Get so we can't miss the Have reply. + let rx = inner.event_subscribe(); + drop(inner); + this.waiting = Some(rx); + } else { + drop(inner); + } + return this.poll_background_and_wait(cx); + } + } + } + this.byte_range_fut = Some(ByteRangeFuture { + inner: this.inner.clone(), + index: this.index, + infos: Vec::new(), + pending_read: None, + }); + // Loop to poll byte_range_fut immediately. + } + } +} + +pub(crate) struct MissingNodesFuture { + inner: Arc>, + merkle_tree_index: u64, + infos: Vec, + pending_read: Option, HypercoreError>>>, +} + +impl Future for MissingNodesFuture { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + loop { + if let Some(fut) = this.pending_read.as_mut() { + match fut.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(new_infos)) => { + this.infos.extend(new_infos); + this.pending_read = None; + } + } + } + + let result = { + let inner = this.inner.lock().unwrap(); + let infos_opt = if this.infos.is_empty() { + None + } else { + Some(this.infos.as_slice()) + }; + inner.tree.missing_nodes(this.merkle_tree_index, infos_opt) + // Lock is dropped here. + }; + + match result { + Err(e) => return Poll::Ready(Err(e)), + Ok(Either::Right(value)) => return Poll::Ready(Ok(value)), + Ok(Either::Left(instructions)) => { + let storage = this.inner.lock().unwrap().storage.clone(); + this.pending_read = Some(storage.read_infos_to_vec(Vec::from(instructions))); + } + } + } + } +} + +pub(crate) struct ByteRangeFuture { + inner: Arc>, + index: u64, + infos: Vec, + pending_read: Option, HypercoreError>>>, +} + +impl Future for ByteRangeFuture { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + loop { + if let Some(fut) = this.pending_read.as_mut() { + match fut.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(new_infos)) => { + this.infos.extend(new_infos); + this.pending_read = None; + } + } + } + + let result = { + let inner = this.inner.lock().unwrap(); + let infos_opt = if this.infos.is_empty() { + None + } else { + Some(this.infos.as_slice()) + }; + inner.tree.byte_range(this.index, infos_opt) + // Lock is dropped here. + }; + + match result { + Err(e) => return Poll::Ready(Err(e)), + Ok(Either::Right(value)) => return Poll::Ready(Ok(value)), + Ok(Either::Left(instructions)) => { + let storage = this.inner.lock().unwrap().storage.clone(); + this.pending_read = Some(storage.read_infos_to_vec(Vec::from(instructions))); + } + } + } + } +} + +pub(crate) struct VerifyProofFuture { + inner: Arc>, + proof: Proof, + // None = first attempt (no read done yet), Some = read completed + infos: Option>, + pending_read: Option, HypercoreError>>>, +} + +impl Future for VerifyProofFuture { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + loop { + // Phase 1: if there's a pending storage read, drive it to completion. + if let Some(fut) = this.pending_read.as_mut() { + match fut.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(infos)) => { + this.infos = Some(infos); + this.pending_read = None; + // Fall through to retry verify_proof. + } + } + } + + // Phase 2: call tree.verify_proof synchronously under the lock. + let result = { + let inner = this.inner.lock().unwrap(); + let public_key = inner.key_pair.public; + let infos_opt = this.infos.as_deref(); + inner.tree.verify_proof(&this.proof, &public_key, infos_opt) + // Lock is dropped here. + }; + + match result { + Err(e) => return Poll::Ready(Err(e)), + Ok(Either::Right(value)) => return Poll::Ready(Ok(value)), + Ok(Either::Left(_)) if this.infos.is_some() => { + // We already read infos and still got Left — the proof can't + // be satisfied, which is an error. + return Poll::Ready(Err(HypercoreError::InvalidOperation { + context: "Could not verify proof from tree".to_string(), + })); + } + Ok(Either::Left(instructions)) => { + let storage = this.inner.lock().unwrap().storage.clone(); + this.pending_read = Some(storage.read_infos_to_vec(Vec::from(instructions))); + // Loop to poll the new future immediately. + } + } + } + } +} + +pub(crate) struct ValuelessProofFuture { + inner: Arc>, + block: Option, + hash: Option, + seek: Option, + upgrade: Option, + infos: Vec, + pending_read: Option, HypercoreError>>>, +} + +impl ValuelessProofFuture { + fn new( + inner: Arc>, + block: Option, + hash: Option, + seek: Option, + upgrade: Option, + infos: Vec, + pending_read: Option, HypercoreError>>>, + ) -> Self { + Self { + inner, + block, + hash, + seek, + upgrade, + infos, + pending_read, + } + } +} + +impl Future for ValuelessProofFuture { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + // ValuelessProofFuture is Unpin (all fields are Unpin), so this is safe. + let this = self.get_mut(); + + loop { + // Phase 1: if there's a pending storage read, drive it to completion. + if let Some(fut) = this.pending_read.as_mut() { + match fut.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(new_infos)) => { + this.infos.extend(new_infos); + this.pending_read = None; + // Fall through to retry create_valueless_proof. + } + } + } + + // Phase 2: call tree.create_valueless_proof synchronously under the lock. + let result = { + let inner = this.inner.lock().unwrap(); + let infos_opt = if this.infos.is_empty() { + None + } else { + Some(this.infos.as_slice()) + }; + inner.tree.create_valueless_proof( + this.block.as_ref(), + this.hash.as_ref(), + this.seek.as_ref(), + this.upgrade.as_ref(), + infos_opt, + ) + // Lock is dropped here. + }; + + match result { + Err(e) => return Poll::Ready(Err(e)), + Ok(Either::Right(value)) => return Poll::Ready(Ok(value)), + Ok(Either::Left(instructions)) => { + // Need more nodes from storage. Clone storage (cheap Arc clone) + // outside the lock so we don't hold it across the async read. + let storage = this.inner.lock().unwrap().storage.clone(); + this.pending_read = Some(storage.read_infos_to_vec(Vec::from(instructions))); + // Loop to poll the new future immediately. + } + } + } + } +} + +pub(crate) struct VerifyAndApplyProofFuture { + inner: Arc>, + proof: Proof, + // Carried between phases + changeset: Option, + bitfield_update: Option, + pending_header: Option
, + // Phase 1: verify the proof + verify_fut: Option, + // Phase 2: read nodes for byte_offset_in_changeset (only if proof.block is Some) + byte_offset_infos: Vec, + byte_offset_read_fut: Option, HypercoreError>>>, + // Phase 3: flush block data to storage + flush_block_fut: Option>>, + // Phase 4: flush oplog + flush_oplog_fut: Option>>, + // Phase 5: flush bitfield+tree+oplog (conditional) + flush_all_fut: Option>>, +} + +impl VerifyAndApplyProofFuture { + // Run oplog.append_changeset synchronously under the lock and return the + // BoxFuture that flushes the resulting infos to storage. + fn start_flush_oplog( + inner: &Arc>, + changeset: &MerkleTreeChangeset, + pending_header: &mut Option
, + bitfield_update: &Option, + ) -> Result>, HypercoreError> { + let (storage, infos) = { + let mut guard = inner.lock().unwrap(); + let OplogCreateHeaderOutcome { + header, + infos_to_flush, + } = { + let HypercoreInnerInner { oplog, header, .. } = &mut *guard; + oplog.append_changeset(changeset, bitfield_update.clone(), false, header)? + }; + *pending_header = Some(header); + let storage = guard.storage.clone(); + (storage, infos_to_flush) + }; + Ok(storage.flush_infos(Vec::from(infos))) + } + + fn emit_events( + inner: &Arc>, + proof: &Proof, + bitfield_update: &Option, + ) { + #[cfg(feature = "replication")] + { + let inner = inner.lock().unwrap(); + if proof.upgrade.is_some() { + let _ = inner + .events + .send(crate::replication::events::DataUpgrade {}); + } + if let Some(bu) = bitfield_update { + let _ = inner + .events + .send(crate::replication::events::Have::from(bu)); + } + } + let _ = (inner, proof, bitfield_update); + } +} + +impl Future for VerifyAndApplyProofFuture { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + loop { + // Phase 5: flush bitfield+tree+oplog. + if let Some(fut) = this.flush_all_fut.as_mut() { + match fut.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(())) => { + this.flush_all_fut = None; + Self::emit_events(&this.inner, &this.proof, &this.bitfield_update); + return Poll::Ready(Ok(true)); + } + } + } + + // Phase 4: flush oplog. + if let Some(fut) = this.flush_oplog_fut.as_mut() { + match fut.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(())) => { + this.flush_oplog_fut = None; + let maybe_flush = { + let mut inner = this.inner.lock().unwrap(); + inner.header = this.pending_header.take().unwrap(); + if let Some(ref bu) = this.bitfield_update { + inner.bitfield.update(bu); + let HypercoreInnerInner { + bitfield, header, .. + } = &mut *inner; + update_contiguous_length(header, bitfield, bu); + } + let changeset = this.changeset.take().unwrap(); + if let Err(e) = inner.tree.commit(changeset) { + return Poll::Ready(Err(e)); + } + if inner.should_flush_bitfield_and_tree_and_oplog() { + Some(inner.flush_bitfield_and_tree_and_oplog(false)) + } else { + None + } + }; + if let Some(fut) = maybe_flush { + this.flush_all_fut = Some(fut); + continue; + } + Self::emit_events(&this.inner, &this.proof, &this.bitfield_update); + return Poll::Ready(Ok(true)); + } + } + } + + // Phase 3: flush block data. + if let Some(fut) = this.flush_block_fut.as_mut() { + match fut.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(())) => { + this.flush_block_fut = None; + match Self::start_flush_oplog( + &this.inner, + this.changeset.as_ref().unwrap(), + &mut this.pending_header, + &this.bitfield_update, + ) { + Err(e) => return Poll::Ready(Err(e)), + Ok(fut) => this.flush_oplog_fut = Some(fut), + } + continue; + } + } + } + + // Phase 2: read nodes for byte_offset_in_changeset. + if let Some(fut) = this.byte_offset_read_fut.as_mut() { + match fut.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(infos)) => { + this.byte_offset_infos.extend(infos); + this.byte_offset_read_fut = None; + let block = this.proof.block.as_ref().unwrap(); + let changeset = this.changeset.as_ref().unwrap(); + let flush_fut = { + let inner = this.inner.lock().unwrap(); + let byte_offset = match inner.tree.byte_offset_in_changeset( + block.index, + changeset, + Some(&this.byte_offset_infos), + ) { + Err(e) => return Poll::Ready(Err(e)), + Ok(Either::Right(v)) => v, + Ok(Either::Left(_)) => { + return Poll::Ready(Err(HypercoreError::InvalidOperation { + context: format!( + "Could not read offset for index {} from tree", + block.index + ), + })); + } + }; + let info = inner.block_store.put(&block.value, byte_offset); + let storage = inner.storage.clone(); + drop(inner); + storage.flush_info(info) + }; + this.bitfield_update = Some(BitfieldUpdate { + drop: false, + start: block.index, + length: 1, + }); + this.flush_block_fut = Some(flush_fut); + continue; + } + } + } + + // Phase 1: verify the proof. + if let Some(fut) = this.verify_fut.as_mut() { + match Pin::new(fut).poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(changeset)) => { + this.verify_fut = None; + { + let inner = this.inner.lock().unwrap(); + if !inner.tree.commitable(&changeset) { + return Poll::Ready(Ok(false)); + } + } + this.changeset = Some(changeset); + + if let Some(block) = this.proof.block.as_ref() { + let changeset = this.changeset.as_ref().unwrap(); + let next = { + let inner = this.inner.lock().unwrap(); + match inner.tree.byte_offset_in_changeset( + block.index, + changeset, + None, + ) { + Err(e) => return Poll::Ready(Err(e)), + Ok(Either::Right(byte_offset)) => { + let info = inner.block_store.put(&block.value, byte_offset); + let storage = inner.storage.clone(); + drop(inner); + let bu = BitfieldUpdate { + drop: false, + start: block.index, + length: 1, + }; + Either::Right((storage.flush_info(info), bu)) + } + Ok(Either::Left(instructions)) => { + let storage = inner.storage.clone(); + drop(inner); + Either::Left( + storage.read_infos_to_vec(Vec::from(instructions)), + ) + } + } + }; + match next { + Either::Right((flush_fut, bu)) => { + this.bitfield_update = Some(bu); + this.flush_block_fut = Some(flush_fut); + } + Either::Left(read_fut) => { + this.byte_offset_read_fut = Some(read_fut); + } + } + } else { + // No block — skip straight to oplog flush. + match Self::start_flush_oplog( + &this.inner, + this.changeset.as_ref().unwrap(), + &mut this.pending_header, + &this.bitfield_update, + ) { + Err(e) => return Poll::Ready(Err(e)), + Ok(fut) => this.flush_oplog_fut = Some(fut), + } + } + continue; + } + } + } + + // Initial: check fork, then start verify. + { + let inner = this.inner.lock().unwrap(); + if this.proof.fork != inner.tree.fork { + return Poll::Ready(Ok(false)); + } + } + this.verify_fut = Some(VerifyProofFuture { + inner: this.inner.clone(), + proof: this.proof.clone(), + infos: None, + pending_read: None, + }); + } + } +} + +pub(crate) fn update_contiguous_length( + header: &mut Header, + bitfield: &Bitfield, + bitfield_update: &BitfieldUpdate, +) { + let end = bitfield_update.start + bitfield_update.length; + let mut c = header.hints.contiguous_length; + if bitfield_update.drop { + if c <= end && c > bitfield_update.start { + c = bitfield_update.start; + } + } else if c <= end && c >= bitfield_update.start { + c = end; + while bitfield.get(c) { + c += 1; + } + } + + if c != header.hints.contiguous_length { + header.hints.contiguous_length = c; + } +} diff --git a/src/core/mod.rs b/src/core/mod.rs new file mode 100644 index 00000000..26c7c8c6 --- /dev/null +++ b/src/core/mod.rs @@ -0,0 +1,850 @@ +//! Hypercore's main abstraction. Exposes an append-only, secure log structure. +pub(crate) mod inner; + +use futures::future::Either; +use tracing::instrument; + +#[cfg(feature = "cache")] +use crate::common::cache::CacheOptions; +use crate::{ + common::{BitfieldUpdate, HypercoreError, StoreInfo}, + core::inner::HypercoreInnerInner, + crypto::PartialKeypair, + storage::Storage, +}; +use hypercore_schema::{Proof, RequestBlock, RequestSeek, RequestUpgrade}; + +pub(crate) use inner::HypercoreInner; +use inner::update_contiguous_length; + +#[derive(Debug)] +pub(crate) struct HypercoreOptions { + pub(crate) key_pair: Option, + pub(crate) open: bool, + #[cfg(feature = "cache")] + pub(crate) node_cache_options: Option, +} + +impl HypercoreOptions { + pub(crate) fn new() -> Self { + Self { + key_pair: None, + open: false, + #[cfg(feature = "cache")] + node_cache_options: None, + } + } +} + +macro_rules! ininner { + ($self:expr) => { + $self.inner.inner.lock().unwrap() + }; +} + +/// Hypercore is an append-only log structure. +#[derive(Debug, Clone)] +pub struct Hypercore { + pub(crate) inner: HypercoreInner, +} + +/// Response from append, matches that of the Javascript result +#[derive(Debug, PartialEq)] +pub struct AppendOutcome { + /// Length of the hypercore after append + pub length: u64, + /// Byte length of the hypercore after append + pub byte_length: u64, +} + +/// Info about the hypercore +#[derive(Debug, PartialEq)] +pub struct Info { + /// Length of the hypercore + pub length: u64, + /// Byte length of the hypercore + pub byte_length: u64, + /// Continuous length of entries in the hypercore with data + /// starting from index 0 + pub contiguous_length: u64, + /// Fork index. 0 if hypercore not forked. + pub fork: u64, + /// True if hypercore is writeable, false if read-only + pub writeable: bool, +} + +impl Hypercore { + /// Creates/opens new hypercore using given storage and options + pub(crate) async fn new( + storage: Storage, + options: HypercoreOptions, + ) -> Result { + Ok(Hypercore { + inner: HypercoreInner::new(storage, options).await?, + }) + } + + /// Gets basic info about the Hypercore + pub fn info(&self) -> Info { + self.inner.info() + } + + /// Appends a data slice to the hypercore. + #[instrument(err, skip_all, fields(data_len = data.len()))] + pub async fn append(&self, data: &[u8]) -> Result { + self.append_batch(&[data]).await + } + + /// Appends a given batch of data slices to the hypercore. + #[instrument(err, skip_all, fields(batch_len = batch.as_ref().len()))] + pub async fn append_batch, B: AsRef<[A]>>( + &self, + batch: B, + ) -> Result { + let secret_key = match self.inner.key_pair().secret { + Some(key) => key, + None => return Err(HypercoreError::NotWritable), + }; + + if batch.as_ref().is_empty() { + return Ok(self.inner.append_outcome()); + } + // Create a changeset for the tree + let mut changeset = ininner!(self).tree.changeset(); + let mut batch_length: usize = 0; + for data in batch.as_ref().iter() { + batch_length += changeset.append(data.as_ref()); + } + changeset.hash_and_sign(&secret_key); + + // Write the received data to the block store + let byte_length = ininner!(self).tree.byte_length; + let info = + ininner!(self) + .block_store + .append_batch(batch.as_ref(), batch_length, byte_length); + { ininner!(self).storage.flush_info(info) }.await?; + + // Append the changeset to the Oplog + let bitfield_update = BitfieldUpdate { + drop: false, + start: changeset.ancestors, + length: changeset.batch_length, + }; + let outcome = { + let HypercoreInnerInner { oplog, header, .. } = &mut *ininner!(self); + oplog.append_changeset(&changeset, Some(bitfield_update.clone()), false, header)? + }; + { + ininner!(self) + .storage + .flush_infos(Vec::from(outcome.infos_to_flush)) + } + .await?; + ininner!(self).header = outcome.header; + + // Write to bitfield + ininner!(self).bitfield.update(&bitfield_update); + + // Contiguous length is known only now + { + let HypercoreInnerInner { + bitfield, header, .. + } = &mut *ininner!(self); + update_contiguous_length(header, bitfield, &bitfield_update); + } + + // Commit changeset to in-memory tree + ininner!(self).tree.commit(changeset)?; + + // Now ready to flush + if self.inner.should_flush_bitfield_and_tree_and_oplog() { + self.inner.flush_bitfield_and_tree_and_oplog(false).await?; + } + + #[cfg(feature = "replication")] + { + use tracing::trace; + + trace!(bitfield_update = ?bitfield_update, "Hppercore.append_batch emit DataUpgrade & Have"); + let _ = ininner!(self) + .events + .send(crate::replication::events::DataUpgrade {}); + let _ = ininner!(self) + .events + .send(crate::replication::events::Have::from(&bitfield_update)); + } + + Ok(self.inner.append_outcome()) + } + + #[cfg(feature = "replication")] + /// Subscribe to core events relevant to replication + pub fn event_subscribe(&self) -> async_broadcast::Receiver { + self.inner.event_subscribe() + } + + /// Check if core has the block at the given `index` locally + #[instrument(ret, skip(self))] + pub fn has(&self, index: u64) -> bool { + self.inner.has(index) + } + + /// Read value at given index, if any. + #[instrument(err, skip(self))] + pub async fn get(&self, index: u64) -> Result>, HypercoreError> { + Ok(self.inner.get(index).await?.map(|b| b.into_vec())) + } + + /// Clear data for entries between start and end (exclusive) indexes. + #[instrument(err, skip(self))] + pub async fn clear(&mut self, start: u64, end: u64) -> Result<(), HypercoreError> { + if start >= end { + // NB: This is what javascript does, so we mimic that here + return Ok(()); + } + // Write to oplog + let infos_to_flush = ininner!(self).oplog.clear(start, end)?; + { + ininner!(self) + .storage + .flush_infos(Vec::from(infos_to_flush)) + } + .await?; + + // Set bitfield + ininner!(self).bitfield.set_range(start, end - start, false); + + // Set contiguous length + if start < ininner!(self).header.hints.contiguous_length { + ininner!(self).header.hints.contiguous_length = start; + } + + // Find the biggest hole that can be punched into the data + let start = if let Some(index) = ininner!(self).bitfield.last_index_of(true, start) { + index + 1 + } else { + 0 + }; + let end = if let Some(index) = ininner!(self).bitfield.index_of(true, end) { + index + } else { + ininner!(self).tree.length + }; + + // Find byte offset for first value + let mut infos: Vec = Vec::new(); + let clear_offset = match { ininner!(self).tree.byte_offset(start, None)? } { + Either::Right(value) => value, + Either::Left(instructions) => { + let new_infos = { + ininner!(self) + .storage + .read_infos_to_vec(Vec::from(instructions)) + } + .await?; + infos.extend(new_infos); + match ininner!(self).tree.byte_offset(start, Some(&infos))? { + Either::Right(value) => value, + Either::Left(_) => { + return Err(HypercoreError::InvalidOperation { + context: format!("Could not read offset for index {start} from tree"), + }); + } + } + } + }; + + // Find byte range for last value + let last_byte_range = self.inner.byte_range(end - 1, infos).await?; + + let clear_length = (last_byte_range.index + last_byte_range.length) - clear_offset; + + // Clear blocks + let info_to_flush = ininner!(self).block_store.clear(clear_offset, clear_length); + { ininner!(self).storage.flush_info(info_to_flush) }.await?; + + // Now ready to flush + if self.inner.should_flush_bitfield_and_tree_and_oplog() { + self.inner.flush_bitfield_and_tree_and_oplog(false).await?; + } + + Ok(()) + } + + /// Access the key pair. + pub fn key_pair(&self) -> PartialKeypair { + self.inner.key_pair() + } + + /// Create a proof for given request + #[instrument(err, skip_all)] + pub async fn create_proof( + &self, + block: Option, + hash: Option, + seek: Option, + upgrade: Option, + ) -> Result, HypercoreError> { + self.inner.create_proof(block, hash, seek, upgrade).await + } + + /// Verify and apply proof received from peer, returns true if changed, false if not + /// possible to apply. + #[instrument(skip_all)] + pub async fn verify_and_apply_proof(&self, proof: Proof) -> Result { + self.inner.verify_and_apply_proof(proof).await + } + + #[allow(dead_code)] + async fn verify_and_apply_proof_old(&self, proof: &Proof) -> Result { + if proof.fork != ininner!(self).tree.fork { + return Ok(false); + } + let changeset = self.inner.verify_proof(proof.clone()).await?; + if !ininner!(self).tree.commitable(&changeset) { + return Ok(false); + } + + // In javascript there's _verifyExclusive and _verifyShared based on changeset.upgraded, but + // here we do only one. _verifyShared groups together many subsequent changesets into a single + // oplog push, and then flushes in the end only for the whole group. + let bitfield_update: Option = if let Some(block) = &proof.block.as_ref() { + let byte_offset = match { + ininner!(self) + .tree + .byte_offset_in_changeset(block.index, &changeset, None)? + } { + Either::Right(value) => value, + Either::Left(instructions) => { + let infos = { + ininner!(self) + .storage + .read_infos_to_vec(Vec::from(instructions)) + } + .await?; + match ininner!(self).tree.byte_offset_in_changeset( + block.index, + &changeset, + Some(&infos), + )? { + Either::Right(value) => value, + Either::Left(_) => { + return Err(HypercoreError::InvalidOperation { + context: format!( + "Could not read offset for index {} from tree", + block.index + ), + }); + } + } + } + }; + + // Write the value to the block store + let info_to_flush = ininner!(self).block_store.put(&block.value, byte_offset); + { ininner!(self).storage.flush_info(info_to_flush) }.await?; + + // Return a bitfield update for the given value + Some(BitfieldUpdate { + drop: false, + start: block.index, + length: 1, + }) + } else { + // Only from DataBlock can there be changes to the bitfield + None + }; + + // Append the changeset to the Oplog + let outcome = { + let HypercoreInnerInner { oplog, header, .. } = &mut *ininner!(self); + oplog.append_changeset(&changeset, bitfield_update.clone(), false, header)? + }; + { + ininner!(self) + .storage + .flush_infos(Vec::from(outcome.infos_to_flush)) + } + .await?; + ininner!(self).header = outcome.header; + + if let Some(bitfield_update) = &bitfield_update { + // Write to bitfield + ininner!(self).bitfield.update(bitfield_update); + + // Contiguous length is known only now + { + let HypercoreInnerInner { + bitfield, header, .. + } = &mut *ininner!(self); + update_contiguous_length(header, bitfield, bitfield_update); + } + } + + // Commit changeset to in-memory tree + ininner!(self).tree.commit(changeset)?; + + // Now ready to flush + if self.inner.should_flush_bitfield_and_tree_and_oplog() { + self.inner.flush_bitfield_and_tree_and_oplog(false).await?; + } + + #[cfg(feature = "replication")] + { + if proof.upgrade.is_some() { + // Notify replicator if we receieved an upgrade + let _ = ininner!(self) + .events + .send(crate::replication::events::DataUpgrade {}); + } + + // Notify replicator if we receieved a bitfield update + if let Some(ref bitfield) = bitfield_update { + let _ = ininner!(self) + .events + .send(crate::replication::events::Have::from(bitfield)); + } + } + Ok(true) + } + + /// Used to fill the nodes field of a `RequestBlock` during + /// synchronization. + #[instrument(err, skip(self))] + pub async fn missing_nodes(&self, index: u64) -> Result { + self.inner + .missing_nodes_from_merkle_tree_index(index * 2) + .await + } + + /// Get missing nodes using a merkle tree index. Advanced variant of missing_nodes + /// that allow for special cases of searching directly from the merkle tree. + #[instrument(err, skip(self))] + pub async fn missing_nodes_from_merkle_tree_index( + &self, + merkle_tree_index: u64, + ) -> Result { + self.inner + .missing_nodes_from_merkle_tree_index(merkle_tree_index) + .await + } + + /// Makes the hypercore read-only by deleting the secret key. Returns true if the + /// hypercore was changed, false if the hypercore was already read-only. This is useful + /// in scenarios where a hypercore should be made immutable after initial values have + /// been stored. + #[instrument(err, skip_all)] + pub async fn make_read_only(&mut self) -> Result { + if ininner!(self).key_pair.secret.is_some() { + ininner!(self).key_pair.secret = None; + ininner!(self).header.key_pair.secret = None; + // Need to flush clearing traces to make sure both oplog slots are cleared + { ininner!(self).flush_bitfield_and_tree_and_oplog(true) }.await?; + Ok(true) + } else { + Ok(false) + } + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use crate::crypto::{PartialKeypair, generate_signing_key}; + + #[tokio::test] + async fn core_create_proof_block_only() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + + let proof = hypercore + .create_proof(Some(RequestBlock { index: 4, nodes: 2 }), None, None, None) + .await? + .unwrap(); + let block = proof.block.unwrap(); + assert_eq!(proof.upgrade, None); + assert_eq!(proof.seek, None); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 2); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(block.nodes[1].index, 13); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_block_and_upgrade() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 4, nodes: 0 }), + None, + None, + Some(RequestUpgrade { + start: 0, + length: 10, + }), + ) + .await? + .unwrap(); + let block = proof.block.unwrap(); + let upgrade = proof.upgrade.unwrap(); + assert_eq!(proof.seek, None); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 3); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(block.nodes[1].index, 13); + assert_eq!(block.nodes[2].index, 3); + assert_eq!(upgrade.start, 0); + assert_eq!(upgrade.length, 10); + assert_eq!(upgrade.nodes.len(), 1); + assert_eq!(upgrade.nodes[0].index, 17); + assert_eq!(upgrade.additional_nodes.len(), 0); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_block_and_upgrade_and_additional() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 4, nodes: 0 }), + None, + None, + Some(RequestUpgrade { + start: 0, + length: 8, + }), + ) + .await? + .unwrap(); + let block = proof.block.unwrap(); + let upgrade = proof.upgrade.unwrap(); + assert_eq!(proof.seek, None); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 3); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(block.nodes[1].index, 13); + assert_eq!(block.nodes[2].index, 3); + assert_eq!(upgrade.start, 0); + assert_eq!(upgrade.length, 8); + assert_eq!(upgrade.nodes.len(), 0); + assert_eq!(upgrade.additional_nodes.len(), 1); + assert_eq!(upgrade.additional_nodes[0].index, 17); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_block_and_upgrade_from_existing_state() -> Result<(), HypercoreError> + { + let hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 1, nodes: 0 }), + None, + None, + Some(RequestUpgrade { + start: 1, + length: 9, + }), + ) + .await? + .unwrap(); + let block = proof.block.unwrap(); + let upgrade = proof.upgrade.unwrap(); + assert_eq!(proof.seek, None); + assert_eq!(block.index, 1); + assert_eq!(block.nodes.len(), 0); + assert_eq!(upgrade.start, 1); + assert_eq!(upgrade.length, 9); + assert_eq!(upgrade.nodes.len(), 3); + assert_eq!(upgrade.nodes[0].index, 5); + assert_eq!(upgrade.nodes[1].index, 11); + assert_eq!(upgrade.nodes[2].index, 17); + assert_eq!(upgrade.additional_nodes.len(), 0); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_block_and_upgrade_from_existing_state_with_additional() + -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 1, nodes: 0 }), + None, + None, + Some(RequestUpgrade { + start: 1, + length: 5, + }), + ) + .await? + .unwrap(); + let block = proof.block.unwrap(); + let upgrade = proof.upgrade.unwrap(); + assert_eq!(proof.seek, None); + assert_eq!(block.index, 1); + assert_eq!(block.nodes.len(), 0); + assert_eq!(upgrade.start, 1); + assert_eq!(upgrade.length, 5); + assert_eq!(upgrade.nodes.len(), 2); + assert_eq!(upgrade.nodes[0].index, 5); + assert_eq!(upgrade.nodes[1].index, 9); + assert_eq!(upgrade.additional_nodes.len(), 2); + assert_eq!(upgrade.additional_nodes[0].index, 13); + assert_eq!(upgrade.additional_nodes[1].index, 17); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_block_and_seek_1_no_upgrade() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 4, nodes: 2 }), + None, + Some(RequestSeek { bytes: 8 }), + None, + ) + .await? + .unwrap(); + let block = proof.block.unwrap(); + assert_eq!(proof.seek, None); // seek included in block + assert_eq!(proof.upgrade, None); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 2); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(block.nodes[1].index, 13); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_block_and_seek_2_no_upgrade() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 4, nodes: 2 }), + None, + Some(RequestSeek { bytes: 10 }), + None, + ) + .await? + .unwrap(); + let block = proof.block.unwrap(); + assert_eq!(proof.seek, None); // seek included in block + assert_eq!(proof.upgrade, None); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 2); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(block.nodes[1].index, 13); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_block_and_seek_3_no_upgrade() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 4, nodes: 2 }), + None, + Some(RequestSeek { bytes: 13 }), + None, + ) + .await? + .unwrap(); + let block = proof.block.unwrap(); + let seek = proof.seek.unwrap(); + assert_eq!(proof.upgrade, None); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 1); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(seek.nodes.len(), 2); + assert_eq!(seek.nodes[0].index, 12); + assert_eq!(seek.nodes[1].index, 14); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_block_and_seek_to_tree_no_upgrade() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(16).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 0, nodes: 4 }), + None, + Some(RequestSeek { bytes: 26 }), + None, + ) + .await? + .unwrap(); + let block = proof.block.unwrap(); + let seek = proof.seek.unwrap(); + assert_eq!(proof.upgrade, None); + assert_eq!(block.nodes.len(), 3); + assert_eq!(block.nodes[0].index, 2); + assert_eq!(block.nodes[1].index, 5); + assert_eq!(block.nodes[2].index, 11); + assert_eq!(seek.nodes.len(), 2); + assert_eq!(seek.nodes[0].index, 19); + assert_eq!(seek.nodes[1].index, 27); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_block_and_seek_with_upgrade() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 4, nodes: 2 }), + None, + Some(RequestSeek { bytes: 13 }), + Some(RequestUpgrade { + start: 8, + length: 2, + }), + ) + .await? + .unwrap(); + let block = proof.block.unwrap(); + let seek = proof.seek.unwrap(); + let upgrade = proof.upgrade.unwrap(); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 1); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(seek.nodes.len(), 2); + assert_eq!(seek.nodes[0].index, 12); + assert_eq!(seek.nodes[1].index, 14); + assert_eq!(upgrade.nodes.len(), 1); + assert_eq!(upgrade.nodes[0].index, 17); + assert_eq!(upgrade.additional_nodes.len(), 0); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_seek_with_upgrade() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + None, + None, + Some(RequestSeek { bytes: 13 }), + Some(RequestUpgrade { + start: 0, + length: 10, + }), + ) + .await? + .unwrap(); + let seek = proof.seek.unwrap(); + let upgrade = proof.upgrade.unwrap(); + assert_eq!(proof.block, None); + assert_eq!(seek.nodes.len(), 4); + assert_eq!(seek.nodes[0].index, 12); + assert_eq!(seek.nodes[1].index, 14); + assert_eq!(seek.nodes[2].index, 9); + assert_eq!(seek.nodes[3].index, 3); + assert_eq!(upgrade.nodes.len(), 1); + assert_eq!(upgrade.nodes[0].index, 17); + assert_eq!(upgrade.additional_nodes.len(), 0); + Ok(()) + } + + #[tokio::test] + async fn core_verify_proof_invalid_signature() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + // Invalid clone hypercore with a different public key + let hypercore_clone = create_hypercore_with_data(0).await?; + let proof = hypercore + .create_proof( + None, + Some(RequestBlock { index: 6, nodes: 0 }), + None, + Some(RequestUpgrade { + start: 0, + length: 10, + }), + ) + .await? + .unwrap(); + assert!(hypercore_clone.verify_and_apply_proof(proof).await.is_err()); + Ok(()) + } + + #[tokio::test] + async fn core_verify_and_apply_proof() -> Result<(), HypercoreError> { + let main = create_hypercore_with_data(10).await?; + let clone = create_hypercore_with_data_and_key_pair( + 0, + PartialKeypair { + public: { ininner!(main).key_pair.public }, + secret: None, + }, + ) + .await?; + let index = 6; + let nodes = clone.missing_nodes(index).await?; + let proof = main + .create_proof( + None, + Some(RequestBlock { index, nodes }), + None, + Some(RequestUpgrade { + start: 0, + length: 10, + }), + ) + .await? + .unwrap(); + assert!(clone.verify_and_apply_proof(proof).await?); + let main_info = main.info(); + let clone_info = clone.info(); + assert_eq!(main_info.byte_length, clone_info.byte_length); + assert_eq!(main_info.length, clone_info.length); + assert!(main.get(6).await?.is_some()); + assert!(clone.get(6).await?.is_none()); + + // Fetch data for index 6 and verify it is found + let index = 6; + let nodes = clone.missing_nodes(index).await?; + let proof = main + .create_proof(Some(RequestBlock { index, nodes }), None, None, None) + .await? + .unwrap(); + assert!(clone.verify_and_apply_proof(proof).await?); + Ok(()) + } + + pub(crate) async fn create_hypercore_with_data( + length: u64, + ) -> Result { + let signing_key = generate_signing_key(); + create_hypercore_with_data_and_key_pair( + length, + PartialKeypair { + public: signing_key.verifying_key(), + secret: Some(signing_key), + }, + ) + .await + } + + pub(crate) async fn create_hypercore_with_data_and_key_pair( + length: u64, + key_pair: PartialKeypair, + ) -> Result { + let storage = Storage::new_memory().await?; + let hypercore = Hypercore::new( + storage, + HypercoreOptions { + key_pair: Some(key_pair), + open: false, + #[cfg(feature = "cache")] + node_cache_options: None, + }, + ) + .await?; + for i in 0..length { + hypercore.append(format!("#{}", i).as_bytes()).await?; + } + Ok(hypercore) + } +} diff --git a/src/lib.rs b/src/lib.rs index c2a4f4d0..b1d467c5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,16 +1,12 @@ -#![forbid( - unsafe_code, - future_incompatible, - rust_2018_idioms, - rust_2018_compatibility, - missing_debug_implementations, - missing_docs -)] +#![forbid(unsafe_code, future_incompatible)] #![doc(test(attr(deny(warnings))))] #![warn( unreachable_pub, redundant_lifetimes, non_local_definitions, + missing_debug_implementations, + //missing_docs, + clippy::unused_async, clippy::needless_pass_by_value, clippy::needless_pass_by_ref_mut, clippy::enum_glob_use @@ -33,33 +29,20 @@ //! //! When using disk storage, clearing values may create sparse files. On by default. //! -//! ### `async-std` (default) -//! -//! Use the async-std runtime, on by default. Either this or `tokio` is mandatory. -//! -//! ### `tokio` -//! -//! Use the tokio runtime. Either this or `async_std` is mandatory. -//! //! ### `cache` //! //! Use a moka cache for merkle tree nodes to speed-up reading. //! //! ## Example //! ```rust -//! # #[cfg(feature = "tokio")] //! # tokio_test::block_on(async { //! # example().await; //! # }); -//! # #[cfg(feature = "async-std")] -//! # async_std::task::block_on(async { -//! # example().await; -//! # }); //! # async fn example() { //! use hypercore::{HypercoreBuilder, Storage}; //! //! // Create an in-memory hypercore using a builder -//! let mut hypercore = HypercoreBuilder::new(Storage::new_memory().await.unwrap()) +//! let hypercore = HypercoreBuilder::new(Storage::new_memory().await.unwrap()) //! .build() //! .await //! .unwrap(); diff --git a/src/replication/events.rs b/src/replication/events.rs index 09f8320f..c1db6794 100644 --- a/src/replication/events.rs +++ b/src/replication/events.rs @@ -45,6 +45,18 @@ impl From<&BitfieldUpdate> for Have { } } +/// Emitted when a peer's advertised sync state changes, or when a peer is added. +/// +/// Deliberately payload-free: this is only a wake-up signal. The event channel drops messages +/// under load, so nothing may depend on receiving any particular one. Authoritative peer state +/// lives in the core's peer registry and must be read from there. +#[derive(Debug, Clone)] +pub struct PeerSync {} + +/// Emitted by [`crate::Hypercore::update`] to ask replicators to (re)send an upgrade request. +#[derive(Debug, Clone)] +pub struct Upgrade {} + #[derive(Debug, Clone)] /// Core events relevant to replication pub enum Event { @@ -54,6 +66,10 @@ pub enum Event { DataUpgrade(DataUpgrade), /// Emmitted when core gets new blocks Have(Have), + /// Emitted when a peer's advertised sync state changes + PeerSync(PeerSync), + /// Emitted when a caller wants peers asked for an upgrade + Upgrade(Upgrade), } /// Derive From for Enum where enum variant and msg have the same name @@ -70,6 +86,8 @@ macro_rules! impl_from_for_enum_variant { impl_from_for_enum_variant!(Event, Get); impl_from_for_enum_variant!(Event, DataUpgrade); impl_from_for_enum_variant!(Event, Have); +impl_from_for_enum_variant!(Event, PeerSync); +impl_from_for_enum_variant!(Event, Upgrade); #[derive(Debug)] pub(crate) struct Events { @@ -116,14 +134,14 @@ mod test { use super::*; use crate::replication::CoreMethodsError; - #[async_std::test] + #[tokio::test] async fn test_events() -> Result<(), CoreMethodsError> { - let mut core = crate::core::tests::create_hypercore_with_data(0).await?; + let core = crate::core::tests::create_hypercore_with_data(0).await?; // Check that appending data emits a DataUpgrade and Have event let mut rx = core.event_subscribe(); - let handle = async_std::task::spawn(async move { + let handle = tokio::task::spawn(async move { let mut out = vec![]; loop { if out.len() == 2 { @@ -135,7 +153,7 @@ mod test { } }); core.append(b"foo").await?; - let (res, mut rx) = handle.await; + let (res, mut rx) = handle.await.unwrap(); assert!(matches!(res[0], Event::DataUpgrade(_))); assert!(matches!( res[1], @@ -150,7 +168,7 @@ mod test { // Check that Hypercore::get for missing data emits a Get event - let handle = async_std::task::spawn(async move { + let handle = tokio::task::spawn(async move { let mut out = vec![]; loop { if out.len() == 1 { @@ -162,7 +180,7 @@ mod test { } }); assert_eq!(core.get(1).await?, None); - let (res, rx) = handle.await; + let (res, rx) = handle.await.unwrap(); assert!(matches!( res[0], Event::Get(Get { diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 35843736..cea404c0 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -1,19 +1,42 @@ //! Hypercore to Hypercore replication pub mod events; -#[cfg(feature = "shared-core")] -pub mod shared_core; +mod update; -#[cfg(feature = "shared-core")] -pub use shared_core::SharedCore; +use std::{ + collections::{BTreeSet, VecDeque}, + future::Future, + io, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicBool, AtomicU64, Ordering}, + }, + task::{Context, Poll}, +}; -use crate::{AppendOutcome, HypercoreError, Info, PartialKeypair}; +use futures::{ + Stream, StreamExt, + stream::{FuturesUnordered, SelectAll}, +}; +use hypercore_handshake::CipherTrait; +use hypercore_protocol::{ + Channel, Message, Protocol, discovery_key, + schema::{Data, Range, Request, Synchronize}, +}; +use tracing::{error, trace, warn}; -use hypercore_schema::{Proof, RequestBlock, RequestSeek, RequestUpgrade}; +use crate::{ + AppendOutcome, Hypercore, HypercoreError, Info, PartialKeypair, + core::inner::{ + CreateProofFuture, HypercoreInner, MissingNodesFuture, VerifyAndApplyProofFuture, + }, +}; +use hypercore_schema::{RequestBlock, RequestSeek, RequestUpgrade}; pub use events::Event; +pub use update::{UpdateFuture, UpdateOptions}; use async_broadcast::Receiver; -use std::future::Future; /// Methods related to just this core's information pub trait CoreInfo { @@ -39,7 +62,7 @@ pub trait ReplicationMethods: CoreInfo + Send { /// ref Core::verify_and_apply_proof fn verify_and_apply_proof( &self, - proof: &Proof, + proof: hypercore_schema::Proof, ) -> impl Future> + Send; /// ref Core::missing_nodes fn missing_nodes( @@ -53,7 +76,7 @@ pub trait ReplicationMethods: CoreInfo + Send { hash: Option, seek: Option, upgrade: Option, - ) -> impl Future, ReplicationMethodsError>> + Send; + ) -> impl Future, ReplicationMethodsError>> + Send; /// subscribe to core events fn event_subscribe(&self) -> impl Future>; } @@ -66,8 +89,7 @@ pub enum CoreMethodsError { HypercoreError(#[from] HypercoreError), } -/// Trait for things that consume [`crate::Hypercore`] can instead use this trait -/// so they can use all Hypercore-like things such as `SharedCore`. +/// Trait for things that consume [`crate::Hypercore`]. pub trait CoreMethods: CoreInfo { /// Check if the core has the block at the given index locally fn has(&self, index: u64) -> impl Future + Send; @@ -90,3 +112,762 @@ pub trait CoreMethods: CoreInfo { batch: B, ) -> impl Future> + Send; } + +// ── Remote bitfield ──────────────────────────────────────────────────────────── + +struct RemoteBitfield(BTreeSet); + +impl RemoteBitfield { + fn new() -> Self { + Self(BTreeSet::new()) + } + + fn get(&self, index: u64) -> bool { + self.0.contains(&index) + } + + fn set_range(&mut self, start: u64, length: u64, value: bool) { + for i in start..(start + length) { + if value { + self.0.insert(i); + } else { + self.0.remove(&i); + } + } + } +} + +// ── PeerSyncState ────────────────────────────────────────────────────────────── + +/// Generate paired getter/setter methods over atomic fields. +macro_rules! atomic_accessors { + ($($get:ident / $set:ident : $ty:ty),* $(,)?) => { + $( + pub(crate) fn $get(&self) -> $ty { + self.$get.load(Ordering::Relaxed) + } + pub(crate) fn $set(&self, value: $ty) { + self.$get.store(value, Ordering::Relaxed) + } + )* + }; +} + +/// The slice of one peer's protocol state that the core itself needs, in order to answer +/// "could any peer still upgrade me?" — see [`crate::Hypercore::update`]. +/// +/// Owned by the [`ChannelState`] that writes it. The core holds only [`std::sync::Weak`] +/// observers, so when a channel goes away its entry simply becomes unreachable and is pruned +/// on the next read; there is no deregistration to get wrong. +#[derive(Debug, Default)] +pub(crate) struct PeerSyncState { + /// The peer has sent us at least one [`Synchronize`]. + remote_synced: AtomicBool, + remote_fork: AtomicU64, + remote_length: AtomicU64, + /// The peer says it can serve us an upgrade starting from the length we last told it. + remote_can_upgrade: AtomicBool, + /// The length of *ours* that the peer has echoed back to us. + length_acked: AtomicU64, + /// An upgrade [`Request`] is on the wire and its [`Data`] has not been applied yet. + upgrade_inflight: AtomicBool, +} + +impl PeerSyncState { + atomic_accessors! { + remote_synced / set_remote_synced: bool, + remote_fork / set_remote_fork: u64, + remote_length / set_remote_length: u64, + remote_can_upgrade / set_remote_can_upgrade: bool, + length_acked / set_length_acked: u64, + upgrade_inflight / set_upgrade_inflight: bool, + } +} + +// ── PeerState ────────────────────────────────────────────────────────────────── + +struct PeerState { + can_upgrade: bool, + remote_bitfield: RemoteBitfield, + remote_uploading: bool, + remote_downloading: bool, + /// The subset of this state the core can observe. See [`PeerSyncState`]. + shared: Arc, +} + +impl Default for PeerState { + fn default() -> Self { + Self { + can_upgrade: true, + remote_bitfield: RemoteBitfield::new(), + remote_uploading: true, + remote_downloading: true, + shared: Arc::new(PeerSyncState::default()), + } + } +} + +// ── DataMeta ─────────────────────────────────────────────────────────────────── + +/// Saved from a Data message; used after verify_and_apply completes to decide +/// what to request next. +struct DataMeta { + has_upgrade: bool, + pre_length: u64, + remote_length: u64, + block_index: Option, +} + +// ── ChannelState ─────────────────────────────────────────────────────────────── + +struct ChannelState { + channel: Channel, + state: PeerState, + synced: bool, + + // Outgoing batches; drained one at a time through pending_send. + outgoing: VecDeque>, + pending_send: Option> + Send>>>, + + // Incoming Request → create_proof + pending_create_proof: Option, + pending_create_proof_id: u64, + pending_create_proof_fork: u64, + + // Incoming Data → verify_and_apply_proof + pending_verify_apply: Option, + pending_data_meta: Option, + + // Block request queue: each index needs a missing_nodes call before we + // can send the Request message. + pending_request_indices: VecDeque, + pending_missing_nodes: Option<(u64, MissingNodesFuture)>, + + // (our fork, our length, their length) of the last upgrade Request we sent, so that + // re-checking every poll does not re-send the same request. + last_upgrade_requested: Option<(u64, u64, u64)>, + + // Core events (Get / Have / DataUpgrade / Upgrade) + core_events: async_broadcast::Receiver, +} + +impl ChannelState { + /// Registers this peer with `inner`, so that [`crate::Hypercore::update`] can see its + /// advertised length. The registry holds a `Weak`, so dropping this `ChannelState` is + /// all the deregistration there is. + fn new(channel: Channel, inner: &HypercoreInner) -> Self { + let state = PeerState::default(); + inner.register_peer(&state.shared); + inner.send_event(events::PeerSync {}); + Self { + channel, + state, + synced: false, + outgoing: VecDeque::new(), + pending_send: None, + pending_create_proof: None, + pending_create_proof_id: 0, + pending_create_proof_fork: 0, + pending_verify_apply: None, + pending_data_meta: None, + pending_request_indices: VecDeque::new(), + pending_missing_nodes: None, + last_upgrade_requested: None, + core_events: inner.event_subscribe(), + } + } + + fn poll( + &mut self, + cx: &mut Context<'_>, + inner: &HypercoreInner, + ) -> Poll> { + // ── Drive pending send ───────────────────────────────────────────────── + if let Some(ref mut fut) = self.pending_send { + match fut.as_mut().poll(cx) { + Poll::Ready(Ok(())) => self.pending_send = None, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e.into())), + Poll::Pending => {} + } + } + if self.pending_send.is_none() + && let Some(batch) = self.outgoing.pop_front() + { + let channel = self.channel.clone(); + self.pending_send = Some(Box::pin(async move { channel.send_batch(&batch).await })); + cx.waker().wake_by_ref(); + } + + // ── Initial sync ─────────────────────────────────────────────────────── + if !self.synced { + let info = inner.info(); + let remote_length = if info.fork == self.state.shared.remote_fork() { + self.state.shared.remote_length() + } else { + 0 + }; + let mut msgs = vec![Message::Synchronize(Synchronize { + fork: info.fork, + length: info.length, + remote_length, + can_upgrade: self.state.can_upgrade, + uploading: true, + downloading: true, + })]; + if info.contiguous_length > 0 { + msgs.push(Message::Range(Range { + drop: false, + start: 0, + length: info.contiguous_length, + })); + } + self.outgoing.push_back(msgs); + self.synced = true; + cx.waker().wake_by_ref(); + return Poll::Pending; + } + + // ── Poll core events ─────────────────────────────────────────────────── + loop { + match Pin::new(&mut self.core_events).poll_next(cx) { + Poll::Ready(Some(event)) => self.on_core_event(event, inner), + Poll::Ready(None) => break, + Poll::Pending => break, + } + } + + // ── Drive pending_missing_nodes ──────────────────────────────────────── + if let Some((index, ref mut fut)) = self.pending_missing_nodes { + match Pin::new(fut).poll(cx) { + Poll::Ready(Ok(nodes)) => { + self.pending_missing_nodes = None; + if self.state.remote_bitfield.get(index) + && self.state.shared.remote_length() > index + { + let info = inner.info(); + self.outgoing.push_back(vec![Message::Request(Request { + id: index + 1, + fork: info.fork, + block: Some(RequestBlock { index, nodes }), + hash: None, + seek: None, + upgrade: None, + manifest: false, + priority: 42, + })]); + cx.waker().wake_by_ref(); + } + if let Some(next) = self.pending_request_indices.pop_front() { + self.pending_missing_nodes = + Some((next, inner.missing_nodes_from_merkle_tree_index(next * 2))); + cx.waker().wake_by_ref(); + } + } + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Pending => {} + } + } else if let Some(index) = self.pending_request_indices.pop_front() { + self.pending_missing_nodes = + Some((index, inner.missing_nodes_from_merkle_tree_index(index * 2))); + cx.waker().wake_by_ref(); + } + + // ── Drive pending_create_proof ───────────────────────────────────────── + if let Some(ref mut fut) = self.pending_create_proof { + match Pin::new(fut).poll(cx) { + Poll::Ready(Ok(maybe_proof)) => { + let req_id = self.pending_create_proof_id; + let req_fork = self.pending_create_proof_fork; + self.pending_create_proof = None; + if let Some(proof) = maybe_proof { + self.outgoing.push_back(vec![Message::Data(Data { + request: req_id, + fork: req_fork, + hash: proof.hash, + block: proof.block, + seek: proof.seek, + upgrade: proof.upgrade, + })]); + cx.waker().wake_by_ref(); + } + } + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Pending => {} + } + } + + // ── Drive pending_verify_apply ───────────────────────────────────────── + if let Some(ref mut fut) = self.pending_verify_apply { + match Pin::new(fut).poll(cx) { + Poll::Ready(Ok(_applied)) => { + self.pending_verify_apply = None; + if let Some(meta) = self.pending_data_meta.take() { + if meta.has_upgrade { + self.state.shared.set_upgrade_inflight(false); + } + let next_index = if meta.has_upgrade { + (meta.pre_length < meta.remote_length).then_some(meta.pre_length) + } else { + meta.block_index + .filter(|&i| i < meta.remote_length.saturating_sub(1)) + .map(|i| i + 1) + }; + if let Some(idx) = next_index { + self.pending_request_indices.push_front(idx); + cx.waker().wake_by_ref(); + } + } + } + Poll::Ready(Err(e)) => { + if self + .pending_data_meta + .take() + .is_some_and(|meta| meta.has_upgrade) + { + self.state.shared.set_upgrade_inflight(false); + } + return Poll::Ready(Err(e)); + } + Poll::Pending => {} + } + } + + // ── Re-check whether we should ask this peer for an upgrade ──────────── + // The peer's Synchronize may have arrived at a moment when we could not act on it + // (e.g. it had not yet acked our length). This is the analogue of JS's `updateAll`. + if self.maybe_request_upgrade(inner) { + cx.waker().wake_by_ref(); + } + + // ── Poll channel for incoming messages ───────────────────────────────── + match Pin::new(&mut self.channel).poll_next(cx) { + Poll::Ready(Some(message)) => { + self.on_message(message, inner); + cx.waker().wake_by_ref(); + } + Poll::Ready(None) => return Poll::Ready(Ok(())), + Poll::Pending => {} + } + + Poll::Pending + } + + fn on_message(&mut self, message: Message, inner: &HypercoreInner) { + match message { + Message::Synchronize(msg) => self.on_synchronize(&msg, inner), + Message::Request(msg) => { + if self.pending_create_proof.is_none() { + self.pending_create_proof_id = msg.id; + self.pending_create_proof_fork = msg.fork; + self.pending_create_proof = + Some(inner.create_proof(msg.block, msg.hash, msg.seek, msg.upgrade)); + } + } + Message::Data(msg) => { + if self.pending_verify_apply.is_none() { + let info = inner.info(); + self.pending_data_meta = Some(DataMeta { + has_upgrade: msg.upgrade.is_some(), + pre_length: info.length, + remote_length: self.state.shared.remote_length(), + block_index: msg.block.as_ref().map(|b| b.index), + }); + self.pending_verify_apply = + Some(inner.verify_and_apply_proof(msg.into_proof())); + } + } + Message::Range(Range { start, length, .. }) => { + self.state.remote_bitfield.set_range(start, length, true); + } + _ => {} + } + } + + fn on_synchronize(&mut self, msg: &Synchronize, inner: &HypercoreInner) { + let info = inner.info(); + let first_sync = !self.state.shared.remote_synced(); + let same_fork = msg.fork == info.fork; + + let shared = &self.state.shared; + shared.set_remote_fork(msg.fork); + shared.set_remote_length(msg.length); + shared.set_remote_can_upgrade(msg.can_upgrade); + shared.set_remote_synced(true); + shared.set_length_acked(if same_fork { msg.remote_length } else { 0 }); + self.state.remote_uploading = msg.uploading; + self.state.remote_downloading = msg.downloading; + + if first_sync { + self.outgoing + .push_back(vec![Message::Synchronize(Synchronize { + fork: info.fork, + length: info.length, + remote_length: self.state.shared.remote_length(), + can_upgrade: self.state.can_upgrade, + uploading: true, + downloading: true, + })]); + } + + self.maybe_request_upgrade(inner); + + // Wake any `Hypercore::update` waiting on this peer's advertised length. + inner.send_event(events::PeerSync {}); + } + + /// Ask this peer to grow our verified length, if it has advertised a longer one and we are + /// not already waiting on an upgrade from it. Returns whether a request was queued. + /// + /// Called both when a [`Synchronize`] arrives and once per poll, because the conditions can + /// become true after the fact — most often `length_acked` catching up to our own length. + /// This is the analogue of JS's `_updatePeerNonPrimary` upgrade branch. + fn maybe_request_upgrade(&mut self, inner: &HypercoreInner) -> bool { + let info = inner.info(); + let shared = &self.state.shared; + if !shared.remote_synced() || shared.upgrade_inflight() { + return false; + } + let remote_length = shared.remote_length(); + if remote_length <= info.length || shared.length_acked() != info.length { + return false; + } + // Don't re-send the same request on every poll. + let request = (info.fork, info.length, remote_length); + if self.last_upgrade_requested == Some(request) { + return false; + } + self.last_upgrade_requested = Some(request); + shared.set_upgrade_inflight(true); + self.outgoing.push_back(vec![Message::Request(Request { + id: 1, + fork: info.fork, + hash: None, + block: None, + seek: None, + upgrade: Some(RequestUpgrade { + start: info.length, + length: remote_length - info.length, + }), + manifest: false, + priority: 42, + })]); + true + } + + fn on_core_event(&mut self, event: events::Event, inner: &HypercoreInner) { + match event { + events::Event::Get(evt) => { + if self.state.shared.remote_length() > evt.index + && self.state.remote_bitfield.get(evt.index) + { + self.pending_request_indices.push_back(evt.index); + } + } + events::Event::Have(evt) => { + self.outgoing.push_back(vec![Message::Range(Range { + drop: evt.drop, + start: evt.start, + length: evt.length, + })]); + } + events::Event::DataUpgrade(_) => { + let info = inner.info(); + self.outgoing + .push_back(vec![Message::Synchronize(Synchronize { + fork: info.fork, + length: info.length, + remote_length: self.state.shared.remote_length(), + downloading: true, + uploading: true, + can_upgrade: self.state.can_upgrade, + })]); + } + // Pure wake-ups. `Upgrade` is handled by the `maybe_request_upgrade` re-check that + // runs every poll, and `PeerSync` is only ever of interest to `Hypercore::update`. + events::Event::PeerSync(_) | events::Event::Upgrade(_) => {} + } + } +} + +// ── ConnectionReplicator ─────────────────────────────────────────────────────── + +/// Drives replication for a single peer connection. Used internally by +/// [`Replicator`]. +struct ConnectionReplicator { + inner: HypercoreInner, + protocol: Protocol, + discovery_key: [u8; 32], + public_key: [u8; 32], + pending_open: Option> + Send>>>, + channel_state: Option, +} + +impl ConnectionReplicator { + fn new(inner: HypercoreInner, stream: impl CipherTrait + 'static) -> Self { + let protocol = Protocol::new(Box::new(stream)); + let public_key = inner.key_pair().public.to_bytes(); + let discovery_key = discovery_key(&public_key); + Self { + inner, + protocol, + discovery_key, + public_key, + pending_open: None, + channel_state: None, + } + } + + fn poll_inner(&mut self, cx: &mut Context<'_>) -> Poll> { + if let Some(ref mut fut) = self.pending_open { + match fut.as_mut().poll(cx) { + Poll::Ready(Ok(())) => self.pending_open = None, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e.into())), + Poll::Pending => return Poll::Pending, + } + } + + match Pin::new(&mut self.protocol).poll_next(cx) { + Poll::Ready(Some(Ok(event))) => { + self.on_protocol_event(event); + cx.waker().wake_by_ref(); + } + Poll::Ready(Some(Err(e))) => return Poll::Ready(Err(e.into())), + Poll::Ready(None) => return Poll::Ready(Ok(())), + Poll::Pending => {} + } + + if let Some(ref mut cs) = self.channel_state { + match cs.poll(cx, &self.inner) { + Poll::Ready(Ok(())) => self.channel_state = None, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Pending => {} + } + } + + Poll::Pending + } + + fn on_protocol_event(&mut self, event: hypercore_protocol::Event) { + match event { + hypercore_protocol::Event::Handshake(_) => { + if self.protocol.is_initiator() { + self.pending_open = Some(Box::pin(self.protocol.open(self.public_key))); + } + } + hypercore_protocol::Event::DiscoveryKey(dkey) => { + if self.discovery_key == dkey { + self.pending_open = Some(Box::pin(self.protocol.open(self.public_key))); + } else { + warn!("Got discovery key for different core: {dkey:?}"); + } + } + hypercore_protocol::Event::Channel(channel) => { + if self.discovery_key == *channel.discovery_key() { + self.channel_state = Some(ChannelState::new(channel, &self.inner)); + } else { + error!("Wrong discovery key?"); + } + } + hypercore_protocol::Event::Close(_) => {} + _ => {} + } + } +} + +impl Future for ConnectionReplicator { + type Output = Result<(), HypercoreError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.get_mut().poll_inner(cx) + } +} + +// ── Replicator ───────────────────────────────────────────────────────────────── + +type BoxReplicatorStream = Pin + Send>>; + +/// Drives replication for one or more peer connections. +/// +/// Created by [`Hypercore::replicator`]. Add connections via +/// [`with_connection`](Replicator::with_connection) or whole connection streams +/// via [`with_connection_stream`](Replicator::with_connection_stream), then +/// `.await` to drive all replication. Resolves when all connections have closed +/// and all connection streams have ended. +pub struct Replicator { + inner: HypercoreInner, + active: FuturesUnordered, + pending: SelectAll, +} + +impl std::fmt::Debug for Replicator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Replicator") + .field("active_count", &self.active.len()) + .finish_non_exhaustive() + } +} + +impl Replicator { + fn new(inner: HypercoreInner) -> Self { + Self { + inner, + active: FuturesUnordered::new(), + pending: SelectAll::new(), + } + } + + /// Add a single connection to replicate over. + pub fn with_connection(self, stream: impl CipherTrait + 'static) -> Self { + self.active + .push(ConnectionReplicator::new(self.inner.clone(), stream)); + self + } + + /// Add a stream of connections. Each connection yielded by the stream will + /// be replicated in parallel with all others. + /// + /// ```rust,ignore + /// core.replicator() + /// .with_connection_stream(swarm.connections().filter_map(|r| async move { + /// r.ok().map(|e| e.connection) + /// })) + /// .await?; + /// ``` + pub fn with_connection_stream(mut self, stream: S) -> Self + where + S: Stream + Send + 'static, + C: CipherTrait + 'static, + { + let inner = self.inner.clone(); + let boxed: BoxReplicatorStream = + Box::pin(stream.map(move |conn| ConnectionReplicator::new(inner.clone(), conn))); + self.pending.push(boxed); + self + } +} + +impl Future for Replicator { + type Output = Result<(), HypercoreError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + // Drain pending connection streams → push new ConnectionReplicators into active. + loop { + let pending_result = Pin::new(&mut this.pending).poll_next(cx); + trace!( + "[replicator] pending.poll_next = {:?}", + match &pending_result { + Poll::Ready(Some(_)) => "Ready(Some(conn))", + Poll::Ready(None) => "Ready(None)", + Poll::Pending => "Pending", + } + ); + match pending_result { + Poll::Ready(Some(rep)) => this.active.push(rep), + Poll::Ready(None) | Poll::Pending => break, + } + } + + // Drive all active connection replicators. + loop { + match Pin::new(&mut this.active).poll_next(cx) { + Poll::Ready(Some(Ok(()))) => {} + Poll::Ready(Some(Err(e))) => return Poll::Ready(Err(e)), + Poll::Ready(None) | Poll::Pending => break, + } + } + + if this.pending.is_empty() && this.active.is_empty() { + Poll::Ready(Ok(())) + } else { + Poll::Pending + } + } +} + +// ── ChannelReplicator ────────────────────────────────────────────────────────── + +/// Drives replication for a single core over a [`Channel`] already opened on a +/// [`Protocol`] someone else owns (e.g. a corestore-like multiplexer serving +/// several cores over one physical connection). +/// +/// Unlike [`ConnectionReplicator`], this does not create or drive a `Protocol` +/// itself — it only runs the per-core wire protocol over an already-open +/// channel. Returned as an opaque `impl Future` from [`Hypercore::attach_channel`]. +struct ChannelReplicator { + inner: HypercoreInner, + state: ChannelState, +} + +impl ChannelReplicator { + fn new(inner: HypercoreInner, channel: Channel) -> Self { + Self { + state: ChannelState::new(channel, &inner), + inner, + } + } +} + +impl Future for ChannelReplicator { + type Output = Result<(), HypercoreError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + this.state.poll(cx, &this.inner) + } +} + +// ── Hypercore::replicator / replicate ───────────────────────────────────────── +impl Hypercore { + /// Create a [`Replicator`] for this core. Add connections or connection + /// streams, then `.await` to drive all replication. + /// + /// ```rust,ignore + /// // Single connection + /// core.replicator().with_connection(stream).await?; + /// + /// // Stream of connections (e.g. from hyperswarm) + /// core.replicator() + /// .with_connection_stream(swarm.connections().filter_map(|r| async move { + /// r.ok().map(|e| e.connection) + /// })) + /// .await?; + /// ``` + pub fn replicator(&self) -> Replicator { + Replicator::new(self.inner.clone()) + } + + /// Shorthand for `self.replicator().with_connection(stream)`. + pub fn replicate(&self, stream: impl CipherTrait + 'static) -> Replicator { + self.replicator().with_connection(stream) + } + + /// Attach a replicator to this core so that [`Hypercore::get`] automatically + /// drives replication while waiting for missing blocks. + /// + /// Once attached, a `get` for a block that is not yet locally available will + /// block (without spinning) until replication delivers it, rather than + /// returning `None` immediately. + /// + /// Only one replicator can be attached at a time; calling this again replaces + /// the previous one. + pub fn attach_replicator(&self, replicator: Replicator) { + *self.inner.background.lock().unwrap() = Some(Box::pin(replicator)); + } + + /// Drive this core's replication over a [`Channel`] already opened on a + /// [`Protocol`] the caller owns and drives itself. + /// + /// Use this (instead of [`Hypercore::replicate`]) when multiplexing several + /// cores over one physical connection: the caller owns a single `Protocol` + /// for the connection, opens a channel per core's discovery key, and calls + /// this once per core with the resulting channel. Resolves when the channel + /// closes. + pub fn attach_channel( + &self, + channel: Channel, + ) -> impl Future> + Send + 'static { + ChannelReplicator::new(self.inner.clone(), channel) + } +} diff --git a/src/replication/shared_core.rs b/src/replication/shared_core.rs deleted file mode 100644 index 80418062..00000000 --- a/src/replication/shared_core.rs +++ /dev/null @@ -1,190 +0,0 @@ -//! Implementation of a Hypercore that can have multiple owners. Along with implementations of all -//! the hypercore traits. -use crate::{AppendOutcome, Hypercore, Info, PartialKeypair}; -use async_broadcast::Receiver; -use async_lock::Mutex; -use hypercore_schema::{Proof, RequestBlock, RequestSeek, RequestUpgrade}; -use std::sync::Arc; - -use super::{ - CoreInfo, CoreMethods, CoreMethodsError, Event, ReplicationMethods, ReplicationMethodsError, -}; - -/// Hypercore that can have multiple owners -#[derive(Debug, Clone)] -pub struct SharedCore(pub Arc>); - -impl From for SharedCore { - fn from(core: Hypercore) -> Self { - SharedCore(Arc::new(Mutex::new(core))) - } -} -impl SharedCore { - /// Create a shared core from a [`Hypercore`] - pub fn from_hypercore(core: Hypercore) -> Self { - SharedCore(Arc::new(Mutex::new(core))) - } -} - -impl CoreInfo for SharedCore { - async fn info(&self) -> Info { - let core = &self.0.lock().await; - core.info() - } - - async fn key_pair(&self) -> PartialKeypair { - let core = &self.0.lock().await; - core.key_pair().clone() - } -} - -impl ReplicationMethods for SharedCore { - async fn verify_and_apply_proof(&self, proof: &Proof) -> Result { - Ok(self.0.lock().await.verify_and_apply_proof(proof).await?) - } - - async fn missing_nodes(&self, index: u64) -> Result { - Ok(self.0.lock().await.missing_nodes(index).await?) - } - - async fn create_proof( - &self, - block: Option, - hash: Option, - seek: Option, - upgrade: Option, - ) -> Result, ReplicationMethodsError> { - Ok(self - .0 - .lock() - .await - .create_proof(block, hash, seek, upgrade) - .await?) - } - - async fn event_subscribe(&self) -> Receiver { - self.0.lock().await.event_subscribe() - } -} - -impl CoreMethods for SharedCore { - async fn has(&self, index: u64) -> bool { - self.0.lock().await.has(index) - } - async fn get(&self, index: u64) -> Result>, CoreMethodsError> { - Ok(self.0.lock().await.get(index).await?) - } - - async fn append(&self, data: &[u8]) -> Result { - Ok(self.0.lock().await.append(data).await?) - } - - async fn append_batch, B: AsRef<[A]> + Send>( - &self, - batch: B, - ) -> Result { - Ok(self.0.lock().await.append_batch(batch).await?) - } -} - -#[cfg(test)] -mod tests { - - use super::*; - - use crate::core::tests::{create_hypercore_with_data, create_hypercore_with_data_and_key_pair}; - #[async_std::test] - async fn shared_core_methods() -> Result<(), CoreMethodsError> { - let core = crate::core::tests::create_hypercore_with_data(0).await?; - let core = SharedCore::from(core); - - // check CoreInfo - let info = core.info().await; - assert_eq!( - info, - crate::core::Info { - length: 0, - byte_length: 0, - contiguous_length: 0, - fork: 0, - writeable: true, - } - ); - - // key_pair is random, nothing to test here - let _kp = core.key_pair().await; - - // check CoreMethods - assert_eq!(core.has(0).await, false); - assert_eq!(core.get(0).await?, None); - let res = core.append(b"foo").await?; - assert_eq!( - res, - AppendOutcome { - length: 1, - byte_length: 3 - } - ); - assert_eq!(core.has(0).await, true); - assert_eq!(core.get(0).await?, Some(b"foo".into())); - let res = core.append_batch([b"hello", b"world"]).await?; - assert_eq!( - res, - AppendOutcome { - length: 3, - byte_length: 13 - } - ); - assert_eq!(core.has(2).await, true); - assert_eq!(core.get(2).await?, Some(b"world".into())); - Ok(()) - } - - #[async_std::test] - async fn shared_core_replication_methods() -> Result<(), ReplicationMethodsError> { - let main = create_hypercore_with_data(10).await?; - let clone = create_hypercore_with_data_and_key_pair( - 0, - PartialKeypair { - public: main.key_pair.public, - secret: None, - }, - ) - .await?; - - let main = SharedCore::from(main); - let clone = SharedCore::from(clone); - - let index = 6; - let nodes = clone.missing_nodes(index).await?; - let proof = main - .create_proof( - None, - Some(RequestBlock { index, nodes }), - None, - Some(RequestUpgrade { - start: 0, - length: 10, - }), - ) - .await? - .unwrap(); - assert!(clone.verify_and_apply_proof(&proof).await?); - let main_info = main.info().await; - let clone_info = clone.info().await; - assert_eq!(main_info.byte_length, clone_info.byte_length); - assert_eq!(main_info.length, clone_info.length); - assert!(main.get(6).await?.is_some()); - assert!(clone.get(6).await?.is_none()); - - // Fetch data for index 6 and verify it is found - let index = 6; - let nodes = clone.missing_nodes(index).await?; - let proof = main - .create_proof(Some(RequestBlock { index, nodes }), None, None, None) - .await? - .unwrap(); - assert!(clone.verify_and_apply_proof(&proof).await?); - Ok(()) - } -} diff --git a/src/replication/update.rs b/src/replication/update.rs new file mode 100644 index 00000000..ed85d921 --- /dev/null +++ b/src/replication/update.rs @@ -0,0 +1,385 @@ +//! Fetching the newest state of a core from its peers. +//! +//! See [`Hypercore::update`]. + +use std::{ + future::Future, + pin::Pin, + sync::Arc, + task::{Context, Poll}, + time::Duration, +}; + +use futures::Stream as _; + +use crate::{ + Hypercore, HypercoreError, Info, + core::inner::{BackgroundFuture, HypercoreInner}, + replication::{PeerSyncState, events}, +}; + +/// Options for [`Hypercore::update`]. +#[derive(Debug, Clone)] +pub struct UpdateOptions { + /// Wait for peers to connect and tell us their length. + /// + /// When `false`, [`Hypercore::update`] answers from what currently-connected peers have + /// already said and never blocks on a peer arriving — so with no peers it resolves + /// `Ok(false)` immediately. + /// + /// Defaults to `true`. + pub wait: bool, + /// Run even on a writable core, which otherwise resolves `Ok(false)` without doing + /// anything. Defaults to `false`. + pub force: bool, + /// Give up after this long and return [`HypercoreError::Timeout`]. + /// + /// `None` (the default) means wait indefinitely. Note that `wait: true` with no timeout + /// and no peers really does block forever, exactly like Javascript's + /// `core.update({ wait: true })`. + pub timeout: Option, +} + +impl Default for UpdateOptions { + fn default() -> Self { + Self { + wait: true, + force: false, + timeout: None, + } + } +} + +/// State captured on the first poll, against which "did we grow?" is judged. +struct Snapshot { + fork: u64, + length: u64, +} + +/// Future returned by [`Hypercore::update`]. +/// +/// Resolves `Ok(true)` when the core's verified length (or fork) moved, `Ok(false)` when the +/// peers confirmed there is nothing newer, and [`HypercoreError::Timeout`] when +/// [`UpdateOptions::timeout`] elapsed with nobody having answered. +pub struct UpdateFuture { + inner: HypercoreInner, + opts: UpdateOptions, + /// `None` until the first poll, which is also what makes the first poll special. + snapshot: Option, + events: Option>, + /// Replicator attached via [`Hypercore::attach_replicator`], if any. Driven in-band, the + /// same way [`crate::Hypercore::get`] does it. + background: BackgroundFuture, + sleep: Option>>, +} + +impl std::fmt::Debug for UpdateFuture { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UpdateFuture") + .field("opts", &self.opts) + .finish_non_exhaustive() + } +} + +impl UpdateFuture { + pub(crate) fn new(inner: HypercoreInner, opts: UpdateOptions) -> Self { + let background = inner.background.clone(); + Self { + inner, + opts, + snapshot: None, + events: None, + background, + sleep: None, + } + } + + /// Drive an attached replicator, so that `update` works under + /// [`Hypercore::attach_replicator`] as well as under an externally driven one. + /// + /// `Ok(None)` means no replicator is attached — replication, if any, is driven elsewhere + /// and we can conclude nothing from that. `Ok(Some(true))` means one was attached and has + /// now run to completion, so no further peer will ever appear through it. + fn poll_background(&self, cx: &mut Context<'_>) -> Result, HypercoreError> { + let mut background = self.background.lock().unwrap(); + let Some(fut) = background.as_mut() else { + return Ok(None); + }; + match fut.as_mut().poll(cx) { + Poll::Ready(Ok(())) => { + *background = None; + Ok(Some(true)) + } + Poll::Ready(Err(e)) => Err(e), + Poll::Pending => Ok(Some(false)), + } + } + + /// Drain pending events. Payloads are ignored on purpose: the event channel drops + /// messages under load, so events are treated purely as "something changed, look again". + fn drain_events(&mut self, cx: &mut Context<'_>) { + let Some(events) = self.events.as_mut() else { + return; + }; + while let Poll::Ready(Some(_)) = Pin::new(&mut *events).poll_next(cx) {} + } + +} + +/// Whether some peer might still upgrade us, i.e. whether it is too early to answer `false`. +/// A port of Javascript's `_checkUpgradeIfAvailable` (`lib/replicator.js`). +/// +/// Every peer gets a say: one peer having nothing newer says nothing about the others, so a +/// useless peer must `continue` rather than end the search. Javascript only consults the +/// first `MAX_PEERS_UPGRADE` (3) peers; we check them all, since a core here has a handful at +/// most. +/// +/// Free-standing (rather than a method) so it can be unit tested against hand-built peer +/// state — the multi-peer cases are impractical to stage deterministically over real +/// connections, because which peer registers first is a race. +fn any_peer_may_upgrade( + peers: &[Arc], + info: &Info, + snapshot: &Snapshot, + wait: bool, +) -> bool { + for peer in peers { + // Hasn't told us anything yet. Under `wait: false` we don't hang around for it. + if !peer.remote_synced() { + if wait { + return true; + } + continue; + } + // We have nothing at all and the peer does: it can certainly upgrade us. + if info.length == 0 && peer.remote_length() > 0 { + return true; + } + // This peer has nothing we don't already have, or is on another fork. + if peer.remote_length() <= snapshot.length || peer.remote_fork() != snapshot.fork { + continue; + } + // It hasn't acked our current length, so its `can_upgrade` claim is stale. + if peer.length_acked() != info.length && peer.remote_fork() == info.fork { + return true; + } + if peer.remote_can_upgrade() || peer.upgrade_inflight() { + return true; + } + } + false +} + +impl Future for UpdateFuture { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + // ── First poll ───────────────────────────────────────────────────────── + if this.snapshot.is_none() { + let info = this.inner.info(); + if info.writeable && !this.opts.force { + return Poll::Ready(Ok(false)); + } + // Subscribe before reading any state, so no wake-up can slip between the two. + this.events = Some(this.inner.event_subscribe()); + this.snapshot = Some(Snapshot { + fork: info.fork, + length: info.length, + }); + // Nudge replicators that are connected but idle into re-checking whether they + // should ask for an upgrade. + this.inner.send_event(events::Upgrade {}); + this.sleep = this + .opts + .timeout + .map(|timeout| Box::pin(tokio::time::sleep(timeout))); + } + + let background = match this.poll_background(cx) { + Ok(background) => background, + Err(e) => return Poll::Ready(Err(e)), + }; + + this.drain_events(cx); + + // ── Did we grow? ─────────────────────────────────────────────────────── + let info = this.inner.info(); + let snapshot = this.snapshot.as_ref().expect("set above"); + if info.length != snapshot.length || info.fork != snapshot.fork { + return Poll::Ready(Ok(true)); + } + + // ── Could anyone still grow us? ──────────────────────────────────────── + let peers = this.inner.peers(); + if peers.is_empty() && !this.opts.wait { + return Poll::Ready(Ok(false)); + } + let snapshot = this.snapshot.as_ref().expect("set above"); + if !peers.is_empty() && !any_peer_may_upgrade(&peers, &info, snapshot, this.opts.wait) { + return Poll::Ready(Ok(false)); + } + + // An attached replicator that ran to completion means every connection it owned has + // closed, so no peer is ever going to show up through it. + if background == Some(true) && peers.is_empty() { + return Poll::Ready(Ok(false)); + } + + // ── Give up? ─────────────────────────────────────────────────────────── + if let Some(sleep) = this.sleep.as_mut() + && sleep.as_mut().poll(cx).is_ready() + { + return Poll::Ready(Err(HypercoreError::Timeout { + context: "Hypercore::update timed out waiting for a peer".to_string(), + })); + } + + Poll::Pending + } +} + +impl Hypercore { + /// Fetch the newest state of this core from its peers. + /// + /// Resolves `Ok(true)` if the core's verified length grew, `Ok(false)` if the peers + /// confirmed there is nothing newer (or the core is writable — pass + /// [`UpdateOptions::force`] to update one anyway), and [`HypercoreError::Timeout`] if + /// [`UpdateOptions::timeout`] elapsed before any peer answered. + /// + /// This is the "is there anything new?" primitive: it asks peers to grow this core's + /// *verified* length, which is a different operation from downloading blocks. Use + /// [`Hypercore::get`] for the latter. + /// + /// Replication must be driven by someone: either a spawned [`Replicator`](super::Replicator) + /// / channel future, or one attached with [`Hypercore::attach_replicator`], in which case + /// this future drives it while it waits. + /// + /// ```rust,ignore + /// // Block until a peer connects and tells us where it is, but no longer than 5s. + /// let grew = core.update(UpdateOptions { + /// timeout: Some(Duration::from_secs(5)), + /// ..Default::default() + /// }).await?; + /// ``` + pub fn update(&self, opts: UpdateOptions) -> UpdateFuture { + UpdateFuture::new(self.inner.clone(), opts) + } + + /// The number of peers currently replicating this core. + pub fn peer_count(&self) -> usize { + self.inner.peers().len() + } +} + +#[cfg(test)] +mod test { + use super::*; + + /// Peer state as it would be after a `Synchronize` from a peer at `remote_length`. + fn synced_peer(remote_length: u64, our_length: u64) -> Arc { + let peer = Arc::new(PeerSyncState::default()); + peer.set_remote_synced(true); + peer.set_remote_length(remote_length); + peer.set_length_acked(our_length); + peer.set_remote_can_upgrade(true); + peer + } + + fn unsynced_peer() -> Arc { + Arc::new(PeerSyncState::default()) + } + + fn info(length: u64) -> Info { + Info { + length, + byte_length: 0, + contiguous_length: length, + fork: 0, + writeable: false, + } + } + + fn snapshot(length: u64) -> Snapshot { + Snapshot { fork: 0, length } + } + + /// The case the peer registry exists for: a peer with nothing newer must not end the + /// search, because a later peer may still have more. Consulting only `peers[0]` here + /// would wrongly answer "nobody can upgrade us". + #[test] + fn useless_peer_does_not_mask_a_useful_later_one() { + let peers = vec![synced_peer(2, 2), synced_peer(5, 2)]; + assert!(any_peer_may_upgrade(&peers, &info(2), &snapshot(2), true)); + // ...and the same two peers in the other order, so this cannot pass by luck. + let peers = vec![synced_peer(5, 2), synced_peer(2, 2)]; + assert!(any_peer_may_upgrade(&peers, &info(2), &snapshot(2), true)); + } + + #[test] + fn all_peers_in_sync_means_nothing_newer() { + let peers = vec![synced_peer(2, 2), synced_peer(2, 2), synced_peer(1, 2)]; + assert!(!any_peer_may_upgrade(&peers, &info(2), &snapshot(2), true)); + } + + #[test] + fn no_peers_means_nothing_newer() { + assert!(!any_peer_may_upgrade(&[], &info(2), &snapshot(2), true)); + } + + /// An unsynced peer is worth waiting for, but only when the caller asked to wait. + #[test] + fn unsynced_peer_is_awaited_only_when_waiting() { + let peers = vec![synced_peer(2, 2), unsynced_peer()]; + assert!(any_peer_may_upgrade(&peers, &info(2), &snapshot(2), true)); + assert!(!any_peer_may_upgrade(&peers, &info(2), &snapshot(2), false)); + } + + /// An empty core with a peer that has anything at all is always worth waiting for. + #[test] + fn empty_core_waits_for_any_peer_with_data() { + let peers = vec![synced_peer(1, 0)]; + assert!(any_peer_may_upgrade(&peers, &info(0), &snapshot(0), true)); + } + + /// A peer on another fork cannot help us, even though it looks longer. + #[test] + fn peer_on_another_fork_cannot_help() { + let peer = synced_peer(9, 2); + peer.set_remote_fork(7); + assert!(!any_peer_may_upgrade(&[peer], &info(2), &snapshot(2), true)); + } + + /// A peer that has more but has not acked our current length: its `can_upgrade` claim is + /// stale, so we wait for the next sync rather than concluding either way. + #[test] + fn peer_with_stale_ack_is_awaited() { + let peer = synced_peer(5, 2); + peer.set_length_acked(1); // has not caught up to our length of 2 + peer.set_remote_can_upgrade(false); + assert!(any_peer_may_upgrade(&[peer], &info(2), &snapshot(2), true)); + } + + /// A peer that has more, has acked us, but says it cannot serve the upgrade, and has + /// nothing in flight, is a dead end. + #[test] + fn peer_that_cannot_upgrade_and_has_nothing_inflight_is_a_dead_end() { + let peer = synced_peer(5, 2); + peer.set_remote_can_upgrade(false); + assert!(!any_peer_may_upgrade( + std::slice::from_ref(&peer), + &info(2), + &snapshot(2), + true + )); + // ...unless we are already waiting on a response from it. + peer.set_upgrade_inflight(true); + assert!(any_peer_may_upgrade( + std::slice::from_ref(&peer), + &info(2), + &snapshot(2), + true + )); + } +} diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 749192b9..aeab3b18 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -4,10 +4,10 @@ use futures::future::FutureExt; #[cfg(not(target_arch = "wasm32"))] use random_access_disk::RandomAccessDisk; use random_access_memory::RandomAccessMemory; -use random_access_storage::{RandomAccess, RandomAccessError}; -use std::fmt::Debug; +use random_access_storage::{BoxFuture, RandomAccess, RandomAccessError}; #[cfg(not(target_arch = "wasm32"))] use std::path::PathBuf; +use std::{fmt::Debug, sync::Arc}; use tracing::instrument; use crate::{ @@ -16,40 +16,16 @@ use crate::{ }; /// Supertrait for Storage -pub trait StorageTraits: RandomAccess + Debug {} -impl StorageTraits for T {} +pub trait StorageTraits: RandomAccess + Debug + Send + Sync {} +impl StorageTraits for T {} /// Save data to a desired storage backend. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Storage { - tree: Box, - data: Box, - bitfield: Box, - oplog: Box, -} - -pub(crate) fn map_random_access_err(err: RandomAccessError) -> HypercoreError { - match err { - RandomAccessError::IO { - return_code, - context, - source, - } => HypercoreError::IO { - context: Some(format!( - "RandomAccess IO error. Context: {context:?}, return_code: {return_code:?}", - )), - source, - }, - RandomAccessError::OutOfBounds { - offset, - end, - length, - } => HypercoreError::InvalidOperation { - context: format!( - "RandomAccess out of bounds. Offset: {offset}, end: {end:?}, length: {length}", - ), - }, - } + tree: Arc, + data: Arc, + bitfield: Arc, + oplog: Arc, } impl Storage { @@ -60,182 +36,182 @@ impl Storage { Store, ) -> std::pin::Pin< Box< - dyn std::future::Future< - Output = Result, RandomAccessError>, - > + Send, + dyn std::future::Future, RandomAccessError>> + + Send, >, >, { - let mut tree = create(Store::Tree).await.map_err(map_random_access_err)?; - let mut data = create(Store::Data).await.map_err(map_random_access_err)?; - let mut bitfield = create(Store::Bitfield) - .await - .map_err(map_random_access_err)?; - let mut oplog = create(Store::Oplog).await.map_err(map_random_access_err)?; + let tree: Arc = create(Store::Tree).await?; + let data: Arc = create(Store::Data).await?; + let bitfield: Arc = create(Store::Bitfield).await?; + let oplog: Arc = create(Store::Oplog).await?; if overwrite { - if tree.len().await.map_err(map_random_access_err)? > 0 { - tree.truncate(0).await.map_err(map_random_access_err)?; + if tree.len() > 0 { + tree.truncate(0).await?; } - if data.len().await.map_err(map_random_access_err)? > 0 { - data.truncate(0).await.map_err(map_random_access_err)?; + if data.len() > 0 { + data.truncate(0).await?; } - if bitfield.len().await.map_err(map_random_access_err)? > 0 { - bitfield.truncate(0).await.map_err(map_random_access_err)?; + if bitfield.len() > 0 { + bitfield.truncate(0).await?; } - if oplog.len().await.map_err(map_random_access_err)? > 0 { - oplog.truncate(0).await.map_err(map_random_access_err)?; + if oplog.len() > 0 { + oplog.truncate(0).await?; } } - let instance = Self { + Ok(Self { tree, data, bitfield, oplog, - }; - - Ok(instance) + }) } - /// Read info from store based on given instruction. Convenience method to `read_infos`. - pub(crate) async fn read_info( - &mut self, + /// Read info from store based on given instruction. + pub(crate) fn read_info( + &self, info_instruction: StoreInfoInstruction, - ) -> Result { - let mut infos = self.read_infos_to_vec(&[info_instruction]).await?; - Ok(infos - .pop() - .expect("Should have gotten one info with one instruction")) + ) -> BoxFuture> { + let fut = self.read_infos_to_vec(vec![info_instruction]); + Box::pin(async move { + Ok(fut + .await? + .pop() + .expect("Should have gotten one info with one instruction")) + }) } /// Read infos from stores based on given instructions - pub(crate) async fn read_infos( - &mut self, - info_instructions: &[StoreInfoInstruction], - ) -> Result, HypercoreError> { - let infos = self.read_infos_to_vec(info_instructions).await?; - Ok(infos.into_boxed_slice()) + pub(crate) fn read_infos( + &self, + info_instructions: Vec, + ) -> BoxFuture, HypercoreError>> { + let fut = self.read_infos_to_vec(info_instructions); + Box::pin(async move { Ok(fut.await?.into_boxed_slice()) }) } /// Reads infos but retains them as a Vec - pub(crate) async fn read_infos_to_vec( - &mut self, - info_instructions: &[StoreInfoInstruction], - ) -> Result, HypercoreError> { - if info_instructions.is_empty() { - return Ok(vec![]); - } - let mut current_store: Store = info_instructions[0].store.clone(); - let mut storage = self.get_random_access_mut(¤t_store); - let mut infos: Vec = Vec::with_capacity(info_instructions.len()); - for instruction in info_instructions.iter() { - if instruction.store != current_store { - current_store = instruction.store.clone(); - storage = self.get_random_access_mut(¤t_store); + pub(crate) fn read_infos_to_vec( + &self, + info_instructions: Vec, + ) -> BoxFuture, HypercoreError>> { + let storage = self.clone(); + let instructions = info_instructions; // TODO rm + Box::pin(async move { + if instructions.is_empty() { + return Ok(vec![]); } - match instruction.info_type { - StoreInfoType::Content => { - let read_length = match instruction.length { - Some(length) => length, - None => storage.len().await.map_err(map_random_access_err)?, - }; - let read_result = storage.read(instruction.index, read_length).await; - let info: StoreInfo = match read_result { - Ok(buf) => Ok(StoreInfo::new_content( + let mut current_store: Store = instructions[0].store.clone(); + let mut ra: Arc = storage.get_random_access(¤t_store).clone(); + let mut infos: Vec = Vec::with_capacity(instructions.len()); + for instruction in instructions.iter() { + if instruction.store != current_store { + current_store = instruction.store.clone(); + ra = storage.get_random_access(¤t_store).clone(); + } + match instruction.info_type { + StoreInfoType::Content => { + let read_length = match instruction.length { + Some(length) => length, + None => ra.len(), + }; + let read_result = ra.read(instruction.index, read_length).await; + let info: StoreInfo = match read_result { + Ok(buf) => Ok(StoreInfo::new_content( + instruction.store.clone(), + instruction.index, + &buf, + )), + Err(RandomAccessError::OutOfBounds { length, .. }) => { + if instruction.allow_miss { + Ok(StoreInfo::new_content_miss( + instruction.store.clone(), + instruction.index, + )) + } else { + Err(HypercoreError::InvalidOperation { + context: format!( + "Could not read from store {}, index {} / length {} is out of bounds for store length {}", + current_store, instruction.index, read_length, length + ), + }) + } + } + Err(e) => Err(HypercoreError::from(e)), + }?; + infos.push(info); + } + StoreInfoType::Size => { + let length = ra.len(); + infos.push(StoreInfo::new_size( instruction.store.clone(), instruction.index, - &buf, - )), - Err(RandomAccessError::OutOfBounds { length, .. }) => { - if instruction.allow_miss { - Ok(StoreInfo::new_content_miss( - instruction.store.clone(), - instruction.index, - )) - } else { - Err(HypercoreError::InvalidOperation { - context: format!( - "Could not read from store {}, index {} / length {} is out of bounds for store length {}", - current_store, instruction.index, read_length, length - ), - }) - } - } - Err(e) => Err(map_random_access_err(e)), - }?; - infos.push(info); - } - StoreInfoType::Size => { - let length = storage.len().await.map_err(map_random_access_err)?; - infos.push(StoreInfo::new_size( - instruction.store.clone(), - instruction.index, - length - instruction.index, - )); + length - instruction.index, + )); + } } } - } - Ok(infos) + Ok(infos) + }) } - /// Flush info to storage. Convenience method to `flush_infos`. - pub(crate) async fn flush_info(&mut self, slice: StoreInfo) -> Result<(), HypercoreError> { - self.flush_infos(&[slice]).await + /// Flush info to storage. + pub(crate) fn flush_info(&self, info: StoreInfo) -> BoxFuture> { + self.flush_infos(vec![info]) } /// Flush infos to storage - pub(crate) async fn flush_infos(&mut self, infos: &[StoreInfo]) -> Result<(), HypercoreError> { - if infos.is_empty() { - return Ok(()); - } - let mut current_store: Store = infos[0].store.clone(); - let mut storage = self.get_random_access_mut(¤t_store); - for info in infos.iter() { - if info.store != current_store { - current_store = info.store.clone(); - storage = self.get_random_access_mut(¤t_store); + pub(crate) fn flush_infos( + &self, + infos: Vec, + ) -> BoxFuture> { + let storage = self.clone(); + Box::pin(async move { + if infos.is_empty() { + return Ok(()); } - match info.info_type { - StoreInfoType::Content => { - if !info.miss { - if let Some(data) = &info.data { - storage - .write(info.index, data) - .await - .map_err(map_random_access_err)?; - } - } else { - storage - .del( + let mut current_store: Store = infos[0].store.clone(); + let mut ra: Arc = storage.get_random_access(¤t_store).clone(); + for info in infos.iter() { + if info.store != current_store { + current_store = info.store.clone(); + ra = storage.get_random_access(¤t_store).clone(); + } + match info.info_type { + StoreInfoType::Content => { + if !info.miss { + if let Some(data) = &info.data { + ra.write(info.index, data).await?; + } + } else { + ra.del( info.index, info.length.expect("When deleting, length must be given"), ) - .await - .map_err(map_random_access_err)?; + .await?; + } } - } - StoreInfoType::Size => { - if info.miss { - storage - .truncate(info.index) - .await - .map_err(map_random_access_err)?; - } else { - panic!("Flushing a size that isn't miss, is not supported"); + StoreInfoType::Size => { + if info.miss { + ra.truncate(info.index).await?; + } else { + panic!("Flushing a size that isn't miss, is not supported"); + } } } } - } - Ok(()) + Ok(()) + }) } - fn get_random_access_mut(&mut self, store: &Store) -> &mut Box { + fn get_random_access(&self, store: &Store) -> &Arc { match store { - Store::Tree => &mut self.tree, - Store::Data => &mut self.data, - Store::Bitfield => &mut self.bitfield, - Store::Oplog => &mut self.oplog, + Store::Tree => &self.tree, + Store::Data => &self.data, + Store::Bitfield => &self.bitfield, + Store::Oplog => &self.oplog, } } @@ -243,10 +219,8 @@ impl Storage { #[instrument(err)] pub async fn new_memory() -> Result { let create = |_| { - async { Ok(Box::new(RandomAccessMemory::default()) as Box) } - .boxed() + async { Ok(Arc::new(RandomAccessMemory::default()) as Arc) }.boxed() }; - // No reason to overwrite, as this is a new memory segment Self::open(create, false).await } @@ -264,8 +238,8 @@ impl Storage { Store::Oplog => "oplog", }; Ok( - Box::new(RandomAccessDisk::open(dir.as_path().join(name)).await?) - as Box, + Arc::new(RandomAccessDisk::open(dir.as_path().join(name)).await?) + as Arc, ) } .boxed() @@ -273,3 +247,67 @@ impl Storage { Self::open(storage, overwrite).await } } + +#[cfg(test)] +mod test { + use super::*; + use crate::common::{StoreInfo, StoreInfoInstruction}; + + #[tokio::test] + async fn test_futures_are_owned() -> Result<(), Box> { + let fut = { + let storage = Storage::new_memory().await?; + + let data = b"hello hypercore"; + + // Write to tree store + storage.flush_info(StoreInfo::new_content(Store::Tree, 0, data)) + }; + fut.await?; + + Ok(()) + } + + #[tokio::test] + async fn test_storage() -> Result<(), Box> { + let storage = Storage::new_memory().await?; + + let data = b"hello hypercore"; + + // Write to tree store + storage + .flush_info(StoreInfo::new_content(Store::Tree, 0, data)) + .await?; + + // Read it back + let info = storage + .read_info(StoreInfoInstruction::new_content( + Store::Tree, + 0, + data.len() as u64, + )) + .await?; + + assert_eq!(info.data.as_deref(), Some(data.as_slice())); + + // Write to two different stores, read back together + storage + .flush_infos(vec![ + StoreInfo::new_content(Store::Data, 0, b"block0"), + StoreInfo::new_content(Store::Bitfield, 0, b"bits"), + ]) + .await?; + + let infos = storage + .read_infos(vec![ + StoreInfoInstruction::new_content(Store::Data, 0, 6), + StoreInfoInstruction::new_content(Store::Bitfield, 0, 4), + ]) + .await?; + + assert_eq!(infos[0].data.as_deref(), Some(b"block0".as_slice())); + assert_eq!(infos[1].data.as_deref(), Some(b"bits".as_slice())); + + Ok(()) + } +} diff --git a/tests/core.rs b/tests/core.rs index 7b39a76b..76c493f1 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -4,21 +4,15 @@ use anyhow::Result; use common::{create_hypercore, get_test_key_pair, open_hypercore, storage_contains_data}; use hypercore::{HypercoreBuilder, Storage}; use tempfile::Builder; -use test_log::test; -#[cfg(feature = "async-std")] -use async_std::test as async_test; -#[cfg(feature = "tokio")] -use tokio::test as async_test; - -#[test(async_test)] +#[tokio::test] async fn hypercore_new() -> Result<()> { let storage = Storage::new_memory().await?; let _hypercore = HypercoreBuilder::new(storage).build(); Ok(()) } -#[test(async_test)] +#[tokio::test] async fn hypercore_new_with_key_pair() -> Result<()> { let storage = Storage::new_memory().await?; let key_pair = get_test_key_pair(); @@ -29,7 +23,7 @@ async fn hypercore_new_with_key_pair() -> Result<()> { Ok(()) } -#[test(async_test)] +#[tokio::test] async fn hypercore_open_with_key_pair_error() -> Result<()> { let storage = Storage::new_memory().await?; let key_pair = get_test_key_pair(); @@ -44,14 +38,14 @@ async fn hypercore_open_with_key_pair_error() -> Result<()> { Ok(()) } -#[test(async_test)] +#[tokio::test] async fn hypercore_make_read_only() -> Result<()> { let dir = Builder::new() .prefix("hypercore_make_read_only") .tempdir() .unwrap(); let write_key_pair = { - let mut hypercore = create_hypercore(&dir.path().to_string_lossy()).await?; + let hypercore = create_hypercore(&dir.path().to_string_lossy()).await?; hypercore.append(b"Hello").await?; hypercore.append(b"World!").await?; hypercore.key_pair().clone() @@ -74,7 +68,7 @@ async fn hypercore_make_read_only() -> Result<()> { &write_key_pair.secret.as_ref().unwrap().to_bytes()[16..], )); - let mut hypercore = open_hypercore(&dir.path().to_string_lossy()).await?; + let hypercore = open_hypercore(&dir.path().to_string_lossy()).await?; assert_eq!(&hypercore.get(0).await?.unwrap(), b"Hello"); assert_eq!(&hypercore.get(1).await?.unwrap(), b"World!"); Ok(()) diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 5d02d737..6b94185f 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -5,12 +5,6 @@ use std::sync::Once; use anyhow::Result; use common::{create_hypercore, create_hypercore_hash, open_hypercore}; use js::{cleanup, install, js_run_step, prepare_test_set}; -use test_log::test; - -#[cfg(feature = "async-std")] -use async_std::test as async_test; -#[cfg(feature = "tokio")] -use tokio::test as async_test; const TEST_SET_JS_FIRST: &str = "jsfirst"; const TEST_SET_RS_FIRST: &str = "rsfirst"; @@ -24,7 +18,7 @@ fn init() { }); } -#[test(async_test)] +#[tokio::test] #[cfg_attr(not(feature = "js_interop_tests"), ignore)] async fn js_interop_js_first() -> Result<()> { init(); @@ -43,7 +37,7 @@ async fn js_interop_js_first() -> Result<()> { Ok(()) } -#[test(async_test)] +#[tokio::test] #[cfg_attr(not(feature = "js_interop_tests"), ignore)] async fn js_interop_rs_first() -> Result<()> { init(); @@ -68,7 +62,7 @@ async fn step_1_create(work_dir: &str) -> Result<()> { } async fn step_2_append_hello_world(work_dir: &str) -> Result<()> { - let mut hypercore = open_hypercore(work_dir).await?; + let hypercore = open_hypercore(work_dir).await?; let batch: &[&[u8]] = &[b"Hello", b"World"]; let append_outcome = hypercore.append_batch(batch).await?; assert_eq!(append_outcome.length, 2); @@ -77,7 +71,7 @@ async fn step_2_append_hello_world(work_dir: &str) -> Result<()> { } async fn step_3_read_and_append_unflushed(work_dir: &str) -> Result<()> { - let mut hypercore = open_hypercore(work_dir).await?; + let hypercore = open_hypercore(work_dir).await?; let hello = hypercore.get(0).await?; assert_eq!(hello.unwrap(), b"Hello"); let world = hypercore.get(1).await?; @@ -109,7 +103,7 @@ async fn step_3_read_and_append_unflushed(work_dir: &str) -> Result<()> { } async fn step_4_append_with_flush(work_dir: &str) -> Result<()> { - let mut hypercore = open_hypercore(work_dir).await?; + let hypercore = open_hypercore(work_dir).await?; for i in 0..5 { let append_outcome = hypercore.append(&[i]).await?; assert_eq!(append_outcome.length, (6 + i + 1) as u64); diff --git a/tests/model.rs b/tests/model.rs index 86a74657..342bc0fe 100644 --- a/tests/model.rs +++ b/tests/model.rs @@ -44,13 +44,6 @@ proptest! { })] #[test] - #[cfg(feature = "async-std")] - fn implementation_matches_model(ops: Vec) { - assert!(async_std::task::block_on(assert_implementation_matches_model(ops))); - } - - #[test] - #[cfg(feature = "tokio")] fn implementation_matches_model(ops: Vec) { let rt = tokio::runtime::Runtime::new().unwrap(); assert!(rt.block_on(async { diff --git a/tests/replication.rs b/tests/replication.rs new file mode 100644 index 00000000..1825350c --- /dev/null +++ b/tests/replication.rs @@ -0,0 +1,434 @@ +#![cfg(feature = "replication")] + +use hypercore::{ + Hypercore, HypercoreBuilder, HypercoreError, PartialKeypair, Storage, + replication::UpdateOptions, +}; +use hypercore_handshake::{ + Cipher, CipherTrait, + state_machine::{SecStream, hc_specific::generate_keypair}, +}; +use std::time::Duration; +use tokio_util::compat::TokioAsyncReadCompatExt; +use uint24le_framing::Uint24LELengthPrefixedFraming; + +/// Create a pair of connected in-memory encrypted streams. +fn connected_pair() -> (impl CipherTrait + 'static, impl CipherTrait + 'static) { + let (a_b, b_a) = tokio::io::duplex(64 * 1024); + let a_b = Uint24LELengthPrefixedFraming::new(a_b.compat()); + let b_a = Uint24LELengthPrefixedFraming::new(b_a.compat()); + let keypair = generate_keypair().unwrap(); + let initiator = Cipher::new( + Some(Box::new(a_b)), + SecStream::new_initiator_xx(&[]).unwrap().into(), + ); + let responder = Cipher::new( + Some(Box::new(b_a)), + SecStream::new_responder_xx(&keypair, &[]).unwrap().into(), + ); + (initiator, responder) +} + +/// Create a writer (with data) and a blank reader sharing the same public key. +async fn make_writer_reader(data: &[&[u8]]) -> (Hypercore, Hypercore) { + let writer = HypercoreBuilder::new(Storage::new_memory().await.unwrap()) + .build() + .await + .unwrap(); + for chunk in data { + writer.append(chunk).await.unwrap(); + } + let public_key = writer.key_pair().public; + let reader = HypercoreBuilder::new(Storage::new_memory().await.unwrap()) + .key_pair(PartialKeypair { + public: public_key, + secret: None, + }) + .build() + .await + .unwrap(); + (writer, reader) +} + +/// Another read-only replica of `core`'s core, for building multi-peer setups. +async fn make_replica(core: &Hypercore) -> Hypercore { + HypercoreBuilder::new(Storage::new_memory().await.unwrap()) + .key_pair(PartialKeypair { + public: core.key_pair().public, + secret: None, + }) + .build() + .await + .unwrap() +} + +/// Get block 0 from `reader`, with `writer`'s replicator spawned to drive the other end. +/// Uses `attach_replicator` so that `reader.get()` itself drives replication (structured +/// concurrency path). +async fn get_via_attached_replicator(writer: &Hypercore, reader: &Hypercore) -> Option> { + let (writer_stream, reader_stream) = connected_pair(); + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + reader.attach_replicator(reader.replicate(reader_stream)); + let block = tokio::time::timeout(Duration::from_secs(5), reader.get(0)) + .await + .expect("timed out waiting for attach_replicator get") + .unwrap(); + writer_rep.abort(); + block +} + +/// Poll until `core.info().contiguous_length >= expected`, with a 5-second timeout. +/// +/// Note this cannot be replaced by `Hypercore::update`: `update` resolves once the *verified +/// length* has caught up, which says nothing about whether the blocks themselves have been +/// downloaded yet. These tests read blocks back, so they need `contiguous_length`. +async fn wait_for_length(core: &Hypercore, expected: u64) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + let mut last_reported = u64::MAX; + loop { + let current = core.info().contiguous_length; + if current >= expected { + return; + } + if current != last_reported { + eprintln!("contiguous_length = {current} (waiting for {expected})"); + last_reported = current; + } + assert!( + tokio::time::Instant::now() < deadline, + "timed out: contiguous_length stuck at {current}, never reached {expected}" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } +} + +/// `UpdateOptions` with a deadline, so a broken `update` fails the test instead of hanging it. +fn update_opts(timeout_ms: u64) -> UpdateOptions { + UpdateOptions { + timeout: Some(Duration::from_millis(timeout_ms)), + ..Default::default() + } +} + +/// `update` drives an attached replicator in-band, exactly as `get` does: nothing else is +/// polling the reader's side of the connection here. +#[tokio::test] +async fn update_drives_attached_replicator() { + let (writer, reader) = make_writer_reader(&[b"hello", b"world"]).await; + let (writer_stream, reader_stream) = connected_pair(); + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + reader.attach_replicator(reader.replicate(reader_stream)); + + assert!(reader.update(update_opts(5000)).await.unwrap()); + assert_eq!(reader.info().length, 2); + + writer_rep.abort(); +} + +/// The reader is at length 0 and stays there until `update` is polled, so the append is +/// guaranteed to be news by the time `update` takes its snapshot. +#[tokio::test] +async fn update_reports_growth_for_late_append() { + let (writer, reader) = make_writer_reader(&[]).await; + let (writer_stream, reader_stream) = connected_pair(); + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + reader.attach_replicator(reader.replicate(reader_stream)); + + writer.append(b"late").await.unwrap(); + + assert!(reader.update(update_opts(5000)).await.unwrap()); + assert_eq!(reader.info().length, 1); + + writer_rep.abort(); +} + +/// Once caught up, a second `update` reports that the peer has nothing newer. +#[tokio::test] +async fn update_returns_false_when_in_sync() { + let (writer, reader) = make_writer_reader(&[b"hello", b"world"]).await; + let (writer_stream, reader_stream) = connected_pair(); + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + reader.attach_replicator(reader.replicate(reader_stream)); + + assert!(reader.update(update_opts(5000)).await.unwrap()); + assert!(!reader.update(update_opts(5000)).await.unwrap()); + assert_eq!(reader.info().length, 2); + + writer_rep.abort(); +} + +/// `wait: false` answers from what is already known rather than waiting for a peer. +#[tokio::test] +async fn update_no_wait_returns_false_without_peers() { + let (_, reader) = make_writer_reader(&[b"hello"]).await; + assert_eq!(reader.peer_count(), 0); + + let updated = reader + .update(UpdateOptions { + wait: false, + ..Default::default() + }) + .await + .unwrap(); + assert!(!updated); +} + +/// `wait: true` with no peers blocks, and the timeout is what ends it. +#[tokio::test] +async fn update_times_out_without_peers() { + let (_, reader) = make_writer_reader(&[b"hello"]).await; + + let err = reader.update(update_opts(100)).await.unwrap_err(); + assert!( + matches!(err, HypercoreError::Timeout { .. }), + "expected Timeout, got {err:?}" + ); +} + +/// A writable core has nothing to fetch, so `update` short-circuits unless forced. +#[tokio::test] +async fn update_on_writable_core_returns_false() { + let (writer, _) = make_writer_reader(&[b"hello"]).await; + assert!(writer.info().writeable); + + // Not forced: returns immediately without waiting for the (nonexistent) peers. + assert!(!writer.update(update_opts(5000)).await.unwrap()); + + // Forced: actually runs, and so hits the timeout waiting for a peer. + let err = writer + .update(UpdateOptions { + force: true, + timeout: Some(Duration::from_millis(100)), + ..Default::default() + }) + .await + .unwrap_err(); + assert!( + matches!(err, HypercoreError::Timeout { .. }), + "expected Timeout, got {err:?}" + ); +} + +/// Two peers where only one has anything, so the per-peer loop has to cope with a mix. +/// +/// This deliberately does *not* assert the returned bool. `update` answers from the peers +/// connected at the moment it decides — matching Javascript's `_checkUpgradeIfAvailable`, +/// which likewise iterates only `this.peers` — so if the empty peer happens to register and +/// sync before the writer's channel opens, `Ok(false)` is the correct answer. What is +/// guaranteed either way is the postcondition: `update` does not return while a peer it can +/// see is known to have more. +#[tokio::test] +async fn update_with_two_peers_only_one_useful() { + let (writer, reader) = make_writer_reader(&[b"hello", b"world"]).await; + let empty_peer = make_replica(&writer).await; + + let (writer_stream, reader_stream_a) = connected_pair(); + let (empty_stream, reader_stream_b) = connected_pair(); + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + let empty_rep = tokio::spawn(empty_peer.replicate(empty_stream)); + + reader.attach_replicator( + reader + .replicator() + .with_connection(reader_stream_a) + .with_connection(reader_stream_b), + ); + + // Retry to absorb the registration race described above: whichever peer wins, the reader + // must end up at the writer's length rather than stalling on the empty peer. + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while reader.info().length < 2 { + assert!( + tokio::time::Instant::now() < deadline, + "reader stuck at length {} with 2 peers attached", + reader.info().length + ); + reader.update(update_opts(1000)).await.unwrap(); + } + assert_eq!(reader.info().length, 2); + assert_eq!(reader.peer_count(), 2, "both peers should be registered"); + + writer_rep.abort(); + empty_rep.abort(); +} + +/// With two peers and nothing newer anywhere, `update` reports false — but only after both +/// have been heard from. +#[tokio::test] +async fn update_with_two_peers_all_in_sync() { + let (writer, reader) = make_writer_reader(&[b"hello", b"world"]).await; + let other = make_replica(&writer).await; + + let (writer_stream, reader_stream_a) = connected_pair(); + let (other_stream, reader_stream_b) = connected_pair(); + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + let other_rep = tokio::spawn(other.replicate(other_stream)); + + reader.attach_replicator( + reader + .replicator() + .with_connection(reader_stream_a) + .with_connection(reader_stream_b), + ); + + assert!(reader.update(update_opts(5000)).await.unwrap()); + assert!(!reader.update(update_opts(5000)).await.unwrap()); + assert_eq!(reader.info().length, 2); + assert_eq!(reader.peer_count(), 2); + + writer_rep.abort(); + other_rep.abort(); +} + +/// `update` also works when replication is driven by someone else entirely (a spawned +/// replicator, as corestore does). Whether it reports growth is a race against the +/// background task, but it must not return before the reader has caught up. +#[tokio::test] +async fn update_with_externally_driven_replication() { + let (writer, reader) = make_writer_reader(&[b"a", b"b", b"c"]).await; + let (writer_stream, reader_stream) = connected_pair(); + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + let reader_rep = tokio::spawn(reader.replicate(reader_stream)); + + reader.update(update_opts(5000)).await.unwrap(); + assert_eq!(reader.info().length, 3); + assert_eq!(reader.peer_count(), 1); + + writer_rep.abort(); + reader_rep.abort(); +} + +#[tokio::test] +async fn replicate_data_before_connect() { + let (writer, reader) = make_writer_reader(&[b"hello", b"world"]).await; + let (writer_stream, reader_stream) = connected_pair(); + + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + let reader_rep = tokio::spawn(reader.replicate(reader_stream)); + + wait_for_length(&reader, 2).await; + + assert_eq!(reader.get(0).await.unwrap(), Some(b"hello".to_vec())); + assert_eq!(reader.get(1).await.unwrap(), Some(b"world".to_vec())); + + writer_rep.abort(); + reader_rep.abort(); +} + +#[tokio::test] +async fn replicate_data_after_connect() { + let (writer, reader) = make_writer_reader(&[]).await; + let (writer_stream, reader_stream) = connected_pair(); + + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + let reader_rep = tokio::spawn(reader.replicate(reader_stream)); + + writer.append(b"late data").await.unwrap(); + + wait_for_length(&reader, 1).await; + + assert_eq!(reader.get(0).await.unwrap(), Some(b"late data".to_vec())); + + writer_rep.abort(); + reader_rep.abort(); +} + +/// Without an attached replicator, `get()` returns `None` immediately for a missing block. +#[tokio::test] +async fn get_returns_none_without_replicator() { + let (_, reader) = make_writer_reader(&[b"hello"]).await; + assert_eq!(reader.get(0).await.unwrap(), None); +} + +/// `attach_replicator` makes `reader.get()` drive replication itself — no spawn needed for +/// the reader side. +#[tokio::test] +async fn attach_replicator_drives_get() { + let (writer, reader) = make_writer_reader(&[b"hello", b"world"]).await; + assert_eq!( + get_via_attached_replicator(&writer, &reader).await, + Some(b"hello".to_vec()) + ); +} + +/// `attach_replicator` still works when the writer appends the block *after* the connection +/// is established. +#[tokio::test] +async fn attach_replicator_drives_get_late_data() { + let (writer, reader) = make_writer_reader(&[]).await; + let (writer_stream, reader_stream) = connected_pair(); + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + reader.attach_replicator(reader.replicate(reader_stream)); + + writer.append(b"late").await.unwrap(); + + let block = tokio::time::timeout(Duration::from_secs(5), reader.get(0)) + .await + .expect("timed out") + .unwrap(); + assert_eq!(block, Some(b"late".to_vec())); + writer_rep.abort(); +} + +/// Sequential gets — after `get(0)` resolves, `get(1)` and `get(2)` must also work. +#[tokio::test] +async fn attach_replicator_gets_sequential_blocks() { + let (writer, reader) = make_writer_reader(&[b"a", b"b", b"c"]).await; + let (writer_stream, reader_stream) = connected_pair(); + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + reader.attach_replicator(reader.replicate(reader_stream)); + + for (i, expected) in [b"a".as_slice(), b"b", b"c"].iter().enumerate() { + let block = tokio::time::timeout(Duration::from_secs(5), reader.get(i as u64)) + .await + .unwrap_or_else(|_| panic!("timed out on block {i}")) + .unwrap(); + assert_eq!(block.as_deref(), Some(*expected), "block {i}"); + } + writer_rep.abort(); +} + +/// Get a non-zero block directly without fetching earlier indices first. +/// Verifies that `Event::Get` fires with the requested index, not always 0. +#[tokio::test] +async fn attach_replicator_get_by_index() { + let data: Vec> = (0u8..5).map(|i| vec![i]).collect(); + let slices: Vec<&[u8]> = data.iter().map(|v| v.as_slice()).collect(); + let (writer, reader) = make_writer_reader(&slices).await; + let (writer_stream, reader_stream) = connected_pair(); + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + reader.attach_replicator(reader.replicate(reader_stream)); + + let block = tokio::time::timeout(Duration::from_secs(5), reader.get(4)) + .await + .expect("timed out") + .unwrap(); + assert_eq!(block, Some(vec![4u8])); + writer_rep.abort(); +} + +#[tokio::test] +async fn replicate_many_blocks() { + let data: Vec> = (0u8..10).map(|i| vec![i]).collect(); + let slices: Vec<&[u8]> = data.iter().map(|d| d.as_slice()).collect(); + + let (writer, reader) = make_writer_reader(&slices).await; + let (writer_stream, reader_stream) = connected_pair(); + + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + let reader_rep = tokio::spawn(reader.replicate(reader_stream)); + + wait_for_length(&reader, 10).await; + + for (i, expected) in data.iter().enumerate() { + assert_eq!( + reader.get(i as u64).await.unwrap().as_ref(), + Some(expected), + "block {i} mismatch" + ); + } + + writer_rep.abort(); + reader_rep.abort(); +} +