Rewrite async methods to return owned futures.

.github/workflows/ci.yml

@@ -34,23 +34,18 @@ jobs:
         components: rustfmt
     - name: Run tests
       run: |
-        cargo check --all-targets --no-default-features --features tokio
-        cargo check --all-targets --no-default-features --features tokio,sparse
-        cargo check --all-targets --no-default-features --features tokio,sparse,cache
-        cargo check --all-targets --no-default-features --features async-std
-        cargo check --all-targets --no-default-features --features async-std,sparse
-        cargo check --all-targets --no-default-features --features async-std,sparse,cache
-        cargo test --no-default-features --features js_interop_tests,tokio
-        cargo test --no-default-features --features js_interop_tests,tokio,shared-core
-        cargo test --no-default-features --features js_interop_tests,tokio,sparse
-        cargo test --no-default-features --features js_interop_tests,tokio,sparse,cache
-        cargo test --no-default-features --features js_interop_tests,async-std
-        cargo test --no-default-features --features js_interop_tests,async-std,shared-core
-        cargo test --no-default-features --features js_interop_tests,async-std,sparse
-        cargo test --no-default-features --features js_interop_tests,async-std,sparse,cache
-        cargo test --benches --no-default-features --features tokio
-        cargo test --benches --no-default-features --features async-std
-
+        cargo check --all-targets
+        cargo check --all-targets --all-features
+        cargo check --all-targets --no-default-features
+        cargo check --all-targets --no-default-features --features sparse
+        cargo check --all-targets --no-default-features --features sparse,cache
+        cargo test
+        cargo test --all-features
+        cargo test --no-default-features
+        cargo test --no-default-features --features js_interop_tests
+        cargo test --no-default-features --features js_interop_tests,sparse
+        cargo test --no-default-features --features js_interop_tests,sparse,cache
+        cargo test --benches --no-default-features
   test-windows:
     runs-on: windows-latest
 
@@ -61,22 +56,17 @@ jobs:
           components: rustfmt
       - name: Run tests
         run: |
-          cargo check --all-targets --no-default-features --features tokio
-          cargo check --all-targets --no-default-features --features tokio,sparse
-          cargo check --all-targets --no-default-features --features tokio,sparse,cache
-          cargo check --all-targets --no-default-features --features async-std
-          cargo check --all-targets --no-default-features --features async-std,sparse
-          cargo check --all-targets --no-default-features --features async-std,sparse,cache
-          cargo test --no-default-features --features tokio
-          cargo test --no-default-features --features tokio,shared-core
-          cargo test --no-default-features --features tokio,sparse
-          cargo test --no-default-features --features tokio,sparse,cache
-          cargo test --no-default-features --features async-std
-          cargo test --no-default-features --features async-std,shared-core
-          cargo test --no-default-features --features async-std,sparse
-          cargo test --no-default-features --features async-std,sparse,cache
-          cargo test --benches --no-default-features --features tokio
-          cargo test --benches --no-default-features --features async-std
+          cargo check --all-targets
+          cargo check --all-targets --all-features
+          cargo check --all-targets --no-default-features
+          cargo check --all-targets --no-default-features --features sparse
+          cargo check --all-targets --no-default-features --features sparse,cache
+          cargo test
+          cargo test --all-features
+          cargo test --no-default-features
+          cargo test --no-default-features --features sparse
+          cargo test --no-default-features --features sparse,cache
+          cargo test --benches --no-default-features
 
   test-macos:
     runs-on: macos-latest
@@ -88,22 +78,18 @@ jobs:
           components: rustfmt
       - name: Run tests
         run: |
-          cargo check --all-targets --no-default-features --features tokio
-          cargo check --all-targets --no-default-features --features tokio,sparse
-          cargo check --all-targets --no-default-features --features tokio,sparse,cache
-          cargo check --all-targets --no-default-features --features async-std
-          cargo check --all-targets --no-default-features --features async-std,sparse
-          cargo check --all-targets --no-default-features --features async-std,sparse,cache
-          cargo test --no-default-features --features js_interop_tests,tokio
-          cargo test --no-default-features --features js_interop_tests,tokio,shared-core
-          cargo test --no-default-features --features js_interop_tests,tokio,sparse
-          cargo test --no-default-features --features js_interop_tests,tokio,sparse,cache
-          cargo test --no-default-features --features js_interop_tests,async-std
-          cargo test --no-default-features --features js_interop_tests,async-std,shared-core
-          cargo test --no-default-features --features js_interop_tests,async-std,sparse
-          cargo test --no-default-features --features js_interop_tests,async-std,sparse,cache
-          cargo test --benches --no-default-features --features tokio
-          cargo test --benches --no-default-features --features async-std
+          cargo check --all-targets
+          cargo check --all-targets --all-features
+          cargo check --all-targets --no-default-features
+          cargo check --all-targets --no-default-features --features sparse
+          cargo check --all-targets --no-default-features --features sparse,cache
+          cargo test
+          cargo test --all-features
+          cargo test --no-default-features
+          cargo test --no-default-features --features js_interop_tests
+          cargo test --no-default-features --features js_interop_tests,sparse
+          cargo test --no-default-features --features js_interop_tests,sparse,cache
+          cargo test --benches --no-default-features
 
   build-extra:
     runs-on: ubuntu-latest
@@ -115,24 +101,17 @@ jobs:
           targets: wasm32-unknown-unknown
       - name: Build WASM
         run: |
-          cargo build --target=wasm32-unknown-unknown --no-default-features --features tokio
-          cargo build --target=wasm32-unknown-unknown --no-default-features --features async-std
+          cargo build --target=wasm32-unknown-unknown --no-default-features
       - name: Build release
         run: |
-          cargo build --release --no-default-features --features tokio
-          cargo build --release --no-default-features --features tokio,sparse
-          cargo build --release --no-default-features --features tokio,sparse,cache
-          cargo build --release --no-default-features --features async-std
-          cargo build --release --no-default-features --features async-std,sparse
-          cargo build --release --no-default-features --features async-std,sparse,cache
+          cargo build --release --no-default-features
+          cargo build --release --no-default-features --features sparse
+          cargo build --release --no-default-features --features sparse,cache
       - name: Run examples
         run: |
-          cargo run --no-default-features --features tokio --example disk 
-          cargo run --no-default-features --features async-std --example disk 
-          cargo run --no-default-features --features tokio --example memory 
-          cargo run --no-default-features --features async-std --example memory 
-          cargo run --no-default-features --features tokio --example replication 
-          cargo run --no-default-features --features async-std --example replication 
+          cargo run --no-default-features --example disk
+          cargo run --no-default-features --example memory
+          cargo run --no-default-features --example replication
 
   lint:
     runs-on: ubuntu-latest

.gitignore

@@ -13,3 +13,5 @@ my-first-dataset/
 feed.db/
 .vscode
 tests/js/work
+.claude
+docs

CHANGELOG.md

@@ -11,10 +11,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+* Replication over the Hypercore wire protocol, behind the (default) `replication` feature:
+  * `Hypercore::replicate` builds a `Replicator` driving a Noise-encrypted connection, and
+    `Hypercore::replicator` builds an unconnected one to attach a connection to later.
+  * `Replicator::with_connection` and `Replicator::with_connection_stream` attach a transport.
+  * `Hypercore::attach_replicator` attaches a `Replicator` to the core.
+  * `Hypercore::attach_channel` joins a channel on an already-multiplexed `hypercore-protocol`
+    connection, for hosting several cores over one stream.
+  * Polling `Hypercore::get` drives replication, so a request for a block the core does not have
+    is served from peers.
+* `HypercoreError::Protocol`, wrapping `hypercore_protocol::Error`.
+* `From<RandomAccessError> for HypercoreError`.
+
 ### Changed
 
+* `Hypercore` is now `Clone`, sharing one underlying core between clones.
+* `append`, `append_batch`, `get`, `create_proof`, `verify_and_apply_proof`, `missing_nodes` and
+  `missing_nodes_from_merkle_tree_index` take `&self` rather than `&mut self`.
+* `key_pair` returns `PartialKeypair` by value instead of `&PartialKeypair`.
+* `verify_and_apply_proof` takes `Proof` by value instead of `&Proof`.
+* `StorageTraits` now requires `Send + Sync`.
+* Internals were reworked to produce owned (`'static`) futures rather than futures borrowing the
+  core, which is what lets replication hold work in flight across calls.
+* `random-access-storage` bumped to `6.0.0-alpha`, `random-access-memory` and `random-access-disk`
+  to `4.0.0-alpha`.
+* The `replication` feature now also pulls in `hypercore-protocol` and `hypercore_handshake`.
+* `src/core.rs` was split into `src/core/mod.rs` and `src/core/inner.rs`.
+
 ### Removed
 
+* The `tokio` and `async-std` features. `async-std` is no longer supported and tokio is used
+  unconditionally, so there is no longer a runtime to select.
+* The `shared-core` feature and its `SharedCore` type. `Hypercore` is now `Clone` and its methods
+  take `&self`, so wrapping it in an `Arc<Mutex<_>>` is no longer needed.
+
 
 
 ## [0.16.0] - 2026-02-18

CLAUDE.md

@@ -0,0 +1,123 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+This is the **hypercore** crate, a Rust implementation of the Hypercore protocol - a secure, distributed append-only log. It aims for binary compatibility with the Javascript LTS version for disk storage while maintaining interoperability with the wire protocol.
+
+Key characteristics:
+- **100% Safe Rust**: Uses `#![forbid(unsafe_code)]`
+- **Structured concurrency**: Avoids `tokio::spawn`; work only happens when polled
+
+## JavaScript implementation
+
+The original JavaScript implementation of hypercore is located in  `/home/blake/git/hyper/js/hypercore/.`.
+
+## Building and Testing
+
+### Basic commands
+```bash
+# Run tests
+cargo test
+
+# Run linting (note: strict lints enabled)
+cargo clippy
+```
+
+### Running specific tests
+```bash
+# Run a single test
+cargo test test_name
+
+# Run tests with specific runtime
+cargo test --no-default-features --features tokio
+cargo test --no-default-features --features async-std
+```
+
+### JavaScript interoperability tests
+To verify compatibility with the original Javascript implementation:
+```bash
+cargo test --features js_interop_tests
+```
+
+These tests use the `rusty_nodejs_repl` crate to run Javascript code side-by-side with Rust, verifying that both implementations produce identical on-disk results.
+
+### Running examples
+```bash
+cargo run --example memory
+cargo run --example disk
+cargo run --example replication
+```
+
+## Architecture
+
+## Style guide
+
+Favor reduced nesting wherever  possible.
+
+### Core Data Structures
+
+The `Hypercore` struct is the main entry point, built using `HypercoreBuilder`. Internally, it consists of:
+
+- **Oplog** (`oplog/`): Append-only operation log with dual-header structure for crash recovery. Headers alternate between two memory areas for atomic updates. Stores entries with CRC checksums.
+
+- **MerkleTree** (`tree/`): Sparse merkle tree storing cryptographic hashes. Uses root indices to track tree state. Supports optional Moka cache for node lookups (with `cache` feature).
+
+- **BlockStore** (`data/`): Stores actual data blocks indexed by position.
+
+- **Bitfield** (`bitfield/`): Tracks which blocks are available locally (for sparse replication). Has `fixed` and `dynamic` implementations.
+
+- **Storage** (`storage/`): Abstraction over storage backends (memory/disk). Splits data into four stores: `tree`, `data`, `bitfield`, and `oplog`. Non-WASM targets use `random-access-disk`; WASM uses `random-access-memory`.
+
+### Replication
+
+The `replication/` module (enabled with `replication` feature, on by default) handles:
+
+- **Events** (`events.rs`): Broadcast events to peers using `async-broadcast`
+- **Peer**: Represents remote peers with protocol state
+
+Replication works by creating proofs in a source hypercore and verifying/applying them to destination hypercores.
+
+### Key Modules
+
+- **crypto/**: Cryptographic operations (signing, verification, hashing) using ed25519-dalek and blake2
+- **encoding.rs**: Compact binary encoding utilities
+- **builder.rs**: Fluent API for constructing Hypercore instances
+- **common/**: Shared types including error types, store definitions, and internal utilities
+
+## Features
+
+Default features: `sparse`, `replication`, `cache`
+
+- `tokio` / `async-std`: Choose async runtime (mutually exclusive, at least one required)
+- `sparse`: Enables sparse file support for disk storage
+- `replication`: Enables replication support and event broadcasting
+- `cache`: Moka-based cache for merkle tree nodes
+- `js_interop_tests`: Enables Javascript interoperability tests (dev-only)
+
+## Code Style and Lints
+
+This crate has strict linting enabled (see `.clippy.toml` and `lib.rs`):
+- `#![forbid(unsafe_code)]` - no unsafe code allowed
+- `#![cfg_attr(test, deny(warnings))]` - warnings become errors in tests
+- Additional lints: `unreachable_pub`, `redundant_lifetimes`, `clippy::needless_pass_by_value`, etc.
+
+All public APIs must have documentation (`missing_docs` warning).
+
+## Testing Strategy
+
+Tests are organized in:
+- `tests/core.rs`: Core functionality tests
+- `tests/model.rs`: Property-based tests using `proptest`
+- `tests/js_interop.rs`: Javascript interoperability verification
+- `tests/common/`: Shared test utilities
+- `tests/js/`: Javascript test harness code
+
+## Important Implementation Details
+
+- **Header structure**: Uses dual-header approach where headers alternate between two 4KB regions for atomic updates
+- **Oplog entries**: Each entry has an 8-byte "leader" (4-byte CRC + 4-byte length/flags)
+- **Storage flush**: Uses `skip_flush_count` for batching writes (mirrors JS `autoFlush`)
+- **Node indexing**: Uses flat-tree indices for merkle tree navigation
+- **Crash recovery**: Oplog design ensures consistency even if process crashes during write

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hypercore"
-version = "0.16.0"
+version = "0.17.0-alpha.1"
 license = "MIT OR Apache-2.0"
 description = "Secure, distributed, append-only log"
 documentation = "https://docs.rs/hypercore"
@@ -32,21 +32,39 @@ flat-tree = "6"
 merkle-tree-stream =  "0.12"
 pretty-hash = "0.4"
 rand = "0.8"
-random-access-memory = "3"
-random-access-storage = "5"
 sha2 = "0.10"
 futures = "0.3"
 crc32fast = "1"
 intmap = "2"
 moka = { version = "0.12", optional = true, features = ["sync"] }
 async-broadcast = { version = "0.7.1", optional = true }
-async-lock = {version = "3.4.0", optional = true }
+futures-lite = "2.6.1"
 
 [dependencies.hypercore_schema]
 version = "0.2.0"
 
-[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-random-access-disk = { version = "3", default-features = false }
+[dependencies.hypercore-protocol]
+version = "0.7.2"
+optional = true
+
+[dependencies.hypercore_handshake]
+version = "0.6.0"
+optional = true
+
+[dependencies.tokio]
+version = "1"
+default-features = false
+features = ["time"]
+
+[dependencies.random-access-storage]
+version = "6.0.0-alpha"
+
+[dependencies.random-access-memory]
+version = "4.0.0-alpha"
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies.random-access-disk]
+version = "4.0.0-alpha"
+default-features = false
 
 [dev-dependencies]
 anyhow = "1.0.70"
@@ -55,21 +73,19 @@ proptest-derive = "0.5.1"
 data-encoding = "2.2.0"
 remove_dir_all = "0.7.0"
 tempfile = "3.1.0"
-async-std = { version = "1.12.0", features = ["attributes"] }
-tokio = { version = "1.27.0", default-features = false, features = ["macros", "rt", "rt-multi-thread"] }
+tokio = { version = "1.27.0", default-features = false, features = ["macros", "rt", "rt-multi-thread", "io-util", "time"] }
+tokio-util = { version = "0.7", features = ["compat"] }
+uint24le_framing = { version = "0.2.0" }
 tokio-test = "0.4"
 sha2 = "0.10"
 criterion = { version = "0.4", features = ["async_std", "async_tokio"] }
 test-log = { version = "0.2.11", default-features = false, features = ["trace"] }
 tracing-subscriber = { version = "0.3.16", features = ["env-filter", "fmt"] }
 
 [features]
-default = ["tokio", "sparse", "replication", "cache"]
-replication = ["dep:async-broadcast"]
-shared-core = ["replication", "dep:async-lock"]
+default = ["sparse", "replication", "cache"]
+replication = ["dep:async-broadcast", "dep:hypercore-protocol", "dep:hypercore_handshake"]
 sparse = ["random-access-disk/sparse"]
-tokio = ["random-access-disk/tokio"]
-async-std = ["random-access-disk/async-std"]
 cache = ["moka"]
 # Used only in interoperability tests under tests/js-interop which use the javascript version of hypercore
 # to verify that this crate works. To run them, use:

README.md

@@ -20,7 +20,7 @@ corresponding wire protocol implementation.
 - [x] Get entries from hypercore
 - [x] Clear range from hypercore, with optional support for sparse files
 - [x] Support basic replication by creating proofs in a source hypercore and verifying and applying them to a destination hypercore
-- [x] Support `tokio` or `async-std` runtimes
+- [x] ~~Support `tokio` or `async-std` runtimes~~ `async-std` support dropped `tokio` used unconditionally
 - [x] Support WASM for in-memory storage
 - [x] Test Javascript interoperability for supported features
 - [x] Add optional read cache