Implement snapshot server

Dockerfile

@@ -66,7 +66,7 @@ useradd -r -g hyperliquid -u 10001 -d /home/hyperliquid -s /bin/bash hyperliquid
 
 # Create base directory structure
 install -d -m 755 -o root -g root /opt/hl
-install -d -m 755 -o hyperliquid -g hyperliquid /home/hyperliquid /data /data/hl /data/hl/data /opt/hl/bin
+install -d -m 755 -o hyperliquid -g hyperliquid /home/hyperliquid /data /data/hl /data/hl/data /data/snapshots /opt/hl/bin
 ln -s /data/hl /home/hyperliquid/hl
 EOF
 
@@ -79,6 +79,7 @@ USER hyperliquid:hyperliquid
 
 VOLUME /opt/hl/bin
 VOLUME /data
+VOLUME /data/snapshots
 WORKDIR /data
 
 # Import Hypqliquid public key. This is also required by hl-visor to verify downloaded binaries
@@ -95,10 +96,17 @@ ENV HL_BOOTSTRAP_OVERRIDE_GOSSIP_CONFIG_MAX_AGE=15m
 ENV HL_BOOTSTRAP_SEED_PEERS_AMOUNT=5
 ENV HL_BOOTSTRAP_SEED_PEERS_MAX_LATENCY=80ms
 ENV HL_BOOTSTRAP_NETWORK=Mainnet
+ENV HL_BOOTSTRAP_METRICS_LISTEN_ADDRESS=0.0.0.0:2112
+ENV HL_BOOTSTRAP_SNAPSHOT_SERVER_LISTEN_ADDRESS=0.0.0.0:2113
 
 # RPC
 EXPOSE 3001/tcp
 # P2P
 EXPOSE 4000-4010/tcp
+# Metrics
+EXPOSE 2112/tcp
+# Snapshot server
+EXPOSE 2113/tcp
+
 ENTRYPOINT ["/usr/bin/catatonit", "--", "hl-bootstrap", "--override-gossip-config-path=/data/override_gossip_config.json", "--"]
 CMD ["run-non-validator", "--write-trades", "--write-fills", "--write-order-statuses", "--serve-eth-rpc", "--serve-info", "--disable-output-file-buffering"]

compose.yaml

@@ -28,6 +28,7 @@ services:
     volumes:
       - "node-bin:/opt/hl/bin"
       - "node-data:/data"
+      - "node-snapshot:/data/snapshot"
     ports:
       - "127.0.0.1:3001:3001"
       - "0.0.0.0:4000-4010:4000-4010"
@@ -41,3 +42,4 @@ services:
 volumes:
   node-bin: {}
   node-data: {}
+  node-snapshot: {}

hl-bootstrap/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2024"
 axum = { version = "0.8.4", default-features = false, features = [
     "tokio",
     "http1",
+    "query",
 ] }
 clap = { version = "4.5.41", features = ["env", "derive"] }
 duration-string = "0.5.2"
@@ -28,6 +29,8 @@ tokio = { version = "1.46.1", features = [
     "rt",
     "rt-multi-thread",
 ] }
+tokio-util = { version = "0.7.18", features = ["io"] }
 tracing = "0.1.41"
 tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }
+uuid = { version = "1.19.0", features = ["serde", "v7"] }
 which = { version = "8.0.0", features = ["tracing"] }

hl-bootstrap/src/axum_ext.rs

@@ -0,0 +1,31 @@
+use axum::response::{IntoResponse, Response};
+use http::StatusCode;
+use tracing::error;
+
+pub struct Report(eyre::Report);
+
+pub type HttpResult<T, E = Report> = eyre::Result<T, E>;
+
+impl std::fmt::Debug for Report {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+impl<E> From<E> for Report
+where
+    E: Into<eyre::Report>,
+{
+    fn from(err: E) -> Self {
+        Self(err.into())
+    }
+}
+
+impl IntoResponse for Report {
+    fn into_response(self) -> Response {
+        let err = self.0;
+
+        error!(?err, "http handler error");
+        (StatusCode::INTERNAL_SERVER_ERROR, format!("{err:?}")).into_response()
+    }
+}

hl-bootstrap/src/main.rs

@@ -8,6 +8,7 @@ use std::{
     os::unix::process::CommandExt,
     path::PathBuf,
     process::Command,
+    time::Duration,
 };
 
 use clap::Parser;
@@ -23,10 +24,12 @@ use tracing_subscriber::{
     util::SubscriberInitExt,
 };
 
+mod axum_ext;
 mod hl_gossip_config;
 mod hl_visor;
 mod monitor;
 mod prune;
+mod snapshot;
 mod speedtest;
 mod sysctl;
 
@@ -134,10 +137,22 @@ struct Cli {
     #[arg(long, env = "HL_BOOTSTRAP_NETWORK", default_value_t = HyperliquidChain::Mainnet)]
     network: HyperliquidChain,
 
+    /// fileSnapshot request server listen address
+    #[arg(long, env = "HL_BOOTSTRAP_SNAPSHOT_SERVER_LISTEN_ADDRESS")]
+    snapshot_server_listen_address: Option<SocketAddr>,
+
     /// Free form args to execute after the setup
     args: Vec<OsString>,
 }
 
+impl Cli {
+    fn should_keep_bootstrap_running(&self) -> bool {
+        self.prune_data_interval.is_some()
+            || self.metrics_listen_address.is_some()
+            || self.snapshot_server_listen_address.is_some()
+    }
+}
+
 fn main() -> eyre::Result<()> {
     let args = Cli::parse();
 
@@ -168,9 +183,7 @@ fn main() -> eyre::Result<()> {
 
     trace!(?args, "args");
 
-    let use_mt = args.prune_data_interval.is_some() || args.metrics_listen_address.is_some();
-
-    let runtime = if use_mt {
+    let runtime = if args.should_keep_bootstrap_running() {
         Builder::new_multi_thread()
     } else {
         Builder::new_current_thread()
@@ -192,7 +205,7 @@ fn main() -> eyre::Result<()> {
 fn run_node(rt: Runtime, args: &Cli) -> eyre::Result<()> {
     info!(args = ?args.args, "setup done, executing hl-visor");
 
-    if args.prune_data_interval.is_none() && args.metrics_listen_address.is_none() {
+    if !args.should_keep_bootstrap_running() {
         drop(rt);
 
         // Just exec into the child
@@ -205,11 +218,12 @@ fn run_node(rt: Runtime, args: &Cli) -> eyre::Result<()> {
     let data_directory = current_dir().wrap_err("failed to get current working directory")?;
 
     let _prune_task = args.prune_data_interval.map(|prune_interval| {
+        let hl_data_directory = data_directory.clone().join("hl/data");
         rt.spawn({
             let prune_data_older_than = args.prune_data_older_than;
 
             prune_worker_task(
-                data_directory,
+                hl_data_directory,
                 prune_interval.into(),
                 prune_data_older_than.into(),
             )
@@ -235,6 +249,29 @@ fn run_node(rt: Runtime, args: &Cli) -> eyre::Result<()> {
         })
     });
 
+    let _snapshot_server = args.snapshot_server_listen_address.map(|address| {
+        let snapshot_directory = data_directory.clone().join("snapshots");
+        rt.spawn(async move {
+            info!(%address, "starting snapshot server");
+            if let Err(err) =
+                crate::snapshot::server::run_snapshot_server(snapshot_directory, address).await
+            {
+                error!(?err, "failed to start snapshot server")
+            }
+        })
+    });
+
+    let _snapshot_prune_task = args.snapshot_server_listen_address.is_some().then(|| {
+        let snapshot_directory = data_directory.join("snapshots");
+        rt.spawn(async move {
+            // Snapshots are rather big, prune them more often
+            let prune_interval = Duration::from_secs(30);
+            let prune_data_older_than = Duration::from_secs(15);
+
+            prune_worker_task(snapshot_directory, prune_interval, prune_data_older_than)
+        });
+    });
+
     let mut child = Command::new("hl-visor")
         .args(&args.args)
         .spawn()

hl-bootstrap/src/prune.rs

@@ -11,7 +11,7 @@ pub async fn prune_worker_task<P: AsRef<Path>>(
     prune_interval: Duration,
     prune_older_than: Duration,
 ) {
-    let base_path = base_path.as_ref().join("hl/data");
+    let base_path = base_path.as_ref();
 
     let mut interval = interval(prune_interval);
     interval.set_missed_tick_behavior(MissedTickBehavior::Skip);