diff --git a/README.md b/README.md index e8975ad..0ebde62 100644 --- a/README.md +++ b/README.md @@ -38,10 +38,11 @@ This repo contains Rust crates for blockchain data processing, indexing, and que | [`sqd-archive`](crates/archive/) | Archive service for ingesting and storing data to S3 with layout management, progress tracking, and Prometheus metrics | | [`sqd-bds`](crates/bds/) | Big Data Service (WIP) - Cassandra-based data storage | | [`sqd-hotblocks`](crates/hotblocks/) | Hotblocks database with portal-like API | +| [`sqd-hotblocks-retain`](crates/hotblocks-retain/) | Retention coordinator that applies Hotblocks retain points based on scheduling status updates | ## Building Docker image The ["docker"](/.github/workflows/docker.yaml) workflow should be triggered manually with the following inputs: -- `target` — either `hotblocks` or `sqd-archive`. +- `target` — one of `hotblocks`, `hotblocks-retain`, or `sqd-archive`. - `tag` — a 8-byte hash of the commit. This will be the published docker tag. - `platforms` — platforms to build for. Using only `linux/amd64` instead of default values can save a lot of building time. diff --git a/crates/hotblocks-retain/src/main.rs b/crates/hotblocks-retain/src/main.rs index 4f18180..0b7e351 100644 --- a/crates/hotblocks-retain/src/main.rs +++ b/crates/hotblocks-retain/src/main.rs @@ -150,13 +150,22 @@ impl HotblocksRetain { let mut all_success = true; for (dataset, props) in &self.datasets { + let network_dataset = props + .as_ref() + .and_then(|p| p.network_dataset.as_deref()) + .unwrap_or(dataset.as_str()); + let dataset_id = if let Some(id) = props.as_ref().and_then(|p| p.id.as_deref()) { id } else { - match self.name_to_id.get(dataset) { + match self.name_to_id.get(network_dataset) { Some(id) => id.as_str(), None => { - tracing::warn!(dataset, "dataset not found in manifest, skipping"); + tracing::warn!( + dataset, + network_dataset, + "dataset not found in manifest, skipping" + ); continue; } } diff --git a/crates/hotblocks-retain/src/types.rs b/crates/hotblocks-retain/src/types.rs index 81ae409..a6ce02a 100644 --- a/crates/hotblocks-retain/src/types.rs +++ b/crates/hotblocks-retain/src/types.rs @@ -6,6 +6,8 @@ pub type DatasetId = String; // s3:// #[derive(Debug, Clone, Deserialize)] pub struct DatasetProps { pub id: Option, + #[serde(rename = "name", alias = "network_dataset")] + pub network_dataset: Option, } pub type DatasetsConfig = HashMap>;