From 1e258d8f2f40555d76102d3fdd2c1f579b9611ac Mon Sep 17 00:00:00 2001 From: tison Date: Mon, 28 Sep 2026 08:13:07 +0800 Subject: [PATCH 1/7] fix: write zero seed hashes for empty Theta and Tuple sketches --- CHANGELOG.md | 1 + datasketches/src/thetafamily/theta/sketch.rs | 4 +- datasketches/src/thetafamily/tuple/sketch.rs | 3 +- tests-integration/tests/serde_tests/theta.rs | 53 ++++++++++++++++++++ tests-integration/tests/serde_tests/tuple.rs | 50 ++++++++++++++++++ 5 files changed, 109 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6614a73..986fbe35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ All significant changes to this project will be documented in this file. ### Bug fixes +* Empty compact Theta and Tuple sketches now serialize with a zero seed hash, matching the canonical cross-language encoding. Previously written empty images remain readable. * `CountMinSketch` updates now panic and merges return `InvalidArgument` if the total absolute weight would exceed the counter type's maximum. Both leave the sketch unchanged, including in release builds. * `CountMinSketch::upper_bound` now clamps to the counter type's maximum instead of overflowing. * `CountMinSketch` deserialization now returns `InvalidData` if the total absolute weight is negative or any counter's magnitude exceeds it. diff --git a/datasketches/src/thetafamily/theta/sketch.rs b/datasketches/src/thetafamily/theta/sketch.rs index c293a775..49bdf29a 100644 --- a/datasketches/src/thetafamily/theta/sketch.rs +++ b/datasketches/src/thetafamily/theta/sketch.rs @@ -563,6 +563,8 @@ impl CompactThetaSketch { } /// Serializes this sketch into the uncompressed compact theta format. + /// + /// Empty sketches write a zero seed hash; non-empty sketches retain their seed hash. pub fn serialize(&self) -> Vec { let retained_hashes = self.retained_hashes(); let mut bytes = SketchBytes::with_capacity(64 + retained_hashes.len() * 8); @@ -584,7 +586,7 @@ impl CompactThetaSketch { } bytes.write_u8(flags); - bytes.write_u16_le(self.seed_hash()); + bytes.write_u16_le(if self.is_empty() { 0 } else { self.seed_hash() }); if pre_longs > 1 { bytes.write_u32_le(retained_hashes.len() as u32); diff --git a/datasketches/src/thetafamily/tuple/sketch.rs b/datasketches/src/thetafamily/tuple/sketch.rs index 2d6b93f4..4a46216f 100644 --- a/datasketches/src/thetafamily/tuple/sketch.rs +++ b/datasketches/src/thetafamily/tuple/sketch.rs @@ -561,6 +561,7 @@ impl CompactTupleSketch { /// Each summary is encoded by its [`TupleSummaryValue`] implementation. The layout matches the /// Java/C++ Tuple sketches, so the output can be read by those implementations given a /// compatible summary encoding. + /// Empty sketches write a zero seed hash; non-empty sketches retain their seed hash. /// /// # Examples /// @@ -600,7 +601,7 @@ impl CompactTupleSketch { flags |= FLAGS_IS_ORDERED; } bytes.write_u8(flags); - bytes.write_u16_le(self.seed_hash()); + bytes.write_u16_le(if self.is_empty() { 0 } else { self.seed_hash() }); if pre_longs > 1 { bytes.write_u32_le(retained_entries.len() as u32); diff --git a/tests-integration/tests/serde_tests/theta.rs b/tests-integration/tests/serde_tests/theta.rs index 0850f733..b0108543 100644 --- a/tests-integration/tests/serde_tests/theta.rs +++ b/tests-integration/tests/serde_tests/theta.rs @@ -159,6 +159,59 @@ fn test_go_compatibility() { test_sketch_file(path, 0, false); } +#[test] +fn empty_images_use_canonical_seed_hash() { + let expected = fs::read(serialization_test_data( + "cpp_generated_files", + "theta_n0_cpp.sk", + )) + .unwrap(); + + for builder in [ + ThetaSketchBuilder::default(), + ThetaSketchBuilder::default() + .seed(123) + .sampling_probability(0.5), + ] { + let sketch = builder.build().unwrap().compact(false); + assert_eq!(sketch.serialize(), expected); + assert_eq!(sketch.serialize_compressed(), expected); + + // Older Rust images stored the configured seed hash even when empty. + let mut legacy = expected.clone(); + legacy[6..8].copy_from_slice(&sketch.seed_hash().to_le_bytes()); + for bytes in [&expected, &legacy] { + let restored = CompactThetaSketch::deserialize_with_seed(bytes, 456).unwrap(); + assert!(restored.is_empty()); + assert_eq!(restored.serialize(), expected); + assert_eq!(restored.serialize_compressed(), expected); + } + } +} + +#[test] +fn non_empty_images_without_entries_preserve_seed_hash() { + let mut sketch = ThetaSketchBuilder::default() + .seed(123) + .sampling_probability(1e-12) + .build() + .unwrap(); + sketch.update("apple"); + let compact = sketch.compact(true); + assert!(!compact.is_empty()); + assert_eq!(compact.num_retained(), 0); + + for bytes in [compact.serialize(), compact.serialize_compressed()] { + let restored = CompactThetaSketch::deserialize_with_seed(&bytes, 123).unwrap(); + assert!(!restored.is_empty()); + assert_eq!(restored.num_retained(), 0); + assert_eq!(restored.theta64(), compact.theta64()); + assert_eq!(restored.seed_hash(), compact.seed_hash()); + let err = CompactThetaSketch::deserialize(&bytes).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidData); + } +} + #[test] fn malformed_input_is_rejected() { let mut sketch = ThetaSketchBuilder::default().lg_k(5).build().unwrap(); diff --git a/tests-integration/tests/serde_tests/tuple.rs b/tests-integration/tests/serde_tests/tuple.rs index fe4c7bea..200bf798 100644 --- a/tests-integration/tests/serde_tests/tuple.rs +++ b/tests-integration/tests/serde_tests/tuple.rs @@ -108,6 +108,56 @@ fn test_go_compatibility() { } } +#[test] +fn empty_images_use_canonical_seed_hash() { + let expected = fs::read(serialization_test_data( + "cpp_generated_files", + "tuple_int_n0_cpp.sk", + )) + .unwrap(); + + for builder in [ + TupleSketchBuilder::new(DefaultUpdatePolicy::::default()), + TupleSketchBuilder::new(DefaultUpdatePolicy::::default()) + .seed(123) + .sampling_probability(0.5), + ] { + let sketch = builder.build().unwrap().compact(false); + assert_eq!(sketch.serialize(), expected); + + // Older Rust images stored the configured seed hash even when empty. + let mut legacy = expected.clone(); + legacy[6..8].copy_from_slice(&sketch.seed_hash().to_le_bytes()); + for bytes in [&expected, &legacy] { + let restored = CompactTupleSketch::::deserialize_with_seed(bytes, 456).unwrap(); + assert!(restored.is_empty()); + assert_eq!(restored.serialize(), expected); + } + } +} + +#[test] +fn non_empty_images_without_entries_preserve_seed_hash() { + let mut sketch = TupleSketchBuilder::new(DefaultUpdatePolicy::::default()) + .seed(123) + .sampling_probability(1e-12) + .build() + .unwrap(); + sketch.update("apple", 1); + let compact = sketch.compact(true); + assert!(!compact.is_empty()); + assert_eq!(compact.num_retained(), 0); + + let bytes = compact.serialize(); + let restored = CompactTupleSketch::::deserialize_with_seed(&bytes, 123).unwrap(); + assert!(!restored.is_empty()); + assert_eq!(restored.num_retained(), 0); + assert_eq!(restored.theta64(), compact.theta64()); + assert_eq!(restored.seed_hash(), compact.seed_hash()); + let err = CompactTupleSketch::::deserialize(&bytes).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidData); +} + #[test] fn round_trip_preserves_summaries() { let mut sketch = TupleSketchBuilder::new(DefaultUpdatePolicy::::default()) From b80d0a211a59411fb6046aaf4c30ff8149cba3d8 Mon Sep 17 00:00:00 2001 From: tison Date: Tue, 29 Sep 2026 08:54:11 +0800 Subject: [PATCH 2/7] perf: serialize empty Theta and Tuple sketches from constants --- datasketches/src/thetafamily/theta/serialization.rs | 3 +++ datasketches/src/thetafamily/theta/sketch.rs | 11 ++++++----- datasketches/src/thetafamily/tuple/serialization.rs | 3 +++ datasketches/src/thetafamily/tuple/sketch.rs | 12 +++++++----- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/datasketches/src/thetafamily/theta/serialization.rs b/datasketches/src/thetafamily/theta/serialization.rs index f9d6e2cd..d10917f0 100644 --- a/datasketches/src/thetafamily/theta/serialization.rs +++ b/datasketches/src/thetafamily/theta/serialization.rs @@ -20,6 +20,9 @@ pub const UNCOMPRESSED_SERIAL_VERSION: u8 = 3; pub const COMPRESSED_SERIAL_VERSION: u8 = 4; +/// Canonical empty compact image, independent of the configured seed. +pub const EMPTY_SKETCH_BYTES: [u8; 8] = [1, 3, 3, 0, 0, 0x1e, 0, 0]; + pub const V2_PREAMBLE_EMPTY: u8 = 1; pub const V2_PREAMBLE_PRECISE: u8 = 2; pub const V2_PREAMBLE_ESTIMATE: u8 = 3; diff --git a/datasketches/src/thetafamily/theta/sketch.rs b/datasketches/src/thetafamily/theta/sketch.rs index 49bdf29a..3263c20c 100644 --- a/datasketches/src/thetafamily/theta/sketch.rs +++ b/datasketches/src/thetafamily/theta/sketch.rs @@ -537,7 +537,7 @@ impl CompactThetaSketch { if self.is_estimation_mode() { 2 } else { 1 } } else if self.is_estimation_mode() { 3 - } else if self.is_empty() || self.num_retained() == 1 { + } else if self.num_retained() == 1 { 1 } else { 2 @@ -566,6 +566,10 @@ impl CompactThetaSketch { /// /// Empty sketches write a zero seed hash; non-empty sketches retain their seed hash. pub fn serialize(&self) -> Vec { + if self.is_empty() { + return serialization::EMPTY_SKETCH_BYTES.to_vec(); + } + let retained_hashes = self.retained_hashes(); let mut bytes = SketchBytes::with_capacity(64 + retained_hashes.len() * 8); @@ -578,15 +582,12 @@ impl CompactThetaSketch { let mut flags = 0u8; flags |= FLAGS_IS_READ_ONLY; flags |= FLAGS_IS_COMPACT; - if self.is_empty() { - flags |= FLAGS_IS_EMPTY; - } if self.is_ordered() { flags |= FLAGS_IS_ORDERED; } bytes.write_u8(flags); - bytes.write_u16_le(if self.is_empty() { 0 } else { self.seed_hash() }); + bytes.write_u16_le(self.seed_hash()); if pre_longs > 1 { bytes.write_u32_le(retained_hashes.len() as u32); diff --git a/datasketches/src/thetafamily/tuple/serialization.rs b/datasketches/src/thetafamily/tuple/serialization.rs index 1f4cfeeb..d44a1049 100644 --- a/datasketches/src/thetafamily/tuple/serialization.rs +++ b/datasketches/src/thetafamily/tuple/serialization.rs @@ -40,6 +40,9 @@ pub const SKETCH_TYPE: u8 = 1; /// Legacy sketch-type byte still accepted on read. pub const SKETCH_TYPE_LEGACY: u8 = 5; +/// Canonical empty compact image, independent of the configured seed and summary type. +pub const EMPTY_SKETCH_BYTES: [u8; 8] = [1, 3, 9, 1, 0, 0x1e, 0, 0]; + /// Trait for values that can be stored as Tuple sketch summaries. /// /// Implement this trait for a summary type to make it (de)serializable by diff --git a/datasketches/src/thetafamily/tuple/sketch.rs b/datasketches/src/thetafamily/tuple/sketch.rs index 4a46216f..e99ae06d 100644 --- a/datasketches/src/thetafamily/tuple/sketch.rs +++ b/datasketches/src/thetafamily/tuple/sketch.rs @@ -53,6 +53,7 @@ use crate::tuple::hash_table::TupleEntry; use crate::tuple::hash_table::TupleHashTable; use crate::tuple::policy::SummaryPolicy; use crate::tuple::policy::SummaryUpdatePolicy; +use crate::tuple::serialization::EMPTY_SKETCH_BYTES; use crate::tuple::serialization::SERIAL_VERSION; use crate::tuple::serialization::SERIAL_VERSION_LEGACY; use crate::tuple::serialization::SKETCH_TYPE; @@ -549,7 +550,7 @@ impl CompactTupleSketch { fn preamble_longs(&self) -> u8 { if self.is_estimation_mode() { 3 - } else if self.is_empty() || self.num_retained() == 1 { + } else if self.num_retained() == 1 { 1 } else { 2 @@ -579,6 +580,10 @@ impl CompactTupleSketch { where S: TupleSummaryValue, { + if self.is_empty() { + return EMPTY_SKETCH_BYTES.to_vec(); + } + let retained_entries = self.retained_entries(); let pre_longs = self.preamble_longs(); let entries_size: usize = retained_entries @@ -594,14 +599,11 @@ impl CompactTupleSketch { bytes.write_u8(0); // unused let mut flags = FLAGS_IS_READ_ONLY | FLAGS_IS_COMPACT; - if self.is_empty() { - flags |= FLAGS_IS_EMPTY; - } if self.is_ordered() { flags |= FLAGS_IS_ORDERED; } bytes.write_u8(flags); - bytes.write_u16_le(if self.is_empty() { 0 } else { self.seed_hash() }); + bytes.write_u16_le(self.seed_hash()); if pre_longs > 1 { bytes.write_u32_le(retained_entries.len() as u32); From a43f0be6e9013e642279e9c0d242d15b642eee43 Mon Sep 17 00:00:00 2001 From: tison Date: Tue, 29 Sep 2026 09:06:49 +0800 Subject: [PATCH 3/7] fix: accept seed-independent empty Theta v2 images --- CHANGELOG.md | 2 +- datasketches/src/thetafamily/theta/sketch.rs | 86 +++++++++----------- tests-integration/tests/serde_tests/theta.rs | 66 +++++++++++++-- 3 files changed, 96 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 986fbe35..0615f1a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ All significant changes to this project will be documented in this file. ### Bug fixes -* Empty compact Theta and Tuple sketches now serialize with a zero seed hash, matching the canonical cross-language encoding. Previously written empty images remain readable. +* Empty compact Theta and Tuple sketches now serialize with a zero seed hash, matching the canonical cross-language encoding. Previously written empty images remain readable. Legacy Theta v2 empty images now accept zero or mismatched seed hashes and normalize them to zero on read, matching Java's v2 conversion. * `CountMinSketch` updates now panic and merges return `InvalidArgument` if the total absolute weight would exceed the counter type's maximum. Both leave the sketch unchanged, including in release builds. * `CountMinSketch::upper_bound` now clamps to the counter type's maximum instead of overflowing. * `CountMinSketch` deserialization now returns `InvalidData` if the total absolute weight is negative or any counter's magnitude exceeds it. diff --git a/datasketches/src/thetafamily/theta/sketch.rs b/datasketches/src/thetafamily/theta/sketch.rs index 3263c20c..b7d45d62 100644 --- a/datasketches/src/thetafamily/theta/sketch.rs +++ b/datasketches/src/thetafamily/theta/sketch.rs @@ -692,18 +692,21 @@ impl CompactThetaSketch { /// /// # Errors /// - /// Returns `InvalidData` if the image is malformed or its seed hash does not match the default - /// seed. + /// Returns `InvalidData` if the image is malformed or a non-empty image's seed hash does not + /// match the default seed. pub fn deserialize(bytes: &[u8]) -> Result { Self::deserialize_with_seed(bytes, DEFAULT_UPDATE_SEED) } /// Deserializes a compact theta sketch from bytes using the provided expected seed. /// + /// Empty images do not require a matching seed hash. Legacy version 2 empty images are + /// normalized to a zero seed hash. + /// /// # Errors /// - /// Returns `InvalidData` if the image is malformed, its seed hash does not match `seed`, or - /// `seed` itself computes to the reserved zero seed hash. + /// Returns `InvalidData` if the image is malformed, a non-empty image's seed hash does not + /// match `seed`, or `seed` itself computes to the reserved zero seed hash. pub fn deserialize_with_seed(bytes: &[u8], seed: u64) -> Result { let expected_seed_hash = compute_seed_hash(seed, ErrorKind::InvalidData)?; let mut cursor = SketchSlice::new(bytes); @@ -812,36 +815,9 @@ impl CompactThetaSketch { let seed_hash = cursor .read_u16_le() .map_err(insufficient_data("seed_hash"))?; - check_seed_hash( - expected_seed_hash, - seed_hash, - "deserialized CompactThetaSketch v2", - ErrorKind::InvalidData, - )?; - - match pre_longs { - V2_PREAMBLE_EMPTY => Ok(Self::from_compact_state(CompactSketchState::empty( - seed_hash, - ))), - V2_PREAMBLE_PRECISE => { - let num_entries = cursor - .read_u32_le() - .map_err(insufficient_data("num_entries"))? - as usize; - cursor - .read_u32_le() - .map_err(insufficient_data(""))?; - let entries = Self::read_entries(&mut cursor, num_entries, MAX_THETA)?; - if num_entries == 0 { - return Ok(Self::from_compact_state(CompactSketchState::empty( - seed_hash, - ))); - } - Ok(Self::from_compact_state(CompactSketchState::non_empty( - entries, MAX_THETA, seed_hash, true, - ))) - } - V2_PREAMBLE_ESTIMATE => { + let (num_entries, theta) = match pre_longs { + V2_PREAMBLE_EMPTY => (0, MAX_THETA), + V2_PREAMBLE_PRECISE | V2_PREAMBLE_ESTIMATE => { let num_entries = cursor .read_u32_le() .map_err(insufficient_data("num_entries"))? @@ -849,23 +825,35 @@ impl CompactThetaSketch { cursor .read_u32_le() .map_err(insufficient_data(""))?; - let theta = Self::deserialize_theta( - cursor - .read_u64_le() - .map_err(insufficient_data("theta_long"))?, - )?; - let entries = Self::read_entries(&mut cursor, num_entries, theta)?; - if num_entries == 0 && theta == MAX_THETA { - return Ok(Self::from_compact_state(CompactSketchState::empty( - seed_hash, - ))); - } - Ok(Self::from_compact_state(CompactSketchState::non_empty( - entries, theta, seed_hash, true, - ))) + let theta = if pre_longs == V2_PREAMBLE_ESTIMATE { + Self::deserialize_theta( + cursor + .read_u64_le() + .map_err(insufficient_data("theta_long"))?, + )? + } else { + MAX_THETA + }; + (num_entries, theta) } - _ => Err(Error::invalid_preamble_longs(&[1, 2, 3], pre_longs)), + _ => return Err(Error::invalid_preamble_longs(&[1, 2, 3], pre_longs)), + }; + + // Java's v2 conversion normalizes empty images to its seed-independent empty sketch. + if num_entries == 0 && theta == MAX_THETA { + return Ok(Self::from_compact_state(CompactSketchState::empty(0))); } + + check_seed_hash( + expected_seed_hash, + seed_hash, + "deserialized CompactThetaSketch v2", + ErrorKind::InvalidData, + )?; + let entries = Self::read_entries(&mut cursor, num_entries, theta)?; + Ok(Self::from_compact_state(CompactSketchState::non_empty( + entries, theta, seed_hash, true, + ))) } fn deserialize_v3( diff --git a/tests-integration/tests/serde_tests/theta.rs b/tests-integration/tests/serde_tests/theta.rs index b0108543..e7a97657 100644 --- a/tests-integration/tests/serde_tests/theta.rs +++ b/tests-integration/tests/serde_tests/theta.rs @@ -288,13 +288,63 @@ fn test_v2_exact_non_empty_compatibility() { } #[test] -fn test_v2_exact_zero_entries_remains_empty() { - let sketch = CompactThetaSketch::deserialize(&serialize_v2_exact(&[])).unwrap(); +fn test_v2_empty_images_ignore_seed_hash() { + let reference = ThetaSketchBuilder::default() + .seed(123) + .build() + .unwrap() + .compact(true); + let expected = reference.serialize(); + + // Java accepts empty v2 images with one, two, or three preamble longs. + for pre_longs in [1, 2, 3] { + let mut bytes = serialize_v2_exact(&[]); + bytes[0] = pre_longs; + if pre_longs == 1 { + bytes.truncate(8); + } else if pre_longs == 3 { + bytes.extend_from_slice(&(i64::MAX as u64).to_le_bytes()); + } - assert!(sketch.is_empty()); - assert!(!sketch.is_estimation_mode()); - assert_eq!(sketch.num_retained(), 0); - assert_eq!(sketch.estimate(), 0.0); - assert_eq!(sketch.lower_bound(NumStdDev::One), 0.0); - assert_eq!(sketch.upper_bound(NumStdDev::One), 0.0); + for seed_hash in [0, reference.seed_hash()] { + bytes[6..8].copy_from_slice(&seed_hash.to_le_bytes()); + for seed in [123, 456] { + let sketch = CompactThetaSketch::deserialize_with_seed(&bytes, seed).unwrap(); + assert!(sketch.is_empty()); + assert!(!sketch.is_estimation_mode()); + assert_eq!(sketch.num_retained(), 0); + assert_eq!(sketch.estimate(), 0.0); + assert_eq!(sketch.lower_bound(NumStdDev::One), 0.0); + assert_eq!(sketch.upper_bound(NumStdDev::One), 0.0); + assert_eq!(sketch.seed_hash(), 0); + assert_eq!(sketch.serialize(), expected); + + let restored = + CompactThetaSketch::deserialize_with_seed(&sketch.serialize(), seed).unwrap(); + assert!(restored.is_empty()); + assert_eq!(restored.seed_hash(), 0); + } + } + } +} + +#[test] +fn test_v2_non_empty_images_validate_seed_hash() { + let exact = serialize_v2_exact(&[1]); + let mut sampled = serialize_v2_exact(&[]); + sampled[0] = 3; + sampled.extend_from_slice(&((i64::MAX as u64) / 2).to_le_bytes()); + + // Zero retained entries with theta < 1 still describe a non-empty sketch. + for mut bytes in [exact, sampled] { + let sketch = CompactThetaSketch::deserialize(&bytes).unwrap(); + assert!(!sketch.is_empty()); + + let err = CompactThetaSketch::deserialize_with_seed(&bytes, 456).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidData); + + bytes[6..8].fill(0); + let err = CompactThetaSketch::deserialize(&bytes).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidData); + } } From 9089402fe94828901e781ae32f93ff891cedda27 Mon Sep 17 00:00:00 2001 From: tison Date: Tue, 29 Sep 2026 09:08:43 +0800 Subject: [PATCH 4/7] test: port Java empty Theta v2 compatibility cases --- tests-integration/tests/serde_tests/theta.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests-integration/tests/serde_tests/theta.rs b/tests-integration/tests/serde_tests/theta.rs index e7a97657..0ef87312 100644 --- a/tests-integration/tests/serde_tests/theta.rs +++ b/tests-integration/tests/serde_tests/theta.rs @@ -288,7 +288,7 @@ fn test_v2_exact_non_empty_compatibility() { } #[test] -fn test_v2_empty_images_ignore_seed_hash() { +fn test_v2_java_empty_layouts() { let reference = ThetaSketchBuilder::default() .seed(123) .build() @@ -296,22 +296,32 @@ fn test_v2_empty_images_ignore_seed_hash() { .compact(true); let expected = reference.serialize(); - // Java accepts empty v2 images with one, two, or three preamble longs. + // Port of Java 8.0.0 ForwardCompatibilityTest's checkSerVer2_{1,2,3}PreLong[s]_Empty: + // https://github.com/apache/datasketches-java/blob/8.0.0/src/test/java/org/apache/datasketches/theta/ForwardCompatibilityTest.java#L90-L137 for pre_longs in [1, 2, 3] { let mut bytes = serialize_v2_exact(&[]); bytes[0] = pre_longs; + bytes[5] = 0x0e; // Java's v2 flags: no rebuild, empty, read-only, little-endian. if pre_longs == 1 { bytes.truncate(8); } else if pre_longs == 3 { bytes.extend_from_slice(&(i64::MAX as u64).to_le_bytes()); } + let sketch = CompactThetaSketch::deserialize(&bytes).unwrap(); + assert!(sketch.is_empty()); + assert!(!sketch.is_estimation_mode()); + assert!(sketch.is_ordered()); + assert_eq!(sketch.seed_hash(), 0); + + // Extend Java's default-seed cases with zero and mismatched seed hashes. for seed_hash in [0, reference.seed_hash()] { bytes[6..8].copy_from_slice(&seed_hash.to_le_bytes()); for seed in [123, 456] { let sketch = CompactThetaSketch::deserialize_with_seed(&bytes, seed).unwrap(); assert!(sketch.is_empty()); assert!(!sketch.is_estimation_mode()); + assert!(sketch.is_ordered()); assert_eq!(sketch.num_retained(), 0); assert_eq!(sketch.estimate(), 0.0); assert_eq!(sketch.lower_bound(NumStdDev::One), 0.0); From ea007c0c47be961fdbde13666550d47eb599a7a6 Mon Sep 17 00:00:00 2001 From: tison Date: Tue, 29 Sep 2026 22:10:36 +0800 Subject: [PATCH 5/7] docs: clarify Theta and Tuple serialization contracts --- datasketches/src/thetafamily/theta/sketch.rs | 18 ++++++------- datasketches/src/thetafamily/tuple/sketch.rs | 27 ++++++++++++-------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/datasketches/src/thetafamily/theta/sketch.rs b/datasketches/src/thetafamily/theta/sketch.rs index b7d45d62..69fc8fe9 100644 --- a/datasketches/src/thetafamily/theta/sketch.rs +++ b/datasketches/src/thetafamily/theta/sketch.rs @@ -495,7 +495,9 @@ impl CompactThetaSketch { self.compact_state.is_ordered() } - /// Returns the 16-bit seed hash. + /// Returns the 16-bit fingerprint used to check seed compatibility. + /// + /// Empty sketches may return `0`, indicating no seed association. pub fn seed_hash(&self) -> u16 { self.compact_state.seed_hash() } @@ -546,8 +548,7 @@ impl CompactThetaSketch { /// Serializes this sketch in compressed form if applicable. /// - /// This uses `serVer = 4` when the sketch is ordered and suitable for compression, and falls - /// back to uncompressed `serVer = 3` otherwise. + /// Falls back to [`serialize`](Self::serialize) when compression is not applicable. pub fn serialize_compressed(&self) -> Vec { if self.is_suitable_for_compression() { self.serialize_v4() @@ -562,9 +563,9 @@ impl CompactThetaSketch { && (self.num_retained() != 1 || self.is_estimation_mode()) } - /// Serializes this sketch into the uncompressed compact theta format. + /// Serializes this sketch into the uncompressed compact Theta format. /// - /// Empty sketches write a zero seed hash; non-empty sketches retain their seed hash. + /// Empty sketches serialize with a zero seed hash. pub fn serialize(&self) -> Vec { if self.is_empty() { return serialization::EMPTY_SKETCH_BYTES.to_vec(); @@ -688,7 +689,7 @@ impl CompactThetaSketch { bits.div_ceil(8) as u8 } - /// Deserializes a compact theta sketch from bytes. + /// Deserializes a compact Theta sketch using the default seed. /// /// # Errors /// @@ -698,10 +699,9 @@ impl CompactThetaSketch { Self::deserialize_with_seed(bytes, DEFAULT_UPDATE_SEED) } - /// Deserializes a compact theta sketch from bytes using the provided expected seed. + /// Deserializes a compact Theta sketch using the expected `seed`. /// - /// Empty images do not require a matching seed hash. Legacy version 2 empty images are - /// normalized to a zero seed hash. + /// Empty sketches do not require a matching seed hash. /// /// # Errors /// diff --git a/datasketches/src/thetafamily/tuple/sketch.rs b/datasketches/src/thetafamily/tuple/sketch.rs index e99ae06d..81e56967 100644 --- a/datasketches/src/thetafamily/tuple/sketch.rs +++ b/datasketches/src/thetafamily/tuple/sketch.rs @@ -504,7 +504,9 @@ impl CompactTupleSketch { self.compact_state.is_ordered() } - /// Returns the 16-bit seed hash. + /// Returns the 16-bit fingerprint used to check seed compatibility. + /// + /// Empty sketches may return `0`, indicating no seed association. pub fn seed_hash(&self) -> u16 { self.compact_state.seed_hash() } @@ -559,10 +561,10 @@ impl CompactTupleSketch { /// Serializes this sketch into the compact Tuple binary format. /// - /// Each summary is encoded by its [`TupleSummaryValue`] implementation. The layout matches the - /// Java/C++ Tuple sketches, so the output can be read by those implementations given a - /// compatible summary encoding. - /// Empty sketches write a zero seed hash; non-empty sketches retain their seed hash. + /// Uses [`TupleSummaryValue`] to encode summaries. Reading the output in Java or C++ requires + /// a compatible summary encoding. + /// + /// Empty sketches serialize with a zero seed hash. /// /// # Examples /// @@ -624,8 +626,9 @@ impl CompactTupleSketch { /// /// # Errors /// - /// Returns `InvalidData` if the image is malformed, its seed hash does not match the default - /// seed, or a summary cannot be decoded by `S`. + /// Returns `InvalidData` if the image is malformed or a non-empty image's seed hash does not + /// match the default seed. Also propagates errors from + /// [`TupleSummaryValue::deserialize_value`]. pub fn deserialize(bytes: &[u8]) -> Result where S: TupleSummaryValue, @@ -633,13 +636,15 @@ impl CompactTupleSketch { Self::deserialize_with_seed(bytes, DEFAULT_UPDATE_SEED) } - /// Deserializes a compact Tuple sketch using the provided expected `seed`. + /// Deserializes a compact Tuple sketch using the expected `seed`. + /// + /// Empty sketches do not require a matching seed hash. /// /// # Errors /// - /// Returns `InvalidData` if the bytes are truncated, the family/serial version/sketch type are - /// unexpected, the seed hash does not match, the supplied seed computes to the reserved zero - /// seed hash, or an entry is corrupted. + /// Returns `InvalidData` if the image is malformed, a non-empty image's seed hash does not + /// match `seed`, or `seed` itself computes to the reserved zero seed hash. Also propagates + /// errors from [`TupleSummaryValue::deserialize_value`]. pub fn deserialize_with_seed(bytes: &[u8], seed: u64) -> Result where S: TupleSummaryValue, From ce9e942918d65d1fe9a6e16028cce0b90782b4ff Mon Sep 17 00:00:00 2001 From: tison Date: Tue, 29 Sep 2026 22:48:30 +0800 Subject: [PATCH 6/7] fix: retain caller seed hashes when reading empty sketches --- CHANGELOG.md | 2 +- datasketches/src/thetafamily/theta/sketch.rs | 17 +++-- datasketches/src/thetafamily/tuple/sketch.rs | 10 ++- tests-integration/tests/serde_tests/theta.rs | 65 +++++++++++++++++--- tests-integration/tests/serde_tests/tuple.rs | 28 +++++++-- 5 files changed, 95 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0615f1a7..51377697 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ All significant changes to this project will be documented in this file. ### Bug fixes -* Empty compact Theta and Tuple sketches now serialize with a zero seed hash, matching the canonical cross-language encoding. Previously written empty images remain readable. Legacy Theta v2 empty images now accept zero or mismatched seed hashes and normalize them to zero on read, matching Java's v2 conversion. +* Empty compact Theta and Tuple sketches now serialize with a zero seed hash, matching the canonical cross-language encoding. Deserialization associates empty sketches with the supplied seed, including legacy Theta v2 images with zero or mismatched seed hashes. * `CountMinSketch` updates now panic and merges return `InvalidArgument` if the total absolute weight would exceed the counter type's maximum. Both leave the sketch unchanged, including in release builds. * `CountMinSketch::upper_bound` now clamps to the counter type's maximum instead of overflowing. * `CountMinSketch` deserialization now returns `InvalidData` if the total absolute weight is negative or any counter's magnitude exceeds it. diff --git a/datasketches/src/thetafamily/theta/sketch.rs b/datasketches/src/thetafamily/theta/sketch.rs index 69fc8fe9..2c873832 100644 --- a/datasketches/src/thetafamily/theta/sketch.rs +++ b/datasketches/src/thetafamily/theta/sketch.rs @@ -495,9 +495,7 @@ impl CompactThetaSketch { self.compact_state.is_ordered() } - /// Returns the 16-bit fingerprint used to check seed compatibility. - /// - /// Empty sketches may return `0`, indicating no seed association. + /// Returns the 16-bit fingerprint of the seed associated with this sketch. pub fn seed_hash(&self) -> u16 { self.compact_state.seed_hash() } @@ -699,9 +697,9 @@ impl CompactThetaSketch { Self::deserialize_with_seed(bytes, DEFAULT_UPDATE_SEED) } - /// Deserializes a compact Theta sketch using the expected `seed`. + /// Deserializes a compact Theta sketch using `seed`. /// - /// Empty sketches do not require a matching seed hash. + /// Empty sketches use the hash of `seed` regardless of the stored seed hash. /// /// # Errors /// @@ -839,9 +837,10 @@ impl CompactThetaSketch { _ => return Err(Error::invalid_preamble_longs(&[1, 2, 3], pre_longs)), }; - // Java's v2 conversion normalizes empty images to its seed-independent empty sketch. if num_entries == 0 && theta == MAX_THETA { - return Ok(Self::from_compact_state(CompactSketchState::empty(0))); + return Ok(Self::from_compact_state(CompactSketchState::empty( + expected_seed_hash, + ))); } check_seed_hash( @@ -872,7 +871,7 @@ impl CompactThetaSketch { let empty = (flags & FLAGS_IS_EMPTY) != 0; if empty { return Ok(Self::from_compact_state(CompactSketchState::empty( - seed_hash, + expected_seed_hash, ))); } @@ -1012,7 +1011,7 @@ impl CompactThetaSketch { let ordered = (flags & FLAGS_IS_ORDERED) != 0; let compact_state = if empty { - CompactSketchState::empty(seed_hash) + CompactSketchState::empty(expected_seed_hash) } else { CompactSketchState::non_empty(entries, theta, seed_hash, ordered) }; diff --git a/datasketches/src/thetafamily/tuple/sketch.rs b/datasketches/src/thetafamily/tuple/sketch.rs index 81e56967..509ef16e 100644 --- a/datasketches/src/thetafamily/tuple/sketch.rs +++ b/datasketches/src/thetafamily/tuple/sketch.rs @@ -504,9 +504,7 @@ impl CompactTupleSketch { self.compact_state.is_ordered() } - /// Returns the 16-bit fingerprint used to check seed compatibility. - /// - /// Empty sketches may return `0`, indicating no seed association. + /// Returns the 16-bit fingerprint of the seed associated with this sketch. pub fn seed_hash(&self) -> u16 { self.compact_state.seed_hash() } @@ -636,9 +634,9 @@ impl CompactTupleSketch { Self::deserialize_with_seed(bytes, DEFAULT_UPDATE_SEED) } - /// Deserializes a compact Tuple sketch using the expected `seed`. + /// Deserializes a compact Tuple sketch using `seed`. /// - /// Empty sketches do not require a matching seed hash. + /// Empty sketches use the hash of `seed` regardless of the stored seed hash. /// /// # Errors /// @@ -688,7 +686,7 @@ impl CompactTupleSketch { if empty { return Ok(Self::from_compact_state(CompactSketchState::empty( - seed_hash, + expected_seed_hash, ))); } diff --git a/tests-integration/tests/serde_tests/theta.rs b/tests-integration/tests/serde_tests/theta.rs index 0ef87312..c7f52dc4 100644 --- a/tests-integration/tests/serde_tests/theta.rs +++ b/tests-integration/tests/serde_tests/theta.rs @@ -166,6 +166,7 @@ fn empty_images_use_canonical_seed_hash() { "theta_n0_cpp.sk", )) .unwrap(); + let default_seed_hash = ThetaSketchBuilder::default().build().unwrap().seed_hash(); for builder in [ ThetaSketchBuilder::default(), @@ -181,10 +182,21 @@ fn empty_images_use_canonical_seed_hash() { let mut legacy = expected.clone(); legacy[6..8].copy_from_slice(&sketch.seed_hash().to_le_bytes()); for bytes in [&expected, &legacy] { - let restored = CompactThetaSketch::deserialize_with_seed(bytes, 456).unwrap(); - assert!(restored.is_empty()); - assert_eq!(restored.serialize(), expected); - assert_eq!(restored.serialize_compressed(), expected); + let restored = CompactThetaSketch::deserialize(bytes).unwrap(); + assert_eq!(restored.seed_hash(), default_seed_hash); + + for seed in [123, 456] { + let expected_seed_hash = ThetaSketchBuilder::default() + .seed(seed) + .build() + .unwrap() + .seed_hash(); + let restored = CompactThetaSketch::deserialize_with_seed(bytes, seed).unwrap(); + assert!(restored.is_empty()); + assert_eq!(restored.seed_hash(), expected_seed_hash); + assert_eq!(restored.serialize(), expected); + assert_eq!(restored.serialize_compressed(), expected); + } } } } @@ -295,6 +307,7 @@ fn test_v2_java_empty_layouts() { .unwrap() .compact(true); let expected = reference.serialize(); + let default_seed_hash = ThetaSketchBuilder::default().build().unwrap().seed_hash(); // Port of Java 8.0.0 ForwardCompatibilityTest's checkSerVer2_{1,2,3}PreLong[s]_Empty: // https://github.com/apache/datasketches-java/blob/8.0.0/src/test/java/org/apache/datasketches/theta/ForwardCompatibilityTest.java#L90-L137 @@ -312,12 +325,17 @@ fn test_v2_java_empty_layouts() { assert!(sketch.is_empty()); assert!(!sketch.is_estimation_mode()); assert!(sketch.is_ordered()); - assert_eq!(sketch.seed_hash(), 0); + assert_eq!(sketch.seed_hash(), default_seed_hash); // Extend Java's default-seed cases with zero and mismatched seed hashes. for seed_hash in [0, reference.seed_hash()] { bytes[6..8].copy_from_slice(&seed_hash.to_le_bytes()); for seed in [123, 456] { + let expected_seed_hash = ThetaSketchBuilder::default() + .seed(seed) + .build() + .unwrap() + .seed_hash(); let sketch = CompactThetaSketch::deserialize_with_seed(&bytes, seed).unwrap(); assert!(sketch.is_empty()); assert!(!sketch.is_estimation_mode()); @@ -326,18 +344,51 @@ fn test_v2_java_empty_layouts() { assert_eq!(sketch.estimate(), 0.0); assert_eq!(sketch.lower_bound(NumStdDev::One), 0.0); assert_eq!(sketch.upper_bound(NumStdDev::One), 0.0); - assert_eq!(sketch.seed_hash(), 0); + assert_eq!(sketch.seed_hash(), expected_seed_hash); assert_eq!(sketch.serialize(), expected); let restored = CompactThetaSketch::deserialize_with_seed(&sketch.serialize(), seed).unwrap(); assert!(restored.is_empty()); - assert_eq!(restored.seed_hash(), 0); + assert_eq!(restored.seed_hash(), expected_seed_hash); } } } } +#[test] +fn empty_v1_and_v4_images_use_requested_seed() { + let mut v1 = vec![0; 24]; + v1[..3].copy_from_slice(&[3, 1, 3]); + v1[16..24].copy_from_slice(&(i64::MAX as u64).to_le_bytes()); + + // The v4 header is followed by one byte encoding an entry count of zero. + let v4 = vec![1, 4, 3, 0, 1, 0x1e, 0, 0, 0]; + let default = ThetaSketchBuilder::default().build().unwrap().compact(true); + let mut legacy_v4 = v4.clone(); + legacy_v4[6..8].copy_from_slice(&default.seed_hash().to_le_bytes()); + + for bytes in [v1, v4, legacy_v4] { + let restored = CompactThetaSketch::deserialize(&bytes).unwrap(); + assert_eq!(restored.seed_hash(), default.seed_hash()); + + for seed in [123, 456] { + let original = ThetaSketchBuilder::default() + .seed(seed) + .build() + .unwrap() + .compact(true); + let restored = CompactThetaSketch::deserialize_with_seed(&bytes, seed).unwrap(); + assert!(restored.is_empty()); + assert_eq!(restored.seed_hash(), original.seed_hash()); + assert_eq!(restored.serialize(), original.serialize()); + let round_trip = + CompactThetaSketch::deserialize_with_seed(&restored.serialize(), seed).unwrap(); + assert_eq!(round_trip.seed_hash(), original.seed_hash()); + } + } +} + #[test] fn test_v2_non_empty_images_validate_seed_hash() { let exact = serialize_v2_exact(&[1]); diff --git a/tests-integration/tests/serde_tests/tuple.rs b/tests-integration/tests/serde_tests/tuple.rs index 200bf798..d3ba0b14 100644 --- a/tests-integration/tests/serde_tests/tuple.rs +++ b/tests-integration/tests/serde_tests/tuple.rs @@ -115,6 +115,10 @@ fn empty_images_use_canonical_seed_hash() { "tuple_int_n0_cpp.sk", )) .unwrap(); + let default_seed_hash = TupleSketchBuilder::new(DefaultUpdatePolicy::::default()) + .build() + .unwrap() + .seed_hash(); for builder in [ TupleSketchBuilder::new(DefaultUpdatePolicy::::default()), @@ -128,10 +132,26 @@ fn empty_images_use_canonical_seed_hash() { // Older Rust images stored the configured seed hash even when empty. let mut legacy = expected.clone(); legacy[6..8].copy_from_slice(&sketch.seed_hash().to_le_bytes()); - for bytes in [&expected, &legacy] { - let restored = CompactTupleSketch::::deserialize_with_seed(bytes, 456).unwrap(); - assert!(restored.is_empty()); - assert_eq!(restored.serialize(), expected); + let mut legacy_format = legacy.clone(); + legacy_format[1] = 1; // legacy serialization version + legacy_format[3] = 5; // legacy compact sketch type + for bytes in [&expected, &legacy, &legacy_format] { + let restored = CompactTupleSketch::::deserialize(bytes).unwrap(); + assert_eq!(restored.seed_hash(), default_seed_hash); + + for seed in [123, 456] { + let expected_seed_hash = + TupleSketchBuilder::new(DefaultUpdatePolicy::::default()) + .seed(seed) + .build() + .unwrap() + .seed_hash(); + let restored = + CompactTupleSketch::::deserialize_with_seed(bytes, seed).unwrap(); + assert!(restored.is_empty()); + assert_eq!(restored.seed_hash(), expected_seed_hash); + assert_eq!(restored.serialize(), expected); + } } } } From 3d4f1e9bf3587506a2813081329fdfefde8bf46a Mon Sep 17 00:00:00 2001 From: tison Date: Tue, 29 Sep 2026 22:55:30 +0800 Subject: [PATCH 7/7] test: focus empty sketch serialization coverage on its contracts --- tests-integration/tests/serde_tests/theta.rs | 167 +++++-------------- tests-integration/tests/serde_tests/tuple.rs | 54 ++---- 2 files changed, 54 insertions(+), 167 deletions(-) diff --git a/tests-integration/tests/serde_tests/theta.rs b/tests-integration/tests/serde_tests/theta.rs index c7f52dc4..b51fc2d5 100644 --- a/tests-integration/tests/serde_tests/theta.rs +++ b/tests-integration/tests/serde_tests/theta.rs @@ -160,43 +160,29 @@ fn test_go_compatibility() { } #[test] -fn empty_images_use_canonical_seed_hash() { +fn empty_sketch_serialization() { let expected = fs::read(serialization_test_data( "cpp_generated_files", "theta_n0_cpp.sk", )) .unwrap(); - let default_seed_hash = ThetaSketchBuilder::default().build().unwrap().seed_hash(); - - for builder in [ - ThetaSketchBuilder::default(), - ThetaSketchBuilder::default() - .seed(123) - .sampling_probability(0.5), - ] { - let sketch = builder.build().unwrap().compact(false); - assert_eq!(sketch.serialize(), expected); - assert_eq!(sketch.serialize_compressed(), expected); - - // Older Rust images stored the configured seed hash even when empty. - let mut legacy = expected.clone(); - legacy[6..8].copy_from_slice(&sketch.seed_hash().to_le_bytes()); - for bytes in [&expected, &legacy] { - let restored = CompactThetaSketch::deserialize(bytes).unwrap(); - assert_eq!(restored.seed_hash(), default_seed_hash); - - for seed in [123, 456] { - let expected_seed_hash = ThetaSketchBuilder::default() - .seed(seed) - .build() - .unwrap() - .seed_hash(); - let restored = CompactThetaSketch::deserialize_with_seed(bytes, seed).unwrap(); - assert!(restored.is_empty()); - assert_eq!(restored.seed_hash(), expected_seed_hash); - assert_eq!(restored.serialize(), expected); - assert_eq!(restored.serialize_compressed(), expected); - } + let sketch = ThetaSketchBuilder::default() + .seed(123) + .sampling_probability(0.5) + .build() + .unwrap() + .compact(false); + assert_eq!(sketch.serialize(), expected); + assert_eq!(sketch.serialize_compressed(), expected); + + let other_seed_hash = ThetaSketchBuilder::default().build().unwrap().seed_hash(); + // v3 and v4 empty inputs; v4 includes a one-byte entry count of zero. + for mut bytes in [expected, vec![1, 4, 3, 0, 1, 0x1e, 0, 0, 0]] { + for stored_seed_hash in [0, other_seed_hash] { + bytes[6..8].copy_from_slice(&stored_seed_hash.to_le_bytes()); + let restored = CompactThetaSketch::deserialize_with_seed(&bytes, 123).unwrap(); + assert!(restored.is_empty()); + assert_eq!(restored.seed_hash(), sketch.seed_hash()); } } } @@ -213,14 +199,19 @@ fn non_empty_images_without_entries_preserve_seed_hash() { assert!(!compact.is_empty()); assert_eq!(compact.num_retained(), 0); - for bytes in [compact.serialize(), compact.serialize_compressed()] { + // v2 and v3 share the count/theta layout for sampled sketches. + for version in [2, 3] { + let mut bytes = compact.serialize(); + bytes[1] = version; let restored = CompactThetaSketch::deserialize_with_seed(&bytes, 123).unwrap(); assert!(!restored.is_empty()); - assert_eq!(restored.num_retained(), 0); assert_eq!(restored.theta64(), compact.theta64()); - assert_eq!(restored.seed_hash(), compact.seed_hash()); let err = CompactThetaSketch::deserialize(&bytes).unwrap_err(); assert_eq!(err.kind(), ErrorKind::InvalidData); + + bytes[6..8].fill(0); + let err = CompactThetaSketch::deserialize_with_seed(&bytes, 123).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidData); } } @@ -301,111 +292,29 @@ fn test_v2_exact_non_empty_compatibility() { #[test] fn test_v2_java_empty_layouts() { - let reference = ThetaSketchBuilder::default() + let expected_seed_hash = ThetaSketchBuilder::default() .seed(123) .build() .unwrap() - .compact(true); - let expected = reference.serialize(); - let default_seed_hash = ThetaSketchBuilder::default().build().unwrap().seed_hash(); + .seed_hash(); + let other_seed_hash = ThetaSketchBuilder::default().build().unwrap().seed_hash(); // Port of Java 8.0.0 ForwardCompatibilityTest's checkSerVer2_{1,2,3}PreLong[s]_Empty: // https://github.com/apache/datasketches-java/blob/8.0.0/src/test/java/org/apache/datasketches/theta/ForwardCompatibilityTest.java#L90-L137 for pre_longs in [1, 2, 3] { let mut bytes = serialize_v2_exact(&[]); bytes[0] = pre_longs; - bytes[5] = 0x0e; // Java's v2 flags: no rebuild, empty, read-only, little-endian. - if pre_longs == 1 { - bytes.truncate(8); - } else if pre_longs == 3 { - bytes.extend_from_slice(&(i64::MAX as u64).to_le_bytes()); - } - - let sketch = CompactThetaSketch::deserialize(&bytes).unwrap(); - assert!(sketch.is_empty()); - assert!(!sketch.is_estimation_mode()); - assert!(sketch.is_ordered()); - assert_eq!(sketch.seed_hash(), default_seed_hash); - - // Extend Java's default-seed cases with zero and mismatched seed hashes. - for seed_hash in [0, reference.seed_hash()] { - bytes[6..8].copy_from_slice(&seed_hash.to_le_bytes()); - for seed in [123, 456] { - let expected_seed_hash = ThetaSketchBuilder::default() - .seed(seed) - .build() - .unwrap() - .seed_hash(); - let sketch = CompactThetaSketch::deserialize_with_seed(&bytes, seed).unwrap(); - assert!(sketch.is_empty()); - assert!(!sketch.is_estimation_mode()); - assert!(sketch.is_ordered()); - assert_eq!(sketch.num_retained(), 0); - assert_eq!(sketch.estimate(), 0.0); - assert_eq!(sketch.lower_bound(NumStdDev::One), 0.0); - assert_eq!(sketch.upper_bound(NumStdDev::One), 0.0); - assert_eq!(sketch.seed_hash(), expected_seed_hash); - assert_eq!(sketch.serialize(), expected); - - let restored = - CompactThetaSketch::deserialize_with_seed(&sketch.serialize(), seed).unwrap(); - assert!(restored.is_empty()); - assert_eq!(restored.seed_hash(), expected_seed_hash); - } + bytes[5] = 0x0e; // Java's v2 empty flags. + bytes.resize(pre_longs as usize * 8, 0); + if pre_longs == 3 { + bytes[16..24].copy_from_slice(&(i64::MAX as u64).to_le_bytes()); } - } -} -#[test] -fn empty_v1_and_v4_images_use_requested_seed() { - let mut v1 = vec![0; 24]; - v1[..3].copy_from_slice(&[3, 1, 3]); - v1[16..24].copy_from_slice(&(i64::MAX as u64).to_le_bytes()); - - // The v4 header is followed by one byte encoding an entry count of zero. - let v4 = vec![1, 4, 3, 0, 1, 0x1e, 0, 0, 0]; - let default = ThetaSketchBuilder::default().build().unwrap().compact(true); - let mut legacy_v4 = v4.clone(); - legacy_v4[6..8].copy_from_slice(&default.seed_hash().to_le_bytes()); - - for bytes in [v1, v4, legacy_v4] { - let restored = CompactThetaSketch::deserialize(&bytes).unwrap(); - assert_eq!(restored.seed_hash(), default.seed_hash()); - - for seed in [123, 456] { - let original = ThetaSketchBuilder::default() - .seed(seed) - .build() - .unwrap() - .compact(true); - let restored = CompactThetaSketch::deserialize_with_seed(&bytes, seed).unwrap(); - assert!(restored.is_empty()); - assert_eq!(restored.seed_hash(), original.seed_hash()); - assert_eq!(restored.serialize(), original.serialize()); - let round_trip = - CompactThetaSketch::deserialize_with_seed(&restored.serialize(), seed).unwrap(); - assert_eq!(round_trip.seed_hash(), original.seed_hash()); + for stored_seed_hash in [0, other_seed_hash] { + bytes[6..8].copy_from_slice(&stored_seed_hash.to_le_bytes()); + let sketch = CompactThetaSketch::deserialize_with_seed(&bytes, 123).unwrap(); + assert!(sketch.is_empty()); + assert_eq!(sketch.seed_hash(), expected_seed_hash); } } } - -#[test] -fn test_v2_non_empty_images_validate_seed_hash() { - let exact = serialize_v2_exact(&[1]); - let mut sampled = serialize_v2_exact(&[]); - sampled[0] = 3; - sampled.extend_from_slice(&((i64::MAX as u64) / 2).to_le_bytes()); - - // Zero retained entries with theta < 1 still describe a non-empty sketch. - for mut bytes in [exact, sampled] { - let sketch = CompactThetaSketch::deserialize(&bytes).unwrap(); - assert!(!sketch.is_empty()); - - let err = CompactThetaSketch::deserialize_with_seed(&bytes, 456).unwrap_err(); - assert_eq!(err.kind(), ErrorKind::InvalidData); - - bytes[6..8].fill(0); - let err = CompactThetaSketch::deserialize(&bytes).unwrap_err(); - assert_eq!(err.kind(), ErrorKind::InvalidData); - } -} diff --git a/tests-integration/tests/serde_tests/tuple.rs b/tests-integration/tests/serde_tests/tuple.rs index d3ba0b14..3a71e25a 100644 --- a/tests-integration/tests/serde_tests/tuple.rs +++ b/tests-integration/tests/serde_tests/tuple.rs @@ -109,50 +109,30 @@ fn test_go_compatibility() { } #[test] -fn empty_images_use_canonical_seed_hash() { +fn empty_sketch_serialization() { let expected = fs::read(serialization_test_data( "cpp_generated_files", "tuple_int_n0_cpp.sk", )) .unwrap(); - let default_seed_hash = TupleSketchBuilder::new(DefaultUpdatePolicy::::default()) + let sketch = TupleSketchBuilder::new(DefaultUpdatePolicy::::default()) + .seed(123) + .sampling_probability(0.5) .build() .unwrap() - .seed_hash(); + .compact(false); + assert_eq!(sketch.serialize(), expected); - for builder in [ - TupleSketchBuilder::new(DefaultUpdatePolicy::::default()), - TupleSketchBuilder::new(DefaultUpdatePolicy::::default()) - .seed(123) - .sampling_probability(0.5), - ] { - let sketch = builder.build().unwrap().compact(false); - assert_eq!(sketch.serialize(), expected); - - // Older Rust images stored the configured seed hash even when empty. - let mut legacy = expected.clone(); - legacy[6..8].copy_from_slice(&sketch.seed_hash().to_le_bytes()); - let mut legacy_format = legacy.clone(); - legacy_format[1] = 1; // legacy serialization version - legacy_format[3] = 5; // legacy compact sketch type - for bytes in [&expected, &legacy, &legacy_format] { - let restored = CompactTupleSketch::::deserialize(bytes).unwrap(); - assert_eq!(restored.seed_hash(), default_seed_hash); - - for seed in [123, 456] { - let expected_seed_hash = - TupleSketchBuilder::new(DefaultUpdatePolicy::::default()) - .seed(seed) - .build() - .unwrap() - .seed_hash(); - let restored = - CompactTupleSketch::::deserialize_with_seed(bytes, seed).unwrap(); - assert!(restored.is_empty()); - assert_eq!(restored.seed_hash(), expected_seed_hash); - assert_eq!(restored.serialize(), expected); - } - } + let other_seed_hash = TupleSketchBuilder::new(DefaultUpdatePolicy::::default()) + .build() + .unwrap() + .seed_hash(); + for stored_seed_hash in [0, other_seed_hash] { + let mut bytes = expected.clone(); + bytes[6..8].copy_from_slice(&stored_seed_hash.to_le_bytes()); + let restored = CompactTupleSketch::::deserialize_with_seed(&bytes, 123).unwrap(); + assert!(restored.is_empty()); + assert_eq!(restored.seed_hash(), sketch.seed_hash()); } } @@ -171,9 +151,7 @@ fn non_empty_images_without_entries_preserve_seed_hash() { let bytes = compact.serialize(); let restored = CompactTupleSketch::::deserialize_with_seed(&bytes, 123).unwrap(); assert!(!restored.is_empty()); - assert_eq!(restored.num_retained(), 0); assert_eq!(restored.theta64(), compact.theta64()); - assert_eq!(restored.seed_hash(), compact.seed_hash()); let err = CompactTupleSketch::::deserialize(&bytes).unwrap_err(); assert_eq!(err.kind(), ErrorKind::InvalidData); }