From eda62dfb6db475ec1b0e19f39de796a1d48ed30c Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 21 Oct 2026 19:02:52 +0300 Subject: [PATCH 1/3] perf(encode): trim the optimal parser's per-search work The binary-tree search behind levels 16-22 cost about 1.4x upstream's instructions per searched position. It now runs at 1.13x, with every frame byte-identical to before. - The tree walk works in one stored coordinate, as upstream walks `matchIndex`: the floor test, the candidate's bytes and the offset are derived from the stored index, the window gate is one compare, and the two open sides are pointers with a dummy for a closed side. - The block chooses the search for its key width once and calls it through a function pointer (upstream `ZSTD_selectBtGetAllMatches`), so the hashes take the width as a constant. - A repeat is gated by one unsigned compare, the hash3 probe drops checks its strategy already guarantees, the search checks the remaining length once, and the pass clamps the insert cursor. - Candidates are written into the room the pass reserves, sized for the most one search can produce, instead of through a growing push. - The search-width documentation now states upstream's hashing: a minMatch of 3 hashes four bytes there; an explicit minMatch of 3 keeps the three-byte key here, where only btultra2 uses hash3. Measured on an x86 host with hardware counters, prebuilt binaries interleaved: level 16-22 cycles fall 7-8% on 10 KiB inputs, 3.5-6% on the 1 MiB corpus file and 7-8% on source text; level 3, which does not run this search, is flat. Closes #545 --- zstd/src/encoding/hc/generator.rs | 286 ++++++++++++--------- zstd/src/encoding/hc/mod.rs | 10 +- zstd/src/encoding/hc/optimal.rs | 201 ++++++++++----- zstd/src/encoding/levels/config.rs | 36 +-- zstd/src/encoding/match_generator/tests.rs | 4 +- 5 files changed, 324 insertions(+), 213 deletions(-) diff --git a/zstd/src/encoding/hc/generator.rs b/zstd/src/encoding/hc/generator.rs index a000ab31..08d7e5d2 100644 --- a/zstd/src/encoding/hc/generator.rs +++ b/zstd/src/encoding/hc/generator.rs @@ -528,6 +528,29 @@ macro_rules! prefix_len_first_word { } pub(crate) use prefix_len_first_word; +/// Append a candidate to the optimal search's ladder without a capacity check +/// (upstream writes `matches[mnum]` the same way): the pass reserved +/// [`HC_OPT_CANDIDATE_CAPACITY`](crate::encoding::hc::HC_OPT_CANDIDATE_CAPACITY) +/// slots, more than one search can write. +macro_rules! push_opt_candidate { + ($out:expr, $candidate:expr) => {{ + let out: &mut alloc::vec::Vec<$crate::encoding::opt::types::MatchCandidate> = $out; + let candidate = $candidate; + let len = out.len(); + debug_assert!( + len < out.capacity(), + "the candidate ladder outgrew its reservation" + ); + // SAFETY: `len < capacity` (the reservation above), so the slot is + // allocated, and the new length covers only initialised elements. + unsafe { + out.as_mut_ptr().add(len).write(candidate); + out.set_len(len + 1); + } + }}; +} +pub(crate) use push_opt_candidate; + /// One repeat-offset probe, expanded per slot. Crate-private (see /// `bt_insert_range_body!`). /// @@ -541,7 +564,6 @@ macro_rules! rep_probe_slot { ( $rep:expr, $abs_pos:ident, - $hist_start:ident, $rbase:ident, $idx:ident, $cur_gate:ident, @@ -557,14 +579,13 @@ macro_rules! rep_probe_slot { ) => { 'slot: { let rep: usize = $rep; - if rep == 0 || rep > $abs_pos { + // A repeat reaches back inside the history when `1 <= rep <= + // idx`, which one unsigned compare asks: a zero wraps past every + // `idx` (upstream `repOffset-1 < curr-dictLimit`, zstd_opt.c). + if rep.wrapping_sub(1) >= $idx { break 'slot; } - let candidate_pos = $abs_pos - rep; - if candidate_pos < $hist_start { - break 'slot; - } - let candidate_idx = candidate_pos - $hist_start; + let candidate_idx = $idx - rep; // SAFETY: `candidate_idx < idx` (rep >= 1) and `idx + 4 <= rlen`, // so the 4-byte read stays inside `concat`. let cand_word = unsafe { @@ -601,11 +622,14 @@ macro_rules! rep_probe_slot { // `if (repLen > bestLength)`, zstd_opt.c:678); the minimum length // was checked just above. if match_len > *$best_len_for_skip { - $out.push($crate::encoding::opt::types::MatchCandidate { - start: $abs_pos, - offset: rep, - match_len, - }); + $crate::encoding::hc::generator::push_opt_candidate!( + &mut *$out, + $crate::encoding::opt::types::MatchCandidate { + start: $abs_pos, + offset: rep, + match_len, + } + ); *$best_len_for_skip = match_len; } // `abs_pos + match_len >= current_abs_end` is `match_len >= @@ -635,6 +659,8 @@ macro_rules! bt_insert_and_collect_matches_body { $reps:ident, $ll0:ident, $use_hash3:expr, + // The tree's key width, a const of the search the block chose. + $mls:expr, $cpl:path, $cmf:path $(,)? ) => {{ @@ -667,7 +693,10 @@ macro_rules! bt_insert_and_collect_matches_body { let hist_start = $table.history_abs_start; let cbase = concat.as_ptr(); let clen = concat.len(); - if idx + 4 <= clen { + // The caller searches only positions with eight bytes of the block + // left, which every read below relies on. + debug_assert!(idx + 8 <= clen); + { let rbase = cbase; let rlen = clen; // Upstream walks the three repeat offsets as plain indices with the @@ -714,14 +743,13 @@ macro_rules! bt_insert_and_collect_matches_body { // 2% alongside them and the control arm flat, so it is added // work rather than layout. Upstream writes the same rejection the // same way, as an intentional unsigned overflow that "discards 0 - // and -1": a `reps[0]` of 0 wraps past `abs_pos` and one of 1 - // becomes the zero the slot's first test rejects. + // and -1": a `reps[0]` of 0 or 1 becomes a value the slot's one + // range compare rejects. macro_rules! probe { ($slot_rep:expr) => { $crate::encoding::hc::generator::rep_probe_slot!( $slot_rep, $abs_pos, - hist_start, rbase, idx, cur_gate, @@ -767,7 +795,7 @@ macro_rules! bt_insert_and_collect_matches_body { target_feature = "sse", any(target_arch = "x86", target_arch = "x86_64") ))] - if $use_hash3 && $table.hash3_log > 13 && idx + 1 + 4 <= clen { + if $use_hash3 && $table.hash3_log > 13 { #[cfg(target_arch = "x86")] use core::arch::x86::{_MM_HINT_T0, _mm_prefetch}; #[cfg(target_arch = "x86_64")] @@ -804,7 +832,10 @@ macro_rules! bt_insert_and_collect_matches_body { "block coordinates are stale" ); let h3_start = $table.next_to_update3; - let h3_ptr: *mut u32 = if h3_log != 0 && h3_start >= hist_start && idx + 4 <= clen { + // A strategy that uses the table has it configured (asserted at the + // top), and this position's four bytes were checked on entry. + debug_assert!(h3_log != 0 && idx + 4 <= clen); + let h3_ptr: *mut u32 = if h3_start >= hist_start { debug_assert_eq!($table.hash3_table().len(), 1usize << h3_log); let table3 = table3_base; let insert = |cursor: usize| { @@ -846,9 +877,7 @@ macro_rules! bt_insert_and_collect_matches_body { // the BT collect's `concat` / `idx` / `tail_limit`. Labeled block so // the probe's early-outs yield None without returning from the walk. let h3_candidate: Option<$crate::encoding::opt::types::MatchCandidate> = - if h3_log == 0 || idx + 4 > clen { - None - } else { + { 'h3: { let hh = $crate::encoding::match_table::storage::MatchTable::hash_position_at( @@ -933,7 +962,7 @@ macro_rules! bt_insert_and_collect_matches_body { debug_assert!( h3.match_len >= $min_match_len && h3.match_len > *$best_len_for_skip ); - $out.push(h3); + $crate::encoding::hc::generator::push_opt_candidate!(&mut *$out, h3); *$best_len_for_skip = h3.match_len; // Same fold as the repeat probe: the block end lives in the // length space the probe already carries. @@ -948,14 +977,11 @@ macro_rules! bt_insert_and_collect_matches_body { if skip_further_match_search { return; } - if idx + 8 > concat.len() { - return; - } let hash = $crate::encoding::match_table::storage::MatchTable::hash_position_at( concat, idx, $table.hash_log, - $table.search_mls, + $mls, ); // Upstream holds `U32* const hashTable = ms->hashTable` and the tree's // `bt` for the whole body (zstd_opt.c:607); the walk below indexes the @@ -997,7 +1023,7 @@ macro_rules! bt_insert_and_collect_matches_body { concat, idx + 1, $table.hash_log, - $table.search_mls, + $mls, ); // SAFETY: prefetch never faults; an out-of-range index is a // harmless no-op hint. @@ -1019,33 +1045,31 @@ macro_rules! bt_insert_and_collect_matches_body { // first BT walk of a fresh frame where `abs_pos < bt_mask`. let bt_low = $abs_pos.saturating_sub(bt_mask); let window_low = $table.window_low_abs_for_target($abs_pos); - // Upstream zstd-style window bound in stored space so the BT-walk loop - // condition rejects out-of-window / HC_EMPTY candidates WITHOUT - // decoding them (mirrors upstream `while ... matchIndex >= matchLow`): - // one range check on `match_stored` instead of decode-then-break, - // dropping the wasted candidate_abs decode on every walk's terminating - // step. candidate_abs(s) = (position_base + s - 1) - index_shift = - // base + s (wrapping); in-window ⟺ candidate_abs - window_low < - // abs_pos - window_low ⟺ s.wrapping_add(win_off) < win_range. - // HC_EMPTY (s = 0) maps to base = (lowest representable abs) - 1 < - // window_low, so it falls out of range and ends the walk. - let win_off = coords.abs_bias.wrapping_sub(window_low); - let win_range = $abs_pos - window_low; - // Decode biases: fold the per-node coordinate conversions into - // loop-invariant additions. The gate-validated chain entry - // `match_stored` maps to its absolute position, its history-relative - // index and its BT pair slot by a single add each, instead of - // re-reading `position_base` / `index_shift` / `history_abs_start` / - // `bt_mask` from `self` and round-tripping `index_shift` through the - // slot computation on every node. Upstream zstd walks one - // window-relative `matchIndex` directly (`match = base + matchIndex`, - // slot `2*(matchIndex & btMask)`); these biases are the - // single-coordinate equivalent. Wrapping throughout: the window gate - // already proved `match_stored ∈ [window_low, abs_pos)` before decode, - // mirroring the `win_off` form above. - let abs_bias = coords.abs_bias; - let idx_bias = coords.idx_bias; + // The tree floor in the stored space the walk already carries, so a + // node is tested as `stored <= bt_low_stored` without decoding it to an + // absolute position (upstream compares `matchIndex <= btLow` in its own + // single coordinate). Every walked candidate is at or above the window + // floor, whose stored index is at least 1 and exact; a floor below the + // window can never be reached, so it sits one under the window instead + // of wrapping. + let win_low_stored = window_low.wrapping_sub(coords.abs_bias); + let bt_low_stored = if bt_low < window_low { + win_low_stored - 1 + } else { + bt_low.wrapping_sub(coords.abs_bias) + }; + // A node is reached through its stored index alone, as upstream walks + // one `matchIndex` (`match = base + matchIndex`, slot + // `2*(matchIndex & btMask)`): its bytes sit at `cand_base + stored` + // and its pair at `(stored + bt_bias) & bt_mask`, and nothing decodes + // it to an absolute position unless it joins the ladder. + // `cand_base` may lie outside the history; only the candidate it is + // offset to is ever read, and the window gate proves that one inside. + debug_assert!(win_low_stored >= 1); let bt_bias = coords.bt_bias; + // SAFETY: `idx + 8 <= concat.len()`, which the search's entry checks. + let cur_ptr = unsafe { cbase.add(idx) }; + let cand_base = cbase.wrapping_add(coords.idx_bias); // Raw `+ 9` is safe here — see `bt_insert_range_body!` // for the full discussion of the upstream `STREAM_ABS_HEADROOM` // cap in `MatchTable::fill_uncommitted`. @@ -1065,8 +1089,14 @@ macro_rules! bt_insert_and_collect_matches_body { // `abs_pos + index_shift`. let pair_idx = 2 * ((stored as usize).wrapping_add(bt_bias) & bt_mask); debug_assert_eq!(pair_idx, $table.bt_pair_index_for_abs($abs_pos)); - let mut smaller_slot = pair_idx; - let mut larger_slot = pair_idx + 1; + // The two open sides of the tree, as pointers into the pair table + // (upstream `smallerPtr` / `largerPtr`). A side the walk closes at the + // tree floor is pointed at `closed_side` instead, so the final store + // below needs no test (upstream `dummy32`). + let mut closed_side: u32 = 0; + // SAFETY: `pair_idx + 1 < chain_table.len()` by the pair invariant. + let mut smaller_slot: *mut u32 = unsafe { chain_ptr.add(pair_idx) }; + let mut larger_slot: *mut u32 = unsafe { chain_ptr.add(pair_idx + 1) }; // SAFETY: `hash` is masked to `hash_log` bits and the table is // `1 << hash_log` slots wide (both asserted at `hash_ptr`), so the slot // is in range by construction. Upstream reads and writes the same slot @@ -1083,22 +1113,26 @@ macro_rules! bt_insert_and_collect_matches_body { ); let mut best_len = (*$best_len_for_skip).max($min_match_len - 1); - // Upstream zstd-form loop condition: the stored-space window range check - // (`s.wrapping_add(win_off) < win_range`) rejects out-of-window and - // HC_EMPTY candidates here, so the terminating step never enters the - // body — no wasted candidate_abs decode, matching upstream's - // `while ... matchIndex >= matchLow`. - while compares_left > 0 && (match_stored as usize).wrapping_add(win_off) < win_range { + // Upstream's `while (nbCompares && matchIndex >= matchLow)`: a node below + // the window floor, and the empty slot 0 below every floor, ends the + // walk. Nothing in the tree lies at or past the searched position: + // the parser searches positions in order and only ever inserts the + // ones before it, and a pass that leaves entries behind (the btultra2 + // seed pass) moves the stored offset past all of them. + while compares_left > 0 && match_stored as usize >= win_low_stored { compares_left -= 1; - // The condition proved this candidate is in `[window_low, - // abs_pos)`, so `match_stored >= 1` (HC_EMPTY is out of range). - // Decode via the precomputed biases (single add each), the - // single-coordinate form of upstream's `matchIndex`. - let stored = match_stored as usize; - let candidate_abs = stored.wrapping_add(abs_bias); + debug_assert!(match_stored < stored, "a tree entry at or past the searched position"); + let node_stored = match_stored as usize; + debug_assert_eq!( + node_stored <= bt_low_stored, + node_stored.wrapping_add(coords.abs_bias) <= bt_low, + ); // `2*(candidate_abs + index_shift & bt_mask)` with `index_shift` // folded away: `candidate_abs + index_shift == stored + bt_bias`. - let next_pair_idx = 2 * (stored.wrapping_add(bt_bias) & bt_mask); + // SAFETY: the pair index is below `chain_table.len() - 1`; + // `chain_ptr` is the hoisted live base, not reallocated during the + // walk. + let next_pair = unsafe { chain_ptr.add(2 * (node_stored.wrapping_add(bt_bias) & bt_mask)) }; // Both children are read ahead of the compare, where upstream reads // only the side it descends, so the next node's load can overlap // the compare. Reading only the chosen side after it measured about @@ -1106,16 +1140,26 @@ macro_rules! bt_insert_and_collect_matches_body { // against a flat level 3 control; that is below what two builds // resolve, so neither order is shown faster and the walk keeps its // existing one. - // SAFETY: `next_pair_idx (+1)` = `2*(candidate_abs & bt_mask) (+1)` - // ≤ `chain_table.len()-1`; `chain_ptr` is the hoisted live base, - // table not realloc'd during the walk. - let next_smaller = unsafe { *chain_ptr.add(next_pair_idx) }; - let next_larger = unsafe { *chain_ptr.add(next_pair_idx + 1) }; + // SAFETY: as for `next_pair`. + let next_smaller = unsafe { *next_pair }; + let next_larger = unsafe { *next_pair.add(1) }; let seed_len = common_length_smaller.min(common_length_larger); - let candidate_idx = stored.wrapping_add(idx_bias); - // SAFETY: BT walk invariant — `candidate_idx + tail_limit ≤ - // concat.len()`. - let match_len = unsafe { $cmf(concat, idx, candidate_idx, tail_limit, seed_len) }; + // Every side's common length is a match the walk did not stop at, + // so it is below the tail limit and the compare below has bytes + // left to read. + debug_assert!(seed_len < tail_limit); + let cand_ptr = cand_base.wrapping_add(node_stored); + // SAFETY: BT walk invariant: the candidate and the current position + // both have `tail_limit` readable bytes, `seed_len` of them already + // known equal. + let match_len = seed_len + + unsafe { + $cpl( + cand_ptr.add(seed_len), + cur_ptr.add(seed_len), + tail_limit - seed_len, + ) + }; if match_len > best_len { // `best_len` is `max(best_len_for_skip, min_match_len - 1)`, @@ -1123,20 +1167,22 @@ macro_rules! bt_insert_and_collect_matches_body { // best: it joins the ladder unconditionally, as upstream's // `matches[mnum]` store does (zstd_opt.c:749-752). debug_assert!(match_len >= $min_match_len && match_len > *$best_len_for_skip); - $out.push($crate::encoding::opt::types::MatchCandidate { - start: $abs_pos, - offset: $abs_pos - candidate_abs, - match_len, - }); + // `abs_pos - candidate_abs`, both in stored space. + let offset = stored.wrapping_sub(match_stored) as usize; + $crate::encoding::hc::generator::push_opt_candidate!( + &mut *$out, + $crate::encoding::opt::types::MatchCandidate { + start: $abs_pos, + offset, + match_len, + } + ); *$best_len_for_skip = match_len; best_len = match_len; - // BT walker invariants: `candidate_abs < abs_pos` and - // `match_len <= tail_limit = current_abs_end - abs_pos`. So - // `candidate_abs + match_len < abs_pos + tail_limit = - // current_abs_end`, which fits in `usize` on every supported - // target (32-bit i686 included) — the addition stays within - // the current block. - let candidate_end = candidate_abs + match_len; + // BT walker invariants: the candidate precedes `abs_pos` and + // `match_len <= tail_limit = current_abs_end - abs_pos`, so the + // candidate's end stays within the current block. + let candidate_end = $abs_pos - offset + match_len; if candidate_end > match_end_abs { match_end_abs = candidate_end; } @@ -1151,49 +1197,38 @@ macro_rules! bt_insert_and_collect_matches_body { break; } - let candidate_next = candidate_idx + match_len; - let current_next = idx + match_len; - // SAFETY: first-differing positions after a match_len-long prefix; - // match_len < tail_limit (break above) + BT-walk bound - // idx/candidate_idx + tail_limit <= concat.len() keep both in range. - if unsafe { - *concat.get_unchecked(candidate_next) < *concat.get_unchecked(current_next) - } { - // SAFETY: `smaller_slot` holds a valid pair index (init - // `pair_idx`, updated to `next_pair_idx + 1`); the `usize::MAX` - // sentinel is set only just before `break`, never written here. - unsafe { *chain_ptr.add(smaller_slot) = match_stored }; + // SAFETY: the first differing bytes after a `match_len`-long + // prefix; `match_len < tail_limit` (break above) keeps both in + // range. + if unsafe { *cand_ptr.add(match_len) < *cur_ptr.add(match_len) } { + // SAFETY: an open side points at a pair slot (initial pair or + // the descended node's), or at `closed_side`. + unsafe { *smaller_slot = match_stored }; common_length_smaller = match_len; - if candidate_abs <= bt_low { - smaller_slot = usize::MAX; + if node_stored <= bt_low_stored { + smaller_slot = &raw mut closed_side; break; } - smaller_slot = next_pair_idx + 1; + // SAFETY: as for `next_pair`. + smaller_slot = unsafe { next_pair.add(1) }; match_stored = next_larger; } else { // SAFETY: as above for `larger_slot`. - unsafe { *chain_ptr.add(larger_slot) = match_stored }; + unsafe { *larger_slot = match_stored }; common_length_larger = match_len; - if candidate_abs <= bt_low { - larger_slot = usize::MAX; + if node_stored <= bt_low_stored { + larger_slot = &raw mut closed_side; break; } - larger_slot = next_pair_idx; + larger_slot = next_pair; match_stored = next_smaller; } } - // SAFETY: both slots, when not the `usize::MAX` sentinel, hold valid - // pair indices into the hoisted `chain_table` base. - if smaller_slot != usize::MAX { - unsafe { - *chain_ptr.add(smaller_slot) = $crate::encoding::match_table::storage::HC_EMPTY - }; - } - if larger_slot != usize::MAX { - unsafe { - *chain_ptr.add(larger_slot) = $crate::encoding::match_table::storage::HC_EMPTY - }; + // SAFETY: each side points at a pair slot or at `closed_side`. + unsafe { + *smaller_slot = $crate::encoding::match_table::storage::HC_EMPTY; + *larger_slot = $crate::encoding::match_table::storage::HC_EMPTY; } // Dict dual-probe (upstream zstd `ZSTD_dictMatchState`, zstd_opt.c:777-813): @@ -1252,11 +1287,14 @@ macro_rules! bt_insert_and_collect_matches_body { debug_assert!( match_len >= $min_match_len && match_len > *$best_len_for_skip ); - $out.push($crate::encoding::opt::types::MatchCandidate { - start: $abs_pos, - offset: idx - dict_idx, - match_len, - }); + $crate::encoding::hc::generator::push_opt_candidate!( + &mut *$out, + $crate::encoding::opt::types::MatchCandidate { + start: $abs_pos, + offset: idx - dict_idx, + match_len, + } + ); *$best_len_for_skip = match_len; best_len = match_len; // Where the match ends in the SOURCE, not at the position diff --git a/zstd/src/encoding/hc/mod.rs b/zstd/src/encoding/hc/mod.rs index 3eb99150..8e740466 100644 --- a/zstd/src/encoding/hc/mod.rs +++ b/zstd/src/encoding/hc/mod.rs @@ -61,10 +61,16 @@ impl HcMatch { } } -/// Hard cap on chain-walk depth, and the capacity the optimal parser reserves -/// for its per-position candidate buffer. +/// Hard cap on chain-walk depth. pub(crate) const MAX_HC_SEARCH_DEPTH: usize = 512; +/// Capacity the optimal parser reserves for its per-position candidate buffer: +/// one search adds at most three repeats, one hash3 match and one candidate per +/// tree compare (the live and the dictionary tree share a budget of at most +/// [`MAX_HC_SEARCH_DEPTH`]), and the long-distance producer one more, so the +/// search writes into it without a capacity check. +pub(crate) const HC_OPT_CANDIDATE_CAPACITY: usize = MAX_HC_SEARCH_DEPTH + 8; + /// Hash-chain matcher state used by the `Lazy2` parse mode (and the /// short-history fast path of the BT cascade's initial pass). /// diff --git a/zstd/src/encoding/hc/optimal.rs b/zstd/src/encoding/hc/optimal.rs index 6a296bb7..45d0a293 100644 --- a/zstd/src/encoding/hc/optimal.rs +++ b/zstd/src/encoding/hc/optimal.rs @@ -16,7 +16,7 @@ use super::generator::HcMatchGenerator; use crate::encoding::Sequence; use crate::encoding::blocks::encode_offset_with_history; -use crate::encoding::hc::MAX_HC_SEARCH_DEPTH; +use crate::encoding::hc::HC_OPT_CANDIDATE_CAPACITY; use crate::encoding::hc::priceset::{ priceset_avx2, priceset_neon, priceset_scalar, priceset_simd128, priceset_sse2, priceset_sse41, }; @@ -47,6 +47,19 @@ use crate::encoding::{ }, }; +/// One position's search, built for the block's strategy, key width and kernel +/// (`collect_optimal_candidates_initialized_`), as the block's pass +/// calls it. +type OptimalCollectFn = unsafe fn( + &mut HcMatchGenerator, + usize, + usize, + usize, + &[u32; 3], + bool, + &mut Vec, +); + macro_rules! build_optimal_plan_impl_body { ( $self:expr, @@ -212,7 +225,8 @@ macro_rules! build_optimal_plan_impl_body { // kernel's target_feature umbrella, entered under the runtime // detector. unsafe { - $self.$collect::<$strategy_ty>( + $collect( + &mut *$self, $current_abs_start + skipped_literals, current_abs_end, profile.sufficient_match_len, @@ -317,7 +331,8 @@ macro_rules! build_optimal_plan_impl_body { // `$collect` kernel variant; the runtime kernel detector already // gated entry into the wrapper. unsafe { - $self.$collect::<$strategy_ty>( + $collect( + &mut *$self, $current_abs_start, current_abs_end, profile.sufficient_match_len, @@ -710,7 +725,8 @@ macro_rules! build_optimal_plan_impl_body { // stays live across it; the post-call reads below are a fresh load // of the same stable `nodes[pos]`. unsafe { - $self.$collect::<$strategy_ty>( + $collect( + &mut *$self, abs_pos, current_abs_end, profile.sufficient_match_len, @@ -1103,6 +1119,25 @@ macro_rules! optimal_block_body { ) => {{ // The block is armed, so the tree's coordinates hold for the whole pass. $self.table.capture_block_coords(); + // The tree's insert cursor starts inside the history, so no search has + // to pull it up: within the pass it only moves forward. + $self.table.skip_insert_until_abs = $self + .table + .skip_insert_until_abs + .max($self.table.history_abs_start); + // The search's key width is fixed for the block, so the search built + // for it is chosen once here and every position calls it directly, as + // upstream picks `ZSTD_btGetAllMatches__` per block + // (`ZSTD_selectBtGetAllMatches`). + let collect: OptimalCollectFn = match $self.table.search_mls { + 3 => HcMatchGenerator::$collect::<$strategy_ty, 3>, + 5 => HcMatchGenerator::$collect::<$strategy_ty, 5>, + 6 => HcMatchGenerator::$collect::<$strategy_ty, 6>, + mls => { + debug_assert_eq!(mls, 4, "the binary tree keys on three to six bytes"); + HcMatchGenerator::$collect::<$strategy_ty, 4> + } + }; // Everything the pass carries between segments lives in `$buffers.pass`, // read at the top of a segment and written at its end, so none of it is // live across the segment body (see `HcOptimalPlanBuffers::pass`). @@ -1139,7 +1174,7 @@ macro_rules! optimal_block_body { segment_stats, $plan, $buffers, - $collect, + collect, $priceset, $current, $opt_state, @@ -1173,10 +1208,15 @@ macro_rules! collect_optimal_candidates_initialized_body { $reps:ident, $ll0:ident, $out:ident, + $mls:expr, $bt_insert_range:ident, $cpl:path, $cmf:path $(,)? ) => {{ + debug_assert_eq!( + $mls, $self.table.search_mls, + "the block chose the search for its key width" + ); // Per-strategy compile-time const: only BtUltra2 drives the // hash3 short-match table. All other monomorphisations drop // the entire hash3 lookup block at codegen time. The relaxed @@ -1203,11 +1243,10 @@ macro_rules! collect_optimal_candidates_initialized_body { // binary tree before this position's search, as one run (upstream // zstd ZSTD_updateTree). `$bt_insert_range` overshoots the target // with no clamp (a long match skips the positions it covers), - // exactly as upstream. - if $self.table.skip_insert_until_abs < $self.table.history_abs_start { - $self.table.skip_insert_until_abs = $self.table.history_abs_start; - } + // exactly as upstream. The pass starts the cursor inside the + // history and only ever moves it forward. let update_abs = $self.table.skip_insert_until_abs; + debug_assert!(update_abs >= $self.table.history_abs_start); // No rebase guard: `arm_block_positions` cleared every position in // this block before the parse began, and the catch-up only inserts // positions below `$abs_pos`, which is in it. @@ -1224,8 +1263,10 @@ macro_rules! collect_optimal_candidates_initialized_body { $self.table.skip_insert_until_abs = $abs_pos; } let current_idx = $abs_pos - $self.table.history_abs_start; - // The pass's live history length, taken with its coordinates. - if current_idx + 4 > $self.table.block_live.1 { + // Every probe below reads up to eight bytes at the position; the parser + // searches only positions with that much of the block left, so this is + // the one length check the search needs. + if current_idx + 8 > $self.table.block_live.1 { return; } let mut best_len_for_skip = 0usize; @@ -1254,6 +1295,7 @@ macro_rules! collect_optimal_candidates_initialized_body { $reps, $ll0, use_hash3, + $mls, $cpl, $cmf, ); @@ -1740,9 +1782,8 @@ impl HcMatchGenerator { if node_prices.len() < HC_OPT_NODE_LEN { node_prices = alloc::boxed::Box::new_uninit_slice(HC_OPT_NODE_LEN); } - if candidates.capacity() < MAX_HC_SEARCH_DEPTH { - candidates.reserve_exact(MAX_HC_SEARCH_DEPTH - candidates.capacity()); - } + candidates.clear(); + candidates.reserve_exact(HC_OPT_CANDIDATE_CAPACITY); if price_arena.len() < HC_OPT_PRICE_ARENA_LEN { price_arena = alloc::vec![[0u32; 2]; HC_OPT_PRICE_ARENA_LEN].into_boxed_slice(); } @@ -2055,6 +2096,14 @@ impl HcMatchGenerator { use crate::encoding::strategy::{self, StrategyTag}; self.table.ensure_tables(); self.table.capture_block_coords(); + // As a block's pass starts it (see `optimal_block_body!`), and with + // the room the search writes its candidates into. + self.table.skip_insert_until_abs = self + .table + .skip_insert_until_abs + .max(self.table.history_abs_start); + out.clear(); + out.reserve_exact(HC_OPT_CANDIDATE_CAPACITY); let reps = &query.reps; let ll0 = query.lit_len == 0; // Dispatch purely from `self.strategy_tag` (set by @@ -2062,42 +2111,50 @@ impl HcMatchGenerator { // way production does — wiring up `table.hash3_log` directly // without setting a matching `strategy_tag` is no longer // allowed. + // The key width picks the search the way the block's pass picks it. + macro_rules! search { + ($strategy:ty) => { + match self.table.search_mls { + 3 => self.collect_optimal_candidates_initialized::<$strategy, 3>( + abs_pos, + current_abs_end, + sufficient_match_len, + reps, + ll0, + out, + ), + 5 => self.collect_optimal_candidates_initialized::<$strategy, 5>( + abs_pos, + current_abs_end, + sufficient_match_len, + reps, + ll0, + out, + ), + 6 => self.collect_optimal_candidates_initialized::<$strategy, 6>( + abs_pos, + current_abs_end, + sufficient_match_len, + reps, + ll0, + out, + ), + _ => self.collect_optimal_candidates_initialized::<$strategy, 4>( + abs_pos, + current_abs_end, + sufficient_match_len, + reps, + ll0, + out, + ), + } + }; + } match self.strategy_tag { - StrategyTag::BtUltra2 => self - .collect_optimal_candidates_initialized::( - abs_pos, - current_abs_end, - sufficient_match_len, - reps, - ll0, - out, - ), - StrategyTag::BtUltra => self - .collect_optimal_candidates_initialized::( - abs_pos, - current_abs_end, - sufficient_match_len, - reps, - ll0, - out, - ), - StrategyTag::Btlazy2 => self - .collect_optimal_candidates_initialized::( - abs_pos, - current_abs_end, - sufficient_match_len, - reps, - ll0, - out, - ), - StrategyTag::BtOpt => self.collect_optimal_candidates_initialized::( - abs_pos, - current_abs_end, - sufficient_match_len, - reps, - ll0, - out, - ), + StrategyTag::BtUltra2 => search!(strategy::BtUltra2), + StrategyTag::BtUltra => search!(strategy::BtUltra), + StrategyTag::Btlazy2 => search!(strategy::Btlazy2), + StrategyTag::BtOpt => search!(strategy::BtOpt), StrategyTag::Fast | StrategyTag::Dfast | StrategyTag::Greedy | StrategyTag::Lazy => { // Optimal candidate collection is binary-tree only (btopt / // btultra / btultra2 / btlazy2). The Fast / Dfast / Greedy / Lazy @@ -2137,7 +2194,10 @@ impl HcMatchGenerator { /// cover the rest. #[allow(dead_code)] #[inline(always)] - pub(crate) fn collect_optimal_candidates_initialized( + pub(crate) fn collect_optimal_candidates_initialized< + S: crate::encoding::strategy::Strategy, + const MLS: usize, + >( &mut self, abs_pos: usize, current_abs_end: usize, @@ -2152,7 +2212,7 @@ impl HcMatchGenerator { feature = "kernel-neon" ))] unsafe { - self.collect_optimal_candidates_initialized_neon::( + self.collect_optimal_candidates_initialized_neon::( abs_pos, current_abs_end, sufficient_match_len, @@ -2167,7 +2227,7 @@ impl HcMatchGenerator { match self.table.kernel { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => unsafe { - self.collect_optimal_candidates_initialized_avx2_bmi2::( + self.collect_optimal_candidates_initialized_avx2_bmi2::( abs_pos, current_abs_end, sufficient_match_len, @@ -2178,7 +2238,7 @@ impl HcMatchGenerator { }, #[cfg(feature = "kernel-sse")] FastpathKernel::Sse2 => unsafe { - self.collect_optimal_candidates_initialized_sse2::( + self.collect_optimal_candidates_initialized_sse2::( abs_pos, current_abs_end, sufficient_match_len, @@ -2189,7 +2249,7 @@ impl HcMatchGenerator { }, #[cfg(feature = "kernel-sse")] FastpathKernel::Sse42 => unsafe { - self.collect_optimal_candidates_initialized_sse42::( + self.collect_optimal_candidates_initialized_sse42::( abs_pos, current_abs_end, sufficient_match_len, @@ -2198,14 +2258,15 @@ impl HcMatchGenerator { out, ) }, - FastpathKernel::Scalar => self.collect_optimal_candidates_initialized_scalar::( - abs_pos, - current_abs_end, - sufficient_match_len, - reps, - ll0, - out, - ), + FastpathKernel::Scalar => self + .collect_optimal_candidates_initialized_scalar::( + abs_pos, + current_abs_end, + sufficient_match_len, + reps, + ll0, + out, + ), } } // wasm resolves `simd128` at compile time (no runtime detection), so @@ -2218,7 +2279,7 @@ impl HcMatchGenerator { // SAFETY: the `cfg` above establishes `simd128` at compile time, which // is exactly the umbrella the callee declares. unsafe { - self.collect_optimal_candidates_initialized_simd128::( + self.collect_optimal_candidates_initialized_simd128::( abs_pos, current_abs_end, sufficient_match_len, @@ -2242,7 +2303,7 @@ impl HcMatchGenerator { ) )))] { - self.collect_optimal_candidates_initialized_scalar::( + self.collect_optimal_candidates_initialized_scalar::( abs_pos, current_abs_end, sufficient_match_len, @@ -2266,6 +2327,7 @@ impl HcMatchGenerator { #[inline(never)] unsafe fn collect_optimal_candidates_initialized_neon< S: crate::encoding::strategy::Strategy, + const MLS: usize, >( &mut self, abs_pos: usize, @@ -2284,6 +2346,7 @@ impl HcMatchGenerator { reps, ll0, out, + MLS, bt_insert_range_neon, crate::encoding::fastpath::neon::common_prefix_len_ptr, crate::encoding::fastpath::neon::count_match_from_indices, @@ -2298,6 +2361,7 @@ impl HcMatchGenerator { #[inline(never)] unsafe fn collect_optimal_candidates_initialized_sse42< S: crate::encoding::strategy::Strategy, + const MLS: usize, >( &mut self, abs_pos: usize, @@ -2316,6 +2380,7 @@ impl HcMatchGenerator { reps, ll0, out, + MLS, bt_insert_range_sse2, crate::encoding::fastpath::sse2::common_prefix_len_ptr, crate::encoding::fastpath::sse2::count_match_from_indices, @@ -2336,6 +2401,7 @@ impl HcMatchGenerator { #[inline(never)] unsafe fn collect_optimal_candidates_initialized_sse2< S: crate::encoding::strategy::Strategy, + const MLS: usize, >( &mut self, abs_pos: usize, @@ -2354,6 +2420,7 @@ impl HcMatchGenerator { reps, ll0, out, + MLS, bt_insert_range_sse2, crate::encoding::fastpath::sse2::common_prefix_len_ptr, crate::encoding::fastpath::sse2::count_match_from_indices, @@ -2368,6 +2435,7 @@ impl HcMatchGenerator { #[inline(never)] unsafe fn collect_optimal_candidates_initialized_avx2_bmi2< S: crate::encoding::strategy::Strategy, + const MLS: usize, >( &mut self, abs_pos: usize, @@ -2386,6 +2454,7 @@ impl HcMatchGenerator { reps, ll0, out, + MLS, bt_insert_range_avx2_bmi2, crate::encoding::fastpath::avx2_bmi2::common_prefix_len_ptr, crate::encoding::fastpath::avx2_bmi2::count_match_from_indices, @@ -2407,6 +2476,7 @@ impl HcMatchGenerator { #[inline(never)] unsafe fn collect_optimal_candidates_initialized_simd128< S: crate::encoding::strategy::Strategy, + const MLS: usize, >( &mut self, abs_pos: usize, @@ -2425,6 +2495,7 @@ impl HcMatchGenerator { reps, ll0, out, + MLS, bt_insert_range_simd128, crate::encoding::fastpath::simd128::common_prefix_len_ptr, crate::encoding::fastpath::simd128::count_match_from_indices, @@ -2442,6 +2513,7 @@ impl HcMatchGenerator { #[inline(never)] pub(crate) fn collect_optimal_candidates_initialized_scalar< S: crate::encoding::strategy::Strategy, + const MLS: usize, >( &mut self, abs_pos: usize, @@ -2460,6 +2532,7 @@ impl HcMatchGenerator { reps, ll0, out, + MLS, bt_insert_range_scalar, crate::encoding::fastpath::scalar::common_prefix_len_ptr, crate::encoding::fastpath::scalar::count_match_from_indices, diff --git a/zstd/src/encoding/levels/config.rs b/zstd/src/encoding/levels/config.rs index ab5ff92b..cff68eac 100644 --- a/zstd/src/encoding/levels/config.rs +++ b/zstd/src/encoding/levels/config.rs @@ -22,19 +22,17 @@ pub(crate) struct HcConfig { pub(crate) chain_log: usize, pub(crate) search_depth: usize, pub(crate) target_len: usize, - /// Binary-tree finder hash width. Upstream uses `mls = BOUNDED(3, minMatch, 6)` - /// (`ZSTD_selectBtGetAllMatches`, zstd_opt.c:896) — i.e. mls=3 on the - /// btultra/btultra2 levels (L18-22, minMatch=3) — and surfaces 3-byte matches - /// through a fallback-only HC3 finder (zstd_opt.c:691-720: distance < 256 KiB, - /// taken only when no longer/repcode match exists). Our optimal parser does - /// NOT yet replicate that 3-byte handling — it emits short matches C prices - /// out, breaking level-22 sequence parity — so the BT hash width is clamped UP - /// to 4 (`cp.min_match.clamp(4, 6)`) to keep the finder from surfacing those - /// 3-byte matches and so match C's output. This is a deliberate workaround, - /// NOT C's finder width; drop the clamp once the optimal parser is C-faithful - /// at minMatch 3 (tracked in #337). Carried explicitly per level so a - /// `target_length` override can't silently change the finder's hashing width. - /// Only the BT body reads it; HC / lazy levels keep it at 4. + /// Key width the binary tree hashes. A level's own `minMatch` is clamped to + /// `4..=6`: upstream templates its finder on `mls = BOUNDED(3, minMatch, 6)` + /// (`ZSTD_selectBtGetAllMatches`, zstd_opt.c), but its `ZSTD_hashPtr` has no + /// 3-byte case and hashes `mls = 3` on four bytes (the `default` arm, + /// zstd_compress_internal.h), taking 3-byte matches from the hash3 table. + /// A `minMatch` of 3 set explicitly keys the tree on three bytes instead: + /// here the hash3 table serves btultra2 only, and on the other optimal + /// strategies a 4-byte key would leave the tree as the only source of the + /// short matches the knob asks for and miss them (z000033 at level 16 with + /// `minMatch` 3: 443,830 bytes against 428,164). Only the BT body reads it; + /// HC / lazy levels keep it at 4. pub(crate) search_mls: usize, } @@ -436,11 +434,8 @@ pub(crate) fn apply_param_overrides( hc.target_len = target_length as usize; } if let Some(min_match) = ov.min_match { - // BT finder hash width, derived from cParams.minMatch exactly - // as upstream zstd: `mls = BOUNDED(3, cParams.minMatch, 6)` - // (zstd_opt.c:896 ZSTD_selectBtGetAllMatches). minMatch=3 - // tiers hash on 3 bytes (btultra/btultra2 path). Only the BT - // body reads `search_mls`; HC/lazy hash on 4 bytes regardless. + // A `minMatch` of 3 asked for explicitly keys the tree on + // three bytes (see `HcConfig::search_mls`). hc.search_mls = (min_match as usize).clamp(3, 6); } } @@ -657,10 +652,7 @@ fn level_params_from_cparams(cp: crate::encoding::cparams::CParams) -> LevelPara chain_log: cp.chain_log as usize, search_depth, target_len, - // Clamp UP to 4: C's BT finder uses mls=3 on L18-22, but our optimal - // parser diverges on the resulting 3-byte matches (breaks level-22 - // sequence parity), so we keep the finder at >=4 as a workaround until - // the parser is C-faithful at minMatch 3. See `HcConfig::search_mls` (#337). + // See `HcConfig::search_mls`: a `minMatch` of 3 hashes four bytes. search_mls: cp.min_match.clamp(4, 6) as usize, }; let row = RowConfig { diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs index e6aa353f..8cdd1936 100644 --- a/zstd/src/encoding/match_generator/tests.rs +++ b/zstd/src/encoding/match_generator/tests.rs @@ -1442,7 +1442,9 @@ fn hc_collect_optimal_candidates_keeps_reps_when_chain_depth_zero() { // reads it; `hc.search_depth` above is the configure-time source and does // not reach the walk on its own. hc.table.search_depth = 0; - hc.table.history = b"xyzxyzxyzxyz".to_vec().into(); + // Eight bytes past the searched position, as the parser leaves every + // position it searches. + hc.table.history = b"xyzxyzxyzxyzxyzxyz".to_vec().into(); hc.table.history_start = 0; hc.table.history_abs_start = 0; From acf1251f8a3b6b8f27f064b21cdc10950fee0289 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 21 Oct 2026 19:39:51 +0300 Subject: [PATCH 2/3] test(encode): cover the search dispatch for every kernel tier - hc_collect_optimal_candidates_agree_across_kernel_tiers runs the candidate search under each kernel tier the CPU can execute and requires the same candidates at every position of a block. - The strategy dispatch test covers Btlazy2. - The search-width doc names both strategies that use the hash3 table. --- zstd/src/encoding/levels/config.rs | 9 ++- zstd/src/encoding/match_generator/tests.rs | 73 ++++++++++++++++++++++ 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/zstd/src/encoding/levels/config.rs b/zstd/src/encoding/levels/config.rs index cff68eac..c4d334ad 100644 --- a/zstd/src/encoding/levels/config.rs +++ b/zstd/src/encoding/levels/config.rs @@ -28,11 +28,10 @@ pub(crate) struct HcConfig { /// 3-byte case and hashes `mls = 3` on four bytes (the `default` arm, /// zstd_compress_internal.h), taking 3-byte matches from the hash3 table. /// A `minMatch` of 3 set explicitly keys the tree on three bytes instead: - /// here the hash3 table serves btultra2 only, and on the other optimal - /// strategies a 4-byte key would leave the tree as the only source of the - /// short matches the knob asks for and miss them (z000033 at level 16 with - /// `minMatch` 3: 443,830 bytes against 428,164). Only the BT body reads it; - /// HC / lazy levels keep it at 4. + /// here only btultra and btultra2 search the hash3 table, and on btopt a + /// 4-byte key would leave no source for the short matches the knob asks for + /// (z000033 at level 16 with `minMatch` 3: 443,830 bytes against 428,164). + /// Only the BT body reads it; HC / lazy levels keep it at 4. pub(crate) search_mls: usize, } diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs index 8cdd1936..061bdd7d 100644 --- a/zstd/src/encoding/match_generator/tests.rs +++ b/zstd/src/encoding/match_generator/tests.rs @@ -1511,6 +1511,7 @@ fn hc_collect_optimal_candidates_dispatches_every_bt_strategy() { // offset 12 while the 4-byte hash of `abcZ` misses `abcQ`. Run under the // scalar kernel so the scalar dispatch arm is exercised too. for tag in [ + StrategyTag::Btlazy2, StrategyTag::BtOpt, StrategyTag::BtUltra, StrategyTag::BtUltra2, @@ -1551,6 +1552,78 @@ fn hc_collect_optimal_candidates_dispatches_every_bt_strategy() { } } +/// The candidate search dispatches on the cached kernel tier, and each tier the +/// CPU can run finds the same candidates at every position of a block: the +/// scalar fallback and the SIMD kernels agree on the search itself, not only on +/// the sequences the parser builds from it. Forcing the tier runs the arms the +/// runtime detection leaves cold on this machine. x86 only, where the tier is a +/// runtime choice. +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[test] +fn hc_collect_optimal_candidates_agree_across_kernel_tiers() { + use crate::encoding::fastpath::FastpathKernel; + use crate::encoding::strategy::StrategyTag; + + // Only tiers the running CPU may execute: each dispatch arm assumes its + // target feature is present. + #[allow(unused_mut)] + let mut tiers = alloc::vec![FastpathKernel::Scalar]; + #[cfg(feature = "kernel-sse")] + if std::is_x86_feature_detected!("sse2") { + tiers.push(FastpathKernel::Sse2); + } + #[cfg(feature = "kernel-sse")] + if std::is_x86_feature_detected!("sse4.2") { + tiers.push(FastpathKernel::Sse42); + } + #[cfg(feature = "kernel-avx2")] + if std::is_x86_feature_detected!("avx2") && std::is_x86_feature_detected!("bmi2") { + tiers.push(FastpathKernel::Avx2Bmi2); + } + + // Mixed redundancy, so the repeat, hash3 and tree probes all find matches. + let data: Vec = (0..4096usize) + .map(|i| ((i * 7 + i / 13) % 67) as u8) + .collect(); + let run = |tier: FastpathKernel| -> Vec> { + let mut hc = HcMatchGenerator::new(1 << 20); + hc.configure(BTULTRA2_HC_CONFIG, StrategyTag::BtUltra2, 26); + hc.table.commit_input(&data); + hc.table.kernel = tier; + let mut out = Vec::new(); + (0..data.len() - 8) + .map(|pos| { + hc.collect_optimal_candidates( + pos, + data.len(), + usize::MAX / 2, + HcCandidateQuery { + reps: [1, 4, 8], + lit_len: 1, + ldm_candidate: None, + }, + &mut out, + ); + out.iter().map(|c| (c.offset, c.match_len)).collect() + }) + .collect() + }; + + let reference = run(tiers[0]); + assert!( + reference.iter().any(|found| !found.is_empty()), + "the search should find candidates on mixed-redundancy input" + ); + for &tier in &tiers[1..] { + assert_eq!( + run(tier), + reference, + "kernel tier {tier:?} found different candidates from {:?}", + tiers[0], + ); + } +} + #[test] fn hc_collect_optimal_candidates_rep_tail_match_skips_chain_probe() { let mut hc = HcMatchGenerator::new(64); From 6ecf51165af32e290c76af25f80ee1f2f856384c Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 21 Oct 2026 19:50:32 +0300 Subject: [PATCH 3/3] refactor(encode): pin the tree strategies' compare budget at compile time The candidate buffer's reserved capacity holds only while no strategy that runs the binary-tree search spends more than MAX_HC_SEARCH_DEPTH compares per position; a const assertion now fails the build if one does. --- zstd/src/encoding/hc/mod.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/zstd/src/encoding/hc/mod.rs b/zstd/src/encoding/hc/mod.rs index 8e740466..52d7a7b6 100644 --- a/zstd/src/encoding/hc/mod.rs +++ b/zstd/src/encoding/hc/mod.rs @@ -71,6 +71,16 @@ pub(crate) const MAX_HC_SEARCH_DEPTH: usize = 512; /// search writes into it without a capacity check. pub(crate) const HC_OPT_CANDIDATE_CAPACITY: usize = MAX_HC_SEARCH_DEPTH + 8; +// The capacity above holds only while no strategy that runs the binary-tree +// search can spend more compares than `MAX_HC_SEARCH_DEPTH`. +const _: () = { + use crate::encoding::strategy::{BtOpt, BtUltra, BtUltra2, Btlazy2, Strategy}; + assert!(BtOpt::MAX_CHAIN_DEPTH <= MAX_HC_SEARCH_DEPTH); + assert!(BtUltra::MAX_CHAIN_DEPTH <= MAX_HC_SEARCH_DEPTH); + assert!(BtUltra2::MAX_CHAIN_DEPTH <= MAX_HC_SEARCH_DEPTH); + assert!(Btlazy2::MAX_CHAIN_DEPTH <= MAX_HC_SEARCH_DEPTH); +}; + /// Hash-chain matcher state used by the `Lazy2` parse mode (and the /// short-history fast path of the BT cascade's initial pass). ///