Derive hashmap filter k from expected element count via envelope.

include/bitcoin/database/impl/primitives/hashhead.ipp

@@ -20,6 +20,7 @@
 #define LIBBITCOIN_DATABASE_PRIMITIVES_HASHHEAD_IPP
 
 #include <algorithm>
+#include <cmath>
 #include <bitcoin/database/define.hpp>
 
 // Heads are not subject to resize/remap and therefore do not require memory
@@ -32,9 +33,10 @@ namespace database {
 // ----------------------------------------------------------------------------
 
 TEMPLATE
-CLASS::hashhead(storage& head, size_t buckets) NOEXCEPT
+CLASS::hashhead(storage& head, size_t buckets, size_t expected) NOEXCEPT
   : file_(head),
-    buckets_(system::possible_narrow_cast<link>(buckets))
+    buckets_(system::possible_narrow_cast<link>(buckets)),
+    k_(optimal_k(expected, buckets))
 {
     BC_ASSERT(buckets <= Link::terminal);
 }
@@ -51,6 +53,12 @@ inline size_t CLASS::buckets() const NOEXCEPT
     return buckets_;
 }
 
+TEMPLATE
+inline size_t CLASS::filter_k() const NOEXCEPT
+{
+    return k_;
+}
+
 TEMPLATE
 bool CLASS::create() NOEXCEPT
 {
@@ -84,6 +92,92 @@ bool CLASS::verify() const NOEXCEPT
     return file_.size() == size();
 }
 
+TEMPLATE
+bool CLASS::set_filter_k(size_t k) NOEXCEPT
+{
+    if constexpr (is_zero(k_max))
+    {
+        return is_zero(k);
+    }
+    else
+    {
+        if (is_zero(k) || (k > k_max))
+            return false;
+
+        k_ = k;
+        return true;
+    }
+}
+
+TEMPLATE
+size_t CLASS::optimal_k(size_t count, size_t buckets) NOEXCEPT
+{
+    if constexpr (is_zero(k_max))
+    {
+        return k_default;
+    }
+    else
+    {
+        if (is_zero(count) || is_zero(buckets))
+            return k_default;
+
+        const auto load_factor = system::to_floating(count) /
+            system::to_floating(buckets);
+
+        auto optimum = one;
+        auto minimum = expected_walk(one, load_factor);
+        for (auto k = two; k <= k_max; ++k)
+        {
+            const auto cost = expected_walk(k, load_factor);
+            if (cost < minimum)
+            {
+                minimum = cost;
+                optimum = k;
+            }
+        }
+
+        return optimum;
+    }
+}
+
+// Bloom false positive rate for n keys of k selections over m bits.
+TEMPLATE
+double CLASS::bloom_false_positive(size_t k, size_t n) NOEXCEPT
+{
+    using namespace system;
+    const auto retain = 1.0 - (1.0 / to_floating(m));
+    const auto value = std::pow(retain, to_floating(k * n));
+    return std::pow(1.0 - value, to_floating(k));
+}
+
+// Poisson mass is negligible past ten deviations of the mean. The minimum
+// floors the bound for small means where deviations are near zero.
+TEMPLATE
+size_t CLASS::poisson_span(double load_factor) NOEXCEPT
+{
+    constexpr auto deviations = 10.0;
+    constexpr auto minimum = 30.0;
+    return system::to_integer<size_t>(std::ceil(load_factor +
+        deviations * std::sqrt(load_factor) + minimum));
+}
+
+// False positive over poisson bucket occupancy, weighted by the occupancy.
+TEMPLATE
+double CLASS::expected_walk(size_t k, double load_factor) NOEXCEPT
+{
+    using namespace system;
+    auto walk = 0.0;
+    auto poisson = std::exp(-load_factor);
+    const auto span = poisson_span(load_factor);
+    for (auto n = one; n <= span; ++n)
+    {
+        poisson *= load_factor / to_floating(n);
+        walk += poisson * to_floating(n) * bloom_false_positive(k, n);
+    }
+
+    return walk;
+}
+
 TEMPLATE
 bool CLASS::get_body_count(Link& count) const NOEXCEPT
 {
@@ -266,8 +360,8 @@ INLINE constexpr CLASS::link CLASS::to_link(cell value) NOEXCEPT
 }
 
 TEMPLATE
-INLINE constexpr CLASS::cell CLASS::next_cell(bool& collision, cell previous,
-    link current, uint64_t entropy) NOEXCEPT
+INLINE CLASS::cell CLASS::next_cell(bool& collision, cell previous,
+    link current, uint64_t entropy) const NOEXCEPT
 {
     if constexpr (filter_t::disabled)
     {
@@ -278,22 +372,22 @@ INLINE constexpr CLASS::cell CLASS::next_cell(bool& collision, cell previous,
     {
         using namespace system;
         const auto prev = to_filter(previous);
-        const auto next = filter_t::screen(prev, entropy);
+        const auto next = filter_t::screen(prev, entropy, k_);
         collision = filter_t::is_collision(prev, next);
         return bit_or<cell>(shift_left<cell>(next, link_bits), current);
     }
 }
 
 TEMPLATE
-INLINE constexpr bool CLASS::screened(cell value, uint64_t entropy) NOEXCEPT
+INLINE bool CLASS::screened(cell value, uint64_t entropy) const NOEXCEPT
 {
     if constexpr (filter_t::disabled)
     {
         return true;
     }
     else
     {
-        return filter_t::is_screened(to_filter(value), entropy);
+        return filter_t::is_screened(to_filter(value), entropy, k_);
     }
 }
 

include/bitcoin/database/impl/primitives/hashmap.ipp

@@ -27,8 +27,9 @@ namespace libbitcoin {
 namespace database {
 
 TEMPLATE
-CLASS::hashmap(storage& header, storage& body, const Link& buckets) NOEXCEPT
-  : head_(header, buckets), body_(body)
+CLASS::hashmap(storage& header, storage& body, const Link& buckets,
+    size_t expected) NOEXCEPT
+  : head_(header, buckets, expected), body_(body)
 {
 }
 
@@ -86,6 +87,18 @@ size_t CLASS::buckets() const NOEXCEPT
     return head_.buckets();
 }
 
+TEMPLATE
+size_t CLASS::filter_k() const NOEXCEPT
+{
+    return head_.filter_k();
+}
+
+TEMPLATE
+bool CLASS::set_filter_k(size_t k) NOEXCEPT
+{
+    return head_.set_filter_k(k);
+}
+
 TEMPLATE
 size_t CLASS::head_size() const NOEXCEPT
 {

include/bitcoin/database/impl/primitives/hashmaps.ipp

@@ -25,8 +25,9 @@ namespace libbitcoin {
 namespace database {
 
 TEMPLATE
-CLASS::hashmaps(storage& header, storage& body, const Link& buckets) NOEXCEPT
-  : head_(header, buckets), body_(body)
+CLASS::hashmaps(storage& header, storage& body, const Link& buckets,
+    size_t expected) NOEXCEPT
+  : head_(header, buckets, expected), body_(body)
 {
 }
 
@@ -84,6 +85,18 @@ size_t CLASS::buckets() const NOEXCEPT
     return head_.buckets();
 }
 
+TEMPLATE
+size_t CLASS::filter_k() const NOEXCEPT
+{
+    return head_.filter_k();
+}
+
+TEMPLATE
+bool CLASS::set_filter_k(size_t k) NOEXCEPT
+{
+    return head_.set_filter_k(k);
+}
+
 TEMPLATE
 size_t CLASS::head_size() const NOEXCEPT
 {

include/bitcoin/database/impl/store/store.ipp

@@ -109,31 +109,42 @@ CLASS::store(const settings& config) NOEXCEPT
     // Tables.
     // ------------------------------------------------------------------------
 
-    header(header_head_, header_body_, config.header.buckets),
+    header(header_head_, header_body_, config.header.buckets, config.header.expected),
     input(input_head_, input_body_),
     output(output_head_, output_body_),
-    ins(ins_head_, ins_body_, config.ins.buckets),
-    outs(outs_head_, outs_body_, config.outs.buckets),
-    tx(tx_head_, tx_body_, config.tx.buckets),
+    ins(ins_head_, ins_body_, config.ins.buckets, config.ins.expected),
+    outs(outs_head_, outs_body_, config.outs.buckets, config.outs.expected),
+    tx(tx_head_, tx_body_, config.tx.buckets, config.tx.expected),
     txs(txs_head_, txs_body_, config.txs.buckets),
 
     candidate(candidate_head_),
     confirmed(confirmed_head_),
-    strong_tx(strong_tx_head_, strong_tx_body_, config.strong_tx.buckets),
+    strong_tx(strong_tx_head_, strong_tx_body_, config.strong_tx.buckets, config.strong_tx.expected),
 
     ecdsa(ecdsa_head_, ecdsa_body_),
     schnorr(schnorr_head_, schnorr_body_),
     silent(silent_head_, silent_body_),
-    duplicate(duplicate_head_, duplicate_body_, config.duplicate.buckets),
+    duplicate(duplicate_head_, duplicate_body_, config.duplicate.buckets, config.duplicate.expected),
     prevalid(prevalid_head_, prevalid_body_),
     prevout(prevout_head_, prevout_body_, config.prevout.buckets),
     validated_bk(validated_bk_head_, validated_bk_body_, config.validated_bk.buckets),
-    validated_tx(validated_tx_head_, validated_tx_body_, config.validated_tx.buckets),
+    validated_tx(validated_tx_head_, validated_tx_body_, config.validated_tx.buckets, config.validated_tx.expected),
 
     filter_bk(filter_bk_head_, filter_bk_body_, config.filter_bk.buckets),
     filter_tx(filter_tx_head_, filter_tx_body_, config.filter_tx.buckets)
 {
     envelope_.set(config);
+
+    // Filter k derives from configured expected/buckets, envelope records it
+    // at create and overrides it at open (stored values govern the store).
+    using namespace system;
+    envelope_.header_k = possible_narrow_cast<uint8_t>(header.filter_k());
+    envelope_.ins_k = possible_narrow_cast<uint8_t>(ins.filter_k());
+    envelope_.outs_k = possible_narrow_cast<uint8_t>(outs.filter_k());
+    envelope_.tx_k = possible_narrow_cast<uint8_t>(tx.filter_k());
+    envelope_.strong_tx_k = possible_narrow_cast<uint8_t>(strong_tx.filter_k());
+    envelope_.duplicate_k = possible_narrow_cast<uint8_t>(duplicate.filter_k());
+    envelope_.validated_tx_k = possible_narrow_cast<uint8_t>(validated_tx.filter_k());
 }
 
 TEMPLATE

include/bitcoin/database/impl/store/store_open.ipp

@@ -85,17 +85,7 @@ code CLASS::open(const event_handler& handler) NOEXCEPT
     verify(ec, filter_tx, table_t::filter_tx_table);
 
     if (!ec)
-    {
-        // The stored creation envelope governs, configuration is not read.
-        table::txs::get_envelope genesis{};
-        if (!is_zero(txs.body_size()))
-        {
-            if (txs.at(zero, genesis))
-                envelope_ = genesis.envelope;
-            else
-                ec = error::verify_table;
-        }
-    }
+        ec = load_envelope();
 
     if (ec)
     {
@@ -111,6 +101,34 @@ code CLASS::open(const event_handler& handler) NOEXCEPT
     return ec;
 }
 
+TEMPLATE
+code CLASS::load_envelope() NOEXCEPT
+{
+    // The stored creation envelope governs, configuration is not read.
+    table::txs::get_envelope genesis{};
+    if (!is_zero(txs.body_size()))
+    {
+        if (txs.at(zero, genesis))
+            envelope_ = genesis.envelope;
+        else
+            return error::verify_table;
+    }
+
+    // The stored filter k values govern filter bit interpretation.
+    if (!header.set_filter_k(envelope_.header_k) ||
+        !ins.set_filter_k(envelope_.ins_k) ||
+        !outs.set_filter_k(envelope_.outs_k) ||
+        !tx.set_filter_k(envelope_.tx_k) ||
+        !strong_tx.set_filter_k(envelope_.strong_tx_k) ||
+        !duplicate.set_filter_k(envelope_.duplicate_k) ||
+        !validated_tx.set_filter_k(envelope_.validated_tx_k))
+    {
+        return error::verify_table;
+    }
+
+    return error::success;
+}
+
 } // namespace database
 } // namespace libbitcoin
 

include/bitcoin/database/impl/store/store_restore.ipp

@@ -156,6 +156,9 @@ code CLASS::restore(const event_handler& handler) NOEXCEPT
         restore(ec, filter_bk, table_t::filter_bk_table);
         restore(ec, filter_tx, table_t::filter_tx_table);
 
+        if (!ec)
+            ec = load_envelope();
+
         if (ec)
             /* code */ unload_close(handler);
     }