diff --git a/builds/gnu/Makefile.am b/builds/gnu/Makefile.am index 3a3819cae..f8143b859 100644 --- a/builds/gnu/Makefile.am +++ b/builds/gnu/Makefile.am @@ -58,10 +58,10 @@ src_libbitcoin_database_la_SOURCES = \ ${srcdir}/../../src/memory/mman.cpp \ ${srcdir}/../../src/memory/mstage.cpp \ ${srcdir}/../../src/memory/utilities.cpp \ - ${srcdir}/../../src/types/envelope.cpp \ + ${srcdir}/../../src/tables/envelope.cpp \ ${srcdir}/../../src/types/history.cpp \ ${srcdir}/../../src/types/multisig_view.cpp \ - ${srcdir}/../../src/types/unspent.cpp + ${srcdir}/../../src/types/unspent_output.cpp include_bitcoindir = \ ${includedir}/bitcoin @@ -85,6 +85,7 @@ include_bitcoin_database_filedir = \ ${includedir}/bitcoin/database/file include_bitcoin_database_file_HEADERS = \ + ${srcdir}/../../include/bitcoin/database/file/advice.hpp \ ${srcdir}/../../include/bitcoin/database/file/file.hpp \ ${srcdir}/../../include/bitcoin/database/file/rotator.hpp \ ${srcdir}/../../include/bitcoin/database/file/utilities.hpp @@ -210,11 +211,18 @@ include_bitcoin_database_impl_store_HEADERS = \ ${srcdir}/../../include/bitcoin/database/impl/store/store_tables.ipp \ ${srcdir}/../../include/bitcoin/database/impl/store/store_unload_close.ipp -include_bitcoin_database_impl_typesdir = \ - ${includedir}/bitcoin/database/impl/types +include_bitcoin_database_impl_unspentdir = \ + ${includedir}/bitcoin/database/impl/unspent -include_bitcoin_database_impl_types_HEADERS = \ - ${srcdir}/../../include/bitcoin/database/impl/types/difference_set.ipp +include_bitcoin_database_impl_unspent_HEADERS = \ + ${srcdir}/../../include/bitcoin/database/impl/unspent/unspent_counter.ipp \ + ${srcdir}/../../include/bitcoin/database/impl/unspent/unspent_matcher.ipp \ + ${srcdir}/../../include/bitcoin/database/impl/unspent/unspent_muhash.ipp \ + ${srcdir}/../../include/bitcoin/database/impl/unspent/unspent_reader.ipp \ + ${srcdir}/../../include/bitcoin/database/impl/unspent/unspent_scanner.ipp \ + ${srcdir}/../../include/bitcoin/database/impl/unspent/unspent_serial.ipp \ + ${srcdir}/../../include/bitcoin/database/impl/unspent/unspent_spans.ipp \ + ${srcdir}/../../include/bitcoin/database/impl/unspent/unspent_writer.ipp include_bitcoin_database_locksdir = \ ${includedir}/bitcoin/database/locks @@ -237,6 +245,7 @@ include_bitcoin_database_memory_HEADERS = \ ${srcdir}/../../include/bitcoin/database/memory/mmaps.hpp \ ${srcdir}/../../include/bitcoin/database/memory/mstage.hpp \ ${srcdir}/../../include/bitcoin/database/memory/reader.hpp \ + ${srcdir}/../../include/bitcoin/database/memory/release.hpp \ ${srcdir}/../../include/bitcoin/database/memory/settings.hpp \ ${srcdir}/../../include/bitcoin/database/memory/streamers.hpp \ ${srcdir}/../../include/bitcoin/database/memory/utilities.hpp @@ -272,6 +281,7 @@ include_bitcoin_database_tablesdir = \ include_bitcoin_database_tables_HEADERS = \ ${srcdir}/../../include/bitcoin/database/tables/context.hpp \ + ${srcdir}/../../include/bitcoin/database/tables/envelope.hpp \ ${srcdir}/../../include/bitcoin/database/tables/event.hpp \ ${srcdir}/../../include/bitcoin/database/tables/names.hpp \ ${srcdir}/../../include/bitcoin/database/tables/schema.hpp \ @@ -326,8 +336,8 @@ include_bitcoin_database_types_HEADERS = \ ${srcdir}/../../include/bitcoin/database/types/associations.hpp \ ${srcdir}/../../include/bitcoin/database/types/block_amounts.hpp \ ${srcdir}/../../include/bitcoin/database/types/block_state.hpp \ + ${srcdir}/../../include/bitcoin/database/types/constants.hpp \ ${srcdir}/../../include/bitcoin/database/types/difference_set.hpp \ - ${srcdir}/../../include/bitcoin/database/types/envelope.hpp \ ${srcdir}/../../include/bitcoin/database/types/fee_rate.hpp \ ${srcdir}/../../include/bitcoin/database/types/header_state.hpp \ ${srcdir}/../../include/bitcoin/database/types/history.hpp \ @@ -338,10 +348,24 @@ include_bitcoin_database_types_HEADERS = \ ${srcdir}/../../include/bitcoin/database/types/tx_state.hpp \ ${srcdir}/../../include/bitcoin/database/types/type.hpp \ ${srcdir}/../../include/bitcoin/database/types/types.hpp \ - ${srcdir}/../../include/bitcoin/database/types/unspent.hpp \ ${srcdir}/../../include/bitcoin/database/types/unspent_coin.hpp \ + ${srcdir}/../../include/bitcoin/database/types/unspent_output.hpp \ ${srcdir}/../../include/bitcoin/database/types/unspent_totals.hpp +include_bitcoin_database_unspentdir = \ + ${includedir}/bitcoin/database/unspent + +include_bitcoin_database_unspent_HEADERS = \ + ${srcdir}/../../include/bitcoin/database/unspent/unspent.hpp \ + ${srcdir}/../../include/bitcoin/database/unspent/unspent_counter.hpp \ + ${srcdir}/../../include/bitcoin/database/unspent/unspent_matcher.hpp \ + ${srcdir}/../../include/bitcoin/database/unspent/unspent_muhash.hpp \ + ${srcdir}/../../include/bitcoin/database/unspent/unspent_reader.hpp \ + ${srcdir}/../../include/bitcoin/database/unspent/unspent_scanner.hpp \ + ${srcdir}/../../include/bitcoin/database/unspent/unspent_serial.hpp \ + ${srcdir}/../../include/bitcoin/database/unspent/unspent_spans.hpp \ + ${srcdir}/../../include/bitcoin/database/unspent/unspent_writer.hpp + # Tests. #============================================================================== # Target test 'test/libbitcoin-database-test' @@ -378,6 +402,7 @@ test_libbitcoin_database_test_SOURCES = \ ${srcdir}/../../test/locks/interprocess_lock.cpp \ ${srcdir}/../../test/memory/accessor.cpp \ ${srcdir}/../../test/memory/mmap.cpp \ + ${srcdir}/../../test/memory/release.cpp \ ${srcdir}/../../test/memory/utilities.cpp \ ${srcdir}/../../test/mocks/blocks.cpp \ ${srcdir}/../../test/primitives/arrayhead.cpp \ @@ -468,9 +493,11 @@ test_libbitcoin_database_test_SOURCES = \ ${srcdir}/../../test/tables/optional/address.cpp \ ${srcdir}/../../test/tables/optional/filter_bk.cpp \ ${srcdir}/../../test/tables/optional/filter_tx.cpp \ + ${srcdir}/../../test/types/difference_set.cpp \ ${srcdir}/../../test/types/history.cpp \ ${srcdir}/../../test/types/span.cpp \ - ${srcdir}/../../test/types/unspent.cpp + ${srcdir}/../../test/types/unspent_output.cpp \ + ${srcdir}/../../test/unspent/unspent_commitment.cpp TESTS = test_runner.sh diff --git a/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj b/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj index b2f4bdab9..c58402f96 100644 --- a/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj +++ b/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj @@ -129,6 +129,7 @@ + $(IntDir)test_memory_utilities.obj @@ -195,9 +196,7 @@ - - $(IntDir)test_query_unspent.obj - + @@ -245,11 +244,11 @@ + - - $(IntDir)test_types_unspent.obj - + + diff --git a/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters b/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters index fdbd9b0d4..67f016b52 100644 --- a/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters +++ b/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters @@ -64,6 +64,9 @@ {5C4DA53A-8C06-4DA6-0000-000000000003} + + {5C4DA53A-8C06-4DA6-0000-000000000004} + @@ -93,6 +96,9 @@ src\memory + + src\memory + src\memory @@ -369,15 +375,21 @@ src + + src\types + src\types src\types - + src\types + + src\unspent + diff --git a/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj b/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj index 570e9b435..525c9929d 100644 --- a/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj +++ b/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj @@ -136,16 +136,17 @@ $(IntDir)src_memory_utilities.obj - + - + + @@ -162,6 +163,7 @@ + @@ -199,6 +201,7 @@ + @@ -213,8 +216,8 @@ + - @@ -225,9 +228,18 @@ - + + + + + + + + + + @@ -307,7 +319,14 @@ - + + + + + + + + diff --git a/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj.filters b/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj.filters index c8b684323..67a4cf5a4 100644 --- a/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj.filters +++ b/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj.filters @@ -49,7 +49,7 @@ {62D7FBEE-4D52-424A-0000-00000000000D} - + {62D7FBEE-4D52-424A-0000-00000000000E} @@ -82,21 +82,27 @@ {62D7FBEE-4D52-424A-0000-000000000009} - + {62D7FBEE-4D52-424A-0000-000000000010} - + {62D7FBEE-4D52-424A-0000-0000000000A1} - + {62D7FBEE-4D52-424A-0000-0000000000B1} - + {62D7FBEE-4D52-424A-0000-0000000000C1} - + {62D7FBEE-4D52-424A-0000-0000000000D1} + + {62D7FBEE-4D52-424A-0000-0000000000E1} + + + {62D7FBEE-4D52-424A-0000-0000000000F1} + @@ -132,8 +138,8 @@ src - - src\types + + src\tables src\types @@ -141,7 +147,7 @@ src\types - + src\types @@ -158,6 +164,9 @@ include\bitcoin\database + + include\bitcoin\database\file + include\bitcoin\database\file @@ -206,6 +215,9 @@ include\bitcoin\database\memory + + include\bitcoin\database\memory + include\bitcoin\database\memory @@ -317,6 +329,9 @@ include\bitcoin\database\tables + + include\bitcoin\database\tables + include\bitcoin\database\tables @@ -359,10 +374,10 @@ include\bitcoin\database\types - + include\bitcoin\database\types - + include\bitcoin\database\types @@ -395,15 +410,42 @@ include\bitcoin\database\types - + include\bitcoin\database\types - + include\bitcoin\database\types include\bitcoin\database\types + + include\bitcoin\database\unspent + + + include\bitcoin\database\unspent + + + include\bitcoin\database\unspent + + + include\bitcoin\database\unspent + + + include\bitcoin\database\unspent + + + include\bitcoin\database\unspent + + + include\bitcoin\database\unspent + + + include\bitcoin\database\unspent + + + include\bitcoin\database\unspent + include\bitcoin\database @@ -637,8 +679,29 @@ include\bitcoin\database\impl\store - - include\bitcoin\database\impl\types + + include\bitcoin\database\impl\unspent + + + include\bitcoin\database\impl\unspent + + + include\bitcoin\database\impl\unspent + + + include\bitcoin\database\impl\unspent + + + include\bitcoin\database\impl\unspent + + + include\bitcoin\database\impl\unspent + + + include\bitcoin\database\impl\unspent + + + include\bitcoin\database\impl\unspent diff --git a/builds/msvc/vs2026/libbitcoin-database-test/libbitcoin-database-test.vcxproj b/builds/msvc/vs2026/libbitcoin-database-test/libbitcoin-database-test.vcxproj index 2869b4756..46d57736e 100644 --- a/builds/msvc/vs2026/libbitcoin-database-test/libbitcoin-database-test.vcxproj +++ b/builds/msvc/vs2026/libbitcoin-database-test/libbitcoin-database-test.vcxproj @@ -129,6 +129,7 @@ + $(IntDir)test_memory_utilities.obj @@ -195,9 +196,7 @@ - - $(IntDir)test_query_unspent.obj - + @@ -245,11 +244,11 @@ + - - $(IntDir)test_types_unspent.obj - + + diff --git a/builds/msvc/vs2026/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters b/builds/msvc/vs2026/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters index fdbd9b0d4..67f016b52 100644 --- a/builds/msvc/vs2026/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters +++ b/builds/msvc/vs2026/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters @@ -64,6 +64,9 @@ {5C4DA53A-8C06-4DA6-0000-000000000003} + + {5C4DA53A-8C06-4DA6-0000-000000000004} + @@ -93,6 +96,9 @@ src\memory + + src\memory + src\memory @@ -369,15 +375,21 @@ src + + src\types + src\types src\types - + src\types + + src\unspent + diff --git a/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj b/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj index c2f545055..0c7659c7b 100644 --- a/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj +++ b/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj @@ -136,16 +136,17 @@ $(IntDir)src_memory_utilities.obj - + - + + @@ -162,6 +163,7 @@ + @@ -199,6 +201,7 @@ + @@ -213,8 +216,8 @@ + - @@ -225,9 +228,18 @@ - + + + + + + + + + + @@ -307,7 +319,14 @@ - + + + + + + + + diff --git a/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj.filters b/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj.filters index c8b684323..67a4cf5a4 100644 --- a/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj.filters +++ b/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj.filters @@ -49,7 +49,7 @@ {62D7FBEE-4D52-424A-0000-00000000000D} - + {62D7FBEE-4D52-424A-0000-00000000000E} @@ -82,21 +82,27 @@ {62D7FBEE-4D52-424A-0000-000000000009} - + {62D7FBEE-4D52-424A-0000-000000000010} - + {62D7FBEE-4D52-424A-0000-0000000000A1} - + {62D7FBEE-4D52-424A-0000-0000000000B1} - + {62D7FBEE-4D52-424A-0000-0000000000C1} - + {62D7FBEE-4D52-424A-0000-0000000000D1} + + {62D7FBEE-4D52-424A-0000-0000000000E1} + + + {62D7FBEE-4D52-424A-0000-0000000000F1} + @@ -132,8 +138,8 @@ src - - src\types + + src\tables src\types @@ -141,7 +147,7 @@ src\types - + src\types @@ -158,6 +164,9 @@ include\bitcoin\database + + include\bitcoin\database\file + include\bitcoin\database\file @@ -206,6 +215,9 @@ include\bitcoin\database\memory + + include\bitcoin\database\memory + include\bitcoin\database\memory @@ -317,6 +329,9 @@ include\bitcoin\database\tables + + include\bitcoin\database\tables + include\bitcoin\database\tables @@ -359,10 +374,10 @@ include\bitcoin\database\types - + include\bitcoin\database\types - + include\bitcoin\database\types @@ -395,15 +410,42 @@ include\bitcoin\database\types - + include\bitcoin\database\types - + include\bitcoin\database\types include\bitcoin\database\types + + include\bitcoin\database\unspent + + + include\bitcoin\database\unspent + + + include\bitcoin\database\unspent + + + include\bitcoin\database\unspent + + + include\bitcoin\database\unspent + + + include\bitcoin\database\unspent + + + include\bitcoin\database\unspent + + + include\bitcoin\database\unspent + + + include\bitcoin\database\unspent + include\bitcoin\database @@ -637,8 +679,29 @@ include\bitcoin\database\impl\store - - include\bitcoin\database\impl\types + + include\bitcoin\database\impl\unspent + + + include\bitcoin\database\impl\unspent + + + include\bitcoin\database\impl\unspent + + + include\bitcoin\database\impl\unspent + + + include\bitcoin\database\impl\unspent + + + include\bitcoin\database\impl\unspent + + + include\bitcoin\database\impl\unspent + + + include\bitcoin\database\impl\unspent diff --git a/include/bitcoin/database.hpp b/include/bitcoin/database.hpp index 94a9bfef0..4f9b1ed90 100644 --- a/include/bitcoin/database.hpp +++ b/include/bitcoin/database.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -37,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -57,6 +59,7 @@ #include #include #include +#include #include #include #include @@ -86,8 +89,8 @@ #include #include #include +#include #include -#include #include #include #include @@ -98,8 +101,17 @@ #include #include #include -#include #include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include #endif diff --git a/include/bitcoin/database/error.hpp b/include/bitcoin/database/error.hpp index e78813b4c..a8f677a2b 100644 --- a/include/bitcoin/database/error.hpp +++ b/include/bitcoin/database/error.hpp @@ -140,6 +140,7 @@ enum error_t : uint8_t depth_limited, invalid_cursor, query_canceled, + branch_inactive, invalid_argument, missing_prevouts, merkle_proof, diff --git a/include/bitcoin/database/file/advice.hpp b/include/bitcoin/database/file/advice.hpp new file mode 100644 index 000000000..d973fb6bf --- /dev/null +++ b/include/bitcoin/database/file/advice.hpp @@ -0,0 +1,51 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_FILE_ADVICE_HPP +#define LIBBITCOIN_DATABASE_FILE_ADVICE_HPP + +#include + +namespace libbitcoin { +namespace database { + +/// Expected read pattern of mapped storage, guiding kernel page advice. This +/// is independent of the write pattern (structural): bodies are appended +/// sequentially but read randomly by validation, so advising the kernel from +/// the write pattern invites eviction of the read set under memory pressure. +enum class advice : uint8_t +{ + /// Let the operating system decide (mixed or unpredictable access). + normal, + + /// Random access (suppresses read ahead), preloaded (small maps only). + random, + + /// One pass access (allows the kernel to free pages behind). + sequential, + + /// Random access (suppresses read ahead) without preload, for maps too + /// large to reside: scattered reads otherwise fault a full read-ahead + /// window each, and that manufactured cache displaces the head set. + scattered +}; + +} // namespace database +} // namespace libbitcoin + +#endif diff --git a/include/bitcoin/database/file/file.hpp b/include/bitcoin/database/file/file.hpp index c25ee580d..93ac5ec86 100644 --- a/include/bitcoin/database/file/file.hpp +++ b/include/bitcoin/database/file/file.hpp @@ -19,6 +19,7 @@ #ifndef LIBBITCOIN_DATABASE_FILE_FILE_HPP #define LIBBITCOIN_DATABASE_FILE_FILE_HPP +#include #include #include diff --git a/include/bitcoin/database/file/utilities.hpp b/include/bitcoin/database/file/utilities.hpp index c1b21151a..deb25c49c 100644 --- a/include/bitcoin/database/file/utilities.hpp +++ b/include/bitcoin/database/file/utilities.hpp @@ -21,7 +21,7 @@ #include #include -#include +#include namespace libbitcoin { namespace database { diff --git a/include/bitcoin/database/impl/memory/mmap_staging.ipp b/include/bitcoin/database/impl/memory/mmap_staging.ipp index a5602bd7c..b94b16525 100644 --- a/include/bitcoin/database/impl/memory/mmap_staging.ipp +++ b/include/bitcoin/database/impl/memory/mmap_staging.ipp @@ -28,6 +28,7 @@ #endif #include #include +#include #include namespace libbitcoin { diff --git a/include/bitcoin/database/impl/primitives/hashhead.ipp b/include/bitcoin/database/impl/primitives/hashhead.ipp index d88b280c6..c901ed1fe 100644 --- a/include/bitcoin/database/impl/primitives/hashhead.ipp +++ b/include/bitcoin/database/impl/primitives/hashhead.ipp @@ -109,6 +109,41 @@ bool CLASS::set_filter_k(size_t k) NOEXCEPT } } +TEMPLATE +uint32_t CLASS::derive_buckets(uint64_t expected, uint32_t low, + uint32_t high) NOEXCEPT +{ + using namespace system; + constexpr auto gigabyte = power2(30u); + constexpr auto megabyte = power2(20u); + constexpr auto uncontested = power2(35u); +#if defined(HAVE_APPLE) + constexpr auto contested = 10u * gigabyte; +#else + constexpr auto contested = 8u * gigabyte; +#endif + + if (is_zero(expected) || is_zero(low) || is_zero(high)) + return {}; + + const auto scaled = ceilinged_multiply(expected, 10); + const auto floored = scaled / low; + const auto ceiled = scaled / high; + const auto memory = system_memory(); + + if (memory <= contested) + return possible_narrow_cast(floored); + + if (memory >= uncontested) + return possible_narrow_cast(ceiled); + + // Megabytes, as the byte product overflows the word. + const auto over = (memory - contested) / megabyte; + const auto rise = floored_subtract(ceiled, floored); + constexpr auto span = (uncontested - contested) / megabyte; + return limit(floored + (ceilinged_multiply(rise, over) / span)); +} + TEMPLATE size_t CLASS::optimal_k(size_t count, size_t buckets) NOEXCEPT { diff --git a/include/bitcoin/database/impl/primitives/hashmap.ipp b/include/bitcoin/database/impl/primitives/hashmap.ipp index 523abe14a..6998e66f0 100644 --- a/include/bitcoin/database/impl/primitives/hashmap.ipp +++ b/include/bitcoin/database/impl/primitives/hashmap.ipp @@ -87,6 +87,13 @@ size_t CLASS::buckets() const NOEXCEPT return head_.buckets(); } +TEMPLATE +uint32_t CLASS::derive_buckets(uint64_t expected, uint32_t low, + uint32_t high) NOEXCEPT +{ + return head::derive_buckets(expected, low, high); +} + TEMPLATE size_t CLASS::filter_k() const NOEXCEPT { diff --git a/include/bitcoin/database/impl/primitives/hashmaps.ipp b/include/bitcoin/database/impl/primitives/hashmaps.ipp index 75d33b65b..ad845e0c6 100644 --- a/include/bitcoin/database/impl/primitives/hashmaps.ipp +++ b/include/bitcoin/database/impl/primitives/hashmaps.ipp @@ -85,6 +85,13 @@ size_t CLASS::buckets() const NOEXCEPT return head_.buckets(); } +TEMPLATE +uint32_t CLASS::derive_buckets(uint64_t expected, uint32_t low, + uint32_t high) NOEXCEPT +{ + return head::derive_buckets(expected, low, high); +} + TEMPLATE size_t CLASS::filter_k() const NOEXCEPT { diff --git a/include/bitcoin/database/impl/primitives/headmap.ipp b/include/bitcoin/database/impl/primitives/headmap.ipp index fc2393cea..8d56949c9 100644 --- a/include/bitcoin/database/impl/primitives/headmap.ipp +++ b/include/bitcoin/database/impl/primitives/headmap.ipp @@ -164,6 +164,44 @@ Link CLASS::at(size_t index) const NOEXCEPT } } +TEMPLATE +memory CLASS::get_memory() const NOEXCEPT +{ + return file_.get(); +} + +TEMPLATE +Link CLASS::at(const memory& ptr, size_t index) const NOEXCEPT +{ + using namespace system; + + // Buckets at or above the logical size are unallocated (count publishes). + if (index >= count()) + return {}; + + const auto raw = ptr.offset(link_to_position(index)); + if (is_null(raw)) + return {}; + + if constexpr (aligned) + { + // Reads full padded word (masked by to_link). + const auto& head = *pointer_cast>(raw); + return to_link(head.load(std::memory_order_relaxed)); + } + else + { + const auto& head = cell_array(raw); + cell value{}; + + mutex_.lock_shared(); + cell_array(value) = head; + mutex_.unlock_shared(); + + return to_link(value); + } +} + // NOT WRITER-WRITER THREAD SAFE (the logical top is read-write). TEMPLATE bool CLASS::push(const Link& link) NOEXCEPT diff --git a/include/bitcoin/database/impl/query/address/address_unspent.ipp b/include/bitcoin/database/impl/query/address/address_unspent.ipp index a291610e5..50a2c2e23 100644 --- a/include/bitcoin/database/impl/query/address/address_unspent.ipp +++ b/include/bitcoin/database/impl/query/address/address_unspent.ipp @@ -35,7 +35,7 @@ namespace database { // ununsed // Only unconfirmed outputs that are not spent (by an unconfirmed tx). TEMPLATE -code CLASS::get_unconfirmed_unspent(const stopper& cancel, unspents& out, +code CLASS::get_unconfirmed_unspent(const stopper& cancel, unspent_outputs& out, const hash_digest& key, bool turbo) const NOEXCEPT { output_links outs{}; @@ -51,7 +51,7 @@ code CLASS::get_unconfirmed_unspent(const stopper& cancel, unspents& out, // this is only unconfirmeds, but checking both via is_spent() is // much faster than calling is_unconfirmed_spent(). if (cancel || fail || is_spent(link)) - return unspent{}; + return unspent_output{}; const auto out = get_tx_unconfirmed_unspent(link); if (out.fault()) fail = true; @@ -62,7 +62,7 @@ code CLASS::get_unconfirmed_unspent(const stopper& cancel, unspents& out, // ununsed // Only confirmed outputs that are not spent by a confirmed tx. TEMPLATE -code CLASS::get_confirmed_unspent(const stopper& cancel, unspents& out, +code CLASS::get_confirmed_unspent(const stopper& cancel, unspent_outputs& out, const hash_digest& key, bool turbo) const NOEXCEPT { output_links outs{}; @@ -76,7 +76,7 @@ code CLASS::get_confirmed_unspent(const stopper& cancel, unspents& out, { // Exclude if spent by confirmed tx, ignore unconfirmed spenders. if (cancel || fail || is_confirmed_spent(link)) - return unspent{}; + return unspent_output{}; const auto out = get_tx_confirmed_unspent(link); if (out.fault()) fail = true; @@ -87,7 +87,7 @@ code CLASS::get_confirmed_unspent(const stopper& cancel, unspents& out, // server/electrum // All outputs that are not spent by any tx (confirmed or unconfirmed). TEMPLATE -code CLASS::get_unspent(const stopper& cancel, unspents& out, +code CLASS::get_unspent(const stopper& cancel, unspent_outputs& out, const hash_digest& key, bool turbo) const NOEXCEPT { output_links outs{}; @@ -101,7 +101,7 @@ code CLASS::get_unspent(const stopper& cancel, unspents& out, { // Exclude if spent by any tx, confirmed or unconfirmed. if (cancel || fail || is_spent(link)) - return unspent{}; + return unspent_output{}; auto out = get_tx_unspent(link); if (out.fault()) fail = true; @@ -113,7 +113,7 @@ code CLASS::get_unspent(const stopper& cancel, unspents& out, // ---------------------------------------------------------------------------- TEMPLATE -unspent CLASS::get_tx_unspent(const output_link& link) const NOEXCEPT +unspent_output CLASS::get_tx_unspent(const output_link& link) const NOEXCEPT { table::output::get_parent_value out{}; if (!store_.output.get(link, out)) @@ -125,8 +125,8 @@ unspent CLASS::get_tx_unspent(const output_link& link) const NOEXCEPT if ((index == point::null_index) || (hash == system::null_hash)) return {}; - auto height = unspent::unused_height; - auto position = unspent::unconfirmed_position; + auto height = unspent_output::unused_height; + auto position = unspent_output::unconfirmed_position; const auto block = find_strong(tx); const auto at = get_confirmed_height(block); @@ -141,9 +141,9 @@ unspent CLASS::get_tx_unspent(const output_link& link) const NOEXCEPT } TEMPLATE -unspent CLASS::get_tx_confirmed_unspent(const output_link& link) const NOEXCEPT +unspent_output CLASS::get_tx_confirmed_unspent(const output_link& link) const NOEXCEPT { - // unspent is invalid in default construction with position. + // unspent_output is invalid in default construction with position. table::output::get_parent_value out{}; if (!store_.output.get(link, out)) return {}; @@ -154,7 +154,7 @@ unspent CLASS::get_tx_confirmed_unspent(const output_link& link) const NOEXCEPT // Invalid with unconfirmed_position signals no fault. if (at.is_terminal()) - return { .position = unspent::unconfirmed_position }; + return { .position = unspent_output::unconfirmed_position }; size_t position{}; auto hash = get_tx_key(tx); @@ -167,9 +167,9 @@ unspent CLASS::get_tx_confirmed_unspent(const output_link& link) const NOEXCEPT } TEMPLATE -unspent CLASS::get_tx_unconfirmed_unspent(const output_link& link) const NOEXCEPT +unspent_output CLASS::get_tx_unconfirmed_unspent(const output_link& link) const NOEXCEPT { - // unspent is invalid in default construction with position. + // unspent_output is invalid in default construction with position. table::output::get_parent_value out{}; if (!store_.output.get(link, out)) return {}; @@ -178,15 +178,15 @@ unspent CLASS::get_tx_unconfirmed_unspent(const output_link& link) const NOEXCEP // Invalid with unconfirmed_position signals no fault. if (!get_confirmed_height(find_strong(tx)).is_terminal()) - return { .position = unspent::unconfirmed_position }; + return { .position = unspent_output::unconfirmed_position }; auto hash = get_tx_key(tx); const auto index = to_output_index(tx, link); if ((index == point::null_index) || (hash == system::null_hash)) return {}; - return { { { std::move(hash), index }, out.value }, unspent::unused_height, - unspent::unconfirmed_position }; + return { { { std::move(hash), index }, out.value }, unspent_output::unused_height, + unspent_output::unconfirmed_position }; } // utilities @@ -196,7 +196,7 @@ unspent CLASS::get_tx_unconfirmed_unspent(const output_link& link) const NOEXCEP TEMPLATE template code CLASS::parallel_unspent_transform(const stopper& cancel, bool turbo, - unspents& out, const output_links& outs, Functor&& functor) NOEXCEPT + unspent_outputs& out, const output_links& outs, Functor&& functor) NOEXCEPT { const auto policy = poolstl::execution::par_if(turbo); stopper fail{}; @@ -215,7 +215,7 @@ code CLASS::parallel_unspent_transform(const stopper& cancel, bool turbo, if (cancel) return error::query_canceled; - unspent::filter_sort_and_dedup(out); + unspent_output::filter_sort_and_dedup(out); return error::success; } diff --git a/include/bitcoin/database/impl/query/unspent.ipp b/include/bitcoin/database/impl/query/unspent.ipp index d482ba0de..427429b7e 100644 --- a/include/bitcoin/database/impl/query/unspent.ipp +++ b/include/bitcoin/database/impl/query/unspent.ipp @@ -19,53 +19,30 @@ #ifndef LIBBITCOIN_DATABASE_QUERY_UNSPENT_IPP #define LIBBITCOIN_DATABASE_QUERY_UNSPENT_IPP -#include -#include -#include #include namespace libbitcoin { namespace database { -// Unspent. -// ---------------------------------------------------------------------------- - -// The unspent set is the symmetric difference of creates and spends over -// the branch: paired events cancel and the residue is the utxo set. -static_assert(tx_link::bits <= bits - difference_set<>::window_bits); - TEMPLATE bool CLASS::is_bip30_exception(bool& out, const header_link& link) const NOEXCEPT { - using namespace system; - - static const chain::checkpoint first - { - "00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec", - 91842 - }; - - static const chain::checkpoint second - { - "00000000000743f190a18c5577a3c2d2a1f610ae9601ac046a38084ccb7cd721", - 91880 - }; - out = false; size_t height{}; if (!get_height(height, link)) return false; - if (height != first.height() && height != second.height()) + const auto first = (height == bip30::first_exception); + const auto second = (height == bip30::second_exception); + if (!first && !second) return true; - // With the rule unconfigured duplicates are unconstrained (no exception). if (!store_.envelope().forks.bip30) return true; - const auto key = get_header_key(link); - out = key == (height == first.height() ? first.hash() : second.hash()); + const auto& hash = first ? bip30::first.hash() : bip30::second.hash(); + out = (get_header_key(link) == hash); return true; } @@ -115,290 +92,60 @@ code CLASS::get_block_amounts(block_amounts& out, } TEMPLATE -code CLASS::scan_unspent(difference_set<>& set, unspent_totals& out, - const stopper& cancel, const header_links& branch, bool turbo) const NOEXCEPT +code CLASS::get_unspent_totals(const stopper& cancel, unspent_totals& out, + const header_links& branch, bool turbo) const NOEXCEPT { - using namespace system; - std::atomic_bool fail{}; - std::atomic unspendable{}; - std::atomic overwritten{}; - const auto parallel = poolstl::execution::par_if(turbo); - - // Reverse order in best effort attempt to minimize set size. - std::for_each(parallel, branch.crbegin(), branch.crend(), - [&](const auto& element) NOEXCEPT - { - if (cancel || fail) - return; - - auto bip30_exception = false; - const header_link link{ element }; - if (!is_bip30_exception(bip30_exception, link)) - { - fail = true; - return; - } - - auto coinbase = true; - table::output::get_spendable value{}; - - // The bip30 exception blocks are coinbase only, their duplicated - // outputs overwrite (destroy) those of the original blocks. - if (bip30_exception) - { - for (const auto& tx: to_transactions(link)) - for (const auto& out_: to_outputs(tx)) - { - if (!store_.output.get(out_, value)) - { - fail = true; - return; - } - - overwritten += value.value; - } - - return; - } - for (const auto& tx: to_transactions(link)) - { - if (cancel || fail) - return; - - uint32_t index{}; - for (const auto& out: to_outputs(tx)) - { - if (!store_.output.get(out, value)) - { - fail = true; - return; - } - - if (value.unspendable) - unspendable += value.value; - else - set.toggle(tx, index); - - ++index; - } - - if (coinbase) - { - coinbase = false; - continue; - } - - for (const auto& in: to_points(tx)) - { - const auto point = get_point_key(in); - auto it = store_.tx.it(point.hash()); - if (!it) - { - fail = true; - return; - } - - // Duplicate txs require the confirmed instance. - auto spent = *it; - if (++it && !is_confirmed_tx(spent)) - { - for (spent = tx_link::terminal; it; ++it) - { - if (is_confirmed_tx(*it)) - { - spent = *it; - break; - } - } - - if (spent.is_terminal()) - { - fail = true; - return; - } - } - - set.toggle(spent, point.index()); - } - } - }); - - if (fail) - return error::integrity; - - if (cancel) - return error::query_canceled; + out = {}; + difference_set set{ outs_records() }; + const unspent_scanner scanner{ *this, store_, cancel, turbo }; + if (const auto ec = scanner.scan(set, out, branch)) + return ec; - out.unspendable = unspendable; - out.bip30 = overwritten; - return error::success; + const unspent_counter counter{ store_, cancel, turbo }; + return counter.count(out, set); } TEMPLATE -code CLASS::count_unspent(unspent_totals& out, const stopper& cancel, - const difference_set<>::entries& survivors) const NOEXCEPT +code CLASS::get_unspent_muhash(const stopper& cancel, unspent_totals& out, + hash_digest& digest, const header_links& branch, bool turbo) const NOEXCEPT { - using namespace system; - - output_links puts{}; - auto previous = max_uint64; - table::output::get_spendable value{}; - for (const auto& [key, mask]: survivors) - { - if (cancel) - return error::query_canceled; - - const auto tx = difference_set<>::to_id(key); - if (tx != previous) - { - previous = tx; - ++out.transactions; - puts = to_outputs(possible_narrow_cast(tx)); - } - - const auto base = difference_set<>::to_index(key); - for (auto bits = mask; !is_zero(bits); - bits = bit_and(bits, sub1(bits))) - { - // A surviving spend with no create implies store inconsistency. - const auto index = base + right_zeros(bits); - if (index >= puts.size() || - !store_.output.get(puts[index], value)) - return error::integrity; - - // Coin serialization is outpoint, height code, value, script. - constexpr auto coin_overhead = hash_size + sizeof(uint32_t) + - sizeof(uint32_t) + sizeof(uint64_t); - - ++out.outputs; - out.value += value.value; - out.script_bytes += value.script_size; - out.coin_bytes += coin_overhead + - variable_size(value.script_size) + value.script_size; - } - } + out = {}; + difference_set set{ outs_records() }; + const unspent_scanner scanner{ *this, store_, cancel, turbo }; + if (const auto ec = scanner.scan(set, out, branch)) + return ec; - return error::success; + const unspent_muhash muhash{ store_, cancel, turbo }; + return muhash.hash(out, digest, set); } TEMPLATE -template -code CLASS::visit_unspent(const Visitor& visit, const stopper& cancel, - const difference_set<>::entries& survivors) const NOEXCEPT +code CLASS::get_unspent_serialized(const stopper& cancel, unspent_totals& out, + hash_digest& digest, const header_links& branch, bool turbo) const NOEXCEPT { - using namespace system; - - unspent_coin coin{}; - output_links puts{}; - auto previous = max_uint64; - const auto bip30 = store_.envelope().forks.bip30; - table::output::get_coin out{}; - for (const auto& [key, mask]: survivors) - { - if (cancel) - return error::query_canceled; - - const auto tx = difference_set<>::to_id(key); - coin.first = (tx != previous); - if (coin.first) - { - previous = tx; - const auto link = possible_narrow_cast(tx); - const auto at = get_confirmed_height(find_strong(link)); - coin.txid = get_tx_key(link); - if (at.is_terminal() || coin.txid == null_hash) - return error::integrity; - - coin.height = at.value; - coin.coinbase = is_coinbase(link); - - // bitcoind retains duplicated coinbases at the overwriting - // heights (bip30 exceptions), the store at the originals. - if (bip30 && coin.coinbase) - coin.height = (coin.height == 91812) ? 91842 : - (coin.height == 91722) ? 91880 : coin.height; - - puts = to_outputs(link); - } - - const auto base = difference_set<>::to_index(key); - for (auto bits = mask; !is_zero(bits); - bits = bit_and(bits, sub1(bits))) - { - // A surviving spend with no create implies store inconsistency. - const auto index = base + right_zeros(bits); - if (index >= puts.size() || !store_.output.get(puts[index], out)) - return error::integrity; - - coin.index = possible_narrow_cast(index); - coin.value = out.value; - std::swap(coin.script, out.script); - visit(coin); - coin.first = false; - } - } + out = {}; + difference_set set{ outs_records() }; + const unspent_scanner scanner{ *this, store_, cancel, turbo }; + if (const auto ec = scanner.scan(set, out, branch)) + return ec; - return error::success; + const unspent_serial serial{ store_, cancel, turbo }; + return serial.hash(out, digest, set); } TEMPLATE -code CLASS::get_unspent_totals(const stopper& cancel, unspent_totals& out, +code CLASS::get_unspent_matches(const stopper& cancel, unspent_coins& out, + size_t& txouts, const std::unordered_set& keys, const header_links& branch, bool turbo) const NOEXCEPT { - out = {}; - difference_set<> set{}; - const auto ec = scan_unspent(set, out, cancel, branch, turbo); - return ec ? ec : count_unspent(out, cancel, set.drain()); -} - -TEMPLATE -template -code CLASS::get_unspent_coins(const stopper& cancel, unspent_totals& out, - const Visitor& visit, const header_links& branch, bool ordered, - bool turbo) const NOEXCEPT -{ - using namespace system; - - out = {}; - difference_set<> set{}; - if (const auto ec = scan_unspent(set, out, cancel, branch, turbo)) + unspent_totals unused{}; + difference_set set{ outs_records() }; + const unspent_scanner scanner{ *this, store_, cancel, turbo }; + if (const auto ec = scanner.scan(set, unused, branch)) return ec; - auto survivors = set.drain(); - if (ordered) - { - // Canonical order is (txid, index), as bitcoind's chainstate cursor. - // The txid sort requires full materialization (memory expensive). - using keyed_entry = std::pair::entry>; - std_vector keyed{}; - keyed.reserve(survivors.size()); - for (const auto& item: survivors) - { - if (cancel) - return error::query_canceled; - - const auto tx = difference_set<>::to_id(item.first); - auto txid = get_tx_key(possible_narrow_cast(tx)); - if (txid == null_hash) - return error::integrity; - - keyed.emplace_back(std::move(txid), item); - } - - // Key order preserves window order within a transaction. - std::sort(poolstl::execution::par_if(turbo), keyed.begin(), - keyed.end(), [](const auto& left, const auto& right) NOEXCEPT - { - return (left.first == right.first) ? - (left.second.first < right.second.first) : - (left.first < right.first); - }); - - std::transform(keyed.begin(), keyed.end(), survivors.begin(), - [](const auto& item) NOEXCEPT { return item.second; }); - } - - return visit_unspent(visit, cancel, survivors); + const unspent_matcher matcher{ store_, cancel, turbo }; + return matcher.match(out, txouts, keys, set); } } // namespace database diff --git a/include/bitcoin/database/impl/types/difference_set.ipp b/include/bitcoin/database/impl/types/difference_set.ipp deleted file mode 100644 index b80bdd7c4..000000000 --- a/include/bitcoin/database/impl/types/difference_set.ipp +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Copyright (c) 2011-2026 libbitcoin developers - * - * This file is part of libbitcoin. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef LIBBITCOIN_DATABASE_TYPES_DIFFERENCE_SET_IPP -#define LIBBITCOIN_DATABASE_TYPES_DIFFERENCE_SET_IPP - -#include -#include -#include - -namespace libbitcoin { -namespace database { - -BC_PUSH_WARNING(NO_THROW_IN_NOEXCEPT) - -TEMPLATE -void CLASS::toggle(uint64_t id, uint32_t index) NOEXCEPT -{ - using namespace system; - constexpr auto golden = 0x9e3779b97f4a7c15_u64; - const auto key = to_key(id, index); - auto& shard = shards_[shift_right(key * golden, key_bits - shard_bits)]; - - const std::lock_guard lock{ shard.mutex }; - auto& mask = shard.map[key]; - mask ^= bit_right(bit_and(index, index_mask)); - if (is_zero(mask)) - shard.map.erase(key); -} - -TEMPLATE -typename CLASS::entries CLASS::drain() NOEXCEPT -{ - size_t count{}; - for (const auto& shard: shards_) - count += shard.map.size(); - - entries out{}; - out.reserve(count); - for (auto& shard: shards_) - { - for (const auto& entry: shard.map) - out.emplace_back(entry.first, entry.second); - - shard.map.clear(); - } - - std::sort(out.begin(), out.end()); - return out; -} - -BC_POP_WARNING() - -} // namespace database -} // namespace libbitcoin - -#endif diff --git a/include/bitcoin/database/impl/unspent/unspent_counter.ipp b/include/bitcoin/database/impl/unspent/unspent_counter.ipp new file mode 100644 index 000000000..07f5b20cd --- /dev/null +++ b/include/bitcoin/database/impl/unspent/unspent_counter.ipp @@ -0,0 +1,108 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_COUNTER_IPP +#define LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_COUNTER_IPP + +#include + +namespace libbitcoin { +namespace database { + +TEMPLATE +CLASS::unspent_counter(const Store& store, const stopper& cancel, + bool turbo) NOEXCEPT + : store_(store), + cancel_(cancel), + reader_(store, cancel), + spans_(store, cancel, turbo) +{ +} + +TEMPLATE +code CLASS::count(unspent_totals& out, + const difference_set& set) const NOEXCEPT +{ + std_vector totals(spans_.count()); + const auto ec = spans_.for_each(set, + [&](size_t index, size_t begin, size_t end) NOEXCEPT + { + return span(totals.at(index), set, begin, end); + }); + + if (ec) + return ec; + + for (const auto& total: totals) + unspent_writer::add(out, total); + + return error::success; +} + +TEMPLATE +code CLASS::span(unspent_totals& out, const difference_set& set, + size_t begin, size_t end) const NOEXCEPT +{ + output_links puts{}; + auto previous = tx_link::terminal; + return reader_.elements(set, begin, end, + [&](const unspent_elements& elements) NOEXCEPT + { + puts.resize(elements.size()); + if (const auto ec = reader_.read_puts(puts, zero, elements, zero, + elements.size())) + return ec; + + return read_totals(out, previous, puts); + }); +} + +TEMPLATE +code CLASS::read_totals(unspent_totals& out, tx_link::integer& previous, + const output_links& puts) const NOEXCEPT +{ + constexpr auto fixed = unspent_writer::fixed_size; + + const auto ptr = store_.output.get_memory(); + for (const auto& put: puts) + { + table::output::get_parent_coin output{}; + if (!store_.output.get(ptr, put, output)) + return error::integrity; + + if (output.parent_fk != previous) + { + previous = output.parent_fk; + ++out.transactions; + } + + const auto variable = variable_size(output.script_size); + + ++out.outputs; + out.value += output.value; + out.script_bytes += output.script_size; + out.coin_bytes += fixed + variable + output.script_size; + } + + return error::success; +} + +} // namespace database +} // namespace libbitcoin + +#endif diff --git a/include/bitcoin/database/impl/unspent/unspent_matcher.ipp b/include/bitcoin/database/impl/unspent/unspent_matcher.ipp new file mode 100644 index 000000000..d020fc4cc --- /dev/null +++ b/include/bitcoin/database/impl/unspent/unspent_matcher.ipp @@ -0,0 +1,86 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_MATCHER_IPP +#define LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_MATCHER_IPP + +#include +#include +#include + +namespace libbitcoin { +namespace database { + +TEMPLATE +CLASS::unspent_matcher(const Store& store, const stopper& cancel, + bool turbo) NOEXCEPT + : reader_(store, cancel), + spans_(store, cancel, turbo) +{ +} + +TEMPLATE +code CLASS::match(unspent_coins& out, size_t& txouts, const keys& keys, + const difference_set& set) const NOEXCEPT +{ + BC_ASSERT(out.empty()); + + txouts = zero; + std_vector totals(spans_.count()); + std_vector matches(spans_.count()); + const auto ec = spans_.for_each(set, + [&](size_t index, size_t begin, size_t end) NOEXCEPT + { + return span(totals.at(index), matches.at(index), keys, set, begin, + end); + }); + + if (ec) + return ec; + + for (auto index = zero; index < matches.size(); ++index) + { + txouts += totals.at(index).outputs; + auto& found = matches.at(index); + out.insert(out.end(), + std::make_move_iterator(found.begin()), + std::make_move_iterator(found.end())); + } + + return error::success; +} + +TEMPLATE +code CLASS::span(unspent_totals& out, unspent_coins& matches, + const keys& keys, const difference_set& set, size_t begin, + size_t end) const NOEXCEPT +{ + using namespace system; + return reader_.batch(set, begin, end, + [&](const unspent_coin& coin) NOEXCEPT + { + unspent_writer::add(out, coin); + if (keys.contains(sha256_hash(coin.script))) + matches.push_back(coin); + }); +} + +} // namespace database +} // namespace libbitcoin + +#endif diff --git a/include/bitcoin/database/impl/unspent/unspent_muhash.ipp b/include/bitcoin/database/impl/unspent/unspent_muhash.ipp new file mode 100644 index 000000000..1d3090a70 --- /dev/null +++ b/include/bitcoin/database/impl/unspent/unspent_muhash.ipp @@ -0,0 +1,113 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_MUHASH_IPP +#define LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_MUHASH_IPP + +#include +#include + +namespace libbitcoin { +namespace database { + +TEMPLATE +CLASS::unspent_muhash(const Store& store, const stopper& cancel, + bool turbo) NOEXCEPT + : reader_(store, cancel), spans_(store, cancel, turbo), store_(store) +{ +} + +TEMPLATE +code CLASS::hash(unspent_totals& out, hash_digest& digest, + const difference_set& set) const NOEXCEPT +{ + using namespace system; + std_vector totals(spans_.count()); + std_vector partials(spans_.count()); + const auto ec = spans_.for_each(set, + [&](size_t index, size_t begin, size_t end) NOEXCEPT + { + return span(totals.at(index), partials.at(index), set, begin, end); + }); + + if (ec) + return ec; + + muhash3072 muhash{}; + for (size_t index{}; index < partials.size(); ++index) + { + unspent_writer::add(out, totals.at(index)); + muhash *= partials.at(index); + } + + digest = muhash.flush(); + return error::success; +} + +TEMPLATE +code CLASS::span(unspent_totals& out, system::muhash3072& partial, + const difference_set& set, size_t begin, size_t end) const NOEXCEPT +{ + output_links puts{}; + unspent_coins coins{}; + auto previous = tx_link::terminal; + + return reader_.elements(set, begin, end, + [&](const unspent_elements& at) NOEXCEPT + { + puts.resize(at.size()); + coins.resize(at.size()); + auto ec = reader_.fill(coins, puts, zero, previous, at, zero, + at.size()); + + if (!ec) + ec = insert(out, partial, coins, puts); + + return ec; + }); +} + +TEMPLATE +code CLASS::insert(unspent_totals& out, system::muhash3072& partial, + const unspent_coins& coins, const output_links& puts) const NOEXCEPT +{ + using namespace system; + const auto ptr = store_.output.get_memory(); + for (size_t at{}; at < coins.size(); ++at) + { + hash_digest digest{}; + stream::out::fast stream{ digest }; + hash::sha256::fast sink{ stream }; + const auto& coin = coins.at(at); + unspent_writer::write(sink, coin); + table::output::write_script tail{ {}, sink }; + if (!store_.output.raw(ptr, puts.at(at), tail)) + return error::integrity; + + sink.flush(); + partial.insert_hash(digest); + unspent_writer::add(out, coin.first, tail.value, tail.length); + } + + return error::success; +} + +} // namespace database +} // namespace libbitcoin + +#endif diff --git a/include/bitcoin/database/impl/unspent/unspent_reader.ipp b/include/bitcoin/database/impl/unspent/unspent_reader.ipp new file mode 100644 index 000000000..a38412f6e --- /dev/null +++ b/include/bitcoin/database/impl/unspent/unspent_reader.ipp @@ -0,0 +1,329 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_READER_IPP +#define LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_READER_IPP + +#include +#include + +namespace libbitcoin { +namespace database { + +TEMPLATE +CLASS::unspent_reader(const Store& store, + const stopper& cancel) NOEXCEPT + : store_(store), cancel_(cancel) +{ +} + +TEMPLATE +template +code CLASS::elements(const difference_set& set, size_t begin, size_t end, + const Flush& flush) const NOEXCEPT +{ + constexpr auto width = difference_set::word_bits; + unspent_elements elements{}; + + using namespace system; + using word = difference_set::word; + const auto last = ceilinged_divide(end, width); + for (auto index = floored_divide(begin, width); index < last; ++index) + { + if (cancel_) + return error::query_canceled; + + const auto base = index * width; + auto bits = set.at(index); + if (base < begin) + bits = bit_and(bits, mask_right(begin - base)); + + if (base + width > end) + bits = bit_and(bits, unmask_right(end - base)); + + for (; !is_zero(bits); bits = bit_and(bits, sub1(bits))) + { + const auto link = base + right_zeros(bits); + elements.push_back(possible_narrow_cast(link)); + } + + if (elements.size() >= batch_size) + { + if (const auto ec = flush(elements)) + return ec; + + elements.clear(); + } + } + + return flush(elements); +} + +TEMPLATE +template +code CLASS::batch(const difference_set& set, size_t begin, size_t end, + const Handler& handle) const NOEXCEPT +{ + output_links puts{}; + unspent_coins coins{}; + auto previous = tx_link::terminal; + + return elements(set, begin, end, [&](const unspent_elements& at) NOEXCEPT + { + puts.resize(at.size()); + coins.resize(at.size()); + auto ec = fill(coins, puts, zero, previous, at, zero, at.size()); + if (!ec) + ec = read_scripts(coins, zero, puts, at.size()); + + if (!ec) + for (const auto& coin: coins) + handle(coin); + + return ec; + }); +} + +TEMPLATE +code CLASS::fill(unspent_coins& out, output_links& puts, size_t offset, + tx_link::integer& previous, const unspent_elements& elements, + size_t begin, size_t end) const NOEXCEPT +{ + if (const auto ec = read_puts(puts, offset, elements, begin, end)) + return ec; + + tx_links parents{}; + if (const auto ec = read_parents(parents, puts, offset, end - begin)) + return ec; + + if (const auto ec = read_transactions(out, offset, previous, parents, + elements, begin)) + return ec; + + header_links blocks{}; + if (const auto ec = read_blocks(blocks, parents)) + return ec; + + std_vector heights{}; + if (const auto ec = read_heights(heights, blocks)) + return ec; + + return confirm(out, offset, blocks, heights); +} + +TEMPLATE +code CLASS::read_scripts(unspent_coins& out, size_t offset, + const output_links& puts, size_t count) const NOEXCEPT +{ + const auto ptr = store_.output.get_memory(); + for (size_t at{}; at < count; ++at) + { + table::output::get_coin output{}; + if (!store_.output.get(ptr, puts.at(offset + at), output)) + return error::integrity; + + auto& coin = out.at(offset + at); + coin.out = { coin.out.point(), output.value }; + std::swap(coin.script, output.script); + } + + return error::success; +} + +TEMPLATE +code CLASS::read_puts(output_links& out, size_t offset, + const unspent_elements& elements, size_t begin, size_t end) const NOEXCEPT +{ + const auto count = end - begin; + + const auto ptr = store_.outs.puts.get_memory(); + for (size_t at{}; at < count; ++at) + { + table::outs::get_output output{}; + if (!store_.outs.puts.get(ptr, elements.at(begin + at), output)) + return error::integrity; + + out.at(offset + at) = output.out_fk; + } + + return error::success; +} + +TEMPLATE +code CLASS::read_parents(tx_links& out, const output_links& puts, + size_t offset, size_t count) const NOEXCEPT +{ + out.resize(count); + + const auto ptr = store_.output.get_memory(); + for (size_t at{}; at < count; ++at) + { + table::output::get_parent output{}; + if (!store_.output.get(ptr, puts.at(offset + at), output)) + return error::integrity; + + out.at(at) = output.parent_fk; + } + + return error::success; +} + +TEMPLATE +code CLASS::read_hashes(system::hashes& out, + const tx_links& parents) const NOEXCEPT +{ + out.resize(parents.size()); + + const auto ptr = store_.tx.get_memory(); + for (size_t at{}; at < parents.size(); ++at) + { + table::transaction::get_hash tx{}; + if (!store_.tx.get(ptr, parents.at(at), tx)) + return error::integrity; + + out.at(at) = tx.hash; + } + + return error::success; +} + +TEMPLATE +code CLASS::read_transactions(unspent_coins& out, size_t offset, + tx_link::integer& previous, const tx_links& parents, + const unspent_elements& elements, size_t begin) const NOEXCEPT +{ + auto parent = tx_link::terminal; + table::transaction::get_puts tx{}; + table::transaction::get_hash hash{}; + + const auto ptr = store_.tx.get_memory(); + for (size_t at{}; at < parents.size(); ++at) + { + if (parents.at(at) != parent) + { + parent = parents.at(at); + if (!store_.tx.get(ptr, parent, tx) || + !store_.tx.get(ptr, parent, hash)) + return error::integrity; + } + + auto& coin = out.at(offset + at); + const auto index = elements.at(begin + at) - tx.outs_fk; + coin.first = (parent != previous); + coin.coinbase = tx.coinbase; + coin.out = { { hash.hash, index }, coin.out.value() }; + previous = parent; + } + + return error::success; +} + +// The branch is inactive if a tx of it is no longer strong (reorganized). +TEMPLATE +code CLASS::read_blocks(header_links& out, + const tx_links& parents) const NOEXCEPT +{ + tx_link parent{}; + auto block = header_link::terminal; + out.resize(parents.size()); + + const auto ptr = store_.strong_tx.get_memory(); + for (size_t at{}; at < parents.size(); ++at) + { + if (parents.at(at) != parent) + { + parent = parents.at(at); + const auto fk = store_.strong_tx.first(ptr, parent); + + table::strong_tx::record strong{}; + if (!store_.strong_tx.get(ptr, fk, strong) || !strong.positive()) + return error::branch_inactive; + + block = strong.header_fk(); + } + + out.at(at) = block; + } + + return error::success; +} + +TEMPLATE +code CLASS::read_heights(std_vector& out, + const header_links& blocks) const NOEXCEPT +{ + size_t height{}; + header_link block{}; + out.resize(blocks.size()); + + const auto ptr = store_.header.get_memory(); + for (size_t at{}; at < blocks.size(); ++at) + { + if (blocks.at(at) != block) + { + block = blocks.at(at); + table::header::get_height get_height{}; + if (!store_.header.get(ptr, block, get_height)) + return error::integrity; + + height = get_height.height; + } + + out.at(at) = height; + } + + return error::success; +} + +// bitcoind retains duplicated coinbases at the overwriting heights (bip30 +// exceptions), the store at the originals. +TEMPLATE +code CLASS::confirm(unspent_coins& out, size_t offset, + const header_links& blocks, const std_vector& heights) const NOEXCEPT +{ + header_link block{}; + const auto fork = store_.envelope().forks.bip30; + + const auto ptr = store_.confirmed.get_memory(); + for (size_t at{}; at < blocks.size(); ++at) + { + if (blocks.at(at) != block) + { + block = blocks.at(at); + if (store_.confirmed.at(ptr, heights.at(at)) != block) + return error::integrity; + } + + auto& coin = out.at(offset + at); + coin.height = heights.at(at); + if (!fork || !coin.coinbase) + continue; + + if (coin.height == bip30::first_original) + coin.height = bip30::first_exception; + else if (coin.height == bip30::second_original) + coin.height = bip30::second_exception; + } + + return error::success; +} + +} // namespace database +} // namespace libbitcoin + +#endif diff --git a/include/bitcoin/database/impl/unspent/unspent_scanner.ipp b/include/bitcoin/database/impl/unspent/unspent_scanner.ipp new file mode 100644 index 000000000..da37f34b7 --- /dev/null +++ b/include/bitcoin/database/impl/unspent/unspent_scanner.ipp @@ -0,0 +1,307 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_SCANNER_IPP +#define LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_SCANNER_IPP + +#include +#include +#include +#include +#include + +namespace libbitcoin { +namespace database { + +TEMPLATE +CLASS::unspent_scanner(const query& query, const Store& store, + const stopper& cancel, bool turbo) NOEXCEPT + : query_(query), store_(store), cancel_(cancel), turbo_(turbo) +{ +} + +TEMPLATE +code CLASS::scan(difference_set& set, unspent_totals& out, + const header_links& branch) const NOEXCEPT +{ + if (branch.empty()) + return error::success; + + std_vector bounds{}; + if (const auto ec = this->bounds(bounds, branch)) + return ec; + + const auto cuts = sub1(bounds.size()); + std_vector unspendables(cuts, zero); + std_vector overwrittens(cuts, zero); + std::atomic_bool fail{}; + + // Static chunking is contiguous, so spans are interleaved across threads. + const auto width = cores(); + std_vector index{}; + index.reserve(cuts); + for (size_t lane{}; lane < width; ++lane) + for (auto at = lane; at < cuts; at += width) + index.push_back(at); + + const auto parallel = poolstl::execution::par_if(turbo_); + std::for_each(parallel, index.begin(), index.end(), + [&](size_t at) NOEXCEPT + { + if (fail) + return; + + if (span(set, unspendables.at(at), overwrittens.at(at), branch, + bounds.at(at), bounds.at(add1(at)))) + fail = true; + }); + + if (fail) + return cancel_ ? error::query_canceled : error::integrity; + + for (size_t at{}; at < cuts; ++at) + { + out.unspendable += unspendables.at(at); + out.bip30 += overwrittens.at(at); + } + + return error::success; +} + +// Spans are cut to equal tx counts, as blocks vary by orders of magnitude. +TEMPLATE +code CLASS::bounds(std_vector& out, + const header_links& branch) const NOEXCEPT +{ + BC_ASSERT(out.empty()); + + std_vector counts(branch.size()); + std_vector blocks(branch.size()); + std::iota(blocks.begin(), blocks.end(), zero); + const auto parallel = poolstl::execution::par_if(turbo_); + + std::for_each(parallel, blocks.begin(), blocks.end(), + [&](size_t block) NOEXCEPT + { + counts.at(block) = query_.get_tx_count(branch.at(block)); + }); + + size_t txs{}; + for (const auto count: counts) + txs += count; + + const auto width = cores(); + const auto spans = std::min(branch.size(), width * per_thread); + const auto per_span = std::max(one, system::ceilinged_divide(txs, spans)); + + out.reserve(add1(spans)); + out.push_back(zero); + size_t carry{}; + + for (size_t block{}; block < branch.size(); ++block) + { + carry += counts.at(block); + if (carry >= per_span && out.size() <= spans) + { + out.push_back(add1(block)); + carry = zero; + } + } + + if (out.back() != branch.size()) + out.push_back(branch.size()); + + return error::success; +} + +TEMPLATE +code CLASS::span(difference_set& set, uint64_t& unspendable, + uint64_t& overwritten, const header_links& branch, size_t begin, + size_t end) const NOEXCEPT +{ + buffers buffer{}; + for (auto at = begin; at < end; ++at) + { + if (cancel_) + return error::query_canceled; + + if (const auto ec = block(set, unspendable, overwritten, buffer, + branch.at(at))) + return ec; + } + + return error::success; +} + +TEMPLATE +code CLASS::block(difference_set& set, uint64_t& unspendable, + uint64_t& overwritten, buffers& buffer, + const header_link& link) const NOEXCEPT +{ + auto bip30_exception = false; + if (!query_.is_bip30_exception(bip30_exception, link)) + return error::integrity; + + if (bip30_exception) + return this->overwritten(overwritten, link); + + if (const auto ec = read_transactions(buffer.txs, link)) + return ec; + + if (const auto ec = toggle_creates(set, buffer.exclusions, buffer.txs)) + return ec; + + if (const auto ec = read_exclusions(unspendable, buffer.exclusions)) + return ec; + + if (const auto ec = read_points(buffer.spends, buffer.txs)) + return ec; + + return toggle_spends(set, buffer.spends); +} + +// The bip30 exception blocks are coinbase only, their duplicated outputs +// overwrite (destroy) those of the original blocks. +TEMPLATE +code CLASS::overwritten(uint64_t& out, const header_link& link) const NOEXCEPT +{ + table::output::get_spendable value{}; + for (const auto& tx: query_.to_transactions(link)) + { + for (const auto& put: query_.to_outputs(tx)) + { + if (!store_.output.get(put, value)) + return error::integrity; + + out += value.value; + } + } + + return error::success; +} + +TEMPLATE +code CLASS::read_transactions(gets& out, + const header_link& link) const NOEXCEPT +{ + const auto links = query_.to_transactions(link); + out.resize(links.size()); + + const auto ptr = store_.tx.get_memory(); + for (size_t at{}; at < links.size(); ++at) + if (!store_.tx.get(ptr, links.at(at), out.at(at))) + return error::integrity; + + return error::success; +} + +TEMPLATE +code CLASS::toggle_creates(difference_set& set, output_links& exclusions, + const gets& txs) const NOEXCEPT +{ + exclusions.clear(); + + const auto ptr = store_.outs.puts.get_memory(); + for (const auto& tx: txs) + { + table::outs::get_excluded outs{ .number = tx.outs_count }; + if (!store_.outs.puts.get(ptr, tx.outs_fk, outs)) + return error::integrity; + + auto excluded = outs.excluded.begin(); + for (uint32_t index{}; index < outs.number; ++index) + { + if (excluded != outs.excluded.end() && excluded->first == index) + exclusions.push_back((excluded++)->second); + else + set.toggle(tx.outs_fk + index); + } + } + + return error::success; +} + +TEMPLATE +code CLASS::read_exclusions(uint64_t& out, + const output_links& exclusions) const NOEXCEPT +{ + const auto ptr = store_.output.get_memory(); + for (const auto& put: exclusions) + { + table::output::get_spendable value{}; + if (!store_.output.get(ptr, put, value)) + return error::integrity; + + out += value.value; + } + + return error::success; +} + +TEMPLATE +code CLASS::read_points(system::chain::points& out, + const gets& txs) const NOEXCEPT +{ + out.clear(); + + const auto ptr = store_.ins.get_memory(); + for (const auto& tx: txs) + { + if (tx.coinbase) + continue; + + for (ins_link::integer in{}; in < tx.ins_count; ++in) + { + table::ins_point::get_composed point{}; + if (!store_.ins.get(ptr, tx.points_fk + in, point)) + return error::integrity; + + out.push_back(point.key); + } + } + + return error::success; +} + +// As validation, the first tx of the hash resolves the spend. +TEMPLATE +code CLASS::toggle_spends(difference_set& set, + const system::chain::points& spends) const NOEXCEPT +{ + const auto ptr = store_.tx.get_memory(); + for (const auto& spend: spends) + { + const auto spent = store_.tx.first(ptr, spend.hash()); + if (spent.is_terminal()) + return error::integrity; + + table::transaction::get_output prevout{ {}, spend.index() }; + if (!store_.tx.get(ptr, spent, prevout) || + prevout.outs_fk == outs_link::terminal) + return error::integrity; + + set.toggle(prevout.outs_fk); + } + + return error::success; +} + +} // namespace database +} // namespace libbitcoin + +#endif diff --git a/include/bitcoin/database/impl/unspent/unspent_serial.ipp b/include/bitcoin/database/impl/unspent/unspent_serial.ipp new file mode 100644 index 000000000..f163ca694 --- /dev/null +++ b/include/bitcoin/database/impl/unspent/unspent_serial.ipp @@ -0,0 +1,352 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_SERIAL_IPP +#define LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_SERIAL_IPP + +#include +#include +#include +#include +#include + +namespace libbitcoin { +namespace database { + +TEMPLATE +CLASS::unspent_serial(const Store& store, const stopper& cancel, + bool turbo) NOEXCEPT + : reader_(store, cancel), + spans_(store, cancel, turbo), + store_(store), + cancel_(cancel), + turbo_(turbo) +{ +} + +// One folder thread for the run, each bucket handed off through 'ordered' +// (ready) and returned (done), so a fold overlaps the fill of the next bucket. +TEMPLATE +code CLASS::hash(unspent_totals& out, hash_digest& digest, + const difference_set& set) const NOEXCEPT +{ + sizes offsets{}; + unspent_elements elements{}; + if (const auto ec = partition(elements, offsets, set)) + return ec; + + code folded{}; + auto stop = false; + auto previous = system::null_hash; + system::stream::out::fast stream{ digest }; + system::hash::sha256x2::fast sink{ stream }; + unspent_coins coins{}, ordered{}; + output_links puts{}, links{}; + std::binary_semaphore ready{ 0 }; + std::binary_semaphore done{ 1 }; + std::thread folder + { + [&]() NOEXCEPT + { + for (ready.acquire(); !stop; ready.acquire()) + { + folded = fold(out, sink, previous, ordered, links); + done.release(); + } + } + }; + + code ec{}; + sizes order{}; + for (size_t bucket{}; bucket < buckets; ++bucket) + { + const auto begin = offsets.at(bucket); + const auto end = offsets.at(add1(bucket)); + ec = fill(coins, puts, elements, begin, end); + done.acquire(); + if (!ec) + ec = folded; + + if (ec) + break; + + this->order(order, coins); + gather(ordered, links, coins, puts, order); + ready.release(); + } + + if (!ec) + { + done.acquire(); + ec = folded; + } + + stop = true; + ready.release(); + folder.join(); + if (ec) + return ec; + + sink.flush(); + return error::success; +} + +// Two passes over the set: count per chunk and bucket, then scatter. +TEMPLATE +code CLASS::partition(unspent_elements& out, sizes& offsets, + const difference_set& set) const NOEXCEPT +{ + const auto chunks = std::max(one, std::min(set.words(), spans_.count())); + const auto span = system::ceilinged_divide(set.words(), chunks); + sizes counts(chunks * buckets, zero); + sizes index(chunks); + std::iota(index.begin(), index.end(), zero); + const auto parallel = poolstl::execution::par_if(turbo_); + std::atomic_bool fail{}; + + std::for_each(parallel, index.begin(), index.end(), + [&](size_t chunk) NOEXCEPT + { + if (fail) + return; + + const auto begin = chunk * span; + const auto end = std::min(set.words(), begin + span); + if (walk(set, begin, end, + [&](size_t slot, outs_link::integer) NOEXCEPT + { + ++counts.at(chunk * buckets + slot); + })) fail = true; + }); + + if (fail) + return cancel_ ? error::query_canceled : error::integrity; + + starts(counts, offsets, chunks); + out.resize(offsets.back()); + + std::for_each(parallel, index.begin(), index.end(), + [&](size_t chunk) NOEXCEPT + { + if (fail) + return; + + const auto begin = chunk * span; + const auto end = std::min(set.words(), begin + span); + if (walk(set, begin, end, + [&](size_t slot, outs_link::integer element) NOEXCEPT + { + out.at(counts.at(chunk * buckets + slot)++) = element; + })) fail = true; + }); + + if (fail) + return cancel_ ? error::query_canceled : error::integrity; + + return error::success; +} + +// emit(slot, element) for the set bits of the word range, in element order. +TEMPLATE +template +code CLASS::walk(const difference_set& set, size_t begin, size_t end, + const Emit& emit) const NOEXCEPT +{ + constexpr auto width = difference_set::word_bits; + unspent_elements elements{}; + + for (auto word = begin; word < end; ++word) + { + if (cancel_) + return error::query_canceled; + + using namespace system; + const auto base = word * width; + for (auto bits = set.at(word); !is_zero(bits); + bits = bit_and(bits, sub1(bits))) + { + const auto link = base + right_zeros(bits); + elements.push_back(possible_narrow_cast(link)); + } + } + + sizes slots{}; + if (const auto ec = this->slots(slots, elements)) + return ec; + + for (size_t at{}; at < elements.size(); ++at) + emit(slots.at(at), elements.at(at)); + + return error::success; +} + +TEMPLATE +code CLASS::slots(sizes& out, + const unspent_elements& elements) const NOEXCEPT +{ + output_links puts(elements.size()); + if (const auto ec = reader_.read_puts(puts, zero, elements, zero, + elements.size())) + return ec; + + tx_links parents{}; + if (const auto ec = reader_.read_parents(parents, puts, zero, puts.size())) + return ec; + + system::hashes hashes{}; + if (const auto ec = reader_.read_hashes(hashes, parents)) + return ec; + + out.resize(hashes.size()); + for (auto at = zero; at < hashes.size(); ++at) + out.at(at) = hashes.at(at).front(); + + return error::success; +} + +// Bucket offsets, then chunk starts within each bucket (counts in place). +TEMPLATE +void CLASS::starts(sizes& counts, sizes& offsets, size_t chunks) NOEXCEPT +{ + offsets.assign(add1(buckets), zero); + for (size_t chunk{}; chunk < chunks; ++chunk) + for (size_t bucket{}; bucket < buckets; ++bucket) + offsets.at(add1(bucket)) += counts.at(chunk * buckets + bucket); + + for (size_t bucket{}; bucket < buckets; ++bucket) + offsets.at(add1(bucket)) += offsets.at(bucket); + + for (size_t bucket{}; bucket < buckets; ++bucket) + { + auto start = offsets.at(bucket); + for (size_t chunk{}; chunk < chunks; ++chunk) + { + auto& slot = counts.at(chunk * buckets + bucket); + const auto count = slot; + slot = start; + start += count; + } + } +} + +TEMPLATE +code CLASS::fill(unspent_coins& out, output_links& puts, + const unspent_elements& elements, size_t begin, size_t end) const NOEXCEPT +{ + const auto size = end - begin; + out.resize(size); + puts.resize(size); + if (is_zero(size)) + return error::success; + + using namespace system; + const auto parallel = poolstl::execution::par_if(turbo_); + const auto chunks = std::max(one, std::min(size, spans_.count())); + const auto span = ceilinged_divide(size, chunks); + sizes index(ceilinged_divide(size, span)); + std::iota(index.begin(), index.end(), zero); + std::atomic_bool fail{}; + + std::for_each(parallel, index.begin(), index.end(), + [&](size_t chunk) NOEXCEPT + { + if (fail) + return; + + auto previous = tx_link::terminal; + const auto first = chunk * span; + const auto last = std::min(size, first + span); + if (reader_.fill(out, puts, first, previous, elements, + begin + first, begin + last)) + fail = true; + }); + + if (fail) + return cancel_ ? error::query_canceled : error::integrity; + + return error::success; +} + +TEMPLATE +void CLASS::order(sizes& out, const unspent_coins& coins) const NOEXCEPT +{ + out.resize(coins.size()); + std::iota(out.begin(), out.end(), zero); + const auto parallel = poolstl::execution::par_if(turbo_); + + std::sort(parallel, out.begin(), out.end(), + [&](size_t left, size_t right) NOEXCEPT + { + const auto& one = coins.at(left).out.point(); + const auto& two = coins.at(right).out.point(); + return (one.hash() == two.hash()) ? + (one.index() < two.index()) : + (one.hash() < two.hash()); + }); +} + +TEMPLATE +void CLASS::gather(unspent_coins& out, output_links& links, + unspent_coins& coins, const output_links& puts, + const sizes& order) const NOEXCEPT +{ + out.resize(coins.size()); + links.resize(coins.size()); + sizes index(coins.size()); + std::iota(index.begin(), index.end(), zero); + const auto parallel = poolstl::execution::par_if(turbo_); + + std::for_each(parallel, index.begin(), index.end(), + [&](size_t at) NOEXCEPT + { + out.at(at) = std::move(coins.at(order.at(at))); + links.at(at) = puts.at(order.at(at)); + }); +} + +TEMPLATE +code CLASS::fold(unspent_totals& out, system::writer& sink, + hash_digest& previous, unspent_coins& coins, + const output_links& puts) const NOEXCEPT +{ + const auto ptr = store_.output.get_memory(); + for (size_t at{}; at < coins.size(); ++at) + { + if (cancel_) + return error::query_canceled; + + auto& coin = coins.at(at); + const auto& hash = coin.out.point().hash(); + coin.first = (hash != previous); + previous = hash; + + unspent_writer::write(sink, coin); + table::output::write_script tail{ {}, sink }; + if (!store_.output.raw(ptr, puts.at(at), tail)) + return error::integrity; + + unspent_writer::add(out, coin.first, tail.value, tail.length); + } + + return error::success; +} + +} // namespace database +} // namespace libbitcoin + +#endif diff --git a/include/bitcoin/database/impl/unspent/unspent_spans.ipp b/include/bitcoin/database/impl/unspent/unspent_spans.ipp new file mode 100644 index 000000000..73946d133 --- /dev/null +++ b/include/bitcoin/database/impl/unspent/unspent_spans.ipp @@ -0,0 +1,143 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_SPANS_IPP +#define LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_SPANS_IPP + +#include +#include +#include +#include +#include + +namespace libbitcoin { +namespace database { + +TEMPLATE +CLASS::unspent_spans(const Store& store, const stopper& cancel, + bool turbo) NOEXCEPT + : store_(store), cancel_(cancel), turbo_(turbo) +{ +} + +TEMPLATE +size_t CLASS::count() const NOEXCEPT +{ + return cores() * per_thread; +} + +TEMPLATE +template +code CLASS::for_each(const difference_set& set, const Span& span) const NOEXCEPT +{ + using namespace system; + std_vector bounds{}; + if (const auto ec = this->bounds(bounds, set)) + return ec; + + // Static chunking is contiguous, so spans are interleaved across threads + // (unspent density rises with height, contiguous lanes are unbalanced). + const auto cuts = sub1(bounds.size()); + const auto width = cores(); + std_vector index{}; + index.reserve(cuts); + for (size_t lane{}; lane < width; ++lane) + for (auto at = lane; at < cuts; at += width) + index.push_back(at); + + std::atomic_bool fail{}; + const auto parallel = poolstl::execution::par_if(turbo_); + + std::for_each(parallel, index.begin(), index.end(), + [&](size_t chunk) NOEXCEPT + { + if (fail) + return; + + if (span(chunk, bounds.at(chunk), bounds.at(add1(chunk)))) + fail = true; + }); + + if (fail) + return cancel_ ? error::query_canceled : error::integrity; + + return error::success; +} + +TEMPLATE +bool CLASS::to_tx_base(size_t& out, size_t element) const NOEXCEPT +{ + using namespace system; + const outs_link link{ possible_narrow_cast(element) }; + + table::outs::get_output outs{}; + if (!store_.outs.puts.get(link, outs)) + return false; + + table::output::get_parent output{}; + if (!store_.output.get(outs.out_fk, output)) + return false; + + table::transaction::get_output tx{}; + if (!store_.tx.get(output.parent_fk, tx)) + return false; + + out = tx.outs_fk; + return true; +} + +// Bounds are snapped to tx boundaries, as tx count is by distinct tx. +TEMPLATE +code CLASS::bounds(std_vector& out, + const difference_set& set) const NOEXCEPT +{ + BC_ASSERT(out.empty()); + out.push_back(zero); + if (is_zero(set.size())) + { + out.push_back(zero); + return error::success; + } + + using namespace system; + const auto chunks = std::min(set.size(), count()); + const auto stride = ceilinged_divide(set.size(), chunks); + out.reserve(add1(chunks)); + + for (auto chunk = one; chunk < chunks; ++chunk) + { + const auto element = chunk * stride; + if (element >= set.size()) + break; + + size_t at{}; + if (!to_tx_base(at, element)) + return error::integrity; + + if (at > out.back()) + out.push_back(at); + } + + out.push_back(set.size()); + return error::success; +} + +} // namespace database +} // namespace libbitcoin + +#endif diff --git a/include/bitcoin/database/impl/unspent/unspent_writer.ipp b/include/bitcoin/database/impl/unspent/unspent_writer.ipp new file mode 100644 index 000000000..e9a04d7ea --- /dev/null +++ b/include/bitcoin/database/impl/unspent/unspent_writer.ipp @@ -0,0 +1,75 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_WRITER_IPP +#define LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_WRITER_IPP + +#include + +namespace libbitcoin { +namespace database { + +inline void unspent_writer::write(system::writer& sink, + const unspent_coin& coin) NOEXCEPT +{ + using namespace system; + const auto coinbase = to_int(coin.coinbase); + const auto height = possible_narrow_cast(coin.height); + + sink.write_bytes(coin.out.point().hash()); + sink.write_4_bytes_little_endian(coin.out.point().index()); + sink.write_4_bytes_little_endian(bit_or(shift_left(height), coinbase)); +} + +inline size_t unspent_writer::size(size_t script) NOEXCEPT +{ + return fixed_size + variable_size(script) + script; +} + +inline void unspent_writer::add(unspent_totals& out, bool first, + uint64_t value, size_t script) NOEXCEPT +{ + if (first) + ++out.transactions; + + ++out.outputs; + out.value += value; + out.script_bytes += script; + out.coin_bytes += size(script); +} + +inline void unspent_writer::add(unspent_totals& out, + const unspent_coin& coin) NOEXCEPT +{ + add(out, coin.first, coin.out.value(), coin.script.size()); +} + +inline void unspent_writer::add(unspent_totals& out, + const unspent_totals& totals) NOEXCEPT +{ + out.transactions += totals.transactions; + out.outputs += totals.outputs; + out.value += totals.value; + out.script_bytes += totals.script_bytes; + out.coin_bytes += totals.coin_bytes; +} + +} // namespace database +} // namespace libbitcoin + +#endif diff --git a/include/bitcoin/database/memory/memory.hpp b/include/bitcoin/database/memory/memory.hpp index b60ba4102..0a1b5fe26 100644 --- a/include/bitcoin/database/memory/memory.hpp +++ b/include/bitcoin/database/memory/memory.hpp @@ -26,6 +26,10 @@ #include #include #include +#include +#include +#include #include +#include #endif diff --git a/include/bitcoin/database/memory/release.hpp b/include/bitcoin/database/memory/release.hpp new file mode 100644 index 000000000..83511e316 --- /dev/null +++ b/include/bitcoin/database/memory/release.hpp @@ -0,0 +1,125 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_MEMORY_RELEASE_HPP +#define LIBBITCOIN_DATABASE_MEMORY_RELEASE_HPP + +#include +#include +#include + +namespace libbitcoin { +namespace database { + +/// Head-release page-run helpers (pure, unit tested). Pages are tracked as +/// bits in 64-bit words (low order bit is the lowest page of the word). A +/// release candidate page is clean (not dirty), cold (not recently written) +/// and not already released. Candidacy is limited to full pages below the +/// logical page count. +/// --------------------------------------------------------------------------- + +/// Candidate bits given page-state words. +constexpr uint64_t release_candidates(uint64_t dirty, uint64_t hot, + uint64_t released) NOEXCEPT +{ + using namespace system; + return bit_not(bit_or(dirty, bit_or(hot, released))); +} + +/// Retain candidacy below the page count, for the word starting at page +/// 'first' (a full word of candidacy above the count clears to zero). +constexpr uint64_t release_below(uint64_t candidates, size_t pages, + size_t first) NOEXCEPT +{ + using namespace system; + constexpr auto bits = to_bits(sizeof(uint64_t)); + + if (pages <= first) + return zero; + + if (pages - first >= bits) + return candidates; + + return bit_and(candidates, unmask_right(pages - first)); +} + +/// Bits for pages [lo, hi) clamped to the word of 64 pages starting at page +/// 'first', empty if the ranges do not intersect. +constexpr uint64_t page_mask(size_t lo, size_t hi, size_t first) NOEXCEPT +{ + using namespace system; + constexpr auto bits = to_bits(sizeof(uint64_t)); + const auto begin = std::max(lo, first); + const auto end = std::min(hi, first + bits); + + if (begin >= end) + return zero; + + const auto high = (end - first >= bits) ? bit_all : + unmask_right(end - first); + + return bit_and(high, mask_right(begin - first)); +} + +/// The maximal run of set bits at or above 'page' within 'pages', as the +/// right-open page range [first, second), empty (equal) if none. +inline std::pair next_run(const uint64_t* words, size_t pages, + size_t page) NOEXCEPT +{ + using namespace system; + constexpr auto bits = to_bits(sizeof(uint64_t)); + + // Find first set bit at or above page. + while ((page < pages) && !get_right(words[page / bits], page % bits)) + ++page; + + // Find first clear bit above start. + auto end = page; + while ((end < pages) && get_right(words[end / bits], end % bits)) + ++end; + + return { page, end }; +} + +/// The maximal run of set bits containing 'page' within 'pages', as the +/// right-open page range [first, second), empty (equal) if page is not set. +inline std::pair bit_run(const uint64_t* words, size_t pages, + size_t page) NOEXCEPT +{ + using namespace system; + constexpr auto bits = to_bits(sizeof(uint64_t)); + + if ((page >= pages) || !get_right(words[page / bits], page % bits)) + return { page, page }; + + auto start = page; + while (!is_zero(start) && + get_right(words[sub1(start) / bits], sub1(start) % bits)) + --start; + + auto end = add1(page); + while ((end < pages) && get_right(words[end / bits], end % bits)) + ++end; + + return { start, end }; +} + +} // namespace database +} // namespace libbitcoin + +#endif diff --git a/include/bitcoin/database/memory/settings.hpp b/include/bitcoin/database/memory/settings.hpp index 5ba490ece..ca152f7c0 100644 --- a/include/bitcoin/database/memory/settings.hpp +++ b/include/bitcoin/database/memory/settings.hpp @@ -20,31 +20,11 @@ #define LIBBITCOIN_DATABASE_MEMORY_SETTINGS_HPP #include +#include namespace libbitcoin { namespace database { -/// Expected read pattern of mapped storage, guiding kernel page advice. This -/// is independent of the write pattern (structural): bodies are appended -/// sequentially but read randomly by validation, so advising the kernel from -/// the write pattern invites eviction of the read set under memory pressure. -enum class advice : uint8_t -{ - /// Let the operating system decide (mixed or unpredictable access). - normal, - - /// Random access (suppresses read ahead), preloaded (small maps only). - random, - - /// One pass access (allows the kernel to free pages behind). - sequential, - - /// Random access (suppresses read ahead) without preload, for maps too - /// large to reside: scattered reads otherwise fault a full read-ahead - /// window each, and that manufactured cache displaces the head set. - scattered -}; - /// Storage tuning consumed by memory map construction: the base of table /// configuration, as network settings bases are derived by server services. /// Future staging tunables land here without constructor signature changes. diff --git a/include/bitcoin/database/memory/utilities.hpp b/include/bitcoin/database/memory/utilities.hpp index 22a8b84d7..2ca4ee2f7 100644 --- a/include/bitcoin/database/memory/utilities.hpp +++ b/include/bitcoin/database/memory/utilities.hpp @@ -19,9 +19,6 @@ #ifndef LIBBITCOIN_DATABASE_MEMORY_UTILITIES_HPP #define LIBBITCOIN_DATABASE_MEMORY_UTILITIES_HPP -#include -#include -#include #include namespace libbitcoin { @@ -33,12 +30,6 @@ BCD_API size_t page_size() NOEXCEPT; /// The bytes of physical memory, zero if failed. BCD_API uint64_t system_memory() NOEXCEPT; -/// Head buckets from expected elements and installed memory, scaled linearly -/// between the contested and uncontested load factors (in tenths). Hashmaps -/// only; zero expected is not derived. -BCD_API uint32_t derive_buckets(uint64_t expected, uint32_t low, - uint32_t high) NOEXCEPT; - /// The bytes of unused physical memory, zero if failed. Scarcity precedes /// pressure: clean file cache is reclaimable, so the pressure level does not /// raise while free memory exhausts. @@ -57,90 +48,8 @@ BCD_API size_t system_pressure() NOEXCEPT; /// if failed or the platform provides no source. BCD_API uint64_t system_compressed() NOEXCEPT; -// TODO: move all below to release.hpp, remove compound ternaries, delint. - -/// Head-release page-run helpers (pure, unit tested). Pages are tracked as -/// bits in 64-bit words (low order bit is the lowest page of the word). A -/// release candidate page is clean (not dirty), cold (not recently written) -/// and not already released. Candidacy is limited to full pages below the -/// logical page count. -/// --------------------------------------------------------------------------- - -/// Candidate bits given page-state words. -constexpr uint64_t release_candidates(uint64_t dirty, uint64_t hot, - uint64_t released) NOEXCEPT -{ - using namespace system; - return bit_not(bit_or(dirty, bit_or(hot, released))); -} - -/// Retain candidacy below the page count, for the word starting at page -/// 'first' (a full word of candidacy above the count clears to zero). -constexpr uint64_t release_below(uint64_t candidates, size_t pages, - size_t first) NOEXCEPT -{ - using namespace system; - return (pages <= first) ? zero : - (pages - first >= to_bits(sizeof(uint64_t))) ? candidates : - bit_and(candidates, unmask_right(pages - first)); -} - -/// Bits for pages [lo, hi) clamped to the word of 64 pages starting at page -/// 'first', empty if the ranges do not intersect. -constexpr uint64_t page_mask(size_t lo, size_t hi, size_t first) NOEXCEPT -{ - using namespace system; - constexpr auto bits = to_bits(sizeof(uint64_t)); - const auto begin = std::max(lo, first); - const auto end = std::min(hi, first + bits); - return (begin >= end) ? zero : bit_and( - (end - first >= bits) ? bit_all : - unmask_right(end - first), - mask_right(begin - first)); -} - -/// The maximal run of set bits at or above 'page' within 'pages', as the -/// right-open page range [first, second), empty (equal) if none. -inline std::pair next_run(const uint64_t* words, size_t pages, - size_t page) NOEXCEPT -{ - using namespace system; - constexpr auto bits = to_bits(sizeof(uint64_t)); - - // Find first set bit at or above page. - while ((page < pages) && !get_right(words[page / bits], page % bits)) - ++page; - - // Find first clear bit above start. - auto end = page; - while ((end < pages) && get_right(words[end / bits], end % bits)) - ++end; - - return { page, end }; -} - -/// The maximal run of set bits containing 'page' within 'pages', as the -/// right-open page range [first, second), empty (equal) if page is not set. -inline std::pair bit_run(const uint64_t* words, size_t pages, - size_t page) NOEXCEPT -{ - using namespace system; - constexpr auto bits = to_bits(sizeof(uint64_t)); - - if ((page >= pages) || !get_right(words[page / bits], page % bits)) - return { page, page }; - - auto start = page; - while (!is_zero(start) && - get_right(words[sub1(start) / bits], sub1(start) % bits)) - --start; - - auto end = add1(page); - while ((end < pages) && get_right(words[end / bits], end % bits)) - ++end; - - return { start, end }; -} +/// Number of cores, no less than one. +BCD_API size_t cores() NOEXCEPT; } // namespace database } // namespace libbitcoin diff --git a/include/bitcoin/database/primitives/hashhead.hpp b/include/bitcoin/database/primitives/hashhead.hpp index 317eb8d4b..10d1150ac 100644 --- a/include/bitcoin/database/primitives/hashhead.hpp +++ b/include/bitcoin/database/primitives/hashhead.hpp @@ -56,6 +56,10 @@ class hashhead /// Optimal filter k-value for expected load factor. static size_t optimal_k(size_t count, size_t buckets) NOEXCEPT; + /// Buckets for expected elements, scaled by memory between load factors. + static uint32_t derive_buckets(uint64_t expected, uint32_t low, + uint32_t high) NOEXCEPT; + /// Create from empty head file (not thread safe). bool create() NOEXCEPT; diff --git a/include/bitcoin/database/primitives/hashmap.hpp b/include/bitcoin/database/primitives/hashmap.hpp index 2c07e6d9d..c7e16389e 100644 --- a/include/bitcoin/database/primitives/hashmap.hpp +++ b/include/bitcoin/database/primitives/hashmap.hpp @@ -67,6 +67,10 @@ class hashmap /// Hash table bucket count. size_t buckets() const NOEXCEPT; + /// Head buckets for expected elements (hashhead::derive_buckets). + static uint32_t derive_buckets(uint64_t expected, uint32_t low, + uint32_t high) NOEXCEPT; + /// Filter selections, derived at construct, overridden by set_filter_k. size_t filter_k() const NOEXCEPT; diff --git a/include/bitcoin/database/primitives/hashmaps.hpp b/include/bitcoin/database/primitives/hashmaps.hpp index 30fdce7c8..10d9f203a 100644 --- a/include/bitcoin/database/primitives/hashmaps.hpp +++ b/include/bitcoin/database/primitives/hashmaps.hpp @@ -76,6 +76,10 @@ class hashmaps /// Hash table bucket count. size_t buckets() const NOEXCEPT; + /// Head buckets for expected elements (hashhead::derive_buckets). + static uint32_t derive_buckets(uint64_t expected, uint32_t low, + uint32_t high) NOEXCEPT; + /// Filter selections, derived at construct, overridden by set_filter_k. size_t filter_k() const NOEXCEPT; diff --git a/include/bitcoin/database/primitives/headmap.hpp b/include/bitcoin/database/primitives/headmap.hpp index f1a7feff0..66ab6dcb7 100644 --- a/include/bitcoin/database/primitives/headmap.hpp +++ b/include/bitcoin/database/primitives/headmap.hpp @@ -88,6 +88,10 @@ class headmap /// Return link at index, terminal if at or above count. Link at(size_t index) const NOEXCEPT; + /// Head accessor (shared lock), for at(ptr, index) over many indexes. + memory get_memory() const NOEXCEPT; + Link at(const memory& ptr, size_t index) const NOEXCEPT; + /// Append link at count into reserved capacity (single writer). bool push(const Link& link) NOEXCEPT; diff --git a/include/bitcoin/database/query.hpp b/include/bitcoin/database/query.hpp index 689508e22..9217b6d5d 100644 --- a/include/bitcoin/database/query.hpp +++ b/include/bitcoin/database/query.hpp @@ -24,6 +24,7 @@ #include #include #include +#include namespace libbitcoin { namespace database { @@ -70,6 +71,8 @@ class query /// Get a transactor object. transactor get_transactor() const NOEXCEPT; + /// The store. + /// Get first fault code, or disk_full if none and full, or success. code get_code() const NOEXCEPT; @@ -719,12 +722,12 @@ class query histories& out, const hash_digest& key, size_t limit=max_size_t, bool turbo=false) const NOEXCEPT; - /// Electrum queries (unspents, deduped, electrum sort). - code get_unconfirmed_unspent(const stopper& cancel, unspents& out, + /// Electrum queries (unspent_outputs, deduped, electrum sort). + code get_unconfirmed_unspent(const stopper& cancel, unspent_outputs& out, const hash_digest& key, bool turbo=false) const NOEXCEPT; - code get_confirmed_unspent(const stopper& cancel, unspents& out, + code get_confirmed_unspent(const stopper& cancel, unspent_outputs& out, const hash_digest& key, bool turbo=false) const NOEXCEPT; - code get_unspent(const stopper& cancel, unspents& out, + code get_unspent(const stopper& cancel, unspent_outputs& out, const hash_digest& key, bool turbo=false) const NOEXCEPT; /// Balance queries (universal, unconfirmed conflict resolution arbitrary). @@ -748,9 +751,9 @@ class query uint32_t index) const NOEXCEPT; /// Unspent queries. - unspent get_tx_unspent(const output_link& link) const NOEXCEPT; - unspent get_tx_confirmed_unspent(const output_link& link) const NOEXCEPT; - unspent get_tx_unconfirmed_unspent(const output_link& link) const NOEXCEPT; + unspent_output get_tx_unspent(const output_link& link) const NOEXCEPT; + unspent_output get_tx_confirmed_unspent(const output_link& link) const NOEXCEPT; + unspent_output get_tx_unconfirmed_unspent(const output_link& link) const NOEXCEPT; /// Filters. /// ----------------------------------------------------------------------- @@ -783,14 +786,26 @@ class query code get_unspent_totals(const stopper& cancel, unspent_totals& out, const header_links& branch, bool turbo=false) const NOEXCEPT; - /// Visit each unspent coin over the branch (administrative scan), in - /// canonical (txid, index) order as required for utxo set serialization, - /// otherwise unordered (ordering requires a full materialized sort). - template - code get_unspent_coins(const stopper& cancel, unspent_totals& out, - const Visitor& visit, const header_links& branch, bool ordered, + /// Unspent totals and bitcoind muhash set commitment (order invariant). + code get_unspent_muhash(const stopper& cancel, unspent_totals& out, + hash_digest& digest, const header_links& branch, + bool turbo=false) const NOEXCEPT; + + /// Unspent totals and bitcoind serialized set commitment (canonical + /// (txid, index) order, a serial stream). + code get_unspent_serialized(const stopper& cancel, unspent_totals& out, + hash_digest& digest, const header_links& branch, bool turbo=false) const NOEXCEPT; + /// Unspent coins with scripts hashing to the keys, and the set size. + code get_unspent_matches(const stopper& cancel, unspent_coins& out, + size_t& txouts, const std::unordered_set& keys, + const header_links& branch, bool turbo=false) const NOEXCEPT; + + /// The block is a bip30 exception (duplicated coinbase, if configured). + bool is_bip30_exception(bool& out, + const header_link& link) const NOEXCEPT; + protected: /// Network /// ----------------------------------------------------------------------- @@ -986,21 +1001,11 @@ class query histories& out, const tx_links& links, Functor&& functor) NOEXCEPT; template static code parallel_unspent_transform(const stopper& cancel, bool turbo, - unspents& out, const output_links& outs, Functor&& functor) NOEXCEPT; + unspent_outputs& out, const output_links& outs, Functor&& functor) NOEXCEPT; static point::cptr make_point(hash_digest&& hash, uint32_t index) NOEXCEPT; - bool is_bip30_exception(bool& out, - const header_link& link) const NOEXCEPT; - code scan_unspent(difference_set<>& set, unspent_totals& out, - const stopper& cancel, const header_links& branch, - bool turbo) const NOEXCEPT; - code count_unspent(unspent_totals& out, const stopper& cancel, - const difference_set<>::entries& survivors) const NOEXCEPT; - template - code visit_unspent(const Visitor& visit, const stopper& cancel, - const difference_set<>::entries& survivors) const NOEXCEPT; // Not thread safe. size_t get_fork_() const NOEXCEPT; diff --git a/include/bitcoin/database/settings.hpp b/include/bitcoin/database/settings.hpp index 8099e0260..855b25773 100644 --- a/include/bitcoin/database/settings.hpp +++ b/include/bitcoin/database/settings.hpp @@ -20,8 +20,8 @@ #define LIBBITCOIN_DATABASE_SETTINGS_HPP #include -#include -#include +#include +#include #include diff --git a/include/bitcoin/database/tables/archives/output.hpp b/include/bitcoin/database/tables/archives/output.hpp index 6739b6b84..95c81ec08 100644 --- a/include/bitcoin/database/tables/archives/output.hpp +++ b/include/bitcoin/database/tables/archives/output.hpp @@ -148,6 +148,31 @@ struct output bool match{}; }; + /// Unstreamed, writes full output directly to the sink. + struct write_script + : public schema::output + { + inline bool from_data(memory::iterator start) NOEXCEPT + { + using namespace system; + + // Skip parent fk, read the value and script size. + const auto* position = std::next(start, tx::size); + value = unsafe_from_variable(position); + const auto script_size = unsafe_from_variable(position); + length = possible_narrow_cast(script_size); + + sink.write_8_bytes_little_endian(value); + sink.write_variable(length); + sink.write_bytes(position, length); + return true; + } + + bytewriter& sink; + uint64_t value{}; + size_t length{}; + }; + /// Reuse an instance across gets to amortize the script allocation. struct get_coin : public schema::output @@ -161,7 +186,7 @@ struct output inline bool from_data(reader& source) NOEXCEPT { using namespace system; - source.skip_bytes(tx::size); + parent_fk = source.read_little_endian(); value = source.read_variable(); script.resize(possible_narrow_cast( source.read_variable())); @@ -169,6 +194,7 @@ struct output return source; } + tx::integer parent_fk{}; uint64_t value{}; system::data_chunk script{}; }; @@ -190,7 +216,7 @@ struct output script_size = possible_narrow_cast( source.read_variable()); - // bitcoind's utxo set exclusion (op_return lead or oversized). + // bitcoind's unspent set exclusion (op_return lead or oversized). constexpr auto op_return = to_value(chain::opcode::op_return); unspendable = (script_size > chain::max_script_size) || (!is_zero(script_size) && (source.read_byte() == op_return)); @@ -222,6 +248,30 @@ struct output uint64_t value{}; }; + struct get_parent_coin + : public schema::output + { + inline link count() const NOEXCEPT + { + BC_ASSERT(false); + return {}; + } + + inline bool from_data(reader& source) NOEXCEPT + { + using namespace system; + parent_fk = source.read_little_endian(); + value = source.read_variable(); + script_size = possible_narrow_cast( + source.read_variable()); + return source; + } + + tx::integer parent_fk{}; + uint64_t value{}; + size_t script_size{}; + }; + struct get_parent : public schema::output { diff --git a/include/bitcoin/database/tables/archives/outs.hpp b/include/bitcoin/database/tables/archives/outs.hpp index 6b3c70abe..79f6d031f 100644 --- a/include/bitcoin/database/tables/archives/outs.hpp +++ b/include/bitcoin/database/tables/archives/outs.hpp @@ -56,12 +56,50 @@ struct outs_address /// Outs column: output fk records (rows aligned with the address spine). struct outs_puts { + using ix = linkage; using out = schema::output::link; using tx = schema::transaction::link; using link = schema::outs::link; using output_links = std::vector; + using excluded_outputs = std::vector>; static constexpr auto width = schema::outs::size; static constexpr auto suffix = "puts"_t; + static constexpr auto offset = out::bits; + static_assert(offset < to_bits(out::size)); + + static constexpr out::integer merge(bool excluded, + out::integer out_fk) NOEXCEPT + { + BC_ASSERT_MSG(!system::get_right(out_fk, offset), "overflow"); + return system::set_right(out_fk, offset, excluded); + } + + static constexpr out::integer to_out_fk(out::integer merged) NOEXCEPT + { + return system::set_right(merged, offset, false); + } + + static constexpr bool is_excluded(out::integer merged) NOEXCEPT + { + return system::get_right(merged, offset); + } + + static inline bool is_excluded(const system::chain::output& out) NOEXCEPT + { + using namespace system; + const auto& script = out.script(); + const auto size = script.serialized_size(false); + return (size > chain::max_script_size) || (!is_zero(size) && + (script.ops().front().code() == chain::opcode::op_return)); + } + + static inline bool peek_excluded(auto& source, size_t bytes) NOEXCEPT + { + using namespace system; + constexpr auto op_return = to_value(chain::opcode::op_return); + return (bytes > chain::max_script_size) || + (!is_zero(bytes) && (source.peek_byte() == op_return)); + } struct record : public schema::outs @@ -75,7 +113,7 @@ struct outs_puts { std::ranges::for_each(out_fks, [&](auto& fk) NOEXCEPT { - fk = source.read_little_endian(); + fk = to_out_fk(source.read_little_endian()); }); BC_ASSERT(!source || source.get_read_position() == count() * out::size); @@ -108,13 +146,41 @@ struct outs_puts inline bool from_data(reader& source) NOEXCEPT { - out_fk = source.read_little_endian(); + out_fk = to_out_fk(source.read_little_endian()); return source; } out::integer out_fk{}; }; + struct get_excluded + : public schema::outs + { + inline link count() const NOEXCEPT + { + return system::possible_narrow_cast(number); + } + + inline bool from_data(reader& source) NOEXCEPT + { + excluded.clear(); + for (ix::integer index{}; index < number; ++index) + { + const auto merged = source.read_little_endian(); + + if (is_excluded(merged)) + excluded.emplace_back(index, to_out_fk(merged)); + } + + return source; + } + + // Output count is assigned by the caller, from the tx record. + ix::integer number{}; + excluded_outputs excluded{}; + }; + struct put_ref : public schema::outs { @@ -133,7 +199,8 @@ struct outs_puts const auto& outs = *tx_.outputs_ptr(); std::ranges::for_each(outs, [&](const auto& out) NOEXCEPT { - sink.write_little_endian(out_fk); + const auto merged = merge(is_excluded(*out), out_fk); + sink.write_little_endian(merged); // Calculate next corresponding output fk from serialized size. // (variable_size(value) + (value + script)) - (value - parent) @@ -165,10 +232,12 @@ struct outs_puts read::bytes::fast source{ stream }; for (size_t out{}; out < tx_.outputs(); ++out) { - sink.write_little_endian(out_fk); const auto value = source.read_8_bytes_little_endian(); const auto bytes = source.read_size(); + const auto excluded = peek_excluded(source, bytes); source.skip_bytes(bytes); + const auto merged = merge(excluded, out_fk); + sink.write_little_endian(merged); out_fk += tx::size + variable_size(value) + variable_size(bytes) + bytes; } @@ -197,8 +266,10 @@ struct outs /// Column element aliases. using record = outs_puts::record; using get_output = outs_puts::get_output; + using get_excluded = outs_puts::get_excluded; using put_ref = outs_puts::put_ref; using put_view = outs_puts::put_view; + using excluded_outputs = outs_puts::excluded_outputs; /// The output fk column (rows aligned with the address spine). column puts{ *this }; diff --git a/include/bitcoin/database/tables/archives/transaction.hpp b/include/bitcoin/database/tables/archives/transaction.hpp index 6373b4a5e..125b30a7b 100644 --- a/include/bitcoin/database/tables/archives/transaction.hpp +++ b/include/bitcoin/database/tables/archives/transaction.hpp @@ -266,6 +266,19 @@ struct transaction bool coinbase{}; }; + struct get_hash + : public schema::transaction + { + inline bool from_data(reader& source) NOEXCEPT + { + source.rewind_bytes(sk); + hash = source.read_hash(); + return source; + } + + key hash{}; + }; + struct get_version : public schema::transaction { diff --git a/include/bitcoin/database/tables/archives/txs.hpp b/include/bitcoin/database/tables/archives/txs.hpp index 2ac8b3677..9b0d40e18 100644 --- a/include/bitcoin/database/tables/archives/txs.hpp +++ b/include/bitcoin/database/tables/archives/txs.hpp @@ -23,7 +23,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/include/bitcoin/database/types/envelope.hpp b/include/bitcoin/database/tables/envelope.hpp similarity index 97% rename from include/bitcoin/database/types/envelope.hpp rename to include/bitcoin/database/tables/envelope.hpp index c87b07937..34392732e 100644 --- a/include/bitcoin/database/types/envelope.hpp +++ b/include/bitcoin/database/tables/envelope.hpp @@ -16,8 +16,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef LIBBITCOIN_DATABASE_TYPES_ENVELOPE_HPP -#define LIBBITCOIN_DATABASE_TYPES_ENVELOPE_HPP +#ifndef LIBBITCOIN_DATABASE_TABLES_ENVELOPE_HPP +#define LIBBITCOIN_DATABASE_TABLES_ENVELOPE_HPP #include #include diff --git a/include/bitcoin/database/tables/schema.hpp b/include/bitcoin/database/tables/schema.hpp index 55a6ac5bf..29dc7e195 100644 --- a/include/bitcoin/database/tables/schema.hpp +++ b/include/bitcoin/database/tables/schema.hpp @@ -20,7 +20,7 @@ #define LIBBITCOIN_DATABASE_TABLES_SCHEMA_HPP #include -#include +#include #include #define TABLE_COLUMN(table, bytes) \ @@ -153,7 +153,7 @@ struct output { static constexpr size_t sk = zero; static constexpr size_t pk = schema::put; - using link = linkage; + using link = linkage; // reduced for outs merge. static constexpr size_t minsize = schema::transaction::pk + // parent->tx (address navigation) one + // value (variable) @@ -232,6 +232,7 @@ struct outs static constexpr size_t pk = schema::outs_; using link = linkage; static constexpr size_t minsize = + ////schema::bit + // excluded (merged into output fk) schema::output::pk; static constexpr size_t minrow = minsize; static constexpr size_t size = minsize; diff --git a/include/bitcoin/database/tables/tables.hpp b/include/bitcoin/database/tables/tables.hpp index fe1177c2f..01dc35130 100644 --- a/include/bitcoin/database/tables/tables.hpp +++ b/include/bitcoin/database/tables/tables.hpp @@ -19,6 +19,7 @@ #ifndef LIBBITCOIN_DATABASE_TABLES_TABLES_HPP #define LIBBITCOIN_DATABASE_TABLES_TABLES_HPP +#include #include #include #include diff --git a/include/bitcoin/database/types/constants.hpp b/include/bitcoin/database/types/constants.hpp new file mode 100644 index 000000000..742084e3e --- /dev/null +++ b/include/bitcoin/database/types/constants.hpp @@ -0,0 +1,51 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_TYPES_CONSTANTS_HPP +#define LIBBITCOIN_DATABASE_TYPES_CONSTANTS_HPP + +#include + +namespace libbitcoin { +namespace database { + +/// The two bip30 exceptions, coinbases duplicating (overwriting) an original. +struct bip30 +{ + static constexpr size_t first_original = 91722; + static constexpr size_t first_exception = 91880; + static constexpr size_t second_original = 91812; + static constexpr size_t second_exception = 91842; + + static inline const system::chain::checkpoint first + { + "00000000000743f190a18c5577a3c2d2a1f610ae9601ac046a38084ccb7cd721", + first_exception + }; + + static inline const system::chain::checkpoint second + { + "00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec", + second_exception + }; +}; + +} // namespace database +} // namespace libbitcoin + +#endif diff --git a/include/bitcoin/database/types/difference_set.hpp b/include/bitcoin/database/types/difference_set.hpp index e7ed7e284..0a1bb27fd 100644 --- a/include/bitcoin/database/types/difference_set.hpp +++ b/include/bitcoin/database/types/difference_set.hpp @@ -19,85 +19,75 @@ #ifndef LIBBITCOIN_DATABASE_TYPES_DIFFERENCE_SET_HPP #define LIBBITCOIN_DATABASE_TYPES_DIFFERENCE_SET_HPP -#include -#include -#include +#include #include namespace libbitcoin { namespace database { -/// Multiset symmetric difference, exact and enumerable. Each presentation of -/// an element toggles its membership, so even multiplicities cancel. Elements -/// are (id, index) pairs, held as Mask-width index windows of an id, with the -/// id and window number packed into a 64 bit key. Thread safe. -template +/// Multiset symmetric difference over a dense element domain. Each +/// presentation of an element toggles its membership, so even multiplicities +/// cancel and the residue is the set. One bit per element, in domain order. +/// Toggle is thread safe, enumeration (word) is not concurrent with toggle. class difference_set { public: DELETE_COPY_MOVE(difference_set); - using entry = std::pair; - using entries = std::vector; + using word = uint64_t; + static constexpr auto word_bits = bits; + static constexpr auto word_shift = system::floored_log2(word_bits); - static constexpr auto key_bits = bits; - static constexpr auto mask_bits = bits; - static constexpr auto window_bits = WindowBits; - static constexpr auto index_bits = system::floored_log2(mask_bits); - static_assert(window_bits < key_bits); - - static constexpr uint64_t to_key(uint64_t id, uint32_t index) NOEXCEPT + /// The domain is [0..size), all elements initially absent. + difference_set(size_t size=zero) NOEXCEPT + : size_(size), words_(system::ceilinged_divide(size, word_bits)) { - using namespace system; - return bit_or(shift_left(id, window_bits), - shift_right(index, index_bits)); } - static constexpr uint64_t to_id(uint64_t key) NOEXCEPT + ~difference_set() = default; + + /// Toggle element membership (thread safe). + void toggle(size_t element) NOEXCEPT { - return system::shift_right(key, window_bits); + BC_ASSERT(element < size_); + const auto value = system::bit_right(to_offset(element)); + words_[to_word(element)].fetch_xor(value, std::memory_order_relaxed); } - static constexpr uint64_t to_index(uint64_t key) NOEXCEPT + /// Element domain size. + size_t size() const NOEXCEPT { - using namespace system; - return shift_left(bit_and(key, unmask_right(window_bits)), - index_bits); + return size_; } - difference_set() NOEXCEPT = default; - ~difference_set() = default; - - /// Toggle element membership (thread safe). - void toggle(uint64_t id, uint32_t index) NOEXCEPT; + /// Number of words spanning the domain. + size_t words() const NOEXCEPT + { + return words_.size(); + } - /// Sorted odd-multiplicity elements, emptying the set (not thread safe). - entries drain() NOEXCEPT; + /// Membership of the word_bits elements based at index * word_bits. + word at(size_t index) const NOEXCEPT + { + return words_[index].load(std::memory_order_relaxed); + } private: - static constexpr auto shard_bits = 4_size; - static constexpr auto index_mask = system::possible_narrow_cast( - sub1(mask_bits)); + static constexpr size_t to_word(size_t element) NOEXCEPT + { + return system::shift_right(element, word_shift); + } - struct shard + static constexpr size_t to_offset(size_t element) NOEXCEPT { - std::mutex mutex{}; - boost::unordered_flat_map map{}; - }; + return system::bit_and(element, sub1(word_bits)); + } - // This is guarded by shard::mutex. - std::array shards_{}; + const size_t size_; + std_vector> words_; }; } // namespace database } // namespace libbitcoin -#define TEMPLATE template -#define CLASS difference_set - -#include - -#undef CLASS -#undef TEMPLATE - #endif diff --git a/include/bitcoin/database/types/types.hpp b/include/bitcoin/database/types/types.hpp index 1cd664bad..20609c3da 100644 --- a/include/bitcoin/database/types/types.hpp +++ b/include/bitcoin/database/types/types.hpp @@ -22,8 +22,8 @@ #include #include #include +#include #include -#include #include #include #include @@ -34,7 +34,7 @@ #include #include #include -#include +#include #include #include diff --git a/include/bitcoin/database/types/unspent_coin.hpp b/include/bitcoin/database/types/unspent_coin.hpp index a431c24a5..dd55d4377 100644 --- a/include/bitcoin/database/types/unspent_coin.hpp +++ b/include/bitcoin/database/types/unspent_coin.hpp @@ -20,22 +20,19 @@ #define LIBBITCOIN_DATABASE_TYPES_UNSPENT_COIN_HPP #include +#include namespace libbitcoin { namespace database { -/// A coin (unspent output) presented during a branch scan. -/// The buffers are reused across visits, copy to retain. +/// An unspent output, used by bitcoind. struct BCD_API unspent_coin { - /// First visited coin of the containing transaction. + /// First coin of the containing transaction, in presentation order. bool first{}; - /// Hash of the containing transaction. - system::hash_digest txid{}; - - /// Output index within the containing transaction. - uint32_t index{}; + /// The output (value carried). + outpoint out{}; /// Confirmed height of the containing transaction. size_t height{}; @@ -43,9 +40,6 @@ struct BCD_API unspent_coin /// The containing transaction is coinbase. bool coinbase{}; - /// Output value. - uint64_t value{}; - /// Serialized script. system::data_chunk script{}; }; diff --git a/include/bitcoin/database/types/unspent.hpp b/include/bitcoin/database/types/unspent_output.hpp similarity index 67% rename from include/bitcoin/database/types/unspent.hpp rename to include/bitcoin/database/types/unspent_output.hpp index 154a1d9bf..4d0c32cb5 100644 --- a/include/bitcoin/database/types/unspent.hpp +++ b/include/bitcoin/database/types/unspent_output.hpp @@ -16,8 +16,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef LIBBITCOIN_DATABASE_TYPES_UNSPENT_HPP -#define LIBBITCOIN_DATABASE_TYPES_UNSPENT_HPP +#ifndef LIBBITCOIN_DATABASE_TYPES_UNSPENT_OUTPUT_HPP +#define LIBBITCOIN_DATABASE_TYPES_UNSPENT_OUTPUT_HPP #include #include @@ -25,40 +25,42 @@ namespace libbitcoin { namespace database { -struct BCD_API unspent +/// Used by native and electrum services. +struct BCD_API unspent_output { static constexpr size_t unused_height = zero; static constexpr size_t unconfirmed_position = max_size_t; /// Filter out invalid unspent output elements and sort. - static void filter_sort_and_dedup(std::vector& unspent) NOEXCEPT; + static void filter_sort_and_dedup( + std::vector& unspent_output) NOEXCEPT; - /// The unspent is valid. + /// The unspent output is valid. bool valid() const NOEXCEPT; - /// A fault occured (and the unspent is not valid). + /// A fault occured (and the unspent output is not valid). bool fault() const NOEXCEPT; /// The tx is confirmed in a block. bool confirmed() const NOEXCEPT; /// Comparison operator based on Electrum unspent status sort. - bool operator<(const unspent& other) const NOEXCEPT; + bool operator<(const unspent_output& other) const NOEXCEPT; /// Equivalence: !LT && !GT. - bool operator==(const unspent& other) const NOEXCEPT; + bool operator==(const unspent_output& other) const NOEXCEPT; /// Tx hash and index of output within the tx. outpoint out{}; - /// Tx's block height if confirmed, or unspent::unused_height. + /// Tx's block height if confirmed, or unspent_output::unused_height. size_t height{}; - /// Tx's position in confirmed block, or unspent::unconfirmed_position. + /// Tx's position in confirmed block, or unspent_output::unconfirmed_position. size_t position{}; }; -using unspents = std::vector; +using unspent_outputs = std::vector; } // namespace database } // namespace libbitcoin diff --git a/include/bitcoin/database/types/unspent_totals.hpp b/include/bitcoin/database/types/unspent_totals.hpp index e2b550bec..a6231ea4a 100644 --- a/include/bitcoin/database/types/unspent_totals.hpp +++ b/include/bitcoin/database/types/unspent_totals.hpp @@ -24,7 +24,7 @@ namespace libbitcoin { namespace database { -/// Spendable unspent output totals, accumulated over a branch scan. +/// Summary of unspent outputs, used by bitcoind. struct BCD_API unspent_totals { /// Unspent outputs. diff --git a/include/bitcoin/database/unspent/unspent.hpp b/include/bitcoin/database/unspent/unspent.hpp new file mode 100644 index 000000000..589638929 --- /dev/null +++ b/include/bitcoin/database/unspent/unspent.hpp @@ -0,0 +1,31 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_HPP +#define LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +#endif diff --git a/include/bitcoin/database/unspent/unspent_counter.hpp b/include/bitcoin/database/unspent/unspent_counter.hpp new file mode 100644 index 000000000..8e68c911f --- /dev/null +++ b/include/bitcoin/database/unspent/unspent_counter.hpp @@ -0,0 +1,65 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_COUNTER_HPP +#define LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_COUNTER_HPP + +#include +#include +#include +#include +#include + +namespace libbitcoin { +namespace database { + +/// Totals of a set. +template +class unspent_counter +{ +public: + unspent_counter(const Store& store, const stopper& cancel, + bool turbo) NOEXCEPT; + + /// The totals of the set. + code count(unspent_totals& out, const difference_set& set) const NOEXCEPT; + +private: + code span(unspent_totals& out, const difference_set& set, size_t begin, + size_t end) const NOEXCEPT; + code read_totals(unspent_totals& out, tx_link::integer& previous, + const output_links& puts) const NOEXCEPT; + + const Store& store_; + const stopper& cancel_; + const unspent_reader reader_; + const unspent_spans spans_; +}; + +} // namespace database +} // namespace libbitcoin + +#define TEMPLATE template +#define CLASS unspent_counter + +#include + +#undef CLASS +#undef TEMPLATE + +#endif diff --git a/include/bitcoin/database/unspent/unspent_matcher.hpp b/include/bitcoin/database/unspent/unspent_matcher.hpp new file mode 100644 index 000000000..e391cf462 --- /dev/null +++ b/include/bitcoin/database/unspent/unspent_matcher.hpp @@ -0,0 +1,64 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_MATCHER_HPP +#define LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_MATCHER_HPP + +#include +#include +#include +#include +#include + +namespace libbitcoin { +namespace database { + +/// Coins of a set with scripts hashing to a key set. +template +class unspent_matcher +{ +public: + using keys = std::unordered_set; + + unspent_matcher(const Store& store, const stopper& cancel, + bool turbo) NOEXCEPT; + + /// The matching coins and the size of the set. + code match(unspent_coins& out, size_t& txouts, const keys& keys, + const difference_set& set) const NOEXCEPT; + +private: + code span(unspent_totals& out, unspent_coins& matches, const keys& keys, + const difference_set& set, size_t begin, size_t end) const NOEXCEPT; + + const unspent_reader reader_; + const unspent_spans spans_; +}; + +} // namespace database +} // namespace libbitcoin + +#define TEMPLATE template +#define CLASS unspent_matcher + +#include + +#undef CLASS +#undef TEMPLATE + +#endif diff --git a/include/bitcoin/database/unspent/unspent_muhash.hpp b/include/bitcoin/database/unspent/unspent_muhash.hpp new file mode 100644 index 000000000..a83cd908b --- /dev/null +++ b/include/bitcoin/database/unspent/unspent_muhash.hpp @@ -0,0 +1,64 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_MUHASH_HPP +#define LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_MUHASH_HPP + +#include +#include +#include +#include + +namespace libbitcoin { +namespace database { + +/// The muhash commitment of a set (order invariant, partial sets combined). +template +class unspent_muhash +{ +public: + unspent_muhash(const Store& store, const stopper& cancel, + bool turbo) NOEXCEPT; + + /// The totals and muhash of the set. + code hash(unspent_totals& out, hash_digest& digest, + const difference_set& set) const NOEXCEPT; + +private: + code span(unspent_totals& out, system::muhash3072& partial, + const difference_set& set, size_t begin, size_t end) const NOEXCEPT; + code insert(unspent_totals& out, system::muhash3072& partial, + const unspent_coins& coins, const output_links& puts) const NOEXCEPT; + + const unspent_reader reader_; + const unspent_spans spans_; + const Store& store_; +}; + +} // namespace database +} // namespace libbitcoin + +#define TEMPLATE template +#define CLASS unspent_muhash + +#include + +#undef CLASS +#undef TEMPLATE + +#endif diff --git a/include/bitcoin/database/unspent/unspent_reader.hpp b/include/bitcoin/database/unspent/unspent_reader.hpp new file mode 100644 index 000000000..e7b64e4a5 --- /dev/null +++ b/include/bitcoin/database/unspent/unspent_reader.hpp @@ -0,0 +1,99 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_READER_HPP +#define LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_READER_HPP + +#include +#include +#include + +namespace libbitcoin { +namespace database { + +/// Set elements (outs table links). +using unspent_elements = std_vector; + +/// Materializes coins from set elements, each table in its own pass. +template +class unspent_reader +{ +public: + unspent_reader(const Store& store, const stopper& cancel) NOEXCEPT; + + /// flush(elements) for each batch of elements in the range. + template + code elements(const difference_set& set, size_t begin, size_t end, + const Flush& flush) const NOEXCEPT; + + /// handle(coin) for each coin (script copied) in the range, in tx order. + template + code batch(const difference_set& set, size_t begin, size_t end, + const Handler& handle) const NOEXCEPT; + + /// Coins (scripts not copied) and output links of the elements into out + /// and puts from offset, tx state carried. + code fill(unspent_coins& out, output_links& puts, size_t offset, + tx_link::integer& previous, const unspent_elements& elements, + size_t begin, size_t end) const NOEXCEPT; + + /// Values and scripts of the output links into out from offset. + code read_scripts(unspent_coins& out, size_t offset, + const output_links& puts, size_t count) const NOEXCEPT; + + /// Output links of the elements into out from offset. + code read_puts(output_links& out, size_t offset, + const unspent_elements& elements, size_t begin, + size_t end) const NOEXCEPT; + + /// Parent tx links of the output links from offset. + code read_parents(tx_links& out, const output_links& puts, size_t offset, + size_t count) const NOEXCEPT; + + /// Hashes of the tx links. + code read_hashes(system::hashes& out, + const tx_links& parents) const NOEXCEPT; + +private: + static constexpr auto batch_size = 4096_size; + + code read_transactions(unspent_coins& out, size_t offset, + tx_link::integer& previous, const tx_links& parents, + const unspent_elements& elements, size_t begin) const NOEXCEPT; + code read_blocks(header_links& out, const tx_links& parents) const NOEXCEPT; + code read_heights(std_vector& out, + const header_links& blocks) const NOEXCEPT; + code confirm(unspent_coins& out, size_t offset, const header_links& blocks, + const std_vector& heights) const NOEXCEPT; + + const Store& store_; + const stopper& cancel_; +}; + +} // namespace database +} // namespace libbitcoin + +#define TEMPLATE template +#define CLASS unspent_reader + +#include + +#undef CLASS +#undef TEMPLATE + +#endif diff --git a/include/bitcoin/database/unspent/unspent_scanner.hpp b/include/bitcoin/database/unspent/unspent_scanner.hpp new file mode 100644 index 000000000..7a563041f --- /dev/null +++ b/include/bitcoin/database/unspent/unspent_scanner.hpp @@ -0,0 +1,93 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_SCANNER_HPP +#define LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_SCANNER_HPP + +#include +#include +#include + +namespace libbitcoin { +namespace database { + +template +class query; + +/// Toggles the creates and spends of a branch into a set, the residue of +/// which is the unspent set (as outs table links). +template +class unspent_scanner +{ +public: + unspent_scanner(const query& query, const Store& store, + const stopper& cancel, bool turbo) NOEXCEPT; + + /// Scan the branch into the set, with the unspendable and bip30 totals. + code scan(difference_set& set, unspent_totals& out, + const header_links& branch) const NOEXCEPT; + +private: + using gets = std_vector; + + static constexpr auto per_thread = 32_size; + + struct buffers + { + gets txs{}; + output_links exclusions{}; + system::chain::points spends{}; + }; + + code bounds(std_vector& out, + const header_links& branch) const NOEXCEPT; + code span(difference_set& set, uint64_t& unspendable, + uint64_t& overwritten, const header_links& branch, size_t begin, + size_t end) const NOEXCEPT; + code block(difference_set& set, uint64_t& unspendable, + uint64_t& overwritten, buffers& buffer, + const header_link& link) const NOEXCEPT; + code overwritten(uint64_t& out, const header_link& link) const NOEXCEPT; + code read_transactions(gets& out, const header_link& link) const NOEXCEPT; + code toggle_creates(difference_set& set, output_links& exclusions, + const gets& txs) const NOEXCEPT; + code read_exclusions(uint64_t& out, + const output_links& exclusions) const NOEXCEPT; + code read_points(system::chain::points& out, + const gets& txs) const NOEXCEPT; + code toggle_spends(difference_set& set, + const system::chain::points& spends) const NOEXCEPT; + + const query& query_; + const Store& store_; + const stopper& cancel_; + const bool turbo_; +}; + +} // namespace database +} // namespace libbitcoin + +#define TEMPLATE template +#define CLASS unspent_scanner + +#include + +#undef CLASS +#undef TEMPLATE + +#endif diff --git a/include/bitcoin/database/unspent/unspent_serial.hpp b/include/bitcoin/database/unspent/unspent_serial.hpp new file mode 100644 index 000000000..f8a8adba5 --- /dev/null +++ b/include/bitcoin/database/unspent/unspent_serial.hpp @@ -0,0 +1,83 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_SERIAL_HPP +#define LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_SERIAL_HPP + +#include +#include +#include +#include +#include + +namespace libbitcoin { +namespace database { + +/// The serialized commitment of a set, a stream in canonical (txid, index) +/// order, folded by bucket of leading txid byte (canonical across buckets). +template +class unspent_serial +{ +public: + unspent_serial(const Store& store, const stopper& cancel, + bool turbo) NOEXCEPT; + + /// The totals and serialized hash of the set. + code hash(unspent_totals& out, hash_digest& digest, + const difference_set& set) const NOEXCEPT; + +private: + using sizes = std_vector; + static constexpr auto buckets = 256_size; + + code partition(unspent_elements& out, sizes& offsets, + const difference_set& set) const NOEXCEPT; + template + code walk(const difference_set& set, size_t begin, size_t end, + const Emit& emit) const NOEXCEPT; + code slots(sizes& out, const unspent_elements& elements) const NOEXCEPT; + static void starts(sizes& counts, sizes& offsets, size_t chunks) NOEXCEPT; + code fill(unspent_coins& out, output_links& puts, + const unspent_elements& elements, size_t begin, + size_t end) const NOEXCEPT; + void order(sizes& out, const unspent_coins& coins) const NOEXCEPT; + void gather(unspent_coins& out, output_links& links, unspent_coins& coins, + const output_links& puts, const sizes& order) const NOEXCEPT; + code fold(unspent_totals& out, system::writer& sink, + hash_digest& previous, unspent_coins& coins, + const output_links& puts) const NOEXCEPT; + + const unspent_reader reader_; + const unspent_spans spans_; + const Store& store_; + const stopper& cancel_; + const bool turbo_; +}; + +} // namespace database +} // namespace libbitcoin + +#define TEMPLATE template +#define CLASS unspent_serial + +#include + +#undef CLASS +#undef TEMPLATE + +#endif diff --git a/include/bitcoin/database/unspent/unspent_spans.hpp b/include/bitcoin/database/unspent/unspent_spans.hpp new file mode 100644 index 000000000..8b4cea539 --- /dev/null +++ b/include/bitcoin/database/unspent/unspent_spans.hpp @@ -0,0 +1,66 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_SPANS_HPP +#define LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_SPANS_HPP + +#include +#include + +namespace libbitcoin { +namespace database { + +/// Parallel iteration of a set in tx-aligned element ranges. +template +class unspent_spans +{ +public: + unspent_spans(const Store& store, const stopper& cancel, + bool turbo) NOEXCEPT; + + /// The span index bound. + size_t count() const NOEXCEPT; + + /// span(index, begin, end) for each element range, in parallel. + template + code for_each(const difference_set& set, const Span& span) const NOEXCEPT; + +private: + static constexpr auto per_thread = 32_size; + + bool to_tx_base(size_t& out, size_t element) const NOEXCEPT; + code bounds(std_vector& out, + const difference_set& set) const NOEXCEPT; + + const Store& store_; + const stopper& cancel_; + const bool turbo_; +}; + +} // namespace database +} // namespace libbitcoin + +#define TEMPLATE template +#define CLASS unspent_spans + +#include + +#undef CLASS +#undef TEMPLATE + +#endif diff --git a/include/bitcoin/database/unspent/unspent_writer.hpp b/include/bitcoin/database/unspent/unspent_writer.hpp new file mode 100644 index 000000000..c83b1f09f --- /dev/null +++ b/include/bitcoin/database/unspent/unspent_writer.hpp @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_WRITER_HPP +#define LIBBITCOIN_DATABASE_UNSPENT_UNSPENT_WRITER_HPP + +#include +#include + +namespace libbitcoin { +namespace database { + +/// The unspent output commitment element (bitcoind coin serialization). +struct unspent_writer +{ + static constexpr auto point_size = system::hash_size + sizeof(uint32_t); + static constexpr auto code_size = sizeof(uint32_t); + static constexpr auto value_size = sizeof(uint64_t); + static constexpr auto fixed_size = point_size + code_size + value_size; + + /// Write the element head (txid, index, height code) of the coin to the + /// sink, the tail (value, script) is streamed by output::wire_script. + static void write(system::writer& sink, const unspent_coin& coin) NOEXCEPT; + + /// The element size given the script size. + static size_t size(size_t script) NOEXCEPT; + + /// Add an output to the totals. + static void add(unspent_totals& out, bool first, uint64_t value, + size_t script) NOEXCEPT; + + /// Add the coin (script copied) to the totals. + static void add(unspent_totals& out, const unspent_coin& coin) NOEXCEPT; + + /// Add the totals to the totals. + static void add(unspent_totals& out, const unspent_totals& totals) NOEXCEPT; +}; + +} // namespace database +} // namespace libbitcoin + +#include + +#endif diff --git a/src/define.cpp b/src/define.cpp index d1b6efd41..820d2076c 100644 --- a/src/define.cpp +++ b/src/define.cpp @@ -1,33 +1,34 @@ -/** - * Copyright (c) 2011-2026 libbitcoin developers - * - * This file is part of libbitcoin. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include - -// version : -// boost : version -// error : boost -// define : error -// settings : define -// /locks : define -// /file : define -// /memory : /file settings -// /primitives : /memory -// /tables : /primitives -// /types : /tables -// store : /types settings /locks -// query : /types settings +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include + +// version : +// boost : version +// error : boost +// define : error +// settings : define +// /locks : define +// /file : define +// /memory : /file settings +// /primitives : /memory +// /tables : /primitives +// /types : /tables +// /unspent : /types +// store : /types settings /locks +// query : /unspent settings diff --git a/src/error.cpp b/src/error.cpp index cd1c40c8c..198cf5384 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -134,6 +134,7 @@ DEFINE_ERROR_T_MESSAGE_MAP(error) { depth_limited, "depth_limited" }, { invalid_cursor, "invalid_cursor" }, { query_canceled, "query_canceled" }, + { branch_inactive, "branch_inactive" }, { invalid_argument, "invalid_argument" }, { missing_prevouts, "missing_prevouts" }, { merkle_proof, "merkle_proof" }, diff --git a/src/memory/utilities.cpp b/src/memory/utilities.cpp index 6cdeae1f4..78d294abc 100644 --- a/src/memory/utilities.cpp +++ b/src/memory/utilities.cpp @@ -248,38 +248,9 @@ uint64_t system_compressed() NOEXCEPT #endif // HAVE_MSC -constexpr auto gigabyte = power2(30u); -constexpr auto megabyte = power2(20u); -constexpr auto uncontested = power2(35u); - -#if defined(HAVE_APPLE) -constexpr auto contested = 10u * gigabyte; -#else -constexpr auto contested = 8u * gigabyte; -#endif - -uint32_t derive_buckets(uint64_t expected, uint32_t low, - uint32_t high) NOEXCEPT +size_t cores() NOEXCEPT { - if (is_zero(expected) || is_zero(low) || is_zero(high)) - return {}; - - const auto scaled = ceilinged_multiply(expected, 10); - const auto floored = scaled / low; - const auto ceiled = scaled / high; - const auto memory = system_memory(); - - if (memory <= contested) - return possible_narrow_cast(floored); - - if (memory >= uncontested) - return possible_narrow_cast(ceiled); - - // Megabytes, as the byte product overflows the word. - const auto over = (memory - contested) / megabyte; - const auto rise = floored_subtract(ceiled, floored); - constexpr auto span = (uncontested - contested) / megabyte; - return limit(floored + (ceilinged_multiply(rise, over) / span)); + return std::max(std::thread::hardware_concurrency(), 1_u32); } } // namespace database diff --git a/src/types/envelope.cpp b/src/tables/envelope.cpp similarity index 99% rename from src/types/envelope.cpp rename to src/tables/envelope.cpp index fdd707588..bbb3c6b23 100644 --- a/src/types/envelope.cpp +++ b/src/tables/envelope.cpp @@ -16,7 +16,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include +#include #include #include diff --git a/src/types/unspent.cpp b/src/types/unspent_output.cpp similarity index 80% rename from src/types/unspent.cpp rename to src/types/unspent_output.cpp index 06ceb75c9..78e7a31e6 100644 --- a/src/types/unspent.cpp +++ b/src/types/unspent_output.cpp @@ -16,7 +16,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include +#include #include #include @@ -26,7 +26,7 @@ namespace database { // local // zero is unconfirmed height, but cannot be a sentinel for unconfirmed. -bool less_than(const unspent& a, const unspent& b) NOEXCEPT +bool less_than(const unspent_output& a, const unspent_output& b) NOEXCEPT { const auto a_point = a.out.point(); const auto b_point = b.out.point(); @@ -55,36 +55,36 @@ bool less_than(const unspent& a, const unspent& b) NOEXCEPT return a_point < b_point; } -bool unspent::valid() const NOEXCEPT +bool unspent_output::valid() const NOEXCEPT { return out.is_valid(); } -bool unspent::fault() const NOEXCEPT +bool unspent_output::fault() const NOEXCEPT { // Invalid with default position implies fault (return {}). return !valid() && is_zero(position); } -bool unspent::confirmed() const NOEXCEPT +bool unspent_output::confirmed() const NOEXCEPT { return position != unconfirmed_position; } -bool unspent::operator<(const unspent& other) const NOEXCEPT +bool unspent_output::operator<(const unspent_output& other) const NOEXCEPT { return less_than(*this, other); } -bool unspent::operator==(const unspent& other) const NOEXCEPT +bool unspent_output::operator==(const unspent_output& other) const NOEXCEPT { return !(*this < other) && !(other < *this); } -void unspent::filter_sort_and_dedup(std::vector& out) NOEXCEPT +void unspent_output::filter_sort_and_dedup(std::vector& out) NOEXCEPT { const auto excluded = std::remove_if(out.begin(), out.end(), - [](const unspent& element) NOEXCEPT + [](const unspent_output& element) NOEXCEPT { return !element.valid(); }); diff --git a/test/memory/release.cpp b/test/memory/release.cpp new file mode 100644 index 000000000..af53a8ec1 --- /dev/null +++ b/test/memory/release.cpp @@ -0,0 +1,174 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "../test.hpp" + +BOOST_AUTO_TEST_SUITE(memory_release_tests) + +// release_candidates + +BOOST_AUTO_TEST_CASE(memory_release__release_candidates__none__all) +{ + BOOST_REQUIRE_EQUAL(release_candidates(0, 0, 0), max_uint64); +} + +BOOST_AUTO_TEST_CASE(memory_release__release_candidates__each__excluded) +{ + BOOST_REQUIRE_EQUAL(release_candidates(0b0001, 0b0010, 0b0100), + system::bit_not(0b0111)); +} + +BOOST_AUTO_TEST_CASE(memory_release__release_candidates__all__none) +{ + BOOST_REQUIRE_EQUAL(release_candidates(max_uint64, 0, 0), zero); + BOOST_REQUIRE_EQUAL(release_candidates(0, max_uint64, 0), zero); + BOOST_REQUIRE_EQUAL(release_candidates(0, 0, max_uint64), zero); +} + +// release_below +// The boundary word masking defect class: candidacy must retain LOW bits +// (pages below the count), never the high complement. + +BOOST_AUTO_TEST_CASE(memory_release__release_below__word_above_count__zero) +{ + BOOST_REQUIRE_EQUAL(release_below(max_uint64, 64, 64), zero); + BOOST_REQUIRE_EQUAL(release_below(max_uint64, 10, 64), zero); +} + +BOOST_AUTO_TEST_CASE(memory_release__release_below__word_below_count__all) +{ + BOOST_REQUIRE_EQUAL(release_below(max_uint64, 64, 0), max_uint64); + BOOST_REQUIRE_EQUAL(release_below(max_uint64, 128, 64), max_uint64); +} + +BOOST_AUTO_TEST_CASE(memory_release__release_below__boundary__low_bits) +{ + // One full page: only bit zero (the historical inversion released 1..63). + BOOST_REQUIRE_EQUAL(release_below(max_uint64, 1, 0), 0b0001u); + BOOST_REQUIRE_EQUAL(release_below(max_uint64, 3, 0), 0b0111u); + + // Sixty-five pages: boundary word (first=64) retains only bit zero. + BOOST_REQUIRE_EQUAL(release_below(max_uint64, 65, 64), 0b0001u); +} + +// page_mask + +BOOST_AUTO_TEST_CASE(memory_release__page_mask__disjoint__zero) +{ + BOOST_REQUIRE_EQUAL(page_mask(0, 10, 64), zero); + BOOST_REQUIRE_EQUAL(page_mask(128, 130, 64), zero); + BOOST_REQUIRE_EQUAL(page_mask(10, 10, 0), zero); +} + +BOOST_AUTO_TEST_CASE(memory_release__page_mask__contained__expected) +{ + BOOST_REQUIRE_EQUAL(page_mask(0, 1, 0), 0b0001u); + BOOST_REQUIRE_EQUAL(page_mask(1, 3, 0), 0b0110u); + BOOST_REQUIRE_EQUAL(page_mask(65, 67, 64), 0b0110u); +} + +BOOST_AUTO_TEST_CASE(memory_release__page_mask__spanning__clamped) +{ + // Run [60, 70) in word zero: bits 60..63; in word one: bits 0..5. + BOOST_REQUIRE_EQUAL(page_mask(60, 70, 0), + system::bit_and(max_uint64, system::mask_right(60))); + BOOST_REQUIRE_EQUAL(page_mask(60, 70, 64), system::unmask_right(6)); +} + +BOOST_AUTO_TEST_CASE(memory_release__page_mask__full_word__all) +{ + BOOST_REQUIRE_EQUAL(page_mask(0, 128, 64), max_uint64); +} + +// next_run + +BOOST_AUTO_TEST_CASE(memory_release__next_run__empty__none) +{ + const uint64_t words[]{ 0, 0 }; + const auto run = next_run(words, 128, 0); + BOOST_REQUIRE_EQUAL(run.first, run.second); +} + +BOOST_AUTO_TEST_CASE(memory_release__next_run__single_page__found) +{ + const uint64_t words[]{ 0b01000, 0 }; + const auto run = next_run(words, 128, 0); + BOOST_REQUIRE_EQUAL(run.first, 3u); + BOOST_REQUIRE_EQUAL(run.second, 4u); +} + +BOOST_AUTO_TEST_CASE(memory_release__next_run__word_spanning__joined) +{ + // Bits 62..63 of word zero and 0..2 of word one: run [62, 67). + const uint64_t words[]{ system::mask_right(62), 0b0111 }; + const auto run = next_run(words, 128, 0); + BOOST_REQUIRE_EQUAL(run.first, 62u); + BOOST_REQUIRE_EQUAL(run.second, 67u); +} + +BOOST_AUTO_TEST_CASE(memory_release__next_run__from_skips__second) +{ + const uint64_t words[]{ 0b0011, 0b0011 }; + const auto run = next_run(words, 128, 2); + BOOST_REQUIRE_EQUAL(run.first, 64u); + BOOST_REQUIRE_EQUAL(run.second, 66u); +} + +BOOST_AUTO_TEST_CASE(memory_release__next_run__page_bound__clipped) +{ + // All set but only 70 pages exist: run [0, 70). + const uint64_t words[]{ max_uint64, max_uint64 }; + const auto run = next_run(words, 70, 0); + BOOST_REQUIRE_EQUAL(run.first, zero); + BOOST_REQUIRE_EQUAL(run.second, 70u); +} + +// bit_run + +BOOST_AUTO_TEST_CASE(memory_release__bit_run__unset_page__empty) +{ + const uint64_t words[]{ 0b0110, 0 }; + const auto run = bit_run(words, 128, 3); + BOOST_REQUIRE_EQUAL(run.first, run.second); +} + +BOOST_AUTO_TEST_CASE(memory_release__bit_run__interior_page__extent) +{ + const uint64_t words[]{ 0b111100, 0 }; + const auto run = bit_run(words, 128, 3); + BOOST_REQUIRE_EQUAL(run.first, 2u); + BOOST_REQUIRE_EQUAL(run.second, 6u); +} + +BOOST_AUTO_TEST_CASE(memory_release__bit_run__word_spanning__extent) +{ + // Bits 63 of word zero and 0..1 of word one: run [63, 66) from page 64. + const uint64_t words[]{ system::bit_left(0), 0b0011 }; + const auto run = bit_run(words, 128, 64); + BOOST_REQUIRE_EQUAL(run.first, 63u); + BOOST_REQUIRE_EQUAL(run.second, 66u); +} + +BOOST_AUTO_TEST_CASE(memory_release__bit_run__beyond_pages__empty) +{ + const uint64_t words[]{ max_uint64, max_uint64 }; + const auto run = bit_run(words, 64, 100); + BOOST_REQUIRE_EQUAL(run.first, run.second); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/test/memory/utilities.cpp b/test/memory/utilities.cpp index f532a1863..66e329920 100644 --- a/test/memory/utilities.cpp +++ b/test/memory/utilities.cpp @@ -35,155 +35,4 @@ BOOST_AUTO_TEST_CASE(memory_utilities__system_memory__always__nonzero) BOOST_REQUIRE(is_nonzero(system_memory())); } -// release_candidates - -BOOST_AUTO_TEST_CASE(memory_utilities__release_candidates__none__all) -{ - BOOST_REQUIRE_EQUAL(release_candidates(0, 0, 0), max_uint64); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__release_candidates__each__excluded) -{ - BOOST_REQUIRE_EQUAL(release_candidates(0b0001, 0b0010, 0b0100), - system::bit_not(0b0111)); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__release_candidates__all__none) -{ - BOOST_REQUIRE_EQUAL(release_candidates(max_uint64, 0, 0), zero); - BOOST_REQUIRE_EQUAL(release_candidates(0, max_uint64, 0), zero); - BOOST_REQUIRE_EQUAL(release_candidates(0, 0, max_uint64), zero); -} - -// release_below -// The boundary word masking defect class: candidacy must retain LOW bits -// (pages below the count), never the high complement. - -BOOST_AUTO_TEST_CASE(memory_utilities__release_below__word_above_count__zero) -{ - BOOST_REQUIRE_EQUAL(release_below(max_uint64, 64, 64), zero); - BOOST_REQUIRE_EQUAL(release_below(max_uint64, 10, 64), zero); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__release_below__word_below_count__all) -{ - BOOST_REQUIRE_EQUAL(release_below(max_uint64, 64, 0), max_uint64); - BOOST_REQUIRE_EQUAL(release_below(max_uint64, 128, 64), max_uint64); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__release_below__boundary__low_bits) -{ - // One full page: only bit zero (the historical inversion released 1..63). - BOOST_REQUIRE_EQUAL(release_below(max_uint64, 1, 0), 0b0001u); - BOOST_REQUIRE_EQUAL(release_below(max_uint64, 3, 0), 0b0111u); - - // Sixty-five pages: boundary word (first=64) retains only bit zero. - BOOST_REQUIRE_EQUAL(release_below(max_uint64, 65, 64), 0b0001u); -} - -// page_mask - -BOOST_AUTO_TEST_CASE(memory_utilities__page_mask__disjoint__zero) -{ - BOOST_REQUIRE_EQUAL(page_mask(0, 10, 64), zero); - BOOST_REQUIRE_EQUAL(page_mask(128, 130, 64), zero); - BOOST_REQUIRE_EQUAL(page_mask(10, 10, 0), zero); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__page_mask__contained__expected) -{ - BOOST_REQUIRE_EQUAL(page_mask(0, 1, 0), 0b0001u); - BOOST_REQUIRE_EQUAL(page_mask(1, 3, 0), 0b0110u); - BOOST_REQUIRE_EQUAL(page_mask(65, 67, 64), 0b0110u); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__page_mask__spanning__clamped) -{ - // Run [60, 70) in word zero: bits 60..63; in word one: bits 0..5. - BOOST_REQUIRE_EQUAL(page_mask(60, 70, 0), - system::bit_and(max_uint64, system::mask_right(60))); - BOOST_REQUIRE_EQUAL(page_mask(60, 70, 64), system::unmask_right(6)); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__page_mask__full_word__all) -{ - BOOST_REQUIRE_EQUAL(page_mask(0, 128, 64), max_uint64); -} - -// next_run - -BOOST_AUTO_TEST_CASE(memory_utilities__next_run__empty__none) -{ - const uint64_t words[]{ 0, 0 }; - const auto run = next_run(words, 128, 0); - BOOST_REQUIRE_EQUAL(run.first, run.second); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__next_run__single_page__found) -{ - const uint64_t words[]{ 0b01000, 0 }; - const auto run = next_run(words, 128, 0); - BOOST_REQUIRE_EQUAL(run.first, 3u); - BOOST_REQUIRE_EQUAL(run.second, 4u); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__next_run__word_spanning__joined) -{ - // Bits 62..63 of word zero and 0..2 of word one: run [62, 67). - const uint64_t words[]{ system::mask_right(62), 0b0111 }; - const auto run = next_run(words, 128, 0); - BOOST_REQUIRE_EQUAL(run.first, 62u); - BOOST_REQUIRE_EQUAL(run.second, 67u); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__next_run__from_skips__second) -{ - const uint64_t words[]{ 0b0011, 0b0011 }; - const auto run = next_run(words, 128, 2); - BOOST_REQUIRE_EQUAL(run.first, 64u); - BOOST_REQUIRE_EQUAL(run.second, 66u); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__next_run__page_bound__clipped) -{ - // All set but only 70 pages exist: run [0, 70). - const uint64_t words[]{ max_uint64, max_uint64 }; - const auto run = next_run(words, 70, 0); - BOOST_REQUIRE_EQUAL(run.first, zero); - BOOST_REQUIRE_EQUAL(run.second, 70u); -} - -// bit_run - -BOOST_AUTO_TEST_CASE(memory_utilities__bit_run__unset_page__empty) -{ - const uint64_t words[]{ 0b0110, 0 }; - const auto run = bit_run(words, 128, 3); - BOOST_REQUIRE_EQUAL(run.first, run.second); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__bit_run__interior_page__extent) -{ - const uint64_t words[]{ 0b111100, 0 }; - const auto run = bit_run(words, 128, 3); - BOOST_REQUIRE_EQUAL(run.first, 2u); - BOOST_REQUIRE_EQUAL(run.second, 6u); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__bit_run__word_spanning__extent) -{ - // Bits 63 of word zero and 0..1 of word one: run [63, 66) from page 64. - const uint64_t words[]{ system::bit_left(0), 0b0011 }; - const auto run = bit_run(words, 128, 64); - BOOST_REQUIRE_EQUAL(run.first, 63u); - BOOST_REQUIRE_EQUAL(run.second, 66u); -} - -BOOST_AUTO_TEST_CASE(memory_utilities__bit_run__beyond_pages__empty) -{ - const uint64_t words[]{ max_uint64, max_uint64 }; - const auto run = bit_run(words, 64, 100); - BOOST_REQUIRE_EQUAL(run.first, run.second); -} - BOOST_AUTO_TEST_SUITE_END() diff --git a/test/mocks/blocks.cpp b/test/mocks/blocks.cpp index 8b63f3e2d..dc1c35d18 100644 --- a/test/mocks/blocks.cpp +++ b/test/mocks/blocks.cpp @@ -517,6 +517,51 @@ const block block1b } } }; + +const block block1c +{ + header + { + 0x31323334, // version + block0_hash, // previous_block_hash + hash_digest{ 0x1c },// merkle_root + 0x41424344, // timestamp + 0x51525354, // bits + 0x61626364 // nonce + }, + transactions + { + // This first transaction is a coinbase with two outputs. + transaction // tx#1 + { + 0xc1, + inputs + { + input + { + point{}, + script{ { { opcode::push_positive_1 } } }, + witness{}, + 0xc1 + } + }, + outputs + { + output + { + 0x18, + script{ { { opcode::pick } } } + }, + output + { + 0x2a, + script{ { { opcode::roll } } } + } + }, + 0xc1 + } + } +}; const block block2b { header diff --git a/test/mocks/blocks.hpp b/test/mocks/blocks.hpp index de68e0912..e1e6aca2f 100644 --- a/test/mocks/blocks.hpp +++ b/test/mocks/blocks.hpp @@ -98,6 +98,7 @@ extern const system::chain::block block3a; extern const system::chain::transaction tx4; extern const system::chain::transaction tx5; extern const system::chain::block block1b; +extern const system::chain::block block1c; extern const system::chain::block block2b; extern const system::chain::transaction tx2b; diff --git a/test/query/address/address_unspent.cpp b/test/query/address/address_unspent.cpp index d91688bb0..3f853bae4 100644 --- a/test/query/address/address_unspent.cpp +++ b/test/query/address/address_unspent.cpp @@ -35,7 +35,7 @@ BOOST_AUTO_TEST_CASE(query_address__get_unspent__genesis__expected) BOOST_REQUIRE(!store.create(test::events_handler)); BOOST_REQUIRE(test::setup_three_block_confirmed_address_store(query)); - unspents out{}; + unspent_outputs out{}; const std::atomic_bool cancel{}; BOOST_REQUIRE(!query.get_unconfirmed_unspent(cancel, out, test::genesis_address0)); BOOST_REQUIRE_EQUAL(out.size(), 0u); @@ -68,7 +68,7 @@ BOOST_AUTO_TEST_CASE(query_address__get_unspent__turbo_genesis__expected) BOOST_REQUIRE(!store.create(test::events_handler)); BOOST_REQUIRE(test::setup_three_block_confirmed_address_store(query)); - unspents out{}; + unspent_outputs out{}; const std::atomic_bool cancel{}; BOOST_REQUIRE(!query.get_unconfirmed_unspent(cancel, out, test::block1a_address0, true)); @@ -77,20 +77,20 @@ BOOST_AUTO_TEST_CASE(query_address__get_unspent__turbo_genesis__expected) // block2b (unconfirmed) contains 1 address output, so 3 total address outputs. BOOST_CHECK_EQUAL(out.size(), 3u); - BOOST_CHECK_EQUAL(out.at(0).height, unspent::unused_height); - BOOST_CHECK_EQUAL(out.at(0).position, unspent::unconfirmed_position); + BOOST_CHECK_EQUAL(out.at(0).height, unspent_output::unused_height); + BOOST_CHECK_EQUAL(out.at(0).position, unspent_output::unconfirmed_position); BOOST_CHECK_EQUAL(out.at(0).out.value(), 0x08u); BOOST_CHECK_EQUAL(out.at(0).out.point().index(), 0u); BOOST_CHECK_EQUAL(out.at(0).out.point().hash(), test::tx4.hash(false)); - BOOST_CHECK_EQUAL(out.at(1).height, unspent::unused_height); - BOOST_CHECK_EQUAL(out.at(1).position, unspent::unconfirmed_position); + BOOST_CHECK_EQUAL(out.at(1).height, unspent_output::unused_height); + BOOST_CHECK_EQUAL(out.at(1).position, unspent_output::unconfirmed_position); BOOST_CHECK_EQUAL(out.at(1).out.value(), 0x85u); BOOST_CHECK_EQUAL(out.at(1).out.point().index(), 0u); BOOST_CHECK_EQUAL(out.at(1).out.point().hash(), test::tx5.hash(false)); - BOOST_CHECK_EQUAL(out.at(2).height, unspent::unused_height); - BOOST_CHECK_EQUAL(out.at(2).position, unspent::unconfirmed_position); + BOOST_CHECK_EQUAL(out.at(2).height, unspent_output::unused_height); + BOOST_CHECK_EQUAL(out.at(2).position, unspent_output::unconfirmed_position); BOOST_CHECK_EQUAL(out.at(2).out.value(), 0xb2u); BOOST_CHECK_EQUAL(out.at(2).out.point().index(), 0u); BOOST_CHECK_EQUAL(out.at(2).out.point().hash(), test::block2b.transactions_ptr()->at(0)->hash(false)); @@ -152,20 +152,20 @@ BOOST_AUTO_TEST_CASE(query_address__get_unspent__turbo_genesis__expected) // same as get_confirmed_unspent() - BOOST_CHECK_EQUAL(out.at(3).height, unspent::unused_height); - BOOST_CHECK_EQUAL(out.at(3).position, unspent::unconfirmed_position); + BOOST_CHECK_EQUAL(out.at(3).height, unspent_output::unused_height); + BOOST_CHECK_EQUAL(out.at(3).position, unspent_output::unconfirmed_position); BOOST_CHECK_EQUAL(out.at(3).out.value(), 0x08u); BOOST_CHECK_EQUAL(out.at(3).out.point().index(), 0u); BOOST_CHECK_EQUAL(out.at(3).out.point().hash(), test::tx4.hash(false)); - BOOST_CHECK_EQUAL(out.at(4).height, unspent::unused_height); - BOOST_CHECK_EQUAL(out.at(4).position, unspent::unconfirmed_position); + BOOST_CHECK_EQUAL(out.at(4).height, unspent_output::unused_height); + BOOST_CHECK_EQUAL(out.at(4).position, unspent_output::unconfirmed_position); BOOST_CHECK_EQUAL(out.at(4).out.value(), 0x85u); BOOST_CHECK_EQUAL(out.at(4).out.point().index(), 0u); BOOST_CHECK_EQUAL(out.at(4).out.point().hash(), test::tx5.hash(false)); - BOOST_CHECK_EQUAL(out.at(5).height, unspent::unused_height); - BOOST_CHECK_EQUAL(out.at(5).position, unspent::unconfirmed_position); + BOOST_CHECK_EQUAL(out.at(5).height, unspent_output::unused_height); + BOOST_CHECK_EQUAL(out.at(5).position, unspent_output::unconfirmed_position); BOOST_CHECK_EQUAL(out.at(5).out.value(), 0xb2u); BOOST_CHECK_EQUAL(out.at(5).out.point().index(), 0u); BOOST_CHECK_EQUAL(out.at(5).out.point().hash(), test::block2b.transactions_ptr()->at(0)->hash(false)); diff --git a/test/query/unspent.cpp b/test/query/unspent.cpp index ef4191928..d9ed43a26 100644 --- a/test/query/unspent.cpp +++ b/test/query/unspent.cpp @@ -49,7 +49,11 @@ BOOST_AUTO_TEST_CASE(query_unspent__get_unspent_totals__two_block_branch__expect BOOST_REQUIRE(!store.create(test::events_handler)); BOOST_REQUIRE(query.initialize(test::genesis)); BOOST_REQUIRE(query.set(test::block1, context{ 0, 1, 0 }, false, false)); + BOOST_REQUIRE(query.set_strong(1)); + BOOST_REQUIRE(query.push_confirmed(1, false)); BOOST_REQUIRE(query.set(test::block2, context{ 0, 2, 0 }, false, false)); + BOOST_REQUIRE(query.set_strong(2)); + BOOST_REQUIRE(query.push_confirmed(2, false)); // Each block is one coinbase with one unspent p2pk output (67 bytes). const stopper cancel{}; @@ -62,6 +66,188 @@ BOOST_AUTO_TEST_CASE(query_unspent__get_unspent_totals__two_block_branch__expect BOOST_REQUIRE_EQUAL(totals.value, 10'000'000'000u); } +// The commitment element (bitcoind coin serialization) of a confirmed output. +static system::data_chunk output_element(const system::chain::transaction& tx, uint32_t index, size_t height) +{ + const auto& output = *tx.outputs_ptr()->at(index); + const auto script = output.script().to_data(false); + const auto txid = tx.hash(false); + const auto coded = system::possible_narrow_cast((height << 1) | 1u); + const auto code = system::to_little_endian(coded); + const auto value = system::to_little_endian(output.value()); + const auto size = system::to_array(system::possible_narrow_cast(script.size())); + return system::build_chunk({ txid, system::to_little_endian(index), code, value, size, script }); +} + +static system::data_chunk coin_element(const system::chain::block& block, size_t height) +{ + return output_element(*block.transactions_ptr()->front(), 0, height); +} + +BOOST_AUTO_TEST_CASE(query_unspent__get_unspent_muhash__two_block_branch__expected) +{ + settings settings{}; + settings.path = TEST_DIRECTORY; + test::chunk_store store{ settings }; + test::query_accessor query{ store }; + BOOST_REQUIRE(!store.create(test::events_handler)); + BOOST_REQUIRE(query.initialize(test::genesis)); + BOOST_REQUIRE(query.set(test::block1, context{ 0, 1, 0 }, false, false)); + BOOST_REQUIRE(query.set_strong(1)); + BOOST_REQUIRE(query.push_confirmed(1, false)); + BOOST_REQUIRE(query.set(test::block2, context{ 0, 2, 0 }, false, false)); + BOOST_REQUIRE(query.set_strong(2)); + BOOST_REQUIRE(query.push_confirmed(2, false)); + + system::muhash3072 expected{}; + expected.insert(coin_element(test::block1, 1)); + expected.insert(coin_element(test::block2, 2)); + + const stopper cancel{}; + hash_digest digest{}; + unspent_totals totals{}; + const header_links branch{ header_link{ 2 }, header_link{ 1 } }; + BOOST_REQUIRE(!query.get_unspent_muhash(cancel, totals, digest, branch)); + BOOST_REQUIRE_EQUAL(digest, expected.flush()); + BOOST_REQUIRE_EQUAL(totals.outputs, 2u); + BOOST_REQUIRE_EQUAL(totals.transactions, 2u); + BOOST_REQUIRE_EQUAL(totals.script_bytes, 134u); + BOOST_REQUIRE_EQUAL(totals.coin_bytes, 2u * (48u + 1u + 67u)); + BOOST_REQUIRE_EQUAL(totals.value, 10'000'000'000u); +} + +BOOST_AUTO_TEST_CASE(query_unspent__get_unspent_serialized__two_block_branch__expected) +{ + settings settings{}; + settings.path = TEST_DIRECTORY; + test::chunk_store store{ settings }; + test::query_accessor query{ store }; + BOOST_REQUIRE(!store.create(test::events_handler)); + BOOST_REQUIRE(query.initialize(test::genesis)); + BOOST_REQUIRE(query.set(test::block1, context{ 0, 1, 0 }, false, false)); + BOOST_REQUIRE(query.set_strong(1)); + BOOST_REQUIRE(query.push_confirmed(1, false)); + BOOST_REQUIRE(query.set(test::block2, context{ 0, 2, 0 }, false, false)); + BOOST_REQUIRE(query.set_strong(2)); + BOOST_REQUIRE(query.push_confirmed(2, false)); + + // Canonical order is by txid, the stream is double sha256 hashed. + const auto first = coin_element(test::block1, 1); + const auto second = coin_element(test::block2, 2); + const auto& tx1 = *test::block1.transactions_ptr()->front(); + const auto& tx2 = *test::block2.transactions_ptr()->front(); + const auto ordered = tx1.hash(false) < tx2.hash(false); + const auto forward = system::build_chunk({ first, second }); + const auto reverse = system::build_chunk({ second, first }); + const auto expected = system::bitcoin_hash(ordered ? forward : reverse); + + const stopper cancel{}; + hash_digest digest{}; + unspent_totals totals{}; + const header_links branch{ header_link{ 2 }, header_link{ 1 } }; + BOOST_REQUIRE(!query.get_unspent_serialized(cancel, totals, digest, branch)); + BOOST_REQUIRE_EQUAL(digest, expected); + BOOST_REQUIRE_EQUAL(totals.outputs, 2u); + BOOST_REQUIRE_EQUAL(totals.transactions, 2u); + BOOST_REQUIRE_EQUAL(totals.script_bytes, 134u); + BOOST_REQUIRE_EQUAL(totals.coin_bytes, 2u * (48u + 1u + 67u)); + BOOST_REQUIRE_EQUAL(totals.value, 10'000'000'000u); +} + +BOOST_AUTO_TEST_CASE(query_unspent__get_unspent_serialized__two_output_coinbase__one_transaction) +{ + settings settings{}; + settings.path = TEST_DIRECTORY; + test::chunk_store store{ settings }; + test::query_accessor query{ store }; + BOOST_REQUIRE(!store.create(test::events_handler)); + BOOST_REQUIRE(query.initialize(test::genesis)); + BOOST_REQUIRE(query.set(test::block1c, context{ 0, 1, 0 }, false, false)); + BOOST_REQUIRE(query.set_strong(1)); + BOOST_REQUIRE(query.push_confirmed(1, false)); + + // Both outputs of the one coinbase are unspent, ordered by index. + const auto& tx = *test::block1c.transactions_ptr()->front(); + const auto first = output_element(tx, 0, 1); + const auto second = output_element(tx, 1, 1); + const auto expected = system::bitcoin_hash(system::build_chunk({ first, second })); + system::muhash3072 muhash{}; + muhash.insert(first); + muhash.insert(second); + + const stopper cancel{}; + hash_digest digest{}; + unspent_totals totals{}; + const header_links branch{ header_link{ 1 } }; + BOOST_REQUIRE(!query.get_unspent_serialized(cancel, totals, digest, branch)); + BOOST_REQUIRE_EQUAL(digest, expected); + BOOST_REQUIRE_EQUAL(totals.outputs, 2u); + BOOST_REQUIRE_EQUAL(totals.transactions, 1u); + BOOST_REQUIRE_EQUAL(totals.value, 66u); + + totals = {}; + BOOST_REQUIRE(!query.get_unspent_muhash(cancel, totals, digest, branch)); + BOOST_REQUIRE_EQUAL(digest, muhash.flush()); + BOOST_REQUIRE_EQUAL(totals.outputs, 2u); + BOOST_REQUIRE_EQUAL(totals.transactions, 1u); +} + +BOOST_AUTO_TEST_CASE(query_unspent__get_unspent_matches__two_block_branch__one_match) +{ + settings settings{}; + settings.path = TEST_DIRECTORY; + test::chunk_store store{ settings }; + test::query_accessor query{ store }; + BOOST_REQUIRE(!store.create(test::events_handler)); + BOOST_REQUIRE(query.initialize(test::genesis)); + BOOST_REQUIRE(query.set(test::block1, context{ 0, 1, 0 }, false, false)); + BOOST_REQUIRE(query.set_strong(1)); + BOOST_REQUIRE(query.push_confirmed(1, false)); + BOOST_REQUIRE(query.set(test::block2, context{ 0, 2, 0 }, false, false)); + BOOST_REQUIRE(query.set_strong(2)); + BOOST_REQUIRE(query.push_confirmed(2, false)); + + const auto& tx = *test::block1.transactions_ptr()->front(); + const auto script = tx.outputs_ptr()->front()->script().to_data(false); + const std::unordered_set keys{ system::sha256_hash(script) }; + + const stopper cancel{}; + size_t txouts{}; + unspent_coins coins{}; + const header_links branch{ header_link{ 2 }, header_link{ 1 } }; + BOOST_REQUIRE(!query.get_unspent_matches(cancel, coins, txouts, keys, branch)); + BOOST_REQUIRE_EQUAL(txouts, 2u); + BOOST_REQUIRE_EQUAL(coins.size(), 1u); + BOOST_REQUIRE_EQUAL(coins.front().out.point().hash(), tx.hash(false)); + BOOST_REQUIRE_EQUAL(coins.front().out.point().index(), 0u); + BOOST_REQUIRE_EQUAL(coins.front().height, 1u); + BOOST_REQUIRE(coins.front().coinbase); + BOOST_REQUIRE_EQUAL(coins.front().out.value(), 5'000'000'000u); + BOOST_REQUIRE_EQUAL(coins.front().script, script); +} + +BOOST_AUTO_TEST_CASE(query_unspent__get_unspent_matches__no_keys__no_matches) +{ + settings settings{}; + settings.path = TEST_DIRECTORY; + test::chunk_store store{ settings }; + test::query_accessor query{ store }; + BOOST_REQUIRE(!store.create(test::events_handler)); + BOOST_REQUIRE(query.initialize(test::genesis)); + BOOST_REQUIRE(query.set(test::block1, context{ 0, 1, 0 }, false, false)); + BOOST_REQUIRE(query.set_strong(1)); + BOOST_REQUIRE(query.push_confirmed(1, false)); + + const stopper cancel{}; + size_t txouts{}; + unspent_coins coins{}; + const std::unordered_set keys{}; + const header_links branch{ header_link{ 1 } }; + BOOST_REQUIRE(!query.get_unspent_matches(cancel, coins, txouts, keys, branch)); + BOOST_REQUIRE_EQUAL(txouts, 1u); + BOOST_REQUIRE(coins.empty()); +} + BOOST_AUTO_TEST_CASE(query_unspent__get_unspent_totals__cancelled__query_canceled) { settings settings{}; @@ -71,6 +257,8 @@ BOOST_AUTO_TEST_CASE(query_unspent__get_unspent_totals__cancelled__query_cancele BOOST_REQUIRE(!store.create(test::events_handler)); BOOST_REQUIRE(query.initialize(test::genesis)); BOOST_REQUIRE(query.set(test::block1, context{ 0, 1, 0 }, false, false)); + BOOST_REQUIRE(query.set_strong(1)); + BOOST_REQUIRE(query.push_confirmed(1, false)); const stopper cancel{ true }; unspent_totals totals{}; diff --git a/test/types/difference_set.cpp b/test/types/difference_set.cpp new file mode 100644 index 000000000..127bb81df --- /dev/null +++ b/test/types/difference_set.cpp @@ -0,0 +1,129 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "../test.hpp" + +BOOST_AUTO_TEST_SUITE(difference_set_tests) + +using namespace system; + +// Members of a set, in element order. +static std_vector members(const difference_set& set) NOEXCEPT +{ + std_vector out{}; + for (size_t index{}; index < set.words(); ++index) + for (auto bits = set.at(index); !is_zero(bits); + bits = bit_and(bits, sub1(bits))) + out.push_back(index * difference_set::word_bits + + right_zeros(bits)); + + return out; +} + +BOOST_AUTO_TEST_CASE(difference_set__size__default__zero) +{ + const difference_set instance{}; + BOOST_REQUIRE_EQUAL(instance.size(), zero); + BOOST_REQUIRE_EQUAL(instance.words(), zero); +} + +BOOST_AUTO_TEST_CASE(difference_set__words__partial_word__ceilinged) +{ + const difference_set instance{ 65 }; + BOOST_REQUIRE_EQUAL(instance.size(), 65u); + BOOST_REQUIRE_EQUAL(instance.words(), 2u); +} + +BOOST_AUTO_TEST_CASE(difference_set__members__unpresented__empty) +{ + const difference_set instance{ 256 }; + BOOST_REQUIRE(members(instance).empty()); +} + +BOOST_AUTO_TEST_CASE(difference_set__toggle__once__member) +{ + difference_set instance{ 256 }; + instance.toggle(42); + BOOST_REQUIRE_EQUAL(members(instance), std_vector{ 42 }); +} + +BOOST_AUTO_TEST_CASE(difference_set__toggle__twice__cancelled) +{ + difference_set instance{ 256 }; + instance.toggle(42); + instance.toggle(42); + BOOST_REQUIRE(members(instance).empty()); +} + +BOOST_AUTO_TEST_CASE(difference_set__toggle__thrice__member) +{ + difference_set instance{ 256 }; + instance.toggle(42); + instance.toggle(42); + instance.toggle(42); + BOOST_REQUIRE_EQUAL(members(instance), std_vector{ 42 }); +} + +BOOST_AUTO_TEST_CASE(difference_set__toggle__distinct__all_members) +{ + difference_set instance{ 256 }; + instance.toggle(0); + instance.toggle(63); + instance.toggle(64); + instance.toggle(255); + const std_vector expected{ 0, 63, 64, 255 }; + BOOST_REQUIRE_EQUAL(members(instance), expected); +} + +BOOST_AUTO_TEST_CASE(difference_set__toggle__paired__residue_only) +{ + difference_set instance{ 256 }; + instance.toggle(10); + instance.toggle(20); + instance.toggle(30); + instance.toggle(20); + instance.toggle(10); + BOOST_REQUIRE_EQUAL(members(instance), std_vector{ 30 }); +} + +BOOST_AUTO_TEST_CASE(difference_set__toggle__word_boundary__independent) +{ + difference_set instance{ 128 }; + instance.toggle(63); + instance.toggle(64); + instance.toggle(63); + BOOST_REQUIRE_EQUAL(members(instance), std_vector{ 64 }); +} + +BOOST_AUTO_TEST_CASE(difference_set__at__unset_word__zero) +{ + const difference_set instance{ 128 }; + BOOST_REQUIRE_EQUAL(instance.at(0), 0_u64); + BOOST_REQUIRE_EQUAL(instance.at(1), 0_u64); +} + +BOOST_AUTO_TEST_CASE(difference_set__at__set_elements__expected_word) +{ + difference_set instance{ 128 }; + instance.toggle(0); + instance.toggle(2); + BOOST_REQUIRE_EQUAL(instance.at(0), 0b101_u64); + BOOST_REQUIRE_EQUAL(instance.at(1), 0_u64); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/test/types/unspent.cpp b/test/types/unspent.cpp deleted file mode 100644 index 0a50c8d78..000000000 --- a/test/types/unspent.cpp +++ /dev/null @@ -1,166 +0,0 @@ -/** - * Copyright (c) 2011-2026 libbitcoin developers - * - * This file is part of libbitcoin. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "../test.hpp" - -BOOST_AUTO_TEST_SUITE(unspent_tests) - -using namespace system; - -// less_than.operator() - -BOOST_AUTO_TEST_CASE(unspent__less_than__confirmed_before_unconfirmed__expected) -{ - const unspent a{ {}, 42, 7 }; - const unspent b{ {}, 0, unspent::unconfirmed_position }; - BOOST_REQUIRE(a < b); - BOOST_REQUIRE(!(b < a)); -} - -BOOST_AUTO_TEST_CASE(unspent__less_than__confirmed_height_ascending__expected) -{ - const unspent a{ {}, 100, 5 }; - const unspent b{ {}, 200, 5 }; - BOOST_REQUIRE(a < b); - BOOST_REQUIRE(!(b < a)); -} - -BOOST_AUTO_TEST_CASE(unspent__less_than__confirmed_position_ascending__expected) -{ - const unspent a{ {}, 100, 3 }; - const unspent b{ {}, 100, 10 }; - BOOST_REQUIRE(a < b); - BOOST_REQUIRE(!(b < a)); -} - -BOOST_AUTO_TEST_CASE(unspent__less_than__confirmed_output_index_ascending__expected) -{ - const outpoint p1{ { {}, 0 }, 42 }; - const outpoint p2{ { {}, 5 }, 42 }; - const unspent a{ p1, 100, 10 }; - const unspent b{ p2, 100, 10 }; - BOOST_REQUIRE(a < b); - BOOST_REQUIRE(!(b < a)); -} - -BOOST_AUTO_TEST_CASE(unspent__less_than__unconfirmed_outpoint_ascending__expected) -{ - const outpoint p1{ { {}, 3 }, 42 }; - const outpoint p2{ { {}, 8 }, 42 }; - const unspent a{ p1, 0, unspent::unconfirmed_position }; - const unspent b{ p2, 0, unspent::unconfirmed_position }; - BOOST_REQUIRE(a < b); - BOOST_REQUIRE(!(b < a)); -} - -BOOST_AUTO_TEST_CASE(unspent__equality__distinct__false) -{ - const outpoint p1{ { {}, 0 }, 42 }; - const outpoint p2{ { {}, 1 }, 42 }; - const unspent a{ p1, 101, 10 }; - const unspent b{ p2, 100, 11 }; - BOOST_REQUIRE(!(a == b)); - BOOST_REQUIRE(!(b == a)); -} - -BOOST_AUTO_TEST_CASE(unspent__equality__same__true) -{ - const outpoint point{ { {}, 0 }, 42 }; - const unspent value{ point, 100, 10 }; - BOOST_REQUIRE(value == value); -} - -BOOST_AUTO_TEST_CASE(unspent__valid__always__expected) -{ - const auto valid1 = unspent{ { { hash_digest{}, 0 }, 42 }, 2, 0 }.valid(); - BOOST_REQUIRE(valid1); - - const auto valid2 = unspent{ { { hash_digest{}, 0 }, 42 }, 2, unspent::unconfirmed_position }.valid(); - BOOST_REQUIRE(valid2); - - const auto invalid1 = !unspent{ {}, 2, 3 }.valid(); - BOOST_REQUIRE(invalid1); - - const auto invalid2 = !unspent{ {}, 2, unspent::unconfirmed_position }.valid(); - BOOST_REQUIRE(invalid2); -} - -BOOST_AUTO_TEST_CASE(unspent__fault__always__expected) -{ - const auto fault1 = unspent{}.fault(); - BOOST_REQUIRE(fault1); - - const auto fault2 = unspent{ {}, 2, 0 }.fault(); - BOOST_REQUIRE(fault2); - - const auto nonfault1 = !unspent{ {}, 2, 3 }.fault(); - BOOST_REQUIRE(nonfault1); - - const auto nonfault2 = !unspent{ {}, 2, unspent::unconfirmed_position }.fault(); - BOOST_REQUIRE(nonfault2); -} - -BOOST_AUTO_TEST_CASE(unspent__confirmed__always__expected) -{ - const auto confirmed = unspent{ { { hash_digest{}, 0 }, 42 }, 2, 3 }.confirmed(); - BOOST_REQUIRE(confirmed); - - const auto unconfirmed = !unspent{ { { hash_digest{}, 0 }, 42 }, 2, unspent::unconfirmed_position }.confirmed(); - BOOST_REQUIRE(unconfirmed); -} - -// filter_sort_and_dedup - -BOOST_AUTO_TEST_CASE(unspent__filter_sort_and_dedup__unsorted_with_duplicates_mixed__sorted_and_deduped) -{ - const outpoint lo{ { {}, 0 }, 42 }; - const outpoint hi{ { {}, 5 }, 42 }; - std::vector values - { - { hi, 0, unspent::unconfirmed_position }, // unconfirmed - { lo, 200, 3 }, // confirmed - { lo, 100, 5 }, // confirmed - { lo, 100, 5 }, // confirmed duplicate - { hi, 0, unspent::unconfirmed_position } // unconfirmed duplicate - }; - - unspent::filter_sort_and_dedup(values); - BOOST_REQUIRE_EQUAL(values.size(), 3u); - BOOST_REQUIRE_EQUAL(values[0].height, 100u); // confirmed, lowest height - BOOST_REQUIRE_EQUAL(values[1].height, 200u); // confirmed - BOOST_REQUIRE_EQUAL(values[2].height, unspent::unused_height); // unconfirmed -} - -BOOST_AUTO_TEST_CASE(unspent__filter_sort_and_dedup__exclusions__removes_excluded_items) -{ - unspents items - { - unspent{ outpoint{}, 10, 42 }, // excluded (default outpoint) - unspent{ outpoint{}, 200, 42 }, // excluded (default outpoint) - unspent{ { {}, 3 }, 50, 10 }, // valid confirmed - unspent{ { {}, 4 }, 50, 5 }, // valid confirmed (same height, lower position) - unspent{ { {}, 3 }, 50, 10 } // duplicate - }; - - unspent::filter_sort_and_dedup(items); - BOOST_REQUIRE_EQUAL(items.size(), 2u); - BOOST_REQUIRE_EQUAL(items[0].position, 5u); - BOOST_REQUIRE_EQUAL(items[1].position, 10u); -} - -BOOST_AUTO_TEST_SUITE_END() diff --git a/test/types/unspent_output.cpp b/test/types/unspent_output.cpp new file mode 100644 index 000000000..aa218879b --- /dev/null +++ b/test/types/unspent_output.cpp @@ -0,0 +1,166 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "../test.hpp" + +BOOST_AUTO_TEST_SUITE(unspent_output_tests) + +using namespace system; + +// less_than.operator() + +BOOST_AUTO_TEST_CASE(unspent_output__less_than__confirmed_before_unconfirmed__expected) +{ + const unspent_output a{ {}, 42, 7 }; + const unspent_output b{ {}, 0, unspent_output::unconfirmed_position }; + BOOST_REQUIRE(a < b); + BOOST_REQUIRE(!(b < a)); +} + +BOOST_AUTO_TEST_CASE(unspent_output__less_than__confirmed_height_ascending__expected) +{ + const unspent_output a{ {}, 100, 5 }; + const unspent_output b{ {}, 200, 5 }; + BOOST_REQUIRE(a < b); + BOOST_REQUIRE(!(b < a)); +} + +BOOST_AUTO_TEST_CASE(unspent_output__less_than__confirmed_position_ascending__expected) +{ + const unspent_output a{ {}, 100, 3 }; + const unspent_output b{ {}, 100, 10 }; + BOOST_REQUIRE(a < b); + BOOST_REQUIRE(!(b < a)); +} + +BOOST_AUTO_TEST_CASE(unspent_output__less_than__confirmed_output_index_ascending__expected) +{ + const outpoint p1{ { {}, 0 }, 42 }; + const outpoint p2{ { {}, 5 }, 42 }; + const unspent_output a{ p1, 100, 10 }; + const unspent_output b{ p2, 100, 10 }; + BOOST_REQUIRE(a < b); + BOOST_REQUIRE(!(b < a)); +} + +BOOST_AUTO_TEST_CASE(unspent_output__less_than__unconfirmed_outpoint_ascending__expected) +{ + const outpoint p1{ { {}, 3 }, 42 }; + const outpoint p2{ { {}, 8 }, 42 }; + const unspent_output a{ p1, 0, unspent_output::unconfirmed_position }; + const unspent_output b{ p2, 0, unspent_output::unconfirmed_position }; + BOOST_REQUIRE(a < b); + BOOST_REQUIRE(!(b < a)); +} + +BOOST_AUTO_TEST_CASE(unspent_output__equality__distinct__false) +{ + const outpoint p1{ { {}, 0 }, 42 }; + const outpoint p2{ { {}, 1 }, 42 }; + const unspent_output a{ p1, 101, 10 }; + const unspent_output b{ p2, 100, 11 }; + BOOST_REQUIRE(!(a == b)); + BOOST_REQUIRE(!(b == a)); +} + +BOOST_AUTO_TEST_CASE(unspent_output__equality__same__true) +{ + const outpoint point{ { {}, 0 }, 42 }; + const unspent_output value{ point, 100, 10 }; + BOOST_REQUIRE(value == value); +} + +BOOST_AUTO_TEST_CASE(unspent_output__valid__always__expected) +{ + const auto valid1 = unspent_output{ { { hash_digest{}, 0 }, 42 }, 2, 0 }.valid(); + BOOST_REQUIRE(valid1); + + const auto valid2 = unspent_output{ { { hash_digest{}, 0 }, 42 }, 2, unspent_output::unconfirmed_position }.valid(); + BOOST_REQUIRE(valid2); + + const auto invalid1 = !unspent_output{ {}, 2, 3 }.valid(); + BOOST_REQUIRE(invalid1); + + const auto invalid2 = !unspent_output{ {}, 2, unspent_output::unconfirmed_position }.valid(); + BOOST_REQUIRE(invalid2); +} + +BOOST_AUTO_TEST_CASE(unspent_output__fault__always__expected) +{ + const auto fault1 = unspent_output{}.fault(); + BOOST_REQUIRE(fault1); + + const auto fault2 = unspent_output{ {}, 2, 0 }.fault(); + BOOST_REQUIRE(fault2); + + const auto nonfault1 = !unspent_output{ {}, 2, 3 }.fault(); + BOOST_REQUIRE(nonfault1); + + const auto nonfault2 = !unspent_output{ {}, 2, unspent_output::unconfirmed_position }.fault(); + BOOST_REQUIRE(nonfault2); +} + +BOOST_AUTO_TEST_CASE(unspent_output__confirmed__always__expected) +{ + const auto confirmed = unspent_output{ { { hash_digest{}, 0 }, 42 }, 2, 3 }.confirmed(); + BOOST_REQUIRE(confirmed); + + const auto unconfirmed = !unspent_output{ { { hash_digest{}, 0 }, 42 }, 2, unspent_output::unconfirmed_position }.confirmed(); + BOOST_REQUIRE(unconfirmed); +} + +// filter_sort_and_dedup + +BOOST_AUTO_TEST_CASE(unspent_output__filter_sort_and_dedup__unsorted_with_duplicates_mixed__sorted_and_deduped) +{ + const outpoint lo{ { {}, 0 }, 42 }; + const outpoint hi{ { {}, 5 }, 42 }; + std::vector values + { + { hi, 0, unspent_output::unconfirmed_position }, // unconfirmed + { lo, 200, 3 }, // confirmed + { lo, 100, 5 }, // confirmed + { lo, 100, 5 }, // confirmed duplicate + { hi, 0, unspent_output::unconfirmed_position } // unconfirmed duplicate + }; + + unspent_output::filter_sort_and_dedup(values); + BOOST_REQUIRE_EQUAL(values.size(), 3u); + BOOST_REQUIRE_EQUAL(values[0].height, 100u); // confirmed, lowest height + BOOST_REQUIRE_EQUAL(values[1].height, 200u); // confirmed + BOOST_REQUIRE_EQUAL(values[2].height, unspent_output::unused_height); // unconfirmed +} + +BOOST_AUTO_TEST_CASE(unspent_output__filter_sort_and_dedup__exclusions__removes_excluded_items) +{ + unspent_outputs items + { + unspent_output{ outpoint{}, 10, 42 }, // excluded (default outpoint) + unspent_output{ outpoint{}, 200, 42 }, // excluded (default outpoint) + unspent_output{ { {}, 3 }, 50, 10 }, // valid confirmed + unspent_output{ { {}, 4 }, 50, 5 }, // valid confirmed (same height, lower position) + unspent_output{ { {}, 3 }, 50, 10 } // duplicate + }; + + unspent_output::filter_sort_and_dedup(items); + BOOST_REQUIRE_EQUAL(items.size(), 2u); + BOOST_REQUIRE_EQUAL(items[0].position, 5u); + BOOST_REQUIRE_EQUAL(items[1].position, 10u); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/test/unspent/unspent_commitment.cpp b/test/unspent/unspent_commitment.cpp new file mode 100644 index 000000000..4a42d863a --- /dev/null +++ b/test/unspent/unspent_commitment.cpp @@ -0,0 +1,108 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "../test.hpp" + +BOOST_AUTO_TEST_SUITE(unspent_writer_tests) + +using namespace system; + +constexpr auto txid = base16_array( + "0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"); + +static unspent_coin coin(size_t height, bool coinbase, uint64_t value, + const data_chunk& script) +{ + unspent_coin out{}; + out.first = true; + out.out = { chain::point{ txid, 7 }, value }; + out.height = height; + out.coinbase = coinbase; + out.script = script; + return out; +} + +BOOST_AUTO_TEST_CASE(unspent_writer__write__coinbase__expected_head) +{ + const auto script = base16_chunk("76a914000000000000000000000000000000000000000088ac"); + const auto subject = coin(1234, true, 5'000'000'000, script); + const auto index = to_little_endian(7); + const auto code = to_little_endian((1234 << 1) | 1); + const auto expected = build_chunk({ txid, index, code }); + + data_chunk head{}; + write::bytes::data sink{ head }; + unspent_writer::write(sink, subject); + sink.flush(); + BOOST_REQUIRE_EQUAL(head, expected); +} + +BOOST_AUTO_TEST_CASE(unspent_writer__write__non_coinbase__expected_head) +{ + const data_chunk script(300, 0x51); + const auto subject = coin(42, false, 1, script); + const auto index = to_little_endian(7); + const auto code = to_little_endian(42 << 1); + const auto expected = build_chunk({ txid, index, code }); + + data_chunk head{}; + write::bytes::data sink{ head }; + unspent_writer::write(sink, subject); + sink.flush(); + BOOST_REQUIRE_EQUAL(head, expected); +} + +BOOST_AUTO_TEST_CASE(unspent_writer__size__script_sizes__expected) +{ + BOOST_REQUIRE_EQUAL(unspent_writer::size(0), 48u + 1u); + BOOST_REQUIRE_EQUAL(unspent_writer::size(25), 48u + 1u + 25u); + BOOST_REQUIRE_EQUAL(unspent_writer::size(300), 48u + 3u + 300u); +} + +BOOST_AUTO_TEST_CASE(unspent_writer__add__coin__accumulated) +{ + const auto script = base16_chunk("51"); + const auto subject = coin(1, false, 100, script); + unspent_totals totals{}; + unspent_writer::add(totals, subject); + unspent_writer::add(totals, subject); + BOOST_REQUIRE_EQUAL(totals.transactions, 2u); + BOOST_REQUIRE_EQUAL(totals.outputs, 2u); + BOOST_REQUIRE_EQUAL(totals.value, 200u); + BOOST_REQUIRE_EQUAL(totals.script_bytes, 2u); + BOOST_REQUIRE_EQUAL(totals.coin_bytes, 2u * (48u + 1u + 1u)); +} + +BOOST_AUTO_TEST_CASE(unspent_writer__add__totals__summed) +{ + unspent_totals left{}; + left.transactions = 1; + left.outputs = 2; + left.value = 3; + left.script_bytes = 4; + left.coin_bytes = 5; + unspent_totals right{ left }; + unspent_writer::add(left, right); + BOOST_REQUIRE_EQUAL(left.transactions, 2u); + BOOST_REQUIRE_EQUAL(left.outputs, 4u); + BOOST_REQUIRE_EQUAL(left.value, 6u); + BOOST_REQUIRE_EQUAL(left.script_bytes, 8u); + BOOST_REQUIRE_EQUAL(left.coin_bytes, 10u); +} + +BOOST_AUTO_TEST_SUITE_END()