From 175e32dec1467ea432f7ed4b01aedcb76e1e1bbc Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Mon, 2 Nov 2026 16:40:08 +0800 Subject: [PATCH 1/5] feat(file-index): support precise file index predicate evaluation --- docs/source/user_guide/file_index.rst | 19 +- include/paimon/file_index/file_index_reader.h | 23 +- include/paimon/file_index/file_indexer.h | 4 + .../file_index/scored_file_index_result.h | 2 +- include/paimon/predicate/function_visitor.h | 12 +- include/paimon/predicate/predicate_utils.h | 9 +- include/paimon/read_context.h | 74 +- src/paimon/CMakeLists.txt | 3 + .../empty/empty_file_index_reader.h | 28 +- .../empty/empty_file_index_reader_test.cpp | 34 +- .../common/file_index/file_index_format.cpp | 7 +- .../common/file_index/file_index_reader.cpp | 13 +- .../file_index/file_index_reader_test.cpp | 41 +- src/paimon/common/file_index/file_indexer.cpp | 30 + .../common/predicate/predicate_utils.cpp | 9 - ...complete_index_score_file_batch_reader.cpp | 111 ++- .../complete_index_score_file_batch_reader.h | 20 +- ...ete_index_score_file_batch_reader_test.cpp | 158 ++++ .../pksorted/pk_sorted_data_file_reader.cpp | 2 +- .../pksorted/pk_sorted_data_file_reader.h | 3 +- src/paimon/core/io/file_index_evaluator.cpp | 364 ++++---- src/paimon/core/io/file_index_evaluator.h | 88 +- .../core/io/file_index_evaluator_test.cpp | 492 ++++++++--- .../core/operation/abstract_split_read.cpp | 76 +- .../core/operation/abstract_split_read.h | 20 +- .../operation/data_evolution_split_read.cpp | 22 +- .../operation/data_evolution_split_read.h | 3 +- .../data_evolution_split_read_test.cpp | 2 +- .../operation/file_index_read_executor.cpp | 553 ++++++++++++ .../core/operation/file_index_read_executor.h | 121 +++ .../file_index_read_executor_test.cpp | 820 ++++++++++++++++++ src/paimon/core/operation/file_store_scan.cpp | 8 +- .../core/operation/internal_read_context.cpp | 52 +- .../core/operation/internal_read_context.h | 16 +- .../operation/internal_read_context_test.cpp | 192 ++-- .../core/operation/merge_file_split_read.cpp | 4 +- .../core/operation/raw_file_split_read.cpp | 172 +--- .../core/operation/raw_file_split_read.h | 48 +- .../operation/raw_file_split_read_test.cpp | 26 +- src/paimon/core/operation/read_context.cpp | 37 +- .../core/operation/read_context_test.cpp | 56 +- .../realtime_offset_file_index_lookup.cpp | 8 +- .../table/source/append_only_table_read.cpp | 8 +- .../table/source/key_value_table_read.cpp | 8 +- .../core/table/source/table_read_test.cpp | 187 +++- src/paimon/core/table/table.cpp | 28 +- .../indexer/lumina/lumina_file_index.cpp | 6 + src/paimon/indexer/lumina/lumina_file_index.h | 6 + .../testing/mock/mock_search_file_index.cpp | 189 +++- .../testing/mock/mock_search_file_index.h | 84 +- test/inte/predicate_abi_inte_test.cpp | 7 +- test/inte/write_and_read_inte_test.cpp | 24 +- .../write_and_read_inte_with_index_test.cpp | 278 +++++- 53 files changed, 3575 insertions(+), 1032 deletions(-) create mode 100644 src/paimon/common/file_index/file_indexer.cpp create mode 100644 src/paimon/core/operation/file_index_read_executor.cpp create mode 100644 src/paimon/core/operation/file_index_read_executor.h create mode 100644 src/paimon/core/operation/file_index_read_executor_test.cpp diff --git a/docs/source/user_guide/file_index.rst b/docs/source/user_guide/file_index.rst index f3dcbad7d..30dd11f3e 100644 --- a/docs/source/user_guide/file_index.rst +++ b/docs/source/user_guide/file_index.rst @@ -21,6 +21,11 @@ File Index Full-text predicates and root TopK ---------------------------------- +Enable ``ReadContextBuilder::EnablePreciseFileIndexEvaluate(true)`` to evaluate +complete predicates within each data file. The option defaults to ``false``. +Full-text predicates and root TopK requests require this option; configuring +them while it is disabled is rejected by ``ReadContextBuilder::Finish()``. + ``PredicateBuilder::FullTextMatch`` creates a ``LeafPredicate`` for a full-text condition without TopK truncation. It holds a shared ``FullTextMatch`` request, whose query uses the same JSON DSL as ``FullTextSearch``. Do not modify a request @@ -45,7 +50,9 @@ positions in that schema. For example, the following constructs std::shared_ptr filter, paimon::PredicateBuilder::Or({text, price})); paimon::ReadContextBuilder builder(table_path); - builder.SetReadFieldNames({"body", "price"}).SetPredicate(filter); + builder.EnablePreciseFileIndexEvaluate(true) + .SetReadFieldNames({"body", "price"}) + .SetPredicate(filter); ``SetVectorSearch`` and ``SetFullTextSearch`` configure a separate root TopK request. They are mutually exclusive and require a positive limit. The intended @@ -54,7 +61,9 @@ field does not need to be projected. A full-text match inside a predicate never has a TopK limit. Score output is requested through an explicit ``__INDEX_SCORE`` -projection. For example, request the score of a full-text +projection. Precise evaluation rejects the unnamed ``_INDEX_SCORE`` column, +whether selected by field name, field ID or an explicit read schema. +For example, request the score of a full-text predicate by projecting ``_body_INDEX_SCORE``: .. code-block:: cpp @@ -65,7 +74,9 @@ predicate by projecting ``_body_INDEX_SCORE``: std::shared_ptr text, paimon::PredicateBuilder::FullTextMatch(0, request)); paimon::ReadContextBuilder builder(table_path); - builder.SetReadFieldNames({"body", "_body_INDEX_SCORE"}).SetPredicate(text); + builder.EnablePreciseFileIndexEvaluate(true) + .SetReadFieldNames({"body", "_body_INDEX_SCORE"}) + .SetPredicate(text); Score fields have type ``float32`` and preserve the source field's exact name, including case. Select them with ``SetReadFieldNames`` or ``SetReadSchema``; @@ -80,4 +91,4 @@ An ordinary scalar predicate on the same field is allowed. For example, when projecting the score of ``match``, ``match AND (p OR q)`` is valid, while ``match OR p`` is rejected. Without that score projection, both OR branches and repeated full-text search fields are allowed. These checks use the effective -read projection, after resolving the read schema. +read projection, after resolving the read schema. \ No newline at end of file diff --git a/include/paimon/file_index/file_index_reader.h b/include/paimon/file_index/file_index_reader.h index 94f758ea9..7799626f3 100644 --- a/include/paimon/file_index/file_index_reader.h +++ b/include/paimon/file_index/file_index_reader.h @@ -20,6 +20,7 @@ #include #include +#include #include #include "paimon/file_index/file_index_result.h" @@ -31,6 +32,7 @@ #include "paimon/visibility.h" namespace paimon { +class LeafPredicate; struct FullTextMatch; /// Evaluates filter predicates against a file-level index to determine file eligibility. /// @@ -42,8 +44,18 @@ struct FullTextMatch; /// of the indexed data; otherwise, index filtering results may be incorrect. class PAIMON_EXPORT FileIndexReader : public FunctionVisitor> { public: + enum class SearchType { FULL_TEXT_MATCH, FULL_TEXT_SEARCH, VECTOR_SEARCH }; + ~FileIndexReader() override = default; + /// Whether this reader can execute the requested search without probing it with a query. + virtual bool SupportsSearch(SearchType search_type) const; + + /// Estimated matching fraction in [0, 1], or unknown. Estimates only affect execution order. + /// Existing scalar indexes use the default unknown estimate. + virtual std::optional EstimateSelectivity( + const std::shared_ptr& predicate) const; + Result> VisitIsNotNull() override; Result> VisitIsNull() override; @@ -77,13 +89,16 @@ class PAIMON_EXPORT FileIndexReader : public FunctionVisitor> VisitVectorSearch( const std::shared_ptr& vector_search); - /// Execute full-text search and return matching file-local physical row positions. - virtual Result> VisitFullTextSearch( + /// Execute full-text TopK search and return scored file-local physical row positions. + /// Scores are required for merging results across files, regardless of score projection. + virtual Result> VisitFullTextSearch( const std::shared_ptr& full_text_search); /// Return every matching file-local physical row position, without TopK truncation. - Result> VisitFullTextMatch( - const std::shared_ptr& full_text_match) override; + /// When needs_scores is true, return a ScoredFileIndexResult, including for empty results. + /// When needs_scores is false, return a BitmapIndexResult, not a ScoredFileIndexResult. + virtual Result> VisitFullTextMatch( + const std::shared_ptr& full_text_match, bool needs_scores = false); }; } // namespace paimon diff --git a/include/paimon/file_index/file_indexer.h b/include/paimon/file_index/file_indexer.h index c6cc77a02..2852bfc9c 100644 --- a/include/paimon/file_index/file_indexer.h +++ b/include/paimon/file_index/file_indexer.h @@ -47,6 +47,10 @@ class PAIMON_EXPORT FileIndexer { return false; } + /// Create a reader for an empty index entry without serialized bytes. + /// Scalar indexers use the default reader. Search indexers must preserve their capabilities. + virtual std::shared_ptr CreateEmptyReader() const; + /// Extra columns required while building this index, in addition to the indexed column. virtual Result>> GetExtraFieldNames() const { return std::optional>(std::nullopt); diff --git a/include/paimon/file_index/scored_file_index_result.h b/include/paimon/file_index/scored_file_index_result.h index a81693f68..55c3d4395 100644 --- a/include/paimon/file_index/scored_file_index_result.h +++ b/include/paimon/file_index/scored_file_index_result.h @@ -29,7 +29,7 @@ namespace paimon { -/// File-local vector search result. Scores correspond to row positions in ascending order. +/// File-local search result. Scores correspond to row positions in ascending order. class PAIMON_EXPORT ScoredFileIndexResult : public FileIndexResult { public: static Result> Create(RoaringBitmap32&& row_positions, diff --git a/include/paimon/predicate/function_visitor.h b/include/paimon/predicate/function_visitor.h index 5e8b54b3e..5a14c9fc6 100644 --- a/include/paimon/predicate/function_visitor.h +++ b/include/paimon/predicate/function_visitor.h @@ -18,20 +18,16 @@ #pragma once -#include #include #include #include "paimon/predicate/leaf_predicate.h" #include "paimon/predicate/literal.h" #include "paimon/result.h" -#include "paimon/status.h" #include "paimon/visibility.h" namespace paimon { -struct FullTextMatch; - -/// A visitor interface for evaluating filter predicates on indexed columns. +/// A visitor interface for evaluating scalar filter predicates on indexed columns. /// @tparam T The result type produced by each visit method (e.g., a file index result or global /// index result). template @@ -80,11 +76,5 @@ class PAIMON_EXPORT FunctionVisitor { /// Evaluates whether string values like the given string. virtual Result VisitLike(const Literal& literal) = 0; - - /// Evaluates a full-text condition without TopK truncation. - /// Visitors without full-text support return NotImplemented. - virtual Result VisitFullTextMatch(const std::shared_ptr& full_text_match) { - return Status::NotImplemented("This visitor does not support complete full-text matching"); - } }; } // namespace paimon diff --git a/include/paimon/predicate/predicate_utils.h b/include/paimon/predicate/predicate_utils.h index 24cc47c08..5380f5316 100644 --- a/include/paimon/predicate/predicate_utils.h +++ b/include/paimon/predicate/predicate_utils.h @@ -64,6 +64,7 @@ class PAIMON_EXPORT PredicateUtils { /// Whether the predicate contains a full-text matching condition. static bool ContainsFullTextMatch(const std::shared_ptr& predicate); + /// Dispatches scalar leaf predicates. Search predicates use dedicated index reader methods. template static Result VisitPredicate(const std::shared_ptr& predicate, const std::shared_ptr>& visitor) { @@ -117,11 +118,6 @@ class PAIMON_EXPORT PredicateUtils { assert(predicate->Literals().size() == 1); return visitor->VisitLike(predicate->Literals()[0]); } - case Function::Type::FULL_TEXT_MATCH: { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr full_text_match, - GetFullTextMatch(predicate)); - return visitor->VisitFullTextMatch(full_text_match); - } default: return Status::Invalid("invalid " + predicate->GetFunction().ToString() + " function in leaf predicate"); @@ -129,9 +125,6 @@ class PAIMON_EXPORT PredicateUtils { } private: - static Result> GetFullTextMatch( - const std::shared_ptr& predicate); - static Result>> ReconstructPredicateWithPickedFields( const std::shared_ptr& predicate, const std::map& picked_field_name_to_idx); diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index 81b0f8fa2..5d04d89d2 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -52,23 +52,26 @@ struct VectorSearch; /// @see ReadContextBuilder class PAIMON_EXPORT ReadContext { public: - ReadContext( - const std::string& path, const std::string& branch, - const std::vector& read_field_names, - const std::vector& read_field_ids, const std::shared_ptr& predicate, - const std::shared_ptr& vector_search, - const std::shared_ptr& full_text_search, bool enable_predicate_filter, - bool enable_prefetch, bool enable_late_materializing, uint32_t prefetch_batch_count, - uint32_t prefetch_max_parallel_num, bool enable_multi_thread_row_to_batch, - uint32_t row_to_batch_thread_number, const std::optional& table_schema, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& fs_scheme_to_identifier_map, - const std::shared_ptr& realtime_context, - const std::map& options, bool read_ahead_cache_enabled, - const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level, - const std::shared_ptr& catalog, const std::optional& identifier); + ReadContext(const std::string& path, const std::string& branch, + const std::vector& read_field_names, + const std::vector& read_field_ids, + const std::shared_ptr& predicate, + const std::shared_ptr& vector_search, + const std::shared_ptr& full_text_search, + bool enable_precise_file_index_evaluate, bool enable_predicate_filter, + bool enable_prefetch, bool enable_late_materializing, uint32_t prefetch_batch_count, + uint32_t prefetch_max_parallel_num, bool enable_multi_thread_row_to_batch, + uint32_t row_to_batch_thread_number, const std::optional& table_schema, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor, + const std::shared_ptr& specific_file_system, + const std::map& fs_scheme_to_identifier_map, + const std::shared_ptr& realtime_context, + const std::map& options, bool read_ahead_cache_enabled, + const CacheConfig& cache_config, const std::shared_ptr& cache, + const std::shared_ptr& format_table, WarmupLevel warmup_level, + const std::shared_ptr& catalog, + const std::optional& identifier); ~ReadContext(); @@ -113,6 +116,10 @@ class PAIMON_EXPORT ReadContext { return vector_search_ != nullptr || full_text_search_ != nullptr; } + bool EnablePreciseFileIndexEvaluate() const { + return enable_precise_file_index_evaluate_; + } + bool EnablePredicateFilter() const { return enable_predicate_filter_; } @@ -208,6 +215,7 @@ class PAIMON_EXPORT ReadContext { std::shared_ptr predicate_; std::shared_ptr vector_search_; std::shared_ptr full_text_search_; + bool enable_precise_file_index_evaluate_; bool enable_predicate_filter_; bool enable_prefetch_; bool enable_late_materializing_; @@ -352,25 +360,23 @@ class PAIMON_EXPORT ReadContextBuilder { /// /// All referenced fields, including full-text match fields, must be projected in the read /// schema. - /// TODO(lisizhuo.lsz): The behavior here will be aligned with Java Paimon in a follow-up - /// change: fields referenced in predicates will no longer be required to appear in the - /// projection. Full-text matches can be combined with scalar predicates using - /// PredicateBuilder::And/Or. Full-text predicate execution is not implemented yet; reading such - /// a predicate returns NotImplemented. + /// TODO(lisizhuo.lsz): The behavior here will be aligned with Java Paimon. + /// Full-text matches can be combined with scalar predicates using PredicateBuilder::And/Or. /// /// @param predicate Shared pointer to the predicate for data filtering. /// @return Reference to this builder for method chaining. + /// @note Predicates containing FullTextMatch require EnablePreciseFileIndexEvaluate(true). ReadContextBuilder& SetPredicate(const std::shared_ptr& predicate); /// Configure the root vector TopK request, logically applied after the predicate filter. /// The search field need not be projected. SetVectorSearch is mutually exclusive with /// SetFullTextSearch and must have a positive limit. /// Scores are required internally for TopK. Project `__INDEX_SCORE` to include them - /// in the result. Per-field score output is not implemented yet; reading such a projection - /// returns NotImplemented. - /// TODO(lisizhuo.lsz): Execute combined read predicates and merge TopK across files. + /// in the result. + /// TODO(lisizhuo.lsz): Merge TopK across files. /// @param vector_search Vector search request, or nullptr to clear it. /// @return Reference to this builder for method chaining. + /// @note Requires EnablePreciseFileIndexEvaluate(true). ReadContextBuilder& SetVectorSearch(const std::shared_ptr& vector_search); /// Configure the root full-text TopK request, logically applied after the predicate filter. @@ -378,13 +384,25 @@ class PAIMON_EXPORT ReadContextBuilder { /// full-text filter without TopK truncation. SetFullTextSearch is mutually exclusive with /// SetVectorSearch and must have a positive limit. /// Scores are required internally for TopK. Project `__INDEX_SCORE` to include them - /// in the result. Per-field score output is not implemented yet; reading such a projection - /// returns NotImplemented. - /// TODO(lisizhuo.lsz): Execute combined read predicates and merge TopK across files. + /// in the result. + /// TODO(lisizhuo.lsz): Merge TopK across files. /// @param full_text_search Full-text search request, or nullptr to clear it. /// @return Reference to this builder for method chaining. + /// @note Requires EnablePreciseFileIndexEvaluate(true). ReadContextBuilder& SetFullTextSearch(const std::shared_ptr& full_text_search); + /// Compute exact file-local result row positions during file-index evaluation, using the + /// configured Predicate and optional VectorSearch or FullTextSearch request. + /// Requires file-index.read.enabled=true. Reads available file indexes and rechecks scalar + /// index candidates against the data. If a scalar predicate's field has no index, reads the + /// data to compute exact matching rows. Only scalar predicates may be evaluated without an + /// index. Score output must use named projections such as __INDEX_SCORE; + /// projecting _INDEX_SCORE by name, field ID or read schema is not supported in this mode. + /// The default is false. + /// @param enabled Whether to compute exact result row positions during file-index evaluation. + /// @return Reference to this builder for method chaining. + ReadContextBuilder& EnablePreciseFileIndexEvaluate(bool enabled); + /// Whether to perform precise filtering according to predicates for data read from format /// reader. /// @param enabled Whether to enable precise filtering (default: false) diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 4db62cb79..da3e4d3e3 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -69,6 +69,7 @@ set(PAIMON_COMMON_SRCS common/factories/io_hook.cpp common/factories/factory_creator.cpp common/file_index/bitmap/bitmap_index_result.cpp + common/file_index/file_indexer.cpp common/file_index/file_indexer_factory.cpp common/file_index/file_index_format.cpp common/file_index/file_index_format_utils.cpp @@ -412,6 +413,7 @@ set(PAIMON_CORE_SRCS core/operation/orphan_files_cleaner.cpp core/operation/orphan_files_cleaner_impl.cpp core/operation/raw_file_split_read.cpp + core/operation/file_index_read_executor.cpp core/operation/read_context.cpp core/operation/scan_context.cpp core/manifest/snapshot_live_manifest_entries.cpp @@ -992,6 +994,7 @@ if(PAIMON_BUILD_TESTS) core/operation/commit/realtime_deduplicate_dv_materializer_test.cpp core/operation/commit/sequence_snapshot_properties_test.cpp core/operation/key_value_file_store_write_test.cpp + core/operation/file_index_read_executor_test.cpp core/operation/internal_read_context_test.cpp core/operation/abstract_split_read_test.cpp core/operation/append_only_file_store_write_test.cpp diff --git a/src/paimon/common/file_index/empty/empty_file_index_reader.h b/src/paimon/common/file_index/empty/empty_file_index_reader.h index 624806e66..8304e8c09 100644 --- a/src/paimon/common/file_index/empty/empty_file_index_reader.h +++ b/src/paimon/common/file_index/empty/empty_file_index_reader.h @@ -18,6 +18,7 @@ #pragma once +#include #include #include #include @@ -37,15 +38,33 @@ class Literal; /// Vector and full-text searches return no matches. class EmptyFileIndexReader : public FileIndexReader { public: + explicit EmptyFileIndexReader(std::vector search_types = {}) + : search_types_(std::move(search_types)) {} + + bool SupportsSearch(SearchType search_type) const override { + return std::find(search_types_.begin(), search_types_.end(), search_type) != + search_types_.end(); + } + + Result> VisitFullTextMatch( + const std::shared_ptr&, bool needs_scores = false) override { + if (needs_scores) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, + ScoredFileIndexResult::Create(RoaringBitmap32(), {})); + return result; + } + return std::make_shared( + []() -> Result { return RoaringBitmap32(); }); + } + Result> VisitVectorSearch( const std::shared_ptr& vector_search) override { return ScoredFileIndexResult::Create(RoaringBitmap32(), {}); } - Result> VisitFullTextSearch( + Result> VisitFullTextSearch( const std::shared_ptr& full_text_search) override { - return std::make_shared( - []() -> Result { return RoaringBitmap32(); }); + return ScoredFileIndexResult::Create(RoaringBitmap32(), {}); } Result> VisitEqual(const Literal& literal) override { @@ -82,6 +101,9 @@ class EmptyFileIndexReader : public FileIndexReader { const std::vector& literals) override { return FileIndexResult::Skip(); } + + private: + std::vector search_types_; }; } // namespace paimon diff --git a/src/paimon/common/file_index/empty/empty_file_index_reader_test.cpp b/src/paimon/common/file_index/empty/empty_file_index_reader_test.cpp index 6d9eb5089..5f3e6c34a 100644 --- a/src/paimon/common/file_index/empty/empty_file_index_reader_test.cpp +++ b/src/paimon/common/file_index/empty/empty_file_index_reader_test.cpp @@ -26,6 +26,7 @@ #include "gtest/gtest.h" #include "paimon/file_index/bitmap_index_result.h" +#include "paimon/predicate/full_text_match.h" #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/vector_search.h" @@ -36,6 +37,10 @@ TEST(EmptyFileIndexReaderTest, TestSimple) { Literal lit0(static_cast(0)); EmptyFileIndexReader reader; + ASSERT_FALSE(reader.SupportsSearch(FileIndexReader::SearchType::FULL_TEXT_MATCH)); + ASSERT_FALSE(reader.SupportsSearch(FileIndexReader::SearchType::FULL_TEXT_SEARCH)); + ASSERT_FALSE(reader.SupportsSearch(FileIndexReader::SearchType::VECTOR_SEARCH)); + ASSERT_FALSE(reader.VisitIsNotNull().value()->IsRemain().value()); ASSERT_FALSE(reader.VisitEqual(lit0).value()->IsRemain().value()); ASSERT_FALSE(reader.VisitStartsWith(lit0).value()->IsRemain().value()); @@ -64,17 +69,22 @@ TEST(EmptyFileIndexReaderTest, TestVectorSearchReturnsNoMatches) { EXPECT_TRUE(result->GetScores().empty()); } -TEST(EmptyFileIndexReaderTest, TestFullTextSearchReturnsNoMatches) { - EmptyFileIndexReader reader; - auto search = - std::make_shared("body", R"({"match":{"query":"word"}})", /*limit=*/1); - ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - reader.VisitFullTextSearch(search)); - ASSERT_OK_AND_ASSIGN(bool is_remain, result->IsRemain()); - EXPECT_FALSE(is_remain); - auto bitmap_result = std::dynamic_pointer_cast(result); - ASSERT_NE(nullptr, bitmap_result); - ASSERT_OK_AND_ASSIGN(const RoaringBitmap32* bitmap, bitmap_result->GetBitmap()); - EXPECT_TRUE(bitmap->IsEmpty()); +TEST(EmptyFileIndexReaderTest, TestFullTextMatchAndScores) { + EmptyFileIndexReader reader({FileIndexReader::SearchType::FULL_TEXT_MATCH, + FileIndexReader::SearchType::FULL_TEXT_SEARCH}); + ASSERT_TRUE(reader.SupportsSearch(FileIndexReader::SearchType::FULL_TEXT_MATCH)); + ASSERT_TRUE(reader.SupportsSearch(FileIndexReader::SearchType::FULL_TEXT_SEARCH)); + ASSERT_FALSE(reader.SupportsSearch(FileIndexReader::SearchType::VECTOR_SEARCH)); + auto match = std::make_shared("body", "word"); + ASSERT_OK_AND_ASSIGN(auto result, reader.VisitFullTextMatch(match, /*needs_scores=*/true)); + auto scored = std::dynamic_pointer_cast(result); + ASSERT_TRUE(scored); + ASSERT_TRUE(scored->IsEmpty()); + ASSERT_TRUE(scored->GetScores().empty()); + ASSERT_OK_AND_ASSIGN(result, reader.VisitFullTextMatch(match)); + auto bitmap = std::dynamic_pointer_cast(result); + ASSERT_TRUE(bitmap); + ASSERT_OK_AND_ASSIGN(const RoaringBitmap32* positions, bitmap->GetBitmap()); + ASSERT_TRUE(positions->IsEmpty()); } } // namespace paimon::test diff --git a/src/paimon/common/file_index/file_index_format.cpp b/src/paimon/common/file_index/file_index_format.cpp index 97c3b772e..8005eb464 100644 --- a/src/paimon/common/file_index/file_index_format.cpp +++ b/src/paimon/common/file_index/file_index_format.cpp @@ -108,13 +108,14 @@ class FileIndexFormatReaderImpl : public FileIndexFormat::Reader { Result> GetFileIndexReader( const std::string& column_name, const std::shared_ptr& arrow_schema, const std::string& index_type, const std::pair& offset_and_length) const { - if (offset_and_length.first == FileIndexFormat::EMPTY_INDEX_FLAG) { - return std::make_shared(); - } const std::map& indexer_options = options_.GetIndexerOptions(column_name, index_type); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_indexer, FileIndexerFactory::Get(index_type, indexer_options)); + if (offset_and_length.first == FileIndexFormat::EMPTY_INDEX_FLAG) { + return file_indexer ? file_indexer->CreateEmptyReader() + : std::make_shared(); + } // assert(file_indexer); if (!file_indexer) { return std::shared_ptr(); diff --git a/src/paimon/common/file_index/file_index_reader.cpp b/src/paimon/common/file_index/file_index_reader.cpp index 7151c5e5a..d51910d2b 100644 --- a/src/paimon/common/file_index/file_index_reader.cpp +++ b/src/paimon/common/file_index/file_index_reader.cpp @@ -24,6 +24,15 @@ #include "paimon/status.h" namespace paimon { +bool FileIndexReader::SupportsSearch(SearchType) const { + return false; +} + +std::optional FileIndexReader::EstimateSelectivity( + const std::shared_ptr&) const { + return std::nullopt; +} + Result> FileIndexReader::VisitIsNotNull() { return FileIndexResult::Remain(); } @@ -107,13 +116,13 @@ Result> FileIndexReader::VisitVectorSearc return Status::NotImplemented("This file index does not support vector search"); } -Result> FileIndexReader::VisitFullTextSearch( +Result> FileIndexReader::VisitFullTextSearch( const std::shared_ptr& full_text_search) { return Status::NotImplemented("This file index does not support full text search"); } Result> FileIndexReader::VisitFullTextMatch( - const std::shared_ptr& full_text_match) { + const std::shared_ptr& full_text_match, bool needs_scores) { return Status::NotImplemented("This file index does not support complete full-text matching"); } } // namespace paimon diff --git a/src/paimon/common/file_index/file_index_reader_test.cpp b/src/paimon/common/file_index/file_index_reader_test.cpp index 78002f718..80e70e817 100644 --- a/src/paimon/common/file_index/file_index_reader_test.cpp +++ b/src/paimon/common/file_index/file_index_reader_test.cpp @@ -87,37 +87,18 @@ TEST(FileIndexReaderTest, TestDefaultIndexReader) { auto full_text_match = std::make_shared("body", R"({"match":{"query":"word"}})"); ASSERT_NOK_WITH_MSG(reader.VisitFullTextMatch(full_text_match), "does not support complete full-text matching"); + ASSERT_NOK_WITH_MSG(reader.VisitFullTextMatch(full_text_match, /*needs_scores=*/true), + "does not support complete full-text matching"); } -TEST(FileIndexReaderTest, TestFullTextMatchVisitorDispatch) { - class FullTextReader : public FileIndexReader { - public: - Result> VisitFullTextMatch( - const std::shared_ptr& full_text_match) override { - received_match_ = full_text_match; - return FileIndexResult::Skip(); - } - - std::shared_ptr received_match_; - }; - - auto match = std::make_shared("body", R"({"match":{"query":"apple"}})"); - ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::FullTextMatch(0, match)); - auto leaf = std::dynamic_pointer_cast(predicate); - ASSERT_TRUE(leaf); - - auto default_reader = std::make_shared(); - auto result = - PredicateUtils::VisitPredicate>(leaf, default_reader); - ASSERT_NOK_WITH_MSG(result, "This file index does not support complete full-text matching"); - ASSERT_TRUE(result.status().IsNotImplemented()); - - auto reader = std::make_shared(); - ASSERT_OK_AND_ASSIGN( - auto matches, - PredicateUtils::VisitPredicate>(leaf, reader)); - ASSERT_EQ(reader->received_match_, match); - ASSERT_OK_AND_ASSIGN(bool is_remain, matches->IsRemain()); - ASSERT_FALSE(is_remain); +TEST(FileIndexReaderTest, TestScalarVisitorAndUnknownSelectivity) { + auto reader = std::make_shared(); + auto predicate = std::dynamic_pointer_cast( + PredicateBuilder::Equal(0, "value", FieldType::INT, Literal(1))); + ASSERT_TRUE(predicate); + ASSERT_FALSE(reader->EstimateSelectivity(predicate).has_value()); + ASSERT_FALSE(reader->SupportsSearch(FileIndexReader::SearchType::FULL_TEXT_MATCH)); + ASSERT_FALSE(reader->SupportsSearch(FileIndexReader::SearchType::FULL_TEXT_SEARCH)); + ASSERT_FALSE(reader->SupportsSearch(FileIndexReader::SearchType::VECTOR_SEARCH)); } } // namespace paimon::test diff --git a/src/paimon/common/file_index/file_indexer.cpp b/src/paimon/common/file_index/file_indexer.cpp new file mode 100644 index 000000000..fb5b9c8f4 --- /dev/null +++ b/src/paimon/common/file_index/file_indexer.cpp @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/file_index/file_indexer.h" + +#include "paimon/common/file_index/empty/empty_file_index_reader.h" + +namespace paimon { + +std::shared_ptr FileIndexer::CreateEmptyReader() const { + return std::make_shared(); +} + +} // namespace paimon diff --git a/src/paimon/common/predicate/predicate_utils.cpp b/src/paimon/common/predicate/predicate_utils.cpp index f8dc25005..83ad478cb 100644 --- a/src/paimon/common/predicate/predicate_utils.cpp +++ b/src/paimon/common/predicate/predicate_utils.cpp @@ -29,15 +29,6 @@ namespace paimon { -Result> PredicateUtils::GetFullTextMatch( - const std::shared_ptr& predicate) { - auto full_text = std::dynamic_pointer_cast(predicate); - if (!full_text) { - return Status::Invalid("FULL_TEXT_MATCH requires a FullTextMatchPredicate"); - } - return full_text->GetFullTextMatch(); -} - bool PredicateUtils::ContainsFullTextMatch(const std::shared_ptr& predicate) { if (std::dynamic_pointer_cast(predicate)) { return true; diff --git a/src/paimon/common/reader/complete_index_score_file_batch_reader.cpp b/src/paimon/common/reader/complete_index_score_file_batch_reader.cpp index 0cf6e418a..0a778d151 100644 --- a/src/paimon/common/reader/complete_index_score_file_batch_reader.cpp +++ b/src/paimon/common/reader/complete_index_score_file_batch_reader.cpp @@ -36,6 +36,17 @@ namespace paimon { CompleteIndexScoreFileBatchReader::CompleteIndexScoreFileBatchReader( std::unique_ptr&& reader, const std::vector& scores, const std::shared_ptr& arrow_pool) + : CompleteIndexScoreFileBatchReader(std::move(reader), + std::map>{}, arrow_pool) { + if (!scores.empty()) { + scores_.emplace(SpecialFields::IndexScore().Name(), scores); + } +} + +CompleteIndexScoreFileBatchReader::CompleteIndexScoreFileBatchReader( + std::unique_ptr&& reader, + const std::map>& scores, + const std::shared_ptr& arrow_pool) : arrow_pool_(arrow_pool), reader_(std::move(reader)), scores_(scores) {} Result CompleteIndexScoreFileBatchReader::NextBatch() { @@ -48,9 +59,12 @@ Result CompleteIndexScoreFileBatchReader::Next PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap, reader_->NextBatchWithBitmap()); if (BatchReader::IsEofBatch(batch_with_bitmap)) { - if (!scores_.empty() && score_cursor_ != scores_.size()) { - return Status::Invalid(fmt::format("Index score count {} does not match rows read {}", - scores_.size(), score_cursor_)); + for (const auto& [field_name, scores] : scores_) { + if (score_cursor_ != scores.size()) { + return Status::Invalid( + fmt::format("Index score count {} for field '{}' does not match rows read {}", + scores.size(), field_name, score_cursor_)); + } } return batch_with_bitmap; } @@ -68,53 +82,67 @@ Result CompleteIndexScoreFileBatchReader::Next } auto struct_array = checked_pointer_cast(arrow_array); auto struct_type = struct_array->struct_type(); - UpdateScoreFieldIndex(struct_type); - - std::unique_ptr index_score_builder; - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::MakeBuilder( - arrow_pool_.get(), SpecialFields::IndexScore().Type(), &index_score_builder)); - if (!index_score_builder || !index_score_builder->type() || - index_score_builder->type()->id() != arrow::Type::FLOAT) { - return Status::Invalid("cannot cast index score builder to FloatBuilder"); - } - auto* typed_builder = checked_cast(index_score_builder.get()); - PAIMON_RETURN_NOT_OK_FROM_ARROW(typed_builder->Reserve(struct_array->length())); - bool all_not_null = (struct_array->length() == bitmap.Cardinality()); - size_t score_count = all_not_null ? static_cast(struct_array->length()) - : static_cast(bitmap.Cardinality()); - if (score_cursor_ > scores_.size() || score_count > scores_.size() - score_cursor_) { - return Status::Invalid(fmt::format("Index score count {} is smaller than rows read {}", - scores_.size(), score_cursor_ + score_count)); - } - for (int64_t i = 0; i < struct_array->length(); i++) { - if (all_not_null || bitmap.Contains(i)) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(typed_builder->Append(scores_[score_cursor_++])); - } else { - PAIMON_RETURN_NOT_OK_FROM_ARROW(typed_builder->AppendNull()); - } - } - std::shared_ptr index_score_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW(typed_builder->Finish(&index_score_array)); arrow::ArrayVector array_vec = struct_array->fields(); - array_vec[index_score_field_idx_] = index_score_array; + for (const auto& [field_name, scores] : scores_) { + PAIMON_ASSIGN_OR_RAISE(int32_t field_index, GetScoreFieldIndex(struct_type, field_name)); + PAIMON_ASSIGN_OR_RAISE( + array_vec[field_index], + PrepareScoreArray(field_name, scores, struct_array->length(), bitmap)); + } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array_with_score, - arrow::StructArray::Make(array_vec, field_names_with_score_)); + arrow::StructArray::Make(array_vec, struct_type->fields())); PAIMON_RETURN_NOT_OK_FROM_ARROW( arrow::ExportArray(*array_with_score, c_array.get(), c_schema.get())); PAIMON_RETURN_NOT_OK(AddArrowArrayLifetime(c_array.get(), c_schema.get(), arrow_pool_)); + score_cursor_ += bitmap.Cardinality(); return batch_with_bitmap; } -void CompleteIndexScoreFileBatchReader::UpdateScoreFieldIndex( - const arrow::StructType* struct_type) { - if (index_score_field_idx_ != -1) { - return; +Result CompleteIndexScoreFileBatchReader::GetScoreFieldIndex( + const arrow::StructType* struct_type, const std::string& field_name) { + auto iter = index_score_field_indices_.find(field_name); + if (iter != index_score_field_indices_.end()) { + return iter->second; } - index_score_field_idx_ = struct_type->GetFieldIndex(SpecialFields::IndexScore().Name()); - field_names_with_score_.reserve(struct_type->num_fields()); - for (const auto& field : struct_type->fields()) { - field_names_with_score_.push_back(field->name()); + const auto field_indices = struct_type->GetAllFieldIndices(field_name); + if (field_indices.empty()) { + return Status::Invalid(fmt::format("Missing index score field '{}'", field_name)); + } + if (field_indices.size() != 1) { + return Status::Invalid(fmt::format("Duplicate index score field '{}'", field_name)); + } + int32_t field_index = field_indices[0]; + if (struct_type->field(field_index)->type()->id() != arrow::Type::FLOAT) { + return Status::Invalid( + fmt::format("Index score field '{}' must have float32 type", field_name)); + } + index_score_field_indices_.emplace(field_name, field_index); + return field_index; +} + +Result> CompleteIndexScoreFileBatchReader::PrepareScoreArray( + const std::string& field_name, const std::vector& scores, int64_t row_count, + const RoaringBitmap32& bitmap) const { + size_t score_count = bitmap.Cardinality(); + if (score_cursor_ > scores.size() || score_count > scores.size() - score_cursor_) { + return Status::Invalid( + fmt::format("Index score count {} for field '{}' is smaller than rows read {}", + scores.size(), field_name, score_cursor_ + score_count)); + } + arrow::FloatBuilder builder(arrow_pool_.get()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(row_count)); + bool all_not_null = (row_count == bitmap.Cardinality()); + size_t cursor = score_cursor_; + for (int64_t i = 0; i < row_count; i++) { + if (all_not_null || bitmap.Contains(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(scores[cursor++])); + } else { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull()); + } } + std::shared_ptr score_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&score_array)); + return score_array; } Result> CompleteIndexScoreFileBatchReader::GetFileSchema() const { @@ -125,8 +153,7 @@ Status CompleteIndexScoreFileBatchReader::SetReadSchema( ::ArrowSchema* read_schema, const std::shared_ptr& predicate, const std::optional& selection_bitmap) { score_cursor_ = 0; - index_score_field_idx_ = -1; - field_names_with_score_.clear(); + index_score_field_indices_.clear(); return reader_->SetReadSchema(read_schema, predicate, selection_bitmap); } diff --git a/src/paimon/common/reader/complete_index_score_file_batch_reader.h b/src/paimon/common/reader/complete_index_score_file_batch_reader.h index 17e2bcd88..8269ca306 100644 --- a/src/paimon/common/reader/complete_index_score_file_batch_reader.h +++ b/src/paimon/common/reader/complete_index_score_file_batch_reader.h @@ -19,7 +19,9 @@ #pragma once +#include #include +#include #include #include #include @@ -36,6 +38,11 @@ class CompleteIndexScoreFileBatchReader : public FileBatchReader { const std::vector& scores, const std::shared_ptr& arrow_pool); + /// Scores are keyed by output field name and aligned with the selected rows in file order. + CompleteIndexScoreFileBatchReader(std::unique_ptr&& reader, + const std::map>& scores, + const std::shared_ptr& arrow_pool); + Result NextBatch() override; Result NextBatchWithBitmap() override; Result> GetFileSchema() const override; @@ -49,13 +56,18 @@ class CompleteIndexScoreFileBatchReader : public FileBatchReader { std::shared_ptr GetReaderMetrics() const override; private: - void UpdateScoreFieldIndex(const arrow::StructType* struct_type); + Result GetScoreFieldIndex(const arrow::StructType* struct_type, + const std::string& field_name); + + Result> PrepareScoreArray(const std::string& field_name, + const std::vector& scores, + int64_t row_count, + const RoaringBitmap32& bitmap) const; size_t score_cursor_ = 0; - int32_t index_score_field_idx_ = -1; - std::vector field_names_with_score_; + std::map index_score_field_indices_; std::shared_ptr arrow_pool_; std::unique_ptr reader_; - std::vector scores_; + std::map> scores_; }; } // namespace paimon diff --git a/src/paimon/common/reader/complete_index_score_file_batch_reader_test.cpp b/src/paimon/common/reader/complete_index_score_file_batch_reader_test.cpp index 76ca34983..5a1def7ea 100644 --- a/src/paimon/common/reader/complete_index_score_file_batch_reader_test.cpp +++ b/src/paimon/common/reader/complete_index_score_file_batch_reader_test.cpp @@ -20,7 +20,9 @@ #include "paimon/common/reader/complete_index_score_file_batch_reader.h" #include +#include #include +#include #include #include @@ -190,4 +192,160 @@ TEST_F(CompleteIndexScoreFileBatchReaderTest, TestSetReadSchemaReordersScoreColu ASSERT_TRUE(expected_array->Equals(*result_array)) << result_array->ToString(); } +TEST_F(CompleteIndexScoreFileBatchReaderTest, TestMultipleScoresFollowSelectedRowsAndProjection) { + arrow::FieldVector fields = {arrow::field("score_z", arrow::float32()), + arrow::field("f0", arrow::int32()), + arrow::field("score_a", arrow::float32())}; + auto type = arrow::struct_(fields); + auto data = arrow::ipc::internal::json::ArrayFromJSON(type, R"([ + [null, 10, null], [null, 11, null], [null, 12, null], + [null, 13, null], [null, 14, null], [null, 15, null], + [null, 16, null], [null, 17, null], [null, 18, null], + [null, 19, null], [null, 20, null], [null, 21, null] + ])") + .ValueOrDie(); + auto selected = RoaringBitmap32::From({0, 2, 8, 9, 10, 11}); + auto inner = std::make_unique(data, type, selected, + /*read_batch_size=*/3); + inner->EnableRandomizeBatchSize(false); + std::map> scores = { + {"score_a", {1.25f, 2.5f, 3.75f, 5.0f, 6.25f, 7.5f}}, + {"score_z", {12.5f, 25.0f, 37.5f, 50.0f, 62.5f, 75.0f}}}; + CompleteIndexScoreFileBatchReader reader(std::move(inner), scores, GetArrowPool(pool_)); + + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap first, reader.NextBatchWithBitmap()); + ASSERT_EQ(RoaringBitmap32::From({0, 2}), first.second); + ASSERT_OK_AND_ASSIGN(std::shared_ptr first_array, + ReadResultCollector::GetArray(std::move(first.first))); + auto expected_first = arrow::ipc::internal::json::ArrayFromJSON( + type, R"([[12.5, 10, 1.25], [null, 11, null], [25, 12, 2.5]])") + .ValueOrDie(); + ASSERT_TRUE(expected_first->Equals(*first_array)) << first_array->ToString(); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr result_array, + ReadResultCollector::CollectResult(&reader)); + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + type, {R"([[37.5, 18, 3.75], [50, 19, 5], [62.5, 20, 6.25], [75, 21, 7.5]])"}, + &expected_array) + .ok()); + ASSERT_TRUE(expected_array->Equals(*result_array)) << result_array->ToString(); + + arrow::FieldVector reordered_fields = {fields[2], fields[0], fields[1]}; + ::ArrowSchema read_schema; + ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(reordered_fields), &read_schema).ok()); + ASSERT_OK(reader.SetReadSchema(&read_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/selected)); + ASSERT_OK_AND_ASSIGN(result_array, ReadResultCollector::CollectResult(&reader)); + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(reordered_fields), + {R"([[1.25, 12.5, 10], [2.5, 25, 12], [3.75, 37.5, 18], + [5, 50, 19], [6.25, 62.5, 20], [7.5, 75, 21]])"}, + &expected_array) + .ok()); + ASSERT_TRUE(expected_array->Equals(*result_array)) << result_array->ToString(); +} + +TEST_F(CompleteIndexScoreFileBatchReaderTest, TestMultipleScoresWithEmptyResult) { + auto type = arrow::struct_( + {arrow::field("score_a", arrow::float32()), arrow::field("score_b", arrow::float32())}); + for (const std::string& json : {"[]", "[[null, null]]"}) { + auto data = arrow::ipc::internal::json::ArrayFromJSON(type, json).ValueOrDie(); + auto inner = std::make_unique(data, type, RoaringBitmap32(), + /*read_batch_size=*/1); + CompleteIndexScoreFileBatchReader reader( + std::move(inner), + std::map>{{"score_a", {}}, {"score_b", {}}}, + GetArrowPool(pool_)); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap eof, reader.NextBatchWithBitmap()); + ASSERT_TRUE(BatchReader::IsEofBatch(eof)); + } +} + +TEST_F(CompleteIndexScoreFileBatchReaderTest, TestRejectsTooFewScoresForEachField) { + auto type = arrow::struct_( + {arrow::field("score_a", arrow::float32()), arrow::field("score_b", arrow::float32())}); + auto data = arrow::ipc::internal::json::ArrayFromJSON(type, "[[null, null], [null, null]]") + .ValueOrDie(); + for (const std::string& field_name : {"score_a", "score_b"}) { + auto inner = std::make_unique(data, type, /*read_batch_size=*/1); + std::map> scores = {{"score_a", {1.0f, 2.0f}}, + {"score_b", {3.0f, 4.0f}}}; + scores[field_name].resize(1); + CompleteIndexScoreFileBatchReader reader(std::move(inner), scores, GetArrowPool(pool_)); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch first, reader.NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr first_array, + ReadResultCollector::GetArray(std::move(first))); + ASSERT_TRUE(first_array); + ASSERT_EQ(1, first_array->length()); + ASSERT_NOK_WITH_MSG(reader.NextBatch(), "Index score count 1 for field '" + field_name + + "' is smaller than rows read 2"); + } +} + +TEST_F(CompleteIndexScoreFileBatchReaderTest, TestRejectsUnusedScoresForEachField) { + auto type = arrow::struct_( + {arrow::field("score_a", arrow::float32()), arrow::field("score_b", arrow::float32())}); + auto data = arrow::ipc::internal::json::ArrayFromJSON(type, "[[null, null]]").ValueOrDie(); + for (const std::string& field_name : {"score_a", "score_b"}) { + auto inner = std::make_unique(data, type, /*read_batch_size=*/1); + std::map> scores = {{"score_a", {1.0f}}, + {"score_b", {3.0f}}}; + scores[field_name].push_back(2.0f); + CompleteIndexScoreFileBatchReader reader(std::move(inner), scores, GetArrowPool(pool_)); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch first, reader.NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr first_array, + ReadResultCollector::GetArray(std::move(first))); + ASSERT_TRUE(first_array); + ASSERT_EQ(1, first_array->length()); + ASSERT_NOK_WITH_MSG(reader.NextBatch(), "Index score count 2 for field '" + field_name + + "' does not match rows read 1"); + } +} + +TEST_F(CompleteIndexScoreFileBatchReaderTest, TestRejectsEmptyScoresWithNonemptyResult) { + auto type = arrow::struct_({arrow::field("score", arrow::float32())}); + auto data = arrow::ipc::internal::json::ArrayFromJSON(type, "[[null]]").ValueOrDie(); + auto inner = std::make_unique(data, type, /*read_batch_size=*/1); + CompleteIndexScoreFileBatchReader reader( + std::move(inner), std::map>{{"score", {}}}, + GetArrowPool(pool_)); + ASSERT_NOK_WITH_MSG(reader.NextBatch(), + "Index score count 0 for field 'score' is smaller than rows read 1"); +} + +TEST_F(CompleteIndexScoreFileBatchReaderTest, TestRejectsInvalidScoreFields) { + auto check_error = [&](const arrow::FieldVector& fields, const std::string& json, + const std::string& message) { + auto type = arrow::struct_(fields); + auto data = arrow::ipc::internal::json::ArrayFromJSON(type, json).ValueOrDie(); + auto inner = std::make_unique(data, type, /*read_batch_size=*/1); + CompleteIndexScoreFileBatchReader reader( + std::move(inner), std::map>{{"score", {1.25f}}}, + GetArrowPool(pool_)); + ASSERT_NOK_WITH_MSG(reader.NextBatch(), message); + }; + auto score = arrow::field("score", arrow::float32()); + check_error({arrow::field("f0", arrow::int32())}, "[[10]]", + "Missing index score field 'score'"); + check_error({score, score}, "[[null, null]]", "Duplicate index score field 'score'"); + check_error({arrow::field("score", arrow::float64())}, "[[null]]", + "Index score field 'score' must have float32 type"); +} + +TEST_F(CompleteIndexScoreFileBatchReaderTest, TestLegacyEmptyScoresPassThrough) { + auto type = arrow::struct_({arrow::field("f0", arrow::int32())}); + auto data = arrow::ipc::internal::json::ArrayFromJSON(type, "[[10], [11]]").ValueOrDie(); + auto inner = std::make_unique(data, type, /*read_batch_size=*/2); + inner->EnableRandomizeBatchSize(false); + CompleteIndexScoreFileBatchReader reader(std::move(inner), std::vector{}, + GetArrowPool(pool_)); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader.NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(data->Equals(*result)) << result->ToString(); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof, reader.NextBatch()); + ASSERT_TRUE(BatchReader::IsEofBatch(eof)); +} + } // namespace paimon::test diff --git a/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp b/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp index 9c3ff443e..1407c55b7 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.cpp @@ -153,7 +153,7 @@ Result> PkSortedDataFileReader::ApplyIndexAndDv const std::shared_ptr&, const std::shared_ptr& read_schema, const std::shared_ptr&, DeletionVector::Factory, const std::optional>&, const std::shared_ptr&, - const std::map&, std::vector*) const { + const std::map&) const { ::ArrowSchema c_read_schema; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, &c_read_schema)); PAIMON_RETURN_NOT_OK(file_reader->SetReadSchema(&c_read_schema, /*predicate=*/nullptr, diff --git a/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.h b/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.h index 2dc74e4b6..3e32eae39 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.h +++ b/src/paimon/core/index/pksorted/pk_sorted_data_file_reader.h @@ -73,8 +73,7 @@ class PkSortedDataFileReader : public RawFileSplitRead { const std::shared_ptr& predicate, DeletionVector::Factory dv_factory, const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory, - const std::map& file_index_options, - std::vector* index_scores) const override; + const std::map& file_index_options) const override; private: PkSortedDataFileReader(const std::shared_ptr& path_factory, diff --git a/src/paimon/core/io/file_index_evaluator.cpp b/src/paimon/core/io/file_index_evaluator.cpp index 59c20f477..98d9b6e21 100644 --- a/src/paimon/core/io/file_index_evaluator.cpp +++ b/src/paimon/core/io/file_index_evaluator.cpp @@ -19,6 +19,7 @@ #include "paimon/core/io/file_index_evaluator.h" #include +#include #include #include #include @@ -30,6 +31,7 @@ #include "arrow/type.h" #include "fmt/format.h" #include "fmt/ranges.h" +#include "paimon/common/predicate/full_text_match_predicate.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/io/data_file_meta.h" @@ -52,60 +54,163 @@ namespace paimon { class MemoryPool; enum class FieldType; -Result> FileIndexEvaluator::Evaluate( +FileIndexEvaluator::FileIndexEvaluator(std::shared_ptr data_schema, + std::shared_ptr file_meta, + std::unique_ptr format_reader) + : data_schema_(std::move(data_schema)), + file_meta_(std::move(file_meta)), + format_reader_(std::move(format_reader)) {} + +Result> FileIndexEvaluator::Create( const std::shared_ptr& data_schema, - const std::map& options, const std::shared_ptr& predicate, - const std::shared_ptr& data_file_path_factory, + const std::map& options, + const std::shared_ptr& path_factory, const std::shared_ptr& file_meta, const std::shared_ptr& file_system, - const std::shared_ptr& pool) { - return Evaluate(/*only_use_embedding_index=*/false, data_schema, options, predicate, - data_file_path_factory, file_meta, file_system, pool); + bool only_use_embedding_index, const std::shared_ptr& pool) { + std::unique_ptr format_reader; + if (file_meta->row_count >= 0 && file_meta->row_count <= RoaringBitmap32::MAX_VALUE) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, + ExtractIndexInputStream(only_use_embedding_index, path_factory, + file_meta, file_system)); + if (input_stream) { + PAIMON_ASSIGN_OR_RAISE(format_reader, + FileIndexFormat::CreateReader(input_stream, pool, options)); + } + } + return std::unique_ptr( + new FileIndexEvaluator(data_schema, file_meta, std::move(format_reader))); } -Result> FileIndexEvaluator::Evaluate( - const std::shared_ptr& data_schema, - const std::map& options, const std::shared_ptr& predicate, - const std::shared_ptr& file_meta, const std::shared_ptr& pool) { - return Evaluate(/*only_use_embedding_index=*/true, data_schema, options, predicate, - /*data_file_path_factory=*/nullptr, file_meta, /*file_system=*/nullptr, pool); +Result>> FileIndexEvaluator::GetReaders( + const std::string& field_name) { + auto iter = field_readers_.find(field_name); + if (iter != field_readers_.end()) { + return iter->second; + } + std::shared_ptr field = data_schema_->GetFieldByName(field_name); + if (!field) { + return Status::Invalid(fmt::format("field '{}' does not exist in data schema", field_name)); + } + std::vector> readers; + if (format_reader_) { + ::ArrowSchema c_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportSchema(*arrow::schema({std::move(field)}), &c_schema)); + PAIMON_ASSIGN_OR_RAISE(readers, format_reader_->ReadColumnIndex(field_name, &c_schema)); + } + field_readers_.emplace(field_name, readers); + return readers; } -Result> FileIndexEvaluator::Evaluate( - bool only_use_embedding_index, const std::shared_ptr& data_schema, - const std::map& options, const std::shared_ptr& predicate, - const std::shared_ptr& data_file_path_factory, - const std::shared_ptr& file_meta, const std::shared_ptr& file_system, - const std::shared_ptr& pool) { - if (predicate == nullptr || file_meta->row_count > RoaringBitmap32::MAX_VALUE) { - return FileIndexResult::Remain(); +Result> FileIndexEvaluator::GetSearchReader( + const std::string& field_name, FileIndexReader::SearchType search_type) { + if (!data_schema_->GetFieldByName(field_name)) { + return Status::Invalid( + fmt::format("Search field '{}' does not exist in data schema", field_name)); + } + if (!format_reader_) { + return Status::Invalid(fmt::format("Data file '{}' has no File Index for search field '{}'", + file_meta_->file_name, field_name)); + } + PAIMON_ASSIGN_OR_RAISE(std::vector> readers, + GetReaders(field_name)); + std::shared_ptr selected; + for (const auto& reader : readers) { + if (!reader->SupportsSearch(search_type)) { + continue; + } + if (selected) { + return Status::Invalid(fmt::format( + "Multiple File Index readers support requested search for field '{}'", field_name)); + } + selected = reader; + } + if (!selected) { + return Status::NotImplemented(fmt::format( + "No File Index reader supports requested search for field '{}'", field_name)); + } + return selected; +} + +Result> FileIndexEvaluator::EvaluateLeaf( + const std::shared_ptr& predicate, bool needs_scores) { + if (predicate->GetFunction().GetType() == Function::Type::FULL_TEXT_MATCH) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr reader, + GetSearchReader(predicate->FieldName(), FileIndexReader::SearchType::FULL_TEXT_MATCH)); + auto full_text = std::dynamic_pointer_cast(predicate); + if (!full_text) { + return Status::Invalid("FullTextMatch requires a full-text match predicate"); + } + return reader->VisitFullTextMatch(full_text->GetFullTextMatch(), needs_scores); + } + PAIMON_ASSIGN_OR_RAISE(std::vector> readers, + GetReaders(predicate->FieldName())); + std::shared_ptr result = FileIndexResult::Remain(); + for (const auto& reader : readers) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr child, + PredicateUtils::VisitPredicate>(predicate, reader)); + PAIMON_ASSIGN_OR_RAISE(result, result->And(child)); + PAIMON_ASSIGN_OR_RAISE(bool remain, result->IsRemain()); + if (!remain) { + break; + } } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, - ExtractIndexInputStream(only_use_embedding_index, data_file_path_factory, - file_meta, file_system)); - if (!input_stream) { - // no index files, just return REMAIN + return result; +} + +Result> FileIndexEvaluator::EstimateSelectivity( + const std::shared_ptr& predicate) { + if (predicate->GetFunction().GetType() == Function::Type::FULL_TEXT_MATCH) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr reader, + GetSearchReader(predicate->FieldName(), FileIndexReader::SearchType::FULL_TEXT_MATCH)); + return reader->EstimateSelectivity(predicate); + } + PAIMON_ASSIGN_OR_RAISE(std::vector> readers, + GetReaders(predicate->FieldName())); + std::optional estimate; + for (const auto& reader : readers) { + auto value = reader->EstimateSelectivity(predicate); + if (value && std::isfinite(*value) && *value >= 0.0 && *value <= 1.0 && + (!estimate || *value < *estimate)) { + estimate = value; + } + } + return estimate; +} + +Result> FileIndexEvaluator::SearchVector( + const std::shared_ptr& search) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr reader, + GetSearchReader(search->field_name, FileIndexReader::SearchType::VECTOR_SEARCH)); + return reader->VisitVectorSearch(search); +} + +Result> FileIndexEvaluator::SearchFullText( + const std::shared_ptr& search) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr reader, + GetSearchReader(search->field_name, FileIndexReader::SearchType::FULL_TEXT_SEARCH)); + return reader->VisitFullTextSearch(search); +} + +Result> FileIndexEvaluator::Evaluate( + const std::shared_ptr& predicate) { + if (!predicate || file_meta_->row_count < 0 || + file_meta_->row_count > RoaringBitmap32::MAX_VALUE || !format_reader_) { return FileIndexResult::Remain(); } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_reader, - FileIndexFormat::CreateReader(input_stream, pool, options)); + // Load every referenced field before short-circuiting, so missing fields and invalid indexes + // are reported even when an earlier branch already excludes the file. std::set required_field_names; PAIMON_RETURN_NOT_OK(PredicateUtils::GetAllNames(predicate, &required_field_names)); - std::map>> - field_name_to_index_readers; for (const auto& field_name : required_field_names) { - auto data_field = data_schema->GetFieldByName(field_name); - if (data_field == nullptr) { - return Status::Invalid(fmt::format("field {} not exist in data schema {}", field_name, - data_schema->ToString())); - } - ArrowSchema c_arrow_schema; - PAIMON_RETURN_NOT_OK_FROM_ARROW( - arrow::ExportSchema(*arrow::schema({data_field}), &c_arrow_schema)); - PAIMON_ASSIGN_OR_RAISE(std::vector> index_readers, - format_reader->ReadColumnIndex(field_name, &c_arrow_schema)); - field_name_to_index_readers.emplace(field_name, std::move(index_readers)); + PAIMON_RETURN_NOT_OK(GetReaders(field_name)); } - return Evaluate(predicate, field_name_to_index_readers); + return EvaluatePredicate(predicate); } Result> FileIndexEvaluator::ExtractIndexInputStream( @@ -151,170 +256,43 @@ Result> FileIndexEvaluator::ExtractIndexInputStream return std::shared_ptr(); } -namespace { - -Result>> OpenSearchReaders( - const std::shared_ptr& data_schema, const std::string& field_name, - const std::map& options, - const std::shared_ptr& file_meta, - const std::shared_ptr& input_stream, const std::shared_ptr& pool) { - std::shared_ptr field = data_schema->GetFieldByName(field_name); - if (!field) { - return Status::Invalid( - fmt::format("Search field '{}' does not exist in data schema", field_name)); - } - if (!input_stream) { - return Status::Invalid(fmt::format("Data file '{}' has no File Index for search field '{}'", - file_meta->file_name, field_name)); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_reader, - FileIndexFormat::CreateReader(input_stream, pool, options)); - ::ArrowSchema c_schema; - PAIMON_RETURN_NOT_OK_FROM_ARROW( - arrow::ExportSchema(*arrow::schema({std::move(field)}), &c_schema)); - return format_reader->ReadColumnIndex(field_name, &c_schema); -} - -Result> ExecuteVectorSearch( - const std::vector>& readers, - const std::shared_ptr& vector_search) { - if (readers.empty()) { - return Status::NotImplemented( - fmt::format("No File Index reader supports vector search for field '{}'", - vector_search->field_name)); - } - if (readers.size() != 1) { - return Status::Invalid( - fmt::format("Multiple File Index readers exist for vector search field '{}'", - vector_search->field_name)); - } - return readers[0]->VisitVectorSearch(vector_search); -} - -Result> ExecuteFullTextSearch( - const std::vector>& readers, - const std::shared_ptr& full_text_search) { - if (readers.empty()) { - return Status::NotImplemented( - fmt::format("No File Index reader supports full-text search for field '{}'", - full_text_search->field_name)); +Result> FileIndexEvaluator::EvaluatePredicate( + const std::shared_ptr& predicate) { + if (auto compound = std::dynamic_pointer_cast(predicate)) { + return EvaluateCompoundPredicate(compound); } - if (readers.size() != 1) { - return Status::Invalid( - fmt::format("Multiple File Index readers exist for full-text search field '{}'", - full_text_search->field_name)); - } - return readers[0]->VisitFullTextSearch(full_text_search); -} - -} // namespace - -Result> FileIndexEvaluator::Evaluate( - const std::shared_ptr& predicate, - const std::map>>& - field_name_to_index_readers) { - if (auto compound_predicate = std::dynamic_pointer_cast(predicate)) { - return EvaluateCompoundPredicate(compound_predicate, field_name_to_index_readers); - } else if (auto leaf_predicate = std::dynamic_pointer_cast(predicate)) { - std::shared_ptr compound_result = FileIndexResult::Remain(); - // TODO(xinyu.lxy): erase predicate, rm predicate which hits bitmap index - auto iter = field_name_to_index_readers.find(leaf_predicate->FieldName()); - if (iter == field_name_to_index_readers.end()) { - assert(false); - return Status::Invalid(fmt::format( - "field name {} in leaf literal does not exist in field_name_to_index_readers", - leaf_predicate->FieldName())); - } - for (const auto& index_reader : iter->second) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr sub_result, - PredicateUtils::VisitPredicate>( - leaf_predicate, index_reader)); - PAIMON_ASSIGN_OR_RAISE(compound_result, compound_result->And(sub_result)); - PAIMON_ASSIGN_OR_RAISE(bool is_remain, compound_result->IsRemain()); - if (!is_remain) { - return compound_result; - } - } - assert(compound_result); - // if index readers is empty (the index type is not registered), return - // FileIndexResult::Remain() - return compound_result; + if (auto leaf = std::dynamic_pointer_cast(predicate)) { + return EvaluateLeaf(leaf, /*needs_scores=*/false); } return Status::Invalid(fmt::format( "cannot cast predicate {} to CompoundPredicate or LeafPredicate", predicate->ToString())); } Result> FileIndexEvaluator::EvaluateCompoundPredicate( - const std::shared_ptr& compound_predicate, - const std::map>>& - field_name_to_index_readers) { - if (compound_predicate->GetFunction().GetType() == Function::Type::OR) { - std::shared_ptr compound_result; - for (const auto& child : compound_predicate->Children()) { - if (compound_result == nullptr) { - PAIMON_ASSIGN_OR_RAISE(compound_result, - Evaluate(child, field_name_to_index_readers)); - } else { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr child_result, - Evaluate(child, field_name_to_index_readers)); - PAIMON_ASSIGN_OR_RAISE(compound_result, compound_result->Or(child_result)); - } + const std::shared_ptr& compound_predicate) { + const Function::Type type = compound_predicate->GetFunction().GetType(); + if (type != Function::Type::AND && type != Function::Type::OR) { + return Status::Invalid("CompoundPredicate only support And/Or function"); + } + std::shared_ptr result; + for (const auto& child : compound_predicate->Children()) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr child_result, + EvaluatePredicate(child)); + if (!result) { + result = std::move(child_result); + } else { + PAIMON_ASSIGN_OR_RAISE(result, type == Function::Type::AND ? result->And(child_result) + : result->Or(child_result)); } - assert(compound_result); - return compound_result; - } else if (compound_predicate->GetFunction().GetType() == Function::Type::AND) { - std::shared_ptr compound_result; - for (const auto& child : compound_predicate->Children()) { - if (compound_result == nullptr) { - PAIMON_ASSIGN_OR_RAISE(compound_result, - Evaluate(child, field_name_to_index_readers)); - } else { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr child_result, - Evaluate(child, field_name_to_index_readers)); - PAIMON_ASSIGN_OR_RAISE(compound_result, compound_result->And(child_result)); - } - // if not remain, no need to test anymore - PAIMON_ASSIGN_OR_RAISE(bool is_remain, compound_result->IsRemain()); + if (type == Function::Type::AND) { + PAIMON_ASSIGN_OR_RAISE(bool is_remain, result->IsRemain()); if (!is_remain) { - return compound_result; + return result; } } - assert(compound_result); - return compound_result; } - return Status::Invalid("CompoundPredicate only support And/Or function"); -} - -Result> FileIndexEvaluator::EvaluateVectorSearch( - const std::shared_ptr& data_schema, - const std::map& options, - const std::shared_ptr& vector_search, - const std::shared_ptr& path_factory, - const std::shared_ptr& file_meta, const std::shared_ptr& file_system, - const std::shared_ptr& pool) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, - ExtractIndexInputStream(/*only_use_embedding_index=*/false, path_factory, - file_meta, file_system)); - PAIMON_ASSIGN_OR_RAISE(std::vector> readers, - OpenSearchReaders(data_schema, vector_search->field_name, options, - file_meta, input_stream, pool)); - return ExecuteVectorSearch(readers, vector_search); -} - -Result> FileIndexEvaluator::EvaluateFullTextSearch( - const std::shared_ptr& data_schema, - const std::map& options, - const std::shared_ptr& full_text_search, - const std::shared_ptr& path_factory, - const std::shared_ptr& file_meta, const std::shared_ptr& file_system, - const std::shared_ptr& pool) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, - ExtractIndexInputStream(/*only_use_embedding_index=*/false, path_factory, - file_meta, file_system)); - PAIMON_ASSIGN_OR_RAISE(std::vector> readers, - OpenSearchReaders(data_schema, full_text_search->field_name, options, - file_meta, input_stream, pool)); - return ExecuteFullTextSearch(readers, full_text_search); + assert(result); + return result; } } // namespace paimon diff --git a/src/paimon/core/io/file_index_evaluator.h b/src/paimon/core/io/file_index_evaluator.h index 82b58ee0f..f046990dc 100644 --- a/src/paimon/core/io/file_index_evaluator.h +++ b/src/paimon/core/io/file_index_evaluator.h @@ -19,11 +19,12 @@ #pragma once #include #include -#include +#include #include #include #include "paimon/core/io/data_file_path_factory.h" +#include "paimon/file_index/file_index_format.h" #include "paimon/file_index/file_index_reader.h" #include "paimon/file_index/file_index_result.h" #include "paimon/fs/file_system.h" @@ -46,53 +47,46 @@ struct DataFileMeta; struct FullTextSearch; struct VectorSearch; -/// Evaluate indexes using predicates and search requests expressed in file field names. -/// Backend options are also expressed in file field names. +/// Evaluates indexes for one data file, sharing its container and field readers across operations. +/// Predicates, search requests and backend options use file field names. class FileIndexEvaluator { public: - FileIndexEvaluator() = delete; - ~FileIndexEvaluator() = delete; - - // for scan process, will only use embedding file to evaluate - static Result> Evaluate( + /// Open one index container and reuse its field readers throughout a single file read. + /// Scan planning uses only embedded indexes; file reads also allow external indexes. + /// Files whose row counts cannot be represented by bitmap32 skip index loading. + static Result> Create( const std::shared_ptr& data_schema, const std::map& options, - const std::shared_ptr& predicate, const std::shared_ptr& file_meta, + const std::shared_ptr& path_factory, + const std::shared_ptr& file_meta, + const std::shared_ptr& file_system, bool only_use_embedding_index, const std::shared_ptr& pool); - // for read process, will use embedding file or extra index file to evaluate - static Result> Evaluate( - const std::shared_ptr& data_schema, - const std::map& options, - const std::shared_ptr& predicate, - const std::shared_ptr& data_file_path_factory, - const std::shared_ptr& file_meta, - const std::shared_ptr& file_system, const std::shared_ptr& pool); + /// Evaluates a predicate tree for index pruning, reusing the cached field readers. + /// This does not perform exact scalar filtering or reorder children by selectivity. + Result> Evaluate(const std::shared_ptr& predicate); - static Result> EvaluateVectorSearch( - const std::shared_ptr& data_schema, - const std::map& options, - const std::shared_ptr& vector_search, - const std::shared_ptr& path_factory, - const std::shared_ptr& file_meta, - const std::shared_ptr& file_system, const std::shared_ptr& pool); + Result> GetSearchReader( + const std::string& field_name, FileIndexReader::SearchType search_type); - static Result> EvaluateFullTextSearch( - const std::shared_ptr& data_schema, - const std::map& options, - const std::shared_ptr& full_text_search, - const std::shared_ptr& path_factory, - const std::shared_ptr& file_meta, - const std::shared_ptr& file_system, const std::shared_ptr& pool); + Result> EvaluateLeaf( + const std::shared_ptr& predicate, bool needs_scores); + + Result> EstimateSelectivity( + const std::shared_ptr& predicate); + + Result> SearchVector( + const std::shared_ptr& search); + + Result> SearchFullText( + const std::shared_ptr& search); private: - static Result> Evaluate( - bool only_use_embedding_index, const std::shared_ptr& data_schema, - const std::map& options, - const std::shared_ptr& predicate, - const std::shared_ptr& data_file_path_factory, - const std::shared_ptr& file_meta, - const std::shared_ptr& file_system, const std::shared_ptr& pool); + FileIndexEvaluator(std::shared_ptr data_schema, + std::shared_ptr file_meta, + std::unique_ptr format_reader); + + Result>> GetReaders(const std::string& field_name); static Result> ExtractIndexInputStream( bool only_use_embedding_index, @@ -100,15 +94,17 @@ class FileIndexEvaluator { const std::shared_ptr& file_meta, const std::shared_ptr& file_system); - static Result> Evaluate( - const std::shared_ptr& predicate, - const std::map>>& - field_name_to_index_readers); + Result> EvaluatePredicate( + const std::shared_ptr& predicate); + + Result> EvaluateCompoundPredicate( + const std::shared_ptr& compound_predicate); - static Result> EvaluateCompoundPredicate( - const std::shared_ptr& compound_predicate, - const std::map>>& - field_name_to_index_readers); + private: + std::shared_ptr data_schema_; + std::shared_ptr file_meta_; + std::unique_ptr format_reader_; + std::map>> field_readers_; }; } // namespace paimon diff --git a/src/paimon/core/io/file_index_evaluator_test.cpp b/src/paimon/core/io/file_index_evaluator_test.cpp index 1a0168efc..a6f51a15d 100644 --- a/src/paimon/core/io/file_index_evaluator_test.cpp +++ b/src/paimon/core/io/file_index_evaluator_test.cpp @@ -28,8 +28,11 @@ #include #include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/common/data/binary_row.h" +#include "paimon/common/file_index/bitmap/bitmap_file_index.h" #include "paimon/common/fs/external_path_provider.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/date_time_utils.h" @@ -40,13 +43,17 @@ #include "paimon/core/stats/simple_stats.h" #include "paimon/data/timestamp.h" #include "paimon/defs.h" +#include "paimon/factories/factory_creator.h" #include "paimon/file_index/bitmap_index_result.h" #include "paimon/file_index/file_index_format.h" +#include "paimon/file_index/file_indexer_factory.h" #include "paimon/file_index/scored_file_index_result.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" +#include "paimon/predicate/full_text_match.h" #include "paimon/predicate/full_text_search.h" +#include "paimon/predicate/leaf_predicate.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/predicate/vector_search.h" @@ -91,23 +98,21 @@ class FileIndexEvaluatorTest : public ::testing::Test { void CheckResult(const std::shared_ptr& data_file_path_factory, const std::shared_ptr& data_file_meta) const { + ASSERT_OK_AND_ASSIGN(auto evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + data_file_path_factory, data_file_meta, fs_, + /*only_use_embedding_index=*/false, pool_)); { auto predicate = PredicateBuilder::IsNull(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT); - ASSERT_OK_AND_ASSIGN( - auto file_index_result, - FileIndexEvaluator::Evaluate(data_schema_, core_options_.ToMap(), predicate, - data_file_path_factory, data_file_meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(predicate)); ASSERT_TRUE(file_index_result->IsRemain().value()); CheckResult(file_index_result, {7}); } { auto predicate = PredicateBuilder::IsNotNull(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT); - ASSERT_OK_AND_ASSIGN( - auto file_index_result, - FileIndexEvaluator::Evaluate(data_schema_, core_options_.ToMap(), predicate, - data_file_path_factory, data_file_meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(predicate)); ASSERT_TRUE(file_index_result->IsRemain().value()); CheckResult(file_index_result, {0, 1, 2, 3, 4, 5, 6}); } @@ -115,10 +120,7 @@ class FileIndexEvaluatorTest : public ::testing::Test { auto predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice", 5)); - ASSERT_OK_AND_ASSIGN( - auto file_index_result, - FileIndexEvaluator::Evaluate(data_schema_, core_options_.ToMap(), predicate, - data_file_path_factory, data_file_meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(predicate)); ASSERT_TRUE(file_index_result->IsRemain().value()); CheckResult(file_index_result, {0, 7}); } @@ -126,50 +128,35 @@ class FileIndexEvaluatorTest : public ::testing::Test { auto predicate = PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice", 5)); - ASSERT_OK_AND_ASSIGN( - auto file_index_result, - FileIndexEvaluator::Evaluate(data_schema_, core_options_.ToMap(), predicate, - data_file_path_factory, data_file_meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(predicate)); ASSERT_TRUE(file_index_result->IsRemain().value()); CheckResult(file_index_result, {1, 2, 3, 4, 5, 6}); } { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/1, /*field_name=*/"f1", FieldType::INT, Literal(10)); - ASSERT_OK_AND_ASSIGN( - auto file_index_result, - FileIndexEvaluator::Evaluate(data_schema_, core_options_.ToMap(), predicate, - data_file_path_factory, data_file_meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(predicate)); ASSERT_TRUE(file_index_result->IsRemain().value()); ASSERT_FALSE(dynamic_cast(file_index_result.get())); } { auto predicate = PredicateBuilder::GreaterOrEqual( /*field_index=*/1, /*field_name=*/"f1", FieldType::INT, Literal(10)); - ASSERT_OK_AND_ASSIGN( - auto file_index_result, - FileIndexEvaluator::Evaluate(data_schema_, core_options_.ToMap(), predicate, - data_file_path_factory, data_file_meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(predicate)); ASSERT_TRUE(file_index_result->IsRemain().value()); ASSERT_FALSE(dynamic_cast(file_index_result.get())); } { auto predicate = PredicateBuilder::LessThan(/*field_index=*/1, /*field_name=*/"f1", FieldType::INT, Literal(10)); - ASSERT_OK_AND_ASSIGN( - auto file_index_result, - FileIndexEvaluator::Evaluate(data_schema_, core_options_.ToMap(), predicate, - data_file_path_factory, data_file_meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(predicate)); ASSERT_TRUE(file_index_result->IsRemain().value()); ASSERT_FALSE(dynamic_cast(file_index_result.get())); } { auto predicate = PredicateBuilder::LessOrEqual(/*field_index=*/1, /*field_name=*/"f1", FieldType::INT, Literal(10)); - ASSERT_OK_AND_ASSIGN( - auto file_index_result, - FileIndexEvaluator::Evaluate(data_schema_, core_options_.ToMap(), predicate, - data_file_path_factory, data_file_meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(predicate)); ASSERT_TRUE(file_index_result->IsRemain().value()); ASSERT_FALSE(dynamic_cast(file_index_result.get())); } @@ -178,10 +165,7 @@ class FileIndexEvaluatorTest : public ::testing::Test { /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, {Literal(FieldType::STRING, "Alice", 5), Literal(FieldType::STRING, "Bob", 3), Literal(FieldType::STRING, "Lucy", 4)}); - ASSERT_OK_AND_ASSIGN( - auto file_index_result, - FileIndexEvaluator::Evaluate(data_schema_, core_options_.ToMap(), predicate, - data_file_path_factory, data_file_meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(predicate)); ASSERT_TRUE(file_index_result->IsRemain().value()); CheckResult(file_index_result, {0, 1, 4, 5, 7}); } @@ -190,10 +174,7 @@ class FileIndexEvaluatorTest : public ::testing::Test { /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, {Literal(FieldType::STRING, "Alice", 5), Literal(FieldType::STRING, "Bob", 3), Literal(FieldType::STRING, "Lucy", 4)}); - ASSERT_OK_AND_ASSIGN( - auto file_index_result, - FileIndexEvaluator::Evaluate(data_schema_, core_options_.ToMap(), predicate, - data_file_path_factory, data_file_meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(predicate)); ASSERT_TRUE(file_index_result->IsRemain().value()); CheckResult(file_index_result, {2, 3, 6}); } @@ -205,10 +186,7 @@ class FileIndexEvaluatorTest : public ::testing::Test { FieldType::INT, Literal(20)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({f0_predicate, f1_predicate})); - ASSERT_OK_AND_ASSIGN( - auto file_index_result, - FileIndexEvaluator::Evaluate(data_schema_, core_options_.ToMap(), predicate, - data_file_path_factory, data_file_meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(predicate)); ASSERT_TRUE(file_index_result->IsRemain().value()); CheckResult(file_index_result, {7}); } @@ -220,10 +198,7 @@ class FileIndexEvaluatorTest : public ::testing::Test { FieldType::INT, Literal(20)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::Or({f0_predicate, f1_predicate})); - ASSERT_OK_AND_ASSIGN( - auto file_index_result, - FileIndexEvaluator::Evaluate(data_schema_, core_options_.ToMap(), predicate, - data_file_path_factory, data_file_meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(predicate)); ASSERT_TRUE(file_index_result->IsRemain().value()); CheckResult(file_index_result, {0, 4, 6, 7}); } @@ -232,10 +207,7 @@ class FileIndexEvaluatorTest : public ::testing::Test { auto predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "unknown", 7)); - ASSERT_OK_AND_ASSIGN( - auto file_index_result, - FileIndexEvaluator::Evaluate(data_schema_, core_options_.ToMap(), predicate, - data_file_path_factory, data_file_meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(predicate)); ASSERT_FALSE(file_index_result->IsRemain().value()); } { @@ -250,18 +222,12 @@ class FileIndexEvaluatorTest : public ::testing::Test { ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({f1_predicate, f2_predicate, f0_predicate})); - ASSERT_OK_AND_ASSIGN( - auto file_index_result, - FileIndexEvaluator::Evaluate(data_schema_, core_options_.ToMap(), predicate, - data_file_path_factory, data_file_meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(predicate)); ASSERT_FALSE(file_index_result->IsRemain().value()); } { // test no predicate - ASSERT_OK_AND_ASSIGN(auto file_index_result, - FileIndexEvaluator::Evaluate( - data_schema_, core_options_.ToMap(), /*predicate=*/nullptr, - data_file_path_factory, data_file_meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(nullptr)); ASSERT_TRUE(file_index_result->IsRemain().value()); ASSERT_FALSE(dynamic_cast(file_index_result.get())); } @@ -269,10 +235,7 @@ class FileIndexEvaluatorTest : public ::testing::Test { // test predicate on f3 (do not have index) auto predicate = PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(14.1)); - ASSERT_OK_AND_ASSIGN(auto file_index_result, - FileIndexEvaluator::Evaluate( - data_schema_, core_options_.ToMap(), /*predicate=*/nullptr, - data_file_path_factory, data_file_meta, fs_, pool_)); + ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(predicate)); ASSERT_TRUE(file_index_result->IsRemain().value()); ASSERT_FALSE(dynamic_cast(file_index_result.get())); } @@ -402,14 +365,16 @@ TEST_F(FileIndexEvaluatorTest, TestTimestampType) { /*field_index=*/6, /*field_name=*/"ts_tz_micro", FieldType::TIMESTAMP, Literal(Timestamp(1745542602001l, 1000))); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({in_predicate, greater_than})); - ASSERT_OK_AND_ASSIGN(auto file_index_result, - FileIndexEvaluator::Evaluate(data_schema, core_options_.ToMap(), predicate, - data_file_path_factory, data_file_meta, - /*file_system=*/fs_, pool_)); + ASSERT_OK_AND_ASSIGN( + auto evaluator, + FileIndexEvaluator::Create(data_schema, core_options_.ToMap(), data_file_path_factory, + data_file_meta, /*file_system=*/fs_, + /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(predicate)); CheckResult(file_index_result, {0, 6}); } -TEST_F(FileIndexEvaluatorTest, TestInvalidEvaluate) { +TEST_F(FileIndexEvaluatorTest, TestCreateRequiresExternalIndexDependencies) { auto data_file_meta = std::make_shared( "data-414509f5-e40c-4245-b992-bbf486778ac9-0.orc", /*file_size=*/689, /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(), @@ -425,12 +390,10 @@ TEST_F(FileIndexEvaluatorTest, TestInvalidEvaluate) { /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); - auto predicate = - PredicateBuilder::IsNull(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT); ASSERT_NOK_WITH_MSG( - FileIndexEvaluator::Evaluate(data_schema_, core_options_.ToMap(), predicate, - /*data_file_path_factory=*/nullptr, data_file_meta, - /*file_system=*/nullptr, pool_), + FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), /*data_file_path_factory=*/nullptr, data_file_meta, + /*file_system=*/nullptr, /*only_use_embedding_index=*/false, pool_), "read process for FileIndexEvaluator must have data_file_path_factory and file_system"); } @@ -439,6 +402,11 @@ class FileIndexSearchEvaluatorTest : public FileIndexEvaluatorTest, protected: void SetUp() override { FileIndexEvaluatorTest::SetUp(); + auto factory = std::make_unique(statistics_); + FactoryCreator::GetInstance()->Register(statistics_index_type_, factory.release()); + auto vector_factory = std::make_unique(statistics_); + FactoryCreator::GetInstance()->Register(vector_statistics_index_type_, + vector_factory.release()); data_schema_ = arrow::schema({arrow::field("embedding", arrow::list(arrow::float32())), arrow::field("title", arrow::utf8())}); dir_ = UniqueTestDirectory::Create(); @@ -447,6 +415,12 @@ class FileIndexSearchEvaluatorTest : public FileIndexEvaluatorTest, /*data_file_prefix=*/"data-", nullptr)); } + void TearDown() override { + FactoryCreator::GetInstance()->TEST_Unregister(statistics_index_type_); + FactoryCreator::GetInstance()->TEST_Unregister(vector_statistics_index_type_); + FileIndexEvaluatorTest::TearDown(); + } + Result> MakeIndex(const std::shared_ptr& bytes) const { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr meta, @@ -485,16 +459,14 @@ class FileIndexSearchEvaluatorTest : public FileIndexEvaluatorTest, void CheckSearchError(SearchIndexType index_type, const std::shared_ptr& meta, const std::string& field, const std::string& message) const { + ASSERT_OK_AND_ASSIGN( + auto evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), path_factory_, meta, + fs_, /*only_use_embedding_index=*/false, pool_)); if (index_type == SearchIndexType::VECTOR) { - ASSERT_NOK_WITH_MSG(FileIndexEvaluator::EvaluateVectorSearch( - data_schema_, core_options_.ToMap(), MakeVectorSearch(field), - path_factory_, meta, fs_, pool_), - message); + ASSERT_NOK_WITH_MSG(evaluator->SearchVector(MakeVectorSearch(field)), message); } else { - ASSERT_NOK_WITH_MSG(FileIndexEvaluator::EvaluateFullTextSearch( - data_schema_, core_options_.ToMap(), MakeFullTextSearch(field), - path_factory_, meta, fs_, pool_), - message); + ASSERT_NOK_WITH_MSG(evaluator->SearchFullText(MakeFullTextSearch(field)), message); } } @@ -519,11 +491,22 @@ class FileIndexSearchEvaluatorTest : public FileIndexEvaluatorTest, FileIndexFormat::ColumnIndexes indexes; ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, MockSearchFileIndex::MakePayload({{3, 0.75f}, {1, 0.25f}}, pool_)); + const std::string another_identifier = "another-" + identifier; + auto* factories = FactoryCreator::GetInstance(); + if (!factories->Create(another_identifier)) { + std::unique_ptr factory; + if (index_type == SearchIndexType::VECTOR) { + factory = std::make_unique(); + } else { + factory = std::make_unique(); + } + factories->Register(another_identifier, factory.release()); + } indexes[field][identifier] = payload; - indexes[field]["another-empty-index"] = nullptr; + indexes[field][another_identifier] = nullptr; ASSERT_OK_AND_ASSIGN(bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(meta, MakeIndex(bytes)); - CheckSearchError(index_type, meta, field, "Multiple File Index readers exist"); + CheckSearchError(index_type, meta, field, "Multiple File Index readers support"); meta->embedded_index.reset(); meta->extra_files.clear(); @@ -533,34 +516,48 @@ class FileIndexSearchEvaluatorTest : public FileIndexEvaluatorTest, std::unique_ptr dir_; std::shared_ptr path_factory_; + const std::string statistics_index_type_ = "evaluator-statistics-full-text"; + const std::string vector_statistics_index_type_ = "evaluator-statistics-vector"; + std::shared_ptr statistics_ = + std::make_shared(); }; TEST_P(FileIndexSearchEvaluatorTest, TestVectorSearchReadsIndexWithScores) { - ASSERT_OK_AND_ASSIGN( - std::shared_ptr bytes, - MockSearchFileIndex::MakeIndex("embedding", MockSearchFileIndex::VECTOR_IDENTIFIER, - {{3, 0.75f}, {1, 0.25f}}, pool_)); + FileIndexFormat::ColumnIndexes indexes; + ASSERT_OK_AND_ASSIGN(indexes["embedding"][vector_statistics_index_type_], + MockSearchFileIndex::MakePayload({{3, 0.75f}, {1, 0.25f}}, pool_)); + indexes["embedding"]["bitmap"] = nullptr; + indexes["embedding"]["unregistered-empty-index"] = nullptr; + ASSERT_OK_AND_ASSIGN(std::shared_ptr bytes, + MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, MakeIndex(bytes)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*only_use_embedding_index=*/false, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - FileIndexEvaluator::EvaluateVectorSearch( - data_schema_, core_options_.ToMap(), MakeVectorSearch("embedding"), - path_factory_, meta, fs_, pool_)); + evaluator->SearchVector(MakeVectorSearch("embedding"))); ASSERT_TRUE(result); EXPECT_EQ(RoaringBitmap32::From({1, 3}), result->GetRowPositions()); EXPECT_EQ(std::vector({0.25f, 0.75f}), result->GetScores()); + ASSERT_EQ(statistics_->executions.size(), 1); + ASSERT_EQ(statistics_->executions.front().field_name, "embedding"); + ASSERT_TRUE(statistics_->executions.front().needs_scores); } -TEST_P(FileIndexSearchEvaluatorTest, TestFullTextSearchReadsIndex) { +TEST_P(FileIndexSearchEvaluatorTest, TestFullTextSearchReadsIndexWithScores) { ASSERT_OK_AND_ASSIGN( std::shared_ptr bytes, MockSearchFileIndex::MakeIndex("title", MockSearchFileIndex::FULL_TEXT_IDENTIFIER, {{3, 0.75f}, {1, 0.25f}}, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(std::shared_ptr result, - FileIndexEvaluator::EvaluateFullTextSearch( - data_schema_, core_options_.ToMap(), MakeFullTextSearch("title"), - path_factory_, meta, fs_, pool_)); - CheckResult(result, {1, 3}); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + evaluator->SearchFullText(MakeFullTextSearch("title"))); + ASSERT_TRUE(result); + ASSERT_EQ(result->GetRowPositions(), RoaringBitmap32::From({1, 3})); + ASSERT_EQ(result->GetScores(), std::vector({0.25f, 0.75f})); } TEST_P(FileIndexSearchEvaluatorTest, TestEmptyVectorSearchIndex) { @@ -569,10 +566,11 @@ TEST_P(FileIndexSearchEvaluatorTest, TestEmptyVectorSearchIndex) { ASSERT_OK_AND_ASSIGN(std::shared_ptr bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, MakeIndex(bytes)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*only_use_embedding_index=*/false, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr vector_result, - FileIndexEvaluator::EvaluateVectorSearch( - data_schema_, core_options_.ToMap(), MakeVectorSearch("embedding"), - path_factory_, meta, fs_, pool_)); + evaluator->SearchVector(MakeVectorSearch("embedding"))); ASSERT_TRUE(vector_result); EXPECT_TRUE(vector_result->IsEmpty()); EXPECT_TRUE(vector_result->GetScores().empty()); @@ -584,11 +582,14 @@ TEST_P(FileIndexSearchEvaluatorTest, TestEmptyFullTextSearchIndex) { ASSERT_OK_AND_ASSIGN(std::shared_ptr bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(std::shared_ptr text_result, - FileIndexEvaluator::EvaluateFullTextSearch( - data_schema_, core_options_.ToMap(), MakeFullTextSearch("title"), - path_factory_, meta, fs_, pool_)); - CheckResult(text_result, {}); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr text_result, + evaluator->SearchFullText(MakeFullTextSearch("title"))); + ASSERT_TRUE(text_result); + ASSERT_TRUE(text_result->IsEmpty()); + ASSERT_TRUE(text_result->GetScores().empty()); } TEST_P(FileIndexSearchEvaluatorTest, TestInvalidVectorSearchIndexes) { @@ -599,12 +600,264 @@ TEST_P(FileIndexSearchEvaluatorTest, TestInvalidFullTextSearchIndexes) { CheckInvalidSearchIndexes(SearchIndexType::FULL_TEXT); } +TEST_P(FileIndexSearchEvaluatorTest, TestEvaluateNullPredicateAndMissingIndex) { + FileIndexFormat::ColumnIndexes indexes; + indexes["title"]["bitmap"] = nullptr; + ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); + ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(auto result, evaluator->Evaluate(nullptr)); + ASSERT_EQ(result, FileIndexResult::Remain()); + + meta->embedded_index.reset(); + meta->extra_files.clear(); + auto missing = PredicateBuilder::IsNull(2, "missing", FieldType::INT); + ASSERT_OK_AND_ASSIGN(evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, meta, + fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(result, evaluator->Evaluate(missing)); + ASSERT_EQ(result, FileIndexResult::Remain()); + ASSERT_OK_AND_ASSIGN(result, evaluator->Evaluate(nullptr)); + ASSERT_EQ(result, FileIndexResult::Remain()); +} + +TEST_P(FileIndexSearchEvaluatorTest, TestEvaluateValidatesFieldsBeforeAndShortCircuit) { + FileIndexFormat::ColumnIndexes indexes; + indexes["title"]["bitmap"] = nullptr; + ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); + ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*only_use_embedding_index=*/false, pool_)); + auto empty = PredicateBuilder::IsNotNull(1, "title", FieldType::STRING); + ASSERT_OK_AND_ASSIGN(auto result, evaluator->Evaluate(empty)); + ASSERT_OK_AND_ASSIGN(bool remain, result->IsRemain()); + ASSERT_FALSE(remain); + + auto missing = PredicateBuilder::IsNull(2, "z_missing", FieldType::INT); + ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({empty, missing})); + ASSERT_NOK_WITH_MSG(evaluator->Evaluate(predicate), "z_missing"); +} + +TEST_P(FileIndexSearchEvaluatorTest, TestSessionReturnsQueryScores) { + MockSearchFileIndex::QueryMatches queries = { + {"apple", {{{3, 0.9f}, {1, 0.5f}, {2, 0.3f}}, 0.75}}, + {"pear", {{{0, 0.8f}, {2, 0.2f}}, std::nullopt}}}; + ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::MakeQueryIndex( + "title", statistics_index_type_, queries, pool_)); + ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH)); + ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_SEARCH)); + ASSERT_TRUE(statistics_->executions.empty()); + ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::FullTextMatch( + 1, std::make_shared("title", "apple"))); + auto leaf = std::dynamic_pointer_cast(predicate); + ASSERT_TRUE(leaf); + ASSERT_OK_AND_ASSIGN(auto estimate, evaluator->EstimateSelectivity(leaf)); + ASSERT_EQ(estimate, 0.75); + ASSERT_OK_AND_ASSIGN(auto result, evaluator->EvaluateLeaf(leaf, true)); + auto scored = std::dynamic_pointer_cast(result); + ASSERT_TRUE(scored); + ASSERT_EQ(scored->GetRowPositions(), RoaringBitmap32::From({1, 2, 3})); + ASSERT_EQ(scored->GetScores(), std::vector({0.5f, 0.3f, 0.9f})); + ASSERT_EQ(statistics_->executions.size(), 1); + ASSERT_TRUE(statistics_->executions.back().needs_scores); + ASSERT_OK_AND_ASSIGN(result, evaluator->EvaluateLeaf(leaf, false)); + CheckResult(result, {1, 2, 3}); + ASSERT_FALSE(statistics_->executions.back().needs_scores); + + auto include = RoaringBitmap64::From({0, 2}); + auto search = std::make_shared("title", "pear", 1, include); + ASSERT_OK_AND_ASSIGN(scored, evaluator->SearchFullText(search)); + ASSERT_TRUE(scored); + ASSERT_EQ(scored->GetRowPositions(), RoaringBitmap32::From({0})); + ASSERT_EQ(scored->GetScores(), std::vector({0.8f})); + ASSERT_TRUE(statistics_->executions.back().needs_scores); + ASSERT_EQ(statistics_->executions.size(), 3); + + ASSERT_OK_AND_ASSIGN(auto pear, PredicateBuilder::FullTextMatch( + 1, std::make_shared("title", "pear"))); + ASSERT_OK_AND_ASSIGN(auto conjunction, PredicateBuilder::And({predicate, pear})); + ASSERT_OK_AND_ASSIGN(result, evaluator->Evaluate(conjunction)); + CheckResult(result, {2}); + ASSERT_OK_AND_ASSIGN(auto disjunction, PredicateBuilder::Or({predicate, pear})); + ASSERT_OK_AND_ASSIGN(result, evaluator->Evaluate(disjunction)); + CheckResult(result, {0, 1, 2, 3}); + ASSERT_EQ(statistics_->executions.size(), 7); + + ASSERT_OK_AND_ASSIGN( + auto restricted, + PredicateBuilder::FullTextMatch( + 1, std::make_shared("title", "apple", RoaringBitmap64::From({1, 3})))); + auto restricted_leaf = std::dynamic_pointer_cast(restricted); + ASSERT_TRUE(restricted_leaf); + ASSERT_OK_AND_ASSIGN(result, evaluator->EvaluateLeaf(restricted_leaf, false)); + CheckResult(result, {1, 3}); + ASSERT_FALSE(statistics_->executions.back().needs_scores); + ASSERT_OK_AND_ASSIGN(result, evaluator->EvaluateLeaf(restricted_leaf, true)); + scored = std::dynamic_pointer_cast(result); + ASSERT_TRUE(scored); + ASSERT_EQ(scored->GetRowPositions(), RoaringBitmap32::From({1, 3})); + ASSERT_EQ(scored->GetScores(), std::vector({0.5f, 0.9f})); + ASSERT_TRUE(statistics_->executions.back().needs_scores); + ASSERT_EQ(statistics_->executions.size(), 9); +} + +TEST_P(FileIndexSearchEvaluatorTest, TestSessionScalarAndSearchReadersCanCoexist) { + auto field = arrow::field("title", arrow::utf8()); + ASSERT_OK_AND_ASSIGN(auto writer, BitmapFileIndexWriter::Create(field, {}, pool_)); + auto array_result = arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({field}), R"([["a"], ["b"], ["a"], ["b"]])"); + ASSERT_TRUE(array_result.ok()) << array_result.status().ToString(); + auto array = array_result.ValueUnsafe(); + ::ArrowArray c_array; + ASSERT_TRUE(arrow::ExportArray(*array, &c_array).ok()); + ASSERT_OK(writer->AddBatch(&c_array)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr bitmap_payload, writer->SerializedBytes()); + ASSERT_OK_AND_ASSIGN(auto search_payload, + MockSearchFileIndex::MakePayload({{3, 0.9f}, {1, 0.5f}}, pool_)); + FileIndexFormat::ColumnIndexes indexes; + indexes["title"]["bitmap"] = bitmap_payload; + indexes["title"][statistics_index_type_] = search_payload; + ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); + ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH)); + ASSERT_NOK_WITH_MSG( + evaluator->GetSearchReader("title", FileIndexReader::SearchType::VECTOR_SEARCH), + "No File Index reader supports"); + ASSERT_TRUE(statistics_->executions.empty()); + auto scalar = std::dynamic_pointer_cast( + PredicateBuilder::Equal(1, "title", FieldType::STRING, Literal(FieldType::STRING, "a", 1))); + ASSERT_TRUE(scalar); + ASSERT_OK_AND_ASSIGN(auto estimate, evaluator->EstimateSelectivity(scalar)); + ASSERT_FALSE(estimate.has_value()); + ASSERT_OK_AND_ASSIGN(auto result, evaluator->EvaluateLeaf(scalar, false)); + CheckResult(result, {0, 2}); + auto other = + PredicateBuilder::Equal(1, "title", FieldType::STRING, Literal(FieldType::STRING, "b", 1)); + ASSERT_OK_AND_ASSIGN(auto disjunction, PredicateBuilder::Or({scalar, other})); + ASSERT_OK_AND_ASSIGN(result, evaluator->Evaluate(disjunction)); + CheckResult(result, {0, 1, 2, 3}); + ASSERT_OK_AND_ASSIGN(auto search_result, + evaluator->SearchFullText(MakeFullTextSearch("title"))); + ASSERT_EQ(search_result->GetRowPositions(), RoaringBitmap32::From({1, 3})); + ASSERT_EQ(search_result->GetScores(), std::vector({0.5f, 0.9f})); + ASSERT_EQ(statistics_->executions.size(), 1); + ASSERT_TRUE(statistics_->executions.back().needs_scores); +} + +TEST_P(FileIndexSearchEvaluatorTest, TestSessionPrecheckAndMissingScalarIndex) { + ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::MakeIndex("title", statistics_index_type_, + {{1, 0.5f}}, pool_)); + ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_NOK_WITH_MSG( + evaluator->GetSearchReader("embedding", FileIndexReader::SearchType::VECTOR_SEARCH), + "No File Index reader supports"); + ASSERT_NOK_WITH_MSG( + evaluator->GetSearchReader("missing", FileIndexReader::SearchType::FULL_TEXT_MATCH), + "does not exist in data schema"); + ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH)); + ASSERT_TRUE(statistics_->executions.empty()); + auto scalar = std::dynamic_pointer_cast( + PredicateBuilder::IsNull(0, "embedding", FieldType::ARRAY)); + ASSERT_TRUE(scalar); + ASSERT_OK_AND_ASSIGN(auto result, evaluator->EvaluateLeaf(scalar, false)); + ASSERT_EQ(result, FileIndexResult::Remain()); + ASSERT_OK_AND_ASSIGN(auto estimate, evaluator->EstimateSelectivity(scalar)); + ASSERT_FALSE(estimate.has_value()); + + meta->embedded_index.reset(); + meta->extra_files.clear(); + ASSERT_OK_AND_ASSIGN(evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, meta, + fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_NOK_WITH_MSG( + evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH), + "has no File Index"); + ASSERT_OK_AND_ASSIGN(result, evaluator->EvaluateLeaf(scalar, false)); + ASSERT_EQ(result, FileIndexResult::Remain()); +} + +TEST_P(FileIndexSearchEvaluatorTest, TestEmptyScalarIndexDoesNotProvideSearchCapability) { + FileIndexFormat::ColumnIndexes indexes; + indexes["title"]["bitmap"] = nullptr; + ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); + ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_NOK_WITH_MSG( + evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH), + "No File Index reader supports"); + ASSERT_NOK_WITH_MSG( + evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_SEARCH), + "No File Index reader supports"); + ASSERT_NOK_WITH_MSG( + evaluator->GetSearchReader("title", FileIndexReader::SearchType::VECTOR_SEARCH), + "No File Index reader supports"); + + ASSERT_OK_AND_ASSIGN(indexes["title"][MockSearchFileIndex::FULL_TEXT_IDENTIFIER], + MockSearchFileIndex::MakePayload({{1, 0.5f}}, pool_)); + ASSERT_OK_AND_ASSIGN(bytes, MockSearchFileIndex::Serialize(indexes, pool_)); + ASSERT_OK_AND_ASSIGN(meta, MakeIndex(bytes)); + ASSERT_OK_AND_ASSIGN(evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, meta, + fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH)); + ASSERT_OK_AND_ASSIGN(auto result, evaluator->SearchFullText(MakeFullTextSearch("title"))); + ASSERT_EQ(result->GetRowPositions(), RoaringBitmap32::From({1})); + ASSERT_EQ(result->GetScores(), std::vector({0.5f})); +} + +TEST_P(FileIndexSearchEvaluatorTest, TestEmptySearchIndexPreservesCapabilityAndScores) { + FileIndexFormat::ColumnIndexes indexes; + indexes["title"][MockSearchFileIndex::FULL_TEXT_IDENTIFIER] = nullptr; + indexes["embedding"][MockSearchFileIndex::VECTOR_IDENTIFIER] = nullptr; + ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); + ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); + ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( + data_schema_, core_options_.ToMap(), path_factory_, + meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH)); + ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_SEARCH)); + ASSERT_NOK_WITH_MSG( + evaluator->GetSearchReader("title", FileIndexReader::SearchType::VECTOR_SEARCH), + "No File Index reader supports"); + ASSERT_OK(evaluator->GetSearchReader("embedding", FileIndexReader::SearchType::VECTOR_SEARCH)); + ASSERT_NOK_WITH_MSG( + evaluator->GetSearchReader("embedding", FileIndexReader::SearchType::FULL_TEXT_MATCH), + "No File Index reader supports"); + ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::FullTextMatch( + 1, std::make_shared("title", "apple"))); + auto leaf = std::dynamic_pointer_cast(predicate); + ASSERT_TRUE(leaf); + ASSERT_OK_AND_ASSIGN(auto result, evaluator->EvaluateLeaf(leaf, true)); + auto scored = std::dynamic_pointer_cast(result); + ASSERT_TRUE(scored); + ASSERT_TRUE(scored->IsEmpty()); + ASSERT_TRUE(scored->GetScores().empty()); + ASSERT_OK_AND_ASSIGN(scored, evaluator->SearchFullText(MakeFullTextSearch("title"))); + ASSERT_TRUE(scored); + ASSERT_TRUE(scored->IsEmpty()); +} + INSTANTIATE_TEST_SUITE_P(IndexStorage, FileIndexSearchEvaluatorTest, ::testing::Bool(), [](const ::testing::TestParamInfo& info) { return info.param ? "Embedded" : "External"; }); -TEST_F(FileIndexEvaluatorTest, TestLargeFileSkips32BitIndex) { +TEST_F(FileIndexEvaluatorTest, TestUnsupportedRowCountSkips32BitIndex) { auto data_file_meta = std::make_shared( "data-414509f5-e40c-4245-b992-bbf486778ac9-0.orc", /*file_size=*/689, /*row_count=*/(1LL << 32), /*min_key=*/BinaryRow::EmptyRow(), @@ -623,11 +876,32 @@ TEST_F(FileIndexEvaluatorTest, TestLargeFileSkips32BitIndex) { auto predicate = PredicateBuilder::IsNull(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT); ASSERT_OK_AND_ASSIGN( - auto result, FileIndexEvaluator::Evaluate(data_schema_, core_options_.ToMap(), predicate, - /*data_file_path_factory=*/nullptr, - data_file_meta, /*file_system=*/nullptr, pool_)); + auto evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), nullptr, data_file_meta, + nullptr, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(auto result, evaluator->Evaluate(predicate)); ASSERT_TRUE(result->IsRemain().value()); ASSERT_EQ(dynamic_cast(result.get()), nullptr); + + data_file_meta->row_count = -1; + ASSERT_OK_AND_ASSIGN( + evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), nullptr, data_file_meta, + nullptr, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(result, evaluator->Evaluate(predicate)); + ASSERT_EQ(result, FileIndexResult::Remain()); + + FileIndexFormat::ColumnIndexes indexes; + indexes["f2"]["bitmap"] = nullptr; + ASSERT_OK_AND_ASSIGN(data_file_meta->embedded_index, + MockSearchFileIndex::Serialize(indexes, pool_)); + ASSERT_OK_AND_ASSIGN( + evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), nullptr, data_file_meta, + nullptr, /*only_use_embedding_index=*/false, pool_)); + auto missing = PredicateBuilder::IsNull(4, "missing", FieldType::INT); + ASSERT_OK_AND_ASSIGN(result, evaluator->Evaluate(missing)); + ASSERT_EQ(result, FileIndexResult::Remain()); } } // namespace paimon::test diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp index b087e20b8..7f3360195 100644 --- a/src/paimon/core/operation/abstract_split_read.cpp +++ b/src/paimon/core/operation/abstract_split_read.cpp @@ -168,7 +168,8 @@ AbstractSplitRead::CreateRawFileReadersWithMetaImpl( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr predicate, predicate_provider(file)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr field_mapping_builder, - FieldMappingBuilder::Create(read_schema, context_->GetPartitionKeys(), predicate)); + FieldMappingBuilder::Create(read_schema, context_->GetPartitionKeys(), + EnablePreciseFileIndexEvaluate() ? nullptr : predicate)); return CreateRawFileReader(partition, file, field_mapping_builder.get(), dv_factory, row_ranges, data_file_path_factory, extra_format_options, attach_blob_context); @@ -222,8 +223,7 @@ Result> AbstractSplitRead::ApplyIndexAndDvReade const std::shared_ptr& read_schema, const std::shared_ptr& predicate, DeletionVector::Factory dv_factory, const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory, - const std::map& file_index_options, - std::vector* index_scores) const { + const std::map& file_index_options) const { std::shared_ptr deletion_vector; if (dv_factory) { PAIMON_ASSIGN_OR_RAISE(deletion_vector, dv_factory(file->file_name)); @@ -238,10 +238,12 @@ Result> AbstractSplitRead::ApplyIndexAndDvReade std::shared_ptr file_index_result; if (options_.FileIndexReadEnabled() && dynamic_cast(deletion_vector.get()) == nullptr) { - PAIMON_ASSIGN_OR_RAISE(file_index_result, - FileIndexEvaluator::Evaluate(data_schema, file_index_options, - predicate, data_file_path_factory, file, - options_.GetFileSystem(), pool_)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr evaluator, + FileIndexEvaluator::Create(data_schema, file_index_options, data_file_path_factory, + file, options_.GetFileSystem(), + /*only_use_embedding_index=*/false, pool_)); + PAIMON_ASSIGN_OR_RAISE(file_index_result, evaluator->Evaluate(predicate)); PAIMON_ASSIGN_OR_RAISE(bool is_remain, file_index_result->IsRemain()); if (!is_remain) { return std::unique_ptr(); @@ -321,6 +323,13 @@ Result AbstractSplitRead::CreateRawFil const std::shared_ptr& data_file_path_factory, const std::map& extra_format_options, bool attach_blob_context) const { + if (EnablePreciseFileIndexEvaluate()) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + CreatePreciseFileIndexReader( + partition, file, field_mapping_builder, std::move(dv_factory), + row_ranges, data_file_path_factory, extra_format_options)); + return RawFileReaderWithMeta{file, std::move(reader)}; + } auto data_file_path = data_file_path_factory->ToPath(file); PAIMON_ASSIGN_OR_RAISE(std::string data_file_identifier, file->FileFormat()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder, @@ -343,6 +352,15 @@ Result AbstractSplitRead::CreateRawFil return RawFileReaderWithMeta{file, std::move(reader), std::move(blob_context)}; } +Result> AbstractSplitRead::CreatePreciseFileIndexReader( + const BinaryRow& partition, const std::shared_ptr& file, + const FieldMappingBuilder* field_mapping_builder, DeletionVector::Factory dv_factory, + const std::optional>& row_ranges, + const std::shared_ptr& path_factory, + const std::map& extra_format_options) const { + return Status::NotImplemented("This read path does not support file predicate evaluation"); +} + bool AbstractSplitRead::NeedCompleteRowTrackingFields( bool row_tracking_enabled, const std::shared_ptr& read_schema) { if (row_tracking_enabled && @@ -400,7 +418,8 @@ Result> AbstractSplitRead::CreateFieldMappingRe const BinaryRow& partition, std::unique_ptr reader_builder, const FieldMappingBuilder* field_mapping_builder, DeletionVector::Factory dv_factory, const std::optional>& row_ranges, - const std::shared_ptr& data_file_path_factory) const { + const std::shared_ptr& data_file_path_factory, + const RoaringBitmap32* exact_selection) const { std::shared_ptr data_schema; if (file_meta->schema_id == context_->GetTableSchema()->Id()) { data_schema = context_->GetTableSchema(); @@ -460,21 +479,26 @@ Result> AbstractSplitRead::CreateFieldMappingRe std::move(file_reader), file_meta->first_row_id, file_meta->max_sequence_number, file_field_names, arrow_pool_); } - const auto& predicate = field_mapping->non_partition_info.non_partition_filter; - auto all_data_schema = DataField::ConvertDataFieldsToArrowSchema(data_schema->Fields()); - auto field_name_mapping = FieldMappingBuilder::CreateFieldNameMapping( - context_->GetTableSchema()->Fields(), data_schema->Fields()); - std::map file_index_options; - if (options_.FileIndexReadEnabled() || context_->HasVectorOrFullTextSearch()) { - PAIMON_ASSIGN_OR_RAISE(file_index_options, FileIndexOptions::RewriteFieldNames( - options_.ToMap(), field_name_mapping)); + std::unique_ptr final_reader; + if (exact_selection) { + PAIMON_ASSIGN_OR_RAISE(final_reader, + ApplyBitmapSelection(std::move(file_reader), read_schema, + /*predicate=*/nullptr, *exact_selection)); + } else { + const auto& predicate = field_mapping->non_partition_info.non_partition_filter; + auto all_data_schema = DataField::ConvertDataFieldsToArrowSchema(data_schema->Fields()); + auto field_name_mapping = FieldMappingBuilder::CreateFieldNameMapping( + context_->GetTableSchema()->Fields(), data_schema->Fields()); + std::map file_index_options; + if (options_.FileIndexReadEnabled()) { + PAIMON_ASSIGN_OR_RAISE(file_index_options, FileIndexOptions::RewriteFieldNames( + options_.ToMap(), field_name_mapping)); + } + PAIMON_ASSIGN_OR_RAISE(final_reader, ApplyIndexAndDvReaderIfNeeded( + std::move(file_reader), file_meta, all_data_schema, + read_schema, predicate, dv_factory, row_ranges, + data_file_path_factory, file_index_options)); } - std::vector index_scores; - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr final_reader, - ApplyIndexAndDvReaderIfNeeded(std::move(file_reader), file_meta, all_data_schema, - read_schema, predicate, dv_factory, row_ranges, - data_file_path_factory, file_index_options, &index_scores)); if (!final_reader) { // file is skipped by index or dv return std::unique_ptr(); @@ -485,13 +509,7 @@ Result> AbstractSplitRead::CreateFieldMappingRe field_mapping_builder->GetReadFieldCount(), std::move(final_reader), partition, std::move(field_mapping), std::move(skip_map_selected_keys_filter_field_ids), arrow_pool_)); - std::unique_ptr result = std::move(mapping_reader); - if (!index_scores.empty() && context_->GetVectorSearch() && - raw_read_schema_->GetFieldIndex(SpecialFields::IndexScore().Name()) >= 0) { - result = std::make_unique(std::move(result), - index_scores, arrow_pool_); - } - return result; + return std::unique_ptr(std::move(mapping_reader)); } Result, std::set>> diff --git a/src/paimon/core/operation/abstract_split_read.h b/src/paimon/core/operation/abstract_split_read.h index d5857d716..30747d5ea 100644 --- a/src/paimon/core/operation/abstract_split_read.h +++ b/src/paimon/core/operation/abstract_split_read.h @@ -56,6 +56,7 @@ class DataFilePathFactory; class DataSplitImpl; class Executor; class FieldMappingBuilder; +class FileIndexReadExecutor; class FileStorePathFactory; class InternalReadContext; class MemoryPool; @@ -85,6 +86,17 @@ class AbstractSplitRead : public SplitRead { const std::map& extra_format_options) const; protected: + virtual bool EnablePreciseFileIndexEvaluate() const { + return false; + } + + virtual Result> CreatePreciseFileIndexReader( + const BinaryRow& partition, const std::shared_ptr& file, + const FieldMappingBuilder* field_mapping_builder, DeletionVector::Factory dv_factory, + const std::optional>& row_ranges, + const std::shared_ptr& path_factory, + const std::map& extra_format_options) const; + struct RawFileReaderWithMeta { std::shared_ptr file; std::unique_ptr reader; @@ -130,8 +142,7 @@ class AbstractSplitRead : public SplitRead { const std::shared_ptr& predicate, DeletionVector::Factory dv_factory, const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory, - const std::map& file_index_options, - std::vector* index_scores) const; + const std::map& file_index_options) const; static Result> ApplyBitmapSelection( std::unique_ptr&& file_reader, @@ -146,6 +157,8 @@ class AbstractSplitRead : public SplitRead { const std::optional>& write_cols); private: + friend class FileIndexReadExecutor; + Result> CreateRawFileReadersWithMetaImpl( const BinaryRow& partition, const std::vector>& data_files, const std::shared_ptr& read_schema, PredicateProvider predicate_provider, @@ -181,7 +194,8 @@ class AbstractSplitRead : public SplitRead { const BinaryRow& partition, std::unique_ptr reader_builder, const FieldMappingBuilder* field_mapping_builder, DeletionVector::Factory dv_factory, const std::optional>& row_ranges, - const std::shared_ptr& data_file_path_factory) const; + const std::shared_ptr& data_file_path_factory, + const RoaringBitmap32* exact_selection = nullptr) const; /// The returned field ID set contains MAP fields handled by shredding read plans. It tells /// FieldMappingReader to skip its generic selected-key filtering because the plans have already diff --git a/src/paimon/core/operation/data_evolution_split_read.cpp b/src/paimon/core/operation/data_evolution_split_read.cpp index 772e63878..21eef0b0e 100644 --- a/src/paimon/core/operation/data_evolution_split_read.cpp +++ b/src/paimon/core/operation/data_evolution_split_read.cpp @@ -583,10 +583,13 @@ Result> DataEvolutionSplitRead::SelectByFileIndex std::map file_index_options; PAIMON_ASSIGN_OR_RAISE(file_index_options, FileIndexOptions::RewriteFieldNames( options_.ToMap(), field_name_mapping)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr evaluator, + FileIndexEvaluator::Create(written_schema, file_index_options, data_file_path_factory, + file, options_.GetFileSystem(), + /*only_use_embedding_index=*/false, pool_)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index_result, - FileIndexEvaluator::Evaluate(written_schema, file_index_options, - data_predicate, data_file_path_factory, - file, options_.GetFileSystem(), pool_)); + evaluator->Evaluate(data_predicate)); PAIMON_ASSIGN_OR_RAISE(bool is_remain, index_result->IsRemain()); if (!is_remain) { return std::optional(RoaringBitmap32()); @@ -619,14 +622,15 @@ Result> DataEvolutionSplitRead::ApplyIndexAndDv const std::shared_ptr& read_schema, const std::shared_ptr& predicate, DeletionVector::Factory dv_factory, const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory, - const std::map& file_index_options, - std::vector* index_scores) const { + const std::map& file_index_options) const { std::shared_ptr file_index_result; if (options_.FileIndexReadEnabled() && !file_indexes_evaluated_) { - PAIMON_ASSIGN_OR_RAISE(file_index_result, - FileIndexEvaluator::Evaluate(data_schema, file_index_options, - predicate, data_file_path_factory, file, - options_.GetFileSystem(), pool_)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr evaluator, + FileIndexEvaluator::Create(data_schema, file_index_options, data_file_path_factory, + file, options_.GetFileSystem(), + /*only_use_embedding_index=*/false, pool_)); + PAIMON_ASSIGN_OR_RAISE(file_index_result, evaluator->Evaluate(predicate)); PAIMON_ASSIGN_OR_RAISE(bool is_remain, file_index_result->IsRemain()); if (!is_remain) { return std::unique_ptr(); diff --git a/src/paimon/core/operation/data_evolution_split_read.h b/src/paimon/core/operation/data_evolution_split_read.h index b7610dc1c..70075fb7e 100644 --- a/src/paimon/core/operation/data_evolution_split_read.h +++ b/src/paimon/core/operation/data_evolution_split_read.h @@ -97,8 +97,7 @@ class DataEvolutionSplitRead : public AbstractSplitRead { const std::shared_ptr& predicate, DeletionVector::Factory dv_factory, const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory, - const std::map& file_index_options, - std::vector* index_scores) const override; + const std::map& file_index_options) const override; private: /// Files for partial field. diff --git a/src/paimon/core/operation/data_evolution_split_read_test.cpp b/src/paimon/core/operation/data_evolution_split_read_test.cpp index 3bd6beb0b..763673e4d 100644 --- a/src/paimon/core/operation/data_evolution_split_read_test.cpp +++ b/src/paimon/core/operation/data_evolution_split_read_test.cpp @@ -1585,7 +1585,7 @@ TEST_F(DataEvolutionSplitReadTest, TestGroupBitmapCombinesWithRowRangesAndDvInFi std::make_unique(data, type, 2), file, schema, schema, /*predicate=*/nullptr, MakeSingleFileDvFactory(file->file_name, MakeBitmapDv({1})), std::vector{Range(101, 103)}, /*data_file_path_factory=*/nullptr, - /*file_index_options=*/{}, /*index_scores=*/nullptr)); + /*file_index_options=*/{})); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, ReadResultCollector::CollectResult(std::move(reader))); auto expected = arrow::ipc::internal::json::ArrayFromJSON(type, R"([[3]])").ValueOrDie(); diff --git a/src/paimon/core/operation/file_index_read_executor.cpp b/src/paimon/core/operation/file_index_read_executor.cpp new file mode 100644 index 000000000..aa389208c --- /dev/null +++ b/src/paimon/core/operation/file_index_read_executor.cpp @@ -0,0 +1,553 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/operation/file_index_read_executor.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/c/abi.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "paimon/common/reader/complete_index_score_file_batch_reader.h" +#include "paimon/common/reader/predicate_batch_reader.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/object_utils.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/core/deletionvectors/bitmap64_deletion_vector.h" +#include "paimon/core/deletionvectors/bitmap_deletion_vector.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/io/file_index_evaluator.h" +#include "paimon/core/io/file_index_options.h" +#include "paimon/core/operation/internal_read_context.h" +#include "paimon/core/operation/raw_file_split_read.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/field_mapping.h" +#include "paimon/file_index/bitmap_index_result.h" +#include "paimon/file_index/file_index_result.h" +#include "paimon/file_index/scored_file_index_result.h" +#include "paimon/format/reader_builder.h" +#include "paimon/predicate/compound_predicate.h" +#include "paimon/predicate/full_text_search.h" +#include "paimon/predicate/function.h" +#include "paimon/predicate/leaf_predicate.h" +#include "paimon/predicate/vector_search.h" +#include "paimon/utils/roaring_bitmap64.h" + +namespace paimon { +namespace { + +Result ResolveSearchFieldName( + const std::string& field_name, const std::map& field_name_mapping, + const std::string& file_name) { + auto iter = field_name_mapping.find(field_name); + if (iter == field_name_mapping.end()) { + return Status::Invalid( + fmt::format("Search field '{}' has no matching field ID in data schema for file '{}'", + field_name, file_name)); + } + return iter->second; +} + +} // namespace + +Result> FileIndexReadExecutor::Create( + const RawFileSplitRead& split_read, const BinaryRow& partition, + const std::shared_ptr& file, const FieldMappingBuilder* field_mapping_builder, + const std::shared_ptr& path_factory, + const std::map& extra_format_options) { + std::unique_ptr executor(new FileIndexReadExecutor( + split_read, partition, file, field_mapping_builder, path_factory, extra_format_options)); + PAIMON_RETURN_NOT_OK(executor->Prepare()); + return executor; +} + +FileIndexReadExecutor::FileIndexReadExecutor( + const RawFileSplitRead& split_read, const BinaryRow& partition, + const std::shared_ptr& file, const FieldMappingBuilder* field_mapping_builder, + const std::shared_ptr& path_factory, + const std::map& extra_format_options) + : split_read_(split_read), + partition_(partition), + file_(file), + field_mapping_builder_(field_mapping_builder), + path_factory_(path_factory), + extra_format_options_(extra_format_options) {} + +Status FileIndexReadExecutor::Prepare() { + std::shared_ptr table_schema = split_read_.context_->GetTableSchema(); + if (file_->schema_id != table_schema->Id()) { + PAIMON_ASSIGN_OR_RAISE(table_schema, + split_read_.schema_manager_->ReadSchema(file_->schema_id)); + } + const auto data_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + if (file_->row_count < 0 || file_->row_count > RoaringBitmap32::MAX_VALUE) { + return Status::NotImplemented( + "Precise file-index evaluation requires bitmap32 physical row positions"); + } + PAIMON_ASSIGN_OR_RAISE(std::vector data_fields, + DataField::ConvertArrowSchemaToDataFields(data_schema)); + PAIMON_ASSIGN_OR_RAISE( + std::vector read_fields, + DataField::ConvertArrowSchemaToDataFields(split_read_.context_->GetReadSchema())); + auto field_names = FieldMappingBuilder::CreateFieldNameMapping( + split_read_.context_->GetTableSchema()->Fields(), data_fields); + std::map index_options; + PAIMON_ASSIGN_OR_RAISE(index_options, + FileIndexOptions::RewriteFieldNames( + split_read_.context_->GetCoreOptions().ToMap(), field_names)); + PAIMON_ASSIGN_OR_RAISE(indexes_, FileIndexEvaluator::Create( + data_schema, index_options, path_factory_, file_, + split_read_.context_->GetCoreOptions().GetFileSystem(), + /*only_use_embedding_index=*/false, split_read_.pool_)); + + std::map read_by_name; + for (const auto& field : read_fields) { + read_by_name.emplace(field.Name(), field); + } + std::map> data_by_id; + for (size_t i = 0; i < data_fields.size(); ++i) { + data_by_id.emplace(data_fields[i].Id(), std::make_pair(i, data_fields[i])); + } + // Precheck the entire tree before an empty candidate set can short-circuit it. + PAIMON_RETURN_NOT_OK( + PreparePredicate(split_read_.context_->GetPredicate(), read_by_name, data_by_id)); + return PrepareSearch(field_names); +} + +Status FileIndexReadExecutor::PreparePredicate( + const std::shared_ptr& predicate, + const std::map& read_by_name, + const std::map>& data_by_id) { + if (!predicate) { + return Status::OK(); + } + if (auto compound = std::dynamic_pointer_cast(predicate)) { + const Function::Type type = compound->GetFunction().GetType(); + if (type != Function::Type::AND && type != Function::Type::OR) { + return Status::Invalid( + fmt::format("unsupported compound file predicate {}", predicate->ToString())); + } + for (const auto& child : compound->Children()) { + PAIMON_RETURN_NOT_OK(PreparePredicate(child, read_by_name, data_by_id)); + } + return Status::OK(); + } + auto leaf = std::dynamic_pointer_cast(predicate); + if (!leaf) { + return Status::Invalid("File predicate must contain leaf or compound predicates"); + } + bool full_text = leaf->GetFunction().GetType() == Function::Type::FULL_TEXT_MATCH; + if (!full_text && + ObjectUtils::Contains(split_read_.context_->GetPartitionKeys(), leaf->FieldName())) { + mapped_leaves_.emplace(leaf, nullptr); + return Status::OK(); + } + PAIMON_ASSIGN_OR_RAISE( + std::optional> mapped, + FieldMappingBuilder::ReconstructPredicateWithDataFields(leaf, read_by_name, data_by_id)); + std::shared_ptr mapped_leaf; + if (mapped) { + mapped_leaf = std::dynamic_pointer_cast(*mapped); + if (!mapped_leaf) { + return Status::Invalid("Mapped file predicate must be a leaf"); + } + if (full_text) { + PAIMON_ASSIGN_OR_RAISE( + [[maybe_unused]] std::shared_ptr reader, + indexes_->GetSearchReader(mapped_leaf->FieldName(), + FileIndexReader::SearchType::FULL_TEXT_MATCH)); + } + } + mapped_leaves_.emplace(leaf, std::move(mapped_leaf)); + return Status::OK(); +} + +Status FileIndexReadExecutor::PrepareSearch(const std::map& field_names) { + if (split_read_.context_->GetVectorSearch()) { + vector_search_ = std::make_shared(*split_read_.context_->GetVectorSearch()); + PAIMON_ASSIGN_OR_RAISE( + vector_search_->field_name, + ResolveSearchFieldName(vector_search_->field_name, field_names, file_->file_name)); + PAIMON_ASSIGN_OR_RAISE( + [[maybe_unused]] std::shared_ptr reader, + indexes_->GetSearchReader(vector_search_->field_name, + FileIndexReader::SearchType::VECTOR_SEARCH)); + } + if (split_read_.context_->GetFullTextSearch()) { + full_text_search_ = + std::make_shared(*split_read_.context_->GetFullTextSearch()); + PAIMON_ASSIGN_OR_RAISE( + full_text_search_->field_name, + ResolveSearchFieldName(full_text_search_->field_name, field_names, file_->file_name)); + PAIMON_ASSIGN_OR_RAISE( + [[maybe_unused]] std::shared_ptr reader, + indexes_->GetSearchReader(full_text_search_->field_name, + FileIndexReader::SearchType::FULL_TEXT_SEARCH)); + } + return Status::OK(); +} + +Result FileIndexReadExecutor::CreateCandidates( + const DeletionVector::Factory& dv_factory, + const std::optional>& row_ranges) const { + RoaringBitmap32 candidates; + if (row_ranges) { + for (const auto& range : *row_ranges) { + candidates.AddRange(static_cast(range.from), + static_cast(range.to + 1)); + } + } else { + candidates.AddRange(0, static_cast(file_->row_count)); + } + if (dv_factory) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr dv, dv_factory(file_->file_name)); + if (dynamic_cast(dv.get())) { + return Status::NotImplemented( + "Precise file-index evaluation does not support bitmap64 deletion vectors"); + } + if (auto bitmap = dynamic_cast(dv.get())) { + candidates -= *bitmap->GetBitmap(); + } else if (dv) { + return Status::NotImplemented( + "Precise file-index evaluation requires bitmap32 deletion vectors"); + } + } + return candidates; +} + +Result FileIndexReadExecutor::Evaluate( + const std::shared_ptr& predicate, const RoaringBitmap32& candidates) const { + if (!predicate || candidates.IsEmpty()) { + return Evaluation{candidates, {}}; + } + if (auto leaf = std::dynamic_pointer_cast(predicate)) { + return EvaluateLeaf(leaf, candidates); + } + if (auto compound = std::dynamic_pointer_cast(predicate)) { + return EvaluateCompound(compound, candidates); + } + return Status::Invalid(fmt::format("unsupported file predicate {}", predicate->ToString())); +} + +Result FileIndexReadExecutor::EvaluateCompound( + const std::shared_ptr& predicate, const RoaringBitmap32& candidates) const { + const Function::Type type = predicate->GetFunction().GetType(); + if (type == Function::Type::OR) { + Evaluation result; + for (const auto& child : predicate->Children()) { + // Every branch must see the same input, including rows matched by earlier branches. + PAIMON_ASSIGN_OR_RAISE(Evaluation child_result, Evaluate(child, candidates)); + result.rows |= child_result.rows; + result.scores.insert(child_result.scores.begin(), child_result.scores.end()); + } + return result; + } + if (type != Function::Type::AND) { + return Status::Invalid( + fmt::format("unsupported compound file predicate {}", predicate->ToString())); + } + + struct Child { + std::shared_ptr predicate; + std::optional selectivity; + }; + std::vector children; + children.reserve(predicate->Children().size()); + for (const auto& child : predicate->Children()) { + PAIMON_ASSIGN_OR_RAISE(std::optional selectivity, EstimateSelectivity(child)); + children.push_back({child, selectivity}); + } + std::stable_sort(children.begin(), children.end(), [](const Child& lhs, const Child& rhs) { + if (!lhs.selectivity) { + return false; + } + return !rhs.selectivity || *lhs.selectivity < *rhs.selectivity; + }); + + Evaluation result{candidates, {}}; + for (const auto& child : children) { + PAIMON_ASSIGN_OR_RAISE(Evaluation child_result, Evaluate(child.predicate, result.rows)); + result.rows = std::move(child_result.rows); + result.scores.insert(child_result.scores.begin(), child_result.scores.end()); + if (result.rows.IsEmpty()) { + break; + } + } + return result; +} + +Result> FileIndexReadExecutor::EstimateSelectivity( + const std::shared_ptr& predicate) const { + if (!predicate) { + return std::optional(); + } + if (auto leaf = std::dynamic_pointer_cast(predicate)) { + PAIMON_ASSIGN_OR_RAISE(std::optional estimate, EstimateLeafSelectivity(leaf)); + if (estimate && (!std::isfinite(*estimate) || *estimate < 0.0 || *estimate > 1.0)) { + return std::optional(); + } + return estimate; + } + auto compound = std::dynamic_pointer_cast(predicate); + if (!compound) { + return Status::Invalid(fmt::format("unsupported file predicate {}", predicate->ToString())); + } + const Function::Type type = compound->GetFunction().GetType(); + if (type != Function::Type::AND && type != Function::Type::OR) { + return Status::Invalid( + fmt::format("unsupported compound file predicate {}", predicate->ToString())); + } + std::optional estimate = + type == Function::Type::AND ? std::nullopt : std::make_optional(0.0); + // These are scheduling hints. Continue visiting all children to propagate errors. + for (const auto& child : compound->Children()) { + PAIMON_ASSIGN_OR_RAISE(std::optional child_estimate, EstimateSelectivity(child)); + if (type == Function::Type::AND) { + if (child_estimate) { + if (estimate.has_value()) { + estimate = std::min(*estimate, *child_estimate); + } else { + estimate = *child_estimate; + } + } + } else if (estimate && child_estimate) { + estimate = std::min(1.0, *estimate + *child_estimate); + } else { + estimate = std::nullopt; + } + } + return estimate; +} + +Result> FileIndexReadExecutor::EstimateLeafSelectivity( + const std::shared_ptr& predicate) const { + auto iter = mapped_leaves_.find(predicate); + if (iter == mapped_leaves_.end()) { + return Status::Invalid("File predicate mapping was not prepared"); + } + if (!iter->second) { + return std::optional(); + } + return indexes_->EstimateSelectivity(iter->second); +} + +Result FileIndexReadExecutor::EvaluateLeaf( + const std::shared_ptr& predicate, const RoaringBitmap32& candidates) const { + bool is_full_text = predicate->GetFunction().GetType() == Function::Type::FULL_TEXT_MATCH; + bool needs_scores = + is_full_text && + split_read_.context_->NeedsScores( + {InternalReadContext::IndexScoreSource::Type::FULL_TEXT_MATCH, predicate->FieldName()}); + std::shared_ptr index_result = FileIndexResult::Remain(); + auto iter = mapped_leaves_.find(predicate); + if (iter == mapped_leaves_.end()) { + return Status::Invalid("File predicate mapping was not prepared"); + } + if (iter->second) { + PAIMON_ASSIGN_OR_RAISE(index_result, indexes_->EvaluateLeaf(iter->second, needs_scores)); + } + if (!index_result) { + return Status::Invalid("file predicate index returned a null result"); + } + Evaluation result{candidates, {}}; + if (needs_scores) { + auto scored = std::dynamic_pointer_cast(index_result); + if (!scored) { + return Status::Invalid( + fmt::format("full-text index for field '{}' did not return requested scores", + predicate->FieldName())); + } + result.rows &= scored->GetRowPositions(); + result.scores.emplace(predicate->FieldName(), std::move(scored)); + return result; + } + + if (auto bitmap = std::dynamic_pointer_cast(index_result)) { + PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap32* rows, bitmap->GetBitmap()); + result.rows &= *rows; + } else if (is_full_text) { + return Status::Invalid(fmt::format("full-text index for field '{}' did not return a bitmap", + predicate->FieldName())); + } else if (std::dynamic_pointer_cast(index_result)) { + return Evaluation{}; + } else if (!std::dynamic_pointer_cast(index_result)) { + return Status::Invalid( + fmt::format("invalid index result for predicate {}", predicate->ToString())); + } + // TODO(lisizhuo.lsz): Whether to ScanScalar depends on whether the index returns exact results. + if (!is_full_text && !result.rows.IsEmpty()) { + // Scalar indexes may contain false positives, even if they return a bitmap. + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 exact_rows, ScanScalar(predicate, result.rows)); + result.rows &= exact_rows; + } + return result; +} + +Result> FileIndexReadExecutor::CreateSelectedFileReader( + const RoaringBitmap32& selection, const FieldMappingBuilder* field_mapping_builder) const { + PAIMON_ASSIGN_OR_RAISE(std::string format, file_->FileFormat()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr builder, + split_read_.PrepareReaderBuilder(format, extra_format_options_)); + return split_read_.CreateFieldMappingReader(path_factory_->ToPath(file_), file_, partition_, + std::move(builder), field_mapping_builder, nullptr, + std::nullopt, path_factory_, &selection); +} + +Result FileIndexReadExecutor::ScanScalar( + const std::shared_ptr& predicate, const RoaringBitmap32& input) const { + auto field = split_read_.context_->GetReadSchema()->GetFieldByName(predicate->FieldName()); + if (!field) { + return Status::Invalid(fmt::format( + "Scalar predicate field '{}' is missing from the read schema", predicate->FieldName())); + } + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr field_mapping_builder, + FieldMappingBuilder::Create(arrow::schema({field}), + split_read_.context_->GetPartitionKeys(), nullptr)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + CreateSelectedFileReader(input, field_mapping_builder.get())); + FileBatchReader& file_reader = *reader; + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr filtered_reader, + PredicateBatchReader::Create(std::move(reader), predicate, + split_read_.context_->GetArrowMemoryPool())); + ScopeGuard close([&]() { filtered_reader->Close(); }); + RoaringBitmap32 matches; + while (true) { + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch, + filtered_reader->NextBatchWithBitmap()); + if (BatchReader::IsEofBatch(batch)) { + break; + } + for (auto iter = batch.second.Begin(); iter != batch.second.End(); ++iter) { + int32_t batch_row = *iter; + if (batch_row < 0 || batch_row >= batch.first.first->length) { + return Status::Invalid("Scalar scan returned an invalid batch row position"); + } + PAIMON_ASSIGN_OR_RAISE(uint64_t row, file_reader.GetPreviousBatchFileRowId(batch_row)); + if (row >= static_cast(file_->row_count)) { + return Status::Invalid("Scalar scan returned an invalid file row position"); + } + matches.Add(static_cast(row)); + } + } + return matches; +} + +Status FileIndexReadExecutor::ApplyRootSearch(Evaluation* evaluated) const { + std::shared_ptr root_scores; + std::string field_name; + if (vector_search_) { + auto eligible = std::make_shared(evaluated->rows); + auto user_filter = vector_search_->pre_filter; + auto search = vector_search_->ReplacePreFilter([eligible, user_filter](int64_t row) { + return row >= 0 && row < RoaringBitmap32::MAX_VALUE && + eligible->Contains(static_cast(row)) && + (!user_filter || user_filter(row)); + }); + PAIMON_ASSIGN_OR_RAISE(root_scores, indexes_->SearchVector(search)); + field_name = split_read_.context_->GetVectorSearch()->field_name; + } else if (full_text_search_) { + RoaringBitmap64 eligible(evaluated->rows); + if (full_text_search_->include_row_ids) { + eligible &= *full_text_search_->include_row_ids; + } + auto search = std::make_shared(*full_text_search_); + search->include_row_ids = std::move(eligible); + PAIMON_ASSIGN_OR_RAISE(root_scores, indexes_->SearchFullText(search)); + field_name = split_read_.context_->GetFullTextSearch()->field_name; + } else { + return Status::OK(); + } + evaluated->rows &= root_scores->GetRowPositions(); + // Score bindings use the original field name, not the historical file schema's name. + evaluated->scores.emplace(field_name, std::move(root_scores)); + return Status::OK(); +} + +Result> FileIndexReadExecutor::AlignScores( + const std::shared_ptr& source, const RoaringBitmap32& rows) { + if (!source) { + return Status::Invalid("Projected search source did not return scores"); + } + std::vector aligned; + aligned.reserve(rows.Cardinality()); + size_t score = 0; + for (auto iter = source->GetRowPositions().Begin(); iter != source->GetRowPositions().End(); + ++iter, ++score) { + if (rows.Contains(*iter)) { + aligned.push_back(source->GetScores()[score]); + } + } + if (aligned.size() != static_cast(rows.Cardinality())) { + return Status::Invalid("Projected scores do not cover all selected file rows"); + } + return aligned; +} + +Result>> FileIndexReadExecutor::CreateOutputScores( + const Evaluation& evaluated) const { + std::map> output_scores; + for (const auto& [position, source] : split_read_.context_->GetIndexScoreBindings()) { + auto found = evaluated.scores.find(source.field_name); + if (found == evaluated.scores.end()) { + return Status::Invalid(fmt::format("Projected search field '{}' did not return scores", + source.field_name)); + } + PAIMON_ASSIGN_OR_RAISE(std::vector aligned, + AlignScores(found->second, evaluated.rows)); + output_scores.emplace(split_read_.context_->GetReadSchema()->field(position)->name(), + std::move(aligned)); + } + return output_scores; +} + +Result> FileIndexReadExecutor::CreateReader( + const DeletionVector::Factory& dv_factory, + const std::optional>& row_ranges) const { + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 candidates, CreateCandidates(dv_factory, row_ranges)); + PAIMON_ASSIGN_OR_RAISE(Evaluation evaluated, + Evaluate(split_read_.context_->GetPredicate(), candidates)); + if (evaluated.rows.IsEmpty()) { + return std::unique_ptr(); + } + PAIMON_RETURN_NOT_OK(ApplyRootSearch(&evaluated)); + if (evaluated.rows.IsEmpty()) { + return std::unique_ptr(); + } + std::map> output_scores; + PAIMON_ASSIGN_OR_RAISE(output_scores, CreateOutputScores(evaluated)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + CreateSelectedFileReader(evaluated.rows, field_mapping_builder_)); + if (!output_scores.empty()) { + return std::unique_ptr(std::make_unique( + std::move(reader), output_scores, split_read_.context_->GetArrowMemoryPool())); + } + return reader; +} + +} // namespace paimon diff --git a/src/paimon/core/operation/file_index_read_executor.h b/src/paimon/core/operation/file_index_read_executor.h new file mode 100644 index 000000000..ed1885876 --- /dev/null +++ b/src/paimon/core/operation/file_index_read_executor.h @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/core/deletionvectors/deletion_vector.h" +#include "paimon/reader/file_batch_reader.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/utils/range.h" +#include "paimon/utils/roaring_bitmap32.h" + +namespace paimon { +class BinaryRow; +class CompoundPredicate; +class DataField; +class DataFilePathFactory; +class FieldMappingBuilder; +class FileIndexEvaluator; +class LeafPredicate; +class Predicate; +class RawFileSplitRead; +class ScoredFileIndexResult; +struct DataFileMeta; +struct FullTextSearch; +struct VectorSearch; + +/// Evaluates a file predicate and root search before opening the selected output rows. +class FileIndexReadExecutor { + public: + static Result> Create( + const RawFileSplitRead& split_read, const BinaryRow& partition, + const std::shared_ptr& file, const FieldMappingBuilder* field_mapping_builder, + const std::shared_ptr& path_factory, + const std::map& extra_format_options); + + /// Finishes evaluation synchronously. The returned reader does not retain this executor. + Result> CreateReader( + const DeletionVector::Factory& dv_factory, + const std::optional>& row_ranges) const; + + private: + struct Evaluation { + RoaringBitmap32 rows; + // Sources retain physical row positions until all filtering and root TopK are complete. + std::map> scores; + }; + + FileIndexReadExecutor(const RawFileSplitRead& split_read, const BinaryRow& partition, + const std::shared_ptr& file, + const FieldMappingBuilder* field_mapping_builder, + const std::shared_ptr& path_factory, + const std::map& extra_format_options); + + Status Prepare(); + Status PreparePredicate(const std::shared_ptr& predicate, + const std::map& read_by_name, + const std::map>& data_by_id); + Status PrepareSearch(const std::map& field_names); + Result CreateCandidates( + const DeletionVector::Factory& dv_factory, + const std::optional>& row_ranges) const; + Result Evaluate(const std::shared_ptr& predicate, + const RoaringBitmap32& candidates) const; + Result EvaluateCompound(const std::shared_ptr& predicate, + const RoaringBitmap32& candidates) const; + Result EvaluateLeaf(const std::shared_ptr& predicate, + const RoaringBitmap32& candidates) const; + Result> EstimateSelectivity( + const std::shared_ptr& predicate) const; + Result> EstimateLeafSelectivity( + const std::shared_ptr& predicate) const; + /// Opens a fresh mapped reader for selected physical rows without re-evaluating indexes, + /// predicates or deletion vectors. + Result> CreateSelectedFileReader( + const RoaringBitmap32& selection, const FieldMappingBuilder* field_mapping_builder) const; + Result ScanScalar(const std::shared_ptr& predicate, + const RoaringBitmap32& candidates) const; + Status ApplyRootSearch(Evaluation* evaluated) const; + Result>> CreateOutputScores( + const Evaluation& evaluated) const; + static Result> AlignScores( + const std::shared_ptr& source, const RoaringBitmap32& rows); + + private: + const RawFileSplitRead& split_read_; + const BinaryRow& partition_; + std::shared_ptr file_; + const FieldMappingBuilder* field_mapping_builder_; + std::shared_ptr path_factory_; + std::map extra_format_options_; + std::unique_ptr indexes_; + std::map, std::shared_ptr> mapped_leaves_; + std::shared_ptr vector_search_; + std::shared_ptr full_text_search_; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/file_index_read_executor_test.cpp b/src/paimon/core/operation/file_index_read_executor_test.cpp new file mode 100644 index 000000000..9e1bb7142 --- /dev/null +++ b/src/paimon/core/operation/file_index_read_executor_test.cpp @@ -0,0 +1,820 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/operation/file_index_read_executor.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/json_simple.h" +#include "gtest/gtest.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/common/file_index/bitmap/bitmap_file_index.h" +#include "paimon/common/predicate/leaf_predicate_impl.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/deletionvectors/bitmap64_deletion_vector.h" +#include "paimon/core/deletionvectors/bitmap_deletion_vector.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/io/file_index_evaluator.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/core/operation/internal_read_context.h" +#include "paimon/core/operation/raw_file_split_read.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/stats/simple_stats.h" +#include "paimon/core/utils/field_mapping.h" +#include "paimon/data/timestamp.h" +#include "paimon/factories/factory_creator.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/full_text_match.h" +#include "paimon/predicate/full_text_search.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/predicate/vector_search.h" +#include "paimon/read_context.h" +#include "paimon/status.h" +#include "paimon/testing/mock/mock_search_file_index.h" +#include "paimon/testing/utils/binary_row_generator.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +namespace { + +class RecordingDoubleFunction final : public LeafFunction { + public: + RecordingDoubleFunction(const LeafFunction& function, std::vector* evaluated) + : function_(function), evaluated_(evaluated) {} + + Type GetType() const override { + return function_.GetType(); + } + + std::string ToString() const override { + return function_.ToString(); + } + + const LeafFunction* Negate() const override { + return function_.Negate(); + } + + Result> Test(const arrow::Array& array, const std::vector& literals, + arrow::MemoryPool* pool) const override { + const auto* values = dynamic_cast(&array); + if (!values) { + return Status::Invalid("Recording function expects a double field"); + } + for (int64_t i = 0; i < values->length(); ++i) { + evaluated_->push_back(values->Value(i)); + } + return function_.Test(array, literals, pool); + } + + Result Test(const Literal& value, const std::vector& literals) const override { + return function_.Test(value, literals); + } + + Result Test(int64_t row_count, const Literal& min_value, const Literal& max_value, + const std::optional& null_count, + const std::vector& literals) const override { + return function_.Test(row_count, min_value, max_value, null_count, literals); + } + + private: + const LeafFunction& function_; + std::vector* evaluated_; +}; + +} // namespace + +class FileIndexReadExecutorTest : public ::testing::Test { + protected: + void SetUp() override { + auto factory = std::make_unique(statistics_); + FactoryCreator::GetInstance()->Register(statistics_index_type_, factory.release()); + } + + void TearDown() override { + FactoryCreator::GetInstance()->TEST_Unregister(statistics_index_type_); + } + + static std::string SearchTablePath() { + return GetDataDir() + "/orc/multi_partition_append_table.db/multi_partition_append_table"; + } + + std::shared_ptr PrepareDataFile() const { + return std::make_shared( + "data-01b6a930-6564-409b-b8f4-ed1307790d72-0.orc", /*file_size=*/575, /*row_count=*/3, + /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), + /*key_stats=*/SimpleStats::EmptyStats(), + BinaryRowGenerator::GenerateStats({std::string("Bob"), 10, 0, 12.1}, + {std::string("Tony"), 10, 0, 14.1}, {0, 0, 0, 0}, + pool_.get()), + /*min_sequence_number=*/0, /*max_sequence_number=*/2, /*schema_id=*/0, + /*level=*/0, /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(1728497439433ll, 0), + /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); + } + + Result> CreateSplitRead( + const std::shared_ptr& read_context, + std::shared_ptr table_schema = nullptr) const { + if (!table_schema) { + SchemaManager schema_manager(std::make_shared(), + read_context->GetPath()); + PAIMON_ASSIGN_OR_RAISE(table_schema, schema_manager.ReadSchema(0)); + } + auto options = table_schema->Options(); + for (const auto& [key, value] : read_context->GetOptions()) { + options[key] = value; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr context, + InternalReadContext::Create(read_context, table_schema, options)); + return std::make_unique(nullptr, context, pool_, nullptr); + } + + Result> ReadWithFileIndex( + const std::shared_ptr& read_context, const std::shared_ptr& index, + const RoaringBitmap32& deleted_rows = RoaringBitmap32(), + std::shared_ptr table_schema = nullptr, + const std::optional>& row_ranges = std::nullopt, + std::shared_ptr deletion_vector = nullptr, + const std::optional& row_count = std::nullopt) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr split_read, + CreateSplitRead(read_context, std::move(table_schema))); + auto file = PrepareDataFile(); + file->embedded_index = index; + if (row_count) { + file->row_count = *row_count; + } + const std::string bucket_path = SearchTablePath() + "/f1=10/f2=0/bucket-0"; + auto path_factory = std::make_shared(); + PAIMON_RETURN_NOT_OK(path_factory->Init(bucket_path, "orc", "data-", nullptr)); + const auto& context = split_read->context_; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr field_mapping_builder, + FieldMappingBuilder::Create(context->GetReadSchema(), + context->GetPartitionKeys(), nullptr)); + const BinaryRow partition = BinaryRowGenerator::GenerateRow({10, 0}, pool_.get()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr executor, + FileIndexReadExecutor::Create(*split_read, partition, file, field_mapping_builder.get(), + path_factory, {})); + if (!deletion_vector) { + deletion_vector = std::make_shared(deleted_rows); + } + DeletionVector::Factory dv_factory = + [deletion_vector](const std::string&) -> Result> { + return deletion_vector; + }; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + executor->CreateReader(dv_factory, row_ranges)); + if (!reader) { + return std::shared_ptr(); + } + return ReadResultCollector::CollectResult(std::move(reader)); + } + + Result> ReadQueryPredicate( + const std::shared_ptr& predicate, + const MockSearchFileIndex::QueryMatches& matches) const { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr index, + MockSearchFileIndex::MakeQueryIndex("f0", statistics_index_type_, matches, pool_)); + ReadContextBuilder builder(SearchTablePath()); + builder.SetReadFieldNames({"f0"}).SetPredicate(predicate).EnablePreciseFileIndexEvaluate( + true); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr context, builder.Finish()); + return ReadWithFileIndex(context, index); + } + + static void CheckSearchResult(const std::shared_ptr& actual, + const arrow::FieldVector& fields, const std::string& json) { + ASSERT_TRUE(actual); + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow::struct_(fields), {json}, + &expected) + .ok()); + ASSERT_TRUE(actual->Equals(expected)) << actual->ToString(); + } + + std::shared_ptr pool_ = GetDefaultPool(); + const std::string statistics_index_type_ = "executor-statistics-full-text"; + std::shared_ptr statistics_ = + std::make_shared(); +}; + +TEST_F(FileIndexReadExecutorTest, TestMixedPredicateNestedAndOrWithPartitionConstants) { + const std::string first_query = R"({"match":{"query":"first"}})"; + const std::string second_query = R"({"match":{"query":"second"}})"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr first, + PredicateBuilder::FullTextMatch(0, std::make_shared("f0", first_query))); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr second, + PredicateBuilder::FullTextMatch(0, std::make_shared("f0", second_query))); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr first_branch, + PredicateBuilder::And( + {first, PredicateBuilder::GreaterThan(3, "f3", FieldType::DOUBLE, Literal(12.5))})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr second_branch, + PredicateBuilder::And({second, PredicateBuilder::Equal( + 2, "f2", FieldType::INT, Literal(0))})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr alternatives, + PredicateBuilder::Or({first_branch, second_branch})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr predicate, + PredicateBuilder::And( + {PredicateBuilder::Equal(1, "f1", FieldType::INT, Literal(10)), alternatives})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr index, + MockSearchFileIndex::MakeQueryIndex( + "f0", statistics_index_type_, + {{first_query, {{{0, 0.2f}, {1, 0.4f}}, 0.2}}, {second_query, {{{2, 0.6f}}, 0.1}}}, + pool_)); + ReadContextBuilder builder(SearchTablePath()); + builder.EnablePreciseFileIndexEvaluate(true); + builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}) + .SetPredicate(predicate) + .AddOption(Options::READ_BATCH_SIZE, "1"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadWithFileIndex(context, index)); + CheckSearchResult(result, + {arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}, + R"([["Emily",10,0,13.1],["Tony",10,0,14.1]])"); + ASSERT_EQ(statistics_->executions.size(), 2); + ASSERT_EQ(statistics_->executions[0].query, first_query); + ASSERT_EQ(statistics_->executions[1].query, second_query); + for (const auto& execution : statistics_->executions) { + ASSERT_FALSE(execution.needs_scores); + } +} + +TEST_F(FileIndexReadExecutorTest, TestMixedPredicateReordersQueries) { + const std::string wide_query = R"({"match":{"query":"wide"}})"; + const std::string narrow_query = R"({"match":{"query":"narrow"}})"; + auto wide_match = + std::make_shared("f0", wide_query, RoaringBitmap64::From({1, 2})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr wide, + PredicateBuilder::FullTextMatch(0, wide_match)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr narrow, + PredicateBuilder::FullTextMatch(0, std::make_shared("f0", narrow_query))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({wide, narrow, + PredicateBuilder::GreaterThan( + 1, "f3", FieldType::DOUBLE, Literal(12.0))})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr index, + MockSearchFileIndex::MakeQueryIndex("f0", statistics_index_type_, + {{wide_query, {{{0, 0.1f}, {1, 0.2f}, {2, 0.3f}}, 0.9}}, + {narrow_query, {{{0, 0.4f}, {2, 0.5f}}, 0.1}}}, + pool_)); + ReadContextBuilder builder(SearchTablePath()); + builder.EnablePreciseFileIndexEvaluate(true); + builder.SetReadFieldNames({"f0", "f3"}).SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadWithFileIndex(context, index)); + CheckSearchResult(result, + {arrow::field("f0", arrow::utf8()), arrow::field("f3", arrow::float64())}, + R"([["Tony",14.1]])"); + ASSERT_EQ(statistics_->executions.size(), 2); + ASSERT_EQ(statistics_->executions[0].query, narrow_query); + ASSERT_EQ(statistics_->executions[1].query, wide_query); + ASSERT_EQ(wide_match->include_row_ids, RoaringBitmap64::From({1, 2})); +} + +TEST_F(FileIndexReadExecutorTest, TestMixedPredicateSelectivityOrder) { + const std::vector queries = { + R"({"match":{"query":"a"}})", R"({"match":{"query":"b"}})", R"({"match":{"query":"c"}})", + R"({"match":{"query":"d"}})", R"({"match":{"query":"e"}})"}; + std::vector> leaves; + for (const auto& query : queries) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr leaf, + PredicateBuilder::FullTextMatch(0, std::make_shared("f0", query))); + leaves.push_back(leaf); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, PredicateBuilder::And(leaves)); + struct Case { + std::vector> estimates; + std::vector order; + }; + const std::vector cases = { + {{std::nullopt, 0.8, 0.2, 0.2, std::nullopt}, {2, 3, 1, 0, 4}}, + {{std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt}, {0, 1, 2, 3, 4}}, + {{std::numeric_limits::quiet_NaN(), -0.1, 1.1, 0.5, + std::numeric_limits::infinity()}, + {3, 0, 1, 2, 4}}}; + for (size_t case_id = 0; case_id < cases.size(); ++case_id) { + const auto& test_case = cases[case_id]; + MockSearchFileIndex::QueryMatches matches; + for (size_t i = 0; i < queries.size(); ++i) { + matches.emplace(queries[i], + MockSearchFileIndex::QueryResult{{{0, 0.1f}, {1, 0.2f}, {2, 0.3f}}, + test_case.estimates[i]}); + } + statistics_->executions.clear(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadQueryPredicate(predicate, matches)); + CheckSearchResult(result, {arrow::field("f0", arrow::utf8())}, + R"([["Bob"],["Emily"],["Tony"]])"); + ASSERT_EQ(statistics_->executions.size(), queries.size()); + for (size_t i = 0; i < queries.size(); ++i) { + ASSERT_EQ(statistics_->executions[i].query, queries[test_case.order[i]]); + } + } +} + +TEST_F(FileIndexReadExecutorTest, TestMixedPredicateZeroEstimateOnlyChangesOrder) { + const std::string a_query = R"({"match":{"query":"a"}})"; + const std::string b_query = R"({"match":{"query":"b"}})"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr a, + PredicateBuilder::FullTextMatch(0, std::make_shared("f0", a_query))); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr b, + PredicateBuilder::FullTextMatch(0, std::make_shared("f0", b_query))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, PredicateBuilder::And({a, b})); + for (bool a_first : {true, false}) { + MockSearchFileIndex::QueryMatches matches = { + {a_query, {{{0, 0.1f}, {1, 0.2f}}, a_first ? 0.0 : 0.9}}, + {b_query, {{{1, 0.3f}, {2, 0.4f}}, a_first ? 0.9 : 0.0}}}; + statistics_->executions.clear(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadQueryPredicate(predicate, matches)); + CheckSearchResult(result, {arrow::field("f0", arrow::utf8())}, R"([["Emily"]])"); + ASSERT_EQ(statistics_->executions.size(), 2); + ASSERT_EQ(statistics_->executions[0].query, a_first ? a_query : b_query); + ASSERT_EQ(statistics_->executions[1].query, a_first ? b_query : a_query); + } +} + +TEST_F(FileIndexReadExecutorTest, TestMixedPredicateCompoundSelectivity) { + const std::string a_query = R"({"match":{"query":"a"}})"; + const std::string b_query = R"({"match":{"query":"b"}})"; + const std::string c_query = R"({"match":{"query":"c"}})"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr a, + PredicateBuilder::FullTextMatch(0, std::make_shared("f0", a_query))); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr b, + PredicateBuilder::FullTextMatch(0, std::make_shared("f0", b_query))); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr c, + PredicateBuilder::FullTextMatch(0, std::make_shared("f0", c_query))); + struct Case { + bool inner_and; + std::optional a_estimate; + std::optional b_estimate; + double c_estimate; + std::vector order; + }; + const std::vector cases = { + {true, 0.2, 0.3, 0.1, {c_query, a_query, b_query}}, + {true, 0.2, 0.3, 0.4, {a_query, b_query, c_query}}, + {true, 0.2, std::nullopt, 0.3, {a_query, b_query, c_query}}, + {true, std::nullopt, 0.2, 0.3, {b_query, a_query, c_query}}, + {true, std::nullopt, std::nullopt, 0.3, {c_query, a_query, b_query}}, + {true, 0.0, std::nullopt, 0.1, {a_query, b_query, c_query}}, + {false, 0.2, 0.3, 0.6, {a_query, b_query, c_query}}, + {false, 0.6, 0.8, 1.0, {a_query, b_query, c_query}}, + {false, 0.2, std::nullopt, 0.6, {c_query, a_query, b_query}}, + {false, std::nullopt, 0.2, 0.6, {c_query, a_query, b_query}}, + {false, std::nullopt, std::nullopt, 0.6, {c_query, a_query, b_query}}}; + for (size_t case_id = 0; case_id < cases.size(); ++case_id) { + const auto& test_case = cases[case_id]; + ASSERT_OK_AND_ASSIGN(std::shared_ptr inner, test_case.inner_and + ? PredicateBuilder::And({a, b}) + : PredicateBuilder::Or({a, b})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({inner, c})); + MockSearchFileIndex::QueryMatches matches = { + {a_query, {{{0, 0.1f}, {1, 0.2f}}, test_case.a_estimate}}, + {b_query, {{{1, 0.3f}, {2, 0.4f}}, test_case.b_estimate}}, + {c_query, {{{0, 0.5f}, {1, 0.6f}, {2, 0.7f}}, test_case.c_estimate}}}; + statistics_->executions.clear(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadQueryPredicate(predicate, matches)); + CheckSearchResult( + result, {arrow::field("f0", arrow::utf8())}, + test_case.inner_and ? R"([["Emily"]])" : R"([["Bob"],["Emily"],["Tony"]])"); + ASSERT_EQ(statistics_->executions.size(), 3); + for (size_t i = 0; i < test_case.order.size(); ++i) { + ASSERT_EQ(statistics_->executions[i].query, test_case.order[i]); + } + } +} + +TEST_F(FileIndexReadExecutorTest, TestMixedPredicateEmptyRowsShortCircuitExecution) { + const std::string empty_query = R"({"match":{"query":"empty"}})"; + const std::string all_query = R"({"match":{"query":"all"}})"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr empty, + PredicateBuilder::FullTextMatch(0, std::make_shared("f0", empty_query))); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr all, + PredicateBuilder::FullTextMatch(0, std::make_shared("f0", all_query))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, PredicateBuilder::And({empty, all})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr index, + MockSearchFileIndex::MakeQueryIndex( + "f0", statistics_index_type_, + {{empty_query, {{}, std::nullopt}}, + {all_query, {{{0, 0.1f}, {1, 0.2f}, {2, 0.3f}}, std::nullopt}}}, + pool_)); + ReadContextBuilder builder(SearchTablePath()); + builder.EnablePreciseFileIndexEvaluate(true); + builder.SetReadFieldNames({"f0"}).SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + for (bool all_deleted : {false, true}) { + statistics_->executions.clear(); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr result, + ReadWithFileIndex(context, index, + all_deleted ? RoaringBitmap32::From({0, 1, 2}) : RoaringBitmap32())); + ASSERT_TRUE(!result || result->length() == 0); + ASSERT_EQ(statistics_->executions.size(), all_deleted ? 0 : 1); + if (!all_deleted) { + ASSERT_EQ(statistics_->executions[0].query, empty_query); + } + } +} + +TEST_F(FileIndexReadExecutorTest, TestMixedPredicateOrEvaluatesAllBranches) { + const std::string all_query = R"({"match":{"query":"all"}})"; + const std::string one_query = R"({"match":{"query":"one"}})"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr all, + PredicateBuilder::FullTextMatch(0, std::make_shared("f0", all_query))); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr one, + PredicateBuilder::FullTextMatch(0, std::make_shared("f0", one_query))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, PredicateBuilder::Or({all, one})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr result, + ReadQueryPredicate(predicate, {{all_query, {{{0, 0.1f}, {1, 0.2f}, {2, 0.3f}}, 0.9}}, + {one_query, {{{1, 0.4f}}, 0.1}}})); + CheckSearchResult(result, {arrow::field("f0", arrow::utf8())}, + R"([["Bob"],["Emily"],["Tony"]])"); + ASSERT_EQ(statistics_->executions.size(), 2); + ASSERT_EQ(statistics_->executions[0].query, all_query); + ASSERT_EQ(statistics_->executions[1].query, one_query); +} + +TEST_F(FileIndexReadExecutorTest, TestMixedPredicateIndexedLeafSkipsUnselectedScalarBatches) { + const std::string query = R"({"match":{"query":"selected"}})"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr indexed, + PredicateBuilder::FullTextMatch(0, std::make_shared("f0", query))); + auto scalar = std::dynamic_pointer_cast( + PredicateBuilder::GreaterThan(1, "f3", FieldType::DOUBLE, Literal(13.0))); + ASSERT_TRUE(scalar); + std::vector evaluated; + RecordingDoubleFunction scalar_function(scalar->GetLeafFunction(), &evaluated); + auto unindexed = std::make_shared( + scalar_function, scalar->FieldIndex(), scalar->FieldName(), scalar->GetFieldType(), + scalar->Literals()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({unindexed, indexed})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr index, + MockSearchFileIndex::MakeQueryIndex("f0", statistics_index_type_, + {{query, {{{0, 0.25f}, {2, 0.75f}}, 0.1}}}, pool_)); + ReadContextBuilder builder(SearchTablePath()); + builder.EnablePreciseFileIndexEvaluate(true); + // Single-row batches let bitmap selection skip the unselected scalar evaluation batch. + builder.SetReadFieldNames({"f0", "f3"}) + .SetPredicate(predicate) + .AddOption(Options::READ_BATCH_SIZE, "1"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadWithFileIndex(context, index)); + CheckSearchResult(result, + {arrow::field("f0", arrow::utf8()), arrow::field("f3", arrow::float64())}, + R"([["Tony",14.1]])"); + ASSERT_EQ(evaluated, std::vector({12.1, 14.1})); + ASSERT_EQ(statistics_->executions.size(), 1); +} + +TEST_F(FileIndexReadExecutorTest, TestMixedPredicateRechecksScalarIndexCandidates) { + auto field = arrow::field("f3", arrow::float64()); + ASSERT_OK_AND_ASSIGN(auto writer, BitmapFileIndexWriter::Create(field, {}, pool_)); + // Include Bob as a false positive: the data file contains 12.1 for row 0. + auto indexed_values = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({field}), + R"([[13.1], [13.1], [14.1]])"); + ASSERT_TRUE(indexed_values.ok()) << indexed_values.status().ToString(); + ::ArrowArray c_array; + ASSERT_TRUE(arrow::ExportArray(*indexed_values.ValueUnsafe(), &c_array).ok()); + ASSERT_OK(writer->AddBatch(&c_array)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr scalar_payload, writer->SerializedBytes()); + + const std::string query = R"({"match":{"query":"all"}})"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr text_payload, + MockSearchFileIndex::MakeQueryPayload( + {{query, {{{0, 0.25f}, {1, 0.5f}, {2, 0.75f}}, std::nullopt}}}, pool_)); + FileIndexFormat::ColumnIndexes indexes; + indexes["f3"]["bitmap"] = scalar_payload; + indexes["f0"][statistics_index_type_] = text_payload; + ASSERT_OK_AND_ASSIGN(std::shared_ptr index, + MockSearchFileIndex::Serialize(indexes, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr text, + PredicateBuilder::FullTextMatch(0, std::make_shared("f0", query))); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr predicate, + PredicateBuilder::And( + {PredicateBuilder::Equal(1, "f3", FieldType::DOUBLE, Literal(13.1)), text})); + ReadContextBuilder builder(SearchTablePath()); + builder.EnablePreciseFileIndexEvaluate(true); + builder.SetReadFieldNames({"f0", "f3", "_f0_INDEX_SCORE"}) + .SetPredicate(predicate) + .AddOption(Options::READ_BATCH_SIZE, "1"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadWithFileIndex(context, index)); + CheckSearchResult(result, + {arrow::field("f0", arrow::utf8()), field, + arrow::field("_f0_INDEX_SCORE", arrow::float32())}, + R"([["Emily",13.1,0.5]])"); + ASSERT_EQ(statistics_->executions.size(), 1); + ASSERT_TRUE(statistics_->executions.front().needs_scores); +} + +TEST_F(FileIndexReadExecutorTest, TestMixedPredicatePrechecksSearchBeforeEmptyCandidates) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr text, + PredicateBuilder::FullTextMatch( + 0, std::make_shared("f0", R"({"match":{"query":"q"}})"))); + for (int32_t case_id : {0, 1, 2}) { + auto scalar = + PredicateBuilder::Equal(1, "f1", FieldType::INT, Literal(case_id == 0 ? 20 : 10)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({scalar, text})); + ReadContextBuilder builder(SearchTablePath()); + builder.EnablePreciseFileIndexEvaluate(true); + builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + const auto deleted = case_id == 1 ? RoaringBitmap32::From({0, 1, 2}) : RoaringBitmap32(); + const std::optional> ranges = + case_id == 2 ? std::make_optional(std::vector()) : std::nullopt; + ASSERT_NOK_WITH_MSG(ReadWithFileIndex(context, nullptr, deleted, nullptr, ranges), + "has no File Index for search field 'f0'"); + } +} + +TEST_F(FileIndexReadExecutorTest, TestMixedPredicateRenamedTextAndMissingScalarColumn) { + SchemaManager schemas(std::make_shared(), SearchTablePath()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr original, schemas.ReadSchema(0)); + auto fields = original->Fields(); + fields[0] = DataField(fields[0].Id(), fields[0].ArrowField()->WithName("renamed_f0")); + fields.emplace_back(4, arrow::field("new_value", arrow::int32())); + auto options = original->Options(); + options[Options::READ_BATCH_SIZE] = "1"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr current, + TableSchema::Create(1, DataField::ConvertDataFieldsToArrowSchema(fields), + original->PartitionKeys(), {}, options)); + const std::string query = R"({"match":{"query":"all"}})"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr text, + PredicateBuilder::FullTextMatch(0, std::make_shared("renamed_f0", query))); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr predicate, + PredicateBuilder::And( + {text, PredicateBuilder::IsNull(1, "new_value", FieldType::INT), + PredicateBuilder::GreaterThan(2, "f3", FieldType::DOUBLE, Literal(13.5))})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr index, + MockSearchFileIndex::MakeQueryIndex( + "f0", MockSearchFileIndex::FULL_TEXT_IDENTIFIER, + {{query, {{{0, 0.25f}, {1, 0.5f}, {2, 0.75f}}, 0.5}}}, pool_)); + ReadContextBuilder builder(SearchTablePath()); + builder.EnablePreciseFileIndexEvaluate(true); + builder.SetReadFieldNames({"renamed_f0", "new_value", "f3", "_renamed_f0_INDEX_SCORE"}) + .SetOptions(options) + .SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadWithFileIndex(context, index, RoaringBitmap32(), current)); + CheckSearchResult( + result, + {arrow::field("renamed_f0", arrow::utf8()), arrow::field("new_value", arrow::int32()), + arrow::field("f3", arrow::float64()), + arrow::field("_renamed_f0_INDEX_SCORE", arrow::float32())}, + R"([["Tony",null,14.1,0.75]])"); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr nonnull, + PredicateBuilder::And({text, PredicateBuilder::IsNotNull(1, "new_value", FieldType::INT)})); + ReadContextBuilder nonnull_builder(SearchTablePath()); + nonnull_builder.EnablePreciseFileIndexEvaluate(true); + nonnull_builder.SetReadFieldNames({"renamed_f0", "new_value", "f3", "_renamed_f0_INDEX_SCORE"}) + .SetOptions(options) + .SetPredicate(nonnull); + ASSERT_OK_AND_ASSIGN(std::shared_ptr nonnull_context, nonnull_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr empty, + ReadWithFileIndex(nonnull_context, index, RoaringBitmap32(), current)); + ASSERT_TRUE(!empty || empty->length() == 0); +} + +TEST_F(FileIndexReadExecutorTest, TestMixedPredicateFullTextAndVectorScoresAfterFiltering) { + const std::string query = R"({"match":{"query":"all"}})"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr text, + PredicateBuilder::FullTextMatch(1, std::make_shared("f0", query))); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr predicate, + PredicateBuilder::And( + {text, PredicateBuilder::GreaterThan(2, "f3", FieldType::DOUBLE, Literal(12.5))})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr text_payload, + MockSearchFileIndex::MakeQueryPayload( + {{query, {{{0, 0.25f}, {1, 0.5f}, {2, 0.75f}}, 0.5}}}, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr vector_payload, + MockSearchFileIndex::MakePayload({{0, 0.125f}, {1, 0.25f}, {2, 0.5f}}, pool_)); + FileIndexFormat::ColumnIndexes indexes; + indexes["f0"][MockSearchFileIndex::FULL_TEXT_IDENTIFIER] = text_payload; + indexes["f3"][MockSearchFileIndex::VECTOR_IDENTIFIER] = vector_payload; + ASSERT_OK_AND_ASSIGN(std::shared_ptr index, + MockSearchFileIndex::Serialize(indexes, pool_)); + auto search = std::make_shared("f3", 1, std::vector{1.0f}, nullptr, + nullptr, VectorSearch::DistanceType::EUCLIDEAN, + std::map{}); + ReadContextBuilder builder(SearchTablePath()); + builder.EnablePreciseFileIndexEvaluate(true); + builder.SetReadFieldNames({"_f3_INDEX_SCORE", "f0", "f3", "_f0_INDEX_SCORE"}) + .SetPredicate(predicate) + .SetVectorSearch(search) + .AddOption(Options::READ_BATCH_SIZE, "1"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + for (int32_t case_id : {0, 1, 2}) { + const auto deleted = case_id == 1 ? RoaringBitmap32::From({1}) : RoaringBitmap32(); + const std::optional> ranges = + case_id == 2 ? std::make_optional(std::vector{Range(2, 2)}) : std::nullopt; + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadWithFileIndex(context, index, deleted, nullptr, ranges)); + CheckSearchResult( + result, + {arrow::field("_f3_INDEX_SCORE", arrow::float32()), arrow::field("f0", arrow::utf8()), + arrow::field("f3", arrow::float64()), + arrow::field("_f0_INDEX_SCORE", arrow::float32())}, + case_id == 0 ? R"([[0.25,"Emily",13.1,0.5]])" : R"([[0.5,"Tony",14.1,0.75]])"); + } +} + +TEST_F(FileIndexReadExecutorTest, TestFullTextRootScoresAndLimitAfterScalarDvAndRanges) { + const std::string query = R"({"match":{"query":"ranked"}})"; + auto search = std::make_shared("f0", query, 1, RoaringBitmap64::From({0, 2})); + ReadContextBuilder builder(SearchTablePath()); + builder.EnablePreciseFileIndexEvaluate(true); + builder.SetReadFieldNames({"f0", "f3", "_f0_INDEX_SCORE"}) + .SetFullTextSearch(search) + .SetPredicate(PredicateBuilder::GreaterThan(1, "f3", FieldType::DOUBLE, Literal(13.5))) + .AddOption(Options::READ_BATCH_SIZE, "1"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr index, + MockSearchFileIndex::MakeQueryIndex( + "f0", MockSearchFileIndex::FULL_TEXT_IDENTIFIER, + {{query, {{{0, 0.25f}, {1, 0.5f}, {2, 0.75f}}, std::nullopt}}}, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadWithFileIndex(context, index, RoaringBitmap32::From({0}), nullptr, + std::vector{Range(1, 2)})); + CheckSearchResult(result, + {arrow::field("f0", arrow::utf8()), arrow::field("f3", arrow::float64()), + arrow::field("_f0_INDEX_SCORE", arrow::float32())}, + R"([["Tony",14.1,0.75]])"); + ASSERT_EQ(search->include_row_ids, RoaringBitmap64::From({0, 2})); +} + +TEST_F(FileIndexReadExecutorTest, TestPreciseEvaluationRejectsBitmap64DeletionVectors) { + const std::string query = R"({"match":{"query":"all"}})"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr text, + MockSearchFileIndex::MakeQueryPayload( + {{query, {{{0, 0.25f}, {1, 0.5f}, {2, 0.75f}}, std::nullopt}}}, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr vector, + MockSearchFileIndex::MakePayload({{0, 0.25f}, {1, 0.5f}, {2, 0.75f}}, pool_)); + FileIndexFormat::ColumnIndexes indexes; + indexes["f0"][MockSearchFileIndex::FULL_TEXT_IDENTIFIER] = text; + indexes["f3"][MockSearchFileIndex::VECTOR_IDENTIFIER] = vector; + ASSERT_OK_AND_ASSIGN(std::shared_ptr index, + MockSearchFileIndex::Serialize(indexes, pool_)); + for (const std::string& query_type : + std::vector{"none", "scalar", "match", "full-text", "vector"}) { + ReadContextBuilder builder(SearchTablePath()); + builder.SetReadFieldNames({"f0", "f3"}).EnablePreciseFileIndexEvaluate(true); + if (query_type == "scalar") { + builder.SetPredicate( + PredicateBuilder::GreaterThan(1, "f3", FieldType::DOUBLE, Literal(13.0))); + } else if (query_type == "match") { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr predicate, + PredicateBuilder::FullTextMatch(0, std::make_shared("f0", query))); + builder.SetPredicate(predicate); + } else if (query_type == "full-text") { + builder.SetFullTextSearch(std::make_shared("f0", query, /*limit=*/1)); + } else if (query_type == "vector") { + builder.SetVectorSearch(std::make_shared( + "f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, + VectorSearch::DistanceType::EUCLIDEAN, std::map{})); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + for (bool empty_dv : {false, true}) { + auto dv = std::make_shared(); + if (!empty_dv) { + ASSERT_OK(dv->Delete(1)); + } + for (bool empty_candidates : {false, true}) { + const std::optional> ranges = + empty_candidates ? std::make_optional(std::vector()) : std::nullopt; + ASSERT_NOK_WITH_MSG( + ReadWithFileIndex(context, index, RoaringBitmap32(), nullptr, ranges, dv), + "Precise file-index evaluation does not support bitmap64 deletion vectors"); + } + } + } +} + +TEST_F(FileIndexReadExecutorTest, TestPreciseEvaluationRejectsRowsAboveBitmap32Limit) { + for (const std::string& search_type : + std::vector{"none", "scalar", "match", "full-text", "vector"}) { + ReadContextBuilder builder(SearchTablePath()); + builder.EnablePreciseFileIndexEvaluate(true); + builder.SetReadFieldNames({"f0", "f3"}); + if (search_type == "scalar") { + builder.SetPredicate(PredicateBuilder::IsNotNull(0, "f0", FieldType::STRING)); + } else if (search_type == "match") { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr predicate, + PredicateBuilder::FullTextMatch( + 0, std::make_shared("f0", R"({"match":{"query":"all"}})"))); + builder.SetPredicate(predicate); + } else if (search_type == "full-text") { + builder.SetFullTextSearch(std::make_shared( + "f0", R"({"match":{"query":"all"}})", /*limit=*/1)); + } else if (search_type == "vector") { + builder.SetVectorSearch(std::make_shared( + "f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, + VectorSearch::DistanceType::EUCLIDEAN, std::map{})); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + ASSERT_NOK_WITH_MSG( + ReadWithFileIndex(context, /*index=*/nullptr, RoaringBitmap32(), nullptr, + /*row_ranges=*/std::nullopt, + /*deletion_vector=*/nullptr, int64_t{RoaringBitmap32::MAX_VALUE} + 1), + "Precise file-index evaluation requires bitmap32 physical row positions"); + } +} + +TEST_F(FileIndexReadExecutorTest, TestPreciseEvaluationAtBitmap32Limit) { + for (bool has_predicate : {false, true}) { + ReadContextBuilder builder(SearchTablePath()); + builder.SetReadFieldNames({"f0", "f3"}).EnablePreciseFileIndexEvaluate(true); + if (has_predicate) { + builder.SetPredicate( + PredicateBuilder::GreaterThan(1, "f3", FieldType::DOUBLE, Literal(13.0))); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr result, + ReadWithFileIndex(context, /*index=*/nullptr, RoaringBitmap32(), nullptr, + std::vector{Range(0, 2)}, + /*deletion_vector=*/nullptr, RoaringBitmap32::MAX_VALUE)); + CheckSearchResult(result, + {arrow::field("f0", arrow::utf8()), arrow::field("f3", arrow::float64())}, + has_predicate ? R"([["Emily",13.1],["Tony",14.1]])" + : R"([["Bob",12.1],["Emily",13.1],["Tony",14.1]])"); + } +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/file_store_scan.cpp b/src/paimon/core/operation/file_store_scan.cpp index d0bb04e47..ceba23f0a 100644 --- a/src/paimon/core/operation/file_store_scan.cpp +++ b/src/paimon/core/operation/file_store_scan.cpp @@ -123,9 +123,13 @@ Result FileStoreScan::TestFileIndex(const std::shared_ptr& pred std::map file_index_options; PAIMON_ASSIGN_OR_RAISE(file_index_options, FileIndexOptions::RewriteFieldNames( core_options_.ToMap(), field_name_mapping)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr evaluator, + FileIndexEvaluator::Create(data_arrow_schema, file_index_options, /*path_factory=*/nullptr, + meta, /*file_system=*/nullptr, /*only_use_embedding_index=*/true, + pool_)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index_result, - FileIndexEvaluator::Evaluate(data_arrow_schema, file_index_options, - data_predicate, meta, pool_)); + evaluator->Evaluate(data_predicate)); return index_result->IsRemain(); } diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index f36a15d71..3f6fc83c5 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -47,7 +47,6 @@ #include "paimon/core/utils/nested_projection_utils.h" #include "paimon/predicate/compound_predicate.h" #include "paimon/predicate/full_text_search.h" -#include "paimon/predicate/predicate_utils.h" #include "paimon/predicate/vector_search.h" #include "paimon/rest/rest_api.h" #include "paimon/status.h" @@ -217,7 +216,7 @@ Result> InternalReadContext::AlignReadFieldWithTab } std::optional InternalReadContext::TryResolveSpecialFieldById( - int32_t field_id, const CoreOptions& core_options, bool has_vector_or_full_text_search) { + int32_t field_id, const CoreOptions& core_options) { if (field_id == SpecialFields::ValueKind().Id()) { return SpecialFields::ValueKind(); } @@ -234,7 +233,7 @@ std::optional InternalReadContext::TryResolveSpecialFieldById( return std::nullopt; } if (field_id == SpecialFields::IndexScore().Id()) { - if (core_options.DataEvolutionEnabled() || has_vector_or_full_text_search) { + if (core_options.DataEvolutionEnabled()) { return SpecialFields::IndexScore(); } return std::nullopt; @@ -243,8 +242,7 @@ std::optional InternalReadContext::TryResolveSpecialFieldById( } std::optional InternalReadContext::TryResolveSpecialFieldByName( - const std::string& name, const CoreOptions& core_options, bool has_vector_or_full_text_search, - int32_t output_field_idx) { + const std::string& name, const CoreOptions& core_options, int32_t output_field_idx) { if (SpecialFields::ParseIndexScoreFieldName(name)) { // These virtual columns have read-local negative IDs, not persistent table IDs. return DataField(-1 - output_field_idx, arrow::field(name, arrow::float32())); @@ -265,7 +263,7 @@ std::optional InternalReadContext::TryResolveSpecialFieldByName( return std::nullopt; } if (name == SpecialFields::IndexScore().Name()) { - if (core_options.DataEvolutionEnabled() || has_vector_or_full_text_search) { + if (core_options.DataEvolutionEnabled()) { return SpecialFields::IndexScore(); } return std::nullopt; @@ -280,23 +278,21 @@ Result> InternalReadContext::Create( CoreOptions::FromMap(options, context->GetSpecificFileSystem(), context->GetFileSystemSchemeToIdentifierMap())); core_options.WithCache(context->GetCache()); - if (context->HasVectorOrFullTextSearch()) { + if (context->EnablePreciseFileIndexEvaluate()) { if (!core_options.FileIndexReadEnabled()) { return Status::Invalid( - "File Index read must be enabled for VectorSearch or FullTextSearch"); + "Precise file-index evaluation requires file-index.read.enabled=true"); } if (!table_schema->PrimaryKeys().empty() && !core_options.DeletionVectorsEnabled()) { return Status::NotImplemented( - "VectorSearch or FullTextSearch on primary-key tables requires deletion vectors"); + "Precise file-index evaluation on primary-key tables requires deletion vectors"); } - if (core_options.DataEvolutionEnabled()) { + if (core_options.DataEvolutionEnabled() || context->GetRealtimeContext()) { return Status::NotImplemented( - "VectorSearch or FullTextSearch does not support data-evolution reads yet"); - } - if (context->GetRealtimeContext()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch does not support real-time reads yet"); + "Precise file-index evaluation does not support data-evolution or real-time reads"); } + } + if (context->HasVectorOrFullTextSearch()) { const std::string& search_field = context->GetVectorSearch() ? context->GetVectorSearch()->field_name : context->GetFullTextSearch()->field_name; @@ -306,7 +302,6 @@ Result> InternalReadContext::Create( // prepare read schema // Priority: projected_arrow_schema > read_field_ids > read_field_names const bool has_projected_read_schema = context->HasReadSchema(); - const bool has_vector_or_full_text_search = context->HasVectorOrFullTextSearch(); std::vector read_data_fields; if (has_projected_read_schema) { // Nested column pruning path: user provided a read C ArrowSchema @@ -317,9 +312,9 @@ Result> InternalReadContext::Create( read_data_fields.reserve(read_schema->num_fields()); // Align special-field validation with read_field_ids/read_field_names branches. for (const auto& read_field : read_schema->fields()) { - if (auto resolved_special_field = TryResolveSpecialFieldByName( - read_field->name(), core_options, has_vector_or_full_text_search, - static_cast(read_data_fields.size()))) { + if (auto resolved_special_field = + TryResolveSpecialFieldByName(read_field->name(), core_options, + static_cast(read_data_fields.size()))) { read_data_fields.push_back(*resolved_special_field); continue; } @@ -344,8 +339,7 @@ Result> InternalReadContext::Create( } else if (!context->GetReadFieldIds().empty()) { read_data_fields.reserve(context->GetReadFieldIds().size()); for (const auto& field_id : context->GetReadFieldIds()) { - if (auto resolved_special_field = TryResolveSpecialFieldById( - field_id, core_options, has_vector_or_full_text_search)) { + if (auto resolved_special_field = TryResolveSpecialFieldById(field_id, core_options)) { read_data_fields.push_back(*resolved_special_field); continue; } @@ -355,9 +349,8 @@ Result> InternalReadContext::Create( } else if (!context->GetReadFieldNames().empty()) { read_data_fields.reserve(context->GetReadFieldNames().size()); for (const auto& name : context->GetReadFieldNames()) { - if (auto resolved_special_field = - TryResolveSpecialFieldByName(name, core_options, has_vector_or_full_text_search, - static_cast(read_data_fields.size()))) { + if (auto resolved_special_field = TryResolveSpecialFieldByName( + name, core_options, static_cast(read_data_fields.size()))) { read_data_fields.push_back(*resolved_special_field); continue; } @@ -377,9 +370,11 @@ Result> InternalReadContext::Create( } else { PAIMON_RETURN_NOT_OK(ArrowSchemaValidator::ValidateSchemaWithFieldId(*read_schema)); } - if (context->GetFullTextSearch() && + if (context->EnablePreciseFileIndexEvaluate() && read_schema->GetFieldIndex(SpecialFields::IndexScore().Name()) >= 0) { - return Status::NotImplemented("File full-text search does not support score output yet"); + return Status::Invalid( + "Precise file-index evaluation does not support _INDEX_SCORE; " + "project __INDEX_SCORE instead"); } // validate predicate if (context->GetPredicate()) { @@ -481,10 +476,9 @@ InternalReadContext::InternalReadContext(const std::shared_ptr& rea Result> InternalReadContext::CreateWithSchema( const std::shared_ptr& original, const std::shared_ptr& new_read_schema) { - if (original->HasVectorOrFullTextSearch() || - PredicateUtils::ContainsFullTextMatch(original->GetPredicate())) { + if (original->EnablePreciseFileIndexEvaluate()) { return Status::NotImplemented( - "CreateWithSchema does not support search requests or index score projections"); + "CreateWithSchema does not support precise file-index evaluation"); } // Create a new InternalReadContext sharing all properties except read_schema. // The new read_schema is the minimal column set for COUNT(*). diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index 9de415d5f..0f1c0f12b 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -102,6 +102,9 @@ class InternalReadContext { bool HasVectorOrFullTextSearch() const { return read_context_->HasVectorOrFullTextSearch(); } + bool EnablePreciseFileIndexEvaluate() const { + return read_context_->EnablePreciseFileIndexEvaluate(); + } bool EnablePredicateFilter() const { return read_context_->EnablePredicateFilter(); } @@ -153,8 +156,8 @@ class InternalReadContext { return read_context_->GetWarmupLevel(); } - /// Create the minimal read schema for PK COUNT(*). Search requests and score - /// projections are unsupported; their bindings cannot be inherited across schemas. + /// Create the minimal read schema for PK COUNT(*). Precise file-index evaluation is + /// unsupported because count readers do not use the file predicate executor. static Result> CreateWithSchema( const std::shared_ptr& original, const std::shared_ptr& new_read_schema); @@ -167,11 +170,10 @@ class InternalReadContext { const std::shared_ptr& arrow_pool); static std::optional TryResolveSpecialFieldById(int32_t field_id, - const CoreOptions& core_options, - bool has_vector_or_full_text_search); - static std::optional TryResolveSpecialFieldByName( - const std::string& name, const CoreOptions& core_options, - bool has_vector_or_full_text_search, int32_t output_field_idx); + const CoreOptions& core_options); + static std::optional TryResolveSpecialFieldByName(const std::string& name, + const CoreOptions& core_options, + int32_t output_field_idx); static Result> AlignReadFieldWithTableFieldIds( const std::shared_ptr& read_field, const std::shared_ptr& table_field); diff --git a/src/paimon/core/operation/internal_read_context_test.cpp b/src/paimon/core/operation/internal_read_context_test.cpp index 6e4957f4e..ab8fab938 100644 --- a/src/paimon/core/operation/internal_read_context_test.cpp +++ b/src/paimon/core/operation/internal_read_context_test.cpp @@ -66,6 +66,52 @@ TEST(InternalReadContext, TestReadWithUnspecifiedSchema) { ASSERT_TRUE(internal_context->GetReadSchema()->Equals(expected_schema)); } +TEST(InternalReadContext, TestPreciseScalarEvaluation) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(0, + arrow::schema({arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8())}), + {}, {}, {})); + for (bool enabled : {false, true}) { + ReadContextBuilder builder("unused-table-path"); + builder.EnablePreciseFileIndexEvaluate(enabled).SetPredicate( + PredicateBuilder::IsNotNull(0, "f0", FieldType::INT)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr internal_context, + InternalReadContext::Create(context, table_schema, table_schema->Options())); + ASSERT_EQ(internal_context->EnablePreciseFileIndexEvaluate(), enabled); + ASSERT_FALSE(internal_context->EnablePredicateFilter()); + ASSERT_TRUE(internal_context->GetCoreOptions().FileIndexReadEnabled()); + if (enabled) { + ASSERT_NOK_WITH_MSG(InternalReadContext::CreateWithSchema( + internal_context, internal_context->GetReadSchema()), + "CreateWithSchema does not support precise file-index evaluation"); + } + } + auto match = std::make_shared("f1", R"({"match":{"query":"apple"}})"); + ASSERT_OK_AND_ASSIGN(auto full_text, PredicateBuilder::FullTextMatch(1, match)); + ReadContextBuilder builder("unused-table-path"); + builder.SetPredicate(full_text); + ASSERT_NOK_WITH_MSG(builder.Finish(), "EnablePreciseFileIndexEvaluate(true)"); + builder.EnablePreciseFileIndexEvaluate(true); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + ASSERT_TRUE(context->EnablePreciseFileIndexEvaluate()); + ASSERT_OK(InternalReadContext::Create(context, table_schema, table_schema->Options())); +} + +TEST(InternalReadContext, TestPreciseEvaluationRequiresFileIndexRead) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(0, arrow::schema({arrow::field("f0", arrow::int32())}), {}, {}, {})); + ReadContextBuilder builder("unused-table-path"); + builder.EnablePreciseFileIndexEvaluate(true); + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + ASSERT_NOK_WITH_MSG(InternalReadContext::Create(context, table_schema, + {{Options::FILE_INDEX_READ_ENABLED, "false"}}), + "requires file-index.read.enabled=true"); +} + TEST(InternalReadContext, TestReadWithSpecifiedSchema) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; ReadContextBuilder context_builder(path); @@ -369,47 +415,50 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { } } -TEST(InternalReadContext, TestVectorSearchIndexScoreProjection) { +TEST(InternalReadContext, TestPreciseEvaluationRejectsUnnamedScoreProjection) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; SchemaManager schema_manager(std::make_shared(), path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); - std::shared_ptr vector_search = std::make_shared( - "f3", /*limit=*/2, std::vector{1.0f}, nullptr, nullptr, - VectorSearch::DistanceType::EUCLIDEAN, std::map{}); - - { - ReadContextBuilder builder(path); - builder.SetReadFieldNames({"f0", "_INDEX_SCORE"}).SetVectorSearch(vector_search); - ASSERT_OK_AND_ASSIGN(auto read_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); - std::shared_ptr expected = DataField::ConvertDataFieldsToArrowSchema( - {DataField(0, arrow::field("f0", arrow::utf8())), SpecialFields::IndexScore()}); - EXPECT_TRUE(internal_context->GetReadSchema()->Equals(expected)); - } - { - ReadContextBuilder builder(path); - builder.SetReadFieldIds({0, SpecialFieldIds::INDEX_SCORE}).SetVectorSearch(vector_search); - ASSERT_OK_AND_ASSIGN(auto read_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); - EXPECT_EQ(arrow::Type::FLOAT, internal_context->GetReadSchema()->field(1)->type()->id()); - } - { - std::shared_ptr projected = arrow::schema( - {arrow::field("_INDEX_SCORE", arrow::float32()), arrow::field("f0", arrow::utf8())}); - auto c_schema = std::make_unique(); - ASSERT_TRUE(arrow::ExportSchema(*projected, c_schema.get()).ok()); - ReadContextBuilder builder(path); - builder.SetReadSchema(std::move(c_schema)).SetVectorSearch(vector_search); - ASSERT_OK_AND_ASSIGN(auto read_context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto internal_context, - InternalReadContext::Create(std::move(read_context), table_schema, - table_schema->Options())); - EXPECT_EQ(std::vector({"_INDEX_SCORE", "f0"}), - internal_context->GetReadSchema()->field_names()); + for (const auto& search_type : + std::vector{"none", "scalar", "vector", "full-text", "full-text-match"}) { + for (const auto& projection : std::vector{"names", "ids", "schema"}) { + ReadContextBuilder builder(path); + builder.EnablePreciseFileIndexEvaluate(true).SetReadFieldNames({"f0"}); + if (projection == "names") { + builder.SetReadFieldNames({"f0", "_INDEX_SCORE"}); + } else if (projection == "ids") { + builder.SetReadFieldIds({0, SpecialFieldIds::INDEX_SCORE}); + } else if (projection == "schema") { + builder.SetReadFieldIds({0}); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema( + *arrow::schema({arrow::field("f0", arrow::utf8()), + arrow::field("_INDEX_SCORE", arrow::float32())}), + c_schema.get()) + .ok()); + builder.SetReadSchema(std::move(c_schema)); + } + if (search_type == "scalar") { + builder.SetPredicate(PredicateBuilder::IsNotNull(0, "f0", FieldType::STRING)); + } else if (search_type == "vector") { + builder.SetVectorSearch(std::make_shared( + "f3", 1, std::vector{1.0f}, nullptr, nullptr, std::nullopt, + std::map{})); + } else if (search_type == "full-text") { + builder.SetFullTextSearch( + std::make_shared("f0", R"({"match":{"query":"apple"}})", 1)); + } else if (search_type == "full-text-match") { + ASSERT_OK_AND_ASSIGN(auto predicate, + PredicateBuilder::FullTextMatch( + 0, std::make_shared( + "f0", R"({"match":{"query":"apple"}})"))); + builder.SetPredicate(predicate); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); + ASSERT_NOK_WITH_MSG( + InternalReadContext::Create(context, table_schema, table_schema->Options()), + "not exist in table schema"); + } } } @@ -422,12 +471,15 @@ TEST(InternalReadContext, TestPrimaryKeyVectorSearchRequiresDeletionVectors) { std::make_shared("f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, std::nullopt, std::map{}); ReadContextBuilder builder(path); - builder.SetReadFieldNames({"f0", "_INDEX_SCORE"}).SetVectorSearch(search); + builder.SetReadFieldNames({"f0", "_f3_INDEX_SCORE"}) + .EnablePreciseFileIndexEvaluate(true) + .SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::unique_ptr internal, InternalReadContext::Create(context, table_schema, table_schema->Options())); EXPECT_TRUE(internal->HasVectorOrFullTextSearch()); + EXPECT_TRUE(internal->EnablePreciseFileIndexEvaluate()); std::map options = table_schema->Options(); options[Options::DELETION_VECTORS_ENABLED] = "false"; @@ -435,19 +487,6 @@ TEST(InternalReadContext, TestPrimaryKeyVectorSearchRequiresDeletionVectors) { "primary-key tables requires deletion vectors"); } -TEST(InternalReadContext, TestFileFullTextSearchScoreIsNotSupported) { - std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - SchemaManager schema_manager(std::make_shared(), path); - ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, schema_manager.ReadSchema(0)); - auto search = - std::make_shared("f0", R"({"match":{"query":"query"}})", /*limit=*/1); - ReadContextBuilder builder(path); - builder.SetReadFieldNames({"f0", "_INDEX_SCORE"}).SetFullTextSearch(search); - ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); - ASSERT_NOK_WITH_MSG(InternalReadContext::Create(context, table_schema, table_schema->Options()), - "does not support score output"); -} - TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; SchemaManager schema_manager(std::make_shared(), path); @@ -472,7 +511,9 @@ TEST(InternalReadContext, TestFullTextMatchValidatesProjection) { ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::FullTextMatch(test_case.field_index, match)); ReadContextBuilder builder(path); - builder.SetReadFieldNames(test_case.projection).SetPredicate(predicate); + builder.EnablePreciseFileIndexEvaluate(true) + .SetReadFieldNames(test_case.projection) + .SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); if (test_case.expected_error.empty()) { ASSERT_OK_AND_ASSIGN( @@ -498,13 +539,23 @@ TEST(InternalReadContext, TestPerFieldTopKScoreBindings) { SchemaManager schema_manager(std::make_shared(), path); ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, schema_manager.ReadSchema(0)); for (bool vector_search : {false, true}) { - for (bool project_score : {false, true}) { + for (const auto& projection_type : std::vector{"none", "names", "schema"}) { + const bool project_score = projection_type != "none"; ReadContextBuilder builder(path); std::vector projection = {"f1"}; if (project_score) { projection.push_back(vector_search ? "_f3_INDEX_SCORE" : "_f0_INDEX_SCORE"); } builder.SetReadFieldNames(projection); + if (projection_type == "schema") { + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema( + *arrow::schema({arrow::field("f1", arrow::int32()), + arrow::field(projection[1], arrow::float32())}), + c_schema.get()) + .ok()); + builder.SetReadSchema(std::move(c_schema)); + } const InternalReadContext::IndexScoreSource source = { vector_search ? SourceType::VECTOR_SEARCH : SourceType::FULL_TEXT_SEARCH, vector_search ? "f3" : "f0"}; @@ -512,11 +563,11 @@ TEST(InternalReadContext, TestPerFieldTopKScoreBindings) { auto search = std::make_shared( "f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, std::nullopt, std::map{}); - builder.SetVectorSearch(search); + builder.EnablePreciseFileIndexEvaluate(true).SetVectorSearch(search); } else { auto search = std::make_shared( "f0", R"({"match":{"query":"apple"}})", /*limit=*/1); - builder.SetFullTextSearch(search); + builder.EnablePreciseFileIndexEvaluate(true).SetFullTextSearch(search); } ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN( @@ -590,7 +641,7 @@ TEST_F(InternalReadContextScoreTest, BindsFullTextScoresByProjection) { if (score_count == 2) { projection.push_back(test_case.reverse_scores ? "_F2_INDEX_SCORE" : "_F1_INDEX_SCORE"); } - builder.SetPredicate(predicate); + builder.EnablePreciseFileIndexEvaluate(true).SetPredicate(predicate); if (test_case.explicit_schema) { arrow::FieldVector fields = {arrow::field("F1", arrow::utf8()), arrow::field("F2", arrow::utf8())}; @@ -645,8 +696,10 @@ TEST_F(InternalReadContextScoreTest, BindsRootAndPredicateScores) { PredicateBuilder::FullTextMatch(0, match)); auto search = std::make_shared("F2", match->query, /*limit=*/10); ReadContextBuilder builder("/tmp/unused-table-path"); - builder.SetPredicate(predicate).SetFullTextSearch(search).SetReadFieldNames( - {"F1", "_F2_INDEX_SCORE", "_F1_INDEX_SCORE"}); + builder.SetPredicate(predicate) + .EnablePreciseFileIndexEvaluate(true) + .SetFullTextSearch(search) + .SetReadFieldNames({"F1", "_F2_INDEX_SCORE", "_F1_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto internal, Create(&builder)); ASSERT_EQ(internal->GetReadSchema()->GetFieldIndex("F2"), -1); const auto& bindings = internal->GetIndexScoreBindings(); @@ -656,8 +709,10 @@ TEST_F(InternalReadContextScoreTest, BindsRootAndPredicateScores) { ASSERT_TRUE(internal->NeedsScores(search_source)); ASSERT_TRUE(internal->NeedsScores(match_source)); - builder.SetPredicate(predicate).SetFullTextSearch(search).SetReadFieldNames( - {"F1", "_F2_INDEX_SCORE", "F2", "_F1_INDEX_SCORE"}); + builder.SetPredicate(predicate) + .EnablePreciseFileIndexEvaluate(true) + .SetFullTextSearch(search) + .SetReadFieldNames({"F1", "_F2_INDEX_SCORE", "F2", "_F1_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(internal, Create(&builder)); const std::map non_contiguous = { {1, search_source}, {3, match_source}}; @@ -665,8 +720,10 @@ TEST_F(InternalReadContextScoreTest, BindsRootAndPredicateScores) { auto scalar = PredicateBuilder::IsNotNull(0, "F1", FieldType::STRING); ASSERT_OK_AND_ASSIGN(predicate, PredicateBuilder::Or({predicate, scalar})); - builder.SetPredicate(predicate).SetFullTextSearch(search).SetReadFieldNames( - {"F1", "_F2_INDEX_SCORE"}); + builder.SetPredicate(predicate) + .EnablePreciseFileIndexEvaluate(true) + .SetFullTextSearch(search) + .SetReadFieldNames({"F1", "_F2_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(internal, Create(&builder)); ASSERT_TRUE(internal->NeedsScores(search_source)); ASSERT_FALSE(internal->NeedsScores(match_source)); @@ -699,7 +756,8 @@ TEST_F(InternalReadContextScoreTest, RejectsAmbiguousOrConditionalScoreProjectio for (const auto& test_case : cases) { for (bool project_score : {false, true}) { ReadContextBuilder builder("/tmp/unused-table-path"); - builder.SetPredicate(test_case.predicate) + builder.EnablePreciseFileIndexEvaluate(true) + .SetPredicate(test_case.predicate) .SetReadFieldNames(project_score ? std::vector{"F1", "_F1_INDEX_SCORE"} : std::vector{"F1"}); if (test_case.root_search) { @@ -735,10 +793,12 @@ TEST_F(InternalReadContextScoreTest, ResolvesScoresOnlyFromEffectiveProjection) auto search = std::make_shared("F2", R"({"match":{"query":"apple"}})", 1); for (bool explicit_schema : {false, true}) { ReadContextBuilder builder("/tmp/unused-table-path"); - builder.SetFullTextSearch(search) - .SetReadFieldNames({"_missing_INDEX_SCORE"}) + builder.EnablePreciseFileIndexEvaluate(true) + .SetFullTextSearch(search) + .SetReadFieldNames({"_INDEX_SCORE", "_missing_INDEX_SCORE"}) .SetReadFieldIds({0}); if (explicit_schema) { + builder.SetReadFieldIds({SpecialFieldIds::INDEX_SCORE}); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema( *arrow::schema({arrow::field("_F2_INDEX_SCORE", arrow::float32())}), diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp index 8a256bc0b..312760db2 100644 --- a/src/paimon/core/operation/merge_file_split_read.cpp +++ b/src/paimon/core/operation/merge_file_split_read.cpp @@ -405,9 +405,9 @@ Result> MergeFileSplitRead::Create( Result> MergeFileSplitRead::CreateReader( const std::shared_ptr& split) { - if (context_->HasVectorOrFullTextSearch()) { + if (context_->EnablePreciseFileIndexEvaluate()) { return Status::NotImplemented( - "VectorSearch or FullTextSearch does not support merge file reads yet"); + "Precise file-index evaluation does not support merge file reads"); } auto data_split = std::dynamic_pointer_cast(split); if (!data_split) { diff --git a/src/paimon/core/operation/raw_file_split_read.cpp b/src/paimon/core/operation/raw_file_split_read.cpp index d81bc4850..823c57fc9 100644 --- a/src/paimon/core/operation/raw_file_split_read.cpp +++ b/src/paimon/core/operation/raw_file_split_read.cpp @@ -26,31 +26,22 @@ #include "fmt/format.h" #include "paimon/common/reader/complete_row_kind_batch_reader.h" #include "paimon/common/reader/concat_batch_reader.h" -#include "paimon/common/types/data_field.h" #include "paimon/common/utils/object_utils.h" #include "paimon/core/core_options.h" -#include "paimon/core/deletionvectors/bitmap_deletion_vector.h" #include "paimon/core/deletionvectors/deletion_vector.h" #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/io/file_index_evaluator.h" +#include "paimon/core/operation/file_index_read_executor.h" #include "paimon/core/operation/internal_read_context.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/data_split_impl.h" -#include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/file_store_path_factory.h" -#include "paimon/file_index/bitmap_index_result.h" -#include "paimon/file_index/file_index_result.h" -#include "paimon/file_index/scored_file_index_result.h" #include "paimon/memory/memory_pool.h" -#include "paimon/predicate/full_text_search.h" -#include "paimon/predicate/vector_search.h" #include "paimon/reader/file_batch_reader.h" #include "paimon/status.h" #include "paimon/table/source/data_split.h" -#include "paimon/utils/roaring_bitmap32.h" -#include "paimon/utils/roaring_bitmap64.h" namespace paimon { class DataFilePathFactory; @@ -59,18 +50,6 @@ class Predicate; namespace { -Result ResolveSearchFieldName( - const std::string& field_name, const std::map& field_name_mapping, - const std::string& file_name) { - auto iter = field_name_mapping.find(field_name); - if (iter == field_name_mapping.end()) { - return Status::Invalid( - fmt::format("Search field '{}' has no matching field ID in data schema for file '{}'", - field_name, file_name)); - } - return iter->second; -} - Status ValidateFileLocalRowRanges(const std::vector>& data_files, const std::optional>& local_row_ranges) { if (local_row_ranges == std::nullopt) { @@ -182,7 +161,9 @@ Result> RawFileSplitRead::CreateReaderWithPredicate std::make_unique(std::move(raw_readers), arrow_pool_); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr batch_reader, - ApplyPredicateFilterIfNeeded(std::move(concat_batch_reader), residual_predicate)); + ApplyPredicateFilterIfNeeded(std::move(concat_batch_reader), + EnablePreciseFileIndexEvaluate() ? nullptr + : residual_predicate)); return std::make_unique(std::move(batch_reader), arrow_pool_); } @@ -231,140 +212,21 @@ Result RawFileSplitRead::Match(const std::shared_ptr& split, return matched; } -Result> RawFileSplitRead::ApplyIndexAndDvReaderIfNeeded( - std::unique_ptr&& file_reader, const std::shared_ptr& file, - const std::shared_ptr& data_schema, - const std::shared_ptr& read_schema, const std::shared_ptr& predicate, - DeletionVector::Factory dv_factory, const std::optional>& ranges, - const std::shared_ptr& data_file_path_factory, - const std::map& file_index_options, - std::vector* index_scores) const { - if (!context_->HasVectorOrFullTextSearch()) { - return AbstractSplitRead::ApplyIndexAndDvReaderIfNeeded( - std::move(file_reader), file, data_schema, read_schema, predicate, dv_factory, ranges, - data_file_path_factory, file_index_options, index_scores); - } - if (predicate) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch does not support combining with read predicates yet"); - } - if (ranges) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch does not support indexed split row ranges yet"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr deletion, - LoadSearchDeletionBitmap(file, dv_factory)); - if (deletion && deletion->Cardinality() == file->row_count) { - return std::unique_ptr(); - } - - PAIMON_ASSIGN_OR_RAISE(std::vector data_fields, - DataField::ConvertArrowSchemaToDataFields(data_schema)); - auto field_name_mapping = FieldMappingBuilder::CreateFieldNameMapping( - context_->GetTableSchema()->Fields(), data_fields); - std::optional search_selection; - if (context_->GetVectorSearch()) { - auto vector_search = std::make_shared(*context_->GetVectorSearch()); - PAIMON_ASSIGN_OR_RAISE( - vector_search->field_name, - ResolveSearchFieldName(vector_search->field_name, field_name_mapping, file->file_name)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr search_result, - EvaluateVectorSearchWithDv(file, data_schema, data_file_path_factory, deletion, - vector_search, file_index_options)); - if (search_result->IsEmpty()) { - return std::unique_ptr(); - } - search_selection = search_result->GetRowPositions(); - *index_scores = search_result->GetScores(); - } else { - auto full_text_search = std::make_shared(*context_->GetFullTextSearch()); - PAIMON_ASSIGN_OR_RAISE(full_text_search->field_name, - ResolveSearchFieldName(full_text_search->field_name, - field_name_mapping, file->file_name)); - PAIMON_ASSIGN_OR_RAISE( - search_selection, - EvaluateFullTextSearchWithDv(file, data_schema, data_file_path_factory, deletion, - full_text_search, file_index_options)); - if (!search_selection) { - return std::unique_ptr(); - } - } - - return ApplyBitmapSelection(std::move(file_reader), read_schema, /*predicate=*/nullptr, - std::move(search_selection)); +bool RawFileSplitRead::EnablePreciseFileIndexEvaluate() const { + return context_->EnablePreciseFileIndexEvaluate(); } -Result> RawFileSplitRead::LoadSearchDeletionBitmap( - const std::shared_ptr& file, const DeletionVector::Factory& dv_factory) { - std::shared_ptr deletion_vector; - if (dv_factory) { - PAIMON_ASSIGN_OR_RAISE(deletion_vector, dv_factory(file->file_name)); - } - if (auto* bitmap_dv = dynamic_cast(deletion_vector.get())) { - // Keep the deletion vector alive for the search callback without copying the bitmap. - return std::shared_ptr(deletion_vector, bitmap_dv->GetBitmap()); - } - if (deletion_vector && !deletion_vector->IsEmpty()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch only supports bitmap32 deletion vectors"); - } - return std::shared_ptr(); -} - -Result> RawFileSplitRead::EvaluateVectorSearchWithDv( - const std::shared_ptr& file, const std::shared_ptr& data_schema, - const std::shared_ptr& data_file_path_factory, - const std::shared_ptr& deletion, - const std::shared_ptr& vector_search, - const std::map& file_index_options) const { - std::shared_ptr search_with_dv = vector_search; - if (deletion && !deletion->IsEmpty()) { - auto user_filter = vector_search->pre_filter; - search_with_dv = vector_search->ReplacePreFilter( - [deletion, user_filter, row_count = file->row_count](int64_t row_id) { - return row_id >= 0 && row_id < row_count && - !deletion->Contains(static_cast(row_id)) && - (!user_filter || user_filter(row_id)); - }); - } - return FileIndexEvaluator::EvaluateVectorSearch(data_schema, file_index_options, search_with_dv, - data_file_path_factory, file, - options_.GetFileSystem(), pool_); -} - -Result> RawFileSplitRead::EvaluateFullTextSearchWithDv( - const std::shared_ptr& file, const std::shared_ptr& data_schema, - const std::shared_ptr& data_file_path_factory, - const std::shared_ptr& deletion, - const std::shared_ptr& full_text_search, - const std::map& file_index_options) const { - std::shared_ptr search_with_dv = full_text_search; - if (deletion && !deletion->IsEmpty()) { - RoaringBitmap64 eligible; - eligible.AddRange(0, file->row_count); - eligible -= RoaringBitmap64(*deletion); - if (full_text_search->include_row_ids) { - eligible &= full_text_search->include_row_ids.value(); - } - if (eligible.IsEmpty()) { - return std::optional(); - } - search_with_dv = - std::make_shared(full_text_search->field_name, full_text_search->query, - full_text_search->limit, std::move(eligible)); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr search_result, - FileIndexEvaluator::EvaluateFullTextSearch( - data_schema, file_index_options, search_with_dv, - data_file_path_factory, file, options_.GetFileSystem(), pool_)); - std::shared_ptr bitmap_result = - std::dynamic_pointer_cast(search_result); - if (!bitmap_result) { - return Status::Invalid("FullTextSearch must return a BitmapIndexResult"); - } - PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap32* bitmap, bitmap_result->GetBitmap()); - return std::optional(*bitmap); +Result> RawFileSplitRead::CreatePreciseFileIndexReader( + const BinaryRow& partition, const std::shared_ptr& file, + const FieldMappingBuilder* field_mapping_builder, DeletionVector::Factory dv_factory, + const std::optional>& row_ranges, + const std::shared_ptr& path_factory, + const std::map& extra_format_options) const { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr executor, + FileIndexReadExecutor::Create(*this, partition, file, field_mapping_builder, path_factory, + extra_format_options)); + return executor->CreateReader(dv_factory, row_ranges); } } // namespace paimon diff --git a/src/paimon/core/operation/raw_file_split_read.h b/src/paimon/core/operation/raw_file_split_read.h index 14df8cd66..262d13896 100644 --- a/src/paimon/core/operation/raw_file_split_read.h +++ b/src/paimon/core/operation/raw_file_split_read.h @@ -46,19 +46,17 @@ class FileStorePathFactory; class InternalReadContext; class MemoryPool; class Predicate; -class ScoredFileIndexResult; struct DataFileMeta; struct DeletionFile; -struct FullTextSearch; -struct VectorSearch; /// If the class name below is enclosed in parentheses, it might be present in the read path; /// otherwise, it must be present in the read path. /// /// Readers Overview: (ConcatBatchReader across splits) -> CompleteRowKindBatchReader -> /// (PredicateBatchReader) -> ConcatBatchReader across files -> (CompleteIndexScoreFileBatchReader) -/// -> FieldMappingReader -> (ApplyBitmapIndexBatchReader) -> (CompleteRowTrackingFieldsBatchReader) -/// -> (ShreddingFileReader) -> (VectorFileBatchReader) -> (DelegatingPrefetchReader) -> +/// -> FieldMappingReader -> (ApplyDeletionVectorBatchReader) -> (ApplyBitmapIndexBatchReader) +/// -> (CompleteRowTrackingFieldsBatchReader) -> (ShreddingFileReader) -> (VectorFileBatchReader) -> +/// (DelegatingPrefetchReader) -> /// (PrefetchFileBatchReader) -> (LateMaterializingFileBatchReader) -> FormatReader class RawFileSplitRead : public AbstractSplitRead { @@ -103,36 +101,16 @@ class RawFileSplitRead : public AbstractSplitRead { const std::shared_ptr& residual_predicate, PredicateProvider predicate_provider); Result Match(const std::shared_ptr& split, bool force_keep_delete) const override; - Result> ApplyIndexAndDvReaderIfNeeded( - std::unique_ptr&& file_reader, const std::shared_ptr& file, - const std::shared_ptr& data_schema, - const std::shared_ptr& read_schema, - const std::shared_ptr& predicate, DeletionVector::Factory dv_factory, - const std::optional>& ranges, - const std::shared_ptr& data_file_path_factory, - const std::map& file_index_options, - std::vector* index_scores) const override; - - private: - static Result> LoadSearchDeletionBitmap( - const std::shared_ptr& file, const DeletionVector::Factory& dv_factory); - - Result> EvaluateVectorSearchWithDv( - const std::shared_ptr& file, - const std::shared_ptr& data_schema, - const std::shared_ptr& data_file_path_factory, - const std::shared_ptr& deletion, - const std::shared_ptr& vector_search, - const std::map& file_index_options) const; - - // nullopt means the file can be skipped, not an unrestricted selection. - Result> EvaluateFullTextSearchWithDv( - const std::shared_ptr& file, - const std::shared_ptr& data_schema, - const std::shared_ptr& data_file_path_factory, - const std::shared_ptr& deletion, - const std::shared_ptr& full_text_search, - const std::map& file_index_options) const; + + protected: + bool EnablePreciseFileIndexEvaluate() const override; + + Result> CreatePreciseFileIndexReader( + const BinaryRow& partition, const std::shared_ptr& file, + const FieldMappingBuilder* field_mapping_builder, DeletionVector::Factory dv_factory, + const std::optional>& row_ranges, + const std::shared_ptr& path_factory, + const std::map& extra_format_options) const override; }; } // namespace paimon diff --git a/src/paimon/core/operation/raw_file_split_read_test.cpp b/src/paimon/core/operation/raw_file_split_read_test.cpp index 04b5462b8..f536696f4 100644 --- a/src/paimon/core/operation/raw_file_split_read_test.cpp +++ b/src/paimon/core/operation/raw_file_split_read_test.cpp @@ -545,7 +545,8 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { "f3", /*limit=*/2, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); ReadContextBuilder context_builder(path); - context_builder.SetReadFieldNames({"_INDEX_SCORE", "f0"}).SetVectorSearch(vector_search); + context_builder.EnablePreciseFileIndexEvaluate(true); + context_builder.SetReadFieldNames({"_f3_INDEX_SCORE", "f0"}).SetVectorSearch(vector_search); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), path); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); @@ -587,13 +588,14 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchScoreProjection) { std::shared_ptr expected; ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), - arrow::field("_INDEX_SCORE", arrow::float32()), + arrow::field("_f3_INDEX_SCORE", arrow::float32()), arrow::field("f0", arrow::utf8())}), {R"([[0, 0.25, "Bob"], [0, 0.75, "Tony"]])"}, &expected) .ok()); ASSERT_TRUE(result->Equals(expected)); ReadContextBuilder without_score_builder(path); + without_score_builder.EnablePreciseFileIndexEvaluate(true); without_score_builder.SetReadFieldNames({"f0"}).SetVectorSearch(vector_search); ASSERT_OK_AND_ASSIGN(std::shared_ptr without_score_context, without_score_builder.Finish()); @@ -623,7 +625,8 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchFiltersDeletionVectorBeforeTopK) { "f3", /*limit=*/1, std::vector{1.0f}, user_filter, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); ReadContextBuilder builder(path); - builder.SetReadFieldNames({"_INDEX_SCORE", "f0"}).SetVectorSearch(search); + builder.EnablePreciseFileIndexEvaluate(true); + builder.SetReadFieldNames({"_f3_INDEX_SCORE", "f0"}).SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( std::shared_ptr result, @@ -632,7 +635,7 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchFiltersDeletionVectorBeforeTopK) { std::shared_ptr expected; ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), - arrow::field("_INDEX_SCORE", arrow::float32()), + arrow::field("_f3_INDEX_SCORE", arrow::float32()), arrow::field("f0", arrow::utf8())}), {R"([[0, 0.75, "Tony"]])"}, &expected) .ok()); @@ -648,6 +651,7 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchFiltersDeletionVectorBeforeLimit) auto search = std::make_shared("f3", R"({"match":{"query":"match"}})", /*limit=*/1, user_filter); ReadContextBuilder builder(path); + builder.EnablePreciseFileIndexEvaluate(true); builder.SetReadFieldNames({"f0"}).SetFullTextSearch(search); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, builder.Finish()); ASSERT_OK_AND_ASSIGN( @@ -693,12 +697,15 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchRenamedUnprojectedFieldInOldFile) { nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); ReadContextBuilder builder(path); - builder.SetOptions(options).SetReadFieldNames({"f0", "_INDEX_SCORE"}).SetVectorSearch(search); + builder.EnablePreciseFileIndexEvaluate(true); + builder.SetOptions(options) + .SetReadFieldNames({"f0", "_renamed_f3_INDEX_SCORE"}) + .SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); - auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), - arrow::field("f0", arrow::utf8()), - arrow::field("_INDEX_SCORE", arrow::float32())}); + auto expected_type = arrow::struct_( + {arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("f0", arrow::utf8()), + arrow::field("_renamed_f3_INDEX_SCORE", arrow::float32())}); const std::vector> cases = { {RoaringBitmap32(), R"([[0,"Bob",0.25],[0,"Tony",0.75]])"}, {RoaringBitmap32::From({0}), R"([[0,"Tony",0.75]])"}}; @@ -722,6 +729,7 @@ TEST_F(RawFileSplitReadTest, TestVectorSearchRenamedUnprojectedFieldInOldFile) { auto missing_search = std::make_shared(*search); missing_search->field_name = "f3"; ReadContextBuilder missing_field_builder(path); + missing_field_builder.EnablePreciseFileIndexEvaluate(true); missing_field_builder.SetOptions(options).SetReadFieldNames({"f0"}).SetVectorSearch( missing_search); ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_context, @@ -760,6 +768,7 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile) auto search = std::make_shared("renamed_f3", R"({"match":{"query":"match"}})", 2); ReadContextBuilder builder(path); + builder.EnablePreciseFileIndexEvaluate(true); builder.SetOptions(options).SetReadFieldNames({"f0"}).SetFullTextSearch(search); ASSERT_OK_AND_ASSIGN(std::shared_ptr context, builder.Finish()); @@ -788,6 +797,7 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile) auto missing_search = std::make_shared(*search); missing_search->field_name = "f3"; ReadContextBuilder missing_field_builder(path); + missing_field_builder.EnablePreciseFileIndexEvaluate(true); missing_field_builder.SetOptions(options).SetReadFieldNames({"f0"}).SetFullTextSearch( missing_search); ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_context, diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index bc65b4661..f2bf754cf 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -30,6 +30,7 @@ #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/predicate/full_text_search.h" +#include "paimon/predicate/predicate_utils.h" #include "paimon/predicate/vector_search.h" #include "paimon/status.h" #include "paimon/table/format/format_table.h" @@ -41,8 +42,9 @@ ReadContext::ReadContext( const std::string& path, const std::string& branch, const std::vector& read_field_names, const std::vector& read_field_ids, const std::shared_ptr& predicate, const std::shared_ptr& vector_search, - const std::shared_ptr& full_text_search, bool enable_predicate_filter, - bool enable_prefetch, bool enable_late_materializing, uint32_t prefetch_batch_count, + const std::shared_ptr& full_text_search, + bool enable_precise_file_index_evaluate, bool enable_predicate_filter, bool enable_prefetch, + bool enable_late_materializing, uint32_t prefetch_batch_count, uint32_t prefetch_max_parallel_num, bool enable_multi_thread_row_to_batch, uint32_t row_to_batch_thread_number, const std::optional& table_schema, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, @@ -60,6 +62,7 @@ ReadContext::ReadContext( predicate_(predicate), vector_search_(vector_search), full_text_search_(full_text_search), + enable_precise_file_index_evaluate_(enable_precise_file_index_evaluate), enable_predicate_filter_(enable_predicate_filter), enable_prefetch_(enable_prefetch), enable_late_materializing_(enable_late_materializing), @@ -113,6 +116,7 @@ class ReadContextBuilder::Impl { predicate_.reset(); vector_search_.reset(); full_text_search_.reset(); + enable_precise_file_index_evaluate_ = false; enable_predicate_filter_ = false; enable_prefetch_ = false; enable_late_materializing_ = false; @@ -150,6 +154,7 @@ class ReadContextBuilder::Impl { std::shared_ptr predicate_; std::shared_ptr vector_search_; std::shared_ptr full_text_search_; + bool enable_precise_file_index_evaluate_ = false; bool enable_predicate_filter_ = false; bool enable_prefetch_ = false; bool enable_late_materializing_ = false; @@ -236,6 +241,11 @@ ReadContextBuilder& ReadContextBuilder::SetFullTextSearch( return *this; } +ReadContextBuilder& ReadContextBuilder::EnablePreciseFileIndexEvaluate(bool enabled) { + impl_->enable_precise_file_index_evaluate_ = enabled; + return *this; +} + ReadContextBuilder& ReadContextBuilder::EnablePredicateFilter(bool enabled) { impl_->enable_predicate_filter_ = enabled; return *this; @@ -338,6 +348,13 @@ ReadContextBuilder& ReadContextBuilder::WithCache(const std::shared_ptr& } Result> ReadContextBuilder::Finish() { + if (!impl_->enable_precise_file_index_evaluate_ && + (impl_->vector_search_ || impl_->full_text_search_ || + PredicateUtils::ContainsFullTextMatch(impl_->predicate_))) { + return Status::Invalid( + "VectorSearch, FullTextSearch and FullTextMatch require " + "EnablePreciseFileIndexEvaluate(true)"); + } if (impl_->vector_search_ && impl_->full_text_search_) { return Status::Invalid("VectorSearch and FullTextSearch cannot be configured together"); } @@ -430,14 +447,14 @@ Result> ReadContextBuilder::Finish() { auto ctx = std::make_unique( impl_->path_, branch, impl_->read_field_names_, impl_->read_field_ids_, impl_->predicate_, - impl_->vector_search_, impl_->full_text_search_, impl_->enable_predicate_filter_, - impl_->enable_prefetch_, impl_->enable_late_materializing_, impl_->prefetch_batch_count_, - impl_->prefetch_max_parallel_num_, impl_->enable_multi_thread_row_to_batch_, - impl_->row_to_batch_thread_number_, impl_->table_schema_, impl_->memory_pool_, - impl_->executor_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, - impl_->realtime_context_, impl_->options_, impl_->read_ahead_cache_enabled_, - impl_->cache_config_, impl_->cache_, impl_->format_table_, impl_->warmup_level_, - impl_->catalog_, impl_->identifier_); + impl_->vector_search_, impl_->full_text_search_, impl_->enable_precise_file_index_evaluate_, + impl_->enable_predicate_filter_, impl_->enable_prefetch_, impl_->enable_late_materializing_, + impl_->prefetch_batch_count_, impl_->prefetch_max_parallel_num_, + impl_->enable_multi_thread_row_to_batch_, impl_->row_to_batch_thread_number_, + impl_->table_schema_, impl_->memory_pool_, impl_->executor_, impl_->specific_file_system_, + impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, impl_->options_, + impl_->read_ahead_cache_enabled_, impl_->cache_config_, impl_->cache_, impl_->format_table_, + impl_->warmup_level_, impl_->catalog_, impl_->identifier_); if (impl_->read_schema_ && impl_->read_schema_->release) { ctx->SetReadSchema(std::move(impl_->read_schema_)); } diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp index 4fb5e00b4..3d5040a8a 100644 --- a/src/paimon/core/operation/read_context_test.cpp +++ b/src/paimon/core/operation/read_context_test.cpp @@ -123,6 +123,7 @@ TEST(ReadContextTest, TestDefaultValue) { ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_FALSE(ctx->GetPredicate()); ASSERT_FALSE(ctx->HasVectorOrFullTextSearch()); + ASSERT_FALSE(ctx->EnablePreciseFileIndexEvaluate()); ASSERT_FALSE(ctx->EnablePredicateFilter()); ASSERT_FALSE(ctx->EnablePrefetch()); ASSERT_TRUE(ctx->ReadAheadCacheEnabled()); @@ -151,6 +152,7 @@ TEST(ReadContextTest, TestSetContent) { auto predicate = PredicateBuilder::IsNull(/*field_index=*/0, /*field_name=*/"f1", FieldType::INT); builder.SetPredicate(predicate); + builder.EnablePreciseFileIndexEvaluate(true); builder.EnablePredicateFilter(true); builder.EnablePrefetch(true); builder.SetReadAheadCacheEnabled(false); @@ -177,6 +179,7 @@ TEST(ReadContextTest, TestSetContent) { ASSERT_EQ(ctx->GetReadFieldNames(), std::vector({"f1", "f2"})); ASSERT_EQ(ctx->GetReadFieldIds(), std::vector({0, 1})); ASSERT_EQ(*predicate, *(ctx->GetPredicate())); + ASSERT_TRUE(ctx->EnablePreciseFileIndexEvaluate()); ASSERT_TRUE(ctx->EnablePredicateFilter()); ASSERT_TRUE(ctx->EnablePrefetch()); ASSERT_FALSE(ctx->ReadAheadCacheEnabled()); @@ -218,6 +221,40 @@ TEST(ReadContextTest, TestSetWarmupLevel) { ASSERT_EQ(WarmupLevel::RAW, second_ctx->GetWarmupLevel()); } +TEST(ReadContextTest, TestSearchRequiresPreciseFileIndexEvaluate) { + auto vector = + std::make_shared("embedding", /*limit=*/3, std::vector{1.0f}, nullptr, + nullptr, std::nullopt, std::map{}); + auto text = std::make_shared("body", R"({"match":{"query":"apple"}})", 3); + auto match = std::make_shared("body", text->query); + ASSERT_OK_AND_ASSIGN(std::shared_ptr full_text, + PredicateBuilder::FullTextMatch(0, match)); + auto scalar = PredicateBuilder::IsNotNull(1, "price", FieldType::INT); + ASSERT_OK_AND_ASSIGN(std::shared_ptr disjunction, + PredicateBuilder::Or({scalar, full_text})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({scalar, disjunction})); + for (int32_t search_type = 0; search_type < 3; ++search_type) { + for (bool enable_before_search : {false, true}) { + ReadContextBuilder builder("table_root_path"); + builder.EnablePreciseFileIndexEvaluate(enable_before_search); + if (search_type == 0) { + builder.SetVectorSearch(vector); + } else if (search_type == 1) { + builder.SetFullTextSearch(text); + } else { + builder.SetPredicate(predicate); + } + if (!enable_before_search) { + ASSERT_NOK_WITH_MSG(builder.Finish(), "EnablePreciseFileIndexEvaluate(true)"); + builder.EnablePreciseFileIndexEvaluate(true); + } + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); + ASSERT_TRUE(context->EnablePreciseFileIndexEvaluate()); + } + } +} + TEST(ReadContextTest, TestSetOptionsOverridesAddedOptions) { ReadContextBuilder builder("table_root_path"); builder.AddOption("old", "value"); @@ -345,6 +382,7 @@ TEST(ReadContextTest, TestFileIndexSearchConfigurationAndReset) { VectorSearch::DistanceType::COSINE, std::map{}); ReadContextBuilder builder("table_root_path"); ASSERT_EQ(&builder, &builder.SetVectorSearch(vector_search)); + builder.EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context, builder.Finish()); EXPECT_EQ(vector_search, vector_context->GetVectorSearch()); EXPECT_FALSE(vector_context->GetFullTextSearch()); @@ -353,6 +391,7 @@ TEST(ReadContextTest, TestFileIndexSearchConfigurationAndReset) { std::shared_ptr full_text_search = std::make_shared("body", R"({"match":{"query":"paimon"}})", /*limit=*/5); ASSERT_EQ(&builder, &builder.SetFullTextSearch(full_text_search)); + builder.EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr full_text_context, builder.Finish()); EXPECT_FALSE(full_text_context->GetVectorSearch()); EXPECT_EQ(full_text_search, full_text_context->GetFullTextSearch()); @@ -361,6 +400,7 @@ TEST(ReadContextTest, TestFileIndexSearchConfigurationAndReset) { // Finish resets search configuration just like every other per-read option. ASSERT_OK_AND_ASSIGN(std::unique_ptr reset_context, builder.Finish()); EXPECT_FALSE(reset_context->HasVectorOrFullTextSearch()); + EXPECT_FALSE(reset_context->EnablePreciseFileIndexEvaluate()); } TEST(ReadContextTest, TestRejectConflictingFileIndexSearch) { @@ -370,7 +410,9 @@ TEST(ReadContextTest, TestRejectConflictingFileIndexSearch) { std::shared_ptr full_text_search = std::make_shared("body", R"({"match":{"query":"paimon"}})", /*limit=*/5); ReadContextBuilder conflicting("table_root_path"); - conflicting.SetVectorSearch(vector_search).SetFullTextSearch(full_text_search); + conflicting.EnablePreciseFileIndexEvaluate(true) + .SetVectorSearch(vector_search) + .SetFullTextSearch(full_text_search); ASSERT_NOK_WITH_MSG(conflicting.Finish(), "cannot be configured together"); } @@ -384,13 +426,17 @@ TEST(ReadContextTest, TestFullTextPredicateAndRootTopKConfiguration) { std::make_shared("embedding", 3, std::vector{1.0f}, nullptr, nullptr, std::nullopt, std::map{}); ReadContextBuilder builder("table_root_path"); - builder.SetReadFieldNames({"body", "price"}).SetPredicate(filter).SetVectorSearch(vector); + builder.EnablePreciseFileIndexEvaluate(true) + .SetReadFieldNames({"body", "price"}) + .SetPredicate(filter) + .SetVectorSearch(vector); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_EQ(context->GetPredicate(), filter); ASSERT_EQ(context->GetVectorSearch(), vector); auto ranked_text = std::make_shared("title", match->query, 2); - builder.SetPredicate(filter).SetFullTextSearch(ranked_text); + builder.EnablePreciseFileIndexEvaluate(true).SetPredicate(filter).SetFullTextSearch( + ranked_text); ASSERT_OK_AND_ASSIGN(context, builder.Finish()); ASSERT_EQ(context->GetPredicate(), filter); ASSERT_EQ(context->GetFullTextSearch(), ranked_text); @@ -401,11 +447,11 @@ TEST(ReadContextTest, TestRootTopKValidation) { std::make_shared("embedding", 0, std::vector{1.0f}, nullptr, nullptr, std::nullopt, std::map{}); ReadContextBuilder vector_builder("table_root_path"); - vector_builder.SetVectorSearch(vector); + vector_builder.EnablePreciseFileIndexEvaluate(true).SetVectorSearch(vector); ASSERT_NOK_WITH_MSG(vector_builder.Finish(), "VectorSearch TopK limit must be positive"); auto text = std::make_shared("body", R"({"match":{"query":"apple"}})", -1); ReadContextBuilder text_builder("table_root_path"); - text_builder.SetFullTextSearch(text); + text_builder.EnablePreciseFileIndexEvaluate(true).SetFullTextSearch(text); ASSERT_NOK_WITH_MSG(text_builder.Finish(), "FullTextSearch TopK limit must be positive"); } diff --git a/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp b/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp index 82eef3a23..96543db30 100644 --- a/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp +++ b/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp @@ -104,9 +104,11 @@ RealtimeOffsetFileIndexLookup::LookupFilePositions(const RoaringBitmap64& offset std::shared_ptr predicate = PredicateBuilder::In( schema->GetFieldIndex(offset_name), offset_name, FieldType::BIGINT, literals); PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr index_result, - FileIndexEvaluator::Evaluate(schema, options_, predicate, path_factory_, file, - file_system_, memory_pool_)); + std::unique_ptr evaluator, + FileIndexEvaluator::Create(schema, options_, path_factory_, file, file_system_, + /*only_use_embedding_index=*/false, memory_pool_)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index_result, + evaluator->Evaluate(predicate)); PAIMON_ASSIGN_OR_RAISE(bool remains, index_result->IsRemain()); if (!remains) { continue; diff --git a/src/paimon/core/table/source/append_only_table_read.cpp b/src/paimon/core/table/source/append_only_table_read.cpp index 47f1c5a32..ca786e206 100644 --- a/src/paimon/core/table/source/append_only_table_read.cpp +++ b/src/paimon/core/table/source/append_only_table_read.cpp @@ -392,13 +392,13 @@ Result> AppendOnlyTableRead::CreateCountReader( "CreateCountReader does not support process-local real-time splits"); } } - if (context_->GetPredicate() != nullptr) { + if (context_->EnablePreciseFileIndexEvaluate()) { return Status::NotImplemented( - "CreateCountReader with predicate pushdown is not supported yet"); + "CreateCountReader does not support precise file-index evaluation"); } - if (context_->HasVectorOrFullTextSearch()) { + if (context_->GetPredicate() != nullptr) { return Status::NotImplemented( - "CreateCountReader with VectorSearch or FullTextSearch is not supported"); + "CreateCountReader with predicate pushdown is not supported yet"); } return std::make_unique(splits, context_->GetCoreOptions().GetFileSystem(), diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp index ae85ec405..eeed68e59 100644 --- a/src/paimon/core/table/source/key_value_table_read.cpp +++ b/src/paimon/core/table/source/key_value_table_read.cpp @@ -351,13 +351,13 @@ Result> KeyValueTableRead::CreateCountReader( "CreateCountReader does not support process-local real-time splits"); } } - if (context_->GetPredicate() != nullptr) { + if (context_->EnablePreciseFileIndexEvaluate()) { return Status::NotImplemented( - "CreateCountReader with predicate pushdown is not supported yet"); + "CreateCountReader does not support precise file-index evaluation"); } - if (context_->HasVectorOrFullTextSearch()) { + if (context_->GetPredicate() != nullptr) { return Status::NotImplemented( - "CreateCountReader with VectorSearch or FullTextSearch is not supported"); + "CreateCountReader with predicate pushdown is not supported yet"); } if (force_keep_delete_) { diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index ead31b17a..3b1226513 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -30,26 +31,32 @@ #include "gtest/gtest.h" #include "paimon/catalog/table.h" #include "paimon/common/catalog/catalog_context.h" +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/catalog/identifier.h" #include "paimon/common/data/binary_row.h" #include "paimon/core/core_options.h" +#include "paimon/core/io/data_file_meta.h" #include "paimon/core/operation/abstract_split_read.h" #include "paimon/core/operation/split_read.h" #include "paimon/core/schema/schema_manager.h" -#include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/key_value_table_read.h" #include "paimon/defs.h" -#include "paimon/fs/local/local_file_system.h" #include "paimon/predicate/full_text_match.h" #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/predicate/vector_search.h" #include "paimon/read_context.h" +#include "paimon/scan_context.h" #include "paimon/status.h" #include "paimon/table/format/format_table.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/mock/mock_search_file_index.h" #include "paimon/testing/utils/binary_row_generator.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -125,50 +132,94 @@ TEST(TableReadTest, TestReadWithSpecifiedInvalidSchema) { "Get field field_no_exist failed: not exist in table schema"); } -TEST(TableReadTest, TestFullTextMatchExecutionIsNotImplemented) { +TEST(TableReadTest, TestFullTextMatchReadEntryAndScoreValidation) { auto match = std::make_shared("f0", R"({"match":{"query":"apple"}})"); ASSERT_OK_AND_ASSIGN(auto full_text, PredicateBuilder::FullTextMatch(/*field_index=*/0, match)); auto scalar = PredicateBuilder::Equal(/*field_index=*/1, "f1", FieldType::INT, Literal(1)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::Or({scalar, full_text})); const std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - auto fs = std::make_shared(); - SchemaManager schema_manager(fs, table_path); - ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); - Table table(fs, table_path, schema, Identifier("append_09.db", "append_09")); - for (bool table_entry : {false, true}) { - ReadContextBuilder builder(table_path); - builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - auto result = - table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); - ASSERT_NOK_WITH_MSG(result, "FullTextMatch predicate execution is not implemented yet"); - ASSERT_TRUE(result.status().IsNotImplemented()); + ReadContextBuilder builder(table_path); + builder.EnablePreciseFileIndexEvaluate(true) + .SetReadFieldNames({"f0", "f1"}) + .SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); + ASSERT_OK(TableRead::Create(std::move(context))); - builder.SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}).SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(context, builder.Finish()); - result = - table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); - ASSERT_NOK_WITH_MSG(result, "Score-producing FullTextMatch cannot appear under OR"); - } + builder.EnablePreciseFileIndexEvaluate(true) + .SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}) + .SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(context, builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(context)), + "Score-producing FullTextMatch cannot appear under OR"); } -TEST(TableReadTest, TestPerFieldScoreExecutionIsNotImplemented) { +TEST(TableReadTest, TestPerFieldScoreReadEntry) { const std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; auto search = std::make_shared("f0", R"({"match":{"query":"apple"}})", /*limit=*/1); - auto fs = std::make_shared(); - SchemaManager schema_manager(fs, path); - ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); - Table table(fs, path, schema, Identifier("append_09.db", "append_09")); - for (bool table_entry : {false, true}) { - ReadContextBuilder builder(path); - builder.SetFullTextSearch(search).SetReadFieldNames({"f1", "_f0_INDEX_SCORE"}); - ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - auto result = - table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); - ASSERT_NOK_WITH_MSG(result, "Per-field index score output is not implemented yet"); - ASSERT_TRUE(result.status().IsNotImplemented()); + ReadContextBuilder builder(path); + builder.EnablePreciseFileIndexEvaluate(true).SetFullTextSearch(search).SetReadFieldNames( + {"f1", "_f0_INDEX_SCORE"}); + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); + ASSERT_OK(TableRead::Create(std::move(context))); +} + +TEST(TableReadTest, TestMixedFilePredicateProducesScoresThroughPublicReadEntry) { + const std::string path = + GetDataDir() + "/orc/multi_partition_append_table.db/multi_partition_append_table"; + ScanContextBuilder scan_builder(path); + scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + std::shared_ptr selected_split; + for (const auto& split : plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + for (const auto& file : data_split->DataFiles()) { + if (file->file_name != "data-01b6a930-6564-409b-b8f4-ed1307790d72-0.orc") { + continue; + } + auto indexed_file = std::make_shared(*file); + ASSERT_OK_AND_ASSIGN( + indexed_file->embedded_index, + MockSearchFileIndex::MakeIndex("f0", MockSearchFileIndex::FULL_TEXT_IDENTIFIER, + {{0, 0.25f}, {2, 0.75f}}, GetDefaultPool())); + DataSplitImpl::Builder split_builder(data_split->Partition(), data_split->Bucket(), + data_split->BucketPath(), {indexed_file}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr single, + split_builder.WithSnapshot(1).RawConvertible(true).Build()); + selected_split = std::dynamic_pointer_cast(single); + } } + ASSERT_TRUE(selected_split); + auto match = std::make_shared("f0", R"({"match":{"query":"names"}})"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr text, + PredicateBuilder::FullTextMatch(2, match)); + auto scalar = PredicateBuilder::GreaterThan(1, "f3", FieldType::DOUBLE, Literal(13.0)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({text, scalar})); + ReadContextBuilder builder(path); + builder.EnablePreciseFileIndexEvaluate(true) + .SetReadFieldNames({"_f0_INDEX_SCORE", "f3", "f0"}) + .SetPredicate(predicate) + .AddOption(Options::READ_BATCH_SIZE, "1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(selected_split)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + ReadResultCollector::CollectResult(std::move(reader))); + ASSERT_TRUE(actual); + ASSERT_EQ(actual->length(), 1); + auto values = std::dynamic_pointer_cast(actual->chunk(0)); + ASSERT_TRUE(values); + auto scores = std::dynamic_pointer_cast(values->field(1)); + auto names = std::dynamic_pointer_cast(values->field(3)); + ASSERT_TRUE(scores); + ASSERT_TRUE(names); + ASSERT_FLOAT_EQ(scores->Value(0), 0.75f); + ASSERT_EQ(names->GetString(0), "Tony"); } TEST(TableReadTest, TestCreateKeyValueTableRead) { @@ -190,10 +241,13 @@ TEST(TableReadTest, TestCreateCountReaderRejectsFileIndexSearch) { auto search = std::make_shared("f0", R"({"match":{"query":"Alice"}})", /*limit=*/1); ReadContextBuilder context_builder(path); - context_builder.SetReadFieldNames({"f0"}).SetFullTextSearch(search); + context_builder.EnablePreciseFileIndexEvaluate(true) + .SetReadFieldNames({"f0"}) + .SetFullTextSearch(search); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - ASSERT_NOK_WITH_MSG(table_read->CreateCountReader({}), "VectorSearch or FullTextSearch"); + ASSERT_NOK_WITH_MSG(table_read->CreateCountReader({}), + "CreateCountReader does not support precise file-index evaluation"); } { std::string path = paimon::test::GetDataDir() + @@ -202,10 +256,64 @@ TEST(TableReadTest, TestCreateCountReaderRejectsFileIndexSearch) { nullptr, nullptr, std::nullopt, std::map{}); ReadContextBuilder context_builder(path); - context_builder.SetReadFieldNames({"f0"}).SetVectorSearch(search); + context_builder.EnablePreciseFileIndexEvaluate(true) + .SetReadFieldNames({"f0"}) + .SetVectorSearch(search); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - ASSERT_NOK_WITH_MSG(table_read->CreateCountReader({}), "VectorSearch or FullTextSearch"); + ASSERT_NOK_WITH_MSG(table_read->CreateCountReader({}), + "CreateCountReader does not support precise file-index evaluation"); + } +} + +TEST(TableReadTest, TestPreciseEvaluationRejectsSystemAndFormatTables) { + std::string path = GetDataDir() + "/orc/append_09.db/append_09"; + ReadContextBuilder system_builder(path + "$snapshots"); + system_builder.EnablePreciseFileIndexEvaluate(true); + ASSERT_OK_AND_ASSIGN(auto system_context, system_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(system_context)), + "Precise file-index evaluation is not supported for system tables"); + + auto directory = UniqueTestDirectory::Create(); + SchemaManager schema_manager(directory->GetFileSystem(), directory->Str()); + ASSERT_OK_AND_ASSIGN( + auto schema, schema_manager.CreateTable(arrow::schema({arrow::field("f0", arrow::int32())}), + /*partition_keys=*/{}, /*primary_keys=*/{}, + {{Options::TYPE, "format-table"}})); + ASSERT_OK_AND_ASSIGN(auto format_table, + FormatTable::Create(directory->GetFileSystem(), directory->Str(), + Identifier("db", "table"), {})); + for (bool loaded_table : {false, true}) { + ReadContextBuilder builder = + loaded_table ? ReadContextBuilder(format_table) : ReadContextBuilder(directory->Str()); + builder.EnablePreciseFileIndexEvaluate(true); + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(context)), + "Precise file-index evaluation is not supported for format tables"); + } +} + +TEST(TableReadTest, TestPreciseScalarEvaluationRejectsUnsupportedTableModes) { + for (bool primary_key : {false, true}) { + const std::string path = + GetDataDir() + + (primary_key ? "/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality" + : "/orc/append_09.db/append_09"); + ReadContextBuilder builder(path); + builder.EnablePreciseFileIndexEvaluate(true).SetReadFieldNames({"f3"}).SetPredicate( + PredicateBuilder::GreaterThan(0, "f3", FieldType::DOUBLE, Literal(0.0))); + if (primary_key) { + builder.AddOption(Options::DELETION_VECTORS_ENABLED, "false"); + } else { + builder.AddOption(Options::DATA_EVOLUTION_ENABLED, "true"); + } + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); + const std::string expected_error = + primary_key ? "Precise file-index evaluation on primary-key tables requires " + "deletion vectors" + : "Precise file-index evaluation does not support data-evolution or " + "real-time reads"; + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(context)), expected_error); } } @@ -216,7 +324,8 @@ TEST(TableReadTest, TestPrimaryKeyFileIndexSearchDoesNotFallBackToMergeReader) { std::make_shared("f3", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, std::nullopt, std::map{}); ReadContextBuilder context_builder(path); - context_builder.SetReadFieldNames({"f0"}).SetVectorSearch(search); + context_builder.EnablePreciseFileIndexEvaluate(true).SetReadFieldNames({"f0"}).SetVectorSearch( + search); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, TableRead::Create(std::move(context))); diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index 7370377fb..c49b77163 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -106,7 +106,6 @@ #include "paimon/file_store_write.h" #include "paimon/format/file_format.h" #include "paimon/fs/file_system.h" -#include "paimon/predicate/predicate_utils.h" #include "paimon/read_context.h" #include "paimon/realtime/realtime_context.h" #include "paimon/result.h" @@ -1008,12 +1007,6 @@ Result> CreateInternalReadContext( Result> CreateTableRead( const std::shared_ptr& internal_context, const std::shared_ptr& memory_pool, const std::shared_ptr& executor) { - if (PredicateUtils::ContainsFullTextMatch(internal_context->GetPredicate())) { - return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); - } - if (!internal_context->GetIndexScoreBindings().empty()) { - return Status::NotImplemented("Per-field index score output is not implemented yet"); - } const auto& core_options = internal_context->GetCoreOptions(); const auto& table_schema = internal_context->GetTableSchema(); if (internal_context->GetRealtimeContext() && !core_options.RealtimeEnabled()) { @@ -1077,8 +1070,9 @@ Result> NewDataTableRead( /// refuses by name what a format table cannot honour. Result> NewFormatTableRead(const std::shared_ptr& table, const std::shared_ptr& context) { - if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { - return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); + if (context->EnablePreciseFileIndexEvaluate()) { + return Status::NotImplemented( + "Precise file-index evaluation is not supported for format tables"); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, FormatTableRead::Create(table, context)); @@ -1103,10 +1097,6 @@ Result> TableRead::Create(std::unique_ptrGetFormatTable() != nullptr) { - if (context->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for format tables"); - } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, FormatTable::Copy(context->GetFormatTable(), context->GetOptions())); return NewFormatTableRead(format_table, context); @@ -1118,13 +1108,9 @@ Result> TableRead::Create(std::unique_ptr system_table_path, SystemTableLoader::TryParsePath(context->GetPath())); if (system_table_path) { - if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { + if (context->EnablePreciseFileIndexEvaluate()) { return Status::NotImplemented( - "FullTextMatch predicate execution is not implemented yet"); - } - if (context->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for system tables"); + "Precise file-index evaluation is not supported for system tables"); } PAIMON_ASSIGN_OR_RAISE( std::shared_ptr system_table, @@ -1144,10 +1130,6 @@ Result> TableRead::Create(std::unique_ptrGetSpecificTableSchema(), /*schema_manager=*/nullptr, &latest_schema)); if (format_table != nullptr) { - if (context->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for format tables"); - } return NewFormatTableRead(format_table, context); } diff --git a/src/paimon/indexer/lumina/lumina_file_index.cpp b/src/paimon/indexer/lumina/lumina_file_index.cpp index 6db365aa7..2308bd7a6 100644 --- a/src/paimon/indexer/lumina/lumina_file_index.cpp +++ b/src/paimon/indexer/lumina/lumina_file_index.cpp @@ -26,6 +26,7 @@ #include "fmt/format.h" #include "lumina/api/LuminaBuilder.h" #include "lumina/core/Types.h" +#include "paimon/common/file_index/empty/empty_file_index_reader.h" #include "paimon/common/io/byte_array_output_stream.h" #include "paimon/common/io/memory_segment_output_stream.h" #include "paimon/common/io/offset_input_stream.h" @@ -56,6 +57,11 @@ Result> ImportVectorSchema(::ArrowSchema* c_schem } // namespace +std::shared_ptr LuminaFileIndexer::CreateEmptyReader() const { + return std::make_shared( + std::vector{FileIndexReader::SearchType::VECTOR_SEARCH}); +} + LuminaFileIndexWriter::LuminaFileIndexWriter(std::string field_name, std::shared_ptr arrow_type, const LuminaIndexInfo& index_info, diff --git a/src/paimon/indexer/lumina/lumina_file_index.h b/src/paimon/indexer/lumina/lumina_file_index.h index 9021756e2..76d893eab 100644 --- a/src/paimon/indexer/lumina/lumina_file_index.h +++ b/src/paimon/indexer/lumina/lumina_file_index.h @@ -41,6 +41,8 @@ class LuminaFileIndexer final : public FileIndexer { explicit LuminaFileIndexer(const std::map& options) : options_(options) {} + std::shared_ptr CreateEmptyReader() const override; + Result>> GetExtraFieldNames() const override; Result> CreateReader( @@ -84,6 +86,10 @@ class LuminaFileIndexReader final : public FileIndexReader { public: explicit LuminaFileIndexReader(std::unique_ptr&& searcher); + bool SupportsSearch(SearchType search_type) const override { + return search_type == SearchType::VECTOR_SEARCH; + } + Result> VisitVectorSearch( const std::shared_ptr& vector_search) override; diff --git a/src/paimon/testing/mock/mock_search_file_index.cpp b/src/paimon/testing/mock/mock_search_file_index.cpp index 27576e4b7..27e6d47e4 100644 --- a/src/paimon/testing/mock/mock_search_file_index.cpp +++ b/src/paimon/testing/mock/mock_search_file_index.cpp @@ -21,24 +21,107 @@ #include #include "arrow/c/bridge.h" +#include "paimon/common/file_index/empty/empty_file_index_reader.h" #include "paimon/common/io/byte_array_output_stream.h" #include "paimon/common/io/data_output_stream.h" #include "paimon/common/io/memory_segment_output_stream.h" #include "paimon/common/io/offset_input_stream.h" +#include "paimon/common/predicate/full_text_match_predicate.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/factories/factory.h" #include "paimon/file_index/bitmap_index_result.h" #include "paimon/file_index/scored_file_index_result.h" #include "paimon/io/data_input_stream.h" #include "paimon/memory/bytes.h" +#include "paimon/predicate/full_text_match.h" #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/vector_search.h" namespace paimon::test { -MockSearchFileIndexReader::MockSearchFileIndexReader(std::string field_name, - SearchIndexType index_type, - MockSearchFileIndex::ScoredRows rows) - : field_name_(std::move(field_name)), index_type_(index_type), rows_(std::move(rows)) {} + +MockSearchFileIndexReader::MockSearchFileIndexReader( + std::string field_name, SearchIndexType index_type, MockSearchFileIndex::ScoredRows rows, + std::shared_ptr statistics) + : field_name_(std::move(field_name)), + index_type_(index_type), + rows_(std::move(rows)), + statistics_(std::move(statistics)) {} + +MockSearchFileIndexReader::MockSearchFileIndexReader( + std::string field_name, SearchIndexType index_type, MockSearchFileIndex::QueryMatches queries, + std::shared_ptr statistics) + : field_name_(std::move(field_name)), + index_type_(index_type), + queries_(std::move(queries)), + statistics_(std::move(statistics)) {} + +bool MockSearchFileIndexReader::SupportsSearch(SearchType search_type) const { + return index_type_ == SearchIndexType::VECTOR ? search_type == SearchType::VECTOR_SEARCH + : search_type == SearchType::FULL_TEXT_MATCH || + search_type == SearchType::FULL_TEXT_SEARCH; +} + +const MockSearchFileIndex::ScoredRows& MockSearchFileIndexReader::GetRows( + const std::string& query) const { + auto iter = queries_.find(query); + return iter == queries_.end() ? rows_ : iter->second.rows; +} + +std::optional MockSearchFileIndexReader::EstimateSelectivity( + const std::shared_ptr& predicate) const { + auto full_text = std::dynamic_pointer_cast(predicate); + if (!full_text) { + return std::nullopt; + } + auto iter = queries_.find(full_text->GetFullTextMatch()->query); + return iter == queries_.end() ? std::nullopt : iter->second.selectivity; +} + +Result> MockSearchFileIndexReader::MakeResult( + const std::map& selected, bool needs_scores) const { + if (needs_scores) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, + MakeScoredResult(selected)); + return result; + } + RoaringBitmap32 positions; + for (const auto& [row_id, score] : selected) { + positions.Add(row_id); + } + return std::make_shared( + [bitmap = std::move(positions)]() -> Result { return bitmap; }); +} + +Result> MockSearchFileIndexReader::MakeScoredResult( + const std::map& selected) const { + RoaringBitmap32 positions; + std::vector scores; + for (const auto& [row_id, score] : selected) { + positions.Add(row_id); + scores.push_back(score); + } + return ScoredFileIndexResult::Create(std::move(positions), std::move(scores)); +} + +Result> MockSearchFileIndexReader::VisitFullTextMatch( + const std::shared_ptr& match, bool needs_scores) { + if (index_type_ != SearchIndexType::FULL_TEXT) { + return FileIndexReader::VisitFullTextMatch(match, needs_scores); + } + if (match->field_name != field_name_) { + return Status::Invalid("Mock full-text match field does not match the indexed field"); + } + if (statistics_) { + statistics_->executions.push_back({field_name_, match->query, needs_scores}); + } + std::map selected; + for (const auto& [row_id, score] : GetRows(match->query)) { + if (!match->include_row_ids || match->include_row_ids->Contains(row_id)) { + selected.emplace(row_id, score); + } + } + return MakeResult(selected, needs_scores); +} Result> MockSearchFileIndexReader::VisitVectorSearch( const std::shared_ptr& search) { @@ -48,6 +131,9 @@ Result> MockSearchFileIndexReader::VisitV if (search->field_name != field_name_) { return Status::Invalid("Mock vector search field does not match the indexed field"); } + if (statistics_) { + statistics_->executions.push_back({field_name_, "", true}); + } std::map selected; for (const auto& [row_id, score] : rows_) { if (selected.size() >= static_cast(search->limit)) { @@ -57,16 +143,10 @@ Result> MockSearchFileIndexReader::VisitV selected.emplace(row_id, score); } } - RoaringBitmap32 positions; - std::vector scores; - for (const auto& [row_id, score] : selected) { - positions.Add(row_id); - scores.push_back(score); - } - return ScoredFileIndexResult::Create(std::move(positions), std::move(scores)); + return MakeScoredResult(selected); } -Result> MockSearchFileIndexReader::VisitFullTextSearch( +Result> MockSearchFileIndexReader::VisitFullTextSearch( const std::shared_ptr& search) { if (index_type_ != SearchIndexType::FULL_TEXT) { return FileIndexReader::VisitFullTextSearch(search); @@ -74,21 +154,33 @@ Result> MockSearchFileIndexReader::VisitFullTex if (search->field_name != field_name_) { return Status::Invalid("Mock full-text search field does not match the indexed field"); } - RoaringBitmap32 positions; - for (const auto& [row_id, score] : rows_) { - if (positions.Cardinality() >= search->limit) { + if (statistics_) { + statistics_->executions.push_back({field_name_, search->query, true}); + } + std::map selected; + for (const auto& [row_id, score] : GetRows(search->query)) { + if (selected.size() >= static_cast(search->limit)) { break; } if (!search->include_row_ids || search->include_row_ids->Contains(row_id)) { - positions.Add(row_id); + selected.emplace(row_id, score); } } - return std::make_shared( - [bitmap = std::move(positions)]() -> Result { return bitmap; }); + return MakeScoredResult(selected); } -MockSearchFileIndexer::MockSearchFileIndexer(SearchIndexType index_type) - : index_type_(index_type) {} +MockSearchFileIndexer::MockSearchFileIndexer( + SearchIndexType index_type, std::shared_ptr statistics) + : index_type_(index_type), statistics_(std::move(statistics)) {} + +std::shared_ptr MockSearchFileIndexer::CreateEmptyReader() const { + using SearchType = FileIndexReader::SearchType; + std::vector search_types = + index_type_ == SearchIndexType::VECTOR + ? std::vector{SearchType::VECTOR_SEARCH} + : std::vector{SearchType::FULL_TEXT_MATCH, SearchType::FULL_TEXT_SEARCH}; + return std::make_shared(std::move(search_types)); +} Result> MockSearchFileIndexer::CreateReader( ::ArrowSchema* c_schema, int64_t start, int64_t length, @@ -99,6 +191,27 @@ Result> MockSearchFileIndexer::CreateReader( OffsetInputStream::Create(input, length, start)); DataInputStream data_input(payload); PAIMON_ASSIGN_OR_RAISE(int32_t count, data_input.ReadValue()); + if (count == -1) { + PAIMON_ASSIGN_OR_RAISE(int32_t query_count, data_input.ReadValue()); + MockSearchFileIndex::QueryMatches queries; + for (int32_t i = 0; i < query_count; ++i) { + PAIMON_ASSIGN_OR_RAISE(std::string query, data_input.ReadString()); + PAIMON_ASSIGN_OR_RAISE(double estimate, data_input.ReadValue()); + PAIMON_ASSIGN_OR_RAISE(int32_t row_count, data_input.ReadValue()); + MockSearchFileIndex::QueryResult result; + if (estimate >= 0.0) { + result.selectivity = estimate; + } + for (int32_t j = 0; j < row_count; ++j) { + PAIMON_ASSIGN_OR_RAISE(int32_t row_id, data_input.ReadValue()); + PAIMON_ASSIGN_OR_RAISE(float score, data_input.ReadValue()); + result.rows.emplace_back(row_id, score); + } + queries.emplace(std::move(query), std::move(result)); + } + return std::make_shared(schema->field(0)->name(), index_type_, + std::move(queries), statistics_); + } MockSearchFileIndex::ScoredRows rows; for (int32_t i = 0; i < count; ++i) { PAIMON_ASSIGN_OR_RAISE(int32_t row_id, data_input.ReadValue()); @@ -106,7 +219,7 @@ Result> MockSearchFileIndexer::CreateReader( rows.emplace_back(row_id, score); } return std::make_shared(schema->field(0)->name(), index_type_, - std::move(rows)); + std::move(rows), statistics_); } Result> MockSearchFileIndexer::CreateWriter( @@ -120,7 +233,7 @@ const char* MockVectorSearchFileIndexFactory::Identifier() const { Result> MockVectorSearchFileIndexFactory::Create( const std::map&) const { - return std::make_unique(SearchIndexType::VECTOR); + return std::make_unique(SearchIndexType::VECTOR, statistics_); } const char* MockFullTextSearchFileIndexFactory::Identifier() const { @@ -129,7 +242,7 @@ const char* MockFullTextSearchFileIndexFactory::Identifier() const { Result> MockFullTextSearchFileIndexFactory::Create( const std::map&) const { - return std::make_unique(SearchIndexType::FULL_TEXT); + return std::make_unique(SearchIndexType::FULL_TEXT, statistics_); } REGISTER_PAIMON_FACTORY(MockVectorSearchFileIndexFactory); @@ -140,7 +253,7 @@ Result> OptionsCheckingMockSearchFileIndexFactory:: if (options != expected_options_) { return Status::Invalid("Mock search backend received unexpected indexer options"); } - return std::make_unique(index_type_); + return std::make_unique(index_type_, statistics_); } namespace { @@ -169,6 +282,34 @@ Result> MockSearchFileIndex::MakePayload( return payload; } +Result> MockSearchFileIndex::MakeQueryPayload( + const QueryMatches& queries, const std::shared_ptr& pool) { + auto output = MakeOutput(pool); + DataOutputStream data_output(output); + PAIMON_RETURN_NOT_OK(data_output.WriteValue(-1)); + PAIMON_RETURN_NOT_OK(data_output.WriteValue(static_cast(queries.size()))); + for (const auto& [query, result] : queries) { + PAIMON_RETURN_NOT_OK(data_output.WriteString(query)); + PAIMON_RETURN_NOT_OK(data_output.WriteValue(result.selectivity.value_or(-1.0))); + PAIMON_RETURN_NOT_OK( + data_output.WriteValue(static_cast(result.rows.size()))); + for (const auto& [row_id, score] : result.rows) { + PAIMON_RETURN_NOT_OK(data_output.WriteValue(row_id)); + PAIMON_RETURN_NOT_OK(data_output.WriteValue(score)); + } + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr payload, output->Finish(pool.get())); + return payload; +} + +Result> MockSearchFileIndex::MakeQueryIndex( + const std::string& field_name, const std::string& index_type, const QueryMatches& queries, + const std::shared_ptr& pool) { + FileIndexFormat::ColumnIndexes indexes; + PAIMON_ASSIGN_OR_RAISE(indexes[field_name][index_type], MakeQueryPayload(queries, pool)); + return Serialize(indexes, pool); +} + Result> MockSearchFileIndex::Serialize( const FileIndexFormat::ColumnIndexes& indexes, const std::shared_ptr& pool) { auto output = MakeOutput(pool); diff --git a/src/paimon/testing/mock/mock_search_file_index.h b/src/paimon/testing/mock/mock_search_file_index.h index 8def6b0cb..06cc91f3f 100644 --- a/src/paimon/testing/mock/mock_search_file_index.h +++ b/src/paimon/testing/mock/mock_search_file_index.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -36,7 +37,7 @@ namespace paimon::test { enum class SearchIndexType { VECTOR, FULL_TEXT }; /// File Index test backend with configured matches in search order. It applies pre-filters -/// before limits and returns vector scores aligned with ascending file-local row positions. +/// before limits and returns search scores aligned with ascending file-local row positions. class MockSearchFileIndex { public: MockSearchFileIndex() = delete; @@ -47,6 +48,30 @@ class MockSearchFileIndex { using ScoredRows = std::vector>; + struct QueryResult { + ScoredRows rows; + std::optional selectivity = std::nullopt; + }; + using QueryMatches = std::map; + + struct Execution { + std::string field_name; + std::string query; + bool needs_scores; + }; + + struct Statistics { + std::vector executions; + }; + + static Result> MakeQueryPayload(const QueryMatches& queries, + const std::shared_ptr& pool); + + static Result> MakeQueryIndex(const std::string& field_name, + const std::string& index_type, + const QueryMatches& queries, + const std::shared_ptr& pool); + /// Serialize configured matches into a mock backend payload, without a FileIndexFormat /// container. Use Serialize() to wrap it before reading through FileIndexFormat. /// @param rows File-local row positions and scores in search order. Full-text search ignores @@ -80,23 +105,49 @@ class MockSearchFileIndex { class MockSearchFileIndexReader final : public FileIndexReader { public: MockSearchFileIndexReader(std::string field_name, SearchIndexType index_type, - MockSearchFileIndex::ScoredRows rows); + MockSearchFileIndex::ScoredRows rows, + std::shared_ptr statistics); + + MockSearchFileIndexReader(std::string field_name, SearchIndexType index_type, + MockSearchFileIndex::QueryMatches queries, + std::shared_ptr statistics); + + bool SupportsSearch(SearchType search_type) const override; + + std::optional EstimateSelectivity( + const std::shared_ptr& predicate) const override; + + Result> VisitFullTextMatch( + const std::shared_ptr& match, bool needs_scores = false) override; Result> VisitVectorSearch( const std::shared_ptr& search) override; - Result> VisitFullTextSearch( + Result> VisitFullTextSearch( const std::shared_ptr& search) override; private: + const MockSearchFileIndex::ScoredRows& GetRows(const std::string& query) const; + + Result> MakeResult(const std::map& selected, + bool needs_scores) const; + + Result> MakeScoredResult( + const std::map& selected) const; + std::string field_name_; SearchIndexType index_type_; MockSearchFileIndex::ScoredRows rows_; + MockSearchFileIndex::QueryMatches queries_; + std::shared_ptr statistics_; }; class MockSearchFileIndexer final : public FileIndexer { public: - explicit MockSearchFileIndexer(SearchIndexType index_type); + MockSearchFileIndexer(SearchIndexType index_type, + std::shared_ptr statistics); + + std::shared_ptr CreateEmptyReader() const override; Result> CreateReader( ::ArrowSchema* c_schema, int64_t start, int64_t length, @@ -108,32 +159,50 @@ class MockSearchFileIndexer final : public FileIndexer { private: SearchIndexType index_type_; + std::shared_ptr statistics_; }; class MockVectorSearchFileIndexFactory final : public FileIndexerFactory { public: + explicit MockVectorSearchFileIndexFactory( + std::shared_ptr statistics = nullptr) + : statistics_(std::move(statistics)) {} + const char* Identifier() const override; Result> Create( const std::map& options) const override; + + private: + std::shared_ptr statistics_; }; class MockFullTextSearchFileIndexFactory final : public FileIndexerFactory { public: + explicit MockFullTextSearchFileIndexFactory( + std::shared_ptr statistics = nullptr) + : statistics_(std::move(statistics)) {} + const char* Identifier() const override; Result> Create( const std::map& options) const override; + + private: + std::shared_ptr statistics_; }; /// Reuse the mock search backend while requiring the exact options supplied to its factory. class OptionsCheckingMockSearchFileIndexFactory final : public FileIndexerFactory { public: - OptionsCheckingMockSearchFileIndexFactory(std::string identifier, SearchIndexType index_type, - std::map expected_options) + OptionsCheckingMockSearchFileIndexFactory( + std::string identifier, SearchIndexType index_type, + std::map expected_options, + std::shared_ptr statistics = nullptr) : identifier_(std::move(identifier)), index_type_(index_type), - expected_options_(std::move(expected_options)) {} + expected_options_(std::move(expected_options)), + statistics_(std::move(statistics)) {} const char* Identifier() const override { return identifier_.c_str(); @@ -146,6 +215,7 @@ class OptionsCheckingMockSearchFileIndexFactory final : public FileIndexerFactor std::string identifier_; SearchIndexType index_type_; std::map expected_options_; + std::shared_ptr statistics_; }; } // namespace paimon::test diff --git a/test/inte/predicate_abi_inte_test.cpp b/test/inte/predicate_abi_inte_test.cpp index 023400593..c1de62f83 100644 --- a/test/inte/predicate_abi_inte_test.cpp +++ b/test/inte/predicate_abi_inte_test.cpp @@ -54,7 +54,6 @@ #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate.h" #include "paimon/predicate/predicate_builder.h" -#include "paimon/predicate/predicate_utils.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -90,9 +89,13 @@ TEST(PredicateAbiInteTest, FullTextMatchCastsToLeafAcrossModule) { ASSERT_TRUE(leaf->Literals().empty()); ASSERT_EQ(leaf->GetFunction().GetType(), Function::Type::FULL_TEXT_MATCH); auto reader = std::make_shared(); - auto result = PredicateUtils::VisitPredicate>(leaf, reader); + auto result = reader->VisitFullTextMatch(match); ASSERT_NOK_WITH_MSG(result, "This file index does not support complete full-text matching"); ASSERT_TRUE(result.status().IsNotImplemented()); + ASSERT_FALSE(reader->SupportsSearch(FileIndexReader::SearchType::FULL_TEXT_MATCH)); + ASSERT_FALSE(reader->EstimateSelectivity(leaf)); + ASSERT_NOK_WITH_MSG(reader->VisitFullTextMatch(match, /*needs_scores=*/true), + "This file index does not support complete full-text matching"); } // A compound predicate built in libpaimon.so must survive dynamic_pointer_cast diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index dbbdf63f6..50772b438 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -814,15 +814,16 @@ TEST_P(WriteAndReadInteTest, TestLuminaSearchAfterRenameAcrossSchemaVersions) { std::map{}); ReadContextBuilder builder(path); builder.SetOptions(current_options) - .SetReadFieldNames({"id", "_INDEX_SCORE"}) - .SetVectorSearch(search); + .SetReadFieldNames({"id", "_vec_INDEX_SCORE"}) + .SetVectorSearch(search) + .EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); ASSERT_OK_AND_ASSIGN(auto read, TableRead::Create(std::move(context))); ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(std::move(reader))); auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("id", arrow::int32()), - arrow::field("_INDEX_SCORE", arrow::float32())}); + arrow::field("_vec_INDEX_SCORE", arrow::float32())}); ASSERT_TRUE(actual); ASSERT_TRUE(actual->type()->Equals(expected_type)); auto expected = std::make_shared( @@ -903,7 +904,7 @@ TEST_P(WriteAndReadInteTest, TestLuminaVectorIndexAfterCompaction) { } auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("id", arrow::int32()), - arrow::field("_INDEX_SCORE", arrow::float32())}); + arrow::field("_embedding_INDEX_SCORE", arrow::float32())}); auto read_search = [&](int32_t limit) -> Result> { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, InnerScan(options)); auto search = std::make_shared( @@ -911,8 +912,9 @@ TEST_P(WriteAndReadInteTest, TestLuminaVectorIndexAfterCompaction) { VectorSearch::DistanceType::EUCLIDEAN, std::map{}); ReadContextBuilder builder(path); builder.SetOptions(options) - .SetReadFieldNames({"id", "_INDEX_SCORE"}) - .SetVectorSearch(search); + .SetReadFieldNames({"id", "_embedding_INDEX_SCORE"}) + .SetVectorSearch(search) + .EnablePreciseFileIndexEvaluate(true); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, TableRead::Create(std::move(context))); @@ -1031,16 +1033,18 @@ TEST_P(WriteAndReadInteTest, TestAppendWithLuminaVectorIndex) { /*predicate=*/nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); ReadContextBuilder builder(PathUtil::JoinPath(root, "foo.db/bar")); - builder.SetReadFieldNames({"id", "_INDEX_SCORE"}).SetVectorSearch(search); + builder.SetReadFieldNames({"id", "_embedding_INDEX_SCORE"}) + .SetVectorSearch(search) + .EnablePreciseFileIndexEvaluate(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read, TableRead::Create(std::move(context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, ReadResultCollector::CollectResult(std::move(reader))); - auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), - arrow::field("id", arrow::int32()), - arrow::field("_INDEX_SCORE", arrow::float32())}); + auto expected_type = arrow::struct_( + {arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("id", arrow::int32()), + arrow::field("_embedding_INDEX_SCORE", arrow::float32())}); auto expected = arrow::ipc::internal::json::ArrayFromJSON(expected_type, R"([[0, 10, 4.0], [0, 12, 0.0], diff --git a/test/inte/write_and_read_inte_with_index_test.cpp b/test/inte/write_and_read_inte_with_index_test.cpp index efda3de03..7a2c64295 100644 --- a/test/inte/write_and_read_inte_with_index_test.cpp +++ b/test/inte/write_and_read_inte_with_index_test.cpp @@ -56,12 +56,14 @@ #include "paimon/data/timestamp.h" #include "paimon/defs.h" #include "paimon/factories/factory_creator.h" -#include "paimon/fs/local/local_file_system.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" #include "paimon/metrics.h" +#include "paimon/predicate/full_text_match.h" +#include "paimon/predicate/full_text_search.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" +#include "paimon/predicate/vector_search.h" #include "paimon/read_context.h" #include "paimon/reader/batch_reader.h" #include "paimon/record_batch.h" @@ -72,6 +74,7 @@ #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_index_path_factory.h" +#include "paimon/testing/mock/mock_search_file_index.h" #include "paimon/testing/utils/io_exception_helper.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/test_helper.h" @@ -126,8 +129,12 @@ class WriteAndReadInteWithIndexTest embedded_index_ = RandomNumber(0, 1) == 0; RecordProperty("index_storage", embedded_index_ ? "embedded" : "external"); } + auto factory = std::make_unique(statistics_); + FactoryCreator::GetInstance()->Register(statistics_index_type_, factory.release()); + } + void TearDown() override { + FactoryCreator::GetInstance()->TEST_Unregister(statistics_index_type_); } - void TearDown() override {} void CheckResult(const std::string& table_path, const std::vector> splits, @@ -1088,6 +1095,97 @@ class WriteAndReadInteWithIndexTest ASSERT_EQ(1, (*split)->DataFiles()[1]->schema_id); } + private: + std::string PkDvTablePath() const { + return data_source_ == IndexDataSource::Java + ? GetDataDir() + "/" + file_format_ + "/pk_with_bitmap_dv.db/pk_with_bitmap_dv/" + : dir_->Str() + "/foo.db/bar"; + } + + void CreatePkDvSplit(std::shared_ptr* split, + const std::shared_ptr& embedded_index = nullptr) const { + std::string path = PkDvTablePath(); + if (data_source_ != IndexDataSource::Java) { + std::vector read_fields = { + SpecialFields::ValueKind(), + DataField(0, arrow::field("key", arrow::int32())), + DataField(1, arrow::field("f0", arrow::utf8())), + DataField(2, arrow::field("f1", arrow::int32())), + DataField(3, arrow::field("f2", arrow::int32())), + DataField(4, arrow::field("f3", arrow::float64()))}; + ASSERT_NO_FATAL_FAILURE(WriteCppTableWithDv(read_fields, &path, split)); + if (embedded_index) { + (*split)->DataFiles()[0]->embedded_index = embedded_index; + } + return; + } + const bool is_orc = file_format_ == "orc"; + const std::string file_name = is_orc + ? "data-0da0dc51-797c-4c82-9b08-269b2ce400dc-0.orc" + : "data-bfa36b28-40e7-4026-893b-d23116973174-0.parquet"; + const std::string deletion_file_path = + path + (is_orc ? "index/index-73ec72e9-489e-49d9-a96c-f6db9b56fa07-0" + : "index/index-daed25d2-8e68-45ba-aaad-ff342666ef27-0"); + auto file = std::make_shared( + file_name, /*file_size=*/1001, /*row_count=*/8, BinaryRow::EmptyRow(), + BinaryRow::EmptyRow(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/0, /*max_sequence_number=*/7, /*schema_id=*/0, /*level=*/5, + std::vector>{file_name + ".index"}, Timestamp(0ll, 0), + /*delete_row_count=*/0, embedded_index, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, + /*column_max_sequence_numbers=*/std::nullopt); + // The fixture's deletion vector removes physical rows 0 (Alice) and 4 (Lucy). + DeletionFile deletion_file(deletion_file_path, /*offset=*/1, /*length=*/24, + /*cardinality=*/2); + DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, path + "bucket-0/", + {file}); + ASSERT_OK_AND_ASSIGN(*split, builder.WithSnapshot(4) + .WithDataDeletionFiles({deletion_file}) + .IsStreaming(false) + .RawConvertible(true) + .Build()); + } + + void CheckPkDvPreciseResult( + const std::shared_ptr& split, const arrow::FieldVector& fields, + const std::string& json, const std::shared_ptr& predicate, + const std::shared_ptr& vector_search = nullptr, + const std::shared_ptr& full_text_search = nullptr) const { + const std::string path = PkDvTablePath(); + arrow::FieldVector output_fields = {SpecialFields::ValueKind().ArrowField()}; + output_fields.insert(output_fields.end(), fields.begin(), fields.end()); + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow::struct_(output_fields), + {json}, &expected) + .ok()); + std::vector field_names; + for (const auto& field : fields) { + field_names.push_back(field->name()); + } + ReadContextBuilder builder(path); + builder.EnablePreciseFileIndexEvaluate(true) + .EnablePredicateFilter(false) + .SetReadFieldNames(field_names) + .SetPredicate(predicate) + .SetVectorSearch(vector_search) + .SetFullTextSearch(full_text_search) + .AddOption("read.batch-size", "1") + .AddOption("test.enable-adaptive-prefetch-strategy", "false") + .EnableLateMaterializing(false); + if (enable_prefetch_) { + builder.EnablePrefetch(true).SetPrefetchBatchCount(3); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(split)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + ReadResultCollector::CollectResult(std::move(reader))); + ASSERT_TRUE(actual); + ASSERT_TRUE(actual->Equals(expected)) << actual->ToString(); + } + protected: std::string file_format_; bool enable_prefetch_ = false; @@ -1095,6 +1193,9 @@ class WriteAndReadInteWithIndexTest IndexDataSource data_source_ = IndexDataSource::Java; std::shared_ptr pool_; std::unique_ptr dir_; + std::string statistics_index_type_ = "read-inte-statistics-full-text"; + std::shared_ptr statistics_ = + std::make_shared(); }; std::vector GetFileFormatsForWriteAndReadInteWithIndexTest() { @@ -1626,46 +1727,9 @@ TEST_P(WriteAndReadInteWithIndexTest, TestBitmapIndexWithDv) { DataField(4, arrow::field("f3", arrow::float64()))}; std::shared_ptr arrow_data_type = DataField::ConvertDataFieldsToArrowStructType(read_fields); - std::string path; + const std::string path = PkDvTablePath(); std::shared_ptr split; - if (data_source_ != IndexDataSource::Java) { - ASSERT_NO_FATAL_FAILURE(WriteCppTableWithDv(read_fields, &path, &split)); - } else { - path = GetDataDir() + "/" + file_format_ + "/pk_with_bitmap_dv.db/pk_with_bitmap_dv/"; - std::string file_name, deletion_file_path; - if (file_format_ == "orc") { - file_name = "data-0da0dc51-797c-4c82-9b08-269b2ce400dc-0.orc"; - deletion_file_path = path + "/index/index-73ec72e9-489e-49d9-a96c-f6db9b56fa07-0"; - } else if (file_format_ == "parquet") { - file_name = "data-bfa36b28-40e7-4026-893b-d23116973174-0.parquet"; - deletion_file_path = path + "/index/index-daed25d2-8e68-45ba-aaad-ff342666ef27-0"; - } - - auto data_file_meta = std::make_shared( - file_name, /*file_size=*/1001, - /*row_count=*/8, /*min_key=*/BinaryRow::EmptyRow(), - /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(), - /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0, - /*max_sequence_number=*/7, /*schema_id=*/0, - /*level=*/5, - /*extra_files=*/ - std::vector>({file_name + ".index"}), - /*creation_time=*/Timestamp(0ll, 0), /*delete_row_count=*/0, - /*embedded_index=*/nullptr, FileSource::Append(), - /*value_stats_cols=*/std::nullopt, - /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, - /*write_cols=*/std::nullopt, - /*column_max_sequence_numbers=*/std::nullopt); - DeletionFile deletion_file(deletion_file_path, - /*offset=*/1, /*length=*/24, /*cardinality=*/2); - DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, - /*bucket_path=*/path + "bucket-0/", {data_file_meta}); - ASSERT_OK_AND_ASSIGN(split, builder.WithSnapshot(4) - .WithDataDeletionFiles({deletion_file}) - .IsStreaming(false) - .RawConvertible(true) - .Build()); - } + ASSERT_NO_FATAL_FAILURE(CreatePkDvSplit(&split)); { // test with non predicate std::shared_ptr expected_array; @@ -1723,6 +1787,138 @@ TEST_P(WriteAndReadInteWithIndexTest, TestBitmapIndexWithDv) { } } +TEST_P(WriteAndReadInteWithIndexTest, TestPkDvPreciseScalarPredicates) { + std::shared_ptr split; + ASSERT_NO_FATAL_FAILURE(CreatePkDvSplit(&split)); + const arrow::FieldVector fields = {arrow::field("key", arrow::int32()), + arrow::field("f0", arrow::utf8()), + arrow::field("f3", arrow::float64())}; + auto name = + PredicateBuilder::Equal(1, "f0", FieldType::STRING, Literal(FieldType::STRING, "Bob", 3)); + auto value = PredicateBuilder::GreaterThan(2, "f3", FieldType::DOUBLE, Literal(15.0)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr conjunction, + PredicateBuilder::And({name, value})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr disjunction, + PredicateBuilder::Or({name, value})); + CheckPkDvPreciseResult(split, fields, R"([[0,5,"Bob",16.1]])", conjunction); + CheckPkDvPreciseResult( + split, fields, + R"([[0,1,"Bob",12.1],[0,5,"Bob",16.1],[0,6,"Tony",17.1],[0,7,"Alice",18.1]])", disjunction); + CheckPkDvPreciseResult(split, fields, + R"([[0,1,"Bob",12.1],[0,2,"Emily",13.1],[0,3,"Tony",14.1], + [0,5,"Bob",16.1],[0,6,"Tony",17.1],[0,7,"Alice",18.1]])", + /*predicate=*/nullptr); +} + +TEST_P(WriteAndReadInteWithIndexTest, TestPkDvMixedFilePredicatesAndScores) { + const std::string query = R"({"match":{"query":"names"}})"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr index, + MockSearchFileIndex::MakeQueryIndex( + "f0", MockSearchFileIndex::FULL_TEXT_IDENTIFIER, + {{query, {{{0, 0.875f}, {4, 0.75f}, {7, 0.5f}, {1, 0.25f}, {6, 0.125f}}, 0.5}}}, + pool_)); + std::shared_ptr split; + ASSERT_NO_FATAL_FAILURE(CreatePkDvSplit(&split, index)); + { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr text, + PredicateBuilder::FullTextMatch(2, std::make_shared("f0", query))); + auto scalar = PredicateBuilder::GreaterThan(3, "f3", FieldType::DOUBLE, Literal(15.0)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({text, scalar})); + CheckPkDvPreciseResult( + split, + {arrow::field("_f0_INDEX_SCORE", arrow::float32()), arrow::field("key", arrow::int32()), + arrow::field("f0", arrow::utf8()), arrow::field("f3", arrow::float64())}, + R"([[0,0.125,6,"Tony",17.1],[0,0.5,7,"Alice",18.1]])", predicate); + } + { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr text, + PredicateBuilder::FullTextMatch(1, std::make_shared("f0", query))); + auto scalar = PredicateBuilder::GreaterThan(2, "f3", FieldType::DOUBLE, Literal(15.0)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::Or({text, scalar})); + CheckPkDvPreciseResult( + split, + {arrow::field("key", arrow::int32()), arrow::field("f0", arrow::utf8()), + arrow::field("f3", arrow::float64())}, + R"([[0,1,"Bob",12.1],[0,5,"Bob",16.1],[0,6,"Tony",17.1],[0,7,"Alice",18.1]])", + predicate); + } +} + +TEST_P(WriteAndReadInteWithIndexTest, TestPkDvFullTextTopKFiltersBeforeLimit) { + const std::string query = R"({"match":{"query":"ranked"}})"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr index, + MockSearchFileIndex::MakeQueryIndex("f0", statistics_index_type_, + {{query, + {{{0, 1.0f}, + {4, 0.875f}, + {7, 0.75f}, + {5, 0.625f}, + {2, 0.5f}, + {1, 0.375f}, + {3, 0.25f}, + {6, 0.125f}}, + std::nullopt}}}, + pool_)); + auto search = std::make_shared("f0", query, /*limit=*/2); + auto predicate = PredicateBuilder::LessThan(2, "f3", FieldType::DOUBLE, Literal(17.0)); + std::shared_ptr split; + ASSERT_NO_FATAL_FAILURE(CreatePkDvSplit(&split, index)); + // Rows 0 and 4 are deleted, and row 7 fails the scalar filter. All three rank ahead of + // the surviving TopK, whose scores must be reordered with the physical output rows. + CheckPkDvPreciseResult( + split, + {arrow::field("key", arrow::int32()), arrow::field("_f0_INDEX_SCORE", arrow::float32()), + arrow::field("f3", arrow::float64())}, + R"([[0,2,0.5,13.1],[0,5,0.625,16.1]])", predicate, /*vector_search=*/nullptr, search); + ASSERT_EQ(statistics_->executions.size(), 1); + ASSERT_TRUE(statistics_->executions.front().needs_scores); + ASSERT_FALSE(search->include_row_ids); +} + +TEST_P(WriteAndReadInteWithIndexTest, TestPkDvFullTextAndVectorTopKScoreAlignment) { + const std::string query = R"({"match":{"query":"names"}})"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr text_payload, + MockSearchFileIndex::MakeQueryPayload( + {{query, {{{0, 1.0f}, {4, 0.875f}, {2, 0.75f}, {6, 0.5f}, {7, 0.25f}}, 0.5}}}, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr vector_payload, + MockSearchFileIndex::MakePayload({{0, 0.125f}, + {4, 0.25f}, + {5, 0.375f}, + {7, 0.5f}, + {2, 0.625f}, + {6, 0.75f}, + {1, 0.875f}, + {3, 1.0f}}, + pool_)); + FileIndexFormat::ColumnIndexes indexes; + indexes["f0"][MockSearchFileIndex::FULL_TEXT_IDENTIFIER] = text_payload; + indexes["f3"][MockSearchFileIndex::VECTOR_IDENTIFIER] = vector_payload; + ASSERT_OK_AND_ASSIGN(std::shared_ptr index, + MockSearchFileIndex::Serialize(indexes, pool_)); + std::shared_ptr split; + ASSERT_NO_FATAL_FAILURE(CreatePkDvSplit(&split, index)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr text, + PredicateBuilder::FullTextMatch(2, std::make_shared("f0", query))); + auto search = std::make_shared( + "f3", /*limit=*/2, std::vector{1.0f}, nullptr, nullptr, + VectorSearch::DistanceType::EUCLIDEAN, std::map{}); + // The DV excludes rows 0 and 4 and FullTextMatch excludes row 5 before vector TopK. + // The vector field itself is not projected, and both score columns follow the final rows. + CheckPkDvPreciseResult( + split, + {arrow::field("_f3_INDEX_SCORE", arrow::float32()), arrow::field("key", arrow::int32()), + arrow::field("f0", arrow::utf8()), arrow::field("_f0_INDEX_SCORE", arrow::float32())}, + R"([[0,0.625,2,"Emily",0.75],[0,0.5,7,"Alice",0.25]])", text, search); + ASSERT_FALSE(search->pre_filter); +} + TEST_P(WriteAndReadInteWithIndexTest, TestWithAlterTable) { std::vector read_fields = {SpecialFields::ValueKind(), DataField(1, arrow::field("f1", arrow::int64())), From 8d822677baa9e53753e0d29bbb70de1a6491e2b7 Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Wed, 4 Nov 2026 20:09:38 +0800 Subject: [PATCH 2/5] fix comment --- docs/source/user_guide/file_index.rst | 15 +- include/paimon/file_index/file_index_reader.h | 7 +- include/paimon/read_context.h | 5 +- src/paimon/CMakeLists.txt | 4 +- .../empty/empty_file_index_reader.h | 4 +- .../empty/empty_file_index_reader_test.cpp | 6 +- .../common/file_index/file_index_format.cpp | 3 +- src/paimon/common/file_index/file_indexer.cpp | 4 +- ...ete_index_score_file_batch_reader_test.cpp | 6 +- src/paimon/core/io/file_index_evaluator.cpp | 9 +- src/paimon/core/io/file_index_evaluator.h | 6 +- .../core/io/file_index_evaluator_test.cpp | 126 +++++----- .../core/operation/abstract_split_read.cpp | 14 +- .../core/operation/abstract_split_read.h | 10 +- .../operation/data_evolution_split_read.cpp | 8 +- src/paimon/core/operation/file_store_scan.cpp | 3 +- .../core/operation/internal_read_context.cpp | 11 + .../operation/internal_read_context_test.cpp | 48 ++++ ...ecutor.cpp => precise_file_index_read.cpp} | 232 ++++++++++-------- ...d_executor.h => precise_file_index_read.h} | 27 +- ...t.cpp => precise_file_index_read_test.cpp} | 48 ++-- .../core/operation/raw_file_split_read.cpp | 20 +- .../core/operation/raw_file_split_read.h | 15 +- .../realtime_offset_file_index_lookup.cpp | 7 +- .../write_and_read_inte_with_index_test.cpp | 36 +++ 25 files changed, 389 insertions(+), 285 deletions(-) rename src/paimon/core/operation/{file_index_read_executor.cpp => precise_file_index_read.cpp} (79%) rename src/paimon/core/operation/{file_index_read_executor.h => precise_file_index_read.h} (83%) rename src/paimon/core/operation/{file_index_read_executor_test.cpp => precise_file_index_read_test.cpp} (95%) diff --git a/docs/source/user_guide/file_index.rst b/docs/source/user_guide/file_index.rst index 30dd11f3e..44955689e 100644 --- a/docs/source/user_guide/file_index.rst +++ b/docs/source/user_guide/file_index.rst @@ -21,8 +21,17 @@ File Index Full-text predicates and root TopK ---------------------------------- -Enable ``ReadContextBuilder::EnablePreciseFileIndexEvaluate(true)`` to evaluate -complete predicates within each data file. The option defaults to ``false``. +Enable ``ReadContextBuilder::EnablePreciseFileIndexEvaluate(true)`` to precisely +filter rows at the file-index-query stage, before reading all columns requested by +the query from each data file. It evaluates the complete predicate to compute +exact matching row positions within that file. + +For a root full-text or vector TopK request, predicate filtering, row-range +restrictions and deletion vectors are applied before the TopK limit. Filtering after +TopK truncation could discard selected rows. The final row positions are then passed +to the data-file reader to restrict the read of all requested columns. + +The option defaults to ``false`` and requires ``file-index.read.enabled=true``. Full-text predicates and root TopK requests require this option; configuring them while it is disabled is rejected by ``ReadContextBuilder::Finish()``. @@ -91,4 +100,4 @@ An ordinary scalar predicate on the same field is allowed. For example, when projecting the score of ``match``, ``match AND (p OR q)`` is valid, while ``match OR p`` is rejected. Without that score projection, both OR branches and repeated full-text search fields are allowed. These checks use the effective -read projection, after resolving the read schema. \ No newline at end of file +read projection, after resolving the read schema. diff --git a/include/paimon/file_index/file_index_reader.h b/include/paimon/file_index/file_index_reader.h index 7799626f3..677617bbf 100644 --- a/include/paimon/file_index/file_index_reader.h +++ b/include/paimon/file_index/file_index_reader.h @@ -51,8 +51,11 @@ class PAIMON_EXPORT FileIndexReader : public FunctionVisitor EstimateSelectivity( const std::shared_ptr& predicate) const; diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index 5d04d89d2..4ca68dd9c 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -360,8 +360,11 @@ class PAIMON_EXPORT ReadContextBuilder { /// /// All referenced fields, including full-text match fields, must be projected in the read /// schema. - /// TODO(lisizhuo.lsz): The behavior here will be aligned with Java Paimon. + /// TODO(lisizhuo.lsz): Align with Java Paimon so fields referenced by predicates no longer need + /// to appear in the projection. /// Full-text matches can be combined with scalar predicates using PredicateBuilder::And/Or. + /// Predicates on virtual index score columns (_INDEX_SCORE or __INDEX_SCORE) are not + /// supported. /// /// @param predicate Shared pointer to the predicate for data filtering. /// @return Reference to this builder for method chaining. diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index da3e4d3e3..4a4c35092 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -413,7 +413,7 @@ set(PAIMON_CORE_SRCS core/operation/orphan_files_cleaner.cpp core/operation/orphan_files_cleaner_impl.cpp core/operation/raw_file_split_read.cpp - core/operation/file_index_read_executor.cpp + core/operation/precise_file_index_read.cpp core/operation/read_context.cpp core/operation/scan_context.cpp core/manifest/snapshot_live_manifest_entries.cpp @@ -994,7 +994,7 @@ if(PAIMON_BUILD_TESTS) core/operation/commit/realtime_deduplicate_dv_materializer_test.cpp core/operation/commit/sequence_snapshot_properties_test.cpp core/operation/key_value_file_store_write_test.cpp - core/operation/file_index_read_executor_test.cpp + core/operation/precise_file_index_read_test.cpp core/operation/internal_read_context_test.cpp core/operation/abstract_split_read_test.cpp core/operation/append_only_file_store_write_test.cpp diff --git a/src/paimon/common/file_index/empty/empty_file_index_reader.h b/src/paimon/common/file_index/empty/empty_file_index_reader.h index 8304e8c09..5de3bde93 100644 --- a/src/paimon/common/file_index/empty/empty_file_index_reader.h +++ b/src/paimon/common/file_index/empty/empty_file_index_reader.h @@ -38,7 +38,7 @@ class Literal; /// Vector and full-text searches return no matches. class EmptyFileIndexReader : public FileIndexReader { public: - explicit EmptyFileIndexReader(std::vector search_types = {}) + explicit EmptyFileIndexReader(std::vector search_types) : search_types_(std::move(search_types)) {} bool SupportsSearch(SearchType search_type) const override { @@ -47,7 +47,7 @@ class EmptyFileIndexReader : public FileIndexReader { } Result> VisitFullTextMatch( - const std::shared_ptr&, bool needs_scores = false) override { + const std::shared_ptr&, bool needs_scores) override { if (needs_scores) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, ScoredFileIndexResult::Create(RoaringBitmap32(), {})); diff --git a/src/paimon/common/file_index/empty/empty_file_index_reader_test.cpp b/src/paimon/common/file_index/empty/empty_file_index_reader_test.cpp index 5f3e6c34a..923d44b07 100644 --- a/src/paimon/common/file_index/empty/empty_file_index_reader_test.cpp +++ b/src/paimon/common/file_index/empty/empty_file_index_reader_test.cpp @@ -35,7 +35,7 @@ namespace paimon::test { TEST(EmptyFileIndexReaderTest, TestSimple) { Literal lit0(static_cast(0)); - EmptyFileIndexReader reader; + EmptyFileIndexReader reader(/*search_types=*/{}); ASSERT_FALSE(reader.SupportsSearch(FileIndexReader::SearchType::FULL_TEXT_MATCH)); ASSERT_FALSE(reader.SupportsSearch(FileIndexReader::SearchType::FULL_TEXT_SEARCH)); @@ -59,7 +59,7 @@ TEST(EmptyFileIndexReaderTest, TestSimple) { } TEST(EmptyFileIndexReaderTest, TestVectorSearchReturnsNoMatches) { - EmptyFileIndexReader reader; + EmptyFileIndexReader reader(/*search_types=*/{}); auto search = std::make_shared( "embedding", /*limit=*/1, std::vector{1.0f}, nullptr, nullptr, VectorSearch::DistanceType::EUCLIDEAN, std::map{}); @@ -81,7 +81,7 @@ TEST(EmptyFileIndexReaderTest, TestFullTextMatchAndScores) { ASSERT_TRUE(scored); ASSERT_TRUE(scored->IsEmpty()); ASSERT_TRUE(scored->GetScores().empty()); - ASSERT_OK_AND_ASSIGN(result, reader.VisitFullTextMatch(match)); + ASSERT_OK_AND_ASSIGN(result, reader.VisitFullTextMatch(match, /*needs_scores=*/false)); auto bitmap = std::dynamic_pointer_cast(result); ASSERT_TRUE(bitmap); ASSERT_OK_AND_ASSIGN(const RoaringBitmap32* positions, bitmap->GetBitmap()); diff --git a/src/paimon/common/file_index/file_index_format.cpp b/src/paimon/common/file_index/file_index_format.cpp index 8005eb464..2585f1b1d 100644 --- a/src/paimon/common/file_index/file_index_format.cpp +++ b/src/paimon/common/file_index/file_index_format.cpp @@ -114,7 +114,8 @@ class FileIndexFormatReaderImpl : public FileIndexFormat::Reader { FileIndexerFactory::Get(index_type, indexer_options)); if (offset_and_length.first == FileIndexFormat::EMPTY_INDEX_FLAG) { return file_indexer ? file_indexer->CreateEmptyReader() - : std::make_shared(); + : std::make_shared( + std::vector{}); } // assert(file_indexer); if (!file_indexer) { diff --git a/src/paimon/common/file_index/file_indexer.cpp b/src/paimon/common/file_index/file_indexer.cpp index fb5b9c8f4..6641af461 100644 --- a/src/paimon/common/file_index/file_indexer.cpp +++ b/src/paimon/common/file_index/file_indexer.cpp @@ -19,12 +19,14 @@ #include "paimon/file_index/file_indexer.h" +#include + #include "paimon/common/file_index/empty/empty_file_index_reader.h" namespace paimon { std::shared_ptr FileIndexer::CreateEmptyReader() const { - return std::make_shared(); + return std::make_shared(std::vector{}); } } // namespace paimon diff --git a/src/paimon/common/reader/complete_index_score_file_batch_reader_test.cpp b/src/paimon/common/reader/complete_index_score_file_batch_reader_test.cpp index 5a1def7ea..9852c2a60 100644 --- a/src/paimon/common/reader/complete_index_score_file_batch_reader_test.cpp +++ b/src/paimon/common/reader/complete_index_score_file_batch_reader_test.cpp @@ -249,7 +249,7 @@ TEST_F(CompleteIndexScoreFileBatchReaderTest, TestMultipleScoresFollowSelectedRo TEST_F(CompleteIndexScoreFileBatchReaderTest, TestMultipleScoresWithEmptyResult) { auto type = arrow::struct_( {arrow::field("score_a", arrow::float32()), arrow::field("score_b", arrow::float32())}); - for (const std::string& json : {"[]", "[[null, null]]"}) { + for (const std::string json : {"[]", "[[null, null]]"}) { auto data = arrow::ipc::internal::json::ArrayFromJSON(type, json).ValueOrDie(); auto inner = std::make_unique(data, type, RoaringBitmap32(), /*read_batch_size=*/1); @@ -267,7 +267,7 @@ TEST_F(CompleteIndexScoreFileBatchReaderTest, TestRejectsTooFewScoresForEachFiel {arrow::field("score_a", arrow::float32()), arrow::field("score_b", arrow::float32())}); auto data = arrow::ipc::internal::json::ArrayFromJSON(type, "[[null, null], [null, null]]") .ValueOrDie(); - for (const std::string& field_name : {"score_a", "score_b"}) { + for (const std::string field_name : {"score_a", "score_b"}) { auto inner = std::make_unique(data, type, /*read_batch_size=*/1); std::map> scores = {{"score_a", {1.0f, 2.0f}}, {"score_b", {3.0f, 4.0f}}}; @@ -287,7 +287,7 @@ TEST_F(CompleteIndexScoreFileBatchReaderTest, TestRejectsUnusedScoresForEachFiel auto type = arrow::struct_( {arrow::field("score_a", arrow::float32()), arrow::field("score_b", arrow::float32())}); auto data = arrow::ipc::internal::json::ArrayFromJSON(type, "[[null, null]]").ValueOrDie(); - for (const std::string& field_name : {"score_a", "score_b"}) { + for (const std::string field_name : {"score_a", "score_b"}) { auto inner = std::make_unique(data, type, /*read_batch_size=*/1); std::map> scores = {{"score_a", {1.0f}}, {"score_b", {3.0f}}}; diff --git a/src/paimon/core/io/file_index_evaluator.cpp b/src/paimon/core/io/file_index_evaluator.cpp index 98d9b6e21..bcb57c249 100644 --- a/src/paimon/core/io/file_index_evaluator.cpp +++ b/src/paimon/core/io/file_index_evaluator.cpp @@ -66,12 +66,11 @@ Result> FileIndexEvaluator::Create( const std::map& options, const std::shared_ptr& path_factory, const std::shared_ptr& file_meta, const std::shared_ptr& file_system, - bool only_use_embedding_index, const std::shared_ptr& pool) { + const std::shared_ptr& pool) { std::unique_ptr format_reader; if (file_meta->row_count >= 0 && file_meta->row_count <= RoaringBitmap32::MAX_VALUE) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, - ExtractIndexInputStream(only_use_embedding_index, path_factory, - file_meta, file_system)); + ExtractIndexInputStream(path_factory, file_meta, file_system)); if (input_stream) { PAIMON_ASSIGN_OR_RAISE(format_reader, FileIndexFormat::CreateReader(input_stream, pool, options)); @@ -214,7 +213,6 @@ Result> FileIndexEvaluator::Evaluate( } Result> FileIndexEvaluator::ExtractIndexInputStream( - bool only_use_embedding_index, const std::shared_ptr& data_file_path_factory, const std::shared_ptr& file_meta, const std::shared_ptr& file_system) { @@ -224,9 +222,6 @@ Result> FileIndexEvaluator::ExtractIndexInputStream return std::make_shared(embedded_index->data(), embedded_index->size()); } - if (only_use_embedding_index) { - return std::shared_ptr(); - } if (!data_file_path_factory || !file_system) { return Status::Invalid( "read process for FileIndexEvaluator must have data_file_path_factory and file_system"); diff --git a/src/paimon/core/io/file_index_evaluator.h b/src/paimon/core/io/file_index_evaluator.h index f046990dc..fed6d97a7 100644 --- a/src/paimon/core/io/file_index_evaluator.h +++ b/src/paimon/core/io/file_index_evaluator.h @@ -52,15 +52,14 @@ struct VectorSearch; class FileIndexEvaluator { public: /// Open one index container and reuse its field readers throughout a single file read. - /// Scan planning uses only embedded indexes; file reads also allow external indexes. + /// Uses the embedded index when present; otherwise, loads the external index if available. /// Files whose row counts cannot be represented by bitmap32 skip index loading. static Result> Create( const std::shared_ptr& data_schema, const std::map& options, const std::shared_ptr& path_factory, const std::shared_ptr& file_meta, - const std::shared_ptr& file_system, bool only_use_embedding_index, - const std::shared_ptr& pool); + const std::shared_ptr& file_system, const std::shared_ptr& pool); /// Evaluates a predicate tree for index pruning, reusing the cached field readers. /// This does not perform exact scalar filtering or reorder children by selectivity. @@ -89,7 +88,6 @@ class FileIndexEvaluator { Result>> GetReaders(const std::string& field_name); static Result> ExtractIndexInputStream( - bool only_use_embedding_index, const std::shared_ptr& data_file_path_factory, const std::shared_ptr& file_meta, const std::shared_ptr& file_system); diff --git a/src/paimon/core/io/file_index_evaluator_test.cpp b/src/paimon/core/io/file_index_evaluator_test.cpp index a6f51a15d..7972cd7e6 100644 --- a/src/paimon/core/io/file_index_evaluator_test.cpp +++ b/src/paimon/core/io/file_index_evaluator_test.cpp @@ -98,10 +98,10 @@ class FileIndexEvaluatorTest : public ::testing::Test { void CheckResult(const std::shared_ptr& data_file_path_factory, const std::shared_ptr& data_file_meta) const { - ASSERT_OK_AND_ASSIGN(auto evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), - data_file_path_factory, data_file_meta, fs_, - /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN( + auto evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), data_file_path_factory, + data_file_meta, fs_, pool_)); { auto predicate = PredicateBuilder::IsNull(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT); @@ -368,8 +368,7 @@ TEST_F(FileIndexEvaluatorTest, TestTimestampType) { ASSERT_OK_AND_ASSIGN( auto evaluator, FileIndexEvaluator::Create(data_schema, core_options_.ToMap(), data_file_path_factory, - data_file_meta, /*file_system=*/fs_, - /*only_use_embedding_index=*/false, pool_)); + data_file_meta, /*file_system=*/fs_, pool_)); ASSERT_OK_AND_ASSIGN(auto file_index_result, evaluator->Evaluate(predicate)); CheckResult(file_index_result, {0, 6}); } @@ -391,9 +390,9 @@ TEST_F(FileIndexEvaluatorTest, TestCreateRequiresExternalIndexDependencies) { /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt, /*column_max_sequence_numbers=*/std::nullopt); ASSERT_NOK_WITH_MSG( - FileIndexEvaluator::Create( - data_schema_, core_options_.ToMap(), /*data_file_path_factory=*/nullptr, data_file_meta, - /*file_system=*/nullptr, /*only_use_embedding_index=*/false, pool_), + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + /*data_file_path_factory=*/nullptr, data_file_meta, + /*file_system=*/nullptr, pool_), "read process for FileIndexEvaluator must have data_file_path_factory and file_system"); } @@ -459,10 +458,9 @@ class FileIndexSearchEvaluatorTest : public FileIndexEvaluatorTest, void CheckSearchError(SearchIndexType index_type, const std::shared_ptr& meta, const std::string& field, const std::string& message) const { - ASSERT_OK_AND_ASSIGN( - auto evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), path_factory_, meta, - fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + path_factory_, meta, fs_, pool_)); if (index_type == SearchIndexType::VECTOR) { ASSERT_NOK_WITH_MSG(evaluator->SearchVector(MakeVectorSearch(field)), message); } else { @@ -531,9 +529,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestVectorSearchReadsIndexWithScores) { ASSERT_OK_AND_ASSIGN(std::shared_ptr bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( - data_schema_, core_options_.ToMap(), path_factory_, - meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + path_factory_, meta, fs_, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, evaluator->SearchVector(MakeVectorSearch("embedding"))); ASSERT_TRUE(result); @@ -550,9 +548,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestFullTextSearchReadsIndexWithScores) { MockSearchFileIndex::MakeIndex("title", MockSearchFileIndex::FULL_TEXT_IDENTIFIER, {{3, 0.75f}, {1, 0.25f}}, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( - data_schema_, core_options_.ToMap(), path_factory_, - meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + path_factory_, meta, fs_, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, evaluator->SearchFullText(MakeFullTextSearch("title"))); ASSERT_TRUE(result); @@ -566,9 +564,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestEmptyVectorSearchIndex) { ASSERT_OK_AND_ASSIGN(std::shared_ptr bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( - data_schema_, core_options_.ToMap(), path_factory_, - meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + path_factory_, meta, fs_, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr vector_result, evaluator->SearchVector(MakeVectorSearch("embedding"))); ASSERT_TRUE(vector_result); @@ -582,9 +580,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestEmptyFullTextSearchIndex) { ASSERT_OK_AND_ASSIGN(std::shared_ptr bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( - data_schema_, core_options_.ToMap(), path_factory_, - meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + path_factory_, meta, fs_, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr text_result, evaluator->SearchFullText(MakeFullTextSearch("title"))); ASSERT_TRUE(text_result); @@ -605,18 +603,17 @@ TEST_P(FileIndexSearchEvaluatorTest, TestEvaluateNullPredicateAndMissingIndex) { indexes["title"]["bitmap"] = nullptr; ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( - data_schema_, core_options_.ToMap(), path_factory_, - meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + path_factory_, meta, fs_, pool_)); ASSERT_OK_AND_ASSIGN(auto result, evaluator->Evaluate(nullptr)); ASSERT_EQ(result, FileIndexResult::Remain()); meta->embedded_index.reset(); meta->extra_files.clear(); auto missing = PredicateBuilder::IsNull(2, "missing", FieldType::INT); - ASSERT_OK_AND_ASSIGN(evaluator, FileIndexEvaluator::Create( - data_schema_, core_options_.ToMap(), path_factory_, meta, - fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(evaluator, FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + path_factory_, meta, fs_, pool_)); ASSERT_OK_AND_ASSIGN(result, evaluator->Evaluate(missing)); ASSERT_EQ(result, FileIndexResult::Remain()); ASSERT_OK_AND_ASSIGN(result, evaluator->Evaluate(nullptr)); @@ -628,9 +625,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestEvaluateValidatesFieldsBeforeAndShortCi indexes["title"]["bitmap"] = nullptr; ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( - data_schema_, core_options_.ToMap(), path_factory_, - meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + path_factory_, meta, fs_, pool_)); auto empty = PredicateBuilder::IsNotNull(1, "title", FieldType::STRING); ASSERT_OK_AND_ASSIGN(auto result, evaluator->Evaluate(empty)); ASSERT_OK_AND_ASSIGN(bool remain, result->IsRemain()); @@ -648,9 +645,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestSessionReturnsQueryScores) { ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::MakeQueryIndex( "title", statistics_index_type_, queries, pool_)); ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( - data_schema_, core_options_.ToMap(), path_factory_, - meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + path_factory_, meta, fs_, pool_)); ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH)); ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_SEARCH)); ASSERT_TRUE(statistics_->executions.empty()); @@ -726,9 +723,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestSessionScalarAndSearchReadersCanCoexist indexes["title"][statistics_index_type_] = search_payload; ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( - data_schema_, core_options_.ToMap(), path_factory_, - meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + path_factory_, meta, fs_, pool_)); ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH)); ASSERT_NOK_WITH_MSG( evaluator->GetSearchReader("title", FileIndexReader::SearchType::VECTOR_SEARCH), @@ -758,9 +755,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestSessionPrecheckAndMissingScalarIndex) { ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::MakeIndex("title", statistics_index_type_, {{1, 0.5f}}, pool_)); ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( - data_schema_, core_options_.ToMap(), path_factory_, - meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + path_factory_, meta, fs_, pool_)); ASSERT_NOK_WITH_MSG( evaluator->GetSearchReader("embedding", FileIndexReader::SearchType::VECTOR_SEARCH), "No File Index reader supports"); @@ -779,9 +776,8 @@ TEST_P(FileIndexSearchEvaluatorTest, TestSessionPrecheckAndMissingScalarIndex) { meta->embedded_index.reset(); meta->extra_files.clear(); - ASSERT_OK_AND_ASSIGN(evaluator, FileIndexEvaluator::Create( - data_schema_, core_options_.ToMap(), path_factory_, meta, - fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(evaluator, FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + path_factory_, meta, fs_, pool_)); ASSERT_NOK_WITH_MSG( evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH), "has no File Index"); @@ -794,9 +790,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestEmptyScalarIndexDoesNotProvideSearchCap indexes["title"]["bitmap"] = nullptr; ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( - data_schema_, core_options_.ToMap(), path_factory_, - meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + path_factory_, meta, fs_, pool_)); ASSERT_NOK_WITH_MSG( evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH), "No File Index reader supports"); @@ -811,9 +807,8 @@ TEST_P(FileIndexSearchEvaluatorTest, TestEmptyScalarIndexDoesNotProvideSearchCap MockSearchFileIndex::MakePayload({{1, 0.5f}}, pool_)); ASSERT_OK_AND_ASSIGN(bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(evaluator, FileIndexEvaluator::Create( - data_schema_, core_options_.ToMap(), path_factory_, meta, - fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(evaluator, FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + path_factory_, meta, fs_, pool_)); ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH)); ASSERT_OK_AND_ASSIGN(auto result, evaluator->SearchFullText(MakeFullTextSearch("title"))); ASSERT_EQ(result->GetRowPositions(), RoaringBitmap32::From({1})); @@ -826,9 +821,9 @@ TEST_P(FileIndexSearchEvaluatorTest, TestEmptySearchIndexPreservesCapabilityAndS indexes["embedding"][MockSearchFileIndex::VECTOR_IDENTIFIER] = nullptr; ASSERT_OK_AND_ASSIGN(auto bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(auto meta, MakeIndex(bytes)); - ASSERT_OK_AND_ASSIGN(auto evaluator, FileIndexEvaluator::Create( - data_schema_, core_options_.ToMap(), path_factory_, - meta, fs_, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), + path_factory_, meta, fs_, pool_)); ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH)); ASSERT_OK(evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_SEARCH)); ASSERT_NOK_WITH_MSG( @@ -875,19 +870,17 @@ TEST_F(FileIndexEvaluatorTest, TestUnsupportedRowCountSkips32BitIndex) { /*column_max_sequence_numbers=*/std::nullopt); auto predicate = PredicateBuilder::IsNull(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT); - ASSERT_OK_AND_ASSIGN( - auto evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), nullptr, data_file_meta, - nullptr, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(auto evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), nullptr, + data_file_meta, nullptr, pool_)); ASSERT_OK_AND_ASSIGN(auto result, evaluator->Evaluate(predicate)); ASSERT_TRUE(result->IsRemain().value()); ASSERT_EQ(dynamic_cast(result.get()), nullptr); data_file_meta->row_count = -1; - ASSERT_OK_AND_ASSIGN( - evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), nullptr, data_file_meta, - nullptr, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), nullptr, + data_file_meta, nullptr, pool_)); ASSERT_OK_AND_ASSIGN(result, evaluator->Evaluate(predicate)); ASSERT_EQ(result, FileIndexResult::Remain()); @@ -895,10 +888,9 @@ TEST_F(FileIndexEvaluatorTest, TestUnsupportedRowCountSkips32BitIndex) { indexes["f2"]["bitmap"] = nullptr; ASSERT_OK_AND_ASSIGN(data_file_meta->embedded_index, MockSearchFileIndex::Serialize(indexes, pool_)); - ASSERT_OK_AND_ASSIGN( - evaluator, - FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), nullptr, data_file_meta, - nullptr, /*only_use_embedding_index=*/false, pool_)); + ASSERT_OK_AND_ASSIGN(evaluator, + FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), nullptr, + data_file_meta, nullptr, pool_)); auto missing = PredicateBuilder::IsNull(4, "missing", FieldType::INT); ASSERT_OK_AND_ASSIGN(result, evaluator->Evaluate(missing)); ASSERT_EQ(result, FileIndexResult::Remain()); diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp index 7f3360195..a9c1470bb 100644 --- a/src/paimon/core/operation/abstract_split_read.cpp +++ b/src/paimon/core/operation/abstract_split_read.cpp @@ -168,8 +168,9 @@ AbstractSplitRead::CreateRawFileReadersWithMetaImpl( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr predicate, predicate_provider(file)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr field_mapping_builder, - FieldMappingBuilder::Create(read_schema, context_->GetPartitionKeys(), - EnablePreciseFileIndexEvaluate() ? nullptr : predicate)); + FieldMappingBuilder::Create( + read_schema, context_->GetPartitionKeys(), + context_->EnablePreciseFileIndexEvaluate() ? nullptr : predicate)); return CreateRawFileReader(partition, file, field_mapping_builder.get(), dv_factory, row_ranges, data_file_path_factory, extra_format_options, attach_blob_context); @@ -236,13 +237,12 @@ Result> AbstractSplitRead::ApplyIndexAndDvReade "vectors in existing data files"); } std::shared_ptr file_index_result; - if (options_.FileIndexReadEnabled() && + if (predicate && options_.FileIndexReadEnabled() && dynamic_cast(deletion_vector.get()) == nullptr) { PAIMON_ASSIGN_OR_RAISE( std::unique_ptr evaluator, FileIndexEvaluator::Create(data_schema, file_index_options, data_file_path_factory, - file, options_.GetFileSystem(), - /*only_use_embedding_index=*/false, pool_)); + file, options_.GetFileSystem(), pool_)); PAIMON_ASSIGN_OR_RAISE(file_index_result, evaluator->Evaluate(predicate)); PAIMON_ASSIGN_OR_RAISE(bool is_remain, file_index_result->IsRemain()); if (!is_remain) { @@ -323,7 +323,7 @@ Result AbstractSplitRead::CreateRawFil const std::shared_ptr& data_file_path_factory, const std::map& extra_format_options, bool attach_blob_context) const { - if (EnablePreciseFileIndexEvaluate()) { + if (context_->EnablePreciseFileIndexEvaluate()) { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, CreatePreciseFileIndexReader( partition, file, field_mapping_builder, std::move(dv_factory), @@ -348,7 +348,7 @@ Result AbstractSplitRead::CreateRawFil std::unique_ptr reader, CreateFieldMappingReader(data_file_path, file, partition, std::move(reader_builder), field_mapping_builder, dv_factory, row_ranges, - data_file_path_factory)); + data_file_path_factory, /*exact_selection=*/nullptr)); return RawFileReaderWithMeta{file, std::move(reader), std::move(blob_context)}; } diff --git a/src/paimon/core/operation/abstract_split_read.h b/src/paimon/core/operation/abstract_split_read.h index 30747d5ea..8a1464c0f 100644 --- a/src/paimon/core/operation/abstract_split_read.h +++ b/src/paimon/core/operation/abstract_split_read.h @@ -56,10 +56,10 @@ class DataFilePathFactory; class DataSplitImpl; class Executor; class FieldMappingBuilder; -class FileIndexReadExecutor; class FileStorePathFactory; class InternalReadContext; class MemoryPool; +class PreciseFileIndexRead; class Predicate; struct BlobReadContext; struct DataFileMeta; @@ -86,10 +86,6 @@ class AbstractSplitRead : public SplitRead { const std::map& extra_format_options) const; protected: - virtual bool EnablePreciseFileIndexEvaluate() const { - return false; - } - virtual Result> CreatePreciseFileIndexReader( const BinaryRow& partition, const std::shared_ptr& file, const FieldMappingBuilder* field_mapping_builder, DeletionVector::Factory dv_factory, @@ -157,7 +153,7 @@ class AbstractSplitRead : public SplitRead { const std::optional>& write_cols); private: - friend class FileIndexReadExecutor; + friend class PreciseFileIndexRead; Result> CreateRawFileReadersWithMetaImpl( const BinaryRow& partition, const std::vector>& data_files, @@ -195,7 +191,7 @@ class AbstractSplitRead : public SplitRead { const FieldMappingBuilder* field_mapping_builder, DeletionVector::Factory dv_factory, const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory, - const RoaringBitmap32* exact_selection = nullptr) const; + const RoaringBitmap32* exact_selection) const; /// The returned field ID set contains MAP fields handled by shredding read plans. It tells /// FieldMappingReader to skip its generic selected-key filtering because the plans have already diff --git a/src/paimon/core/operation/data_evolution_split_read.cpp b/src/paimon/core/operation/data_evolution_split_read.cpp index 21eef0b0e..9facbad4c 100644 --- a/src/paimon/core/operation/data_evolution_split_read.cpp +++ b/src/paimon/core/operation/data_evolution_split_read.cpp @@ -586,8 +586,7 @@ Result> DataEvolutionSplitRead::SelectByFileIndex PAIMON_ASSIGN_OR_RAISE( std::unique_ptr evaluator, FileIndexEvaluator::Create(written_schema, file_index_options, data_file_path_factory, - file, options_.GetFileSystem(), - /*only_use_embedding_index=*/false, pool_)); + file, options_.GetFileSystem(), pool_)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index_result, evaluator->Evaluate(data_predicate)); PAIMON_ASSIGN_OR_RAISE(bool is_remain, index_result->IsRemain()); @@ -624,12 +623,11 @@ Result> DataEvolutionSplitRead::ApplyIndexAndDv const std::shared_ptr& data_file_path_factory, const std::map& file_index_options) const { std::shared_ptr file_index_result; - if (options_.FileIndexReadEnabled() && !file_indexes_evaluated_) { + if (predicate && options_.FileIndexReadEnabled() && !file_indexes_evaluated_) { PAIMON_ASSIGN_OR_RAISE( std::unique_ptr evaluator, FileIndexEvaluator::Create(data_schema, file_index_options, data_file_path_factory, - file, options_.GetFileSystem(), - /*only_use_embedding_index=*/false, pool_)); + file, options_.GetFileSystem(), pool_)); PAIMON_ASSIGN_OR_RAISE(file_index_result, evaluator->Evaluate(predicate)); PAIMON_ASSIGN_OR_RAISE(bool is_remain, file_index_result->IsRemain()); if (!is_remain) { diff --git a/src/paimon/core/operation/file_store_scan.cpp b/src/paimon/core/operation/file_store_scan.cpp index ceba23f0a..6a1a395b9 100644 --- a/src/paimon/core/operation/file_store_scan.cpp +++ b/src/paimon/core/operation/file_store_scan.cpp @@ -126,8 +126,7 @@ Result FileStoreScan::TestFileIndex(const std::shared_ptr& pred PAIMON_ASSIGN_OR_RAISE( std::unique_ptr evaluator, FileIndexEvaluator::Create(data_arrow_schema, file_index_options, /*path_factory=*/nullptr, - meta, /*file_system=*/nullptr, /*only_use_embedding_index=*/true, - pool_)); + meta, /*file_system=*/nullptr, pool_)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index_result, evaluator->Evaluate(data_predicate)); return index_result->IsRemain(); diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 3f6fc83c5..09661843e 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "arrow/api.h" @@ -47,6 +48,7 @@ #include "paimon/core/utils/nested_projection_utils.h" #include "paimon/predicate/compound_predicate.h" #include "paimon/predicate/full_text_search.h" +#include "paimon/predicate/predicate_utils.h" #include "paimon/predicate/vector_search.h" #include "paimon/rest/rest_api.h" #include "paimon/status.h" @@ -378,6 +380,15 @@ Result> InternalReadContext::Create( } // validate predicate if (context->GetPredicate()) { + std::set field_names; + PAIMON_RETURN_NOT_OK(PredicateUtils::GetAllNames(context->GetPredicate(), &field_names)); + for (const auto& field_name : field_names) { + if (field_name == SpecialFields::IndexScore().Name() || + SpecialFields::ParseIndexScoreFieldName(field_name)) { + return Status::Invalid(fmt::format( + "Predicates on virtual index score column '{}' are not supported", field_name)); + } + } PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( *read_schema, context->GetPredicate(), /*validate_field_idx=*/true)); } diff --git a/src/paimon/core/operation/internal_read_context_test.cpp b/src/paimon/core/operation/internal_read_context_test.cpp index ab8fab938..2147e8242 100644 --- a/src/paimon/core/operation/internal_read_context_test.cpp +++ b/src/paimon/core/operation/internal_read_context_test.cpp @@ -39,6 +39,7 @@ #include "paimon/fs/local/local_file_system.h" #include "paimon/predicate/full_text_match.h" #include "paimon/predicate/full_text_search.h" +#include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/predicate/vector_search.h" #include "paimon/status.h" @@ -778,6 +779,53 @@ TEST_F(InternalReadContextScoreTest, RejectsAmbiguousOrConditionalScoreProjectio } } +TEST_F(InternalReadContextScoreTest, RejectsPredicatesOnFullTextMatchScores) { + auto match = std::make_shared("F1", R"({"match":{"query":"apple"}})"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr text, + PredicateBuilder::FullTextMatch(0, match)); + const std::vector> score_predicates = { + PredicateBuilder::GreaterThan(1, "_F1_INDEX_SCORE", FieldType::FLOAT, Literal(0.5f)), + PredicateBuilder::IsNull(1, "_F1_INDEX_SCORE", FieldType::FLOAT), + PredicateBuilder::IsNotNull(1, "_F1_INDEX_SCORE", FieldType::FLOAT)}; + for (const auto& score_predicate : score_predicates) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({text, score_predicate})); + ReadContextBuilder builder("/tmp/unused-table-path"); + builder.EnablePreciseFileIndexEvaluate(true) + .SetReadFieldNames({"F1", "_F1_INDEX_SCORE"}) + .SetPredicate(predicate); + ASSERT_NOK_WITH_MSG( + Create(&builder), + "Predicates on virtual index score column '_F1_INDEX_SCORE' are not supported"); + } +} + +TEST_F(InternalReadContextScoreTest, RejectsNestedPredicatesOnRootSearchScores) { + auto search = + std::make_shared("F1", R"({"match":{"query":"apple"}})", /*limit=*/2); + auto scalar = PredicateBuilder::IsNotNull(0, "F1", FieldType::STRING); + auto score = PredicateBuilder::IsNull(1, "_F1_INDEX_SCORE", FieldType::FLOAT); + ASSERT_OK_AND_ASSIGN(std::shared_ptr child, PredicateBuilder::Or({scalar, score})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({scalar, child})); + ReadContextBuilder builder("/tmp/unused-table-path"); + builder.EnablePreciseFileIndexEvaluate(true) + .SetFullTextSearch(search) + .SetReadFieldNames({"F1", "_F1_INDEX_SCORE"}) + .SetPredicate(predicate); + ASSERT_NOK_WITH_MSG( + Create(&builder), + "Predicates on virtual index score column '_F1_INDEX_SCORE' are not supported"); +} + +TEST_F(InternalReadContextScoreTest, RejectsPredicatesOnUnnamedIndexScores) { + ReadContextBuilder builder("/tmp/unused-table-path"); + builder.SetPredicate(PredicateBuilder::IsNull(0, "_INDEX_SCORE", FieldType::FLOAT)); + ASSERT_NOK_WITH_MSG( + Create(&builder), + "Predicates on virtual index score column '_INDEX_SCORE' are not supported"); +} + TEST_F(InternalReadContextScoreTest, RejectsMissingScoreSource) { for (const auto& name : {"_F1_INDEX_SCORE", "_missing_INDEX_SCORE"}) { ReadContextBuilder builder("/tmp/unused-table-path"); diff --git a/src/paimon/core/operation/file_index_read_executor.cpp b/src/paimon/core/operation/precise_file_index_read.cpp similarity index 79% rename from src/paimon/core/operation/file_index_read_executor.cpp rename to src/paimon/core/operation/precise_file_index_read.cpp index aa389208c..4fcdca57e 100644 --- a/src/paimon/core/operation/file_index_read_executor.cpp +++ b/src/paimon/core/operation/precise_file_index_read.cpp @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "paimon/core/operation/file_index_read_executor.h" +#include "paimon/core/operation/precise_file_index_read.h" #include #include @@ -32,6 +32,7 @@ #include "fmt/format.h" #include "paimon/common/reader/complete_index_score_file_batch_reader.h" #include "paimon/common/reader/predicate_batch_reader.h" +#include "paimon/common/reader/reader_utils.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/object_utils.h" #include "paimon/common/utils/scope_guard.h" @@ -72,76 +73,12 @@ Result ResolveSearchFieldName( return iter->second; } -} // namespace - -Result> FileIndexReadExecutor::Create( - const RawFileSplitRead& split_read, const BinaryRow& partition, - const std::shared_ptr& file, const FieldMappingBuilder* field_mapping_builder, - const std::shared_ptr& path_factory, - const std::map& extra_format_options) { - std::unique_ptr executor(new FileIndexReadExecutor( - split_read, partition, file, field_mapping_builder, path_factory, extra_format_options)); - PAIMON_RETURN_NOT_OK(executor->Prepare()); - return executor; -} - -FileIndexReadExecutor::FileIndexReadExecutor( - const RawFileSplitRead& split_read, const BinaryRow& partition, - const std::shared_ptr& file, const FieldMappingBuilder* field_mapping_builder, - const std::shared_ptr& path_factory, - const std::map& extra_format_options) - : split_read_(split_read), - partition_(partition), - file_(file), - field_mapping_builder_(field_mapping_builder), - path_factory_(path_factory), - extra_format_options_(extra_format_options) {} - -Status FileIndexReadExecutor::Prepare() { - std::shared_ptr table_schema = split_read_.context_->GetTableSchema(); - if (file_->schema_id != table_schema->Id()) { - PAIMON_ASSIGN_OR_RAISE(table_schema, - split_read_.schema_manager_->ReadSchema(file_->schema_id)); - } - const auto data_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); - if (file_->row_count < 0 || file_->row_count > RoaringBitmap32::MAX_VALUE) { - return Status::NotImplemented( - "Precise file-index evaluation requires bitmap32 physical row positions"); - } - PAIMON_ASSIGN_OR_RAISE(std::vector data_fields, - DataField::ConvertArrowSchemaToDataFields(data_schema)); - PAIMON_ASSIGN_OR_RAISE( - std::vector read_fields, - DataField::ConvertArrowSchemaToDataFields(split_read_.context_->GetReadSchema())); - auto field_names = FieldMappingBuilder::CreateFieldNameMapping( - split_read_.context_->GetTableSchema()->Fields(), data_fields); - std::map index_options; - PAIMON_ASSIGN_OR_RAISE(index_options, - FileIndexOptions::RewriteFieldNames( - split_read_.context_->GetCoreOptions().ToMap(), field_names)); - PAIMON_ASSIGN_OR_RAISE(indexes_, FileIndexEvaluator::Create( - data_schema, index_options, path_factory_, file_, - split_read_.context_->GetCoreOptions().GetFileSystem(), - /*only_use_embedding_index=*/false, split_read_.pool_)); - - std::map read_by_name; - for (const auto& field : read_fields) { - read_by_name.emplace(field.Name(), field); - } - std::map> data_by_id; - for (size_t i = 0; i < data_fields.size(); ++i) { - data_by_id.emplace(data_fields[i].Id(), std::make_pair(i, data_fields[i])); - } - // Precheck the entire tree before an empty candidate set can short-circuit it. - PAIMON_RETURN_NOT_OK( - PreparePredicate(split_read_.context_->GetPredicate(), read_by_name, data_by_id)); - return PrepareSearch(field_names); -} - -Status FileIndexReadExecutor::PreparePredicate( +Status PreparePredicate( const std::shared_ptr& predicate, const std::map& read_by_name, - const std::map>& data_by_id) { + const std::map>& data_by_id, + const std::vector& partition_keys, FileIndexEvaluator* indexes, + std::map, std::shared_ptr>* mapped_leaves) { if (!predicate) { return Status::OK(); } @@ -152,7 +89,8 @@ Status FileIndexReadExecutor::PreparePredicate( fmt::format("unsupported compound file predicate {}", predicate->ToString())); } for (const auto& child : compound->Children()) { - PAIMON_RETURN_NOT_OK(PreparePredicate(child, read_by_name, data_by_id)); + PAIMON_RETURN_NOT_OK(PreparePredicate(child, read_by_name, data_by_id, partition_keys, + indexes, mapped_leaves)); } return Status::OK(); } @@ -161,9 +99,8 @@ Status FileIndexReadExecutor::PreparePredicate( return Status::Invalid("File predicate must contain leaf or compound predicates"); } bool full_text = leaf->GetFunction().GetType() == Function::Type::FULL_TEXT_MATCH; - if (!full_text && - ObjectUtils::Contains(split_read_.context_->GetPartitionKeys(), leaf->FieldName())) { - mapped_leaves_.emplace(leaf, nullptr); + if (!full_text && ObjectUtils::Contains(partition_keys, leaf->FieldName())) { + mapped_leaves->emplace(leaf, nullptr); return Status::OK(); } PAIMON_ASSIGN_OR_RAISE( @@ -178,40 +115,120 @@ Status FileIndexReadExecutor::PreparePredicate( if (full_text) { PAIMON_ASSIGN_OR_RAISE( [[maybe_unused]] std::shared_ptr reader, - indexes_->GetSearchReader(mapped_leaf->FieldName(), - FileIndexReader::SearchType::FULL_TEXT_MATCH)); + indexes->GetSearchReader(mapped_leaf->FieldName(), + FileIndexReader::SearchType::FULL_TEXT_MATCH)); } } - mapped_leaves_.emplace(leaf, std::move(mapped_leaf)); + mapped_leaves->emplace(leaf, std::move(mapped_leaf)); return Status::OK(); } -Status FileIndexReadExecutor::PrepareSearch(const std::map& field_names) { - if (split_read_.context_->GetVectorSearch()) { - vector_search_ = std::make_shared(*split_read_.context_->GetVectorSearch()); +Status PrepareSearch(const std::shared_ptr& context, + const std::map& field_names, + const std::string& file_name, FileIndexEvaluator* indexes, + std::shared_ptr* vector_search, + std::shared_ptr* full_text_search) { + if (context->GetVectorSearch()) { + *vector_search = std::make_shared(*context->GetVectorSearch()); PAIMON_ASSIGN_OR_RAISE( - vector_search_->field_name, - ResolveSearchFieldName(vector_search_->field_name, field_names, file_->file_name)); + (*vector_search)->field_name, + ResolveSearchFieldName((*vector_search)->field_name, field_names, file_name)); PAIMON_ASSIGN_OR_RAISE( [[maybe_unused]] std::shared_ptr reader, - indexes_->GetSearchReader(vector_search_->field_name, - FileIndexReader::SearchType::VECTOR_SEARCH)); + indexes->GetSearchReader((*vector_search)->field_name, + FileIndexReader::SearchType::VECTOR_SEARCH)); } - if (split_read_.context_->GetFullTextSearch()) { - full_text_search_ = - std::make_shared(*split_read_.context_->GetFullTextSearch()); + if (context->GetFullTextSearch()) { + *full_text_search = std::make_shared(*context->GetFullTextSearch()); PAIMON_ASSIGN_OR_RAISE( - full_text_search_->field_name, - ResolveSearchFieldName(full_text_search_->field_name, field_names, file_->file_name)); + (*full_text_search)->field_name, + ResolveSearchFieldName((*full_text_search)->field_name, field_names, file_name)); PAIMON_ASSIGN_OR_RAISE( [[maybe_unused]] std::shared_ptr reader, - indexes_->GetSearchReader(full_text_search_->field_name, - FileIndexReader::SearchType::FULL_TEXT_SEARCH)); + indexes->GetSearchReader((*full_text_search)->field_name, + FileIndexReader::SearchType::FULL_TEXT_SEARCH)); } return Status::OK(); } -Result FileIndexReadExecutor::CreateCandidates( +} // namespace + +Result> PreciseFileIndexRead::Create( + const RawFileSplitRead& split_read, const BinaryRow& partition, + const std::shared_ptr& file, const FieldMappingBuilder* field_mapping_builder, + const std::shared_ptr& path_factory, + const std::map& extra_format_options) { + std::shared_ptr table_schema = split_read.context_->GetTableSchema(); + if (file->schema_id != table_schema->Id()) { + PAIMON_ASSIGN_OR_RAISE(table_schema, + split_read.schema_manager_->ReadSchema(file->schema_id)); + } + const auto data_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + if (file->row_count < 0 || file->row_count > RoaringBitmap32::MAX_VALUE) { + return Status::NotImplemented( + "Precise file-index evaluation requires bitmap32 physical row positions"); + } + PAIMON_ASSIGN_OR_RAISE(std::vector data_fields, + DataField::ConvertArrowSchemaToDataFields(data_schema)); + PAIMON_ASSIGN_OR_RAISE( + std::vector read_fields, + DataField::ConvertArrowSchemaToDataFields(split_read.context_->GetReadSchema())); + auto field_names = FieldMappingBuilder::CreateFieldNameMapping( + split_read.context_->GetTableSchema()->Fields(), data_fields); + std::map index_options; + PAIMON_ASSIGN_OR_RAISE(index_options, + FileIndexOptions::RewriteFieldNames( + split_read.context_->GetCoreOptions().ToMap(), field_names)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr indexes, + FileIndexEvaluator::Create(data_schema, index_options, path_factory, file, + split_read.context_->GetCoreOptions().GetFileSystem(), + split_read.pool_)); + + std::map read_by_name; + for (const auto& field : read_fields) { + read_by_name.emplace(field.Name(), field); + } + // Field ID -> (zero-based position in data_fields, field definition from the file's schema). + std::map> data_by_id; + for (size_t i = 0; i < data_fields.size(); ++i) { + data_by_id.emplace(data_fields[i].Id(), std::make_pair(i, data_fields[i])); + } + std::map, std::shared_ptr> mapped_leaves; + // Precheck the entire tree before an empty candidate set can short-circuit it. + PAIMON_RETURN_NOT_OK(PreparePredicate(split_read.context_->GetPredicate(), read_by_name, + data_by_id, split_read.context_->GetPartitionKeys(), + indexes.get(), &mapped_leaves)); + std::shared_ptr vector_search; + std::shared_ptr full_text_search; + PAIMON_RETURN_NOT_OK(PrepareSearch(split_read.context_, field_names, file->file_name, + indexes.get(), &vector_search, &full_text_search)); + return std::unique_ptr(new PreciseFileIndexRead( + split_read, partition, file, field_mapping_builder, path_factory, extra_format_options, + std::move(indexes), mapped_leaves, vector_search, full_text_search)); +} + +PreciseFileIndexRead::PreciseFileIndexRead( + const RawFileSplitRead& split_read, const BinaryRow& partition, + const std::shared_ptr& file, const FieldMappingBuilder* field_mapping_builder, + const std::shared_ptr& path_factory, + const std::map& extra_format_options, + std::unique_ptr indexes, + const std::map, std::shared_ptr>& mapped_leaves, + const std::shared_ptr& vector_search, + const std::shared_ptr& full_text_search) + : split_read_(split_read), + partition_(partition), + file_(file), + field_mapping_builder_(field_mapping_builder), + path_factory_(path_factory), + extra_format_options_(extra_format_options), + indexes_(std::move(indexes)), + mapped_leaves_(mapped_leaves), + vector_search_(vector_search), + full_text_search_(full_text_search) {} + +Result PreciseFileIndexRead::CreateCandidates( const DeletionVector::Factory& dv_factory, const std::optional>& row_ranges) const { RoaringBitmap32 candidates; @@ -225,10 +242,6 @@ Result FileIndexReadExecutor::CreateCandidates( } if (dv_factory) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr dv, dv_factory(file_->file_name)); - if (dynamic_cast(dv.get())) { - return Status::NotImplemented( - "Precise file-index evaluation does not support bitmap64 deletion vectors"); - } if (auto bitmap = dynamic_cast(dv.get())) { candidates -= *bitmap->GetBitmap(); } else if (dv) { @@ -239,7 +252,7 @@ Result FileIndexReadExecutor::CreateCandidates( return candidates; } -Result FileIndexReadExecutor::Evaluate( +Result PreciseFileIndexRead::Evaluate( const std::shared_ptr& predicate, const RoaringBitmap32& candidates) const { if (!predicate || candidates.IsEmpty()) { return Evaluation{candidates, {}}; @@ -253,7 +266,7 @@ Result FileIndexReadExecutor::Evaluate( return Status::Invalid(fmt::format("unsupported file predicate {}", predicate->ToString())); } -Result FileIndexReadExecutor::EvaluateCompound( +Result PreciseFileIndexRead::EvaluateCompound( const std::shared_ptr& predicate, const RoaringBitmap32& candidates) const { const Function::Type type = predicate->GetFunction().GetType(); if (type == Function::Type::OR) { @@ -300,7 +313,7 @@ Result FileIndexReadExecutor::EvaluateCompoun return result; } -Result> FileIndexReadExecutor::EstimateSelectivity( +Result> PreciseFileIndexRead::EstimateSelectivity( const std::shared_ptr& predicate) const { if (!predicate) { return std::optional(); @@ -343,7 +356,7 @@ Result> FileIndexReadExecutor::EstimateSelectivity( return estimate; } -Result> FileIndexReadExecutor::EstimateLeafSelectivity( +Result> PreciseFileIndexRead::EstimateLeafSelectivity( const std::shared_ptr& predicate) const { auto iter = mapped_leaves_.find(predicate); if (iter == mapped_leaves_.end()) { @@ -355,7 +368,7 @@ Result> FileIndexReadExecutor::EstimateLeafSelectivity( return indexes_->EstimateSelectivity(iter->second); } -Result FileIndexReadExecutor::EvaluateLeaf( +Result PreciseFileIndexRead::EvaluateLeaf( const std::shared_ptr& predicate, const RoaringBitmap32& candidates) const { bool is_full_text = predicate->GetFunction().GetType() == Function::Type::FULL_TEXT_MATCH; bool needs_scores = @@ -368,6 +381,9 @@ Result FileIndexReadExecutor::EvaluateLeaf( return Status::Invalid("File predicate mapping was not prepared"); } if (iter->second) { + // TODO(lisizhuo.lsz): Pass candidates through FileIndexEvaluator::EvaluateLeaf to index + // readers so they can skip rows excluded by earlier predicates, row ranges, or deletion + // vectors. PAIMON_ASSIGN_OR_RAISE(index_result, indexes_->EvaluateLeaf(iter->second, needs_scores)); } if (!index_result) { @@ -407,7 +423,7 @@ Result FileIndexReadExecutor::EvaluateLeaf( return result; } -Result> FileIndexReadExecutor::CreateSelectedFileReader( +Result> PreciseFileIndexRead::CreateSelectedFileReader( const RoaringBitmap32& selection, const FieldMappingBuilder* field_mapping_builder) const { PAIMON_ASSIGN_OR_RAISE(std::string format, file_->FileFormat()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr builder, @@ -417,13 +433,15 @@ Result> FileIndexReadExecutor::CreateSelectedFi std::nullopt, path_factory_, &selection); } -Result FileIndexReadExecutor::ScanScalar( +Result PreciseFileIndexRead::ScanScalar( const std::shared_ptr& predicate, const RoaringBitmap32& input) const { auto field = split_read_.context_->GetReadSchema()->GetFieldByName(predicate->FieldName()); if (!field) { return Status::Invalid(fmt::format( "Scalar predicate field '{}' is missing from the read schema", predicate->FieldName())); } + // TODO(lisizhuo.lsz): Push the scalar predicate to use min/max statistics for pruning row + // groups/pages. PAIMON_ASSIGN_OR_RAISE( std::unique_ptr field_mapping_builder, FieldMappingBuilder::Create(arrow::schema({field}), @@ -443,6 +461,8 @@ Result FileIndexReadExecutor::ScanScalar( if (BatchReader::IsEofBatch(batch)) { break; } + ScopeGuard release_batch( + [&batch]() { ReaderUtils::ReleaseReadBatch(std::move(batch.first)); }); for (auto iter = batch.second.Begin(); iter != batch.second.End(); ++iter) { int32_t batch_row = *iter; if (batch_row < 0 || batch_row >= batch.first.first->length) { @@ -458,7 +478,7 @@ Result FileIndexReadExecutor::ScanScalar( return matches; } -Status FileIndexReadExecutor::ApplyRootSearch(Evaluation* evaluated) const { +Status PreciseFileIndexRead::ApplyRootSearch(Evaluation* evaluated) const { std::shared_ptr root_scores; std::string field_name; if (vector_search_) { @@ -489,7 +509,7 @@ Status FileIndexReadExecutor::ApplyRootSearch(Evaluation* evaluated) const { return Status::OK(); } -Result> FileIndexReadExecutor::AlignScores( +Result> PreciseFileIndexRead::AlignScores( const std::shared_ptr& source, const RoaringBitmap32& rows) { if (!source) { return Status::Invalid("Projected search source did not return scores"); @@ -509,7 +529,7 @@ Result> FileIndexReadExecutor::AlignScores( return aligned; } -Result>> FileIndexReadExecutor::CreateOutputScores( +Result>> PreciseFileIndexRead::CreateOutputScores( const Evaluation& evaluated) const { std::map> output_scores; for (const auto& [position, source] : split_read_.context_->GetIndexScoreBindings()) { @@ -526,7 +546,7 @@ Result>> FileIndexReadExecutor::CreateO return output_scores; } -Result> FileIndexReadExecutor::CreateReader( +Result> PreciseFileIndexRead::CreateReader( const DeletionVector::Factory& dv_factory, const std::optional>& row_ranges) const { PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 candidates, CreateCandidates(dv_factory, row_ranges)); diff --git a/src/paimon/core/operation/file_index_read_executor.h b/src/paimon/core/operation/precise_file_index_read.h similarity index 83% rename from src/paimon/core/operation/file_index_read_executor.h rename to src/paimon/core/operation/precise_file_index_read.h index ed1885876..3c373620c 100644 --- a/src/paimon/core/operation/file_index_read_executor.h +++ b/src/paimon/core/operation/precise_file_index_read.h @@ -36,7 +36,6 @@ namespace paimon { class BinaryRow; class CompoundPredicate; -class DataField; class DataFilePathFactory; class FieldMappingBuilder; class FileIndexEvaluator; @@ -49,15 +48,15 @@ struct FullTextSearch; struct VectorSearch; /// Evaluates a file predicate and root search before opening the selected output rows. -class FileIndexReadExecutor { +class PreciseFileIndexRead { public: - static Result> Create( + static Result> Create( const RawFileSplitRead& split_read, const BinaryRow& partition, const std::shared_ptr& file, const FieldMappingBuilder* field_mapping_builder, const std::shared_ptr& path_factory, const std::map& extra_format_options); - /// Finishes evaluation synchronously. The returned reader does not retain this executor. + /// Finishes evaluation synchronously. The returned reader does not retain this instance. Result> CreateReader( const DeletionVector::Factory& dv_factory, const std::optional>& row_ranges) const; @@ -69,17 +68,17 @@ class FileIndexReadExecutor { std::map> scores; }; - FileIndexReadExecutor(const RawFileSplitRead& split_read, const BinaryRow& partition, - const std::shared_ptr& file, - const FieldMappingBuilder* field_mapping_builder, - const std::shared_ptr& path_factory, - const std::map& extra_format_options); + PreciseFileIndexRead(const RawFileSplitRead& split_read, const BinaryRow& partition, + const std::shared_ptr& file, + const FieldMappingBuilder* field_mapping_builder, + const std::shared_ptr& path_factory, + const std::map& extra_format_options, + std::unique_ptr indexes, + const std::map, + std::shared_ptr>& mapped_leaves, + const std::shared_ptr& vector_search, + const std::shared_ptr& full_text_search); - Status Prepare(); - Status PreparePredicate(const std::shared_ptr& predicate, - const std::map& read_by_name, - const std::map>& data_by_id); - Status PrepareSearch(const std::map& field_names); Result CreateCandidates( const DeletionVector::Factory& dv_factory, const std::optional>& row_ranges) const; diff --git a/src/paimon/core/operation/file_index_read_executor_test.cpp b/src/paimon/core/operation/precise_file_index_read_test.cpp similarity index 95% rename from src/paimon/core/operation/file_index_read_executor_test.cpp rename to src/paimon/core/operation/precise_file_index_read_test.cpp index 9e1bb7142..db99474f5 100644 --- a/src/paimon/core/operation/file_index_read_executor_test.cpp +++ b/src/paimon/core/operation/precise_file_index_read_test.cpp @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "paimon/core/operation/file_index_read_executor.h" +#include "paimon/core/operation/precise_file_index_read.h" #include #include @@ -111,7 +111,7 @@ class RecordingDoubleFunction final : public LeafFunction { } // namespace -class FileIndexReadExecutorTest : public ::testing::Test { +class PreciseFileIndexReadTest : public ::testing::Test { protected: void SetUp() override { auto factory = std::make_unique(statistics_); @@ -183,9 +183,9 @@ class FileIndexReadExecutorTest : public ::testing::Test { context->GetPartitionKeys(), nullptr)); const BinaryRow partition = BinaryRowGenerator::GenerateRow({10, 0}, pool_.get()); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr executor, - FileIndexReadExecutor::Create(*split_read, partition, file, field_mapping_builder.get(), - path_factory, {})); + std::unique_ptr precise_read, + PreciseFileIndexRead::Create(*split_read, partition, file, field_mapping_builder.get(), + path_factory, {})); if (!deletion_vector) { deletion_vector = std::make_shared(deleted_rows); } @@ -194,7 +194,7 @@ class FileIndexReadExecutorTest : public ::testing::Test { return deletion_vector; }; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, - executor->CreateReader(dv_factory, row_ranges)); + precise_read->CreateReader(dv_factory, row_ranges)); if (!reader) { return std::shared_ptr(); } @@ -225,12 +225,12 @@ class FileIndexReadExecutorTest : public ::testing::Test { } std::shared_ptr pool_ = GetDefaultPool(); - const std::string statistics_index_type_ = "executor-statistics-full-text"; + const std::string statistics_index_type_ = "precise-read-statistics-full-text"; std::shared_ptr statistics_ = std::make_shared(); }; -TEST_F(FileIndexReadExecutorTest, TestMixedPredicateNestedAndOrWithPartitionConstants) { +TEST_F(PreciseFileIndexReadTest, TestMixedPredicateNestedAndOrWithPartitionConstants) { const std::string first_query = R"({"match":{"query":"first"}})"; const std::string second_query = R"({"match":{"query":"second"}})"; ASSERT_OK_AND_ASSIGN( @@ -278,7 +278,7 @@ TEST_F(FileIndexReadExecutorTest, TestMixedPredicateNestedAndOrWithPartitionCons } } -TEST_F(FileIndexReadExecutorTest, TestMixedPredicateReordersQueries) { +TEST_F(PreciseFileIndexReadTest, TestMixedPredicateReordersQueries) { const std::string wide_query = R"({"match":{"query":"wide"}})"; const std::string narrow_query = R"({"match":{"query":"narrow"}})"; auto wide_match = @@ -313,7 +313,7 @@ TEST_F(FileIndexReadExecutorTest, TestMixedPredicateReordersQueries) { ASSERT_EQ(wide_match->include_row_ids, RoaringBitmap64::From({1, 2})); } -TEST_F(FileIndexReadExecutorTest, TestMixedPredicateSelectivityOrder) { +TEST_F(PreciseFileIndexReadTest, TestMixedPredicateSelectivityOrder) { const std::vector queries = { R"({"match":{"query":"a"}})", R"({"match":{"query":"b"}})", R"({"match":{"query":"c"}})", R"({"match":{"query":"d"}})", R"({"match":{"query":"e"}})"}; @@ -355,7 +355,7 @@ TEST_F(FileIndexReadExecutorTest, TestMixedPredicateSelectivityOrder) { } } -TEST_F(FileIndexReadExecutorTest, TestMixedPredicateZeroEstimateOnlyChangesOrder) { +TEST_F(PreciseFileIndexReadTest, TestMixedPredicateZeroEstimateOnlyChangesOrder) { const std::string a_query = R"({"match":{"query":"a"}})"; const std::string b_query = R"({"match":{"query":"b"}})"; ASSERT_OK_AND_ASSIGN( @@ -379,7 +379,7 @@ TEST_F(FileIndexReadExecutorTest, TestMixedPredicateZeroEstimateOnlyChangesOrder } } -TEST_F(FileIndexReadExecutorTest, TestMixedPredicateCompoundSelectivity) { +TEST_F(PreciseFileIndexReadTest, TestMixedPredicateCompoundSelectivity) { const std::string a_query = R"({"match":{"query":"a"}})"; const std::string b_query = R"({"match":{"query":"b"}})"; const std::string c_query = R"({"match":{"query":"c"}})"; @@ -435,7 +435,7 @@ TEST_F(FileIndexReadExecutorTest, TestMixedPredicateCompoundSelectivity) { } } -TEST_F(FileIndexReadExecutorTest, TestMixedPredicateEmptyRowsShortCircuitExecution) { +TEST_F(PreciseFileIndexReadTest, TestMixedPredicateEmptyRowsShortCircuitExecution) { const std::string empty_query = R"({"match":{"query":"empty"}})"; const std::string all_query = R"({"match":{"query":"all"}})"; ASSERT_OK_AND_ASSIGN( @@ -469,7 +469,7 @@ TEST_F(FileIndexReadExecutorTest, TestMixedPredicateEmptyRowsShortCircuitExecuti } } -TEST_F(FileIndexReadExecutorTest, TestMixedPredicateOrEvaluatesAllBranches) { +TEST_F(PreciseFileIndexReadTest, TestMixedPredicateOrEvaluatesAllBranches) { const std::string all_query = R"({"match":{"query":"all"}})"; const std::string one_query = R"({"match":{"query":"one"}})"; ASSERT_OK_AND_ASSIGN( @@ -490,7 +490,7 @@ TEST_F(FileIndexReadExecutorTest, TestMixedPredicateOrEvaluatesAllBranches) { ASSERT_EQ(statistics_->executions[1].query, one_query); } -TEST_F(FileIndexReadExecutorTest, TestMixedPredicateIndexedLeafSkipsUnselectedScalarBatches) { +TEST_F(PreciseFileIndexReadTest, TestMixedPredicateIndexedLeafSkipsUnselectedScalarBatches) { const std::string query = R"({"match":{"query":"selected"}})"; ASSERT_OK_AND_ASSIGN( std::shared_ptr indexed, @@ -525,7 +525,7 @@ TEST_F(FileIndexReadExecutorTest, TestMixedPredicateIndexedLeafSkipsUnselectedSc ASSERT_EQ(statistics_->executions.size(), 1); } -TEST_F(FileIndexReadExecutorTest, TestMixedPredicateRechecksScalarIndexCandidates) { +TEST_F(PreciseFileIndexReadTest, TestMixedPredicateRechecksScalarIndexCandidates) { auto field = arrow::field("f3", arrow::float64()); ASSERT_OK_AND_ASSIGN(auto writer, BitmapFileIndexWriter::Create(field, {}, pool_)); // Include Bob as a false positive: the data file contains 12.1 for row 0. @@ -570,7 +570,7 @@ TEST_F(FileIndexReadExecutorTest, TestMixedPredicateRechecksScalarIndexCandidate ASSERT_TRUE(statistics_->executions.front().needs_scores); } -TEST_F(FileIndexReadExecutorTest, TestMixedPredicatePrechecksSearchBeforeEmptyCandidates) { +TEST_F(PreciseFileIndexReadTest, TestMixedPredicatePrechecksSearchBeforeEmptyCandidates) { ASSERT_OK_AND_ASSIGN( std::shared_ptr text, PredicateBuilder::FullTextMatch( @@ -592,7 +592,7 @@ TEST_F(FileIndexReadExecutorTest, TestMixedPredicatePrechecksSearchBeforeEmptyCa } } -TEST_F(FileIndexReadExecutorTest, TestMixedPredicateRenamedTextAndMissingScalarColumn) { +TEST_F(PreciseFileIndexReadTest, TestMixedPredicateRenamedTextAndMissingScalarColumn) { SchemaManager schemas(std::make_shared(), SearchTablePath()); ASSERT_OK_AND_ASSIGN(std::shared_ptr original, schemas.ReadSchema(0)); auto fields = original->Fields(); @@ -645,7 +645,7 @@ TEST_F(FileIndexReadExecutorTest, TestMixedPredicateRenamedTextAndMissingScalarC ASSERT_TRUE(!empty || empty->length() == 0); } -TEST_F(FileIndexReadExecutorTest, TestMixedPredicateFullTextAndVectorScoresAfterFiltering) { +TEST_F(PreciseFileIndexReadTest, TestMixedPredicateFullTextAndVectorScoresAfterFiltering) { const std::string query = R"({"match":{"query":"all"}})"; ASSERT_OK_AND_ASSIGN( std::shared_ptr text, @@ -690,7 +690,7 @@ TEST_F(FileIndexReadExecutorTest, TestMixedPredicateFullTextAndVectorScoresAfter } } -TEST_F(FileIndexReadExecutorTest, TestFullTextRootScoresAndLimitAfterScalarDvAndRanges) { +TEST_F(PreciseFileIndexReadTest, TestFullTextRootScoresAndLimitAfterScalarDvAndRanges) { const std::string query = R"({"match":{"query":"ranked"}})"; auto search = std::make_shared("f0", query, 1, RoaringBitmap64::From({0, 2})); ReadContextBuilder builder(SearchTablePath()); @@ -715,7 +715,7 @@ TEST_F(FileIndexReadExecutorTest, TestFullTextRootScoresAndLimitAfterScalarDvAnd ASSERT_EQ(search->include_row_ids, RoaringBitmap64::From({0, 2})); } -TEST_F(FileIndexReadExecutorTest, TestPreciseEvaluationRejectsBitmap64DeletionVectors) { +TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsBitmap64DeletionVectors) { const std::string query = R"({"match":{"query":"all"}})"; ASSERT_OK_AND_ASSIGN( std::shared_ptr text, @@ -759,13 +759,13 @@ TEST_F(FileIndexReadExecutorTest, TestPreciseEvaluationRejectsBitmap64DeletionVe empty_candidates ? std::make_optional(std::vector()) : std::nullopt; ASSERT_NOK_WITH_MSG( ReadWithFileIndex(context, index, RoaringBitmap32(), nullptr, ranges, dv), - "Precise file-index evaluation does not support bitmap64 deletion vectors"); + "Precise file-index evaluation requires bitmap32 deletion vectors"); } } } } -TEST_F(FileIndexReadExecutorTest, TestPreciseEvaluationRejectsRowsAboveBitmap32Limit) { +TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationRejectsRowsAboveBitmap32Limit) { for (const std::string& search_type : std::vector{"none", "scalar", "match", "full-text", "vector"}) { ReadContextBuilder builder(SearchTablePath()); @@ -796,7 +796,7 @@ TEST_F(FileIndexReadExecutorTest, TestPreciseEvaluationRejectsRowsAboveBitmap32L } } -TEST_F(FileIndexReadExecutorTest, TestPreciseEvaluationAtBitmap32Limit) { +TEST_F(PreciseFileIndexReadTest, TestPreciseEvaluationAtBitmap32Limit) { for (bool has_predicate : {false, true}) { ReadContextBuilder builder(SearchTablePath()); builder.SetReadFieldNames({"f0", "f3"}).EnablePreciseFileIndexEvaluate(true); diff --git a/src/paimon/core/operation/raw_file_split_read.cpp b/src/paimon/core/operation/raw_file_split_read.cpp index 823c57fc9..659709bbc 100644 --- a/src/paimon/core/operation/raw_file_split_read.cpp +++ b/src/paimon/core/operation/raw_file_split_read.cpp @@ -32,8 +32,8 @@ #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/io/file_index_evaluator.h" -#include "paimon/core/operation/file_index_read_executor.h" #include "paimon/core/operation/internal_read_context.h" +#include "paimon/core/operation/precise_file_index_read.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/data_split_impl.h" @@ -161,9 +161,9 @@ Result> RawFileSplitRead::CreateReaderWithPredicate std::make_unique(std::move(raw_readers), arrow_pool_); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr batch_reader, - ApplyPredicateFilterIfNeeded(std::move(concat_batch_reader), - EnablePreciseFileIndexEvaluate() ? nullptr - : residual_predicate)); + ApplyPredicateFilterIfNeeded( + std::move(concat_batch_reader), + context_->EnablePreciseFileIndexEvaluate() ? nullptr : residual_predicate)); return std::make_unique(std::move(batch_reader), arrow_pool_); } @@ -212,10 +212,6 @@ Result RawFileSplitRead::Match(const std::shared_ptr& split, return matched; } -bool RawFileSplitRead::EnablePreciseFileIndexEvaluate() const { - return context_->EnablePreciseFileIndexEvaluate(); -} - Result> RawFileSplitRead::CreatePreciseFileIndexReader( const BinaryRow& partition, const std::shared_ptr& file, const FieldMappingBuilder* field_mapping_builder, DeletionVector::Factory dv_factory, @@ -223,10 +219,10 @@ Result> RawFileSplitRead::CreatePreciseFileInde const std::shared_ptr& path_factory, const std::map& extra_format_options) const { PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr executor, - FileIndexReadExecutor::Create(*this, partition, file, field_mapping_builder, path_factory, - extra_format_options)); - return executor->CreateReader(dv_factory, row_ranges); + std::unique_ptr precise_read, + PreciseFileIndexRead::Create(*this, partition, file, field_mapping_builder, path_factory, + extra_format_options)); + return precise_read->CreateReader(dv_factory, row_ranges); } } // namespace paimon diff --git a/src/paimon/core/operation/raw_file_split_read.h b/src/paimon/core/operation/raw_file_split_read.h index 262d13896..135b7cdcf 100644 --- a/src/paimon/core/operation/raw_file_split_read.h +++ b/src/paimon/core/operation/raw_file_split_read.h @@ -52,12 +52,13 @@ struct DeletionFile; /// If the class name below is enclosed in parentheses, it might be present in the read path; /// otherwise, it must be present in the read path. /// -/// Readers Overview: (ConcatBatchReader across splits) -> CompleteRowKindBatchReader -> -/// (PredicateBatchReader) -> ConcatBatchReader across files -> (CompleteIndexScoreFileBatchReader) -/// -> FieldMappingReader -> (ApplyDeletionVectorBatchReader) -> (ApplyBitmapIndexBatchReader) -/// -> (CompleteRowTrackingFieldsBatchReader) -> (ShreddingFileReader) -> (VectorFileBatchReader) -> -/// (DelegatingPrefetchReader) -> -/// (PrefetchFileBatchReader) -> (LateMaterializingFileBatchReader) -> FormatReader +/// Readers Overview: (ConcatBatchReader across splits) -> CompleteRowKindBatchReader +/// -> (PredicateBatchReader) -> ConcatBatchReader across files +/// -> (CompleteIndexScoreFileBatchReader) -> FieldMappingReader +/// -> (ApplyDeletionVectorBatchReader) -> (ApplyBitmapIndexBatchReader) +/// -> (CompleteRowTrackingFieldsBatchReader) -> (ShreddingFileReader) +/// -> (VectorFileBatchReader) -> (DelegatingPrefetchReader) -> (PrefetchFileBatchReader) +/// -> (LateMaterializingFileBatchReader) -> FormatReader class RawFileSplitRead : public AbstractSplitRead { public: @@ -103,8 +104,6 @@ class RawFileSplitRead : public AbstractSplitRead { Result Match(const std::shared_ptr& split, bool force_keep_delete) const override; protected: - bool EnablePreciseFileIndexEvaluate() const override; - Result> CreatePreciseFileIndexReader( const BinaryRow& partition, const std::shared_ptr& file, const FieldMappingBuilder* field_mapping_builder, DeletionVector::Factory dv_factory, diff --git a/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp b/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp index 96543db30..185976387 100644 --- a/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp +++ b/src/paimon/core/realtime/realtime_offset_file_index_lookup.cpp @@ -103,10 +103,9 @@ RealtimeOffsetFileIndexLookup::LookupFilePositions(const RoaringBitmap64& offset std::shared_ptr predicate = PredicateBuilder::In( schema->GetFieldIndex(offset_name), offset_name, FieldType::BIGINT, literals); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr evaluator, - FileIndexEvaluator::Create(schema, options_, path_factory_, file, file_system_, - /*only_use_embedding_index=*/false, memory_pool_)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr evaluator, + FileIndexEvaluator::Create(schema, options_, path_factory_, file, + file_system_, memory_pool_)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index_result, evaluator->Evaluate(predicate)); PAIMON_ASSIGN_OR_RAISE(bool remains, index_result->IsRemain()); diff --git a/test/inte/write_and_read_inte_with_index_test.cpp b/test/inte/write_and_read_inte_with_index_test.cpp index 7a2c64295..99bd90e2e 100644 --- a/test/inte/write_and_read_inte_with_index_test.cpp +++ b/test/inte/write_and_read_inte_with_index_test.cpp @@ -2313,6 +2313,42 @@ TEST_P(WriteAndReadInteWithIndexTest, TestWithAlterTable) { ASSERT_TRUE(array_status.ok()); CheckResult(path, {split}, predicate, expected_array); } + { + // f1 evolved from INT to BIGINT. The second literal overflows the old file's INT type, + // so its predicate cannot use the index and must be evaluated against the data. + auto predicate = + PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f1", FieldType::BIGINT, + {Literal(10l), Literal(int64_t{2147483648})}); + ReadContextBuilder builder(path); + builder.EnablePreciseFileIndexEvaluate(true) + .EnablePredicateFilter(false) + .SetPredicate(predicate) + .AddOption("read.batch-size", "2") + .AddOption("test.enable-adaptive-prefetch-strategy", "false") + .EnableLateMaterializing(false); + if (enable_prefetch_) { + builder.EnablePrefetch(true).SetPrefetchBatchCount(3); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, + TableRead::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(split)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + ReadResultCollector::CollectResult(std::move(reader))); + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(arrow_data_type, {R"([ + [0, 10, "Alice", 11.1, null], + [0, 10, "Bob", 12.1, null], + [0, 10, "Emily", 13.1, null], + [0, 10, "Tony", 14.1, null], + [0, 10, "Bob", 16.1, null], + [0, 10, "Bob", 23.1, 100] + ])"}, + &expected) + .ok()); + ASSERT_TRUE(actual); + ASSERT_TRUE(actual->Equals(expected)) << actual->ToString(); + } } TEST_P(WriteAndReadInteWithIndexTest, TestWithBsiIndex) { From 9c6a8e040caabd5cb0e9996daf5f5ddcaa2758fe Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Thu, 5 Nov 2026 09:27:24 +0800 Subject: [PATCH 3/5] fix ci --- .../predicate/full_text_match_predicate.h | 3 +- .../core/table/source/table_read_test.cpp | 5 ++- test/inte/realtime_write_inte_test.cpp | 33 ++++--------------- 3 files changed, 10 insertions(+), 31 deletions(-) diff --git a/src/paimon/common/predicate/full_text_match_predicate.h b/src/paimon/common/predicate/full_text_match_predicate.h index 61ae13a70..ea8a17431 100644 --- a/src/paimon/common/predicate/full_text_match_predicate.h +++ b/src/paimon/common/predicate/full_text_match_predicate.h @@ -23,13 +23,14 @@ #include #include "paimon/predicate/leaf_predicate.h" +#include "paimon/visibility.h" namespace paimon { struct FullTextMatch; /// A full-text leaf evaluated by an index, rather than scalar PredicateFilter::Test. /// Use PredicateBuilder::FullTextMatch to construct it. -class FullTextMatchPredicate : public LeafPredicate { +class PAIMON_EXPORT FullTextMatchPredicate : public LeafPredicate { public: FullTextMatchPredicate(int32_t field_index, const std::shared_ptr& match); diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 3b1226513..2e38731c6 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -27,13 +27,12 @@ #include #include +#include "arrow/api.h" #include "arrow/type.h" #include "gtest/gtest.h" +#include "paimon/catalog/identifier.h" #include "paimon/catalog/table.h" #include "paimon/common/catalog/catalog_context.h" -#include "arrow/api.h" -#include "gtest/gtest.h" -#include "paimon/catalog/identifier.h" #include "paimon/common/data/binary_row.h" #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_meta.h" diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index e520d43e9..90f2e89e2 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -47,6 +47,7 @@ #include "paimon/catalog/identifier.h" #include "paimon/commit_context.h" #include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/predicate/equal.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/path_util.h" @@ -75,7 +76,7 @@ #include "paimon/fs/local/local_file_system.h" #include "paimon/memory/memory_pool.h" #include "paimon/orphan_files_cleaner.h" -#include "paimon/predicate/function.h" +#include "paimon/predicate/leaf_predicate.h" #include "paimon/predicate/predicate.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/read_context.h" @@ -336,39 +337,17 @@ constexpr char kTruncateCommitUser[] = "truncate_commit_user"; } // namespace -class UnsupportedFunction : public Function { +// Pass read-context validation, but fail predicate reconstruction when creating the reader. +class UnsupportedPredicate : public LeafPredicate { public: - Type GetType() const override { - return Type::EQUAL; - } + UnsupportedPredicate() + : LeafPredicate(Equal::Instance(), 0, "id", FieldType::BIGINT, {Literal(int64_t{0})}) {} std::string ToString() const override { return "unsupported"; } }; -class UnsupportedPredicate : public Predicate { - public: - bool operator==(const Predicate&) const override { - return false; - } - - const Function& GetFunction() const override { - return function_; - } - - std::shared_ptr Negate() const override { - return nullptr; - } - - std::string ToString() const override { - return function_.ToString(); - } - - private: - UnsupportedFunction function_; -}; - class ConcurrentTestState { public: void WaitForStart() { From 6713dccd1de1170381eeea1934676eba162d30e9 Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Thu, 5 Nov 2026 12:02:25 +0800 Subject: [PATCH 4/5] fix ziyi comment --- src/paimon/core/io/file_index_evaluator.cpp | 23 ++++++++++++++++--- .../core/io/file_index_evaluator_test.cpp | 19 +++++++++++---- .../data_evolution_split_read_test.cpp | 3 +-- .../operation/precise_file_index_read.cpp | 8 +++---- 4 files changed, 39 insertions(+), 14 deletions(-) diff --git a/src/paimon/core/io/file_index_evaluator.cpp b/src/paimon/core/io/file_index_evaluator.cpp index bcb57c249..796b46e4e 100644 --- a/src/paimon/core/io/file_index_evaluator.cpp +++ b/src/paimon/core/io/file_index_evaluator.cpp @@ -54,6 +54,21 @@ namespace paimon { class MemoryPool; enum class FieldType; +namespace { +std::string SearchTypeToString(const FileIndexReader::SearchType& search_type) { + switch (search_type) { + case FileIndexReader::SearchType::FULL_TEXT_MATCH: + return "FULL_TEXT_MATCH"; + case FileIndexReader::SearchType::FULL_TEXT_SEARCH: + return "FULL_TEXT_SEARCH"; + case FileIndexReader::SearchType::VECTOR_SEARCH: + return "VECTOR_SEARCH"; + default: + return "UNKNOWN"; + } +} +} // namespace + FileIndexEvaluator::FileIndexEvaluator(std::shared_ptr data_schema, std::shared_ptr file_meta, std::unique_ptr format_reader) @@ -120,13 +135,15 @@ Result> FileIndexEvaluator::GetSearchReader( } if (selected) { return Status::Invalid(fmt::format( - "Multiple File Index readers support requested search for field '{}'", field_name)); + "Multiple File Index readers support {} for field '{}' in data file '{}'", + SearchTypeToString(search_type), field_name, file_meta_->file_name)); } selected = reader; } if (!selected) { - return Status::NotImplemented(fmt::format( - "No File Index reader supports requested search for field '{}'", field_name)); + return Status::NotImplemented( + fmt::format("No File Index reader supports {} for field '{}' in data file '{}'", + SearchTypeToString(search_type), field_name, file_meta_->file_name)); } return selected; } diff --git a/src/paimon/core/io/file_index_evaluator_test.cpp b/src/paimon/core/io/file_index_evaluator_test.cpp index 7972cd7e6..067212590 100644 --- a/src/paimon/core/io/file_index_evaluator_test.cpp +++ b/src/paimon/core/io/file_index_evaluator_test.cpp @@ -470,6 +470,8 @@ class FileIndexSearchEvaluatorTest : public FileIndexEvaluatorTest, void CheckInvalidSearchIndexes(SearchIndexType index_type) const { const std::string field = index_type == SearchIndexType::VECTOR ? "embedding" : "title"; + const std::string search_type = + index_type == SearchIndexType::VECTOR ? "VECTOR_SEARCH" : "FULL_TEXT_SEARCH"; const std::string identifier = index_type == SearchIndexType::VECTOR ? MockSearchFileIndex::VECTOR_IDENTIFIER : MockSearchFileIndex::FULL_TEXT_IDENTIFIER; @@ -484,7 +486,9 @@ class FileIndexSearchEvaluatorTest : public FileIndexEvaluatorTest, MockSearchFileIndex::MakeIndex("another-field", identifier, {{3, 0.75f}, {1, 0.25f}}, pool_)); ASSERT_OK_AND_ASSIGN(meta, MakeIndex(bytes)); - CheckSearchError(index_type, meta, field, "No File Index reader supports"); + CheckSearchError(index_type, meta, field, + "No File Index reader supports " + search_type + " for field '" + field + + "' in data file '" + meta->file_name + "'"); FileIndexFormat::ColumnIndexes indexes; ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, @@ -504,7 +508,9 @@ class FileIndexSearchEvaluatorTest : public FileIndexEvaluatorTest, indexes[field][another_identifier] = nullptr; ASSERT_OK_AND_ASSIGN(bytes, MockSearchFileIndex::Serialize(indexes, pool_)); ASSERT_OK_AND_ASSIGN(meta, MakeIndex(bytes)); - CheckSearchError(index_type, meta, field, "Multiple File Index readers support"); + CheckSearchError(index_type, meta, field, + "Multiple File Index readers support " + search_type + " for field '" + + field + "' in data file '" + meta->file_name + "'"); meta->embedded_index.reset(); meta->extra_files.clear(); @@ -795,13 +801,16 @@ TEST_P(FileIndexSearchEvaluatorTest, TestEmptyScalarIndexDoesNotProvideSearchCap path_factory_, meta, fs_, pool_)); ASSERT_NOK_WITH_MSG( evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_MATCH), - "No File Index reader supports"); + "No File Index reader supports FULL_TEXT_MATCH for field 'title' in data file " + "'data-search.orc'"); ASSERT_NOK_WITH_MSG( evaluator->GetSearchReader("title", FileIndexReader::SearchType::FULL_TEXT_SEARCH), - "No File Index reader supports"); + "No File Index reader supports FULL_TEXT_SEARCH for field 'title' in data file " + "'data-search.orc'"); ASSERT_NOK_WITH_MSG( evaluator->GetSearchReader("title", FileIndexReader::SearchType::VECTOR_SEARCH), - "No File Index reader supports"); + "No File Index reader supports VECTOR_SEARCH for field 'title' in data file " + "'data-search.orc'"); ASSERT_OK_AND_ASSIGN(indexes["title"][MockSearchFileIndex::FULL_TEXT_IDENTIFIER], MockSearchFileIndex::MakePayload({{1, 0.5f}}, pool_)); diff --git a/src/paimon/core/operation/data_evolution_split_read_test.cpp b/src/paimon/core/operation/data_evolution_split_read_test.cpp index 763673e4d..4cdfce10b 100644 --- a/src/paimon/core/operation/data_evolution_split_read_test.cpp +++ b/src/paimon/core/operation/data_evolution_split_read_test.cpp @@ -1497,8 +1497,7 @@ TEST_F(DataEvolutionSplitReadTest, TestFileIndexSelectionOnlyPushesDownNormalGro split_read.ApplyIndexAndDvReaderIfNeeded( std::make_unique(data, data->type(), 2), normal, index_schema, index_schema, predicate, /*dv_factory=*/nullptr, row_ranges, - /*data_file_path_factory=*/nullptr, /*file_index_options=*/{}, - /*index_scores=*/nullptr)); + /*data_file_path_factory=*/nullptr, /*file_index_options=*/{})); if (expected_json.empty()) { ASSERT_FALSE(reader); return; diff --git a/src/paimon/core/operation/precise_file_index_read.cpp b/src/paimon/core/operation/precise_file_index_read.cpp index 4fcdca57e..79daca660 100644 --- a/src/paimon/core/operation/precise_file_index_read.cpp +++ b/src/paimon/core/operation/precise_file_index_read.cpp @@ -158,12 +158,12 @@ Result> PreciseFileIndexRead::Create( const std::shared_ptr& file, const FieldMappingBuilder* field_mapping_builder, const std::shared_ptr& path_factory, const std::map& extra_format_options) { - std::shared_ptr table_schema = split_read.context_->GetTableSchema(); - if (file->schema_id != table_schema->Id()) { - PAIMON_ASSIGN_OR_RAISE(table_schema, + std::shared_ptr file_schema = split_read.context_->GetTableSchema(); + if (file->schema_id != file_schema->Id()) { + PAIMON_ASSIGN_OR_RAISE(file_schema, split_read.schema_manager_->ReadSchema(file->schema_id)); } - const auto data_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + const auto data_schema = DataField::ConvertDataFieldsToArrowSchema(file_schema->Fields()); if (file->row_count < 0 || file->row_count > RoaringBitmap32::MAX_VALUE) { return Status::NotImplemented( "Precise file-index evaluation requires bitmap32 physical row positions"); From f92f9fa0be084c1cb1162922709528d4aed0068b Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Thu, 5 Nov 2026 14:11:19 +0800 Subject: [PATCH 5/5] fix clang-tidy --- src/paimon/core/io/file_index_evaluator_test.cpp | 2 +- src/paimon/core/operation/precise_file_index_read_test.cpp | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/paimon/core/io/file_index_evaluator_test.cpp b/src/paimon/core/io/file_index_evaluator_test.cpp index 067212590..55775950c 100644 --- a/src/paimon/core/io/file_index_evaluator_test.cpp +++ b/src/paimon/core/io/file_index_evaluator_test.cpp @@ -391,7 +391,7 @@ TEST_F(FileIndexEvaluatorTest, TestCreateRequiresExternalIndexDependencies) { /*column_max_sequence_numbers=*/std::nullopt); ASSERT_NOK_WITH_MSG( FileIndexEvaluator::Create(data_schema_, core_options_.ToMap(), - /*data_file_path_factory=*/nullptr, data_file_meta, + /*path_factory=*/nullptr, data_file_meta, /*file_system=*/nullptr, pool_), "read process for FileIndexEvaluator must have data_file_path_factory and file_system"); } diff --git a/src/paimon/core/operation/precise_file_index_read_test.cpp b/src/paimon/core/operation/precise_file_index_read_test.cpp index db99474f5..640179ef9 100644 --- a/src/paimon/core/operation/precise_file_index_read_test.cpp +++ b/src/paimon/core/operation/precise_file_index_read_test.cpp @@ -335,8 +335,7 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateSelectivityOrder) { {{std::numeric_limits::quiet_NaN(), -0.1, 1.1, 0.5, std::numeric_limits::infinity()}, {3, 0, 1, 2, 4}}}; - for (size_t case_id = 0; case_id < cases.size(); ++case_id) { - const auto& test_case = cases[case_id]; + for (const auto& test_case : cases) { MockSearchFileIndex::QueryMatches matches; for (size_t i = 0; i < queries.size(); ++i) { matches.emplace(queries[i], @@ -411,8 +410,7 @@ TEST_F(PreciseFileIndexReadTest, TestMixedPredicateCompoundSelectivity) { {false, 0.2, std::nullopt, 0.6, {c_query, a_query, b_query}}, {false, std::nullopt, 0.2, 0.6, {c_query, a_query, b_query}}, {false, std::nullopt, std::nullopt, 0.6, {c_query, a_query, b_query}}}; - for (size_t case_id = 0; case_id < cases.size(); ++case_id) { - const auto& test_case = cases[case_id]; + for (const auto& test_case : cases) { ASSERT_OK_AND_ASSIGN(std::shared_ptr inner, test_case.inner_and ? PredicateBuilder::And({a, b}) : PredicateBuilder::Or({a, b}));