diff --git a/README.md b/README.md index d59dfb3fa..276bcb6e0 100644 --- a/README.md +++ b/README.md @@ -38,17 +38,19 @@ Paimon C++ currently provides: - **File systems**: file system abstraction with built-in local, Jindo, OSS, and S3 file system support. - **File formats**: file format abstraction with built-in Parquet, Blob, ORC, and Avro support, plus optional Lance and Mosaic formats. - **Runtime utilities**: memory pool and thread pool abstractions with default implementations. -- **AI-Oriented Features**: supports RowTracking and DataEvolution mode and provides Global Index - capabilities including B-tree index, DiskANN-based vector search with Lumina, and Lucene-based - full-text search. +- **AI-oriented features**: row tracking and data evolution modes, plus global index + capabilities including B-tree and bitmap indexing, vector search with Lumina, and + full-text search with Lucene or Paimon's Tantivy-based implementation. - **Compatibility**: compatibility with Apache Paimon Java format and communication protocols, including commit messages, data splits, and manifests. -Note: Linux `x86_64` and `aarch64` builds are supported and verified in CI. See the supported -platform matrix in [docs/source/building.rst](docs/source/building.rst) for other platforms. - ## Building +Building requires a C++17-compatible compiler and CMake 3.22 or newer. +Linux `x86_64` and `aarch64` builds are supported and verified in CI. +See the [build guide](docs/source/building.rst) for platform support, system requirements, +dependency configuration, optional components, and instructions for building and running tests. + ```bash git clone https://github.com/apache/paimon-cpp.git cd paimon-cpp @@ -60,18 +62,26 @@ Build with CMake: cmake -B build cmake --build build ``` + ### Dev Containers We provide Dev Container configuration file templates. To use a Dev Container as your development environment, follow the steps below, then select `Dev Containers: Reopen in Container` from VS Code's Command Palette. -``` +```bash cd .devcontainer cp Dockerfile.template Dockerfile cp devcontainer.json.template devcontainer.json ``` +## Examples + +Start with the [read and write example](examples/read_write_demo.cpp) to see how to write, +commit, scan, and read table data. The [cleanup example](examples/clean_demo.cpp) demonstrates +table maintenance. Both examples have a [CMake project](examples/CMakeLists.txt) that uses +installed Paimon and Arrow packages. + ## Collaboration Paimon C++ is an active open-source project and we welcome people who want to contribute or share good ideas! diff --git a/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp b/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp index 3501e6190..a95fd91a9 100644 --- a/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp +++ b/src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp @@ -248,6 +248,82 @@ class LateMaterializingFileBatchReaderTest : public ::testing::Test { std::shared_ptr full_type_; }; +// Restrict only the payload pass to model a format reader that omits rows or returns +// an invalid selection without changing the probe data. +class PayloadSelectionReader : public MockFileBatchReader { + public: + PayloadSelectionReader(const std::shared_ptr& data, + const std::shared_ptr& schema, int32_t batch_size, + const RoaringBitmap32& payload_bitmap, bool empty_bitmap = false) + : MockFileBatchReader(data, schema, batch_size), + payload_bitmap_(payload_bitmap), + empty_bitmap_(empty_bitmap) {} + + Status SetReadSchema(::ArrowSchema* schema, const std::shared_ptr& predicate, + const std::optional& selection) override { + PAIMON_RETURN_NOT_OK(MockFileBatchReader::SetReadSchema(schema, predicate, selection)); + if (++schema_count_ == 2) { + bitmap_ = payload_bitmap_; + } + return Status::OK(); + } + + Result NextBatchWithBitmap() override { + PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch, + MockFileBatchReader::NextBatchWithBitmap()); + if (schema_count_ == 2 && empty_bitmap_ && !BatchReader::IsEofBatch(batch)) { + batch.second = RoaringBitmap32(); + } + return batch; + } + + private: + RoaringBitmap32 payload_bitmap_; + bool empty_bitmap_; + int32_t schema_count_ = 0; +}; + +TEST_F(LateMaterializingFileBatchReaderTest, PayloadEarlyEofReturnsError) { + auto data = BuildData({0, 1, 2, 3}); + RoaringBitmap32 payload_bitmap; + payload_bitmap.Add(2); + auto mock = std::make_unique(data, full_type_, 2, payload_bitmap); + ASSERT_OK_AND_ASSIGN(auto reader, LateMaterializingFileBatchReader::Create( + std::move(mock), GetArrowPool(pool_))); + auto predicate = PredicateBuilder::GreaterThan(0, "k", FieldType::BIGINT, Literal(int64_t{1})); + ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt)); + ASSERT_NOK_WITH_MSG(Collect(reader.get()), "does not match probe data length"); +} + +TEST_F(LateMaterializingFileBatchReaderTest, EmptyPayloadBitmapReturnsError) { + auto data = BuildData({0, 1, 2, 3}); + RoaringBitmap32 payload_bitmap; + payload_bitmap.AddRange(0, 4); + auto mock = std::make_unique(data, full_type_, 2, payload_bitmap, + /*empty_bitmap=*/true); + ASSERT_OK_AND_ASSIGN(auto reader, LateMaterializingFileBatchReader::Create( + std::move(mock), GetArrowPool(pool_))); + auto predicate = PredicateBuilder::GreaterThan(0, "k", FieldType::BIGINT, Literal(int64_t{1})); + ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt)); + ASSERT_NOK_WITH_MSG(Collect(reader.get()), "inner read bitmap is empty"); +} + +TEST_F(LateMaterializingFileBatchReaderTest, SkipsPayloadBatchesWithoutMatches) { + auto data = BuildData({0, 1, 2, 3, 4, 5}); + auto mock = std::make_unique(data, full_type_, 2); + ASSERT_OK_AND_ASSIGN(auto reader, LateMaterializingFileBatchReader::Create( + std::move(mock), GetArrowPool(pool_))); + auto predicate = PredicateBuilder::GreaterThan(0, "k", FieldType::BIGINT, Literal(int64_t{3})); + ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::vector rows, Collect(reader.get())); + ASSERT_EQ(rows.size(), 2u); + for (size_t i = 0; i < rows.size(); ++i) { + ASSERT_EQ(rows[i].k, static_cast(i + 4)); + ASSERT_EQ(rows[i].v, "v_" + std::to_string(i + 4)); + ASSERT_EQ(rows[i].file_row, i + 4); + } +} + // No predicate: the reader must pass through the inner reader unchanged (all rows, all columns). TEST_F(LateMaterializingFileBatchReaderTest, PassThroughWhenNoPredicate) { auto data = BuildData({0, 1, 2, 3, 4}); diff --git a/src/paimon/format/parquet/column_index_filter_test.cpp b/src/paimon/format/parquet/column_index_filter_test.cpp index f1b916bb7..0f3621c16 100644 --- a/src/paimon/format/parquet/column_index_filter_test.cpp +++ b/src/paimon/format/parquet/column_index_filter_test.cpp @@ -322,6 +322,112 @@ class ColumnIndexFilterTest : public ::testing::Test { int64_t row_group_row_count_ = 0; }; +// Supply optional or malformed statistics while keeping the real file's offset index. +class TestColumnIndex : public ::parquet::ColumnIndex { + public: + const std::vector& null_pages() const override { + return null_pages_; + } + + const std::vector& encoded_min_values() const override { + return min_values_; + } + + const std::vector& encoded_max_values() const override { + return max_values_; + } + + const std::vector& null_counts() const override { + return null_counts_; + } + + const std::vector& non_null_page_indices() const override { + return non_null_pages_; + } + + ::parquet::BoundaryOrder::type boundary_order() const override { + return ::parquet::BoundaryOrder::Unordered; + } + bool has_null_counts() const override { + return has_null_counts_; + } + + std::vector null_pages_ = std::vector(10, false); + std::vector min_values_ = std::vector(10, std::string(4, '\0')); + std::vector max_values_ = min_values_; + std::vector null_counts_ = std::vector(10, 0); + std::vector non_null_pages_{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + bool has_null_counts_ = true; +}; + +class TestRowGroupPageIndexReader : public ::parquet::RowGroupPageIndexReader { + public: + TestRowGroupPageIndexReader(const std::shared_ptr& column_index, + const std::shared_ptr<::parquet::OffsetIndex>& offset_index) + : column_index_(column_index), offset_index_(offset_index) {} + + std::shared_ptr<::parquet::ColumnIndex> GetColumnIndex(int32_t) override { + return column_index_; + } + std::shared_ptr<::parquet::OffsetIndex> GetOffsetIndex(int32_t) override { + return offset_index_; + } + + private: + std::shared_ptr column_index_; + std::shared_ptr<::parquet::OffsetIndex> offset_index_; +}; + +TEST_F(ColumnIndexFilterTest, InWithNullKeepsPagesWithPossibleNulls) { + auto index = std::make_shared(); + index->null_pages_[0] = true; + index->null_counts_[0] = 10; + index->null_counts_[1] = 1; + index->non_null_pages_.erase(index->non_null_pages_.begin()); + auto page_reader = std::make_shared( + index, page_index_reader_->RowGroup(0)->GetOffsetIndex(0)); + auto predicate = PredicateBuilder::In(0, "val", FieldType::INT, + {Literal(FieldType::INT), Literal(int32_t{999})}); + ASSERT_OK_AND_ASSIGN(auto ranges, ColumnIndexFilter::CalculateRowRanges( + predicate, page_reader, column_name_to_index_, + parquet_reader_->metadata()->schema(), 100)); + ASSERT_EQ(ranges.RowCount(), 20); + ASSERT_EQ(ranges.GetRanges().size(), 1u); + ASSERT_EQ(ranges.GetRanges()[0].from, 0); + ASSERT_EQ(ranges.GetRanges()[0].to, 19); + + // Without null counts every non-null page may still contain nulls. + index->has_null_counts_ = false; + index->null_counts_.clear(); + ASSERT_OK_AND_ASSIGN( + ranges, ColumnIndexFilter::CalculateRowRanges(predicate, page_reader, column_name_to_index_, + parquet_reader_->metadata()->schema(), 100)); + ASSERT_EQ(ranges.RowCount(), 100); +} + +TEST_F(ColumnIndexFilterTest, NotInWithNullRejectsEveryPage) { + auto predicate = PredicateBuilder::NotIn(0, "val", FieldType::INT, + {Literal(int32_t{55}), Literal(FieldType::INT)}); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(predicate)); + ASSERT_TRUE(ranges.IsEmpty()); +} + +TEST_F(ColumnIndexFilterTest, TruncatedIntegerBoundsKeepPages) { + for (bool truncate_min : {true, false}) { + SCOPED_TRACE(truncate_min); + auto index = std::make_shared(); + auto& bounds = truncate_min ? index->min_values_ : index->max_values_; + bounds.assign(10, std::string(3, '\0')); + auto page_reader = std::make_shared( + index, page_index_reader_->RowGroup(0)->GetOffsetIndex(0)); + auto predicate = PredicateBuilder::Equal(0, "val", FieldType::INT, Literal(int32_t{999})); + ASSERT_OK_AND_ASSIGN(auto ranges, ColumnIndexFilter::CalculateRowRanges( + predicate, page_reader, column_name_to_index_, + parquet_reader_->metadata()->schema(), 100)); + ASSERT_EQ(ranges.RowCount(), 100); + } +} + /// EQUAL: val = 55 → should match only page 5 (rows [50,59]) TEST_F(ColumnIndexFilterTest, EqualMatchSinglePage) { auto pred =