test(read): cover reader edge cases and clarify build guidance

README.md

@@ -38,17 +38,19 @@ Paimon C++ currently provides:
 - **File systems**: file system abstraction with built-in local, Jindo, OSS, and S3 file system support.
 - **File formats**: file format abstraction with built-in Parquet, Blob, ORC, and Avro support, plus optional Lance and Mosaic formats.
 - **Runtime utilities**: memory pool and thread pool abstractions with default implementations.
-- **AI-Oriented Features**: supports RowTracking and DataEvolution mode and provides Global Index
-  capabilities including B-tree index, DiskANN-based vector search with Lumina, and Lucene-based
-  full-text search.
+- **AI-oriented features**: row tracking and data evolution modes, plus global index
+  capabilities including B-tree and bitmap indexing, vector search with Lumina, and
+  full-text search with Lucene or Paimon's Tantivy-based implementation.
 - **Compatibility**: compatibility with Apache Paimon Java format and communication protocols,
   including commit messages, data splits, and manifests.
 
-Note: Linux `x86_64` and `aarch64` builds are supported and verified in CI. See the supported
-platform matrix in [docs/source/building.rst](docs/source/building.rst) for other platforms.
-
 ## Building
 
+Building requires a C++17-compatible compiler and CMake 3.22 or newer.
+Linux `x86_64` and `aarch64` builds are supported and verified in CI.
+See the [build guide](docs/source/building.rst) for platform support, system requirements,
+dependency configuration, optional components, and instructions for building and running tests.
+
 ```bash
 git clone https://github.com/apache/paimon-cpp.git
 cd paimon-cpp
@@ -60,18 +62,26 @@ Build with CMake:
 cmake -B build
 cmake --build build
 ```
+
 ### Dev Containers
 
 We provide Dev Container configuration file templates.
 
 To use a Dev Container as your development environment, follow the steps below, then select `Dev Containers: Reopen in Container` from VS Code's Command Palette.
 
-```
+```bash
 cd .devcontainer
 cp Dockerfile.template Dockerfile
 cp devcontainer.json.template devcontainer.json
 ```
 
+## Examples
+
+Start with the [read and write example](examples/read_write_demo.cpp) to see how to write,
+commit, scan, and read table data. The [cleanup example](examples/clean_demo.cpp) demonstrates
+table maintenance. Both examples have a [CMake project](examples/CMakeLists.txt) that uses
+installed Paimon and Arrow packages.
+
 ## Collaboration
 
 Paimon C++ is an active open-source project and we welcome people who want to contribute or share good ideas!

src/paimon/common/reader/late_materializing_file_batch_reader_test.cpp

@@ -248,6 +248,82 @@ class LateMaterializingFileBatchReaderTest : public ::testing::Test {
     std::shared_ptr<arrow::DataType> full_type_;
 };
 
+// Restrict only the payload pass to model a format reader that omits rows or returns
+// an invalid selection without changing the probe data.
+class PayloadSelectionReader : public MockFileBatchReader {
+ public:
+    PayloadSelectionReader(const std::shared_ptr<arrow::Array>& data,
+                           const std::shared_ptr<arrow::DataType>& schema, int32_t batch_size,
+                           const RoaringBitmap32& payload_bitmap, bool empty_bitmap = false)
+        : MockFileBatchReader(data, schema, batch_size),
+          payload_bitmap_(payload_bitmap),
+          empty_bitmap_(empty_bitmap) {}
+
+    Status SetReadSchema(::ArrowSchema* schema, const std::shared_ptr<Predicate>& predicate,
+                         const std::optional<RoaringBitmap32>& selection) override {
+        PAIMON_RETURN_NOT_OK(MockFileBatchReader::SetReadSchema(schema, predicate, selection));
+        if (++schema_count_ == 2) {
+            bitmap_ = payload_bitmap_;
+        }
+        return Status::OK();
+    }
+
+    Result<ReadBatchWithBitmap> NextBatchWithBitmap() override {
+        PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch,
+                               MockFileBatchReader::NextBatchWithBitmap());
+        if (schema_count_ == 2 && empty_bitmap_ && !BatchReader::IsEofBatch(batch)) {
+            batch.second = RoaringBitmap32();
+        }
+        return batch;
+    }
+
+ private:
+    RoaringBitmap32 payload_bitmap_;
+    bool empty_bitmap_;
+    int32_t schema_count_ = 0;
+};
+
+TEST_F(LateMaterializingFileBatchReaderTest, PayloadEarlyEofReturnsError) {
+    auto data = BuildData({0, 1, 2, 3});
+    RoaringBitmap32 payload_bitmap;
+    payload_bitmap.Add(2);
+    auto mock = std::make_unique<PayloadSelectionReader>(data, full_type_, 2, payload_bitmap);
+    ASSERT_OK_AND_ASSIGN(auto reader, LateMaterializingFileBatchReader::Create(
+                                          std::move(mock), GetArrowPool(pool_)));
+    auto predicate = PredicateBuilder::GreaterThan(0, "k", FieldType::BIGINT, Literal(int64_t{1}));
+    ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt));
+    ASSERT_NOK_WITH_MSG(Collect(reader.get()), "does not match probe data length");
+}
+
+TEST_F(LateMaterializingFileBatchReaderTest, EmptyPayloadBitmapReturnsError) {
+    auto data = BuildData({0, 1, 2, 3});
+    RoaringBitmap32 payload_bitmap;
+    payload_bitmap.AddRange(0, 4);
+    auto mock = std::make_unique<PayloadSelectionReader>(data, full_type_, 2, payload_bitmap,
+                                                         /*empty_bitmap=*/true);
+    ASSERT_OK_AND_ASSIGN(auto reader, LateMaterializingFileBatchReader::Create(
+                                          std::move(mock), GetArrowPool(pool_)));
+    auto predicate = PredicateBuilder::GreaterThan(0, "k", FieldType::BIGINT, Literal(int64_t{1}));
+    ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt));
+    ASSERT_NOK_WITH_MSG(Collect(reader.get()), "inner read bitmap is empty");
+}
+
+TEST_F(LateMaterializingFileBatchReaderTest, SkipsPayloadBatchesWithoutMatches) {
+    auto data = BuildData({0, 1, 2, 3, 4, 5});
+    auto mock = std::make_unique<MockFileBatchReader>(data, full_type_, 2);
+    ASSERT_OK_AND_ASSIGN(auto reader, LateMaterializingFileBatchReader::Create(
+                                          std::move(mock), GetArrowPool(pool_)));
+    auto predicate = PredicateBuilder::GreaterThan(0, "k", FieldType::BIGINT, Literal(int64_t{3}));
+    ASSERT_OK(SetReadSchema(reader.get(), arrow::schema(full_fields_), predicate, std::nullopt));
+    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, Collect(reader.get()));
+    ASSERT_EQ(rows.size(), 2u);
+    for (size_t i = 0; i < rows.size(); ++i) {
+        ASSERT_EQ(rows[i].k, static_cast<int64_t>(i + 4));
+        ASSERT_EQ(rows[i].v, "v_" + std::to_string(i + 4));
+        ASSERT_EQ(rows[i].file_row, i + 4);
+    }
+}
+
 // No predicate: the reader must pass through the inner reader unchanged (all rows, all columns).
 TEST_F(LateMaterializingFileBatchReaderTest, PassThroughWhenNoPredicate) {
     auto data = BuildData({0, 1, 2, 3, 4});

src/paimon/format/parquet/column_index_filter_test.cpp

@@ -322,6 +322,112 @@ class ColumnIndexFilterTest : public ::testing::Test {
     int64_t row_group_row_count_ = 0;
 };
 
+// Supply optional or malformed statistics while keeping the real file's offset index.
+class TestColumnIndex : public ::parquet::ColumnIndex {
+ public:
+    const std::vector<bool>& null_pages() const override {
+        return null_pages_;
+    }
+
+    const std::vector<std::string>& encoded_min_values() const override {
+        return min_values_;
+    }
+
+    const std::vector<std::string>& encoded_max_values() const override {
+        return max_values_;
+    }
+
+    const std::vector<int64_t>& null_counts() const override {
+        return null_counts_;
+    }
+
+    const std::vector<int32_t>& non_null_page_indices() const override {
+        return non_null_pages_;
+    }
+
+    ::parquet::BoundaryOrder::type boundary_order() const override {
+        return ::parquet::BoundaryOrder::Unordered;
+    }
+    bool has_null_counts() const override {
+        return has_null_counts_;
+    }
+
+    std::vector<bool> null_pages_ = std::vector<bool>(10, false);
+    std::vector<std::string> min_values_ = std::vector<std::string>(10, std::string(4, '\0'));
+    std::vector<std::string> max_values_ = min_values_;
+    std::vector<int64_t> null_counts_ = std::vector<int64_t>(10, 0);
+    std::vector<int32_t> non_null_pages_{0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    bool has_null_counts_ = true;
+};
+
+class TestRowGroupPageIndexReader : public ::parquet::RowGroupPageIndexReader {
+ public:
+    TestRowGroupPageIndexReader(const std::shared_ptr<TestColumnIndex>& column_index,
+                                const std::shared_ptr<::parquet::OffsetIndex>& offset_index)
+        : column_index_(column_index), offset_index_(offset_index) {}
+
+    std::shared_ptr<::parquet::ColumnIndex> GetColumnIndex(int32_t) override {
+        return column_index_;
+    }
+    std::shared_ptr<::parquet::OffsetIndex> GetOffsetIndex(int32_t) override {
+        return offset_index_;
+    }
+
+ private:
+    std::shared_ptr<TestColumnIndex> column_index_;
+    std::shared_ptr<::parquet::OffsetIndex> offset_index_;
+};
+
+TEST_F(ColumnIndexFilterTest, InWithNullKeepsPagesWithPossibleNulls) {
+    auto index = std::make_shared<TestColumnIndex>();
+    index->null_pages_[0] = true;
+    index->null_counts_[0] = 10;
+    index->null_counts_[1] = 1;
+    index->non_null_pages_.erase(index->non_null_pages_.begin());
+    auto page_reader = std::make_shared<TestRowGroupPageIndexReader>(
+        index, page_index_reader_->RowGroup(0)->GetOffsetIndex(0));
+    auto predicate = PredicateBuilder::In(0, "val", FieldType::INT,
+                                          {Literal(FieldType::INT), Literal(int32_t{999})});
+    ASSERT_OK_AND_ASSIGN(auto ranges, ColumnIndexFilter::CalculateRowRanges(
+                                          predicate, page_reader, column_name_to_index_,
+                                          parquet_reader_->metadata()->schema(), 100));
+    ASSERT_EQ(ranges.RowCount(), 20);
+    ASSERT_EQ(ranges.GetRanges().size(), 1u);
+    ASSERT_EQ(ranges.GetRanges()[0].from, 0);
+    ASSERT_EQ(ranges.GetRanges()[0].to, 19);
+
+    // Without null counts every non-null page may still contain nulls.
+    index->has_null_counts_ = false;
+    index->null_counts_.clear();
+    ASSERT_OK_AND_ASSIGN(
+        ranges, ColumnIndexFilter::CalculateRowRanges(predicate, page_reader, column_name_to_index_,
+                                                      parquet_reader_->metadata()->schema(), 100));
+    ASSERT_EQ(ranges.RowCount(), 100);
+}
+
+TEST_F(ColumnIndexFilterTest, NotInWithNullRejectsEveryPage) {
+    auto predicate = PredicateBuilder::NotIn(0, "val", FieldType::INT,
+                                             {Literal(int32_t{55}), Literal(FieldType::INT)});
+    ASSERT_OK_AND_ASSIGN(auto ranges, Filter(predicate));
+    ASSERT_TRUE(ranges.IsEmpty());
+}
+
+TEST_F(ColumnIndexFilterTest, TruncatedIntegerBoundsKeepPages) {
+    for (bool truncate_min : {true, false}) {
+        SCOPED_TRACE(truncate_min);
+        auto index = std::make_shared<TestColumnIndex>();
+        auto& bounds = truncate_min ? index->min_values_ : index->max_values_;
+        bounds.assign(10, std::string(3, '\0'));
+        auto page_reader = std::make_shared<TestRowGroupPageIndexReader>(
+            index, page_index_reader_->RowGroup(0)->GetOffsetIndex(0));
+        auto predicate = PredicateBuilder::Equal(0, "val", FieldType::INT, Literal(int32_t{999}));
+        ASSERT_OK_AND_ASSIGN(auto ranges, ColumnIndexFilter::CalculateRowRanges(
+                                              predicate, page_reader, column_name_to_index_,
+                                              parquet_reader_->metadata()->schema(), 100));
+        ASSERT_EQ(ranges.RowCount(), 100);
+    }
+}
+
 /// EQUAL: val = 55 → should match only page 5 (rows [50,59])
 TEST_F(ColumnIndexFilterTest, EqualMatchSinglePage) {
     auto pred =